docker-compose.yml

services:
  traefik:
    image: traefik:v3.7.13
    container_name: sysndd_traefik
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
        compress: "true"
    command:
      # Enable API for healthcheck CLI but disable dashboard (security)
      # Internal API uses 'traefik' entrypoint, our routes use 'web' - no conflict
      - "--api=true"
      - "--api.insecure=true"
      - "--api.dashboard=false"
      - "--entryPoints.traefik.address=:8080"
      - "--entryPoints.traefik.observability.accessLogs=false"
      # Docker provider
      - "--providers.docker=true"
      - "--providers.docker.exposedbydefault=false"
      - "--providers.docker.network=sysndd_proxy"
      # HTTP entrypoint
      - "--entryPoints.web.address=:80"
      # Bound slow/incomplete bodies. Enable privacy-bounded access logs globally,
      # opt the shared web entrypoint out, then opt only MCP routers back in.
      - "--entryPoints.web.transport.respondingTimeouts.readTimeout=60s"
      - "--accesslog=true"
      - "--accesslog.format=json"
      - "--accesslog.bufferingSize=100"
      - "--accesslog.fields.headers.defaultmode=drop"
      - "--accesslog.fields.queryparameters.defaultmode=drop"
      - "--accesslog.fields.names.ClientAddr=drop"
      - "--accesslog.fields.names.ClientHost=drop"
      - "--entryPoints.web.observability.accessLogs=false"
      # RFC 3986 encoded characters - allow all (backend handles decoding)
      # See: https://doc.traefik.io/traefik/v3.6/security/request-path/
      - "--entryPoints.web.http.encodedCharacters.allowEncodedSlash=true"
      - "--entryPoints.web.http.encodedCharacters.allowEncodedBackSlash=true"
      - "--entryPoints.web.http.encodedCharacters.allowEncodedPercent=true"
      # Enable ping for health checks (on API port for traefik healthcheck CLI)
      - "--ping=true"
      # Minimal logging in production
      - "--log.level=WARN"
    ports:
      - "80:80"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
    networks:
      - proxy
      - mcp_edge
    healthcheck:
      test: ["CMD", "traefik", "healthcheck", "--ping"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.25'

  mysql:
    image: mysql:8.4.11
    container_name: sysndd_mysql
    restart: unless-stopped
    oom_score_adj: -500
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
        compress: "true"
    command:
      - --authentication-policy=caching_sha2_password
      - --character-set-server=utf8mb4
      - --collation-server=utf8mb4_unicode_ci
      - --innodb-buffer-pool-size=512M
      - --innodb-buffer-pool-instances=1
      - --performance-schema=OFF
      - --temptable-max-ram=128M
      - --innodb-log-buffer-size=16M
      - --binlog-row-image=MINIMAL
      - --binlog-expire-logs-seconds=604800
      - --max-connections=60
    environment:
      MYSQL_DATABASE: ${MYSQL_DATABASE}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
    volumes:
      - mysql_data:/var/lib/mysql
      - mysql_backup:/backup
    networks:
      - backend
    healthcheck:
      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "${MYSQL_USER}", "-p${MYSQL_PASSWORD}"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    deploy:
      resources:
        limits:
          memory: 1536M
          cpus: '1.0'
        reservations:
          memory: 512M

  mysql-cron-backup:
    image: fradelg/mysql-cron-backup
    container_name: sysndd_mysql_backup
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
        compress: "true"
    depends_on:
      - mysql
    volumes:
      - mysql_backup:/backup
    environment:
      MYSQL_HOST: mysql
      MYSQL_USER: root
      MYSQL_PASS: ${MYSQL_ROOT_PASSWORD}
      MAX_BACKUPS: 60
      INIT_BACKUP: 1
      # Every day at 03:00
      CRON_TIME: 0 3 * * *
      # Make it small
      GZIP_LEVEL: 9
    networks:
      - backend
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.5'

  api:
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        # UID/GID for non-root user - matches most Linux hosts (uid 1000)
        # Override with HOST_UID/HOST_GID env vars for different host users
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    restart: unless-stopped
    init: true
    stop_grace_period: 30s
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "5"
        compress: "true"
    depends_on:
      mysql:
        condition: service_healthy
    # Run from /app where packages are installed, with selective mounts for dev
    volumes:
      - ./api/endpoints:/app/endpoints
      - ./api/functions:/app/functions
      - ./api/core:/app/core
      - ./api/services:/app/services
      - ./api/bootstrap:/app/bootstrap
      - ./api/config:/app/config
      - ./api/scripts:/app/scripts:ro
      - ./api/layout/gene-network-fcose-layout.mjs:/app/layout/gene-network-fcose-layout.mjs:ro
      - ./api/layout/package.json:/app/layout/package.json:ro
      - ./api/layout/package-lock.json:/app/layout/package-lock.json:ro
      - ./api/data:/app/data
      - ./api/results:/app/results
      - ./db/migrations:/app/db/migrations:ro
      - api_cache:/app/cache
      - mysql_backup:/backup:rw
      - ./api/config.yml:/app/config.yml:ro
      - ./api/version_spec.json:/app/version_spec.json
      - ./api/start_sysndd_api.R:/app/start_sysndd_api.R
    environment:
      ENVIRONMENT: production
      PASSWORD: ${PASSWORD}
      SMTP_PASSWORD: ${SMTP_PASSWORD}
      DB_POOL_SIZE: ${DB_POOL_SIZE:-5}
      MIRAI_WORKERS: ${MIRAI_WORKERS:-0}
      GEMINI_API_KEY: ${GEMINI_API_KEY:-}
      # NCBI E-utilities identity/rate-limit (#494/#499): raises the per-IP EUtils
      # cap from the anonymous 3 req/s to 10 req/s so publication ingestion and the
      # publication_date_backfill do not 429. Inert when unset (anonymous is valid).
      NCBI_API_KEY: ${NCBI_API_KEY:-}
      NCBI_EUTILS_EMAIL: ${NCBI_EUTILS_EMAIL:-}
      # CORS configuration: comma-separated list of allowed origins
      # Critical for Firefox compatibility (issue #143)
      CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-}
      CACHE_VERSION: ${CACHE_VERSION:-3}
      # #535 S6: per-caller throttle on the public clustering submit routes. Compose
      # uses explicit env maps, so these must be passed through to be tunable via .env.
      # ASYNC_PUBLIC_JOB_CAP is the global queue-depth backstop (invalid -> default 8).
      ASYNC_PUBLIC_JOB_CAP: ${ASYNC_PUBLIC_JOB_CAP:-8}
      CLUSTERING_SUBMIT_PER_CALLER_MAX: ${CLUSTERING_SUBMIT_PER_CALLER_MAX:-5}
      CLUSTERING_SUBMIT_WINDOW_SECONDS: ${CLUSTERING_SUBMIT_WINDOW_SECONDS:-60}
      CLUSTERING_SUBMIT_MAX_TRACKED: ${CLUSTERING_SUBMIT_MAX_TRACKED:-20000}
      # Trusted reverse-proxy source CIDRs (comma-separated). Empty for the shipped
      # single-Traefik direct-edge topology; set to the front-proxy CIDR(s) ONLY when an
      # additional trusted proxy sits in front of Traefik (see 09-deployment.qmd).
      CLUSTERING_SUBMIT_TRUSTED_PROXY_CIDRS: ${CLUSTERING_SUBMIT_TRUSTED_PROXY_CIDRS:-}
      # #550: same bounded, fail-closed per-caller admission control for public
      # signup/signin/password-reset requests. See 09-deployment.qmd for proxy trust.
      AUTH_ENDPOINT_PER_CALLER_MAX: ${AUTH_ENDPOINT_PER_CALLER_MAX:-5}
      AUTH_ENDPOINT_WINDOW_SECONDS: ${AUTH_ENDPOINT_WINDOW_SECONDS:-60}
      AUTH_ENDPOINT_MAX_TRACKED: ${AUTH_ENDPOINT_MAX_TRACKED:-20000}
      AUTH_ENDPOINT_TRUSTED_PROXY_CIDRS: ${AUTH_ENDPOINT_TRUSTED_PROXY_CIDRS:-}
      # OMIM download key for genemap2.txt (required for ontology/comparisons)
      OMIM_DOWNLOAD_KEY: ${OMIM_DOWNLOAD_KEY}
      # NDDScore release source; defaults match api/config.yml.
      NDDSCORE_ZENODO_RECORD_ID: "${NDDSCORE_ZENODO_RECORD_ID:-20258027}"
      NDDSCORE_ZENODO_API_BASE_URL: "${NDDSCORE_ZENODO_API_BASE_URL:-https://zenodo.org/api/records}"
      # Database version surface (issue #22). Optional release-time injection;
      # when set, the API refreshes the db_version row at startup. The migration
      # seeds a baseline so the surface works even when these are unset.
      DB_VERSION: "${DB_VERSION:-}"
      DB_COMMIT: "${DB_COMMIT:-}"
      # Auto-bootstrap public analysis snapshots on startup (#420). Default on;
      # set to false to disable the post-migration enqueue of missing presets.
      ANALYSIS_SNAPSHOT_BOOTSTRAP_ON_STARTUP: "${ANALYSIS_SNAPSHOT_BOOTSTRAP_ON_STARTUP:-true}"
    networks:
      - proxy
      - backend
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:7777/api/health/ready"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    deploy:
      # #344: core lane floor. A single API container gives zero cross-request
      # isolation; two replicas is the baseline. Override for load tests with
      # `--scale api=N`.
      replicas: 2
      resources:
        limits:
          memory: 4608M
          cpus: '2.0'
    develop:
      watch:
        - action: sync
          path: ./api/endpoints
          target: /app/endpoints
        - action: sync
          path: ./api/functions
          target: /app/functions
        - action: sync
          path: ./api/core
          target: /app/core
        - action: sync
          path: ./api/services
          target: /app/services
        - action: sync
          path: ./api/bootstrap
          target: /app/bootstrap
        - action: rebuild
          path: ./api/renv.lock
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.api.rule=Host(`sysndd.dbmr.unibe.ch`) && PathPrefix(`/api`)"
      - "traefik.http.routers.api.entrypoints=web"
      - "traefik.http.routers.api.priority=100"
      # #535 S6: strip client-supplied X-Forwarded-For *underscore aliases* so only
      # Traefik's canonical (hyphenated) X-Forwarded-For chain reaches the API. Plumber
      # folds `X_Forwarded_For` (and mixed `X-Forwarded_For` / `X_Forwarded-For`) into
      # the SAME CGI field as the canonical header; without this an attacker could
      # inject a rotating IP that collides with the proxy-owned value and defeats the
      # per-caller submit throttle. Empty value = delete the header before forwarding.
      - "traefik.http.middlewares.api-strip-xff-alias.headers.customRequestHeaders.X_Forwarded_For="
      - "traefik.http.middlewares.api-strip-xff-alias.headers.customRequestHeaders.X-Forwarded_For="
      - "traefik.http.middlewares.api-strip-xff-alias.headers.customRequestHeaders.X_Forwarded-For="
      - "traefik.http.routers.api.middlewares=api-strip-xff-alias@docker"
      - "traefik.http.services.api.loadbalancer.server.port=7777"
      # #344: sticky sessions removed — job state is durable in `async_jobs`
      # (MySQL), so no request needs the same instance. Sticky harmed load
      # distribution (the exact thing the 2026-05-18 mitigation had to undo).

  # #344: synchronous "enrichment" lane. Same image + full router as `api`, but
  # Traefik sends it ONLY /api/external/* (live upstream provider I/O). Isolating
  # that surface into its own process pool means a slow upstream can never
  # head-of-line-block cheap/core routes (health/auth/statistics/entity/gene/
  # analysis/jobs), which stay on `api`. Mirrors the worker/worker-maintenance
  # async-lane split. Scale independently: --scale api-enrichment=N.
  api-enrichment:
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    command: ["Rscript", "start_sysndd_api.R"]
    restart: unless-stopped
    init: true
    stop_grace_period: 30s
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "5"
        compress: "true"
    depends_on:
      api:
        condition: service_healthy   # migrations + startup bootstraps done by core first
    volumes:
      - ./api/endpoints:/app/endpoints
      - ./api/functions:/app/functions
      - ./api/core:/app/core
      - ./api/services:/app/services
      - ./api/bootstrap:/app/bootstrap
      - ./api/config:/app/config
      - ./api/scripts:/app/scripts:ro
      - ./api/data:/app/data
      - ./api/results:/app/results
      - ./db/migrations:/app/db/migrations:ro
      - api_cache:/app/cache
      - ./api/config.yml:/app/config.yml:ro
      - ./api/version_spec.json:/app/version_spec.json
      - ./api/start_sysndd_api.R:/app/start_sysndd_api.R
    environment:
      ENVIRONMENT: production
      API_LANE: enrichment            # #344: skip startup bootstraps (core owns them)
      PASSWORD: ${PASSWORD}
      SMTP_PASSWORD: ${SMTP_PASSWORD}
      DB_POOL_SIZE: ${API_ENRICHMENT_DB_POOL_SIZE:-3}
      MIRAI_WORKERS: ${API_ENRICHMENT_MIRAI_WORKERS:-0}
      GEMINI_API_KEY: ${GEMINI_API_KEY:-}
      NCBI_API_KEY: ${NCBI_API_KEY:-}
      NCBI_EUTILS_EMAIL: ${NCBI_EUTILS_EMAIL:-}
      CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-}
      CACHE_VERSION: ${CACHE_VERSION:-3}
      OMIM_DOWNLOAD_KEY: ${OMIM_DOWNLOAD_KEY}
      # Belt-and-braces: the API_LANE gate already skips these, but pin the flags
      # off too so a future ungated bootstrap cannot double-run on this lane.
      ANALYSIS_SNAPSHOT_BOOTSTRAP_ON_STARTUP: "false"
      DISEASE_ONTOLOGY_MAPPING_BOOTSTRAP_ON_STARTUP: "false"
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:7777/api/health/ready"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    deploy:
      replicas: 1
      resources:
        limits:
          memory: 2048M
          cpus: '1.0'
    networks:
      - proxy      # egress for external providers
      - backend    # DB for cache reads/writes (e.g. RGD id lookup)
    labels:
      - "traefik.enable=true"
      # Higher priority than the core `api` router (100) so /api/external wins.
      - "traefik.http.routers.api-enrichment.rule=Host(`sysndd.dbmr.unibe.ch`) && PathPrefix(`/api/external`)"
      - "traefik.http.routers.api-enrichment.entrypoints=web"
      - "traefik.http.routers.api-enrichment.priority=200"
      - "traefik.http.routers.api-enrichment.middlewares=api-strip-xff-alias@docker"
      - "traefik.http.services.api-enrichment.loadbalancer.server.port=7777"
      # No sticky cookie: the enrichment lane is stateless (durable MySQL jobs).

  worker:
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    command:
      - bash
      - -lc
      - >
          drain_file="$${ASYNC_JOB_DRAIN_FILE:-/tmp/sysndd_async_worker_drain}";
          rm -f "$$drain_file";
          trap 'touch "$$drain_file"' TERM INT;
          Rscript /app/start_async_worker.R &
          child=$$!;
          wait "$$child"
    restart: unless-stopped
    init: true
    stop_grace_period: 60s
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "5"
        compress: "true"
    depends_on:
      api:
        condition: service_healthy
    volumes:
      - ./api/functions:/app/functions
      - ./api/core:/app/core
      - ./api/services:/app/services
      - ./api/bootstrap:/app/bootstrap
      - ./api/config:/app/config
      - ./api/scripts:/app/scripts:ro
      - ./api/layout/gene-network-fcose-layout.mjs:/app/layout/gene-network-fcose-layout.mjs:ro
      - ./api/layout/package.json:/app/layout/package.json:ro
      - ./api/layout/package-lock.json:/app/layout/package-lock.json:ro
      - ./api/data:/app/data
      - ./api/results:/app/results
      - ./db/migrations:/app/db/migrations:ro
      - api_cache:/app/cache
      - mysql_backup:/backup:rw
      - ./api/config.yml:/app/config.yml:ro
      - ./api/start_async_worker.R:/app/start_async_worker.R
    environment:
      ENVIRONMENT: production
      PASSWORD: ${PASSWORD}
      SMTP_PASSWORD: ${SMTP_PASSWORD}
      DB_POOL_SIZE: ${DB_POOL_SIZE:-5}
      GEMINI_API_KEY: ${GEMINI_API_KEY:-}
      CACHE_VERSION: ${CACHE_VERSION:-3}
      OMIM_DOWNLOAD_KEY: ${OMIM_DOWNLOAD_KEY}
      NDDSCORE_ZENODO_RECORD_ID: "${NDDSCORE_ZENODO_RECORD_ID:-20258027}"
      NDDSCORE_ZENODO_API_BASE_URL: "${NDDSCORE_ZENODO_API_BASE_URL:-https://zenodo.org/api/records}"
      ASYNC_JOB_LEASE_SECONDS: ${ASYNC_JOB_LEASE_SECONDS:-60}
      ASYNC_JOB_IDLE_SLEEP_SECONDS: ${ASYNC_JOB_IDLE_SLEEP_SECONDS:-2}
      MAX_JOBS_PER_WORKER: ${MAX_JOBS_PER_WORKER:-50}
      MAX_WORKER_LIFETIME: ${MAX_WORKER_LIFETIME:-3600}
      # Interactive lane only (#486): heavy/bulk maintenance jobs run on the
      # sibling `worker-maintenance` container's `maintenance` lane so they never
      # head-of-line block clustering / llm_generation / snapshot->LLM jobs on
      # this worker.
      ASYNC_JOB_QUEUES: ${ASYNC_JOB_QUEUES:-default}
      # NCBI E-utilities key (#494/#499). The dev override drains the maintenance
      # lane on this worker too, so the backfill runs here locally.
      NCBI_API_KEY: ${NCBI_API_KEY:-}
      NCBI_EUTILS_EMAIL: ${NCBI_EUTILS_EMAIL:-}
      ASYNC_JOB_DRAIN_FILE: ${ASYNC_JOB_DRAIN_FILE:-/tmp/sysndd_async_worker_drain}
    healthcheck:
      test: ["CMD-SHELL", "ps -ef | grep '[s]tart_async_worker.R' >/dev/null || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s
    networks:
      # Backend is internal-only; worker also needs proxy egress for external
      # provider APIs such as Gemini, PubMed, and PubTator.
      - proxy
      - backend
    deploy:
      resources:
        limits:
          memory: 3072M
          cpus: '2.0'
    develop:
      watch:
        - action: sync
          path: ./api/functions
          target: /app/functions
        - action: sync
          path: ./api/core
          target: /app/core
        - action: sync
          path: ./api/services
          target: /app/services
        - action: sync
          path: ./api/bootstrap
          target: /app/bootstrap
        - action: rebuild
          path: ./api/renv.lock

  # Maintenance-lane async worker (#486). This is a deliberate mirror of `worker`
  # and MUST stay in sync with it (same image / command / volumes / networks /
  # env / restart / depends_on) — only ASYNC_JOB_QUEUES differs. It drains the
  # `maintenance` lane so heavy/bulk/external jobs (publication_date_backfill,
  # omim_update, publication_refresh, disease_ontology_mapping_refresh, backups,
  # etc.) run in parallel with the interactive `worker` instead of blocking it.
  # Like `worker`, it needs BOTH the internal `backend` (DB) and egress-capable
  # `proxy` (PubMed / OMIM / Zenodo / MONDO) networks.
  worker-maintenance:
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    command:
      - bash
      - -lc
      - >
          drain_file="$${ASYNC_JOB_DRAIN_FILE:-/tmp/sysndd_async_worker_drain}";
          rm -f "$$drain_file";
          trap 'touch "$$drain_file"' TERM INT;
          Rscript /app/start_async_worker.R &
          child=$$!;
          wait "$$child"
    restart: unless-stopped
    init: true
    stop_grace_period: 60s
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "5"
        compress: "true"
    depends_on:
      api:
        condition: service_healthy
    volumes:
      - ./api/functions:/app/functions
      - ./api/core:/app/core
      - ./api/services:/app/services
      - ./api/bootstrap:/app/bootstrap
      - ./api/config:/app/config
      - ./api/scripts:/app/scripts:ro
      - ./api/layout/gene-network-fcose-layout.mjs:/app/layout/gene-network-fcose-layout.mjs:ro
      - ./api/layout/package.json:/app/layout/package.json:ro
      - ./api/layout/package-lock.json:/app/layout/package-lock.json:ro
      - ./api/data:/app/data
      - ./api/results:/app/results
      - ./db/migrations:/app/db/migrations:ro
      - api_cache:/app/cache
      - mysql_backup:/backup:rw
      - ./api/config.yml:/app/config.yml:ro
      - ./api/start_async_worker.R:/app/start_async_worker.R
    environment:
      ENVIRONMENT: production
      PASSWORD: ${PASSWORD}
      SMTP_PASSWORD: ${SMTP_PASSWORD}
      DB_POOL_SIZE: ${DB_POOL_SIZE:-5}
      GEMINI_API_KEY: ${GEMINI_API_KEY:-}
      CACHE_VERSION: ${CACHE_VERSION:-3}
      OMIM_DOWNLOAD_KEY: ${OMIM_DOWNLOAD_KEY}
      NDDSCORE_ZENODO_RECORD_ID: "${NDDSCORE_ZENODO_RECORD_ID:-20258027}"
      NDDSCORE_ZENODO_API_BASE_URL: "${NDDSCORE_ZENODO_API_BASE_URL:-https://zenodo.org/api/records}"
      ASYNC_JOB_LEASE_SECONDS: ${ASYNC_JOB_LEASE_SECONDS:-60}
      ASYNC_JOB_IDLE_SLEEP_SECONDS: ${ASYNC_JOB_IDLE_SLEEP_SECONDS:-2}
      MAX_JOBS_PER_WORKER: ${MAX_JOBS_PER_WORKER:-50}
      MAX_WORKER_LIFETIME: ${MAX_WORKER_LIFETIME:-3600}
      # Maintenance lane only. Override ASYNC_JOB_MAINTENANCE_QUEUES to widen it.
      ASYNC_JOB_QUEUES: ${ASYNC_JOB_MAINTENANCE_QUEUES:-maintenance}
      # NCBI E-utilities key (#494/#499): this container runs publication_date_backfill.
      NCBI_API_KEY: ${NCBI_API_KEY:-}
      NCBI_EUTILS_EMAIL: ${NCBI_EUTILS_EMAIL:-}
      ASYNC_JOB_DRAIN_FILE: ${ASYNC_JOB_DRAIN_FILE:-/tmp/sysndd_async_worker_drain}
    healthcheck:
      test: ["CMD-SHELL", "ps -ef | grep '[s]tart_async_worker.R' >/dev/null || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s
    networks:
      # Backend is internal-only; the maintenance worker also needs proxy egress
      # for external provider APIs such as PubMed, OMIM, Zenodo, and MONDO.
      - proxy
      - backend
    deploy:
      resources:
        limits:
          memory: 2048M
          cpus: '1.5'
    develop:
      watch:
        - action: sync
          path: ./api/functions
          target: /app/functions
        - action: sync
          path: ./api/core
          target: /app/core
        - action: sync
          path: ./api/services
          target: /app/services
        - action: sync
          path: ./api/bootstrap
          target: /app/bootstrap
        - action: rebuild
          path: ./api/renv.lock

  mcp:
    profiles: ["mcp"]
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    command: ["Rscript", "start_sysndd_mcp.R"]
    restart: unless-stopped
    init: true
    stop_grace_period: 30s
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "20m"
        max-file: "3"
        compress: "true"
    depends_on:
      mysql:
        condition: service_healthy
    volumes:
      - ./api/functions:/app/functions:ro
      - ./api/core:/app/core:ro
      - ./api/services:/app/services:ro
      - ./api/bootstrap:/app/bootstrap:ro
      - ./api/config/mcp:/app/config/mcp:ro
      - ./api/scripts:/app/scripts:ro
      - type: bind
        source: ${MCP_DB_PASSWORD_OUTPUT_FILE:-./secrets/mcp-db-password}
        target: /run/secrets/mcp_db_password
        read_only: true
        bind:
          create_host_path: false
      - ./api/version_spec.json:/app/version_spec.json:ro
      - ./api/start_sysndd_mcp.R:/app/start_sysndd_mcp.R:ro
    environment:
      ENVIRONMENT: production
      SYSNDD_RUNTIME: mcp
      MCP_DB_HOST: ${MCP_DB_HOST:-mysql}
      MCP_DB_PORT: ${MCP_DB_PORT:-3306}
      MCP_DB_NAME: ${MCP_DB_NAME:-sysndd_db}
      MCP_DB_USER: sysndd_mcp
      MCP_DB_PASSWORD_FILE: /run/secrets/mcp_db_password
      MCP_DB_POOL_SIZE: ${MCP_DB_POOL_SIZE:-2}
      MCP_HOST: 0.0.0.0
      MCP_PORT: 8787
      MCP_OUTPUT_MODE: ${MCP_OUTPUT_MODE:-json_text}
      MCP_ALLOWED_ORIGINS: ${MCP_ALLOWED_ORIGINS:-https://sysndd.dbmr.unibe.ch}
    networks:
      # Public ingress arrives only through the dedicated internal edge network;
      # MCP keeps its internal DB path and receives no internet egress.
      - mcp_edge
      - backend
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=sysndd_mcp_edge"
      - "traefik.http.routers.mcp-post.rule=Host(`sysndd.dbmr.unibe.ch`) && Path(`/mcp`) && Method(`POST`)"
      - "traefik.http.routers.mcp-post.entrypoints=web"
      - "traefik.http.routers.mcp-post.priority=200"
      - "traefik.http.routers.mcp-post.service=mcp"
      - "traefik.http.routers.mcp-post.middlewares=mcp-shared-rate,mcp-post-body,mcp-post-inflight,mcp-strip"
      - "traefik.http.routers.mcp-post.observability.accesslogs=true"
      - "traefik.http.routers.mcp-get.rule=Host(`sysndd.dbmr.unibe.ch`) && Path(`/mcp`) && Method(`GET`) && HeaderRegexp(`Accept`, `(?i)(^|[[:space:],])text/event-stream([[:space:];,]|$)`)"
      - "traefik.http.routers.mcp-get.entrypoints=web"
      - "traefik.http.routers.mcp-get.priority=200"
      - "traefik.http.routers.mcp-get.service=mcp"
      - "traefik.http.routers.mcp-get.middlewares=mcp-shared-rate,mcp-strip"
      - "traefik.http.routers.mcp-get.observability.accesslogs=true"
      - "traefik.http.routers.mcp-delete.rule=Host(`sysndd.dbmr.unibe.ch`) && Path(`/mcp`) && Method(`DELETE`)"
      - "traefik.http.routers.mcp-delete.entrypoints=web"
      - "traefik.http.routers.mcp-delete.priority=200"
      - "traefik.http.routers.mcp-delete.service=mcp"
      - "traefik.http.routers.mcp-delete.middlewares=mcp-shared-rate,mcp-strip"
      - "traefik.http.routers.mcp-delete.observability.accesslogs=true"
      - "traefik.http.middlewares.mcp-shared-rate.ratelimit.average=120"
      - "traefik.http.middlewares.mcp-shared-rate.ratelimit.period=1m"
      - "traefik.http.middlewares.mcp-shared-rate.ratelimit.burst=20"
      - "traefik.http.middlewares.mcp-shared-rate.ratelimit.sourcecriterion.requesthost=true"
      - "traefik.http.middlewares.mcp-post-body.buffering.maxrequestbodybytes=262144"
      - "traefik.http.middlewares.mcp-post-inflight.inflightreq.amount=4"
      - "traefik.http.middlewares.mcp-post-inflight.inflightreq.sourcecriterion.requesthost=true"
      - "traefik.http.middlewares.mcp-strip.stripprefix.prefixes=/mcp"
      - "traefik.http.services.mcp.loadbalancer.server.port=8787"
    healthcheck:
      test: ["CMD", "Rscript", "scripts/mcp-healthcheck.R"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s
    deploy:
      resources:
        limits:
          memory: 1024M
          cpus: '0.5'
    develop:
      watch:
        - action: sync
          path: ./api/functions
          target: /app/functions
        - action: sync
          path: ./api/services
          target: /app/services
        - action: sync
          path: ./api/bootstrap
          target: /app/bootstrap
        - action: rebuild
          path: ./api/renv.lock

  # Explicit one-off operator path for creating/rotating the SELECT-only MCP
  # principal. This service never starts with the application or MCP profiles.
  mcp-provisioner:
    profiles: ["mcp-provision"]
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    command: ["Rscript", "scripts/provision-mcp-readonly-principal.R"]
    restart: "no"
    security_opt:
      - no-new-privileges:true
    volumes:
      - ./api/functions:/app/functions:ro
      - ./api/scripts:/app/scripts:ro
      - ./db/migrations:/app/db/migrations:ro
      - type: bind
        source: ./secrets
        target: /run/secrets/sysndd
        read_only: false
        bind:
          create_host_path: false
    environment:
      MCP_PROVISION_SERIALIZED: "true"
      MCP_ADMIN_DB_HOST: ${MCP_ADMIN_DB_HOST:-mysql}
      MCP_ADMIN_DB_PORT: ${MCP_ADMIN_DB_PORT:-3306}
      MCP_ADMIN_DB_NAME: ${MCP_ADMIN_DB_NAME:-sysndd_db}
      MCP_ADMIN_DB_USER: ${MCP_ADMIN_DB_USER:-root}
      MCP_ADMIN_DB_PASSWORD_FILE: /run/secrets/sysndd/mcp-admin-db-password
      MCP_EXPECTED_VIEW_DEFINER: ${MCP_EXPECTED_VIEW_DEFINER:-}
      MCP_DB_PASSWORD_OUTPUT_FILE: /run/secrets/sysndd/mcp-db-password
      MCP_MIGRATION_PATH: /app/db/migrations/044_mcp_public_read_projections.sql
    networks:
      - backend

  # Scheduled retention cleanup of the operational request log table (`logging`).
  # Reuses the API image so renv deps + RMariaDB + the existing DB connection
  # helpers/config are available — no separate r-base/RMySQL container. It runs
  # a small no-root cron loop that invokes scripts/delete_old_logs.R once a day
  # at LOG_CLEANUP_AT (HH:MM, container UTC). Attached to `backend` only: it
  # needs the DB but no outbound egress. See db/migrations/000 (`logging` table)
  # and AGENTS.md (network boundary).
  log-cleanup:
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    container_name: sysndd_log_cleanup
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
        compress: "true"
    depends_on:
      mysql:
        condition: service_healthy
    # The API image's baked-in healthcheck probes the Plumber port, but this
    # service only runs the bash scheduler loop below — disable it; if the
    # loop (PID 1) dies the container exits and `restart` takes over.
    healthcheck:
      disable: true
    # Daily scheduler loop (no root / system cron needed). Sleeps until the next
    # LOG_CLEANUP_AT (HH:MM) and then runs the cleanup scripts; on script failure
    # it logs and continues to the next day rather than crash-looping. The
    # fully-bounded async_jobs prune runs FIRST so the legacy unbounded log prune
    # can never delay/starve it.
    command:
      - bash
      - -lc
      - >
          set -u;
          at="$${LOG_CLEANUP_AT:-04:00}";
          if ! printf '%s' "$$at" | grep -Eq '^([01][0-9]|2[0-3]):[0-5][0-9]$$'; then
            echo "[log-cleanup] FATAL: invalid LOG_CLEANUP_AT '$$at' (expected HH:MM, 00:00-23:59); refusing to start to avoid a tight destructive-cleanup loop";
            exit 1;
          fi;
          echo "[log-cleanup] scheduler started; daily run at $$at (container time), retention=$${LOG_RETENTION_DAYS:-30}d, dry_run=$${LOG_CLEANUP_DRY_RUN:-false}";
          while true; do
            now=$$(date +%s);
            target=$$(date -d "today $$at" +%s);
            if [ "$$target" -le "$$now" ]; then target=$$(date -d "tomorrow $$at" +%s); fi;
            sleep_for=$$(( target - now ));
            echo "[log-cleanup] sleeping $$sleep_for s until next run";
            sleep "$$sleep_for";
            echo "[log-cleanup] running scripts/delete_old_jobs.R";
            Rscript scripts/delete_old_jobs.R || echo "[log-cleanup] job-retention run failed (exit $$?); will retry next cycle";
            echo "[log-cleanup] running scripts/delete_old_logs.R";
            Rscript scripts/delete_old_logs.R || echo "[log-cleanup] log run failed (exit $$?); will retry next cycle";
          done
    volumes:
      - ./api/functions:/app/functions:ro
      - ./api/core:/app/core:ro
      - ./api/bootstrap:/app/bootstrap:ro
      - ./api/config:/app/config:ro
      - ./api/scripts:/app/scripts:ro
      - ./api/config.yml:/app/config.yml:ro
    environment:
      ENVIRONMENT: production
      # Single connection is plenty for a once-a-day prune.
      DB_POOL_SIZE: ${LOG_CLEANUP_DB_POOL_SIZE:-1}
      # Retention window in days (rows older than this are deleted).
      LOG_RETENTION_DAYS: ${LOG_RETENTION_DAYS:-30}
      # Daily run time, HH:MM in container (UTC) time.
      # 04:00 (not 03:00) so the daily prune does not compete with the 03:00
      # mysql-cron-backup dump for DB I/O.
      LOG_CLEANUP_AT: ${LOG_CLEANUP_AT:-04:00}
      # Set to 1/true to count-only and never delete (safe verification).
      LOG_CLEANUP_DRY_RUN: ${LOG_CLEANUP_DRY_RUN:-false}
      # #535 S7: same daily loop also prunes terminal async_jobs rows older than
      # this window (result_json/async_job_events grow unbounded otherwise).
      ASYNC_JOB_RETENTION_DAYS: ${ASYNC_JOB_RETENTION_DAYS:-90}
      ASYNC_JOB_RETENTION_DRY_RUN: ${ASYNC_JOB_RETENTION_DRY_RUN:-false}
      # Candidate PKs read + deleted per batch (also bounds the FK cascade into
      # async_job_events); lower it for job families that emit many events.
      ASYNC_JOB_RETENTION_BATCH_SIZE: ${ASYNC_JOB_RETENTION_BATCH_SIZE:-1000}
      # Per-batch InnoDB row-lock + metadata-lock wait ceiling (seconds).
      ASYNC_JOB_RETENTION_LOCK_WAIT_SECONDS: ${ASYNC_JOB_RETENTION_LOCK_WAIT_SECONDS:-10}
    networks:
      # DB access only; no proxy/egress needed for a DB-local cleanup.
      - backend
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.25'
    develop:
      watch:
        - action: sync
          path: ./api/functions
          target: /app/functions
        - action: sync
          path: ./api/scripts
          target: /app/scripts
        - action: rebuild
          path: ./api/renv.lock

  # PubtatorNDD nightly scheduler. A dumb sidecar: once a night it ENQUEUES one
  # durable `pubtatornidd_nightly` job and exits the run. The durable `worker`
  # service (which has PubTator/PubMed egress) claims and runs the actual
  # refresh; all retries/locking/history live there. Needs the DB only (to
  # enqueue) — no egress here. See functions/pubtatornidd-nightly.R and
  # scripts/pubtatornidd_nightly_enqueue.R.
  pubtatornidd-cron:
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    container_name: sysndd_pubtatornidd_cron
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
    depends_on:
      mysql:
        condition: service_healthy
    # Only runs the bash scheduler loop below; disable the image's Plumber
    # healthcheck. If the loop (PID 1) dies the container exits and restarts.
    healthcheck:
      disable: true
    # Daily scheduler loop (no root / system cron). Sleeps until the next
    # PUBTATORNDD_NIGHTLY_AT (HH:MM, container/UTC time) then enqueues one job;
    # on enqueue failure it logs and continues to the next day.
    command:
      - bash
      - -lc
      - >
          set -u;
          at="$${PUBTATORNDD_NIGHTLY_AT:-02:30}";
          echo "[pubtatornidd-cron] scheduler started; daily enqueue at $$at (container time)";
          while true; do
            now=$$(date +%s);
            target=$$(date -d "today $$at" +%s 2>/dev/null || date +%s);
            if [ "$$target" -le "$$now" ]; then target=$$(date -d "tomorrow $$at" +%s); fi;
            sleep_for=$$(( target - now ));
            echo "[pubtatornidd-cron] sleeping $$sleep_for s until next run";
            sleep "$$sleep_for";
            echo "[pubtatornidd-cron] enqueuing scripts/pubtatornidd_nightly_enqueue.R";
            Rscript scripts/pubtatornidd_nightly_enqueue.R || echo "[pubtatornidd-cron] enqueue failed (exit $$?); will retry next cycle";
          done
    volumes:
      - ./api/functions:/app/functions:ro
      - ./api/core:/app/core:ro
      - ./api/bootstrap:/app/bootstrap:ro
      - ./api/config:/app/config:ro
      - ./api/scripts:/app/scripts:ro
      - ./api/config.yml:/app/config.yml:ro
    environment:
      ENVIRONMENT: production
      # One connection is plenty for a once-a-day enqueue.
      DB_POOL_SIZE: ${PUBTATORNDD_CRON_DB_POOL_SIZE:-1}
      # Daily enqueue time, HH:MM in container (UTC) time.
      PUBTATORNDD_NIGHTLY_AT: ${PUBTATORNDD_NIGHTLY_AT:-02:30}
      # Optional standing-query override; empty => worker uses the cached query.
      PUBTATORNDD_NIGHTLY_QUERY: ${PUBTATORNDD_NIGHTLY_QUERY:-}
      # Optional page cap for the incremental fetch.
      PUBTATORNDD_NIGHTLY_MAX_PAGES: ${PUBTATORNDD_NIGHTLY_MAX_PAGES:-}
    networks:
      # DB access only; the worker (not this sidecar) makes the external calls.
      - backend
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.25'
    develop:
      watch:
        - action: sync
          path: ./api/functions
          target: /app/functions
        - action: sync
          path: ./api/scripts
          target: /app/scripts
        - action: rebuild
          path: ./api/renv.lock

  ontology-mapping-cron:
    image: ${SYSNDD_API_IMAGE:-sysndd-api:latest}
    build:
      context: ./api/
      args:
        UID: ${HOST_UID:-1000}
        GID: ${HOST_GID:-1000}
    container_name: sysndd_ontology_mapping_cron
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
        compress: "true"
    depends_on:
      mysql:
        condition: service_healthy
    # Only runs the bash scheduler loop below; disable the image's Plumber
    # healthcheck. If the loop (PID 1) dies the container exits and restarts.
    healthcheck:
      disable: true
    # Weekly scheduler loop (no root / system cron). Sleeps until the next
    # ONTOLOGY_MAPPING_REFRESH_DOW (0=Sun..6=Sat) at ONTOLOGY_MAPPING_REFRESH_AT
    # (HH:MM, container/UTC time), then enqueues one job; on enqueue failure it
    # logs and continues to the next cycle. The worker (egress-capable) runs the
    # job; this sidecar only enqueues over the DB-only backend network.
    command:
      - bash
      - -lc
      - >
          set -u;
          at="$${ONTOLOGY_MAPPING_REFRESH_AT:-04:30}";
          dow="$${ONTOLOGY_MAPPING_REFRESH_DOW:-0}";
          echo "[ontology-mapping-cron] scheduler started; weekly enqueue on dow=$$dow at $$at (container time)";
          while true; do
            now=$$(date +%s);
            target=0;
            for i in 0 1 2 3 4 5 6 7; do
              cand=$$(date -d "today +$$i days $$at" +%s 2>/dev/null || echo 0);
              cand_dow=$$(date -d "@$$cand" +%w 2>/dev/null || echo -1);
              if [ "$$cand" -gt "$$now" ] && [ "$$cand_dow" = "$$dow" ]; then target=$$cand; break; fi;
            done;
            if [ "$$target" -le "$$now" ]; then target=$$(( now + 86400 )); fi;
            sleep_for=$$(( target - now ));
            echo "[ontology-mapping-cron] sleeping $$sleep_for s until next run";
            sleep "$$sleep_for";
            echo "[ontology-mapping-cron] enqueuing scripts/ontology_mapping_refresh_enqueue.R";
            Rscript scripts/ontology_mapping_refresh_enqueue.R || echo "[ontology-mapping-cron] enqueue failed (exit $$?); will retry next cycle";
          done
    volumes:
      - ./api/functions:/app/functions:ro
      - ./api/services:/app/services:ro
      - ./api/core:/app/core:ro
      - ./api/bootstrap:/app/bootstrap:ro
      - ./api/config:/app/config:ro
      - ./api/scripts:/app/scripts:ro
      - ./api/config.yml:/app/config.yml:ro
    environment:
      ENVIRONMENT: production
      # One connection is plenty for a once-a-week enqueue.
      DB_POOL_SIZE: ${ONTOLOGY_MAPPING_CRON_DB_POOL_SIZE:-1}
      # Weekly enqueue time, HH:MM in container (UTC) time.
      ONTOLOGY_MAPPING_REFRESH_AT: ${ONTOLOGY_MAPPING_REFRESH_AT:-04:30}
      # Day of week to enqueue: 0=Sun .. 6=Sat.
      ONTOLOGY_MAPPING_REFRESH_DOW: ${ONTOLOGY_MAPPING_REFRESH_DOW:-0}
    networks:
      # DB access only; the worker (not this sidecar) makes the external calls.
      - backend
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.25'
    develop:
      watch:
        - action: sync
          path: ./api/functions
          target: /app/functions
        - action: sync
          path: ./api/services
          target: /app/services
        - action: sync
          path: ./api/scripts
          target: /app/scripts
        - action: rebuild
          path: ./api/renv.lock

  app:
    build:
      context: ./app/
      args:
        VUE_MODE: production
    container_name: sysndd_app
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
    networks:
      - proxy
    healthcheck:
      # Use 127.0.0.1 instead of localhost to avoid IPv6 resolution issues in Alpine
      test: ["CMD", "wget", "--spider", "--tries=1", "--no-verbose", "http://127.0.0.1:8080/"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.5'
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.app.rule=Host(`sysndd.dbmr.unibe.ch`) && PathPrefix(`/`)"
      - "traefik.http.routers.app.entrypoints=web"
      - "traefik.http.services.app.loadbalancer.server.port=8080"
      - "traefik.http.routers.app.priority=1"

  seo-prerender:
    profiles: ["ops"]
    build:
      context: ./app/
      target: builder
      args:
        VUE_MODE: production
        SEO_GENERATE: "false"
    container_name: sysndd_seo_prerender
    depends_on:
      api:
        condition: service_healthy
    environment:
      SEO_API_BASE_URL: http://api:7777/api
      SEO_PUBLIC_BASE_URL: ${SEO_PUBLIC_BASE_URL:-https://sysndd.dbmr.unibe.ch}
    command: sh -lc 'npm run seo:generate'
    networks:
      - backend

networks:
  proxy:
    name: sysndd_proxy
    driver: bridge
  mcp_edge:
    name: sysndd_mcp_edge
    driver: bridge
    internal: true
  backend:
    name: sysndd_backend
    driver: bridge
    internal: true

volumes:
  mysql_data:
    name: sysndd_mysql_data
  mysql_backup:
    name: sysndd_mysql_backup
  api_cache:
    name: sysndd_api_cache