docker-compose.yaml

# This docker-compose is for developer convenience, not for running in production.

volumes:
  condor-token:

services:

  # For running the FastAPI server
  cdm_task_service:
    build:
      context: .
      dockerfile: Dockerfile
    ports:
      - 5000:5000
    extra_hosts:
      # See https://stackoverflow.com/a/43541732/643675
      - "host.docker.internal:host-gateway"
    depends_on:
      mongodb:
        condition: service_started
      minio:
        condition: service_started
      kafka:
        condition: service_healthy
      htcondor-mini:
        condition: service_started
      cdm_refdata_service:
        condition: service_started
    environment:
      - _CONDOR_COLLECTOR_HOST=htcondor-mini:9618
      - _CONDOR_SEC_TOKEN_DIRECTORY=/etc/condor/tokens.d
      - KBCTS_PORT=5000
      - KBCTS_KBASE_AUTH2_URL=https://ci.kbase.us/services/auth
      - KBCTS_KBASE_AUTH2_ADMIN_ROLES=CDM_TASK_SERVICE_ADMIN
      - KBCTS_CTS_USER_ROLES=KBASE_STAFF
      - KBCTS_HAS_NERSC_ACCOUNT_ROLE=HAS_NERSC_ACCOUNT
      - KBCTS_NERSC_JAWS_USER=cdm_ts
      - KBCTS_NERSC_JAWS_REFDATA_DIR=/global/dna/kbase/reference/jaws
      - KBCTS_NERSC_JAWS_PERMUTTER_STAGING_DIR=/pscratch/sd/k/kbjaws/kbase-prod/
      - KBCTS_SFAPI_CRED_PATH=/creds/sfapi_creds
      - KBCTS_NERSC_REMOTE_CODE_DIR=/global/cfs/cdirs/kbase/cdm_task_service
      - KBCTS_JAWS_URL=https://jaws-api.jgi.doe.gov/
      # Don't commit your token to github please
      - KBCTS_JAWS_TOKEN=dont_check_tokens_into_vcs_please
      - KBCTS_JAWS_GROUP=kbase
      # From ./docker_compose/htcondor/start_condor_and_gen_token.sh
      - KBCTS_HTC_INITIAL_DIR=/condor_workdir
      # From ./docker_compose/htcondor/condor_config.local
      - KBCTS_HTC_CLIENT_GROUP=test_cg
      - KBCTS_HTC_ADDL_PATH=/home/submituser/.local/bin
      - KBCTS_HTC_CACHE_DIR=/home/submituser/job_cache
      - KBCTS_REFDATA_SERVER_URL=http://cdm_refdata_service:5001
      - KBCTS_REFDATA_SERVER_TOKEN=token_with_the_cts_role
      - KBCTS_REFDATA_HOST_PATH=<absolute_path_to_repo>/refdata_mount/data
      - KBCTS_EXTERNAL_JOB_MOUNT_PREFIX_OVERRIDE=/var/lib/condor/execute:<path to condor_share>/condor_share
      # Note that the host minio must be bound to 0.0.0.0:9002 or 172.17.0.1:9002 (the docker
      # bridge interface). The latter is safer but only works for bridge networks
      - KBCTS_S3_URL=http://host.docker.internal:9002
      # local Minio, only useful if not running jobs on NERSC
      #- KBCTS_S3_URL=http://minio:9000
      - KBCTS_S3_EXTERNAL_URL=https://ci.berkeley.kbase.us:9000
      - KBCTS_VERIFY_S3_EXTERNAL_URL=false
      - KBCTS_S3_ACCESS_KEY=miniouser
      - KBCTS_S3_ACCESS_SECRET=miniopassword
      - KBCTS_S3_ALLOW_INSECURE=false
      - KBCTS_S3_ALLOW_INSECURE_EXTERNAL=true
      - KBCTS_MONGO_HOST=mongodb://mongodb:27017
      - KBCTS_MONGO_DB=cdmtaskservice
      # TODO MONGOAUTH need to add a user to the cmdtaskservice db before this works out
      #                of the box.
      #                The env vars in mongodb below only create a user in admin and only if
      #                the data directory is empty.
      # - KBCTS_MONGO_USER=root
      # - KBCTS_MONGO_PWD=secret
      - KBCTS_MONGO_RETRYWRITES=false
      - KBCTS_ALLOWED_S3_PATHS=cts/io
      - KBCTS_CONTAINER_S3_LOG_DIR=cts/logs
      - KBCTS_JOB_MAX_CPU_HOURS=200
      - KBCTS_KAFKA_BOOTSTRAP_SERVERS=kafka:9092
      - KBCTS_KAFKA_TOPIC_JOBS=cts-jobs
      - KBCTS_SERVICE_ROOT_URL=http://cdm_task_service:5000
      - KBCTS_SERVICE_GROUP=local_development
    volumes:
      - ./tmp_creds:/creds
      - condor-token:/etc/condor/tokens.d
      - ./test_manual:/cts/test_manual

  cdm_refdata_service:
    build:
      context: .
      dockerfile: Dockerfile
    ports:
      - 5001:5001
    extra_hosts:
      # See https://stackoverflow.com/a/43541732/643675
      - "host.docker.internal:host-gateway"
    depends_on:
      minio:
        condition: service_started
    environment:
      - KBCTS_MODE=refdata
      - KBCTS_PORT=5001
      - KBCTS_KBASE_AUTH2_URL=https://ci.kbase.us/services/auth
      - KBCTS_KBASE_AUTH2_ADMIN_ROLES=CDM_TASK_SERVICE_ADMIN
      # Note that the host minio must be bound to 0.0.0.0:9002 or 172.17.0.1:9002 (the docker
      # bridge interface). The latter is safer but only works for bridge networks
      - KBCTS_S3_URL=http://host.docker.internal:9002
      # local Minio, only useful if not running jobs on NERSC
      #- KBCTS_S3_URL=http://minio:9000
      - KBCTS_S3_ACCESS_KEY=miniouser
      - KBCTS_S3_ACCESS_SECRET=miniopassword
      - KBCTS_S3_ALLOW_INSECURE=true
      - KBCTS_REFDATA_LOCAL_PATH=/refdata/data
      - KBCTS_REFDATA_METADATA_PATH=/refdata/meta
      - KBCTS_SERVICE_ROOT_URL=http://cdm_task_service:5000
      - KBCTS_REFDATA_SERVICE_TOKEN=token_with_refservice_role
      - KBCTS_SERVICE_GROUP=localdev
    volumes:
      - ./refdata_mount:/refdata

  kafka:
    # Note the image should sync with the test.cfg.example and GHA test files.
    image: apache/kafka-native:4.0.0
    ports:
      - 9092:9092
    environment:
      # default config
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://localhost:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_NUM_PARTITIONS: 3
      
      # custom config
      KAFKA_DELETE_TOPIC_ENABLE: true
    healthcheck:
      test: ["CMD-SHELL", "nc -z localhost 9092"]
      interval: 1s
      timeout: 3s
      retries: 30

  minio:
    # minio/minio was removed from Docker Hub; mirrored from ghcr.io/coollabsio/minio in the
    # interim. See https://github.com/coollabsio/minio
    image: ghcr.io/coollabsio/minio:RELEASE.2025-04-22T22-12-26Z
    environment:
      MINIO_ROOT_USER: miniouser
      MINIO_ROOT_PASSWORD: miniopassword
    ports:
      - 9000:9000
      - 9001:9001
    command: 'server /data --console-address ":9001"'

  mongodb:
    image: mongo:7.0.14
    ports:
      - 27017:27017
    # environment:
    #  - MONGO_INITDB_ROOT_USERNAME=root
    #  - MONGO_INITDB_ROOT_PASSWORD=secret

  htcondor-mini:
    # TODO TEST try and get gpu testing working
    build:
      context: .
      dockerfile: ./docker_compose/htcondor/Dockerfile
      args:
        DOCKER_GID: 999  # replace with your host's docker group GID
    extra_hosts:
      # See https://stackoverflow.com/a/43541732/643675
      - "host.docker.internal:host-gateway"
    environment:
      CTS_EXCECUTOR_TOKEN_FOR_FILE: dont_check_tokens_into_vcs_please
      # Note this will need to be replaced with the external minio secret
      # if not using the docker compose instance
      CTS_EXECUTOR_S3_ACCESS_SECRET_FOR_FILE: miniopassword
    ports:
      - "9618:9618"
    volumes:
      - <path to chondor_share>/condor_share:/var/lib/condor/execute
      - ./docker_compose/htcondor/submit_test_script:/opt/submit_test_script
      - ./docker_compose/htcondor/start_condor_and_gen_token.sh:/opt/start_condor_and_gen_token.sh
      - ./docker_compose/htcondor/condor_config.local:/etc/condor/condor_config.local
      - ./docker_compose/htcondor/signal_test.py:/signal_test.py
      - ./docker_compose/htcondor/signal_test.sh:/signal_test.sh
      - condor-token:/etc/condor/tokens.d   # shared token directory
      - /var/run/docker.sock:/var/run/docker.sock
    # for some reason the script is failing without the `sh`, not sure why
    command: sh /opt/start_condor_and_gen_token.sh