diff --git a/.copier-answers.yml b/.copier-answers.yml index d77dfca..cf7177d 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: v9.6.1 +_commit: v10.0.5 _src_path: gh:Tecnativa/doodba-copier-template backup_deletion: false backup_dst: boto3+s3://man18-backup @@ -54,8 +54,6 @@ smtp_relay_user: manmanufacturing.com smtp_relay_version: latest traefik_version: 3 use_refurb: true -whitelist_docker_project_devel: man18 -whitelist_docker_project_test: demo-man18-19 whitelisted_hosts_devel: - cdnjs.cloudflare.com - fonts.googleapis.com diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57bdcd0..ffe213e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,7 +64,6 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: debug-statements - - id: check-case-conflict - id: check-docstring-first - id: check-executables-have-shebangs diff --git a/.ruff.toml b/.ruff.toml index 744e0e7..6c79fe1 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,7 +1,6 @@ target-version = "py310" fix = true cache-dir = "~/.cache/ruff" - [lint] extend-select = [ "B", diff --git a/README.md b/README.md index a2c60c3..38c35f9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Doodba deployment](https://img.shields.io/badge/deployment-doodba-informational)](https://github.com/Tecnativa/doodba) -[![Last template update](https://img.shields.io/badge/last%20template%20update-v9.6.1-informational)](https://github.com/Tecnativa/doodba-copier-template/tree/v9.6.1) +[![Last template update](https://img.shields.io/badge/last%20template%20update-v10.0.5-informational)](https://github.com/Tecnativa/doodba-copier-template/tree/v10.0.5) [![Odoo](https://img.shields.io/badge/odoo-v19.0-a3478a)](https://github.com/odoo/odoo/tree/19.0) [![Deployment data](https://img.shields.io/badge/%F0%9F%8C%90%20prod-manmanufacturing.com-green)](http://manmanufacturing.com) [![Deployment data](https://img.shields.io/badge/%E2%9A%92%20demo-test.manmanufacturing.com-yellow)](http://test.manmanufacturing.com) diff --git a/common.yaml b/common.yaml index 097fc28..2c4c1d5 100644 --- a/common.yaml +++ b/common.yaml @@ -8,11 +8,13 @@ services: ODOO_VERSION: "19.0" UID: "${UID:-1000}" GID: "${GID:-1000}" + GITHUB_TOKEN: "${DOODBA_GITHUB_TOKEN:-}" environment: EMAIL_FROM: "notifications@manmanufacturing.com" PGDATABASE: &dbname prod PGUSER: &dbuser "odoo" PROXY_MODE: "true" + GITHUB_TOKEN: "${DOODBA_GITHUB_TOKEN:-}" tty: true volumes: - filestore:/var/lib/odoo @@ -22,7 +24,15 @@ services: traefik.docker.network: "inverseproxy_shared" traefik.frontend.passHostHeader: "true" traefik.port: "8069" - + gatekeeper: + image: ghcr.io/tecnativa/egress-gatekeeper:latest + cap_add: + - NET_ADMIN + labels: + traefik.backend.buffering.retryExpression: IsNetworkError() && Attempts() < 5 + traefik.docker.network: "inverseproxy_shared" + traefik.frontend.passHostHeader: "true" + traefik.port: "8069" db: image: ghcr.io/tecnativa/postgres-autoconf:18-alpine shm_size: 4gb @@ -34,6 +44,13 @@ services: work_mem = 512MB volumes: - db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -q -d $$POSTGRES_DB -U $$POSTGRES_USER"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 15s + start_interval: 1s smtpfake: image: docker.io/axllent/mailpit diff --git a/devel.yaml b/devel.yaml index b2788a7..6d7709b 100644 --- a/devel.yaml +++ b/devel.yaml @@ -12,7 +12,7 @@ services: - "127.0.0.1:${PORT_PREFIX:-19}072:8072" environment: PORT: "6899 8069 8072" - TARGET: odoo + TARGET: gatekeeper odoo: extends: @@ -20,7 +20,7 @@ services: service: odoo build: args: - # To aggregate in development, use `setup-devel.yaml` + # To aggregate in development, run the devel-setup service AGGREGATE: "false" # Export these variables to own files created by odoo in your filesystem UID: "${UID:-1000}" @@ -48,13 +48,15 @@ services: - ./odoo/custom:/opt/odoo/custom:ro,z - ./odoo/auto:/opt/odoo/auto:rw,z depends_on: - - db - - smtp - - wdb - - proxy_general - networks: - default: - whitelist: + db: + condition: service_healthy + smtp: + condition: service_started + wdb: + condition: service_started + gatekeeper: + condition: service_healthy + network_mode: service:gatekeeper command: - odoo - --limit-memory-soft=0 @@ -64,6 +66,34 @@ services: - --dev=reload,qweb,werkzeug,xml,access - --from-filter=manmanufacturing.com + devel-setup: + profiles: ["devel-setup"] + extends: odoo + entrypoint: autoaggregate + network_mode: host + environment: + DEPTH_DEFAULT: 100 + # XXX Export these variables before running setup to own the files + UID: "${DOODBA_GITAGGREGATE_UID:-1000}" + GID: "${DOODBA_GITAGGREGATE_GID:-1000}" + UMASK: "$DOODBA_UMASK" + depends_on: !override [] + user: root + volumes: !override + - ./odoo/custom/src:/opt/odoo/custom/src:rw,z + gatekeeper: + extends: + file: common.yaml + service: gatekeeper + environment: + REFRESH_INTERVAL: 300 + ALLOWED_HOSTS: >- + cdnjs.cloudflare.com fonts.googleapis.com fonts.gstatic.com www.google.com + www.googleapis.com www.gravatar.com + networks: + default: + public: + db: extends: file: common.yaml @@ -71,11 +101,6 @@ services: environment: POSTGRES_DB: *dbname POSTGRES_PASSWORD: odoopassword - healthcheck: - test: ["CMD-SHELL", "pg_isready -q -d $$POSTGRES_DB -U $$POSTGRES_USER"] - interval: 5s - timeout: 5s - retries: 5 pgweb: image: docker.io/sosedoff/pgweb @@ -137,49 +162,10 @@ services: - "127.0.0.1:${PORT_PREFIX:-19}984:1984" # HACK https://github.com/Kozea/wdb/issues/136 init: true - - odoo_net_setup: - image: ghcr.io/tecnativa/docker-whitelist-gateway-service:edge - network_mode: "service:odoo" - cap_add: - - NET_ADMIN - depends_on: - - odoo - - proxy_general - healthcheck: - disable: true - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - environment: - MODE_RUN: "net_setup" - GATEWAY_NAME: "proxy_general" - DNS_LOCAL: "1" - DNS_INTERNAL_FROM_DOCKER: "1" - DOCKER_PROJECT: "${COMPOSE_PROJECT_NAME:-man18}" - DOCKER_PRIMARY_SERVICE: "odoo" - ALLOWED_HOSTS: &proxy_allowed_hosts >- - cdnjs.cloudflare.com fonts.googleapis.com fonts.gstatic.com www.google.com - www.googleapis.com www.gravatar.com - - proxy_general: - image: ghcr.io/tecnativa/docker-whitelist-gateway-service:edge - cap_add: - - NET_ADMIN - sysctls: - net.ipv4.ip_forward: "1" - networks: - whitelist: - aliases: - - proxy_general - public: - environment: - MODE_RUN: "gateway" - ALLOWED_HOSTS: *proxy_allowed_hosts networks: default: internal: ${DOODBA_NETWORK_INTERNAL-true} public: - whitelist: volumes: filestore: diff --git a/migration.yaml b/migration.yaml index 3dfec66..15adbb8 100644 --- a/migration.yaml +++ b/migration.yaml @@ -11,7 +11,7 @@ services: LIST_DB: "true" ODOO_BUS_PUBLIC_SAMESITE_WS: 1 # To install demo data export DOODBA_WITHOUT_DEMO=false - WITHOUT_DEMO: "${DOODBA_WITHOUT_DEMO-all}" + WITHOUT_DEMO: "${DOODBA_WITHOUT_DEMO-True}" SMTP_PORT: "1025" SMTP_SERVER: smtplocal # Just in case you use queue_job @@ -19,20 +19,39 @@ services: PGHOST: man18-19-0-test-db restart: unless-stopped depends_on: - - db - - smtp - - proxy_general - networks: - default: - globalwhitelist_shared: - whitelist: + db: + condition: service_healthy + smtp: + condition: service_started + gatekeeper: + condition: service_healthy + network_mode: service:gatekeeper labels: doodba.domain.main: "" command: - odoo - --workers=3 - --max-cron-threads=1 - + gatekeeper: + extends: + file: common.yaml + service: gatekeeper + environment: + REFRESH_INTERVAL: 300 + ALLOWED_HOSTS: >- + accounts.google.com api.openrouteservice.org api.unsplash.com apis.google.com + cdnjs.cloudflare.com download.geonames.org ec.europa.eu fonts.cdnfonts.com + fonts.googleapis.com fonts.gstatic.com images.unsplash.com iap-services.odoo.com + maps.googleapis.com media-api.odoo.com olg.api.odoo.com www.google.com + www.googleapis.com www.gravatar.com undraw.co updates.maxmind.com + www.ecb.europa.eu www.xe.com sis-t.redsys.es prewww1.aeat.es + www2.agenciatributaria.gob.es www.openstreetmap.org nominatim.openstreetmap.org + www.bizkaia.eus sii.araba.eus vies.api.odoo.com + networks: + default: + public: + labels: + doodba.domain.main: "" db: extends: file: common.yaml @@ -64,64 +83,15 @@ services: command: - test -r /etc/mailhog/auth && export MH_AUTH_FILE=/etc/mailhog/auth; exec MailHog - odoo_net_setup: - image: ghcr.io/tecnativa/docker-whitelist-gateway-service:edge - network_mode: "service:odoo" - cap_add: - - NET_ADMIN - depends_on: - - odoo - - proxy_general - healthcheck: - disable: true - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - environment: - MODE_RUN: "net_setup" - GATEWAY_NAME: "proxy_general" - DNS_LOCAL: "1" - DNS_INTERNAL_FROM_DOCKER: "1" - DOCKER_PROJECT: "${COMPOSE_PROJECT_NAME:-demo-man18-19}" - DOCKER_PRIMARY_SERVICE: "odoo" - ALLOWED_HOSTS: &proxy_allowed_hosts >- - accounts.google.com api.openrouteservice.org api.unsplash.com apis.google.com - cdnjs.cloudflare.com download.geonames.org ec.europa.eu fonts.cdnfonts.com - fonts.googleapis.com fonts.gstatic.com images.unsplash.com iap-services.odoo.com - maps.googleapis.com media-api.odoo.com olg.api.odoo.com www.google.com - www.googleapis.com www.gravatar.com undraw.co updates.maxmind.com - www.ecb.europa.eu www.xe.com sis-t.redsys.es prewww1.aeat.es - www2.agenciatributaria.gob.es www.openstreetmap.org nominatim.openstreetmap.org - www.bizkaia.eus sii.araba.eus vies.api.odoo.com - - proxy_general: - image: ghcr.io/tecnativa/docker-whitelist-gateway-service:edge - restart: unless-stopped - cap_add: - - NET_ADMIN - sysctls: - net.ipv4.ip_forward: "1" - networks: - whitelist: - aliases: - - proxy_general - public: - environment: - MODE_RUN: "gateway" - ALLOWED_HOSTS: *proxy_allowed_hosts - networks: default: internal: ${DOODBA_NETWORK_INTERNAL-true} driver_opts: encrypted: 1 - globalwhitelist_shared: - external: true - inverseproxy_shared: external: true public: - whitelist: volumes: filestore: diff --git a/setup-devel.yaml b/setup-devel.yaml deleted file mode 100644 index aad7917..0000000 --- a/setup-devel.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Use this environment to download all repositories from `repos.yaml` file: -# -# export DOODBA_GITAGGREGATE_UID="$(id -u $USER)" DOODBA_GITAGGREGATE_GID="$(id -g $USER)" DOODBA_UMASK="$(umask)" -# docker-compose -f setup-devel.yaml run --rm odoo -# -# You can clean your git project before if you want to have all really clean: -# -# git clean -ffd - -services: - odoo: - image: ghcr.io/pyxiris/man18:19.0 - build: - context: ./odoo - args: - AGGREGATE: "false" - DEPTH_DEFAULT: 100 - ODOO_VERSION: "19.0" - PYTHONOPTIMIZE: "" - PIP_INSTALL_ODOO: "false" - CLEAN: "false" - COMPILE: "false" - UID: "${UID:-1000}" - GID: "${GID:-1000}" - networks: - - public - volumes: - - ./odoo/custom/src:/opt/odoo/custom/src:rw,z - environment: - DEPTH_DEFAULT: 100 - # XXX Export these variables before running setup to own the files - UID: "${DOODBA_GITAGGREGATE_UID:-1000}" - GID: "${DOODBA_GITAGGREGATE_GID:-1000}" - UMASK: "$DOODBA_UMASK" - user: root - entrypoint: autoaggregate - -networks: - public: diff --git a/tasks.py b/tasks.py index a4a9adf..3af4af6 100644 --- a/tasks.py +++ b/tasks.py @@ -30,6 +30,11 @@ except ImportError: from invoke.util import yaml +yaml.SafeLoader.add_constructor( + "!override", + lambda loader, node: loader.construct_sequence(node), +) + PROJECT_ROOT = Path(__file__).parent.absolute() SRC_PATH = PROJECT_ROOT / "odoo" / "custom" / "src" UID_ENV = { @@ -47,6 +52,10 @@ ), } ) +yaml.SafeLoader.add_constructor( + "!override", + lambda loader, node: loader.construct_sequence(node), +) SERVICES_WAIT_TIME = int(os.environ.get("SERVICES_WAIT_TIME", 4)) ODOO_VERSION = float( yaml.safe_load((PROJECT_ROOT / "common.yaml").read_text())["services"]["odoo"][ @@ -636,7 +645,7 @@ def git_aggregate(c): """ with c.cd(str(PROJECT_ROOT)): c.run( - DOCKER_COMPOSE_CMD + " --file setup-devel.yaml run --rm -T odoo", + DOCKER_COMPOSE_CMD + " run --rm -T devel-setup", env=UID_ENV, ) write_code_workspace_file(c) @@ -669,10 +678,13 @@ def img_build(c, pull=True): @task() -def img_pull(c): +def img_pull(c, ignore_buildable=True): """Pull docker images.""" + cmd = DOCKER_COMPOSE_CMD + " pull" + if ignore_buildable: + cmd += " --ignore-buildable" with c.cd(str(PROJECT_ROOT)): - c.run(DOCKER_COMPOSE_CMD + " pull", pty=True) + c.run(cmd, pty=True) @task() @@ -1123,6 +1135,29 @@ def stop(c, purge=False): c.run(cmd, pty=True) +def _db_run_cmd(odoo_command): + """Return the compose command running a DB management command in odoo.""" + cmd = f"{DOCKER_COMPOSE_CMD} run --rm -l traefik.enable=false" + if ODOO_VERSION >= 20: + # `odoo db` rejects the `--load-language` the entrypoint adds for a missing DB + cmd += " -e INITIAL_LANG=" + return f"{cmd} odoo {odoo_command}" + + +def _dropdb_cmd(dbname): + """Return the command to drop a DB and its filestore.""" + if ODOO_VERSION >= 20: + return _db_run_cmd(f"odoo db drop {dbname}") + return _db_run_cmd(f"click-odoo-dropdb {dbname}") + + +def _copydb_cmd(source_db, destination_db): + """Return the command to copy a DB and its filestore.""" + if ODOO_VERSION >= 20: + return _db_run_cmd(f"odoo db duplicate {source_db} {destination_db}") + return _db_run_cmd(f"click-odoo-copydb {source_db} {destination_db}") + + @task( help={ "dbname": "The DB that will be DESTROYED and recreated. Default: 'devel'.", @@ -1135,6 +1170,7 @@ def stop(c, purge=False): " Default: True", "dependencies": "Install only the dependencies of the specified addons." "Default: False", + "demo": "Create the DB with demo data. Default: True.", }, ) def resetdb( @@ -1147,6 +1183,7 @@ def resetdb( dbname="devel", populate=True, dependencies=False, + demo=True, ): """Reset the specified database with the specified modules. @@ -1163,15 +1200,16 @@ def resetdb( c.run(f"{DOCKER_COMPOSE_CMD} stop odoo", pty=True) _run = f"{DOCKER_COMPOSE_CMD} run --rm -l traefik.enable=false odoo" c.run( - f"{_run} click-odoo-dropdb {dbname}", + _dropdb_cmd(dbname), env=UID_ENV, warn=True, pty=True, ) lang = os.getenv("INITIAL_LANG") lang_opt = f" --lang {lang}" if lang else "" + demo_opt = " --demo" if demo else " --no-demo" c.run( - f"{_run} click-odoo-initdb -n {dbname} -m {modules}{lang_opt}", + f"{_run} click-odoo-initdb -n {dbname} -m {modules}{lang_opt}{demo_opt}", env=UID_ENV, pty=True, ) @@ -1279,16 +1317,15 @@ def snapshot( ): """Snapshot current database and filestore. - Uses click-odoo-copydb behind the scenes to make a snapshot. + Uses the DB management commands of the odoo image behind the scenes. """ if not destination_db: destination_db = f"{source_db}-{datetime.now().strftime('%Y_%m_%d-%H_%M')}" with c.cd(str(PROJECT_ROOT)): cur_state = c.run(f"{DOCKER_COMPOSE_CMD} stop odoo db", pty=True).stdout _logger.info("Snapshoting current %s DB to %s", (source_db, destination_db)) - _run = f"{DOCKER_COMPOSE_CMD} run --rm -l traefik.enable=false odoo" c.run( - f"{_run} click-odoo-copydb {source_db} {destination_db}", + _copydb_cmd(source_db, destination_db), env=UID_ENV, pty=True, ) @@ -1312,7 +1349,7 @@ def restore_snapshot( ): """Restore database and filestore snapshot. - Uses click-odoo-copydb behind the scenes to restore a DB snapshot. + Uses the DB management commands of the odoo image behind the scenes. """ with c.cd(str(PROJECT_ROOT)): cur_state = c.run(f"{DOCKER_COMPOSE_CMD} stop odoo db", pty=True).stdout @@ -1343,15 +1380,14 @@ def restore_snapshot( f"No snapshot found for destination_db {destination_db}" ) _logger.info("Restoring snapshot %s to %s", (snapshot_name, destination_db)) - _run = f"{DOCKER_COMPOSE_CMD} run --rm -l traefik.enable=false odoo" c.run( - f"{_run} click-odoo-dropdb {destination_db}", + _dropdb_cmd(destination_db), env=UID_ENV, warn=True, pty=True, ) c.run( - f"{_run} click-odoo-copydb {snapshot_name} {destination_db}", + _copydb_cmd(snapshot_name, destination_db), env=UID_ENV, pty=True, ) diff --git a/test.yaml b/test.yaml index 87062a3..d4fb0f0 100644 --- a/test.yaml +++ b/test.yaml @@ -11,23 +11,47 @@ services: LIST_DB: "true" ODOO_BUS_PUBLIC_SAMESITE_WS: 1 # To install demo data export DOODBA_WITHOUT_DEMO=false - WITHOUT_DEMO: "${DOODBA_WITHOUT_DEMO-all}" + WITHOUT_DEMO: "${DOODBA_WITHOUT_DEMO-True}" SMTP_PORT: "1025" SMTP_SERVER: smtplocal # Just in case you use queue_job ODOO_QUEUE_JOB_CHANNELS: "root:1" PGHOST: man18-19-0-test-db restart: unless-stopped - hostname: "test.manmanufacturing.com" depends_on: - - db - - smtp - - proxy_general + db: + condition: service_healthy + smtp: + condition: service_started + gatekeeper: + condition: service_healthy + network_mode: service:gatekeeper + labels: + doodba.domain.main: "test.manmanufacturing.com" + command: + - odoo + - --workers=3 + - --max-cron-threads=1 + gatekeeper: + extends: + file: common.yaml + service: gatekeeper + hostname: "test.manmanufacturing.com" + environment: + REFRESH_INTERVAL: 300 + ALLOWED_HOSTS: >- + accounts.google.com api.openrouteservice.org api.unsplash.com apis.google.com + cdnjs.cloudflare.com download.geonames.org ec.europa.eu fonts.cdnfonts.com + fonts.googleapis.com fonts.gstatic.com images.unsplash.com iap-services.odoo.com + maps.googleapis.com media-api.odoo.com olg.api.odoo.com www.google.com + www.googleapis.com www.gravatar.com undraw.co updates.maxmind.com + www.ecb.europa.eu www.xe.com sis-t.redsys.es prewww1.aeat.es + www2.agenciatributaria.gob.es www.openstreetmap.org nominatim.openstreetmap.org + www.bizkaia.eus sii.araba.eus vies.api.odoo.com networks: default: - globalwhitelist_shared: + public: inverseproxy_shared: - whitelist: labels: doodba.domain.main: "test.manmanufacturing.com" traefik.enable: "true" @@ -92,11 +116,6 @@ services: traefik.http.routers.man18-19-0-test-redirect-secure-1.priority: 10 traefik.http.routers.man18-19-0-test-redirect-secure-1.tls: "true" traefik.http.routers.man18-19-0-test-redirect-secure-1.tls.certResolver: letsencrypt - command: - - odoo - - --workers=3 - - --max-cron-threads=1 - db: extends: file: common.yaml @@ -161,64 +180,15 @@ services: command: - test -r /etc/mailhog/auth && export MH_AUTH_FILE=/etc/mailhog/auth; exec MailHog - odoo_net_setup: - image: ghcr.io/tecnativa/docker-whitelist-gateway-service:edge - network_mode: "service:odoo" - cap_add: - - NET_ADMIN - depends_on: - - odoo - - proxy_general - healthcheck: - disable: true - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - environment: - MODE_RUN: "net_setup" - GATEWAY_NAME: "proxy_general" - DNS_LOCAL: "1" - DNS_INTERNAL_FROM_DOCKER: "1" - DOCKER_PROJECT: "${COMPOSE_PROJECT_NAME:-demo-man18-19}" - DOCKER_PRIMARY_SERVICE: "odoo" - ALLOWED_HOSTS: &proxy_allowed_hosts >- - accounts.google.com api.openrouteservice.org api.unsplash.com apis.google.com - cdnjs.cloudflare.com download.geonames.org ec.europa.eu fonts.cdnfonts.com - fonts.googleapis.com fonts.gstatic.com images.unsplash.com iap-services.odoo.com - maps.googleapis.com media-api.odoo.com olg.api.odoo.com www.google.com - www.googleapis.com www.gravatar.com undraw.co updates.maxmind.com - www.ecb.europa.eu www.xe.com sis-t.redsys.es prewww1.aeat.es - www2.agenciatributaria.gob.es www.openstreetmap.org nominatim.openstreetmap.org - www.bizkaia.eus sii.araba.eus vies.api.odoo.com - - proxy_general: - image: ghcr.io/tecnativa/docker-whitelist-gateway-service:edge - restart: unless-stopped - cap_add: - - NET_ADMIN - sysctls: - net.ipv4.ip_forward: "1" - networks: - whitelist: - aliases: - - proxy_general - public: - environment: - MODE_RUN: "gateway" - ALLOWED_HOSTS: *proxy_allowed_hosts - networks: default: internal: ${DOODBA_NETWORK_INTERNAL-true} driver_opts: encrypted: 1 - globalwhitelist_shared: - external: true - inverseproxy_shared: external: true public: - whitelist: volumes: filestore: