Enhance SENAITE configuration and performance tuning in Docker

.github/workflows/build.yml

@@ -19,6 +19,8 @@ jobs:
       DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_REGISTRY_USERNAME }}
       DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_REGISTRY_PASSWORD }}
   build-publish-sso:
+    # senaite-sso is built FROM the base image, so wait for it to be published
+    needs: build-publish
     uses: mekomsolutions/shared-github-workflow/.github/workflows/docker-build-publish.yml@main
     with:
       context: senaite-sso/

senaite-sso/Dockerfile

@@ -1,11 +1,21 @@
 FROM mekomsolutions/senaite
 
 # Copy Buildout
+# known-good-versions.cfg comes from the base image
 COPY requirements.txt buildout.cfg develop.cfg $SENAITE_INSTANCE_HOME/
 
 # Copy the build dependencies and startup scripts
 COPY build_deps.txt run_deps.txt  /
 
+# Debian buster is EOL: its repositories moved to archive.debian.org
+# (repeated here in case the base image predates this fix)
+RUN sed -i \
+    -e 's|deb.debian.org/debian|archive.debian.org/debian|g' \
+    -e 's|security.debian.org/debian-security|archive.debian.org/debian-security|g' \
+    -e '/buster-updates/d' \
+    /etc/apt/sources.list \
+  && echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99archive
+
 # Install moreutils and gettext
 RUN apt-get update \
     && apt-get install -y --no-install-recommends moreutils gettext \

senaite-sso/buildout.cfg

@@ -1,7 +1,8 @@
 [buildout]
-index = https://pypi.python.org/simple/
+index = https://pypi.org/simple/
 extends =
   buildout-base.cfg
+  known-good-versions.cfg
 extensions = mr.developer
 
 var-dir=/data

senaite-sso/docker-entrypoint.sh

@@ -18,8 +18,15 @@ for f in /data/**/*.json; do \
     envsubst < "$f" | sponge "$f"; \
 done
 
-if [ -e "custom.cfg" ]; then
+# Opt-in: skip re-running buildout when custom.cfg is unchanged since the last
+# successful run (note: plonesite profiles are then not re-applied on restart)
+BUILDOUT_STAMP=".custom.cfg.sha256"
+if [ -e "custom.cfg" ] && [[ "${BUILDOUT_SKIP_IF_UNCHANGED,,}" == "true" ]] \
+  && sha256sum --status -c "$BUILDOUT_STAMP" 2>/dev/null; then
+  echo "custom.cfg unchanged, skipping buildout"
+elif [ -e "custom.cfg" ]; then
   buildout -c custom.cfg
+  sha256sum custom.cfg > "$BUILDOUT_STAMP"
   find /data  -not -user senaite -exec chown senaite:senaite {} \+
   find /home/senaite -not -user senaite -exec chown senaite:senaite {} \+
   gosu senaite python /docker-initialize.py

senaite-sso/docker-initialize.py

@@ -13,14 +13,16 @@ def __init__(
         custom_conf="/home/senaite/senaitelims/custom.cfg",
         zeopack_conf="/home/senaite/senaitelims/bin/zeopack",
         zeoserver_conf="/home/senaite/senaitelims/parts/zeoserver/etc/zeo.conf",
-        cors_conf="/home/senaite/senaitelims/parts/instance/etc/package-includes/999-additional-overrides.zcml"
+        cors_conf="/home/senaite/senaitelims/parts/instance/etc/package-includes/999-additional-overrides.zcml",
+        wsgi_conf="/home/senaite/senaitelims/parts/instance/etc/wsgi.ini"
     ):
         self.env = env
         self.zope_conf = zope_conf
         self.custom_conf = custom_conf
         self.zeopack_conf = zeopack_conf
         self.zeoserver_conf = zeoserver_conf
         self.cors_conf = cors_conf
+        self.wsgi_conf = wsgi_conf
 
     def zeoclient(self):
         """ ZEO Client
@@ -94,6 +96,27 @@ def zeoserver(self):
             with open(self.zeoserver_conf, 'w') as cfile:
                 cfile.write(text)
 
+    def performance(self):
+        """ Zope performance tuning (only applied when the variables are set)
+        """
+        zodb_cache_size = self.env.get("ZODB_CACHE_SIZE", "").strip()
+        if zodb_cache_size and os.path.exists(self.zope_conf):
+            with open(self.zope_conf, "r") as cfile:
+                config = cfile.read()
+            pattern = re.compile(r"(<zodb_db main>.*?cache-size\s+)\d+", re.DOTALL)
+            config = re.sub(pattern, r"\g<1>%s" % zodb_cache_size, config, count=1)
+            with open(self.zope_conf, "w") as cfile:
+                cfile.write(config)
+
+        wsgi_threads = self.env.get("WSGI_THREADS", "").strip()
+        if wsgi_threads and os.path.exists(self.wsgi_conf):
+            with open(self.wsgi_conf, "r") as cfile:
+                config = cfile.read()
+            pattern = re.compile(r"^threads\s*=\s*\d+", re.MULTILINE)
+            config = re.sub(pattern, "threads = %s" % wsgi_threads, config, count=1)
+            with open(self.wsgi_conf, "w") as cfile:
+                cfile.write(config)
+
     def cors(self):
         """ Configure CORS Policies
         """
@@ -285,6 +308,7 @@ def setup(self, **kwargs):
         self.zeoclient()
         self.zeopack()
         self.zeoserver()
+        self.performance()
 
     __call__ = setup
 

senaite/Dockerfile

@@ -25,20 +25,28 @@ RUN useradd --system -m -d $SENAITE_HOME -U -u 500 $SENAITE_USER
 RUN mkdir -p $SENAITE_INSTANCE_HOME $SENAITE_FILESTORAGE $SENAITE_BLOBSTORAGE
 
 # Copy Buildout
-COPY resources/requirements.txt resources/versions.cfg resources/buildout.cfg resources/develop.cfg $SENAITE_INSTANCE_HOME/
+COPY resources/requirements.txt resources/versions.cfg resources/known-good-versions.cfg resources/buildout.cfg resources/develop.cfg $SENAITE_INSTANCE_HOME/
 
 #RUN chown -R senaite:senaite $SENAITE_INSTANCE_HOME $SENAITE_FILESTORAGE $SENAITE_BLOBSTORAGE
 
 # Copy the build dependencies and startup scripts
 COPY resources/build_deps.txt resources/run_deps.txt /
 
+# Debian buster is EOL: its repositories moved to archive.debian.org
+RUN sed -i \
+    -e 's|deb.debian.org/debian|archive.debian.org/debian|g' \
+    -e 's|security.debian.org/debian-security|archive.debian.org/debian-security|g' \
+    -e '/buster-updates/d' \
+    /etc/apt/sources.list \
+  && echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99archive
+
 # Note: we concatenate all commands to avoid multiple layer generation and reduce the image size
 RUN apt-get update \
   # Install system packages
   && apt-get install -y --no-install-recommends $(grep -vE "^\s*#" /build_deps.txt | tr "\n" " ") \
   && apt-get install -y --no-install-recommends $(grep -vE "^\s*#" /run_deps.txt | tr "\n" " ") \
   # Fetch unified installer
-  && wget -O Plone.tgz https://launchpad.net/plone/$PLONE_MAJOR/$PLONE_VERSION/+download/$PLONE_UNIFIED_INSTALLER.tgz \
+  && wget --tries=5 --retry-connrefused --waitretry=10 --timeout=60 -O Plone.tgz https://launchpad.net/plone/$PLONE_MAJOR/$PLONE_VERSION/+download/$PLONE_UNIFIED_INSTALLER.tgz \
   && echo "$PLONE_MD5 Plone.tgz" | md5sum -c - \
   && tar -xzf /Plone.tgz \
   && cp -rv $PLONE_UNIFIED_INSTALLER/base_skeleton/* $SENAITE_INSTANCE_HOME \

senaite/README.md

@@ -25,6 +25,10 @@ Additionally, the image will accept the following environment variable when run:
 | `ADMIN_USER`                       |         | Name of the administrator user   
 | `ADMIN_PASSWORD`                      |  | Password of the administrator  user name                        
 | `SITE` |         | The 'site' name, eg, 'clinic1' |
+| `ZODB_CACHE_SIZE` | `30000` | Number of objects kept in the ZODB cache per connection. Raising it (e.g. `100000`–`300000`) speeds up large SENAITE sites at the cost of RAM |
+| `WSGI_THREADS` | `4` | Number of WSGI worker threads per instance (each thread holds its own ZODB cache) |
+| `ZEO_CLIENT_CACHE_SIZE` | `128MB` | ZEO client cache size (only when `ZEO_ADDRESS` is set) |
+| `BUILDOUT_SKIP_IF_UNCHANGED` | `false` | When `true`, skip re-running buildout at startup if `custom.cfg` has not changed since the last successful run. Faster restarts, but `plonesite` profiles are no longer re-applied on every restart |
 
 ### Volumes
 

senaite/resources/buildout.cfg

@@ -1,7 +1,8 @@
 [buildout]
-index = https://pypi.python.org/simple/
+index = https://pypi.org/simple/
 extends =
   buildout-base.cfg
+  known-good-versions.cfg
 extensions = mr.developer
 
 var-dir=/data

senaite/resources/docker-entrypoint.sh

@@ -28,8 +28,15 @@ function git_fixture {
 # https://github.com/senaite/senaite.docker/issues/17
 git_fixture
 
-if [ -e "custom.cfg" ]; then
+# Opt-in: skip re-running buildout when custom.cfg is unchanged since the last
+# successful run (note: plonesite profiles are then not re-applied on restart)
+BUILDOUT_STAMP=".custom.cfg.sha256"
+if [ -e "custom.cfg" ] && [[ "${BUILDOUT_SKIP_IF_UNCHANGED,,}" == "true" ]] \
+  && sha256sum --status -c "$BUILDOUT_STAMP" 2>/dev/null; then
+  echo "custom.cfg unchanged, skipping buildout"
+elif [ -e "custom.cfg" ]; then
     buildout -c custom.cfg
+    sha256sum custom.cfg > "$BUILDOUT_STAMP"
     find /data  -not -user senaite -exec chown senaite:senaite {} \+
     find /home/senaite -not -user senaite -exec chown senaite:senaite {} \+
     gosu senaite python /docker-initialize.py

senaite/resources/docker-initialize.py

@@ -13,14 +13,16 @@ def __init__(
             custom_conf="/home/senaite/senaitelims/custom.cfg",
             zeopack_conf="/home/senaite/senaitelims/bin/zeopack",
             zeoserver_conf="/home/senaite/senaitelims/parts/zeoserver/etc/zeo.conf",
-            cors_conf="/home/senaite/senaitelims/parts/instance/etc/package-includes/999-additional-overrides.zcml"
+            cors_conf="/home/senaite/senaitelims/parts/instance/etc/package-includes/999-additional-overrides.zcml",
+            wsgi_conf="/home/senaite/senaitelims/parts/instance/etc/wsgi.ini"
     ):
         self.env = env
         self.zope_conf = zope_conf
         self.custom_conf = custom_conf
         self.zeopack_conf = zeopack_conf
         self.zeoserver_conf = zeoserver_conf
         self.cors_conf = cors_conf
+        self.wsgi_conf = wsgi_conf
 
     def zeoclient(self):
         """ ZEO Client
@@ -94,6 +96,27 @@ def zeoserver(self):
             with open(self.zeoserver_conf, 'w') as cfile:
                 cfile.write(text)
 
+    def performance(self):
+        """ Zope performance tuning (only applied when the variables are set)
+        """
+        zodb_cache_size = self.env.get("ZODB_CACHE_SIZE", "").strip()
+        if zodb_cache_size and os.path.exists(self.zope_conf):
+            with open(self.zope_conf, "r") as cfile:
+                config = cfile.read()
+            pattern = re.compile(r"(<zodb_db main>.*?cache-size\s+)\d+", re.DOTALL)
+            config = re.sub(pattern, r"\g<1>%s" % zodb_cache_size, config, count=1)
+            with open(self.zope_conf, "w") as cfile:
+                cfile.write(config)
+
+        wsgi_threads = self.env.get("WSGI_THREADS", "").strip()
+        if wsgi_threads and os.path.exists(self.wsgi_conf):
+            with open(self.wsgi_conf, "r") as cfile:
+                config = cfile.read()
+            pattern = re.compile(r"^threads\s*=\s*\d+", re.MULTILINE)
+            config = re.sub(pattern, "threads = %s" % wsgi_threads, config, count=1)
+            with open(self.wsgi_conf, "w") as cfile:
+                cfile.write(config)
+
     def cors(self):
         """ Configure CORS Policies
         """
@@ -277,6 +300,7 @@ def setup(self, **kwargs):
         self.zeoclient()
         self.zeopack()
         self.zeoserver()
+        self.performance()
 
     __call__ = setup