From aea2bee41534b0442ff6379c52e973870b901c74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:18:42 +0000 Subject: [PATCH 1/6] Bump vite in /src/apps/dashboard/ui in the dashboard-versions group Bumps the dashboard-versions group in /src/apps/dashboard/ui with 1 update: [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite). Updates `vite` from 8.3.3 to 8.3.4 - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v8.3.4/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-version: 8.3.4 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: dashboard-versions ... Signed-off-by: dependabot[bot] --- src/apps/dashboard/ui/package-lock.json | 28 ++++++++++++------------- src/apps/dashboard/ui/package.json | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/apps/dashboard/ui/package-lock.json b/src/apps/dashboard/ui/package-lock.json index f7158ad..7f47b8c 100644 --- a/src/apps/dashboard/ui/package-lock.json +++ b/src/apps/dashboard/ui/package-lock.json @@ -15,7 +15,7 @@ "@vitejs/plugin-react": "6.1.2", "bootstrap": "5.3.8", "echarts": "6.1.0", - "vite": "8.3.3" + "vite": "8.3.4" }, "engines": { "node": ">=22.12" @@ -708,9 +708,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.19", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", - "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", "dev": true, "funding": [ { @@ -747,9 +747,9 @@ } }, "node_modules/postcss": { - "version": "8.5.28", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", - "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "version": "8.5.29", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.29.tgz", + "integrity": "sha512-49cGhUbXj8Qenv0iTMxA1cFBzxXoctpC9Ujd77t1WcbJIr6nF/eI7g/8MgxrYldFRuAXvja7xQRwavoW7kgrxQ==", "dev": true, "funding": [ { @@ -767,9 +767,9 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.18", + "nanoid": "^3.3.19", "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" + "source-map-js": "^1.2.2" }, "engines": { "node": "^10 || ^12 || >=14" @@ -864,16 +864,16 @@ } }, "node_modules/vite": { - "version": "8.3.3", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.3.tgz", - "integrity": "sha512-cTAldKPImjg6c+gk48U19POPn3GCBzZwpdsN8ZMEEcbpes+6/wvfqUd0C2y3qYY4wsj8PwgFwrZ/1jBPVttMSg==", + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.4.tgz", + "integrity": "sha512-IftLE8phJhdNRIGNEV4JCCvfNScoZQVfcXaN8TQropWGJtOHcu+iAB9pHcuGvmEb/kddw0LBe/jS8ztdC6xdiQ==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", - "postcss": "^8.5.28", - "rolldown": "~1.2.11", + "postcss": "^8.5.29", + "rolldown": "~1.2.12", "tinyglobby": "^0.2.17" }, "bin": { diff --git a/src/apps/dashboard/ui/package.json b/src/apps/dashboard/ui/package.json index f6e482c..de1bfaf 100644 --- a/src/apps/dashboard/ui/package.json +++ b/src/apps/dashboard/ui/package.json @@ -20,7 +20,7 @@ "@vitejs/plugin-react": "6.1.2", "bootstrap": "5.3.8", "echarts": "6.1.0", - "vite": "8.3.3" + "vite": "8.3.4" }, "engines": { "node": ">=22.12" From d1408bf8c0473f311b8b332ca5dead5adbd6de04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 06:36:11 +0000 Subject: [PATCH 2/6] Bump the python-versions group across 1 directory with 7 updates Bumps the python-versions group with 7 updates in the /src/apps/mcp directory: | Package | From | To | | --- | --- | --- | | [mcp](https://github.com/modelcontextprotocol/python-sdk) | `2.2.0` | `2.3.0` | | [mcp-types](https://github.com/modelcontextprotocol/python-sdk) | `2.2.0` | `2.3.0` | | [opentelemetry-api](https://github.com/open-telemetry/opentelemetry-python) | `1.45.0` | `1.45.1` | | [pycparser](https://github.com/eliben/pycparser) | `3.0` | `3.11` | | [pydantic](https://github.com/pydantic/pydantic) | `2.13.5` | `2.14.1` | | [pydantic-core](https://github.com/pydantic/pydantic) | `2.46.5` | `2.50.1` | | [truststore](https://github.com/sethmlarson/truststore) | `0.10.4` | `0.10.5` | Updates `mcp` from 2.2.0 to 2.3.0 - [Release notes](https://github.com/modelcontextprotocol/python-sdk/releases) - [Changelog](https://github.com/modelcontextprotocol/python-sdk/blob/main/RELEASE.md) - [Commits](https://github.com/modelcontextprotocol/python-sdk/compare/v2.2.0...v2.3.0) Updates `mcp-types` from 2.2.0 to 2.3.0 - [Release notes](https://github.com/modelcontextprotocol/python-sdk/releases) - [Changelog](https://github.com/modelcontextprotocol/python-sdk/blob/main/RELEASE.md) - [Commits](https://github.com/modelcontextprotocol/python-sdk/compare/v2.2.0...v2.3.0) Updates `opentelemetry-api` from 1.45.0 to 1.45.1 - [Release notes](https://github.com/open-telemetry/opentelemetry-python/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-python/compare/v1.45.0...v1.45.1) Updates `pycparser` from 3.0 to 3.11 - [Release notes](https://github.com/eliben/pycparser/releases) - [Commits](https://github.com/eliben/pycparser/compare/release_v3.00...release_v3.11) Updates `pydantic` from 2.13.5 to 2.14.1 - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.13.5...v2.14.1) Updates `pydantic-core` from 2.46.5 to 2.50.1 - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/core-v2.46.5...core-v2.50.1) Updates `truststore` from 0.10.4 to 0.10.5 - [Release notes](https://github.com/sethmlarson/truststore/releases) - [Changelog](https://github.com/sethmlarson/truststore/blob/main/CHANGELOG.md) - [Commits](https://github.com/sethmlarson/truststore/compare/v0.10.4...v0.10.5) --- updated-dependencies: - dependency-name: mcp dependency-version: 2.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-versions - dependency-name: mcp-types dependency-version: 2.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-versions - dependency-name: opentelemetry-api dependency-version: 1.45.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-versions - dependency-name: pycparser dependency-version: '3.11' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-versions - dependency-name: pydantic dependency-version: 2.14.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-versions - dependency-name: pydantic-core dependency-version: 2.50.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-versions - dependency-name: truststore dependency-version: 0.10.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-versions ... Signed-off-by: dependabot[bot] --- src/apps/mcp/requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/apps/mcp/requirements.txt b/src/apps/mcp/requirements.txt index d36eb88..36da7dd 100644 --- a/src/apps/mcp/requirements.txt +++ b/src/apps/mcp/requirements.txt @@ -25,13 +25,13 @@ Jinja2==3.1.6 jsonschema-specifications==2025.9.1 jsonschema==4.26.0 MarkupSafe==3.0.4 -mcp==2.2.0 -mcp-types==2.2.0 -opentelemetry-api==1.45.0 +mcp==2.3.0 +mcp-types==2.3.0 +opentelemetry-api==1.45.1 packaging==26.3 -pycparser==3.0 -pydantic==2.13.5 -pydantic_core==2.46.5 +pycparser==3.11 +pydantic==2.14.1 +pydantic_core==2.50.1 PyJWT==2.15.1 python-multipart==0.0.32 referencing==0.37.0 @@ -40,7 +40,7 @@ requests==2.34.2 rpds-py==2026.9.1 sse-starlette==3.5.0 starlette==1.7.0 -truststore==0.10.4 +truststore==0.10.5 typing-inspection==0.4.4 typing_extensions==4.16.0 urllib3==2.8.0 From 80bccbafc09b476ec4b125eea55c1cb75811932e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?eW=C9=98yn?= <5607939+Llewellynvdm@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:28:06 +0200 Subject: [PATCH 3/6] Prepare dashboard dependency update for release 3.2.11 --- .env.example | 2 +- Dockerfile | 2 +- VERSION | 2 +- compose.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index b5599fb..05f757d 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ # Nonempty GETBIBLE_* application settings override values saved by the menu. # Empty optional values leave the saved setting/default in control. GETBIBLE_IMAGE_REPOSITORY=ghcr.io/getbible/api -GETBIBLE_IMAGE_TAG=3.2.10 +GETBIBLE_IMAGE_TAG=3.2.11 GETBIBLE_DATA_ROOT=/srv/getbible-data GETBIBLE_HOSTNAME=getbible TZ=UTC diff --git a/Dockerfile b/Dockerfile index dfafbf4..4244b84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 FROM ubuntu:24.04 -ARG GETBIBLE_VERSION=3.2.10 +ARG GETBIBLE_VERSION=3.2.11 ARG GETBIBLE_REVISION=unknown ARG GETBIBLE_IMAGE_PYTHONS= LABEL org.opencontainers.image.title="getBible API" \ diff --git a/VERSION b/VERSION index f15386a..17ce918 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.10 +3.2.11 diff --git a/compose.yaml b/compose.yaml index bf0534e..c4d1d95 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,7 +4,7 @@ services: getbible: # Production deployment only pulls the prebuilt image. No checkout # or Dockerfile is required on the server. See docs/DOCKER.md for login. - image: ${GETBIBLE_IMAGE_REPOSITORY:-ghcr.io/getbible/api}:${GETBIBLE_IMAGE_TAG:-3.2.10} + image: ${GETBIBLE_IMAGE_REPOSITORY:-ghcr.io/getbible/api}:${GETBIBLE_IMAGE_TAG:-3.2.11} restart: unless-stopped hostname: ${GETBIBLE_HOSTNAME:-getbible} user: "0:0" From 17407735e05455cf08be75859f591016f95f8b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?eW=C9=98yn?= <5607939+Llewellynvdm@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:28:07 +0200 Subject: [PATCH 4/6] Keep MCP SDK pins compatible and prepare release 3.2.12 --- .env.example | 2 +- Dockerfile | 2 +- VERSION | 2 +- compose.yaml | 2 +- src/apps/mcp/requirements.txt | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index b5599fb..0e9412c 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ # Nonempty GETBIBLE_* application settings override values saved by the menu. # Empty optional values leave the saved setting/default in control. GETBIBLE_IMAGE_REPOSITORY=ghcr.io/getbible/api -GETBIBLE_IMAGE_TAG=3.2.10 +GETBIBLE_IMAGE_TAG=3.2.12 GETBIBLE_DATA_ROOT=/srv/getbible-data GETBIBLE_HOSTNAME=getbible TZ=UTC diff --git a/Dockerfile b/Dockerfile index dfafbf4..588fc12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 FROM ubuntu:24.04 -ARG GETBIBLE_VERSION=3.2.10 +ARG GETBIBLE_VERSION=3.2.12 ARG GETBIBLE_REVISION=unknown ARG GETBIBLE_IMAGE_PYTHONS= LABEL org.opencontainers.image.title="getBible API" \ diff --git a/VERSION b/VERSION index f15386a..275e51e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.10 +3.2.12 diff --git a/compose.yaml b/compose.yaml index bf0534e..27f4c33 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,7 +4,7 @@ services: getbible: # Production deployment only pulls the prebuilt image. No checkout # or Dockerfile is required on the server. See docs/DOCKER.md for login. - image: ${GETBIBLE_IMAGE_REPOSITORY:-ghcr.io/getbible/api}:${GETBIBLE_IMAGE_TAG:-3.2.10} + image: ${GETBIBLE_IMAGE_REPOSITORY:-ghcr.io/getbible/api}:${GETBIBLE_IMAGE_TAG:-3.2.12} restart: unless-stopped hostname: ${GETBIBLE_HOSTNAME:-getbible} user: "0:0" diff --git a/src/apps/mcp/requirements.txt b/src/apps/mcp/requirements.txt index 36da7dd..bf81d4f 100644 --- a/src/apps/mcp/requirements.txt +++ b/src/apps/mcp/requirements.txt @@ -25,8 +25,8 @@ Jinja2==3.1.6 jsonschema-specifications==2025.9.1 jsonschema==4.26.0 MarkupSafe==3.0.4 -mcp==2.3.0 -mcp-types==2.3.0 +mcp==2.2.0 +mcp-types==2.2.0 opentelemetry-api==1.45.1 packaging==26.3 pycparser==3.11 From e9bf7c96a9585b93fcaef85e7c608181981dad97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?eW=C9=98yn?= <5607939+Llewellynvdm@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:34:33 +0200 Subject: [PATCH 5/6] Add reviewed hardware realignment and resumable storage accounting --- .env.example | 2 +- Dockerfile | 2 +- README.md | 4 + VERSION | 2 +- compose.yaml | 2 +- docs/DOCKER.md | 8 + docs/RUNTIME_RESOURCES.md | 97 ++++- getbible.sh | 9 +- .../telemetry/getbible_telemetry/collector.py | 46 ++- .../telemetry/getbible_telemetry/health.py | 31 +- .../getbible_telemetry/service_alerts.py | 215 +++++++++++ src/bin/getbible-resources | 267 +++++++++++++- src/bin/getbible-storage-guard | 341 +++++++++++++++--- src/bin/getbible-sync | 6 +- src/lib/config.sh | 1 + src/lib/dashboard.sh | 10 +- src/lib/deployment.sh | 4 +- src/lib/resources.sh | 141 +++++++- src/systemd/getbible-alert@.service.tmpl | 8 +- src/systemd/getbible-storage.service.tmpl | 4 +- src/systemd/getbible-storage.timer.tmpl | 5 +- tests/cli/test_dashboard_management.sh | 8 + tests/cli/test_resource_realign.sh | 103 ++++++ tests/python/test_resources.py | 97 +++++ tests/python/test_service_alerts.py | 207 +++++++++++ tests/python/test_static_sync.py | 20 + tests/python/test_storage_guard.py | 176 +++++++++ 27 files changed, 1742 insertions(+), 74 deletions(-) create mode 100644 src/apps/telemetry/getbible_telemetry/service_alerts.py create mode 100644 tests/cli/test_resource_realign.sh create mode 100644 tests/python/test_service_alerts.py diff --git a/.env.example b/.env.example index 0e9412c..a29c8e8 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ # Nonempty GETBIBLE_* application settings override values saved by the menu. # Empty optional values leave the saved setting/default in control. GETBIBLE_IMAGE_REPOSITORY=ghcr.io/getbible/api -GETBIBLE_IMAGE_TAG=3.2.12 +GETBIBLE_IMAGE_TAG=3.3.0 GETBIBLE_DATA_ROOT=/srv/getbible-data GETBIBLE_HOSTNAME=getbible TZ=UTC diff --git a/Dockerfile b/Dockerfile index 588fc12..514e78e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 FROM ubuntu:24.04 -ARG GETBIBLE_VERSION=3.2.12 +ARG GETBIBLE_VERSION=3.3.0 ARG GETBIBLE_REVISION=unknown ARG GETBIBLE_IMAGE_PYTHONS= LABEL org.opencontainers.image.title="getBible API" \ diff --git a/README.md b/README.md index 3e2e020..646d4e7 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,10 @@ describes the official MCP service at [https://mcp.getbible.net/](https://mcp.ge retention, analytics with total calls and unique callers, Telegram notifications for every change, Cloudflare integration, manager updates from the menu or one command, application updates and generation rollback. +- **Hardware alignment**: `getbible resources realign` reviews effective + CPU/RAM, filesystem headroom, configuration ownership and proposed runtime + allocations. Add `--apply` to activate a freshly checked plan through healthy + runtime generations. See [resource controls](docs/RUNTIME_RESOURCES.md#review-and-realign-hardware-allocation). - **Staged deployments**: a domain can be installed and verified on a server its DNS does not point to yet, with a placeholder certificate in managed TLS mode or the HTTP origin in external TLS mode, and diff --git a/VERSION b/VERSION index 275e51e..15a2799 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.2.12 +3.3.0 diff --git a/compose.yaml b/compose.yaml index 27f4c33..9f85098 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,7 +4,7 @@ services: getbible: # Production deployment only pulls the prebuilt image. No checkout # or Dockerfile is required on the server. See docs/DOCKER.md for login. - image: ${GETBIBLE_IMAGE_REPOSITORY:-ghcr.io/getbible/api}:${GETBIBLE_IMAGE_TAG:-3.2.12} + image: ${GETBIBLE_IMAGE_REPOSITORY:-ghcr.io/getbible/api}:${GETBIBLE_IMAGE_TAG:-3.3.0} restart: unless-stopped hostname: ${GETBIBLE_HOSTNAME:-getbible} user: "0:0" diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 4ac81b7..d8b8750 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -332,8 +332,16 @@ process that reaches a hard ceiling. docker compose exec --user root getbible getbible resources show docker compose exec -T --user root getbible getbible resources show --json docker compose exec --user root getbible getbible resources apply +docker compose exec --user root getbible getbible resources realign +docker compose exec --user root getbible getbible resources realign --apply ``` +`resources realign` adds a read-only hardware review, selected-versus-proposed +settings and current storage/telemetry diagnostics. Its explicit `--apply` +rechecks and applies changed runtime allocations. Environment-owned settings +and outer container limits remain controlled by Compose; the report identifies +these instead of silently replacing them. See [hardware alignment](RUNTIME_RESOURCES.md#review-and-realign-hardware-allocation). + The report distinguishes calculated targets from running generations. Configuration changes recalculate targets; `resources apply` activates them through the normal candidate/readiness/reload workflow. Deploying an additional diff --git a/docs/RUNTIME_RESOURCES.md b/docs/RUNTIME_RESOURCES.md index d212775..7d36eab 100644 --- a/docs/RUNTIME_RESOURCES.md +++ b/docs/RUNTIME_RESOURCES.md @@ -126,6 +126,56 @@ is exactly 30 days. CDN cache hits are not visible in origin traffic statistics. ## Operator controls +### Review and realign hardware allocation + +```sh +getbible resources realign +getbible resources realign --json +getbible resources realign --apply +``` + +The first two commands are read-only. They report effective CPU capacity, +physical and cgroup-limited RAM, available memory, filesystem byte/inode +headroom, configuration ownership, and selected versus proposed runtime +settings. They do not walk the Bible corpus or initialize an installation. +The existing capacity observations also distinguish collection/reclamation +problems from a request for more storage. Missing or stale observations are +reported explicitly. + +Docker uses its finite effective cgroup memory allowance; the container cannot +raise the deployment's CPU or memory ceiling. An existing explicit application +budget is retained. Native installations with `MEMORY_BUDGET=auto` receive a +conservative proposal derived from both total and currently available memory, +leaving at least 25% of effective RAM (with a 256-MiB floor) for the host. +Existing runtime memory is not assumed reclaimable. The runtime allocator +then reserves infrastructure and deployment-overlap capacity within that +application budget. This is a starting allocation, not a throughput benchmark. +An operator can supply a deliberate shared-host budget with +`resources realign --memory-budget 4G` and review it before adding `--apply`. +After changing native hardware, `resources realign --memory-budget auto` +requests a new automatic recommendation instead of retaining a saved budget. + +Every proposal has a plan identifier. `--expected-plan` accepts that identifier +to reject changed hardware, configuration or selected-generation settings. +When replaying a native automatic proposal later, pass its displayed proposed +`--memory-budget` as well as `--expected-plan`; otherwise changed available RAM +may produce a different proposal. The menu preserves both values internally. +Apply rechecks the plan under the manager lock, saves only an editable memory +budget when needed, and applies changed runtime domains through the existing +candidate-readiness, nginx validation/reload and drain transactions. Reductions +are applied before increases. As with `resources apply`, these transactions use +the installed manager's current runtime code and configuration for the selected +domains. A failed candidate retains its serving generation; +the command reports an incomplete application rather than claiming success. +Environment-owned settings must be changed at their deployment source. +Deliberately configured endpoint workers, cache policies and limits remain +subject to their existing precedence and hardware bounds. + +The same review and apply controls are available under Settings > Deployment +settings > Hardware alignment and runtime memory budget. Alignment does not +increase telemetry allowances to conceal failed cleanup, remove data, change +storage admission policy, or change the host's container configuration. + ```sh getbible runtime query.example.org cache v2 info getbible runtime query.example.org cache v2 warm kjv @@ -156,15 +206,40 @@ them. Public query/search HTTP routes never expose these controls. `GETBIBLE_STORAGE_MAX_GIB=0` disables application storage admission. With a positive cap, an independent root timer records managed allocated disk blocks without reading file contents, deduplicating hard-linked releases. It samples -again 30 seconds after the previous sample finishes, independently of resource -applications, and has a 75-second execution timeout. It is inactive when the -storage cap is disabled. Startup obtains the first snapshot before sync services -can publish. +in bounded slices, 30 seconds after the previous invocation finishes, +independently of resource applications. Each slice has a 75-second systemd +deadline and normally yields after 20 seconds. A disk-backed directory queue, +checkpointed totals and hard-link index preserve completed work across timer +runs and process interruption without retaining the corpus's inode identities +in Python memory. Incomplete scans never replace the last complete snapshot. +It exits without scanning when the storage cap is disabled. Startup begins +the first slice; budgeted publication waits until a complete snapshot exists. + +`STORAGE_SCAN_MAX_SECONDS` (default `900`, supported range `120`–`86400`) +limits the wall-clock horizon of a whole scan, including timer waits. Its +deployment override is `GETBIBLE_STORAGE_SCAN_MAX_SECONDS`. Exceeding the +horizon or encountering an unreadable directory records a failure and retains +the last complete accounting. The next invocation starts a new scan. Progress +and the last error are recorded in `progress.json`; scan work is private under +the storage state's `.scan` directory. No partial count authorizes publication. + +A complete snapshot records its start, completion and duration. Freshness is +measured from its start and is bounded by twice the configured scan horizon +plus 40 seconds for timer/scheduling margin. This allows consecutive bounded +scans to complete without the old fixed 120-second freshness window expiring +mid-scan. Legacy snapshots retain the 120-second window. Increasing the scan +horizon therefore also increases the tolerated accounting age; do so only +when the observed corpus traversal needs it. This admission estimate is not +a strict physical disk quota. Trusted sync services reserve conservative projected export space under one shared lock before export, and recheck before publication. Stale accounting or insufficient budget refuses publication and preserves current and rollback -releases. Successful/failed sync exits release their reservation; crashed leases -expire after 24 hours. Snapshot and reservations live in +releases. Failed exports release their reservation. Successful publications +retain a committed charge until a complete snapshot whose scan started after +publication covers the added storage. Sequential publications therefore cannot +reuse unaccounted headroom between scans. Uncommitted crashed leases expire +after 24 hours; committed charges do not expire merely with time. +Snapshot and reservations live in `/var/lib/getbible/storage`. This is an application admission guard, not a filesystem quota. Git fetches, @@ -174,6 +249,16 @@ physical bound requires a host-supported filesystem quota. Never remove the live corpus to reclaim room. Telemetry has its own oldest-first retention cap; the dashboard reports both budgets and measured usage. +Service failure alerts share durable incident records under +`/var/lib/getbible/telemetry/service-alerts` between systemd and the collector. +The first failure reports the unit's result and exit status; +continuing failures send bounded reminders instead of one message per retry. +A retry in `activating` state or `reset-failed` is not recovery. Storage recovery +also requires a newer complete, fresh accounting snapshot. An ingested rotated +telemetry spool is not necessarily closed: `getbible capacity --json` reports +the reclamation reason before any safe removal. Keep unread files and files +with writable descriptors intact. + ## Verification The CI suite checks allocation and update overlap, environment precedence, diff --git a/getbible.sh b/getbible.sh index e721c27..645acb8 100755 --- a/getbible.sh +++ b/getbible.sh @@ -137,6 +137,9 @@ Platform settings environment show configuration values and their sources (secrets hidden) settings set KEY VALUE set a system setting; explicit environment values take priority resources [show|apply] [--json] inspect or apply the shared runtime memory budget + resources realign [--json] review hardware, ownership and proposed allocation + [--memory-budget SIZE] [--apply] [--expected-plan SHA256] + apply a reviewed plan through healthy generations telegram enable|disable|test cloudflare ... see: getbible.sh cloudflare help doctor check this host @@ -546,7 +549,11 @@ main() { local GB_IMAGE_UPDATE_WAIT=true gb_management_lock || return "$?" update_image "$@" ;; - resources) gb_system_init; resources_cli "$@" ;; + resources) + # Hardware planning is read-only and must not initialize services, + # acquire the deployment lock or save defaults merely to inspect. + if [[ "${1:-}" != realign ]]; then gb_system_init; fi + resources_cli "$@" ;; dashboard) gb_system_init; dashboard_cli "$@" ;; mcp) gb_system_init; mcp_cli "$@" ;; list) ep_list ;; diff --git a/src/apps/telemetry/getbible_telemetry/collector.py b/src/apps/telemetry/getbible_telemetry/collector.py index 2156332..25d06a7 100644 --- a/src/apps/telemetry/getbible_telemetry/collector.py +++ b/src/apps/telemetry/getbible_telemetry/collector.py @@ -28,6 +28,7 @@ from .store import TelemetryStore from .settings import SETTING_NAMES, numeric_setting from .capacity import CapacityTracker, incident_update +from .service_alerts import ServiceIncidents, boot_id def read_settings(path: str) -> dict[str, int]: @@ -73,6 +74,7 @@ def __init__(self, store: TelemetryStore, log_root: str, *, batch_size: int = 10 self._committed_records = 0 self._instance = uuid.uuid4().hex self._capacity = CapacityTracker(store) + self._service_incidents = ServiceIncidents(store.path.parent / "service-alerts") self._history_max_bytes = 10 * 1024**3 self._lock = None self._storage_alert_at = 0.0 @@ -469,7 +471,7 @@ def state(self) -> dict[str, Any]: - self.store.db.execute("PRAGMA freelist_count").fetchone()[0]) * page problems = {row[0]: json.loads(row[1]) for row in self.store.db.execute( "SELECT key,value FROM metadata WHERE key LIKE 'blocked_source:%' OR key LIKE 'source_error:%' " - "OR key IN ('nginx_reopen_failed','spool_cleanup')")} + "OR key IN ('nginx_reopen_failed','spool_cleanup','service_alert_error')")} return {"spool_bytes": total, "unread_bytes": unread, "spool_max_bytes": self.spool_max_bytes, "budgeted_spool_bytes": budgeted, "active_bytes": active, "active_files": active_files, "rotated_spool_bytes": rotated_bytes, "consumed_rotated_bytes": consumed, @@ -594,12 +596,41 @@ def health(self, sample: dict[str, Any], state: dict[str, Any], *, now: float | "telemetry_spool": (usage, allowance, details), } services = sample.get("services", {}) - if services.get("available"): - for key in alerts: - if key.startswith(("service:", "sync:")): - conditions[key] = (0, 1, "Previously reported service condition cleared.") + fresh_services = services.get("available") and 0 <= now - services.get("sampled_at", now) <= 90 + if fresh_services: for key, condition in services.get("conditions", {}).items(): - conditions[key] = (int(condition["unhealthy"]), 1, condition["message"]) + if key.startswith("sync:"): + conditions[key] = (int(condition["unhealthy"]), 1, condition["message"]) + # Share OnFailure's durable incident instead of sending a second alert + # through the generic capacity path. Absence from list-units --failed + # means retrying or unknown, never proof of a successful execution. + try: + incidents = self._service_incidents.snapshot() + service_keys = {key for key in alerts if key.startswith("service:")} + service_keys.update("service:" + unit for unit in incidents) + if fresh_services: + service_keys.update(key for key in services.get("conditions", {}) if key.startswith("service:")) + for key in sorted(service_keys): + unit = key.removeprefix("service:") + condition = services.get("conditions", {}).get(key, {}) if fresh_services else {} + values = condition.get("unit_state") + current, event = self._service_incidents.observe( + unit, values, failed=bool(condition.get("unhealthy")), now=now, + monotonic=services.get("monotonic", time.monotonic()), boot=services.get("boot_id") or boot_id(), + hold=self.alert_settings["ALERT_HOLD_SECONDS"], cooldown=self.alert_settings["ALERT_COOLDOWN_SECONDS"], + reminder=self.alert_settings["ALERT_REMINDER_SECONDS"], legacy=alerts.get(key)) + alerts[key] = current + if event == "recovered": + self._notify("info", "getBible recovered: " + key, + "A successful execution after the failure has been confirmed.") + elif event: + self._notify("fail", "Service failed: " + unit, current["message"] + + (" Persistent incident reminder." if event == "reminder" else "")) + with self.store.db: + self.store.db.execute("DELETE FROM metadata WHERE key='service_alert_error'") + except (OSError, ValueError, TypeError) as exc: + with self.store.db: + self.store.set_metadata("service_alert_error", {"time": now, "error": str(exc)[:512]}) for key, (value, threshold, message) in conditions.items(): if value is None: continue # Missing sensors are not evidence of recovery. @@ -627,7 +658,8 @@ def run(self, *, flush_seconds: int = 1, metrics_seconds: int = 5, self.lock() self._history_max_bytes = max_bytes sampler = MetricsSampler(cgroup_root=cgroup_root, disks=[str(self.root), str(self.store.path.parent)]) - inspector = HealthInspector(systemctl) + inspector = HealthInspector(systemctl, incident_directory=self._service_incidents.directory, + storage_directory=self.store.path.parent.parent / "storage") next_metric = next_prune = next_cleanup = next_rotate = next_settings = 0.0 next_ingest = next_rollup = 0.0 try: diff --git a/src/apps/telemetry/getbible_telemetry/health.py b/src/apps/telemetry/getbible_telemetry/health.py index 5537477..d483e23 100644 --- a/src/apps/telemetry/getbible_telemetry/health.py +++ b/src/apps/telemetry/getbible_telemetry/health.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import json import re import shlex import subprocess @@ -13,14 +14,19 @@ from typing import Any from .store import timestamp +from .service_alerts import PROPERTIES, ServiceIncidents, boot_id, failure_message, parse_units class HealthInspector: - def __init__(self, executable: str = "/usr/bin/systemctl") -> None: + def __init__(self, executable: str = "/usr/bin/systemctl", *, + incident_directory: str | Path = "/var/lib/getbible/telemetry/service-alerts", + storage_directory: str | Path = "/var/lib/getbible/storage") -> None: self.executable = executable self.last_sample: dict[str, Any] = {"available": False, "conditions": {}} self.next_check = 0.0 self._worker: threading.Thread | None = None + self.incidents = ServiceIncidents(incident_directory) + self.storage_directory = Path(storage_directory) def current(self, *, grace_seconds: float = 3600) -> dict[str, Any]: """Inspect off the ingest loop; a stalled service bus cannot block logs.""" @@ -47,10 +53,30 @@ def sample(self, *, grace_seconds: float = 3600, now: float | None = None) -> di self.last_sample = {"available": False, "conditions": {}, "reason": "systemd state unavailable"} return self.last_sample conditions = {} + failed_units = set() for line in failed.splitlines(): unit = line.split()[0] if line.split() else "" if unit == "nginx.service" or re.fullmatch(r"getbible-[A-Za-z0-9_.@-]+\.service", unit): + # The notification helper itself must not create an alert loop. + if unit.startswith("getbible-alert@"): + continue + failed_units.add(unit) conditions["service:" + unit] = {"unhealthy": True, "message": "Managed service failed: " + unit} + try: + tracked = {unit for unit, state in self.incidents.snapshot().items() if state.get("active")} + except (OSError, ValueError): + tracked = set() # The collector exposes incident journal errors. + units = sorted(failed_units | tracked) + if units: + details = self._run("show", *units, "--no-pager", "--property=" + ",".join(PROPERTIES)) + for unit, values in parse_units(details or "").items(): + if unit == "getbible-storage.service": + try: + values["storage_usage"] = json.loads((self.storage_directory / "usage.json").read_text()) + except (OSError, ValueError): + pass + conditions["service:" + unit] = {"unhealthy": values.get("ActiveState") == "failed", + "message": failure_message(unit, values), "unit_state": values} timers = self._run("list-timers", "--all", "--plain", "--no-legend", "--no-pager", "getbible-sync-*.timer") or "" syncs = [] for line in timers.splitlines(): @@ -89,5 +115,6 @@ def sample(self, *, grace_seconds: float = 3600, now: float | None = None) -> di syncs.append(item) conditions["sync:" + service] = {"unhealthy": stale, "message": "Scheduled sync has no successful LAST_CHECK after its timer trigger: " + service} - self.last_sample = {"available": True, "conditions": conditions, "syncs": syncs, "sampled_at": now} + self.last_sample = {"available": True, "conditions": conditions, "syncs": syncs, "sampled_at": now, + "monotonic": time.monotonic(), "boot_id": boot_id()} return self.last_sample diff --git a/src/apps/telemetry/getbible_telemetry/service_alerts.py b/src/apps/telemetry/getbible_telemetry/service_alerts.py new file mode 100644 index 0000000..7c0fd23 --- /dev/null +++ b/src/apps/telemetry/getbible_telemetry/service_alerts.py @@ -0,0 +1,215 @@ +"""Shared service incidents for systemd OnFailure and telemetry health checks. + +Keep the small incident journal independent of the reporting database: a failed +collector or database must still produce one immediate, actionable notification. +Updates are locked and atomic; Telegram delivery never holds the state lock. +""" +from __future__ import annotations + +import argparse +import fcntl +import json +import math +import os +from pathlib import Path +import re +import subprocess +import tempfile +import time +from typing import Any + + +UNIT = re.compile(r"(?:getbible-[A-Za-z0-9_.@-]+|nginx)\.service") +PROPERTIES = ("Id", "LoadState", "ActiveState", "SubState", "Type", "Result", + "ExecMainCode", "ExecMainStatus", "ExecMainExitTimestampMonotonic", + "ActiveEnterTimestampMonotonic", "InvocationID") + + +def boot_id() -> str: + return Path("/proc/sys/kernel/random/boot_id").read_text().strip() + + +def parse_units(text: str) -> dict[str, dict[str, str]]: + result = {} + for block in text.strip().split("\n\n"): + values = dict(line.split("=", 1) for line in block.splitlines() if "=" in line) + if UNIT.fullmatch(values.get("Id", "")): + result[values["Id"]] = values + return result + + +def failure_message(unit: str, values: dict[str, Any]) -> str: + result = values.get("Result") or "unknown" + status = values.get("ExecMainStatus", "unknown") + return (f"{unit}: result={result}, exit status={status}, " + f"state={values.get('ActiveState', 'unknown')}/{values.get('SubState', 'unknown')}. " + f"Inspect journalctl -u {unit} --since '1 hour ago'.") + + +def successful(values: dict[str, Any]) -> float: + """Return the kernel monotonic timestamp of proven successful execution.""" + if values.get("LoadState") != "loaded" or values.get("Result") != "success": + return 0 + state = values.get("ActiveState") + if state == "active" and values.get("SubState") == "running": + stamp = values.get("ActiveEnterTimestampMonotonic", "0") + elif (state in {"inactive", "active"} and values.get("SubState") in {"dead", "exited"} + and values.get("ExecMainCode") == "1" and values.get("ExecMainStatus") == "0"): + stamp = values.get("ExecMainExitTimestampMonotonic", "0") + else: + return 0 # Retrying, reset-failed, stopped-before-start and missing units. + try: + return max(0, int(stamp)) / 1_000_000 + except (ValueError, TypeError): + return 0 + + +class ServiceIncidents: + def __init__(self, directory: str | Path) -> None: + self.directory = Path(directory) + + def snapshot(self) -> dict[str, dict[str, Any]]: + result = {} + for path in self.directory.glob("*.json"): + if UNIT.fullmatch(path.stem): + state = json.loads(path.read_text()) + if not isinstance(state, dict): + raise ValueError("Invalid service incident: " + path.name) + result[path.stem] = state + return result + + def observe(self, unit: str, values: dict[str, Any] | None, *, now: float, + monotonic: float, boot: str, failed: bool = False, hold: float = 60, + reminder: float = 86400, cooldown: float = 900, + legacy: dict[str, Any] | None = None) -> tuple[dict[str, Any], str | None]: + if not UNIT.fullmatch(unit): + raise ValueError("Not a managed service unit") + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + path = self.directory / (unit + ".json") + with (self.directory / (unit + ".lock")).open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + try: + state = json.loads(path.read_text()) + except FileNotFoundError: + # Older telemetry incidents lack execution identity. Require a + # new successful run rather than accepting a historical exit + # retained by reset-failed or a dependency/start-limit failure. + active_legacy = bool((legacy or {}).get("active")) + state = {**(legacy or {}), "boot_id": boot, + "failure_monotonic": monotonic if active_legacy else 0} + if active_legacy: + state["failed_at"] = now + if not isinstance(state, dict): + raise ValueError("Invalid service incident: " + path.name) + original = json.dumps(state, sort_keys=True) + state.setdefault("active", False) + event = None + values = values or {} + if failed or values.get("ActiveState") == "failed": + first = not state["active"] + state.update(active=True, clear_since=None, observed_at=now) + if values or not state.get("message"): + state["message"] = failure_message(unit, values) + try: + failure_stamp = int(values.get("ExecMainExitTimestampMonotonic", "0")) / 1_000_000 + except (ValueError, TypeError): + failure_stamp = 0 + if first or state.get("boot_id") != boot or failure_stamp > state.get("failure_monotonic", 0): + state.update(boot_id=boot, failure_monotonic=failure_stamp or monotonic, + failed_at=now) + if values.get("failure_invocation") or values.get("InvocationID"): + state["failure_invocation"] = values.get("failure_invocation") or values["InvocationID"] + if first: + state.update(since=now, episodes=state.get("episodes", 0) + 1) + event = "onset" + elif now - state.get("last_sent", 0) >= max(reminder, cooldown): + event = "reminder" + elif state["active"]: + stamp = successful(values) + new_invocation = (state.get("failure_invocation") and values.get("InvocationID") + and state["failure_invocation"] != values["InvocationID"] + and monotonic >= state.get("failure_monotonic", 0)) + newer = stamp > 0 and (state.get("boot_id") != boot or new_invocation + or stamp > state.get("failure_monotonic", 0)) + if unit == "getbible-storage.service": + usage = values.get("storage_usage") or {} + if not isinstance(usage, dict): + usage = {} + completed = usage.get("completed_at", usage.get("generated_at", 0)) + generated = usage.get("generated_at", 0) + maximum = usage.get("max_age_seconds", 120) + numeric = all(isinstance(value, (int, float)) and not isinstance(value, bool) + and math.isfinite(value) for value in (completed, generated, maximum)) + newer = (newer and numeric and maximum > 0 + and completed > state.get("failed_at", 0) + and 0 <= now - generated <= maximum) + if newer: + if state.get("clear_since") is None: + state["clear_since"] = now + # A completed oneshot is direct proof of successful work. + # Timer idle intervals can be shorter than the normal hold; + # requiring a long inactive window would prevent recovery. + completed_run = values.get("Type") == "oneshot" and values.get("SubState") in {"dead", "exited"} + if completed_run or now - state["clear_since"] >= hold: + state.update(active=False, since=None, clear_since=None, recovered_at=now) + event = "recovered" + else: + state["clear_since"] = None + if event: + state.update(last_sent=now, last_event=event) + if json.dumps(state, sort_keys=True) == original and path.exists(): + return state, event + descriptor, temporary = tempfile.mkstemp(prefix=".incident-", dir=self.directory) + try: + with os.fdopen(descriptor, "w") as stream: + json.dump(state, stream) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + return state, event + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("unit") + parser.add_argument("--directory", required=True) + parser.add_argument("--systemctl", default="/usr/bin/systemctl") + parser.add_argument("--notify", default="/usr/local/lib/getbible/getbible-notify") + args = parser.parse_args() + if not UNIT.fullmatch(args.unit): + parser.error("not a managed service unit") + values: dict[str, Any] = {} + try: + result = subprocess.run([args.systemctl, "show", args.unit, "--no-pager", + "--property=" + ",".join(PROPERTIES)], + capture_output=True, text=True, timeout=3) + values = parse_units(result.stdout).get(args.unit, {}) + except (OSError, subprocess.TimeoutExpired): + pass + # systemd supplies these for OnFailure even if a fast retry already changed + # the queried unit state. Older systemd versions fall back to show above. + for name, target in (("MONITOR_SERVICE_RESULT", "Result"), ("MONITOR_EXEC_MAIN_STATUS", "ExecMainStatus")): + if os.environ.get(name): + values[target] = os.environ[name] + if os.environ.get("MONITOR_INVOCATION_ID"): + values["failure_invocation"] = os.environ["MONITOR_INVOCATION_ID"] + if values.get("Result") == "success": + values["Result"] = "unavailable (unit has already left its failed state)" + settings = {} + for key, default in (("ALERT_REMINDER_SECONDS", 86400), ("ALERT_COOLDOWN_SECONDS", 900)): + value = os.environ.get("GETBIBLE_" + key, str(default)) + settings[key] = int(value) if re.fullmatch(r"[1-9][0-9]{0,5}", value) else default + state, event = ServiceIncidents(args.directory).observe( + args.unit, values, failed=True, now=time.time(), monotonic=time.monotonic(), boot=boot_id(), + reminder=settings["ALERT_REMINDER_SECONDS"], cooldown=settings["ALERT_COOLDOWN_SECONDS"]) + if event: + subprocess.run([args.notify, "fail", "Service failed: " + args.unit, + state["message"] + (" Persistent incident reminder." if event == "reminder" else "")], + check=False, timeout=18) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/bin/getbible-resources b/src/bin/getbible-resources index f1c9285..904f336 100755 --- a/src/bin/getbible-resources +++ b/src/bin/getbible-resources @@ -9,6 +9,7 @@ from __future__ import annotations import argparse from decimal import Decimal +import hashlib import json import math import os @@ -41,6 +42,29 @@ def config(path: Path) -> dict[str, str]: if re.match(r"^[A-Z][A-Z0-9_]*=", line)) +def host_memory(proc: Path) -> int | None: + """Physical RAM is an upper bound, never a container's missing budget.""" + try: + for line in (proc / "meminfo").read_text().splitlines(): + match = re.fullmatch(r"MemTotal:\s+(\d+)\s+kB", line) + if match: + return int(match[1]) * 1024 + except FileNotFoundError: + pass + return None + + +def available_memory(proc: Path) -> int | None: + try: + for line in (proc / "meminfo").read_text().splitlines(): + match = re.fullmatch(r"MemAvailable:\s+(\d+)\s+kB", line) + if match: + return int(match[1]) * 1024 + except FileNotFoundError: + pass + return None + + def cgroup_limits(proc: Path, override: Path | None = None, owner: str = "self") -> dict: """Read this process's cgroup and every ancestor visible in its namespace. @@ -350,10 +374,13 @@ def plan(args) -> dict: if args.mode == "docker" and cap is None: raise ValueError("Docker requires a finite cgroup v2 memory.max; set the Compose memory limit before starting getBible.") requested = None if args.budget in {"", "auto"} else memory_bytes(args.budget) - budget = min(value for value in (cap, requested) if value is not None) + physical = host_memory(Path(args.proc_root)) + budget = min(value for value in (cap, requested, physical) if value is not None) source = "cgroup" if requested is None else "configured budget" if requested is not None and cap is not None and cap < requested: source = "cgroup (lower than configured budget)" + if physical is not None and physical == budget and all(value is None or value > physical for value in (cap, requested)): + source = "physical memory (lower than configured or cgroup budget)" policy = policy_values(args) infrastructure = max(256 * MIB, budget * bounded(policy["RESOURCE_RESERVE_PERCENT"], "RESOURCE_RESERVE_PERCENT", 5, 75) // 100) # Each MCP domain has a 256-MiB process ceiling. Reserve its @@ -429,10 +456,239 @@ def plan(args) -> dict: "endpoints": effective} +def selected_settings(item: dict) -> dict: + """Read only resource fields from the selected immutable generation.""" + root = Path(item["root"]) / "active" + result = {} + limits = root / "limits.conf" + mapping = {"MemoryMax": "MEMORY_MAX", "MemoryHigh": "MEMORY_HIGH", "CPUQuota": "CPU_QUOTA", + "TasksMax": "TASKS_MAX", "LimitNOFILE": "NOFILE"} + if limits.is_file(): + for line in limits.read_text().splitlines(): + key, separator, value = line.partition("=") + if separator and key in mapping: + if key.startswith("Memory") and value not in {"", "infinity"}: + value = memory_bytes(value) + elif key in {"TasksMax", "LimitNOFILE"} and value.isdigit(): + value = int(value) + result[mapping[key]] = value + prefix = item["kind"].upper() + env_mapping = {f"{prefix}_{key}": key for key in ("WORKERS", "THREADS", "WARM_TRANSLATIONS")} + env_mapping.update({"GETBIBLE_" + key: key for key in ( + "SHARED_CORPUS_LIMIT", "SHARED_CORPUS_BYTES", "CHAPTER_CACHE_BYTES", "TRANSLATION_CACHE_BYTES", + "TRANSLATION_CACHE_LIMIT", "CHAPTER_CACHE_LIMIT", "REFERENCE_CACHE_LIMIT", "SEARCH_CORPUS_LIMIT")}) + env_mapping.update(GETBIBLE_CACHE_TTL_SECONDS="MEMORY_CACHE_TTL", GETBIBLE_CACHE_TTL_JITTER="CACHE_TTL_JITTER", + SEARCH_MAX_CONCURRENT="SEARCH_MAX_CONCURRENT", SEARCH_MAX_CONCURRENT_EXPENSIVE="EXPENSIVE_CONCURRENT") + for key, raw in config(root / "runtime.env").items(): + if key not in env_mapping: + continue + value = re.sub(r'\\([\\"])', r'\1', raw[1:-1]) if raw.startswith('"') and raw.endswith('"') else raw + target = env_mapping[key] + if target == "CACHE_TTL_JITTER": + value = float(value) + elif target != "WARM_TRANSLATIONS" and value.isdigit(): + value = int(value) + result[target] = value + return result + + +def management_observations(args) -> dict: + """Read bounded sampler metadata and the existing read-only capacity view.""" + storage = {"state": "unavailable", "progress": None, "usage": None} + state = Path(getattr(args, "storage_state", "/var/lib/getbible/storage")) + now = time.time() + for filename, key in (("progress.json", "progress"), ("usage.json", "usage")): + try: + with (state / filename).open() as stream: + data = json.loads(stream.read(65537)) + if not isinstance(data, dict): + continue + allowed = {"status", "started_at", "completed_at", "generated_at", "updated_at", "used_bytes", + "max_age_seconds", "error", "entries", "pending", "files", "directories", + "allocated_bytes", "pending_directories", "duration_seconds"} + storage[key] = {name: value for name, value in data.items() if name in allowed} + except (OSError, ValueError): + continue + usage = storage["usage"] or {} + observed = usage.get("started_at", usage.get("generated_at")) + if isinstance(observed, (float, int)): + age = now - observed + horizon = usage.get("max_age_seconds", 120) + storage["age_seconds"] = max(0, age) + storage["state"] = "fresh" if isinstance(horizon, (int, float)) and 0 <= age <= horizon and usage.get("status", "complete") == "complete" else "stale" + root = Path(__file__).resolve().parent + for path in (root / "apps/telemetry", root.parent / "apps/telemetry"): + if path.is_dir(): + sys.path.insert(0, str(path)) + break + try: + from getbible_telemetry.diagnostics import read_capacity + capacity = read_capacity(getattr(args, "telemetry_db", "/var/lib/getbible/telemetry/traffic.sqlite3"), + getattr(args, "config", "/etc/getbible/getbible.conf"), + getattr(args, "environment", "/run/getbible/environment.conf")) + except ImportError: + capacity = {"state": "unavailable", "note": "Capacity reader is not installed; inspect getbible capacity after updating management services."} + return {"storage_accounting": storage, "capacity": capacity} + + +def realignment(args) -> dict: + """Offer a conservative, reviewable allocation without changing policy.""" + proc = Path(args.proc_root) + limits = cgroup_limits(proc, Path(args.cgroup_root) if args.cgroup_root else None, owner="1") + physical, available = host_memory(proc), available_memory(proc) + totals = [value for value in (physical, limits["memory"]) if value is not None] + total = min(totals) if totals else None + # MemAvailable can describe the host in a container. The current usage at + # the limiting ancestor supplies a stricter bound when that value exists. + if limits["memory_path"]: + usage = Path(limits["memory_path"]).with_name("memory.current") + if usage.is_file(): + remaining = max(0, limits["memory"] - int(usage.read_text())) + available = min(remaining, available) if available is not None else remaining + hardware = {"physical_memory_bytes": physical, "effective_memory_bytes": total, + "available_memory_bytes": available, "cgroup_memory_bytes": limits["memory"], + "cpu_capacity": limits["cpus"], "storage": []} + for entry in getattr(args, "storage_path", []) or []: + name, separator, raw = entry.partition("=") + if not separator or not name or not raw: + raise ValueError("Storage observations require NAME=PATH.") + path = Path(raw) + observation = {"name": name, "path": str(path)} + try: + stats = os.statvfs(path) + observation.update(status="observed", total_bytes=stats.f_blocks * stats.f_frsize, + available_bytes=stats.f_bavail * stats.f_frsize, + total_inodes=stats.f_files, available_inodes=stats.f_favail) + except OSError as error: + observation.update(status="unavailable", error=error.strerror) + hardware["storage"].append(observation) + saved = config(Path(getattr(args, "config", "/etc/getbible/getbible.conf"))) + authoritative = set(getattr(args, "authoritative", []) or []) + owner = "environment" if "MEMORY_BUDGET" in authoritative else "saved global" if "MEMORY_BUDGET" in saved else "default" + proposed = getattr(args, "realign_budget", None) or args.budget + reason, assumption = "", "Existing explicit settings remain authoritative; targets are bounded by effective hardware." + if args.mode == "native" and proposed in {"", "auto", "off"}: + assumption = ("Native recommendation reserves 25% of effective RAM (at least 256 MiB) from both total and " + "currently available RAM. Existing runtime allocations are not counted as reclaimable. " + "This is a conservative starting point, not a workload benchmark; use --memory-budget for a deliberate shared-host allocation.") + reserve = max(256 * MIB, total // 4) if total is not None else None + if total is None or available is None: + reason = "Cannot recommend a native budget without total and available memory; provide --memory-budget." + else: + recommendation = max(0, min(total, available) - reserve) // MIB + if recommendation: + proposed = f"{recommendation}M" + else: + reason = "Available memory does not leave the native host reserve; free capacity or provide a deliberate --memory-budget." + budget_changed = proposed != args.budget + if budget_changed and owner == "environment": + reason = "MEMORY_BUDGET is environment-controlled; change GETBIBLE_MEMORY_BUDGET at its deployment source and rerun the plan." + allocation = None + proposal = argparse.Namespace(**vars(args)) + proposal.budget = proposed + try: + if not reason: + allocation = plan(proposal) + if not allocation["enabled"]: + reason = "Aggregate runtime budgeting is not enabled." + except (ValueError, OSError, subprocess.SubprocessError) as error: + reason = str(error) + comparisons = [] + entries = inventory(Path(args.registry), Path(args.runtime_root)) + policies = policy_values(args) + for item in entries: + selected = selected_settings(item) + target = next((row["settings"] for row in (allocation or {}).get("endpoints", []) + if row["domain"] == item["domain"] and row["label"] == item["label"]), {}) + owners = {} + for key in policies: + if key.startswith("_") or key.startswith(("QUERY_", "SEARCH_")) and not key.startswith(item["kind"].upper() + "_"): + continue + endpoint_key = key.removeprefix(item["kind"].upper() + "_") + owners[endpoint_key] = ("environment" if key in authoritative else "endpoint" if endpoint_key in item["config"] + else "saved global" if key in saved else "default") + for key in ("WORKERS", "THREADS"): + owners[key] = "endpoint" if key in item["config"] else "automatic default" + ignored = {"WORKERS_MIN", "WORKERS_MAX"} + if item["kind"] == "query": + ignored.update(("SEARCH_MAX_CONCURRENT", "EXPENSIVE_CONCURRENT")) + differences = {key: {"current": selected.get(key), "proposed": value} for key, value in target.items() + if key not in ignored and selected.get(key) != value} + comparisons.append({"domain": item["domain"], "label": item["label"], "kind": item["kind"], + "selected": selected, "target": target, "differences": differences, "policy_owners": owners}) + changed = {row["domain"] for row in comparisons if row["differences"]} + reductions = domain_changes(allocation, True) if allocation else [] + domains = [name for name in reductions if name in changed] + sorted(changed - set(reductions)) + # Exclude changing utilization, filesystem free space and timestamps. The + # accepted budget and every selected/target resource field remain covered. + inputs = {"mode": args.mode, "memory": total, "cgroup": limits["memory"], "cpus": limits["cpus"], + "budget": proposed, "configured_budget": args.budget, "owner": owner, + "policy": {key: value for key, value in policies.items() if key != "_authoritative"}, + "authoritative": sorted(authoritative), "endpoints": comparisons} + plan_id = hashlib.sha256(json.dumps(inputs, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + return {"plan_id": plan_id, "ready": not reason, "status": "blocked" if reason else "ready", "reason": reason, + "hardware": hardware, "memory_budget": {"current": args.budget, "proposed": proposed, "owner": owner, + "change": budget_changed, "assumption": assumption}, + "allocation": allocation, "endpoints": comparisons, "domains": domains, + "management": management_observations(args), + "limitations": ["Storage figures are filesystem availability, not corpus usage; no data tree is scanned and no storage quota is changed.", + "Storage guard and telemetry collector retain separate service limits. Runtime realignment does not repair their failures or raise those limits.", + "Deployment CPU/memory caps remain operator-owned. Measurements under representative traffic determine throughput; this plan is not a universal optimum.", + "Apply uses normal runtime deployments for the listed domains and the installed manager's current code/configuration; it may also bring those domains to the installed runtime code version.", + "Selected settings describe the active generation files; normal readiness and live cgroup overlap checks remain required during apply."]} + + +def print_realignment(result: dict) -> None: + hardware, budget = result["hardware"], result["memory_budget"] + def mib(value): + return "unknown" if value is None else f"{value // MIB} MiB" + print(f"Hardware realignment: {result['status']}; plan {result['plan_id']}") + print(f"Effective CPU: {hardware['cpu_capacity']:g}; RAM: {mib(hardware['effective_memory_bytes'])}; currently available: {mib(hardware['available_memory_bytes'])}") + print(f"Memory budget: {budget['current']} -> {budget['proposed']} (owner: {budget['owner']})") + print(budget["assumption"]) + for row in result["endpoints"]: + print(f" {row['domain']} {row['label']} ({row['kind']}): {len(row['differences'])} resource changes") + for key, change in row["differences"].items(): + print(f" {key}: {change['current'] if change['current'] is not None else 'unknown'} -> {change['proposed']}") + print(" Policy owners: " + ", ".join(f"{key}={value}" for key, value in row["policy_owners"].items() + if value in {"environment", "endpoint"})) + for row in hardware["storage"]: + if row["status"] == "observed": + print(f" Storage {row['name']}: {mib(row['available_bytes'])} available / {mib(row['total_bytes'])}; {row['available_inodes']} free inodes") + else: + print(f" Storage {row['name']}: observation unavailable ({row['error']})") + management = result.get("management", {}) + storage = management.get("storage_accounting", {}) + print(f"Storage accounting: {storage.get('state', 'unavailable')}; sampler progress: {(storage.get('progress') or {}).get('status', 'unavailable')}; age: {storage.get('age_seconds', 'unknown')} seconds") + capacity = management.get("capacity", {}) + collection = capacity.get("collection", {}) + reclamation = collection.get("reclamation", {}) + print(f"Capacity evidence: {capacity.get('state', 'unavailable')}; collection: {collection.get('state', 'unknown')}; reclamation: {reclamation.get('state', 'unknown')}") + if capacity.get("note"): + print(capacity["note"]) + if reclamation.get("reason"): + print("Reclamation: " + reclamation["reason"]) + for row in capacity.get("limits", []): + advice = row.get("recommendation", {}) + print(f" {row.get('label', row.get('id', 'Capacity'))}: {advice.get('status', 'unknown')}; {advice.get('reason', '')}") + if result["reason"]: + print("Blocked: " + result["reason"]) + for note in result["limitations"]: + print(note) + print("No settings changed. Use resources realign --apply to apply a freshly checked plan.") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--mode", choices=("native", "docker"), default="native") parser.add_argument("--budget", default="auto") + parser.add_argument("--realign", action="store_true", help="Read-only hardware and selected-versus-target review.") + parser.add_argument("--realign-budget", help="Explicit budget to review; does not save configuration.") + parser.add_argument("--config", default="/etc/getbible/getbible.conf") + parser.add_argument("--storage-path", action="append", default=[], metavar="NAME=PATH") + parser.add_argument("--storage-state", default="/var/lib/getbible/storage") + parser.add_argument("--telemetry-db", default="/var/lib/getbible/telemetry/traffic.sqlite3") + parser.add_argument("--environment", default="/run/getbible/environment.conf") parser.add_argument("--policy", action="append", default=[], metavar="KEY=VALUE") parser.add_argument("--authoritative", action="append", default=[]) parser.add_argument("--adaptive-state", default="/var/lib/getbible/adaptive.json") @@ -450,6 +706,15 @@ def main() -> int: parser.add_argument("--format", choices=("json", "text", "env", "domains", "reductions", "changes"), default="text") args = parser.parse_args() try: + if args.realign: + if args.format not in {"json", "text"}: + raise ValueError("Realignment supports text or JSON output.") + result = realignment(args) + if args.format == "json": + print(json.dumps(result, indent=2)) + else: + print_realignment(result) + return 0 if args.wait_drain: wait_drain(args) return 0 diff --git a/src/bin/getbible-storage-guard b/src/bin/getbible-storage-guard index 7a3dde7..9651d86 100755 --- a/src/bin/getbible-storage-guard +++ b/src/bin/getbible-storage-guard @@ -8,12 +8,16 @@ hard physical storage limit is required. """ from __future__ import annotations import argparse +from contextlib import closing import fcntl import json +import math import os from pathlib import Path import re +import sqlite3 import stat +import struct import subprocess import sys import tempfile @@ -21,6 +25,9 @@ import time DEFAULT_ROOTS = ("/srv/getbible", "/opt/getbible", "/var/cache/getbible", "/var/cache/nginx/getbible", "/var/log/getbible", "/var/lib/getbible", "/var/backups/getbible", "/var/www/getbible", "/etc/getbible") +SCAN_MAX_SECONDS = 900 +SCAN_SLICE_SECONDS = 20 +SAMPLE_INTERVAL_SECONDS = 30 def write_json(path, data, mode): @@ -36,30 +43,250 @@ def write_json(path, data, mode): Path(temporary).unlink(missing_ok=True) +def scan_roots(roots): + """A union of lexical paths, without resolving/following a root symlink.""" + paths = list(dict.fromkeys(os.path.abspath(root) for root in roots)) + return [path for path in paths if not any( + other != path and os.path.commonpath((path, other)) == other for other in paths)] + + +def inode_key(metadata): + return struct.pack("!QQ", metadata.st_dev, metadata.st_ino) + + +def directory_entries(descriptor, cursor): + """Resume Linux readdir at the start of its last buffered block. + + scandir duplicates the directory descriptor and shares its kernel offset. + That offset points AFTER readdir's buffer, not after the returned entry. + Replaying the buffer's START therefore cannot skip its unread entries. + Persisted entry names suppress this bounded replay after a checkpoint. + """ + os.lseek(descriptor, cursor, os.SEEK_SET) + offset = buffer_start = cursor + with os.scandir(descriptor) as entries: + for entry in entries: + following = os.lseek(descriptor, 0, os.SEEK_CUR) + if following != offset: + buffer_start, offset = offset, following + yield entry, buffer_start + + +def scan_progress(connection, started_at, status): + allocated, files = connection.execute("SELECT COALESCE(SUM(allocated),0), COALESCE(SUM(files),0) FROM roots").fetchone() + pending = connection.execute("SELECT COUNT(*) FROM directories WHERE done=0").fetchone()[0] + return {"status": status, "started_at": started_at, "updated_at": time.time(), + "allocated_bytes": allocated, "files": files, "pending_directories": pending} + + +def account_file(connection, root_id, metadata): + identity = inode_key(metadata) + allocated, logical = metadata.st_blocks * 512, metadata.st_size + previous = connection.execute("SELECT root,allocated,logical FROM inodes WHERE inode=?", (identity,)).fetchone() + if previous is None: + connection.execute("INSERT INTO inodes VALUES (?,?,?,?)", (identity, root_id, allocated, logical)) + connection.execute("UPDATE roots SET allocated=allocated+?, logical=logical+?, files=files+1 WHERE id=?", + (allocated, logical, root_id)) + else: + owner, old_allocated, old_logical = previous + # Replaying a changed directory neither double-counts unchanged files + # nor loses observed growth. Removed/replaced allocations are retained + # conservatively until the next complete scan. + allocated, logical = max(allocated, old_allocated), max(logical, old_logical) + if (allocated, logical) != (old_allocated, old_logical): + connection.execute("UPDATE inodes SET allocated=?,logical=? WHERE inode=?", (allocated, logical, identity)) + connection.execute("UPDATE roots SET allocated=allocated+?,logical=logical+? WHERE id=?", + (allocated - old_allocated, logical - old_logical, owner)) + + +def prepare_scan(connection, path, roots, max_seconds): + # This is disposable scan work, never serving data. Disk-backed queues and + # identities keep Python memory independent of corpus size. WAL transactions + # retain committed progress if systemd terminates an interrupted slice. + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + connection.execute("PRAGMA cache_size=-4096") + connection.execute("PRAGMA temp_store=FILE") + connection.execute("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + saved = dict(connection.execute("SELECT key,value FROM metadata")) + expected = {"schema": "3", "roots": json.dumps(roots), "max_seconds": str(max_seconds)} + if saved and any(saved.get(key) != value for key, value in expected.items()): + connection.close() + path.unlink() + return scan_database(path, roots, max_seconds) + if not saved: + connection.executescript(""" + CREATE TABLE IF NOT EXISTS roots (id INTEGER PRIMARY KEY, path BLOB NOT NULL, + allocated INTEGER NOT NULL DEFAULT 0, logical INTEGER NOT NULL DEFAULT 0, + files INTEGER NOT NULL DEFAULT 0); + CREATE TABLE IF NOT EXISTS directories (id INTEGER PRIMARY KEY, root INTEGER NOT NULL, + path BLOB NOT NULL, inode BLOB UNIQUE NOT NULL, done INTEGER NOT NULL DEFAULT 0, + cursor INTEGER NOT NULL DEFAULT 0, mtime INTEGER NOT NULL DEFAULT 0, ctime INTEGER NOT NULL DEFAULT 0); + CREATE INDEX IF NOT EXISTS pending_directories ON directories(done,id); + CREATE TABLE IF NOT EXISTS entries (directory INTEGER NOT NULL, name BLOB NOT NULL, + PRIMARY KEY(directory,name)) WITHOUT ROWID; + CREATE TABLE IF NOT EXISTS inodes (inode BLOB PRIMARY KEY, root INTEGER NOT NULL, + allocated INTEGER NOT NULL, logical INTEGER NOT NULL) WITHOUT ROWID; + """) + for table in ("roots", "directories", "entries", "inodes"): + connection.execute(f"DELETE FROM {table}") + expected["started_at"] = str(time.time()) + connection.executemany("INSERT INTO metadata VALUES (?,?)", expected.items()) + for number, root in enumerate(roots): + connection.execute("INSERT INTO roots(id,path) VALUES (?,?)", (number, os.fsencode(root))) + try: + metadata = os.stat(root, follow_symlinks=False) + except FileNotFoundError: + continue + if stat.S_ISDIR(metadata.st_mode): + connection.execute("INSERT OR IGNORE INTO directories(root,path,inode) VALUES (?,?,?)", + (number, os.fsencode(root), inode_key(metadata))) + connection.commit() + saved = expected + return connection, float(saved["started_at"]) + + +def scan_database(path, roots, max_seconds): + connection = sqlite3.connect(path) + try: + return prepare_scan(connection, path, roots, max_seconds) + except BaseException: + connection.close() + raise + + +def sample(roots, state, *, max_seconds=SCAN_MAX_SECONDS, slice_seconds=SCAN_SLICE_SECONDS): + """Resume bounded work; only a complete scan can replace usage.json. + + Each checkpoint commits the directory cursor, names, hard-link identities + and totals together, so resuming cannot count a partial directory twice. + """ + if not 120 <= max_seconds <= 86400 or not 0 < slice_seconds <= 60: + raise ValueError("Storage scan horizon must be 120..86400 seconds and slice must be at most 60 seconds.") + state = Path(state).absolute() + work = state / ".scan" + work.mkdir(mode=0o700, exist_ok=True) + info = work.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid(): + raise ValueError("Storage scan workspace must be a directory owned by the sampler.") + os.chmod(work, 0o700) + descriptor = os.open(work / "lock", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return {"status": "in_progress", "reason": "another sampler owns the active scan"} + database = work / "scan.sqlite3" + connection = None + started_at = time.time() + try: + connection, started_at = scan_database(database, scan_roots(roots), max_seconds) + elapsed = time.time() - started_at + if elapsed < -5 or elapsed >= max_seconds: + raise TimeoutError(f"Storage scan did not complete within its {max_seconds}-second horizon; last complete accounting was retained.") + deadline = time.monotonic() + min(slice_seconds, max_seconds - elapsed) + checkpoint_at = time.monotonic() + processed = 0 + while row := connection.execute("SELECT id,root,path,inode,cursor,mtime,ctime FROM directories WHERE done=0 ORDER BY id LIMIT 1").fetchone(): + directory_id, root_id, raw_path, identity, cursor, mtime, ctime = row + directory = os.fsdecode(raw_path) + try: + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + except FileNotFoundError: + connection.execute("UPDATE directories SET done=1 WHERE id=?", (directory_id,)) + connection.execute("DELETE FROM entries WHERE directory=?", (directory_id,)) + continue + try: + current = os.fstat(descriptor) + if inode_key(current) != identity: + raise ValueError("A directory was replaced during storage sampling; restart accounting and retain the last complete snapshot.") + if (current.st_mtime_ns, current.st_ctime_ns) != (mtime, ctime): + cursor = 0 + connection.execute("DELETE FROM entries WHERE directory=?", (directory_id,)) + connection.execute("UPDATE directories SET mtime=?,ctime=? WHERE id=?", + (current.st_mtime_ns, current.st_ctime_ns, directory_id)) + with closing(directory_entries(descriptor, cursor)) as entries: + for entry, buffer_start in entries: + if time.monotonic() >= deadline: + connection.execute("UPDATE directories SET cursor=? WHERE id=?", (buffer_start, directory_id)) + connection.commit() + if time.time() - started_at >= max_seconds: + raise TimeoutError(f"Storage scan exceeded its {max_seconds}-second horizon; last complete accounting was retained.") + progress = scan_progress(connection, started_at, "in_progress") + write_json(state / "progress.json", progress, 0o640) + return progress + name = os.fsencode(entry.name) + if connection.execute("SELECT 1 FROM entries WHERE directory=? AND name=?", (directory_id, name)).fetchone(): + continue + path = os.path.join(directory, entry.name) + # Never include the sampler's own growing queue/index + # or its temporary atomic-write files in its usage. + own_work = path == str(work) or (directory == str(state) and + (entry.name == "progress.json" or entry.name.startswith(".storage-"))) + if not own_work: + try: + metadata = entry.stat(follow_symlinks=False) + except FileNotFoundError: + metadata = None + if metadata is not None and stat.S_ISDIR(metadata.st_mode): + connection.execute("INSERT OR IGNORE INTO directories(root,path,inode) VALUES (?,?,?)", + (root_id, os.fsencode(path), inode_key(metadata))) + elif metadata is not None and stat.S_ISREG(metadata.st_mode): + account_file(connection, root_id, metadata) + connection.execute("INSERT INTO entries VALUES (?,?)", (directory_id, name)) + processed += 1 + if processed % 256 == 0 or time.monotonic() - checkpoint_at >= 1: + connection.execute("UPDATE directories SET cursor=? WHERE id=?", (buffer_start, directory_id)) + connection.commit() + checkpoint_at = time.monotonic() + after = os.fstat(descriptor) + if (after.st_mtime_ns, after.st_ctime_ns) != (current.st_mtime_ns, current.st_ctime_ns): + # Directory cookies may have changed. Replay names + # to discover additions without double accounting. + connection.execute("UPDATE directories SET cursor=0 WHERE id=?", (directory_id,)) + connection.execute("DELETE FROM entries WHERE directory=?", (directory_id,)) + else: + connection.execute("UPDATE directories SET done=1 WHERE id=?", (directory_id,)) + connection.execute("DELETE FROM entries WHERE directory=?", (directory_id,)) + connection.commit() + finally: + os.close(descriptor) + completed_at = time.time() + if completed_at - started_at > max_seconds: + raise TimeoutError(f"Storage scan exceeded its {max_seconds}-second horizon; last complete accounting was retained.") + totals = [{"path": os.fsdecode(path), "allocated_bytes": allocated, "logical_bytes": logical, "files": files} + for path, allocated, logical, files in connection.execute("SELECT path,allocated,logical,files FROM roots ORDER BY id")] + report = {"status": "complete", "generated_at": started_at, "started_at": started_at, + "completed_at": completed_at, "duration_seconds": completed_at - started_at, + "max_age_seconds": 2 * max_seconds + SAMPLE_INTERVAL_SECONDS + 10, + "used_bytes": sum(item["allocated_bytes"] for item in totals), "roots": totals, + "measurement": "allocated_blocks_deduplicated_by_inode", "hard_quota": False} + write_json(state / "usage.json", report, 0o640) + write_json(state / "progress.json", scan_progress(connection, started_at, "complete"), 0o640) + connection.close() + connection = None + database.unlink() + return report + except (OSError, ValueError, sqlite3.Error) as error: + if connection is not None: + connection.close() + connection = None + database.unlink(missing_ok=True) + write_json(state / "progress.json", {"status": "failed", "started_at": started_at, + "updated_at": time.time(), "error": str(error)}, 0o640) + raise + finally: + if connection is not None: + connection.close() + + def snapshot(roots): - seen = set() - totals = [] - for root in roots: - allocated = logical = files = 0 - root = Path(root) - if root.exists() and not root.is_symlink(): - for directory, names, filenames in os.walk(root, followlinks=False): - names[:] = [name for name in names if not (Path(directory) / name).is_symlink()] - for name in filenames: - try: - metadata = (Path(directory) / name).lstat() - except FileNotFoundError: - continue - identity = (metadata.st_dev, metadata.st_ino) - if identity in seen or not stat.S_ISREG(metadata.st_mode): - continue - seen.add(identity) - allocated += metadata.st_blocks * 512 - logical += metadata.st_size - files += 1 - totals.append({"path": str(root), "allocated_bytes": allocated, "logical_bytes": logical, "files": files}) - return {"generated_at": time.time(), "used_bytes": sum(item["allocated_bytes"] for item in totals), - "roots": totals, "measurement": "allocated_blocks_deduplicated_by_inode", "hard_quota": False} + """Synchronous diagnostic snapshot; periodic production sampling resumes.""" + with tempfile.TemporaryDirectory(prefix="getbible-storage-") as state: + while True: + report = sample(roots, state) + if report["status"] == "complete": + return report def estimate_export(checkout, revision): @@ -93,16 +320,40 @@ def admission(args): except FileNotFoundError: reservations = {} if args.action == "release": + if reservations.get(args.id, {}).get("committed_at") is not None: + return {"released": False, "retained_committed": args.id} reservations.pop(args.id, None) write_json(directory / "reservations.json", reservations, 0o660) return {"released": args.id} + if args.action == "commit": + if args.id in reservations: + reservations[args.id].setdefault("committed_at", time.time()) + write_json(directory / "reservations.json", reservations, 0o660) + return {"committed": args.id} usage = json.loads((directory / "usage.json").read_text()) now = time.time() - if now - usage["generated_at"] > args.max_age or usage["generated_at"] > now + 5: + max_age = args.max_age if args.max_age is not None else usage.get("max_age_seconds", 120) + observed_at = usage.get("started_at", usage["generated_at"]) + completed_at = usage.get("completed_at", usage["generated_at"]) + if (not isinstance(max_age, (float, int)) or not 0 < max_age <= 2 * 86400 + 40 + or not isinstance(observed_at, (float, int)) or not math.isfinite(observed_at) + or not isinstance(completed_at, (float, int)) or not math.isfinite(completed_at) + or completed_at < observed_at or completed_at > now + 5 + or usage.get("status", "complete") != "complete"): + raise ValueError("Storage accounting is incomplete or invalid; keep the current publication.") + if now - observed_at > max_age or observed_at > now + 5: raise ValueError("Storage accounting is stale; keep the current publication and retry after the sampler runs.") - # A crashed sync's lease eventually expires; normal exits release it in - # their trap. Retained corpus generations themselves are never deleted. - reservations = {key: value for key, value in reservations.items() if now - value["created_at"] < 86400} + # Failed exports release their estimates. Successful publications keep + # their charge until a complete scan STARTED after publication. A scan + # already in progress might have visited that tree before it changed. + # Only uncommitted crashed leases expire by age; published data remains. + reservations = {key: value for key, value in reservations.items() + if (value["committed_at"] > observed_at if value.get("committed_at") is not None + else now - value["created_at"] < 86400)} + if reservations.get(args.id, {}).get("committed_at") is not None: + raise ValueError("A committed storage reservation cannot be reused until complete accounting covers its publication.") + if args.action == "check" and args.id not in reservations: + raise ValueError("Storage reservation is missing or expired; keep the current publication and retry.") estimated = args.bytes if args.checkout: estimated = estimate_export(args.checkout, args.revision) @@ -117,36 +368,40 @@ def admission(args): return {"reserved_bytes": estimated, "used_bytes": usage["used_bytes"], "budget_bytes": budget, "hard_quota": False} -def storage_budget(config_path, environment_path): - value = "0" +def storage_setting(config_path, environment_path, key, default, minimum, maximum): + value = str(default) for path in (config_path, environment_path): try: lines = Path(path).read_text().splitlines() except FileNotFoundError: continue for line in lines: - key, separator, configured = line.partition("=") - if separator and key in {"STORAGE_MAX_GIB", "GETBIBLE_STORAGE_MAX_GIB"} and configured: + configured_key, separator, configured = line.partition("=") + if separator and configured_key in {key, f"GETBIBLE_{key}"} and configured: value = configured # Empty deployment values preserve saved settings, like the manager. - if os.environ.get("GETBIBLE_STORAGE_MAX_GIB"): - value = os.environ["GETBIBLE_STORAGE_MAX_GIB"] - budget = int(value) - if not 0 <= budget <= 999999: - raise ValueError("STORAGE_MAX_GIB must be between 0 and 999999.") - return budget + if os.environ.get(f"GETBIBLE_{key}"): + value = os.environ[f"GETBIBLE_{key}"] + setting = int(value) + if not minimum <= setting <= maximum: + raise ValueError(f"{key} must be between {minimum} and {maximum}.") + return setting + + +def storage_budget(config_path, environment_path): + return storage_setting(config_path, environment_path, "STORAGE_MAX_GIB", 0, 0, 999999) def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("snapshot", "sample", "reserve", "check", "release")) + parser.add_argument("action", choices=("snapshot", "sample", "reserve", "check", "commit", "release")) parser.add_argument("--state", default="/var/lib/getbible/storage") parser.add_argument("--config", default="/etc/getbible/getbible.conf") parser.add_argument("--environment", default="/run/getbible/environment.conf") parser.add_argument("--root", action="append") parser.add_argument("--id", default="") parser.add_argument("--max-gib", type=float, default=0) - parser.add_argument("--max-age", type=int, default=120) + parser.add_argument("--max-age", type=int, help="Override the completed snapshot's admission freshness window") parser.add_argument("--bytes", type=int, default=0) parser.add_argument("--checkout") parser.add_argument("--revision", default="HEAD") @@ -156,17 +411,17 @@ def main(): print(json.dumps({"status": "disabled", "reason": "STORAGE_MAX_GIB=0"})) return 0 if args.action in {"snapshot", "sample"}: - report = snapshot(args.root or DEFAULT_ROOTS) - write_json(Path(args.state) / "usage.json", report, 0o640) + horizon = storage_setting(args.config, args.environment, "STORAGE_SCAN_MAX_SECONDS", SCAN_MAX_SECONDS, 120, 86400) + report = sample(args.root or DEFAULT_ROOTS, args.state, max_seconds=horizon) else: if not re.fullmatch(r"[A-Za-z0-9_.:-]{1,200}", args.id): raise ValueError("Invalid reservation identifier.") - if args.max_gib <= 0 and args.action != "release": + if args.max_gib <= 0 and args.action not in {"release", "commit"}: return 0 report = admission(args) print(json.dumps(report)) return 0 - except (OSError, ValueError, KeyError, subprocess.SubprocessError) as error: + except (OSError, ValueError, KeyError, sqlite3.Error, subprocess.SubprocessError) as error: print(f"Storage admission: {error}", file=sys.stderr) return 1 diff --git a/src/bin/getbible-sync b/src/bin/getbible-sync index dee28e8..502b3fd 100755 --- a/src/bin/getbible-sync +++ b/src/bin/getbible-sync @@ -63,7 +63,11 @@ freshness_record() { } cleanup() { if [[ "$STORAGE_RESERVED" == true ]]; then - "$GB_STORAGE_GUARD" release --state "$GB_SYNC_STORAGE_STATE" --id "$STORAGE_ID" >/dev/null || true + local storage_action=release + # A completed publication still occupies space absent from an older + # snapshot. Retain its charge until a newer complete scan covers it. + [[ "$PUBLISHED" != true ]] || storage_action=commit + "$GB_STORAGE_GUARD" "$storage_action" --state "$GB_SYNC_STORAGE_STATE" --id "$STORAGE_ID" >/dev/null || true fi [[ -z "$SWAP" ]] || rm -f -- "$SWAP" if [[ "$PUBLISHED" == false && -n "$RELEASE" ]]; then rm -rf -- "$RELEASE"; fi diff --git a/src/lib/config.sh b/src/lib/config.sh index 4d48adf..c1c9408 100644 --- a/src/lib/config.sh +++ b/src/lib/config.sh @@ -199,6 +199,7 @@ GB_GLOBAL_DEFAULTS=( "DASHBOARD_IDLE_SECONDS=60" "DASHBOARD_TOKEN_SECONDS=60" "STORAGE_MAX_GIB=0" + "STORAGE_SCAN_MAX_SECONDS=900" "ADAPTIVE_ALLOW_IDLE_SHRINK=false" "ALERT_SYNC_GRACE_SECONDS=3600" "TELEMETRY_SPOOL_MAX_GIB=1" diff --git a/src/lib/dashboard.sh b/src/lib/dashboard.sh index 701300e..b5308ab 100644 --- a/src/lib/dashboard.sh +++ b/src/lib/dashboard.sh @@ -153,6 +153,7 @@ infrastructure_environment_write() { gb_install_file "$(gb_tmpdir)/$stage.env" "$GB_RUN/$stage.env" 0640 root:getbible-dashboard || return 1 done printf 'GETBIBLE_STORAGE_MAX_GIB=%s\n' "$(gb_global STORAGE_MAX_GIB 0)" > "$(gb_tmpdir)/storage.env" + printf 'GETBIBLE_STORAGE_SCAN_MAX_SECONDS=%s\n' "$(gb_global STORAGE_SCAN_MAX_SECONDS 900)" >> "$(gb_tmpdir)/storage.env" gb_install_file "$(gb_tmpdir)/storage.env" "$GB_RUN/storage.env" 0640 "root:$GB_READERS_GROUP" || return 1 # Native and container services use the same readable effective snapshot; # the original configuration remains root controlled under /etc/getbible. @@ -190,14 +191,19 @@ infrastructure_storage_initial_sample() { # Native systemd enforces the sampler's own finite resource limits. if ! sd_start getbible-storage.service; then gb_warn 'Initial storage accounting failed. Existing APIs remain available; new budgeted publications wait for a fresh sample.' + elif [[ "$(gb_global STORAGE_MAX_GIB 0)" != 0 && ! -s "$GB_VAR/storage/usage.json" ]]; then + gb_warn 'Initial storage accounting is still in progress. The timer resumes the scan; new budgeted publications wait for its first complete sample.' fi return fi - # Container first boot has no running systemd yet. Finish the bounded local - # snapshot before any static service can request storage admission. + # Container first boot has no running systemd yet. Start one bounded slice; + # the independent timer resumes a large scan after boot. Admission refuses + # publication until the first complete snapshot exists. if ! timeout --kill-after=5s 75s prlimit --as=536870912 --cpu=60:65 -- "$GB_PYTHON" "$GB_LIBEXEC/getbible-storage-guard" "${args[@]}" >/dev/null; then gb_warn 'Initial storage accounting failed. Existing APIs remain available; new budgeted publications wait for a fresh sample.' tg_notify fail 'Initial storage accounting failed' 'Existing APIs remain available. New publications with a storage budget wait for the independent sampler to recover.' + elif [[ "$(gb_global STORAGE_MAX_GIB 0)" != 0 && ! -s "$GB_VAR/storage/usage.json" ]]; then + gb_warn 'Initial storage accounting is still in progress. The timer resumes the scan; new budgeted publications wait for its first complete sample.' fi return 0 } diff --git a/src/lib/deployment.sh b/src/lib/deployment.sh index 69eae07..28bd609 100644 --- a/src/lib/deployment.sh +++ b/src/lib/deployment.sh @@ -101,6 +101,8 @@ gb_setting_validate() { DASHBOARD_IDLE_SECONDS) [[ "$value" =~ ^[0-9]{1,4}$ ]] && (( 10#$value >= 10 && 10#$value <= 3600 )) ;; TELEMETRY_MAX_GIB|TELEMETRY_SPOOL_MAX_GIB|TELEMETRY_SPOOL_ROTATE_MIB|ALERT_COOLDOWN_SECONDS|ALERT_REMINDER_SECONDS|ALERT_HOLD_SECONDS|ALERT_SYNC_GRACE_SECONDS) [[ "$value" =~ ^[1-9][0-9]{0,5}$ ]] ;; STORAGE_MAX_GIB) [[ "$value" =~ ^[0-9]{1,6}$ ]] ;; + STORAGE_SCAN_MAX_SECONDS) + [[ "$value" =~ ^[0-9]{3,5}$ ]] && (( 10#$value >= 120 && 10#$value <= 86400 )) ;; DASHBOARD_SESSION_DAYS) [[ "$value" =~ ^[0-9]{1,2}$ ]] && (( 10#$value >= 1 && 10#$value <= 30 )) ;; DASHBOARD_TOKEN_SECONDS) [[ "$value" =~ ^[0-9]{1,2}$ ]] && (( 10#$value >= 10 && 10#$value <= 60 )) ;; DASHBOARD_DOMAIN) [[ -z "$value" ]] || gb_valid_domain "$value" ;; @@ -195,7 +197,7 @@ gb_settings_menu() { choice="$(ui_menu 'Deployment settings' "Execution: $(gb_execution_mode). Environment values are shown in the configuration report and edited in Compose." \ report 'Show effective settings and their sources' \ edit 'Set one system setting' \ - resources 'Runtime memory budget' \ + resources 'Hardware alignment and runtime memory budget' \ back 'Back')" || return 0 case "$choice" in report) ui_msg 'Effective configuration' "$(gb_environment_status)" ;; diff --git a/src/lib/resources.sh b/src/lib/resources.sh index e8bcdef..1fe8fc0 100644 --- a/src/lib/resources.sh +++ b/src/lib/resources.sh @@ -11,6 +11,9 @@ resources_plan() { budget="${GETBIBLE_MEMORY_BUDGET:-$(gb_global MEMORY_BUDGET auto)}" args=(--mode "$mode" --budget "$budget" --registry "$GB_ENDPOINTS" --runtime-root "$GB_OPT" --systemctl "$GB_SYSTEMCTL") args+=(--adaptive-state "$GB_VAR/adaptive.json") + if declare -F gb_environment_managed >/dev/null && gb_environment_managed "$GB_GLOBAL_CONF" MEMORY_BUDGET; then + args+=(--authoritative MEMORY_BUDGET) + fi for key in MEMORY_CACHE_TTL:2592000 CACHE_TTL_JITTER:0 QUERY_CPU_QUOTA:auto SEARCH_CPU_QUOTA:auto \ QUERY_WORKERS_MIN:1 QUERY_WORKERS_MAX:12 SEARCH_WORKERS_MIN:1 SEARCH_WORKERS_MAX:12 \ QUERY_THREADS_MIN:1 QUERY_THREADS_MAX:16 SEARCH_THREADS_MIN:1 SEARCH_THREADS_MAX:8 \ @@ -132,6 +135,139 @@ resources_status() { if [[ "${1:-}" == --json ]]; then resources_plan --format json; else resources_plan --format text; fi } +resources_realign_plan() { + local budget="${1:-}" format="${2:-json}" + local -a args=(--realign --config "$GB_GLOBAL_CONF" --format "$format" + --telemetry-db "$GB_VAR/telemetry/traffic.sqlite3" --environment "$GB_ENVIRONMENT_CONF" --storage-state "$GB_VAR/storage" + --storage-path "data=$GB_SRV" --storage-path "runtime=$GB_OPT" --storage-path "telemetry=$GB_VAR" + --storage-path "cache=$GB_CACHE") + [[ -z "$budget" ]] || args+=(--realign-budget "$budget") + resources_plan "${args[@]}" +} + +resources_realign_render() { + "$GB_PYTHON" - "$GB_TOOLS/getbible-resources" "$1" <<'PY' +import importlib.machinery, importlib.util, json, sys +loader = importlib.machinery.SourceFileLoader("resource_report", sys.argv[1]) +spec = importlib.util.spec_from_loader(loader.name, loader) +module = importlib.util.module_from_spec(spec) +loader.exec_module(module) +module.print_realignment(json.load(open(sys.argv[2]))) +PY +} + +resources_realign_record() { + local plan="$1" status="$2" error="$3" budget_saved="$4" + shift 4 + "$GB_PYTHON" - "$plan" "$status" "$error" "$budget_saved" "$@" <<'PY' +import json, sys +path, status, error, saved, *applied = sys.argv[1:] +with open(path) as stream: + report = json.load(stream) +report["application"] = {"status": status, "error": error, "budget_saved": saved == "true", + "applied_domains": applied, + "pending_domains": [name for name in report["domains"] if name not in applied]} +with open(path, "w") as stream: + json.dump(report, stream, indent=2) + stream.write("\n") +PY +} + +resources_realign_apply() { + local plan="$1" expected="$2" fresh budget plan_id ready changed domain error='' budget_saved=false + local -a fields=() domains=() applied=() + [[ "${GB_DRY_RUN:-false}" != true ]] || { resources_realign_render "$plan"; return 0; } + gb_require_root + [[ -f "$GB_GLOBAL_CONF" ]] || { gb_warn 'Initialize this installation before applying hardware realignment.'; return 1; } + mapfile -t fields < <("$GB_PYTHON" -c 'import json,sys; p=json.load(open(sys.argv[1])); print(p["plan_id"]); print(p["memory_budget"]["proposed"]); print(str(p["ready"]).lower()); print(str(p["memory_budget"]["change"]).lower())' "$plan") + [[ ${#fields[@]} == 4 ]] || return 1 + plan_id="${fields[0]}"; budget="${fields[1]}"; ready="${fields[2]}"; changed="${fields[3]}" + [[ -z "$expected" || "$expected" == "$plan_id" ]] || { gb_warn 'The reviewed resource plan is stale; review resources realign again.'; return 1; } + [[ "$ready" == true ]] || { gb_warn 'Hardware realignment is blocked; resolve the reason in the plan.'; return 1; } + gb_management_lock || return "$?" + fresh="$(gb_tmpdir)/resources-realign-fresh.json" + resources_realign_plan "$budget" json > "$fresh" || return 1 + "$GB_PYTHON" -c 'import json,sys; p=json.load(open(sys.argv[1])); sys.exit(0 if p["ready"] and p["plan_id"] == sys.argv[2] else 1)' "$fresh" "$plan_id" || { + gb_warn 'Hardware or resource settings changed before apply; review resources realign again.'; return 1; + } + mapfile -t domains < <("$GB_PYTHON" -c 'import json,sys; print("\n".join(json.load(open(sys.argv[1]))["domains"]))' "$plan") + # Check the first overlapping candidate under the proposed aggregate cap + # before saving anything. Subsequent transactions repeat live admission. + if [[ -n "${domains[0]:-}" ]]; then + resources_plan --budget "$budget" --candidate-domain "${domains[0]}" --format json >/dev/null || return 1 + fi + tg_notify start 'Hardware realignment started' 'Applying reviewed resource targets through normal runtime generation transactions.' + if [[ "$changed" == true ]]; then + if gb_global_set MEMORY_BUDGET "$budget"; then + budget_saved=true + else + [[ "$(cfg_get_raw "$GB_GLOBAL_CONF" MEMORY_BUDGET)" != "$budget" ]] || budget_saved=true + error='The memory budget could not be saved or its service settings could not be refreshed. Inspect settings environment before retrying.' + fi + fi + if [[ -z "$error" ]]; then + local GB_RESOURCE_RECONCILING=true + for domain in "${domains[@]}"; do + [[ -n "$domain" ]] || continue + if endpoint_apply "$domain" && resources_plan --wait-drain "$domain"; then + applied+=("$domain") + else + error="Resource application stopped at $domain; completed domains remain applied and remaining domains are pending. Rerun resources realign to review current state." + break + fi + done + fi + if [[ -n "$error" ]]; then + resources_realign_record "$plan" partial "$error" "$budget_saved" "${applied[@]}" || return 1 + tg_notify fail 'Hardware realignment incomplete' "$error" + gb_warn "$error" + return 1 + fi + resources_realign_record "$plan" complete '' "$budget_saved" "${applied[@]}" || return 1 + tg_notify ok 'Hardware realignment complete' 'Reviewed runtime resource settings were applied; management service limits and deployment caps remain unchanged.' +} + +resources_realign() { + local budget='' expected='' apply=false json=false plan result=0 + while (( $# )); do + case "$1" in + --memory-budget) [[ $# -ge 2 && -n "$2" ]] || { gb_warn '--memory-budget requires a size.'; return 1; }; budget="$2"; shift 2 ;; + --expected-plan) [[ $# -ge 2 && "$2" =~ ^[a-f0-9]{64}$ ]] || { gb_warn '--expected-plan requires a SHA-256 plan identifier.'; return 1; }; expected="$2"; shift 2 ;; + --apply) apply=true; shift ;; + --json) json=true; shift ;; + *) gb_warn 'Usage: getbible resources realign [--memory-budget SIZE] [--json] [--apply] [--expected-plan SHA256]'; return 1 ;; + esac + done + plan="$(gb_tmpdir)/resources-realign.json" + resources_realign_plan "$budget" json > "$plan" || return 1 + [[ "${GB_DRY_RUN:-false}" != true ]] || apply=false + if [[ "$apply" == true ]]; then + [[ "$json" == true ]] || resources_realign_render "$plan" + resources_realign_apply "$plan" "$expected" >&2 || result=$? + fi + if [[ "$json" == true ]]; then + cat "$plan" + elif [[ "$apply" != true ]]; then + resources_realign_render "$plan" + elif (( result == 0 )); then + printf 'Hardware realignment completed.\n' + fi + return "$result" +} + +resources_realign_menu() { + local budget plan report expected + budget="$(ui_input 'Hardware realignment' 'Memory budget (auto recalculates a native recommendation or uses the Docker allowance)' "$(gb_global MEMORY_BUDGET auto)")" || return 0 + plan="$(gb_tmpdir)/resources-realign-menu.json"; report="$(gb_tmpdir)/resources-realign-menu.txt" + resources_realign_plan "$budget" json > "$plan" || return 1 + resources_realign_render "$plan" > "$report" || return 1 + ui_textbox 'Review hardware realignment' "$report" + "$GB_PYTHON" -c 'import json,sys; sys.exit(0 if json.load(open(sys.argv[1]))["ready"] else 1)' "$plan" || return 0 + ui_yesno 'Apply hardware realignment' 'Apply this reviewed memory budget and the displayed runtime generation changes?' no || return 0 + expected="$("$GB_PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["plan_id"])' "$plan")" || return 1 + ui_run 'Apply hardware realignment' resources_realign_apply "$plan" "$expected" +} + # Before the outer nginx transaction, shrink other domains when a new endpoint # changes the global split. Their old workers drain before the new allocation # is consumed. An empty target (after removal) also grows remaining domains. @@ -175,8 +311,9 @@ resources_cli() { case "${1:-show}" in show|status) shift || true; resources_status "${1:-}" ;; --json) resources_status --json ;; + realign) shift; resources_realign "$@" ;; apply) resources_apply ;; - *) gb_die "Usage: getbible resources [show|apply] [--json]" ;; + *) gb_die "Usage: getbible resources [show|apply|realign] [--json]" ;; esac } @@ -184,8 +321,10 @@ resources_menu() { local choice while true; do choice="$(ui_menu "Runtime resources" "Memory allocation across all enabled runtime endpoints" \ + realign "Review and align settings with available hardware" \ show "Show effective resource targets" apply "Apply targets through healthy runtime generations")" || return 0 case "$choice" in + realign) resources_realign_menu || true ;; show) ui_run "Runtime resource targets" resources_status || true ;; apply) ui_run "Apply runtime resources" resources_apply || true ;; esac diff --git a/src/systemd/getbible-alert@.service.tmpl b/src/systemd/getbible-alert@.service.tmpl index f7f8aa1..cf0a94c 100644 --- a/src/systemd/getbible-alert@.service.tmpl +++ b/src/systemd/getbible-alert@.service.tmpl @@ -6,13 +6,17 @@ Type=oneshot User=root Group={{NOTIFY_GROUP}} Environment=GB_TELEGRAM_CONF={{RUN}}/telegram.conf -ExecStart={{LIBEXEC}}/getbible-notify fail "Service failed: %I" "A managed getBible service failed. Inspect its journal and the dashboard before retrying." +EnvironmentFile=-{{RUN}}/telemetry.env +# Keep the literal instance: %I unescapes hyphens into path separators. +ExecStart={{PYTHON}} {{MANAGEMENT_RELEASE}}/apps/telemetry/getbible_telemetry/service_alerts.py %i --directory {{VAR}}/telemetry/service-alerts --systemctl {{SYSTEMCTL}} --notify {{LIBEXEC}}/getbible-notify +UMask=0077 NoNewPrivileges=true ProtectSystem=strict ProtectHome=true PrivateTmp=true PrivateDevices=true RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 -TimeoutStartSec=20 +ReadWritePaths={{VAR}}/telemetry +TimeoutStartSec=30 MemoryMax=64M TasksMax=16 diff --git a/src/systemd/getbible-storage.service.tmpl b/src/systemd/getbible-storage.service.tmpl index edb347c..c967a9a 100644 --- a/src/systemd/getbible-storage.service.tmpl +++ b/src/systemd/getbible-storage.service.tmpl @@ -17,7 +17,9 @@ EnvironmentFile={{RUN}}/storage.env ExecStart={{PYTHON}} {{MANAGEMENT_RELEASE}}/bin/getbible-storage-guard sample --config {{ETC}}/getbible.conf --environment {{RUN}}/storage.env --state {{VAR}}/storage UMask=0027 # This sampler is independent of long-running deployment/adaptation jobs. -# It reads inode metadata only and exits immediately when the budget is off. +# It reads inode metadata in resumable 20-second slices, checkpoints progress, +# and exits immediately when the budget is off. The deadline bounds stuck I/O; +# completed checkpoints survive a forced termination. Nice=19 IOSchedulingClass=idle CPUQuota=50% diff --git a/src/systemd/getbible-storage.timer.tmpl b/src/systemd/getbible-storage.timer.tmpl index 24d6ac8..780032a 100644 --- a/src/systemd/getbible-storage.timer.tmpl +++ b/src/systemd/getbible-storage.timer.tmpl @@ -3,8 +3,9 @@ Description=Keep getBible storage admission accounting current [Timer] OnBootSec=10s -# With the 75-second service deadline, this interval keeps a successful sample -# within the admission guard's 120-second freshness window. +# Resume unfinished accounting without discarding completed scan work. +# Complete snapshots carry a freshness window derived from the bounded whole +# scan horizon, this 30-second interval and scheduling allowance. OnUnitInactiveSec=30s AccuracySec=1s Unit=getbible-storage.service diff --git a/tests/cli/test_dashboard_management.sh b/tests/cli/test_dashboard_management.sh index 50fe549..7d39f6e 100644 --- a/tests/cli/test_dashboard_management.sh +++ b/tests/cli/test_dashboard_management.sh @@ -70,6 +70,14 @@ gb_global_set TELEMETRY_MAX_GIB 25 grep -qFx 'GETBIBLE_TELEMETRY_MAX_GIB=25' "$GB_RUN/telemetry.env" || fail 'saved retention must reach the collector immediately' gb_global_set STORAGE_MAX_GIB 50 grep -qFx 'GETBIBLE_STORAGE_MAX_GIB=50' "$GB_RUN/storage.env" || fail 'saved storage budget must reach the independent sampler immediately' +gb_global_set STORAGE_SCAN_MAX_SECONDS 1800 +grep -qFx 'GETBIBLE_STORAGE_SCAN_MAX_SECONDS=1800' "$GB_RUN/storage.env" || fail 'saved scan horizon must reach the resumable sampler' +GETBIBLE_STORAGE_SCAN_MAX_SECONDS=2400 infrastructure_environment +grep -qFx 'GETBIBLE_STORAGE_SCAN_MAX_SECONDS=2400' "$GB_RUN/storage.env" || fail 'deployment scan horizon must override the saved value' +if gb_setting_validate STORAGE_SCAN_MAX_SECONDS 119 2>/dev/null || gb_setting_validate STORAGE_SCAN_MAX_SECONDS 86401 2>/dev/null; then + fail 'scan horizon must remain within its finite supported range' +fi +infrastructure_environment gb_global_set QUERY_WORKERS_MAX 6 if grep -q '^GETBIBLE_\(QUERY\|SEARCH\)_' "$GB_RUN/adaptive.env"; then fail 'saved resource defaults must not become authoritative deployment overrides in the controller' diff --git a/tests/cli/test_resource_realign.sh b/tests/cli/test_resource_realign.sh new file mode 100644 index 0000000..a6ad24f --- /dev/null +++ b/tests/cli/test_resource_realign.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Hardware planning stays read-only; reviewed apply uses live admission and +# reports partial work without hiding a failed generation transaction. +set -Eeuo pipefail +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +GB_PREFIX="$(mktemp -d)" +export GB_PREFIX GB_REPO_DIR="$ROOT" +for lib in core config registry resources; do + # shellcheck source=/dev/null + source "$ROOT/src/lib/$lib.sh" +done +trap 'rm -rf -- "$GB_PREFIX"; gb_cleanup' EXIT +mkdir -p "$GB_ETC" "$GB_VAR" "$GB_PREFIX/proc/1" "$GB_PREFIX/proc/self" "$GB_PREFIX/cgroup" \ + "$GB_ENDPOINTS/query.example.test/versions" "$GB_ENDPOINTS/search.example.test/versions" +printf '0::/\n' > "$GB_PREFIX/proc/1/cgroup" +printf '10 9 0:2 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n' > "$GB_PREFIX/proc/self/mountinfo" +printf '4294967296\n' > "$GB_PREFIX/cgroup/memory.max" +printf '200000 100000\n' > "$GB_PREFIX/cgroup/cpu.max" +printf 'MemTotal: 8388608 kB\nMemAvailable: 3145728 kB\n' > "$GB_PREFIX/proc/meminfo" +printf 'MEMORY_BUDGET=auto\n' > "$GB_GLOBAL_CONF" +for kind in query search; do + printf 'TYPE=runtime\nKIND=%s\nENABLED=true\n' "$kind" > "$GB_ENDPOINTS/$kind.example.test/endpoint.conf" + printf 'ENABLED=true\nWORKERS=auto\n' > "$GB_ENDPOINTS/$kind.example.test/versions/v2.conf" +done +events="$GB_PREFIX/events" +: > "$events" +resources_plan() { + local argument + local -a extra=() + [[ "${OWNED_BUDGET:-false}" != true ]] || extra+=(--authoritative MEMORY_BUDGET) + for argument in "$@"; do + if [[ "$argument" == --candidate-domain ]]; then + printf 'admission\n' >> "$events" + [[ "${BLOCK_ADMISSION:-false}" != true ]] || return 1 + fi + done + "$GB_PYTHON" "$GB_TOOLS/getbible-resources" --mode native --budget "$(cfg_get "$GB_GLOBAL_CONF" MEMORY_BUDGET auto)" \ + --proc-root "$GB_PREFIX/proc" --cgroup-root "$GB_PREFIX/cgroup" --registry "$GB_ENDPOINTS" \ + --runtime-root "$GB_OPT" --offline "${extra[@]}" "$@" +} +gb_require_root() { printf 'root\n' >> "$events"; } +gb_management_lock() { printf 'lock\n' >> "$events"; } +gb_global_set() { printf 'save %s=%s\n' "$1" "$2" >> "$events"; cfg_set "$GB_GLOBAL_CONF" "$1" "$2"; } +tg_notify() { printf 'notify %s\n' "$1" >> "$events"; } +endpoint_apply() { + printf 'apply %s\n' "$1" >> "$events" + [[ "$GB_RESOURCE_RECONCILING" == true ]] + [[ "$1" != "${FAIL_DOMAIN:-}" ]] +} + +resources_realign --json > "$GB_PREFIX/plan.json" +[[ ! -s "$events" ]] +[[ "$(cfg_get "$GB_GLOBAL_CONF" MEMORY_BUDGET)" == auto ]] +[[ ! -e "$GB_VAR/telemetry/traffic.sqlite3" ]] +GB_DRY_RUN=true resources_realign --apply --json > "$GB_PREFIX/dry-run.json" +[[ ! -s "$events" ]] +[[ "$(cfg_get "$GB_GLOBAL_CONF" MEMORY_BUDGET)" == auto ]] +plan_id="$("$GB_PYTHON" -c 'import json,sys; p=json.load(open(sys.argv[1])); assert p["ready"]; assert p["memory_budget"]["proposed"] == "2048M"; print(p["plan_id"])' "$GB_PREFIX/plan.json")" +printf 'THREADS=3\n' >> "$GB_ENDPOINTS/query.example.test/versions/v2.conf" +if resources_realign --apply --json --expected-plan "$plan_id" > "$GB_PREFIX/stale.json" 2>/dev/null; then + printf 'stale review was accepted\n' >&2; exit 1 +fi +[[ "$(cat "$events")" == root ]] +[[ "$(cfg_get "$GB_GLOBAL_CONF" MEMORY_BUDGET)" == auto ]] + +: > "$events" +if BLOCK_ADMISSION=true resources_realign --apply --json > "$GB_PREFIX/blocked.json" 2>/dev/null; then + printf 'failed live admission was ignored\n' >&2; exit 1 +fi +[[ "$(cat "$events")" == $'root\nlock\nadmission' ]] +[[ "$(cfg_get "$GB_GLOBAL_CONF" MEMORY_BUDGET)" == auto ]] + +: > "$events" +if OWNED_BUDGET=true resources_realign --apply --memory-budget 2G --json > "$GB_PREFIX/owned.json" 2>/dev/null; then + printf 'environment-owned setting was overwritten\n' >&2; exit 1 +fi +[[ "$(cat "$events")" == root ]] +[[ "$(cfg_get "$GB_GLOBAL_CONF" MEMORY_BUDGET)" == auto ]] + +: > "$events" +if FAIL_DOMAIN=search.example.test resources_realign --apply --json > "$GB_PREFIX/partial.json" 2>/dev/null; then + printf 'failed generation reported success\n' >&2; exit 1 +fi +"$GB_PYTHON" - "$GB_PREFIX/partial.json" <<'PY' +import json, sys +result = json.load(open(sys.argv[1]))["application"] +assert result["status"] == "partial" +assert result["budget_saved"] is True +assert result["applied_domains"] == ["query.example.test"] +assert result["pending_domains"] == ["search.example.test"] +PY +[[ "$(cat "$events")" == $'root\nlock\nadmission\nnotify start\nsave MEMORY_BUDGET=2048M\napply query.example.test\napply search.example.test\nnotify fail' ]] + +: > "$events" +resources_realign --apply --json > "$GB_PREFIX/applied.json" 2>/dev/null +"$GB_PYTHON" - "$GB_PREFIX/applied.json" <<'PY' +import json, sys +result = json.load(open(sys.argv[1]))["application"] +assert result["status"] == "complete" +assert result["pending_domains"] == [] +PY +[[ "$(tail -n 1 "$events")" == 'notify ok' ]] +printf 'hardware realignment previews, ownership, stale plans, admission and partial apply: ok\n' diff --git a/tests/python/test_resources.py b/tests/python/test_resources.py index b6d854b..9f92b4c 100644 --- a/tests/python/test_resources.py +++ b/tests/python/test_resources.py @@ -232,6 +232,103 @@ def test_drain_timeout_keeps_old_generation_alive(self): self.assertEqual(call.call_count, 1) self.assertEqual(call.call_args.args[0][1], "show") + def realign_args(self, **overrides): + return self.args(config=str(self.root / "global.conf"), storage_path=["data=" + str(self.root)], + storage_state=str(self.root / "storage"), telemetry_db=str(self.root / "missing.sqlite3"), + environment=str(self.root / "environment.conf"), **overrides) + + def test_native_realign_recommends_available_memory_less_host_reserve_without_mutation(self): + (self.cg / "memory.max").write_text("max") + (self.proc / "meminfo").write_text("MemTotal: 8388608 kB\nMemAvailable: 6291456 kB\n") + self.endpoint("query.example.test", "query") + result = planner.realignment(self.realign_args(mode="native")) + self.assertTrue(result["ready"]) + self.assertEqual(result["memory_budget"]["proposed"], "4096M") + self.assertFalse((self.root / "global.conf").exists()) + self.assertFalse((self.root / "missing.sqlite3").exists()) + self.assertFalse(self.runtime.exists()) + self.assertEqual(result["management"]["capacity"]["state"], "unavailable") + self.assertEqual(result["hardware"]["storage"][0]["status"], "observed") + fresh = planner.realignment(self.realign_args(mode="native", realign_budget="4096M")) + self.assertEqual(result["plan_id"], fresh["plan_id"]) + + def test_realign_respects_cgroup_headroom_and_environment_ownership(self): + (self.proc / "meminfo").write_text("MemTotal: 8388608 kB\nMemAvailable: 7340032 kB\n") + (self.cg / "memory.current").write_text(str(2048 * MIB)) + result = planner.realignment(self.realign_args(mode="native")) + self.assertEqual(result["memory_budget"]["proposed"], "1024M") + self.assertEqual(result["hardware"]["available_memory_bytes"], 2048 * MIB) + blocked = planner.realignment(self.realign_args(mode="native", authoritative=["MEMORY_BUDGET"])) + self.assertFalse(blocked["ready"]) + self.assertIn("environment-controlled", blocked["reason"]) + self.assertEqual(blocked["memory_budget"]["owner"], "environment") + + def test_realign_keeps_explicit_native_budget_and_clamps_physical_ram(self): + (self.cg / "memory.max").write_text("max") + (self.proc / "meminfo").write_text("MemTotal: 2097152 kB\nMemAvailable: 1572864 kB\n") + result = planner.realignment(self.realign_args(mode="native", budget="3G")) + self.assertEqual(result["memory_budget"]["proposed"], "3G") + self.assertFalse(result["memory_budget"]["change"]) + self.assertEqual(result["allocation"]["budget_bytes"], 2048 * MIB) + + def test_realign_cannot_guess_native_budget_or_docker_limit(self): + native = planner.realignment(self.realign_args(mode="native")) + self.assertFalse(native["ready"]) + self.assertIn("available memory", native["reason"]) + (self.cg / "memory.max").write_text("max") + (self.proc / "meminfo").write_text("MemTotal: 8388608 kB\nMemAvailable: 7340032 kB\n") + docker = planner.realignment(self.realign_args()) + self.assertFalse(docker["ready"]) + self.assertIn("finite cgroup v2", docker["reason"]) + + def test_realign_reports_budget_failure_and_deliberate_endpoint_policy(self): + self.endpoint("search.example.test", "search", WORKERS="2", CPU_QUOTA="125%", MEMORY_MAX="768M") + result = planner.realignment(self.realign_args(budget="2G", policy=["SEARCH_WARM_TRANSLATIONS=asv"], + authoritative=["SEARCH_WARM_TRANSLATIONS"])) + self.assertTrue(result["ready"]) + row = result["endpoints"][0] + self.assertEqual(row["policy_owners"]["CPU_QUOTA"], "endpoint") + self.assertEqual(row["policy_owners"]["WARM_TRANSLATIONS"], "environment") + self.assertEqual(row["target"]["CPU_QUOTA"], "125%") + failed = planner.realignment(self.realign_args(budget="512M")) + self.assertFalse(failed["ready"]) + self.assertIn("cannot fit", failed["reason"]) + + def test_realign_unchanged_query_does_not_compare_search_only_concurrency(self): + self.endpoint("query.example.test", "query") + args = self.realign_args() + settings = planner.plan(args)["endpoints"][0]["settings"] + selected = {key: value for key, value in settings.items() + if key not in {"WORKERS_MIN", "WORKERS_MAX", "SEARCH_MAX_CONCURRENT", "EXPENSIVE_CONCURRENT"}} + with patch.object(planner, "selected_settings", return_value=selected): + result = planner.realignment(args) + self.assertEqual(result["domains"], []) + self.assertEqual(result["endpoints"][0]["differences"], {}) + + def test_realign_reads_selected_limits_and_never_emits_unrelated_environment(self): + self.endpoint("query.example.test", "query") + active = self.runtime / "query/v2/active" + active.mkdir(parents=True) + (active / "limits.conf").write_text("[Service]\nMemoryMax=512M\nMemoryHigh=400M\nCPUQuota=200%\nTasksMax=64\n") + (active / "runtime.env").write_text('QUERY_WORKERS="2"\nQUERY_THREADS="4"\nQUERY_WARM_TRANSLATIONS="kjv"\nPRIVATE_PASSWORD="secret-value"\n') + result = planner.realignment(self.realign_args()) + self.assertEqual(result["endpoints"][0]["selected"]["MEMORY_MAX"], 512 * MIB) + self.assertEqual(result["endpoints"][0]["selected"]["WORKERS"], 2) + self.assertNotIn("secret-value", json.dumps(result)) + original = result["plan_id"] + (active / "limits.conf").write_text("[Service]\nMemoryMax=768M\n") + self.assertNotEqual(planner.realignment(self.realign_args())["plan_id"], original) + + def test_realign_shows_stale_storage_and_progress_without_resampling(self): + state = self.root / "storage" + state.mkdir() + (state / "usage.json").write_text(json.dumps({"generated_at": 1, "used_bytes": 100})) + (state / "progress.json").write_text(json.dumps({"status": "in_progress", "updated_at": 50})) + with patch.object(planner.time, "time", return_value=1000): + result = planner.realignment(self.realign_args()) + self.assertEqual(result["management"]["storage_accounting"]["state"], "stale") + self.assertEqual(result["management"]["storage_accounting"]["progress"]["status"], "in_progress") + if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_service_alerts.py b/tests/python/test_service_alerts.py new file mode 100644 index 0000000..059acc7 --- /dev/null +++ b/tests/python/test_service_alerts.py @@ -0,0 +1,207 @@ +"""Failures persist through retries; only real successful work can recover them.""" +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "src/apps/telemetry")) + +from getbible_telemetry.collector import Collector +from getbible_telemetry.health import HealthInspector +from getbible_telemetry.service_alerts import ServiceIncidents, main, parse_units, successful +from getbible_telemetry.store import TelemetryStore + + +UNIT = "getbible-example.service" + + +def observation(*, state="failed", result="timeout", code="2", status="15", exit_at=100, + started_at=0, substate=None): + return {"Id": UNIT, "LoadState": "loaded", "ActiveState": state, + "SubState": substate or {"failed": "failed", "inactive": "dead", "activating": "start", + "active": "running"}[state], + "Type": "oneshot", "Result": result, "ExecMainCode": code, "ExecMainStatus": status, + "ExecMainExitTimestampMonotonic": str(int(exit_at * 1_000_000)), + "ActiveEnterTimestampMonotonic": str(int(started_at * 1_000_000))} + + +class ServiceAlertsTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.incidents = ServiceIncidents(self.root / "service-alerts") + + def observe(self, values=None, now=100, unit=UNIT, **kwargs): + return self.incidents.observe(unit, values, now=now, monotonic=now, boot="boot-one", **kwargs) + + def test_immediate_failure_and_reminders_survive_new_process_state(self): + state, event = self.observe(observation()) + self.assertEqual(event, "onset") + self.assertIn("result=timeout", state["message"]) + self.incidents = ServiceIncidents(self.root / "service-alerts") + for now in (205, 310, 415, 1000, 8000): + self.assertIsNone(self.observe(observation(exit_at=now), now=now)[1]) + self.assertEqual(self.observe(observation(exit_at=86501), now=86501)[1], "reminder") + self.assertIsNone(self.observe(observation(exit_at=86606), now=86606)[1]) + + def test_activating_success_default_is_not_recovery(self): + self.observe(observation()) + for now in (130, 195, 1000): + state, event = self.observe(observation(state="activating", result="success", code="0", status="0", exit_at=0), now=now) + self.assertTrue(state["active"]) + self.assertIsNone(event) + + def test_reset_failed_is_not_successful_execution(self): + self.observe(observation()) + for now in (130, 195): + state, event = self.observe(observation(state="inactive", result="success", code="0", status="0"), now=now) + self.assertTrue(state["active"]) + self.assertIsNone(event) + + def test_completed_oneshot_recovers_once_without_waiting_past_next_timer(self): + self.observe(observation()) + success = observation(state="inactive", result="success", code="1", status="0", exit_at=160) + state, event = self.observe(success, now=170) + self.assertEqual(event, "recovered") + self.assertFalse(state["active"]) + self.assertIsNone(self.observe(success, now=300)[1]) + + def test_missing_sample_breaks_recovery_hold(self): + self.observe(observation()) + success = observation(state="active", result="success", started_at=150) + self.observe(success, now=160) + self.observe(None, now=200) + self.assertIsNone(self.observe(success, now=230)[1]) + self.assertEqual(self.observe(success, now=291)[1], "recovered") + + def test_new_boot_running_service_can_recover(self): + self.observe(observation(exit_at=10000), now=10000) + success = observation(state="active", result="success", started_at=10) + self.incidents.observe(UNIT, success, now=11000, monotonic=20, boot="boot-two") + self.assertEqual(self.incidents.observe(UNIT, success, now=11061, monotonic=81, boot="boot-two")[1], "recovered") + + def test_storage_partial_success_does_not_clear_incident(self): + unit = "getbible-storage.service" + self.observe(observation(), unit=unit) + success = observation(state="inactive", result="success", code="1", status="0", exit_at=160) + success["storage_usage"] = {"generated_at": 50, "completed_at": 90, "max_age_seconds": 900} + for now in (170, 231): + self.assertTrue(self.observe(success, unit=unit, now=now)[0]["active"]) + success["storage_usage"] = {"generated_at": 110, "completed_at": 220, "max_age_seconds": 900} + self.assertEqual(self.observe(success, unit=unit, now=240)[1], "recovered") + + def test_old_or_missing_storage_snapshot_is_not_recovery(self): + unit = "getbible-storage.service" + self.observe(observation(), unit=unit) + success = observation(state="inactive", result="success", code="1", status="0", exit_at=160) + for now in (170, 231): + self.assertIsNone(self.observe(success, unit=unit, now=now)[1]) + success["storage_usage"] = {"generated_at": 110, "completed_at": 160, "max_age_seconds": 120} + for now in (300, 361): + self.assertIsNone(self.observe(success, unit=unit, now=now)[1]) + + def test_inspector_explicitly_checks_tracked_retry_and_completed_units(self): + self.observe(observation()) + inspector = HealthInspector(incident_directory=self.incidents.directory) + current = observation(state="activating", result="success", code="0", status="0", exit_at=0) + def command(*args): + if args[0] == "show": + self.assertIn(UNIT, args) + return "\n".join(f"{key}={value}" for key, value in current.items()) + return "" + with patch.object(inspector, "_run", side_effect=command): + sample = inspector.sample(now=200) + values = sample["conditions"]["service:" + UNIT]["unit_state"] + self.assertEqual(successful(values), 0) + self.assertEqual(parse_units("Id=nginx.service\nResult=success\n\nId=not-managed.service\nResult=success"), + {"nginx.service": {"Id": "nginx.service", "Result": "success"}}) + + def test_collector_deduplicates_onfailure_and_preserves_missing_unit(self): + self.observe(observation()) # OnFailure already sent the immediate alert. + with TelemetryStore(self.root / "traffic.sqlite3") as store: + collector = Collector(store, str(self.root)) + with patch.object(collector, "_notify") as notify: + for now in (110, 180, 250): + collector.health({"services": {"available": True, "conditions": {}, "sampled_at": now, + "boot_id": "boot-one", "monotonic": now}}, {}, now=now) + self.assertEqual(notify.call_count, 0) + alerts = json.loads(store.db.execute("SELECT value FROM metadata WHERE key='health_alerts'").fetchone()[0]) + self.assertTrue(alerts["service:" + UNIT]["active"]) + + def test_existing_sqlite_service_incident_is_imported_without_second_alert(self): + with TelemetryStore(self.root / "traffic.sqlite3") as store: + store.set_metadata("health_alerts", {"service:" + UNIT: {"active": True, "last_sent": 100, "since": 40}}) + collector = Collector(store, str(self.root)) + with patch.object(collector, "_notify") as notify: + collector.health({"services": {"available": True, "sampled_at": 200, + "conditions": {"service:" + UNIT: {"unhealthy": True, "unit_state": observation()}}}}, {}, now=200) + self.assertEqual(notify.call_count, 0) + self.assertTrue(self.incidents.snapshot()[UNIT]["active"]) + + def test_alert_template_uses_literal_unit_and_shared_incident_journal(self): + template = ROOT / "src/systemd/getbible-alert@.service.tmpl" + result = subprocess.run([sys.executable, str(ROOT / "src/bin/getbible-render"), str(template), + "PYTHON=/usr/bin/python3", "MANAGEMENT_RELEASE=/opt/getbible/management/current", + "VAR=/var/lib/getbible", "SYSTEMCTL=/usr/bin/systemctl"], capture_output=True, text=True, check=True) + self.assertIn("service_alerts.py %i --directory /var/lib/getbible/telemetry/service-alerts", result.stdout) + self.assertNotIn('"Service failed: %I"', result.stdout) + + + def test_onfailure_entry_point_reports_reason_once_without_database(self): + details = "\n".join(f"{key}={value}" for key, value in observation().items()) + result = subprocess.CompletedProcess([], 0, stdout=details) + with patch("sys.argv", ["service_alerts.py", UNIT, "--directory", str(self.incidents.directory)]), \ + patch("getbible_telemetry.service_alerts.subprocess.run", return_value=result) as run: + main() + self.assertEqual(run.call_count, 2) + command = run.call_args.args[0] + self.assertEqual(command[1:3], ["fail", "Service failed: " + UNIT]) + self.assertIn("result=timeout", command[3]) + main() + self.assertEqual(run.call_count, 3) + + def test_stale_services_sample_cannot_clear_incident(self): + self.observe(observation()) + with TelemetryStore(self.root / "traffic.sqlite3") as store: + collector = Collector(store, str(self.root)) + sample = {"services": {"available": True, "sampled_at": 150, "boot_id": "boot-one", + "conditions": {"service:" + UNIT: {"unhealthy": False, + "unit_state": observation(state="inactive", result="success", code="1", status="0", exit_at=140)}}}} + with patch.object(collector, "_notify") as notify: + collector.health(sample, {}, now=300) + self.assertEqual(notify.call_count, 0) + self.assertTrue(self.incidents.snapshot()[UNIT]["active"]) + + def test_cached_previous_invocation_cannot_clear_a_later_failure(self): + self.observe({**observation(), "InvocationID": "failed-run"}) + old = {**observation(state="inactive", result="success", code="1", status="0", exit_at=50), + "InvocationID": "previous-success"} + state, event = self.incidents.observe(UNIT, old, now=120, monotonic=90, boot="boot-one") + self.assertTrue(state["active"]) + self.assertIsNone(event) + + def test_legacy_incident_requires_execution_after_import(self): + old = observation(state="inactive", result="success", code="1", status="0", exit_at=50) + state, event = self.observe(old, now=100, legacy={"active": True, "last_sent": 40}) + self.assertTrue(state["active"]) + self.assertIsNone(event) + current = observation(state="inactive", result="success", code="1", status="0", exit_at=110) + self.assertEqual(self.observe(current, now=120)[1], "recovered") + + def test_fast_restart_after_monitored_failure_is_recognized(self): + self.observe({**observation(state="active", result="timeout", started_at=99, exit_at=0), + "InvocationID": "restarted", "failure_invocation": "failed-run"}, failed=True) + current = {**observation(state="active", result="success", started_at=99), "InvocationID": "restarted"} + self.assertIsNone(self.observe(current, now=110)[1]) + self.assertEqual(self.observe(current, now=171)[1], "recovered") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_static_sync.py b/tests/python/test_static_sync.py index e620dfa..e0c1260 100644 --- a/tests/python/test_static_sync.py +++ b/tests/python/test_static_sync.py @@ -144,6 +144,26 @@ def test_effective_storage_limit_overrides_installed_limit(self) -> None: self.sync(GB_SYNC_STORAGE_MAX_GIB="1", GETBIBLE_STORAGE_MAX_GIB="000000", **settings) self.assertNotEqual(self.live.resolve(), first) + def test_published_storage_charge_survives_until_a_new_sample(self) -> None: + import time + state = self.root / "storage" + state.mkdir() + (state / "usage.json").write_text(json.dumps({"generated_at": time.time(), "used_bytes": 0})) + settings = {"GB_SYNC_STORAGE_MAX_GIB": "1", "GB_SYNC_STORAGE_STATE": str(state), + "GB_STORAGE_GUARD": str(BIN / "getbible-storage-guard")} + self.sync(**settings) + charges = json.loads((state / "reservations.json").read_text()) + self.assertEqual(len(charges), 1) + charge = next(iter(charges.values())) + self.assertGreater(charge["committed_at"], 0) + self.assertGreater(charge["bytes"], 0) + self.payload("next publication") + self.commit() + self.sync(**settings) + charges = json.loads((state / "reservations.json").read_text()) + self.assertEqual(len(charges), 2) + self.assertTrue(all(value.get("committed_at") for value in charges.values())) + def test_invalid_storage_limit_fails_before_creating_sync_directories(self) -> None: for value in ("-1", "1.5", "nan", "1000000", "1e2", "1;true", ""): with self.subTest(value=value): diff --git a/tests/python/test_storage_guard.py b/tests/python/test_storage_guard.py index bc3380b..71c7379 100644 --- a/tests/python/test_storage_guard.py +++ b/tests/python/test_storage_guard.py @@ -5,6 +5,7 @@ import json import os from pathlib import Path +import sqlite3 import tempfile import time from types import SimpleNamespace @@ -74,6 +75,181 @@ def test_snapshot_counts_hardlinked_releases_once(self): self.assertEqual(result["roots"][0]["files"], 1) self.assertFalse(result["hard_quota"]) + def corpus(self, count=5): + corpus = self.root / "corpus" + corpus.mkdir() + for number in range(count): + (corpus / str(number)).write_bytes(b"x" * 4097) + return corpus + + def first_slice(self, corpus, horizon=900): + # One entry is committed; the next entry reaches the slice deadline. + with patch.object(guard.time, "monotonic", side_effect=[0, 0, 0, 0, 1]): + return guard.sample([str(corpus)], self.root, max_seconds=horizon, slice_seconds=.5) + + def test_partial_scan_retains_usage_and_resumes_without_double_counting(self): + corpus = self.corpus() + previous = (self.root / "usage.json").read_bytes() + partial = self.first_slice(corpus) + self.assertEqual(partial["status"], "in_progress") + self.assertEqual(partial["files"], 1) + self.assertEqual((self.root / "usage.json").read_bytes(), previous) + report = guard.sample([str(corpus)], self.root) + self.assertEqual(report["status"], "complete") + self.assertEqual(report["roots"][0]["files"], 5) + self.assertEqual(report["used_bytes"], sum(path.stat().st_blocks * 512 for path in corpus.iterdir())) + self.assertEqual(report["started_at"], partial["started_at"]) + self.assertEqual(report["generated_at"], report["started_at"]) + self.assertGreaterEqual(report["completed_at"], report["started_at"]) + self.assertEqual(report["max_age_seconds"], 1840) + self.assertFalse((self.root / ".scan/scan.sqlite3").exists()) + + def test_interrupted_transaction_replays_only_uncommitted_work(self): + corpus = self.corpus(300) + iterator = guard.directory_entries + + def interrupt(descriptor, cursor): + for number, entry in enumerate(iterator(descriptor, cursor)): + if number == 270: + raise KeyboardInterrupt + yield entry + + with patch.object(guard, "directory_entries", side_effect=interrupt): + with self.assertRaises(KeyboardInterrupt): + guard.sample([str(corpus)], self.root) + with sqlite3.connect(self.root / ".scan/scan.sqlite3") as connection: + self.assertEqual(connection.execute("SELECT SUM(files) FROM roots").fetchone()[0], 256) + report = guard.sample([str(corpus)], self.root) + self.assertEqual(report["roots"][0]["files"], 300) + + def test_directory_buffer_cookie_resumes_a_wide_directory(self): + corpus = self.corpus(3000) + iterator = guard.directory_entries + visited = 0 + + def observe(descriptor, cursor): + nonlocal visited + for entry in iterator(descriptor, cursor): + visited += 1 + yield entry + + with patch.object(guard, "directory_entries", side_effect=observe), patch.object( + guard.time, "monotonic", side_effect=lambda: 1 if visited >= 1500 else 0): + partial = guard.sample([str(corpus)], self.root, slice_seconds=.5) + self.assertEqual(partial["status"], "in_progress") + with sqlite3.connect(self.root / ".scan/scan.sqlite3") as connection: + self.assertGreater(connection.execute("SELECT cursor FROM directories WHERE done=0").fetchone()[0], 0) + visited = 0 + with patch.object(guard, "directory_entries", side_effect=observe): + report = guard.sample([str(corpus)], self.root) + self.assertEqual(report["roots"][0]["files"], 3000) + self.assertLess(visited, 3000) + + def test_directory_mutation_replays_names_and_discovers_new_files(self): + corpus = self.corpus() + self.first_slice(corpus) + (corpus / "added").write_bytes(b"new") + report = guard.sample([str(corpus)], self.root) + self.assertEqual(report["roots"][0]["files"], 6) + + def test_same_name_replacement_is_counted_during_directory_replay(self): + corpus = self.corpus() + self.first_slice(corpus) + with sqlite3.connect(self.root / ".scan/scan.sqlite3") as connection: + name = os.fsdecode(connection.execute("SELECT name FROM entries LIMIT 1").fetchone()[0]) + replaced = corpus / name + old_allocated = replaced.stat().st_blocks * 512 + # Retain the old inode elsewhere to rule out filesystem inode reuse. + os.link(replaced, self.root / "retained-old-inode") + replaced.unlink() + replaced.write_bytes(b"replacement" * 2000) + report = guard.sample([str(corpus)], self.root) + current_allocated = sum(path.stat().st_blocks * 512 for path in corpus.iterdir()) + self.assertEqual(report["roots"][0]["files"], 6) + self.assertEqual(report["used_bytes"], current_allocated + old_allocated) + + def test_replaced_directory_is_not_promoted_as_a_complete_scan(self): + corpus = self.corpus() + previous = (self.root / "usage.json").read_bytes() + self.first_slice(corpus) + corpus.rename(self.root / "previous") + corpus.mkdir() + (corpus / "replacement").write_bytes(b"different") + with self.assertRaisesRegex(ValueError, "directory was replaced"): + guard.sample([str(corpus)], self.root) + self.assertEqual((self.root / "usage.json").read_bytes(), previous) + self.assertEqual(json.loads((self.root / "progress.json").read_text())["status"], "failed") + self.assertEqual(guard.sample([str(corpus)], self.root)["roots"][0]["files"], 1) + + def test_scan_horizon_fails_without_promoting_partial_accounting(self): + corpus = self.corpus() + previous = (self.root / "usage.json").read_bytes() + self.first_slice(corpus, 120) + with sqlite3.connect(self.root / ".scan/scan.sqlite3") as connection: + connection.execute("UPDATE metadata SET value=? WHERE key='started_at'", (str(time.time() - 121),)) + with self.assertRaisesRegex(TimeoutError, "120-second horizon"): + guard.sample([str(corpus)], self.root, max_seconds=120) + self.assertEqual((self.root / "usage.json").read_bytes(), previous) + self.assertFalse((self.root / ".scan/scan.sqlite3").exists()) + + def test_permission_failure_retains_previous_complete_snapshot(self): + corpus = self.corpus() + previous = (self.root / "usage.json").read_bytes() + original_open = guard.os.open + + def refuse(path, *args, **kwargs): + if str(path) == str(corpus): + raise PermissionError("denied directory") + return original_open(path, *args, **kwargs) + + with patch.object(guard.os, "open", side_effect=refuse): + with self.assertRaisesRegex(PermissionError, "denied directory"): + guard.sample([str(corpus)], self.root) + self.assertEqual((self.root / "usage.json").read_bytes(), previous) + + def test_overlapping_roots_hardlinks_symlinks_and_nonascii_names(self): + corpus = self.corpus(1) + nested = corpus / "écriture" + nested.mkdir() + (nested / "verset").write_bytes(b"text") + os.link(corpus / "0", nested / "copie") + (corpus / "loop").symlink_to(corpus, target_is_directory=True) + report = guard.snapshot([str(nested), str(corpus), str(corpus)]) + self.assertEqual(len(report["roots"]), 1) + self.assertEqual(report["roots"][0]["files"], 2) + + def test_admission_uses_scan_start_and_completed_snapshot_window(self): + now = time.time() + usage = {"generated_at": now - 200, "started_at": now - 200, "completed_at": now, + "max_age_seconds": 1840, "used_bytes": 0, "status": "complete"} + (self.root / "usage.json").write_text(json.dumps(usage)) + guard.admission(self.args(max_age=None)) + with self.assertRaisesRegex(ValueError, "stale"): + guard.admission(self.args(max_age=120, id="explicit")) + usage["started_at"] = usage["generated_at"] = now - 1841 + (self.root / "usage.json").write_text(json.dumps(usage)) + with self.assertRaisesRegex(ValueError, "stale"): + guard.admission(self.args(max_age=None)) + + def test_sequential_publications_keep_budget_charges_until_observed(self): + guard.admission(self.args()) + guard.admission(self.args(action="commit")) + guard.admission(self.args(action="release")) + with self.assertRaisesRegex(ValueError, "committed storage reservation"): + guard.admission(self.args()) + with self.assertRaisesRegex(ValueError, "exceeds STORAGE_MAX_GIB"): + guard.admission(self.args(id="two")) + reservations = json.loads((self.root / "reservations.json").read_text()) + stamp = reservations["one"]["committed_at"] + # Completion alone is insufficient: the sample must START after commit. + (self.root / "usage.json").write_text(json.dumps({"generated_at": stamp - 1, + "started_at": stamp - 1, "completed_at": time.time(), "used_bytes": 0})) + with self.assertRaisesRegex(ValueError, "exceeds STORAGE_MAX_GIB"): + guard.admission(self.args(id="two")) + (self.root / "usage.json").write_text(json.dumps({"generated_at": time.time(), "used_bytes": 0})) + guard.admission(self.args(id="two")) + self.assertNotIn("one", json.loads((self.root / "reservations.json").read_text())) + if __name__ == "__main__": unittest.main() From 11fb7426dbd842da2d1eab9f51e27092fce77c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?eW=C9=98yn?= <5607939+Llewellynvdm@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:40:03 +0200 Subject: [PATCH 6/6] Verify hardware realignment in deployments and retain failed cleanup charges --- docs/RUNTIME_RESOURCES.md | 10 ++++--- src/bin/getbible-sync | 21 +++++++++---- tests/integration/docker.sh | 51 ++++++++++++++++++++++++++++++++ tests/integration/systemd.sh | 43 +++++++++++++++++++++++++++ tests/python/test_static_sync.py | 51 ++++++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 9 deletions(-) diff --git a/docs/RUNTIME_RESOURCES.md b/docs/RUNTIME_RESOURCES.md index 7d36eab..091d654 100644 --- a/docs/RUNTIME_RESOURCES.md +++ b/docs/RUNTIME_RESOURCES.md @@ -234,10 +234,12 @@ a strict physical disk quota. Trusted sync services reserve conservative projected export space under one shared lock before export, and recheck before publication. Stale accounting or insufficient budget refuses publication and preserves current and rollback -releases. Failed exports release their reservation. Successful publications -retain a committed charge until a complete snapshot whose scan started after -publication covers the added storage. Sequential publications therefore cannot -reuse unaccounted headroom between scans. Uncommitted crashed leases expire +releases. Failed exports release their reservation only after their staged files +are removed; failed cleanup retains a committed charge for the orphaned files. +Successful publications retain a committed charge until a complete snapshot +whose scan started after publication covers the added storage. Sequential +publications therefore cannot reuse unaccounted headroom between scans. +Uncommitted crashed leases expire after 24 hours; committed charges do not expire merely with time. Snapshot and reservations live in `/var/lib/getbible/storage`. diff --git a/src/bin/getbible-sync b/src/bin/getbible-sync index 502b3fd..8af6cd7 100755 --- a/src/bin/getbible-sync +++ b/src/bin/getbible-sync @@ -62,15 +62,26 @@ freshness_record() { mv -f -- "$temporary" "$marker" } cleanup() { + local storage_action=release + [[ "$PUBLISHED" != true ]] || storage_action=commit + if [[ -n "$SWAP" ]] && ! rm -f -- "$SWAP"; then + log "WARNING: could not remove temporary publication link: $SWAP" >&2 + fi + if [[ "$PUBLISHED" == false && -n "$RELEASE" ]]; then + # Keep the estimate charged until removal succeeds. An orphaned export + # still occupies space, so retain its charge like a published release + # until complete accounting observes it; do not let its lease expire. + if ! rm -rf -- "$RELEASE" || [[ -e "$RELEASE" || -L "$RELEASE" ]]; then + storage_action=commit + log "WARNING: unpublished release cleanup failed; retaining its storage charge: $RELEASE" >&2 + fi + fi if [[ "$STORAGE_RESERVED" == true ]]; then - local storage_action=release # A completed publication still occupies space absent from an older # snapshot. Retain its charge until a newer complete scan covers it. - [[ "$PUBLISHED" != true ]] || storage_action=commit - "$GB_STORAGE_GUARD" "$storage_action" --state "$GB_SYNC_STORAGE_STATE" --id "$STORAGE_ID" >/dev/null || true + "$GB_STORAGE_GUARD" "$storage_action" --state "$GB_SYNC_STORAGE_STATE" --id "$STORAGE_ID" >/dev/null \ + || log "WARNING: storage reservation cleanup could not complete: $STORAGE_ID" >&2 fi - [[ -z "$SWAP" ]] || rm -f -- "$SWAP" - if [[ "$PUBLISHED" == false && -n "$RELEASE" ]]; then rm -rf -- "$RELEASE"; fi } on_error() { local code=$? diff --git a/tests/integration/docker.sh b/tests/integration/docker.sh index c6b453b..8e23167 100755 --- a/tests/integration/docker.sh +++ b/tests/integration/docker.sh @@ -287,6 +287,57 @@ container bash -Eeuo pipefail -c 'for kind in query search; do "/opt/getbible/$kind/v2/current/.venv/bin/python" -m pip check done' container getbible resources --json | python3 -c 'import json,sys; p=json.load(sys.stdin); assert p["enabled"] and p["cgroup_limit_bytes"] == 4*1024**3 and p["budget_bytes"] <= 4*1024**3; assert len(p["endpoints"]) == 2 and p["mcp_reserve_bytes"] >= 512*1024**2' +# A saved cache policy creates a real alignment change without changing the +# container cap or the later recreation fixture's authoritative 35% policy. +manager settings set CACHE_MEMORY_PERCENT 45 +REALIGN_CONFIG="$(container sha256sum /etc/getbible/getbible.conf)" +manager resources realign --json > "$TEST_ROOT/realign-before.json" +[[ "$(container sha256sum /etc/getbible/getbible.conf)" == "$REALIGN_CONFIG" ]] +REALIGN_PLAN="$(python3 - "$TEST_ROOT/realign-before.json" <<'PYREALIGN' +import json, sys +plan = json.load(open(sys.argv[1])) +assert plan["ready"], plan +assert len(plan["endpoints"]) == 2 and len(plan["domains"]) == 2, plan +assert plan["memory_budget"]["proposed"] == "auto" and not plan["memory_budget"]["change"], plan +print(plan["plan_id"]) +PYREALIGN +)" +manager resources realign --expected-plan "$REALIGN_PLAN" --apply --json > "$TEST_ROOT/realign-applied.json" +manager resources realign --json > "$TEST_ROOT/realign-after.json" +python3 - "$TEST_ROOT/realign-applied.json" "$TEST_ROOT/realign-after.json" <<'PYREALIGN' +import json, sys +application = json.load(open(sys.argv[1]))["application"] +assert application["status"] == "complete" and not application["pending_domains"], application +assert len(application["applied_domains"]) == 2, application +plan = json.load(open(sys.argv[2])) +assert plan["ready"] and not plan["domains"], plan +assert all(not item["differences"] for item in plan["endpoints"]), plan +print("Docker hardware alignment has no pending allocation differences") +PYREALIGN +container /usr/bin/python3 - <<'PYREALIGN' +from pathlib import Path +import subprocess + +for kind in ("query", "search"): + active = Path(f"/opt/getbible/{kind}/v2/active").resolve() + unit = f"getbible-{kind}-v2-{active.name}.service" + def unit_value(key): + return subprocess.check_output(["systemctl", "show", "--property=" + key, "--value", unit], text=True).strip() + values = dict(entry.split("=", 1) for entry in Path(f"/proc/{unit_value('MainPID')}/environ").read_text().split("\0") if "=" in entry) + cache = sum(int(values[name]) for name in ("GETBIBLE_SHARED_CORPUS_BYTES", "GETBIBLE_CHAPTER_CACHE_BYTES", "GETBIBLE_TRANSLATION_CACHE_BYTES")) + assert cache == int(unit_value("MemoryMax")) // int(values[kind.upper() + "_WORKERS"]) * 45 // 100, (kind, cache) + print(f"Docker {kind} process uses the realigned cache allocation") +PYREALIGN +runtime_generations > "$TEST_ROOT/realign-generations" +manager resources realign --apply --json > "$TEST_ROOT/realign-noop.json" +python3 -c 'import json,sys; p=json.load(open(sys.argv[1]))["application"]; assert p["status"] == "complete" and not p["applied_domains"] and not p["pending_domains"],p' "$TEST_ROOT/realign-noop.json" +runtime_generations > "$TEST_ROOT/realign-generations-noop" +cmp "$TEST_ROOT/realign-generations" "$TEST_ROOT/realign-generations-noop" +for kind in query search; do + container curl --fail --silent -H "Host: $kind.example.test" http://127.0.0.1/readyz \ + | python3 -c 'import json,sys; assert json.load(sys.stdin)["status"] == "ready"' +done +container /usr/share/getbible/api/docker/healthcheck.sh # Write from inside the running mount namespace: systemd's private temporary # mounts must not hide a fixture copied through Docker's archive endpoint. container sh -c 'cat > /run/getbible/infrastructure-ci.py' < "$ROOT/tests/integration/infrastructure.py" diff --git a/tests/integration/systemd.sh b/tests/integration/systemd.sh index 27415dc..52d23a5 100755 --- a/tests/integration/systemd.sh +++ b/tests/integration/systemd.sh @@ -378,5 +378,48 @@ check 'MCP removal retires every managed service' '' "$(systemctl list-units --t check 'MCP purge removes its runtime tree' no "$([[ -e "$MCP_ROOT" ]] && echo yes || echo no)" check 'MCP removal preserves nginx master' "$NGINX_PID" "$(systemctl show --property=MainPID --value nginx)" +# Exercise hardware alignment against actual serving generations after the +# rollback fixtures finish. These small fixture ceilings leave room for the +# existing native search generation and its replacement under a 6-GiB budget. +"$GB" settings set QUERY_MEMORY_MAX 512M +"$GB" settings set SEARCH_MEMORY_MAX 1G +REALIGN_CONFIG="$(sha256sum /etc/getbible/getbible.conf)" +"$GB" resources realign --memory-budget 6G --json > /srv/getbible-ci/realign-before.json +check 'hardware preview preserves saved configuration' "$REALIGN_CONFIG" "$(sha256sum /etc/getbible/getbible.conf)" +REALIGN_PLAN="$(/usr/bin/python3 - /srv/getbible-ci/realign-before.json <<'PYREALIGN' +import json, sys +plan = json.load(open(sys.argv[1])) +assert plan["ready"], plan +assert len(plan["endpoints"]) == 2 and plan["domains"], plan +assert plan["memory_budget"]["proposed"] == "6G", plan +print(plan["plan_id"]) +PYREALIGN +)" +start_probe +"$GB" resources realign --memory-budget 6G --expected-plan "$REALIGN_PLAN" --apply --json > /srv/getbible-ci/realign-applied.json +stop_probe +"$GB" resources realign --memory-budget 6G --json > /srv/getbible-ci/realign-after.json +/usr/bin/python3 - /srv/getbible-ci/realign-applied.json /srv/getbible-ci/realign-after.json <<'PYREALIGN' +import json, sys +application = json.load(open(sys.argv[1]))["application"] +assert application["status"] == "complete" and not application["pending_domains"], application +assert application["applied_domains"], application +plan = json.load(open(sys.argv[2])) +assert plan["ready"] and not plan["domains"], plan +assert all(not item["differences"] for item in plan["endpoints"]), plan +print("ok: native hardware alignment applied the reviewed allocation without pending differences") +PYREALIGN +REALIGN_QUERY="$(deployment query)" +REALIGN_SEARCH="$(deployment search)" +"$GB" resources realign --memory-budget 6G --apply --json > /srv/getbible-ci/realign-noop.json +/usr/bin/python3 -c 'import json,sys; p=json.load(open(sys.argv[1]))["application"]; assert p["status"] == "complete" and not p["applied_domains"] and not p["pending_domains"],p' /srv/getbible-ci/realign-noop.json +check 'repeated hardware alignment preserves query generation' "$REALIGN_QUERY" "$(deployment query)" +check 'repeated hardware alignment preserves search generation' "$REALIGN_SEARCH" "$(deployment search)" +check 'hardware alignment preserves nginx master' "$NGINX_PID" "$(systemctl show --property=MainPID --value nginx)" +check 'query enforces aligned fixture memory ceiling' 536870912 "$(systemctl show --property=MemoryMax --value "$(unit query)")" +check 'search enforces aligned fixture memory ceiling' 1073741824 "$(systemctl show --property=MemoryMax --value "$(unit search)")" +check 'query remains ready after hardware alignment' '{"status":"ready"}' "$(request "$Q" /readyz | tr -d '\n')" +check 'search remains ready after hardware alignment' '{"status":"ready"}' "$(request "$S" /readyz | tr -d '\n')" + printf '\n== %d production deployment checks passed ==\n' "$PASS" diff --git a/tests/python/test_static_sync.py b/tests/python/test_static_sync.py index e0c1260..9f78e3b 100644 --- a/tests/python/test_static_sync.py +++ b/tests/python/test_static_sync.py @@ -164,6 +164,57 @@ def test_published_storage_charge_survives_until_a_new_sample(self) -> None: self.assertEqual(len(charges), 2) self.assertTrue(all(value.get("committed_at") for value in charges.values())) + def test_failed_export_keeps_charge_until_cleanup_succeeds(self) -> None: + import time + self.sync() + current = self.live.resolve() + self.payload("failed replacement") + self.commit() + exporter = self.root / "failed-exporter" + exporter.write_text('#!/usr/bin/env bash\nprintf "partial export\\n" > "$4/partial.json"\nexit 23\n') + exporter.chmod(0o755) + commands = self.root / "cleanup-bin" + commands.mkdir() + remover = commands / "rm" + remover.write_text("""#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +if sys.argv[-1].startswith(os.environ["SYNC_TEST_RELEASES"] + "/"): + state = Path(os.environ["SYNC_TEST_STORAGE"]) + charges = json.loads((state / "reservations.json").read_text()) + # The reservation must still protect the staged files during removal. + (state / "charges-at-removal.json").write_text(json.dumps(charges)) + if os.environ["SYNC_TEST_REFUSE_REMOVAL"] == "1": + sys.exit(1) +os.execv(os.environ["SYNC_TEST_REAL_RM"], ["rm", *sys.argv[1:]]) +""") + remover.chmod(0o755) + for refuse in (False, True): + with self.subTest(refuse_removal=refuse): + state = self.root / f"storage-cleanup-{refuse}" + state.mkdir() + (state / "usage.json").write_text(json.dumps({"generated_at": time.time(), "used_bytes": 0})) + result = self.sync(success=False, GB_SYNC_STORAGE_MAX_GIB="1", GB_SYNC_STORAGE_STATE=str(state), + GB_STORAGE_GUARD=str(BIN / "getbible-storage-guard"), GB_EXPORT=str(exporter), + PATH=str(commands) + os.pathsep + os.environ["PATH"], + SYNC_TEST_RELEASES=str(self.data / "releases/v2"), SYNC_TEST_STORAGE=str(state), + SYNC_TEST_REFUSE_REMOVAL="1" if refuse else "0", SYNC_TEST_REAL_RM=shutil.which("rm")) + protected = json.loads((state / "charges-at-removal.json").read_text()) + self.assertEqual(len(protected), 1) + self.assertNotIn("committed_at", next(iter(protected.values()))) + charges = json.loads((state / "reservations.json").read_text()) + if refuse: + self.assertEqual(len(charges), 1) + self.assertGreater(next(iter(charges.values()))["committed_at"], 0) + self.assertIn("unpublished release cleanup failed", result.stderr) + else: + self.assertEqual(charges, {}) + self.assertEqual(self.live.resolve(), current) + self.assertEqual(len(list((self.data / "releases/v2").iterdir())), 2 if refuse else 1) + def test_invalid_storage_limit_fails_before_creating_sync_directories(self) -> None: for value in ("-1", "1.5", "nan", "1000000", "1e2", "1;true", ""): with self.subTest(value=value):