From 6b7d7b4a800a2bf52d4468f6101cc07f70f2dc2e Mon Sep 17 00:00:00 2001 From: Wynne Pirini Date: Thu, 15 Oct 2026 14:07:02 +1300 Subject: [PATCH] packaging(ns8): wire up the robot suite Splits tests/calnode.robot into the NS8 file-naming convention so ordering is explicit and the [ui] test can be skipped on its own, and adds a workflow to run the suite in CI. Dry-run clean: 5/5 under Robot 7.5, order verified as install -> configure -> route -> screenshots -> remove, and --exclude ui correctly drops to 4. Do not reach for the stock NethServer/ns8-github-actions test-module.yml@v1 here. It derives the image under test from the repository name (basename, minus an ns8- prefix) and forces main to :latest, which for this repo resolves to ghcr.io/calnode/calnode:latest. That is the app image, which carries no NS8 module manifest, so add-module would fail; and this repo deliberately never publishes a :latest module tag, since :latest stays pinned to releases. So this calls test-on-digitalocean-infra.yml@v1 directly, which is the reusable workflow test-module.yml delegates to anyway, and supplies the image itself. The workflow is manual-only: it needs a do_token secret this repo does not have, and a workflow_run trigger today would just fail red on every publish. Both the missing trigger and the image reasoning are documented in the header. Also git-ignores tests/outputs/, which robot writes output.xml, log.html and report.html into, and adds a README Testing section. --- .github/workflows/test-module.yml | 84 +++++++++++++++++++++++++++ .gitignore | 3 + packaging/ns8/README.md | 43 ++++++++++++-- packaging/ns8/tests/10__calnode.robot | 25 ++++++++ packaging/ns8/tests/15__ui.robot | 37 ++++++++++++ packaging/ns8/tests/20__remove.robot | 8 +++ packaging/ns8/tests/calnode.robot | 57 ------------------ 7 files changed, 194 insertions(+), 63 deletions(-) create mode 100644 .github/workflows/test-module.yml create mode 100644 packaging/ns8/tests/10__calnode.robot create mode 100644 packaging/ns8/tests/15__ui.robot create mode 100644 packaging/ns8/tests/20__remove.robot delete mode 100644 packaging/ns8/tests/calnode.robot diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml new file mode 100644 index 00000000..a05cd5bf --- /dev/null +++ b/.github/workflows/test-module.yml @@ -0,0 +1,84 @@ +name: Test module (robot) + +# Robot Framework integration tests for the NS8 module. The suite in +# packaging/ns8/tests installs the module on a real NS8 cluster, configures it, +# checks the route, screenshots the UI and removes it again. +# +# WHY THIS DOES NOT USE THE STOCK NS8 WORKFLOW +# +# NethServer/ns8-github-actions provides test-module.yml@v1, the one-line +# entry point every NS8 module uses. It cannot be used here. It derives the +# image under test from the repository name: +# +# name = basename(repository), with any leading "ns8-" stripped +# owner = lowercased repo owner +# tag = head branch, with "main" forced to "latest" +# image = ghcr.io//: +# +# For this repository that yields ghcr.io/calnode/calnode:latest, which is two +# wrong answers at once: +# * the module image is ghcr.io/calnode/ns8-calnode, not ghcr.io/calnode/calnode +# (the latter is the Calnode application image, which has no NS8 module +# manifest, so add-module would fail); +# * this repo deliberately never publishes a :latest module tag. :latest stays +# pinned to releases, and main publishes :edge, exactly as docker-publish.yml +# does for the app image. +# +# So this workflow calls test-on-digitalocean-infra.yml@v1 directly, which is +# the reusable workflow test-module.yml delegates to anyway, and supplies the +# image URL itself via the `args` input. Everything else (DigitalOcean +# provisioning, NS8 install, create-cluster, the test runner, artifact upload, +# tmate debugging) is still theirs and unmodified. +# +# NOT RUNNING AUTOMATICALLY YET +# +# It needs a `do_token` repository secret (DigitalOcean API token) for the +# cluster provisioning, and this repo does not have one. Declaring the workflow +# with a workflow_run trigger now would just fail red on every publish, so it is +# manual-only until that is sorted out. +on: + workflow_dispatch: + inputs: + image: + description: 'Module image under test' + required: false + type: string + default: 'ghcr.io/calnode/ns8-calnode:edge' + run_ui_tests: + description: 'Run the [ui] tests (slower, needs the Playwright image)' + required: false + type: boolean + default: false + debug_shell: + description: 'Hand out a tmate debug shell on failure' + required: false + type: boolean + default: false +# To run this on every successful publish instead of by hand, add: +# workflow_run: +# workflows: ['NS8 module publish'] +# types: [completed] +# ...after setting the do_token secret. + +jobs: + test: + name: Run tests + permissions: + contents: read + pull-requests: write + statuses: write + uses: NethServer/ns8-github-actions/.github/workflows/test-on-digitalocean-infra.yml@v1 + with: + # packaging/ns8 holds both the module definition and tests/, so this is the + # module root for the purposes of the runner. + path: packaging/ns8 + # `args` is the second positional argument to run-ns8-tests, i.e. the + # IMAGE_URL that the suite passes to add-module. See the header note. + args: ${{ inputs.image }} + # One leader node. The default is "dn1 rl1" (two droplets); the suite is + # single-node, so a second one only doubles the cost. + leader_nodes: rl1 + run_ui_tests: ${{ inputs.run_ui_tests }} + debug_shell: ${{ inputs.debug_shell }} + secrets: + do_token: ${{ secrets.do_token }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 79fbf573..e32f2a3a 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,6 @@ pnpm-debug.log* # Vitest browser-mode artifacts (failure screenshots / attachments) __screenshots__/ .vitest-attachments/ + +# NS8 Robot Framework test output (output.xml, log.html, report.html, screenshots) +packaging/ns8/tests/outputs/ diff --git a/packaging/ns8/README.md b/packaging/ns8/README.md index 7ba9258c..ee70894f 100644 --- a/packaging/ns8/README.md +++ b/packaging/ns8/README.md @@ -114,8 +114,11 @@ byte-identical across reconfigures, `remove-module --no-preserve` with automatic route cleanup, and a full cluster backup → destroy → restore cycle: after restoring, the app served 200, logged "keyvault: DEK unwrapped from keystore", the encryption-key fingerprint was unchanged, and an event type -created before the backup was still there. Still untested live: the -automated `tests/` robot suite, which needs a test cluster. +created before the backup was still there. + +The `tests/` robot suite is wired up and dry-run clean (5/5, Robot 7.5), but it +has still never been executed against a live cluster. Every live check above +was done by hand. See [Testing](#testing). ## Uninstall @@ -168,10 +171,38 @@ reference into the module environment as `CALNODE_IMAGE`, which is what the systemd unit and configure-module actually run. The label is the single source of truth; nothing is stamped at build time. -Running the test suite needs an NS8 test cluster; see the -[NS8 testing docs](https://github.com/NethServer/ns8-github-actions/blob/v1/README.md#running-tests-locally). -From this directory: `run-ns8-tests ghcr.io/calnode/ns8-calnode:edge`. -Not yet run in CI: no test cluster is wired up. +## Testing + +The suite in `tests/` is Robot Framework. It installs the module on a real NS8 +cluster, configures it, checks the route responds, screenshots the settings UI +and removes the module again: + +| file | what it does | +|---|---| +| `10__calnode.robot` | install, configure, route responds | +| `15__ui.robot` | screenshots of the Status and Settings pages, tagged `ui` | +| `20__remove.robot` | removal | + +Locally, against a cluster you already have (see the +[NS8 testing docs](https://github.com/NethServer/ns8-github-actions/blob/v1/README.md#running-tests-locally)): + +``` +run-ns8-tests ghcr.io/calnode/ns8-calnode:edge +``` + +`NODE_ADDR` must be reachable by SSH as root, using the key at `~/.ssh/id_ecdsa` +or wherever `SSH_KEYFILE` points. Add `RUN_UI_TESTS=true` to include the `ui` +tests; without it they are skipped and the runner uses a lighter image. + +In CI, `.github/workflows/test-module.yml` does the same thing and provisions +the cluster on DigitalOcean. It is manual-only for now because it needs a +`do_token` repository secret this repo does not have yet. It also deliberately +does not use the stock `test-module.yml@v1`, which derives the image under test +from the repository name and would resolve it to `ghcr.io/calnode/calnode:latest` +instead of `ghcr.io/calnode/ns8-calnode:edge`; the workflow header explains the +full reasoning. + +Output lands in `tests/outputs/` and is git-ignored. ## UI translation diff --git a/packaging/ns8/tests/10__calnode.robot b/packaging/ns8/tests/10__calnode.robot new file mode 100644 index 00000000..08672200 --- /dev/null +++ b/packaging/ns8/tests/10__calnode.robot @@ -0,0 +1,25 @@ +*** Settings *** +Library SSHLibrary + +*** Variables *** +${TEST_HOST} calnode.example.test +${module_id} ${EMPTY} + +*** Test Cases *** +Check if ns8-calnode is installed correctly + ${output} ${rc} = Execute Command add-module ${IMAGE_URL} 1 + ... return_rc=True + Should Be Equal As Integers ${rc} 0 + &{output} = Evaluate ${output} + Set Global Variable ${module_id} ${output.module_id} + +Check if ns8-calnode can be configured + ${rc} = Execute Command api-cli run module/${module_id}/configure-module --data '{"host": "${TEST_HOST}", "lets_encrypt": false, "http2https": false}' + ... return_rc=True return_stdout=False + Should Be Equal As Integers ${rc} 0 + +Check if ns8-calnode works as expected + # Host-based route: pass the hostname explicitly since it does not resolve here + ${rc} = Execute Command curl -f -H 'Host: ${TEST_HOST}' http://127.0.0.1/ + ... return_rc=True return_stdout=False + Should Be Equal As Integers ${rc} 0 \ No newline at end of file diff --git a/packaging/ns8/tests/15__ui.robot b/packaging/ns8/tests/15__ui.robot new file mode 100644 index 00000000..b6696f5b --- /dev/null +++ b/packaging/ns8/tests/15__ui.robot @@ -0,0 +1,37 @@ +*** Settings *** +Documentation Settings-UI screenshots. Excluded automatically unless the runner +... is started with RUN_UI_TESTS=true. + +*** Variables *** +${ADMIN_USER} admin +${ADMIN_PASSWORD} Nethesis,1234 + +*** Keywords *** +Login to cluster-admin + New Page https://${NODE_ADDR}/cluster-admin/ + Fill Text text="Username" ${ADMIN_USER} + Click button >> text="Continue" + Fill Text text="Password" ${ADMIN_PASSWORD} + Click button >> text="Log in" + Wait For Elements State css=#main-content visible timeout=10s + +*** Test Cases *** +Take screenshots + [Tags] ui + # Imported here rather than in *** Settings *** on purpose: the runner uses a + # different image, without robotframework-browser, whenever RUN_UI_TESTS is + # unset. A suite-level import would fail the whole run before the [ui] + # exclusion could skip this one test. + Import Library Browser + New Browser chromium headless=True + New Context ignoreHTTPSErrors=True + Login to cluster-admin + Go To https://${NODE_ADDR}/cluster-admin/#/apps/${module_id} + Wait For Elements State iframe >>> h2 >> text="Status" visible timeout=10s + Sleep 5s + Take Screenshot filename=${OUTPUT DIR}/browser/screenshot/1._Status.png + Go To https://${NODE_ADDR}/cluster-admin/#/apps/${module_id}?page=settings + Wait For Elements State iframe >>> h2 >> text="Settings" visible timeout=10s + Sleep 5s + Take Screenshot filename=${OUTPUT DIR}/browser/screenshot/2._Settings.png + Close Browser \ No newline at end of file diff --git a/packaging/ns8/tests/20__remove.robot b/packaging/ns8/tests/20__remove.robot new file mode 100644 index 00000000..fddd61c7 --- /dev/null +++ b/packaging/ns8/tests/20__remove.robot @@ -0,0 +1,8 @@ +*** Settings *** +Library SSHLibrary + +*** Test Cases *** +Check if ns8-calnode is removed correctly + ${rc} = Execute Command remove-module --no-preserve ${module_id} + ... return_rc=True return_stdout=False + Should Be Equal As Integers ${rc} 0 \ No newline at end of file diff --git a/packaging/ns8/tests/calnode.robot b/packaging/ns8/tests/calnode.robot deleted file mode 100644 index 1eafc657..00000000 --- a/packaging/ns8/tests/calnode.robot +++ /dev/null @@ -1,57 +0,0 @@ -*** Settings *** -Library SSHLibrary - -*** Variables *** -${ADMIN_USER} admin -${ADMIN_PASSWORD} Nethesis,1234 -${TEST_HOST} calnode.example.test -${module_id} ${EMPTY} - -*** Keywords *** -Login to cluster-admin - New Page https://${NODE_ADDR}/cluster-admin/ - Fill Text text="Username" ${ADMIN_USER} - Click button >> text="Continue" - Fill Text text="Password" ${ADMIN_PASSWORD} - Click button >> text="Log in" - Wait For Elements State css=#main-content visible timeout=10s - -*** Test Cases *** -Check if ns8-calnode is installed correctly - ${output} ${rc} = Execute Command add-module ${IMAGE_URL} 1 - ... return_rc=True - Should Be Equal As Integers ${rc} 0 - &{output} = Evaluate ${output} - Set Global Variable ${module_id} ${output.module_id} - -Take screenshots - [Tags] ui - Import Library Browser - New Browser chromium headless=True - New Context ignoreHTTPSErrors=True - Login to cluster-admin - Go To https://${NODE_ADDR}/cluster-admin/#/apps/${module_id} - Wait For Elements State iframe >>> h2 >> text="Status" visible timeout=10s - Sleep 5s - Take Screenshot filename=${OUTPUT DIR}/browser/screenshot/1._Status.png - Go To https://${NODE_ADDR}/cluster-admin/#/apps/${module_id}?page=settings - Wait For Elements State iframe >>> h2 >> text="Settings" visible timeout=10s - Sleep 5s - Take Screenshot filename=${OUTPUT DIR}/browser/screenshot/2._Settings.png - Close Browser - -Check if ns8-calnode can be configured - ${rc} = Execute Command api-cli run module/${module_id}/configure-module --data '{"host": "${TEST_HOST}", "lets_encrypt": false, "http2https": false}' - ... return_rc=True return_stdout=False - Should Be Equal As Integers ${rc} 0 - -Check if ns8-calnode works as expected - # Host-based route: pass the hostname explicitly since it does not resolve here - ${rc} = Execute Command curl -f -H 'Host: ${TEST_HOST}' http://127.0.0.1/ - ... return_rc=True return_stdout=False - Should Be Equal As Integers ${rc} 0 - -Check if ns8-calnode is removed correctly - ${rc} = Execute Command remove-module --no-preserve ${module_id} - ... return_rc=True return_stdout=False - Should Be Equal As Integers ${rc} 0