From b8b6878e00b27cdc5d435cd1b846ac12dbb5516c Mon Sep 17 00:00:00 2001 From: balajikumaran-c-s Date: Sat, 10 Oct 2026 00:57:03 +0530 Subject: [PATCH 1/3] test(orchestrator): update verification library and unit tests Prepare the shared layer that the lifecycle suites build on. - omnia_auto lists skip reasons under the summary table, drops the expected/unexpected/framework skip classification, and the wheel is rebuilt. add_session_result takes an optional reason, so other domains are unchanged. - The test case registry gains the OIM negative cases, the local etcd media case, renumbered duplicate IDs, and per-suite order blocks. - The verification library adds local etcd media, control-plane reboot recovery, OIM negative inputs, and LDAP identity gating. Node removal keeps the heredoc delimiter off the PXE mapping and needs at least two Slurm compute nodes. Recovery helpers check the reboot authorization. Host-key cleanup runs through run_on_host, and progress and cleanup messages use omnia_auto log. Unused imports are removed. - Unit tests cover the etcd contracts, heredoc writes, the two-node removal minimum, host-key cleanup and progress logging. - The test automation standard and test_config documentation describe the new rules and settings. Refs #4849 Signed-off-by: balajikumaran-c-s --- docs/code-style/test_automation.md | 221 ++++++++++++++- test/orchestrator/docs/test_config.md | 9 + .../library/functions/__init__.py | 14 + .../library/functions/_powervault_helpers.py | 2 - .../library/functions/_pxeboot_helpers.py | 10 +- .../library/functions/_workload_helpers.py | 14 + .../functions/apptainer_jobs_pxeboot_func.py | 24 +- .../apptainer_recovery_pxeboot_func.py | 4 +- .../apptainer_runtime_pxeboot_func.py | 19 +- .../library/functions/dcgm_pxeboot_func.py | 6 +- .../functions/hpc_benchmarks_pxeboot_func.py | 2 - .../kubernetes_recovery_pxeboot_func.py | 83 +++++- .../kubernetes_runtime_pxeboot_func.py | 268 +++++++++++------- .../functions/minimal_os_pxeboot_func.py | 1 - .../functions/mount_config_precheck_func.py | 1 - .../functions/mount_config_pxeboot_func.py | 3 - .../functions/oim_readiness_precheck_func.py | 83 +++--- .../functions/powervault_pxeboot_func.py | 7 +- .../functions/slurm_auth_pxeboot_func.py | 96 ++++--- .../functions/slurm_jobs_pxeboot_func.py | 35 +-- .../functions/slurm_lifecycle_pxeboot_func.py | 47 +-- .../functions/slurm_recovery_pxeboot_func.py | 10 +- .../functions/vast_storage_pxeboot_func.py | 3 - .../library/messages/precheck_msgs.py | 8 + test/orchestrator/library/vars/__init__.py | 6 + .../library/vars/precheck_vars.py | 14 + .../orchestrator/library/vars/pxeboot_vars.py | 25 +- .../library/vars/test_case_vars.py | 156 ++++++++-- test/orchestrator/test_config.yml | 15 +- test/orchestrator/ut/README.md | 8 + .../test_additional_cloud_init_contracts.py | 12 +- .../orchestrator/ut/test_cleanup_contracts.py | 13 +- .../ut/test_kubernetes_etcd_contracts.py | 213 ++++++++++++++ .../orchestrator/ut/test_pxeboot_contracts.py | 137 ++++++++- .../dist/omnia_auto-1.0.0-py3-none-any.whl | Bin 88984 -> 88267 bytes test/plugins/docs/06_report.md | 1 - .../omnia_auto/functions/formatting_func.py | 13 + .../omnia_auto/functions/report_func.py | 45 --- .../omnia_auto/functions/report_html.py | 38 +-- 39 files changed, 1254 insertions(+), 412 deletions(-) create mode 100644 test/orchestrator/ut/test_kubernetes_etcd_contracts.py diff --git a/docs/code-style/test_automation.md b/docs/code-style/test_automation.md index 30f7896afe..91147fe739 100644 --- a/docs/code-style/test_automation.md +++ b/docs/code-style/test_automation.md @@ -161,9 +161,10 @@ test// │ └── _msgs.py [required] ├── fvt/ │ ├── README.md [required] -│ └── / -│ ├── test_.py [when the tag itself is verified] -│ └── /test_.py [when suites divide the tag] +│ └── / [one per supported playbook tag] +│ ├── test_playbook.py [required; playbook execution only] +│ └── / [one per functional component] +│ └── test_.py [verification of that component] ├── nft/ │ ├── README.md [when NFT is supported] │ └── test_.py [when NFT is supported] @@ -181,6 +182,93 @@ test// Do not create empty placeholder capabilities. For example, omit `nft/` when there is no defined non-functional contract. +### 3.1 FVT tag folders + +Derive the tag folders from the product entry playbook, for example +`src//playbooks/.yml`: + +1. Every tag that a user can select with `--tags ` gets one + `fvt//` folder and one `FVT_TAGS` entry. The folder name is the + playbook tag, unchanged. +2. `always` and `never` are Ansible control tags, not folders. A tag that + exists only under `never` (for example `cleanup`) is still a folder when + users run it, and belongs in `EXCLUDE_TAGS`. +3. A pure alias that runs the same plays as other tags (for example the + Orchestrator `deploy` or `execute` tags) is not a separate folder. The + README names the alias and the folders it maps to. +4. A verification-only capability with no playbook tag needs a documented + reason in the README before it gets a folder. + +Every tag folder MUST contain `test_playbook.py`. It holds only the +execution (`E`) case for that tag: it runs the entry playbook with +`--tags ` through `omnia_auto.run_playbook`, renders the result, and +asserts it. It contains no verification checks. + +If the playbook needs extra variables for a non-interactive run, add them to +`test_config.yml` with a comment that states the supported values and +default. A domain helper reads and validates them and returns the +`extra_vars` mapping. Never hardcode them in the test file. Secrets are never +extra variables; the product reads them from its domain credential file. + +Suite folders under a tag are functional components of that tag, for +example `fvt/prepare/openldap/` or `fvt/pxeboot/kubernetes_etcd/`. Each suite +name is listed in `SUITES[]`. Verification (`V`) cases live only in +suite folders. + +Example from the Orchestrator domain: + +```text +fvt/ +├── precheck/ +│ ├── test_playbook.py # orchestrator.yml --tags precheck +│ ├── environment/ +│ └── storage/ +├── prepare/ +│ ├── test_playbook.py # orchestrator.yml --tags prepare +│ ├── openchami/ +│ └── openldap/ +└── cleanup/ # never-tag; listed in EXCLUDE_TAGS + ├── test_playbook.py # orchestrator.yml --tags cleanup + └── kubernetes/ +``` + +### 3.2 Naming conventions + +| Item | Convention | Example | +|---|---|---| +| Tag folder | Playbook tag, unchanged | `fvt/pxeboot/` | +| Suite folder | Component name, lowercase `snake_case` | `kubernetes_etcd/` | +| Execution test file | `test_playbook.py` | `fvt/prepare/test_playbook.py` | +| Verification test file | `test_.py` or `test__.py` | `test_local_etcd.py` | +| Execution test function | `test_deploy_` | `test_deploy_pxeboot` | +| Verification test function | `test__` | `test_kubernetes_local_etcd_media` | +| Negative test function | `test_neg__`; file `test__negative.py` when split | `test_neg_pxe_nic_missing` | +| Registry key | Test function name without `test_` | `kubernetes_local_etcd_media` | +| Function module | `__func.py`; private helpers start with `_` | `kubernetes_runtime_pxeboot_func.py` | +| Check function | `check__` | `check_kubernetes_local_etcd_media` | +| Variable module | `_vars.py` for tag data; `test_case_vars.py` for metadata | `pxeboot_vars.py` | +| Message module | `_msgs.py` with `TEST_LOG_MSGS` and `TEST_ASSERT_MSGS`, exported as `_TEST_LOG_MSGS` | `pxeboot_msgs.py` | +| Constant | `UPPER_SNAKE_CASE` | `ETCD_HEALTH_WAIT_TIMEOUT_SECONDS` | +| Marker | Lowercase `snake_case`, registered before use | `kubernetes_etcd` | + +Names describe the behavior under test, not the ticket, lab, or author. +A test function name and its registry key stay stable after release; rename +only with the ID migration described in section 10. + +### 3.3 Execution order + +When a domain orders tests with `@pytest.mark.order`, give every value one +owner. Allocate a block per lifecycle tag and a sub-block of 100 per suite: + +```text +order = lifecycle base + suite slot * 100 + position in suite +``` + +Record the bases and suite slots in one table in `test_case_vars.py`. A new +test takes the next free number in its own suite; a new suite appends a new +block. Never reuse an order value. Suites that reboot, drain, or remove nodes +run last in their tag, and destructive cases run last in their suite. + Generated local files MUST be ignored: ```text @@ -263,6 +351,39 @@ Optional NFT and UT use: Replace placeholders with the real domain name in its README and state its safe lifecycle order. Identify destructive cleanup explicitly. +### 4.1 README contents + +Write each README for the operator who runs the tests. Keep content that +code already enforces out of the README; link to the owning file instead. + +`test//README.md` MUST contain, in this order: + +1. purpose and safety model, including every destructive tag; +2. prerequisites on the development host and the execution OIM; +3. quick start using the first-use workflow above; +4. environment setup: install modes, then the test and domain credential + stores with their `setup_env.sh` options and fields; +5. local and remote execution, and input/output synchronization; +6. tags and suites: one row per tag with its suites and contract, plus any + playbook alias such as `deploy`; +7. options and marker expressions; +8. lifecycle examples per tag, with the `test_config.yml` keys each one uses; +9. NFT, batch execution, reports, and UT entry points; +10. directory structure. + +`fvt/README.md` MUST contain the test-case ID format, the order allocation, +and one table per tag and suite with these columns: order, ID, function, +suite, markers, what the test checks, and the pass condition. Record +retired or reassigned IDs and release traceability in their own tables. + +`nft/README.md` lists each contract, its threshold source and unit, and its +order. `ut/README.md` lists each UT file with the product behavior it +covers and the command that runs it. + +`docs/test_config.md`, `docs/test_run_config.md`, and `docs/test_creds.md` +describe every key or field with its type, default, and effect. Update them +in the same change as the key. + --- ## 5. Credential Contract @@ -274,6 +395,20 @@ safe lifecycle order. Identify destructive cleanup explicitly. | Transport | Reach execution OIM | Local encrypted credential pair | Developer or CI job | | Domain | Authenticate product targets | Project input on execution OIM | Project operator | +The two setup options map to these scopes: + +| Option | Writes | Contains | Read by | +|---|---|---|---| +| `--set-creds` | `test_creds.yml` and `.test_creds.key` in `test//` | Secrets that only the test automation needs: the OIM SSH password, test-only identities such as an LDAP login-test user, and secrets for test-side setup that the product playbook never reads | The test framework | +| `--set-domain-creds` | `_credentials.yml` in the project input directory on the execution OIM | The credentials the Omnia playbook itself reads, with the same file name, fields, and vault format the product expects | The product playbook | + +Decide by asking who reads the value. If the playbook reads it, it is a domain +credential, even when a test also needs it; the test reads it from the domain +store. If only the test reads it, it is a test credential. Never copy a value +into both stores. When a test credential is needed only for an optional +feature, the prompt asks for it only when `test_config.yml` enables that +feature. + The local transport pair is `test_creds.yml` and `.test_creds.key`. Both MUST have mode `0600`, be encrypted at rest, and be gitignored. Passwordless SSH may make transport credentials optional. @@ -360,6 +495,26 @@ channel. Fake examples MUST be unmistakably synthetic. `setup_env.sh` owns environment preparation and credential provisioning. It MUST NOT run a domain playbook or validation suite. +Logic: + +```text +parse and reject incompatible options + -> select install mode (--user, or .venv/ with --venv) + -> install requirements.txt, including the omnia_auto wheel + -> credential option given? + --set-creds / --update-creds / --creds-stdin + -> prompt or read the test credential fields + -> encrypt into test_creds.yml with .test_creds.key (0600) + --set-domain-creds / --update-domain-creds / --domain-creds-stdin + -> resolve $OMNIA_DATA_PATH//input/$OMNIA_PROJECT_NAME/ + -> prompt or read the product credential fields + -> write _credentials.yml in the product vault format + -> print the next safe commands +``` + +Credential prompting, encryption, and vault writes come from `omnia_auto`. +The domain script supplies only its field specification and file names. + Required behavior: - default: install missing requirements into the active or documented user @@ -466,7 +621,7 @@ For every FVT tag, all of these MUST agree: 1. product playbook tag or documented verification-only capability; 2. entry in `FVT_TAGS`; -3. physical `fvt//` directory; +3. physical `fvt//` directory with its `test_playbook.py`; 4. `SUITES[]` entries and physical suite directories; 5. registered pytest markers used beneath the tag; 6. `test_run_config.yml` entry; @@ -643,6 +798,18 @@ atomically. Register every marker before use. Markers represent useful selection axes such as architecture, source, sink, feature, or `sanity`, not temporary labs. +A run without a marker expression runs every collected test, including +functional and negative tests, except tests that reboot nodes or remove and +re-add nodes; the `exec`/`verify` phase rule for execution cases still applies. +A test that reboots a node MUST carry `reboot`, and a test that removes or +re-adds a node MUST carry `node_lifecycle`. Such tests are deselected unless +the marker expression selects their marker, even when their suite is named +explicitly. Apart from `reboot` and `node_lifecycle`, markers narrow a run; +they are not required to authorize one. +`sanity` marks only the baseline positive checks that must pass on a healthy +system. A negative test, one that expects a rejection or failure, MUST NOT +carry `sanity`; mark it `negative`. Do not add `sanity` to an existing test +without the owner's agreement. `domain_vars.py`, pytest configuration, README, batch examples, and decorators must agree. @@ -717,6 +884,13 @@ UT exercises isolated parsing, validation, formatting, and decisions. It does not require SSH, a cluster, appliance, live API, or real credentials. Mock at the I/O boundary, not inside the tested logic. +UT covers product behavior and the logic of domain helpers, such as parsers, +classifiers, result decisions, and skip gates. Do not write UT that only +checks the test framework's own layout: naming conventions, ID formats, +order numbers, marker registration, registry lookups, or README tables. +Those belong in code review and the `test-domain-compliance` skill. Give +every UT a unique `_UT_` ID in its docstring. + A UT may read product files only when the domain explicitly declares a source-contract capability. That exception does not expand compliance scoring to product source or make source inspection mandatory for other domains. @@ -760,11 +934,23 @@ interface even when the declared version is unchanged. python -c "import omnia_auto; print(sorted(omnia_auto.__all__))" ``` -11. Run focused checks for every affected domain. -12. Commit plugin source, docs, wheel, and required consumers together. +11. Find every consumer of each changed or removed symbol, field, or output + shape across all domains: + + ```bash + grep -rn '' test --include='*.py' --include='*.sh' \ + --exclude-dir=plugins + ``` + + Update each consumer in the same change, then run its import check, + collection (`pytest --collect-only`), and UT. A domain that does not use + the changed behavior still needs a collection check with the new wheel. +12. Commit and push plugin source, docs, the rebuilt wheel, and the required + consumers together, in one change. Never push source without its rebuilt + wheel; `setup_env.sh` installs the wheel, not the source. -Do not publish to PyPI, change the version, commit a venv, or push unless the -maintainer explicitly requests it. +Do not publish to PyPI, change the version, or commit a venv unless the +release owner requests it. --- @@ -880,8 +1066,10 @@ Create a human-authored DCO commit: git commit -s -m "test(): describe the behavior change" ``` -Configured `user.name` and `user.email` identify the human author. Never -fabricate another person's signoff or add an AI identity trailer. +Configured `user.name` and `user.email` identify the human author and must be +the author's GitHub user name and email, so the DCO `Signed-off-by` matches the +commit author. Never fabricate another person's signoff or add an AI identity +trailer, such as an AI `Co-Authored-By` line. --- @@ -894,14 +1082,20 @@ fabricate another person's signoff or add an AI identity trailer. - [ ] Directory tree contains only supported capabilities. - [ ] README first-use flow is complete and safe. - [ ] Domain, tags, suites, markers, directories, and batch config agree. +- [ ] Each supported playbook tag has `fvt//test_playbook.py` with + only the execution case; verification lives in suite folders. +- [ ] Playbook extra variables come from `test_config.yml`. +- [ ] Names follow section 3.2 and order values are unique per section 3.3. - [ ] Destructive tags are excluded from implicit execution. -- [ ] Transport and domain credentials are separate and encrypted. +- [ ] Test credentials (`--set-creds`) and playbook credentials + (`--set-domain-creds`) are separate and encrypted. - [ ] Non-interactive secrets use bounded standard input only. - [ ] Tests orchestrate; helpers contain reusable logic. - [ ] Commands, variables, messages, and metadata are centralized. - [ ] IDs and titles are unique and not hardcoded at call sites. - [ ] Datasets contain no credentials and sync in the declared direction. -- [ ] FVT, NFT, and UT boundaries are respected. +- [ ] FVT, NFT, and UT boundaries are respected; UT does not test naming, + order, or README layout. - [ ] Documentation matches commands and defaults. ### `omnia_auto` checklist @@ -914,7 +1108,8 @@ fabricate another person's signoff or add an AI identity trailer. - [ ] Version was preserved unless a bump was requested. - [ ] Exact wheel was force-reinstalled and its API verified. - [ ] Plugin, wheel, and consumers remain synchronized. -- [ ] Nothing was published or pushed without authorization. +- [ ] Every consumer domain was searched, updated, collected, and unit-tested. +- [ ] Source and rebuilt wheel are pushed together; nothing was published. ### Handoff checklist diff --git a/test/orchestrator/docs/test_config.md b/test/orchestrator/docs/test_config.md index 4d83a3f54b..764156bd6c 100644 --- a/test/orchestrator/docs/test_config.md +++ b/test/orchestrator/docs/test_config.md @@ -66,6 +66,15 @@ always passed to the cleanup playbook: | `cleanup_k8s` | Delete Kubernetes shared data, then detach storage. | Preserve data, then detach storage. | | `cleanup_benchmark_tools` | Remove HPC benchmark staged artifacts after verification. | Retain artifacts for inspection. | +## Kubernetes local-etcd disk media + +`expected_etcd_disk_media` defaults to `""`, which skips the media check. +Set it to `ssd`, `hdd`, or `nvme` to require that every Kubernetes control +plane places `/var/lib/etcd` on that media when `etcd_on_local_disk` is +enabled. The selected disk is classified from `lsblk`: `TRAN=nvme` is NVMe, +`ROTA=1` is HDD, and `ROTA=0` is SSD. Any other value fails before the check +runs. + ## NFT performance thresholds `nft_performance_threshold_seconds` is a required mapping when NFT executes. diff --git a/test/orchestrator/library/functions/__init__.py b/test/orchestrator/library/functions/__init__.py index bd3b834753..772edb5e00 100644 --- a/test/orchestrator/library/functions/__init__.py +++ b/test/orchestrator/library/functions/__init__.py @@ -194,6 +194,7 @@ from .kubernetes_runtime_pxeboot_func import ( check_kubernetes_configured_versions, check_kubernetes_local_etcd_integrity, + check_kubernetes_local_etcd_media, check_kubernetes_local_etcd_provisioning, check_kubernetes_version_compatibility, ) @@ -267,6 +268,7 @@ from .oim_readiness_precheck_func import ( check_oim_cpu_threshold, check_oim_disk_threshold, + check_oim_forced_pxe_public_overlap, check_oim_internet_reachability, check_oim_memory_threshold, check_oim_os_version, @@ -523,6 +525,7 @@ def run_playbook(tag: str | None = None, **kwargs): "check_kubernetes_etcd_topology", "check_kubernetes_local_etcd", "check_kubernetes_local_etcd_integrity", + "check_kubernetes_local_etcd_media", "check_kubernetes_local_etcd_provisioning", "check_kubernetes_local_etcd_recovery", "check_kubernetes_nfs_dynamic_provisioning", @@ -575,6 +578,17 @@ def run_playbook(tag: str | None = None, **kwargs): "check_node_os_version", "check_node_ping", "check_node_ssh", + "check_oim_cpu_threshold", + "check_oim_disk_threshold", + "check_oim_forced_pxe_public_overlap", + "check_oim_internet_reachability", + "check_oim_memory_threshold", + "check_oim_os_version", + "check_oim_public_nic_present", + "check_oim_pxe_nic_ipv4", + "check_oim_pxe_nic_present", + "check_oim_pxe_public_overlap", + "check_oim_ssh_preflight", "check_precheck_admin_ipv4", "check_precheck_dependencies", "check_precheck_hostname_domain", diff --git a/test/orchestrator/library/functions/_powervault_helpers.py b/test/orchestrator/library/functions/_powervault_helpers.py index 37267ba31f..9347ba2c93 100644 --- a/test/orchestrator/library/functions/_powervault_helpers.py +++ b/test/orchestrator/library/functions/_powervault_helpers.py @@ -38,9 +38,7 @@ from ..vars.pxeboot_vars import ( POWERVAULT_DEFAULT_FS_TYPE, - POWERVAULT_DEFAULT_ISCSI_PORT, POWERVAULT_DEFAULT_MOUNT_OPTS, - POWERVAULT_DEFAULT_NODE_KEY, POWERVAULT_ISCSI_INITIATOR_PATH, POWERVAULT_LOG_COMPLETE_MSG, POWERVAULT_LOG_TEMPLATE, diff --git a/test/orchestrator/library/functions/_pxeboot_helpers.py b/test/orchestrator/library/functions/_pxeboot_helpers.py index d0a4652596..6bec2ea6ff 100644 --- a/test/orchestrator/library/functions/_pxeboot_helpers.py +++ b/test/orchestrator/library/functions/_pxeboot_helpers.py @@ -23,7 +23,7 @@ from concurrent.futures import TimeoutError as FutureTimeoutError from typing import Any -from omnia_auto import read_remote_env, run_on_host, run_ssh_command +from omnia_auto import log, read_remote_env, run_on_host, run_ssh_command from ..vars.pxeboot_vars import ( CATALOG_ROLE_PREFIXES, @@ -443,10 +443,8 @@ def report_poll_progress( f" ↻ {label}: still in progress; retrying status check " f"(attempt {attempt}, elapsed {elapsed}s, remaining {remaining}s){suffix}" ) - # INFO logging is intentionally hidden in normal validation runs. Long-running - # pull/reboot polls must remain visible so the operator can distinguish active - # progress from a hung test. - print(message, flush=True) + # Poll progress is available in verbose output without bypassing structured logging. + log(message, "INFO") def run_remote_with_progress( @@ -471,7 +469,7 @@ def run_remote_with_progress( try: result = future.result(timeout=min(poll_seconds, remaining)) elapsed = int(time.monotonic() - started) - print(f" ✓ {label}: completed after {elapsed}s", flush=True) + log(f"{label}: completed after {elapsed}s", "OK") return result except FutureTimeoutError: attempt += 1 diff --git a/test/orchestrator/library/functions/_workload_helpers.py b/test/orchestrator/library/functions/_workload_helpers.py index 88649ba787..e9893f4c43 100644 --- a/test/orchestrator/library/functions/_workload_helpers.py +++ b/test/orchestrator/library/functions/_workload_helpers.py @@ -29,6 +29,7 @@ SLURM_SUBMISSION_PREFIXES, ) from ._provision_helpers import load_context +from .external_ldap_func import load_external_ldap_settings from ._pxeboot_helpers import ( first_row, load_workload_context, @@ -159,6 +160,19 @@ def slurm_submission_rows(rows): ] +def ldap_identity_skip(context) -> str: + """Return why LDAP-identity checks must skip, or an empty string. + + The LDAP test identity lives in the external directory proxied by + omnia_auth, so these checks also require validate_external_ldap. + """ + if not context["features"].get("openldap", False): + return "OpenLDAP is not enabled" + if not load_external_ldap_settings()["validation_enabled"]: + return "validate_external_ldap is false; LDAP identity checks are not requested" + return "" + + def ldap_test_username() -> str: """Return the validated non-secret LDAP test identity.""" username = str(load_test_credentials().get("ldap_username") or "") diff --git a/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py b/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py index 50280c7322..ad3d21a7b0 100644 --- a/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py +++ b/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py @@ -18,6 +18,8 @@ import re import shlex +from omnia_auto import log + from ..vars.pxeboot_vars import ( APPTAINER_ARRAY_SIZE, APPTAINER_CONCURRENT_JOB_COUNT, @@ -27,7 +29,6 @@ ) from ._apptainer_helpers import ( _wait_for_array_accounting, - _wait_for_job_accounting, apptainer_context, command_error, grouped_node_fields, @@ -37,7 +38,12 @@ safe_node_name, ) from ._pxeboot_helpers import remote_command, runtime_exception, runtime_result -from ._workload_helpers import ldap_test_username, optional_skip, require_functional +from ._workload_helpers import ( + ldap_identity_skip, + ldap_test_username, + optional_skip, + require_functional, +) def _functional_context(host, summary): @@ -179,10 +185,9 @@ def check_apptainer_ldap_job(host): ) if isinstance(_result, dict) and "skipped" in _result: return _result - if not context["features"].get("openldap", False): - return optional_skip( - summary, "OpenLDAP is not selected for the mapped Slurm roles" - ) + reason = ldap_identity_skip(context) + if reason: + return optional_skip(summary, reason) return _targeted_job_check(host, summary, username=ldap_test_username()) except (OSError, RuntimeError, TypeError, ValueError) as exc: return runtime_exception(summary, exc) @@ -354,10 +359,9 @@ def check_apptainer_restricted_sif(host): % shlex.quote(restricted), ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: restricted SIF removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: restricted SIF removal failed: {str(exc)[:100]}", + "WARN", ) diff --git a/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py b/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py index 481069f3ce..e6b4be426d 100644 --- a/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py +++ b/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py @@ -103,8 +103,8 @@ def _prepare_recovery(host): _RECOVERY_HOST = host_id gated = require_marker( "Apptainer compute-node reboot", - "disruptive", - "Select reboot or disruptive to authorize a compute-node reboot", + "reboot", + "Select the reboot marker to authorize a compute-node reboot", ) if gated: _RECOVERY_STATE.update({"skipped_result": gated}) diff --git a/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py b/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py index 2dadc262a3..71d9868bd0 100644 --- a/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py +++ b/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py @@ -37,7 +37,12 @@ runtime_exception, runtime_result, ) -from ._workload_helpers import ldap_test_username, optional_skip, require_functional +from ._workload_helpers import ( + ldap_identity_skip, + ldap_test_username, + optional_skip, + require_functional, +) def _computes_or_skip(host, summary): @@ -285,10 +290,9 @@ def check_apptainer_ldap_readability(host): context, _control, computes, skipped = _computes_or_skip(host, summary) if skipped: return skipped - if not context["features"].get("openldap", False): - return optional_skip( - summary, "OpenLDAP is not selected for the mapped Slurm roles" - ) + reason = ldap_identity_skip(context) + if reason: + return optional_skip(summary, reason) username = ldap_test_username() outcomes = {} for row in computes: @@ -325,9 +329,12 @@ def check_apptainer_non_root_execution(host): gated = require_functional(summary) if gated: return gated - _context, _control, computes, skipped = _computes_or_skip(host, summary) + context, _control, computes, skipped = _computes_or_skip(host, summary) if skipped: return skipped + reason = ldap_identity_skip(context) + if reason: + return optional_skip(summary, reason) username = ldap_test_username() outcomes = {} for row in computes: diff --git a/test/orchestrator/library/functions/dcgm_pxeboot_func.py b/test/orchestrator/library/functions/dcgm_pxeboot_func.py index d3d5f2c1f3..cccaca7abf 100644 --- a/test/orchestrator/library/functions/dcgm_pxeboot_func.py +++ b/test/orchestrator/library/functions/dcgm_pxeboot_func.py @@ -21,11 +21,7 @@ import re import time -from ..vars.pxeboot_vars import ( - PXEBOOT_COMMANDS, - SLURM_COMPILER_PREFIX, - SLURM_COMPUTE_PREFIX, -) +from ..vars.pxeboot_vars import PXEBOOT_COMMANDS, SLURM_COMPILER_PREFIX from ._pxeboot_helpers import ( first_row, remote_command, diff --git a/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py b/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py index a6a0332e56..3d106ba320 100644 --- a/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py +++ b/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py @@ -38,7 +38,6 @@ HPC_TOOLS_DIRECTORY_MODE, HPC_TOOLS_NVIDIA_SDK_DIRECTORY, HPC_TOOLS_PLATFORMS_DIRECTORY, - HPC_TOOLS_SCRIPTS_DIRECTORY, PXEBOOT_COMMANDS, ) from ._hpc_benchmarks_helpers import ( @@ -71,7 +70,6 @@ snapshot_directories, snapshot_tool_fingerprints, source_only_scan_paths, - staged_tool_directories, tools_list_deployed, wait_for_pid, ) diff --git a/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py b/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py index 4f28749af7..26b654269f 100644 --- a/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py +++ b/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py @@ -18,6 +18,9 @@ import time from ..vars.pxeboot_vars import ( + ETCD_HEALTH_WAIT_TIMEOUT_SECONDS, + ETCD_SETUP_LOG, + ETCD_UPDATE_LOG, PXEBOOT_COMMANDS, RECOVERY_POLL_SECONDS, RECOVERY_WAIT_TIMEOUT_SECONDS, @@ -36,7 +39,10 @@ from ._workload_helpers import kubernetes_context as _context from ._workload_helpers import optional_skip as _skip from .kubernetes_etcd_pxeboot_func import check_kubernetes_etcd_health -from .kubernetes_runtime_pxeboot_func import check_kubernetes_local_etcd_integrity +from .kubernetes_runtime_pxeboot_func import ( + check_kubernetes_local_etcd_integrity, + etcd_boot_script_state, +) def _vip_configuration(context, config) -> tuple[bool, str]: @@ -103,6 +109,36 @@ def _wait_for_new_boot(host, row, previous_boot_id: str) -> bool: return False +def _wait_for_etcd_health(host) -> dict: + """Poll etcd endpoint health until every member reports healthy.""" + started = time.monotonic() + deadline = started + ETCD_HEALTH_WAIT_TIMEOUT_SECONDS + attempt = 0 + result = check_kubernetes_etcd_health(host) + while not result["success"] and time.monotonic() < deadline: + attempt += 1 + report_poll_progress( + "etcd endpoint health", + attempt, + started, + ETCD_HEALTH_WAIT_TIMEOUT_SECONDS, + result.get("error", ""), + ) + time.sleep(RECOVERY_POLL_SECONDS) + result = check_kubernetes_etcd_health(host) + return result + + +def _post_reboot_boot_scripts(host, row) -> tuple[bool, bool]: + """Return whether fstab-update completed and disk setup did not rerun.""" + states, boot_time = etcd_boot_script_state(host, row) + update_time, update_done = states.get(ETCD_UPDATE_LOG, (None, False)) + setup_time, _setup_done = states.get(ETCD_SETUP_LOG, (None, False)) + update_ok = update_done and update_time is not None and update_time >= boot_time + setup_not_rerun = setup_time is None or setup_time < boot_time + return update_ok, setup_not_rerun + + def _cluster_node_name(host, control, row) -> str: """Return the node name used by the Kubernetes API.""" payload = remote_json(host, control, PXEBOOT_COMMANDS["kubernetes_nodes"]) @@ -123,10 +159,10 @@ def check_kubernetes_control_plane_recovery(host): """Reboot the VIP owner and verify failover, cloud-init, and node recovery.""" summary = "Kubernetes control-plane recovery" try: - if not marker_is_authorized("disruptive"): + if not marker_is_authorized("reboot"): return _skip( summary, - "Select the disruptive marker to authorize a node reboot", + "Select the reboot marker to authorize a node reboot", ) context, rows, _control, config = _context(host) if not rows: @@ -197,6 +233,7 @@ def check_kubernetes_control_plane_recovery(host): ) final_owners = _vip_owners(host, rows, vip) final_vip_ok = len(final_owners) == 1 + etcd = _wait_for_etcd_health(host) if ready_ok else {"success": False} local_etcd = check_kubernetes_local_etcd_integrity(host) local_etcd_required = bool(config.get("etcd_on_local_disk", False)) local_etcd_ok = ( @@ -208,6 +245,7 @@ def check_kubernetes_control_plane_recovery(host): and cloud_init_ok and ready_ok and final_vip_ok + and etcd["success"] and local_etcd_ok ) error_parts = [] @@ -223,6 +261,8 @@ def check_kubernetes_control_plane_recovery(host): error_parts.append( f"final VIP owners: {', '.join(final_owners) or 'none'}" ) + if not etcd["success"]: + error_parts.append("etcd endpoint health check failed") if not local_etcd_ok: error_parts.append("local-etcd integrity failed") return runtime_result( @@ -236,6 +276,7 @@ def check_kubernetes_control_plane_recovery(host): ("VIP failover", "passed" if failover_ok else "FAILED"), ("Cloud-init state", "passed" if cloud_init_ok else "FAILED"), ("Kubernetes Ready", "passed" if ready_ok else "FAILED"), + ("etcd endpoint health", "passed" if etcd["success"] else "FAILED"), ( "Local etcd persistence", ( @@ -256,16 +297,24 @@ def check_kubernetes_local_etcd_recovery(host): """Reboot one control plane and prove local-etcd identity is preserved.""" summary = "Kubernetes local-etcd reboot persistence" try: - if not marker_is_authorized("disruptive"): + if not marker_is_authorized("reboot"): return _skip( summary, - "Select the disruptive marker to authorize a node reboot", + "Select the reboot marker to authorize a node reboot", ) _context_data, rows, control, config = _context(host) if not rows or not bool(config.get("etcd_on_local_disk", False)): return _skip(summary, "etcd_on_local_disk is disabled") + control_rows = [ + row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] + ] + if len(control_rows) < 2: + return _skip(summary, "At least two control-plane nodes are required") + watcher = next( + row for row in control_rows if row["HOSTNAME"] != control["HOSTNAME"] + ) before_integrity = check_kubernetes_local_etcd_integrity(host) - cluster_node_name = _cluster_node_name(host, control, control) + cluster_node_name = _cluster_node_name(host, watcher, control) before = remote_command( host, control, @@ -294,11 +343,14 @@ def check_kubernetes_local_etcd_recovery(host): if ssh_ok: ready_ok, _detail = wait_for_remote_command( host, - control, + watcher, PXEBOOT_COMMANDS["kubernetes_node_ready"] % cluster_node_name, RECOVERY_WAIT_TIMEOUT_SECONDS, RECOVERY_POLL_SECONDS, ) + update_ok, setup_not_rerun = ( + _post_reboot_boot_scripts(host, control) if ssh_ok else (False, False) + ) after_integrity = check_kubernetes_local_etcd_integrity(host) after = remote_command( host, @@ -310,11 +362,13 @@ def check_kubernetes_local_etcd_recovery(host): and bool(after.stdout.strip()) and after.stdout.strip() == before_identity ) - etcd = check_kubernetes_etcd_health(host) + etcd = _wait_for_etcd_health(host) if ready_ok else {"success": False} ok = ( ssh_ok and cloud_ok and ready_ok + and update_ok + and setup_not_rerun and after_integrity["success"] and identity_preserved and etcd["success"] @@ -326,6 +380,10 @@ def check_kubernetes_local_etcd_recovery(host): error_parts.append("cloud-init did not complete") if not ready_ok: error_parts.append("Kubernetes node not Ready") + if not update_ok: + error_parts.append("etcd-fstab-update.sh did not complete on this boot") + if not setup_not_rerun: + error_parts.append("etcd-disk-setup.sh ran again after reboot") if not identity_preserved: error_parts.append("mount source/UUID changed across reboot") if not after_integrity["success"]: @@ -338,9 +396,18 @@ def check_kubernetes_local_etcd_recovery(host): [ ("Rebooted node", control["HOSTNAME"]), ("Kubernetes identity", cluster_node_name), + ("Readiness observer", watcher["HOSTNAME"]), ("New boot observed", "passed" if ssh_ok else "FAILED"), ("Cloud-init completion", "passed" if cloud_ok else "FAILED"), ("Kubernetes Ready", "passed" if ready_ok else "FAILED"), + ( + "etcd-fstab-update.sh this boot", + "passed" if update_ok else "FAILED", + ), + ( + "etcd-disk-setup.sh not rerun", + "passed" if setup_not_rerun else "FAILED", + ), ( "Mount source and UUID preserved", "passed" if identity_preserved else "FAILED", diff --git a/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py b/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py index 7b22735f9b..fb61fb76b7 100644 --- a/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py +++ b/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py @@ -20,7 +20,14 @@ from collections.abc import Mapping from typing import Any -from ..vars.pxeboot_vars import PXEBOOT_COMMANDS +from omnia_auto import load_test_config + +from ..vars.pxeboot_vars import ( + ETCD_MEDIA_TYPES, + ETCD_SETUP_LOG, + ETCD_UPDATE_LOG, + PXEBOOT_COMMANDS, +) from ._pxeboot_helpers import ( group_fields, remote_command, @@ -349,17 +356,8 @@ def _root_disk_name(root_source: str, by_name: Mapping[str, Mapping[str, Any]]) return name -def _local_etcd_node_state(host, row) -> tuple[bool, str]: - mount_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_mount"]) - filesystems = ( - mount_payload.get("filesystems", []) if isinstance(mount_payload, dict) else [] - ) - if len(filesystems) != 1: - return False, f"mount records={len(filesystems)}" - mount = filesystems[0] - mount_source = str(mount.get("source") or "") - mount_type = str(mount.get("fstype") or "").lower() - +def _etcd_disk_layout(host, row) -> tuple[dict[str, Any], dict[str, Any], dict, str]: + """Return the etcd partition, its parent disk, all devices, and the root disk.""" block_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_block_devices"]) devices = _flatten_devices(block_payload.get("blockdevices", [])) by_name = { @@ -371,17 +369,86 @@ def _local_etcd_node_state(host, row) -> tuple[bool, str]: (device for device in devices if _mountpoint_matches(device, "/var/lib/etcd")), {}, ) - selected_disk = _parent_disk(selected, by_name) if selected else {} - selected_disk_name = str(selected_disk.get("name") or "") - + selected_disk = dict(_parent_disk(selected, by_name)) if selected else {} root = remote_command(host, row, PXEBOOT_COMMANDS["etcd_root_source"]) root_disk_name = _root_disk_name(root.stdout, by_name) if root.rc == 0 else "" + return selected, selected_disk, by_name, root_disk_name + + +def _disk_media(disk: Mapping[str, Any]) -> str: + """Classify a whole disk as nvme, hdd, ssd, or unknown.""" + name = str(disk.get("name") or "") + if str(disk.get("tran") or "").lower() == "nvme" or name.startswith("nvme"): + return "nvme" + rotational = str(disk.get("rota")) + if rotational in {"1", "True"}: + return "hdd" + if rotational in {"0", "False"}: + return "ssd" + return "unknown" + + +def _parse_boot_scripts(text: str) -> dict[str, tuple[int | None, bool]]: + """Parse ``||`` lines into per-log state.""" + states: dict[str, tuple[int | None, bool]] = {} + for line in text.splitlines(): + parts = line.strip().split("|") + if len(parts) != 3: + continue + path, mtime, done = parts + states[path] = ( + int(mtime) if mtime.isdigit() else None, + done.isdigit() and int(done) > 0, + ) + return states + + +def _completed_this_boot( + states: Mapping[str, tuple[int | None, bool]], + boot_time: int, + paths: tuple[str, ...] = (ETCD_UPDATE_LOG, ETCD_SETUP_LOG), +) -> str: + """Return the first log in ``paths`` that finished with DONE on this boot.""" + for path in paths: + mtime, done = states.get(path, (None, False)) + if done and mtime is not None and mtime >= boot_time: + return path + return "" + + +def _boot_script_label(completed: str) -> str: + """Render which boot script completed on this boot.""" + return f"{os.path.basename(completed)} DONE" if completed else "stale/incomplete" + + +def etcd_boot_script_state(host, row) -> tuple[dict[str, tuple[int | None, bool]], int]: + """Return etcd boot-script log state and the current boot time for one node.""" + scripts = remote_command(host, row, PXEBOOT_COMMANDS["etcd_boot_scripts"]) + boot_time = remote_command(host, row, PXEBOOT_COMMANDS["node_boot_time"]) + if scripts.rc != 0 or boot_time.rc != 0 or not boot_time.stdout.strip().isdigit(): + raise RuntimeError(f"{row['HOSTNAME']} etcd boot-script state is unavailable") + return _parse_boot_scripts(scripts.stdout), int(boot_time.stdout.strip()) + + +def _local_etcd_node_state(host, row) -> tuple[bool, str]: + mount_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_mount"]) + filesystems = ( + mount_payload.get("filesystems", []) if isinstance(mount_payload, dict) else [] + ) + if len(filesystems) != 1: + return False, f"mount records={len(filesystems)}" + mount = filesystems[0] + mount_source = str(mount.get("source") or "") + mount_type = str(mount.get("fstype") or "").lower() + + selected, selected_disk, by_name, root_disk_name = _etcd_disk_layout(host, row) + selected_disk_name = str(selected_disk.get("name") or "") fstab = remote_command(host, row, PXEBOOT_COMMANDS["etcd_fstab"]) fstab_lines = [line.strip() for line in fstab.stdout.splitlines() if line.strip()] permissions = remote_command(host, row, PXEBOOT_COMMANDS["etcd_permissions"]) manifest = remote_command(host, row, PXEBOOT_COMMANDS["etcd_manifest_data_dir"]) - log_time = remote_command(host, row, PXEBOOT_COMMANDS["etcd_boot_log"]) - boot_time = remote_command(host, row, PXEBOOT_COMMANDS["node_boot_time"]) + states, boot_time = etcd_boot_script_state(host, row) + completed = _completed_this_boot(states, boot_time) uuid = str(selected.get("uuid") or "") label = str(selected.get("label") or "") @@ -395,22 +462,14 @@ def _local_etcd_node_state(host, row) -> tuple[bool, str]: ) boss_disks = [ device - for device in devices + for device in by_name.values() if str(device.get("type")) == "disk" and str(device.get("name") or "") != root_disk_name and "boss" in str(device.get("model") or "").lower() ] boss_ok = not boss_disks or "boss" in str(selected_disk.get("model") or "").lower() - media_ok = str(selected_disk.get("type") or "") == "disk" and str( - selected_disk.get("rota") - ) in {"0", "1", "False", "True"} - log_current = ( - log_time.rc == 0 - and boot_time.rc == 0 - and log_time.stdout.strip().isdigit() - and boot_time.stdout.strip().isdigit() - and int(log_time.stdout.strip()) >= int(boot_time.stdout.strip()) - ) + media = _disk_media(selected_disk) + media_ok = str(selected_disk.get("type") or "") == "disk" and media != "unknown" ok = all( ( bool(mount_source), @@ -425,51 +484,53 @@ def _local_etcd_node_state(host, row) -> tuple[bool, str]: manifest.rc == 0, boss_ok, media_ok, - log_current, + bool(completed), ) ) - media = ( - "NVMe" - if str(selected_disk.get("tran") or "") == "nvme" - else ("HDD" if str(selected_disk.get("rota")) in {"1", "True"} else "SSD") - ) detail = ( f"source={mount_source or 'missing'} | disk={selected_disk_name or 'missing'} " - f"({media}) | fs={fstype or 'missing'} | label={label or 'missing'} | " + f"({media.upper()}) | fs={fstype or 'missing'} | label={label or 'missing'} | " f"UUID={'present' if uuid else 'missing'} | fstab={'valid' if fstab_ok else 'invalid'} | " f"permissions={permissions.stdout.strip() or 'invalid'} | " - f"boot-script={'current' if log_current else 'stale/missing'}" + f"boot-script={_boot_script_label(completed)}" ) return ok, detail +def _control_plane_check(host, summary: str, node_state, failure: str): + """Apply one per-node local-etcd contract to every mapped control plane.""" + _runtime, rows, _control, config = _context(host) + if not rows: + return _skip(summary, "No Kubernetes nodes are mapped") + if not bool(config.get("etcd_on_local_disk", False)): + return _skip(summary, "etcd_on_local_disk is disabled") + control_rows = [ + row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] + ] + outcomes = {row["HOSTNAME"]: node_state(host, row) for row in control_rows} + failures = [name for name, outcome in outcomes.items() if not outcome[0]] + return runtime_result( + bool(control_rows) and not failures, + summary, + group_fields(control_rows, outcomes), + ( + f"{failure}: " + ", ".join(failures) + if failures + else ( + "No Kubernetes control-plane nodes are mapped" + if not control_rows + else "" + ) + ), + ) + + def check_kubernetes_local_etcd_integrity(host): """Verify local-etcd disk selection, filesystem, mount, and boot persistence.""" summary = "Kubernetes local-etcd storage integrity" try: - _runtime, rows, _control, config = _context(host) - if not rows or not bool(config.get("etcd_on_local_disk", False)): - return _skip(summary, "etcd_on_local_disk is disabled") - control_rows = [ - row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] - ] - outcomes = { - row["HOSTNAME"]: _local_etcd_node_state(host, row) for row in control_rows - } - failures = [name for name, outcome in outcomes.items() if not outcome[0]] - return runtime_result( - bool(control_rows) and not failures, - summary, - group_fields(control_rows, outcomes), - ( - "Invalid local-etcd storage on: " + ", ".join(failures) - if failures - else ( - "No Kubernetes control-plane nodes are mapped" - if not control_rows - else "" - ) - ), + return _control_plane_check( + host, summary, _local_etcd_node_state, "Invalid local-etcd storage on" ) except (OSError, RuntimeError, TypeError, ValueError) as exc: return runtime_exception(summary, exc) @@ -477,25 +538,12 @@ def check_kubernetes_local_etcd_integrity(host): def _local_etcd_provisioning_state(host, row) -> tuple[bool, str]: """Return the local-etcd disk-layout and provisioning-artifact state.""" - block_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_block_devices"]) - devices = _flatten_devices(block_payload.get("blockdevices", [])) - by_name = { - str(device.get("name")): device - for device in devices - if str(device.get("name") or "") - } - selected = next( - (device for device in devices if _mountpoint_matches(device, "/var/lib/etcd")), - {}, - ) - selected_disk = _parent_disk(selected, by_name) if selected else {} + selected, selected_disk, by_name, root_name = _etcd_disk_layout(host, row) selected_name = str(selected_disk.get("name") or "") pttype = str(selected_disk.get("pttype") or "").lower() - root = remote_command(host, row, PXEBOOT_COMMANDS["etcd_root_source"]) - root_name = _root_disk_name(root.stdout, by_name) if root.rc == 0 else "" boss_disks = [ device - for device in devices + for device in by_name.values() if str(device.get("type")) == "disk" and "boss" in str(device.get("model") or "").lower() ] @@ -507,29 +555,28 @@ def _local_etcd_provisioning_state(host, row) -> tuple[bool, str]: ) setup_script = remote_command(host, row, PXEBOOT_COMMANDS["etcd_setup_script"]) update_script = remote_command(host, row, PXEBOOT_COMMANDS["etcd_update_script"]) - setup_log = remote_command(host, row, PXEBOOT_COMMANDS["etcd_setup_log"]) - update_log = remote_command(host, row, PXEBOOT_COMMANDS["etcd_update_log"]) selection_log = remote_command(host, row, PXEBOOT_COMMANDS["etcd_selection_log"]) + states, boot_time = etcd_boot_script_state(host, row) + completed = _completed_this_boot(states, boot_time) scripts_ok = setup_script.rc == 0 and update_script.rc == 0 - logs_ok = setup_log.rc == 0 and update_log.rc == 0 logged_disks = re.findall( r"(?:found|candidate|disk):\s*/dev/([A-Za-z0-9._-]+)", selection_log.stdout, ) - selection_logged = selection_log.rc == 0 and selected_name in logged_disks + selection_logged = selected_name in logged_disks ok = ( bool(selected.get("pkname")) and selection_ok and pttype == "gpt" and scripts_ok - and logs_ok + and bool(completed) and selection_logged ) detail = ( f"disk={selected_name or 'missing'} | selection={selection_mode} | " f"partition-table={pttype or 'missing'} | " f"scripts={'valid' if scripts_ok else 'missing/not executable'} | " - f"logs={'complete' if logs_ok else 'missing/incomplete'} | " + f"boot-script={_boot_script_label(completed)} | " f"logged-selection={'matched' if selection_logged else 'mismatch'}" ) return ok, detail @@ -539,32 +586,43 @@ def check_kubernetes_local_etcd_provisioning(host): """Verify GPT selection and local-etcd provisioning artifacts.""" summary = "Kubernetes local-etcd provisioning contract" try: - _runtime, rows, _control, config = _context(host) - if not rows: - return _skip(summary, "No Kubernetes nodes are mapped") - if not bool(config.get("etcd_on_local_disk", False)): - return _skip(summary, "etcd_on_local_disk is disabled") - control_rows = [ - row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] - ] - outcomes = { - row["HOSTNAME"]: _local_etcd_provisioning_state(host, row) - for row in control_rows - } - failures = [name for name, outcome in outcomes.items() if not outcome[0]] - return runtime_result( - bool(control_rows) and not failures, + return _control_plane_check( + host, summary, - group_fields(control_rows, outcomes), - ( - "Invalid local-etcd provisioning on: " + ", ".join(failures) - if failures - else ( - "No Kubernetes control-plane nodes are mapped" - if not control_rows - else "" - ) - ), + _local_etcd_provisioning_state, + "Invalid local-etcd provisioning on", + ) + except (OSError, RuntimeError, TypeError, ValueError) as exc: + return runtime_exception(summary, exc) + + +def check_kubernetes_local_etcd_media(host): + """Verify every control plane places etcd on the configured disk media.""" + summary = "Kubernetes local-etcd disk media" + try: + expected = str(load_test_config().get("expected_etcd_disk_media") or "") + expected = expected.strip().lower() + if not expected: + return _skip(summary, "expected_etcd_disk_media is not configured") + if expected not in ETCD_MEDIA_TYPES: + raise ValueError( + "expected_etcd_disk_media must be one of: " + + ", ".join(ETCD_MEDIA_TYPES) + ) + + def node_state(host, row): + _selected, disk, _by_name, _root = _etcd_disk_layout(host, row) + media = _disk_media(disk) + return media == expected, ( + f"disk={disk.get('name') or 'missing'} | " + f"media={media.upper()} | expected={expected.upper()}" + ) + + return _control_plane_check( + host, + summary, + node_state, + f"etcd is not on {expected.upper()} media on", ) except (OSError, RuntimeError, TypeError, ValueError) as exc: return runtime_exception(summary, exc) diff --git a/test/orchestrator/library/functions/minimal_os_pxeboot_func.py b/test/orchestrator/library/functions/minimal_os_pxeboot_func.py index 44e92eecf2..68275dc973 100644 --- a/test/orchestrator/library/functions/minimal_os_pxeboot_func.py +++ b/test/orchestrator/library/functions/minimal_os_pxeboot_func.py @@ -19,7 +19,6 @@ software (Slurm, Kubernetes, container runtimes, GPU drivers, MPI stacks). """ -import re from typing import Any from ..vars.pxeboot_vars import PXEBOOT_COMMANDS diff --git a/test/orchestrator/library/functions/mount_config_precheck_func.py b/test/orchestrator/library/functions/mount_config_precheck_func.py index 93918b9a9a..514072801a 100644 --- a/test/orchestrator/library/functions/mount_config_precheck_func.py +++ b/test/orchestrator/library/functions/mount_config_precheck_func.py @@ -32,7 +32,6 @@ """ import os -import re from typing import Any from ..vars.pxeboot_vars import STORAGE_CONFIG diff --git a/test/orchestrator/library/functions/mount_config_pxeboot_func.py b/test/orchestrator/library/functions/mount_config_pxeboot_func.py index b7e77f6a98..1738e72d12 100644 --- a/test/orchestrator/library/functions/mount_config_pxeboot_func.py +++ b/test/orchestrator/library/functions/mount_config_pxeboot_func.py @@ -22,7 +22,6 @@ """ import os -import re from typing import Any from ..vars.pxeboot_vars import ( @@ -31,9 +30,7 @@ STORAGE_CONFIG, ) from ._pxeboot_helpers import ( - group_fields, load_runtime_context, - load_workload_context, remote_command, rows_matching, runtime_exception, diff --git a/test/orchestrator/library/functions/oim_readiness_precheck_func.py b/test/orchestrator/library/functions/oim_readiness_precheck_func.py index 2a10a52d02..f4c8347d89 100644 --- a/test/orchestrator/library/functions/oim_readiness_precheck_func.py +++ b/test/orchestrator/library/functions/oim_readiness_precheck_func.py @@ -19,6 +19,8 @@ Every function is read-only and never mutates the target environment. """ +import csv +import io import posixpath import re from typing import Any @@ -58,6 +60,17 @@ def _admin_network(host) -> dict[str, Any]: raise ValueError("network_spec.yml does not define admin_network") +def _default_route_nic(host) -> tuple[str, str]: + """Return the default IPv4 route line and its device, or empty strings.""" + result = run_on_host(host, OIM_READINESS_COMMANDS["default_route"]) + line = result.stdout.strip() if result.rc == 0 else "" + parts = line.split() + for index, token in enumerate(parts): + if token == "dev" and index + 1 < len(parts): + return line, parts[index + 1] + return line, "" + + # ── hardware checks ──────────────────────────────────────────────── @@ -143,11 +156,12 @@ def check_oim_disk_threshold( # ── network checks ───────────────────────────────────────────────── -def check_oim_pxe_nic_present(host) -> dict[str, Any]: - """Verify the configured admin NIC exists and is operationally UP.""" +def check_oim_pxe_nic_present(host, *, nic_name: str = "") -> dict[str, Any]: + """Verify the configured (or explicitly named) admin NIC exists and is UP.""" try: - admin = _admin_network(host) - nic_name = str(admin.get("oim_nic_name") or "").strip() + if not nic_name: + admin = _admin_network(host) + nic_name = str(admin.get("oim_nic_name") or "").strip() except (OSError, TypeError, ValueError) as exc: return prepare_result( False, "Unable to resolve admin NIC", [], str(exc) @@ -199,15 +213,12 @@ def check_oim_public_nic_present(host) -> dict[str, Any]: or PXE-only environments do not require a public interface). """ try: - result = run_on_host( - host, "ip -4 route show default 2>/dev/null | head -1" - ) + line, nic_name = _default_route_nic(host) except (OSError, RuntimeError) as exc: return prepare_result( False, "Unable to query default route", [], str(exc) ) - line = result.stdout.strip() if not line: return prepare_result( True, @@ -215,13 +226,6 @@ def check_oim_public_nic_present(host) -> dict[str, Any]: [("Default route", "none"), ("Public NIC check", "skipped")], ) - parts = line.split() - nic_name = "" - for i, token in enumerate(parts): - if token == "dev" and i + 1 < len(parts): - nic_name = parts[i + 1] - break - if not nic_name: return prepare_result( False, @@ -314,18 +318,7 @@ def check_oim_pxe_public_overlap( [("PXE NIC", "not configured")], ) - route_result = run_on_host( - host, "ip -4 route show default 2>/dev/null | head -1" - ) - route_line = route_result.stdout.strip() - public_nic = "" - if route_line: - parts = route_line.split() - for i, token in enumerate(parts): - if token == "dev" and i + 1 < len(parts): - public_nic = parts[i + 1] - break - + _route_line, public_nic = _default_route_nic(host) if not public_nic: return prepare_result( True, @@ -353,15 +346,34 @@ def check_oim_pxe_public_overlap( ) +def check_oim_forced_pxe_public_overlap(host) -> dict[str, Any]: + """Run the overlap check with the PXE NIC forced to the public NIC.""" + try: + _route_line, public_nic = _default_route_nic(host) + except (OSError, RuntimeError) as exc: + return prepare_result(False, "Unable to query default route", [], str(exc)) + if not public_nic: + return prepare_result( + True, + "No default route; forced overlap is not applicable", + [("Reason", "No default IPv4 route on the OIM")], + skipped=True, + ) + return check_oim_pxe_public_overlap(host, nic_name=public_nic) + + # ── prerequisite checks ──────────────────────────────────────────── -def check_oim_ssh_preflight(host) -> dict[str, Any]: +def check_oim_ssh_preflight(host, *, target: str = "") -> dict[str, Any]: """Verify passwordless SSH from OIM to a configured target node. - Uses the first mapped node from the PXE mapping CSV as the target. - Skips if no mapping is configured or no nodes are present. + Uses the first mapped node from the PXE mapping CSV as the target unless + ``target`` is given. Skips if no mapping is configured or no nodes are + present. """ + if target: + return _ssh_preflight_result(host, target) try: input_dir = resolve_target_input_project_path(host) config = read_yaml_mapping( @@ -384,7 +396,7 @@ def check_oim_ssh_preflight(host) -> dict[str, Any]: ) # Read the first ADMIN_IP from the mapping - raw = run_on_host(host, "head -2 %s", mapping_path) + raw = run_on_host(host, OIM_READINESS_COMMANDS["mapping_head"], mapping_path) lines = raw.stdout.strip().split("\n") if raw.rc == 0 else [] if len(lines) < 2: return prepare_result( @@ -393,9 +405,6 @@ def check_oim_ssh_preflight(host) -> dict[str, Any]: [("PXE mapping", mapping_path), ("SSH preflight", "skipped")], ) - import csv - import io - reader = csv.DictReader(io.StringIO(raw.stdout)) row = next(reader, None) if not row or not row.get("ADMIN_IP", "").strip(): @@ -405,7 +414,11 @@ def check_oim_ssh_preflight(host) -> dict[str, Any]: [("PXE mapping", mapping_path), ("SSH preflight", "skipped")], ) - target = row["ADMIN_IP"].strip() + return _ssh_preflight_result(host, row["ADMIN_IP"].strip()) + + +def _ssh_preflight_result(host, target: str) -> dict[str, Any]: + """Return the structured passwordless-SSH result for one target.""" ssh_result = run_on_host( host, OIM_READINESS_COMMANDS["ssh_check"], target ) diff --git a/test/orchestrator/library/functions/powervault_pxeboot_func.py b/test/orchestrator/library/functions/powervault_pxeboot_func.py index 7c50a0dab1..45c079b4d2 100644 --- a/test/orchestrator/library/functions/powervault_pxeboot_func.py +++ b/test/orchestrator/library/functions/powervault_pxeboot_func.py @@ -35,11 +35,7 @@ from typing import Any -from ..vars.pxeboot_vars import ( - POWERVAULT_DEFAULT_ISCSI_PORT, - POWERVAULT_DEFAULT_NODE_KEY, - PXEBOOT_COMMANDS, -) +from ..vars.pxeboot_vars import POWERVAULT_DEFAULT_ISCSI_PORT from ._powervault_helpers import ( SLURM_MANDATORY_BIND_TARGETS, error_result, @@ -48,7 +44,6 @@ get_powervault_entries, get_target_nodes, optional_skip, - resolve_node_key_value, resolve_pv_fs_type, resolve_pv_mount_opts, skip_if_no_powervault, diff --git a/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py b/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py index b1ee5afc26..ef1a2aebb0 100644 --- a/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py @@ -20,7 +20,6 @@ import secrets import shlex import socket -import subprocess import time from pathlib import Path @@ -28,6 +27,8 @@ from omnia_auto import ( connection_params, load_test_credentials, + log, + run_on_host, ) from ..vars.pxeboot_vars import ( @@ -47,6 +48,7 @@ runtime_exception, runtime_result, ) +from ._workload_helpers import ldap_identity_skip as _ldap_identity_skip from ._workload_helpers import ldap_test_username as _ldap_username from ._workload_helpers import optional_skip as _skip from ._workload_helpers import require_functional as _require_functional @@ -114,6 +116,7 @@ def _mapped_host_key_matches(exc: paramiko.BadHostKeyException, target: str) -> def _remove_stale_mapped_host_key( + host, target: str, replacement_key: paramiko.PKey, ) -> tuple[bool, str]: @@ -124,16 +127,15 @@ def _remove_stale_mapped_host_key( if not os.path.isfile(known_hosts): return False, f"known-hosts file does not exist: {known_hosts}" try: - result = subprocess.run( - ["ssh-keygen", "-R", target, "-f", known_hosts], - capture_output=True, - text=True, - timeout=10, - check=False, + result = run_on_host( + host, + PXEBOOT_COMMANDS["ssh_remove_host_key"], + shlex.quote(target), + shlex.quote(known_hosts), ) - except (OSError, subprocess.TimeoutExpired) as exc: + except (OSError, RuntimeError, TypeError, ValueError) as exc: return False, f"unable to remove the stale SSH host key: {exc}" - if result.returncode != 0: + if result.rc != 0: detail = (result.stderr or result.stdout).strip() return False, detail or "ssh-keygen could not remove the stale host key" key_type = replacement_key.get_name() @@ -154,7 +156,7 @@ def _remove_stale_mapped_host_key( def _ldap_ssh_identity( - target: str, username: str, auth_secret: str + host, target: str, username: str, auth_secret: str ) -> tuple[bool, str]: if not _HOST_RE.fullmatch(target) or not _IDENTITY_RE.fullmatch(username): raise ValueError("LDAP SSH target or identity is invalid") @@ -222,7 +224,7 @@ def _ldap_ssh_identity( return True, detail except paramiko.BadHostKeyException as exc: if attempt == 0 and _mapped_host_key_matches(exc, target): - removed, detail = _remove_stale_mapped_host_key(target, exc.key) + removed, detail = _remove_stale_mapped_host_key(host, target, exc.key) if removed: host_key_reconciled = True continue @@ -242,7 +244,9 @@ def _ldap_ssh_identity( return False, "SSH host-key reconciliation did not establish a session" -def _open_ldap_persistent_session(target: str, username: str, auth_secret: str): +def _open_ldap_persistent_session( + host, target: str, username: str, auth_secret: str +): """Open an LDAP SSH command that must be killed when its job ends.""" if not _HOST_RE.fullmatch(target) or not _IDENTITY_RE.fullmatch(username): raise ValueError("LDAP SSH target or identity is invalid") @@ -315,7 +319,7 @@ def _open_ldap_persistent_session(target: str, username: str, auth_secret: str): return session, detail except paramiko.BadHostKeyException as exc: if attempt == 0 and _mapped_host_key_matches(exc, target): - removed, detail = _remove_stale_mapped_host_key(target, exc.key) + removed, detail = _remove_stale_mapped_host_key(host, target, exc.key) if removed: host_key_reconciled = True continue @@ -607,17 +611,24 @@ def _check_slurm_role_ldap_authentication( try: context, rows, _control, _config = _context(host) if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + return _skip( + summary, "OpenLDAP is not enabled" if rows else "No Slurm nodes are mapped" + ) role_rows = _role_rows(rows, role_prefix) if not role_rows: return _skip(summary, f"No {missing_role} nodes are mapped") + reason = _ldap_identity_skip(context) + if reason: + return _skip(summary, reason) username, auth_secret = _ldap_credentials() credential = ( "invalid-" + secrets.token_urlsafe(24) if invalid_password else auth_secret ) outcomes = {} for row in role_rows: - accepted, detail = _ldap_ssh_identity(row["ADMIN_IP"], username, credential) + accepted, detail = _ldap_ssh_identity( + host, row["ADMIN_IP"], username, credential + ) success = not accepted if invalid_password else accepted outcomes[row["HOSTNAME"]] = { "success": success, @@ -750,8 +761,9 @@ def check_slurm_pam_no_job_access(host): summary = "Slurm PAM no-job access policy" try: context, rows, control, _config = _context(host) - if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + reason = _ldap_identity_skip(context) if rows else "No Slurm nodes are mapped" + if reason: + return _skip(summary, reason) username, auth_secret = _ldap_credentials() computes = _compute_rows(rows) if not computes: @@ -782,7 +794,7 @@ def check_slurm_pam_no_job_access(host): outcomes = {} for row in computes: allowed, _detail = _ldap_ssh_identity( - row["ADMIN_IP"], username, auth_secret + host, row["ADMIN_IP"], username, auth_secret ) outcomes[row["HOSTNAME"]] = ( not allowed, @@ -840,13 +852,18 @@ def _check_slurm_role_pam_job_access( return _skip(summary, "Select the functional marker to authorize test jobs") context, rows, control, slurm_config = _context(host) if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + return _skip( + summary, "OpenLDAP is not enabled" if rows else "No Slurm nodes are mapped" + ) computes = _compute_rows(rows) submit_rows = _role_rows(rows, role_prefix) if not submit_rows: return _skip(summary, f"No {missing_role} nodes are mapped") if not computes: return _skip(summary, "At least one Slurm compute node is required") + reason = _ldap_identity_skip(context) + if reason: + return _skip(summary, reason) username, auth_secret = _ldap_credentials() workspace_info = _prepare_ldap_workspace( host, @@ -904,6 +921,7 @@ def _check_slurm_role_pam_job_access( ) if running: session, login_detail = _open_ldap_persistent_session( + host, compute["ADMIN_IP"], username, auth_secret, @@ -927,7 +945,7 @@ def _check_slurm_role_pam_job_access( persistent_sessions.remove(session) time.sleep(PAM_ACCESS_SETTLE_SECONDS) access_after, post_login_detail = _ldap_ssh_identity( - compute["ADMIN_IP"], username, auth_secret + host, compute["ADMIN_IP"], username, auth_secret ) stdout, stderr = _job_artifacts(host, control, workspace, job_id) output_lines = [ @@ -1110,10 +1128,10 @@ def _check_slurm_role_pam_job_access( PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel PAM job {job_id}: " + log( + f"cleanup: failed to cancel PAM job {job_id}: " f"{str(exc)[:100]}", - flush=True, + "WARN", ) if control is not None and workspace_info is not None and username: try: @@ -1124,10 +1142,9 @@ def _check_slurm_role_pam_job_access( username, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: LDAP workspace removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: LDAP workspace removal failed: {str(exc)[:100]}", + "WARN", ) @@ -1169,8 +1186,9 @@ def check_slurm_invalid_ldap_identity(host): summary = "Slurm invalid LDAP identity rejection" try: context, rows, _control, _config = _context(host) - if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + reason = _ldap_identity_skip(context) if rows else "No Slurm nodes are mapped" + if reason: + return _skip(summary, reason) invalid_user = "omnia_missing_" + secrets.token_hex(6) allowed = _allowed_rows(rows) outcomes = {} @@ -1215,13 +1233,18 @@ def _check_slurm_role_ldap_jobs( return gated context, rows, control, slurm_config = _context(host) if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + return _skip( + summary, "OpenLDAP is not enabled" if rows else "No Slurm nodes are mapped" + ) submit_rows = _role_rows(rows, role_prefix) if not submit_rows: return _skip(summary, f"No {missing_role} nodes are mapped") computes = _compute_rows(rows) if not computes: return _skip(summary, "At least one Slurm compute node is required") + reason = _ldap_identity_skip(context) + if reason: + return _skip(summary, reason) username = _ldap_username() workspace_info = _prepare_ldap_workspace( host, @@ -1409,10 +1432,10 @@ def _check_slurm_role_ldap_jobs( PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel LDAP job {job_id}: " + log( + f"cleanup: failed to cancel LDAP job {job_id}: " f"{str(exc)[:100]}", - flush=True, + "WARN", ) if control is not None and workspace_info is not None and username: try: @@ -1423,10 +1446,9 @@ def _check_slurm_role_ldap_jobs( username, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: LDAP workspace removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: LDAP workspace removal failed: {str(exc)[:100]}", + "WARN", ) diff --git a/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py b/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py index c66f8666ff..17701122ae 100644 --- a/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py @@ -21,6 +21,8 @@ import time from pathlib import Path +from omnia_auto import log + from ..vars.pxeboot_vars import ( PXEBOOT_COMMANDS, RECOVERY_POLL_SECONDS, @@ -36,7 +38,6 @@ ) from ._pxeboot_helpers import ( first_row, - marker_is_authorized, remote_command, runtime_exception, runtime_result, @@ -574,10 +575,9 @@ def check_slurm_concurrent_jobs(host): PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel job {job_id}: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: failed to cancel job {job_id}: {str(exc)[:100]}", + "WARN", ) try: remote_command( @@ -1093,10 +1093,9 @@ def check_slurm_job_queueing(host): PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel job {job_id}: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: failed to cancel job {job_id}: {str(exc)[:100]}", + "WARN", ) for row in computes: for job_id in jobs: @@ -1129,11 +1128,6 @@ def check_slurm_drain_queue_recovery(host): control = None compute = None try: - if not marker_is_authorized("disruptive"): - return _skip( - summary, - "Select the disruptive marker to authorize node draining", - ) _runtime, rows, control, _config = _context(host) computes = _compute_rows(rows) if not computes: @@ -1501,16 +1495,15 @@ def check_slurm_openmpi_job(host): f"rmdir {shlex.quote(workspace)} 2>/dev/null || true", ) if cleanup.rc != 0: - print( - f" ! cleanup: OpenMPI workspace {workspace} removal " + log( + f"cleanup: OpenMPI workspace {workspace} removal " f"returned rc={cleanup.rc}", - flush=True, + "WARN", ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: OpenMPI workspace removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: OpenMPI workspace removal failed: {str(exc)[:100]}", + "WARN", ) diff --git a/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py b/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py index 9bb3617a01..210f6fdb16 100644 --- a/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py @@ -33,17 +33,11 @@ from omnia_auto import load_test_config, run_on_host -from ..vars.pxeboot_vars import PXEBOOT_COMMANDS, SLURM_COMPUTE_PREFIX -from ._pxeboot_helpers import ( - remote_command, - runtime_exception, - runtime_result, -) +from ..vars.pxeboot_vars import PXEBOOT_COMMANDS +from ._pxeboot_helpers import remote_command, runtime_exception from ._provision_helpers import ( api_json, - group_members, interface_ips, - load_context, normalise_mac, resource_list, ) @@ -56,6 +50,9 @@ # Functional group prefix for Slurm compute nodes. _SLURM_NODE_PREFIX = "slurm_node_" +# Removal needs at least this many slurm_node rows in the PXE mapping. +_MIN_SLURM_NODES = 2 + def _result( success: bool, @@ -140,8 +137,8 @@ def _resolve_target_nodes( Returns: ``(admin_ips, all_rows, header_line)`` when target nodes are identified, or ``None`` when the test should be skipped because - no ``slurm_node_*`` rows exist and the user did not provide - explicit IPs. + the mapping has fewer than ``_MIN_SLURM_NODES`` ``slurm_node_*`` + rows. Raises: ValueError: When the user supplied explicit IPs that are invalid @@ -160,6 +157,8 @@ def _resolve_target_nodes( row for row in all_rows if row.get("FUNCTIONAL_GROUP_NAME", "").startswith(_SLURM_NODE_PREFIX) ] + if len(slurm_node_rows) < _MIN_SLURM_NODES: + return None if configured: # User provided specific Admin IPs — validation failures are errors. @@ -184,14 +183,18 @@ def _resolve_target_nodes( ) return requested_ips, all_rows, header - # Default: remove the last slurm_node entry. Skip when none exist. - if not slurm_node_rows: - return None - + # Default: remove the last slurm_node entry. last_node = slurm_node_rows[-1] return [last_node["ADMIN_IP"]], all_rows, header +def _write_remote_file(host, path: str, content: str) -> Any: + """Write content through a heredoc; the delimiter must be on its own line.""" + if not content.endswith("\n"): + content += "\n" + return run_on_host(host, f"cat > {path} << 'OMNIA_EOF'\n{content}OMNIA_EOF") + + def _write_mapping(host, path: str, header: str, rows: list[dict[str, str]]) -> None: """Write the mapping file back with the given rows.""" if not rows: @@ -203,10 +206,8 @@ def _write_mapping(host, path: str, header: str, rows: list[dict[str, str]]) -> ) writer.writeheader() writer.writerows(rows) - content = output.getvalue() # Write via shell to handle remote/local transparently. - escaped = content.replace("'", "'\\''") - result = run_on_host(host, f"cat > {path} << 'OMNIA_EOF'\n{escaped}OMNIA_EOF") + result = _write_remote_file(host, path, output.getvalue()) if result.rc != 0: raise RuntimeError(f"Failed to write PXE mapping: {result.stderr}") @@ -231,8 +232,7 @@ def restore_pxe_mapping(host) -> None: raise FileNotFoundError(f"PXE mapping backup not found: {source}") with open(source, encoding="utf-8") as fh: content = fh.read() - escaped = content.replace("'", "'\\''") - result = run_on_host(host, f"cat > {dest} << 'OMNIA_EOF'\n{escaped}OMNIA_EOF") + result = _write_remote_file(host, dest, content) if result.rc != 0: raise RuntimeError(f"Failed to restore PXE mapping: {result.stderr}") @@ -437,7 +437,14 @@ def check_slurm_node_remove(host) -> dict[str, Any]: mapping_path = _mapping_path(host) resolved = _resolve_target_nodes(host, mapping_path) if resolved is None: - return _skip(summary, "No slurm_node entries in PXE mapping") + # A stale backup would make the add test restore an old mapping. + if os.path.isfile(_backup_path()): + os.remove(_backup_path()) + return _skip( + summary, + f"At least {_MIN_SLURM_NODES} slurm_node entries are required " + "in the PXE mapping", + ) remove_ips, all_rows, header = resolved removed_rows = [row for row in all_rows if row["ADMIN_IP"] in remove_ips] diff --git a/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py b/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py index 6315c4eb92..bbbae942ff 100644 --- a/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py @@ -33,6 +33,7 @@ runtime_result, wait_for_cloud_init, ) +from ._workload_helpers import ldap_identity_skip as _ldap_identity_skip from ._workload_helpers import ldap_test_username as _ldap_username from ._workload_helpers import optional_skip as _skip from ._workload_helpers import slurm_context as _context @@ -108,10 +109,10 @@ def check_slurm_cluster_recovery(host): """ summary = "Slurm cluster reboot recovery" try: - if not marker_is_authorized("disruptive"): + if not marker_is_authorized("reboot"): return _skip( summary, - "Select the disruptive marker to authorize a cluster reboot", + "Select the reboot marker to authorize a cluster reboot", ) context, rows, control, _config = _context(host) if not rows: @@ -188,8 +189,9 @@ def check_slurm_cluster_recovery(host): accounting_ok = accounting_state.startswith("COMPLETED") ldap_ok = True - ldap_detail = "not applicable" - if context["features"].get("openldap", False): + ldap_skip = _ldap_identity_skip(context) + ldap_detail = f"not applicable ({ldap_skip})" if ldap_skip else "not applicable" + if not ldap_skip: username = _ldap_username() submit_rows = _submit_rows(rows) if submit_rows: diff --git a/test/orchestrator/library/functions/vast_storage_pxeboot_func.py b/test/orchestrator/library/functions/vast_storage_pxeboot_func.py index 7066dcc349..f3204e797b 100644 --- a/test/orchestrator/library/functions/vast_storage_pxeboot_func.py +++ b/test/orchestrator/library/functions/vast_storage_pxeboot_func.py @@ -40,7 +40,6 @@ """ import os -import re import time from typing import Any @@ -48,7 +47,6 @@ from ..vars.pxeboot_vars import ( OMNIA_CONFIG, - PXEBOOT_COMMANDS, SLURM_COMPILER_PREFIX, SLURM_COMPUTE_PREFIX, SLURM_CONTROL_PREFIX, @@ -59,7 +57,6 @@ from ._provision_helpers import load_context from ._pxeboot_helpers import ( first_row, - remote_command, rows_matching, runtime_exception, runtime_result, diff --git a/test/orchestrator/library/messages/precheck_msgs.py b/test/orchestrator/library/messages/precheck_msgs.py index b366c9f589..2ba4bc7bf9 100644 --- a/test/orchestrator/library/messages/precheck_msgs.py +++ b/test/orchestrator/library/messages/precheck_msgs.py @@ -19,6 +19,9 @@ "playbook_failed": "Orchestrator precheck failed", "check_passed": "{component} verification passed", "check_failed": "{component} verification failed", + "check_skipped": "{component} verification skipped", + "rejection_passed": "{component} rejected the invalid condition", + "rejection_failed": "{component} accepted an invalid condition", } TEST_ASSERT_MSGS = { @@ -30,4 +33,9 @@ "{component} verification failed: {error}. Correct the reported " "environment, input, storage, or dependency state and rerun precheck." ), + "rejection_missing": ( + "{component} did not reject the invalid condition or returned no " + "actionable error. HOW TO FIX: review the checker failure path and " + "rerun the precheck negative suite." + ), } diff --git a/test/orchestrator/library/vars/__init__.py b/test/orchestrator/library/vars/__init__.py index e5ffa0ef06..56b3b23178 100644 --- a/test/orchestrator/library/vars/__init__.py +++ b/test/orchestrator/library/vars/__init__.py @@ -21,14 +21,17 @@ LDAP_READY_DELAY_SECONDS, LDAP_READY_RETRIES, ) +from .precheck_vars import OIM_NEGATIVE_INPUTS from .test_case_vars import ( CLEANUP_TEST_CASES, NFT_TEST_CASES, + ORDER_BLOCK_SIZE, PRECHECK_TEST_CASES, PREPARE_TEST_CASES, PROVISION_TEST_CASES, PXEBOOT_TEST_CASES, TEST_CASES, + TEST_ORDER_BLOCKS, ) __all__ = [ @@ -39,9 +42,12 @@ "LDAP_READY_DELAY_SECONDS", "LDAP_READY_RETRIES", "NFT_TEST_CASES", + "OIM_NEGATIVE_INPUTS", + "ORDER_BLOCK_SIZE", "PRECHECK_TEST_CASES", "PREPARE_TEST_CASES", "PROVISION_TEST_CASES", "PXEBOOT_TEST_CASES", "TEST_CASES", + "TEST_ORDER_BLOCKS", ] diff --git a/test/orchestrator/library/vars/precheck_vars.py b/test/orchestrator/library/vars/precheck_vars.py index 5776020d85..95474b7b64 100644 --- a/test/orchestrator/library/vars/precheck_vars.py +++ b/test/orchestrator/library/vars/precheck_vars.py @@ -59,6 +59,8 @@ "disk_gb": "df -BG / | tail -1 | awk '{print $2}' | tr -d 'G'", "nic_operstate": "cat /sys/class/net/%s/operstate 2>/dev/null", "nic_ipv4": "ip -4 -o addr show %s scope global", + "default_route": "ip -4 route show default 2>/dev/null | head -1", + "mapping_head": "head -2 %s", "ssh_check": ( "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 " "-o BatchMode=yes %s whoami 2>/dev/null" @@ -68,3 +70,15 @@ } INTERNET_CHECK_HOSTS: tuple[str, ...] = ("8.8.8.8", "1.1.1.1", "208.67.222.222") + +# Deliberately unattainable inputs used by the read-only OIM negative cases. +# 192.0.2.1 is RFC 5737 TEST-NET-1 and is never routable. +OIM_NEGATIVE_INPUTS: dict[str, object] = { + "min_cores": 99999, + "min_memory_gb": 99999, + "min_disk_gb": 99999, + "missing_nic": "omnia_fvt_missing0", + "unreachable_target": "192.0.2.1", + "os_id": "omnia_fvt_missing_os", + "os_version": "0.0", +} diff --git a/test/orchestrator/library/vars/pxeboot_vars.py b/test/orchestrator/library/vars/pxeboot_vars.py index 4d28dc3568..46886a7d46 100644 --- a/test/orchestrator/library/vars/pxeboot_vars.py +++ b/test/orchestrator/library/vars/pxeboot_vars.py @@ -39,6 +39,12 @@ PAM_SESSION_TERMINATION_TIMEOUT_SECONDS = 30 SLURM_DRAIN_REASON = "omnia_fvt_validation" ETCD_RAFT_INDEX_DELTA_MAX = 100 +# A rebooted etcd member must resynchronize its raft log before every +# endpoint reports healthy; five minutes covers a full snapshot transfer. +ETCD_HEALTH_WAIT_TIMEOUT_SECONDS = 300 +ETCD_MEDIA_TYPES = ("ssd", "hdd", "nvme") +ETCD_SETUP_LOG = "/var/log/etcd-disk-setup.log" +ETCD_UPDATE_LOG = "/var/log/diskless-etcd-mount.log" APPTAINER_IMAGE_DIRECTORY = "/hpc_tools/container_images" APPTAINER_SCRIPT_DIRECTORY = "/hpc_tools/scripts" APPTAINER_DOWNLOAD_SCRIPT = "/hpc_tools/scripts/download_container_image.sh" @@ -398,7 +404,7 @@ "etcd_mount": "findmnt -J /var/lib/etcd", "etcd_mount_identity": "findmnt -n -o SOURCE,UUID,FSTYPE /var/lib/etcd", "etcd_block_devices": ( - "lsblk -J -b -o NAME,PATH,PKNAME,TYPE,FSTYPE,LABEL,UUID," + "lsblk -J -b -o NAME,PATH,PKNAME,TYPE,PTTYPE,FSTYPE,LABEL,UUID," "MOUNTPOINTS,MODEL,ROTA,TRAN" ), "etcd_root_source": "findmnt -n -o SOURCE /", @@ -411,17 +417,19 @@ "grep -Eq -- '--data-dir(=|[[:space:]]+)/var/lib/etcd([[:space:]]|$)' " "/etc/kubernetes/manifests/etcd.yaml" ), - "etcd_boot_log": ( - "stat -c '%Y' /var/log/diskless-etcd-mount.log 2>/dev/null || " - "stat -c '%Y' /var/log/etcd-disk-setup.log" + # One line per boot script log: ||. + "etcd_boot_scripts": ( + "for log in /var/log/etcd-disk-setup.log /var/log/diskless-etcd-mount.log; " + "do if [ -s \"$log\" ]; then printf '%s|%s|%s\\n' \"$log\" " + "\"$(stat -c %Y \"$log\")\" " + "\"$(tail -n 5 \"$log\" | grep -c '===== DONE =====')\"; " + "else printf '%s|missing|0\\n' \"$log\"; fi; done" ), "etcd_setup_script": "test -x /usr/local/bin/etcd-disk-setup.sh", "etcd_update_script": "test -x /usr/local/bin/etcd-fstab-update.sh", - "etcd_setup_log": "test -s /var/log/etcd-disk-setup.log", - "etcd_update_log": "test -s /var/log/diskless-etcd-mount.log", "etcd_selection_log": ( "cat /var/log/etcd-disk-setup.log /var/log/diskless-etcd-mount.log " - "2>/dev/null" + "2>/dev/null || true" ), "node_boot_time": 'date -d "$(uptime -s)" +%s', "node_boot_id": "cat /proc/sys/kernel/random/boot_id", @@ -482,7 +490,7 @@ ), "slurm_drain_node": ("scontrol update NodeName=%s State=DRAIN Reason=%s"), "slurm_drain_state": ( - "sinfo --noheader --nodes=%s --format='%T' | head -1" + "sinfo --noheader --nodes=%s --format='%%T' | head -1" ), "slurm_resume_node": "scontrol update NodeName=%s State=RESUME", "slurm_cancel_job": "scancel %s", @@ -523,6 +531,7 @@ "slurm_hardware": "scontrol show nodes --oneliner", "openmpi": _OPENMPI_DISCOVERY_COMMAND + '"$mpirun_path" --version', "slurm_mpi_plugins": "srun --mpi=list 2>&1", + "ssh_remove_host_key": "ssh-keygen -R %s -f %s", "openmpi_compile": ("srun --nodes=1 --ntasks=1 --nodelist=%s bash -lc %s"), "openmpi_job": ( "srun --nodes=%s --ntasks=%s --ntasks-per-node=1 " diff --git a/test/orchestrator/library/vars/test_case_vars.py b/test/orchestrator/library/vars/test_case_vars.py index 7a6b607cf6..b74355ad18 100644 --- a/test/orchestrator/library/vars/test_case_vars.py +++ b/test/orchestrator/library/vars/test_case_vars.py @@ -129,6 +129,46 @@ "title": "Verify OIM OS matches expected distribution and version", "component": "OIM OS version", }, + "oim_cpu_rejection": { + "id": "ORCH_FVT_PRECHECK_V110", + "title": "Reject an unattainable OIM CPU threshold", + "component": "OIM CPU threshold", + }, + "oim_memory_rejection": { + "id": "ORCH_FVT_PRECHECK_V111", + "title": "Reject an unattainable OIM memory threshold", + "component": "OIM memory threshold", + }, + "oim_disk_rejection": { + "id": "ORCH_FVT_PRECHECK_V112", + "title": "Reject an unattainable OIM disk threshold", + "component": "OIM disk threshold", + }, + "oim_missing_nic_rejection": { + "id": "ORCH_FVT_PRECHECK_V113", + "title": "Reject a nonexistent admin NIC", + "component": "OIM admin NIC", + }, + "oim_nic_overlap_rejection": { + "id": "ORCH_FVT_PRECHECK_V114", + "title": "Reject a PXE NIC that equals the public NIC", + "component": "PXE/public NIC overlap", + }, + "oim_airgapped_internet": { + "id": "ORCH_FVT_PRECHECK_V115", + "title": "Accept missing internet access in air-gapped mode", + "component": "Internet reachability", + }, + "oim_unreachable_ssh_rejection": { + "id": "ORCH_FVT_PRECHECK_V116", + "title": "Reject SSH to an unreachable target", + "component": "OIM SSH preflight", + }, + "oim_os_mismatch_rejection": { + "id": "ORCH_FVT_PRECHECK_V117", + "title": "Reject a mismatched OIM OS identity", + "component": "OIM OS version", + }, } PREPARE_TEST_CASES: dict[str, dict[str, str]] = { @@ -164,10 +204,6 @@ "id": "ORCH_FVT_PREPARE_V007", "title": "Verify CoreDHCP and CoreDNS match network_spec.yml", }, - "external_ldap_proxy": { - "id": "ORCH_FVT_PREPARE_V008", - "title": "Reconcile and verify the external LDAP proxy", - }, "openldap_runtime": { "id": "ORCH_FVT_PREPARE_V009", "title": "Verify OpenLDAP service and container health", @@ -180,10 +216,6 @@ "id": "ORCH_FVT_PREPARE_V011", "title": "Verify OpenLDAP listeners and local endpoint", }, - "external_ldap_backend": { - "id": "ORCH_FVT_PREPARE_V012", - "title": "Verify external LDAP backend reachability", - }, "postgresql_readiness": { "id": "ORCH_FVT_PREPARE_V013", "title": "Verify PostgreSQL and SMD database readiness", @@ -288,7 +320,7 @@ "component": "Kubernetes version compatibility", }, "kubernetes_configured_versions": { - "id": "ORCH_FVT_PXEBOOT_V101", + "id": "ORCH_FVT_PXEBOOT_V519", "title": "Verify configured Kubernetes component versions", "component": "Kubernetes configured versions", }, @@ -333,10 +365,15 @@ "component": "Kubernetes local-etcd disk integrity", }, "kubernetes_local_etcd_provisioning": { - "id": "ORCH_FVT_PXEBOOT_V102", + "id": "ORCH_FVT_PXEBOOT_V520", "title": "Verify Kubernetes local-etcd provisioning contract", "component": "Kubernetes local-etcd provisioning", }, + "kubernetes_local_etcd_media": { + "id": "ORCH_FVT_PXEBOOT_V523", + "title": "Verify Kubernetes local-etcd disk media", + "component": "Kubernetes local-etcd disk media", + }, "kubernetes_storage": { "id": "ORCH_FVT_PXEBOOT_V016", "title": "Verify Kubernetes NFS and CSI storage", @@ -353,7 +390,7 @@ "component": "Kubernetes snapshot components", }, "kubernetes_nfs_contract": { - "id": "ORCH_FVT_PXEBOOT_V103", + "id": "ORCH_FVT_PXEBOOT_V521", "title": "Verify Kubernetes NFS provisioner and backend contract", "component": "Kubernetes NFS provisioner", }, @@ -457,6 +494,16 @@ "title": "Verify Slurm drain queue and resume behavior", "component": "Slurm drain and queue", }, + "external_ldap_proxy": { + "id": "ORCH_FVT_PXEBOOT_V524", + "title": "Reconcile and verify the external LDAP proxy", + "component": "External LDAP proxy", + }, + "external_ldap_backend": { + "id": "ORCH_FVT_PXEBOOT_V525", + "title": "Verify external LDAP backend reachability", + "component": "External LDAP backend", + }, "slurm_pam": { "id": "ORCH_FVT_PXEBOOT_V039", "title": "Verify Slurm pam_slurm_adopt integration", @@ -748,17 +795,17 @@ "component": "HPC benchmarks staging cleanup", }, "hpc_benchmarks_prerequisites": { - "id": "ORCH_FVT_PXEBOOT_V095", + "id": "ORCH_FVT_PXEBOOT_V225", "title": "Verify benchmark prerequisites: script, platform, tools list, mount (NFS export informational)", "component": "HPC benchmarks prerequisites", }, "hpc_benchmarks_staging_fingerprint_idempotency": { - "id": "ORCH_FVT_PXEBOOT_V096", + "id": "ORCH_FVT_PXEBOOT_V226", "title": "Verify staging idempotency with SHA-256, size, and mtime fingerprints", "component": "HPC benchmarks staging fingerprint idempotency", }, "hpc_benchmarks_concurrent_staging": { - "id": "ORCH_FVT_PXEBOOT_V097", + "id": "ORCH_FVT_PXEBOOT_V227", "title": "Verify concurrent staging from two compute nodes on shared storage", "component": "HPC benchmarks concurrent staging", }, @@ -817,7 +864,6 @@ "title": "Verify no compile or build commands are staged", "component": "HPC benchmarks source-only delivery", }, - "hpc_benchmarks_e2e_provisioning": { "id": "ORCH_FVT_PXEBOOT_V208", "title": "Verify end-to-end benchmark provisioning pipeline", @@ -1104,7 +1150,7 @@ }, # ── DCGM / CUDA verification (ORCH_FVT_PXEBOOT_V501-V518) ───────── "dcgm_cuda_validation": { - "id": "ORCH_FVT_PXEBOOT_V501", + "id": "ORCH_FVT_PXEBOOT_V522", "title": "Verify NVIDIA driver and CUDA toolkit on GPU nodes", "component": "CUDA driver and toolkit validation", }, @@ -1547,3 +1593,81 @@ **CLEANUP_TEST_CASES, **NFT_TEST_CASES, } + + +# ===================================================================== +# Pytest execution order blocks +# ===================================================================== +# Every test's @pytest.mark.order value lies inside its suite's block: +# +# block start = lifecycle base + suite slot * ORDER_BLOCK_SIZE +# order = block start + position in suite (1..99) +# +# Lifecycle bases are 10000 apart, so values are unique across lifecycles. +# "root" holds the lifecycle execution case at the block start itself. Suites +# are listed in dependency order: reachability and node state first, then +# cluster health, temporary workloads, and finally reboot or node-removal +# suites. Within a suite, reboot and destructive cases come last. Add a +# test by taking the next free number in its suite block; add a suite by +# appending a new block. +ORDER_BLOCK_SIZE = 100 + +TEST_ORDER_BLOCKS: dict[str, dict[str, int]] = { + "precheck": { + "root": 10000, + "environment": 10100, + "oim_readiness": 10200, + "storage": 10300, + "dependencies": 10400, + "inputs": 10500, + }, + "prepare": { + "root": 20000, + "openchami": 20100, + "network": 20200, + "openldap": 20300, + }, + "provision": { + "root": 30000, + "openchami": 30100, + }, + "pxeboot": { + "root": 40000, + "connectivity": 40100, + "cloudinit": 40200, + "additional_cloud_init": 40300, + "mount_config": 40400, + "minimal_os": 40500, + "powervault": 40600, + "coredns_coredhcp": 40700, + "kubernetes_cluster": 40800, + "kubernetes_etcd": 40900, + "kubernetes_storage": 41000, + "slurm_cluster": 41100, + "slurm_jobs": 41200, + "slurm_ldap": 41300, + "slurm_gpu": 41400, + "slurm_dcgm": 41500, + "slurm_openmpi": 41600, + "slurm_ucx": 41700, + "slurm_infiniband": 41800, + "slurm_apptainer": 41900, + "slurm_hpc_benchmarks": 42000, + "vast_storage": 42100, + "kubernetes_recovery": 42200, + "slurm_recovery": 42300, + "slurm_lifecycle": 42400, + }, + "cleanup": { + "root": 50000, + "openchami": 50100, + "openldap": 50200, + "slurm": 50300, + "kubernetes": 50400, + "artifacts": 50500, + "credentials": 50600, + }, + "nft": { + "root": 60000, + }, +} diff --git a/test/orchestrator/test_config.yml b/test/orchestrator/test_config.yml index 4fbcbfe02c..3c25898d7e 100644 --- a/test/orchestrator/test_config.yml +++ b/test/orchestrator/test_config.yml @@ -265,7 +265,7 @@ cleanup_benchmark_tools: true # The test automatically removes the LAST slurm_node entry from the PXE # mapping file, runs provision, verifies removal, then restores the # original mapping, runs provision again, and verifies re-addition. -# If no slurm_node is present in the mapping, the test is skipped. +# If the mapping has fewer than two slurm_node entries, the test is skipped. # # When one or more Admin IPs are provided (comma-separated): # The test validates that every IP exists in the PXE mapping AND belongs @@ -275,6 +275,19 @@ cleanup_benchmark_tools: true # Example: "182.10.0.50,182.10.0.52" slurm_lifecycle_remove_add_nodes: "" +# ============================================================================= +# KUBERNETES LOCAL-ETCD DISK MEDIA +# ============================================================================= +# Disk media expected under /var/lib/etcd on every Kubernetes control plane +# when etcd_on_local_disk is enabled. The pxeboot kubernetes_etcd suite +# classifies the selected disk from lsblk (TRAN=nvme -> nvme, ROTA=1 -> hdd, +# ROTA=0 -> ssd) and fails when any control plane differs. +# +# Supported values: ssd, hdd, nvme +# When empty (default), the media check is skipped and the detected media is +# still reported by the local-etcd integrity check. +expected_etcd_disk_media: "" + # ============================================================================= # NON-FUNCTIONAL PERFORMANCE CONTRACTS # ============================================================================= diff --git a/test/orchestrator/ut/README.md b/test/orchestrator/ut/README.md index 5d8fbae901..b25c9b6c7c 100644 --- a/test/orchestrator/ut/README.md +++ b/test/orchestrator/ut/README.md @@ -11,6 +11,10 @@ an OIM, calling OpenCHAMI, or executing an Orchestrator playbook. | `test_pxeboot_contracts.py` | Kubernetes version parsing/skew, Slurm parsers and shared-storage selection, catalog feature resolution, mutation authorization, cloud-init interpretation, and Apptainer image-path validation | | `test_cleanup_contracts.py` | Cleanup after Kubernetes apply failure and Slurm drain-test restoration | | `test_nft_contracts.py` | NFT threshold validation and Ansible idempotency-recap interpretation | +| `test_additional_cloud_init_contracts.py` | Additional cloud-init input validation: disabled paths, allowed sections, prohibited keys, list types, and functional-group names | +| `test_hpc_platform_scripts.py` | Per-node RHEL HPC storage and Pulp paths, OS-dependent Slurm scripts, and offline benchmark downloads | +| `test_orchestrator_status_reconcile.py` | Lifecycle status reconciliation for external and platform-managed PXE boot results | +| `test_kubernetes_etcd_contracts.py` | Local-etcd boot-script DONE detection, disk-media classification, root-disk exclusion, post-reboot setup-rerun detection, media expectation, and the external-LDAP identity gate | Parameterized cases count as separate pytest results. Function names and docstrings carry `ORCH_UT_*` identifiers where a stable unit contract has been @@ -44,6 +48,10 @@ python3 -m pytest --confcutdir=ut ut/test_prepare_contracts.py -q python3 -m pytest --confcutdir=ut ut/test_pxeboot_contracts.py -q python3 -m pytest --confcutdir=ut ut/test_cleanup_contracts.py -q python3 -m pytest --confcutdir=ut ut/test_nft_contracts.py -q +python3 -m pytest --confcutdir=ut ut/test_additional_cloud_init_contracts.py -q +python3 -m pytest --confcutdir=ut ut/test_hpc_platform_scripts.py -q +python3 -m pytest --confcutdir=ut ut/test_orchestrator_status_reconcile.py -q +python3 -m pytest --confcutdir=ut ut/test_kubernetes_etcd_contracts.py -q ``` Unit tests must remain isolated: mock external commands at the helper boundary diff --git a/test/orchestrator/ut/test_additional_cloud_init_contracts.py b/test/orchestrator/ut/test_additional_cloud_init_contracts.py index 9d858d6235..38da00fd6c 100644 --- a/test/orchestrator/ut/test_additional_cloud_init_contracts.py +++ b/test/orchestrator/ut/test_additional_cloud_init_contracts.py @@ -17,8 +17,6 @@ import sys from pathlib import Path -import pytest - # The validator lives in the src orchestrator plugins tree. The UT conftest # already adds the test plugin root; we also need the *src* plugin root so # that ``module_utils.orchestrator_validation`` resolves. @@ -71,7 +69,7 @@ def _write_mapping(tmp_path: Path, groups: list[str]) -> str: # --------------------------------------------------------------------------- def test_empty_path_disables_validation(tmp_path): - """ORCH_UT_040: Empty config path disables validation with no errors.""" + """ORCH_UT_085: Empty config path disables validation with no errors.""" errors = validator.validate( {"additional_cloud_init_config_file": ""}, str(tmp_path) ) @@ -79,20 +77,20 @@ def test_empty_path_disables_validation(tmp_path): def test_missing_key_disables_validation(tmp_path): - """ORCH_UT_041: Absent config key disables validation with no errors.""" + """ORCH_UT_086: Absent config key disables validation with no errors.""" errors = validator.validate({}, str(tmp_path)) assert errors == [] def test_null_yaml_file_passes(tmp_path): - """ORCH_UT_042: An empty/null YAML file passes (feature disabled).""" + """ORCH_UT_087: An empty/null YAML file passes (feature disabled).""" path = _write_yaml(tmp_path, "---\n") errors = validator.validate(_config(path), str(tmp_path)) assert errors == [] def test_valid_common_section(tmp_path): - """ORCH_UT_043: Valid common section with write_files and runcmd passes.""" + """ORCH_UT_088: Valid common section with write_files and runcmd passes.""" content = ( "common:\n" " write_files:\n" @@ -108,7 +106,7 @@ def test_valid_common_section(tmp_path): def test_valid_groups_section(tmp_path): - """ORCH_UT_044: Valid per-FG groups section passes when FG exists.""" + """ORCH_UT_089: Valid per-FG groups section passes when FG exists.""" _write_mapping(tmp_path, ["slurm_node_rhel_10_0_x86_64"]) content = ( "groups:\n" diff --git a/test/orchestrator/ut/test_cleanup_contracts.py b/test/orchestrator/ut/test_cleanup_contracts.py index 100efbe436..d1c3c9b48d 100644 --- a/test/orchestrator/ut/test_cleanup_contracts.py +++ b/test/orchestrator/ut/test_cleanup_contracts.py @@ -67,18 +67,23 @@ def test_slurm_drain_probe_always_cancels_job_and_resumes_node(monkeypatch): "ADMIN_IP": "192.0.2.21", "EXPECTED_FUNCTIONAL_GROUP": "slurm_node_rhel_10_0_x86_64", } - monkeypatch.setattr( - slurm_jobs, "marker_is_authorized", lambda marker: marker == "disruptive" - ) monkeypatch.setattr( slurm_jobs, "_context", lambda _host: ({}, [control, compute], control, {}), ) + node_state = ["idle"] + def remote_command(_host, _row, command): commands.append(command) - if "sbatch" in command: + if "State=DRAIN" in command: + node_state[0] = "drained" + elif "State=RESUME" in command: + node_state[0] = "idle" + elif "sinfo" in command: + return _result(stdout=node_state[0]) + elif "sbatch" in command: return _result(stdout="12345|PENDING") return _result() diff --git a/test/orchestrator/ut/test_kubernetes_etcd_contracts.py b/test/orchestrator/ut/test_kubernetes_etcd_contracts.py new file mode 100644 index 0000000000..5424d01ed7 --- /dev/null +++ b/test/orchestrator/ut/test_kubernetes_etcd_contracts.py @@ -0,0 +1,213 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local-etcd boot-script, disk-media, recovery, and LDAP-gate contracts.""" + +import subprocess + +import pytest +from library.functions import _workload_helpers as workload +from library.functions import kubernetes_recovery_pxeboot_func as recovery +from library.functions import kubernetes_runtime_pxeboot_func as runtime +from library.vars.pxeboot_vars import ( + ETCD_SETUP_LOG, + ETCD_UPDATE_LOG, + PXEBOOT_COMMANDS, +) + +BOOT = 1_700_000_000 + + +def test_boot_script_lines_are_parsed(): + """ORCH_UT_075: Boot-script state lines parse into mtime and DONE flags.""" + states = runtime._parse_boot_scripts( + f"{ETCD_SETUP_LOG}|missing|0\n{ETCD_UPDATE_LOG}|{BOOT}|2\nnoise\n" + ) + + assert states[ETCD_SETUP_LOG] == (None, False) + assert states[ETCD_UPDATE_LOG] == (BOOT, True) + + +@pytest.mark.parametrize( + ("states", "expected"), + [ + # First boot: the early fstab-update run fails, disk setup completes. + ( + {ETCD_SETUP_LOG: (BOOT + 60, True), ETCD_UPDATE_LOG: (BOOT + 30, False)}, + ETCD_SETUP_LOG, + ), + # Later boot: setup log is gone, fstab-update completes. + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT + 30, True)}, ETCD_UPDATE_LOG), + # A DONE marker left by an earlier boot is stale. + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT - 5, True)}, ""), + # A current log without DONE means the script failed. + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT + 30, False)}, ""), + ], +) +def test_completed_boot_script_requires_current_done(states, expected): + """ORCH_UT_076: Only a DONE marker written on this boot counts.""" + assert runtime._completed_this_boot(states, BOOT) == expected + + +@pytest.mark.parametrize( + ("disk", "media"), + [ + ({"name": "nvme0n1", "tran": "nvme", "rota": False}, "nvme"), + ({"name": "nvme1n1", "tran": None, "rota": "0"}, "nvme"), + ({"name": "sdb", "tran": "sata", "rota": True}, "hdd"), + ({"name": "sdb", "tran": "sas", "rota": "1"}, "hdd"), + ({"name": "sdc", "tran": "sata", "rota": False}, "ssd"), + ({"name": "sdc", "tran": "sata", "rota": None}, "unknown"), + ], +) +def test_disk_media_classification(disk, media): + """ORCH_UT_077: Disk media follows lsblk TRAN and ROTA like the 2.2 suite.""" + assert runtime._disk_media(disk) == media + + +def test_partition_resolves_to_parent_and_root_disk(): + """ORCH_UT_078: Partitions resolve to whole disks for root exclusion.""" + devices = runtime._flatten_devices( + [ + {"name": "sda", "type": "disk", "children": [{"name": "sda2", "pkname": "sda"}]}, + { + "name": "nvme0n1", + "type": "disk", + "pttype": "gpt", + "children": [ + { + "name": "nvme0n1p1", + "pkname": "nvme0n1", + "mountpoints": ["/var/lib/etcd"], + } + ], + }, + ] + ) + by_name = {device["name"]: device for device in devices} + selected = by_name["nvme0n1p1"] + + assert runtime._mountpoint_matches(selected, "/var/lib/etcd") + assert runtime._parent_disk(selected, by_name)["name"] == "nvme0n1" + assert runtime._root_disk_name("/dev/sda2", by_name) == "sda" + + +@pytest.mark.parametrize( + ("states", "expected"), + [ + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT + 30, True)}, (True, True)), + ({ETCD_SETUP_LOG: (BOOT + 40, True), ETCD_UPDATE_LOG: (BOOT + 30, True)}, (True, False)), + ({ETCD_SETUP_LOG: (BOOT - 900, True), ETCD_UPDATE_LOG: (BOOT + 30, False)}, (False, True)), + ], +) +def test_post_reboot_detects_disk_setup_rerun(monkeypatch, states, expected): + """ORCH_UT_079: After reboot, fstab-update must finish and setup must not rerun.""" + monkeypatch.setattr(recovery, "etcd_boot_script_state", lambda _host, _row: (states, BOOT)) + + assert recovery._post_reboot_boot_scripts(object(), {"HOSTNAME": "cp1"}) == expected + + +def _media_context(monkeypatch, expected, media_by_host): + rows = [ + { + "HOSTNAME": name, + "ADMIN_IP": f"192.0.2.{index}", + "EXPECTED_FUNCTIONAL_GROUP": "service_kube_control_plane_x86_64", + } + for index, name in enumerate(media_by_host, start=10) + ] + monkeypatch.setattr(runtime, "load_test_config", lambda: {"expected_etcd_disk_media": expected}) + monkeypatch.setattr( + runtime, "_context", lambda _host: ({}, rows, rows[0], {"etcd_on_local_disk": True}) + ) + monkeypatch.setattr( + runtime, + "_etcd_disk_layout", + lambda _host, row: ({}, media_by_host[row["HOSTNAME"]], {}, "sda"), + ) + + +def test_media_check_skips_when_not_configured(monkeypatch): + """ORCH_UT_080: An empty expected media is a documented skip.""" + _media_context(monkeypatch, "", {"cp1": {"name": "sdb", "rota": "0"}}) + + result = runtime.check_kubernetes_local_etcd_media(object()) + + assert result["success"] and result["skipped"] + + +def test_media_check_rejects_unknown_media_value(monkeypatch): + """ORCH_UT_081: An unsupported expected media fails closed.""" + _media_context(monkeypatch, "tape", {"cp1": {"name": "sdb", "rota": "0"}}) + + result = runtime.check_kubernetes_local_etcd_media(object()) + + assert not result["success"] and not result["skipped"] + assert "expected_etcd_disk_media" in result["error"] + + +def test_media_check_requires_every_control_plane(monkeypatch): + """ORCH_UT_082: One control plane on other media fails the check.""" + _media_context( + monkeypatch, + "nvme", + {"cp1": {"name": "nvme0n1", "tran": "nvme"}, "cp2": {"name": "sdb", "rota": "0"}}, + ) + + result = runtime.check_kubernetes_local_etcd_media(object()) + + assert not result["success"] + assert "cp2" in result["error"] and "cp1" not in result["error"] + + +@pytest.mark.parametrize( + ("openldap", "validate", "reason"), + [ + (False, True, "OpenLDAP is not enabled"), + (True, False, "validate_external_ldap is false"), + (True, True, ""), + ], +) +def test_ldap_identity_checks_require_external_validation(monkeypatch, openldap, validate, reason): + """ORCH_UT_083: LDAP-identity checks skip unless external LDAP is validated.""" + monkeypatch.setattr( + workload, "load_external_ldap_settings", lambda: {"validation_enabled": validate} + ) + + result = workload.ldap_identity_skip({"features": {"openldap": openldap}}) + + assert result.startswith(reason) if reason else result == "" + + +def test_boot_script_command_output_is_parseable(tmp_path): + """ORCH_UT_084: The remote boot-script probe emits the parsed line format.""" + setup_log = tmp_path / "etcd-disk-setup.log" + update_log = tmp_path / "diskless-etcd-mount.log" + update_log.write_text("[INFO] start\n2026-01-01_00:00:00 [OK] ===== DONE =====\n") + command = ( + PXEBOOT_COMMANDS["etcd_boot_scripts"] + .replace(ETCD_SETUP_LOG, str(setup_log)) + .replace(ETCD_UPDATE_LOG, str(update_log)) + ) + + output = subprocess.run( + ["bash", "-c", command], capture_output=True, text=True, check=True, timeout=10 + ).stdout + states = runtime._parse_boot_scripts( + output.replace(str(setup_log), ETCD_SETUP_LOG).replace(str(update_log), ETCD_UPDATE_LOG) + ) + + assert states[ETCD_SETUP_LOG] == (None, False) + assert states[ETCD_UPDATE_LOG][1] is True + assert isinstance(states[ETCD_UPDATE_LOG][0], int) diff --git a/test/orchestrator/ut/test_pxeboot_contracts.py b/test/orchestrator/ut/test_pxeboot_contracts.py index e8c1146be3..69dee14b49 100644 --- a/test/orchestrator/ut/test_pxeboot_contracts.py +++ b/test/orchestrator/ut/test_pxeboot_contracts.py @@ -15,6 +15,7 @@ """Focused unit contracts for PXE parsers, features, and safety gates.""" import json +import subprocess from types import SimpleNamespace import pytest @@ -27,6 +28,7 @@ from library.functions import kubernetes_storage_pxeboot_func as kubernetes_storage from library.functions import slurm_auth_pxeboot_func as slurm_auth from library.functions import slurm_configuration_pxeboot_func as slurm_config +from library.functions import slurm_lifecycle_pxeboot_func as slurm_lifecycle from library.functions import slurm_pxeboot_func as slurm @@ -201,11 +203,11 @@ def test_marker_authorization_requires_exact_explicit_selection(monkeypatch): """ORCH_UT_006: Mutation helpers trust only exact authorized markers.""" monkeypatch.setenv( "OMNIA_FVT_AUTHORIZED_MARKERS", - "functional,disruptive", + "functional,reboot", ) assert helpers.marker_is_authorized("functional") - assert helpers.marker_is_authorized("disruptive") - assert not helpers.marker_is_authorized("reboot") + assert helpers.marker_is_authorized("reboot") + assert not helpers.marker_is_authorized("image_download") monkeypatch.delenv("OMNIA_FVT_AUTHORIZED_MARKERS") assert not helpers.marker_is_authorized("functional") @@ -703,3 +705,132 @@ def unexpected_remote_command(*_args, **_kwargs): assert dict(result["details"]["fields"])["Reason"] == ( "No Slurm compute nodes are mapped" ) + + +@pytest.mark.parametrize("content", ["H,A\nr1,x\n", "H,A\nr1,x", "H,A\nr1,'q'"]) +def test_slurm_lifecycle_mapping_write_keeps_content_exact( + monkeypatch, tmp_path, content +): + """ORCH_UT_090: Heredoc writes never append the delimiter to the data.""" + def run_locally(_host, command): + proc = subprocess.run( + ["bash", "-c", command], capture_output=True, text=True, check=False + ) + return SimpleNamespace(rc=proc.returncode, stdout=proc.stdout, + stderr=proc.stderr) + + monkeypatch.setattr(slurm_lifecycle, "run_on_host", run_locally) + target = tmp_path / "pxe_mapping_file.csv" + result = slurm_lifecycle._write_remote_file(None, str(target), content) + + assert result.rc == 0 + written = target.read_text(encoding="utf-8") + assert "OMNIA_EOF" not in written + assert written.rstrip("\n") == content.rstrip("\n") + + +_LIFECYCLE_HEADER = "FUNCTIONAL_GROUP_NAME,GROUP_NAME,HOSTNAME,ADMIN_MAC,ADMIN_IP" + + +@pytest.mark.parametrize( + ("slurm_nodes", "configured", "expected"), + [ + (1, "", None), + (1, "192.0.2.11", None), + (2, "", ["192.0.2.12"]), + (2, "192.0.2.11", ["192.0.2.11"]), + ], +) +def test_slurm_node_remove_requires_two_slurm_nodes( + monkeypatch, slurm_nodes, configured, expected +): + """ORCH_UT_091: Node removal skips with fewer than two slurm_node rows.""" + rows = ["slurm_control_node_x86_64,grp2,ctl,aa:bb:cc:dd:ee:00,192.0.2.10"] + rows += [ + f"slurm_node_x86_64,grp3,node{i},aa:bb:cc:dd:ee:0{i},192.0.2.1{i}" + for i in range(1, slurm_nodes + 1) + ] + content = "\n".join([_LIFECYCLE_HEADER, *rows]) + "\n" + host = SimpleNamespace( + file=lambda _path: SimpleNamespace(is_file=True, content_string=content) + ) + monkeypatch.setattr( + slurm_lifecycle, + "load_test_config", + lambda: {"slurm_lifecycle_remove_add_nodes": configured}, + ) + + resolved = slurm_lifecycle._resolve_target_nodes(host, "mapping.csv") + + assert (resolved[0] if resolved else None) == expected + + +def test_slurm_node_remove_skip_discards_stale_backup(monkeypatch, tmp_path): + """ORCH_UT_092: A skipped removal leaves no backup for the add test.""" + backup = tmp_path / "pxe_mapping_file.csv.backup" + backup.write_text("stale", encoding="utf-8") + monkeypatch.setattr(slurm_lifecycle, "_backup_path", lambda: str(backup)) + monkeypatch.setattr(slurm_lifecycle, "_mapping_path", lambda _host: "m.csv") + monkeypatch.setattr( + slurm_lifecycle, "_resolve_target_nodes", lambda _host, _path: None + ) + + result = slurm_lifecycle.check_slurm_node_remove(None) + + assert result["skipped"] and result["success"] + assert "At least 2 slurm_node" in result["details"]["fields"][0][1] + assert not backup.exists() + + +def test_stale_host_key_removal_uses_central_runner(monkeypatch, tmp_path): + """ORCH_UT_093: Host-key reconciliation uses the centralized local runner.""" + known_hosts = tmp_path / "known_hosts" + known_hosts.write_text("old key\n", encoding="utf-8") + monkeypatch.setenv("OMNIA_SSH_KNOWN_HOSTS", str(known_hosts)) + calls = [] + + def run_locally(host, command, *args): + calls.append((host, command, args)) + return SimpleNamespace(rc=0, stdout="", stderr="") + + monkeypatch.setattr(slurm_auth, "run_on_host", run_locally) + key = SimpleNamespace( + get_name=lambda: "ssh-ed25519", + get_base64=lambda: "A" * 68, + ) + host = object() + + success, _detail = slurm_auth._remove_stale_mapped_host_key( + host, "192.0.2.10", key + ) + + assert success + assert calls == [ + ( + host, + slurm_auth.PXEBOOT_COMMANDS["ssh_remove_host_key"], + ("192.0.2.10", str(known_hosts)), + ) + ] + assert known_hosts.read_text(encoding="utf-8").endswith( + f"192.0.2.10 ssh-ed25519 {'A' * 68}\n" + ) + + +def test_poll_progress_uses_structured_logging(monkeypatch): + """ORCH_UT_094: Poll progress is emitted through structured logging.""" + messages = [] + monkeypatch.setattr( + helpers, "log", lambda message, level: messages.append((message, level)) + ) + monkeypatch.setattr(helpers.time, "monotonic", lambda: 12.0) + + helpers.report_poll_progress("node reboot", 2, 10.0, 30, "booting") + + assert messages == [ + ( + " ↻ node reboot: still in progress; retrying status check " + "(attempt 2, elapsed 2s, remaining 28s) | last state: booting", + "INFO", + ) + ] diff --git a/test/plugins/dist/omnia_auto-1.0.0-py3-none-any.whl b/test/plugins/dist/omnia_auto-1.0.0-py3-none-any.whl index 59ca5d09b97db580f25493c7fd6abca296f76a53..c30291e0c9c608705d999940f83d76a65cc2aab6 100644 GIT binary patch delta 25759 zcmV)dK&QW$w*||w1r1P30|XQR000O8)oezw4GjhZ)oezy9R>vg1J!IslO+f+1J!Is zvsehM1OwGk7Z;>++p<^(VZWIrXB@=Q)^` zMfH&ZZg2BPhO2S!sy*(xz4P;zx3_i2IHyW#n1<*tz<-y~LkM>?h#r(+iUNVclY|su z;f*$vj@8U1WxSiJ7q+J2+!$Aal3^WwFlJIwSj|i-<|8Ycr`s^Lg?!B{HEy9bTabDp z5!-6KuCoRm2$j#NsuzD>8J6!C4NaqNKEITXqP+ z%FAWS5E1;>WHeMa$YK%Gg*3?$p2_vq)wnmAxPO!D%ggrolg5R?lG-dLXf|Ks?=&7_ zOm<>5&hgg~cle@JFPycKx}de+-b4h*QGa#{rMaGN&7JQ~o3N#i-hon0UA+*~N3Sxi z&x=v&NTtz`zITsS{_Hl!$M}X-g7>%jl?)K&Uy$&Dos>g;jkA(aDeVvu=$~g8eR^nnM}4#|8nj@hng_58COZk}glB2^ zbR(PfRH$hw*QArmXD?Q|mXc21V5R*c?YK-%XxGTz-@)V@c~@ParDU4U~#$s<^5W z$dc(Et+D2OC=5rNO;Q^KG=DsK${X;*Ih^HoQQw1Kkj1XJ=4HA@?HN~X{Upwhieb>- z?z}+i(p+VHJXzWCEKg8xs!r-4M@3`9ttl^G-)VPW6ltT#q-jsMRyq0lPJ8?3tWB** zN>#b}`Yv_&2Q6T3y@0Kh!A^U3bI#so;!y{&Zi)_ZyDInQr5h!^XJqh3QQk)9?)97W zmNZv4OGb{qJ(sE8nNif>m5%9*XOn{}di)E>9X7!i-JopNJmB^Y9KQVrP-6d62T-fg zY)Q=u+VSjE?cwyMQSOkbJaWLH`>kI4KTt~p1QY-O00;opY(}$jOC=lw)oezynPBO4pl=EtUda z9K84nj%ggjD4F#j&439GJip>BVkBdf_uw#&;T)kkoKsG-8yfXmJFOjmIAJrIa2i3L zL^K1jq;R+zI*bM&Xu;rR+vG(rfL&kGhHUN)VO+fmyoVKstBffCx!2{w;-A_xJa2 zZ*O}<65C6&#eS>+`Tog&=xBI49`3#99Y~b-NlZD1jQ*6fj7D&^0kU4lY(}nP3Ng9G zIU|dV(g;L~^xkGnuw>B%p3cQB$!Kc_BF06=u5#f`mZ6Su4c^52K6k-o3wo9}dsY4^J;f!!f)&hog6=$D@nUyVEhedjp4mr$55?qtoLqP$rf% z1A4d4kbYcPoJP5HFs9TaV4f<1_?phxoXsF6$s#8U3XAlHW(i9cu%_9Haa>FeB#Bx( z5VI8%M97lLpn9!V5CpAZB51bG7^iSSxqx%JPBQ`D(u8J2q?znGT_ud{kzAx85QyhY zP~K~`G#KUZpW}Cbr`Rogck%WFGT923{LpXh!5LfAm?do%ft{l*9nnk>ef7=;FBEq!>2sJ=QcDN=uLu$>lJmHcVGPl5hUX z)5LtI+oQFY^Gt5|B(GGfd1VD{NISif+#?- zStG-0u)}22g=01oT{vM}bm83^`yp{lLHG0kmSPnfoP`z`S=d5J7ewS7hs^~^1V`fv|8ce>39^L4`Bcq?V({{F>PnT<=yjd+OPYQ zKkxn5<>g*D{dhT<9PGWGKAXVa{`A@8&wKwf?F5|`&d5Li@%MlJCO;Z%P0fr|`hFX+I3-q-k}-$Dx*1i?3mW%MN4TZHemWc3eA znxHsJtTSoEqDR$4EWq?ehNyYC>Nm;`rc+0L!0)0 z7QHTf_<&;%S0o_|nuQWd&Blih9X$`{RODIW^rIWc?(CdN-J0eC=$$|XnkGFn$MRoi zI$!`P@6qIjWogn|P|*(j`3&^v*xW@c`rm;!X@={7y0WilM4OQU-hyY6ewYXMPVGZ@ zZ>Q@)k*J$JG3}xsW?_KvJ#@)x=Cb8~bYkIM5i2apFAku4^z?{)If*TU&4OA6f;jnhwuUV%(xxAe|YrpOr zqw`UIL?4%gFKHC&U#h>4;WgcSl%0LtEj2dFXhajiNc@qknFQo~DgRw(>;`Qhev@1* zL)k({_FQh}l0TdD2h&UbtTPFJE_bJ|+n3`nUw7KC2l%u5a5=r?(~tk&DeyX7M0@Vc zU1j1<-;9AT(MypY(v#;Vne$M&GNDA5zDb3`$$tlzyO$hkn||pn=woypE01a)jFa?v zOW1zh*WNR>@zHu*zX+vMBXC;Ktmw1_y%W-;RjirPEM+Ns%x^VH?M_&K(Obmn6^Wsu z3SbA;o0uh{kAWJx?*j$8#|05|MH68(H%79O(y|`LLIYIkfN`m83Zxos)U1Le7U~&Y z)J4l4lbIw8O?bJrq93FR1_rUm^Q*Rl+m-d5R(TkXlDDq^KDQ-wka3=WQ{Kkds9+=X(j5(+C&C6wGJr6_QaYRES%x7W#|bs@%>K%Z z8Jo3yr1?O0Rpx{HV1@~T8Hs~F9CRVLB{IOqUk?&EU#}_pZL&%ELoYGfa1H|n(Btcv ziT3Vgvg<9-z3tHLi07;JbIBudf&)-Zu{rFT;qJ=f#F0{mGDP=(R6%`PdbV^^m~_~k zVA)icI7(E0KDeAT!c;**V)%~67-Ihx-GoP&Eu%b*Z>Vv{BbK$DD6?L94HJ%$G6tvG zACFi@XClouK<^kAGTmZU7qfHZil=cdly$K@m-60-UTPhD8_;j>O@u_CYz}p)=5et)ipZ$dSo(-gYIocIZcG)^2ml| z;!k?Xa%6u^S>h1UMOW4#;h~3tdeAWvXVBE7wVgJd&_ZOJzQ@SYeNNXjaS^%@WI-2b zGD{|(@ibsRnr%bn;!6;v?<>0z7Uv_~d=mrj6bWPr6fP=8pAu_fONy6Dy_hFUZP{7;* zP(CAT3Y`9blq+T5o~3b`0j^lB%b=ITWG18q4FY30H5sZ^nUvW=$58bxGn_ILOJbQ* zBr`!XUhc8b$C4gxwcEk(f-VHVBdhhFf`SK&>Q}Nl7WK7X%i2YK?QXD(wLj<@lzY znBdhh@==WW&tF|FKUsUvB)!D|9PMoL*D6g(-8@@=n!!K+{tugr#BBne$AT9*Nu5i; zEfl+jpxkX( ziBeaZ0p2IruwV_{t&LvW+TV(lULKuEH(=am(UC+y_Nrd0;11cL%TDhiymvj=z7qy+jBSw z1};DOU=EXD#JJrb?bPDxE}UOJ7#AcHaytzMb}OToq_^#Idvf`)DfJ=qU|s_~F0wg) zezb!xe*9wfMHGJV-4}1a7zY)~q*m?AeUORo(x9wPrarvgZBnsbooM?AdR0QPSvM$` zh7O`UlZ)pd&Eu2^i)Qh3NLc3SZ43_s)tg;LQ_;4f*lCp(M^_~C6b5h~ur&s62D%G@ zX`(1LB@d;Rq@)a#;NaSobJy?Qo{kQG!}0sk#Sona6369bQ^JGnsy-SUd3$*N{qP)H zDL3JY5e%T+C}YiyAH93~_VDyLy!i2KSn8rYd*M1`i3pdJL^9@IIXi6L>{ifBHQJ=2Z`wYX@N`B0b!2kledxC8RwZVf^-yHsEShb~piPMJg zR~sbA;j+m=Az>z*(I`u*;uzBcVY=YmcuQbon|7QQZ}M?He8YQ&)yDr~+6w)mPp)gN9wfNCUgIPr;nzI}K6{$#klPuJV{dA*$v z&)%J1!032;huY9TsV~+1^WQsD{>RJ2Y<)>aSWppqDqL{`N|eagG4HHzcp-M8VJ(aF$Z2-i+D;%SA0y2jGuw87>@*HUqR`DqJNSCX5G z;DQ%L8>~-FT&O`M0`tTJm@j)6TsVf@gNrFi!UEb2u61cWqTDxBR$o~H%wbMyLFJ;V zGeA_HH3EspgKjgpIf9Wapx5g)Ku&8*EtjQ9yn%r~#99o@dakZ9n^_%I{q%xbbA{w} z&CyA!rlj zWp{~jCQmudaIa9$S6A-vbjp4aVto}(H{vw^3b)EAzVIvXAb5B>n^XfXIPzpheBH0IQ{)^fx~ z&gH<_DgvW-LmeD{6cgyfsP2M(m4p(Y4eEn_DN5`yo~o#~-6Qq(G={zbg{xIbfdJ^PyT+0>7x+590$T~>*z>lB^Fv&EgpMvabZXx3GM z^>`(pB9!Nfs?6zD`rPl>ZV##pG2>KiS6VJG8ynXF zVN=_ty3(}8`vhO1IBtvk^}mBNbvxxZ5#rRuI=CfO-&XV(i*f zrf{{#QR6h>ZHS-UeA!`fvg_Auz5dzl8$ZqIz?F&WW=y?PQ-$kR_ho8bHWkTL6NV)N zUQo3IcuX62PUCa|>PVmzUz0+VRLWeOmnqbLkfqI_+A;cL^N_2j5v%A9u{x0}58qTA zH&rS3ar~Tb)sND=Y+#)YxaLsIh(yr0vh>RZZi=vcwIUgsv#Pxdq|^m{h-uPxxv<7o z8K1weMY0A$2~*J5QdI+?n*nsoT*uXkIHU?6_=gJ1+tl64=Civ6Xt%p6 z+nW>I+njpp#-4~Xo?7Od-4a^SUGs4KS|+#)K!yjSmK`F%=Il&j@y=cv7Ea`1~61xL3ilh z(Yw;lQdNDkYQ%&WmAlIN&;$=c5kcEXUNx9RgQWZ>)oai}qvY&JZHwak#H4SxDx+a4 zU(sN>!h9{S7`tfIvEmpe<(8*iSIKv+sx6r>JIygK5rFN~+6RzoO3h2(os>F%<@Z@8 z0zTPHz+-A3K2i0qT_9&^gGfq|Mip zO`6lxv`^-hL89aC`g(-8;9!r`;K8*RX(DWV?#i z+pm*&oaM0YFmqMZz3I2x-<3mu&NU;$v%kMsC)il6(_QQc+y0(a^BI=#589 zx0AKsY70~fKg$!9zW0>BDKcub*fx>qeyaby>;<0*sha47-kO7@*lu}$m&__F>)qr@ z#imRwls;cqwe+hr`Xwy=s$%J#zH4p%O6N*c{8MP1x$XQhl<~p>|S6 zpO#T3HgK5sKZckZo#+);vAjUSA~kNxKOu~@O*=uyv^JuxHLQ;i4VB~~ ziQ~{D9tNeUtB%+B6w`WtH^}uUoQAi?l9jZG`FmblYHg*{el6e=l0bGwwnei4hxFBL zdo|JLI8gt01Xj{e4HOn0Sgj-Nh_dm_9R2KvRFlH z((2hEtj4pz=9{yHe)Gmaeq!@mvW0D}j!h?WzvTX*qf0eAbm72%m+zXT? zDocf$u2N~~fRH|eS=X45V!O_(U>E$azv}RYjA1_w&^olVD131usM^{w!^Ht!iC#S+<9ym3( zERtcWb6{Q{I}L5s?`>7}puGetpFF`08kSVDnfN-9oY4mxyWcD7A8ZhRuV?bXOlfWC zD;HAhp-MM)l_&kuSoYv6-=W0w*)(Zej1GKVMe#7?3|si7*uRN1%Vm$bI(jIH^u1Iy58ayEBOaD7_gk>Gr~^AyB*JWeR~S>&Io0vPo)jflH&u{Ps{tys z6}f94blo;Tfo;^)#fByECl)78cJTy8{W)TUM=JW&d37~t4U>HJGvo(PYSi@TD#p+4 zvH3z#@iM_%>GJHI&>I@dCo$=7>v&A_v6;}iZ)z{I2nCAA3a0*3br@6lSZ(Rj*J0Xs z4$!@Sm(LFroI9i9c;Tc~g?)A@qNz^|jQU8oYopWwp53mL1j@!2TBmPBh z{rqV}*KxWDx~+om&gFQ36c2foUmLbh8RJ#2C9Qet;fTbsd3wOSunByLZ>jTH#@6^0 zG(L8U*t+$#`%bb-Z(Y%OiVp>Du!H>@nz8vtQ-N=w0M&u&BZzw6)fH1 zlV787*ahGTk5L#_l7C}-Suv8^C8J|KF#jD^XIKT#;oli<--*;aET+yM%8i$Tcgo}R z*%FVSsXM7lB228-{_sKZ>4y*aJT0z&7=BSP>vjAmWq06cNoUvc&I>-5ulIULjCVIT zuq4GZG#*PxWV>B7sD0~8Z5UP#-2-46~ff6$x_%&<$p5|NK`i@YL8pX#77$G++t12qxJxJL)ImYs+&!JP5Vsd z4_DSAnoWzEL`(e-@cjiO; z)T$O-r^-T`C6rJ*Ob+wAPv{IQo9x2*i%Ja9^%jHu~ zTbIpDn|eF+ksdW&abgZDND^wn41qloMeR7LpO=D0v%^;hS)+L_lRzEUQP@6CbvN;` zR@74Ly*%ibS78S(vJEliNo`V^R%0@YB9dTjh@ifvGBfJC1FS7^ue!Y}FH=AgC^1&HX-nGB}PT zbIND=j6>Lmd!zkJNuQ2ux_fmu|Lrx)!Mr&B;TyBiw_xSni9y3xI{<^a>;Hq&Ky7`R zRnmM-w$(hcqE@$8O7sNEUzJ7&xk7i;b8fXc6NL>lca3VY|V9iRS*eDJfTP~2>D|b-#QO*p71=$t!{uMK!TE!nW@@8 zj%U}gNT3^NG=AuAG@vbV8YhoDn9T(l^alInjD;b&h`cU|ISE8c(&Z!#{DAU+rCoA= z5{BdwN~h$KrHtP(zuVYqY?1eY$D)+^WEuI46ESDxB%$7%Wp6sw?fBP}T4 zs8>#V)Yu}A@scd)BZ*=`mMJ4*9;9R%gp9BUk0pWx5%JNGVgbHjh{LF=aNQT?n_xJZ*ijD2Ye6}6Rg6a1Ai_`NDSLg1a z+ruiqL?KI4!r4EU0cSp$JQA8DVc^k8$Vf=XlX>5^yTwMS3@t14zkCV%P%aae+ z7w1>x<0U!$_~Gp0`r_ks)vB_Hn?r(sAw zom^d=pKZT8xp;qm2IR#uB@-G&j5l0zwOlMHemnMeI(4P++0ct1IHg zQQ)~VPW^yIg7|T?6vUfTE|5wf7@2TJzxeTeLLkG)+OhLJoSP@ji%u7ztT46LZ3sA!rC^ zgtL%7FrWBK7)-(z6XyHON9EgAvKA~7UDVD-ut0U>tTE;Bf`D9@Cly}3^4XLwL*WOW zkY#)-mkY0D5xwb9urVXRqP~s?2xm-6CLtCNJXPDKKXEUF}^$}M})I3 zU|i(b4pWX58*UvXMJ@?J+%Nycu-@d&0tNZ_M+fKUyvs@~i z7e}DkB5jPXu1~Iixf+9Bk|W|IG)*1j>G+pVpUyAg9egU@UH$#y6Fg171c~{yr~#jf zccAp}bV`HJd|K3iPfnwOPk*^QA76YpyEr|${&)$kyJ-M_T4R0YAN8FMah4Hh)SDxg z;Dad@G#tvmpxBVyI2~DU5r@oMh77uY$M}p5i7W%9^Otvw2h+#mRjDpa6z%10qtWo$ zl#BtZlQE6bpe5LY7?M=*HgS*P-=U=8IG1b@-!TH?NLcF8gpri}a|sxc5~%|0L3BaU zpay4R$s>|~je9p;PFjxB!9wkfv7|Gei$%DG^A$8tGWKGhrGQ>h)&|5`1bP}Gicn85 zp62ZUvgD}J$F|amvd0|@sAx=2P&J1O0s1mjWKYUcxYDoYT~X6%7v1#k^aYe2HBKR3DTS~_kt zKTkgUtA1za>2tUKk4dxLAsxSyroP|Xt%QUT@n^+6_qaJmLf+T1dSSV#`LjxOs z$x&7j0bQ$`A$z$S8mPUGXXGMHmyDdoOR!V_Ea>_Zuuvi1G7{r3o{aRv%R+L=#q$3Z|r)b*_nz zBWszy?2X3C|LMh*bW5Dnv-RPYYi$+s0a z%Dg~QICJPODPu>vi^|cd_=i$xe#zDAkhJQubWkgE)N?H>F%rwA-|v!lAx!^&NFpTl zc$}sLd_%chq-2}$Wd#2$Vjta<=L!oLWkimUe0Rn~%Tdpq4p}`(ZKYv221_;_qHl3y zO7j+M?ORC>gfj-%O?8>WUrG79hh!za2|3Tu=$C(@#~}aom(r()k6kqZ zWeXGKWdc;BPhQ=_&A8B@i+{?NXMR~f~)T2u$`4mI@zB%5vqrBue@TEicC5rDOO%U*tX|h&-MLm-Ajlv4Q zvpO)r92J^ypq+dfne8z-+^i}uiFDzhgflER!Dw~IUN>Wpm2dKow>8C68OPCiWsh(U z)>O-8!#HJL9QmnUU&Mlj<4TDR>D5s)W{^(Gd{I~3ix&$T`D5{z7(z3iTPi=@HjHPW zU{cU9c50Za9m9H|0-wd2#MRNNw`G!aYAa>$V3C|JcM(bhEJsnAS@c7;eVH5iv9 zBrzUofno+x9XeSh$>}{7%R6^dqJL|x1}0J%-=i&(k?7)sR<-@47BO$|H{ztV7W_C$ zL#mLGF|JV(qA7|qBrRQrrk;Rl;^YBCr=3|=Yv`&N%aG9Mu@yvrzOwU_P(ZEJLrbt| z9bIX6IzwZ`3xia)wQ8qs5XDnfGZ9)5DGJS3aH;Uag00z)3=NMty(dTD_dx}PB=|;& zR0fg@d0J@&Q-hoe<(;27cGkpP_mu|S$kqIo{CGtA)_N?gGJ*);r*Gzj7Q$Li-uETdC*Qr1-bT9(J)|HbgH`GRVO=-=j^Ot;GLEL?)K^}b!R5>)T*7b*R9`lXl zWT}nBq&Wng&h5oDgW`exQyLIY*+gr{K$%WMWmVOnOS|NMMhvWPP~^6!q0Ses+C8l0+w&Capf-YnJ|Sc%MWJt93s^L!;U z2%xVGu{eE~DwtLnXdIB3zPBVN4m3X%bdgBM828`0QGDNOcT>Tq@Ui8*cuQXR9rEHg z^5SCn;%D+AQErU#3&$#e2|el%r~4mq5VgwwR~wgq+tNeT#}C^lc|72(*s~=xnt|`q zilgw6r~wC?Q%b1=nGfO8hrL0iVzOCdh$Y(9LJlLFq59Nm)L4+0WM{uulm>?A)~kv? z8Wq)`=@m^>+Ub(F%OK2*87s4EO?kI1$t`~9_XtZ&zBK6}V@;aNOMImUjS?E=@5Hz;!X#wy}sY;hLy)qKH!@>u@}^qc0!&!OPQrM2}?_%fV#BrPEq zt*7Hv#{21TTli}#c^F)Dj#sI8UQr@^lrB`s#_IVsap4}38nu0T7UUW_Kx`!Yi<9j=tpJvqnf?cuf`+#P3s zzsoH<+!p*8Mzb{B`3$y>HndN*M4|R@8nTBG4TD+aVt{WbyE&TCWZ3T|52FR;vmkOO zu@LcMc#u4doQjarghmwaP?2zWT*)E=Jt0zxR0Rda&Z=hmQDurI* zj!qEC%tatx9b$+cKU=x7@e_r;EVoVU@o@3HXt>^%8(FxbXtI-Xmf@BaGy_tdFj`;R zO`ukznxu&KLu)O12r=G@7G_j`D|)Ck-Xe{0I)@#$BlWRTqNEdi8GW-lxGSBNt{qZ% zpBQ&RSfK1dDgrO9b_(oT*z9FdEAM2xQq1sT`-MgnaskKYCX18Y>K;jzn(321ms}P} ztJ2GTR62>z(45Ba2@vi~9SLRO>cS#jjTx-lqkp1E=z<+U$ ze^~f5osY7bwvC|cQ*v>CmSq`L-V1M>a5fDd5O8i*X6EEg6T@}Q+xF8fk%0<5iq}@Y zjZ@_dT~=sTokm3M1j|0!M-d1llncu&)JT*^qdxxaGf`fZ#Z6xE)*ierD*8-I--Ky_ z*|fUh+w|fv=EIW5XxO{^z22yKjJ4H>Ip4$=)O0gn^e-)ZvG#X=@r7x)nJ+rO7QR?} zGv8vs(RDdYIMb&|!odmk890>*XUEMADxoH0y+Sa~Eeav4-QElbyHvUqyvjxKROhT3bSsW)_LDEWHrLhCRak$a zuVj^aPY#JAJQZ@}$ipY^<5{bj3!vGyLkPi?CP7`fk{vmcmV5+;*O61Wr?v7r6#nrl zN7AP+18Id`oa5PGu?~w;<0yv_hs@2)b>w;UF%7Q7aYUGnaMK(e<6~loR z>JYICeN1BRGcF74Bo8Ew!@wsJ@#MSqD4{;Ypk*PfRSl1n+97*d(b&sn)M~CwHvkkj zYp%ECY0QUzNgQB+Mlr^Sj26gHt(&d43eQb_*NwPU?xH+@Nk?*BS9KaHZfetKrHQM;<9Skh zD9>KT7?pYF?F(JT;hw z$Fqxh)AoGHMs zBl$jmp4AdWB!`#Ka%-PJmU7@=9fw5$pIC+qCH9Q*D#cn!eXy~*0O%u~4^_Sf$R%Dq zUbD(usnj6&m3rT5u1pBG2h6ib^~8dL&qu(_!ZgC@RxDd%N@(%VC+n=-my=R2m}OVh zh3$22h5R7~Teo7Aflc2K+JIhG?9#V|HUN}=6`u6Xp^Xq!i|&>ky*j+AKDEsWn?`Cd zwhqEp70c;qJ-fQij=gSiJs&E$BlbNhY6JopSm;$D!PnLohj1?_aR6o{`25>V%mJh0qIWHr58Y_*L++!Vo zq@|8$LEf37c0Fg0S06uQOdvnOqY;!_rn1S{Z$&7IMSe&k zg@e`UJiaGMkT5vC0^zh7Lq&zmT@{h4(>}ha#)7Hc!cDC$ied@4wS*A<#53S#UzlFi6iHi>v!&flRXB6 zJ96#Q2-|+-PEIS6uR33G9nNmldJ1_epp`}{c*`h zxcK03TNY{@A{4@e*nUZl;0JFS_?7YiR3Ig-_t5sXdiQd>LCBEDu^=m%tO%xm_>gQ( z52keLjqsxj7a71a{87))T8D@I>E879NY#}m3%B}g@Oo!5QV(Da8uiI~TNRJGX2P{*6ec#(}o8;VZ1{A@vBmF(7vty^3^Bh!Y?v-gigg_Jg zHZnk(dv=rHeL6HX%- zhMmd5V7fojpSu1RAnxxnzZf4}e}S*m6W^P9?A6G8Gle(({r$b2U9%p4G)?IjV*@rA znQyS(>(~8Be`3~y)@SvmJD$c5-^hA{-CmEG_2BTGy&k@?)Pn#FRwz^$9$`tPlBy)* zJSRPvVprgi-iNGIrqfgAQI8f)g!@7YSpuR`o?i#j>hJ9NY}O%Let$~Y9_hX4kS+RZ zy5ob6^j@?_B_hbWx4$=k-Q6$o$mL$kLym{!A)8s@Q8z?j{>>qm&wvfUjO?-CG)hA% z*l#U&H+g6igtu0|+Z&)dKRvC%E4Zu0fU6rFiQ#ykb6Wzh-n9Q}Kuz!}#!~~Yot@qO z-d+j36r!2nr9h_)UJBF7;HBtVz-#x_-roM}Zvn5J4d6BScI?`J*??UYlAEDxXCrhC zY{<2<>+igNT|%zu!K?nOei^wYlj&gBhFtqSx;Ld3@V)Bckt$;l%|bl(jpTTVnk#7zOui@+A$i6RrI z=-zXh4B^**h+v8#giGbXO7NEL%dxjTB1sI#RpMv}mr}vGl@X@8Ps8|rI1hXu*r5t2 z=?sSxHjOz0DP^r|HZ?kQlE&duun~b6U2jA%!qFQMyy{OX8pg(Zqjh55l~P7j9_;OP zvj2nOiK0erPbh~AFqDW@@ZM#2EE1{QhBF#Z=KWlMi!q)&fRH`WcU52Yu*q;)h6_@M zVRC!jJt$|9l|daY*Fe|CWP+KQBnD#QaoFp=9uX~yS!uy8v|s}(SKgbrT9x$1rWl|I zxB;_~D7t{F72W{iBB>$r8Ie*1Uhm9ac?#~@sABv^Yc6i!YV+_~zu7Lm4}>>YlM-zw zSG`$(t!$l;d8hl>N7alSsriL|Srlo|23^EOW03DZS<3C^<+6<)DbCWCk<>^v3Br){ zx;rUhG-U?L7Lx3hC%tX(q$G7242N_oBuZ_Ve^M-ieN8O#gV~wA-U~xg5~6IK@s^Xo zb0_Ry0pqP+ceeuLugJgLAoAJ6aG-i=GnFf&zkWn_+Rvq> zNl|V?T^Fe+NW&7B6Kvj`$Miqi)`?DVI+-B#AXe$l{80$_n; z5{Gu!MJyfD3Jl7%AU9^P{}zp{S2sm}PA!9#p^%2v_h8ZUts3;Wf4@zWt$w9dr)<@{ z?x9tq*)CQS2OLzbOxtK|j!0Ny8uLY#CYJjG?Rul{AvCCUaf=ND?$G(til*s>Jg#XP zHam96QBwfi^`1#1q!SjFjH7<8HAR1L@5{rbvuyE0h@)MBwO;q2+!RQU(>p4Evy?%- zbER|3Np_jSY1LFh^ZI%@aG8|~nblUv`KXNQ^?Y({GAq^dDZnh_lVyUhXC$)?3r1P3 zdQN$boD&|>tFK^@b$*7}rfd>CEe>FT)N@Q3R4@%6vJL5~p^QfHZXF($eJRyOB$=V4 zIu}KX6InaGUTOM&6MXKjUz+QGt;=%jG@E6e=T}TOJ#a8c9@>@;&v%NrK_;vIfz1N3 zj8M(A#4Ds-V?Et$5SKk;-OJI9IC($yYNf2;zTq9HPZm4avT$hQY2Ja|D$QpC;7(TA zN#>_gm;mRZ`C5L07kXip{ARv{v(G-8twHu;P-PGD@vZ6#3dgpP@Ao^`t7MJm0_H(_v;;Hn%KhoVuFw6cPucDy?gi z)jcXFNvgWCmRxGIiMgwPYrt-`WxINGEQm1HP>vpCsX@hYP-RV*5|fHm4=gon#;Hv% zCD=oG)|iu}B&O#_Xv|SU-%;Y}I>DChHT$|%YEv<0Mo6ku7E_7r>!w_{^!z31=#+y* zC5l-LSu^@;2cc9|WCm&xxX#D^6UZVPa1_jU_%J0SV8c>e8GC(y;^eieuq$`+p0_${ zH8fqy^#hD+)yy^eVS2l@(*AQ|&%HtijBd;aB^dk`vq6seJN$+SvaP-`LJhwB-Uw_r zkHZ?}U1Q&=`V$f9q6SIpxKtxXwQE|@uY(OHVU2%Nc95PEYh3RVSEO0*b5)$Hy<#W} zA=&H)-IDC1C0U7o1ykM}H z^~hsaswC{4=~{G9;Q4oJQF%u}#dimrybjCMs&^*hr2sJ-ngjn# zPVcEB)3ZJC)QIweSSj!X6GAQ{TvNV6QU#oAUK)N=bmDz5>}f*_L_dGu$Bk`F&u@Iy za|Ws~o+ZDL^6sK1*Ty<8EdTCmIUUS=@D}zMM`8^UUD^MQ6*hlT*NAmn>*3I{h8)nk zH4``W*zeNyXaCO=@S!~T7N7zLa!&+7L-V`jO%MNWENaD|p~Z~rf3MigaxJs6pFH8u zRCQ28Q<1~!Q6eDfZm8Bhdt|&YEEGA4fG=KAjoWIlyFiohhupcq^u5TyZP9^|xrU%k z>`HPP1_NHN^v@aGQN%513#!7(ZzV7)>0d%fje(_Pg=4`lIjHC|sEi8`6ZXJ?&*2vN zN2(@I*UaN^v=Sd3jOC9b44RKn6f5e#nRV5Zb~t;gI8NsS@HN>T$~VT=jmGwgtt+t| zr8>DlD@g`4#?E&Q)hL_2kTk7YG{3=H+VIN3yD_log|$b6dZlrWQnJScIyFzY_tXLV zGA!TyJ$u9$cdmsQh6D>pegeXTMBuDp`73?(S5suacPRzt+eK-&S?tGr#;iqkT&LRr zkC4z@BbAOMx>2N7KwnoFM)1viAbh^0$1rwZ{z?dTl8g8d)9i4Rpe5>i{21ZTd3mA{ZRFn;r;K?6Hz|wh{Kq_=q=-Ay9<6L{9Xt~5h#|0QS9wU5AIwgInWojBj3J|jUtYEY8VCl z8*9nqwO>py(C}fW+zYF!GF2@QJar&YCxDZ69r#5Uf8 zF6wPe{Oc4!o*vUn1BFj%O2h6J?B}H(+wTDADfX8G!-Aj;?I&^h&y2&ybp8D*nc-p_ zUk&vw<##bDda)fM%2_P7HFf9xrb#SJn5AIK=4fn(K#&DGErWaq$FpWCo2U(CP|tc7 z&8AN=qS&g>%OtNr(fQf_eC`q4>;iRs?|G`Y&}g!Fi&F5(G?(-uP36l8jSV|$!kSoV zI>!?o>VcL^4me|m4*h>;&k2!%m4ABZ-l`F+mywn+_cP)AWT9icFI-m9Y?+|v zecatRo8$EU!Tw?Xp-;&P$vEwpP!CeBB8a?#97iQHJ8~G>;h%GWGt66~z;TeMyuujkwqcV8V&}0wja|yHMn7U= zl``95qzR2(`>+ONHvW7tUm>aJR6*?y|FLd=@~uywu)y$GsC-ev%L1yFHOttCC#U+w z39sz3HCb!0m)Ccb-D=PJG(xTn>meq&!eTwfvf;k@f%Dk54DW{?_<=#;j2_mUix$i6 zo39{&FKIq0-c`R>ue|hs1=%k~93GL?+aZKZ=%9QX&|Z{@s&%)D%6nt3dnhNa=Wk48 z51RxFKd&P?`zVrk%+Fj-HdaPV8{RW}%i%MN)WFwvx7qS`Z^Jj!B5qOmVpqc`Zhw2_ zhQ5iAct|hMpIaY?l7nz}!K{{hf|5&JqL&mTo#6iq_m5P(vqKY=E2c4cW4rDI+$o9~Z5Zi-Li%FVde0JO>q2wN zqdoB&I4$zbKl(w@i5eYuE*-%FL|LI)#mznSJB&~gDs+s$|jK$K1Q)ty%hYAU|WD-2=psN(=PE@r& z9d&NzokKs`(<@3e>`0irsOgeK1l1AhEL`o^oWjZ{AOilt;1N_bZbs9en}Golk&`?g z?@>`(?L~bW0%wmfp?X<~STPiq!kSN?wOOCsx`ekv5;MLT6c?F@aLg}S3=S7@F^PMP zb-O9zQoZ-e5d~L?dG%<@fkKYtCJ5wUdT|tsL-8lBYu3~;hkk1N`IwzM=!-nnhI}x| zHcfsQ;p8>3Ag8&@>#(v)Zcq^tjWMI2nHyU7EZyXWtz`20NM4T=Q?4f@m9a=5hmuIC zpxAC&wWA_+$MM#Ie}W=PW`jU1tJbMB<`e9?^Fu~gWsI+l5zMF;7N}~A{gET;8mkkW z;mO(W&MSu(aZ(G|$7G`QFA2F%69tUdmc@#k7gv;Tu9Ov|;yooXC z@&jq~Yp!zTTC9B*STG^&Xu2GE@FikT2FX#uv02Z=_1H=GYX)eKvZBm)k<-SO?qk$j zB%aElhHd!7QjEy)gNTDy#V8DKl7nzE@l(c|NVi1B;;voUKf<`iuKHzav(sUIp_9zF znWn#*FbQH?5dE{;E`b%M3J*^3bZyCivuwTTa7A z{^FLmD5Ij3tWe-QF=o4LS@dr-fms>rsp$v~tACa_0;MiEgW$(_`{Ln?kO@a^)80AS z)yyWCu{4qUpzHbjNGna##)ZweV=rcg7jBxeGh|4PIS`34jn~h7%iT=CE|rw@v`XskQmt;bCO64pG?n1y z%Xm6Jy4ZLejXdd8lio1d2=|PcF7oilbx+!!Q#r3VhnHS3Gmlrku1T+(c2&wUR1A%9 ziiN$eT5rY*aSR8stpv<}_3ls6OohtB8&04vHSnV+FmbEeBc%@8T_W*kY!t}ZwH_QL zXgr~(mc9sswZ2zxvQ9N#%~j&q;ne_{06uLyi5@+tn}?{p-#xZH@A7Vzq-LK*`H2-D z;8rIeqk|+Fi7$Cx`jJCCzQLS#UxD!oHS*{<+5U~noN*0E5kbsEHc8nlN1Nns*P0J8 z?8}DoWS`PzP6Z@VO3I>1oPeI0C*OKOMO^>XT%*h?AzrLhM-^wEAe8T+b?Z(J`OSr2 zWR+8i^shwb12q?ZiM38??^;$7Jq|PLR!+QjISHGg5L`DsZIKd1;S|I?tFW>x){69b z;zF89%Fm0S3$-CJ?Mv(HUmyGCq5AuUg&fSRjlR@Ou2p{To*ckqpv!HwCORsNKUHqK zIt$y|W}Tmk=hqrV8;}3vEA%!Ye~sCk|NL}*yyrb-URi3Ysrsi6DS4+%>ZA%T3z{M$ z4=qsIb(N`^xZ@mgGcOLjTxiBsCpdnYZ+b<;Wi1_EppSbP9e&X=Dvyo?7k?6U zk(;v2HeA7woT}AZg#gWFzgrQScLkKiNuUyYC34vt}>-WEPr<)grTu3oV2U9 z58;PFJnHL0-P!OJK4TlWKkA3?*c-8PqK5`ug8W8L=7;NzJ;0TyomtWWK2V&4iP*&9 zo#Wphk#rZxd6%u{&XW^S;TA)=G(5H!IHKtS@?1+%W^c-PZL7Vq|6uM52od=(yemDl zV>@CJBj)@;s8_=z?kQZ03A;|~c$IL?=0@?S2-8=7cFv|gJwh-ZyJ8b681DK~Gz3&t z0aDA>n^taHR$uro>fSGPbX~8J{M!R@x#A5Q#vrpu&R-fe@WD&vElz_-H2Ksq%mqm;(2^jrE$2nH1=wC z+lj;~VZRG^9X_i%A)BkCQQp}@d!wT$DC6Q)jT40} zC5udy8V@p&x&qG2u~E+zoNcJ4?>M70+AMn%!|WI1YHY-6e6(E2p^FQS7@t%Md_Zk3 zf)>F`>;$N`O1v)YBGqVkleMQc!X;@PyQI*Vj9)~L0_ww05o)l>D$lwK$xmnhz}Q@qGlGBC3vz1XAYW+<<|234lCy%^-8v_V zys}6+R4L%1YSko}-FWQX^D-422?nv)O?^J-ba8zz4RPWU9|@^Y+9ld77$pl7_VQ2) z2sJ8Mg;8yliFTd~H-Rjqole|9%jCneQhcoR%8bCRxSRfFXRccL2Eu3mM4nLrTfKO! z(qLdgzlT1oPFt7uE}}Dk%ZfyC_DV8qAJT#JY$+^rvz2_Rc-%7K0cD!}SqDT^v}N*? zuX^7vqTJawBdoPgH!R6aSCSJ@bs=xYLZpo%-Whhf+p~ImJDSD3_qk+#jjvl8SV%RW zS4AY5^O@?nTY}0AjK*Ba`g_qQW>HJ%4uNN$@N9Q8ki=;Vp2!Y(ztg$WW8jEhQGiUt ztl?qy`=(mMOZ3K(5^J)?UJaDIn|e$vsE4OEm{)sZ-^Kv;Pr7|CtIVm}DW9$%|MS?I@A9D#7 z9$)Kk-*fj=TFAMc4gk{}Sv9(-%6hY{m7U3lk#DaEQjJr|E>jIIl<`3R`GifhJ5ECz z)a2vO<@ARSfFt>t|Z(Rj?aDd_2>l-O0B}tp^B(t zkIDP4@5sqJFqqK*m&g8_7T+84sau)1G!)12bKDdEBzmhc4XkVY!1w!tg-Z0XKY`Au zxG+iZbRYjA-F2iB-iN$^_X=}l?C9Mh(ek8!LZPwlQWkjvqVpL zUiTei+X`qPZ7N_hL#yM@;EKy+)VZ>^5<_PrFbhpma2LO(B|-fu_XGb;=(QLEOY-ec z&0eT$Jr+Z=0(XlIxk~3LenzBl>^gX_^i5FU4c*Yy7E0L3NQNN62tAw2`lMKg`k}U0 z`cEcBUwnqfDdNm67dB>EJztF~Ud_NY)83CN>uW;nWEK*cCB>?&ZUQv73q`KJahqKf zbYTK5oI^0`HJ`*$@Q~w`Nek}$yc4~E!Q_>M9hJRT%qNL*`blwYD7kd32*<1z{IW1> z2vug_OIRQ$a>Fj9{&2wqx`w)}vj4H+CXpR4qZ2`X{nl ze~Wep8B}|J#WaTQrABWYGdgIn06dTd=nMV|55z|I2n?!D+*hJaJhy$5lfVd$g$Gg_ zBCZvzp3a_lUA#pzRdpki_Xz7d|MEc;PDNp{w_w3XL18&RKB1v(I?i!3B+A|ak2_zO z3)o`wuG-_mMLoroHFu&4yL_NfKZ7U_6~5HUsaVv%vZ6+Rf`KC)lYqRBPQ?*Sga9Og z3UvkxAOLYeari+y<&(&ZYXXm&J&eDpG?HP3-y=Azlyzg-t94@du>>nS$Uh&_vM#(|{~f%u?A@c@Q|E~M32T(^>HoK*%~BBhM%v3T`zn3I$M zDn>y+>{Supk$k(e9-A)0*|_{11BE?7D;Fvn1OkYo`{v8c8_GB$PY(7QvU)zn2x02Y z9@Uzn0^O7}>AQnaOu4;ADtVqEVm$`iDvAg|lZNZ98k&I`G&hK8oLrar zRxPka(5&N*Wva)OCuqgisK^~vGPE!cQr9swZJ5`?f0RBh6SUEz!RMkFF-ku##~x<3rp3iX*7JwuAZ z_;~qCMa6S&RVu+}bCor^rCF9Bj6WNl-btm9Lu<{v2bOsSexwHfQH{g|8Md^@kkTK` zI5Yjc{hXD=tG0kQRzlfyINuf*a0LykpGhBfvRkC-cEv8$^zdvEysbe$)!tc6Fx%U7 zuOTIJE<~r^k>reH!49dc+Az8@Sk>~7Uz=QWWZEYP^c8M8v=xwKEF$i%bdb)hF zkrr{(&4{^DUY_P6yF==8?Hw_uyC!YmsELdw59X_d*#G$&U_Ye9@bvWKdLZt~NwCyr zylFpX2V%QFRr^&jHTqE?+M7;Gjqt@r>;^(qAHpdS zK~6&q*VY1{7ggT+^;gP{k=si)D5j3t>){z#SZ4k#8(!~@nQ0xH#p?6QrE8q(>#x6> z+PhAA#?NzB3%$<^?w23$3E;tiB{dqEQcE*((4{Hp{@bDw@8z{qOrEE>T{w}o zxZ5z|J9>!c*Eep;yV5CWzlL(bxa?Nn&_xX_Zc&Vc;Asjh(;B%E57UWMO3L@4$+)y& zvmq6-hu9Q$N~}X_E&IB9MNmO1@pW=;KhDI_ec0g}Rr+@(C;yvl>*?G(kYYn#=3C}~x;4jVJ^$wn{Yb4`f{8qy-?lv)#sq1w=G z8sB|ibri3uQ3(jmMn{39xO6UGE!whqEl#<1SM>MoAYQ;o)j9o4^GK2wO_wvFw*sV~ z8Pua>oReNXt#_rn+};iwhLc+^8d%e!8P%JBo`1(|)|&k+Aoj|SjPPk0DuO1MH^W75 z;s+Lj+n-K=T0|%f*&@Y9-DcJer(SNav~!Hs>E> zh{Fe3A^m#{tT!szGMo^TMtSFHKk}O7T#$@hYlmSPXKS8a7SscwM?(%?j-X`xZizJe z_Y&Mg+FR0)&i;aGSd?|+6ZUuWwvQkxu8;My3f1jyeDh1)+H3r&b~9*KN9oZl&m{kt zU8#FU2a3Ih->#Fu1pa=FyIqHd3v~aN6iNX6{jX95VxS<>-)Wb+SZZM6ze*VyfQeAh zBgPVSeN4bi^jFDR5a@{VR|2oe0Ku<(($N?Ea1GU+W zIB+XZJ~-;I->lZzznnmYB9Vj9pve}lgC28pLtS=m`Js9i#ZA7>;b!A!WA5O%#6 z63i|-q;s{fvl}V&AD;$NC&;4kV$KI}|?VntwkHYaQl|O}o zeiAO%CIBzmEm(72DQd{6Hqh3925Pn(KJXlvUs+GCkLec#XcQGVXWGhtkGWZ{_IV<8 zL?73~X0o^|D&H9D2-)3fw&B-gPWZ|v)MN{5wM?K}4;RQ*rXA`YSXm(>8N&PE(suLZ z)FG>jLiqfA;lMxZmmvj*dQ9<#z&G%Ht?L?5loQfd@*B86{cz%x9)9y;pj0Z=7VI%= z71+C2L37e;My+S|MHbOexk#kbgmLE$nqSBVT+H2K-|n9In^(k3IJIhhI(8G&Rg+Qu zYn@>6gUE9BD(7fZzGA11+A<_zaz}VZQbEfaNc7i-__TVR;!zKQOZHG*HYZuqxPNc| zDBR^+&bYcXeCvl+yXHR-2ug;k9Q=L^&B|WQLS_O6p)r3q+p4o9AR3Sp{Z-!twD03d zs-4|mA`>QE#2Hps3j3L_9*rA#4TSC3U^`Ud9?Z-r6lxn3&Xg@=uG}I=?+X;z9S^5I zT4x`HmUIloGST(;d<`v4+Kmy^#~o+vBgI{k!oZT&1Uvb4sPK`}7-S}T&W{D@iq9SF z>kS@k77-ewNx=QZW%ltGF{)m_#9nQwOnY<5A0RJ%Cj0nXx_wH4GE2@m$CuCw)6ej`5m_N$cvQZLf^ zP(8ZQbo;8~rt!(JmnTS+@8uGNHU0d3+%DzuZ5o{3LhxwSOzPOv7_`mjifnoiTfEy* z%<#qThNd6S4P#`9?>1hGCUJwlgNll)1QVRsRVzKjQ`Ft(eY*<70=VBDQAn6WVdD03(q>CN3{=WH2c@8`YB8Sb$?1 z#6G=MUdkV)wep$s2CL;FFmSxQfY{EKpJ!o_1G1t#unhs}i(v&l2oNh)N;+aFr4Cl; z?SQlHMR6lpv&v%|YM^H0t<3ngDu)9(XJ*7BryoC{$4wsOuN*1T=QMLBnhGYOZ@p(= z$r@l35lm+FBMQ3dv2B`Qka&*AkW%}=wy86GN~Px}o3Ix~>6?w5a`NL{(#tZiHtyzo zN$Lbx*82ibREv))$7RV=2OTdl-zYkcg8relGOs<=RTJlhc)7M!&kuotH+A1yxKSu+ z+8+7ZW|qXKUE5YxDkY|dy!JS=3i?s0pnZD`OXU|KS+^T5sC#UuhOHF5c@GuFGtmYS z9?C8-peXoTsw{5KX(!Z&^+bAMTdOV3=-fqJt&}2V*#bJnKT1wts`J|Nas_eZzmRa8 zprxq!;0XkbCnqM$(CMV@KSbdtZ5S2w1uHtOrIM&%&}_r#hd!>uS$n(x>P{=0CaK#054Gk z{+n|BNX=hJ+}JiVEb!1@`k#TrUuXCK7k%jbmj=I41>(Pw{}YDb|B?;S{~_VQ=BmI~ z?*Q0G_2r`(w(+&3@}-T}Qyl)Q%pZDHLaPDay$<*v?t}jh007AKV_W=7#*(gJ)Bhh6 z>jdNOnj%9102$B#0Lecpk)wi_)qptQLNy>R$zRjyRda^~zyJU)C;-5Rf1sA&=obiV zjS0m3|2p8{Oj+W4skMg)0FeKq77O;jI*?Ka;*sR^? z2u>yd5`r_;fheyQC|3s(Ksylt!3M-Y67a!))b}rHs9H+!I}PCLCLCZHjTcUV76=w# z0KNzR(0Hl-_o1HhqJ}!42V-agiNKQ@Ktj^L&vS*d_0JqHXGid2i5C~me^ahsB;Q|@ z<&_F2wq($LDbsp6NdL?HwKJ^LzZwK-0x@3Ev2WmTkrzbj<)Hfqn(=RRNE1l-I- uml4rP|EqxyT0p$l243GVU;%*o7wi4+%eC6i=ijK877z#S?lbVE_5TN#vt7Xe delta 26462 zcmY(oQ*fYN*R>nlw(XA7v2EM7ojdB-Hak|wwr$%^I_~&?zV~SFqjjxS>tI&RS+hos zFlq9E+~DQ81+=vl2eWGS z9up^b2{T}J_1O37LeUG}^+LRG)|I9E^1>BtYv*JZR`YO2-FM5?_wf@2r@x31w!iANo?Nbo-flhWfF- z$&tM6JzaM3g4TLC$>q}~L}m|^UjzrFBROOmhXg>VQ#i8Z(*Q1Q1-#C?yu!f6$kxKq zmjnzytuFBAg`GsRM8!^$GyN0-?hBijA%ptr(}RSu(LhJqzZpwMTn%l5Vm;8cm`_nO z{};Td7$xjf)LOPT8jWa{$fkFY9Ia)7vFNcav;rj za43L$_1n&ca9iq~-Nx-$7Q5-M@MM5D>F&H8Y!yZ(C%^He-LH-wcJ5&;fhzO<7o%*gZz#bY zI*hLrtU{GK}0)YPNzdbJ;xTBsSe*x1>F0ROmU_)k$=rf;VZuP?16YNuYwF#dnfYiVUwv1_&x#NN%veRl*3vFN z|FUg=<=?Sn$))`JI>$$syw}(HjhxM7wCcwvmz_)d#7%+iPhc%7P`uaM(>%V}k&=W% zKFf)ze%9U3!2fScHbTl#68vxVFE9$yBlJ{LJaXGyV$Ne3tnKrfby9!FPDbla5AKsy_0?3`y7(sfE;T=nM5mi20Wxp3t^k_H=eTL@tY*MdW- zvPG5rF=@)=^CzFeDQpf5m}%R4WsLW!sae4y7)8&6BkHG%^+Ix)F$Tt|TpesW{$l4v zbIb0lJ0fqupqy0}Jd~|qRX1WXtl=%l4ahhpJZSZ?D{%jPhc%RmP|YxlHb-l88`x(` zbG?w@TClb6f9Z(4(rMs^gi5w+53{=%08M5$`j{%wd-n9ay}9uLVxTrB@7EoQK!$Jb z{_dR}eOR5UJH{*Rl)++yxqJ=M&3Hqs+8RxoqbaVr50DyrwprMvxe^6UIdyq@GaS0D zKqkr`6Ha>D!WU_c@%0!;JJwH~ne2m_(f+{*{&dC3gZ{hx_4>DKb0hdR412fB7XaP8 z&dQAJXc`x$2q5cx9L1_#9UbEi7iMo zJTFiz(@TLvNs0&7d_kcljhNZa8Ro5giid7cvU$uVLD{r{CCaw&NnWV3R29EjD-5bI zA%rh!Quz!|>i3-be}%I3Ou4`sKBSz=S@5{$GA?9X|4E$!gvb)KhLfTVs!ICX0>1?S z&*1F=E^Nnm(lQN*dC}fx@UpukYUc@0oZ8mfuoOn8z3T{Rq6x45!ICh13F8z-t(-8p zrp1jQ(X`zwsv!)Q8EgVG&TPu8lO(%=B-hO2HY(ZQ)}-D)_iG5bxdp>%WHr&KTrHp= zWy8bXt?9)yjB0%Jqt5ae0 zYYSxFDjbU46Hw>>d2&zN2I)Ob#sm{dnuho@PBBUTA#DY1IT*d)|8;lc`EE%)m|SN} zG<(9n%bmMV<9I+RheidV0%A|!^!k~km_X%eocdD`DCN=p+sV+|pUQh$5n=8(NotZL zJVVZ@d^V^$UsBRGc`xPWOvaP0Ms-?rh!ik!V3z8d@uy@6*yI`A50^zmu>o1$Pukz0 zvl-W{AYq+&O=2U+-`F(a*QsHXXOSY*7BIE`C_XSpv*d|$^UQX>_C$lq^Ei=E+Q1y z=@OOs=2@iLrP{EBzHh+31P>z3;FN09r7rO71ob`lS@Oo12u4Lb_P5dAjiqIUUmhV& zToQ3|GA%gh>ufh0kics+vr{67bOeyA_y0=#^~D$eN4p9^5%0H(X+bNHxM)bIUtES!JzR?P#S z$q&X}9uVMT-pKJO-22yjZC${t`nx;uc$)j&-n9igO3Wl#+?~66h)uq>PS}g{>3s@( zkK`6C4q(2H?tVYV_kK@D{6VMGSKIGh1!^>g%AC5RlBMALP+)ugryvYY;dvoaDQBx zn{NnR(P>?|=HT?=0-ii)ko%4GV;~MVo~)|MT9GE9wzVQAK>oDQ6SFGAWpEXUz2&V; z{Pd%ObFv(;e4g4X(29b`r7OLy!G>+}c3im}SjL>;*ZY$)?L~x^q7z$H6=x96#-!QaapbYCk$@#o?$OH!|5SS;+j$-7;|byM>YxIM)_ClN zbZj7a0W~)_tcwr`g!6*WTl{B?$H(0qzaOGsE6E|YeN}Ib&A|;1E_YSHs@a&vScVRN z+o0uKL~s!4VxgE~nCpI#LCeScb4#!dM)8H2DQ-qV=_=QZRsm+6VQGL$?UXDJFX^XJ+B6UeRoE5#+y`_O^7jANNl_vR5el##e>3bX}gZ|rA^ANV+;`?Pi zl?>yc-C%>GK98wj+xhwPrb#m0^AP)b)!+5ZuYh?zoCn>N=+|5N#VsJ}HMmM#-iW+e zGD5`@??q>t=4UjS0rD$jv3HWMXP6D8w=p#*d`TqShl$IT)*$9B+8FZ~@q#sUH4R!&_#;p*%0MH-2|Fyxd+Jfv$4?BqJxp4_l0}&@N+q z?fy+(v9t(~Ob2tii_(qh!EaG>@)KO{N~6bN~Snk6xN(ZwD~ z;Acqaqz!6W1=0}?1&7K9sgcKKiK7@7=FE%h^R1(E76$F38UPM8XV8+6;2~@1RhKOx znd-w#s9qd5d0fr5|HDOWZMqHbjGZ|cN1w7wuqhAPGiDfqw)Y=f5 zKeW}vP{q`qboS&bD~T+R*`k1+zCHfgbrt=RS_{0NstNvjPsjZeVt2kh8?hVi-kjhs z5U_t38|<|D8^Af`I^iq_D9_NQDA!2Cb_I&FQVgZiVXG*tM#E4A=8z~}6}re$woX^lvVb7>L)Yj*9No&0scq_z zI9GXXgu;?X`@Tagnz$Q-LBG}UCc)cEIR)@JjCQ6IeBz%8rqJ zgpHv{SwoTJn=|wADN~Ho8qFDX%$1|zo#|?$HIyXi8=EU$rG0<~xsqf-T2Sq1g1ut_ zn@>_g09dlEDQY)h)&E=movO~DS%HvjhczNn-DXw1S;l=j7rR%UQKhCsjN;85Lrjw` zr{Nv9!w^JoMZnYv&t_@2LgbdUT2XcXCIEr{=&zuK_GnxO-%m)sOZw$QvA66pu1uwp z{3sfY!p;hTfFY#FkgV0x%DiZQ3*qf#ReB~>4PafTi-Ac;p)1J`tFCb_ZkHY`DU;_U zl6I6;Ewz?t%5wcOaqoyW#obR8kB*3f2NRaAaEMBvB&Es9C4SS6R>HOXc9Gl0qoT>JtEb+5coiT2pz#0g??pk{{jbJ5& zhUpTEV6v>(Ac_Y$e^!2g{?USkUCoiR!nB-6#i-XIAv?%5)3K$DFNd2x$`QAMm^aR> zh6n%EgVO!$>>h3~SPVk3bhy73<1P3!I_utAy`rLjo+;H!dqk_`%&^%R6TlEPy?(AT z!e<^$-ov5tgOjQlRTMTpf{*=zl;w(BK%EvwL0H7;HjzS^qJ-_9S8~qR#QNA0WAN0U zlxd&Yzp%+P-HS^u-V#L1safZu6Dp+Nn)aTN7P?J>5LW45B&oEQKs>HG{kLKJ{2F}B z#w4%z$0xuE-LMKz8f#}7}N}3Yygbk+FXpU|O1n;E& zC%g~`N(w`)QWsBnq!uYm7mBp`?9tM!ebD;{g`O;0=hd1R={4so2^3Obm zwj4`k=ONBp=~x$NiFpbFYJq~`G}L^i&36cNX3Xyl;t)^>0ApvkC0OUMzrv9dV6UEN z?=u(ucgcY?R@g9a&~H4FGf(4Mdxf(I7VX~O`G6P}Yrmna(=|N$`4@RdwVLxyvs!VV zI8{=Q)O#{q0rrvJu-4$y*pWF9`|m5UJ*E3{9#lH5x*`$E8)|Rwmg|{lkoRGJC6pm3 zpItt2?&t9tP~E%zeQ;bQ2_xf#Y#v5nKRS8s{2eXWcX z?siT+s%3A0W{UF27i!2nbSg(B7`^R5+C_+uCkS=|W+Nr|7@d?>*6z6NhZc)$(e!#Y z2bNO*l@7G+Y0pDlni68$s8kFIt$>t15aD6Z2;cNm=9CzNUgNEHG!Z4D;V)IxDS+*_ zUrgo;m2kPp6hjQ4$5D5IIgsI=BlvV1fFHHBEUDn~96^6!*#4NiY;RK*c)G8?NXK5v zqo=3?LM;AKkCx{4MrSKL{LGr&k%*{=m9P&j>YCDlVVvup_u!YH&0W8EoO7rvH65_d zM5up*$|i*wl9mwtOk9w7b)*(AP4Hq6$)}I2A2*>kQI$YBVZ>rh^@H|rn`wZ6`Wr?$ zQ11kPosy!0(`Vo~TfzBbo%+fQv&?^hsZ4APC@VNRf{hOZj&3EDjm;LZOzs=qQ_`19 zw3G;#a*bF+4CL4XM^Q8!2#|tltQ-uH8v@AEn2*^ddH(I)C`6bgWxcCO))O$tfk^?z!GVu zxU(UuMMgIqoe)wDlcWSTv-Lc^l{O63CefX;@bZ?a_8?A24)dP`-3WFq0a}j{3NFC zt&}}z)(_NpB9eJ@G0N@_X4W{D&)P0AOvAcQN6Av0?h8hH<&0j7CAYTIt!d*3VuJN{XTTV)( zhMlC9qK+C)W+zF$tJTjYOqm3XR0x(=&Q#}0%%H%i*b0j^a`_SBB#Ng_YZ&L$dX+S= znvg%+W_y=24kIIm`kC9rkkjFKa%%ZtaJxD$8I+s2Tyq!+YRLGv{2lgLJa}U_dnTf- z2;P#)lcBaEnGrEAAGEe}9iYfJf9ORN!?oVDF`N?Z(@^jWsj>88{Cb*TYv+ZRhb0l+_TtI z$#0{>`9}ewc|DDj0rioQ`O_t>6>rU^-iU|jU|Cgwq!#`y;cpXgtzsN2#gV39*l30r zI%9upTU!}4_Yo-jyXTc9zNG$gJ_B=EP*!NC1y3n|${!+q3$aUA1B`hL27BVGoPq7c z<0bcx9juUL$jf%QZ6;2RQU$2nq@MeBeWO$;@<;WYOP+9R693+alYLhMT0W7nlh0Ko ze)fXZ_eU3!YYOM5A&O@0aqW`*#(qKXg5Z zmAMh|R&LAZgQ`TxYYZYs@mbW#YLakHt z?^9;IrVBl1q+*xJ!&>^IPK~9sIOfi&NE$?t%?f_vz@ZeK^qx){_kmk#Le#(WqY!+Q z&->)!f_$Py1?B39eT;2Wn?M&;{gur#P&S;IzxOn9IRa!v)s|c4K@+Uxa+oDt>KYI zAxNfs^~zwZyuV!H5;0iL%pxz~1h*B~egl<0Vr3sQW>BV`$1G)-8XBG2C8=!YW`%CS zW>3Ej&E}b%7ntd&yB9sLGkfQV$jrYnRLYpDFar+y9jR8m{xY-XW?jU1s7Vl~D@&|6 z;k?iQ@;~nt+vvI+MPp*+9;2$X@<@#n@fzv!j7K&~35!kjZ0A>qsudCN=$O>gC<>kV z5M*REbL-}5N1kjFs+74A9h)fW9Z(;41W#Lg-g ze7S>Hw-w?^XjvIGM>P#jNU*scH9D%q!cC_GWRDyjbÙKP__NUvCBITIL*2kW?6 zu@*toT)yz1C)1#>iPMgMv09qBIdhd#&i3nzWjT1e&y za)sALP20MKn1r@s_-2W(7Yb0DZ;#^ExWNP*^of!gEgRK)MP=)PQQaR-;X(eETMybObQ!Bre zV}qAzSZy@}8t=N-&jKdaJXVK05-0tDLJ4d+>f9MUq!8m-K!->zD4Aj=%uNvopZXKnWI36UZKV|41YEP5mdsk(93nn)_9G=c%7L~4t z!l#JgEf+5>l|HJ$ffKKsI0cc5QC#S|y~UXRO)c*sc%Q`_2gHvu0$G$b9RAh-c{4$! zo2jSw67#erE54y-B*LaAA78{cOAM(n4(!-{(1o5w>uRj0%p)XJt=q z8z#@3+wbEur_S%;5Uzw#Nfu2WJO?Lk!tS?ENxdvm&XpHwo7!X?D&|;dkG*4hOC^r% z$0dEN;kaj}$&)Gu;%(n15kwGx#~Y7N3wPd^!ylvHHQdW;4o{-|{U2=cBEy#00JpCs z0e_m-TaehaY3s>beE9LhCsZ}!K_3%l^xP&U6@j^ed(R)y>Z6HLI93@Bp`aF{m+S7@ z%@fIl*bd)~Ka^hs2LsC3vm{ITgAy4k^L4dV^uo3gM-Tb^f*BRci^cQ+4qy{M%{35J zM!AzT(datm=q(b@^suYpBj>syXSK|QImsXwbkH1G|BXYpfgzxgt@id)?p+D8^5rog zR0P5g^l8Pnsp#G;ayR@&PmPz0oVmlDDAGHi_2fyi_sLT5OKXG;b$(-XZ09U|0r^q% zy!R*RQWDgg`A$CJuiXH^l6lBF);Cbvyo&wQj$Ms zxm-!Vk^PwHcAqZ86%g+nLGp0)j#JdARNmW#RG#&PB~F(48*ea|f*C7eQIOU1%BZOR zZGEy6v@kAzVvKuRgX%~=?c20-Np{n@wS!K+<1kq0uK_VVcc7eTN10Bq-M>I6@Xmu8 zt_PBLmlZ)u`SzdxzuArYOExUr$K*?Pccuw0YjBn}?AGjrGP>eJ6s`-SNj0xbF)bsw zlnHY-lf6af{X%BG!bc3l6QS3c=iL26Qml2)Xoi~0%^LPQB|7fu4+QgWt?^8hXDDA1 zi5Wf<-jcvdDOgyl8Nt@D(oyThljL%+htruG+Wpew(xMn}Gx_^yMSq?bSPMkNv3&`- z#S~esy;E5$xCaoT{E}_tia2L;{D`%GZ%1<&#BWT!K#j^`sUOPA7g9a+Pd7(UIVd06 zR?OzELNnOSC(>Tc(2?2=Y%ogZ4fOHoZ@Pc*mCK{JdHCOqzcQAXA$PURPe@O1VoK{4uj)&jBh`sv* z28#m)pG^F>OMH;eiX9g(DPJ0$CpB025gzoc&QIEUj!GR8pocf%PZ}X5U(kqi7ZF)x zAl%~?VHTeo^0EEO zOap@DYaiqv>4L+ZZE+&cdKzVX^gah+3{yBBd+PpNIR<~Uhph3hH8{A${$U8%dO7y% zi=*3@5|dvv&n{Z&ZScCS`$ijDYdhtc#AW<0aCr^D3D9`dTOF}yn=IV{{EH;{YbiKh z;n;yt9Tv7eqUa)zq?J0^|7?Jxs*vhLzI5vXRu7DpcvVCif#s-;sNC7-w`2Xa;Ap>C zpAigKmb>_x5_CTnh3zu&E;#fyf&^E-MDL|B5NVIHJm}|$&O6fZ>5rm+M>nu5IO;XX zKH6JdEP5sJP+=8t0le|@$jar=GeBgx@nyWCZuQwRkc5^M1e7chYXz zA<=es$148S=R$we7|0!RjyN#w>GL!H-K){Mr3QryP2!T2gV zO=>C<;cvKYDS8bl0OiZnBBky zNS%wl3I4IW;d=jlMVOZ&$=$$ul@E>>$$29!9Ev@lMw5T^33Z{s39!f+4Y|G!4-ZEY z2TXXCS+|kIhTYx=htFR}s~$Xh7l)|~2LelIrPlI`xZ*v*zOs&D7nj*bo;k8ZJvXLUkf z{YKCRhz=pn!h;|=59Si-M-afM5=m%cGcOhW(jnM$n9foXt=(pm=8(%MEXKDKjagJh zMC)3_!GIgriTdA5v_j~DkO5=Rb3A)1%6s9})5Yb4V*5=e+zQuxM+}kZg62o3z@?!; zf-IIKEHz+CDvQFJgf%Ui-*O%-8QPeV3}IW6t0S*rMq6lxrlhd&YgXG=PM8k-4TF(8 zQ}t%109q&)T|H6uJ%A>l&o$6T<%08X<(Dr%ZA?IChQrQjW6N?1J#dv6fi3nlctk;q zbBUZgpvXGdi?>b^@x&<$0CAzq$Y26vLM@A-y0OzO*!CzUDR7325zV+Cb%1QJ8-; zm|n=^&bbc{ES-MwlM@}MP)rpwt^5^5$>lCkWTD66GZ2awqz)L{Odhf<#O4cP5HUQU zkpm6IdDOWMA!bGU<1}FjP-sJtkiwF~!k>{pM6?F>Q`iYu0icYdAWX7Q$@p`$~` zb!#iLqm=La^YOId=J##OIlC>(kFKcV&;6o ze8T}=uZdm#&(8z4lv2+CyR;L=Jz4--1i1wL0@}iQ2P`i~2T`_HFE5@+AWuIhPY>vT zxNxv3sHx|Fi1PsyqPe44jF)l?nfilakJ)h|dJ8rJx40SS9f%;O^dS&BfY~Hkd=zD9 zo2v8|hC6D-*D4X+44RF}O`V#PdRW|`+Bivexlbs|ct0VPJa*}KoG(HcoH*Cm9q7`!Ga9777y@saLirV#*f)Y`bjB223fclsZnR?sWl z7<$C^m?^DQ9ZNCj69b(OUIA7ilcjX!8OWcb@}<#I1v7BshK=0NVNiGN!B^Ofcl6v* z(XxulqdMd}_r7b#_`B<+ua=vo{cAz&L?2|ib+YeI!|&JK@2C7MBpz@=mkKv2a4eLo zMm0+L^&-VbScJyKAcK3Qb znf_!k38#)p_T_U}vun5I=ilbP?!^nTeUC!9!v%Z{zH=6z$9R*!rm%7ue{7z`@%tan z6Tqc7t^^yiZ|=dCr@G@ zngiZj?w&4|H!TjYQTbZ(dUpck9eY3PEOu(wgDv=DVd=R^cnK92Q1M2_X>#X7``2b< z$^pl(x`*a34#?lFbr*ucWp^HX;;t9r#R)U(t=cIr|B7FIm1s+-hE-rUWR{`SqggB5 z@ZhR*9ApH_gxO0$X{S+DTF}LpIdSCdW<=tyKVf}frBV(OF??vI=u%Ps(asEXC7Nex z%*xwAekO`DlBuJiKw-(CZuHH?9?{rFzybQU+1Ae{Ty7;$8jBUy9G}@KtHfmxzbeti z1Y`S*th}teN@-;O-0(6EBvHuR2PcJ&8wjGxfQT{FtQ^oSj~Ru8fEyX0hI1cbT9bfj zZ(KplgUcQn*#s~3i!-VAZAfuPC-9F<9Z#YEqk_$C45w+w-{z+|6vLtUA~&Q!zyj)0 z?!>DzaTi)dRrc0n2Vfru#;MzrH&eX3y^n&PK-CnA#3TL;M3@b`)yiQ)psJF@QGfNa z(_yg8SOw<=F(f7%xCDnO#&$v{m~uD|7w;Iu15393c`@3QH$uc(FpWCT!6Cw&cV*FH zDPuf!JEhS$85Ykd26Mg_&`KOtnLy{hb|i9`14)oP&M#+Bri+5#POh&!{uKa$4g3S1;kWc5=-GF z6pA6eng9=eWfdHuXPSsoa^j9e&c4YhY~sIl?DW>CF2^oK;2G_?iI+@CVnD`PkCS@p zM8X3RD0X|dn_k358@m(K?@h89@ZoZ3K7P||_-#>#2BxK^mektHV3pg(I zhPjJ2GIPLApq4~?7l(tP7Btcey`rnk{?r^jP-=u@pQy~1^%z?zfV|{V@~|E7T@fEc z10uhwO(;Sw=t2Ys2*J?J0s*E+`H5sKum7S?bING!M82z#Ak1^D$i5_m$P_Z^U~2euqoW2ARz66FTA%P4GVKT zH#Bf&l!!2)B!V}>J8U2Hj(P-ZD1S#p?0c`CoVuWZCGCzz3G>|QgIz?%j-4UDIWGRg z2`v_kc+2}rdjfr((0tFF<51(7IxdvhV(Ri=&a~>b&KQD|%Wp2!t?8Zq`vUK#FR|u{ z{WX~DIc|%>RhrG?3E*nBuz}>$*r{@N+Cn}EB#a`O<+gi;4Mns}+`%E6YQhXP&JP(( zBU)*h;583sd+XP{X!#GY09NUhtT8gs-ExK}&lM=D4Dyf+q`6E6kz^wD!Rdy_qzEJA zB_}9xG=xgs=N4QaJHKKgNBYV#vV*F_i6|!|4faWkK^k+T0N}TeFOf4|j0}g)Z(Ydg z4B?+N58~>sTv@Z$XR-fO?O3L1TG<17YH-a-j1~%>2JJE?4#O9-3C7`44^`6ijJsREK*rp>I<0{@|dI|qr1jFK}L`)gQJ(^I3r1um7yPteNf; z6LEy1iiI$4;{t|>dF8qO;XnOu#{A$`5t*PP7u6%B5JShX6vF$7EYBkhEpN}rJw4T+ z1FG4f3EouoIhX8_X9gFM%Fq;=s@ad}QO3q1!N(;9S!2%Yj!2(3nG{)x$sx}4s-iFR zX*^%E9=|4rM~U%K9cD$s0o#+vfT^p%BUtn2BN9{r>JylqCE&F&seM-ob{Mh=kToDo z%xflupNKH3+k5ad6@fRVEIpuWLn6x}Mk{JAYz>U$ge99mbkOCf&7J1#n&=bW+99hq z)qK`!@u|zGRze;OE=c8&dAP@NMk*QMt6jJit5)3`L4r80x`0i@w<_)|YOz@Ijy6*l zM_s75B?9~r%=p>=Xgo=Z39JOEq9!l*h4Mh;YCIoAO?U=Wr^)y?RJshcf`NJV2h$-` z(AQL^GM3zNBzS253b4TJObZTC! zFSiy!)S7ehi@LQW?w?iA?Y0D>W#~XucH-`EgCz8W9upFh- zeU8GrjEih1P7KOf`cu-)DI7V z-UF%%r}a)X&UL-$63DSs5b4x^*PINx$eFvayzlV6oqu@U!F!|1N=o^pk+*Au@!O!5 zJrE}nb4C&Ap<`c(6kZ*UUZZ0wb5A~q!XTFIeX_GG&_WYbYxI|3nQ249Bj2O^j(}~D zK+$cD{)HpKDSlUxRE|o`JU$b^Rfh=kg9RYL?_tH85MIR3(y=ZR-C_J7VQYPGOQ-Nz z<3_oMJ|`yK4#WXhYy%3raTb3o#XW%S8drl*!AP{>zwx!3DzeDBJWvkvXs{5Xh}_`( zW>ILr_Z4>$4+G$Hf2*%Pg*EFF;VffH7^qvpBPNKe_aQs=BRaXB=269ZY$7Ic_5oH0 zdg5oJl1K8K(ho91joRn#?Pw^QZWx@7iGdQhptDp@C535FxjRQZ(=VRd7jb4&BT{8! zuT|d3!}S4>e=kp{9!M4A;B7BSU&Ms|O0B8bH}%G)W)+aDBKjsilDqMM1JwJ zP_hYer`Ur(+;2;1N;^&%mK=7^Z06m2-QBdHn{_Pf4zsE>=kaBK>w&4QJ_C8u`u()+LJh6&BmEq1o*3WOP|H|>3lE;QxS$m9-&-;u=4>E6eD zAu&t3zXd9}9+K>}@2`&NjG{jf#pIm+v&6R0gBkV4lA!jENyGjM@igV->-z$hOiVOS{1x|E|j z?vK#q_7iO<`Y|lijEObC^y(69$P%V95e9qbYcD`q1pRrZec5i>qWxoQ+?nwUae3UL zB5Qy6*(Gx96)bLA9t*H;U*=C@WnZM;#?ZrxrXN%laM>wtQSi_m)uHE*+}uh)6TQGx zK2FG;b7`NTSD$MtUJ!2bbRtwrO02gKHL>aILs`k(5CEP;%%0;*;JC z?xi!Z+555NN}0#k+(|H*GmjB`bQBR&Ok;&{;EB_~8SIoMMF)g!>BsOb{i>meO(Dr@ z0T=9YjRT(6L)d|fwpqIR107S&V7Z)Uf)DJ|e`W2Gi|Z`5)|)Z|Vtg>Gk)rVC&LK`j zJ#>@2^)lL=Q)jX1!NKbNK=~fj`Gr1yDhWFY(gH>GBW9uqV}(?65PPz^{XGQBGQ{dj z6bt!t!mgEn7hsQjq&_s`KB90jgd?WhGu3vVgWouz7l1B^WO^YSYC0E9JT8#6Bg+=Q zQ&9QZC4*9{I6kbMwxQ!QyRDq_C-OUjepVh;p75~cY4hZP5fzo5KW8&Vp;mPSzJWQt zS>`oqzyX^81Z6Q)cc~t#h?}vhRfTUP`UJOE8PW0D6v*Uy%#Otf3x8?Kw#JUYjR0E0M7>S8^i?v-{r~c0aTLsnMEO^ zmiZ8(2$d-0OeE*e;Tv9Jf=;^*iB^&pga~_QH|}HS_6=MGjY?H}0Zb7;64}cPkAR#v z+OI|6&{d8|U@I-&aF9|q$?{xPLm|))DmBOF^E83_XI0W!KWPs&QCiPSMpfq2CvtH@ zDxnX0DRlU#(NbIv|Kd1Q+x=3eYnNKJS8bFIj*~2E%$4N2VGB-JNtIEAu;DnXdp;rB zFE5O{F;r?IL9s;%{T~H8O;sqt>&`hX!#bY85sX$548vb?>UdSP`^yC1QP~pdeA$kl zyR%~Y`qMGn0SEeQx-H_AcIt7Nm}e;bjMBHqWOhsGrH(KnIsYe@*=3-dk4%@C z8nH>_%EVhK+!sCMM6uW#1^Ozs#DA_eOT4Z?!hD-1x&Zen)8E2)s_D(6F@h{u8)_4U zjBRBGXI%x63wr-7JI6r&8|NCX_jh2skl%zRuqc=@TfJKV30i7(cs($~RqZ8rJ_}8! zV=Eo5OM;>))K!1ieKMfpW5p6xUu#*NFbtt|^GI9ffYmc3|9E8Mq;Foy09ucGch;rK z!FA6{z7Cwp|M}Ezv(AO#YW~PYsFe?QH1UPvsE)zgobO=Gr^Pa9`0h{B8ZIzLwt~_D z0wJ@v)<9jnLu@E2^-7-GIA|}xD=_5x6844(U3O=WnP8I4R`It=C?KLV6+}6CP_|%e zgLxzqp58 zz0Dt+a5fou z&GqPy-1ouwZ*u5(S{^0n^t3^qLkzYMm67-y=GEIxXax-fyqmH1mvwNKt#JpD#P@DV zqRtn6DvuJ~A!L8hi3~b)h4_g&RE2* zmqPMyi01Z(3_7tG31uv@kgsZwZ@WIZS%MfUwJhozgsPuV3l%i_Pv`J|Yhl@L1qgr{ zuFj>>TK1Lv4uZL^bCZA7=$oU*h^RJ*?OLGS^=;1%{}{aTCc@ZCwtfejz!(V3B_PzI zl}{+wA*#NFB?}A%+O?ZH#y>ScoyYjh4+gr8*G7mYaY9fepJ#A(Fn-i6+O|lgKORa} zs|M_DJ7(=|^ka|&n@jrm&RL8isqSB;8{J?U3qXOOe{e2HX83w!fZ49<6n9n?Sj_Vp zd3t`F6mU^^TOs0buWxT}Ps17i2%O*W9qceA9O5n{MF0k8{uKzW6JM0r-y#M2c1}%> zLa!S3(H{-DQZxDTv+v67$vXmoqvfg{^l#}`lj_+w(Fep9--lAJo7ia&A^#PJhC`cw ze`BpycdtD%*O7v+pFRGKEFcB{SR=tUrWwdmp)i+EqkwY*g7t&Qx{gl zL*c7j41i#XlE{wu%K6RphqWg0mad)92L1*Ac{1I{C$mv--18nQBzPedcTd2vq0_Lh z@Sj1XXiRt?=@`R)IUX6yw?)F9bvfc3uJ#W_s|=`=#$CMY?SJ*sO5Gejxemm`udm*} zdV(hh5Zv91ZB4j2F1D~zG(8u5x64ZJCq`9O4FF!=&hGB+;QJ)JX*C0tbBxsVC2R3x z%6*-b)tk%i?oY1W;x3OBn!2x-%i3Iyjw%S3?u!lWs*X3#>E_1E>n?BAIlbhN)$Y~q zq7%G$>QDXbla1SjZrElkriukRc1=$pT{?)ys@8wRAl^{Zx^^`=Z}q`0%l2S_zeBU6 z9WWrix^k!HoaVS&`vR?5&>Qw>KVjXZolceg{yE*;xmu!cW5u){UUZU_`(-dq>BZ0# zr6fnsu9ylENDe^Br^fu}ELC!WQu4FulSqOCn z%*gzqQWF?744w&vgUD&XU-oiCA+_Qi8({Dlw>np#BxhRp)(HO#`@&%V@SWzU<_$Aj zBF3-1yp^fgd8k<9(-LcotqnBD&kSC}F_%IBQHK_B(F3lv9x;;gY=Z#JKODhv2A8aE zldYbA z{e_3@a9BBrutR4ZqQR(fHwk9m`%!a29*0(7RPrL0%#HDFsN`hvYimwK9;7RDBSIv zSbh^N<096_$u7cSVS%Nq+a~8nEK3M;-UeXJ9XoG6v)}RS8xA}q^xbk=KHyY$wctZW zduJe_VX;l%2iEZ40p7MZ;Z6co&~Ww3*%Z^IO;FFSU~EW%kV(!2XMY;CxJxNHXjZ_? zr;NTC(R3Ld&~o&kWJO64s)*;svvoPnoWBFIsdPW#subocy%)u+BpUUrsgStKb#hCE zs~4oSOwlMG7|i8f!KQ7BZ-1~9xI0B7g5S)&4n&oX|Kc9B1@Jg|JyOZbK-w<#(ItdN z8E9}M`6)TP0jJx^;gUwHJJE<@Z57+?8R0|>snkVYI+a!7uGG2aXjsq8KI>_-wD$`S z8RyJcmppY&B`>qAPa8fNuFTlVyQ2w)!DK>Ytyn2DqLYaKSJqcR#nCit@8Us&y9R;= zcMZYa-Q6L$1lZv2vIKW`&*By&!QI{6HE>DZd;fdi@85IgY<1PsTh%jrc4oV)pQ=@# zPEkb)dZS>k&Q*P;P0Y~rCQUID2c8);JH;kTnrKZxQj4-*n-eL1Zl5H`@Sq$o4)e`Q z%2P9XtN+RkvfO?3!g&3?#mlO)kuhS*it{JkC{vcb*IG8I5a$Qt%;PP`fJlut=rmYk z{G4#BuvKPD!oz*9faX2?L;Yy`-Y_&u2Ji&p3I2+ockNtPa*zn2!sR>UsBUl2`|PDb zR=$;IT?EWEU3P;AYL6U^O~^h{JCYI(KB(!ZN~ElUgPJ=S>5<4mJ&eN$K& z1D4?%Q8)rXz_i*idIleQcI_Gvxhd6_Ukh|iz5UMPEd-J}kG+L6AE}~&nVG(6s9K=L zuC<(J9G`DXuN_t(o|e)AfjLU1U7XG{@eQpBHohX0K^YoFkg$|ZY(k88+tt5RG%uvBZL+-^FaV4#m(DC zrUW#`5t)g69ulTu-~Q&83ST)r2ZGZ#g^rHA2a*;V!jx-;%a~Lt5KWhsyYNVbvH2**{iydR)c)aKap%_5`TIo)?sm~FKxvP@YdwJEM8ejQ+Fs@JUBf$ zjO&knD>cVAS+GYoyDP*X)<+DBzj|DC$;+Tpw(u8@M3|zi zlC(#gUSXFpwwbTK!i6&j_ql)yoJjvSmb__0#MQlz`D4X=F^LeSmanY1o?@Z-5F;G) z%1}>LuHLeRG7&t`O)&rdR{!QuT>Y!Ek1=}t$83$2HV&fU7cd1Ip<52Weq-F}XX~DA&!2ir9}Ps3l%Xl5 z&lNZ}^5LVg#hS4D!x>Q?1)Xa0x?dEqDJ+-F4U9SYNRwM>YFu3FZend_ZRS^1CJ(v(9j*wkVwS1pi0Hxsq7ZL|X_#Q4gxjlhPWMt`QW zhV#Z>z-N0AzrpyEU{T5zv~^VkLPRjiz2#9b91WbHaF31P6+a4h zH4o67D*imn@F&Y+8)udLtcEW2kVqUcbd7yN%=7B^O*@_8Ty^3RkdH=?UqZrBbI-gM z*+vAOsRM_Sjd17q_w<4I+3P^+N~|XQAaqjQeDYBhKlG$fwfYkH z#>uY&%3w69>ojCX^3u799`Xt&|5QiPNT^(8o6kyTJyW-HUsojB!EA7-IGc{mJ$9bT z3hAen#LorG?}f6m(lZR=-5#3@NtKc;3f7UAb9i;VxDV$Q)`;y#y=udKf2o;Q@b-f| z_+!!;JlJsw&rj=++NZ{+pJFhuDQKr@-;J57WPHO0Ynt|#RrO-;-Mb0~9a17{{&&L*QfhwbK=orcg|LDfPTRF01*tQ zVmHm#pM4e^S;o}y$kMLi_Cgq43*zL7{dxy8yp9U*eAHvZ*R0R!3J`mgjPaR z0VUQr3(VYXYgnbzZStpS(Lp}38O$WA-?pEyrz)}!==c^oQp=AxN=%P`#G3dFs@Fl0Zjcq<`$le~Y12>33KeOC^%Sk&JZfYA} zo$#Z#)|%#l6-6P9Rf=}@}m}`n(MuwG&cDAVvazFk4K-s4mX__JKw(duxjG&-+uG<0+Xem18aFp`` z3G?9KXWuoCr@N=`#ciF!%bn#nt-VW4^Tc+IV0+R-Q?`_YHs1?I}dnq>i) zTNLux(g?rL-^#q+EO9!Z)T%-ambDJhAj~^Eui8v7UW9;isOTvZjaaR z`P<*F5N}-)c$B3k*rU)x#J)T|++wWQh6o$G_W^Hf0VaRN^JpG(Mf1q!>xotoI=p zMy$~NEHA5OO;D)q7BHvl{_6_?t@mveYTJnl5NUjN>(6iE}N08 zep&F_yvm{lhl4-rHcL=XM^^z|$}zfb)N1pVyHr5(LWNs?bDirvWFv?T40MW`_y zq1WA2k5ouXMQ+_n={?y<{9F*w-{)~D=Q6!eZpD-T&v$Wum^HT8NQ}9 z@Nqlx-n-8Xi7~Ieg>(V?`;Z%3KabUDX6A8CjIFV;Z$wbPUxq~?JE$K>2`R-5zXLI9 zUbyr@?SL+5c;5Jn>-`Ka<>E`A;VC*{AU$`)+>|0tk39PpC*S>=Mq@J-iCt0!IFYE%oiTc0lJL$ksh0HVcy@;Twp?7?9;BzsSVc? zM+a)2^p($d7n?|X3`LlA$vNffH2Afu%X!zBoV3cGlBeW;=LNSEho0FBunQT&N0mA> z88d@=3e!}d3dzc`Q|WbKgxL&4#cEv};3^5tz4a!=j60+Nd-?Vluw7yU@9x|rpz8_J z#=i(j!{sxcoEA4BoV*tS7QLwgbCYSr%C6M%RURh?K%}lhW-yCsZ%Wl&^mN&^sjDQ7 zh)Z=p*Yg-^l{j$#4c~gHD2|_#cSg)&k;H)@t-n+UrGyLS$Os=y1+Lnbi6CY$YZLuR zzd|CKUqVhnM60gw6TE{3@Fi?S4a;<#bPf$GSQadN{10yhs<#pg%g;@0hIf8e+RX>B zey&y!3yzag=to<}Y-F$_qaDWPy#&paU3gu$;E>HVCHgL|HzL3i+izCnALuUzo{N!S ztm?5G5WbeB>%J!-$|NUFW`XqxzJYHeen^h3*HIyB&v z60v|A`be~ZZKWc?6l}bh2RGq;2$GLds%A_b@k*TW5uxvFF((r3;?laq`lMv4vO~Q} zV;AbnOQ_IIfwiyYN}_&mSuSm-?7LVw4riklO2PUn~sk22J8wh7l@$jXu{n7 zU?JCr#;JU_5#Czb^;(rHLETwEQ(#fxO#Y(rYi+G!>=OPWo~m6#GefT9wUhWy#1bZ* z5I$>c;pwWj_`HYjzZzZR#?{9M&u~Qy2e4BO%?NMv{&F~!+bNfmnHGu z+5Kv0Fbc5y`c>r?WvWb zL(5@#SF5Gh|H!2iJpp|PU5ON*!o1ei7t(y^wS*Tnyjf=C6Up&(!^Ze+rNFl5bFpPk z)>jm^_dybyXY9ru{RUsG`+D>uob6@Ngdehs>v|yy ziXjF_;x>+qHI5_3Gh-`6yPBoD+qTuvNWz>;VX{GW9d3m7kWGbc4cqTX zxm`X2N*k-TM@f6L9Mh*OGhzKcb4*LxY)tf>L;v*dmW!S;>4G*XxD`)D}SH)gGTY5 zsroJlYLpLAgd98Niex_exT6bpifONWbt7pl;xdySw9>qciO2S}_hNZa%M{p)&GRxy zJ@w<|j2zZXHn%cTI4=`<5e$m`gHAMgD-5J@ygQ;eze=Av$jna=bm-E7!fET8`o>hXbtJNMt^uWtP z4qVsE`)CQ!WBu?=>0}yaX+@0fMX8yqU>)gRPYRbG&5cn@3p$k=EK$ID?_5cE%*Ev1 zo+!Cso0fPV%gkpe`qctS!{cBvwUZGK>nFeIS!n#_S6(N@{s_(AHr-t8^jZjNncSeZ zu{o%PliSjzYIdCK5DSK+MS0|YBbCbrv=ZAd`D3miHKQO9M-G*@^RP$Bt-iGYkZriA zGFtVL6{JF+w@qHUZeW{Jah+sAnPNCI-lgSLgqe%V2$#3scXzdU_Vl!;h<8mr)8|ZL zG7g+dIHp@d%^xpHaNo>JsFQ)m)571d7aBaN@Bjs-+o9fHU38=IUIs)ref7CAI@_%N z9x?}-48lrbXk#<>USJTUh=;_79l$~8hg}|gVjU1XM{n}&w# zq1_D)TQnL8D);4xI~qHnF9(+)2TEo6(4KjFs!{o9C_T@<528KpD8(pv|Bm%QF5)Cg zj9#cdFNNqM=Or(wCLXcbeQ7VmI6SYLnkjr51P5)Y z@4nnP@mORr2InDh^WJ+r*q+h4|4Gf{E1tUQN@u4ss!SA7goJdwDrOl@c z^a-LVeP9b{;0GuQ32-tr@Ewk_N)tfL_4uoC8sV=MNtbz}Iwa&vUhpV1kPyn%4SWO* zB!Y6_1S7%#xftKuO31P(7$$U&e=hm$P~?M^;u+gW7_0e_ywTsP%Si^s(O)^^!;ck+ z4~_->H?RW?kOlY?oC5>ILMsdoMov4dV@tWTf18)=3GRgf(ip356s@1npLt(lqnfLG zkjr~T3|=bksKcqHFG~(D`6?-`7A7TsubfSE-VTd(oS@(?{KN%rv%{ull0qH$f*oLixS*rsO7pRl z_+*?xGXjf}VxzLzh*61b=LUC=Cg~-nI>`6K!eog%&Mn)Ys#v}{teQ8Hc3tlL$X82g zKlFcNg_J(GZ zD^lL2O}MZ#NdLANKB0s)63JnzYONVfHR2_=NQ9z*0ZR-iArw5#(dk}K+0>~>W$7s$ zOX=PA(iY8cmi5^s_o5qe9WuTQ`N`8fW_H)QUNy+R0qvS;_aeQ2DXqTUmDLpb@o;w{ zqQ9N`l{EuTdy{b+u;6O0UpdDV*J3*5;rL$Bf_!8e4Qv#BlI?&RD&#A$!Op@XqJXL| z5k+l_vf>&brJE9rQ08^4L4#qpyHvi_{QRqwxb1bnKy&B*0dHds6M{b*tGVhM*NW7`+KQhK=zCCdVn8T2h$c}4?<4nlbx(O^Ca z-3G0xQxT79)t1`|4132wiHdFyX)?&}etj>Zy0)dn5st z!0l<96v{$IqgA`35~rC;Z^kJHti^_VglRTPxsL0Z3P)K1(o2kTdQ6pbUlaY4ROED( zRzl8mPm!o@-I)%D6>E`(qQf4(YkiLTdI0jWwSrR`z;|!K&5zl^g1p8372)^GVO}L~ zynoD#L)&2mD9{1HsMsBYynkc(!eKT)t7Dfn+LgedpTBjz%a%hl2!+;gYis!1ld+Y? zcmw~emXp>5#Vc92jB#&}rbxp)?!zsHTR+Cf;w{9+v6cI86g;r%3% zD(m+FOR`>`!gbMrPLXjkpGvaWzmKKFZ#7_h8LeIGWh0(iGPISQwg7ni^|VmM@IK6STm3%`7jb zdFafBSaChstsZms%-Z=_5S6h;WqwqMb&-KzECH8@Zq2LysD-QEe;fSsj0es5R7n5G z3Y_)AFhi3&L;Ry~gE?3SbfdtaW&Z{4%R!gL!`|NPVmfbijgQ-T^&;k1FZ`n{tW*dnN%=4D9v$pyxSjdJ}QH%TJZ`p zpn{m@)fOMBx)akV(>pvgO>NOL%vf#eJmAoEj^=>*Qdu4XRQpCtMFR%B#w+8|;3?pYK~`KXx!@peqxCcM)ubo+x?A4R+HatDS$G?05{%f|Si3tTL_Loq; z$sG;Y`HI)v%5DG;|5@?(m2-Rzlwfla*_oVTQMzu$WN)f8qY`qwkIOdUWkkL2}GxF*KHvVO3 z_BSDF0XwjhS!AGzM7gG4Bx0taNhZ$ah|)09H2-+@{y>GlBD?tWxPY^qYCUPQT9sYc z>y|`Cq?zB@M-XpxC;e4 zzg-%V^eH8?R%a!_0{2ymOm^INwh~5v0g?TwAJuyc=i9y#b@|P@(S~ysF)D4hA>O8+ z8cDP&BX`NUM;BMZZYayhD7#`;JFS!gs`oP2#3O6xib0gTQqvO}B#E>M^V_Z0|QEGMoXSEA0?d}#a0VD%a9+7jU?J6!G=D@K>zW+};N*~hw{&iJF6#*@SJC>_~A zNFjKUCfxoW1c=RoR4LkRK@c{=j|l9td!xX1HxSLViC?UNk-t`pt>iWSSh@cimYlD1 zBlKR&y3Ffz?S$g8!#FcK#|&(nVyX*MAF?!hd!hLJj$mKEd_oiAr!%}uw#=SF-)@DH z=5%q1VyuNr4swgSYiPZhqHROIDwxrF4A6ZsEolp!&`N0RsS-H#w ziP3>f$BeC#cb-0u$P>PISldvfUHF+h29wUJxKdYNh@b0%iV)I2V5*?ta_eDoa1&0; z4i=^*HA(f0RP^Nnb1iJ#kL6HvcyL}mV}jTXiSm$JTa6wzuXn;Lq2%%$q_u6-<^pr* zTfACJE*c;;lxZaW#k#D4)N|mYN$^7C{G}Yg$_|aVbQ%qF5S;Qw`+LE~4v!&0%hEPh zqeJyYYfmD%h-7-d{>4k?F16S7Yz9pQC-w$yb=kK7pAvR6s_H=6(L;{6Eq10tPoSS; zs8`kVo}ON$~3ReQn+VHfqelhS&vohUK%g94Qytbh#Kp345w=3fY(!jk#5 zUA*k}`-dw+c8;v|-Xw?fCGm{xk%6*N$74UT45pxyb3?s+5<*#~=Iy{`@zT4hSa)XB z;F)CEtg5vCG{Wd_^5h(#^n~)IXC>Y^&~m(Rxe5|0%;$eT>q}1? zoY!g*c!VwEYQKt^xIKQ`_m;nq47lmf%k7F6a=@CJK|Ivfj}Vv4D_)FTOP*wYuIqO8NZwK!e&NtQd;!H4Az$JE(`?MsK|P9A#x4^ya>%{XM}x zagio+Xhr6Htl2HR)+8pXoT`#$xs2sS?Uo(-O@Fu1YzRk+@b6zTT|G_g#XlK`n$!vv zMQUZV99#};Y4E6rilq#BpRp3&`M)#+b>d`o#AAh?p%i6+P*~9auTlv3jUrI^&kr?N z?tg?hMWEnc&4eOQ=CAOYZ}N{|uLM;1tLan%D*s2V5HOiCP~xxATKV-KsHsc>IRqD+ zsSL#XtNi{)8ENrXd8`a1edXf)zjXTmz`NJtCI9ov_tSEHMdE=)Re<<^k^jpC@t??{ z_&>;m<2V$w{~TM5%B#k_f!++TH{&#%<03iP#zpwB4euiUzf=rknM*df)|8Mc& zU!9Iu|8_E8#sYs+1tOFFsiP8tyKjyJ1ps700|2D|LSKN<)quF*MO7d!`5&Y~J$G0D z3;^JU3;?|U7m^nP2o^#CVuRDsfEZw2H6ZfevnizpB!mXy{#+ZQT@8f( zff{K5vHn7%NUT5Tz6PW7dXW7qsA-wMP!zaIZSPQU^H&2RpF None: """Append a test result for the session summary table. @@ -354,6 +355,7 @@ def add_session_result( status: ``PASSED``, ``FAILED``, or ``SKIPPED``. duration: Duration in seconds. tc_id: Test case ID (e.g. ``IMGBM_FVT_PREPARE_E001``). + reason: Skip reason shown below the summary table. """ result = { "test_name": test_name, @@ -361,6 +363,8 @@ def add_session_result( "status": status, "duration": duration, } + if reason: + result["reason"] = " ".join(str(reason).split()) _SESSION_RESULTS.set((*_SESSION_RESULTS.get(), result)) @@ -473,5 +477,14 @@ def _render_summary(results) -> None: f"/ {total} total " f"({total_dur:.2f}s)" ) + skip_reasons = [r for r in skipped if r.get("reason")] + if skip_reasons: + print(f"\n {Colors.YELLOW}Skip reasons{Colors.RESET}") + for r in skip_reasons: + label = r.get("tc_id") or r["test_name"] + print( + f" {Colors.CYAN}{label:<{tc_id_width}}{Colors.RESET} " + f"{TestLogger._truncate(r['reason'], MAX_LINE_WIDTH)}" + ) print(sep) print() diff --git a/test/plugins/omnia_auto/functions/report_func.py b/test/plugins/omnia_auto/functions/report_func.py index 9c64d0e113..5648ced117 100644 --- a/test/plugins/omnia_auto/functions/report_func.py +++ b/test/plugins/omnia_auto/functions/report_func.py @@ -22,7 +22,6 @@ - Suite breakdown tables with progress bars - Collapsible scenario sections with search/filter - Playbook execution logs accordion with CRITICAL/WARNING highlighting -- Skip classification (expected / unexpected / framework) - Sensitive data redaction (IPs, passwords, secrets, paths) - Playbook failure capture (record_playbook_failure) - Separate test runs for repeated domain execution @@ -118,37 +117,6 @@ def _redact_sensitive(text: str) -> str: return text -# ── Skip Classification ───────────────────────────────────────────────────── - -def _classify_skip(details: str = "", error: str = "") -> str: - """Classify a skipped test into expected / unexpected / framework. - - Returns one of: ``'expected'``, ``'unexpected'``, ``'framework'``. - """ - text = ((details or "") + " " + (error or "")).lower() - if not text.strip(): - return "unexpected" - - framework_kw = [ - "fixture", "setup failed", "import error", "conftest", - "collection error", "module not found", "no module named", - "parametrize", "setup error", "teardown error", - ] - if any(kw in text for kw in framework_kw): - return "framework" - - expected_kw = [ - "not enabled", "not configured", "disabled", "not supported", - "not available", "not applicable", "not installed", "feature not", - "requires", "only applies", "not present", "skipping", - "condition", "marker", "prerequisite", - ] - if any(kw in text for kw in expected_kw): - return "expected" - - return "unexpected" - - # ── Playbook / Molecule Log Analysis ──────────────────────────────────────── def _count_log_issues(logs: str) -> dict: @@ -350,12 +318,6 @@ def add_result( if error: result["error"] = error - # Add skip classification for skipped tests - if normalized_status == "SKIPPED": - result["skip_type"] = _classify_skip( - details or "", error or "" - ) - self.results.append(result) def _add_dict_result(self, payload: dict): @@ -412,13 +374,6 @@ def _add_dict_result(self, payload: dict): if payload.get("markers"): result["markers"] = payload.get("markers") - # Add skip classification for skipped tests - if normalized_status == "SKIPPED": - result["skip_type"] = _classify_skip( - payload.get("details", ""), - payload.get("error", ""), - ) - self.results.append(result) # ── Save & Persist ─────────────────────────────────────────────── diff --git a/test/plugins/omnia_auto/functions/report_html.py b/test/plugins/omnia_auto/functions/report_html.py index c018e250fe..fe3c7a3c66 100644 --- a/test/plugins/omnia_auto/functions/report_html.py +++ b/test/plugins/omnia_auto/functions/report_html.py @@ -100,31 +100,6 @@ def _redact_sensitive(text: str) -> str: return text -# ── Skip Classification ───────────────────────────────────────────────────── - -def _classify_skip(details: str = "", error: str = "") -> str: - """Classify a skipped test.""" - text = ((details or "") + " " + (error or "")).lower() - if not text.strip(): - return "unexpected" - framework_kw = [ - "fixture", "setup failed", "import error", "conftest", - "collection error", "module not found", "no module named", - "parametrize", "setup error", "teardown error", - ] - if any(kw in text for kw in framework_kw): - return "framework" - expected_kw = [ - "not enabled", "not configured", "disabled", "not supported", - "not available", "not applicable", "not installed", "feature not", - "requires", "only applies", "not present", "skipping", - "condition", "marker", "prerequisite", - ] - if any(kw in text for kw in expected_kw): - return "expected" - return "unexpected" - - # ── Log Issue Counter ──────────────────────────────────────────────────────── def _count_log_issues(logs: str) -> dict: @@ -208,7 +183,6 @@ def generate_html(data: Dict[str, Any]) -> str: "details": _redact_sensitive(result.get("details", "")), "error": _redact_sensitive(result.get("error", "")), "server": server_ip, - "skip_type": result.get("skip_type", ""), "tc_id": result.get("tc_id", ""), "run_id": run_id, }) @@ -313,16 +287,6 @@ def generate_html(data: Dict[str, Any]) -> str: has_log = bool(t.get("details") or t.get("error")) log_icon = 'View' if has_log else '—' - # Skip type badge - skip_badge = "" - if status_raw == "SKIPPED" and t.get("skip_type"): - skip_cls = { - "expected": "badge-skip", - "unexpected": "badge-warn", - "framework": "badge-fail", - }.get(t["skip_type"], "badge-skip") - skip_badge = f' {_escape_html(t["skip_type"])}' - # TC ID display tc_prefix = f"[{_escape_html(t['tc_id'])}] " if t.get("tc_id") else "" @@ -347,7 +311,7 @@ def generate_html(data: Dict[str, Any]) -> str: {tc_prefix}{_escape_html(t["test_name"])} - {sl}{skip_badge} + {sl} {t["duration"]:.2f}s {log_icon} {log_section}''' From 700c16befe88ec76133330b594ebd5d4ab7d9ee7 Mon Sep 17 00:00:00 2001 From: balajikumaran-c-s Date: Sat, 10 Oct 2026 00:58:01 +0530 Subject: [PATCH 2/3] test(orchestrator): adopt per-test logger contract outside pxeboot Each test now resolves its registry entry, creates its own TestLogger, and passes both to verify_precheck, verify_precheck_rejection, verify_cleanup or verify_nft. The result helpers only render the final record. Tests move into their suite order blocks (precheck 10000, prepare 20000, provision 30000, cleanup 50000, NFT 60000). Precheck adds the OIM negative cases: CPU, memory and disk thresholds, a missing admin NIC, PXE/public NIC overlap, air-gapped internet, an unreachable SSH target and an OS mismatch. They use verify_precheck_rejection, which passes only when the precheck rejects the input with an error. The provision boot image checks carry a boot_image marker so they run by default. verify_pxeboot keeps its (host, key, checker) form for now so the pxeboot suites are unchanged; the next change moves them to the same contract. The FVT README lists the new order blocks and test inventory. Refs #4849 Signed-off-by: balajikumaran-c-s --- test/orchestrator/fvt/README.md | 998 +++++++++++------- .../fvt/cleanup/artifacts/test_artifacts.py | 14 +- .../cleanup/credentials/test_credentials.py | 17 +- .../fvt/cleanup/kubernetes/test_kubernetes.py | 17 +- .../fvt/cleanup/openchami/test_openchami.py | 14 +- .../fvt/cleanup/openldap/test_openldap.py | 12 +- .../fvt/cleanup/slurm/test_slurm.py | 12 +- .../orchestrator/fvt/cleanup/test_playbook.py | 2 +- .../dependencies/test_dependencies.py | 20 +- .../precheck/environment/test_environment.py | 14 +- .../fvt/precheck/inputs/test_inputs.py | 12 +- .../oim_readiness/test_oim_hardware.py | 73 +- .../oim_readiness/test_oim_network.py | 94 +- .../oim_readiness/test_oim_prerequisites.py | 87 +- .../fvt/precheck/storage/test_storage.py | 12 +- .../storage/test_storage_config_validation.py | 62 +- .../fvt/precheck/test_playbook.py | 2 +- .../fvt/prepare/network/test_network.py | 4 +- .../fvt/prepare/openchami/test_openchami.py | 12 +- .../orchestrator/fvt/prepare/test_playbook.py | 2 +- .../openchami/test_provision_state.py | 76 +- .../fvt/provision/test_playbook.py | 2 +- test/orchestrator/fvt/result.py | 66 +- test/orchestrator/nft/README.md | 28 +- test/orchestrator/nft/result.py | 7 +- test/orchestrator/nft/test_idempotency.py | 20 +- test/orchestrator/nft/test_lifecycle.py | 20 +- test/orchestrator/nft/test_performance.py | 30 +- test/orchestrator/nft/test_security.py | 26 +- 29 files changed, 1073 insertions(+), 682 deletions(-) diff --git a/test/orchestrator/fvt/README.md b/test/orchestrator/fvt/README.md index 731d922e57..d7e05fcd1e 100644 --- a/test/orchestrator/fvt/README.md +++ b/test/orchestrator/fvt/README.md @@ -25,17 +25,57 @@ ORCH_FVT__ | Lifecycle | Execution ID | Verification IDs | Suites | |---|---|---|---| -| `precheck` | `ORCH_FVT_PRECHECK_E001` | `V001`–`V012`, `V100`–`V109` | `environment`, `storage`, `dependencies`, `inputs`, `oim_readiness` | -| `prepare` | `ORCH_FVT_PREPARE_E001` | `V001`–`V013` | `openchami`, `network`, `openldap` | -| `provision` | `ORCH_FVT_PROVISION_E001` | `V001`–`V008` | `openchami` | -| `pxeboot` | `ORCH_FVT_PXEBOOT_E001` | `V001`–`V103`, `V095`–`V097`, `V219`–`V224`, `V501`–`V518` | `connectivity`, `cloudinit`, `kubernetes_*`, `slurm_*`, `slurm_dcgm`, `additional_cloud_init` | +| `precheck` | `ORCH_FVT_PRECHECK_E001` | `V001`–`V012`, `V100`–`V105`, `V107`–`V117` | `environment`, `oim_readiness`, `storage`, `dependencies`, `inputs` | +| `prepare` | `ORCH_FVT_PREPARE_E001` | `V001`–`V007`, `V009`–`V011`, `V013` | `openchami`, `network`, `openldap` | +| `provision` | `ORCH_FVT_PROVISION_E001` | `V001`–`V010` | `openchami` | +| `pxeboot` | `ORCH_FVT_PXEBOOT_E001` | `V001`–`V115`, `V117`–`V128`, `V200`–`V206`, `V208`–`V227`, `V300`–`V311`, `V400`–`V429`, `V500`–`V525`, `V700`–`V716` | see `SUITES` in `library/vars/domain_vars.py` | | `cleanup` | `ORCH_FVT_CLEANUP_E001` | `V001`–`V006` | `openchami`, `openldap`, `slurm`, `kubernetes`, `artifacts`, `credentials` | -The detailed registry below is the authoritative inventory. Its `Order` -column defines execution sequence independently of the stable public test-case -IDs, so files and test functions can be reorganized without renumbering IDs. +The detailed tables below are the authoritative inventory. Each test's +`Order` defines execution sequence independently of its stable public ID, so +files and functions can be reorganized without renumbering IDs. -## Effective execution order +### Retired and reassigned IDs + +Retired IDs are never reused. + +| Test | Previous ID | Current ID | Reason | +|---|---|---|---| +| `test_external_ldap_proxy` | `ORCH_FVT_PREPARE_V008` (retired) | `ORCH_FVT_PXEBOOT_V524` | Moved from prepare to the pxeboot `slurm_ldap` suite. | +| `test_external_ldap_backend` | `ORCH_FVT_PREPARE_V012` (retired) | `ORCH_FVT_PXEBOOT_V525` | Moved from prepare to the pxeboot `slurm_ldap` suite. | +| `test_kubernetes_configured_versions` | `ORCH_FVT_PXEBOOT_V101` (duplicate) | `ORCH_FVT_PXEBOOT_V519` | V101 remains with `test_mount_config_mount_point`, which registered it first. | +| `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V102` (duplicate) | `ORCH_FVT_PXEBOOT_V520` | V102 remains with `test_mount_config_volume_mounted`. | +| `test_kubernetes_nfs_provisioner_contract` | `ORCH_FVT_PXEBOOT_V103` (duplicate) | `ORCH_FVT_PXEBOOT_V521` | V103 remains with `test_mount_config_mount_options`. | +| `test_dcgm_cuda_validation` | `ORCH_FVT_PXEBOOT_V501` (duplicate) | `ORCH_FVT_PXEBOOT_V522` | V501 remains with `test_slurm_node_add`, which registered it first. | +| `test_hpc_benchmarks_prerequisites` | `ORCH_FVT_PXEBOOT_V095` (duplicate) | `ORCH_FVT_PXEBOOT_V225` | V095 remains with `test_additional_cloud_init_smd_groups`, which registered it first. | +| `test_hpc_benchmarks_staging_fingerprint_idempotency` | `ORCH_FVT_PXEBOOT_V096` (duplicate) | `ORCH_FVT_PXEBOOT_V226` | V096 remains with `test_additional_cloud_init_metadata_groups`. | +| `test_hpc_benchmarks_concurrent_staging` | `ORCH_FVT_PXEBOOT_V097` (duplicate) | `ORCH_FVT_PXEBOOT_V227` | V097 remains with `test_node_architecture`. | +| `test_hpc_benchmarks_per_tool_staging_report` | `ORCH_FVT_PXEBOOT_V207` (retired) | None | Removed upstream in #5491; staging is exercised by `test_hpc_benchmarks_run_staging` and the fingerprint idempotency case. | + +### Omnia 2.2 Kubernetes etcd and reboot traceability + +The 2.2 `molecule/kubernetes` cases below are covered by these 2.3 cases. +2.3 performs one reboot per recovery case and reports every 2.2 step as a +named field of that case instead of sharing state between tests. + +| 2.2 test | 2.3 test | 2.3 ID | Covered by | +|---|---|---|---| +| `test_tc_f03_disk_partitioning` | `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V520` | GPT partition table on the selected disk, which is not the root disk | +| `test_tc_f04_filesystem_creation` | `test_kubernetes_local_etcd_integrity` | `ORCH_FVT_PXEBOOT_V015` | ext4 filesystem labelled `etcd_data` | +| `test_tc_f05_fstab_update_and_mount` | `test_kubernetes_local_etcd_integrity` | `ORCH_FVT_PXEBOOT_V015` | One UUID-based fstab entry and an active mount | +| `test_tc_f06_etcd_configuration_to_local_disk` | `test_kubernetes_local_etcd`, `test_kubernetes_local_etcd_integrity` | `ORCH_FVT_PXEBOOT_V014`, `V015` | Non-NFS ext4 mount, `etcd:etcd 700`, and the etcd manifest data directory | +| `test_tc_f01_boss_card_detection`, `test_tc_f07_fallback_disk_detection` | `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V520` | BOSS disk selected when present, otherwise a fallback non-root disk, and the selection is logged | +| `test_tc_f08_first_boot_disk_setup` | `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V520` | Both scripts executable and this boot's script finished with `DONE` | +| `test_tc_f10_ssd_disk_support`, `test_tc_f11_hdd_disk_support`, `test_tc_f12_nvme_disk_support` | `test_kubernetes_local_etcd_media` | `ORCH_FVT_PXEBOOT_V523` | Every control plane uses the `expected_etcd_disk_media` media | +| `test_etcd_reboot_control_plane`, `test_etcd_wait_node_online`, `test_etcd_wait_cloud_init` | `test_kubernetes_local_etcd_recovery` | `ORCH_FVT_PXEBOOT_V021` | Reboot, new boot ID, and cloud-init completion | +| `test_tc_f09_subsequent_boot_post_reboot` | `test_kubernetes_local_etcd_recovery` | `ORCH_FVT_PXEBOOT_V021` | `etcd-fstab-update.sh` finished with `DONE` on the new boot and `etcd-disk-setup.sh` did not run again | +| `test_tc_f05_fstab_mount_post_reboot`, `test_tc_f06_etcd_local_disk_post_reboot` | `test_kubernetes_local_etcd_recovery` | `ORCH_FVT_PXEBOOT_V021` | Local-etcd integrity after reboot and unchanged mount source and UUID | +| `test_etcd_cluster_health_post_reboot` | `test_kubernetes_local_etcd_recovery`, `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V021`, `V022` | Every etcd endpoint healthy within `ETCD_HEALTH_WAIT_TIMEOUT_SECONDS` | +| `test_reboot_vip_control_plane`, `test_verify_vip_failover` | `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V022` | VIP owner rebooted and the VIP moves to exactly one other control plane | +| `test_verify_cloud_init_after_reboot` | `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V022` | Cloud-init completes on the rebooted node | +| `test_verify_node_ready_after_reboot` | `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V022` | Rebooted node returns to Ready, observed from another control plane | + +## Execution order blocks The default non-cleanup lifecycle is: @@ -43,424 +83,637 @@ The default non-cleanup lifecycle is: precheck -> prepare -> provision -> pxeboot ``` -Within a lifecycle, pytest follows each case's `order` marker. This preserves -dependency order: connectivity and cloud-init before workload clusters, -cluster health before temporary jobs, and non-disruptive checks before any -explicit recovery operation. +Every `@pytest.mark.order` value lies in its suite's block, defined by +`TEST_ORDER_BLOCKS` in `library/vars/test_case_vars.py`: + +```text +order = lifecycle base + suite slot * 100 + position in suite +``` -PXE capability suites are flat directories. Kubernetes runs as -`kubernetes_cluster`, `kubernetes_etcd`, `kubernetes_storage`, and -`kubernetes_recovery`. Slurm runs as `slurm_cluster`, `slurm_jobs`, -`slurm_ldap`, `slurm_gpu`, `slurm_openmpi`, `slurm_ucx`, -`slurm_infiniband`, `slurm_recovery`, `slurm_apptainer`, and `slurm_dcgm`. +| Lifecycle | Base | Block of the execution case | +|---|---:|---:| +| `precheck` | 10000 | 10000 | +| `prepare` | 20000 | 20000 | +| `provision` | 30000 | 30000 | +| `pxeboot` | 40000 | 40000 | +| `cleanup` | 50000 | 50000 | +| NFT | 60000 | — | + +Each suite owns 100 numbers, so a new test takes the next free number in its +own suite and never renumbers another suite. A new suite appends a block. +Suites run in dependency order: reachability and node state first, then +cluster health, temporary workloads, storage contracts such as +`vast_storage`, and finally the reboot, drain, and node-removal suites +(`kubernetes_recovery`, `slurm_recovery`, `slurm_lifecycle`). Within a suite, +reboot and destructive cases come last. Never reuse an order value. Cleanup is never part of an implicit lifecycle run. ## Precheck test cases -These cases execute and verify OIM identity, selected storage, dependency -outputs, required inputs, boot artifacts, and published repositories. +These cases execute and verify OIM identity and readiness, selected storage, dependency outputs, required inputs, boot artifacts, and published repositories. + +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10000 | `ORCH_FVT_PRECHECK_E001` | `test_deploy_precheck` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags precheck`` exactly once. | The selected Orchestrator lifecycle exits successfully. | + +### Environment (`environment`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10101 | `ORCH_FVT_PRECHECK_V001` | `test_precheck_hostname_domain` | `environment` | `sanity` | Require the host identity to match omnia.env exactly. | Every required item satisfies the stated condition. | +| 10102 | `ORCH_FVT_PRECHECK_V002` | `test_precheck_admin_ipv4` | `environment` | `sanity` | Require the configured administrative IPv4 on a global interface. | Every required item satisfies the stated condition. | + +### OIM readiness (`oim_readiness`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 0 | `ORCH_FVT_PRECHECK_E001` | `test_deploy_precheck` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags precheck`` exactly once. | The selected Orchestrator lifecycle exits successfully. | -| 1 | `ORCH_FVT_PRECHECK_V001` | `test_precheck_hostname_domain` | `environment` | `sanity` | Require the host identity to match omnia.env exactly. | Every required item satisfies the stated condition. | -| 2 | `ORCH_FVT_PRECHECK_V002` | `test_precheck_admin_ipv4` | `environment` | `sanity` | Require the configured administrative IPv4 on a global interface. | Every required item satisfies the stated condition. | -| 3 | `ORCH_FVT_PRECHECK_V003` | `test_precheck_nfs_servers` | `storage` | `sanity` | Require every mapping-selected NFS server to answer ICMP from the OIM. | Every required item satisfies the stated condition. | -| 4 | `ORCH_FVT_PRECHECK_V004` | `test_precheck_dependencies` | `dependencies` | `sanity` | Require both configured/default dependency outputs to be usable. | Every required item satisfies the stated condition. | -| 5 | `ORCH_FVT_PRECHECK_V005` | `test_precheck_inputs` | `inputs` | `sanity` | Require every source-selected Orchestrator input to be non-empty. | Every required item satisfies the stated condition. | -| 6 | `ORCH_FVT_PRECHECK_V006` | `test_precheck_s3_artifacts` | `dependencies` | `sanity` | Require every kernel, initrd, and rootfs artifact to be reachable. | Every required item satisfies the stated condition. | -| 7 | `ORCH_FVT_PRECHECK_V007` | `test_precheck_repositories` | `dependencies` | `sanity` | Require every published RPM and file repository to be reachable. | Every required item satisfies the stated condition. | +| 10201 | `ORCH_FVT_PRECHECK_V100` | `test_oim_cpu_threshold` | `oim_readiness` | `sanity` | Require OIM CPU core count to meet the configured minimum. | Every required item satisfies the stated condition. | +| 10202 | `ORCH_FVT_PRECHECK_V101` | `test_oim_memory_threshold` | `oim_readiness` | `sanity` | Require OIM memory to meet the configured minimum. | Every required item satisfies the stated condition. | +| 10203 | `ORCH_FVT_PRECHECK_V102` | `test_oim_disk_threshold` | `oim_readiness` | `sanity` | Require OIM root filesystem to meet the configured minimum. | Every required item satisfies the stated condition. | +| 10204 | `ORCH_FVT_PRECHECK_V103` | `test_oim_pxe_nic_present` | `oim_readiness` | `sanity` | Require the configured admin NIC to exist and be UP. | Every required item satisfies the stated condition. | +| 10205 | `ORCH_FVT_PRECHECK_V104` | `test_oim_public_nic_present` | `oim_readiness` | `sanity` | Require the public/default-route NIC to exist and be UP. | Every required item satisfies the stated condition. | +| 10206 | `ORCH_FVT_PRECHECK_V105` | `test_oim_pxe_nic_ipv4` | `oim_readiness` | `sanity` | Require the admin NIC to carry the configured IPv4 address. | Every required item satisfies the stated condition. | +| 10207 | `ORCH_FVT_PRECHECK_V107` | `test_oim_ssh_preflight` | `oim_readiness` | `sanity` | Require passwordless SSH from OIM to a mapped target node. | Every required item satisfies the stated condition. | +| 10208 | `ORCH_FVT_PRECHECK_V108` | `test_oim_internet_reachability` | `oim_readiness` | `sanity` | Require internet reachability when not in air-gapped mode. | Every required item satisfies the stated condition. | +| 10209 | `ORCH_FVT_PRECHECK_V109` | `test_oim_os_version` | `oim_readiness` | `sanity` | Require the OIM OS to match the expected distribution and version. | Every required item satisfies the stated condition. | +| 10210 | `ORCH_FVT_PRECHECK_V110` | `test_neg_cpu_below_threshold` | `oim_readiness` | `negative` | Detect failure when CPU threshold exceeds actual cores. | The expected rejection occurs and no prohibited state is accepted. | +| 10211 | `ORCH_FVT_PRECHECK_V111` | `test_neg_memory_below_threshold` | `oim_readiness` | `negative` | Detect failure when memory threshold exceeds actual RAM. | The expected rejection occurs and no prohibited state is accepted. | +| 10212 | `ORCH_FVT_PRECHECK_V112` | `test_neg_disk_below_threshold` | `oim_readiness` | `negative` | Detect failure when disk threshold exceeds actual capacity. | The expected rejection occurs and no prohibited state is accepted. | +| 10213 | `ORCH_FVT_PRECHECK_V113` | `test_neg_pxe_nic_missing` | `oim_readiness` | `negative` | Detect failure when a nonexistent NIC name is checked. | The expected rejection occurs and no prohibited state is accepted. | +| 10214 | `ORCH_FVT_PRECHECK_V114` | `test_neg_pxe_public_overlap` | `oim_readiness` | `negative` | Detect overlap when PXE NIC is forced to match the public NIC. | The expected rejection occurs and no prohibited state is accepted. | +| 10215 | `ORCH_FVT_PRECHECK_V115` | `test_neg_internet_airgapped` | `oim_readiness` | `negative` | Verify air-gapped mode passes even without internet. | The expected rejection occurs and no prohibited state is accepted. | +| 10216 | `ORCH_FVT_PRECHECK_V116` | `test_neg_ssh_unreachable_target` | `oim_readiness` | `negative` | Detect SSH failure to a bogus target address. | The expected rejection occurs and no prohibited state is accepted. | +| 10217 | `ORCH_FVT_PRECHECK_V117` | `test_neg_os_version_mismatch` | `oim_readiness` | `negative` | Detect failure when expected OS version does not match actual. | The expected rejection occurs and no prohibited state is accepted. | + +Every check is read-only. Thresholds default to CPU >= 4, RAM >= 16 GB, and +root disk >= 100 GB. Negative cases use the unattainable inputs in +`OIM_NEGATIVE_INPUTS` (`library/vars/precheck_vars.py`) and pass only when +the checker rejects them with an actionable error. + +### Storage (`storage`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10301 | `ORCH_FVT_PRECHECK_V003` | `test_precheck_nfs_servers` | `storage` | `sanity` | Require every mapping-selected NFS server to answer ICMP from the OIM. | Every required item satisfies the stated condition. | +| 10302 | `ORCH_FVT_PRECHECK_V008` | `test_mount_missing_mount_point` | `storage` | `mount_config`, `negative` | TC-CI-NEG-001: Every mount entry must have a valid absolute mount_point. | The expected rejection occurs and no prohibited state is accepted. | +| 10303 | `ORCH_FVT_PRECHECK_V009` | `test_mount_missing_targeting` | `storage` | `mount_config`, `negative` | TC-CI-NEG-002: Every mount entry must have targeting (prefix or groups). | The expected rejection occurs and no prohibited state is accepted. | +| 10304 | `ORCH_FVT_PRECHECK_V010` | `test_mount_invalid_mount_params` | `storage` | `mount_config`, `negative` | TC-CI-NEG-003: mount_params must reference an existing profile. | The expected rejection occurs and no prohibited state is accepted. | +| 10305 | `ORCH_FVT_PRECHECK_V011` | `test_mount_missing_source` | `storage` | `mount_config`, `negative` | TC-CI-NEG-004: Every mount entry must have a non-empty source. | The expected rejection occurs and no prohibited state is accepted. | +| 10306 | `ORCH_FVT_PRECHECK_V012` | `test_mount_node_key_without_mount_point` | `storage` | `mount_config`, `negative` | TC-CI-NEG-005: node_mount_point is mandatory when node_key is set. | The expected rejection occurs and no prohibited state is accepted. | The NFS case mirrors the production mount-selection contract. It evaluates `functional_group_prefix` against mapped functional groups, evaluates `groups` against mapped PXE `GROUP_NAME` values, and excludes the stock -`vast_storage` entry unless `omnia_config.yml` selects it. Unrelated storage -entries are reported as ignored, not as validated. - -### OIM readiness (oim_readiness) - -Hardware, network-interface, and prerequisite checks that bring parity with -the Omnia 2.2 automation prerequisite runner. Every check is read-only and -never mutates the target environment. Negative tests use impossible -thresholds or bogus names to validate structured failure reporting. - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 10 | `ORCH_FVT_PRECHECK_V100` | `test_oim_cpu_threshold` | `oim_readiness` | `sanity` | Require OIM CPU core count to meet the configured minimum. | Every required item satisfies the stated condition. | -| 11 | `ORCH_FVT_PRECHECK_V101` | `test_oim_memory_threshold` | `oim_readiness` | `sanity` | Require OIM memory to meet the configured minimum. | Every required item satisfies the stated condition. | -| 12 | `ORCH_FVT_PRECHECK_V102` | `test_oim_disk_threshold` | `oim_readiness` | `sanity` | Require OIM root filesystem to meet the configured minimum. | Every required item satisfies the stated condition. | -| 13 | `ORCH_FVT_PRECHECK_V103` | `test_oim_pxe_nic_present` | `oim_readiness` | `sanity` | Require the configured admin NIC to exist and be UP. | Every required item satisfies the stated condition. | -| 14 | `ORCH_FVT_PRECHECK_V104` | `test_oim_public_nic_present` | `oim_readiness` | `sanity` | Require the public/default-route NIC to exist and be UP. | Every required item satisfies the stated condition. | -| 15 | `ORCH_FVT_PRECHECK_V105` | `test_oim_pxe_nic_ipv4` | `oim_readiness` | `sanity` | Require the admin NIC to carry the configured IPv4 address. | Every required item satisfies the stated condition. | -| 16 | `ORCH_FVT_PRECHECK_V107` | `test_oim_ssh_preflight` | `oim_readiness` | `sanity` | Require passwordless SSH from OIM to a mapped target node. | Every required item satisfies the stated condition. | -| 18 | `ORCH_FVT_PRECHECK_V108` | `test_oim_internet_reachability` | `oim_readiness` | `sanity` | Require internet reachability when not in air-gapped mode. | Every required item satisfies the stated condition. | -| 19 | `ORCH_FVT_PRECHECK_V109` | `test_oim_os_version` | `oim_readiness` | `sanity` | Require the OIM OS to match the expected distribution and version. | Every required item satisfies the stated condition. | -| 20 | `ORCH_FVT_PRECHECK_V100` | `test_neg_cpu_below_threshold` | `oim_readiness` | `sanity`, `negative` | Detect failure when CPU threshold exceeds actual cores. | The expected rejection occurs and no prohibited state is accepted. | -| 21 | `ORCH_FVT_PRECHECK_V101` | `test_neg_memory_below_threshold` | `oim_readiness` | `sanity`, `negative` | Detect failure when memory threshold exceeds actual RAM. | The expected rejection occurs and no prohibited state is accepted. | -| 22 | `ORCH_FVT_PRECHECK_V102` | `test_neg_disk_below_threshold` | `oim_readiness` | `sanity`, `negative` | Detect failure when disk threshold exceeds actual capacity. | The expected rejection occurs and no prohibited state is accepted. | -| 23 | `ORCH_FVT_PRECHECK_V103` | `test_neg_pxe_nic_missing` | `oim_readiness` | `sanity`, `negative` | Detect failure when a nonexistent NIC name is checked. | The expected rejection occurs and no prohibited state is accepted. | -| 24 | `ORCH_FVT_PRECHECK_V105` | `test_neg_pxe_public_overlap` | `oim_readiness` | `sanity`, `negative` | Detect overlap when PXE NIC is forced to match the public NIC. | The expected rejection occurs and no prohibited state is accepted. | -| 25 | `ORCH_FVT_PRECHECK_V108` | `test_neg_internet_airgapped` | `oim_readiness` | `sanity`, `negative` | Verify air-gapped mode passes even without internet. | The expected rejection occurs and no prohibited state is accepted. | -| 27 | `ORCH_FVT_PRECHECK_V107` | `test_neg_ssh_unreachable_target` | `oim_readiness` | `sanity`, `negative` | Detect SSH failure to a bogus target address. | The expected rejection occurs and no prohibited state is accepted. | -| 28 | `ORCH_FVT_PRECHECK_V109` | `test_neg_os_version_mismatch` | `oim_readiness` | `sanity`, `negative` | Detect failure when expected OS version does not match actual. | The expected rejection occurs and no prohibited state is accepted. | - -Thresholds default to CPU >= 4, RAM >= 16 GB, root disk >= 100 GB (matching -Omnia 2.2 automation defaults). Negative tests set impossible thresholds to -validate structured failure messages without mutating the environment. +`vast_storage` entry unless `omnia_config.yml` selects it. + +### Dependencies (`dependencies`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10401 | `ORCH_FVT_PRECHECK_V004` | `test_precheck_dependencies` | `dependencies` | `sanity` | Require both configured/default dependency outputs to be usable. | Every required item satisfies the stated condition. | +| 10402 | `ORCH_FVT_PRECHECK_V006` | `test_precheck_s3_artifacts` | `dependencies` | `sanity` | Require every kernel, initrd, and rootfs artifact to be reachable. | Every required item satisfies the stated condition. | +| 10403 | `ORCH_FVT_PRECHECK_V007` | `test_precheck_repositories` | `dependencies` | `sanity` | Require every published RPM and file repository to be reachable. | Every required item satisfies the stated condition. | + +### Inputs (`inputs`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10501 | `ORCH_FVT_PRECHECK_V005` | `test_precheck_inputs` | `inputs` | `sanity` | Require every source-selected Orchestrator input to be non-empty. | Every required item satisfies the stated condition. | ## Prepare test cases These cases verify the infrastructure created by the `prepare` lifecycle. +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 20000 | `ORCH_FVT_PREPARE_E001` | `test_deploy_prepare` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags prepare``. | The selected Orchestrator lifecycle exits successfully. | + +### OpenCHAMI (`openchami`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 20101 | `ORCH_FVT_PREPARE_V001` | `test_openchami_containers_running` | `openchami` | `sanity` | Verify all long-running OpenCHAMI containers. | All stated checks pass for every applicable target. | +| 20102 | `ORCH_FVT_PREPARE_V002` | `test_openchami_services_ready` | `openchami` | `sanity` | Verify OpenCHAMI units and successful SMD initialization. | All stated checks pass for every applicable target. | +| 20103 | `ORCH_FVT_PREPARE_V003` | `test_openchami_apis_ready` | `openchami` | `functional`, `sanity` | Verify the authenticated SMD, Boot and Metadata APIs. | All stated checks pass for every applicable target. | +| 20104 | `ORCH_FVT_PREPARE_V004` | `test_openchami_persistent_storage_and_tls` | `openchami` | `sanity` | Verify persistent data volumes and HAProxy certificates. | All stated checks pass for every applicable target. | +| 20105 | `ORCH_FVT_PREPARE_V005` | `test_openchami_packages_and_artifacts` | `openchami` | `sanity` | Verify installed packages and generated configuration files. | All stated checks pass for every applicable target. | +| 20106 | `ORCH_FVT_PREPARE_V013` | `test_postgresql_readiness` | `openchami` | `functional`, `sanity` | Verify PostgreSQL and its required SMD database contract. | All stated checks pass for every applicable target. | + +### Network (`network`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 20201 | `ORCH_FVT_PREPARE_V006` | `test_firewall_and_podman_network_policy` | `network` | `sanity` | Verify OpenCHAMI ports and trusted Podman interfaces. | All stated checks pass for every applicable target. | +| 20202 | `ORCH_FVT_PREPARE_V007` | `test_coredhcp_and_coredns_configuration` | `network` | `functional`, `sanity` | Verify rendered DHCP/DNS configuration and additional routes. | All stated checks pass for every applicable target. | + +### OpenLDAP (`openldap`) + | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 0 | `ORCH_FVT_PREPARE_E001` | `test_deploy_prepare` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags prepare``. | The selected Orchestrator lifecycle exits successfully. | -| 1 | `ORCH_FVT_PREPARE_V001` | `test_openchami_containers_running` | `openchami` | `sanity` | Verify all long-running OpenCHAMI containers. | All stated checks pass for every applicable target. | -| 2 | `ORCH_FVT_PREPARE_V002` | `test_openchami_services_ready` | `openchami` | `sanity` | Verify OpenCHAMI units and successful SMD initialization. | All stated checks pass for every applicable target. | -| 3 | `ORCH_FVT_PREPARE_V003` | `test_openchami_apis_ready` | `openchami` | `functional`, `sanity` | Verify the authenticated SMD, Boot and Metadata APIs. | All stated checks pass for every applicable target. | -| 4 | `ORCH_FVT_PREPARE_V004` | `test_openchami_persistent_storage_and_tls` | `openchami` | `sanity` | Verify persistent data volumes and HAProxy certificates. | All stated checks pass for every applicable target. | -| 5 | `ORCH_FVT_PREPARE_V005` | `test_openchami_packages_and_artifacts` | `openchami` | `sanity` | Verify installed packages and generated configuration files. | All stated checks pass for every applicable target. | -| 6 | `ORCH_FVT_PREPARE_V006` | `test_firewall_and_podman_network_policy` | `network` | `sanity` | Verify OpenCHAMI ports and trusted Podman interfaces. | All stated checks pass for every applicable target. | -| 7 | `ORCH_FVT_PREPARE_V007` | `test_coredhcp_and_coredns_configuration` | `network` | `functional`, `sanity` | Verify rendered DHCP/DNS configuration and additional routes. | All stated checks pass for every applicable target. | -| 8 | `ORCH_FVT_PREPARE_V008` | `test_external_ldap_proxy` | `openldap` | `openldap`, `sanity` | Reconcile and verify the explicitly enabled LDAP meta-proxy. | Reconciliation is idempotent and every postcondition passes. | -| 9 | `ORCH_FVT_PREPARE_V009` | `test_openldap_runtime` | `openldap` | `openldap`, `sanity` | Verify the enabled service and container are healthy. | All stated checks pass for every applicable target. | -| 10 | `ORCH_FVT_PREPARE_V010` | `test_openldap_artifacts` | `openldap` | `openldap`, `sanity` | Verify configuration modes, syntax and TLS lifetime. | All stated checks pass for every applicable target. | -| 11 | `ORCH_FVT_PREPARE_V011` | `test_openldap_endpoint` | `openldap` | `functional`, `openldap`, `sanity` | Verify the local LDAP endpoint and published listeners. | All stated checks pass for every applicable target. | -| 12 | `ORCH_FVT_PREPARE_V012` | `test_external_ldap_backend` | `openldap` | `functional`, `openldap`, `sanity` | Verify external LDAP reachability from the omnia_auth container. | All stated checks pass for every applicable target. | -| 13 | `ORCH_FVT_PREPARE_V013` | `test_postgresql_readiness` | `openchami` | `functional`, `sanity` | Verify PostgreSQL and its required SMD database contract. | All stated checks pass for every applicable target. | +| 20301 | `ORCH_FVT_PREPARE_V009` | `test_openldap_runtime` | `openldap` | `openldap`, `sanity` | Verify the enabled service and container are healthy. | All stated checks pass for every applicable target. | +| 20302 | `ORCH_FVT_PREPARE_V010` | `test_openldap_artifacts` | `openldap` | `openldap`, `sanity` | Verify configuration modes, syntax and TLS lifetime. | All stated checks pass for every applicable target. | +| 20303 | `ORCH_FVT_PREPARE_V011` | `test_openldap_endpoint` | `openldap` | `functional`, `openldap`, `sanity` | Verify the local LDAP endpoint and published listeners. | All stated checks pass for every applicable target. | OpenLDAP runtime, artifacts, and local endpoint checks follow authoritative -generated `openldap_support` state. External LDAP V008 and V012 additionally -require `validate_external_ldap: true`; otherwise they skip before mutation. -`configure_external_ldap` controls only whether V008 may reconcile -`slapd.conf`. With validation enabled and reconciliation disabled, V008 -checks the existing deployed proxy and reports mismatches as failures. +generated `openldap_support` state. The prepare lifecycle only starts the +local `omnia_auth` container; the external LDAP proxy cases run in the +pxeboot `slurm_ldap` suite. ## Provision test cases -These controller-side cases compare generated reports and live OpenCHAMI -state with the active PXE mapping. +These controller-side cases compare generated reports and live OpenCHAMI state with the active PXE mapping. + +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 30000 | `ORCH_FVT_PROVISION_E001` | `test_deploy_provision` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags provision``. | The selected Orchestrator lifecycle exits successfully. | + +### OpenCHAMI (`openchami`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 100 | `ORCH_FVT_PROVISION_E001` | `test_deploy_provision` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags provision``. | The selected Orchestrator lifecycle exits successfully. | -| 101 | `ORCH_FVT_PROVISION_V001` | `test_provision_reports` | `openchami` | `sanity` | Verify the provision report and generated inventory contracts. | All stated checks pass for every applicable target. | -| 102 | `ORCH_FVT_PROVISION_V002` | `test_smd_identity` | `openchami` | `sanity` | Verify XNAME, administrative MAC, and IP identity in SMD. | All stated checks pass for every applicable target. | -| 103 | `ORCH_FVT_PROVISION_V003` | `test_smd_group_membership` | `openchami` | `sanity` | Verify expected groups and reject competing cloud-init groups. | All stated checks pass for every applicable target. | -| 104 | `ORCH_FVT_PROVISION_V004` | `test_boot_service_configurations` | `openchami` | `sanity` | Verify BootConfigurations and their mapped administrative MACs. | All stated checks pass for every applicable target. | -| 105 | `ORCH_FVT_PROVISION_V005` | `test_boot_service_node_identity` | `openchami` | `sanity` | Verify synchronized XNAME-to-bootMac records. | All stated checks pass for every applicable target. | -| 106 | `ORCH_FVT_PROVISION_V006` | `test_metadata_service_groups` | `openchami` | `sanity` | Verify one usable cloud-init template per functional group. | All stated checks pass for every applicable target. | -| 107 | `ORCH_FVT_PROVISION_V007` | `test_metadata_service_instances` | `openchami` | `sanity` | Verify unique per-node hostname metadata. | All stated checks pass for every applicable target. | -| 108 | `ORCH_FVT_PROVISION_V008` | `test_coredhcp_and_coredns_inventory` | `openchami` | `sanity` | Verify the SMD identity records consumed by CoreDHCP/CoreDNS. | All stated checks pass for every applicable target. | -| 109 | `ORCH_FVT_PROVISION_V009` | `test_boot_image_identity` | `openchami` | `sanity` | Verify Boot Service kernel/initrd paths match build_status.yml per functional group. | All stated checks pass for every applicable target. | -| 110 | `ORCH_FVT_PROVISION_V010` | `test_boot_image_architecture` | `openchami` | `sanity` | Verify build_status.yml architecture keys are consistent with functional group name suffixes. | All stated checks pass for every applicable target. | +| 30101 | `ORCH_FVT_PROVISION_V001` | `test_provision_reports` | `openchami` | `sanity` | Verify the provision report and generated inventory contracts. | All stated checks pass for every applicable target. | +| 30102 | `ORCH_FVT_PROVISION_V002` | `test_smd_identity` | `openchami` | `sanity` | Verify XNAME, administrative MAC, and IP identity in SMD. | All stated checks pass for every applicable target. | +| 30103 | `ORCH_FVT_PROVISION_V003` | `test_smd_group_membership` | `openchami` | `sanity` | Verify expected groups and reject competing cloud-init groups. | All stated checks pass for every applicable target. | +| 30104 | `ORCH_FVT_PROVISION_V004` | `test_boot_service_configurations` | `openchami` | `sanity` | Verify BootConfigurations and their mapped administrative MACs. | All stated checks pass for every applicable target. | +| 30105 | `ORCH_FVT_PROVISION_V005` | `test_boot_service_node_identity` | `openchami` | `sanity` | Verify synchronized XNAME-to-bootMac records. | All stated checks pass for every applicable target. | +| 30106 | `ORCH_FVT_PROVISION_V006` | `test_metadata_service_groups` | `openchami` | `sanity` | Verify one usable cloud-init template per functional group. | All stated checks pass for every applicable target. | +| 30107 | `ORCH_FVT_PROVISION_V007` | `test_metadata_service_instances` | `openchami` | `sanity` | Verify unique per-node hostname metadata. | All stated checks pass for every applicable target. | +| 30108 | `ORCH_FVT_PROVISION_V008` | `test_coredhcp_and_coredns_inventory` | `openchami` | `sanity` | Verify the SMD identity records consumed by CoreDHCP/CoreDNS. | All stated checks pass for every applicable target. | +| 30109 | `ORCH_FVT_PROVISION_V009` | `test_boot_image_identity` | `openchami` | `boot_image` | Verify Boot Service kernel/initrd paths match build_status.yml. | All stated checks pass for every applicable target. | +| 30110 | `ORCH_FVT_PROVISION_V010` | `test_boot_image_architecture` | `openchami` | `boot_image` | Verify build_status.yml architecture keys match functional group names. | All stated checks pass for every applicable target. | Provision verification is read-only. It resolves XNAMEs from live SMD interfaces and groups and obtains fresh OpenCHAMI credentials for API reads. ## PXE post-boot test cases -The following tables are in effective pytest `order` sequence. Optional role -and feature cases skip only when the active mapping or catalog proves that the -target is not applicable. Negative cases pass only when the invalid operation -is rejected. Reboot and scheduler-state cases require their explicit markers. +Optional role and feature cases skip only when the active mapping, catalog, or `test_config.yml` proves that the target is not applicable. Negative cases pass only when the invalid operation is rejected. Reboot, drain, and node-removal suites run last; reboot and node-removal cases require their explicit opt-in markers, while `scheduler_state` only narrows selection. -### PXE lifecycle execution +### Lifecycle execution | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 200 | `ORCH_FVT_PXEBOOT_E001` | `test_deploy_pxeboot` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags pxeboot``. | The selected Orchestrator lifecycle exits successfully. | +| 40000 | `ORCH_FVT_PXEBOOT_E001` | `test_deploy_pxeboot` | `root` | `buildstream`, `deploy`, `sanity` | Run ``orchestrator.yml --tags pxeboot``. | The selected Orchestrator lifecycle exits successfully. | -### Connectivity +### Connectivity (`connectivity`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 201 | `ORCH_FVT_PXEBOOT_V001` | `test_node_ping` | `connectivity` | `connectivity`, `sanity` | Verify ping from the OIM to every mapped administrative IP. | All stated checks pass for every applicable target. | -| 202 | `ORCH_FVT_PXEBOOT_V002` | `test_node_ssh` | `connectivity` | `connectivity`, `sanity` | Verify passwordless root SSH from the OIM to every mapped node. | All stated checks pass for every applicable target. | -| 203 | `ORCH_FVT_PXEBOOT_V003` | `test_node_hostname_ssh` | `connectivity` | `connectivity`, `sanity` | Verify mapped hostnames resolve and support passwordless root SSH. | All stated checks pass for every applicable target. | +| 40101 | `ORCH_FVT_PXEBOOT_V001` | `test_node_ping` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify ping from the OIM to every mapped administrative IP. | All stated checks pass for every applicable target. | +| 40102 | `ORCH_FVT_PXEBOOT_V002` | `test_node_ssh` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify passwordless root SSH from the OIM to every mapped node. | All stated checks pass for every applicable target. | +| 40103 | `ORCH_FVT_PXEBOOT_V003` | `test_node_hostname_ssh` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify mapped hostnames resolve and support passwordless root SSH. | All stated checks pass for every applicable target. | +| 40104 | `ORCH_FVT_PXEBOOT_V097` | `test_node_architecture` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify each node's live architecture matches its functional group. | All stated checks pass for every applicable target. | +| 40105 | `ORCH_FVT_PXEBOOT_V098` | `test_node_os_version` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify each node's live OS version matches its functional group. | All stated checks pass for every applicable target. | -### Cloud-init +### Cloud-init (`cloudinit`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 204 | `ORCH_FVT_PXEBOOT_V004` | `test_node_cloud_init` | `cloudinit` | `cloudinit`, `sanity` | Verify PXE report freshness and direct cloud-init JSON state. | All stated checks pass for every applicable target. | +| 40201 | `ORCH_FVT_PXEBOOT_V004` | `test_node_cloud_init` | `cloudinit` | `buildstream`, `cloudinit`, `sanity` | Verify PXE report freshness and direct cloud-init JSON state. | All stated checks pass for every applicable target. | + +Cloud-init is accepted only when its structured status satisfies the +product contract. A generated script success message is not treated as +authoritative cloud-init state. -### Architecture and OS identity +### Additional cloud-init (`additional_cloud_init`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 298 | `ORCH_FVT_PXEBOOT_V097` | `test_node_architecture` | `connectivity` | `connectivity`, `sanity` | Verify each node's live architecture matches its functional group name suffix. | All stated checks pass for every applicable target. | -| 299 | `ORCH_FVT_PXEBOOT_V098` | `test_node_os_version` | `connectivity` | `connectivity`, `sanity` | Verify each node's live OS version matches the expected image from build_status.yml. | All stated checks pass for every applicable target. | +| 40301 | `ORCH_FVT_PXEBOOT_V095` | `test_additional_cloud_init_smd_groups` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify SMD groups exist for additional cloud-init configuration. | All stated checks pass for every applicable target. | +| 40302 | `ORCH_FVT_PXEBOOT_V096` | `test_additional_cloud_init_metadata_groups` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify metadata-service groups and templates for additional cloud-init. | All stated checks pass for every applicable target. | +| 40303 | `ORCH_FVT_PXEBOOT_V099` | `test_additional_cloud_init_write_files` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify write_files entries were applied on provisioned nodes. | All stated checks pass for every applicable target. | +| 40304 | `ORCH_FVT_PXEBOOT_V100` | `test_additional_cloud_init_runcmd` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify runcmd entries executed during cloud-init on provisioned nodes. | All stated checks pass for every applicable target. | -Cloud-init is accepted only when its structured status satisfies the -product contract. A generated script success message is not treated as -authoritative cloud-init state. +These cases skip when `additional_cloud_init_config_file` is empty or not +configured in `orchestrator_config.yml`. + +### Mount configuration (`mount_config`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40401 | `ORCH_FVT_PXEBOOT_V101` | `test_mount_config_mount_point` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mount point directories exist on all target nodes. | All stated checks pass for every applicable target. | +| 40402 | `ORCH_FVT_PXEBOOT_V102` | `test_mount_config_volume_mounted` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS volumes are actively mounted on all target nodes. | All stated checks pass for every applicable target. | +| 40403 | `ORCH_FVT_PXEBOOT_V103` | `test_mount_config_mount_options` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mount options match storage_config.yml on target nodes. | All stated checks pass for every applicable target. | +| 40404 | `ORCH_FVT_PXEBOOT_V104` | `test_mount_config_fstab` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS fstab entries are persistent on all target nodes. | All stated checks pass for every applicable target. | +| 40405 | `ORCH_FVT_PXEBOOT_V105` | `test_mount_config_bind_mounts` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS bind mount targets are active on all target nodes. | All stated checks pass for every applicable target. | +| 40406 | `ORCH_FVT_PXEBOOT_V106` | `test_mount_config_bind_fstab` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS bind mount fstab entries are persistent on target nodes. | All stated checks pass for every applicable target. | +| 40407 | `ORCH_FVT_PXEBOOT_V107` | `test_mount_config_node_subdirectory` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify per-node subdirectory exists under NFS mount point. | All stated checks pass for every applicable target. | +| 40408 | `ORCH_FVT_PXEBOOT_V108` | `test_mount_config_permissions` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mount permissions match storage_config.yml on nodes. | All stated checks pass for every applicable target. | +| 40409 | `ORCH_FVT_PXEBOOT_V109` | `test_mount_config_fg_targeting` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mounts are present on target FGs and absent on others. | All stated checks pass for every applicable target. | +| 40410 | `ORCH_FVT_PXEBOOT_V110` | `test_mount_config_no_duplicate_fstab` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify no duplicate NFS fstab entries on target nodes. | All stated checks pass for every applicable target. | +| 40411 | `ORCH_FVT_PXEBOOT_V111` | `test_mount_config_writable` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify all configured NFS mounts are writable on target nodes. | All stated checks pass for every applicable target. | +| 40412 | `ORCH_FVT_PXEBOOT_V112` | `test_mount_config_oim_mount` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS storage is mounted on the OIM when mount_on_oim is true. | All stated checks pass for every applicable target. | + +### Minimal OS (`minimal_os`) -### Kubernetes - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 205 | `ORCH_FVT_PXEBOOT_V005` | `test_kubernetes_nodes` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify mapped Kubernetes membership and Ready state. | All stated checks pass for every applicable target. | -| 206 | `ORCH_FVT_PXEBOOT_V006` | `test_kubernetes_node_services` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify required services on every Kubernetes role. | All stated checks pass for every applicable target. | -| 207 | `ORCH_FVT_PXEBOOT_V007` | `test_kubernetes_version_compatibility` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify Kubernetes, kubeadm, and CRI-O version alignment. | All stated checks pass for every applicable target. | -| 208 | `ORCH_FVT_PXEBOOT_V101` | `test_kubernetes_configured_versions` | `kubernetes_cluster` | `kubernetes`, `sanity` | Compare deployed Kubernetes components with the selected catalog version. | Kubernetes components match the configured version and CRI-O matches its major/minor release. | -| 209 | `ORCH_FVT_PXEBOOT_V008` | `test_kubernetes_control_plane` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify API readiness and the configured control plane. | All stated checks pass for every applicable target. | -| 210 | `ORCH_FVT_PXEBOOT_V009` | `test_kubernetes_system_pods` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify required system, CNI, storage, and HA workloads. | All stated checks pass for every applicable target. | -| 211 | `ORCH_FVT_PXEBOOT_V010` | `test_kubernetes_virtual_ip` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify exactly one owner for the configured Kubernetes VIP. | All stated checks pass for every applicable target. | -| 212 | `ORCH_FVT_PXEBOOT_V011` | `test_kubernetes_workload_scheduling` | `kubernetes_cluster` | `functional`, `kubernetes`, `sanity` | Create, verify, and remove an isolated scheduling probe. | The isolated probe becomes ready, satisfies the stated contract, and is removed. | -| 213 | `ORCH_FVT_PXEBOOT_V012` | `test_kubernetes_etcd_health` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify health for all etcd endpoints. | All stated checks pass for every applicable target. | -| 214 | `ORCH_FVT_PXEBOOT_V013` | `test_kubernetes_etcd_topology` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify etcd membership, leader election, and raft consistency. | All stated checks pass for every applicable target. | -| 215 | `ORCH_FVT_PXEBOOT_V014` | `test_kubernetes_local_etcd` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify each control plane has the configured etcd mount. | All stated checks pass for every applicable target. | -| 216 | `ORCH_FVT_PXEBOOT_V015` | `test_kubernetes_local_etcd_integrity` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify disk selection, ext4 label, UUID fstab, and boot persistence. | All stated checks pass for every applicable target. | -| 217 | `ORCH_FVT_PXEBOOT_V102` | `test_kubernetes_local_etcd_provisioning` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify GPT selection and local-etcd scripts, logs, and selected disk. | Every enabled control plane satisfies the provisioning contract. | -| 218 | `ORCH_FVT_PXEBOOT_V016` | `test_kubernetes_storage` | `kubernetes_storage` | `kubernetes`, `sanity` | Verify configured NFS and PowerScale storage objects. | All stated checks pass for every applicable target. | -| 219 | `ORCH_FVT_PXEBOOT_V017` | `test_kubernetes_default_storage_class` | `kubernetes_storage` | `kubernetes`, `sanity` | Verify exactly one expected default StorageClass. | All stated checks pass for every applicable target. | -| 220 | `ORCH_FVT_PXEBOOT_V018` | `test_kubernetes_snapshot_controller` | `kubernetes_storage` | `kubernetes`, `sanity` | Verify PowerScale snapshot components when configured. | All stated checks pass for every applicable target. | -| 221 | `ORCH_FVT_PXEBOOT_V103` | `test_kubernetes_nfs_provisioner_contract` | `kubernetes_storage` | `kubernetes`, `sanity` | Compare the NFS StorageClass, provisioner backend, and node mounts with active configuration. | Every NFS contract field and mapped-node mount matches the selected storage entry. | -| 222 | `ORCH_FVT_PXEBOOT_V019` | `test_kubernetes_nfs_dynamic_provisioning` | `kubernetes_storage` | `functional`, `kubernetes`, `sanity` | Create and remove an isolated NFS-backed workload. | All stated checks pass for every applicable target. | -| 223 | `ORCH_FVT_PXEBOOT_V020` | `test_kubernetes_csi_dynamic_provisioning` | `kubernetes_storage` | `functional`, `kubernetes`, `sanity` | Create and remove an isolated PowerScale-backed workload. | All stated checks pass for every applicable target. | -| 224 | `ORCH_FVT_PXEBOOT_V021` | `test_kubernetes_local_etcd_recovery` | `kubernetes_recovery` | `disruptive`, `kubernetes`, `reboot` | Reboot a control plane and prove its local-etcd UUID is preserved. | The node returns within the bounded wait and every stated post-reboot check passes. | -| 225 | `ORCH_FVT_PXEBOOT_V022` | `test_kubernetes_control_plane_recovery` | `kubernetes_recovery` | `disruptive`, `kubernetes`, `reboot` | Reboot the VIP owner and verify control-plane recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40501 | `ORCH_FVT_PXEBOOT_V113` | `test_minimal_os_base_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify base OS packages are installed on all OS-only nodes. | All stated checks pass for every applicable target. | +| 40502 | `ORCH_FVT_PXEBOOT_V114` | `test_minimal_os_ldms_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify LDMS monitoring packages and binary on OS-only nodes. | All stated checks pass for every applicable target. | +| 40503 | `ORCH_FVT_PXEBOOT_V115` | `test_minimal_os_required_services` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify required services are active on all OS-only nodes. | All stated checks pass for every applicable target. | +| 40504 | `ORCH_FVT_PXEBOOT_V117` | `test_minimal_os_excluded_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify workload-specific packages are NOT installed on OS-only nodes. | All stated checks pass for every applicable target. | +| 40505 | `ORCH_FVT_PXEBOOT_V118` | `test_minimal_os_excluded_services` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify workload-specific services are NOT active on OS-only nodes. | All stated checks pass for every applicable target. | +| 40506 | `ORCH_FVT_PXEBOOT_V119` | `test_minimal_os_package_manager` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify package manager is functional on OS-only nodes. | All stated checks pass for every applicable target. | +| 40507 | `ORCH_FVT_PXEBOOT_V120` | `test_minimal_os_kernel_version` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify kernel version is consistent across OS-only nodes per FG. | All stated checks pass for every applicable target. | +| 40508 | `ORCH_FVT_PXEBOOT_V121` | `test_minimal_os_network_identity` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify admin IP is configured on all OS-only nodes. | All stated checks pass for every applicable target. | +| 40509 | `ORCH_FVT_PXEBOOT_V122` | `test_minimal_os_functional_group_schema` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify functional-group definitions are valid on OS-only nodes. | All stated checks pass for every applicable target. | +| 40510 | `ORCH_FVT_PXEBOOT_V123` | `test_minimal_os_additional_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify configured extra packages are installed on OS-only nodes. | All stated checks pass for every applicable target. | +| 40511 | `ORCH_FVT_PXEBOOT_V124` | `test_minimal_os_additional_packages_fallback` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify absent additional_packages config is handled on OS-only nodes. | All stated checks pass for every applicable target. | +| 40512 | `ORCH_FVT_PXEBOOT_V125` | `test_minimal_os_ldms_service_state` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify LDMS service is installed but inactive at handoff. | All stated checks pass for every applicable target. | +| 40513 | `ORCH_FVT_PXEBOOT_V126` | `test_minimal_os_ssh_key_access` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify SSH key authentication is proven on OS-only nodes. | All stated checks pass for every applicable target. | +| 40514 | `ORCH_FVT_PXEBOOT_V127` | `test_minimal_os_network_isolation` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify default route is on management network on OS-only nodes. | All stated checks pass for every applicable target. | +| 40515 | `ORCH_FVT_PXEBOOT_V128` | `test_minimal_os_no_embedded_credentials` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify no plaintext secrets are present in the OS image. | All stated checks pass for every applicable target. | + +### PowerVault iSCSI storage (`powervault`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40601 | `ORCH_FVT_PXEBOOT_V400` | `test_powervault_iscsi_service` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iscsid is active and enabled on all PowerVault target nodes. | All stated checks pass for every applicable target. | +| 40602 | `ORCH_FVT_PXEBOOT_V401` | `test_powervault_iscsi_initiator_name` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI initiator name matches config on all target nodes. | All stated checks pass for every applicable target. | +| 40603 | `ORCH_FVT_PXEBOOT_V402` | `test_powervault_iscsi_discovery` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI target discovery succeeds from all portal IPs. | All stated checks pass for every applicable target. | +| 40604 | `ORCH_FVT_PXEBOOT_V403` | `test_powervault_iscsi_sessions` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI sessions are active on all target nodes. | All stated checks pass for every applicable target. | +| 40605 | `ORCH_FVT_PXEBOOT_V404` | `test_powervault_iscsi_startup_automatic` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI node startup is automatic on all target nodes. | All stated checks pass for every applicable target. | +| 40606 | `ORCH_FVT_PXEBOOT_V405` | `test_powervault_portal_reachability` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI portal ports are reachable and sessions healthy. | All stated checks pass for every applicable target. | +| 40607 | `ORCH_FVT_PXEBOOT_V406` | `test_powervault_multipath_service` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify multipathd is active and enabled on all target nodes. | All stated checks pass for every applicable target. | +| 40608 | `ORCH_FVT_PXEBOOT_V407` | `test_powervault_multipath_device` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify multipath device exists and matches volume_id. | All stated checks pass for every applicable target. | +| 40609 | `ORCH_FVT_PXEBOOT_V408` | `test_powervault_multipath_redundancy` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify multipath device has multiple paths for redundancy. | All stated checks pass for every applicable target. | +| 40610 | `ORCH_FVT_PXEBOOT_V409` | `test_powervault_gpt_partition` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify GPT partition exists on multipath device. | All stated checks pass for every applicable target. | +| 40611 | `ORCH_FVT_PXEBOOT_V410` | `test_powervault_filesystem_type` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify filesystem formatted with correct type. | All stated checks pass for every applicable target. | +| 40612 | `ORCH_FVT_PXEBOOT_V411` | `test_powervault_mount_point_directory` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify mount point directory exists on all target nodes. | All stated checks pass for every applicable target. | +| 40613 | `ORCH_FVT_PXEBOOT_V412` | `test_powervault_volume_mounted` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify PowerVault volume is actively mounted on all target nodes. | All stated checks pass for every applicable target. | +| 40614 | `ORCH_FVT_PXEBOOT_V413` | `test_powervault_mount_options` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify mount options applied correctly on all target nodes. | All stated checks pass for every applicable target. | +| 40615 | `ORCH_FVT_PXEBOOT_V414` | `test_powervault_fstab_entry` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify persistent fstab entry created on all target nodes. | All stated checks pass for every applicable target. | +| 40616 | `ORCH_FVT_PXEBOOT_V415` | `test_powervault_node_subdirectory` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify per-node subdirectory exists under mount point. | All stated checks pass for every applicable target. | +| 40617 | `ORCH_FVT_PXEBOOT_V416` | `test_powervault_bind_mounts` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify bind mount targets are active on all target nodes. | All stated checks pass for every applicable target. | +| 40618 | `ORCH_FVT_PXEBOOT_V417` | `test_powervault_bind_fstab_entries` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify bind mount fstab entries are persistent on all target nodes. | All stated checks pass for every applicable target. | +| 40619 | `ORCH_FVT_PXEBOOT_V418` | `test_powervault_bind_isolation` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify per-node data separation via bind mounts. | All stated checks pass for every applicable target. | +| 40620 | `ORCH_FVT_PXEBOOT_V419` | `test_powervault_functional_group_targeting` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify PV mount only on correct functional groups. | All stated checks pass for every applicable target. | +| 40621 | `ORCH_FVT_PXEBOOT_V420` | `test_powervault_multiple_prefix_targeting` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify multiple prefixes target all groups correctly. | All stated checks pass for every applicable target. | +| 40622 | `ORCH_FVT_PXEBOOT_V421` | `test_powervault_setup_log` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify cloud-init runcmd log exists and shows completion. | All stated checks pass for every applicable target. | +| 40623 | `ORCH_FVT_PXEBOOT_V422` | `test_powervault_cloud_init_groups_dict` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify rendered iSCSI setup scripts deployed on target nodes. | All stated checks pass for every applicable target. | +| 40624 | `ORCH_FVT_PXEBOOT_V423` | `test_powervault_no_duplicate_fstab` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify no duplicate fstab entries on all target nodes. | All stated checks pass for every applicable target. | +| 40625 | `ORCH_FVT_PXEBOOT_V424` | `test_powervault_all_mounts_writable` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify all PV mounts (main + bind) are writable. | All stated checks pass for every applicable target. | +| 40626 | `ORCH_FVT_PXEBOOT_V425` | `test_powervault_permissions` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify permissions on mount point match config. | All stated checks pass for every applicable target. | +| 40627 | `ORCH_FVT_PXEBOOT_V426` | `test_powervault_io_write_read` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify write-read I/O succeeds on every PV mount point. | All stated checks pass for every applicable target. | +| 40628 | `ORCH_FVT_PXEBOOT_V427` | `test_powervault_bind_io` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify bind-mount I/O reaches the PV backing store. | All stated checks pass for every applicable target. | +| 40629 | `ORCH_FVT_PXEBOOT_V428` | `test_powervault_slurm_mandatory_bind_mounts` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify /var/lib/mysql and /var/spool/slurm configured as bind targets. | All stated checks pass for every applicable target. | +| 40630 | `ORCH_FVT_PXEBOOT_V429` | `test_powervault_mysql_data_on_mount` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify MySQL/MariaDB datadir is on a PowerVault mount. | All stated checks pass for every applicable target. | + +PowerVault cases skip when `powervault_config` is absent or empty in +`storage_config.yml`. + +### CoreDNS/CoreDHCP (`coredns_coredhcp`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40701 | `ORCH_FVT_PXEBOOT_V300` | `test_coredns_container_state` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity` | TC-01: coresmd containers match the dns_enabled dataset (enabled+disabled). | All stated checks pass for every applicable target. | +| 40702 | `ORCH_FVT_PXEBOOT_V301` | `test_coredns_forward_resolution` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-02: dig FQDN from OIM for every mapped node; compare to ADMIN_IP. | All stated checks pass for every applicable target. | +| 40703 | `ORCH_FVT_PXEBOOT_V302` | `test_coredns_reverse_resolution` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-03: dig -x from OIM for every mapped ADMIN_IP; compare to FQDN. | All stated checks pass for every applicable target. | +| 40704 | `ORCH_FVT_PXEBOOT_V303` | `test_coredns_dns_forwarders` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-04: verify DNS forwarders are configured and can resolve external domains. | The operation completes successfully and returns the expected result. | +| 40705 | `ORCH_FVT_PXEBOOT_V304` | `test_coredns_corefile_config` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-05: verify Corefile configuration is correctly rendered. | The operation completes successfully and returns the expected result. | +| 40706 | `ORCH_FVT_PXEBOOT_V305` | `test_coredhcp_config_file` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-06: verify coredhcp.yaml configuration file is correctly rendered. | The operation completes successfully and returns the expected result. | +| 40707 | `ORCH_FVT_PXEBOOT_V306` | `test_coredhcp_multisubnet_running_image` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity` | TC-07: multi-subnet dataset -> coresmd containers use expected image. | All stated checks pass for every applicable target. | +| 40708 | `ORCH_FVT_PXEBOOT_V309` | `test_coredns_idempotency` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity` | TC-08: coresmd state stability (no-drift) across a settle window. | State is identical across the settle window with no spontaneous changes. | +| 40709 | `ORCH_FVT_PXEBOOT_V307` | `test_dns_compute_resolv_conf` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-09: /etc/resolv.conf on every compute has CoreDNS as primary. | All stated checks pass for every applicable target. | +| 40710 | `ORCH_FVT_PXEBOOT_V308` | `test_dns_compute_forward_getent` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | TC-10: getent hosts on every compute resolves every mapped peer. | All stated checks pass for every applicable target. | +| 40711 | `ORCH_FVT_PXEBOOT_V310` | `test_dns_node_addition_pipeline` | `coredns_coredhcp` | `destructive`, `functional` | TC-11: prove SMD-to-CoreDNS pipeline resolves every mapped node. | The operation completes successfully and returns the expected result. | +| 40712 | `ORCH_FVT_PXEBOOT_V311` | `test_dns_smd_unreachable_cached_resolution` | `coredns_coredhcp` | `destructive`, `functional` | TC-12: pause SMD briefly; CoreDNS must keep serving cached records. | The node returns within the bounded wait and every stated postcondition check passes. | + +### Kubernetes cluster (`kubernetes_cluster`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40801 | `ORCH_FVT_PXEBOOT_V005` | `test_kubernetes_nodes` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify mapped Kubernetes membership and Ready state. | All stated checks pass for every applicable target. | +| 40802 | `ORCH_FVT_PXEBOOT_V006` | `test_kubernetes_node_services` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify required services on every Kubernetes role. | All stated checks pass for every applicable target. | +| 40803 | `ORCH_FVT_PXEBOOT_V007` | `test_kubernetes_version_compatibility` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify Kubernetes, kubeadm, and CRI-O version alignment. | All stated checks pass for every applicable target. | +| 40804 | `ORCH_FVT_PXEBOOT_V519` | `test_kubernetes_configured_versions` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify deployed component versions against the selected catalog. | Kubernetes components match the configured version and CRI-O matches its major/minor release. | +| 40805 | `ORCH_FVT_PXEBOOT_V008` | `test_kubernetes_control_plane` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify API readiness and the configured control plane. | All stated checks pass for every applicable target. | +| 40806 | `ORCH_FVT_PXEBOOT_V009` | `test_kubernetes_system_pods` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify required system, CNI, storage, and HA workloads. | All stated checks pass for every applicable target. | +| 40807 | `ORCH_FVT_PXEBOOT_V010` | `test_kubernetes_virtual_ip` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify exactly one owner for the configured Kubernetes VIP. | All stated checks pass for every applicable target. | +| 40808 | `ORCH_FVT_PXEBOOT_V011` | `test_kubernetes_workload_scheduling` | `kubernetes_cluster` | `buildstream`, `functional`, `kubernetes`, `sanity` | Create, verify, and remove an isolated scheduling probe. | The isolated probe becomes ready, satisfies the stated contract, and is removed. | + +### Kubernetes etcd (`kubernetes_etcd`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40901 | `ORCH_FVT_PXEBOOT_V012` | `test_kubernetes_etcd_health` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify health for all etcd endpoints. | All stated checks pass for every applicable target. | +| 40902 | `ORCH_FVT_PXEBOOT_V013` | `test_kubernetes_etcd_topology` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify etcd membership, leader election, and raft consistency. | All stated checks pass for every applicable target. | +| 40903 | `ORCH_FVT_PXEBOOT_V014` | `test_kubernetes_local_etcd` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify each control plane has the configured etcd mount. | All stated checks pass for every applicable target. | +| 40904 | `ORCH_FVT_PXEBOOT_V015` | `test_kubernetes_local_etcd_integrity` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify disk selection, ext4 label, UUID fstab, and boot persistence. | All stated checks pass for every applicable target. | +| 40905 | `ORCH_FVT_PXEBOOT_V520` | `test_kubernetes_local_etcd_provisioning` | `kubernetes_etcd` | `kubernetes` | Verify GPT selection and local-etcd scripts, logs, and selected disk. | Every enabled control plane satisfies the provisioning contract. | +| 40906 | `ORCH_FVT_PXEBOOT_V523` | `test_kubernetes_local_etcd_media` | `kubernetes_etcd` | `kubernetes` | Verify every control plane places etcd on the configured disk media. | All stated checks pass for every applicable target. | + +Local-etcd cases skip when `etcd_on_local_disk` is disabled. Integrity and +provisioning require the boot script for the current boot to finish with +`===== DONE =====`: `etcd-disk-setup.sh` on first boot, or +`etcd-fstab-update.sh` on later boots. The media case skips unless +`expected_etcd_disk_media` is set to `ssd`, `hdd`, or `nvme`. + +### Kubernetes storage (`kubernetes_storage`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41001 | `ORCH_FVT_PXEBOOT_V016` | `test_kubernetes_storage` | `kubernetes_storage` | `buildstream`, `kubernetes`, `sanity` | Verify configured NFS and PowerScale storage objects. | All stated checks pass for every applicable target. | +| 41002 | `ORCH_FVT_PXEBOOT_V017` | `test_kubernetes_default_storage_class` | `kubernetes_storage` | `buildstream`, `kubernetes`, `sanity` | Verify exactly one expected default StorageClass. | All stated checks pass for every applicable target. | +| 41003 | `ORCH_FVT_PXEBOOT_V018` | `test_kubernetes_snapshot_controller` | `kubernetes_storage` | `buildstream`, `kubernetes`, `sanity` | Verify PowerScale snapshot components when configured. | All stated checks pass for every applicable target. | +| 41004 | `ORCH_FVT_PXEBOOT_V019` | `test_kubernetes_nfs_dynamic_provisioning` | `kubernetes_storage` | `buildstream`, `functional`, `kubernetes`, `sanity` | Create and remove an isolated NFS-backed workload. | All stated checks pass for every applicable target. | +| 41005 | `ORCH_FVT_PXEBOOT_V020` | `test_kubernetes_csi_dynamic_provisioning` | `kubernetes_storage` | `buildstream`, `functional`, `kubernetes`, `sanity` | Create and remove an isolated PowerScale-backed workload. | All stated checks pass for every applicable target. | Temporary Kubernetes resources use unique namespaces and are removed by -the creating test. CSI, snapshot, and local-etcd checks are selected from -the active configuration. - -### Slurm - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 226 | `ORCH_FVT_PXEBOOT_V023` | `test_slurm_membership` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify mapped membership, healthy state, and basic hardware fields. | All stated checks pass for every applicable target. | -| 227 | `ORCH_FVT_PXEBOOT_V024` | `test_slurm_scheduler` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify mapped compute nodes have healthy, available partitions. | All stated checks pass for every applicable target. | -| 228 | `ORCH_FVT_PXEBOOT_V025` | `test_slurm_services` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify role and feature-specific Slurm services. | All stated checks pass for every applicable target. | -| 229 | `ORCH_FVT_PXEBOOT_V026` | `test_slurm_cross_node_ssh` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify every mapped Slurm role can reach every peer over root SSH. | All stated checks pass for every applicable target. | -| 230 | `ORCH_FVT_PXEBOOT_V027` | `test_slurm_configless_mode` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify configless controller access and expected cluster identity. | All stated checks pass for every applicable target. | -| 231 | `ORCH_FVT_PXEBOOT_V028` | `test_slurm_configuration_consistency` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Compare authoritative Slurm files with every configless client cache. | Every compared value matches its authoritative source. | -| 232 | `ORCH_FVT_PXEBOOT_V029` | `test_slurm_reconfigure` | `slurm_cluster` | `functional`, `non_disruptive`, `sanity`, `slurm` | Reconfigure Slurm and verify membership remains healthy. | All stated checks pass for every applicable target. | -| 233 | `ORCH_FVT_PXEBOOT_V030` | `test_slurm_hardware_discovery` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify runtime hardware matches the configured discovery strategy. | All stated checks pass for every applicable target. | -| 234 | `ORCH_FVT_PXEBOOT_V031` | `test_slurm_custom_configuration` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify custom values, NFS delivery, and effective visibility. | All stated checks pass for every applicable target. | -| 235 | `ORCH_FVT_PXEBOOT_V032` | `test_slurm_control_node_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every Slurm control node. | The operation completes successfully and returns the expected result. | -| 236 | `ORCH_FVT_PXEBOOT_V033` | `test_slurm_login_node_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every mapped login node. | The operation completes successfully and returns the expected result. | -| 237 | `ORCH_FVT_PXEBOOT_V034` | `test_slurm_compiler_node_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every login compiler node. | The operation completes successfully and returns the expected result. | -| 238 | `ORCH_FVT_PXEBOOT_V035` | `test_slurm_concurrent_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Submit concurrent jobs and verify final accounting state. | The submission completes with the expected final state and output. | -| 239 | `ORCH_FVT_PXEBOOT_V036` | `test_slurm_insufficient_resources` | `slurm_jobs` | `functional`, `negative`, `non_disruptive`, `sanity`, `slurm` | Verify an impossible immediate allocation is rejected. | The expected rejection occurs and no prohibited state is accepted. | -| 240 | `ORCH_FVT_PXEBOOT_V037` | `test_slurm_job_queueing` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Saturate idle computes and verify one follower queues then completes. | The follower is pending under saturation and completes after resources are released. | -| 241 | `ORCH_FVT_PXEBOOT_V038` | `test_slurm_drain_queue_recovery` | `slurm_jobs` | `disruptive`, `sanity`, `scheduler_state`, `slurm` | Drain one compute node, verify queuing, and restore it. | The job queues while the node is drained, then the node is resumed and the job completes. | -| 242 | `ORCH_FVT_PXEBOOT_V039` | `test_slurm_pam_policy` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify SSHD, the PAM module, and pam_slurm_adopt account policy. | All stated checks pass for every applicable target. | -| 243 | `ORCH_FVT_PXEBOOT_V040` | `test_slurm_control_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on the Slurm control node. | All stated checks pass for every applicable target. | -| 244 | `ORCH_FVT_PXEBOOT_V041` | `test_slurm_control_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify an invalid LDAP password is rejected on the control node. | The expected rejection occurs and no prohibited state is accepted. | -| 245 | `ORCH_FVT_PXEBOOT_V042` | `test_slurm_login_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every mapped login node. | All stated checks pass for every applicable target. | -| 246 | `ORCH_FVT_PXEBOOT_V043` | `test_slurm_login_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify an invalid LDAP password is rejected on every login node. | The expected rejection occurs and no prohibited state is accepted. | -| 247 | `ORCH_FVT_PXEBOOT_V044` | `test_slurm_compiler_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every login-compiler node. | All stated checks pass for every applicable target. | -| 248 | `ORCH_FVT_PXEBOOT_V045` | `test_slurm_compiler_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify invalid LDAP passwords are rejected on login-compiler nodes. | The expected rejection occurs and no prohibited state is accepted. | -| 249 | `ORCH_FVT_PXEBOOT_V046` | `test_slurm_pam_no_job_access` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify LDAP compute login is denied without an active job. | The expected rejection occurs and no prohibited state is accepted. | -| 250 | `ORCH_FVT_PXEBOOT_V047` | `test_slurm_invalid_ldap_identity` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a generated missing directory identity is rejected. | The expected rejection occurs and no prohibited state is accepted. | -| 251 | `ORCH_FVT_PXEBOOT_V048` | `test_slurm_control_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from the control node. | The submission completes with the expected final state and output. | -| 252 | `ORCH_FVT_PXEBOOT_V049` | `test_slurm_control_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify control-submitted PAM access during and after a job. | All stated checks pass for every applicable target. | -| 253 | `ORCH_FVT_PXEBOOT_V050` | `test_slurm_login_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from every login node. | The submission completes with the expected final state and output. | -| 254 | `ORCH_FVT_PXEBOOT_V051` | `test_slurm_login_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-node PAM access during and after a job. | All stated checks pass for every applicable target. | -| 255 | `ORCH_FVT_PXEBOOT_V052` | `test_slurm_compiler_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit an LDAP-owned job from every login-compiler node. | The submission completes with the expected final state and output. | -| 256 | `ORCH_FVT_PXEBOOT_V053` | `test_slurm_compiler_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-compiler PAM access during and after a job. | All stated checks pass for every applicable target. | -| 257 | `ORCH_FVT_PXEBOOT_V054` | `test_slurm_gpu_inventory` | `slurm_gpu` | `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA runtime state on scheduler-declared GPU nodes. | All stated checks pass for every applicable target. | -| 258 | `ORCH_FVT_PXEBOOT_V055` | `test_slurm_gpu_job` | `slurm_gpu` | `functional`, `non_disruptive`, `sanity`, `slurm` | Allocate a GPU through Slurm and query the device. | The allocation succeeds and the requested device is visible. | -| 259 | `ORCH_FVT_PXEBOOT_V056` | `test_slurm_gpu_memory_stress` | `slurm_gpu` | `functional`, `non_disruptive`, `sanity`, `slurm` | Compile and run a bounded GPU memory workload through Slurm. | Compilation and bounded workload execution complete successfully. | -| 260 | `ORCH_FVT_PXEBOOT_V057` | `test_slurm_openmpi_installation` | `slurm_openmpi` | `non_disruptive`, `sanity`, `slurm` | Verify OpenMPI discovery and version on every compute node. | All stated checks pass for every applicable target. | -| 261 | `ORCH_FVT_PXEBOOT_V058` | `test_slurm_openmpi_job` | `slurm_openmpi` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run an OpenMPI-backed job when OpenMPI is configured. | The operation completes successfully and returns the expected result. | -| 262 | `ORCH_FVT_PXEBOOT_V059` | `test_slurm_ucx_transport` | `slurm_ucx` | `non_disruptive`, `sanity`, `slurm` | Verify UCX exposes an InfiniBand-capable transport. | All stated checks pass for every applicable target. | -| 263 | `ORCH_FVT_PXEBOOT_V060` | `test_slurm_infiniband_configuration` | `slurm_infiniband` | `non_disruptive`, `sanity`, `slurm` | Verify mapped IB interface, address, prefix, link, MTU, and OFED. | All stated checks pass for every applicable target. | -| 264 | `ORCH_FVT_PXEBOOT_V061` | `test_slurm_infiniband_connectivity` | `slurm_infiniband` | `non_disruptive`, `sanity`, `slurm` | Verify every mapped IB endpoint can reach every mapped peer. | All stated checks pass for every applicable target. | -| 265 | `ORCH_FVT_PXEBOOT_V062` | `test_slurm_cluster_recovery` | `slurm_recovery` | `disruptive`, `functional`, `reboot`, `slurm` | Reboot mapped Slurm nodes and verify scheduler and workload recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | - -Login and login-compiler cases skip before credential loading when the -corresponding role is absent. Feature checks use only catalog layers -selected by mapped functional groups. - -### Slurm Apptainer - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 266 | `ORCH_FVT_PXEBOOT_V063` | `test_apptainer_runtime` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the Apptainer executable and version on every compute node. | All stated checks pass for every applicable target. | -| 267 | `ORCH_FVT_PXEBOOT_V064` | `test_apptainer_shared_artifacts` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify shared image directories and downloader artifacts. | All stated checks pass for every applicable target. | -| 268 | `ORCH_FVT_PXEBOOT_V065` | `test_apptainer_pulp_policy` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the generated downloader uses only the configured Pulp source. | All stated checks pass for every applicable target. | -| 269 | `ORCH_FVT_PXEBOOT_V066` | `test_apptainer_shared_storage` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify /hpc_tools is a shared mounted filesystem on every compute. | All stated checks pass for every applicable target. | -| 270 | `ORCH_FVT_PXEBOOT_V067` | `test_apptainer_download` | `slurm_apptainer` | `apptainer`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the deployed downloader and require at least one usable SIF. | The operation completes successfully and returns the expected result. | -| 271 | `ORCH_FVT_PXEBOOT_V068` | `test_apptainer_download_idempotency` | `slurm_apptainer` | `apptainer`, `functional`, `image_download`, `non_disruptive`, `sanity` | Rerun the downloader and verify existing image metadata is unchanged. | The repeated operation succeeds without changing protected state. | -| 272 | `ORCH_FVT_PXEBOOT_V069` | `test_apptainer_download_memory` | `slurm_apptainer` | `apptainer`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the downloader and enforce a bounded peak resident-memory use. | The operation completes successfully and returns the expected result. | -| 273 | `ORCH_FVT_PXEBOOT_V070` | `test_apptainer_image_inventory` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify every compute sees one consistent non-empty SIF inventory. | All stated checks pass for every applicable target. | -| 274 | `ORCH_FVT_PXEBOOT_V071` | `test_apptainer_sif_format` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify each discovered image is a valid inspectable SIF. | All stated checks pass for every applicable target. | -| 275 | `ORCH_FVT_PXEBOOT_V072` | `test_apptainer_sif_permissions` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify shared SIF files are non-empty and world-readable. | All stated checks pass for every applicable target. | -| 276 | `ORCH_FVT_PXEBOOT_V073` | `test_apptainer_sif_integrity` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the selected SIF has the same checksum on every compute. | All stated checks pass for every applicable target. | -| 277 | `ORCH_FVT_PXEBOOT_V074` | `test_apptainer_ldap_readability` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the LDAP test identity can read a shared SIF when enabled. | All stated checks pass for every applicable target. | -| 278 | `ORCH_FVT_PXEBOOT_V075` | `test_apptainer_non_root_execution` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify an unprivileged local identity can execute a shared SIF. | All stated checks pass for every applicable target. | -| 279 | `ORCH_FVT_PXEBOOT_V076` | `test_apptainer_missing_image_contract` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify the downloader records pull failures and exits non-zero. | The expected rejection occurs and no prohibited state is accepted. | -| 280 | `ORCH_FVT_PXEBOOT_V077` | `test_apptainer_single_node_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run one exact-node container job on every mapped compute. | The operation completes successfully and returns the expected result. | -| 281 | `ORCH_FVT_PXEBOOT_V078` | `test_apptainer_multi_node_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run one container allocation spanning all mapped computes. | The operation completes successfully and returns the expected result. | -| 282 | `ORCH_FVT_PXEBOOT_V079` | `test_apptainer_ldap_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run targeted container jobs as the configured LDAP test identity. | The operation completes successfully and returns the expected result. | -| 283 | `ORCH_FVT_PXEBOOT_V080` | `test_apptainer_concurrent_jobs` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run bounded concurrent container jobs on distinct computes. | The operation completes successfully and returns the expected result. | -| 284 | `ORCH_FVT_PXEBOOT_V081` | `test_apptainer_invalid_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a nonexistent SIF fails through the Slurm execution path. | The expected rejection occurs and no prohibited state is accepted. | -| 285 | `ORCH_FVT_PXEBOOT_V082` | `test_apptainer_restricted_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify an unprivileged identity cannot execute a mode-0600 SIF. | The expected rejection occurs and no prohibited state is accepted. | -| 286 | `ORCH_FVT_PXEBOOT_V083` | `test_apptainer_nfs_visibility` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify containers can read the shared image path on every compute. | All stated checks pass for every applicable target. | -| 287 | `ORCH_FVT_PXEBOOT_V084` | `test_apptainer_slurm_environment` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify Slurm allocation variables propagate into containers. | All stated checks pass for every applicable target. | -| 288 | `ORCH_FVT_PXEBOOT_V085` | `test_apptainer_job_array` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Submit and wait for a bounded Apptainer Slurm job array. | The submission completes with the expected final state and output. | -| 289 | `ORCH_FVT_PXEBOOT_V086` | `test_apptainer_failure_cleanup` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a failed image launch leaves no matching runtime process. | The expected rejection occurs and no prohibited state is accepted. | -| 290 | `ORCH_FVT_PXEBOOT_V087` | `test_apptainer_gpu_access` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify scheduler-declared GPU nodes expose GPUs in the container. | All stated checks pass for every applicable target. | -| 291 | `ORCH_FVT_PXEBOOT_V088` | `test_apptainer_gpu_count` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify each container sees the same GPU count as its host. | All stated checks pass for every applicable target. | -| 292 | `ORCH_FVT_PXEBOOT_V089` | `test_apptainer_cuda_workload` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Execute a bounded NVIDIA device query in each GPU container. | The operation completes successfully and returns the expected result. | -| 293 | `ORCH_FVT_PXEBOOT_V090` | `test_apptainer_gpu_memory` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify the GPU query leaves no material device-memory allocation. | All stated checks pass for every applicable target. | -| 294 | `ORCH_FVT_PXEBOOT_V091` | `test_apptainer_infiniband` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify mapped compute nodes expose InfiniBand devices in containers. | All stated checks pass for every applicable target. | -| 295 | `ORCH_FVT_PXEBOOT_V092` | `test_apptainer_reboot_storage` | `slurm_apptainer` | `apptainer`, `disruptive`, `reboot` | Reboot one compute and verify the shared mount and SIF checksum. | The node returns within the bounded wait and every stated post-reboot check passes. | -| 296 | `ORCH_FVT_PXEBOOT_V093` | `test_apptainer_reboot_job` | `slurm_apptainer` | `apptainer`, `disruptive`, `reboot` | Run an exact-node container job after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | -| 297 | `ORCH_FVT_PXEBOOT_V094` | `test_apptainer_reboot_artifacts` | `slurm_apptainer` | `apptainer`, `disruptive`, `reboot` | Verify downloader artifacts and policy after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | +the creating test. CSI and snapshot checks are selected from the active +configuration. + +### Slurm cluster (`slurm_cluster`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41101 | `ORCH_FVT_PXEBOOT_V023` | `test_slurm_membership` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify mapped membership, healthy state, and basic hardware fields. | All stated checks pass for every applicable target. | +| 41102 | `ORCH_FVT_PXEBOOT_V024` | `test_slurm_scheduler` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify mapped compute nodes have healthy, available partitions. | All stated checks pass for every applicable target. | +| 41103 | `ORCH_FVT_PXEBOOT_V025` | `test_slurm_services` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify role and feature-specific Slurm services. | All stated checks pass for every applicable target. | +| 41104 | `ORCH_FVT_PXEBOOT_V026` | `test_slurm_cross_node_ssh` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify every mapped Slurm role can reach every peer over root SSH. | All stated checks pass for every applicable target. | +| 41105 | `ORCH_FVT_PXEBOOT_V027` | `test_slurm_configless_mode` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify configless controller access and expected cluster identity. | All stated checks pass for every applicable target. | +| 41106 | `ORCH_FVT_PXEBOOT_V028` | `test_slurm_configuration_consistency` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Compare authoritative Slurm files with every configless client cache. | Every compared value matches its authoritative source. | +| 41107 | `ORCH_FVT_PXEBOOT_V029` | `test_slurm_reconfigure` | `slurm_cluster` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Reconfigure Slurm and verify membership remains healthy. | All stated checks pass for every applicable target. | +| 41108 | `ORCH_FVT_PXEBOOT_V030` | `test_slurm_hardware_discovery` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify runtime hardware matches the configured discovery strategy. | All stated checks pass for every applicable target. | +| 41109 | `ORCH_FVT_PXEBOOT_V031` | `test_slurm_custom_configuration` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify custom values, NFS delivery, and effective visibility. | All stated checks pass for every applicable target. | + +### Slurm jobs (`slurm_jobs`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41201 | `ORCH_FVT_PXEBOOT_V032` | `test_slurm_control_node_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every Slurm control node. | The operation completes successfully and returns the expected result. | +| 41202 | `ORCH_FVT_PXEBOOT_V033` | `test_slurm_login_node_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every mapped login node. | The operation completes successfully and returns the expected result. | +| 41203 | `ORCH_FVT_PXEBOOT_V034` | `test_slurm_compiler_node_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every login compiler node. | The operation completes successfully and returns the expected result. | +| 41204 | `ORCH_FVT_PXEBOOT_V035` | `test_slurm_concurrent_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Submit concurrent jobs and verify final accounting state. | The submission completes with the expected final state and output. | +| 41205 | `ORCH_FVT_PXEBOOT_V036` | `test_slurm_insufficient_resources` | `slurm_jobs` | `buildstream`, `functional`, `negative`, `non_disruptive`, `slurm` | Verify an impossible immediate allocation is rejected. | The expected rejection occurs and no prohibited state is accepted. | +| 41206 | `ORCH_FVT_PXEBOOT_V037` | `test_slurm_job_queueing` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Saturate idle computes and verify one follower queues then completes. | The follower is pending under saturation and completes after resources are released. | +| 41207 | `ORCH_FVT_PXEBOOT_V038` | `test_slurm_drain_queue_recovery` | `slurm_jobs` | `buildstream`, `sanity`, `scheduler_state`, `slurm` | Drain one compute node, verify queuing, and restore it. | The job queues while the node is drained, then the node is resumed and the job completes. | + +### Slurm LDAP (`slurm_ldap`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41301 | `ORCH_FVT_PXEBOOT_V524` | `test_external_ldap_proxy` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Reconcile and verify the explicitly enabled LDAP meta-proxy. | Reconciliation is idempotent and every postcondition passes. | +| 41302 | `ORCH_FVT_PXEBOOT_V525` | `test_external_ldap_backend` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify external LDAP reachability from the omnia_auth container. | All stated checks pass for every applicable target. | +| 41303 | `ORCH_FVT_PXEBOOT_V039` | `test_slurm_pam_policy` | `slurm_ldap` | `buildstream`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify SSHD, the PAM module, and pam_slurm_adopt account policy. | All stated checks pass for every applicable target. | +| 41304 | `ORCH_FVT_PXEBOOT_V040` | `test_slurm_control_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on the Slurm control node. | All stated checks pass for every applicable target. | +| 41305 | `ORCH_FVT_PXEBOOT_V041` | `test_slurm_control_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `slurm` | Verify an invalid LDAP password is rejected on the control node. | The expected rejection occurs and no prohibited state is accepted. | +| 41306 | `ORCH_FVT_PXEBOOT_V042` | `test_slurm_login_ldap_authentication` | `slurm_ldap` | `buildstream`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every mapped login node. | All stated checks pass for every applicable target. | +| 41307 | `ORCH_FVT_PXEBOOT_V043` | `test_slurm_login_ldap_invalid_password` | `slurm_ldap` | `buildstream`, `negative`, `non_disruptive`, `openldap`, `slurm` | Verify an invalid LDAP password is rejected on every login node. | The expected rejection occurs and no prohibited state is accepted. | +| 41308 | `ORCH_FVT_PXEBOOT_V044` | `test_slurm_compiler_ldap_authentication` | `slurm_ldap` | `buildstream`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every login-compiler node. | All stated checks pass for every applicable target. | +| 41309 | `ORCH_FVT_PXEBOOT_V045` | `test_slurm_compiler_ldap_invalid_password` | `slurm_ldap` | `buildstream`, `negative`, `non_disruptive`, `openldap`, `slurm` | Verify invalid LDAP passwords are rejected on login-compiler nodes. | The expected rejection occurs and no prohibited state is accepted. | +| 41310 | `ORCH_FVT_PXEBOOT_V046` | `test_slurm_pam_no_job_access` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `slurm` | Verify LDAP compute login is denied without an active job. | The expected rejection occurs and no prohibited state is accepted. | +| 41311 | `ORCH_FVT_PXEBOOT_V047` | `test_slurm_invalid_ldap_identity` | `slurm_ldap` | `buildstream`, `negative`, `non_disruptive`, `openldap`, `slurm` | Verify a generated missing directory identity is rejected. | The expected rejection occurs and no prohibited state is accepted. | +| 41312 | `ORCH_FVT_PXEBOOT_V048` | `test_slurm_control_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from the control node. | The submission completes with the expected final state and output. | +| 41313 | `ORCH_FVT_PXEBOOT_V049` | `test_slurm_control_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify control-submitted PAM access during and after a job. | All stated checks pass for every applicable target. | +| 41314 | `ORCH_FVT_PXEBOOT_V050` | `test_slurm_login_ldap_jobs` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from every login node. | The submission completes with the expected final state and output. | +| 41315 | `ORCH_FVT_PXEBOOT_V051` | `test_slurm_login_pam_job_access` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-node PAM access during and after a job. | All stated checks pass for every applicable target. | +| 41316 | `ORCH_FVT_PXEBOOT_V052` | `test_slurm_compiler_ldap_jobs` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit an LDAP-owned job from every login-compiler node. | The submission completes with the expected final state and output. | +| 41317 | `ORCH_FVT_PXEBOOT_V053` | `test_slurm_compiler_pam_job_access` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-compiler PAM access during and after a job. | All stated checks pass for every applicable target. | + +The two external LDAP cases run first because the LDAP authentication and +job cases depend on the proxied directory. Every case that logs in or runs +jobs as the LDAP test identity skips unless OpenLDAP is enabled and +`validate_external_ldap: true`. `configure_external_ldap` controls only +whether the proxy case may reconcile `slapd.conf`; with it false, the case +checks the existing deployed proxy and reports mismatches as failures. +`test_slurm_pam_policy` verifies only the PAM/SSHD configuration and runs +whenever OpenLDAP is enabled. + +### Slurm GPU (`slurm_gpu`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41401 | `ORCH_FVT_PXEBOOT_V054` | `test_slurm_gpu_inventory` | `slurm_gpu` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA runtime state on scheduler-declared GPU nodes. | All stated checks pass for every applicable target. | +| 41402 | `ORCH_FVT_PXEBOOT_V055` | `test_slurm_gpu_job` | `slurm_gpu` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Allocate a GPU through Slurm and query the device. | The allocation succeeds and the requested device is visible. | +| 41403 | `ORCH_FVT_PXEBOOT_V056` | `test_slurm_gpu_memory_stress` | `slurm_gpu` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Compile and run a bounded GPU memory workload through Slurm. | Compilation and bounded workload execution complete successfully. | + +### DCGM / CUDA (`slurm_dcgm`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41501 | `ORCH_FVT_PXEBOOT_V522` | `test_dcgm_cuda_validation` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA driver and CUDA toolkit are installed on GPU nodes. | All stated checks pass for every applicable target. | +| 41502 | `ORCH_FVT_PXEBOOT_V502` | `test_dcgm_cuda_atomic_lock` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit is installed to /hpc_tools/cuda via atomic lock. | All stated checks pass for every applicable target. | +| 41503 | `ORCH_FVT_PXEBOOT_V508` | `test_dcgm_cuda_compute_node` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify both CUDA toolkit and CUDA driver on compute nodes. | All stated checks pass for every applicable target. | +| 41504 | `ORCH_FVT_PXEBOOT_V507` | `test_dcgm_cuda_login_compiler` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit accessible on login_compiler nodes. | All stated checks pass for every applicable target. | +| 41505 | `ORCH_FVT_PXEBOOT_V503` | `test_dcgm_package_installed` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify datacenter-gpu-manager RPM and dcgmi binary on GPU nodes. | All stated checks pass for every applicable target. | +| 41506 | `ORCH_FVT_PXEBOOT_V504` | `test_dcgm_daemon_running` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify nvidia-dcgm service is active and enabled on GPU nodes. | All stated checks pass for every applicable target. | +| 41507 | `ORCH_FVT_PXEBOOT_V505` | `test_dcgm_gpu_discovery` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery enumerates GPUs with unique UUIDs. | All stated checks pass for every applicable target. | +| 41508 | `ORCH_FVT_PXEBOOT_V506` | `test_dcgm_gpu_metrics` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify dcgmi dmon returns metric samples for each GPU node. | All stated checks pass for every applicable target. | +| 41509 | `ORCH_FVT_PXEBOOT_V509` | `test_dcgm_multi_gpu_discovery` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery on multi-GPU nodes. | All stated checks pass for every applicable target. | +| 41510 | `ORCH_FVT_PXEBOOT_V510` | `test_dcgm_multi_gpu_no_login_compiler` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify GPU nodes work without login_compiler present. | All stated checks pass for every applicable target. | +| 41511 | `ORCH_FVT_PXEBOOT_V511` | `test_dcgm_multi_login_compiler_lock` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit install uses atomic lock with multiple login_compilers. | All stated checks pass for every applicable target. | +| 41512 | `ORCH_FVT_PXEBOOT_V512` | `test_dcgm_toolkit_nfs_storage` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify /hpc_tools is NFS-mounted and CUDA toolkit accessible. | All stated checks pass for every applicable target. | +| 41513 | `ORCH_FVT_PXEBOOT_V513` | `test_dcgm_rhel_compatibility` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify GPU node OS is a supported RHEL version. | All stated checks pass for every applicable target. | +| 41514 | `ORCH_FVT_PXEBOOT_V514` | `test_dcgm_cuda_version_compatibility` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit and DCGM daemon version compatibility. | All stated checks pass for every applicable target. | +| 41515 | `ORCH_FVT_PXEBOOT_V515` | `test_dcgm_neg_cuda_prerequisite` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Verify DCGM deployment requires CUDA driver as a prerequisite. | All stated checks pass for every applicable target. | +| 41516 | `ORCH_FVT_PXEBOOT_V516` | `test_dcgm_neg_daemon_recovery` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Simulate DCGM daemon crash via SIGKILL and verify systemd restarts it. | The node returns within the bounded wait and every stated postcondition check passes. | +| 41517 | `ORCH_FVT_PXEBOOT_V517` | `test_dcgm_neg_socket_inaccessible` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Remove DCGM Unix socket and verify dcgmi returns a clear error. | The expected rejection occurs and no prohibited state is accepted. | +| 41518 | `ORCH_FVT_PXEBOOT_V518` | `test_dcgm_neg_package_install_failure` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Verify error handling when datacenter-gpu-manager package is unavailable. | The expected rejection occurs and no prohibited state is accepted. | + +DCGM cases skip when no GPU GRES nodes are found in the Slurm inventory. + +### Slurm OpenMPI (`slurm_openmpi`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41601 | `ORCH_FVT_PXEBOOT_V057` | `test_slurm_openmpi_installation` | `slurm_openmpi` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify OpenMPI discovery and version on every compute node. | All stated checks pass for every applicable target. | +| 41602 | `ORCH_FVT_PXEBOOT_V058` | `test_slurm_openmpi_job` | `slurm_openmpi` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run an OpenMPI-backed job when OpenMPI is configured. | The operation completes successfully and returns the expected result. | + +### Slurm UCX (`slurm_ucx`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41701 | `ORCH_FVT_PXEBOOT_V059` | `test_slurm_ucx_transport` | `slurm_ucx` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify UCX exposes an InfiniBand-capable transport. | All stated checks pass for every applicable target. | + +### Slurm InfiniBand (`slurm_infiniband`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41801 | `ORCH_FVT_PXEBOOT_V060` | `test_slurm_infiniband_configuration` | `slurm_infiniband` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify mapped IB interface, address, prefix, link, MTU, and OFED. | All stated checks pass for every applicable target. | +| 41802 | `ORCH_FVT_PXEBOOT_V061` | `test_slurm_infiniband_connectivity` | `slurm_infiniband` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify every mapped IB endpoint can reach every mapped peer. | All stated checks pass for every applicable target. | + +### Slurm Apptainer (`slurm_apptainer`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41901 | `ORCH_FVT_PXEBOOT_V063` | `test_apptainer_runtime` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify the Apptainer executable and version on every compute node. | All stated checks pass for every applicable target. | +| 41902 | `ORCH_FVT_PXEBOOT_V064` | `test_apptainer_shared_artifacts` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify shared image directories and downloader artifacts. | All stated checks pass for every applicable target. | +| 41903 | `ORCH_FVT_PXEBOOT_V065` | `test_apptainer_pulp_policy` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify the generated downloader uses only the configured Pulp source. | All stated checks pass for every applicable target. | +| 41904 | `ORCH_FVT_PXEBOOT_V066` | `test_apptainer_shared_storage` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify /hpc_tools is a shared mounted filesystem on every compute. | All stated checks pass for every applicable target. | +| 41905 | `ORCH_FVT_PXEBOOT_V067` | `test_apptainer_download` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the deployed downloader and require at least one usable SIF. | The operation completes successfully and returns the expected result. | +| 41906 | `ORCH_FVT_PXEBOOT_V068` | `test_apptainer_download_idempotency` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `image_download`, `non_disruptive`, `sanity` | Rerun the downloader and verify existing image metadata is unchanged. | The repeated operation succeeds without changing protected state. | +| 41907 | `ORCH_FVT_PXEBOOT_V069` | `test_apptainer_download_memory` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the downloader and enforce a bounded peak resident-memory use. | The operation completes successfully and returns the expected result. | +| 41908 | `ORCH_FVT_PXEBOOT_V070` | `test_apptainer_image_inventory` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify every compute sees one consistent non-empty SIF inventory. | All stated checks pass for every applicable target. | +| 41909 | `ORCH_FVT_PXEBOOT_V071` | `test_apptainer_sif_format` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify each discovered image is a valid inspectable SIF. | All stated checks pass for every applicable target. | +| 41910 | `ORCH_FVT_PXEBOOT_V072` | `test_apptainer_sif_permissions` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify shared SIF files are non-empty and world-readable. | All stated checks pass for every applicable target. | +| 41911 | `ORCH_FVT_PXEBOOT_V073` | `test_apptainer_sif_integrity` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify the selected SIF has the same checksum on every compute. | All stated checks pass for every applicable target. | +| 41912 | `ORCH_FVT_PXEBOOT_V074` | `test_apptainer_ldap_readability` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the LDAP test identity can read a shared SIF when enabled. | All stated checks pass for every applicable target. | +| 41913 | `ORCH_FVT_PXEBOOT_V075` | `test_apptainer_non_root_execution` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify an unprivileged local identity can execute a shared SIF. | All stated checks pass for every applicable target. | +| 41914 | `ORCH_FVT_PXEBOOT_V076` | `test_apptainer_missing_image_contract` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify the downloader records pull failures and exits non-zero. | The expected rejection occurs and no prohibited state is accepted. | +| 41915 | `ORCH_FVT_PXEBOOT_V077` | `test_apptainer_single_node_job` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Run one exact-node container job on every mapped compute. | The operation completes successfully and returns the expected result. | +| 41916 | `ORCH_FVT_PXEBOOT_V078` | `test_apptainer_multi_node_job` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Run one container allocation spanning all mapped computes. | The operation completes successfully and returns the expected result. | +| 41917 | `ORCH_FVT_PXEBOOT_V079` | `test_apptainer_ldap_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run targeted container jobs as the configured LDAP test identity. | The operation completes successfully and returns the expected result. | +| 41918 | `ORCH_FVT_PXEBOOT_V080` | `test_apptainer_concurrent_jobs` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Run bounded concurrent container jobs on distinct computes. | The operation completes successfully and returns the expected result. | +| 41919 | `ORCH_FVT_PXEBOOT_V081` | `test_apptainer_invalid_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a nonexistent SIF fails through the Slurm execution path. | The expected rejection occurs and no prohibited state is accepted. | +| 41920 | `ORCH_FVT_PXEBOOT_V082` | `test_apptainer_restricted_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify an unprivileged identity cannot execute a mode-0600 SIF. | The expected rejection occurs and no prohibited state is accepted. | +| 41921 | `ORCH_FVT_PXEBOOT_V083` | `test_apptainer_nfs_visibility` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify containers can read the shared image path on every compute. | All stated checks pass for every applicable target. | +| 41922 | `ORCH_FVT_PXEBOOT_V084` | `test_apptainer_slurm_environment` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify Slurm allocation variables propagate into containers. | All stated checks pass for every applicable target. | +| 41923 | `ORCH_FVT_PXEBOOT_V085` | `test_apptainer_job_array` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Submit and wait for a bounded Apptainer Slurm job array. | The submission completes with the expected final state and output. | +| 41924 | `ORCH_FVT_PXEBOOT_V086` | `test_apptainer_failure_cleanup` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a failed image launch leaves no matching runtime process. | The expected rejection occurs and no prohibited state is accepted. | +| 41925 | `ORCH_FVT_PXEBOOT_V087` | `test_apptainer_gpu_access` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify scheduler-declared GPU nodes expose GPUs in the container. | All stated checks pass for every applicable target. | +| 41926 | `ORCH_FVT_PXEBOOT_V088` | `test_apptainer_gpu_count` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify each container sees the same GPU count as its host. | All stated checks pass for every applicable target. | +| 41927 | `ORCH_FVT_PXEBOOT_V089` | `test_apptainer_cuda_workload` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Execute a bounded NVIDIA device query in each GPU container. | The operation completes successfully and returns the expected result. | +| 41928 | `ORCH_FVT_PXEBOOT_V090` | `test_apptainer_gpu_memory` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify the GPU query leaves no material device-memory allocation. | All stated checks pass for every applicable target. | +| 41929 | `ORCH_FVT_PXEBOOT_V091` | `test_apptainer_infiniband` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify mapped compute nodes expose InfiniBand devices in containers. | All stated checks pass for every applicable target. | +| 41930 | `ORCH_FVT_PXEBOOT_V092` | `test_apptainer_reboot_storage` | `slurm_apptainer` | `apptainer`, `reboot` | Reboot one compute and verify the shared mount and SIF checksum. | The node returns within the bounded wait and every stated post-reboot check passes. | +| 41931 | `ORCH_FVT_PXEBOOT_V093` | `test_apptainer_reboot_job` | `slurm_apptainer` | `apptainer`, `reboot` | Run an exact-node container job after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | +| 41932 | `ORCH_FVT_PXEBOOT_V094` | `test_apptainer_reboot_artifacts` | `slurm_apptainer` | `apptainer`, `reboot` | Verify downloader artifacts and policy after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | Image download has a 20-minute ceiling with polling progress every 20 -seconds. The reboot cases share one reboot state instead of rebooting the -same compute node independently for each postcondition. - -### Slurm HPC Benchmarks - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 295 | `ORCH_FVT_PXEBOOT_V095` | `test_hpc_benchmarks_prerequisites` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `functional`, `benchmark`, `slurm` | Verify script presence/executability, platform helper, tool list, and /hpc_tools mount (NFS export check is informational). | Every mapped compute passes before downloads begin. | -| 296 | `ORCH_FVT_PXEBOOT_V096` | `test_hpc_benchmarks_staging_fingerprint_idempotency` | `slurm_hpc_benchmarks` | `non_disruptive`, `functional`, `benchmark`, `slurm` | Pull all applicable tools per live platform; validate tarballs and repeat. | Initial downloads succeed; the rerun performs zero downloads and preserves SHA-256, size, and mtime. | -| 297 | `ORCH_FVT_PXEBOOT_V097` | `test_hpc_benchmarks_concurrent_staging` | `slurm_hpc_benchmarks` | `non_disruptive`, `functional`, `benchmark`, `slurm` | Start two same-platform compute pulls together on shared storage. | One process downloads each tool once; both see the same archives. | -| 330 | `ORCH_FVT_PXEBOOT_V223` | `test_hpc_benchmarks_run_staging` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Execute pull_benchmarks.sh to populate staged artifacts before verification TCs. | All stated checks pass for every applicable target. | -| 331 | `ORCH_FVT_PXEBOOT_V200` | `test_hpc_benchmarks_json_declaration` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify benchmark_tools.list JSON syntax and required fields. | All stated checks pass for every applicable target. | -| 332 | `ORCH_FVT_PXEBOOT_V201` | `test_hpc_benchmarks_local_repo_sync` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify offline repo sync status on the OIM. | All stated checks pass for every applicable target. | -| 333 | `ORCH_FVT_PXEBOOT_V202` | `test_hpc_benchmarks_tools_dir_creation` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify /hpc_tools directory layout and permissions. | All stated checks pass for every applicable target. | -| 334 | `ORCH_FVT_PXEBOOT_V203` | `test_hpc_benchmarks_artifact_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify declared benchmark artifacts are staged per tool. | All stated checks pass for every applicable target. | -| 335 | `ORCH_FVT_PXEBOOT_V219` | `test_hpc_benchmarks_platform_script` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify omnia_platform.sh deployment and platform detection. | All stated checks pass for every applicable target. | -| 336 | `ORCH_FVT_PXEBOOT_V220` | `test_hpc_benchmarks_container_image_list` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify container_image.list deployment and content validation. | All stated checks pass for every applicable target. | -| 337 | `ORCH_FVT_PXEBOOT_V221` | `test_hpc_benchmarks_platform_directory_structure` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify platform-specific directory structure exists per architecture. | All stated checks pass for every applicable target. | -| 338 | `ORCH_FVT_PXEBOOT_V222` | `test_hpc_benchmarks_offline_package_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify offline packages are copied to slurm_config_path/packages/{arch}/. | All stated checks pass for every applicable target. | -| 339 | `ORCH_FVT_PXEBOOT_V204` | `test_hpc_benchmarks_msr_safe_arch_boundary` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify msr-safe is staged only for x86_64. | All stated checks pass for every applicable target. | -| 340 | `ORCH_FVT_PXEBOOT_V211` | `test_hpc_benchmarks_post_staging_validation` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify post-staging validation of benchmark tool directories. | All stated checks pass for every applicable target. | -| 341 | `ORCH_FVT_PXEBOOT_V212` | `test_hpc_benchmarks_rhel_compatibility` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify benchmark staging on RHEL 10.x. | All stated checks pass for every applicable target. | -| 342 | `ORCH_FVT_PXEBOOT_V205` | `test_hpc_benchmarks_container_first_guidance` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify pull_benchmarks.sh and benchmark_tools.list are deployed. | All stated checks pass for every applicable target. | -| 343 | `ORCH_FVT_PXEBOOT_V206` | `test_hpc_benchmarks_source_only_delivery` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify no compile or build commands are staged. | All stated checks pass for every applicable target. | -| 344 | `ORCH_FVT_PXEBOOT_V209` | `test_hpc_benchmarks_nfs_accessibility` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify /hpc_tools NFS is mounted and readable on compute nodes. | All stated checks pass for every applicable target. | -| 345 | `ORCH_FVT_PXEBOOT_V208` | `test_hpc_benchmarks_e2e_provisioning` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify end-to-end benchmark provisioning pipeline. | All stated checks pass for every applicable target. | -| 346 | `ORCH_FVT_PXEBOOT_V224` | `test_hpc_benchmarks_cleanup_staging` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Remove staged benchmark tool directories after verification (controlled by cleanup_benchmark_tools in test_config.yml). | All stated checks pass for every applicable target. | -| 347 | `ORCH_FVT_PXEBOOT_V213` | `test_hpc_benchmarks_cuda_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify CUDA flow is unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 348 | `ORCH_FVT_PXEBOOT_V214` | `test_hpc_benchmarks_nvhpc_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify NVIDIA HPC SDK flow is unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 349 | `ORCH_FVT_PXEBOOT_V215` | `test_hpc_benchmarks_container_image_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify container image flow is unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 350 | `ORCH_FVT_PXEBOOT_V216` | `test_hpc_benchmarks_openmpi_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify OpenMPI/UCX are unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 351 | `ORCH_FVT_PXEBOOT_V210` | `test_hpc_benchmarks_airgapped_staging` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify benchmark staging completes without external egress. | The operation completes successfully and returns the expected result. | -| 352 | `ORCH_FVT_PXEBOOT_V217` | `test_hpc_benchmarks_existing_dirs_preserved` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify pre-existing /hpc_tools directories are preserved. | The operation completes successfully and returns the expected result. | -| 353 | `ORCH_FVT_PXEBOOT_V218` | `test_hpc_benchmarks_staging_idempotency` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify benchmark staging is idempotent. | The operation completes successfully and returns the expected result. | - -### CoreDNS/CoreDHCP - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 350 | `ORCH_FVT_PXEBOOT_V300` | `test_coredns_container_state` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify coresmd containers run with expected image; observe dns_enabled. | All stated checks pass for every applicable target. | -| 351 | `ORCH_FVT_PXEBOOT_V301` | `test_coredns_forward_resolution` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Query CoreDNS on the OIM for every SMD-mapped node. | All stated checks pass for every applicable target. | -| 352 | `ORCH_FVT_PXEBOOT_V302` | `test_coredns_reverse_resolution` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Query CoreDNS on the OIM for PTR records. | All stated checks pass for every applicable target. | -| 353 | `ORCH_FVT_PXEBOOT_V303` | `test_coredns_dns_forwarders` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify DNS forwarders are configured and can resolve external domains. | All stated checks pass for every applicable target. | -| 354 | `ORCH_FVT_PXEBOOT_V304` | `test_coredns_corefile_config` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify Corefile configuration is correctly rendered. | All stated checks pass for every applicable target. | -| 355 | `ORCH_FVT_PXEBOOT_V305` | `test_coredhcp_config_file` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify coredhcp.yaml configuration file is correctly rendered. | All stated checks pass for every applicable target. | -| 356 | `ORCH_FVT_PXEBOOT_V306` | `test_coredhcp_multisubnet_running_image` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify the running coresmd-coredhcp image on multi-subnet datasets. | All stated checks pass for every applicable target. | -| 357 | `ORCH_FVT_PXEBOOT_V307` | `test_dns_compute_resolv_conf` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify every mapped compute node has CoreDNS as primary nameserver. | All stated checks pass for every applicable target. | -| 358 | `ORCH_FVT_PXEBOOT_V308` | `test_dns_compute_forward_getent` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Run `getent hosts` on every compute for SMD-derived candidate FQDNs. | All stated checks pass for every applicable target. | -| 359 | `ORCH_FVT_PXEBOOT_V309` | `test_coredns_idempotency` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify CoreDNS/CoreDHCP state stability (no-drift). | State is identical across the settle window with no spontaneous changes. | -| 360 | `ORCH_FVT_PXEBOOT_V310` | `test_dns_node_addition_pipeline` | `coredns_coredhcp` | `destructive`, `sanity` | Prove the SMD-to-CoreDNS pipeline resolves every SMD-registered mapped node. | The operation completes successfully and returns the expected result. | -| 361 | `ORCH_FVT_PXEBOOT_V311` | `test_dns_smd_unreachable_cached_resolution` | `coredns_coredhcp` | `destructive`, `sanity` | Pause the SMD container briefly; require CoreDNS to keep serving cached. | The node returns within the bounded wait and every stated postcondition check passes. | - -### PowerVault iSCSI Storage - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 301 | `ORCH_FVT_PXEBOOT_V400` | `test_powervault_iscsi_service` | `powervault` | `non_disruptive`, `sanity` | Verify iscsid is active and enabled on all PowerVault target nodes. | All stated checks pass for every applicable target. | -| 302 | `ORCH_FVT_PXEBOOT_V401` | `test_powervault_iscsi_initiator_name` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI initiator name matches config on all target nodes. | All stated checks pass for every applicable target. | -| 303 | `ORCH_FVT_PXEBOOT_V402` | `test_powervault_iscsi_discovery` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI target discovery succeeds from all portal IPs. | All stated checks pass for every applicable target. | -| 304 | `ORCH_FVT_PXEBOOT_V403` | `test_powervault_iscsi_sessions` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI sessions are active on all target nodes. | All stated checks pass for every applicable target. | -| 305 | `ORCH_FVT_PXEBOOT_V404` | `test_powervault_iscsi_startup_automatic` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI node startup is automatic on all target nodes. | All stated checks pass for every applicable target. | -| 306 | `ORCH_FVT_PXEBOOT_V405` | `test_powervault_portal_reachability` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI portal ports are reachable and sessions healthy. | All stated checks pass for every applicable target. | -| 307 | `ORCH_FVT_PXEBOOT_V406` | `test_powervault_multipath_service` | `powervault` | `non_disruptive`, `sanity` | Verify multipathd is active and enabled on all target nodes. | All stated checks pass for every applicable target. | -| 308 | `ORCH_FVT_PXEBOOT_V407` | `test_powervault_multipath_device` | `powervault` | `non_disruptive`, `sanity` | Verify multipath device exists and matches volume_id. | All stated checks pass for every applicable target. | -| 309 | `ORCH_FVT_PXEBOOT_V408` | `test_powervault_multipath_redundancy` | `powervault` | `non_disruptive`, `sanity` | Verify multipath device has multiple paths for redundancy. | All stated checks pass for every applicable target. | -| 310 | `ORCH_FVT_PXEBOOT_V409` | `test_powervault_gpt_partition` | `powervault` | `non_disruptive`, `sanity` | Verify GPT partition exists on multipath device. | All stated checks pass for every applicable target. | -| 311 | `ORCH_FVT_PXEBOOT_V410` | `test_powervault_filesystem_type` | `powervault` | `non_disruptive`, `sanity` | Verify filesystem formatted with correct type. | All stated checks pass for every applicable target. | -| 312 | `ORCH_FVT_PXEBOOT_V411` | `test_powervault_mount_point_directory` | `powervault` | `non_disruptive`, `sanity` | Verify mount point directory exists on all target nodes. | All stated checks pass for every applicable target. | -| 313 | `ORCH_FVT_PXEBOOT_V412` | `test_powervault_volume_mounted` | `powervault` | `non_disruptive`, `sanity` | Verify PowerVault volume is actively mounted on all target nodes. | All stated checks pass for every applicable target. | -| 314 | `ORCH_FVT_PXEBOOT_V413` | `test_powervault_mount_options` | `powervault` | `non_disruptive`, `sanity` | Verify mount options applied correctly on all target nodes. | All stated checks pass for every applicable target. | -| 315 | `ORCH_FVT_PXEBOOT_V414` | `test_powervault_fstab_entry` | `powervault` | `non_disruptive`, `sanity` | Verify persistent fstab entry created on all target nodes. | All stated checks pass for every applicable target. | -| 316 | `ORCH_FVT_PXEBOOT_V415` | `test_powervault_node_subdirectory` | `powervault` | `non_disruptive`, `sanity` | Verify per-node subdirectory exists under mount point. | All stated checks pass for every applicable target. | -| 317 | `ORCH_FVT_PXEBOOT_V416` | `test_powervault_bind_mounts` | `powervault` | `non_disruptive`, `sanity` | Verify bind mount targets are active on all target nodes. | All stated checks pass for every applicable target. | -| 318 | `ORCH_FVT_PXEBOOT_V417` | `test_powervault_bind_fstab_entries` | `powervault` | `non_disruptive`, `sanity` | Verify bind mount fstab entries are persistent on all target nodes. | All stated checks pass for every applicable target. | -| 319 | `ORCH_FVT_PXEBOOT_V418` | `test_powervault_bind_isolation` | `powervault` | `non_disruptive`, `sanity` | Verify per-node data separation via bind mounts. | All stated checks pass for every applicable target. | -| 320 | `ORCH_FVT_PXEBOOT_V419` | `test_powervault_functional_group_targeting` | `powervault` | `non_disruptive`, `sanity` | Verify PV mount only on correct functional groups. | All stated checks pass for every applicable target. | -| 321 | `ORCH_FVT_PXEBOOT_V420` | `test_powervault_multiple_prefix_targeting` | `powervault` | `non_disruptive`, `sanity` | Verify multiple prefixes target all groups correctly. | All stated checks pass for every applicable target. | -| 322 | `ORCH_FVT_PXEBOOT_V421` | `test_powervault_setup_log` | `powervault` | `non_disruptive`, `sanity` | Verify cloud-init runcmd log exists and shows completion. | All stated checks pass for every applicable target. | -| 323 | `ORCH_FVT_PXEBOOT_V422` | `test_powervault_cloud_init_groups_dict` | `powervault` | `non_disruptive`, `sanity` | Verify rendered iSCSI setup scripts deployed on target nodes. | All stated checks pass for every applicable target. | -| 324 | `ORCH_FVT_PXEBOOT_V423` | `test_powervault_no_duplicate_fstab` | `powervault` | `non_disruptive`, `sanity` | Verify no duplicate fstab entries on all target nodes. | All stated checks pass for every applicable target. | -| 325 | `ORCH_FVT_PXEBOOT_V424` | `test_powervault_all_mounts_writable` | `powervault` | `non_disruptive`, `sanity` | Verify all PV mounts (main + bind) are writable. | All stated checks pass for every applicable target. | -| 326 | `ORCH_FVT_PXEBOOT_V425` | `test_powervault_permissions` | `powervault` | `non_disruptive`, `sanity` | Verify permissions on mount point match config. | All stated checks pass for every applicable target. | -| 327 | `ORCH_FVT_PXEBOOT_V426` | `test_powervault_io_write_read` | `powervault` | `non_disruptive`, `sanity` | Verify write-read I/O on PV mount points. | All stated checks pass for every applicable target. | -| 328 | `ORCH_FVT_PXEBOOT_V427` | `test_powervault_bind_io` | `powervault` | `non_disruptive`, `sanity` | Verify bind-mount I/O reaches PV backing store. | All stated checks pass for every applicable target. | -| 329 | `ORCH_FVT_PXEBOOT_V428` | `test_powervault_slurm_mandatory_bind_mounts` | `powervault` | `non_disruptive`, `sanity` | Verify /var/lib/mysql and /var/spool/slurm configured as bind targets. | All stated checks pass for every applicable target. | -| 330 | `ORCH_FVT_PXEBOOT_V429` | `test_powervault_mysql_data_on_mount` | `powervault` | `non_disruptive`, `sanity` | Verify MySQL datadir is on PowerVault mount. | All stated checks pass for every applicable target. | - -PowerVault tests skip when `powervault_config` is absent or empty in `storage_config.yml`. The suite validates the iSCSI/multipath/bind-mount contract deployed by `setup_iscsi_storage.sh.j2` and functional_group_prefix targeting. -### Additional cloud-init - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 300 | `ORCH_FVT_PXEBOOT_V095` | `test_additional_cloud_init_smd_groups` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify SMD groups exist for additional cloud-init configuration. | All stated checks pass for every applicable target. | -| 301 | `ORCH_FVT_PXEBOOT_V096` | `test_additional_cloud_init_metadata_groups` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify metadata-service groups and templates for additional cloud-init. | All stated checks pass for every applicable target. | -| 302 | `ORCH_FVT_PXEBOOT_V099` | `test_additional_cloud_init_write_files` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify write_files entries were applied on provisioned nodes. | All stated checks pass for every applicable target. | -| 303 | `ORCH_FVT_PXEBOOT_V100` | `test_additional_cloud_init_runcmd` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify runcmd entries executed during cloud-init on provisioned nodes. | All stated checks pass for every applicable target. | - -Additional cloud-init tests skip automatically when -`additional_cloud_init_config_file` is empty or not configured in -`orchestrator_config.yml`. When enabled, V095 and V096 verify controller-side -state (SMD groups and metadata-service templates); V099 and V100 verify -node-side artifacts (files created by `write_files` and cloud-init completion -confirming `runcmd` execution). - -### DCGM / CUDA - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 501 | `ORCH_FVT_PXEBOOT_V501` | `test_dcgm_cuda_validation` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA driver and CUDA toolkit on GPU nodes. | All stated checks pass for every applicable target. | -| 502 | `ORCH_FVT_PXEBOOT_V502` | `test_dcgm_cuda_atomic_lock` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit installed via atomic lock. | All stated checks pass for every applicable target. | -| 503 | `ORCH_FVT_PXEBOOT_V503` | `test_dcgm_package_installed` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify datacenter-gpu-manager RPM and DCGM binaries. | All stated checks pass for every applicable target. | -| 504 | `ORCH_FVT_PXEBOOT_V504` | `test_dcgm_daemon_running` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify nvidia-dcgm service is active and enabled. | All stated checks pass for every applicable target. | -| 505 | `ORCH_FVT_PXEBOOT_V505` | `test_dcgm_gpu_discovery` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery enumerates GPUs with unique UUIDs. | All stated checks pass for every applicable target. | -| 506 | `ORCH_FVT_PXEBOOT_V506` | `test_dcgm_gpu_metrics` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify dcgmi dmon returns metric samples for each GPU. | All stated checks pass for every applicable target. | -| 507 | `ORCH_FVT_PXEBOOT_V507` | `test_dcgm_cuda_login_compiler` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit accessible on login_compiler nodes. | All stated checks pass for every applicable target. | -| 508 | `ORCH_FVT_PXEBOOT_V508` | `test_dcgm_cuda_compute_node` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit and driver on compute nodes. | All stated checks pass for every applicable target. | -| 509 | `ORCH_FVT_PXEBOOT_V509` | `test_dcgm_multi_gpu_discovery` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery on multi-GPU nodes. | All stated checks pass for every applicable target. | -| 510 | `ORCH_FVT_PXEBOOT_V510` | `test_dcgm_multi_gpu_no_login_compiler` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify GPU nodes work without login_compiler present. | All stated checks pass for every applicable target. | -| 511 | `ORCH_FVT_PXEBOOT_V511` | `test_dcgm_multi_login_compiler_lock` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit install uses atomic lock with multiple login_compilers. | All stated checks pass for every applicable target. | -| 512 | `ORCH_FVT_PXEBOOT_V512` | `test_dcgm_toolkit_nfs_storage` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit is NFS-mounted and accessible. | All stated checks pass for every applicable target. | -| 513 | `ORCH_FVT_PXEBOOT_V513` | `test_dcgm_rhel_compatibility` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify GPU node OS is a supported RHEL version. | All stated checks pass for every applicable target. | -| 514 | `ORCH_FVT_PXEBOOT_V514` | `test_dcgm_cuda_version_compatibility` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit and DCGM daemon version compatibility. | All stated checks pass for every applicable target. | -| 515 | `ORCH_FVT_PXEBOOT_V515` | `test_dcgm_neg_cuda_prerequisite` | `slurm_dcgm` | `negative`, `non_disruptive`, `sanity`, `slurm` | Verify DCGM deployment requires CUDA prerequisites. | All stated checks pass for every applicable target. | -| 516 | `ORCH_FVT_PXEBOOT_V516` | `test_dcgm_neg_daemon_recovery` | `slurm_dcgm` | `destructive`, `sanity`, `slurm` | Verify DCGM daemon auto-recovery after SIGKILL. | The node returns within the bounded wait and every stated postcondition check passes. | -| 517 | `ORCH_FVT_PXEBOOT_V517` | `test_dcgm_neg_socket_inaccessible` | `slurm_dcgm` | `destructive`, `sanity`, `slurm` | Verify dcgmi returns clear error when socket is removed. | The expected rejection occurs and no prohibited state is accepted. | -| 518 | `ORCH_FVT_PXEBOOT_V518` | `test_dcgm_neg_package_install_failure` | `slurm_dcgm` | `negative`, `non_disruptive`, `sanity`, `slurm` | Verify error handling when DCGM package is unavailable. | The expected rejection occurs and no prohibited state is accepted. | - -DCGM tests use platform-aware CUDA toolkit paths resolved via -`omnia_platform.sh`. Compute nodes access the toolkit through the -`/usr/local/cuda` bind mount; the login_compiler accesses it directly -from the NFS-shared platform path. Tests skip when no GPU GRES nodes -are found in the Slurm inventory. +seconds. The reboot cases share one reboot state and run last in the suite. + +### Slurm HPC benchmarks (`slurm_hpc_benchmarks`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42001 | `ORCH_FVT_PXEBOOT_V225` | `test_hpc_benchmarks_prerequisites` | `slurm_hpc_benchmarks` | `benchmark`, `functional`, `non_disruptive`, `sanity`, `slurm` | V225: Verify script presence, platform helper, tool list, and /hpc_tools mount (NFS export check is informational). | Every mapped compute passes before downloads begin. | +| 42002 | `ORCH_FVT_PXEBOOT_V226` | `test_hpc_benchmarks_staging_fingerprint_idempotency` | `slurm_hpc_benchmarks` | `benchmark`, `functional`, `non_disruptive`, `slurm` | V226: Pull all applicable tools per live platform; validate tarballs and repeat. | Initial downloads succeed; the rerun performs zero downloads and preserves SHA-256, size, and mtime. | +| 42003 | `ORCH_FVT_PXEBOOT_V227` | `test_hpc_benchmarks_concurrent_staging` | `slurm_hpc_benchmarks` | `benchmark`, `functional`, `non_disruptive`, `slurm` | V227: Start two same-platform compute pulls together on shared storage. | One process downloads each tool once; both see the same archives. | +| 42004 | `ORCH_FVT_PXEBOOT_V223` | `test_hpc_benchmarks_run_staging` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-00: Execute pull_benchmarks.sh to populate staged artifacts before the verification cases. | All stated checks pass for every applicable target. | +| 42005 | `ORCH_FVT_PXEBOOT_V200` | `test_hpc_benchmarks_json_declaration` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-01: Verify benchmark_tools.list is deployed and non-empty per arch. | All stated checks pass for every applicable target. | +| 42006 | `ORCH_FVT_PXEBOOT_V201` | `test_hpc_benchmarks_local_repo_sync` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-02: Verify each declared tool has files under the Pulp offline URL. | All stated checks pass for every applicable target. | +| 42007 | `ORCH_FVT_PXEBOOT_V202` | `test_hpc_benchmarks_tools_dir_creation` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-03: Verify /hpc_tools directory layout and 0755 permissions. | All stated checks pass for every applicable target. | +| 42008 | `ORCH_FVT_PXEBOOT_V203` | `test_hpc_benchmarks_artifact_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-04: Verify declared benchmark artifacts are staged per tool. | The operation completes successfully and returns the expected result. | +| 42009 | `ORCH_FVT_PXEBOOT_V204` | `test_hpc_benchmarks_msr_safe_arch_boundary` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-05: Verify msr-safe is staged only for x86_64 nodes. | All stated checks pass for every applicable target. | +| 42010 | `ORCH_FVT_PXEBOOT_V211` | `test_hpc_benchmarks_post_staging_validation` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-12: Verify post-staging validation of benchmark tool directories. | All stated checks pass for every applicable target. | +| 42011 | `ORCH_FVT_PXEBOOT_V212` | `test_hpc_benchmarks_rhel_compatibility` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-13: Verify every compute node runs the targeted RHEL major. | All stated checks pass for every applicable target. | +| 42012 | `ORCH_FVT_PXEBOOT_V219` | `test_hpc_benchmarks_platform_script` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-06a: Verify omnia_platform.sh deployment and platform detection. | All stated checks pass for every applicable target. | +| 42013 | `ORCH_FVT_PXEBOOT_V220` | `test_hpc_benchmarks_container_image_list` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06b: Verify container_image.list deployment and content validation. | All stated checks pass for every applicable target. | +| 42014 | `ORCH_FVT_PXEBOOT_V221` | `test_hpc_benchmarks_platform_directory_structure` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06c: Verify platform-specific directory structure exists per architecture. | All stated checks pass for every applicable target. | +| 42015 | `ORCH_FVT_PXEBOOT_V222` | `test_hpc_benchmarks_offline_package_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06d: Verify offline packages are copied to slurm_config_path/packages/{arch}/. | All stated checks pass for every applicable target. | +| 42016 | `ORCH_FVT_PXEBOOT_V205` | `test_hpc_benchmarks_container_first_guidance` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06: Verify pull_benchmarks.sh and benchmark_tools.list are deployed. | All stated checks pass for every applicable target. | +| 42017 | `ORCH_FVT_PXEBOOT_V206` | `test_hpc_benchmarks_source_only_delivery` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-07: Verify no compile/build commands are staged with the artifacts. | All stated checks pass for every applicable target. | +| 42018 | `ORCH_FVT_PXEBOOT_V209` | `test_hpc_benchmarks_nfs_accessibility` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-10: Verify /hpc_tools NFS is mounted and readable on compute nodes. | All stated checks pass for every applicable target. | +| 42019 | `ORCH_FVT_PXEBOOT_V208` | `test_hpc_benchmarks_e2e_provisioning` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-09: Verify the end-to-end benchmark provisioning pipeline. | All stated checks pass for every applicable target. | +| 42020 | `ORCH_FVT_PXEBOOT_V224` | `test_hpc_benchmarks_cleanup_staging` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-00b: Remove staged benchmark tool directories after verification (controlled by `cleanup_benchmark_tools` in `test_config.yml`). | All stated checks pass for every applicable target. | +| 42021 | `ORCH_FVT_PXEBOOT_V213` | `test_hpc_benchmarks_cuda_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-14: Verify /hpc_tools/cuda is unchanged after benchmark staging. | All stated checks pass for every applicable target. | +| 42022 | `ORCH_FVT_PXEBOOT_V214` | `test_hpc_benchmarks_nvhpc_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-15: Verify /hpc_tools/nvidia_sdk is unchanged after staging. | All stated checks pass for every applicable target. | +| 42023 | `ORCH_FVT_PXEBOOT_V215` | `test_hpc_benchmarks_container_image_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-16: Verify /hpc_tools/container_images is unchanged after staging. | All stated checks pass for every applicable target. | +| 42024 | `ORCH_FVT_PXEBOOT_V216` | `test_hpc_benchmarks_openmpi_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-17: Verify OpenMPI/UCX discovery is stable across a staging run. | All stated checks pass for every applicable target. | +| 42025 | `ORCH_FVT_PXEBOOT_V210` | `test_hpc_benchmarks_airgapped_staging` | `slurm_hpc_benchmarks` | `destructive`, `slurm` | TC-11: Verify staging succeeds while external egress is unavailable. | All stated checks pass for every applicable target. | +| 42026 | `ORCH_FVT_PXEBOOT_V217` | `test_hpc_benchmarks_existing_dirs_preserved` | `slurm_hpc_benchmarks` | `destructive`, `slurm` | TC-18: Verify pre-existing /hpc_tools subdirs survive a staging run. | The operation completes successfully and returns the expected result. | +| 42027 | `ORCH_FVT_PXEBOOT_V218` | `test_hpc_benchmarks_staging_idempotency` | `slurm_hpc_benchmarks` | `destructive`, `slurm` | TC-19: Verify a second staging run keeps the /hpc_tools snapshot stable. | The repeated operation succeeds without changing protected state. | + +### VAST storage (`vast_storage`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42101 | `ORCH_FVT_PXEBOOT_V700` | `test_vast_vastnfs_installation` | `vast_storage` | `sanity`, `vast_installation` | Verify vastnfs-ctl status on compute nodes. | All stated checks pass for every applicable target. | +| 42102 | `ORCH_FVT_PXEBOOT_V701` | `test_vast_mount_points` | `vast_storage` | `sanity`, `vast_mounts` | Verify /scratch, /home, /apps, /projects directories. | All stated checks pass for every applicable target. | +| 42103 | `ORCH_FVT_PXEBOOT_V702` | `test_vast_scratch_hostname_isolation` | `vast_storage` | `sanity`, `vast_scratch` | Verify /scratch// per node; file isolation. | All stated checks pass for every applicable target. | +| 42104 | `ORCH_FVT_PXEBOOT_V703` | `test_vast_mount_options` | `vast_storage` | `sanity`, `vast_mounts` | Verify proto=rdma and port=20049 in /proc/mounts. | All stated checks pass for every applicable target. | +| 42105 | `ORCH_FVT_PXEBOOT_V704` | `test_vast_ldapuser_scratch_directory` | `vast_storage` | `sanity`, `vast_scratch` | Verify /scratch// on login_compiler nodes. | All stated checks pass for every applicable target. | +| 42106 | `ORCH_FVT_PXEBOOT_V705` | `test_vast_ldapuser_subdirectories` | `vast_storage` | `sanity`, `vast_scratch` | Verify data/, jobs/, results/, tmp/ subdirectories. | All stated checks pass for every applicable target. | +| 42107 | `ORCH_FVT_PXEBOOT_V706` | `test_vast_ldapuser_permissions` | `vast_storage` | `sanity`, `vast_scratch` | Verify cross-user permission isolation. | All stated checks pass for every applicable target. | +| 42108 | `ORCH_FVT_PXEBOOT_V707` | `test_vast_scratch_isolation` | `vast_storage` | `sanity`, `vast_scratch` | Verify file isolation between scratch subdirectories. | All stated checks pass for every applicable target. | +| 42109 | `ORCH_FVT_PXEBOOT_V708` | `test_vast_control_node_no_vast` | `vast_storage` | `sanity`, `vast_targeting` | Verify Slurm control node has no VAST storage. | All stated checks pass for every applicable target. | +| 42110 | `ORCH_FVT_PXEBOOT_V709` | `test_vast_vastnfs_rpm_and_module` | `vast_storage` | `sanity`, `vast_installation` | Verify vastnfs RPM, kernel module, and service. | All stated checks pass for every applicable target. | +| 42111 | `ORCH_FVT_PXEBOOT_V710` | `test_vast_fstab_entries` | `vast_storage` | `sanity`, `vast_mounts` | Verify /etc/fstab entries with proto=rdma. | All stated checks pass for every applicable target. | +| 42112 | `ORCH_FVT_PXEBOOT_V711` | `test_vast_rdma_mount` | `vast_storage` | `sanity`, `vast_mounts` | Verify RDMA transport and 1 GB I/O checksum. | All stated checks pass for every applicable target. | +| 42113 | `ORCH_FVT_PXEBOOT_V712` | `test_vast_qss_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify VAST on compute/login only; controller has none. | All stated checks pass for every applicable target. | +| 42114 | `ORCH_FVT_PXEBOOT_V713` | `test_vast_slurm_logs_persistence` | `vast_storage` | `sanity`, `vast_persistence` | Verify Slurm logs on persistent storage; sacct accessible. | All stated checks pass for every applicable target. | +| 42115 | `ORCH_FVT_PXEBOOT_V714` | `test_vast_control_node_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify controller mount table (NFS/PV only, no VAST). | All stated checks pass for every applicable target. | +| 42116 | `ORCH_FVT_PXEBOOT_V715` | `test_vast_compute_node_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify compute node mount table (NFS + VAST). | All stated checks pass for every applicable target. | +| 42117 | `ORCH_FVT_PXEBOOT_V716` | `test_vast_login_node_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify login/compiler node mount table. | All stated checks pass for every applicable target. | + +VAST checks apply only when `slurm_cluster[0].vast_storage_name` selects a VAST +storage entry. LDAP scratch checks use the `ldap_username` test credential. + +### Kubernetes recovery (`kubernetes_recovery`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42201 | `ORCH_FVT_PXEBOOT_V021` | `test_kubernetes_local_etcd_recovery` | `kubernetes_recovery` | `kubernetes`, `reboot` | Reboot a control plane and prove its local-etcd UUID is preserved. | The node returns within the bounded wait and every stated post-reboot check passes. | +| 42202 | `ORCH_FVT_PXEBOOT_V022` | `test_kubernetes_control_plane_recovery` | `kubernetes_recovery` | `kubernetes`, `reboot` | Reboot the VIP owner and verify control-plane recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | + +Both cases require two control planes and observe readiness from a node +that is not rebooted. After the reboot they wait, within bounded limits, +for a new boot ID, cloud-init, Kubernetes Ready, and healthy etcd +endpoints. The local-etcd case also requires `etcd-fstab-update.sh` to +finish on the new boot, `etcd-disk-setup.sh` not to run again, and the +mount source and UUID to be unchanged. + +### Slurm recovery (`slurm_recovery`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42301 | `ORCH_FVT_PXEBOOT_V062` | `test_slurm_cluster_recovery` | `slurm_recovery` | `functional`, `reboot`, `slurm` | Reboot mapped Slurm nodes and verify scheduler and workload recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | + +### Slurm node lifecycle (`slurm_lifecycle`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42401 | `ORCH_FVT_PXEBOOT_V500` | `test_slurm_node_remove` | `slurm_lifecycle` | `functional`, `node_lifecycle`, `slurm` | Remove Slurm compute node(s) from PXE mapping, provision, verify. | The operation completes successfully and returns the expected result. | +| 42402 | `ORCH_FVT_PXEBOOT_V501` | `test_slurm_node_add` | `slurm_lifecycle` | `functional`, `node_lifecycle`, `slurm` | Restore removed node(s) to PXE mapping, provision, verify re-addition. | The operation completes successfully and returns the expected result. | + +Node removal and re-addition use `slurm_lifecycle_remove_add_nodes` from +`test_config.yml`; they run only when `--marker` selects `node_lifecycle`. +Removal skips when the PXE mapping has fewer than two `slurm_node_*` entries, +and re-addition then skips because no backup is left. ## Cleanup test cases -These cases execute the explicitly selected full cleanup and verify each -component against the configured deletion or preservation policy. +These cases execute the explicitly selected full cleanup and verify each component against the configured deletion or preservation policy. + +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50000 | `ORCH_FVT_CLEANUP_E001` | `test_deploy_cleanup` | `root` | `deploy`, `destructive`, `sanity` | Run ``orchestrator.yml --tags cleanup`` exactly once. | The selected Orchestrator lifecycle exits successfully. | + +### OpenCHAMI (`openchami`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50101 | `ORCH_FVT_CLEANUP_V001` | `test_openchami_removed` | `openchami` | `destructive`, `sanity` | Verify OpenCHAMI runtime, volumes, packages and state are removed. | All stated checks pass for every applicable target. | + +### OpenLDAP (`openldap`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50201 | `ORCH_FVT_CLEANUP_V002` | `test_openldap_removed` | `openldap` | `destructive`, `sanity` | Verify the OpenLDAP proxy service, container and state are removed. | All stated checks pass for every applicable target. | + +### Slurm (`slurm`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50301 | `ORCH_FVT_CLEANUP_V003` | `test_slurm_cleanup` | `slurm` | `destructive`, `sanity` | Verify Slurm's selected data policy and configured storage detachment. | All stated checks pass for every applicable target. | + +### Kubernetes (`kubernetes`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50401 | `ORCH_FVT_CLEANUP_V004` | `test_kubernetes_cleanup` | `kubernetes` | `destructive`, `sanity` | Verify Kubernetes's selected data policy and storage detachment. | All stated checks pass for every applicable target. | + +### Artifacts (`artifacts`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50501 | `ORCH_FVT_CLEANUP_V005` | `test_artifacts_removed_and_inputs_preserved` | `artifacts` | `destructive`, `sanity` | Verify generated state is removed without deleting required inputs. | All stated checks pass for every applicable target. | + +### Credentials (`credentials`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 0 | `ORCH_FVT_CLEANUP_E001` | `test_deploy_cleanup` | `root` | `deploy`, `destructive`, `sanity` | Run ``orchestrator.yml --tags cleanup`` exactly once. | The selected Orchestrator lifecycle exits successfully. | -| 1 | `ORCH_FVT_CLEANUP_V001` | `test_openchami_removed` | `openchami` | `destructive`, `sanity` | Verify OpenCHAMI runtime, volumes, packages and state are removed. | All stated checks pass for every applicable target. | -| 2 | `ORCH_FVT_CLEANUP_V002` | `test_openldap_removed` | `openldap` | `destructive`, `sanity` | Verify the OpenLDAP proxy service, container and state are removed. | All stated checks pass for every applicable target. | -| 3 | `ORCH_FVT_CLEANUP_V003` | `test_slurm_cleanup` | `slurm` | `destructive`, `sanity` | Verify Slurm's selected data policy and configured storage detachment. | All stated checks pass for every applicable target. | -| 4 | `ORCH_FVT_CLEANUP_V004` | `test_kubernetes_cleanup` | `kubernetes` | `destructive`, `sanity` | Verify Kubernetes's selected data policy and storage detachment. | All stated checks pass for every applicable target. | -| 5 | `ORCH_FVT_CLEANUP_V005` | `test_artifacts_removed_and_inputs_preserved` | `artifacts` | `destructive`, `sanity` | Verify generated state is removed without deleting required inputs. | All stated checks pass for every applicable target. | -| 6 | `ORCH_FVT_CLEANUP_V006` | `test_credentials_follow_selected_policy` | `credentials` | `destructive`, `sanity` | Verify credentials are removed or preserved as selected. | All stated checks pass for every applicable target. | +| 50601 | `ORCH_FVT_CLEANUP_V006` | `test_credentials_follow_selected_policy` | `credentials` | `destructive`, `sanity` | Verify credentials are removed or preserved as selected. | All stated checks pass for every applicable target. | Cleanup verification follows the configured policy; it does not assume every input file, credential, or externally managed data path must always be deleted. @@ -469,18 +722,21 @@ input file, credential, or externally managed data path must always be deleted. | Marker | Purpose | |---|---| -| `sanity` | Default positive PXE and lifecycle coverage | +| `sanity` | Baseline positive PXE and lifecycle coverage | +| `buildstream` | BuildStream validation subset | | `functional` | Temporary workload or job behavior | -| `openldap`, `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer`, `additional_cloud_init` | Capability selectors | +| `openldap`, `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer`, `benchmark`, `additional_cloud_init`, `mount_config`, `minimal_os`, `boot_image` | Capability selectors | +| `powervault_infrastructure`, `powervault_mounts`, `powervault_binds`, `powervault_cloudinit` | PowerVault subset selectors | +| `vast_installation`, `vast_mounts`, `vast_scratch`, `vast_targeting`, `vast_persistence` | VAST storage subset selectors | | `image_download` | Explicit authorization to modify shared Apptainer image storage | | `negative` | Expected rejection and error-path behavior | | `non_disruptive` | Work that does not reboot or drain cluster nodes | -| `disruptive` | Maintenance-window recovery behavior | -| `reboot` | Reboot subset of disruptive cases | -| `scheduler_state` | Scheduler drain/resume subset of disruptive cases | +| `reboot` | Node reboot cases; they run only when `--marker` selects `reboot` | +| `node_lifecycle` | Node remove/add cases; they run only when `--marker` selects `node_lifecycle` | +| `scheduler_state` | Scheduler drain/resume case | | `destructive` | Destructive cleanup selector | -A comma is OR; a plus is AND. For example, `sanity,functional` selects either +With no marker every test runs except reboot and node_lifecycle cases, which are deselected. A comma is OR; a plus is AND. For example, `sanity,functional` selects either class, while `slurm+non_disruptive` selects tests carrying both markers. ## Commands @@ -516,7 +772,7 @@ class, while `slurm+non_disruptive` selects tests carrying both markers. ./run_validation.sh fvt_orchestrator pxeboot verify \ --suite slurm_apptainer --marker functional+image_download ./run_validation.sh fvt_orchestrator pxeboot verify \ - --marker disruptive+reboot + --marker reboot,node_lifecycle # Explicit full cleanup. ./run_validation.sh fvt_orchestrator cleanup test --marker sanity diff --git a/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py b/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py index 20fc280b5a..0ceae9c489 100644 --- a/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py +++ b/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py @@ -15,16 +15,20 @@ """Artifact postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_artifacts +from library.functions import ( + TestLogger, + check_cleanup_artifacts, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(5) +@pytest.mark.order(50501) def test_artifacts_removed_and_inputs_preserved(host): """Verify generated state is removed without deleting required inputs.""" - verify_cleanup( - host, "cleanup_artifacts", "Artifact cleanup", check_cleanup_artifacts - ) + tc = TC["cleanup_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Artifact cleanup", check_cleanup_artifacts) diff --git a/test/orchestrator/fvt/cleanup/credentials/test_credentials.py b/test/orchestrator/fvt/cleanup/credentials/test_credentials.py index 87abdf845b..1298c99dad 100644 --- a/test/orchestrator/fvt/cleanup/credentials/test_credentials.py +++ b/test/orchestrator/fvt/cleanup/credentials/test_credentials.py @@ -15,19 +15,20 @@ """Credential postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_credentials +from library.functions import ( + TestLogger, + check_cleanup_credentials, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(6) +@pytest.mark.order(50601) def test_credentials_follow_selected_policy(host): """Verify credentials are removed or preserved as selected.""" - verify_cleanup( - host, - "cleanup_credentials", - "Credential cleanup", - check_cleanup_credentials, - ) + tc = TC["cleanup_credentials"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Credential cleanup", check_cleanup_credentials) diff --git a/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py b/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py index 5ccdf9150b..bb38a12240 100644 --- a/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py +++ b/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py @@ -15,19 +15,20 @@ """Kubernetes storage postconditions produced by full cleanup.""" import pytest -from library.functions import check_cleanup_kubernetes +from library.functions import ( + TestLogger, + check_cleanup_kubernetes, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(4) +@pytest.mark.order(50401) def test_kubernetes_cleanup(host): """Verify Kubernetes's selected data policy and storage detachment.""" - verify_cleanup( - host, - "cleanup_kubernetes", - "Kubernetes cleanup", - check_cleanup_kubernetes, - ) + tc = TC["cleanup_kubernetes"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Kubernetes cleanup", check_cleanup_kubernetes) diff --git a/test/orchestrator/fvt/cleanup/openchami/test_openchami.py b/test/orchestrator/fvt/cleanup/openchami/test_openchami.py index 8d67a31aaa..41ef0e0f26 100644 --- a/test/orchestrator/fvt/cleanup/openchami/test_openchami.py +++ b/test/orchestrator/fvt/cleanup/openchami/test_openchami.py @@ -15,16 +15,20 @@ """OpenCHAMI postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_openchami +from library.functions import ( + TestLogger, + check_cleanup_openchami, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(1) +@pytest.mark.order(50101) def test_openchami_removed(host): """Verify OpenCHAMI runtime, volumes, packages and state are removed.""" - verify_cleanup( - host, "cleanup_openchami", "OpenCHAMI cleanup", check_cleanup_openchami - ) + tc = TC["cleanup_openchami"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "OpenCHAMI cleanup", check_cleanup_openchami) diff --git a/test/orchestrator/fvt/cleanup/openldap/test_openldap.py b/test/orchestrator/fvt/cleanup/openldap/test_openldap.py index ff025b9cb2..a95e875cad 100644 --- a/test/orchestrator/fvt/cleanup/openldap/test_openldap.py +++ b/test/orchestrator/fvt/cleanup/openldap/test_openldap.py @@ -15,14 +15,20 @@ """OpenLDAP postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_openldap +from library.functions import ( + TestLogger, + check_cleanup_openldap, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(2) +@pytest.mark.order(50201) def test_openldap_removed(host): """Verify the OpenLDAP proxy service, container and state are removed.""" - verify_cleanup(host, "cleanup_openldap", "OpenLDAP cleanup", check_cleanup_openldap) + tc = TC["cleanup_openldap"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "OpenLDAP cleanup", check_cleanup_openldap) diff --git a/test/orchestrator/fvt/cleanup/slurm/test_slurm.py b/test/orchestrator/fvt/cleanup/slurm/test_slurm.py index 5e9d104e88..f2a0d03b9f 100644 --- a/test/orchestrator/fvt/cleanup/slurm/test_slurm.py +++ b/test/orchestrator/fvt/cleanup/slurm/test_slurm.py @@ -15,14 +15,20 @@ """Slurm storage postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_slurm +from library.functions import ( + TestLogger, + check_cleanup_slurm, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(3) +@pytest.mark.order(50301) def test_slurm_cleanup(host): """Verify Slurm's selected data policy and configured storage detachment.""" - verify_cleanup(host, "cleanup_slurm", "Slurm cleanup", check_cleanup_slurm) + tc = TC["cleanup_slurm"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Slurm cleanup", check_cleanup_slurm) diff --git a/test/orchestrator/fvt/cleanup/test_playbook.py b/test/orchestrator/fvt/cleanup/test_playbook.py index 073d1041c5..d2141e801c 100644 --- a/test/orchestrator/fvt/cleanup/test_playbook.py +++ b/test/orchestrator/fvt/cleanup/test_playbook.py @@ -29,7 +29,7 @@ @pytest.mark.destructive @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(0) +@pytest.mark.order(50000) def test_deploy_cleanup(host): """Run ``orchestrator.yml --tags cleanup`` exactly once.""" tc = TC["deploy_cleanup"] diff --git a/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py b/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py index d77cb9b976..0fe43d1ec6 100644 --- a/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py +++ b/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py @@ -16,30 +16,38 @@ import pytest from library.functions import ( + TestLogger, check_precheck_dependencies, check_precheck_repositories, check_precheck_s3_artifacts, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(4) +@pytest.mark.order(10401) def test_precheck_dependencies(host): """Require both configured/default dependency outputs to be usable.""" - verify_precheck(host, "precheck_dependencies", check_precheck_dependencies) + tc = TC["precheck_dependencies"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_dependencies) @pytest.mark.sanity -@pytest.mark.order(6) +@pytest.mark.order(10402) def test_precheck_s3_artifacts(host): """Require every kernel, initrd, and rootfs artifact to be reachable.""" - verify_precheck(host, "precheck_s3_artifacts", check_precheck_s3_artifacts) + tc = TC["precheck_s3_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_s3_artifacts) @pytest.mark.sanity -@pytest.mark.order(7) +@pytest.mark.order(10403) def test_precheck_repositories(host): """Require every published RPM and file repository to be reachable.""" - verify_precheck(host, "precheck_repositories", check_precheck_repositories) + tc = TC["precheck_repositories"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_repositories) diff --git a/test/orchestrator/fvt/precheck/environment/test_environment.py b/test/orchestrator/fvt/precheck/environment/test_environment.py index df54bf8134..d38079b921 100644 --- a/test/orchestrator/fvt/precheck/environment/test_environment.py +++ b/test/orchestrator/fvt/precheck/environment/test_environment.py @@ -16,22 +16,28 @@ import pytest from library.functions import ( + TestLogger, check_precheck_admin_ipv4, check_precheck_hostname_domain, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(1) +@pytest.mark.order(10101) def test_precheck_hostname_domain(host): """Require the host identity to match omnia.env exactly.""" - verify_precheck(host, "precheck_hostname_domain", check_precheck_hostname_domain) + tc = TC["precheck_hostname_domain"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_hostname_domain) @pytest.mark.sanity -@pytest.mark.order(2) +@pytest.mark.order(10102) def test_precheck_admin_ipv4(host): """Require the configured administrative IPv4 on a global interface.""" - verify_precheck(host, "precheck_admin_ipv4", check_precheck_admin_ipv4) + tc = TC["precheck_admin_ipv4"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_admin_ipv4) diff --git a/test/orchestrator/fvt/precheck/inputs/test_inputs.py b/test/orchestrator/fvt/precheck/inputs/test_inputs.py index b48e55a298..bc9cfc117d 100644 --- a/test/orchestrator/fvt/precheck/inputs/test_inputs.py +++ b/test/orchestrator/fvt/precheck/inputs/test_inputs.py @@ -15,13 +15,19 @@ """Required and conditionally selected project-input contracts.""" import pytest -from library.functions import check_precheck_inputs +from library.functions import ( + TestLogger, + check_precheck_inputs, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(5) +@pytest.mark.order(10501) def test_precheck_inputs(host): """Require every source-selected Orchestrator input to be non-empty.""" - verify_precheck(host, "precheck_inputs", check_precheck_inputs) + tc = TC["precheck_inputs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_inputs) diff --git a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py index 0bbb68b774..9c193caae2 100644 --- a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py +++ b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py @@ -14,70 +14,87 @@ """OIM CPU, memory, and disk-space readiness contracts.""" +from functools import partial + import pytest from library.functions import ( + TestLogger, check_oim_cpu_threshold, check_oim_disk_threshold, check_oim_memory_threshold, ) +from library.vars import OIM_NEGATIVE_INPUTS as NEG +from library.vars import TEST_CASES as TC -from fvt.result import verify_precheck - -pytestmark = [pytest.mark.sanity] - +from fvt.result import verify_precheck, verify_precheck_rejection # ── positive tests ────────────────────────────────────────────────── -@pytest.mark.order(10) +@pytest.mark.sanity +@pytest.mark.order(10201) def test_oim_cpu_threshold(host): """Require OIM CPU core count to meet the configured minimum.""" - verify_precheck(host, "oim_cpu_threshold", check_oim_cpu_threshold) + tc = TC["oim_cpu_threshold"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_cpu_threshold) -@pytest.mark.order(11) +@pytest.mark.sanity +@pytest.mark.order(10202) def test_oim_memory_threshold(host): """Require OIM memory to meet the configured minimum.""" - verify_precheck(host, "oim_memory_threshold", check_oim_memory_threshold) + tc = TC["oim_memory_threshold"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_memory_threshold) -@pytest.mark.order(12) +@pytest.mark.sanity +@pytest.mark.order(10203) def test_oim_disk_threshold(host): """Require OIM root filesystem to meet the configured minimum.""" - verify_precheck(host, "oim_disk_threshold", check_oim_disk_threshold) - - -# ── negative tests ────────────────────────────────────────────────── + tc = TC["oim_disk_threshold"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_disk_threshold) @pytest.mark.negative -@pytest.mark.order(20) +@pytest.mark.order(10210) def test_neg_cpu_below_threshold(host): """Detect failure when CPU threshold exceeds actual cores.""" - result = check_oim_cpu_threshold(host, min_cores=99999) - assert not result["success"], ( - "CPU check should fail when threshold exceeds actual cores" + tc = TC["oim_cpu_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_cpu_threshold, min_cores=NEG["min_cores"]), ) - assert result["error"], "Failure must include an actionable message" @pytest.mark.negative -@pytest.mark.order(21) +@pytest.mark.order(10211) def test_neg_memory_below_threshold(host): """Detect failure when memory threshold exceeds actual RAM.""" - result = check_oim_memory_threshold(host, min_memory_gb=99999) - assert not result["success"], ( - "Memory check should fail when threshold exceeds actual RAM" + tc = TC["oim_memory_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_memory_threshold, min_memory_gb=NEG["min_memory_gb"]), ) - assert result["error"], "Failure must include an actionable message" @pytest.mark.negative -@pytest.mark.order(22) +@pytest.mark.order(10212) def test_neg_disk_below_threshold(host): """Detect failure when disk threshold exceeds actual capacity.""" - result = check_oim_disk_threshold(host, min_disk_gb=99999) - assert not result["success"], ( - "Disk check should fail when threshold exceeds actual capacity" + tc = TC["oim_disk_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_disk_threshold, min_disk_gb=NEG["min_disk_gb"]), ) - assert result["error"], "Failure must include an actionable message" diff --git a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py index e77bde4a5a..932b5322d7 100644 --- a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py +++ b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py @@ -14,95 +14,71 @@ """OIM admin/public NIC presence, IPv4, and overlap contracts.""" +from functools import partial + import pytest from library.functions import ( + TestLogger, + check_oim_forced_pxe_public_overlap, check_oim_pxe_nic_ipv4, check_oim_pxe_nic_present, - check_oim_pxe_public_overlap, check_oim_public_nic_present, ) +from library.vars import OIM_NEGATIVE_INPUTS as NEG +from library.vars import TEST_CASES as TC -from fvt.result import verify_precheck - -pytestmark = [pytest.mark.sanity] - +from fvt.result import verify_precheck, verify_precheck_rejection # ── positive tests ────────────────────────────────────────────────── -@pytest.mark.order(13) +@pytest.mark.sanity +@pytest.mark.order(10204) def test_oim_pxe_nic_present(host): """Require the configured admin NIC to exist and be UP.""" - verify_precheck(host, "oim_pxe_nic_present", check_oim_pxe_nic_present) + tc = TC["oim_pxe_nic_present"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_pxe_nic_present) -@pytest.mark.order(14) +@pytest.mark.sanity +@pytest.mark.order(10205) def test_oim_public_nic_present(host): """Require the public/default-route NIC to exist and be UP.""" - verify_precheck( - host, "oim_public_nic_present", check_oim_public_nic_present - ) + tc = TC["oim_public_nic_present"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_public_nic_present) -@pytest.mark.order(15) +@pytest.mark.sanity +@pytest.mark.order(10206) def test_oim_pxe_nic_ipv4(host): """Require the admin NIC to carry the configured IPv4 address.""" - verify_precheck(host, "oim_pxe_nic_ipv4", check_oim_pxe_nic_ipv4) - - -# ── negative tests ────────────────────────────────────────────────── + tc = TC["oim_pxe_nic_ipv4"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_pxe_nic_ipv4) @pytest.mark.negative -@pytest.mark.order(23) +@pytest.mark.order(10213) def test_neg_pxe_nic_missing(host): """Detect failure when a nonexistent NIC name is checked.""" - result = check_oim_pxe_nic_present.__wrapped__(host) if hasattr( - check_oim_pxe_nic_present, "__wrapped__" - ) else None - # Directly call the underlying function with a bogus NIC name - from library.functions.oim_readiness_precheck_func import ( - OIM_READINESS_COMMANDS, - prepare_result, - run_on_host, - ) - - nic_name = "nonexistent_nic_000" - cmd_result = run_on_host( - host, OIM_READINESS_COMMANDS["nic_operstate"], nic_name - ) - state = cmd_result.stdout.strip() if cmd_result.rc == 0 else "" - assert not (cmd_result.rc == 0 and state), ( - f"Interface {nic_name} should not exist" + tc = TC["oim_missing_nic_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_pxe_nic_present, nic_name=NEG["missing_nic"]), ) @pytest.mark.negative -@pytest.mark.order(24) +@pytest.mark.order(10214) def test_neg_pxe_public_overlap(host): """Detect overlap when PXE NIC is forced to match the public NIC.""" - # Find the actual public NIC from default route - from omnia_auto import run_on_host as _run - - route = _run(host, "ip -4 route show default 2>/dev/null | head -1") - route_line = route.stdout.strip() - if not route_line: - pytest.skip("No default route; overlap test not applicable") - - parts = route_line.split() - public_nic = "" - for i, token in enumerate(parts): - if token == "dev" and i + 1 < len(parts): - public_nic = parts[i + 1] - break - - if not public_nic: - pytest.skip("Cannot parse public NIC from default route") - - result = check_oim_pxe_public_overlap(host, nic_name=public_nic) - assert not result["success"], ( - "Overlap check should fail when PXE NIC equals public NIC" - ) - assert "overlap" in result["error"].lower() or "share" in result["error"].lower(), ( - "Error message must mention the overlap" + tc = TC["oim_nic_overlap_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, tc, host, check_oim_forced_pxe_public_overlap ) diff --git a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py index 5fe1cbeecb..280e6d2c73 100644 --- a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py +++ b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py @@ -14,80 +14,91 @@ """OIM SSH, internet, and OS-version readiness contracts.""" +from functools import partial + import pytest from library.functions import ( + TestLogger, check_oim_internet_reachability, check_oim_os_version, check_oim_ssh_preflight, ) +from library.vars import OIM_NEGATIVE_INPUTS as NEG +from library.vars import TEST_CASES as TC -from fvt.result import verify_precheck - -pytestmark = [pytest.mark.sanity] - +from fvt.result import verify_precheck, verify_precheck_rejection # ── positive tests ────────────────────────────────────────────────── -@pytest.mark.order(16) +@pytest.mark.sanity +@pytest.mark.order(10207) def test_oim_ssh_preflight(host): """Require passwordless SSH from OIM to a mapped target node.""" - verify_precheck(host, "oim_ssh_preflight", check_oim_ssh_preflight) + tc = TC["oim_ssh_preflight"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_ssh_preflight) -@pytest.mark.order(18) +@pytest.mark.sanity +@pytest.mark.order(10208) def test_oim_internet_reachability(host): """Require internet reachability when not in air-gapped mode.""" - verify_precheck( - host, "oim_internet_reachability", check_oim_internet_reachability - ) + tc = TC["oim_internet_reachability"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_internet_reachability) -@pytest.mark.order(19) +@pytest.mark.sanity +@pytest.mark.order(10209) def test_oim_os_version(host): """Require the OIM OS to match the expected distribution and version.""" - verify_precheck(host, "oim_os_version", check_oim_os_version) - - -# ── negative tests ────────────────────────────────────────────────── + tc = TC["oim_os_version"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_os_version) @pytest.mark.negative -@pytest.mark.order(25) +@pytest.mark.order(10215) def test_neg_internet_airgapped(host): """Verify air-gapped mode passes even without internet.""" - result = check_oim_internet_reachability(host, require_internet=False) - assert result["success"], ( - "Air-gapped mode must pass regardless of internet availability" + tc = TC["oim_airgapped_internet"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck( + test_log, + tc, + host, + partial(check_oim_internet_reachability, require_internet=False), ) @pytest.mark.negative -@pytest.mark.order(27) +@pytest.mark.order(10216) def test_neg_ssh_unreachable_target(host): """Detect SSH failure to a bogus target address.""" - from library.functions.oim_readiness_precheck_func import ( - OIM_READINESS_COMMANDS, - run_on_host, - ) - - bogus_target = "192.0.2.1" # RFC 5737 TEST-NET, guaranteed unreachable - result = run_on_host( - host, OIM_READINESS_COMMANDS["ssh_check"], bogus_target - ) - assert result.rc != 0, ( - f"SSH to {bogus_target} should fail (unreachable TEST-NET address)" + tc = TC["oim_unreachable_ssh_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_ssh_preflight, target=NEG["unreachable_target"]), ) @pytest.mark.negative -@pytest.mark.order(28) +@pytest.mark.order(10217) def test_neg_os_version_mismatch(host): """Detect failure when expected OS version does not match actual.""" - result = check_oim_os_version( - host, expected_id="nonexistent_os", expected_version="0.0" - ) - assert not result["success"], ( - "OS check should fail when expected ID/version do not match" + tc = TC["oim_os_mismatch_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial( + check_oim_os_version, + expected_id=NEG["os_id"], + expected_version=NEG["os_version"], + ), ) - assert result["error"], "Failure must include an actionable message" diff --git a/test/orchestrator/fvt/precheck/storage/test_storage.py b/test/orchestrator/fvt/precheck/storage/test_storage.py index fe37cfecdf..fd5f2c8bd5 100644 --- a/test/orchestrator/fvt/precheck/storage/test_storage.py +++ b/test/orchestrator/fvt/precheck/storage/test_storage.py @@ -15,13 +15,19 @@ """Configured NFS endpoint reachability contract.""" import pytest -from library.functions import check_precheck_nfs_servers +from library.functions import ( + TestLogger, + check_precheck_nfs_servers, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(3) +@pytest.mark.order(10301) def test_precheck_nfs_servers(host): """Require every mapping-selected NFS server to answer ICMP from the OIM.""" - verify_precheck(host, "precheck_nfs_servers", check_precheck_nfs_servers) + tc = TC["precheck_nfs_servers"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_nfs_servers) diff --git a/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py b/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py index b0ca1e814c..e51c58f41c 100644 --- a/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py +++ b/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py @@ -21,63 +21,67 @@ import pytest from library.functions import ( + TestLogger, check_precheck_mount_invalid_mount_params, check_precheck_mount_missing_mount_point, check_precheck_mount_missing_source, check_precheck_mount_missing_targeting, check_precheck_mount_node_key_without_mount_point, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck -pytestmark = [pytest.mark.sanity, pytest.mark.negative, pytest.mark.mount_config] - - -@pytest.mark.order(30) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10302) def test_mount_missing_mount_point(host): """TC-CI-NEG-001: Every mount entry must have a valid absolute mount_point.""" - verify_precheck( - host, - "precheck_mount_missing_mount_point", - check_precheck_mount_missing_mount_point, - ) + tc = TC["precheck_mount_missing_mount_point"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_missing_mount_point) -@pytest.mark.order(31) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10303) def test_mount_missing_targeting(host): """TC-CI-NEG-002: Every mount entry must have targeting (prefix or groups).""" - verify_precheck( - host, - "precheck_mount_missing_targeting", - check_precheck_mount_missing_targeting, - ) + tc = TC["precheck_mount_missing_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_missing_targeting) -@pytest.mark.order(32) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10304) def test_mount_invalid_mount_params(host): """TC-CI-NEG-003: mount_params must reference an existing profile.""" - verify_precheck( - host, - "precheck_mount_invalid_mount_params", - check_precheck_mount_invalid_mount_params, - ) + tc = TC["precheck_mount_invalid_mount_params"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_invalid_mount_params) -@pytest.mark.order(33) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10305) def test_mount_missing_source(host): """TC-CI-NEG-004: Every mount entry must have a non-empty source.""" - verify_precheck( - host, - "precheck_mount_missing_source", - check_precheck_mount_missing_source, - ) + tc = TC["precheck_mount_missing_source"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_missing_source) -@pytest.mark.order(34) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10306) def test_mount_node_key_without_mount_point(host): """TC-CI-NEG-005: node_mount_point is mandatory when node_key is set.""" + tc = TC["precheck_mount_node_key_without_mount_point"] + test_log = TestLogger(tc["title"], tc["id"]) verify_precheck( + test_log, + tc, host, - "precheck_mount_node_key_without_mount_point", check_precheck_mount_node_key_without_mount_point, ) diff --git a/test/orchestrator/fvt/precheck/test_playbook.py b/test/orchestrator/fvt/precheck/test_playbook.py index 5ac7ec653d..550a127155 100644 --- a/test/orchestrator/fvt/precheck/test_playbook.py +++ b/test/orchestrator/fvt/precheck/test_playbook.py @@ -23,7 +23,7 @@ @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(0) +@pytest.mark.order(10000) def test_deploy_precheck(host): """Run ``orchestrator.yml --tags precheck`` exactly once.""" tc = TC["deploy_precheck"] diff --git a/test/orchestrator/fvt/prepare/network/test_network.py b/test/orchestrator/fvt/prepare/network/test_network.py index 745cdf7efc..a8f4afaf09 100644 --- a/test/orchestrator/fvt/prepare/network/test_network.py +++ b/test/orchestrator/fvt/prepare/network/test_network.py @@ -39,7 +39,7 @@ def _assert_result(test_log, component, result): @pytest.mark.sanity -@pytest.mark.order(6) +@pytest.mark.order(20201) def test_firewall_and_podman_network_policy(host): """Verify OpenCHAMI ports and trusted Podman interfaces.""" tc = TC["firewall_network"] @@ -53,7 +53,7 @@ def test_firewall_and_podman_network_policy(host): @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(7) +@pytest.mark.order(20202) def test_coredhcp_and_coredns_configuration(host): """Verify rendered DHCP/DNS configuration and additional routes.""" tc = TC["coredhcp_network"] diff --git a/test/orchestrator/fvt/prepare/openchami/test_openchami.py b/test/orchestrator/fvt/prepare/openchami/test_openchami.py index aff419b465..5db5a4cff8 100644 --- a/test/orchestrator/fvt/prepare/openchami/test_openchami.py +++ b/test/orchestrator/fvt/prepare/openchami/test_openchami.py @@ -45,7 +45,7 @@ def _assert_result(test_log, component, result): @pytest.mark.sanity -@pytest.mark.order(1) +@pytest.mark.order(20101) def test_openchami_containers_running(host): """Verify all long-running OpenCHAMI containers.""" tc = TC["openchami_containers"] @@ -58,7 +58,7 @@ def test_openchami_containers_running(host): @pytest.mark.sanity -@pytest.mark.order(2) +@pytest.mark.order(20102) def test_openchami_services_ready(host): """Verify OpenCHAMI units and successful SMD initialization.""" tc = TC["openchami_services"] @@ -72,7 +72,7 @@ def test_openchami_services_ready(host): @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(3) +@pytest.mark.order(20103) def test_openchami_apis_ready(host): """Verify the authenticated SMD, Boot and Metadata APIs.""" tc = TC["openchami_apis"] @@ -85,7 +85,7 @@ def test_openchami_apis_ready(host): @pytest.mark.sanity -@pytest.mark.order(4) +@pytest.mark.order(20104) def test_openchami_persistent_storage_and_tls(host): """Verify persistent data volumes and HAProxy certificates.""" tc = TC["openchami_storage"] @@ -98,7 +98,7 @@ def test_openchami_persistent_storage_and_tls(host): @pytest.mark.sanity -@pytest.mark.order(5) +@pytest.mark.order(20105) def test_openchami_packages_and_artifacts(host): """Verify installed packages and generated configuration files.""" tc = TC["openchami_artifacts"] @@ -112,7 +112,7 @@ def test_openchami_packages_and_artifacts(host): @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(13) +@pytest.mark.order(20106) def test_postgresql_readiness(host): """Verify PostgreSQL and its required SMD database contract.""" tc = TC["postgresql_readiness"] diff --git a/test/orchestrator/fvt/prepare/test_playbook.py b/test/orchestrator/fvt/prepare/test_playbook.py index 5270115559..a31fe85782 100644 --- a/test/orchestrator/fvt/prepare/test_playbook.py +++ b/test/orchestrator/fvt/prepare/test_playbook.py @@ -27,7 +27,7 @@ @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(0) +@pytest.mark.order(20000) def test_deploy_prepare(host): """Run ``orchestrator.yml --tags prepare``.""" tc = TC["deploy_prepare"] diff --git a/test/orchestrator/fvt/provision/openchami/test_provision_state.py b/test/orchestrator/fvt/provision/openchami/test_provision_state.py index c6737ec265..e5dc028f29 100644 --- a/test/orchestrator/fvt/provision/openchami/test_provision_state.py +++ b/test/orchestrator/fvt/provision/openchami/test_provision_state.py @@ -52,83 +52,91 @@ def _assert_result(test_log, component, result): ) -def _run_check(host, registry_key, check): - """Create the registered logger and run one provision check.""" - tc = TC[registry_key] - _assert_result(TestLogger(tc["title"], tc["id"]), tc["component"], check(host)) - - @pytest.mark.sanity -@pytest.mark.order(101) +@pytest.mark.order(30101) def test_provision_reports(host): """Verify the provision report and generated inventory contracts.""" - _run_check(host, "provision_reports", check_provision_reports) + tc = TC["provision_reports"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_provision_reports(host)) @pytest.mark.sanity -@pytest.mark.order(102) +@pytest.mark.order(30102) def test_smd_identity(host): """Verify XNAME, administrative MAC, and IP identity in SMD.""" - _run_check(host, "smd_identity", check_smd_identity) + tc = TC["smd_identity"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_smd_identity(host)) @pytest.mark.sanity -@pytest.mark.order(103) +@pytest.mark.order(30103) def test_smd_group_membership(host): """Verify expected groups and reject competing cloud-init groups.""" - _run_check(host, "smd_groups", check_smd_groups) + tc = TC["smd_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_smd_groups(host)) @pytest.mark.sanity -@pytest.mark.order(104) +@pytest.mark.order(30104) def test_boot_service_configurations(host): """Verify BootConfigurations and their mapped administrative MACs.""" - _run_check( - host, - "boot_configurations", - check_boot_configurations, - ) + tc = TC["boot_configurations"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_configurations(host)) @pytest.mark.sanity -@pytest.mark.order(105) +@pytest.mark.order(30105) def test_boot_service_node_identity(host): """Verify synchronized XNAME-to-bootMac records.""" - _run_check(host, "boot_nodes", check_boot_nodes) + tc = TC["boot_nodes"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_nodes(host)) @pytest.mark.sanity -@pytest.mark.order(106) +@pytest.mark.order(30106) def test_metadata_service_groups(host): """Verify one usable cloud-init template per functional group.""" - _run_check(host, "metadata_groups", check_metadata_groups) + tc = TC["metadata_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_metadata_groups(host)) @pytest.mark.sanity -@pytest.mark.order(107) +@pytest.mark.order(30107) def test_metadata_service_instances(host): """Verify unique per-node hostname metadata.""" - _run_check( - host, - "metadata_instances", - check_metadata_instances, - ) + tc = TC["metadata_instances"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_metadata_instances(host)) @pytest.mark.sanity -@pytest.mark.order(108) +@pytest.mark.order(30108) def test_coredhcp_and_coredns_inventory(host): """Verify the SMD identity records consumed by CoreDHCP/CoreDNS.""" - _run_check(host, "network_inventory", check_network_inventory) + tc = TC["network_inventory"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_network_inventory(host)) -@pytest.mark.order(109) +@pytest.mark.boot_image +@pytest.mark.order(30109) def test_boot_image_identity(host): """Verify Boot Service kernel/initrd paths match build_status.yml.""" - _run_check(host, "boot_image_identity", check_boot_image_identity) + tc = TC["boot_image_identity"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_image_identity(host)) -@pytest.mark.order(110) +@pytest.mark.boot_image +@pytest.mark.order(30110) def test_boot_image_architecture(host): """Verify build_status.yml architecture keys match functional group names.""" - _run_check(host, "boot_image_architecture", check_boot_image_architecture) + tc = TC["boot_image_architecture"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_image_architecture(host)) diff --git a/test/orchestrator/fvt/provision/test_playbook.py b/test/orchestrator/fvt/provision/test_playbook.py index 158579e1a4..4727826fd6 100644 --- a/test/orchestrator/fvt/provision/test_playbook.py +++ b/test/orchestrator/fvt/provision/test_playbook.py @@ -23,7 +23,7 @@ @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(100) +@pytest.mark.order(30000) def test_deploy_provision(host): """Run ``orchestrator.yml --tags provision``.""" tc = TC["deploy_provision"] diff --git a/test/orchestrator/fvt/result.py b/test/orchestrator/fvt/result.py index d8482341cb..14f540035b 100644 --- a/test/orchestrator/fvt/result.py +++ b/test/orchestrator/fvt/result.py @@ -12,7 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test-layer rendering and assertion policy for structured FVT results.""" +"""Test-layer rendering and assertion policy for structured FVT results. + +Each test resolves its registry entry, creates its own ``TestLogger`` start +record, and passes both here. These helpers only render the final record and +enforce the result contract. +""" import pytest from library.functions import TestLogger @@ -25,10 +30,9 @@ from library.vars import TEST_CASES -def _report(case, result, fields, component, messages): +def _report(test_log, result, fields, component, messages): """Render one final result and enforce its success contract.""" log_messages, assert_messages = messages - test_log = TestLogger(case["title"], case["id"]) if result["success"]: test_log.passed_fields( log_messages["check_passed"].format(component=component), @@ -45,12 +49,24 @@ def _report(case, result, fields, component, messages): ) -def verify_precheck(host, key, checker): +def _skip_if_requested(test_log, case, result, fields, log_messages): + """Record and raise a documented skip for an optional capability.""" + if not result.get("skipped"): + return + test_log.skipped_fields( + log_messages["check_skipped"].format(component=case["component"]), + fields, + ) + reason = dict(fields).get("Reason") or (fields[0][1] if fields else "") + pytest.skip(str(reason or f"{case['component']} is not enabled")) + + +def verify_precheck(test_log, case, host, checker): """Execute and enforce one precheck verification result.""" - case = TEST_CASES[key] result = checker(host) + _skip_if_requested(test_log, case, result, result["fields"], PRECHECK_LOG) _report( - case, + test_log, result, result["fields"], case["component"], @@ -58,12 +74,33 @@ def verify_precheck(host, key, checker): ) -def verify_cleanup(host, key, component, checker): +def verify_precheck_rejection(test_log, case, host, checker): + """Require one precheck to reject an invalid condition with an error.""" + result = checker(host) + fields = result["fields"] + _skip_if_requested(test_log, case, result, fields, PRECHECK_LOG) + component = case["component"] + rejected = not result["success"] and bool(result["error"]) + if rejected: + test_log.passed_fields( + PRECHECK_LOG["rejection_passed"].format(component=component), + [*fields, ("Rejection", result["error"])], + ) + else: + test_log.failed_fields( + PRECHECK_LOG["rejection_failed"].format(component=component), + fields, + ) + assert rejected, PRECHECK_ASSERT["rejection_missing"].format( + component=component + ) + + +def verify_cleanup(test_log, host, component, checker): """Execute and enforce one cleanup verification result.""" - case = TEST_CASES[key] result = checker(host) _report( - case, + test_log, result, result["fields"], component, @@ -74,17 +111,12 @@ def verify_cleanup(host, key, component, checker): def verify_pxeboot(host, key, checker): """Execute and enforce one PXE post-boot verification result.""" case = TEST_CASES[key] + test_log = TestLogger(case["title"], case["id"]) result = checker(host) fields = result["details"]["fields"] - if result.get("skipped"): - TestLogger(case["title"], case["id"]).skipped_fields( - PXEBOOT_LOG["check_skipped"].format(component=case["component"]), - fields, - ) - reason = fields[0][1] if fields else f"{case['component']} is not enabled" - pytest.skip(str(reason)) + _skip_if_requested(test_log, case, result, fields, PXEBOOT_LOG) _report( - case, + test_log, result, fields, case["component"], diff --git a/test/orchestrator/nft/README.md b/test/orchestrator/nft/README.md index 82275c9be4..6ac87633df 100644 --- a/test/orchestrator/nft/README.md +++ b/test/orchestrator/nft/README.md @@ -9,22 +9,28 @@ tests. | Order | TC ID | Test | Marker | Contract | |---:|---|---|---|---| -| 10 | `ORCH_NFT_001` | `test_precheck_performance` | `performance` | Precheck completes within its configured threshold. | -| 20 | `ORCH_NFT_002` | `test_prepare_performance` | `performance`, `destructive` | Prepare completes within its configured threshold. | -| 30 | `ORCH_NFT_005` | `test_prepare_idempotency` | `idempotency`, `destructive` | The second prepare has no persistent changes, preserves container identity, and leaves all required OpenCHAMI services ready. | -| 40 | `ORCH_NFT_003` | `test_provision_performance` | `performance`, `destructive` | Provision completes within its configured threshold. | -| 50 | `ORCH_NFT_006` | `test_precheck_idempotency` | `idempotency` | The second precheck succeeds without persistent changes. | -| 60 | `ORCH_NFT_008` | `test_credential_file_permissions` | `security` | The encrypted credential file is root-owned and mode `0600` or `0640`. | -| 61 | `ORCH_NFT_009` | `test_ssh_private_key_permissions` | `security` | `/root/.ssh/oim_rsa` is root-owned and mode `0600`. | -| 62 | `ORCH_NFT_010` | `test_log_file_permissions` | `security` | Orchestrator log files are root-owned and have no permissions for other users. | -| 63 | `ORCH_NFT_011` | `test_vault_encryption` | `security` | Product credentials have a supported Ansible Vault header and a root-owned mode-`0600` key. | -| 90 | `ORCH_NFT_004` | `test_cleanup_performance` | `performance`, `destructive` | Full cleanup completes within its configured threshold. | -| 91 | `ORCH_NFT_007` | `test_cleanup_idempotency` | `idempotency`, `destructive` | Two cleanup executions succeed, the second has no persistent changes, and every cleanup FVT postcondition passes. | +| 60001 | `ORCH_NFT_001` | `test_precheck_performance` | `performance` | Precheck completes within its configured threshold. | +| 60002 | `ORCH_NFT_002` | `test_prepare_performance` | `performance`, `destructive` | Prepare completes within its configured threshold. | +| 60003 | `ORCH_NFT_005` | `test_prepare_idempotency` | `idempotency`, `destructive` | The second prepare has no persistent changes, preserves container identity, and leaves all required OpenCHAMI services ready. | +| 60004 | `ORCH_NFT_003` | `test_provision_performance` | `performance`, `destructive` | Provision completes within its configured threshold. | +| 60005 | `ORCH_NFT_006` | `test_precheck_idempotency` | `idempotency` | The second precheck succeeds without persistent changes. | +| 60006 | `ORCH_NFT_008` | `test_credential_file_permissions` | `security` | The encrypted credential file is root-owned and mode `0600` or `0640`. | +| 60007 | `ORCH_NFT_009` | `test_ssh_private_key_permissions` | `security` | `/root/.ssh/oim_rsa` is root-owned and mode `0600`. | +| 60008 | `ORCH_NFT_010` | `test_log_file_permissions` | `security` | Orchestrator log files are root-owned and have no permissions for other users. | +| 60009 | `ORCH_NFT_011` | `test_vault_encryption` | `security` | Product credentials have a supported Ansible Vault header and a root-owned mode-`0600` key. | +| 60010 | `ORCH_NFT_004` | `test_cleanup_performance` | `performance`, `destructive` | Full cleanup completes within its configured threshold. | +| 60011 | `ORCH_NFT_007` | `test_cleanup_idempotency` | `idempotency`, `destructive` | Two cleanup executions succeed, the second has no persistent changes, and every cleanup FVT postcondition passes. | +| 60012 | `ORCH_NFT_012` | `test_clean_baseline` | `lifecycle`, `destructive` | The OIM starts from a clean baseline before a fresh install. | +| 60013 | `ORCH_NFT_013` | `test_lifecycle_fresh_install` | `lifecycle`, `destructive` | The complete fresh-install lifecycle succeeds from the clean baseline. | +| 60014 | `ORCH_NFT_014` | `test_lifecycle_provision_verify` | `lifecycle` | Provision and node state are valid after the fresh-install lifecycle. | `ORCH_NFT_001` and `ORCH_NFT_006` use the current `precheck` lifecycle. They retain the stable IDs formerly associated with the retired standalone validation operation. +Orders use the NFT block `60000` in `TEST_ORDER_BLOCKS`; the sequence follows +the lifecycle phase each contract exercises. + ## Performance thresholds `test_config.yml` defines the accepted wall-clock limits in seconds: diff --git a/test/orchestrator/nft/result.py b/test/orchestrator/nft/result.py index 2af6926ccf..e1a723bf14 100644 --- a/test/orchestrator/nft/result.py +++ b/test/orchestrator/nft/result.py @@ -14,15 +14,10 @@ """Render and enforce structured Orchestrator NFT results.""" -from library.functions import TestLogger -from library.vars import TEST_CASES - -def verify_nft(host, key, checker, *args): +def verify_nft(test_log, case, host, checker, *args): """Execute one NFT contract, report safe fields, and enforce success.""" - case = TEST_CASES[key] result = checker(host, *args) - test_log = TestLogger(case["title"], case["id"]) if result["success"]: test_log.passed_fields(result["details"], result["fields"]) else: diff --git a/test/orchestrator/nft/test_idempotency.py b/test/orchestrator/nft/test_idempotency.py index 78eab1ab2b..21e9a46d02 100644 --- a/test/orchestrator/nft/test_idempotency.py +++ b/test/orchestrator/nft/test_idempotency.py @@ -16,10 +16,12 @@ import pytest from library.functions import ( + TestLogger, check_cleanup_idempotency, check_precheck_idempotency, check_prepare_idempotency, ) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @@ -27,24 +29,30 @@ @pytest.mark.nft @pytest.mark.idempotency @pytest.mark.destructive -@pytest.mark.order(30) +@pytest.mark.order(60003) def test_prepare_idempotency(host): """Require repeated prepare to preserve runtime identity and readiness.""" - verify_nft(host, "prepare_idempotency", check_prepare_idempotency) + tc = TC["prepare_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_prepare_idempotency) @pytest.mark.nft @pytest.mark.idempotency -@pytest.mark.order(50) +@pytest.mark.order(60005) def test_precheck_idempotency(host): """Require repeated precheck to remain read-only.""" - verify_nft(host, "precheck_idempotency", check_precheck_idempotency) + tc = TC["precheck_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_precheck_idempotency) @pytest.mark.nft @pytest.mark.idempotency @pytest.mark.destructive -@pytest.mark.order(91) +@pytest.mark.order(60011) def test_cleanup_idempotency(host): """Require repeated full cleanup to remain successful and unchanged.""" - verify_nft(host, "cleanup_idempotency", check_cleanup_idempotency) + tc = TC["cleanup_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_cleanup_idempotency) diff --git a/test/orchestrator/nft/test_lifecycle.py b/test/orchestrator/nft/test_lifecycle.py index b0f3e85372..078fb0119c 100644 --- a/test/orchestrator/nft/test_lifecycle.py +++ b/test/orchestrator/nft/test_lifecycle.py @@ -23,10 +23,12 @@ import pytest from library.functions import ( + TestLogger, check_clean_baseline, check_lifecycle_fresh_install, check_lifecycle_provision_verify, ) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @@ -34,24 +36,30 @@ @pytest.mark.nft @pytest.mark.lifecycle @pytest.mark.destructive -@pytest.mark.order(100) +@pytest.mark.order(60012) def test_clean_baseline(host): """Require a provably clean OIM state before the fresh-install lifecycle.""" - verify_nft(host, "clean_baseline", check_clean_baseline) + tc = TC["clean_baseline"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_clean_baseline) @pytest.mark.nft @pytest.mark.lifecycle @pytest.mark.destructive -@pytest.mark.order(101) +@pytest.mark.order(60013) def test_lifecycle_fresh_install(host): """Require the complete fresh-install lifecycle to succeed from clean baseline.""" - verify_nft(host, "lifecycle_fresh_install", check_lifecycle_fresh_install) + tc = TC["lifecycle_fresh_install"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_fresh_install) @pytest.mark.nft @pytest.mark.lifecycle -@pytest.mark.order(102) +@pytest.mark.order(60014) def test_lifecycle_provision_verify(host): """Verify provision state and node connectivity after the fresh-install lifecycle.""" - verify_nft(host, "lifecycle_provision_verify", check_lifecycle_provision_verify) + tc = TC["lifecycle_provision_verify"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_provision_verify) diff --git a/test/orchestrator/nft/test_performance.py b/test/orchestrator/nft/test_performance.py index c852978b54..9f3a176c79 100644 --- a/test/orchestrator/nft/test_performance.py +++ b/test/orchestrator/nft/test_performance.py @@ -15,41 +15,53 @@ """Orchestrator lifecycle-duration contracts.""" import pytest -from library.functions import check_lifecycle_performance +from library.functions import ( + TestLogger, + check_lifecycle_performance, +) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @pytest.mark.nft @pytest.mark.performance -@pytest.mark.order(10) +@pytest.mark.order(60001) def test_precheck_performance(host): """Require precheck to complete within its configured threshold.""" - verify_nft(host, "precheck_performance", check_lifecycle_performance, "precheck") + tc = TC["precheck_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "precheck") @pytest.mark.nft @pytest.mark.performance @pytest.mark.destructive -@pytest.mark.order(20) +@pytest.mark.order(60002) def test_prepare_performance(host): """Require prepare to complete within its configured threshold.""" - verify_nft(host, "prepare_performance", check_lifecycle_performance, "prepare") + tc = TC["prepare_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "prepare") @pytest.mark.nft @pytest.mark.performance @pytest.mark.destructive -@pytest.mark.order(40) +@pytest.mark.order(60004) def test_provision_performance(host): """Require provision to complete within its configured threshold.""" - verify_nft(host, "provision_performance", check_lifecycle_performance, "provision") + tc = TC["provision_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "provision") @pytest.mark.nft @pytest.mark.performance @pytest.mark.destructive -@pytest.mark.order(90) +@pytest.mark.order(60010) def test_cleanup_performance(host): """Require full cleanup to complete within its configured threshold.""" - verify_nft(host, "cleanup_performance", check_lifecycle_performance, "cleanup") + tc = TC["cleanup_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "cleanup") diff --git a/test/orchestrator/nft/test_security.py b/test/orchestrator/nft/test_security.py index a9e73532a1..2731e034bf 100644 --- a/test/orchestrator/nft/test_security.py +++ b/test/orchestrator/nft/test_security.py @@ -16,42 +16,52 @@ import pytest from library.functions import ( + TestLogger, check_credential_file_permissions, check_log_file_permissions, check_ssh_private_key_permissions, check_vault_encryption, ) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(60) +@pytest.mark.order(60006) def test_credential_file_permissions(host): """Require restricted permissions on encrypted product credentials.""" - verify_nft(host, "credential_file_permissions", check_credential_file_permissions) + tc = TC["credential_file_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_credential_file_permissions) @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(61) +@pytest.mark.order(60007) def test_ssh_private_key_permissions(host): """Require root-only access to the OIM private key.""" - verify_nft(host, "ssh_private_key_permissions", check_ssh_private_key_permissions) + tc = TC["ssh_private_key_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_ssh_private_key_permissions) @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(62) +@pytest.mark.order(60008) def test_log_file_permissions(host): """Require Orchestrator logs to remain unavailable to other users.""" - verify_nft(host, "log_file_permissions", check_log_file_permissions) + tc = TC["log_file_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_log_file_permissions) @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(63) +@pytest.mark.order(60009) def test_vault_encryption(host): """Require supported vault encryption and a protected vault key.""" - verify_nft(host, "vault_encryption", check_vault_encryption) + tc = TC["vault_encryption"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_vault_encryption) From fbd1b990034246a36375362acd8cef5c8b960370 Mon Sep 17 00:00:00 2001 From: balajikumaran-c-s Date: Sat, 10 Oct 2026 00:58:38 +0530 Subject: [PATCH 3/3] test(orchestrator): adopt per-test logger contract in pxeboot suites The pxeboot suites now create their own TestLogger, resolve their registry entry, and call verify_pxeboot(test_log, case, host, checker). Each suite moves into its order block (40100 to 42400) so cases run in dependency order, with reboot and node removal suites last. Negative cases leave the sanity marker set. Reboot and node remove/add tests carry the reboot and node_lifecycle markers and are deselected unless --marker selects them, even when the suite is named. Without a marker every other test runs; the runner no longer forces a sanity default. Register the benchmark, boot_image, powervault_* and node_lifecycle markers and drop the disruptive marker. Restore the Kubernetes local etcd media check and control-plane reboot recovery. Move the external LDAP proxy and backend checks from prepare to the slurm_ldap suite ahead of the LDAP login tests, and skip LDAP identity checks when validate_external_ldap is false. Update the README and run configuration reference for the marker rules. Refs #4849 Signed-off-by: balajikumaran-c-s --- test/orchestrator/README.md | 61 ++++-- test/orchestrator/_run.py | 16 +- test/orchestrator/conftest.py | 115 ++++++---- test/orchestrator/docs/test_run_config.md | 22 +- .../fvt/prepare/openldap/test_openldap.py | 42 +--- .../test_additional_cloud_init.py | 42 ++-- .../fvt/pxeboot/cloudinit/test_cloudinit.py | 2 +- .../pxeboot/connectivity/test_connectivity.py | 56 ++--- .../coredns_coredhcp/test_destructive.py | 25 ++- .../pxeboot/coredns_coredhcp/test_nodes.py | 25 ++- .../pxeboot/coredns_coredhcp/test_runtime.py | 86 ++++---- .../kubernetes_cluster/test_cluster_health.py | 32 ++- .../kubernetes_cluster/test_versions.py | 18 +- .../kubernetes_cluster/test_workloads.py | 12 +- .../kubernetes_etcd/test_etcd_health.py | 14 +- .../kubernetes_etcd/test_local_etcd.py | 40 +++- .../kubernetes_recovery/test_reboot.py | 20 +- .../kubernetes_storage/test_storage.py | 40 ++-- .../fvt/pxeboot/minimal_os/test_minimal_os.py | 187 +++++++++------- .../pxeboot/mount_config/test_mount_config.py | 129 +++++++---- .../fvt/pxeboot/powervault/conftest.py | 33 --- .../fvt/pxeboot/powervault/test_binds.py | 60 ++++-- .../fvt/pxeboot/powervault/test_cloudinit.py | 32 ++- .../pxeboot/powervault/test_infrastructure.py | 56 +++-- .../fvt/pxeboot/powervault/test_mounts.py | 44 ++-- .../slurm_apptainer/test_accelerators.py | 32 ++- .../pxeboot/slurm_apptainer/test_images.py | 24 ++- .../fvt/pxeboot/slurm_apptainer/test_jobs.py | 46 ++-- .../pxeboot/slurm_apptainer/test_negative.py | 30 +-- .../pxeboot/slurm_apptainer/test_reboot.py | 23 +- .../pxeboot/slurm_apptainer/test_runtime.py | 64 ++++-- .../slurm_cluster/test_cluster_health.py | 40 ++-- .../slurm_cluster/test_configuration.py | 14 +- .../pxeboot/slurm_cluster/test_discovery.py | 12 +- .../fvt/pxeboot/slurm_dcgm/test_dcgm.py | 162 +++++++++----- .../pxeboot/slurm_dcgm/test_dcgm_negative.py | 56 ++--- .../fvt/pxeboot/slurm_gpu/test_gpu.py | 20 +- .../pxeboot/slurm_hpc_benchmarks/conftest.py | 7 - .../slurm_hpc_benchmarks/test_contracts.py | 87 ++++---- .../slurm_hpc_benchmarks/test_destructive.py | 89 ++++---- .../slurm_hpc_benchmarks/test_pipeline.py | 41 ++-- .../slurm_hpc_benchmarks/test_staging.py | 115 +++++----- .../slurm_infiniband/test_infiniband.py | 14 +- .../fvt/pxeboot/slurm_jobs/test_root_jobs.py | 48 +++-- .../pxeboot/slurm_ldap/test_authentication.py | 85 ++++---- .../pxeboot/slurm_ldap/test_external_ldap.py | 55 +++++ .../pxeboot/slurm_ldap/test_jobs_and_pam.py | 50 ++--- .../slurm_lifecycle/test_slurm_lifecycle.py | 27 ++- .../fvt/pxeboot/slurm_openmpi/test_openmpi.py | 14 +- .../fvt/pxeboot/slurm_recovery/test_reboot.py | 13 +- .../fvt/pxeboot/slurm_ucx/test_ucx.py | 12 +- .../orchestrator/fvt/pxeboot/test_playbook.py | 2 +- .../pxeboot/vast_storage/test_installation.py | 17 +- .../fvt/pxeboot/vast_storage/test_mounts.py | 33 ++- .../pxeboot/vast_storage/test_persistence.py | 16 +- .../fvt/pxeboot/vast_storage/test_scratch.py | 51 ++--- .../pxeboot/vast_storage/test_targeting.py | 41 ++-- test/orchestrator/fvt/result.py | 6 +- .../library/functions/__init__.py | 8 + .../functions/external_ldap_pxeboot_func.py | 201 ++++++++++++++++++ .../functions/openldap_prepare_func.py | 161 +------------- test/orchestrator/library/vars/domain_vars.py | 49 +++-- .../orchestrator/ut/test_prepare_contracts.py | 33 +-- 63 files changed, 1776 insertions(+), 1231 deletions(-) delete mode 100644 test/orchestrator/fvt/pxeboot/powervault/conftest.py create mode 100644 test/orchestrator/fvt/pxeboot/slurm_ldap/test_external_ldap.py create mode 100644 test/orchestrator/library/functions/external_ldap_pxeboot_func.py diff --git a/test/orchestrator/README.md b/test/orchestrator/README.md index 37da13a64d..95bf69e8d2 100644 --- a/test/orchestrator/README.md +++ b/test/orchestrator/README.md @@ -18,10 +18,12 @@ The runner separates product execution from verification: | `verify` | Verify existing state; do not execute the product playbook | | `test` | Execute the selected tag once, then verify only if execution succeeds | -Most verification is observational. Tests that create workloads, download -images, drain scheduler nodes, reboot machines, or delete deployment state -are protected by explicit markers. Cleanup is excluded from every implicit -full-lifecycle run and must be selected by name. +Most verification is observational. Tests that reboot machines run only when +`--marker` selects `reboot`, and node remove/add tests only when it selects +`node_lifecycle`. Other markers narrow test selection but do not authorize +workloads or downloads. Destructive tests retain their separate opt-in, and +cleanup is excluded from every implicit full-lifecycle run and must be selected +by name. ## Prerequisites @@ -138,7 +140,8 @@ without putting secrets in process arguments: } ``` -LDAP identity fields are required by the selected Slurm/PAM LDAP tests. The +LDAP identity fields are required by the selected Slurm/PAM LDAP tests, which +skip unless `validate_external_ldap: true`. The external bind secret is additionally required when `validate_external_ldap: true` and `configure_external_ldap: true`. Product- domain credentials continue to use `--domain-creds-stdin`. @@ -207,15 +210,15 @@ operations and includes destructive prepare, provision, and cleanup cases. | Tag | Suites | Main contract | |---|---|---| -| `precheck` | `environment`, `storage`, `dependencies`, `inputs` | OIM identity, selected NFS reachability, upstream artifacts, and required inputs | -| `prepare` | `openchami`, `network`, `openldap` | OpenCHAMI, PostgreSQL, networking, DNS/DHCP, and LDAP readiness | +| `precheck` | `environment`, `oim_readiness`, `storage`, `dependencies`, `inputs` | OIM identity and readiness, selected NFS reachability, upstream artifacts, and required inputs | +| `prepare` | `openchami`, `network`, `openldap` | OpenCHAMI, PostgreSQL, networking, DNS/DHCP, and local OpenLDAP readiness | | `provision` | `openchami` | Provision reports plus SMD, Boot Service, Metadata Service, and network inventory | -| `pxeboot` | `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer` | Node boot completion and workload-cluster behavior | +| `pxeboot` | See `SUITES` in `library/vars/domain_vars.py` | Node boot completion, node features, and workload-cluster behavior; reboot and node-removal suites run last | | `cleanup` | `openchami`, `openldap`, `slurm`, `kubernetes`, `artifacts`, `credentials` | Explicit full-cleanup postconditions | An untagged FVT flow uses `precheck -> prepare -> provision -> pxeboot`. -Cleanup is always explicit. Untagged verification excludes negative and -disruptive cases. +Cleanup is always explicit. Reboot cases never run without `--marker reboot`, +and node remove/add cases never run without `--marker node_lifecycle`. ### Options @@ -237,6 +240,14 @@ current directories. | Comma | Logical OR | `--marker sanity,functional` | | Plus | Logical AND | `--marker slurm+non_disruptive` | +Without `--marker`, every collected test runs, including functional, +negative, image-download, and scheduler-drain tests, except reboot and node +remove/add tests. Those are deselected (not listed in the run), even with +`--suite`, until `--marker` selects `reboot` or `node_lifecycle`; run them in a +maintenance window. Lifecycle execution cases +still run only in the `exec` phase. Pass a marker such as `--marker sanity` to +narrow the run. + `sanity+functional` selects only tests carrying both markers; it does not mean “run sanity, then functional.” Use `sanity,functional` for that union. @@ -245,12 +256,16 @@ Registered selectors include: - Baseline and capability: `sanity`, `functional`, `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `openldap`, and `apptainer`. - Controlled mutation: `image_download`, `negative`, and `non_disruptive`. -- Maintenance-window operations: `disruptive`, `reboot`, and +- Maintenance-window operations: `reboot` (required to run reboot tests), + `node_lifecycle` (required to run node remove/add tests), and `scheduler_state`. - Cleanup authorization: `destructive`. - Non-functional contracts: `nft`, `performance`, `idempotency`, and `security`. +`sanity` marks the baseline positive checks only. Negative tests, which +expect a rejection or failure, are never marked `sanity`. + `deploy` is attached to lifecycle execution cases and is normally managed by the runner rather than selected manually. @@ -281,9 +296,12 @@ active PXE mapping, or `groups` matches a mapped `GROUP_NAME`. The stock ./run_validation.sh fvt_orchestrator prepare verify --suite openldap ``` -Set `validate_external_ldap: true` to run external proxy and backend checks. -Set `configure_external_ldap: true` only when those checks may also reconcile -the local `omnia_auth` proxy configuration. An unchanged desired configuration +Prepare only starts the local `omnia_auth` container. The external LDAP proxy +and backend checks run first in the pxeboot `slurm_ldap` suite, immediately +before the LDAP login and job checks. Set `validate_external_ldap: true` to run +them and every LDAP-identity check. Set `configure_external_ldap: true` only +when the proxy check may also reconcile the local `omnia_auth` proxy +configuration. An unchanged desired configuration is an idempotent no-op; a failed changed configuration is rolled back. ### Provision @@ -300,7 +318,8 @@ access token. ### PXE boot -PXE verification defaults to `sanity` when no marker is supplied: +PXE verification runs every test except reboot and node remove/add cases when +no marker is supplied. Use `--marker sanity` for the baseline only: ```bash ./run_validation.sh fvt_orchestrator pxeboot test @@ -316,6 +335,7 @@ PXE verification defaults to `sanity` when no marker is supplied: ./run_validation.sh fvt_orchestrator pxeboot verify --suite slurm_hpc_benchmarks ./run_validation.sh fvt_orchestrator pxeboot verify --suite coredns_coredhcp ./run_validation.sh fvt_orchestrator pxeboot verify --suite powervault +./run_validation.sh fvt_orchestrator pxeboot verify --suite vast_storage ``` Focused workload and image examples: @@ -337,13 +357,16 @@ Focused workload and image examples: --suite slurm_apptainer --marker functional+image_download ``` -Run disruptive checks only in an approved maintenance window: +Run reboot and node remove/add checks only in an approved maintenance window: ```bash +./run_validation.sh fvt_orchestrator pxeboot verify --marker reboot +./run_validation.sh fvt_orchestrator pxeboot verify \ + --suite slurm_recovery --marker reboot ./run_validation.sh fvt_orchestrator pxeboot verify \ - --marker disruptive+reboot + --suite slurm_lifecycle --marker node_lifecycle ./run_validation.sh fvt_orchestrator pxeboot verify \ - --suite slurm_jobs --marker disruptive+scheduler_state + --suite slurm_jobs --marker scheduler_state ``` The authorization checks are enforced even when pytest is invoked directly. @@ -384,7 +407,7 @@ artifact permissions: A complete NFT run provisions state and finishes with full cleanup. Duration limits come from `nft_performance_threshold_seconds` in `test_config.yml`. Run NFT separately from FVT cleanup and review the cleanup policy first. See -[nft/README.md](nft/README.md) for its 11 contracts and execution order. +[nft/README.md](nft/README.md) for its 14 contracts and execution order. ## Batch execution diff --git a/test/orchestrator/_run.py b/test/orchestrator/_run.py index 5300ee240a..0267d5d078 100755 --- a/test/orchestrator/_run.py +++ b/test/orchestrator/_run.py @@ -46,20 +46,6 @@ def _runner_all_exec_tags(args, lifecycle_tags): return lifecycle_tags -def _apply_safe_pxeboot_marker_default(args): - """Default PXE verification to positive, non-disruptive sanity checks.""" - normalized = list(args) - if ( - len(normalized) >= 3 - and normalized[0] == "fvt_orchestrator" - and normalized[1] == "pxeboot" - and normalized[2] in {"test", "verify"} - and "--marker" not in normalized - ): - normalized.extend(["--marker", "sanity"]) - return normalized - - def main(): """Load domain config and run ValidationRunner.""" script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -81,7 +67,7 @@ def main(): ) from omnia_auto.functions.validation_runner import ValidationRunner - args = _apply_safe_pxeboot_marker_default(sys.argv[1:]) + args = sys.argv[1:] runner = ValidationRunner( domain=DOMAIN_NAME, script_dir=script_dir, diff --git a/test/orchestrator/conftest.py b/test/orchestrator/conftest.py index 71a5db0804..b248c80668 100644 --- a/test/orchestrator/conftest.py +++ b/test/orchestrator/conftest.py @@ -94,6 +94,7 @@ ) from library.vars import TEST_CASES from omnia_auto import ( + TestLogger, TestReport, add_session_result, build_report_name, @@ -194,6 +195,19 @@ ) +_TC_TITLE_MAP = {test_case["id"]: test_case["title"] for test_case in TEST_CASES.values()} + + +def _skip_reason(result) -> str: + """Return the skip reason recorded by pytest, without its prefix.""" + longrepr = result.longrepr + if isinstance(longrepr, tuple) and len(longrepr) == 3: + text = str(longrepr[2]) + else: + text = str(longrepr or "") + return text.removeprefix("Skipped: ").strip() + + def _registered_test_case_id(item) -> str: """Return a deterministic TC ID without relying on logger state.""" return _TC_ID_MAP.get(item.name, "") @@ -241,9 +255,15 @@ def pytest_configure(config): "kubernetes": "Kubernetes post-boot checks", "slurm": "Slurm post-boot checks", "apptainer": "Apptainer runtime, image, and Slurm integration checks", + "benchmark": "HPC benchmark staging and execution tests", "additional_cloud_init": "Additional cloud-init post-boot verification checks", "mount_config": "NFS mount_config post-boot verification checks", "minimal_os": "Minimal OS validation for OS-only provisioned nodes", + "boot_image": "Provisioned boot image identity and architecture checks", + "powervault_infrastructure": "PowerVault iSCSI and multipath infrastructure checks", + "powervault_mounts": "PowerVault partition, filesystem, and mount checks", + "powervault_binds": "PowerVault bind-mount and targeting checks", + "powervault_cloudinit": "PowerVault cloud-init and setup-log checks", "vast_installation": "VAST NFS client installation tests", "vast_mounts": "VAST mount point and mount option validation", "vast_targeting": "VAST functional group targeting validation", @@ -252,8 +272,10 @@ def pytest_configure(config): "image_download": "Explicitly authorized Apptainer image download checks", "negative": "Expected-failure and rejection behavior checks", "non_disruptive": "Checks that do not reboot or drain cluster nodes", - "disruptive": "Explicitly enabled reboot or scheduler-state checks", - "reboot": "Node reboot and post-reboot recovery checks", + "reboot": "Node reboot checks; run only when --marker selects reboot", + "node_lifecycle": ( + "Node remove/add checks; run only when --marker selects node_lifecycle" + ), "scheduler_state": "Scheduler drain, queue, and resume checks", "destructive": "Explicitly selected destructive cleanup checks", "nft": "Non-functional quality-contract checks", @@ -288,6 +310,14 @@ def _item_has_marker(item, marker_name): return item.get_closest_marker(marker_name) is not None +OPT_IN_MARKERS = ("reboot", "node_lifecycle") + + +def _opt_in_markers(item): + """Return the opt-in markers a test carries.""" + return {name for name in OPT_IN_MARKERS if _item_has_marker(item, name)} + + def pytest_collection_modifyitems(session, config, items): """Filter markers, apply safe defaults, and sort by order marker.""" marker_expr = config.getoption("--marker", default="") @@ -308,35 +338,26 @@ def pytest_collection_modifyitems(session, config, items): "negative", } ) - disruptive_authorized = bool( - explicitly_enabled & {"disruptive", "reboot", "scheduler_state"} - ) - - # Only apply the normal deploy auto-skip if no marker expression is given. + # Without a marker expression every collected test runs except opt-in + # (reboot, node_lifecycle) cases, which are deselected. Deploy cases still + # run only in the exec phase, which creates the verified state. if mode == "none": + selected = [] + deselected = [] for item in items: + if _opt_in_markers(item): + deselected.append(item) + continue if _item_has_marker(item, "deploy") and command_type != "exec": item.add_marker( pytest.mark.skip( "Deploy tests run only during the runner exec phase" ) ) - elif _item_has_marker(item, "disruptive"): - item.add_marker( - pytest.mark.skip( - "Select a disruptive, reboot, or scheduler_state marker " - "to authorize this test" - ) - ) - elif _item_has_marker(item, "functional") and not _item_has_marker( - item, "sanity" - ): - item.add_marker( - pytest.mark.skip( - "Select a functional or workload capability marker " - "to authorize this test" - ) - ) + selected.append(item) + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected else: # Feature filters apply to verification cases. The execution phase # still needs its one deploy test to create the state being verified. @@ -352,8 +373,9 @@ def pytest_collection_modifyitems(session, config, items): else: match = _item_has_marker(item, markers[0]) - if match and _item_has_marker(item, "disruptive"): - match = disruptive_authorized + opt_in = _opt_in_markers(item) + if match and opt_in: + match = bool(opt_in & explicitly_enabled) elif match and _item_has_marker(item, "functional"): match = functional_authorized if ( @@ -386,10 +408,28 @@ def _get_order(item): def pytest_runtest_setup(item): - """Expose only explicitly selected mutation markers to runtime helpers.""" + """Expose mutation authorization to runtime helpers. + + With no marker every gate a test carries is authorized; opt-in cases are + deselected at collection. With a marker expression only the selected + mutation markers are. + """ marker_expr = item.config.getoption("--marker", default="") _mode, markers = _parse_marker_expression(marker_expr) selected = set(markers) + if not selected: + # No marker selected: authorize every gate the test carries. Opt-in + # cases are deselected at collection. + authorized = { + marker + for marker in ("functional", "image_download") + if _item_has_marker(item, marker) + } + if authorized: + os.environ["OMNIA_FVT_AUTHORIZED_MARKERS"] = ",".join(sorted(authorized)) + else: + os.environ.pop("OMNIA_FVT_AUTHORIZED_MARKERS", None) + return sanity_authorized = _item_has_marker(item, "sanity") and ( not selected or "sanity" in selected or "buildstream" in selected ) @@ -407,12 +447,7 @@ def pytest_runtest_setup(item): } ): authorized.add("functional") - if _item_has_marker(item, "disruptive") and selected & { - "disruptive", - "reboot", - "scheduler_state", - }: - authorized.add("disruptive") + authorized |= _opt_in_markers(item) & selected if _item_has_marker(item, "image_download") and ( "image_download" in selected or sanity_authorized ): @@ -578,13 +613,7 @@ def pytest_runtest_makereport(item, call): skip_reason = "" if result.skipped: - if hasattr(result, "wasxfail"): - status = "SKIPPED" - rep_text = str(result.longrepr) if result.longrepr else "" - if "Skipped:" in rep_text: - skip_reason = rep_text.split("Skipped:", 1)[-1].strip() - elif "SKIP" in rep_text: - skip_reason = rep_text.split("SKIP", 1)[-1].strip() + skip_reason = _skip_reason(result) if status == "SKIPPED" and skip_reason: details = (details + "\n" if details else "") + f"SKIPPED: {skip_reason}" @@ -600,11 +629,19 @@ def pytest_runtest_makereport(item, call): if not tc_id: tc_id = get_last_tc_id() + if result.when == "setup": + # Marker and fixture skips happen before the test body creates its + # TestLogger, so record the start and skip lines here. + skip_log = TestLogger(_TC_TITLE_MAP.get(tc_id, item.name), tc_id) + skip_log.skipped(skip_reason or "Skipped before the test started") + details = skip_log.get_output() + (f"\nSKIPPED: {skip_reason}" if skip_reason else "") + add_session_result( test_name=item.name, status=status, duration=getattr(result, "duration", 0), tc_id=tc_id, + reason=skip_reason, ) report = get_current_report() diff --git a/test/orchestrator/docs/test_run_config.md b/test/orchestrator/docs/test_run_config.md index 17a395375a..fcb50c7c4e 100644 --- a/test/orchestrator/docs/test_run_config.md +++ b/test/orchestrator/docs/test_run_config.md @@ -44,10 +44,10 @@ non-zero aggregate result; set it to `true` to stop after the first failure. | Tag | Suites | |---|---| -| `precheck` | `environment`, `storage`, `dependencies`, `inputs` | +| `precheck` | `environment`, `oim_readiness`, `storage`, `dependencies`, `inputs` | | `prepare` | `openchami`, `network`, `openldap` | | `provision` | `openchami` | -| `pxeboot` | `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer`, `slurm_hpc_benchmarks`, `coredns_coredhcp`, `powervault` | +| `pxeboot` | `connectivity`, `cloudinit`, `additional_cloud_init`, `mount_config`, `minimal_os`, `powervault`, `coredns_coredhcp`, `kubernetes_cluster`, `kubernetes_etcd`, `kubernetes_storage`, `slurm_cluster`, `slurm_jobs`, `slurm_ldap`, `slurm_gpu`, `slurm_dcgm`, `slurm_openmpi`, `slurm_ucx`, `slurm_infiniband`, `slurm_apptainer`, `slurm_hpc_benchmarks`, `vast_storage`, `kubernetes_recovery`, `slurm_recovery`, `slurm_lifecycle` | | `cleanup` | `openchami`, `openldap`, `slurm`, `kubernetes`, `artifacts`, `credentials` | Discover the live catalog before editing the batch file: @@ -58,14 +58,24 @@ Discover the live catalog before editing the batch file: ## Markers -Registered markers include `sanity`, `functional`, `openldap`, `connectivity`, -`cloudinit`, `kubernetes`, `slurm`, `apptainer`, `benchmark`, `image_download`, -`negative`, `non_disruptive`, `disruptive`, `reboot`, `scheduler_state`, and -`destructive`. +Registered markers are listed in `MARKERS` in `library/vars/domain_vars.py`. +They include `sanity`, `functional`, `openldap`, `connectivity`, `cloudinit`, +`kubernetes`, `slurm`, `apptainer`, `benchmark`, `additional_cloud_init`, +`mount_config`, `minimal_os`, `boot_image`, the PowerVault selectors +`powervault_infrastructure`, `powervault_mounts`, `powervault_binds`, and +`powervault_cloudinit`, the VAST selectors `vast_installation`, `vast_mounts`, +`vast_scratch`, `vast_targeting`, and `vast_persistence`, `image_download`, +`negative`, `non_disruptive`, `reboot`, `node_lifecycle`, `scheduler_state`, +and `destructive`. Reboot tests run only when the expression selects `reboot`, +and node remove/add tests only when it selects `node_lifecycle`; otherwise they +are deselected. Examples: ```yaml +marker: "" # empty: every test except reboot/node_lifecycle +marker: "reboot" # reboot tests only +marker: "node_lifecycle" # node remove/add tests only marker: "sanity" # one marker marker: "sanity,functional" # OR marker: "slurm+non_disruptive" # AND diff --git a/test/orchestrator/fvt/prepare/openldap/test_openldap.py b/test/orchestrator/fvt/prepare/openldap/test_openldap.py index 3968905e2e..73153efbc2 100644 --- a/test/orchestrator/fvt/prepare/openldap/test_openldap.py +++ b/test/orchestrator/fvt/prepare/openldap/test_openldap.py @@ -3,16 +3,17 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -"""OpenLDAP postconditions produced by Orchestrator prepare.""" +"""Local OpenLDAP postconditions produced by Orchestrator prepare. + +External LDAP proxy cases run in the pxeboot slurm_ldap suite. +""" import pytest from library.functions import TestLogger from library.functions.openldap_prepare_func import ( - check_prepare_external_ldap_backend, check_prepare_openldap_artifacts, check_prepare_openldap_endpoint, check_prepare_openldap_runtime, - reconcile_prepare_external_ldap_proxy, ) from library.messages import ( PREPARE_TEST_ASSERT_MSGS as ASSERT, @@ -49,21 +50,7 @@ def _assert_result(test_log, component, result): @pytest.mark.openldap @pytest.mark.sanity -@pytest.mark.order(8) -def test_external_ldap_proxy(host): - """Reconcile and verify the explicitly enabled LDAP meta-proxy.""" - tc = TC["external_ldap_proxy"] - test_log = TestLogger(tc["title"], tc["id"]) - _assert_result( - test_log, - "External LDAP proxy", - reconcile_prepare_external_ldap_proxy(host), - ) - - -@pytest.mark.openldap -@pytest.mark.sanity -@pytest.mark.order(9) +@pytest.mark.order(20301) def test_openldap_runtime(host): """Verify the enabled service and container are healthy.""" tc = TC["openldap_runtime"] @@ -77,7 +64,7 @@ def test_openldap_runtime(host): @pytest.mark.openldap @pytest.mark.sanity -@pytest.mark.order(10) +@pytest.mark.order(20302) def test_openldap_artifacts(host): """Verify configuration modes, syntax and TLS lifetime.""" tc = TC["openldap_artifacts"] @@ -92,7 +79,7 @@ def test_openldap_artifacts(host): @pytest.mark.openldap @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(11) +@pytest.mark.order(20303) def test_openldap_endpoint(host): """Verify the local LDAP endpoint and published listeners.""" tc = TC["openldap_endpoint"] @@ -102,18 +89,3 @@ def test_openldap_endpoint(host): "OpenLDAP endpoint", check_prepare_openldap_endpoint(host), ) - - -@pytest.mark.openldap -@pytest.mark.functional -@pytest.mark.sanity -@pytest.mark.order(12) -def test_external_ldap_backend(host): - """Verify external LDAP reachability from the omnia_auth container.""" - tc = TC["external_ldap_backend"] - test_log = TestLogger(tc["title"], tc["id"]) - _assert_result( - test_log, - "External LDAP backend", - check_prepare_external_ldap_backend(host), - ) diff --git a/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py b/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py index de65f5e676..e33a17e653 100644 --- a/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py +++ b/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py @@ -27,11 +27,13 @@ import pytest from library.functions import ( + TestLogger, check_additional_cloud_init_metadata_groups, check_additional_cloud_init_runcmd, check_additional_cloud_init_smd_groups, check_additional_cloud_init_write_files, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -40,53 +42,45 @@ @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(300) +@pytest.mark.order(40301) def test_additional_cloud_init_smd_groups(host): """Verify SMD groups exist for additional cloud-init configuration.""" - verify_pxeboot( - host, - "additional_cloud_init_smd_groups", - check_additional_cloud_init_smd_groups, - ) + tc = TC["additional_cloud_init_smd_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_smd_groups) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(301) +@pytest.mark.order(40302) def test_additional_cloud_init_metadata_groups(host): """Verify metadata-service groups and templates for additional cloud-init.""" - verify_pxeboot( - host, - "additional_cloud_init_metadata_groups", - check_additional_cloud_init_metadata_groups, - ) + tc = TC["additional_cloud_init_metadata_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_metadata_groups) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(302) +@pytest.mark.order(40303) def test_additional_cloud_init_write_files(host): """Verify write_files entries were applied on provisioned nodes.""" - verify_pxeboot( - host, - "additional_cloud_init_write_files", - check_additional_cloud_init_write_files, - ) + tc = TC["additional_cloud_init_write_files"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_write_files) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(303) +@pytest.mark.order(40304) def test_additional_cloud_init_runcmd(host): """Verify runcmd entries executed during cloud-init on provisioned nodes.""" - verify_pxeboot( - host, - "additional_cloud_init_runcmd", - check_additional_cloud_init_runcmd, - ) + tc = TC["additional_cloud_init_runcmd"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_runcmd) diff --git a/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py b/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py index 9c773a38f0..6a4a8c218a 100644 --- a/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py +++ b/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py @@ -24,7 +24,7 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.cloudinit -@pytest.mark.order(204) +@pytest.mark.order(40201) def test_node_cloud_init(host): """Verify PXE report freshness and direct cloud-init JSON state.""" tc = TC["node_cloud_init"] diff --git a/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py b/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py index b5580906d3..b0bb106c30 100644 --- a/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py +++ b/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py @@ -23,75 +23,61 @@ check_node_ping, check_node_ssh, ) -from library.messages import PXEBOOT_TEST_ASSERT_MSGS as ASSERT -from library.messages import PXEBOOT_TEST_LOG_MSGS as LOG from library.vars import TEST_CASES as TC - -def _check(host, key, callback): - tc = TC[key] - test_log = TestLogger(tc["title"], tc["id"]) - result = callback(host) - fields = result["details"]["fields"] - if result.get("skipped"): - test_log.passed_fields( - LOG["check_passed"].format(component=tc["component"]), fields - ) - pytest.skip(result.get("error") or "Not applicable to this configuration") - if result["success"]: - test_log.passed_fields( - LOG["check_passed"].format(component=tc["component"]), fields - ) - else: - test_log.failed_fields( - LOG["check_failed"].format(component=tc["component"]), - [*fields, ("Error", result["error"])], - ) - assert result["success"], ASSERT["verification_failed"].format( - component=tc["component"], error=result["error"] - ) +from fvt.result import verify_pxeboot @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(201) +@pytest.mark.order(40101) def test_node_ping(host): """Verify ping from the OIM to every mapped administrative IP.""" - _check(host, "node_ping", check_node_ping) + tc = TC["node_ping"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_ping) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(202) +@pytest.mark.order(40102) def test_node_ssh(host): """Verify passwordless root SSH from the OIM to every mapped node.""" - _check(host, "node_ssh", check_node_ssh) + tc = TC["node_ssh"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_ssh) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(203) +@pytest.mark.order(40103) def test_node_hostname_ssh(host): """Verify mapped hostnames resolve and support passwordless root SSH.""" - _check(host, "node_hostname_ssh", check_node_hostname_ssh) + tc = TC["node_hostname_ssh"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_hostname_ssh) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(298) +@pytest.mark.order(40104) def test_node_architecture(host): """Verify each node's live architecture matches its functional group.""" - _check(host, "node_architecture", check_node_architecture) + tc = TC["node_architecture"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_architecture) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(299) +@pytest.mark.order(40105) def test_node_os_version(host): """Verify each node's live OS version matches its functional group.""" - _check(host, "node_os_version", check_node_os_version) + tc = TC["node_os_version"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_os_version) diff --git a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py index 662b863d07..b19d27e75a 100644 --- a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py +++ b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py @@ -21,30 +21,29 @@ import pytest from library.functions import ( + TestLogger, check_dns_node_addition_pipeline, check_dns_smd_unreachable_cached_resolution, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.destructive] - - @pytest.mark.functional -@pytest.mark.order(360) +@pytest.mark.destructive +@pytest.mark.order(40711) def test_dns_node_addition_pipeline(host): """TC-11: prove SMD-to-CoreDNS pipeline resolves every mapped node.""" - verify_pxeboot( - host, "dns_node_addition_pipeline", check_dns_node_addition_pipeline - ) + tc = TC["dns_node_addition_pipeline"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_node_addition_pipeline) @pytest.mark.functional -@pytest.mark.order(361) +@pytest.mark.destructive +@pytest.mark.order(40712) def test_dns_smd_unreachable_cached_resolution(host): """TC-12: pause SMD briefly; CoreDNS must keep serving cached records.""" - verify_pxeboot( - host, - "dns_smd_unreachable_cached_resolution", - check_dns_smd_unreachable_cached_resolution, - ) + tc = TC["dns_smd_unreachable_cached_resolution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_smd_unreachable_cached_resolution) diff --git a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py index f52d5ad703..24fd0d0d8e 100644 --- a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py +++ b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py @@ -22,29 +22,34 @@ import pytest from library.functions import ( + TestLogger, check_dns_compute_forward_getent, check_dns_compute_resolv_conf, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(358) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(40709) def test_dns_compute_resolv_conf(host): """TC-09: /etc/resolv.conf on every compute has CoreDNS as primary.""" - verify_pxeboot(host, "dns_compute_resolv_conf", check_dns_compute_resolv_conf) + tc = TC["dns_compute_resolv_conf"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_compute_resolv_conf) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(359) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(40710) def test_dns_compute_forward_getent(host): """TC-10: getent hosts on every compute resolves every mapped peer.""" - verify_pxeboot( - host, "dns_compute_forward_getent", check_dns_compute_forward_getent - ) + tc = TC["dns_compute_forward_getent"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_compute_forward_getent) diff --git a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py index 53dc430a88..6298bed164 100644 --- a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py +++ b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py @@ -22,6 +22,7 @@ import pytest from library.functions import ( + TestLogger, check_coredhcp_config_file, check_coredhcp_multisubnet_running_image, check_coredns_container_state, @@ -31,91 +32,98 @@ check_coredns_idempotency, check_coredns_reverse_resolution, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.non_disruptive] - - @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(350) +@pytest.mark.non_disruptive +@pytest.mark.order(40701) def test_coredns_container_state(host): """TC-01: coresmd containers match the dns_enabled dataset (enabled+disabled).""" - verify_pxeboot(host, "coredns_container_state", check_coredns_container_state) + tc = TC["coredns_container_state"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_container_state) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(351) +@pytest.mark.non_disruptive +@pytest.mark.order(40702) def test_coredns_forward_resolution(host): """TC-02: dig FQDN from OIM for every mapped node; compare to ADMIN_IP.""" - verify_pxeboot( - host, "coredns_forward_resolution", check_coredns_forward_resolution - ) + tc = TC["coredns_forward_resolution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_forward_resolution) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(352) +@pytest.mark.non_disruptive +@pytest.mark.order(40703) def test_coredns_reverse_resolution(host): """TC-03: dig -x from OIM for every mapped ADMIN_IP; compare to FQDN.""" - verify_pxeboot( - host, "coredns_reverse_resolution", check_coredns_reverse_resolution - ) - + tc = TC["coredns_reverse_resolution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_reverse_resolution) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(353) +@pytest.mark.non_disruptive +@pytest.mark.order(40704) def test_coredns_dns_forwarders(host): """TC-04: verify DNS forwarders are configured and can resolve external domains.""" - verify_pxeboot( - host, "coredns_dns_forwarders", check_coredns_dns_forwarders - ) + tc = TC["coredns_dns_forwarders"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_dns_forwarders) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(354) +@pytest.mark.non_disruptive +@pytest.mark.order(40705) def test_coredns_corefile_config(host): """TC-05: verify Corefile configuration is correctly rendered.""" - verify_pxeboot( - host, "coredns_corefile_config", check_coredns_corefile_config - ) + tc = TC["coredns_corefile_config"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_corefile_config) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(355) +@pytest.mark.non_disruptive +@pytest.mark.order(40706) def test_coredhcp_config_file(host): """TC-06: verify coredhcp.yaml configuration file is correctly rendered.""" - verify_pxeboot( - host, "coredhcp_config_file", check_coredhcp_config_file - ) + tc = TC["coredhcp_config_file"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredhcp_config_file) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(356) +@pytest.mark.non_disruptive +@pytest.mark.order(40707) def test_coredhcp_multisubnet_running_image(host): """TC-07: multi-subnet dataset -> coresmd containers use expected image.""" - verify_pxeboot( - host, - "coredhcp_multisubnet_running_image", - check_coredhcp_multisubnet_running_image, - ) + tc = TC["coredhcp_multisubnet_running_image"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredhcp_multisubnet_running_image) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(357) +@pytest.mark.non_disruptive +@pytest.mark.order(40708) def test_coredns_idempotency(host): """TC-08: coresmd state stability (no-drift) across a settle window.""" - verify_pxeboot(host, "coredns_idempotency", check_coredns_idempotency) + tc = TC["coredns_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_idempotency) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py index bed06f31f9..ff15c1018c 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py @@ -16,12 +16,14 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_control_plane, check_kubernetes_node_services, check_kubernetes_nodes, check_kubernetes_system_pods, check_kubernetes_virtual_ip, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -29,43 +31,53 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(205) +@pytest.mark.order(40801) def test_kubernetes_nodes(host): """Verify mapped Kubernetes membership and Ready state.""" - verify_pxeboot(host, "kubernetes_nodes", check_kubernetes_nodes) + tc = TC["kubernetes_nodes"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_nodes) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(206) +@pytest.mark.order(40802) def test_kubernetes_node_services(host): """Verify required services on every Kubernetes role.""" - verify_pxeboot(host, "kubernetes_services", check_kubernetes_node_services) + tc = TC["kubernetes_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_node_services) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(209) +@pytest.mark.order(40805) def test_kubernetes_control_plane(host): """Verify API readiness and the configured control plane.""" - verify_pxeboot(host, "kubernetes_control_plane", check_kubernetes_control_plane) + tc = TC["kubernetes_control_plane"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_control_plane) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(210) +@pytest.mark.order(40806) def test_kubernetes_system_pods(host): """Verify required system, CNI, storage, and HA workloads.""" - verify_pxeboot(host, "kubernetes_system_pods", check_kubernetes_system_pods) + tc = TC["kubernetes_system_pods"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_system_pods) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(211) +@pytest.mark.order(40807) def test_kubernetes_virtual_ip(host): """Verify exactly one owner for the configured Kubernetes VIP.""" - verify_pxeboot(host, "kubernetes_virtual_ip", check_kubernetes_virtual_ip) + tc = TC["kubernetes_virtual_ip"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_virtual_ip) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py index 6aa8e94107..80661e8a62 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_configured_versions, check_kubernetes_version_compatibility, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -26,20 +28,20 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(207) +@pytest.mark.order(40803) def test_kubernetes_version_compatibility(host): """Verify Kubernetes, kubeadm, and CRI-O version alignment.""" - verify_pxeboot(host, "kubernetes_versions", check_kubernetes_version_compatibility) + tc = TC["kubernetes_versions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_version_compatibility) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(208) +@pytest.mark.order(40804) def test_kubernetes_configured_versions(host): """Verify deployed component versions against the selected catalog.""" - verify_pxeboot( - host, - "kubernetes_configured_versions", - check_kubernetes_configured_versions, - ) + tc = TC["kubernetes_configured_versions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_configured_versions) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py index 2b2762dee7..3fbd7653b4 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py @@ -15,7 +15,11 @@ """Kubernetes workload scheduling contract.""" import pytest -from library.functions import check_kubernetes_workload_scheduling +from library.functions import ( + TestLogger, + check_kubernetes_workload_scheduling, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -24,7 +28,9 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(212) +@pytest.mark.order(40808) def test_kubernetes_workload_scheduling(host): """Create, verify, and remove an isolated scheduling probe.""" - verify_pxeboot(host, "kubernetes_workload", check_kubernetes_workload_scheduling) + tc = TC["kubernetes_workload"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_workload_scheduling) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py index 80e5aeddf8..40d7b9ca3e 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_etcd_health, check_kubernetes_etcd_topology, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -26,16 +28,20 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(213) +@pytest.mark.order(40901) def test_kubernetes_etcd_health(host): """Verify health for all etcd endpoints.""" - verify_pxeboot(host, "kubernetes_etcd_health", check_kubernetes_etcd_health) + tc = TC["kubernetes_etcd_health"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_etcd_health) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(214) +@pytest.mark.order(40902) def test_kubernetes_etcd_topology(host): """Verify etcd membership, leader election, and raft consistency.""" - verify_pxeboot(host, "kubernetes_etcd_topology", check_kubernetes_etcd_topology) + tc = TC["kubernetes_etcd_topology"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_etcd_topology) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py index cfc8153779..4b9d7f6f57 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py @@ -12,13 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Kubernetes local-etcd mount and disk safety contracts.""" +"""Kubernetes local-etcd mount, provisioning, and disk-media contracts.""" import pytest from library.functions import ( + TestLogger, check_kubernetes_local_etcd, check_kubernetes_local_etcd_integrity, + check_kubernetes_local_etcd_media, + check_kubernetes_local_etcd_provisioning, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -26,20 +30,38 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(215) +@pytest.mark.order(40903) def test_kubernetes_local_etcd(host): """Verify each control plane has the configured etcd mount.""" - verify_pxeboot(host, "kubernetes_local_etcd", check_kubernetes_local_etcd) + tc = TC["kubernetes_local_etcd"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(216) +@pytest.mark.order(40904) def test_kubernetes_local_etcd_integrity(host): """Verify disk selection, ext4 label, UUID fstab, and boot persistence.""" - verify_pxeboot( - host, - "kubernetes_local_etcd_integrity", - check_kubernetes_local_etcd_integrity, - ) + tc = TC["kubernetes_local_etcd_integrity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_integrity) + + +@pytest.mark.kubernetes +@pytest.mark.order(40905) +def test_kubernetes_local_etcd_provisioning(host): + """Verify GPT selection and local-etcd scripts, logs, and selected disk.""" + tc = TC["kubernetes_local_etcd_provisioning"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_provisioning) + + +@pytest.mark.kubernetes +@pytest.mark.order(40906) +def test_kubernetes_local_etcd_media(host): + """Verify every control plane places etcd on the configured disk media.""" + tc = TC["kubernetes_local_etcd_media"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_media) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py b/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py index 21b7d9e62b..ccac59568a 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py @@ -16,30 +16,30 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_control_plane_recovery, check_kubernetes_local_etcd_recovery, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -@pytest.mark.disruptive @pytest.mark.reboot @pytest.mark.kubernetes -@pytest.mark.order(224) +@pytest.mark.order(42201) def test_kubernetes_local_etcd_recovery(host): """Reboot a control plane and prove its local-etcd UUID is preserved.""" - verify_pxeboot( - host, - "kubernetes_local_etcd_recovery", - check_kubernetes_local_etcd_recovery, - ) + tc = TC["kubernetes_local_etcd_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_recovery) -@pytest.mark.disruptive @pytest.mark.reboot @pytest.mark.kubernetes -@pytest.mark.order(225) +@pytest.mark.order(42202) def test_kubernetes_control_plane_recovery(host): """Reboot the VIP owner and verify control-plane recovery.""" - verify_pxeboot(host, "kubernetes_recovery", check_kubernetes_control_plane_recovery) + tc = TC["kubernetes_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_control_plane_recovery) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py b/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py index ba64d37594..da01275806 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py @@ -16,12 +16,14 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_csi_dynamic_provisioning, check_kubernetes_default_storage_class, check_kubernetes_nfs_dynamic_provisioning, check_kubernetes_snapshot_controller, check_kubernetes_storage, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -29,53 +31,55 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(218) +@pytest.mark.order(41001) def test_kubernetes_storage(host): """Verify configured NFS and PowerScale storage objects.""" - verify_pxeboot(host, "kubernetes_storage", check_kubernetes_storage) + tc = TC["kubernetes_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_storage) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(219) +@pytest.mark.order(41002) def test_kubernetes_default_storage_class(host): """Verify exactly one expected default StorageClass.""" - verify_pxeboot( - host, "kubernetes_default_storage", check_kubernetes_default_storage_class - ) + tc = TC["kubernetes_default_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_default_storage_class) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(220) +@pytest.mark.order(41003) def test_kubernetes_snapshot_controller(host): """Verify PowerScale snapshot components when configured.""" - verify_pxeboot( - host, "kubernetes_snapshot_controller", check_kubernetes_snapshot_controller - ) + tc = TC["kubernetes_snapshot_controller"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_snapshot_controller) @pytest.mark.functional @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(222) +@pytest.mark.order(41004) def test_kubernetes_nfs_dynamic_provisioning(host): """Create and remove an isolated NFS-backed workload.""" - verify_pxeboot( - host, "kubernetes_nfs_dynamic", check_kubernetes_nfs_dynamic_provisioning - ) + tc = TC["kubernetes_nfs_dynamic"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_nfs_dynamic_provisioning) @pytest.mark.functional @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(223) +@pytest.mark.order(41005) def test_kubernetes_csi_dynamic_provisioning(host): """Create and remove an isolated PowerScale-backed workload.""" - verify_pxeboot( - host, "kubernetes_csi_dynamic", check_kubernetes_csi_dynamic_provisioning - ) + tc = TC["kubernetes_csi_dynamic"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_csi_dynamic_provisioning) diff --git a/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py b/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py index 6714a60063..8f3990d0b0 100644 --- a/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py +++ b/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py @@ -24,6 +24,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_minimal_os_additional_packages, check_minimal_os_additional_packages_fallback, check_minimal_os_base_packages, @@ -40,143 +41,169 @@ check_minimal_os_required_services, check_minimal_os_ssh_key_access, ) +from library.vars import TEST_CASES as TC -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.minimal_os, -] - -@pytest.mark.order(600) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40501) def test_minimal_os_base_packages(host): """Verify base OS packages are installed on all OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_base_packages", check_minimal_os_base_packages - ) + tc = TC["minimal_os_base_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_base_packages) -@pytest.mark.order(601) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40502) def test_minimal_os_ldms_packages(host): """Verify LDMS monitoring packages and binary on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_ldms_packages", check_minimal_os_ldms_packages - ) + tc = TC["minimal_os_ldms_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_ldms_packages) -@pytest.mark.order(602) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40503) def test_minimal_os_required_services(host): """Verify required services are active on all OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_required_services", check_minimal_os_required_services - ) + tc = TC["minimal_os_required_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_required_services) -@pytest.mark.order(604) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40504) def test_minimal_os_excluded_packages(host): """Verify workload-specific packages are NOT installed on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_excluded_packages", check_minimal_os_excluded_packages - ) + tc = TC["minimal_os_excluded_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_excluded_packages) -@pytest.mark.order(605) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40505) def test_minimal_os_excluded_services(host): """Verify workload-specific services are NOT active on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_excluded_services", check_minimal_os_excluded_services - ) + tc = TC["minimal_os_excluded_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_excluded_services) -@pytest.mark.order(606) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40506) def test_minimal_os_package_manager(host): """Verify package manager is functional on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_package_manager", check_minimal_os_package_manager - ) + tc = TC["minimal_os_package_manager"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_package_manager) -@pytest.mark.order(607) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40507) def test_minimal_os_kernel_version(host): """Verify kernel version is consistent across OS-only nodes per FG.""" - verify_pxeboot( - host, "minimal_os_kernel_version", check_minimal_os_kernel_version - ) + tc = TC["minimal_os_kernel_version"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_kernel_version) -@pytest.mark.order(608) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40508) def test_minimal_os_network_identity(host): """Verify admin IP is configured on all OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_network_identity", check_minimal_os_network_identity - ) + tc = TC["minimal_os_network_identity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_network_identity) -@pytest.mark.order(609) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40509) def test_minimal_os_functional_group_schema(host): """Verify functional-group definitions are valid on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_functional_group_schema", - check_minimal_os_functional_group_schema, - ) + tc = TC["minimal_os_functional_group_schema"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_functional_group_schema) -@pytest.mark.order(610) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40510) def test_minimal_os_additional_packages(host): """Verify configured extra packages are installed on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_additional_packages", - check_minimal_os_additional_packages, - ) + tc = TC["minimal_os_additional_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_additional_packages) -@pytest.mark.order(611) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40511) def test_minimal_os_additional_packages_fallback(host): """Verify absent additional_packages config is handled on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_additional_packages_fallback", - check_minimal_os_additional_packages_fallback, - ) + tc = TC["minimal_os_additional_packages_fallback"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_additional_packages_fallback) -@pytest.mark.order(612) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40512) def test_minimal_os_ldms_service_state(host): """Verify LDMS service is installed but inactive at handoff.""" - verify_pxeboot( - host, - "minimal_os_ldms_service_state", - check_minimal_os_ldms_service_state, - ) + tc = TC["minimal_os_ldms_service_state"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_ldms_service_state) -@pytest.mark.order(613) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40513) def test_minimal_os_ssh_key_access(host): """Verify SSH key authentication is proven on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_ssh_key_access", - check_minimal_os_ssh_key_access, - ) + tc = TC["minimal_os_ssh_key_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_ssh_key_access) -@pytest.mark.order(614) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40514) def test_minimal_os_network_isolation(host): """Verify default route is on management network on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_network_isolation", - check_minimal_os_network_isolation, - ) + tc = TC["minimal_os_network_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_network_isolation) -@pytest.mark.order(615) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40515) def test_minimal_os_no_embedded_credentials(host): """Verify no plaintext secrets are present in the OS image.""" - verify_pxeboot( - host, - "minimal_os_no_embedded_credentials", - check_minimal_os_no_embedded_credentials, - ) + tc = TC["minimal_os_no_embedded_credentials"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_no_embedded_credentials) diff --git a/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py b/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py index 89dedd9bec..627af85537 100644 --- a/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py +++ b/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py @@ -25,6 +25,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_mount_config_bind_fstab, check_mount_config_bind_mounts, check_mount_config_fg_targeting, @@ -38,93 +39,135 @@ check_mount_config_volume_mounted, check_mount_config_writable, ) +from library.vars import TEST_CASES as TC -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.mount_config, -] - - -@pytest.mark.order(500) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40401) def test_mount_config_mount_point(host): """Verify NFS mount point directories exist on all target nodes.""" - verify_pxeboot(host, "mount_config_mount_point", check_mount_config_mount_point) + tc = TC["mount_config_mount_point"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_mount_point) -@pytest.mark.order(501) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40402) def test_mount_config_volume_mounted(host): """Verify NFS volumes are actively mounted on all target nodes.""" - verify_pxeboot( - host, "mount_config_volume_mounted", check_mount_config_volume_mounted - ) + tc = TC["mount_config_volume_mounted"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_volume_mounted) -@pytest.mark.order(502) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40403) def test_mount_config_mount_options(host): """Verify NFS mount options match storage_config.yml on target nodes.""" - verify_pxeboot( - host, "mount_config_mount_options", check_mount_config_mount_options - ) + tc = TC["mount_config_mount_options"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_mount_options) -@pytest.mark.order(503) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40404) def test_mount_config_fstab(host): """Verify NFS fstab entries are persistent on all target nodes.""" - verify_pxeboot(host, "mount_config_fstab", check_mount_config_fstab) + tc = TC["mount_config_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_fstab) -@pytest.mark.order(504) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40405) def test_mount_config_bind_mounts(host): """Verify NFS bind mount targets are active on all target nodes.""" - verify_pxeboot(host, "mount_config_bind_mounts", check_mount_config_bind_mounts) + tc = TC["mount_config_bind_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_bind_mounts) -@pytest.mark.order(505) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40406) def test_mount_config_bind_fstab(host): """Verify NFS bind mount fstab entries are persistent on target nodes.""" - verify_pxeboot(host, "mount_config_bind_fstab", check_mount_config_bind_fstab) + tc = TC["mount_config_bind_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_bind_fstab) -@pytest.mark.order(506) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40407) def test_mount_config_node_subdirectory(host): """Verify per-node subdirectory exists under NFS mount point.""" - verify_pxeboot( - host, - "mount_config_node_subdirectory", - check_mount_config_node_subdirectory, - ) + tc = TC["mount_config_node_subdirectory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_node_subdirectory) -@pytest.mark.order(507) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40408) def test_mount_config_permissions(host): """Verify NFS mount permissions match storage_config.yml on nodes.""" - verify_pxeboot(host, "mount_config_permissions", check_mount_config_permissions) + tc = TC["mount_config_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_permissions) -@pytest.mark.order(508) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40409) def test_mount_config_fg_targeting(host): """Verify NFS mounts are present on target FGs and absent on others.""" - verify_pxeboot(host, "mount_config_fg_targeting", check_mount_config_fg_targeting) + tc = TC["mount_config_fg_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_fg_targeting) -@pytest.mark.order(509) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40410) def test_mount_config_no_duplicate_fstab(host): """Verify no duplicate NFS fstab entries on target nodes.""" - verify_pxeboot( - host, - "mount_config_no_duplicate_fstab", - check_mount_config_no_duplicate_fstab, - ) + tc = TC["mount_config_no_duplicate_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_no_duplicate_fstab) -@pytest.mark.order(510) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40411) def test_mount_config_writable(host): """Verify all configured NFS mounts are writable on target nodes.""" - verify_pxeboot(host, "mount_config_writable", check_mount_config_writable) + tc = TC["mount_config_writable"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_writable) -@pytest.mark.order(511) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40412) def test_mount_config_oim_mount(host): """Verify NFS storage is mounted on the OIM when mount_on_oim is true.""" - verify_pxeboot(host, "mount_config_oim_mount", check_mount_config_oim_mount) + tc = TC["mount_config_oim_mount"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_oim_mount) diff --git a/test/orchestrator/fvt/pxeboot/powervault/conftest.py b/test/orchestrator/fvt/pxeboot/powervault/conftest.py deleted file mode 100644 index efccdb72a2..0000000000 --- a/test/orchestrator/fvt/pxeboot/powervault/conftest.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Pytest fixtures for PowerVault iSCSI storage verification.""" - -import pytest - - -def pytest_configure(config): - """Register custom markers for PowerVault tests.""" - config.addinivalue_line( - "markers", "powervault_infrastructure: iSCSI and multipath infrastructure tests" - ) - config.addinivalue_line( - "markers", "powervault_mounts: partition, filesystem, and mount validation" - ) - config.addinivalue_line( - "markers", "powervault_binds: bind mount and targeting validation" - ) - config.addinivalue_line( - "markers", "powervault_cloudinit: cloud-init and log validation" - ) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_binds.py b/test/orchestrator/fvt/pxeboot/powervault/test_binds.py index 9db9a2cbae..40784e4404 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_binds.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_binds.py @@ -18,6 +18,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_node_subdirectory, check_powervault_bind_mounts, check_powervault_bind_fstab_entries, @@ -28,88 +29,103 @@ check_powervault_slurm_mandatory_bind_mounts, check_powervault_mysql_data_on_mount, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(316) +@pytest.mark.order(40616) @pytest.mark.powervault_binds def test_powervault_node_subdirectory(host): """Verify per-node subdirectory exists under mount point.""" - verify_pxeboot(host, "powervault_node_subdirectory", check_powervault_node_subdirectory) + tc = TC["powervault_node_subdirectory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_node_subdirectory) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(317) +@pytest.mark.order(40617) @pytest.mark.powervault_binds def test_powervault_bind_mounts(host): """Verify bind mount targets are active on all target nodes.""" - verify_pxeboot(host, "powervault_bind_mounts", check_powervault_bind_mounts) + tc = TC["powervault_bind_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_mounts) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(318) +@pytest.mark.order(40618) @pytest.mark.powervault_binds def test_powervault_bind_fstab_entries(host): """Verify bind mount fstab entries are persistent on all target nodes.""" - verify_pxeboot(host, "powervault_bind_fstab_entries", check_powervault_bind_fstab_entries) + tc = TC["powervault_bind_fstab_entries"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_fstab_entries) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(319) +@pytest.mark.order(40619) @pytest.mark.powervault_binds def test_powervault_bind_isolation(host): """Verify per-node data separation via bind mounts.""" - verify_pxeboot(host, "powervault_bind_isolation", check_powervault_bind_isolation) + tc = TC["powervault_bind_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_isolation) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(320) +@pytest.mark.order(40620) @pytest.mark.powervault_binds def test_powervault_functional_group_targeting(host): """Verify PV mount only on correct functional groups.""" - verify_pxeboot(host, "powervault_functional_group_targeting", check_powervault_functional_group_targeting) + tc = TC["powervault_functional_group_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_functional_group_targeting) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(321) +@pytest.mark.order(40621) @pytest.mark.powervault_binds def test_powervault_multiple_prefix_targeting(host): """Verify multiple prefixes target all groups correctly.""" - verify_pxeboot(host, "powervault_multiple_prefix_targeting", check_powervault_multiple_prefix_targeting) + tc = TC["powervault_multiple_prefix_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multiple_prefix_targeting) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(328) +@pytest.mark.order(40628) @pytest.mark.powervault_binds def test_powervault_bind_io(host): """Verify bind-mount I/O reaches the PV backing store.""" - verify_pxeboot(host, "powervault_bind_io", check_powervault_bind_io) + tc = TC["powervault_bind_io"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_io) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(329) +@pytest.mark.order(40629) @pytest.mark.powervault_binds def test_powervault_slurm_mandatory_bind_mounts(host): """Verify /var/lib/mysql and /var/spool/slurm configured as bind targets.""" - verify_pxeboot( - host, - "powervault_slurm_mandatory_bind_mounts", - check_powervault_slurm_mandatory_bind_mounts, - ) + tc = TC["powervault_slurm_mandatory_bind_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_slurm_mandatory_bind_mounts) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(330) +@pytest.mark.order(40630) @pytest.mark.powervault_binds def test_powervault_mysql_data_on_mount(host): """Verify MySQL/MariaDB datadir is on a PowerVault mount.""" - verify_pxeboot(host, "powervault_mysql_data_on_mount", check_powervault_mysql_data_on_mount) + tc = TC["powervault_mysql_data_on_mount"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_mysql_data_on_mount) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py b/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py index f4188acfce..7c90c7e616 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py @@ -18,54 +18,66 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_setup_log, check_powervault_cloud_init_groups_dict, check_powervault_no_duplicate_fstab, check_powervault_all_mounts_writable, check_powervault_permissions, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(322) +@pytest.mark.order(40622) @pytest.mark.powervault_cloudinit def test_powervault_setup_log(host): """Verify cloud-init runcmd log exists and shows completion.""" - verify_pxeboot(host, "powervault_setup_log", check_powervault_setup_log) + tc = TC["powervault_setup_log"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_setup_log) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(323) +@pytest.mark.order(40623) @pytest.mark.powervault_cloudinit def test_powervault_cloud_init_groups_dict(host): """Verify rendered iSCSI setup scripts deployed on target nodes.""" - verify_pxeboot(host, "powervault_cloud_init_groups_dict", check_powervault_cloud_init_groups_dict) + tc = TC["powervault_cloud_init_groups_dict"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_cloud_init_groups_dict) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(324) +@pytest.mark.order(40624) @pytest.mark.powervault_cloudinit def test_powervault_no_duplicate_fstab(host): """Verify no duplicate fstab entries on all target nodes.""" - verify_pxeboot(host, "powervault_no_duplicate_fstab", check_powervault_no_duplicate_fstab) + tc = TC["powervault_no_duplicate_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_no_duplicate_fstab) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(325) +@pytest.mark.order(40625) @pytest.mark.powervault_cloudinit def test_powervault_all_mounts_writable(host): """Verify all PV mounts (main + bind) are writable.""" - verify_pxeboot(host, "powervault_all_mounts_writable", check_powervault_all_mounts_writable) + tc = TC["powervault_all_mounts_writable"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_all_mounts_writable) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(326) +@pytest.mark.order(40626) @pytest.mark.powervault_cloudinit def test_powervault_permissions(host): """Verify permissions on mount point match config.""" - verify_pxeboot(host, "powervault_permissions", check_powervault_permissions) + tc = TC["powervault_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_permissions) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py b/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py index d695219703..08ba7f567d 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py @@ -18,6 +18,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_iscsi_service, check_powervault_iscsi_initiator_name, check_powervault_iscsi_discovery, @@ -28,84 +29,103 @@ check_powervault_multipath_device, check_powervault_multipath_redundancy, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(301) +@pytest.mark.order(40601) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_service(host): """Verify iscsid is active and enabled on all PowerVault target nodes.""" - verify_pxeboot(host, "powervault_iscsi_service", check_powervault_iscsi_service) + tc = TC["powervault_iscsi_service"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_service) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(302) +@pytest.mark.order(40602) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_initiator_name(host): """Verify iSCSI initiator name matches config on all target nodes.""" - verify_pxeboot(host, "powervault_iscsi_initiator_name", check_powervault_iscsi_initiator_name) + tc = TC["powervault_iscsi_initiator_name"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_initiator_name) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(303) +@pytest.mark.order(40603) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_discovery(host): """Verify iSCSI target discovery succeeds from all portal IPs.""" - verify_pxeboot(host, "powervault_iscsi_discovery", check_powervault_iscsi_discovery) + tc = TC["powervault_iscsi_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_discovery) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(304) +@pytest.mark.order(40604) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_sessions(host): """Verify iSCSI sessions are active on all target nodes.""" - verify_pxeboot(host, "powervault_iscsi_sessions", check_powervault_iscsi_sessions) + tc = TC["powervault_iscsi_sessions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_sessions) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(305) +@pytest.mark.order(40605) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_startup_automatic(host): """Verify iSCSI node startup is automatic on all target nodes.""" - verify_pxeboot(host, "powervault_iscsi_startup_automatic", check_powervault_iscsi_startup_automatic) + tc = TC["powervault_iscsi_startup_automatic"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_startup_automatic) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(306) +@pytest.mark.order(40606) @pytest.mark.powervault_infrastructure def test_powervault_portal_reachability(host): """Verify iSCSI portal ports are reachable and sessions healthy.""" - verify_pxeboot(host, "powervault_portal_reachability", check_powervault_portal_reachability) + tc = TC["powervault_portal_reachability"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_portal_reachability) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(307) +@pytest.mark.order(40607) @pytest.mark.powervault_infrastructure def test_powervault_multipath_service(host): """Verify multipathd is active and enabled on all target nodes.""" - verify_pxeboot(host, "powervault_multipath_service", check_powervault_multipath_service) + tc = TC["powervault_multipath_service"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multipath_service) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(308) +@pytest.mark.order(40608) @pytest.mark.powervault_infrastructure def test_powervault_multipath_device(host): """Verify multipath device exists and matches volume_id.""" - verify_pxeboot(host, "powervault_multipath_device", check_powervault_multipath_device) + tc = TC["powervault_multipath_device"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multipath_device) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(309) +@pytest.mark.order(40609) @pytest.mark.powervault_infrastructure def test_powervault_multipath_redundancy(host): """Verify multipath device has multiple paths for redundancy.""" - verify_pxeboot(host, "powervault_multipath_redundancy", check_powervault_multipath_redundancy) + tc = TC["powervault_multipath_redundancy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multipath_redundancy) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py b/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py index 76ad20834c..d3c25b8ac5 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py @@ -18,6 +18,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_gpt_partition, check_powervault_filesystem_type, check_powervault_mount_point_directory, @@ -26,66 +27,81 @@ check_powervault_fstab_entry, check_powervault_io_write_read, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(310) +@pytest.mark.order(40610) @pytest.mark.powervault_mounts def test_powervault_gpt_partition(host): """Verify GPT partition exists on multipath device.""" - verify_pxeboot(host, "powervault_gpt_partition", check_powervault_gpt_partition) + tc = TC["powervault_gpt_partition"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_gpt_partition) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(311) +@pytest.mark.order(40611) @pytest.mark.powervault_mounts def test_powervault_filesystem_type(host): """Verify filesystem formatted with correct type.""" - verify_pxeboot(host, "powervault_filesystem_type", check_powervault_filesystem_type) + tc = TC["powervault_filesystem_type"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_filesystem_type) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(312) +@pytest.mark.order(40612) @pytest.mark.powervault_mounts def test_powervault_mount_point_directory(host): """Verify mount point directory exists on all target nodes.""" - verify_pxeboot(host, "powervault_mount_point_directory", check_powervault_mount_point_directory) + tc = TC["powervault_mount_point_directory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_mount_point_directory) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(313) +@pytest.mark.order(40613) @pytest.mark.powervault_mounts def test_powervault_volume_mounted(host): """Verify PowerVault volume is actively mounted on all target nodes.""" - verify_pxeboot(host, "powervault_volume_mounted", check_powervault_volume_mounted) + tc = TC["powervault_volume_mounted"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_volume_mounted) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(314) +@pytest.mark.order(40614) @pytest.mark.powervault_mounts def test_powervault_mount_options(host): """Verify mount options applied correctly on all target nodes.""" - verify_pxeboot(host, "powervault_mount_options", check_powervault_mount_options) + tc = TC["powervault_mount_options"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_mount_options) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(315) +@pytest.mark.order(40615) @pytest.mark.powervault_mounts def test_powervault_fstab_entry(host): """Verify persistent fstab entry created on all target nodes.""" - verify_pxeboot(host, "powervault_fstab_entry", check_powervault_fstab_entry) + tc = TC["powervault_fstab_entry"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_fstab_entry) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(327) +@pytest.mark.order(40627) @pytest.mark.powervault_mounts def test_powervault_io_write_read(host): """Verify write-read I/O succeeds on every PV mount point.""" - verify_pxeboot(host, "powervault_io_write_read", check_powervault_io_write_read) + tc = TC["powervault_io_write_read"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_io_write_read) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py index 433607a5a8..93f02d21ce 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py @@ -16,12 +16,14 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_cuda_workload, check_apptainer_gpu_access, check_apptainer_gpu_count, check_apptainer_gpu_memory, check_apptainer_infiniband, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -31,10 +33,12 @@ @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(290) +@pytest.mark.order(41925) def test_apptainer_gpu_access(host): """Verify scheduler-declared GPU nodes expose GPUs in the container.""" - verify_pxeboot(host, "apptainer_gpu_access", check_apptainer_gpu_access) + tc = TC["apptainer_gpu_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_gpu_access) @pytest.mark.apptainer @@ -42,10 +46,12 @@ def test_apptainer_gpu_access(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(291) +@pytest.mark.order(41926) def test_apptainer_gpu_count(host): """Verify each container sees the same GPU count as its host.""" - verify_pxeboot(host, "apptainer_gpu_count", check_apptainer_gpu_count) + tc = TC["apptainer_gpu_count"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_gpu_count) @pytest.mark.apptainer @@ -53,10 +59,12 @@ def test_apptainer_gpu_count(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(292) +@pytest.mark.order(41927) def test_apptainer_cuda_workload(host): """Execute a bounded NVIDIA device query in each GPU container.""" - verify_pxeboot(host, "apptainer_cuda_workload", check_apptainer_cuda_workload) + tc = TC["apptainer_cuda_workload"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_cuda_workload) @pytest.mark.apptainer @@ -64,10 +72,12 @@ def test_apptainer_cuda_workload(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(293) +@pytest.mark.order(41928) def test_apptainer_gpu_memory(host): """Verify the GPU query leaves no material device-memory allocation.""" - verify_pxeboot(host, "apptainer_gpu_memory", check_apptainer_gpu_memory) + tc = TC["apptainer_gpu_memory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_gpu_memory) @pytest.mark.apptainer @@ -75,7 +85,9 @@ def test_apptainer_gpu_memory(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(294) +@pytest.mark.order(41929) def test_apptainer_infiniband(host): """Verify mapped compute nodes expose InfiniBand devices in containers.""" - verify_pxeboot(host, "apptainer_infiniband", check_apptainer_infiniband) + tc = TC["apptainer_infiniband"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_infiniband) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py index 6dd4267557..8453ebd05c 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py @@ -16,10 +16,12 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_download, check_apptainer_download_idempotency, check_apptainer_download_memory, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -30,10 +32,12 @@ @pytest.mark.functional @pytest.mark.image_download @pytest.mark.non_disruptive -@pytest.mark.order(270) +@pytest.mark.order(41905) def test_apptainer_download(host): """Run the deployed downloader and require at least one usable SIF.""" - verify_pxeboot(host, "apptainer_download", check_apptainer_download) + tc = TC["apptainer_download"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_download) @pytest.mark.apptainer @@ -42,14 +46,12 @@ def test_apptainer_download(host): @pytest.mark.functional @pytest.mark.image_download @pytest.mark.non_disruptive -@pytest.mark.order(271) +@pytest.mark.order(41906) def test_apptainer_download_idempotency(host): """Rerun the downloader and verify existing image metadata is unchanged.""" - verify_pxeboot( - host, - "apptainer_download_idempotency", - check_apptainer_download_idempotency, - ) + tc = TC["apptainer_download_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_download_idempotency) @pytest.mark.apptainer @@ -58,7 +60,9 @@ def test_apptainer_download_idempotency(host): @pytest.mark.functional @pytest.mark.image_download @pytest.mark.non_disruptive -@pytest.mark.order(272) +@pytest.mark.order(41907) def test_apptainer_download_memory(host): """Run the downloader and enforce a bounded peak resident-memory use.""" - verify_pxeboot(host, "apptainer_download_memory", check_apptainer_download_memory) + tc = TC["apptainer_download_memory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_download_memory) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py index 8449e9a287..ba005df083 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_concurrent_jobs, check_apptainer_job_array, check_apptainer_ldap_job, @@ -24,6 +25,7 @@ check_apptainer_single_node_job, check_apptainer_slurm_environment, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -33,10 +35,12 @@ @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(280) +@pytest.mark.order(41915) def test_apptainer_single_node_job(host): """Run one exact-node container job on every mapped compute.""" - verify_pxeboot(host, "apptainer_single_node_job", check_apptainer_single_node_job) + tc = TC["apptainer_single_node_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_single_node_job) @pytest.mark.apptainer @@ -44,20 +48,24 @@ def test_apptainer_single_node_job(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(281) +@pytest.mark.order(41916) def test_apptainer_multi_node_job(host): """Run one container allocation spanning all mapped computes.""" - verify_pxeboot(host, "apptainer_multi_node_job", check_apptainer_multi_node_job) + tc = TC["apptainer_multi_node_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_multi_node_job) @pytest.mark.apptainer @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(282) +@pytest.mark.order(41917) def test_apptainer_ldap_job(host): """Run targeted container jobs as the configured LDAP test identity.""" - verify_pxeboot(host, "apptainer_ldap_job", check_apptainer_ldap_job) + tc = TC["apptainer_ldap_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_ldap_job) @pytest.mark.apptainer @@ -65,10 +73,12 @@ def test_apptainer_ldap_job(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(283) +@pytest.mark.order(41918) def test_apptainer_concurrent_jobs(host): """Run bounded concurrent container jobs on distinct computes.""" - verify_pxeboot(host, "apptainer_concurrent_jobs", check_apptainer_concurrent_jobs) + tc = TC["apptainer_concurrent_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_concurrent_jobs) @pytest.mark.apptainer @@ -76,10 +86,12 @@ def test_apptainer_concurrent_jobs(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(286) +@pytest.mark.order(41921) def test_apptainer_nfs_visibility(host): """Verify containers can read the shared image path on every compute.""" - verify_pxeboot(host, "apptainer_nfs_visibility", check_apptainer_nfs_visibility) + tc = TC["apptainer_nfs_visibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_nfs_visibility) @pytest.mark.apptainer @@ -87,12 +99,12 @@ def test_apptainer_nfs_visibility(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(287) +@pytest.mark.order(41922) def test_apptainer_slurm_environment(host): """Verify Slurm allocation variables propagate into containers.""" - verify_pxeboot( - host, "apptainer_slurm_environment", check_apptainer_slurm_environment - ) + tc = TC["apptainer_slurm_environment"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_slurm_environment) @pytest.mark.apptainer @@ -100,7 +112,9 @@ def test_apptainer_slurm_environment(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(288) +@pytest.mark.order(41923) def test_apptainer_job_array(host): """Submit and wait for a bounded Apptainer Slurm job array.""" - verify_pxeboot(host, "apptainer_job_array", check_apptainer_job_array) + tc = TC["apptainer_job_array"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_job_array) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py index 0a59521ef4..8ffabd8bac 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py @@ -16,11 +16,13 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_failure_cleanup, check_apptainer_invalid_sif, check_apptainer_missing_image_contract, check_apptainer_restricted_sif, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -29,41 +31,45 @@ @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(279) +@pytest.mark.order(41914) def test_apptainer_missing_image_contract(host): """Verify the downloader records pull failures and exits non-zero.""" - verify_pxeboot( - host, - "apptainer_missing_image_contract", - check_apptainer_missing_image_contract, - ) + tc = TC["apptainer_missing_image_contract"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_missing_image_contract) @pytest.mark.apptainer @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(284) +@pytest.mark.order(41919) def test_apptainer_invalid_sif(host): """Verify a nonexistent SIF fails through the Slurm execution path.""" - verify_pxeboot(host, "apptainer_invalid_sif", check_apptainer_invalid_sif) + tc = TC["apptainer_invalid_sif"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_invalid_sif) @pytest.mark.apptainer @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(285) +@pytest.mark.order(41920) def test_apptainer_restricted_sif(host): """Verify an unprivileged identity cannot execute a mode-0600 SIF.""" - verify_pxeboot(host, "apptainer_restricted_sif", check_apptainer_restricted_sif) + tc = TC["apptainer_restricted_sif"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_restricted_sif) @pytest.mark.apptainer @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(289) +@pytest.mark.order(41924) def test_apptainer_failure_cleanup(host): """Verify a failed image launch leaves no matching runtime process.""" - verify_pxeboot(host, "apptainer_failure_cleanup", check_apptainer_failure_cleanup) + tc = TC["apptainer_failure_cleanup"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_failure_cleanup) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py index 5d9c1faa32..ae4a00bcac 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py @@ -16,36 +16,41 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_reboot_artifacts, check_apptainer_reboot_job, check_apptainer_reboot_storage, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @pytest.mark.apptainer -@pytest.mark.disruptive @pytest.mark.reboot -@pytest.mark.order(295) +@pytest.mark.order(41930) def test_apptainer_reboot_storage(host): """Reboot one compute and verify the shared mount and SIF checksum.""" - verify_pxeboot(host, "apptainer_reboot_storage", check_apptainer_reboot_storage) + tc = TC["apptainer_reboot_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_reboot_storage) @pytest.mark.apptainer -@pytest.mark.disruptive @pytest.mark.reboot -@pytest.mark.order(296) +@pytest.mark.order(41931) def test_apptainer_reboot_job(host): """Run an exact-node container job after the authorized reboot.""" - verify_pxeboot(host, "apptainer_reboot_job", check_apptainer_reboot_job) + tc = TC["apptainer_reboot_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_reboot_job) @pytest.mark.apptainer -@pytest.mark.disruptive @pytest.mark.reboot -@pytest.mark.order(297) +@pytest.mark.order(41932) def test_apptainer_reboot_artifacts(host): """Verify downloader artifacts and policy after the authorized reboot.""" - verify_pxeboot(host, "apptainer_reboot_artifacts", check_apptainer_reboot_artifacts) + tc = TC["apptainer_reboot_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_reboot_artifacts) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py index 425c9931e5..cd8ba8f4fc 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_image_inventory, check_apptainer_ldap_readability, check_apptainer_non_root_execution, @@ -27,6 +28,7 @@ check_apptainer_sif_integrity, check_apptainer_sif_permissions, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -35,98 +37,116 @@ @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(266) +@pytest.mark.order(41901) def test_apptainer_runtime(host): """Verify the Apptainer executable and version on every compute node.""" - verify_pxeboot(host, "apptainer_runtime", check_apptainer_runtime) + tc = TC["apptainer_runtime"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_runtime) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(267) +@pytest.mark.order(41902) def test_apptainer_shared_artifacts(host): """Verify shared image directories and downloader artifacts.""" - verify_pxeboot(host, "apptainer_shared_artifacts", check_apptainer_shared_artifacts) + tc = TC["apptainer_shared_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_shared_artifacts) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(268) +@pytest.mark.order(41903) def test_apptainer_pulp_policy(host): """Verify the generated downloader uses only the configured Pulp source.""" - verify_pxeboot(host, "apptainer_pulp_policy", check_apptainer_pulp_policy) + tc = TC["apptainer_pulp_policy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_pulp_policy) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(269) +@pytest.mark.order(41904) def test_apptainer_shared_storage(host): """Verify /hpc_tools is a shared mounted filesystem on every compute.""" - verify_pxeboot(host, "apptainer_shared_storage", check_apptainer_shared_storage) + tc = TC["apptainer_shared_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_shared_storage) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(273) +@pytest.mark.order(41908) def test_apptainer_image_inventory(host): """Verify every compute sees one consistent non-empty SIF inventory.""" - verify_pxeboot(host, "apptainer_image_inventory", check_apptainer_image_inventory) + tc = TC["apptainer_image_inventory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_image_inventory) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(274) +@pytest.mark.order(41909) def test_apptainer_sif_format(host): """Verify each discovered image is a valid inspectable SIF.""" - verify_pxeboot(host, "apptainer_sif_format", check_apptainer_sif_format) + tc = TC["apptainer_sif_format"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_sif_format) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(275) +@pytest.mark.order(41910) def test_apptainer_sif_permissions(host): """Verify shared SIF files are non-empty and world-readable.""" - verify_pxeboot(host, "apptainer_sif_permissions", check_apptainer_sif_permissions) + tc = TC["apptainer_sif_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_sif_permissions) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(276) +@pytest.mark.order(41911) def test_apptainer_sif_integrity(host): """Verify the selected SIF has the same checksum on every compute.""" - verify_pxeboot(host, "apptainer_sif_integrity", check_apptainer_sif_integrity) + tc = TC["apptainer_sif_integrity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_sif_integrity) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.sanity -@pytest.mark.order(277) +@pytest.mark.order(41912) def test_apptainer_ldap_readability(host): """Verify the LDAP test identity can read a shared SIF when enabled.""" - verify_pxeboot(host, "apptainer_ldap_readability", check_apptainer_ldap_readability) + tc = TC["apptainer_ldap_readability"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_ldap_readability) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.sanity @pytest.mark.functional -@pytest.mark.order(278) +@pytest.mark.order(41913) def test_apptainer_non_root_execution(host): """Verify an unprivileged local identity can execute a shared SIF.""" - verify_pxeboot( - host, "apptainer_non_root_execution", check_apptainer_non_root_execution - ) + tc = TC["apptainer_non_root_execution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_non_root_execution) diff --git a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py index 982b4858dd..13ea104268 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py +++ b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_configless_mode, check_slurm_configuration_consistency, check_slurm_cross_node_ssh, @@ -23,6 +24,7 @@ check_slurm_scheduler, check_slurm_services, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -31,59 +33,69 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(226) +@pytest.mark.order(41101) def test_slurm_membership(host): """Verify mapped membership, healthy state, and basic hardware fields.""" - verify_pxeboot(host, "slurm_membership", check_slurm_membership) + tc = TC["slurm_membership"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_membership) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(227) +@pytest.mark.order(41102) def test_slurm_scheduler(host): """Verify mapped compute nodes have healthy, available partitions.""" - verify_pxeboot(host, "slurm_scheduler", check_slurm_scheduler) + tc = TC["slurm_scheduler"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_scheduler) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(228) +@pytest.mark.order(41103) def test_slurm_services(host): """Verify role and feature-specific Slurm services.""" - verify_pxeboot(host, "slurm_services", check_slurm_services) + tc = TC["slurm_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_services) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(229) +@pytest.mark.order(41104) def test_slurm_cross_node_ssh(host): """Verify every mapped Slurm role can reach every peer over root SSH.""" - verify_pxeboot(host, "slurm_cross_ssh", check_slurm_cross_node_ssh) + tc = TC["slurm_cross_ssh"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_cross_node_ssh) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(230) +@pytest.mark.order(41105) def test_slurm_configless_mode(host): """Verify configless controller access and expected cluster identity.""" - verify_pxeboot(host, "slurm_configless", check_slurm_configless_mode) + tc = TC["slurm_configless"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_configless_mode) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(231) +@pytest.mark.order(41106) def test_slurm_configuration_consistency(host): """Compare authoritative Slurm files with every configless client cache.""" - verify_pxeboot( - host, "slurm_config_consistency", check_slurm_configuration_consistency - ) + tc = TC["slurm_config_consistency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_configuration_consistency) diff --git a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py index 36c2a2783d..a42761bae6 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py +++ b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_slurm_custom_configuration, check_slurm_reconfigure, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -28,17 +30,21 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm -@pytest.mark.order(232) +@pytest.mark.order(41107) def test_slurm_reconfigure(host): """Reconfigure Slurm and verify membership remains healthy.""" - verify_pxeboot(host, "slurm_reconfigure", check_slurm_reconfigure) + tc = TC["slurm_reconfigure"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_reconfigure) @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm -@pytest.mark.order(234) +@pytest.mark.order(41109) def test_slurm_custom_configuration(host): """Verify custom values, NFS delivery, and effective visibility.""" - verify_pxeboot(host, "slurm_custom_configuration", check_slurm_custom_configuration) + tc = TC["slurm_custom_configuration"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_custom_configuration) diff --git a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py index 803ab6bc19..6c3a63bfde 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py +++ b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py @@ -15,7 +15,11 @@ """Slurm homogeneous and heterogeneous hardware-discovery contracts.""" import pytest -from library.functions import check_slurm_hardware_discovery +from library.functions import ( + TestLogger, + check_slurm_hardware_discovery, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -24,7 +28,9 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(233) +@pytest.mark.order(41108) def test_slurm_hardware_discovery(host): """Verify runtime hardware matches the configured discovery strategy.""" - verify_pxeboot(host, "slurm_hardware_discovery", check_slurm_hardware_discovery) + tc = TC["slurm_hardware_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_hardware_discovery) diff --git a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py index a8b6d2ef27..6e776a2573 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py +++ b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py @@ -22,6 +22,7 @@ import pytest from library.functions import ( + TestLogger, check_dcgm_cuda_atomic_lock, check_dcgm_cuda_compute_node, check_dcgm_cuda_login_compiler, @@ -37,117 +38,182 @@ check_dcgm_rhel_compatibility, check_dcgm_toolkit_nfs_storage, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.slurm, - pytest.mark.non_disruptive, -] - - # -- CUDA installation checks ------------------------------------------------ -@pytest.mark.order(300) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41501) def test_dcgm_cuda_validation(host): """Verify NVIDIA driver and CUDA toolkit are installed on GPU nodes.""" - verify_pxeboot(host, "dcgm_cuda_validation", check_dcgm_cuda_validation) + tc = TC["dcgm_cuda_validation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_validation) -@pytest.mark.order(301) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41502) def test_dcgm_cuda_atomic_lock(host): """Verify CUDA toolkit is installed to /hpc_tools/cuda via atomic lock.""" - verify_pxeboot(host, "dcgm_cuda_atomic_lock", check_dcgm_cuda_atomic_lock) + tc = TC["dcgm_cuda_atomic_lock"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_atomic_lock) -@pytest.mark.order(302) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41503) def test_dcgm_cuda_compute_node(host): """Verify both CUDA toolkit and CUDA driver on compute nodes.""" - verify_pxeboot(host, "dcgm_cuda_compute_node", check_dcgm_cuda_compute_node) + tc = TC["dcgm_cuda_compute_node"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_compute_node) -@pytest.mark.order(303) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41504) def test_dcgm_cuda_login_compiler(host): """Verify CUDA toolkit accessible on login_compiler nodes.""" - verify_pxeboot(host, "dcgm_cuda_login_compiler", check_dcgm_cuda_login_compiler) + tc = TC["dcgm_cuda_login_compiler"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_login_compiler) # -- DCGM service and package checks ----------------------------------------- -@pytest.mark.order(304) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41505) def test_dcgm_package_installed(host): """Verify datacenter-gpu-manager RPM and dcgmi binary on GPU nodes.""" - verify_pxeboot(host, "dcgm_package_installed", check_dcgm_package_installed) + tc = TC["dcgm_package_installed"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_package_installed) -@pytest.mark.order(305) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41506) def test_dcgm_daemon_running(host): """Verify nvidia-dcgm service is active and enabled on GPU nodes.""" - verify_pxeboot(host, "dcgm_daemon_running", check_dcgm_daemon_running) + tc = TC["dcgm_daemon_running"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_daemon_running) # -- GPU discovery and monitoring checks -------------------------------------- -@pytest.mark.order(306) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41507) def test_dcgm_gpu_discovery(host): """Verify dcgmi discovery enumerates GPUs with unique UUIDs.""" - verify_pxeboot(host, "dcgm_gpu_discovery", check_dcgm_gpu_discovery) + tc = TC["dcgm_gpu_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_gpu_discovery) -@pytest.mark.order(307) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41508) def test_dcgm_gpu_metrics(host): """Verify dcgmi dmon returns metric samples for each GPU node.""" - verify_pxeboot(host, "dcgm_gpu_metrics", check_dcgm_gpu_metrics) + tc = TC["dcgm_gpu_metrics"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_gpu_metrics) -@pytest.mark.order(308) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41509) def test_dcgm_multi_gpu_discovery(host): """Verify dcgmi discovery on multi-GPU nodes.""" - verify_pxeboot(host, "dcgm_multi_gpu_discovery", check_dcgm_multi_gpu_discovery) + tc = TC["dcgm_multi_gpu_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_multi_gpu_discovery) # -- Multi-node configuration checks ----------------------------------------- -@pytest.mark.order(309) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41510) def test_dcgm_multi_gpu_no_login_compiler(host): """Verify GPU nodes work without login_compiler present.""" - verify_pxeboot( - host, - "dcgm_multi_gpu_no_login_compiler", - check_dcgm_multi_gpu_no_login_compiler, - ) + tc = TC["dcgm_multi_gpu_no_login_compiler"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_multi_gpu_no_login_compiler) -@pytest.mark.order(310) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41511) def test_dcgm_multi_login_compiler_lock(host): """Verify CUDA toolkit install uses atomic lock with multiple login_compilers.""" - verify_pxeboot( - host, - "dcgm_multi_login_compiler_lock", - check_dcgm_multi_login_compiler_lock, - ) + tc = TC["dcgm_multi_login_compiler_lock"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_multi_login_compiler_lock) # -- Platform compatibility checks ------------------------------------------- -@pytest.mark.order(311) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41512) def test_dcgm_toolkit_nfs_storage(host): """Verify /hpc_tools is NFS-mounted and CUDA toolkit accessible.""" - verify_pxeboot(host, "dcgm_toolkit_nfs_storage", check_dcgm_toolkit_nfs_storage) + tc = TC["dcgm_toolkit_nfs_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_toolkit_nfs_storage) -@pytest.mark.order(312) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41513) def test_dcgm_rhel_compatibility(host): """Verify GPU node OS is a supported RHEL version.""" - verify_pxeboot(host, "dcgm_rhel_compatibility", check_dcgm_rhel_compatibility) + tc = TC["dcgm_rhel_compatibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_rhel_compatibility) -@pytest.mark.order(313) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41514) def test_dcgm_cuda_version_compatibility(host): """Verify CUDA toolkit and DCGM daemon version compatibility.""" - verify_pxeboot( - host, - "dcgm_cuda_version_compatibility", - check_dcgm_cuda_version_compatibility, - ) + tc = TC["dcgm_cuda_version_compatibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_version_compatibility) diff --git a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py index 50ef9fdcc5..adb74bcf5e 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py +++ b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py @@ -25,50 +25,54 @@ import pytest from library.functions import ( + TestLogger, check_dcgm_neg_cuda_prerequisite, check_dcgm_neg_daemon_recovery, check_dcgm_neg_package_install_failure, check_dcgm_neg_socket_inaccessible, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.slurm, - pytest.mark.functional, -] - - -@pytest.mark.order(320) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41515) def test_dcgm_neg_cuda_prerequisite(host): """Verify DCGM deployment requires CUDA driver as a prerequisite.""" - verify_pxeboot( - host, "dcgm_neg_cuda_prerequisite", check_dcgm_neg_cuda_prerequisite, - ) + tc = TC["dcgm_neg_cuda_prerequisite"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_cuda_prerequisite) -@pytest.mark.order(321) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41516) def test_dcgm_neg_daemon_recovery(host): """Simulate DCGM daemon crash via SIGKILL and verify systemd restarts it.""" - verify_pxeboot( - host, "dcgm_neg_daemon_recovery", check_dcgm_neg_daemon_recovery, - ) + tc = TC["dcgm_neg_daemon_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_daemon_recovery) -@pytest.mark.order(322) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41517) def test_dcgm_neg_socket_inaccessible(host): """Remove DCGM Unix socket and verify dcgmi returns a clear error.""" - verify_pxeboot( - host, "dcgm_neg_socket_inaccessible", check_dcgm_neg_socket_inaccessible, - ) + tc = TC["dcgm_neg_socket_inaccessible"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_socket_inaccessible) -@pytest.mark.order(323) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41518) def test_dcgm_neg_package_install_failure(host): """Verify error handling when datacenter-gpu-manager package is unavailable.""" - verify_pxeboot( - host, - "dcgm_neg_package_install_failure", - check_dcgm_neg_package_install_failure, - ) + tc = TC["dcgm_neg_package_install_failure"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_package_install_failure) diff --git a/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py b/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py index af3064298a..78a2372a49 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py +++ b/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py @@ -16,10 +16,12 @@ import pytest from library.functions import ( + TestLogger, check_slurm_gpu_inventory, check_slurm_gpu_job, check_slurm_gpu_memory_stress, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -28,10 +30,12 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(257) +@pytest.mark.order(41401) def test_slurm_gpu_inventory(host): """Verify NVIDIA runtime state on scheduler-declared GPU nodes.""" - verify_pxeboot(host, "slurm_gpu_inventory", check_slurm_gpu_inventory) + tc = TC["slurm_gpu_inventory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_gpu_inventory) @pytest.mark.buildstream @@ -39,10 +43,12 @@ def test_slurm_gpu_inventory(host): @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.functional -@pytest.mark.order(258) +@pytest.mark.order(41402) def test_slurm_gpu_job(host): """Allocate a GPU through Slurm and query the device.""" - verify_pxeboot(host, "slurm_gpu_job", check_slurm_gpu_job) + tc = TC["slurm_gpu_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_gpu_job) @pytest.mark.buildstream @@ -50,7 +56,9 @@ def test_slurm_gpu_job(host): @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.functional -@pytest.mark.order(259) +@pytest.mark.order(41403) def test_slurm_gpu_memory_stress(host): """Compile and run a bounded GPU memory workload through Slurm.""" - verify_pxeboot(host, "slurm_gpu_memory", check_slurm_gpu_memory_stress) + tc = TC["slurm_gpu_memory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_gpu_memory_stress) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py index affd69de51..061af00cef 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py @@ -41,13 +41,6 @@ def pytest_addoption(parser): ) -def pytest_configure(config): - """Register custom markers for the HPC benchmarks suite.""" - config.addinivalue_line( - "markers", "benchmark: HPC benchmark staging and execution tests" - ) - - def _destructive_authorized(config) -> bool: """Return True when destructive HPC benchmarks tests may run.""" if config.getoption("--run-destructive", default=False): diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py index e7ab8af5af..a9eb3d28b4 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_container_first_guidance, check_hpc_benchmarks_container_image_list, check_hpc_benchmarks_nfs_accessibility, @@ -24,59 +25,59 @@ check_hpc_benchmarks_platform_script, check_hpc_benchmarks_source_only_delivery, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(335) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42012) def test_hpc_benchmarks_platform_script(host): """TC-06a: Verify omnia_platform.sh deployment and platform detection.""" - verify_pxeboot( - host, - "hpc_benchmarks_platform_script", - check_hpc_benchmarks_platform_script, - ) + tc = TC["hpc_benchmarks_platform_script"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_platform_script) @pytest.mark.sanity -@pytest.mark.order(336) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42013) def test_hpc_benchmarks_container_image_list(host): """TC-06b: Verify container_image.list deployment and content validation.""" - verify_pxeboot( - host, - "hpc_benchmarks_container_image_list", - check_hpc_benchmarks_container_image_list, - ) + tc = TC["hpc_benchmarks_container_image_list"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_container_image_list) @pytest.mark.sanity -@pytest.mark.order(337) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42014) def test_hpc_benchmarks_platform_directory_structure(host): """TC-06c: Verify platform-specific directory structure exists per architecture.""" - verify_pxeboot( - host, - "hpc_benchmarks_platform_directory_structure", - check_hpc_benchmarks_platform_directory_structure, - ) + tc = TC["hpc_benchmarks_platform_directory_structure"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_platform_directory_structure) @pytest.mark.sanity -@pytest.mark.order(338) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42015) def test_hpc_benchmarks_offline_package_copy(host): """TC-06d: Verify offline packages are copied to slurm_config_path/packages/{arch}/.""" - verify_pxeboot( - host, - "hpc_benchmarks_offline_package_copy", - check_hpc_benchmarks_offline_package_copy, - ) + tc = TC["hpc_benchmarks_offline_package_copy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_offline_package_copy) @pytest.mark.sanity -@pytest.mark.order(342) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42016) def test_hpc_benchmarks_container_first_guidance(host): """TC-06: Verify pull_benchmarks.sh and benchmark_tools.list are deployed. @@ -84,30 +85,30 @@ def test_hpc_benchmarks_container_first_guidance(host): deployment contract actually enforced by hpc_tools.yml. See the check function's docstring for the rationale. """ - verify_pxeboot( - host, - "hpc_benchmarks_container_first_guidance", - check_hpc_benchmarks_container_first_guidance, - ) + tc = TC["hpc_benchmarks_container_first_guidance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_container_first_guidance) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(343) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42017) def test_hpc_benchmarks_source_only_delivery(host): """TC-07: Verify no compile/build commands are staged with the artifacts.""" - verify_pxeboot( - host, - "hpc_benchmarks_source_only_delivery", - check_hpc_benchmarks_source_only_delivery, - ) + tc = TC["hpc_benchmarks_source_only_delivery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_source_only_delivery) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(344) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42018) def test_hpc_benchmarks_nfs_accessibility(host): """TC-10: Verify /hpc_tools NFS is mounted and readable on compute nodes.""" - verify_pxeboot( - host, "hpc_benchmarks_nfs_accessibility", check_hpc_benchmarks_nfs_accessibility - ) + tc = TC["hpc_benchmarks_nfs_accessibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_nfs_accessibility) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py index 72446aa49f..b253e147e6 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py @@ -31,6 +31,7 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_airgapped_staging, check_hpc_benchmarks_container_image_unaffected, check_hpc_benchmarks_cuda_flow_unaffected, @@ -39,87 +40,85 @@ check_hpc_benchmarks_openmpi_unaffected, check_hpc_benchmarks_staging_idempotency, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.destructive] - - # --- Invariance checks (staging run + snapshot diff) --- @pytest.mark.functional -@pytest.mark.order(347) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42021) def test_hpc_benchmarks_cuda_flow_unaffected(host): """TC-14: Verify /hpc_tools/cuda is unchanged after benchmark staging.""" - verify_pxeboot( - host, - "hpc_benchmarks_cuda_flow_unaffected", - check_hpc_benchmarks_cuda_flow_unaffected, - ) + tc = TC["hpc_benchmarks_cuda_flow_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_cuda_flow_unaffected) @pytest.mark.functional -@pytest.mark.order(348) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42022) def test_hpc_benchmarks_nvhpc_flow_unaffected(host): """TC-15: Verify /hpc_tools/nvidia_sdk is unchanged after staging.""" - verify_pxeboot( - host, - "hpc_benchmarks_nvhpc_flow_unaffected", - check_hpc_benchmarks_nvhpc_flow_unaffected, - ) + tc = TC["hpc_benchmarks_nvhpc_flow_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_nvhpc_flow_unaffected) @pytest.mark.functional -@pytest.mark.order(349) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42023) def test_hpc_benchmarks_container_image_unaffected(host): """TC-16: Verify /hpc_tools/container_images is unchanged after staging.""" - verify_pxeboot( - host, - "hpc_benchmarks_container_image_unaffected", - check_hpc_benchmarks_container_image_unaffected, - ) + tc = TC["hpc_benchmarks_container_image_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_container_image_unaffected) @pytest.mark.functional -@pytest.mark.order(350) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42024) def test_hpc_benchmarks_openmpi_unaffected(host): """TC-17: Verify OpenMPI/UCX discovery is stable across a staging run.""" - verify_pxeboot( - host, - "hpc_benchmarks_openmpi_unaffected", - check_hpc_benchmarks_openmpi_unaffected, - ) + tc = TC["hpc_benchmarks_openmpi_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_openmpi_unaffected) # --- Direct staging execution checks --- -@pytest.mark.order(351) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42025) def test_hpc_benchmarks_airgapped_staging(host): """TC-11: Verify staging succeeds while external egress is unavailable.""" - verify_pxeboot( - host, - "hpc_benchmarks_airgapped_staging", - check_hpc_benchmarks_airgapped_staging, - ) + tc = TC["hpc_benchmarks_airgapped_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_airgapped_staging) -@pytest.mark.order(352) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42026) def test_hpc_benchmarks_existing_dirs_preserved(host): """TC-18: Verify pre-existing /hpc_tools subdirs survive a staging run.""" - verify_pxeboot( - host, - "hpc_benchmarks_existing_dirs_preserved", - check_hpc_benchmarks_existing_dirs_preserved, - ) + tc = TC["hpc_benchmarks_existing_dirs_preserved"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_existing_dirs_preserved) -@pytest.mark.order(353) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42027) def test_hpc_benchmarks_staging_idempotency(host): """TC-19: Verify a second staging run keeps the /hpc_tools snapshot stable.""" - verify_pxeboot( - host, - "hpc_benchmarks_staging_idempotency", - check_hpc_benchmarks_staging_idempotency, - ) + tc = TC["hpc_benchmarks_staging_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_staging_idempotency) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py index 2171975967..71736170db 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py @@ -21,44 +21,47 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_concurrent_staging, check_hpc_benchmarks_e2e_provisioning, check_hpc_benchmarks_staging_fingerprint_idempotency, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.functional @pytest.mark.benchmark -@pytest.mark.order(296) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42002) def test_hpc_benchmarks_staging_fingerprint_idempotency(host): - """V096: Pull tools, validate tarballs, repeat — SHA-256, size, mtime must hold.""" + """V226: Pull tools, validate tarballs, repeat — SHA-256, size, mtime must hold.""" + tc = TC["hpc_benchmarks_staging_fingerprint_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) verify_pxeboot( - host, - "hpc_benchmarks_staging_fingerprint_idempotency", - check_hpc_benchmarks_staging_fingerprint_idempotency, + test_log, tc, host, check_hpc_benchmarks_staging_fingerprint_idempotency ) @pytest.mark.functional @pytest.mark.benchmark -@pytest.mark.order(297) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42003) def test_hpc_benchmarks_concurrent_staging(host): - """V097: Start two same-platform pulls together; both see identical archives.""" - verify_pxeboot( - host, - "hpc_benchmarks_concurrent_staging", - check_hpc_benchmarks_concurrent_staging, - ) + """V227: Start two same-platform pulls together; both see identical archives.""" + tc = TC["hpc_benchmarks_concurrent_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_concurrent_staging) @pytest.mark.sanity -@pytest.mark.order(345) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42019) def test_hpc_benchmarks_e2e_provisioning(host): """TC-09: Verify the end-to-end benchmark provisioning pipeline.""" - verify_pxeboot( - host, "hpc_benchmarks_e2e_provisioning", check_hpc_benchmarks_e2e_provisioning - ) + tc = TC["hpc_benchmarks_e2e_provisioning"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_e2e_provisioning) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py index fce2568dd5..77c20cd048 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_artifact_copy, check_hpc_benchmarks_cleanup_staging, check_hpc_benchmarks_json_declaration, @@ -27,26 +28,28 @@ check_hpc_benchmarks_run_staging, check_hpc_benchmarks_tools_dir_creation, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.sanity @pytest.mark.functional @pytest.mark.benchmark -@pytest.mark.order(295) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42001) def test_hpc_benchmarks_prerequisites(host): - """V095: Verify script presence, platform helper, tool list, mount (NFS export informational).""" - verify_pxeboot( - host, "hpc_benchmarks_prerequisites", check_hpc_benchmarks_prerequisites - ) + """V225: Verify script presence, platform helper, tool list, mount (NFS export informational).""" + tc = TC["hpc_benchmarks_prerequisites"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_prerequisites) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(330) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42004) def test_hpc_benchmarks_run_staging(host): """TC-00: Execute pull_benchmarks.sh to populate staged artifacts. @@ -54,94 +57,102 @@ def test_hpc_benchmarks_run_staging(host): not skip with "staging not yet executed". Skips gracefully when pull_benchmarks.sh is not deployed. """ - verify_pxeboot( - host, "hpc_benchmarks_run_staging", check_hpc_benchmarks_run_staging - ) + tc = TC["hpc_benchmarks_run_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_run_staging) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(331) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42005) def test_hpc_benchmarks_json_declaration(host): """TC-01: Verify benchmark_tools.list is deployed and non-empty per arch.""" - verify_pxeboot( - host, "hpc_benchmarks_json_declaration", check_hpc_benchmarks_json_declaration - ) + tc = TC["hpc_benchmarks_json_declaration"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_json_declaration) @pytest.mark.sanity -@pytest.mark.order(332) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42006) def test_hpc_benchmarks_local_repo_sync(host): """TC-02: Verify each declared tool has files under the Pulp offline URL.""" - verify_pxeboot( - host, "hpc_benchmarks_local_repo_sync", check_hpc_benchmarks_local_repo_sync - ) + tc = TC["hpc_benchmarks_local_repo_sync"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_local_repo_sync) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(333) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42007) def test_hpc_benchmarks_tools_dir_creation(host): """TC-03: Verify /hpc_tools directory layout and 0755 permissions.""" - verify_pxeboot( - host, - "hpc_benchmarks_tools_dir_creation", - check_hpc_benchmarks_tools_dir_creation, - ) + tc = TC["hpc_benchmarks_tools_dir_creation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_tools_dir_creation) @pytest.mark.sanity -@pytest.mark.order(334) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42008) def test_hpc_benchmarks_artifact_copy(host): """TC-04: Verify declared benchmark artifacts are staged per tool.""" - verify_pxeboot( - host, "hpc_benchmarks_artifact_copy", check_hpc_benchmarks_artifact_copy - ) + tc = TC["hpc_benchmarks_artifact_copy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_artifact_copy) @pytest.mark.sanity -@pytest.mark.order(337) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42009) def test_hpc_benchmarks_msr_safe_arch_boundary(host): """TC-05: Verify msr-safe is staged only for x86_64 nodes.""" - verify_pxeboot( - host, - "hpc_benchmarks_msr_safe_arch_boundary", - check_hpc_benchmarks_msr_safe_arch_boundary, - ) + tc = TC["hpc_benchmarks_msr_safe_arch_boundary"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_msr_safe_arch_boundary) @pytest.mark.sanity -@pytest.mark.order(338) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42010) def test_hpc_benchmarks_post_staging_validation(host): """TC-12: Verify post-staging validation of benchmark tool directories.""" - verify_pxeboot( - host, - "hpc_benchmarks_post_staging_validation", - check_hpc_benchmarks_post_staging_validation, - ) + tc = TC["hpc_benchmarks_post_staging_validation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_post_staging_validation) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(339) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42011) def test_hpc_benchmarks_rhel_compatibility(host): """TC-13: Verify every compute node runs the targeted RHEL major.""" - verify_pxeboot( - host, "hpc_benchmarks_rhel_compatibility", check_hpc_benchmarks_rhel_compatibility - ) + tc = TC["hpc_benchmarks_rhel_compatibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_rhel_compatibility) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(346) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42020) def test_hpc_benchmarks_cleanup_staging(host): """TC-00b: Remove staged benchmark tool directories after verification. Controlled by ``cleanup_benchmark_tools`` in ``test_config.yml``. When false, staged artifacts are retained for manual inspection. """ - verify_pxeboot( - host, - "hpc_benchmarks_cleanup_staging", - check_hpc_benchmarks_cleanup_staging, - ) + tc = TC["hpc_benchmarks_cleanup_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_cleanup_staging) diff --git a/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py b/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py index c6833e5866..643e47d9aa 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py +++ b/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_slurm_infiniband_configuration, check_slurm_infiniband_connectivity, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -27,17 +29,21 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(263) +@pytest.mark.order(41801) def test_slurm_infiniband_configuration(host): """Verify mapped IB interface, address, prefix, link, MTU, and OFED.""" - verify_pxeboot(host, "slurm_ib_configuration", check_slurm_infiniband_configuration) + tc = TC["slurm_ib_configuration"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_infiniband_configuration) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(264) +@pytest.mark.order(41802) def test_slurm_infiniband_connectivity(host): """Verify every mapped IB endpoint can reach every mapped peer.""" - verify_pxeboot(host, "slurm_ib_connectivity", check_slurm_infiniband_connectivity) + tc = TC["slurm_ib_connectivity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_infiniband_connectivity) diff --git a/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py b/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py index 7434de7361..645e13535d 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py +++ b/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_compiler_node_jobs, check_slurm_concurrent_jobs, check_slurm_control_node_jobs, @@ -24,6 +25,7 @@ check_slurm_job_queueing, check_slurm_login_node_jobs, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -33,10 +35,12 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(235) +@pytest.mark.order(41201) def test_slurm_control_node_jobs(host): """Run one targeted job per compute from every Slurm control node.""" - verify_pxeboot(host, "slurm_basic_jobs", check_slurm_control_node_jobs) + tc = TC["slurm_basic_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_node_jobs) @pytest.mark.functional @@ -44,10 +48,12 @@ def test_slurm_control_node_jobs(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(236) +@pytest.mark.order(41202) def test_slurm_login_node_jobs(host): """Run one targeted job per compute from every mapped login node.""" - verify_pxeboot(host, "slurm_login_jobs", check_slurm_login_node_jobs) + tc = TC["slurm_login_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_node_jobs) @pytest.mark.functional @@ -55,10 +61,12 @@ def test_slurm_login_node_jobs(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(237) +@pytest.mark.order(41203) def test_slurm_compiler_node_jobs(host): """Run one targeted job per compute from every login compiler node.""" - verify_pxeboot(host, "slurm_compiler_jobs", check_slurm_compiler_node_jobs) + tc = TC["slurm_compiler_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_node_jobs) @pytest.mark.functional @@ -66,24 +74,25 @@ def test_slurm_compiler_node_jobs(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(238) +@pytest.mark.order(41204) def test_slurm_concurrent_jobs(host): """Submit concurrent jobs and verify final accounting state.""" - verify_pxeboot(host, "slurm_concurrent_jobs", check_slurm_concurrent_jobs) + tc = TC["slurm_concurrent_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_concurrent_jobs) @pytest.mark.functional @pytest.mark.negative @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(239) +@pytest.mark.order(41205) def test_slurm_insufficient_resources(host): """Verify an impossible immediate allocation is rejected.""" - verify_pxeboot( - host, "slurm_insufficient_resources", check_slurm_insufficient_resources - ) + tc = TC["slurm_insufficient_resources"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_insufficient_resources) @pytest.mark.functional @@ -91,18 +100,21 @@ def test_slurm_insufficient_resources(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(240) +@pytest.mark.order(41206) def test_slurm_job_queueing(host): """Saturate idle computes and verify one follower queues then completes.""" - verify_pxeboot(host, "slurm_job_queueing", check_slurm_job_queueing) + tc = TC["slurm_job_queueing"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_job_queueing) -@pytest.mark.disruptive @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.scheduler_state @pytest.mark.slurm -@pytest.mark.order(241) +@pytest.mark.order(41207) def test_slurm_drain_queue_recovery(host): """Drain one compute node, verify queuing, and restore it.""" - verify_pxeboot(host, "slurm_drain_queue", check_slurm_drain_queue_recovery) + tc = TC["slurm_drain_queue"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_drain_queue_recovery) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py index 874407f297..2c31e08b31 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py +++ b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_compiler_ldap_authentication, check_slurm_compiler_ldap_invalid_password, check_slurm_control_ldap_authentication, @@ -26,6 +27,7 @@ check_slurm_pam_no_job_access, check_slurm_pam_policy, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -35,39 +37,36 @@ @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(242) +@pytest.mark.order(41303) def test_slurm_pam_policy(host): """Verify SSHD, the PAM module, and pam_slurm_adopt account policy.""" - verify_pxeboot(host, "slurm_pam", check_slurm_pam_policy) + tc = TC["slurm_pam"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_pam_policy) @pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(243) +@pytest.mark.order(41304) def test_slurm_control_ldap_authentication(host): """Verify a valid LDAP password on the Slurm control node.""" - verify_pxeboot( - host, - "slurm_control_ldap_auth", - check_slurm_control_ldap_authentication, - ) + tc = TC["slurm_control_ldap_auth"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_ldap_authentication) -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(244) +@pytest.mark.order(41305) def test_slurm_control_ldap_invalid_password(host): """Verify an invalid LDAP password is rejected on the control node.""" - verify_pxeboot( - host, - "slurm_control_ldap_invalid_password", - check_slurm_control_ldap_invalid_password, - ) + tc = TC["slurm_control_ldap_invalid_password"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_ldap_invalid_password) @pytest.mark.buildstream @@ -75,30 +74,25 @@ def test_slurm_control_ldap_invalid_password(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(245) +@pytest.mark.order(41306) def test_slurm_login_ldap_authentication(host): """Verify a valid LDAP password on every mapped login node.""" - verify_pxeboot( - host, - "slurm_login_ldap_auth", - check_slurm_login_ldap_authentication, - ) + tc = TC["slurm_login_ldap_auth"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_ldap_authentication) @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(246) +@pytest.mark.order(41307) def test_slurm_login_ldap_invalid_password(host): """Verify an invalid LDAP password is rejected on every login node.""" - verify_pxeboot( - host, - "slurm_login_ldap_invalid_password", - check_slurm_login_ldap_invalid_password, - ) + tc = TC["slurm_login_ldap_invalid_password"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_ldap_invalid_password) @pytest.mark.buildstream @@ -106,50 +100,47 @@ def test_slurm_login_ldap_invalid_password(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(247) +@pytest.mark.order(41308) def test_slurm_compiler_ldap_authentication(host): """Verify a valid LDAP password on every login-compiler node.""" - verify_pxeboot( - host, - "slurm_compiler_ldap_auth", - check_slurm_compiler_ldap_authentication, - ) + tc = TC["slurm_compiler_ldap_auth"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_ldap_authentication) @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(248) +@pytest.mark.order(41309) def test_slurm_compiler_ldap_invalid_password(host): """Verify invalid LDAP passwords are rejected on login-compiler nodes.""" - verify_pxeboot( - host, - "slurm_compiler_ldap_invalid_password", - check_slurm_compiler_ldap_invalid_password, - ) + tc = TC["slurm_compiler_ldap_invalid_password"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_ldap_invalid_password) -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(249) +@pytest.mark.order(41310) def test_slurm_pam_no_job_access(host): """Verify LDAP compute login is denied without an active job.""" - verify_pxeboot(host, "slurm_pam_no_job", check_slurm_pam_no_job_access) + tc = TC["slurm_pam_no_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_pam_no_job_access) @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(250) +@pytest.mark.order(41311) def test_slurm_invalid_ldap_identity(host): """Verify a generated missing directory identity is rejected.""" - verify_pxeboot(host, "slurm_invalid_ldap", check_slurm_invalid_ldap_identity) + tc = TC["slurm_invalid_ldap"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_invalid_ldap_identity) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_external_ldap.py b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_external_ldap.py new file mode 100644 index 0000000000..7c775a88a1 --- /dev/null +++ b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_external_ldap.py @@ -0,0 +1,55 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""External LDAP proxy setup that LDAP login checks depend on. + +These cases run first in the slurm_ldap suite. Prepare only starts the local +omnia_auth container, so pointing it at the external directory belongs here, +immediately before the LDAP authentication and job cases. +""" + +import pytest +from library.functions import ( + TestLogger, + check_external_ldap_backend, + reconcile_external_ldap_proxy, +) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot + + +@pytest.mark.sanity +@pytest.mark.openldap +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41301) +def test_external_ldap_proxy(host): + """Reconcile and verify the explicitly enabled LDAP meta-proxy.""" + tc = TC["external_ldap_proxy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, reconcile_external_ldap_proxy) + + +@pytest.mark.functional +@pytest.mark.sanity +@pytest.mark.openldap +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41302) +def test_external_ldap_backend(host): + """Verify external LDAP reachability from the omnia_auth container.""" + tc = TC["external_ldap_backend"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_external_ldap_backend) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py index 9dac46530c..84a9db3601 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py +++ b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_compiler_ldap_jobs, check_slurm_compiler_pam_job_access, check_slurm_control_ldap_jobs, @@ -23,6 +24,7 @@ check_slurm_login_ldap_jobs, check_slurm_login_pam_job_access, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -32,10 +34,12 @@ @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(251) +@pytest.mark.order(41312) def test_slurm_control_ldap_jobs(host): """Submit and complete an LDAP-owned job from the control node.""" - verify_pxeboot(host, "slurm_control_ldap_jobs", check_slurm_control_ldap_jobs) + tc = TC["slurm_control_ldap_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_ldap_jobs) @pytest.mark.functional @@ -43,14 +47,12 @@ def test_slurm_control_ldap_jobs(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(252) +@pytest.mark.order(41313) def test_slurm_control_pam_job_access(host): """Verify control-submitted PAM access during and after a job.""" - verify_pxeboot( - host, - "slurm_control_pam_job_access", - check_slurm_control_pam_job_access, - ) + tc = TC["slurm_control_pam_job_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_pam_job_access) @pytest.mark.functional @@ -59,10 +61,12 @@ def test_slurm_control_pam_job_access(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(253) +@pytest.mark.order(41314) def test_slurm_login_ldap_jobs(host): """Submit and complete an LDAP-owned job from every login node.""" - verify_pxeboot(host, "slurm_login_ldap_jobs", check_slurm_login_ldap_jobs) + tc = TC["slurm_login_ldap_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_ldap_jobs) @pytest.mark.functional @@ -71,14 +75,12 @@ def test_slurm_login_ldap_jobs(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(254) +@pytest.mark.order(41315) def test_slurm_login_pam_job_access(host): """Verify login-node PAM access during and after a job.""" - verify_pxeboot( - host, - "slurm_login_pam_job_access", - check_slurm_login_pam_job_access, - ) + tc = TC["slurm_login_pam_job_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_pam_job_access) @pytest.mark.functional @@ -87,10 +89,12 @@ def test_slurm_login_pam_job_access(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(255) +@pytest.mark.order(41316) def test_slurm_compiler_ldap_jobs(host): """Submit an LDAP-owned job from every login-compiler node.""" - verify_pxeboot(host, "slurm_compiler_ldap_jobs", check_slurm_compiler_ldap_jobs) + tc = TC["slurm_compiler_ldap_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_ldap_jobs) @pytest.mark.functional @@ -99,11 +103,9 @@ def test_slurm_compiler_ldap_jobs(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(256) +@pytest.mark.order(41317) def test_slurm_compiler_pam_job_access(host): """Verify login-compiler PAM access during and after a job.""" - verify_pxeboot( - host, - "slurm_compiler_pam_job_access", - check_slurm_compiler_pam_job_access, - ) + tc = TC["slurm_compiler_pam_job_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_pam_job_access) diff --git a/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py b/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py index c348a4c57c..a9705a8854 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py +++ b/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py @@ -23,26 +23,31 @@ import pytest from library.functions import ( + TestLogger, check_slurm_node_add, check_slurm_node_remove, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [ - pytest.mark.slurm, - pytest.mark.functional, - pytest.mark.disruptive, -] - - -@pytest.mark.order(290) +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.node_lifecycle +@pytest.mark.order(42401) def test_slurm_node_remove(host): """Remove Slurm compute node(s) from PXE mapping, provision, verify.""" - verify_pxeboot(host, "slurm_node_remove", check_slurm_node_remove) + tc = TC["slurm_node_remove"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_node_remove) -@pytest.mark.order(291) +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.node_lifecycle +@pytest.mark.order(42402) def test_slurm_node_add(host): """Restore removed node(s) to PXE mapping, provision, verify re-addition.""" - verify_pxeboot(host, "slurm_node_add", check_slurm_node_add) + tc = TC["slurm_node_add"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_node_add) diff --git a/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py b/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py index fa55a5c484..822e12cde7 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py +++ b/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_slurm_openmpi_installation, check_slurm_openmpi_job, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -27,10 +29,12 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(260) +@pytest.mark.order(41601) def test_slurm_openmpi_installation(host): """Verify OpenMPI discovery and version on every compute node.""" - verify_pxeboot(host, "slurm_openmpi_installation", check_slurm_openmpi_installation) + tc = TC["slurm_openmpi_installation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_openmpi_installation) @pytest.mark.buildstream @@ -38,7 +42,9 @@ def test_slurm_openmpi_installation(host): @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.functional -@pytest.mark.order(261) +@pytest.mark.order(41602) def test_slurm_openmpi_job(host): """Run an OpenMPI-backed job when OpenMPI is configured.""" - verify_pxeboot(host, "slurm_openmpi_job", check_slurm_openmpi_job) + tc = TC["slurm_openmpi_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_openmpi_job) diff --git a/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py b/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py index 79ca8ed3e8..9f1bf98a6d 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py +++ b/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py @@ -15,16 +15,21 @@ """Explicitly authorized full Slurm cluster reboot recovery.""" import pytest -from library.functions import check_slurm_cluster_recovery +from library.functions import ( + TestLogger, + check_slurm_cluster_recovery, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -@pytest.mark.disruptive @pytest.mark.functional @pytest.mark.reboot @pytest.mark.slurm -@pytest.mark.order(265) +@pytest.mark.order(42301) def test_slurm_cluster_recovery(host): """Reboot mapped Slurm nodes and verify scheduler and workload recovery.""" - verify_pxeboot(host, "slurm_recovery", check_slurm_cluster_recovery) + tc = TC["slurm_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_cluster_recovery) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py b/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py index 52734597aa..3c6ce002f7 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py +++ b/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py @@ -15,7 +15,11 @@ """Slurm UCX transport contract.""" import pytest -from library.functions import check_slurm_ucx_transport +from library.functions import ( + TestLogger, + check_slurm_ucx_transport, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -24,7 +28,9 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(262) +@pytest.mark.order(41701) def test_slurm_ucx_transport(host): """Verify UCX exposes an InfiniBand-capable transport.""" - verify_pxeboot(host, "slurm_ucx_transport", check_slurm_ucx_transport) + tc = TC["slurm_ucx_transport"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_ucx_transport) diff --git a/test/orchestrator/fvt/pxeboot/test_playbook.py b/test/orchestrator/fvt/pxeboot/test_playbook.py index 838111a4f4..5527ccfc78 100644 --- a/test/orchestrator/fvt/pxeboot/test_playbook.py +++ b/test/orchestrator/fvt/pxeboot/test_playbook.py @@ -24,7 +24,7 @@ @pytest.mark.deploy @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(200) +@pytest.mark.order(40000) def test_deploy_pxeboot(host): """Run ``orchestrator.yml --tags pxeboot``.""" tc = TC["deploy_pxeboot"] diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py index f706fabc9d..776cd1ed8d 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py @@ -16,24 +16,31 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_vast_vastnfs_installation, check_vast_vastnfs_rpm_and_module, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(700) @pytest.mark.vast_installation +@pytest.mark.order(42101) def test_vast_vastnfs_installation(host): """Verify vastnfs-ctl status on compute nodes.""" - verify_pxeboot(host, "vast_vastnfs_installation", check_vast_vastnfs_installation) + tc = TC["vast_vastnfs_installation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_vastnfs_installation) @pytest.mark.sanity -@pytest.mark.order(709) @pytest.mark.vast_installation +@pytest.mark.order(42110) def test_vast_vastnfs_rpm_and_module(host): """Verify vastnfs RPM, kernel module, and service.""" - verify_pxeboot(host, "vast_vastnfs_rpm_and_module", check_vast_vastnfs_rpm_and_module) + tc = TC["vast_vastnfs_rpm_and_module"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_vastnfs_rpm_and_module) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py index 023781ba9f..748010113e 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py @@ -16,42 +16,53 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( - check_vast_mount_points, - check_vast_mount_options, + TestLogger, check_vast_fstab_entries, + check_vast_mount_options, + check_vast_mount_points, check_vast_rdma_mount, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(701) @pytest.mark.vast_mounts +@pytest.mark.order(42102) def test_vast_mount_points(host): """Verify /scratch, /home, /apps, /projects directories.""" - verify_pxeboot(host, "vast_mount_points", check_vast_mount_points) + tc = TC["vast_mount_points"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_mount_points) @pytest.mark.sanity -@pytest.mark.order(703) @pytest.mark.vast_mounts +@pytest.mark.order(42104) def test_vast_mount_options(host): """Verify proto=rdma and port=20049 in /proc/mounts.""" - verify_pxeboot(host, "vast_mount_options", check_vast_mount_options) + tc = TC["vast_mount_options"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_mount_options) @pytest.mark.sanity -@pytest.mark.order(710) @pytest.mark.vast_mounts +@pytest.mark.order(42111) def test_vast_fstab_entries(host): """Verify /etc/fstab entries with proto=rdma.""" - verify_pxeboot(host, "vast_fstab_entries", check_vast_fstab_entries) + tc = TC["vast_fstab_entries"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_fstab_entries) @pytest.mark.sanity -@pytest.mark.order(711) @pytest.mark.vast_mounts +@pytest.mark.order(42112) def test_vast_rdma_mount(host): """Verify RDMA transport and 1 GB I/O checksum.""" - verify_pxeboot(host, "vast_rdma_mount", check_vast_rdma_mount) + tc = TC["vast_rdma_mount"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_rdma_mount) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py index 57aec1d8dd..5b6c6e7d3f 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py @@ -16,16 +16,20 @@ import pytest +from library.functions import ( + TestLogger, + check_vast_slurm_logs_persistence, +) +from library.vars import TEST_CASES as TC + from fvt.result import verify_pxeboot -from library.functions import check_vast_slurm_logs_persistence @pytest.mark.sanity -@pytest.mark.order(713) @pytest.mark.vast_persistence +@pytest.mark.order(42114) def test_vast_slurm_logs_persistence(host): """Verify Slurm logs on persistent storage; sacct accessible.""" - verify_pxeboot( - host, "vast_slurm_logs_persistence", - check_vast_slurm_logs_persistence, - ) + tc = TC["vast_slurm_logs_persistence"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_slurm_logs_persistence) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py index cafd2504a9..e51adff42d 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py @@ -16,63 +16,64 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( - check_vast_scratch_hostname_isolation, + TestLogger, + check_vast_ldapuser_permissions, check_vast_ldapuser_scratch_directory, check_vast_ldapuser_subdirectories, - check_vast_ldapuser_permissions, + check_vast_scratch_hostname_isolation, check_vast_scratch_isolation, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(702) @pytest.mark.vast_scratch +@pytest.mark.order(42103) def test_vast_scratch_hostname_isolation(host): """Verify /scratch// per node; file isolation.""" - verify_pxeboot( - host, "vast_scratch_hostname_isolation", - check_vast_scratch_hostname_isolation, - ) + tc = TC["vast_scratch_hostname_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_scratch_hostname_isolation) @pytest.mark.sanity -@pytest.mark.order(704) @pytest.mark.vast_scratch +@pytest.mark.order(42105) def test_vast_ldapuser_scratch_directory(host): """Verify /scratch// on login_compiler nodes.""" - verify_pxeboot( - host, "vast_ldapuser_scratch_directory", - check_vast_ldapuser_scratch_directory, - ) + tc = TC["vast_ldapuser_scratch_directory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_ldapuser_scratch_directory) @pytest.mark.sanity -@pytest.mark.order(705) @pytest.mark.vast_scratch +@pytest.mark.order(42106) def test_vast_ldapuser_subdirectories(host): """Verify data/, jobs/, results/, tmp/ subdirectories.""" - verify_pxeboot( - host, "vast_ldapuser_subdirectories", - check_vast_ldapuser_subdirectories, - ) + tc = TC["vast_ldapuser_subdirectories"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_ldapuser_subdirectories) @pytest.mark.sanity -@pytest.mark.order(706) @pytest.mark.vast_scratch +@pytest.mark.order(42107) def test_vast_ldapuser_permissions(host): """Verify cross-user permission isolation.""" - verify_pxeboot( - host, "vast_ldapuser_permissions", - check_vast_ldapuser_permissions, - ) + tc = TC["vast_ldapuser_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_ldapuser_permissions) @pytest.mark.sanity -@pytest.mark.order(707) @pytest.mark.vast_scratch +@pytest.mark.order(42108) def test_vast_scratch_isolation(host): """Verify file isolation between scratch subdirectories.""" - verify_pxeboot(host, "vast_scratch_isolation", check_vast_scratch_isolation) + tc = TC["vast_scratch_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_scratch_isolation) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py index c2d216d1d3..d8ac1e4a41 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py @@ -16,51 +16,64 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( - check_vast_control_node_no_vast, - check_vast_qss_mounts, - check_vast_control_node_mounts, + TestLogger, check_vast_compute_node_mounts, + check_vast_control_node_mounts, + check_vast_control_node_no_vast, check_vast_login_node_mounts, + check_vast_qss_mounts, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(708) @pytest.mark.vast_targeting +@pytest.mark.order(42109) def test_vast_control_node_no_vast(host): """Verify Slurm control node has no VAST storage.""" - verify_pxeboot(host, "vast_control_node_no_vast", check_vast_control_node_no_vast) + tc = TC["vast_control_node_no_vast"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_control_node_no_vast) @pytest.mark.sanity -@pytest.mark.order(712) @pytest.mark.vast_targeting +@pytest.mark.order(42113) def test_vast_qss_mounts(host): """Verify VAST on compute/login only; controller has none.""" - verify_pxeboot(host, "vast_qss_mounts", check_vast_qss_mounts) + tc = TC["vast_qss_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_qss_mounts) @pytest.mark.sanity -@pytest.mark.order(714) @pytest.mark.vast_targeting +@pytest.mark.order(42115) def test_vast_control_node_mounts(host): """Verify controller mount table (NFS/PV only, no VAST).""" - verify_pxeboot(host, "vast_control_node_mounts", check_vast_control_node_mounts) + tc = TC["vast_control_node_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_control_node_mounts) @pytest.mark.sanity -@pytest.mark.order(715) @pytest.mark.vast_targeting +@pytest.mark.order(42116) def test_vast_compute_node_mounts(host): """Verify compute node mount table (NFS + VAST).""" - verify_pxeboot(host, "vast_compute_node_mounts", check_vast_compute_node_mounts) + tc = TC["vast_compute_node_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_compute_node_mounts) @pytest.mark.sanity -@pytest.mark.order(716) @pytest.mark.vast_targeting +@pytest.mark.order(42117) def test_vast_login_node_mounts(host): """Verify login/compiler node mount table.""" - verify_pxeboot(host, "vast_login_node_mounts", check_vast_login_node_mounts) + tc = TC["vast_login_node_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_login_node_mounts) diff --git a/test/orchestrator/fvt/result.py b/test/orchestrator/fvt/result.py index 14f540035b..16e707d109 100644 --- a/test/orchestrator/fvt/result.py +++ b/test/orchestrator/fvt/result.py @@ -20,14 +20,12 @@ """ import pytest -from library.functions import TestLogger from library.messages import CLEANUP_TEST_ASSERT_MSGS as CLEANUP_ASSERT from library.messages import CLEANUP_TEST_LOG_MSGS as CLEANUP_LOG from library.messages import PRECHECK_TEST_ASSERT_MSGS as PRECHECK_ASSERT from library.messages import PRECHECK_TEST_LOG_MSGS as PRECHECK_LOG from library.messages import PXEBOOT_TEST_ASSERT_MSGS as PXEBOOT_ASSERT from library.messages import PXEBOOT_TEST_LOG_MSGS as PXEBOOT_LOG -from library.vars import TEST_CASES def _report(test_log, result, fields, component, messages): @@ -108,10 +106,8 @@ def verify_cleanup(test_log, host, component, checker): ) -def verify_pxeboot(host, key, checker): +def verify_pxeboot(test_log, case, host, checker): """Execute and enforce one PXE post-boot verification result.""" - case = TEST_CASES[key] - test_log = TestLogger(case["title"], case["id"]) result = checker(host) fields = result["details"]["fields"] _skip_if_requested(test_log, case, result, fields, PXEBOOT_LOG) diff --git a/test/orchestrator/library/functions/__init__.py b/test/orchestrator/library/functions/__init__.py index 772edb5e00..5412fd2031 100644 --- a/test/orchestrator/library/functions/__init__.py +++ b/test/orchestrator/library/functions/__init__.py @@ -174,6 +174,11 @@ cleanup_extra_vars, cleanup_selection_fields, ) +from .external_ldap_pxeboot_func import ( + check_external_ldap_backend, + check_external_ldap_proxy, + reconcile_external_ldap_proxy, +) from .kubernetes_etcd_pxeboot_func import ( check_kubernetes_etcd_health, check_kubernetes_etcd_topology, @@ -518,6 +523,8 @@ def run_playbook(tag: str | None = None, **kwargs): "check_hpc_benchmarks_tools_dir_creation", "check_kubernetes_configured_versions", "check_kubernetes_control_plane", + "check_external_ldap_backend", + "check_external_ldap_proxy", "check_kubernetes_control_plane_recovery", "check_kubernetes_csi_dynamic_provisioning", "check_kubernetes_default_storage_class", @@ -672,5 +679,6 @@ def run_playbook(tag: str | None = None, **kwargs): "cleanup_selection_fields", "persistent_changed_count", "resolve_nft_thresholds", + "reconcile_external_ldap_proxy", "run_playbook", ] diff --git a/test/orchestrator/library/functions/external_ldap_pxeboot_func.py b/test/orchestrator/library/functions/external_ldap_pxeboot_func.py new file mode 100644 index 0000000000..8546512060 --- /dev/null +++ b/test/orchestrator/library/functions/external_ldap_pxeboot_func.py @@ -0,0 +1,201 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""External LDAP meta-proxy reconciliation and verification after PXE boot. + +The prepare lifecycle only starts the local omnia_auth container. Pointing it +at an external directory is a test-side configuration step, so it runs in the +pxeboot slurm_ldap suite immediately before the LDAP login checks. +""" + +import ipaddress +import re +from typing import Any + +from ._pxeboot_helpers import runtime_result +from .external_ldap_func import ( + configure_external_ldap_proxy, + domain_to_dn, + load_external_ldap_settings, + resolve_proxy_config_path, + verify_external_ldap_backend, +) +from .openldap_prepare_func import openldap_enabled + + +def _openldap_skip(component: str) -> dict[str, Any]: + """Return a consistent expected skip for catalog-disabled OpenLDAP.""" + return runtime_result( + True, + "OpenLDAP is disabled by the active catalog", + [ + ("Reason", "OpenLDAP is disabled by the active catalog"), + ("Component", component), + ("openldap_support", False), + ], + skipped=True, + ) + + +def reconcile_external_ldap_proxy(host) -> dict[str, Any]: + """Reconcile and verify the opt-in external LDAP proxy configuration.""" + try: + if not openldap_enabled(host): + return _openldap_skip("External LDAP proxy") + except (TypeError, ValueError) as exc: + return runtime_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) + + try: + settings = load_external_ldap_settings() + if not settings["validation_enabled"]: + return runtime_result( + True, + "External LDAP validation is not requested", + [ + ("Reason", "validate_external_ldap is false"), + ("validate_external_ldap", False), + ], + skipped=True, + ) + except (TypeError, ValueError) as exc: + return runtime_result(False, "External LDAP settings are invalid", [], str(exc)) + + reconciliation = configure_external_ldap_proxy(host) + fields = [ + ("Configuration changed", reconciliation.get("changed", False)), + ("Reconciliation", reconciliation["details"]), + ] + if not reconciliation["success"]: + return runtime_result( + False, + reconciliation["details"], + fields, + reconciliation["error"], + ) + + verification = check_external_ldap_proxy(host) + return runtime_result( + verification["success"], + "External LDAP proxy reconciled and verified", + [*fields, *verification["details"]["fields"]], + verification["error"], + ) + + +def check_external_ldap_proxy(host) -> dict[str, Any]: + """Verify the deployed external LDAP meta-proxy configuration.""" + try: + if not openldap_enabled(host): + return _openldap_skip("External LDAP proxy") + settings = load_external_ldap_settings() + if not settings["validation_enabled"]: + return runtime_result( + True, + "External LDAP validation is not requested", + [ + ("Reason", "validate_external_ldap is false"), + ("validate_external_ldap", False), + ], + skipped=True, + ) + path = resolve_proxy_config_path(host, settings) + deployed = host.file(path) + if not deployed.is_file: + return runtime_result( + False, + "External LDAP proxy config missing", + [("Path", path)], + path, + ) + content = deployed.content_string + external_dn = domain_to_dn(str(settings["domain"])) + endpoint_host = str(settings["server_ip"]) + try: + if ipaddress.ip_address(endpoint_host).version == 6: + endpoint_host = f"[{endpoint_host}]" + except ValueError: + pass + expected = { + "database meta": bool( + re.search(r"^database\s+meta\s*$", content, re.MULTILINE) + ), + "back_ldap module": "back_ldap" in content, + "back_meta module": "back_meta" in content, + "external endpoint": ( + f"ldap://{endpoint_host}:{settings['server_port']}/" in content + ), + "suffix massage": external_dn in content, + "idassert bind": bool( + re.search(r"^idassert-bind\s*$", content, re.MULTILINE) + ), + } + except (OSError, TypeError, ValueError) as exc: + return runtime_result( + False, "External LDAP proxy verification failed", [], str(exc) + ) + + fields = [ + (name, "present" if value else "missing") for name, value in expected.items() + ] + failures = [name for name, value in expected.items() if not value] + fields.extend( + [ + ("Config owner", f"{deployed.user}:{deployed.group}"), + ("Config mode", oct(deployed.mode)), + ] + ) + if deployed.user != "root" or deployed.group != "root": + failures.append("config owner") + if deployed.mode != 0o600: + failures.append("config mode") + return runtime_result( + not failures, + "External LDAP meta-proxy configuration checked", + fields, + "; ".join(failures), + ) + + +def check_external_ldap_backend(host) -> dict[str, Any]: + """Verify external LDAP is reachable from the omnia_auth container.""" + try: + if not openldap_enabled(host): + return _openldap_skip("External LDAP backend") + settings = load_external_ldap_settings() + if not settings["validation_enabled"]: + return runtime_result( + True, + "External LDAP validation is not requested", + [ + ("Reason", "validate_external_ldap is false"), + ("validate_external_ldap", False), + ], + skipped=True, + ) + except (TypeError, ValueError) as exc: + return runtime_result(False, "External LDAP settings are invalid", [], str(exc)) + + probe = verify_external_ldap_backend(host) + return runtime_result( + probe["success"], + probe["details"], + [ + ("Endpoint", probe.get("endpoint", "unavailable")), + ( + "LDAP protocol reachability", + "passed" if probe["success"] else "failed", + ), + ], + probe["error"], + ) diff --git a/test/orchestrator/library/functions/openldap_prepare_func.py b/test/orchestrator/library/functions/openldap_prepare_func.py index d2e0197bbc..e03108114a 100644 --- a/test/orchestrator/library/functions/openldap_prepare_func.py +++ b/test/orchestrator/library/functions/openldap_prepare_func.py @@ -14,9 +14,7 @@ """OpenLDAP postcondition helpers for the prepare lifecycle.""" -import ipaddress import posixpath -import re import time from typing import Any @@ -37,20 +35,13 @@ PREPARE_COMMANDS, ) from ._prepare_helpers import prepare_result, read_yaml_mapping -from .external_ldap_func import ( - configure_external_ldap_proxy, - domain_to_dn, - load_external_ldap_settings, - resolve_proxy_config_path, - verify_external_ldap_backend, -) from .project_func import ( resolve_target_omnia_data_path, resolve_target_output_project_path, ) -def _openldap_enabled(host) -> bool: +def openldap_enabled(host) -> bool: """Read the authoritative catalog decision from Orchestrator state.""" state_path = posixpath.join( resolve_target_output_project_path(host), "orchestrator_state.yml" @@ -72,52 +63,10 @@ def _openldap_skip(component: str) -> dict[str, Any]: ) -def reconcile_prepare_external_ldap_proxy(host) -> dict[str, Any]: - """Reconcile and verify the opt-in external LDAP proxy configuration.""" - try: - if not _openldap_enabled(host): - return _openldap_skip("External LDAP proxy") - except (TypeError, ValueError) as exc: - return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) - - try: - settings = load_external_ldap_settings() - if not settings["validation_enabled"]: - return prepare_result( - True, - "External LDAP validation is not requested", - [("validate_external_ldap", False)], - skipped=True, - ) - except (TypeError, ValueError) as exc: - return prepare_result(False, "External LDAP settings are invalid", [], str(exc)) - - reconciliation = configure_external_ldap_proxy(host) - fields = [ - ("Configuration changed", reconciliation.get("changed", False)), - ("Reconciliation", reconciliation["details"]), - ] - if not reconciliation["success"]: - return prepare_result( - False, - reconciliation["details"], - fields, - reconciliation["error"], - ) - - verification = check_prepare_external_ldap_proxy(host) - return prepare_result( - verification["success"], - "External LDAP proxy reconciled and verified", - [*fields, *verification["fields"]], - verification["error"], - ) - - def check_prepare_openldap_runtime(host) -> dict[str, Any]: """Verify enabled OpenLDAP has a healthy service and container.""" try: - if not _openldap_enabled(host): + if not openldap_enabled(host): return _openldap_skip("OpenLDAP runtime") except (TypeError, ValueError) as exc: return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) @@ -165,7 +114,7 @@ def check_prepare_openldap_runtime(host) -> dict[str, Any]: def check_prepare_openldap_artifacts(host) -> dict[str, Any]: """Verify OpenLDAP secret modes, certificate lifetime and slapd syntax.""" try: - if not _openldap_enabled(host): + if not openldap_enabled(host): return _openldap_skip("OpenLDAP artifacts") except (TypeError, ValueError) as exc: return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) @@ -218,7 +167,7 @@ def check_prepare_openldap_artifacts(host) -> dict[str, Any]: def check_prepare_openldap_endpoint(host) -> dict[str, Any]: """Verify local LDAP health and both published listeners.""" try: - if not _openldap_enabled(host): + if not openldap_enabled(host): return _openldap_skip("OpenLDAP endpoint") except (TypeError, ValueError) as exc: return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) @@ -245,105 +194,3 @@ def check_prepare_openldap_endpoint(host) -> dict[str, Any]: fields, "; ".join(failures), ) - - -def check_prepare_external_ldap_proxy(host) -> dict[str, Any]: - """Verify the deployed external LDAP meta-proxy configuration.""" - try: - if not _openldap_enabled(host): - return _openldap_skip("External LDAP proxy") - settings = load_external_ldap_settings() - if not settings["validation_enabled"]: - return prepare_result( - True, - "External LDAP validation is not requested", - [("validate_external_ldap", False)], - skipped=True, - ) - path = resolve_proxy_config_path(host, settings) - deployed = host.file(path) - if not deployed.is_file: - return prepare_result( - False, - "External LDAP proxy config missing", - [("Path", path)], - path, - ) - content = deployed.content_string - external_dn = domain_to_dn(str(settings["domain"])) - endpoint_host = str(settings["server_ip"]) - try: - if ipaddress.ip_address(endpoint_host).version == 6: - endpoint_host = f"[{endpoint_host}]" - except ValueError: - pass - expected = { - "database meta": bool( - re.search(r"^database\s+meta\s*$", content, re.MULTILINE) - ), - "back_ldap module": "back_ldap" in content, - "back_meta module": "back_meta" in content, - "external endpoint": ( - f"ldap://{endpoint_host}:{settings['server_port']}/" in content - ), - "suffix massage": external_dn in content, - "idassert bind": bool( - re.search(r"^idassert-bind\s*$", content, re.MULTILINE) - ), - } - except (OSError, TypeError, ValueError) as exc: - return prepare_result( - False, "External LDAP proxy verification failed", [], str(exc) - ) - - fields = [ - (name, "present" if value else "missing") for name, value in expected.items() - ] - failures = [name for name, value in expected.items() if not value] - fields.extend( - [ - ("Config owner", f"{deployed.user}:{deployed.group}"), - ("Config mode", oct(deployed.mode)), - ] - ) - if deployed.user != "root" or deployed.group != "root": - failures.append("config owner") - if deployed.mode != 0o600: - failures.append("config mode") - return prepare_result( - not failures, - "External LDAP meta-proxy configuration checked", - fields, - "; ".join(failures), - ) - - -def check_prepare_external_ldap_backend(host) -> dict[str, Any]: - """Verify external LDAP is reachable from the omnia_auth container.""" - try: - if not _openldap_enabled(host): - return _openldap_skip("External LDAP backend") - settings = load_external_ldap_settings() - if not settings["validation_enabled"]: - return prepare_result( - True, - "External LDAP validation is not requested", - [("validate_external_ldap", False)], - skipped=True, - ) - except (TypeError, ValueError) as exc: - return prepare_result(False, "External LDAP settings are invalid", [], str(exc)) - - probe = verify_external_ldap_backend(host) - return prepare_result( - probe["success"], - probe["details"], - [ - ("Endpoint", probe.get("endpoint", "unavailable")), - ( - "LDAP protocol reachability", - "passed" if probe["success"] else "failed", - ), - ], - probe["error"], - ) diff --git a/test/orchestrator/library/vars/domain_vars.py b/test/orchestrator/library/vars/domain_vars.py index 67567babba..2b1b95eb30 100644 --- a/test/orchestrator/library/vars/domain_vars.py +++ b/test/orchestrator/library/vars/domain_vars.py @@ -48,14 +48,25 @@ "kubernetes", "slurm", "apptainer", + "benchmark", "additional_cloud_init", "mount_config", "minimal_os", + "boot_image", + "powervault_infrastructure", + "powervault_mounts", + "powervault_binds", + "powervault_cloudinit", + "vast_installation", + "vast_mounts", + "vast_targeting", + "vast_scratch", + "vast_persistence", "image_download", "negative", "non_disruptive", - "disruptive", "reboot", + "node_lifecycle", "scheduler_state", "destructive", "nft", @@ -70,16 +81,32 @@ # ===================================================================== SUITES: dict[str, list[str]] = { - "precheck": ["environment", "storage", "dependencies", "inputs"], - "prepare": ["openchami", "network", "openldap"], - "provision": ["openchami"], + "precheck": [ + "environment", + "oim_readiness", + "storage", + "dependencies", + "inputs", + ], + "prepare": [ + "openchami", + "network", + "openldap", + ], + "provision": [ + "openchami", + ], "pxeboot": [ "connectivity", "cloudinit", + "additional_cloud_init", + "mount_config", + "minimal_os", + "powervault", + "coredns_coredhcp", "kubernetes_cluster", "kubernetes_etcd", "kubernetes_storage", - "kubernetes_recovery", "slurm_cluster", "slurm_jobs", "slurm_ldap", @@ -88,16 +115,12 @@ "slurm_openmpi", "slurm_ucx", "slurm_infiniband", - "slurm_recovery", "slurm_apptainer", "slurm_hpc_benchmarks", - "coredns_coredhcp", - "powervault", "vast_storage", + "kubernetes_recovery", + "slurm_recovery", "slurm_lifecycle", - "additional_cloud_init", - "mount_config", - "minimal_os", ], "cleanup": [ "openchami", @@ -111,8 +134,8 @@ # Each lifecycle owns one playbook execution followed by independent checks. ALL_EXEC_TAGS: list[str] = ["precheck", "prepare", "provision", "pxeboot"] -ALL_EXEC_MARKER: str = "sanity" -ALL_VERIFY_EXCLUDE_MARKERS: list[str] = ["disruptive", "negative"] +ALL_EXEC_MARKER: str = "" +ALL_VERIFY_EXCLUDE_MARKERS: list[str] = [] VERIFY_ONLY_TAGS: list[str] = [] diff --git a/test/orchestrator/ut/test_prepare_contracts.py b/test/orchestrator/ut/test_prepare_contracts.py index 03d0c9b730..d45291bc69 100644 --- a/test/orchestrator/ut/test_prepare_contracts.py +++ b/test/orchestrator/ut/test_prepare_contracts.py @@ -18,6 +18,7 @@ import pytest from library.functions import external_ldap_func as external_ldap +from library.functions import external_ldap_pxeboot_func as external_ldap_pxeboot from library.functions import openchami_prepare_func as openchami from library.functions import openldap_prepare_func as openldap from library.functions import postgres_prepare_func as postgres @@ -174,7 +175,7 @@ def test_postgresql_readiness_accepts_complete_read_only_probe(monkeypatch): def test_explicitly_disabled_optional_openldap_skips(monkeypatch): """ORCH_UT_021: Catalog-disabled OpenLDAP is an intentional skip.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: False) + monkeypatch.setattr(openldap, "openldap_enabled", lambda _host: False) verification = openldap.check_prepare_openldap_runtime(object()) @@ -184,7 +185,7 @@ def test_explicitly_disabled_optional_openldap_skips(monkeypatch): def test_enabled_optional_openldap_missing_runtime_fails(monkeypatch): """ORCH_UT_022: Enabled OpenLDAP cannot skip a missing runtime.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: True) + monkeypatch.setattr(openldap, "openldap_enabled", lambda _host: True) monkeypatch.setattr( openldap, "run_on_host", @@ -246,9 +247,9 @@ def test_external_ldap_validation_requires_complete_endpoint(): def test_external_ldap_validation_opt_out_skips_before_mutation(monkeypatch): """ORCH_UT_028: Disabled validation cannot reconcile the LDAP proxy.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: True) + monkeypatch.setattr(external_ldap_pxeboot, "openldap_enabled", lambda _host: True) monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "load_external_ldap_settings", lambda: { "validation_enabled": False, @@ -260,23 +261,23 @@ def unexpected_reconciliation(_host): raise AssertionError("LDAP reconciliation must not run") monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "configure_external_ldap_proxy", unexpected_reconciliation, ) - verification = openldap.reconcile_prepare_external_ldap_proxy(object()) + verification = external_ldap_pxeboot.reconcile_external_ldap_proxy(object()) assert verification["success"] assert verification["skipped"] - assert dict(verification["fields"])["validate_external_ldap"] is False + assert dict(verification["details"]["fields"])["validate_external_ldap"] is False def test_external_ldap_validation_does_not_hide_existing_proxy_failure(monkeypatch): """ORCH_UT_029: Validation-only mode reports a deployed proxy failure.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: True) + monkeypatch.setattr(external_ldap_pxeboot, "openldap_enabled", lambda _host: True) monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "load_external_ldap_settings", lambda: { "validation_enabled": True, @@ -284,7 +285,7 @@ def test_external_ldap_validation_does_not_hide_existing_proxy_failure(monkeypat }, ) monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "configure_external_ldap_proxy", lambda _host: { "success": True, @@ -295,18 +296,20 @@ def test_external_ldap_validation_does_not_hide_existing_proxy_failure(monkeypat }, ) monkeypatch.setattr( - openldap, - "check_prepare_external_ldap_proxy", + external_ldap_pxeboot, + "check_external_ldap_proxy", lambda _host: { "success": False, "skipped": False, - "details": "External LDAP proxy verification failed", - "fields": [], + "details": { + "summary": "External LDAP proxy verification failed", + "fields": [], + }, "error": "deployed proxy does not match", }, ) - verification = openldap.reconcile_prepare_external_ldap_proxy(object()) + verification = external_ldap_pxeboot.reconcile_external_ldap_proxy(object()) assert not verification["success"] assert not verification["skipped"]