diff --git a/docs/code-style/test_automation.md b/docs/code-style/test_automation.md index 30f7896afe..91147fe739 100644 --- a/docs/code-style/test_automation.md +++ b/docs/code-style/test_automation.md @@ -161,9 +161,10 @@ test// │ └── _msgs.py [required] ├── fvt/ │ ├── README.md [required] -│ └── / -│ ├── test_.py [when the tag itself is verified] -│ └── /test_.py [when suites divide the tag] +│ └── / [one per supported playbook tag] +│ ├── test_playbook.py [required; playbook execution only] +│ └── / [one per functional component] +│ └── test_.py [verification of that component] ├── nft/ │ ├── README.md [when NFT is supported] │ └── test_.py [when NFT is supported] @@ -181,6 +182,93 @@ test// Do not create empty placeholder capabilities. For example, omit `nft/` when there is no defined non-functional contract. +### 3.1 FVT tag folders + +Derive the tag folders from the product entry playbook, for example +`src//playbooks/.yml`: + +1. Every tag that a user can select with `--tags ` gets one + `fvt//` folder and one `FVT_TAGS` entry. The folder name is the + playbook tag, unchanged. +2. `always` and `never` are Ansible control tags, not folders. A tag that + exists only under `never` (for example `cleanup`) is still a folder when + users run it, and belongs in `EXCLUDE_TAGS`. +3. A pure alias that runs the same plays as other tags (for example the + Orchestrator `deploy` or `execute` tags) is not a separate folder. The + README names the alias and the folders it maps to. +4. A verification-only capability with no playbook tag needs a documented + reason in the README before it gets a folder. + +Every tag folder MUST contain `test_playbook.py`. It holds only the +execution (`E`) case for that tag: it runs the entry playbook with +`--tags ` through `omnia_auto.run_playbook`, renders the result, and +asserts it. It contains no verification checks. + +If the playbook needs extra variables for a non-interactive run, add them to +`test_config.yml` with a comment that states the supported values and +default. A domain helper reads and validates them and returns the +`extra_vars` mapping. Never hardcode them in the test file. Secrets are never +extra variables; the product reads them from its domain credential file. + +Suite folders under a tag are functional components of that tag, for +example `fvt/prepare/openldap/` or `fvt/pxeboot/kubernetes_etcd/`. Each suite +name is listed in `SUITES[]`. Verification (`V`) cases live only in +suite folders. + +Example from the Orchestrator domain: + +```text +fvt/ +├── precheck/ +│ ├── test_playbook.py # orchestrator.yml --tags precheck +│ ├── environment/ +│ └── storage/ +├── prepare/ +│ ├── test_playbook.py # orchestrator.yml --tags prepare +│ ├── openchami/ +│ └── openldap/ +└── cleanup/ # never-tag; listed in EXCLUDE_TAGS + ├── test_playbook.py # orchestrator.yml --tags cleanup + └── kubernetes/ +``` + +### 3.2 Naming conventions + +| Item | Convention | Example | +|---|---|---| +| Tag folder | Playbook tag, unchanged | `fvt/pxeboot/` | +| Suite folder | Component name, lowercase `snake_case` | `kubernetes_etcd/` | +| Execution test file | `test_playbook.py` | `fvt/prepare/test_playbook.py` | +| Verification test file | `test_.py` or `test__.py` | `test_local_etcd.py` | +| Execution test function | `test_deploy_` | `test_deploy_pxeboot` | +| Verification test function | `test__` | `test_kubernetes_local_etcd_media` | +| Negative test function | `test_neg__`; file `test__negative.py` when split | `test_neg_pxe_nic_missing` | +| Registry key | Test function name without `test_` | `kubernetes_local_etcd_media` | +| Function module | `__func.py`; private helpers start with `_` | `kubernetes_runtime_pxeboot_func.py` | +| Check function | `check__` | `check_kubernetes_local_etcd_media` | +| Variable module | `_vars.py` for tag data; `test_case_vars.py` for metadata | `pxeboot_vars.py` | +| Message module | `_msgs.py` with `TEST_LOG_MSGS` and `TEST_ASSERT_MSGS`, exported as `_TEST_LOG_MSGS` | `pxeboot_msgs.py` | +| Constant | `UPPER_SNAKE_CASE` | `ETCD_HEALTH_WAIT_TIMEOUT_SECONDS` | +| Marker | Lowercase `snake_case`, registered before use | `kubernetes_etcd` | + +Names describe the behavior under test, not the ticket, lab, or author. +A test function name and its registry key stay stable after release; rename +only with the ID migration described in section 10. + +### 3.3 Execution order + +When a domain orders tests with `@pytest.mark.order`, give every value one +owner. Allocate a block per lifecycle tag and a sub-block of 100 per suite: + +```text +order = lifecycle base + suite slot * 100 + position in suite +``` + +Record the bases and suite slots in one table in `test_case_vars.py`. A new +test takes the next free number in its own suite; a new suite appends a new +block. Never reuse an order value. Suites that reboot, drain, or remove nodes +run last in their tag, and destructive cases run last in their suite. + Generated local files MUST be ignored: ```text @@ -263,6 +351,39 @@ Optional NFT and UT use: Replace placeholders with the real domain name in its README and state its safe lifecycle order. Identify destructive cleanup explicitly. +### 4.1 README contents + +Write each README for the operator who runs the tests. Keep content that +code already enforces out of the README; link to the owning file instead. + +`test//README.md` MUST contain, in this order: + +1. purpose and safety model, including every destructive tag; +2. prerequisites on the development host and the execution OIM; +3. quick start using the first-use workflow above; +4. environment setup: install modes, then the test and domain credential + stores with their `setup_env.sh` options and fields; +5. local and remote execution, and input/output synchronization; +6. tags and suites: one row per tag with its suites and contract, plus any + playbook alias such as `deploy`; +7. options and marker expressions; +8. lifecycle examples per tag, with the `test_config.yml` keys each one uses; +9. NFT, batch execution, reports, and UT entry points; +10. directory structure. + +`fvt/README.md` MUST contain the test-case ID format, the order allocation, +and one table per tag and suite with these columns: order, ID, function, +suite, markers, what the test checks, and the pass condition. Record +retired or reassigned IDs and release traceability in their own tables. + +`nft/README.md` lists each contract, its threshold source and unit, and its +order. `ut/README.md` lists each UT file with the product behavior it +covers and the command that runs it. + +`docs/test_config.md`, `docs/test_run_config.md`, and `docs/test_creds.md` +describe every key or field with its type, default, and effect. Update them +in the same change as the key. + --- ## 5. Credential Contract @@ -274,6 +395,20 @@ safe lifecycle order. Identify destructive cleanup explicitly. | Transport | Reach execution OIM | Local encrypted credential pair | Developer or CI job | | Domain | Authenticate product targets | Project input on execution OIM | Project operator | +The two setup options map to these scopes: + +| Option | Writes | Contains | Read by | +|---|---|---|---| +| `--set-creds` | `test_creds.yml` and `.test_creds.key` in `test//` | Secrets that only the test automation needs: the OIM SSH password, test-only identities such as an LDAP login-test user, and secrets for test-side setup that the product playbook never reads | The test framework | +| `--set-domain-creds` | `_credentials.yml` in the project input directory on the execution OIM | The credentials the Omnia playbook itself reads, with the same file name, fields, and vault format the product expects | The product playbook | + +Decide by asking who reads the value. If the playbook reads it, it is a domain +credential, even when a test also needs it; the test reads it from the domain +store. If only the test reads it, it is a test credential. Never copy a value +into both stores. When a test credential is needed only for an optional +feature, the prompt asks for it only when `test_config.yml` enables that +feature. + The local transport pair is `test_creds.yml` and `.test_creds.key`. Both MUST have mode `0600`, be encrypted at rest, and be gitignored. Passwordless SSH may make transport credentials optional. @@ -360,6 +495,26 @@ channel. Fake examples MUST be unmistakably synthetic. `setup_env.sh` owns environment preparation and credential provisioning. It MUST NOT run a domain playbook or validation suite. +Logic: + +```text +parse and reject incompatible options + -> select install mode (--user, or .venv/ with --venv) + -> install requirements.txt, including the omnia_auto wheel + -> credential option given? + --set-creds / --update-creds / --creds-stdin + -> prompt or read the test credential fields + -> encrypt into test_creds.yml with .test_creds.key (0600) + --set-domain-creds / --update-domain-creds / --domain-creds-stdin + -> resolve $OMNIA_DATA_PATH//input/$OMNIA_PROJECT_NAME/ + -> prompt or read the product credential fields + -> write _credentials.yml in the product vault format + -> print the next safe commands +``` + +Credential prompting, encryption, and vault writes come from `omnia_auto`. +The domain script supplies only its field specification and file names. + Required behavior: - default: install missing requirements into the active or documented user @@ -466,7 +621,7 @@ For every FVT tag, all of these MUST agree: 1. product playbook tag or documented verification-only capability; 2. entry in `FVT_TAGS`; -3. physical `fvt//` directory; +3. physical `fvt//` directory with its `test_playbook.py`; 4. `SUITES[]` entries and physical suite directories; 5. registered pytest markers used beneath the tag; 6. `test_run_config.yml` entry; @@ -643,6 +798,18 @@ atomically. Register every marker before use. Markers represent useful selection axes such as architecture, source, sink, feature, or `sanity`, not temporary labs. +A run without a marker expression runs every collected test, including +functional and negative tests, except tests that reboot nodes or remove and +re-add nodes; the `exec`/`verify` phase rule for execution cases still applies. +A test that reboots a node MUST carry `reboot`, and a test that removes or +re-adds a node MUST carry `node_lifecycle`. Such tests are deselected unless +the marker expression selects their marker, even when their suite is named +explicitly. Apart from `reboot` and `node_lifecycle`, markers narrow a run; +they are not required to authorize one. +`sanity` marks only the baseline positive checks that must pass on a healthy +system. A negative test, one that expects a rejection or failure, MUST NOT +carry `sanity`; mark it `negative`. Do not add `sanity` to an existing test +without the owner's agreement. `domain_vars.py`, pytest configuration, README, batch examples, and decorators must agree. @@ -717,6 +884,13 @@ UT exercises isolated parsing, validation, formatting, and decisions. It does not require SSH, a cluster, appliance, live API, or real credentials. Mock at the I/O boundary, not inside the tested logic. +UT covers product behavior and the logic of domain helpers, such as parsers, +classifiers, result decisions, and skip gates. Do not write UT that only +checks the test framework's own layout: naming conventions, ID formats, +order numbers, marker registration, registry lookups, or README tables. +Those belong in code review and the `test-domain-compliance` skill. Give +every UT a unique `_UT_` ID in its docstring. + A UT may read product files only when the domain explicitly declares a source-contract capability. That exception does not expand compliance scoring to product source or make source inspection mandatory for other domains. @@ -760,11 +934,23 @@ interface even when the declared version is unchanged. python -c "import omnia_auto; print(sorted(omnia_auto.__all__))" ``` -11. Run focused checks for every affected domain. -12. Commit plugin source, docs, wheel, and required consumers together. +11. Find every consumer of each changed or removed symbol, field, or output + shape across all domains: + + ```bash + grep -rn '' test --include='*.py' --include='*.sh' \ + --exclude-dir=plugins + ``` + + Update each consumer in the same change, then run its import check, + collection (`pytest --collect-only`), and UT. A domain that does not use + the changed behavior still needs a collection check with the new wheel. +12. Commit and push plugin source, docs, the rebuilt wheel, and the required + consumers together, in one change. Never push source without its rebuilt + wheel; `setup_env.sh` installs the wheel, not the source. -Do not publish to PyPI, change the version, commit a venv, or push unless the -maintainer explicitly requests it. +Do not publish to PyPI, change the version, or commit a venv unless the +release owner requests it. --- @@ -880,8 +1066,10 @@ Create a human-authored DCO commit: git commit -s -m "test(): describe the behavior change" ``` -Configured `user.name` and `user.email` identify the human author. Never -fabricate another person's signoff or add an AI identity trailer. +Configured `user.name` and `user.email` identify the human author and must be +the author's GitHub user name and email, so the DCO `Signed-off-by` matches the +commit author. Never fabricate another person's signoff or add an AI identity +trailer, such as an AI `Co-Authored-By` line. --- @@ -894,14 +1082,20 @@ fabricate another person's signoff or add an AI identity trailer. - [ ] Directory tree contains only supported capabilities. - [ ] README first-use flow is complete and safe. - [ ] Domain, tags, suites, markers, directories, and batch config agree. +- [ ] Each supported playbook tag has `fvt//test_playbook.py` with + only the execution case; verification lives in suite folders. +- [ ] Playbook extra variables come from `test_config.yml`. +- [ ] Names follow section 3.2 and order values are unique per section 3.3. - [ ] Destructive tags are excluded from implicit execution. -- [ ] Transport and domain credentials are separate and encrypted. +- [ ] Test credentials (`--set-creds`) and playbook credentials + (`--set-domain-creds`) are separate and encrypted. - [ ] Non-interactive secrets use bounded standard input only. - [ ] Tests orchestrate; helpers contain reusable logic. - [ ] Commands, variables, messages, and metadata are centralized. - [ ] IDs and titles are unique and not hardcoded at call sites. - [ ] Datasets contain no credentials and sync in the declared direction. -- [ ] FVT, NFT, and UT boundaries are respected. +- [ ] FVT, NFT, and UT boundaries are respected; UT does not test naming, + order, or README layout. - [ ] Documentation matches commands and defaults. ### `omnia_auto` checklist @@ -914,7 +1108,8 @@ fabricate another person's signoff or add an AI identity trailer. - [ ] Version was preserved unless a bump was requested. - [ ] Exact wheel was force-reinstalled and its API verified. - [ ] Plugin, wheel, and consumers remain synchronized. -- [ ] Nothing was published or pushed without authorization. +- [ ] Every consumer domain was searched, updated, collected, and unit-tested. +- [ ] Source and rebuilt wheel are pushed together; nothing was published. ### Handoff checklist diff --git a/test/orchestrator/README.md b/test/orchestrator/README.md index 37da13a64d..95bf69e8d2 100644 --- a/test/orchestrator/README.md +++ b/test/orchestrator/README.md @@ -18,10 +18,12 @@ The runner separates product execution from verification: | `verify` | Verify existing state; do not execute the product playbook | | `test` | Execute the selected tag once, then verify only if execution succeeds | -Most verification is observational. Tests that create workloads, download -images, drain scheduler nodes, reboot machines, or delete deployment state -are protected by explicit markers. Cleanup is excluded from every implicit -full-lifecycle run and must be selected by name. +Most verification is observational. Tests that reboot machines run only when +`--marker` selects `reboot`, and node remove/add tests only when it selects +`node_lifecycle`. Other markers narrow test selection but do not authorize +workloads or downloads. Destructive tests retain their separate opt-in, and +cleanup is excluded from every implicit full-lifecycle run and must be selected +by name. ## Prerequisites @@ -138,7 +140,8 @@ without putting secrets in process arguments: } ``` -LDAP identity fields are required by the selected Slurm/PAM LDAP tests. The +LDAP identity fields are required by the selected Slurm/PAM LDAP tests, which +skip unless `validate_external_ldap: true`. The external bind secret is additionally required when `validate_external_ldap: true` and `configure_external_ldap: true`. Product- domain credentials continue to use `--domain-creds-stdin`. @@ -207,15 +210,15 @@ operations and includes destructive prepare, provision, and cleanup cases. | Tag | Suites | Main contract | |---|---|---| -| `precheck` | `environment`, `storage`, `dependencies`, `inputs` | OIM identity, selected NFS reachability, upstream artifacts, and required inputs | -| `prepare` | `openchami`, `network`, `openldap` | OpenCHAMI, PostgreSQL, networking, DNS/DHCP, and LDAP readiness | +| `precheck` | `environment`, `oim_readiness`, `storage`, `dependencies`, `inputs` | OIM identity and readiness, selected NFS reachability, upstream artifacts, and required inputs | +| `prepare` | `openchami`, `network`, `openldap` | OpenCHAMI, PostgreSQL, networking, DNS/DHCP, and local OpenLDAP readiness | | `provision` | `openchami` | Provision reports plus SMD, Boot Service, Metadata Service, and network inventory | -| `pxeboot` | `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer` | Node boot completion and workload-cluster behavior | +| `pxeboot` | See `SUITES` in `library/vars/domain_vars.py` | Node boot completion, node features, and workload-cluster behavior; reboot and node-removal suites run last | | `cleanup` | `openchami`, `openldap`, `slurm`, `kubernetes`, `artifacts`, `credentials` | Explicit full-cleanup postconditions | An untagged FVT flow uses `precheck -> prepare -> provision -> pxeboot`. -Cleanup is always explicit. Untagged verification excludes negative and -disruptive cases. +Cleanup is always explicit. Reboot cases never run without `--marker reboot`, +and node remove/add cases never run without `--marker node_lifecycle`. ### Options @@ -237,6 +240,14 @@ current directories. | Comma | Logical OR | `--marker sanity,functional` | | Plus | Logical AND | `--marker slurm+non_disruptive` | +Without `--marker`, every collected test runs, including functional, +negative, image-download, and scheduler-drain tests, except reboot and node +remove/add tests. Those are deselected (not listed in the run), even with +`--suite`, until `--marker` selects `reboot` or `node_lifecycle`; run them in a +maintenance window. Lifecycle execution cases +still run only in the `exec` phase. Pass a marker such as `--marker sanity` to +narrow the run. + `sanity+functional` selects only tests carrying both markers; it does not mean “run sanity, then functional.” Use `sanity,functional` for that union. @@ -245,12 +256,16 @@ Registered selectors include: - Baseline and capability: `sanity`, `functional`, `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `openldap`, and `apptainer`. - Controlled mutation: `image_download`, `negative`, and `non_disruptive`. -- Maintenance-window operations: `disruptive`, `reboot`, and +- Maintenance-window operations: `reboot` (required to run reboot tests), + `node_lifecycle` (required to run node remove/add tests), and `scheduler_state`. - Cleanup authorization: `destructive`. - Non-functional contracts: `nft`, `performance`, `idempotency`, and `security`. +`sanity` marks the baseline positive checks only. Negative tests, which +expect a rejection or failure, are never marked `sanity`. + `deploy` is attached to lifecycle execution cases and is normally managed by the runner rather than selected manually. @@ -281,9 +296,12 @@ active PXE mapping, or `groups` matches a mapped `GROUP_NAME`. The stock ./run_validation.sh fvt_orchestrator prepare verify --suite openldap ``` -Set `validate_external_ldap: true` to run external proxy and backend checks. -Set `configure_external_ldap: true` only when those checks may also reconcile -the local `omnia_auth` proxy configuration. An unchanged desired configuration +Prepare only starts the local `omnia_auth` container. The external LDAP proxy +and backend checks run first in the pxeboot `slurm_ldap` suite, immediately +before the LDAP login and job checks. Set `validate_external_ldap: true` to run +them and every LDAP-identity check. Set `configure_external_ldap: true` only +when the proxy check may also reconcile the local `omnia_auth` proxy +configuration. An unchanged desired configuration is an idempotent no-op; a failed changed configuration is rolled back. ### Provision @@ -300,7 +318,8 @@ access token. ### PXE boot -PXE verification defaults to `sanity` when no marker is supplied: +PXE verification runs every test except reboot and node remove/add cases when +no marker is supplied. Use `--marker sanity` for the baseline only: ```bash ./run_validation.sh fvt_orchestrator pxeboot test @@ -316,6 +335,7 @@ PXE verification defaults to `sanity` when no marker is supplied: ./run_validation.sh fvt_orchestrator pxeboot verify --suite slurm_hpc_benchmarks ./run_validation.sh fvt_orchestrator pxeboot verify --suite coredns_coredhcp ./run_validation.sh fvt_orchestrator pxeboot verify --suite powervault +./run_validation.sh fvt_orchestrator pxeboot verify --suite vast_storage ``` Focused workload and image examples: @@ -337,13 +357,16 @@ Focused workload and image examples: --suite slurm_apptainer --marker functional+image_download ``` -Run disruptive checks only in an approved maintenance window: +Run reboot and node remove/add checks only in an approved maintenance window: ```bash +./run_validation.sh fvt_orchestrator pxeboot verify --marker reboot +./run_validation.sh fvt_orchestrator pxeboot verify \ + --suite slurm_recovery --marker reboot ./run_validation.sh fvt_orchestrator pxeboot verify \ - --marker disruptive+reboot + --suite slurm_lifecycle --marker node_lifecycle ./run_validation.sh fvt_orchestrator pxeboot verify \ - --suite slurm_jobs --marker disruptive+scheduler_state + --suite slurm_jobs --marker scheduler_state ``` The authorization checks are enforced even when pytest is invoked directly. @@ -384,7 +407,7 @@ artifact permissions: A complete NFT run provisions state and finishes with full cleanup. Duration limits come from `nft_performance_threshold_seconds` in `test_config.yml`. Run NFT separately from FVT cleanup and review the cleanup policy first. See -[nft/README.md](nft/README.md) for its 11 contracts and execution order. +[nft/README.md](nft/README.md) for its 14 contracts and execution order. ## Batch execution diff --git a/test/orchestrator/_run.py b/test/orchestrator/_run.py index 5300ee240a..0267d5d078 100755 --- a/test/orchestrator/_run.py +++ b/test/orchestrator/_run.py @@ -46,20 +46,6 @@ def _runner_all_exec_tags(args, lifecycle_tags): return lifecycle_tags -def _apply_safe_pxeboot_marker_default(args): - """Default PXE verification to positive, non-disruptive sanity checks.""" - normalized = list(args) - if ( - len(normalized) >= 3 - and normalized[0] == "fvt_orchestrator" - and normalized[1] == "pxeboot" - and normalized[2] in {"test", "verify"} - and "--marker" not in normalized - ): - normalized.extend(["--marker", "sanity"]) - return normalized - - def main(): """Load domain config and run ValidationRunner.""" script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -81,7 +67,7 @@ def main(): ) from omnia_auto.functions.validation_runner import ValidationRunner - args = _apply_safe_pxeboot_marker_default(sys.argv[1:]) + args = sys.argv[1:] runner = ValidationRunner( domain=DOMAIN_NAME, script_dir=script_dir, diff --git a/test/orchestrator/conftest.py b/test/orchestrator/conftest.py index 71a5db0804..b248c80668 100644 --- a/test/orchestrator/conftest.py +++ b/test/orchestrator/conftest.py @@ -94,6 +94,7 @@ ) from library.vars import TEST_CASES from omnia_auto import ( + TestLogger, TestReport, add_session_result, build_report_name, @@ -194,6 +195,19 @@ ) +_TC_TITLE_MAP = {test_case["id"]: test_case["title"] for test_case in TEST_CASES.values()} + + +def _skip_reason(result) -> str: + """Return the skip reason recorded by pytest, without its prefix.""" + longrepr = result.longrepr + if isinstance(longrepr, tuple) and len(longrepr) == 3: + text = str(longrepr[2]) + else: + text = str(longrepr or "") + return text.removeprefix("Skipped: ").strip() + + def _registered_test_case_id(item) -> str: """Return a deterministic TC ID without relying on logger state.""" return _TC_ID_MAP.get(item.name, "") @@ -241,9 +255,15 @@ def pytest_configure(config): "kubernetes": "Kubernetes post-boot checks", "slurm": "Slurm post-boot checks", "apptainer": "Apptainer runtime, image, and Slurm integration checks", + "benchmark": "HPC benchmark staging and execution tests", "additional_cloud_init": "Additional cloud-init post-boot verification checks", "mount_config": "NFS mount_config post-boot verification checks", "minimal_os": "Minimal OS validation for OS-only provisioned nodes", + "boot_image": "Provisioned boot image identity and architecture checks", + "powervault_infrastructure": "PowerVault iSCSI and multipath infrastructure checks", + "powervault_mounts": "PowerVault partition, filesystem, and mount checks", + "powervault_binds": "PowerVault bind-mount and targeting checks", + "powervault_cloudinit": "PowerVault cloud-init and setup-log checks", "vast_installation": "VAST NFS client installation tests", "vast_mounts": "VAST mount point and mount option validation", "vast_targeting": "VAST functional group targeting validation", @@ -252,8 +272,10 @@ def pytest_configure(config): "image_download": "Explicitly authorized Apptainer image download checks", "negative": "Expected-failure and rejection behavior checks", "non_disruptive": "Checks that do not reboot or drain cluster nodes", - "disruptive": "Explicitly enabled reboot or scheduler-state checks", - "reboot": "Node reboot and post-reboot recovery checks", + "reboot": "Node reboot checks; run only when --marker selects reboot", + "node_lifecycle": ( + "Node remove/add checks; run only when --marker selects node_lifecycle" + ), "scheduler_state": "Scheduler drain, queue, and resume checks", "destructive": "Explicitly selected destructive cleanup checks", "nft": "Non-functional quality-contract checks", @@ -288,6 +310,14 @@ def _item_has_marker(item, marker_name): return item.get_closest_marker(marker_name) is not None +OPT_IN_MARKERS = ("reboot", "node_lifecycle") + + +def _opt_in_markers(item): + """Return the opt-in markers a test carries.""" + return {name for name in OPT_IN_MARKERS if _item_has_marker(item, name)} + + def pytest_collection_modifyitems(session, config, items): """Filter markers, apply safe defaults, and sort by order marker.""" marker_expr = config.getoption("--marker", default="") @@ -308,35 +338,26 @@ def pytest_collection_modifyitems(session, config, items): "negative", } ) - disruptive_authorized = bool( - explicitly_enabled & {"disruptive", "reboot", "scheduler_state"} - ) - - # Only apply the normal deploy auto-skip if no marker expression is given. + # Without a marker expression every collected test runs except opt-in + # (reboot, node_lifecycle) cases, which are deselected. Deploy cases still + # run only in the exec phase, which creates the verified state. if mode == "none": + selected = [] + deselected = [] for item in items: + if _opt_in_markers(item): + deselected.append(item) + continue if _item_has_marker(item, "deploy") and command_type != "exec": item.add_marker( pytest.mark.skip( "Deploy tests run only during the runner exec phase" ) ) - elif _item_has_marker(item, "disruptive"): - item.add_marker( - pytest.mark.skip( - "Select a disruptive, reboot, or scheduler_state marker " - "to authorize this test" - ) - ) - elif _item_has_marker(item, "functional") and not _item_has_marker( - item, "sanity" - ): - item.add_marker( - pytest.mark.skip( - "Select a functional or workload capability marker " - "to authorize this test" - ) - ) + selected.append(item) + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected else: # Feature filters apply to verification cases. The execution phase # still needs its one deploy test to create the state being verified. @@ -352,8 +373,9 @@ def pytest_collection_modifyitems(session, config, items): else: match = _item_has_marker(item, markers[0]) - if match and _item_has_marker(item, "disruptive"): - match = disruptive_authorized + opt_in = _opt_in_markers(item) + if match and opt_in: + match = bool(opt_in & explicitly_enabled) elif match and _item_has_marker(item, "functional"): match = functional_authorized if ( @@ -386,10 +408,28 @@ def _get_order(item): def pytest_runtest_setup(item): - """Expose only explicitly selected mutation markers to runtime helpers.""" + """Expose mutation authorization to runtime helpers. + + With no marker every gate a test carries is authorized; opt-in cases are + deselected at collection. With a marker expression only the selected + mutation markers are. + """ marker_expr = item.config.getoption("--marker", default="") _mode, markers = _parse_marker_expression(marker_expr) selected = set(markers) + if not selected: + # No marker selected: authorize every gate the test carries. Opt-in + # cases are deselected at collection. + authorized = { + marker + for marker in ("functional", "image_download") + if _item_has_marker(item, marker) + } + if authorized: + os.environ["OMNIA_FVT_AUTHORIZED_MARKERS"] = ",".join(sorted(authorized)) + else: + os.environ.pop("OMNIA_FVT_AUTHORIZED_MARKERS", None) + return sanity_authorized = _item_has_marker(item, "sanity") and ( not selected or "sanity" in selected or "buildstream" in selected ) @@ -407,12 +447,7 @@ def pytest_runtest_setup(item): } ): authorized.add("functional") - if _item_has_marker(item, "disruptive") and selected & { - "disruptive", - "reboot", - "scheduler_state", - }: - authorized.add("disruptive") + authorized |= _opt_in_markers(item) & selected if _item_has_marker(item, "image_download") and ( "image_download" in selected or sanity_authorized ): @@ -578,13 +613,7 @@ def pytest_runtest_makereport(item, call): skip_reason = "" if result.skipped: - if hasattr(result, "wasxfail"): - status = "SKIPPED" - rep_text = str(result.longrepr) if result.longrepr else "" - if "Skipped:" in rep_text: - skip_reason = rep_text.split("Skipped:", 1)[-1].strip() - elif "SKIP" in rep_text: - skip_reason = rep_text.split("SKIP", 1)[-1].strip() + skip_reason = _skip_reason(result) if status == "SKIPPED" and skip_reason: details = (details + "\n" if details else "") + f"SKIPPED: {skip_reason}" @@ -600,11 +629,19 @@ def pytest_runtest_makereport(item, call): if not tc_id: tc_id = get_last_tc_id() + if result.when == "setup": + # Marker and fixture skips happen before the test body creates its + # TestLogger, so record the start and skip lines here. + skip_log = TestLogger(_TC_TITLE_MAP.get(tc_id, item.name), tc_id) + skip_log.skipped(skip_reason or "Skipped before the test started") + details = skip_log.get_output() + (f"\nSKIPPED: {skip_reason}" if skip_reason else "") + add_session_result( test_name=item.name, status=status, duration=getattr(result, "duration", 0), tc_id=tc_id, + reason=skip_reason, ) report = get_current_report() diff --git a/test/orchestrator/docs/test_config.md b/test/orchestrator/docs/test_config.md index 4d83a3f54b..764156bd6c 100644 --- a/test/orchestrator/docs/test_config.md +++ b/test/orchestrator/docs/test_config.md @@ -66,6 +66,15 @@ always passed to the cleanup playbook: | `cleanup_k8s` | Delete Kubernetes shared data, then detach storage. | Preserve data, then detach storage. | | `cleanup_benchmark_tools` | Remove HPC benchmark staged artifacts after verification. | Retain artifacts for inspection. | +## Kubernetes local-etcd disk media + +`expected_etcd_disk_media` defaults to `""`, which skips the media check. +Set it to `ssd`, `hdd`, or `nvme` to require that every Kubernetes control +plane places `/var/lib/etcd` on that media when `etcd_on_local_disk` is +enabled. The selected disk is classified from `lsblk`: `TRAN=nvme` is NVMe, +`ROTA=1` is HDD, and `ROTA=0` is SSD. Any other value fails before the check +runs. + ## NFT performance thresholds `nft_performance_threshold_seconds` is a required mapping when NFT executes. diff --git a/test/orchestrator/docs/test_run_config.md b/test/orchestrator/docs/test_run_config.md index 17a395375a..fcb50c7c4e 100644 --- a/test/orchestrator/docs/test_run_config.md +++ b/test/orchestrator/docs/test_run_config.md @@ -44,10 +44,10 @@ non-zero aggregate result; set it to `true` to stop after the first failure. | Tag | Suites | |---|---| -| `precheck` | `environment`, `storage`, `dependencies`, `inputs` | +| `precheck` | `environment`, `oim_readiness`, `storage`, `dependencies`, `inputs` | | `prepare` | `openchami`, `network`, `openldap` | | `provision` | `openchami` | -| `pxeboot` | `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer`, `slurm_hpc_benchmarks`, `coredns_coredhcp`, `powervault` | +| `pxeboot` | `connectivity`, `cloudinit`, `additional_cloud_init`, `mount_config`, `minimal_os`, `powervault`, `coredns_coredhcp`, `kubernetes_cluster`, `kubernetes_etcd`, `kubernetes_storage`, `slurm_cluster`, `slurm_jobs`, `slurm_ldap`, `slurm_gpu`, `slurm_dcgm`, `slurm_openmpi`, `slurm_ucx`, `slurm_infiniband`, `slurm_apptainer`, `slurm_hpc_benchmarks`, `vast_storage`, `kubernetes_recovery`, `slurm_recovery`, `slurm_lifecycle` | | `cleanup` | `openchami`, `openldap`, `slurm`, `kubernetes`, `artifacts`, `credentials` | Discover the live catalog before editing the batch file: @@ -58,14 +58,24 @@ Discover the live catalog before editing the batch file: ## Markers -Registered markers include `sanity`, `functional`, `openldap`, `connectivity`, -`cloudinit`, `kubernetes`, `slurm`, `apptainer`, `benchmark`, `image_download`, -`negative`, `non_disruptive`, `disruptive`, `reboot`, `scheduler_state`, and -`destructive`. +Registered markers are listed in `MARKERS` in `library/vars/domain_vars.py`. +They include `sanity`, `functional`, `openldap`, `connectivity`, `cloudinit`, +`kubernetes`, `slurm`, `apptainer`, `benchmark`, `additional_cloud_init`, +`mount_config`, `minimal_os`, `boot_image`, the PowerVault selectors +`powervault_infrastructure`, `powervault_mounts`, `powervault_binds`, and +`powervault_cloudinit`, the VAST selectors `vast_installation`, `vast_mounts`, +`vast_scratch`, `vast_targeting`, and `vast_persistence`, `image_download`, +`negative`, `non_disruptive`, `reboot`, `node_lifecycle`, `scheduler_state`, +and `destructive`. Reboot tests run only when the expression selects `reboot`, +and node remove/add tests only when it selects `node_lifecycle`; otherwise they +are deselected. Examples: ```yaml +marker: "" # empty: every test except reboot/node_lifecycle +marker: "reboot" # reboot tests only +marker: "node_lifecycle" # node remove/add tests only marker: "sanity" # one marker marker: "sanity,functional" # OR marker: "slurm+non_disruptive" # AND diff --git a/test/orchestrator/fvt/README.md b/test/orchestrator/fvt/README.md index 731d922e57..d7e05fcd1e 100644 --- a/test/orchestrator/fvt/README.md +++ b/test/orchestrator/fvt/README.md @@ -25,17 +25,57 @@ ORCH_FVT__ | Lifecycle | Execution ID | Verification IDs | Suites | |---|---|---|---| -| `precheck` | `ORCH_FVT_PRECHECK_E001` | `V001`–`V012`, `V100`–`V109` | `environment`, `storage`, `dependencies`, `inputs`, `oim_readiness` | -| `prepare` | `ORCH_FVT_PREPARE_E001` | `V001`–`V013` | `openchami`, `network`, `openldap` | -| `provision` | `ORCH_FVT_PROVISION_E001` | `V001`–`V008` | `openchami` | -| `pxeboot` | `ORCH_FVT_PXEBOOT_E001` | `V001`–`V103`, `V095`–`V097`, `V219`–`V224`, `V501`–`V518` | `connectivity`, `cloudinit`, `kubernetes_*`, `slurm_*`, `slurm_dcgm`, `additional_cloud_init` | +| `precheck` | `ORCH_FVT_PRECHECK_E001` | `V001`–`V012`, `V100`–`V105`, `V107`–`V117` | `environment`, `oim_readiness`, `storage`, `dependencies`, `inputs` | +| `prepare` | `ORCH_FVT_PREPARE_E001` | `V001`–`V007`, `V009`–`V011`, `V013` | `openchami`, `network`, `openldap` | +| `provision` | `ORCH_FVT_PROVISION_E001` | `V001`–`V010` | `openchami` | +| `pxeboot` | `ORCH_FVT_PXEBOOT_E001` | `V001`–`V115`, `V117`–`V128`, `V200`–`V206`, `V208`–`V227`, `V300`–`V311`, `V400`–`V429`, `V500`–`V525`, `V700`–`V716` | see `SUITES` in `library/vars/domain_vars.py` | | `cleanup` | `ORCH_FVT_CLEANUP_E001` | `V001`–`V006` | `openchami`, `openldap`, `slurm`, `kubernetes`, `artifacts`, `credentials` | -The detailed registry below is the authoritative inventory. Its `Order` -column defines execution sequence independently of the stable public test-case -IDs, so files and test functions can be reorganized without renumbering IDs. +The detailed tables below are the authoritative inventory. Each test's +`Order` defines execution sequence independently of its stable public ID, so +files and functions can be reorganized without renumbering IDs. -## Effective execution order +### Retired and reassigned IDs + +Retired IDs are never reused. + +| Test | Previous ID | Current ID | Reason | +|---|---|---|---| +| `test_external_ldap_proxy` | `ORCH_FVT_PREPARE_V008` (retired) | `ORCH_FVT_PXEBOOT_V524` | Moved from prepare to the pxeboot `slurm_ldap` suite. | +| `test_external_ldap_backend` | `ORCH_FVT_PREPARE_V012` (retired) | `ORCH_FVT_PXEBOOT_V525` | Moved from prepare to the pxeboot `slurm_ldap` suite. | +| `test_kubernetes_configured_versions` | `ORCH_FVT_PXEBOOT_V101` (duplicate) | `ORCH_FVT_PXEBOOT_V519` | V101 remains with `test_mount_config_mount_point`, which registered it first. | +| `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V102` (duplicate) | `ORCH_FVT_PXEBOOT_V520` | V102 remains with `test_mount_config_volume_mounted`. | +| `test_kubernetes_nfs_provisioner_contract` | `ORCH_FVT_PXEBOOT_V103` (duplicate) | `ORCH_FVT_PXEBOOT_V521` | V103 remains with `test_mount_config_mount_options`. | +| `test_dcgm_cuda_validation` | `ORCH_FVT_PXEBOOT_V501` (duplicate) | `ORCH_FVT_PXEBOOT_V522` | V501 remains with `test_slurm_node_add`, which registered it first. | +| `test_hpc_benchmarks_prerequisites` | `ORCH_FVT_PXEBOOT_V095` (duplicate) | `ORCH_FVT_PXEBOOT_V225` | V095 remains with `test_additional_cloud_init_smd_groups`, which registered it first. | +| `test_hpc_benchmarks_staging_fingerprint_idempotency` | `ORCH_FVT_PXEBOOT_V096` (duplicate) | `ORCH_FVT_PXEBOOT_V226` | V096 remains with `test_additional_cloud_init_metadata_groups`. | +| `test_hpc_benchmarks_concurrent_staging` | `ORCH_FVT_PXEBOOT_V097` (duplicate) | `ORCH_FVT_PXEBOOT_V227` | V097 remains with `test_node_architecture`. | +| `test_hpc_benchmarks_per_tool_staging_report` | `ORCH_FVT_PXEBOOT_V207` (retired) | None | Removed upstream in #5491; staging is exercised by `test_hpc_benchmarks_run_staging` and the fingerprint idempotency case. | + +### Omnia 2.2 Kubernetes etcd and reboot traceability + +The 2.2 `molecule/kubernetes` cases below are covered by these 2.3 cases. +2.3 performs one reboot per recovery case and reports every 2.2 step as a +named field of that case instead of sharing state between tests. + +| 2.2 test | 2.3 test | 2.3 ID | Covered by | +|---|---|---|---| +| `test_tc_f03_disk_partitioning` | `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V520` | GPT partition table on the selected disk, which is not the root disk | +| `test_tc_f04_filesystem_creation` | `test_kubernetes_local_etcd_integrity` | `ORCH_FVT_PXEBOOT_V015` | ext4 filesystem labelled `etcd_data` | +| `test_tc_f05_fstab_update_and_mount` | `test_kubernetes_local_etcd_integrity` | `ORCH_FVT_PXEBOOT_V015` | One UUID-based fstab entry and an active mount | +| `test_tc_f06_etcd_configuration_to_local_disk` | `test_kubernetes_local_etcd`, `test_kubernetes_local_etcd_integrity` | `ORCH_FVT_PXEBOOT_V014`, `V015` | Non-NFS ext4 mount, `etcd:etcd 700`, and the etcd manifest data directory | +| `test_tc_f01_boss_card_detection`, `test_tc_f07_fallback_disk_detection` | `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V520` | BOSS disk selected when present, otherwise a fallback non-root disk, and the selection is logged | +| `test_tc_f08_first_boot_disk_setup` | `test_kubernetes_local_etcd_provisioning` | `ORCH_FVT_PXEBOOT_V520` | Both scripts executable and this boot's script finished with `DONE` | +| `test_tc_f10_ssd_disk_support`, `test_tc_f11_hdd_disk_support`, `test_tc_f12_nvme_disk_support` | `test_kubernetes_local_etcd_media` | `ORCH_FVT_PXEBOOT_V523` | Every control plane uses the `expected_etcd_disk_media` media | +| `test_etcd_reboot_control_plane`, `test_etcd_wait_node_online`, `test_etcd_wait_cloud_init` | `test_kubernetes_local_etcd_recovery` | `ORCH_FVT_PXEBOOT_V021` | Reboot, new boot ID, and cloud-init completion | +| `test_tc_f09_subsequent_boot_post_reboot` | `test_kubernetes_local_etcd_recovery` | `ORCH_FVT_PXEBOOT_V021` | `etcd-fstab-update.sh` finished with `DONE` on the new boot and `etcd-disk-setup.sh` did not run again | +| `test_tc_f05_fstab_mount_post_reboot`, `test_tc_f06_etcd_local_disk_post_reboot` | `test_kubernetes_local_etcd_recovery` | `ORCH_FVT_PXEBOOT_V021` | Local-etcd integrity after reboot and unchanged mount source and UUID | +| `test_etcd_cluster_health_post_reboot` | `test_kubernetes_local_etcd_recovery`, `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V021`, `V022` | Every etcd endpoint healthy within `ETCD_HEALTH_WAIT_TIMEOUT_SECONDS` | +| `test_reboot_vip_control_plane`, `test_verify_vip_failover` | `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V022` | VIP owner rebooted and the VIP moves to exactly one other control plane | +| `test_verify_cloud_init_after_reboot` | `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V022` | Cloud-init completes on the rebooted node | +| `test_verify_node_ready_after_reboot` | `test_kubernetes_control_plane_recovery` | `ORCH_FVT_PXEBOOT_V022` | Rebooted node returns to Ready, observed from another control plane | + +## Execution order blocks The default non-cleanup lifecycle is: @@ -43,424 +83,637 @@ The default non-cleanup lifecycle is: precheck -> prepare -> provision -> pxeboot ``` -Within a lifecycle, pytest follows each case's `order` marker. This preserves -dependency order: connectivity and cloud-init before workload clusters, -cluster health before temporary jobs, and non-disruptive checks before any -explicit recovery operation. +Every `@pytest.mark.order` value lies in its suite's block, defined by +`TEST_ORDER_BLOCKS` in `library/vars/test_case_vars.py`: + +```text +order = lifecycle base + suite slot * 100 + position in suite +``` -PXE capability suites are flat directories. Kubernetes runs as -`kubernetes_cluster`, `kubernetes_etcd`, `kubernetes_storage`, and -`kubernetes_recovery`. Slurm runs as `slurm_cluster`, `slurm_jobs`, -`slurm_ldap`, `slurm_gpu`, `slurm_openmpi`, `slurm_ucx`, -`slurm_infiniband`, `slurm_recovery`, `slurm_apptainer`, and `slurm_dcgm`. +| Lifecycle | Base | Block of the execution case | +|---|---:|---:| +| `precheck` | 10000 | 10000 | +| `prepare` | 20000 | 20000 | +| `provision` | 30000 | 30000 | +| `pxeboot` | 40000 | 40000 | +| `cleanup` | 50000 | 50000 | +| NFT | 60000 | — | + +Each suite owns 100 numbers, so a new test takes the next free number in its +own suite and never renumbers another suite. A new suite appends a block. +Suites run in dependency order: reachability and node state first, then +cluster health, temporary workloads, storage contracts such as +`vast_storage`, and finally the reboot, drain, and node-removal suites +(`kubernetes_recovery`, `slurm_recovery`, `slurm_lifecycle`). Within a suite, +reboot and destructive cases come last. Never reuse an order value. Cleanup is never part of an implicit lifecycle run. ## Precheck test cases -These cases execute and verify OIM identity, selected storage, dependency -outputs, required inputs, boot artifacts, and published repositories. +These cases execute and verify OIM identity and readiness, selected storage, dependency outputs, required inputs, boot artifacts, and published repositories. + +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10000 | `ORCH_FVT_PRECHECK_E001` | `test_deploy_precheck` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags precheck`` exactly once. | The selected Orchestrator lifecycle exits successfully. | + +### Environment (`environment`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10101 | `ORCH_FVT_PRECHECK_V001` | `test_precheck_hostname_domain` | `environment` | `sanity` | Require the host identity to match omnia.env exactly. | Every required item satisfies the stated condition. | +| 10102 | `ORCH_FVT_PRECHECK_V002` | `test_precheck_admin_ipv4` | `environment` | `sanity` | Require the configured administrative IPv4 on a global interface. | Every required item satisfies the stated condition. | + +### OIM readiness (`oim_readiness`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 0 | `ORCH_FVT_PRECHECK_E001` | `test_deploy_precheck` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags precheck`` exactly once. | The selected Orchestrator lifecycle exits successfully. | -| 1 | `ORCH_FVT_PRECHECK_V001` | `test_precheck_hostname_domain` | `environment` | `sanity` | Require the host identity to match omnia.env exactly. | Every required item satisfies the stated condition. | -| 2 | `ORCH_FVT_PRECHECK_V002` | `test_precheck_admin_ipv4` | `environment` | `sanity` | Require the configured administrative IPv4 on a global interface. | Every required item satisfies the stated condition. | -| 3 | `ORCH_FVT_PRECHECK_V003` | `test_precheck_nfs_servers` | `storage` | `sanity` | Require every mapping-selected NFS server to answer ICMP from the OIM. | Every required item satisfies the stated condition. | -| 4 | `ORCH_FVT_PRECHECK_V004` | `test_precheck_dependencies` | `dependencies` | `sanity` | Require both configured/default dependency outputs to be usable. | Every required item satisfies the stated condition. | -| 5 | `ORCH_FVT_PRECHECK_V005` | `test_precheck_inputs` | `inputs` | `sanity` | Require every source-selected Orchestrator input to be non-empty. | Every required item satisfies the stated condition. | -| 6 | `ORCH_FVT_PRECHECK_V006` | `test_precheck_s3_artifacts` | `dependencies` | `sanity` | Require every kernel, initrd, and rootfs artifact to be reachable. | Every required item satisfies the stated condition. | -| 7 | `ORCH_FVT_PRECHECK_V007` | `test_precheck_repositories` | `dependencies` | `sanity` | Require every published RPM and file repository to be reachable. | Every required item satisfies the stated condition. | +| 10201 | `ORCH_FVT_PRECHECK_V100` | `test_oim_cpu_threshold` | `oim_readiness` | `sanity` | Require OIM CPU core count to meet the configured minimum. | Every required item satisfies the stated condition. | +| 10202 | `ORCH_FVT_PRECHECK_V101` | `test_oim_memory_threshold` | `oim_readiness` | `sanity` | Require OIM memory to meet the configured minimum. | Every required item satisfies the stated condition. | +| 10203 | `ORCH_FVT_PRECHECK_V102` | `test_oim_disk_threshold` | `oim_readiness` | `sanity` | Require OIM root filesystem to meet the configured minimum. | Every required item satisfies the stated condition. | +| 10204 | `ORCH_FVT_PRECHECK_V103` | `test_oim_pxe_nic_present` | `oim_readiness` | `sanity` | Require the configured admin NIC to exist and be UP. | Every required item satisfies the stated condition. | +| 10205 | `ORCH_FVT_PRECHECK_V104` | `test_oim_public_nic_present` | `oim_readiness` | `sanity` | Require the public/default-route NIC to exist and be UP. | Every required item satisfies the stated condition. | +| 10206 | `ORCH_FVT_PRECHECK_V105` | `test_oim_pxe_nic_ipv4` | `oim_readiness` | `sanity` | Require the admin NIC to carry the configured IPv4 address. | Every required item satisfies the stated condition. | +| 10207 | `ORCH_FVT_PRECHECK_V107` | `test_oim_ssh_preflight` | `oim_readiness` | `sanity` | Require passwordless SSH from OIM to a mapped target node. | Every required item satisfies the stated condition. | +| 10208 | `ORCH_FVT_PRECHECK_V108` | `test_oim_internet_reachability` | `oim_readiness` | `sanity` | Require internet reachability when not in air-gapped mode. | Every required item satisfies the stated condition. | +| 10209 | `ORCH_FVT_PRECHECK_V109` | `test_oim_os_version` | `oim_readiness` | `sanity` | Require the OIM OS to match the expected distribution and version. | Every required item satisfies the stated condition. | +| 10210 | `ORCH_FVT_PRECHECK_V110` | `test_neg_cpu_below_threshold` | `oim_readiness` | `negative` | Detect failure when CPU threshold exceeds actual cores. | The expected rejection occurs and no prohibited state is accepted. | +| 10211 | `ORCH_FVT_PRECHECK_V111` | `test_neg_memory_below_threshold` | `oim_readiness` | `negative` | Detect failure when memory threshold exceeds actual RAM. | The expected rejection occurs and no prohibited state is accepted. | +| 10212 | `ORCH_FVT_PRECHECK_V112` | `test_neg_disk_below_threshold` | `oim_readiness` | `negative` | Detect failure when disk threshold exceeds actual capacity. | The expected rejection occurs and no prohibited state is accepted. | +| 10213 | `ORCH_FVT_PRECHECK_V113` | `test_neg_pxe_nic_missing` | `oim_readiness` | `negative` | Detect failure when a nonexistent NIC name is checked. | The expected rejection occurs and no prohibited state is accepted. | +| 10214 | `ORCH_FVT_PRECHECK_V114` | `test_neg_pxe_public_overlap` | `oim_readiness` | `negative` | Detect overlap when PXE NIC is forced to match the public NIC. | The expected rejection occurs and no prohibited state is accepted. | +| 10215 | `ORCH_FVT_PRECHECK_V115` | `test_neg_internet_airgapped` | `oim_readiness` | `negative` | Verify air-gapped mode passes even without internet. | The expected rejection occurs and no prohibited state is accepted. | +| 10216 | `ORCH_FVT_PRECHECK_V116` | `test_neg_ssh_unreachable_target` | `oim_readiness` | `negative` | Detect SSH failure to a bogus target address. | The expected rejection occurs and no prohibited state is accepted. | +| 10217 | `ORCH_FVT_PRECHECK_V117` | `test_neg_os_version_mismatch` | `oim_readiness` | `negative` | Detect failure when expected OS version does not match actual. | The expected rejection occurs and no prohibited state is accepted. | + +Every check is read-only. Thresholds default to CPU >= 4, RAM >= 16 GB, and +root disk >= 100 GB. Negative cases use the unattainable inputs in +`OIM_NEGATIVE_INPUTS` (`library/vars/precheck_vars.py`) and pass only when +the checker rejects them with an actionable error. + +### Storage (`storage`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10301 | `ORCH_FVT_PRECHECK_V003` | `test_precheck_nfs_servers` | `storage` | `sanity` | Require every mapping-selected NFS server to answer ICMP from the OIM. | Every required item satisfies the stated condition. | +| 10302 | `ORCH_FVT_PRECHECK_V008` | `test_mount_missing_mount_point` | `storage` | `mount_config`, `negative` | TC-CI-NEG-001: Every mount entry must have a valid absolute mount_point. | The expected rejection occurs and no prohibited state is accepted. | +| 10303 | `ORCH_FVT_PRECHECK_V009` | `test_mount_missing_targeting` | `storage` | `mount_config`, `negative` | TC-CI-NEG-002: Every mount entry must have targeting (prefix or groups). | The expected rejection occurs and no prohibited state is accepted. | +| 10304 | `ORCH_FVT_PRECHECK_V010` | `test_mount_invalid_mount_params` | `storage` | `mount_config`, `negative` | TC-CI-NEG-003: mount_params must reference an existing profile. | The expected rejection occurs and no prohibited state is accepted. | +| 10305 | `ORCH_FVT_PRECHECK_V011` | `test_mount_missing_source` | `storage` | `mount_config`, `negative` | TC-CI-NEG-004: Every mount entry must have a non-empty source. | The expected rejection occurs and no prohibited state is accepted. | +| 10306 | `ORCH_FVT_PRECHECK_V012` | `test_mount_node_key_without_mount_point` | `storage` | `mount_config`, `negative` | TC-CI-NEG-005: node_mount_point is mandatory when node_key is set. | The expected rejection occurs and no prohibited state is accepted. | The NFS case mirrors the production mount-selection contract. It evaluates `functional_group_prefix` against mapped functional groups, evaluates `groups` against mapped PXE `GROUP_NAME` values, and excludes the stock -`vast_storage` entry unless `omnia_config.yml` selects it. Unrelated storage -entries are reported as ignored, not as validated. - -### OIM readiness (oim_readiness) - -Hardware, network-interface, and prerequisite checks that bring parity with -the Omnia 2.2 automation prerequisite runner. Every check is read-only and -never mutates the target environment. Negative tests use impossible -thresholds or bogus names to validate structured failure reporting. - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 10 | `ORCH_FVT_PRECHECK_V100` | `test_oim_cpu_threshold` | `oim_readiness` | `sanity` | Require OIM CPU core count to meet the configured minimum. | Every required item satisfies the stated condition. | -| 11 | `ORCH_FVT_PRECHECK_V101` | `test_oim_memory_threshold` | `oim_readiness` | `sanity` | Require OIM memory to meet the configured minimum. | Every required item satisfies the stated condition. | -| 12 | `ORCH_FVT_PRECHECK_V102` | `test_oim_disk_threshold` | `oim_readiness` | `sanity` | Require OIM root filesystem to meet the configured minimum. | Every required item satisfies the stated condition. | -| 13 | `ORCH_FVT_PRECHECK_V103` | `test_oim_pxe_nic_present` | `oim_readiness` | `sanity` | Require the configured admin NIC to exist and be UP. | Every required item satisfies the stated condition. | -| 14 | `ORCH_FVT_PRECHECK_V104` | `test_oim_public_nic_present` | `oim_readiness` | `sanity` | Require the public/default-route NIC to exist and be UP. | Every required item satisfies the stated condition. | -| 15 | `ORCH_FVT_PRECHECK_V105` | `test_oim_pxe_nic_ipv4` | `oim_readiness` | `sanity` | Require the admin NIC to carry the configured IPv4 address. | Every required item satisfies the stated condition. | -| 16 | `ORCH_FVT_PRECHECK_V107` | `test_oim_ssh_preflight` | `oim_readiness` | `sanity` | Require passwordless SSH from OIM to a mapped target node. | Every required item satisfies the stated condition. | -| 18 | `ORCH_FVT_PRECHECK_V108` | `test_oim_internet_reachability` | `oim_readiness` | `sanity` | Require internet reachability when not in air-gapped mode. | Every required item satisfies the stated condition. | -| 19 | `ORCH_FVT_PRECHECK_V109` | `test_oim_os_version` | `oim_readiness` | `sanity` | Require the OIM OS to match the expected distribution and version. | Every required item satisfies the stated condition. | -| 20 | `ORCH_FVT_PRECHECK_V100` | `test_neg_cpu_below_threshold` | `oim_readiness` | `sanity`, `negative` | Detect failure when CPU threshold exceeds actual cores. | The expected rejection occurs and no prohibited state is accepted. | -| 21 | `ORCH_FVT_PRECHECK_V101` | `test_neg_memory_below_threshold` | `oim_readiness` | `sanity`, `negative` | Detect failure when memory threshold exceeds actual RAM. | The expected rejection occurs and no prohibited state is accepted. | -| 22 | `ORCH_FVT_PRECHECK_V102` | `test_neg_disk_below_threshold` | `oim_readiness` | `sanity`, `negative` | Detect failure when disk threshold exceeds actual capacity. | The expected rejection occurs and no prohibited state is accepted. | -| 23 | `ORCH_FVT_PRECHECK_V103` | `test_neg_pxe_nic_missing` | `oim_readiness` | `sanity`, `negative` | Detect failure when a nonexistent NIC name is checked. | The expected rejection occurs and no prohibited state is accepted. | -| 24 | `ORCH_FVT_PRECHECK_V105` | `test_neg_pxe_public_overlap` | `oim_readiness` | `sanity`, `negative` | Detect overlap when PXE NIC is forced to match the public NIC. | The expected rejection occurs and no prohibited state is accepted. | -| 25 | `ORCH_FVT_PRECHECK_V108` | `test_neg_internet_airgapped` | `oim_readiness` | `sanity`, `negative` | Verify air-gapped mode passes even without internet. | The expected rejection occurs and no prohibited state is accepted. | -| 27 | `ORCH_FVT_PRECHECK_V107` | `test_neg_ssh_unreachable_target` | `oim_readiness` | `sanity`, `negative` | Detect SSH failure to a bogus target address. | The expected rejection occurs and no prohibited state is accepted. | -| 28 | `ORCH_FVT_PRECHECK_V109` | `test_neg_os_version_mismatch` | `oim_readiness` | `sanity`, `negative` | Detect failure when expected OS version does not match actual. | The expected rejection occurs and no prohibited state is accepted. | - -Thresholds default to CPU >= 4, RAM >= 16 GB, root disk >= 100 GB (matching -Omnia 2.2 automation defaults). Negative tests set impossible thresholds to -validate structured failure messages without mutating the environment. +`vast_storage` entry unless `omnia_config.yml` selects it. + +### Dependencies (`dependencies`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10401 | `ORCH_FVT_PRECHECK_V004` | `test_precheck_dependencies` | `dependencies` | `sanity` | Require both configured/default dependency outputs to be usable. | Every required item satisfies the stated condition. | +| 10402 | `ORCH_FVT_PRECHECK_V006` | `test_precheck_s3_artifacts` | `dependencies` | `sanity` | Require every kernel, initrd, and rootfs artifact to be reachable. | Every required item satisfies the stated condition. | +| 10403 | `ORCH_FVT_PRECHECK_V007` | `test_precheck_repositories` | `dependencies` | `sanity` | Require every published RPM and file repository to be reachable. | Every required item satisfies the stated condition. | + +### Inputs (`inputs`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 10501 | `ORCH_FVT_PRECHECK_V005` | `test_precheck_inputs` | `inputs` | `sanity` | Require every source-selected Orchestrator input to be non-empty. | Every required item satisfies the stated condition. | ## Prepare test cases These cases verify the infrastructure created by the `prepare` lifecycle. +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 20000 | `ORCH_FVT_PREPARE_E001` | `test_deploy_prepare` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags prepare``. | The selected Orchestrator lifecycle exits successfully. | + +### OpenCHAMI (`openchami`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 20101 | `ORCH_FVT_PREPARE_V001` | `test_openchami_containers_running` | `openchami` | `sanity` | Verify all long-running OpenCHAMI containers. | All stated checks pass for every applicable target. | +| 20102 | `ORCH_FVT_PREPARE_V002` | `test_openchami_services_ready` | `openchami` | `sanity` | Verify OpenCHAMI units and successful SMD initialization. | All stated checks pass for every applicable target. | +| 20103 | `ORCH_FVT_PREPARE_V003` | `test_openchami_apis_ready` | `openchami` | `functional`, `sanity` | Verify the authenticated SMD, Boot and Metadata APIs. | All stated checks pass for every applicable target. | +| 20104 | `ORCH_FVT_PREPARE_V004` | `test_openchami_persistent_storage_and_tls` | `openchami` | `sanity` | Verify persistent data volumes and HAProxy certificates. | All stated checks pass for every applicable target. | +| 20105 | `ORCH_FVT_PREPARE_V005` | `test_openchami_packages_and_artifacts` | `openchami` | `sanity` | Verify installed packages and generated configuration files. | All stated checks pass for every applicable target. | +| 20106 | `ORCH_FVT_PREPARE_V013` | `test_postgresql_readiness` | `openchami` | `functional`, `sanity` | Verify PostgreSQL and its required SMD database contract. | All stated checks pass for every applicable target. | + +### Network (`network`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 20201 | `ORCH_FVT_PREPARE_V006` | `test_firewall_and_podman_network_policy` | `network` | `sanity` | Verify OpenCHAMI ports and trusted Podman interfaces. | All stated checks pass for every applicable target. | +| 20202 | `ORCH_FVT_PREPARE_V007` | `test_coredhcp_and_coredns_configuration` | `network` | `functional`, `sanity` | Verify rendered DHCP/DNS configuration and additional routes. | All stated checks pass for every applicable target. | + +### OpenLDAP (`openldap`) + | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 0 | `ORCH_FVT_PREPARE_E001` | `test_deploy_prepare` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags prepare``. | The selected Orchestrator lifecycle exits successfully. | -| 1 | `ORCH_FVT_PREPARE_V001` | `test_openchami_containers_running` | `openchami` | `sanity` | Verify all long-running OpenCHAMI containers. | All stated checks pass for every applicable target. | -| 2 | `ORCH_FVT_PREPARE_V002` | `test_openchami_services_ready` | `openchami` | `sanity` | Verify OpenCHAMI units and successful SMD initialization. | All stated checks pass for every applicable target. | -| 3 | `ORCH_FVT_PREPARE_V003` | `test_openchami_apis_ready` | `openchami` | `functional`, `sanity` | Verify the authenticated SMD, Boot and Metadata APIs. | All stated checks pass for every applicable target. | -| 4 | `ORCH_FVT_PREPARE_V004` | `test_openchami_persistent_storage_and_tls` | `openchami` | `sanity` | Verify persistent data volumes and HAProxy certificates. | All stated checks pass for every applicable target. | -| 5 | `ORCH_FVT_PREPARE_V005` | `test_openchami_packages_and_artifacts` | `openchami` | `sanity` | Verify installed packages and generated configuration files. | All stated checks pass for every applicable target. | -| 6 | `ORCH_FVT_PREPARE_V006` | `test_firewall_and_podman_network_policy` | `network` | `sanity` | Verify OpenCHAMI ports and trusted Podman interfaces. | All stated checks pass for every applicable target. | -| 7 | `ORCH_FVT_PREPARE_V007` | `test_coredhcp_and_coredns_configuration` | `network` | `functional`, `sanity` | Verify rendered DHCP/DNS configuration and additional routes. | All stated checks pass for every applicable target. | -| 8 | `ORCH_FVT_PREPARE_V008` | `test_external_ldap_proxy` | `openldap` | `openldap`, `sanity` | Reconcile and verify the explicitly enabled LDAP meta-proxy. | Reconciliation is idempotent and every postcondition passes. | -| 9 | `ORCH_FVT_PREPARE_V009` | `test_openldap_runtime` | `openldap` | `openldap`, `sanity` | Verify the enabled service and container are healthy. | All stated checks pass for every applicable target. | -| 10 | `ORCH_FVT_PREPARE_V010` | `test_openldap_artifacts` | `openldap` | `openldap`, `sanity` | Verify configuration modes, syntax and TLS lifetime. | All stated checks pass for every applicable target. | -| 11 | `ORCH_FVT_PREPARE_V011` | `test_openldap_endpoint` | `openldap` | `functional`, `openldap`, `sanity` | Verify the local LDAP endpoint and published listeners. | All stated checks pass for every applicable target. | -| 12 | `ORCH_FVT_PREPARE_V012` | `test_external_ldap_backend` | `openldap` | `functional`, `openldap`, `sanity` | Verify external LDAP reachability from the omnia_auth container. | All stated checks pass for every applicable target. | -| 13 | `ORCH_FVT_PREPARE_V013` | `test_postgresql_readiness` | `openchami` | `functional`, `sanity` | Verify PostgreSQL and its required SMD database contract. | All stated checks pass for every applicable target. | +| 20301 | `ORCH_FVT_PREPARE_V009` | `test_openldap_runtime` | `openldap` | `openldap`, `sanity` | Verify the enabled service and container are healthy. | All stated checks pass for every applicable target. | +| 20302 | `ORCH_FVT_PREPARE_V010` | `test_openldap_artifacts` | `openldap` | `openldap`, `sanity` | Verify configuration modes, syntax and TLS lifetime. | All stated checks pass for every applicable target. | +| 20303 | `ORCH_FVT_PREPARE_V011` | `test_openldap_endpoint` | `openldap` | `functional`, `openldap`, `sanity` | Verify the local LDAP endpoint and published listeners. | All stated checks pass for every applicable target. | OpenLDAP runtime, artifacts, and local endpoint checks follow authoritative -generated `openldap_support` state. External LDAP V008 and V012 additionally -require `validate_external_ldap: true`; otherwise they skip before mutation. -`configure_external_ldap` controls only whether V008 may reconcile -`slapd.conf`. With validation enabled and reconciliation disabled, V008 -checks the existing deployed proxy and reports mismatches as failures. +generated `openldap_support` state. The prepare lifecycle only starts the +local `omnia_auth` container; the external LDAP proxy cases run in the +pxeboot `slurm_ldap` suite. ## Provision test cases -These controller-side cases compare generated reports and live OpenCHAMI -state with the active PXE mapping. +These controller-side cases compare generated reports and live OpenCHAMI state with the active PXE mapping. + +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 30000 | `ORCH_FVT_PROVISION_E001` | `test_deploy_provision` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags provision``. | The selected Orchestrator lifecycle exits successfully. | + +### OpenCHAMI (`openchami`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 100 | `ORCH_FVT_PROVISION_E001` | `test_deploy_provision` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags provision``. | The selected Orchestrator lifecycle exits successfully. | -| 101 | `ORCH_FVT_PROVISION_V001` | `test_provision_reports` | `openchami` | `sanity` | Verify the provision report and generated inventory contracts. | All stated checks pass for every applicable target. | -| 102 | `ORCH_FVT_PROVISION_V002` | `test_smd_identity` | `openchami` | `sanity` | Verify XNAME, administrative MAC, and IP identity in SMD. | All stated checks pass for every applicable target. | -| 103 | `ORCH_FVT_PROVISION_V003` | `test_smd_group_membership` | `openchami` | `sanity` | Verify expected groups and reject competing cloud-init groups. | All stated checks pass for every applicable target. | -| 104 | `ORCH_FVT_PROVISION_V004` | `test_boot_service_configurations` | `openchami` | `sanity` | Verify BootConfigurations and their mapped administrative MACs. | All stated checks pass for every applicable target. | -| 105 | `ORCH_FVT_PROVISION_V005` | `test_boot_service_node_identity` | `openchami` | `sanity` | Verify synchronized XNAME-to-bootMac records. | All stated checks pass for every applicable target. | -| 106 | `ORCH_FVT_PROVISION_V006` | `test_metadata_service_groups` | `openchami` | `sanity` | Verify one usable cloud-init template per functional group. | All stated checks pass for every applicable target. | -| 107 | `ORCH_FVT_PROVISION_V007` | `test_metadata_service_instances` | `openchami` | `sanity` | Verify unique per-node hostname metadata. | All stated checks pass for every applicable target. | -| 108 | `ORCH_FVT_PROVISION_V008` | `test_coredhcp_and_coredns_inventory` | `openchami` | `sanity` | Verify the SMD identity records consumed by CoreDHCP/CoreDNS. | All stated checks pass for every applicable target. | -| 109 | `ORCH_FVT_PROVISION_V009` | `test_boot_image_identity` | `openchami` | `sanity` | Verify Boot Service kernel/initrd paths match build_status.yml per functional group. | All stated checks pass for every applicable target. | -| 110 | `ORCH_FVT_PROVISION_V010` | `test_boot_image_architecture` | `openchami` | `sanity` | Verify build_status.yml architecture keys are consistent with functional group name suffixes. | All stated checks pass for every applicable target. | +| 30101 | `ORCH_FVT_PROVISION_V001` | `test_provision_reports` | `openchami` | `sanity` | Verify the provision report and generated inventory contracts. | All stated checks pass for every applicable target. | +| 30102 | `ORCH_FVT_PROVISION_V002` | `test_smd_identity` | `openchami` | `sanity` | Verify XNAME, administrative MAC, and IP identity in SMD. | All stated checks pass for every applicable target. | +| 30103 | `ORCH_FVT_PROVISION_V003` | `test_smd_group_membership` | `openchami` | `sanity` | Verify expected groups and reject competing cloud-init groups. | All stated checks pass for every applicable target. | +| 30104 | `ORCH_FVT_PROVISION_V004` | `test_boot_service_configurations` | `openchami` | `sanity` | Verify BootConfigurations and their mapped administrative MACs. | All stated checks pass for every applicable target. | +| 30105 | `ORCH_FVT_PROVISION_V005` | `test_boot_service_node_identity` | `openchami` | `sanity` | Verify synchronized XNAME-to-bootMac records. | All stated checks pass for every applicable target. | +| 30106 | `ORCH_FVT_PROVISION_V006` | `test_metadata_service_groups` | `openchami` | `sanity` | Verify one usable cloud-init template per functional group. | All stated checks pass for every applicable target. | +| 30107 | `ORCH_FVT_PROVISION_V007` | `test_metadata_service_instances` | `openchami` | `sanity` | Verify unique per-node hostname metadata. | All stated checks pass for every applicable target. | +| 30108 | `ORCH_FVT_PROVISION_V008` | `test_coredhcp_and_coredns_inventory` | `openchami` | `sanity` | Verify the SMD identity records consumed by CoreDHCP/CoreDNS. | All stated checks pass for every applicable target. | +| 30109 | `ORCH_FVT_PROVISION_V009` | `test_boot_image_identity` | `openchami` | `boot_image` | Verify Boot Service kernel/initrd paths match build_status.yml. | All stated checks pass for every applicable target. | +| 30110 | `ORCH_FVT_PROVISION_V010` | `test_boot_image_architecture` | `openchami` | `boot_image` | Verify build_status.yml architecture keys match functional group names. | All stated checks pass for every applicable target. | Provision verification is read-only. It resolves XNAMEs from live SMD interfaces and groups and obtains fresh OpenCHAMI credentials for API reads. ## PXE post-boot test cases -The following tables are in effective pytest `order` sequence. Optional role -and feature cases skip only when the active mapping or catalog proves that the -target is not applicable. Negative cases pass only when the invalid operation -is rejected. Reboot and scheduler-state cases require their explicit markers. +Optional role and feature cases skip only when the active mapping, catalog, or `test_config.yml` proves that the target is not applicable. Negative cases pass only when the invalid operation is rejected. Reboot, drain, and node-removal suites run last; reboot and node-removal cases require their explicit opt-in markers, while `scheduler_state` only narrows selection. -### PXE lifecycle execution +### Lifecycle execution | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 200 | `ORCH_FVT_PXEBOOT_E001` | `test_deploy_pxeboot` | `root` | `deploy`, `sanity` | Run ``orchestrator.yml --tags pxeboot``. | The selected Orchestrator lifecycle exits successfully. | +| 40000 | `ORCH_FVT_PXEBOOT_E001` | `test_deploy_pxeboot` | `root` | `buildstream`, `deploy`, `sanity` | Run ``orchestrator.yml --tags pxeboot``. | The selected Orchestrator lifecycle exits successfully. | -### Connectivity +### Connectivity (`connectivity`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 201 | `ORCH_FVT_PXEBOOT_V001` | `test_node_ping` | `connectivity` | `connectivity`, `sanity` | Verify ping from the OIM to every mapped administrative IP. | All stated checks pass for every applicable target. | -| 202 | `ORCH_FVT_PXEBOOT_V002` | `test_node_ssh` | `connectivity` | `connectivity`, `sanity` | Verify passwordless root SSH from the OIM to every mapped node. | All stated checks pass for every applicable target. | -| 203 | `ORCH_FVT_PXEBOOT_V003` | `test_node_hostname_ssh` | `connectivity` | `connectivity`, `sanity` | Verify mapped hostnames resolve and support passwordless root SSH. | All stated checks pass for every applicable target. | +| 40101 | `ORCH_FVT_PXEBOOT_V001` | `test_node_ping` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify ping from the OIM to every mapped administrative IP. | All stated checks pass for every applicable target. | +| 40102 | `ORCH_FVT_PXEBOOT_V002` | `test_node_ssh` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify passwordless root SSH from the OIM to every mapped node. | All stated checks pass for every applicable target. | +| 40103 | `ORCH_FVT_PXEBOOT_V003` | `test_node_hostname_ssh` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify mapped hostnames resolve and support passwordless root SSH. | All stated checks pass for every applicable target. | +| 40104 | `ORCH_FVT_PXEBOOT_V097` | `test_node_architecture` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify each node's live architecture matches its functional group. | All stated checks pass for every applicable target. | +| 40105 | `ORCH_FVT_PXEBOOT_V098` | `test_node_os_version` | `connectivity` | `buildstream`, `connectivity`, `sanity` | Verify each node's live OS version matches its functional group. | All stated checks pass for every applicable target. | -### Cloud-init +### Cloud-init (`cloudinit`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 204 | `ORCH_FVT_PXEBOOT_V004` | `test_node_cloud_init` | `cloudinit` | `cloudinit`, `sanity` | Verify PXE report freshness and direct cloud-init JSON state. | All stated checks pass for every applicable target. | +| 40201 | `ORCH_FVT_PXEBOOT_V004` | `test_node_cloud_init` | `cloudinit` | `buildstream`, `cloudinit`, `sanity` | Verify PXE report freshness and direct cloud-init JSON state. | All stated checks pass for every applicable target. | + +Cloud-init is accepted only when its structured status satisfies the +product contract. A generated script success message is not treated as +authoritative cloud-init state. -### Architecture and OS identity +### Additional cloud-init (`additional_cloud_init`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 298 | `ORCH_FVT_PXEBOOT_V097` | `test_node_architecture` | `connectivity` | `connectivity`, `sanity` | Verify each node's live architecture matches its functional group name suffix. | All stated checks pass for every applicable target. | -| 299 | `ORCH_FVT_PXEBOOT_V098` | `test_node_os_version` | `connectivity` | `connectivity`, `sanity` | Verify each node's live OS version matches the expected image from build_status.yml. | All stated checks pass for every applicable target. | +| 40301 | `ORCH_FVT_PXEBOOT_V095` | `test_additional_cloud_init_smd_groups` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify SMD groups exist for additional cloud-init configuration. | All stated checks pass for every applicable target. | +| 40302 | `ORCH_FVT_PXEBOOT_V096` | `test_additional_cloud_init_metadata_groups` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify metadata-service groups and templates for additional cloud-init. | All stated checks pass for every applicable target. | +| 40303 | `ORCH_FVT_PXEBOOT_V099` | `test_additional_cloud_init_write_files` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify write_files entries were applied on provisioned nodes. | All stated checks pass for every applicable target. | +| 40304 | `ORCH_FVT_PXEBOOT_V100` | `test_additional_cloud_init_runcmd` | `additional_cloud_init` | `additional_cloud_init`, `buildstream`, `non_disruptive`, `sanity` | Verify runcmd entries executed during cloud-init on provisioned nodes. | All stated checks pass for every applicable target. | -Cloud-init is accepted only when its structured status satisfies the -product contract. A generated script success message is not treated as -authoritative cloud-init state. +These cases skip when `additional_cloud_init_config_file` is empty or not +configured in `orchestrator_config.yml`. + +### Mount configuration (`mount_config`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40401 | `ORCH_FVT_PXEBOOT_V101` | `test_mount_config_mount_point` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mount point directories exist on all target nodes. | All stated checks pass for every applicable target. | +| 40402 | `ORCH_FVT_PXEBOOT_V102` | `test_mount_config_volume_mounted` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS volumes are actively mounted on all target nodes. | All stated checks pass for every applicable target. | +| 40403 | `ORCH_FVT_PXEBOOT_V103` | `test_mount_config_mount_options` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mount options match storage_config.yml on target nodes. | All stated checks pass for every applicable target. | +| 40404 | `ORCH_FVT_PXEBOOT_V104` | `test_mount_config_fstab` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS fstab entries are persistent on all target nodes. | All stated checks pass for every applicable target. | +| 40405 | `ORCH_FVT_PXEBOOT_V105` | `test_mount_config_bind_mounts` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS bind mount targets are active on all target nodes. | All stated checks pass for every applicable target. | +| 40406 | `ORCH_FVT_PXEBOOT_V106` | `test_mount_config_bind_fstab` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS bind mount fstab entries are persistent on target nodes. | All stated checks pass for every applicable target. | +| 40407 | `ORCH_FVT_PXEBOOT_V107` | `test_mount_config_node_subdirectory` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify per-node subdirectory exists under NFS mount point. | All stated checks pass for every applicable target. | +| 40408 | `ORCH_FVT_PXEBOOT_V108` | `test_mount_config_permissions` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mount permissions match storage_config.yml on nodes. | All stated checks pass for every applicable target. | +| 40409 | `ORCH_FVT_PXEBOOT_V109` | `test_mount_config_fg_targeting` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS mounts are present on target FGs and absent on others. | All stated checks pass for every applicable target. | +| 40410 | `ORCH_FVT_PXEBOOT_V110` | `test_mount_config_no_duplicate_fstab` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify no duplicate NFS fstab entries on target nodes. | All stated checks pass for every applicable target. | +| 40411 | `ORCH_FVT_PXEBOOT_V111` | `test_mount_config_writable` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify all configured NFS mounts are writable on target nodes. | All stated checks pass for every applicable target. | +| 40412 | `ORCH_FVT_PXEBOOT_V112` | `test_mount_config_oim_mount` | `mount_config` | `buildstream`, `mount_config`, `sanity` | Verify NFS storage is mounted on the OIM when mount_on_oim is true. | All stated checks pass for every applicable target. | + +### Minimal OS (`minimal_os`) -### Kubernetes - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 205 | `ORCH_FVT_PXEBOOT_V005` | `test_kubernetes_nodes` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify mapped Kubernetes membership and Ready state. | All stated checks pass for every applicable target. | -| 206 | `ORCH_FVT_PXEBOOT_V006` | `test_kubernetes_node_services` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify required services on every Kubernetes role. | All stated checks pass for every applicable target. | -| 207 | `ORCH_FVT_PXEBOOT_V007` | `test_kubernetes_version_compatibility` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify Kubernetes, kubeadm, and CRI-O version alignment. | All stated checks pass for every applicable target. | -| 208 | `ORCH_FVT_PXEBOOT_V101` | `test_kubernetes_configured_versions` | `kubernetes_cluster` | `kubernetes`, `sanity` | Compare deployed Kubernetes components with the selected catalog version. | Kubernetes components match the configured version and CRI-O matches its major/minor release. | -| 209 | `ORCH_FVT_PXEBOOT_V008` | `test_kubernetes_control_plane` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify API readiness and the configured control plane. | All stated checks pass for every applicable target. | -| 210 | `ORCH_FVT_PXEBOOT_V009` | `test_kubernetes_system_pods` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify required system, CNI, storage, and HA workloads. | All stated checks pass for every applicable target. | -| 211 | `ORCH_FVT_PXEBOOT_V010` | `test_kubernetes_virtual_ip` | `kubernetes_cluster` | `kubernetes`, `sanity` | Verify exactly one owner for the configured Kubernetes VIP. | All stated checks pass for every applicable target. | -| 212 | `ORCH_FVT_PXEBOOT_V011` | `test_kubernetes_workload_scheduling` | `kubernetes_cluster` | `functional`, `kubernetes`, `sanity` | Create, verify, and remove an isolated scheduling probe. | The isolated probe becomes ready, satisfies the stated contract, and is removed. | -| 213 | `ORCH_FVT_PXEBOOT_V012` | `test_kubernetes_etcd_health` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify health for all etcd endpoints. | All stated checks pass for every applicable target. | -| 214 | `ORCH_FVT_PXEBOOT_V013` | `test_kubernetes_etcd_topology` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify etcd membership, leader election, and raft consistency. | All stated checks pass for every applicable target. | -| 215 | `ORCH_FVT_PXEBOOT_V014` | `test_kubernetes_local_etcd` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify each control plane has the configured etcd mount. | All stated checks pass for every applicable target. | -| 216 | `ORCH_FVT_PXEBOOT_V015` | `test_kubernetes_local_etcd_integrity` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify disk selection, ext4 label, UUID fstab, and boot persistence. | All stated checks pass for every applicable target. | -| 217 | `ORCH_FVT_PXEBOOT_V102` | `test_kubernetes_local_etcd_provisioning` | `kubernetes_etcd` | `kubernetes`, `sanity` | Verify GPT selection and local-etcd scripts, logs, and selected disk. | Every enabled control plane satisfies the provisioning contract. | -| 218 | `ORCH_FVT_PXEBOOT_V016` | `test_kubernetes_storage` | `kubernetes_storage` | `kubernetes`, `sanity` | Verify configured NFS and PowerScale storage objects. | All stated checks pass for every applicable target. | -| 219 | `ORCH_FVT_PXEBOOT_V017` | `test_kubernetes_default_storage_class` | `kubernetes_storage` | `kubernetes`, `sanity` | Verify exactly one expected default StorageClass. | All stated checks pass for every applicable target. | -| 220 | `ORCH_FVT_PXEBOOT_V018` | `test_kubernetes_snapshot_controller` | `kubernetes_storage` | `kubernetes`, `sanity` | Verify PowerScale snapshot components when configured. | All stated checks pass for every applicable target. | -| 221 | `ORCH_FVT_PXEBOOT_V103` | `test_kubernetes_nfs_provisioner_contract` | `kubernetes_storage` | `kubernetes`, `sanity` | Compare the NFS StorageClass, provisioner backend, and node mounts with active configuration. | Every NFS contract field and mapped-node mount matches the selected storage entry. | -| 222 | `ORCH_FVT_PXEBOOT_V019` | `test_kubernetes_nfs_dynamic_provisioning` | `kubernetes_storage` | `functional`, `kubernetes`, `sanity` | Create and remove an isolated NFS-backed workload. | All stated checks pass for every applicable target. | -| 223 | `ORCH_FVT_PXEBOOT_V020` | `test_kubernetes_csi_dynamic_provisioning` | `kubernetes_storage` | `functional`, `kubernetes`, `sanity` | Create and remove an isolated PowerScale-backed workload. | All stated checks pass for every applicable target. | -| 224 | `ORCH_FVT_PXEBOOT_V021` | `test_kubernetes_local_etcd_recovery` | `kubernetes_recovery` | `disruptive`, `kubernetes`, `reboot` | Reboot a control plane and prove its local-etcd UUID is preserved. | The node returns within the bounded wait and every stated post-reboot check passes. | -| 225 | `ORCH_FVT_PXEBOOT_V022` | `test_kubernetes_control_plane_recovery` | `kubernetes_recovery` | `disruptive`, `kubernetes`, `reboot` | Reboot the VIP owner and verify control-plane recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40501 | `ORCH_FVT_PXEBOOT_V113` | `test_minimal_os_base_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify base OS packages are installed on all OS-only nodes. | All stated checks pass for every applicable target. | +| 40502 | `ORCH_FVT_PXEBOOT_V114` | `test_minimal_os_ldms_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify LDMS monitoring packages and binary on OS-only nodes. | All stated checks pass for every applicable target. | +| 40503 | `ORCH_FVT_PXEBOOT_V115` | `test_minimal_os_required_services` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify required services are active on all OS-only nodes. | All stated checks pass for every applicable target. | +| 40504 | `ORCH_FVT_PXEBOOT_V117` | `test_minimal_os_excluded_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify workload-specific packages are NOT installed on OS-only nodes. | All stated checks pass for every applicable target. | +| 40505 | `ORCH_FVT_PXEBOOT_V118` | `test_minimal_os_excluded_services` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify workload-specific services are NOT active on OS-only nodes. | All stated checks pass for every applicable target. | +| 40506 | `ORCH_FVT_PXEBOOT_V119` | `test_minimal_os_package_manager` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify package manager is functional on OS-only nodes. | All stated checks pass for every applicable target. | +| 40507 | `ORCH_FVT_PXEBOOT_V120` | `test_minimal_os_kernel_version` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify kernel version is consistent across OS-only nodes per FG. | All stated checks pass for every applicable target. | +| 40508 | `ORCH_FVT_PXEBOOT_V121` | `test_minimal_os_network_identity` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify admin IP is configured on all OS-only nodes. | All stated checks pass for every applicable target. | +| 40509 | `ORCH_FVT_PXEBOOT_V122` | `test_minimal_os_functional_group_schema` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify functional-group definitions are valid on OS-only nodes. | All stated checks pass for every applicable target. | +| 40510 | `ORCH_FVT_PXEBOOT_V123` | `test_minimal_os_additional_packages` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify configured extra packages are installed on OS-only nodes. | All stated checks pass for every applicable target. | +| 40511 | `ORCH_FVT_PXEBOOT_V124` | `test_minimal_os_additional_packages_fallback` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify absent additional_packages config is handled on OS-only nodes. | All stated checks pass for every applicable target. | +| 40512 | `ORCH_FVT_PXEBOOT_V125` | `test_minimal_os_ldms_service_state` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify LDMS service is installed but inactive at handoff. | All stated checks pass for every applicable target. | +| 40513 | `ORCH_FVT_PXEBOOT_V126` | `test_minimal_os_ssh_key_access` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify SSH key authentication is proven on OS-only nodes. | All stated checks pass for every applicable target. | +| 40514 | `ORCH_FVT_PXEBOOT_V127` | `test_minimal_os_network_isolation` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify default route is on management network on OS-only nodes. | All stated checks pass for every applicable target. | +| 40515 | `ORCH_FVT_PXEBOOT_V128` | `test_minimal_os_no_embedded_credentials` | `minimal_os` | `buildstream`, `minimal_os`, `sanity` | Verify no plaintext secrets are present in the OS image. | All stated checks pass for every applicable target. | + +### PowerVault iSCSI storage (`powervault`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40601 | `ORCH_FVT_PXEBOOT_V400` | `test_powervault_iscsi_service` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iscsid is active and enabled on all PowerVault target nodes. | All stated checks pass for every applicable target. | +| 40602 | `ORCH_FVT_PXEBOOT_V401` | `test_powervault_iscsi_initiator_name` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI initiator name matches config on all target nodes. | All stated checks pass for every applicable target. | +| 40603 | `ORCH_FVT_PXEBOOT_V402` | `test_powervault_iscsi_discovery` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI target discovery succeeds from all portal IPs. | All stated checks pass for every applicable target. | +| 40604 | `ORCH_FVT_PXEBOOT_V403` | `test_powervault_iscsi_sessions` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI sessions are active on all target nodes. | All stated checks pass for every applicable target. | +| 40605 | `ORCH_FVT_PXEBOOT_V404` | `test_powervault_iscsi_startup_automatic` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI node startup is automatic on all target nodes. | All stated checks pass for every applicable target. | +| 40606 | `ORCH_FVT_PXEBOOT_V405` | `test_powervault_portal_reachability` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify iSCSI portal ports are reachable and sessions healthy. | All stated checks pass for every applicable target. | +| 40607 | `ORCH_FVT_PXEBOOT_V406` | `test_powervault_multipath_service` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify multipathd is active and enabled on all target nodes. | All stated checks pass for every applicable target. | +| 40608 | `ORCH_FVT_PXEBOOT_V407` | `test_powervault_multipath_device` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify multipath device exists and matches volume_id. | All stated checks pass for every applicable target. | +| 40609 | `ORCH_FVT_PXEBOOT_V408` | `test_powervault_multipath_redundancy` | `powervault` | `buildstream`, `powervault_infrastructure`, `sanity` | Verify multipath device has multiple paths for redundancy. | All stated checks pass for every applicable target. | +| 40610 | `ORCH_FVT_PXEBOOT_V409` | `test_powervault_gpt_partition` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify GPT partition exists on multipath device. | All stated checks pass for every applicable target. | +| 40611 | `ORCH_FVT_PXEBOOT_V410` | `test_powervault_filesystem_type` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify filesystem formatted with correct type. | All stated checks pass for every applicable target. | +| 40612 | `ORCH_FVT_PXEBOOT_V411` | `test_powervault_mount_point_directory` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify mount point directory exists on all target nodes. | All stated checks pass for every applicable target. | +| 40613 | `ORCH_FVT_PXEBOOT_V412` | `test_powervault_volume_mounted` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify PowerVault volume is actively mounted on all target nodes. | All stated checks pass for every applicable target. | +| 40614 | `ORCH_FVT_PXEBOOT_V413` | `test_powervault_mount_options` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify mount options applied correctly on all target nodes. | All stated checks pass for every applicable target. | +| 40615 | `ORCH_FVT_PXEBOOT_V414` | `test_powervault_fstab_entry` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify persistent fstab entry created on all target nodes. | All stated checks pass for every applicable target. | +| 40616 | `ORCH_FVT_PXEBOOT_V415` | `test_powervault_node_subdirectory` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify per-node subdirectory exists under mount point. | All stated checks pass for every applicable target. | +| 40617 | `ORCH_FVT_PXEBOOT_V416` | `test_powervault_bind_mounts` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify bind mount targets are active on all target nodes. | All stated checks pass for every applicable target. | +| 40618 | `ORCH_FVT_PXEBOOT_V417` | `test_powervault_bind_fstab_entries` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify bind mount fstab entries are persistent on all target nodes. | All stated checks pass for every applicable target. | +| 40619 | `ORCH_FVT_PXEBOOT_V418` | `test_powervault_bind_isolation` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify per-node data separation via bind mounts. | All stated checks pass for every applicable target. | +| 40620 | `ORCH_FVT_PXEBOOT_V419` | `test_powervault_functional_group_targeting` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify PV mount only on correct functional groups. | All stated checks pass for every applicable target. | +| 40621 | `ORCH_FVT_PXEBOOT_V420` | `test_powervault_multiple_prefix_targeting` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify multiple prefixes target all groups correctly. | All stated checks pass for every applicable target. | +| 40622 | `ORCH_FVT_PXEBOOT_V421` | `test_powervault_setup_log` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify cloud-init runcmd log exists and shows completion. | All stated checks pass for every applicable target. | +| 40623 | `ORCH_FVT_PXEBOOT_V422` | `test_powervault_cloud_init_groups_dict` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify rendered iSCSI setup scripts deployed on target nodes. | All stated checks pass for every applicable target. | +| 40624 | `ORCH_FVT_PXEBOOT_V423` | `test_powervault_no_duplicate_fstab` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify no duplicate fstab entries on all target nodes. | All stated checks pass for every applicable target. | +| 40625 | `ORCH_FVT_PXEBOOT_V424` | `test_powervault_all_mounts_writable` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify all PV mounts (main + bind) are writable. | All stated checks pass for every applicable target. | +| 40626 | `ORCH_FVT_PXEBOOT_V425` | `test_powervault_permissions` | `powervault` | `buildstream`, `powervault_cloudinit`, `sanity` | Verify permissions on mount point match config. | All stated checks pass for every applicable target. | +| 40627 | `ORCH_FVT_PXEBOOT_V426` | `test_powervault_io_write_read` | `powervault` | `buildstream`, `powervault_mounts`, `sanity` | Verify write-read I/O succeeds on every PV mount point. | All stated checks pass for every applicable target. | +| 40628 | `ORCH_FVT_PXEBOOT_V427` | `test_powervault_bind_io` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify bind-mount I/O reaches the PV backing store. | All stated checks pass for every applicable target. | +| 40629 | `ORCH_FVT_PXEBOOT_V428` | `test_powervault_slurm_mandatory_bind_mounts` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify /var/lib/mysql and /var/spool/slurm configured as bind targets. | All stated checks pass for every applicable target. | +| 40630 | `ORCH_FVT_PXEBOOT_V429` | `test_powervault_mysql_data_on_mount` | `powervault` | `buildstream`, `powervault_binds`, `sanity` | Verify MySQL/MariaDB datadir is on a PowerVault mount. | All stated checks pass for every applicable target. | + +PowerVault cases skip when `powervault_config` is absent or empty in +`storage_config.yml`. + +### CoreDNS/CoreDHCP (`coredns_coredhcp`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40701 | `ORCH_FVT_PXEBOOT_V300` | `test_coredns_container_state` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity` | TC-01: coresmd containers match the dns_enabled dataset (enabled+disabled). | All stated checks pass for every applicable target. | +| 40702 | `ORCH_FVT_PXEBOOT_V301` | `test_coredns_forward_resolution` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-02: dig FQDN from OIM for every mapped node; compare to ADMIN_IP. | All stated checks pass for every applicable target. | +| 40703 | `ORCH_FVT_PXEBOOT_V302` | `test_coredns_reverse_resolution` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-03: dig -x from OIM for every mapped ADMIN_IP; compare to FQDN. | All stated checks pass for every applicable target. | +| 40704 | `ORCH_FVT_PXEBOOT_V303` | `test_coredns_dns_forwarders` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-04: verify DNS forwarders are configured and can resolve external domains. | The operation completes successfully and returns the expected result. | +| 40705 | `ORCH_FVT_PXEBOOT_V304` | `test_coredns_corefile_config` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-05: verify Corefile configuration is correctly rendered. | The operation completes successfully and returns the expected result. | +| 40706 | `ORCH_FVT_PXEBOOT_V305` | `test_coredhcp_config_file` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity` | TC-06: verify coredhcp.yaml configuration file is correctly rendered. | The operation completes successfully and returns the expected result. | +| 40707 | `ORCH_FVT_PXEBOOT_V306` | `test_coredhcp_multisubnet_running_image` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity` | TC-07: multi-subnet dataset -> coresmd containers use expected image. | All stated checks pass for every applicable target. | +| 40708 | `ORCH_FVT_PXEBOOT_V309` | `test_coredns_idempotency` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity` | TC-08: coresmd state stability (no-drift) across a settle window. | State is identical across the settle window with no spontaneous changes. | +| 40709 | `ORCH_FVT_PXEBOOT_V307` | `test_dns_compute_resolv_conf` | `coredns_coredhcp` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-09: /etc/resolv.conf on every compute has CoreDNS as primary. | All stated checks pass for every applicable target. | +| 40710 | `ORCH_FVT_PXEBOOT_V308` | `test_dns_compute_forward_getent` | `coredns_coredhcp` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | TC-10: getent hosts on every compute resolves every mapped peer. | All stated checks pass for every applicable target. | +| 40711 | `ORCH_FVT_PXEBOOT_V310` | `test_dns_node_addition_pipeline` | `coredns_coredhcp` | `destructive`, `functional` | TC-11: prove SMD-to-CoreDNS pipeline resolves every mapped node. | The operation completes successfully and returns the expected result. | +| 40712 | `ORCH_FVT_PXEBOOT_V311` | `test_dns_smd_unreachable_cached_resolution` | `coredns_coredhcp` | `destructive`, `functional` | TC-12: pause SMD briefly; CoreDNS must keep serving cached records. | The node returns within the bounded wait and every stated postcondition check passes. | + +### Kubernetes cluster (`kubernetes_cluster`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40801 | `ORCH_FVT_PXEBOOT_V005` | `test_kubernetes_nodes` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify mapped Kubernetes membership and Ready state. | All stated checks pass for every applicable target. | +| 40802 | `ORCH_FVT_PXEBOOT_V006` | `test_kubernetes_node_services` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify required services on every Kubernetes role. | All stated checks pass for every applicable target. | +| 40803 | `ORCH_FVT_PXEBOOT_V007` | `test_kubernetes_version_compatibility` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify Kubernetes, kubeadm, and CRI-O version alignment. | All stated checks pass for every applicable target. | +| 40804 | `ORCH_FVT_PXEBOOT_V519` | `test_kubernetes_configured_versions` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify deployed component versions against the selected catalog. | Kubernetes components match the configured version and CRI-O matches its major/minor release. | +| 40805 | `ORCH_FVT_PXEBOOT_V008` | `test_kubernetes_control_plane` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify API readiness and the configured control plane. | All stated checks pass for every applicable target. | +| 40806 | `ORCH_FVT_PXEBOOT_V009` | `test_kubernetes_system_pods` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify required system, CNI, storage, and HA workloads. | All stated checks pass for every applicable target. | +| 40807 | `ORCH_FVT_PXEBOOT_V010` | `test_kubernetes_virtual_ip` | `kubernetes_cluster` | `buildstream`, `kubernetes`, `sanity` | Verify exactly one owner for the configured Kubernetes VIP. | All stated checks pass for every applicable target. | +| 40808 | `ORCH_FVT_PXEBOOT_V011` | `test_kubernetes_workload_scheduling` | `kubernetes_cluster` | `buildstream`, `functional`, `kubernetes`, `sanity` | Create, verify, and remove an isolated scheduling probe. | The isolated probe becomes ready, satisfies the stated contract, and is removed. | + +### Kubernetes etcd (`kubernetes_etcd`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 40901 | `ORCH_FVT_PXEBOOT_V012` | `test_kubernetes_etcd_health` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify health for all etcd endpoints. | All stated checks pass for every applicable target. | +| 40902 | `ORCH_FVT_PXEBOOT_V013` | `test_kubernetes_etcd_topology` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify etcd membership, leader election, and raft consistency. | All stated checks pass for every applicable target. | +| 40903 | `ORCH_FVT_PXEBOOT_V014` | `test_kubernetes_local_etcd` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify each control plane has the configured etcd mount. | All stated checks pass for every applicable target. | +| 40904 | `ORCH_FVT_PXEBOOT_V015` | `test_kubernetes_local_etcd_integrity` | `kubernetes_etcd` | `buildstream`, `kubernetes`, `sanity` | Verify disk selection, ext4 label, UUID fstab, and boot persistence. | All stated checks pass for every applicable target. | +| 40905 | `ORCH_FVT_PXEBOOT_V520` | `test_kubernetes_local_etcd_provisioning` | `kubernetes_etcd` | `kubernetes` | Verify GPT selection and local-etcd scripts, logs, and selected disk. | Every enabled control plane satisfies the provisioning contract. | +| 40906 | `ORCH_FVT_PXEBOOT_V523` | `test_kubernetes_local_etcd_media` | `kubernetes_etcd` | `kubernetes` | Verify every control plane places etcd on the configured disk media. | All stated checks pass for every applicable target. | + +Local-etcd cases skip when `etcd_on_local_disk` is disabled. Integrity and +provisioning require the boot script for the current boot to finish with +`===== DONE =====`: `etcd-disk-setup.sh` on first boot, or +`etcd-fstab-update.sh` on later boots. The media case skips unless +`expected_etcd_disk_media` is set to `ssd`, `hdd`, or `nvme`. + +### Kubernetes storage (`kubernetes_storage`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41001 | `ORCH_FVT_PXEBOOT_V016` | `test_kubernetes_storage` | `kubernetes_storage` | `buildstream`, `kubernetes`, `sanity` | Verify configured NFS and PowerScale storage objects. | All stated checks pass for every applicable target. | +| 41002 | `ORCH_FVT_PXEBOOT_V017` | `test_kubernetes_default_storage_class` | `kubernetes_storage` | `buildstream`, `kubernetes`, `sanity` | Verify exactly one expected default StorageClass. | All stated checks pass for every applicable target. | +| 41003 | `ORCH_FVT_PXEBOOT_V018` | `test_kubernetes_snapshot_controller` | `kubernetes_storage` | `buildstream`, `kubernetes`, `sanity` | Verify PowerScale snapshot components when configured. | All stated checks pass for every applicable target. | +| 41004 | `ORCH_FVT_PXEBOOT_V019` | `test_kubernetes_nfs_dynamic_provisioning` | `kubernetes_storage` | `buildstream`, `functional`, `kubernetes`, `sanity` | Create and remove an isolated NFS-backed workload. | All stated checks pass for every applicable target. | +| 41005 | `ORCH_FVT_PXEBOOT_V020` | `test_kubernetes_csi_dynamic_provisioning` | `kubernetes_storage` | `buildstream`, `functional`, `kubernetes`, `sanity` | Create and remove an isolated PowerScale-backed workload. | All stated checks pass for every applicable target. | Temporary Kubernetes resources use unique namespaces and are removed by -the creating test. CSI, snapshot, and local-etcd checks are selected from -the active configuration. - -### Slurm - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 226 | `ORCH_FVT_PXEBOOT_V023` | `test_slurm_membership` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify mapped membership, healthy state, and basic hardware fields. | All stated checks pass for every applicable target. | -| 227 | `ORCH_FVT_PXEBOOT_V024` | `test_slurm_scheduler` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify mapped compute nodes have healthy, available partitions. | All stated checks pass for every applicable target. | -| 228 | `ORCH_FVT_PXEBOOT_V025` | `test_slurm_services` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify role and feature-specific Slurm services. | All stated checks pass for every applicable target. | -| 229 | `ORCH_FVT_PXEBOOT_V026` | `test_slurm_cross_node_ssh` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify every mapped Slurm role can reach every peer over root SSH. | All stated checks pass for every applicable target. | -| 230 | `ORCH_FVT_PXEBOOT_V027` | `test_slurm_configless_mode` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify configless controller access and expected cluster identity. | All stated checks pass for every applicable target. | -| 231 | `ORCH_FVT_PXEBOOT_V028` | `test_slurm_configuration_consistency` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Compare authoritative Slurm files with every configless client cache. | Every compared value matches its authoritative source. | -| 232 | `ORCH_FVT_PXEBOOT_V029` | `test_slurm_reconfigure` | `slurm_cluster` | `functional`, `non_disruptive`, `sanity`, `slurm` | Reconfigure Slurm and verify membership remains healthy. | All stated checks pass for every applicable target. | -| 233 | `ORCH_FVT_PXEBOOT_V030` | `test_slurm_hardware_discovery` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify runtime hardware matches the configured discovery strategy. | All stated checks pass for every applicable target. | -| 234 | `ORCH_FVT_PXEBOOT_V031` | `test_slurm_custom_configuration` | `slurm_cluster` | `non_disruptive`, `sanity`, `slurm` | Verify custom values, NFS delivery, and effective visibility. | All stated checks pass for every applicable target. | -| 235 | `ORCH_FVT_PXEBOOT_V032` | `test_slurm_control_node_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every Slurm control node. | The operation completes successfully and returns the expected result. | -| 236 | `ORCH_FVT_PXEBOOT_V033` | `test_slurm_login_node_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every mapped login node. | The operation completes successfully and returns the expected result. | -| 237 | `ORCH_FVT_PXEBOOT_V034` | `test_slurm_compiler_node_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every login compiler node. | The operation completes successfully and returns the expected result. | -| 238 | `ORCH_FVT_PXEBOOT_V035` | `test_slurm_concurrent_jobs` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Submit concurrent jobs and verify final accounting state. | The submission completes with the expected final state and output. | -| 239 | `ORCH_FVT_PXEBOOT_V036` | `test_slurm_insufficient_resources` | `slurm_jobs` | `functional`, `negative`, `non_disruptive`, `sanity`, `slurm` | Verify an impossible immediate allocation is rejected. | The expected rejection occurs and no prohibited state is accepted. | -| 240 | `ORCH_FVT_PXEBOOT_V037` | `test_slurm_job_queueing` | `slurm_jobs` | `functional`, `non_disruptive`, `sanity`, `slurm` | Saturate idle computes and verify one follower queues then completes. | The follower is pending under saturation and completes after resources are released. | -| 241 | `ORCH_FVT_PXEBOOT_V038` | `test_slurm_drain_queue_recovery` | `slurm_jobs` | `disruptive`, `sanity`, `scheduler_state`, `slurm` | Drain one compute node, verify queuing, and restore it. | The job queues while the node is drained, then the node is resumed and the job completes. | -| 242 | `ORCH_FVT_PXEBOOT_V039` | `test_slurm_pam_policy` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify SSHD, the PAM module, and pam_slurm_adopt account policy. | All stated checks pass for every applicable target. | -| 243 | `ORCH_FVT_PXEBOOT_V040` | `test_slurm_control_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on the Slurm control node. | All stated checks pass for every applicable target. | -| 244 | `ORCH_FVT_PXEBOOT_V041` | `test_slurm_control_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify an invalid LDAP password is rejected on the control node. | The expected rejection occurs and no prohibited state is accepted. | -| 245 | `ORCH_FVT_PXEBOOT_V042` | `test_slurm_login_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every mapped login node. | All stated checks pass for every applicable target. | -| 246 | `ORCH_FVT_PXEBOOT_V043` | `test_slurm_login_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify an invalid LDAP password is rejected on every login node. | The expected rejection occurs and no prohibited state is accepted. | -| 247 | `ORCH_FVT_PXEBOOT_V044` | `test_slurm_compiler_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every login-compiler node. | All stated checks pass for every applicable target. | -| 248 | `ORCH_FVT_PXEBOOT_V045` | `test_slurm_compiler_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify invalid LDAP passwords are rejected on login-compiler nodes. | The expected rejection occurs and no prohibited state is accepted. | -| 249 | `ORCH_FVT_PXEBOOT_V046` | `test_slurm_pam_no_job_access` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify LDAP compute login is denied without an active job. | The expected rejection occurs and no prohibited state is accepted. | -| 250 | `ORCH_FVT_PXEBOOT_V047` | `test_slurm_invalid_ldap_identity` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a generated missing directory identity is rejected. | The expected rejection occurs and no prohibited state is accepted. | -| 251 | `ORCH_FVT_PXEBOOT_V048` | `test_slurm_control_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from the control node. | The submission completes with the expected final state and output. | -| 252 | `ORCH_FVT_PXEBOOT_V049` | `test_slurm_control_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify control-submitted PAM access during and after a job. | All stated checks pass for every applicable target. | -| 253 | `ORCH_FVT_PXEBOOT_V050` | `test_slurm_login_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from every login node. | The submission completes with the expected final state and output. | -| 254 | `ORCH_FVT_PXEBOOT_V051` | `test_slurm_login_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-node PAM access during and after a job. | All stated checks pass for every applicable target. | -| 255 | `ORCH_FVT_PXEBOOT_V052` | `test_slurm_compiler_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit an LDAP-owned job from every login-compiler node. | The submission completes with the expected final state and output. | -| 256 | `ORCH_FVT_PXEBOOT_V053` | `test_slurm_compiler_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-compiler PAM access during and after a job. | All stated checks pass for every applicable target. | -| 257 | `ORCH_FVT_PXEBOOT_V054` | `test_slurm_gpu_inventory` | `slurm_gpu` | `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA runtime state on scheduler-declared GPU nodes. | All stated checks pass for every applicable target. | -| 258 | `ORCH_FVT_PXEBOOT_V055` | `test_slurm_gpu_job` | `slurm_gpu` | `functional`, `non_disruptive`, `sanity`, `slurm` | Allocate a GPU through Slurm and query the device. | The allocation succeeds and the requested device is visible. | -| 259 | `ORCH_FVT_PXEBOOT_V056` | `test_slurm_gpu_memory_stress` | `slurm_gpu` | `functional`, `non_disruptive`, `sanity`, `slurm` | Compile and run a bounded GPU memory workload through Slurm. | Compilation and bounded workload execution complete successfully. | -| 260 | `ORCH_FVT_PXEBOOT_V057` | `test_slurm_openmpi_installation` | `slurm_openmpi` | `non_disruptive`, `sanity`, `slurm` | Verify OpenMPI discovery and version on every compute node. | All stated checks pass for every applicable target. | -| 261 | `ORCH_FVT_PXEBOOT_V058` | `test_slurm_openmpi_job` | `slurm_openmpi` | `functional`, `non_disruptive`, `sanity`, `slurm` | Run an OpenMPI-backed job when OpenMPI is configured. | The operation completes successfully and returns the expected result. | -| 262 | `ORCH_FVT_PXEBOOT_V059` | `test_slurm_ucx_transport` | `slurm_ucx` | `non_disruptive`, `sanity`, `slurm` | Verify UCX exposes an InfiniBand-capable transport. | All stated checks pass for every applicable target. | -| 263 | `ORCH_FVT_PXEBOOT_V060` | `test_slurm_infiniband_configuration` | `slurm_infiniband` | `non_disruptive`, `sanity`, `slurm` | Verify mapped IB interface, address, prefix, link, MTU, and OFED. | All stated checks pass for every applicable target. | -| 264 | `ORCH_FVT_PXEBOOT_V061` | `test_slurm_infiniband_connectivity` | `slurm_infiniband` | `non_disruptive`, `sanity`, `slurm` | Verify every mapped IB endpoint can reach every mapped peer. | All stated checks pass for every applicable target. | -| 265 | `ORCH_FVT_PXEBOOT_V062` | `test_slurm_cluster_recovery` | `slurm_recovery` | `disruptive`, `functional`, `reboot`, `slurm` | Reboot mapped Slurm nodes and verify scheduler and workload recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | - -Login and login-compiler cases skip before credential loading when the -corresponding role is absent. Feature checks use only catalog layers -selected by mapped functional groups. - -### Slurm Apptainer - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 266 | `ORCH_FVT_PXEBOOT_V063` | `test_apptainer_runtime` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the Apptainer executable and version on every compute node. | All stated checks pass for every applicable target. | -| 267 | `ORCH_FVT_PXEBOOT_V064` | `test_apptainer_shared_artifacts` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify shared image directories and downloader artifacts. | All stated checks pass for every applicable target. | -| 268 | `ORCH_FVT_PXEBOOT_V065` | `test_apptainer_pulp_policy` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the generated downloader uses only the configured Pulp source. | All stated checks pass for every applicable target. | -| 269 | `ORCH_FVT_PXEBOOT_V066` | `test_apptainer_shared_storage` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify /hpc_tools is a shared mounted filesystem on every compute. | All stated checks pass for every applicable target. | -| 270 | `ORCH_FVT_PXEBOOT_V067` | `test_apptainer_download` | `slurm_apptainer` | `apptainer`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the deployed downloader and require at least one usable SIF. | The operation completes successfully and returns the expected result. | -| 271 | `ORCH_FVT_PXEBOOT_V068` | `test_apptainer_download_idempotency` | `slurm_apptainer` | `apptainer`, `functional`, `image_download`, `non_disruptive`, `sanity` | Rerun the downloader and verify existing image metadata is unchanged. | The repeated operation succeeds without changing protected state. | -| 272 | `ORCH_FVT_PXEBOOT_V069` | `test_apptainer_download_memory` | `slurm_apptainer` | `apptainer`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the downloader and enforce a bounded peak resident-memory use. | The operation completes successfully and returns the expected result. | -| 273 | `ORCH_FVT_PXEBOOT_V070` | `test_apptainer_image_inventory` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify every compute sees one consistent non-empty SIF inventory. | All stated checks pass for every applicable target. | -| 274 | `ORCH_FVT_PXEBOOT_V071` | `test_apptainer_sif_format` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify each discovered image is a valid inspectable SIF. | All stated checks pass for every applicable target. | -| 275 | `ORCH_FVT_PXEBOOT_V072` | `test_apptainer_sif_permissions` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify shared SIF files are non-empty and world-readable. | All stated checks pass for every applicable target. | -| 276 | `ORCH_FVT_PXEBOOT_V073` | `test_apptainer_sif_integrity` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the selected SIF has the same checksum on every compute. | All stated checks pass for every applicable target. | -| 277 | `ORCH_FVT_PXEBOOT_V074` | `test_apptainer_ldap_readability` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the LDAP test identity can read a shared SIF when enabled. | All stated checks pass for every applicable target. | -| 278 | `ORCH_FVT_PXEBOOT_V075` | `test_apptainer_non_root_execution` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify an unprivileged local identity can execute a shared SIF. | All stated checks pass for every applicable target. | -| 279 | `ORCH_FVT_PXEBOOT_V076` | `test_apptainer_missing_image_contract` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify the downloader records pull failures and exits non-zero. | The expected rejection occurs and no prohibited state is accepted. | -| 280 | `ORCH_FVT_PXEBOOT_V077` | `test_apptainer_single_node_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run one exact-node container job on every mapped compute. | The operation completes successfully and returns the expected result. | -| 281 | `ORCH_FVT_PXEBOOT_V078` | `test_apptainer_multi_node_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run one container allocation spanning all mapped computes. | The operation completes successfully and returns the expected result. | -| 282 | `ORCH_FVT_PXEBOOT_V079` | `test_apptainer_ldap_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run targeted container jobs as the configured LDAP test identity. | The operation completes successfully and returns the expected result. | -| 283 | `ORCH_FVT_PXEBOOT_V080` | `test_apptainer_concurrent_jobs` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run bounded concurrent container jobs on distinct computes. | The operation completes successfully and returns the expected result. | -| 284 | `ORCH_FVT_PXEBOOT_V081` | `test_apptainer_invalid_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a nonexistent SIF fails through the Slurm execution path. | The expected rejection occurs and no prohibited state is accepted. | -| 285 | `ORCH_FVT_PXEBOOT_V082` | `test_apptainer_restricted_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify an unprivileged identity cannot execute a mode-0600 SIF. | The expected rejection occurs and no prohibited state is accepted. | -| 286 | `ORCH_FVT_PXEBOOT_V083` | `test_apptainer_nfs_visibility` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify containers can read the shared image path on every compute. | All stated checks pass for every applicable target. | -| 287 | `ORCH_FVT_PXEBOOT_V084` | `test_apptainer_slurm_environment` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify Slurm allocation variables propagate into containers. | All stated checks pass for every applicable target. | -| 288 | `ORCH_FVT_PXEBOOT_V085` | `test_apptainer_job_array` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Submit and wait for a bounded Apptainer Slurm job array. | The submission completes with the expected final state and output. | -| 289 | `ORCH_FVT_PXEBOOT_V086` | `test_apptainer_failure_cleanup` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a failed image launch leaves no matching runtime process. | The expected rejection occurs and no prohibited state is accepted. | -| 290 | `ORCH_FVT_PXEBOOT_V087` | `test_apptainer_gpu_access` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify scheduler-declared GPU nodes expose GPUs in the container. | All stated checks pass for every applicable target. | -| 291 | `ORCH_FVT_PXEBOOT_V088` | `test_apptainer_gpu_count` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify each container sees the same GPU count as its host. | All stated checks pass for every applicable target. | -| 292 | `ORCH_FVT_PXEBOOT_V089` | `test_apptainer_cuda_workload` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Execute a bounded NVIDIA device query in each GPU container. | The operation completes successfully and returns the expected result. | -| 293 | `ORCH_FVT_PXEBOOT_V090` | `test_apptainer_gpu_memory` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify the GPU query leaves no material device-memory allocation. | All stated checks pass for every applicable target. | -| 294 | `ORCH_FVT_PXEBOOT_V091` | `test_apptainer_infiniband` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify mapped compute nodes expose InfiniBand devices in containers. | All stated checks pass for every applicable target. | -| 295 | `ORCH_FVT_PXEBOOT_V092` | `test_apptainer_reboot_storage` | `slurm_apptainer` | `apptainer`, `disruptive`, `reboot` | Reboot one compute and verify the shared mount and SIF checksum. | The node returns within the bounded wait and every stated post-reboot check passes. | -| 296 | `ORCH_FVT_PXEBOOT_V093` | `test_apptainer_reboot_job` | `slurm_apptainer` | `apptainer`, `disruptive`, `reboot` | Run an exact-node container job after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | -| 297 | `ORCH_FVT_PXEBOOT_V094` | `test_apptainer_reboot_artifacts` | `slurm_apptainer` | `apptainer`, `disruptive`, `reboot` | Verify downloader artifacts and policy after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | +the creating test. CSI and snapshot checks are selected from the active +configuration. + +### Slurm cluster (`slurm_cluster`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41101 | `ORCH_FVT_PXEBOOT_V023` | `test_slurm_membership` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify mapped membership, healthy state, and basic hardware fields. | All stated checks pass for every applicable target. | +| 41102 | `ORCH_FVT_PXEBOOT_V024` | `test_slurm_scheduler` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify mapped compute nodes have healthy, available partitions. | All stated checks pass for every applicable target. | +| 41103 | `ORCH_FVT_PXEBOOT_V025` | `test_slurm_services` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify role and feature-specific Slurm services. | All stated checks pass for every applicable target. | +| 41104 | `ORCH_FVT_PXEBOOT_V026` | `test_slurm_cross_node_ssh` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify every mapped Slurm role can reach every peer over root SSH. | All stated checks pass for every applicable target. | +| 41105 | `ORCH_FVT_PXEBOOT_V027` | `test_slurm_configless_mode` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify configless controller access and expected cluster identity. | All stated checks pass for every applicable target. | +| 41106 | `ORCH_FVT_PXEBOOT_V028` | `test_slurm_configuration_consistency` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Compare authoritative Slurm files with every configless client cache. | Every compared value matches its authoritative source. | +| 41107 | `ORCH_FVT_PXEBOOT_V029` | `test_slurm_reconfigure` | `slurm_cluster` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Reconfigure Slurm and verify membership remains healthy. | All stated checks pass for every applicable target. | +| 41108 | `ORCH_FVT_PXEBOOT_V030` | `test_slurm_hardware_discovery` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify runtime hardware matches the configured discovery strategy. | All stated checks pass for every applicable target. | +| 41109 | `ORCH_FVT_PXEBOOT_V031` | `test_slurm_custom_configuration` | `slurm_cluster` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify custom values, NFS delivery, and effective visibility. | All stated checks pass for every applicable target. | + +### Slurm jobs (`slurm_jobs`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41201 | `ORCH_FVT_PXEBOOT_V032` | `test_slurm_control_node_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every Slurm control node. | The operation completes successfully and returns the expected result. | +| 41202 | `ORCH_FVT_PXEBOOT_V033` | `test_slurm_login_node_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every mapped login node. | The operation completes successfully and returns the expected result. | +| 41203 | `ORCH_FVT_PXEBOOT_V034` | `test_slurm_compiler_node_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run one targeted job per compute from every login compiler node. | The operation completes successfully and returns the expected result. | +| 41204 | `ORCH_FVT_PXEBOOT_V035` | `test_slurm_concurrent_jobs` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Submit concurrent jobs and verify final accounting state. | The submission completes with the expected final state and output. | +| 41205 | `ORCH_FVT_PXEBOOT_V036` | `test_slurm_insufficient_resources` | `slurm_jobs` | `buildstream`, `functional`, `negative`, `non_disruptive`, `slurm` | Verify an impossible immediate allocation is rejected. | The expected rejection occurs and no prohibited state is accepted. | +| 41206 | `ORCH_FVT_PXEBOOT_V037` | `test_slurm_job_queueing` | `slurm_jobs` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Saturate idle computes and verify one follower queues then completes. | The follower is pending under saturation and completes after resources are released. | +| 41207 | `ORCH_FVT_PXEBOOT_V038` | `test_slurm_drain_queue_recovery` | `slurm_jobs` | `buildstream`, `sanity`, `scheduler_state`, `slurm` | Drain one compute node, verify queuing, and restore it. | The job queues while the node is drained, then the node is resumed and the job completes. | + +### Slurm LDAP (`slurm_ldap`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41301 | `ORCH_FVT_PXEBOOT_V524` | `test_external_ldap_proxy` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Reconcile and verify the explicitly enabled LDAP meta-proxy. | Reconciliation is idempotent and every postcondition passes. | +| 41302 | `ORCH_FVT_PXEBOOT_V525` | `test_external_ldap_backend` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify external LDAP reachability from the omnia_auth container. | All stated checks pass for every applicable target. | +| 41303 | `ORCH_FVT_PXEBOOT_V039` | `test_slurm_pam_policy` | `slurm_ldap` | `buildstream`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify SSHD, the PAM module, and pam_slurm_adopt account policy. | All stated checks pass for every applicable target. | +| 41304 | `ORCH_FVT_PXEBOOT_V040` | `test_slurm_control_ldap_authentication` | `slurm_ldap` | `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on the Slurm control node. | All stated checks pass for every applicable target. | +| 41305 | `ORCH_FVT_PXEBOOT_V041` | `test_slurm_control_ldap_invalid_password` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `slurm` | Verify an invalid LDAP password is rejected on the control node. | The expected rejection occurs and no prohibited state is accepted. | +| 41306 | `ORCH_FVT_PXEBOOT_V042` | `test_slurm_login_ldap_authentication` | `slurm_ldap` | `buildstream`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every mapped login node. | All stated checks pass for every applicable target. | +| 41307 | `ORCH_FVT_PXEBOOT_V043` | `test_slurm_login_ldap_invalid_password` | `slurm_ldap` | `buildstream`, `negative`, `non_disruptive`, `openldap`, `slurm` | Verify an invalid LDAP password is rejected on every login node. | The expected rejection occurs and no prohibited state is accepted. | +| 41308 | `ORCH_FVT_PXEBOOT_V044` | `test_slurm_compiler_ldap_authentication` | `slurm_ldap` | `buildstream`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify a valid LDAP password on every login-compiler node. | All stated checks pass for every applicable target. | +| 41309 | `ORCH_FVT_PXEBOOT_V045` | `test_slurm_compiler_ldap_invalid_password` | `slurm_ldap` | `buildstream`, `negative`, `non_disruptive`, `openldap`, `slurm` | Verify invalid LDAP passwords are rejected on login-compiler nodes. | The expected rejection occurs and no prohibited state is accepted. | +| 41310 | `ORCH_FVT_PXEBOOT_V046` | `test_slurm_pam_no_job_access` | `slurm_ldap` | `negative`, `non_disruptive`, `openldap`, `slurm` | Verify LDAP compute login is denied without an active job. | The expected rejection occurs and no prohibited state is accepted. | +| 41311 | `ORCH_FVT_PXEBOOT_V047` | `test_slurm_invalid_ldap_identity` | `slurm_ldap` | `buildstream`, `negative`, `non_disruptive`, `openldap`, `slurm` | Verify a generated missing directory identity is rejected. | The expected rejection occurs and no prohibited state is accepted. | +| 41312 | `ORCH_FVT_PXEBOOT_V048` | `test_slurm_control_ldap_jobs` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from the control node. | The submission completes with the expected final state and output. | +| 41313 | `ORCH_FVT_PXEBOOT_V049` | `test_slurm_control_pam_job_access` | `slurm_ldap` | `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify control-submitted PAM access during and after a job. | All stated checks pass for every applicable target. | +| 41314 | `ORCH_FVT_PXEBOOT_V050` | `test_slurm_login_ldap_jobs` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit and complete an LDAP-owned job from every login node. | The submission completes with the expected final state and output. | +| 41315 | `ORCH_FVT_PXEBOOT_V051` | `test_slurm_login_pam_job_access` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-node PAM access during and after a job. | All stated checks pass for every applicable target. | +| 41316 | `ORCH_FVT_PXEBOOT_V052` | `test_slurm_compiler_ldap_jobs` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Submit an LDAP-owned job from every login-compiler node. | The submission completes with the expected final state and output. | +| 41317 | `ORCH_FVT_PXEBOOT_V053` | `test_slurm_compiler_pam_job_access` | `slurm_ldap` | `buildstream`, `functional`, `non_disruptive`, `openldap`, `sanity`, `slurm` | Verify login-compiler PAM access during and after a job. | All stated checks pass for every applicable target. | + +The two external LDAP cases run first because the LDAP authentication and +job cases depend on the proxied directory. Every case that logs in or runs +jobs as the LDAP test identity skips unless OpenLDAP is enabled and +`validate_external_ldap: true`. `configure_external_ldap` controls only +whether the proxy case may reconcile `slapd.conf`; with it false, the case +checks the existing deployed proxy and reports mismatches as failures. +`test_slurm_pam_policy` verifies only the PAM/SSHD configuration and runs +whenever OpenLDAP is enabled. + +### Slurm GPU (`slurm_gpu`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41401 | `ORCH_FVT_PXEBOOT_V054` | `test_slurm_gpu_inventory` | `slurm_gpu` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA runtime state on scheduler-declared GPU nodes. | All stated checks pass for every applicable target. | +| 41402 | `ORCH_FVT_PXEBOOT_V055` | `test_slurm_gpu_job` | `slurm_gpu` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Allocate a GPU through Slurm and query the device. | The allocation succeeds and the requested device is visible. | +| 41403 | `ORCH_FVT_PXEBOOT_V056` | `test_slurm_gpu_memory_stress` | `slurm_gpu` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Compile and run a bounded GPU memory workload through Slurm. | Compilation and bounded workload execution complete successfully. | + +### DCGM / CUDA (`slurm_dcgm`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41501 | `ORCH_FVT_PXEBOOT_V522` | `test_dcgm_cuda_validation` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA driver and CUDA toolkit are installed on GPU nodes. | All stated checks pass for every applicable target. | +| 41502 | `ORCH_FVT_PXEBOOT_V502` | `test_dcgm_cuda_atomic_lock` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit is installed to /hpc_tools/cuda via atomic lock. | All stated checks pass for every applicable target. | +| 41503 | `ORCH_FVT_PXEBOOT_V508` | `test_dcgm_cuda_compute_node` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify both CUDA toolkit and CUDA driver on compute nodes. | All stated checks pass for every applicable target. | +| 41504 | `ORCH_FVT_PXEBOOT_V507` | `test_dcgm_cuda_login_compiler` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit accessible on login_compiler nodes. | All stated checks pass for every applicable target. | +| 41505 | `ORCH_FVT_PXEBOOT_V503` | `test_dcgm_package_installed` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify datacenter-gpu-manager RPM and dcgmi binary on GPU nodes. | All stated checks pass for every applicable target. | +| 41506 | `ORCH_FVT_PXEBOOT_V504` | `test_dcgm_daemon_running` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify nvidia-dcgm service is active and enabled on GPU nodes. | All stated checks pass for every applicable target. | +| 41507 | `ORCH_FVT_PXEBOOT_V505` | `test_dcgm_gpu_discovery` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery enumerates GPUs with unique UUIDs. | All stated checks pass for every applicable target. | +| 41508 | `ORCH_FVT_PXEBOOT_V506` | `test_dcgm_gpu_metrics` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify dcgmi dmon returns metric samples for each GPU node. | All stated checks pass for every applicable target. | +| 41509 | `ORCH_FVT_PXEBOOT_V509` | `test_dcgm_multi_gpu_discovery` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery on multi-GPU nodes. | All stated checks pass for every applicable target. | +| 41510 | `ORCH_FVT_PXEBOOT_V510` | `test_dcgm_multi_gpu_no_login_compiler` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify GPU nodes work without login_compiler present. | All stated checks pass for every applicable target. | +| 41511 | `ORCH_FVT_PXEBOOT_V511` | `test_dcgm_multi_login_compiler_lock` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit install uses atomic lock with multiple login_compilers. | All stated checks pass for every applicable target. | +| 41512 | `ORCH_FVT_PXEBOOT_V512` | `test_dcgm_toolkit_nfs_storage` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify /hpc_tools is NFS-mounted and CUDA toolkit accessible. | All stated checks pass for every applicable target. | +| 41513 | `ORCH_FVT_PXEBOOT_V513` | `test_dcgm_rhel_compatibility` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify GPU node OS is a supported RHEL version. | All stated checks pass for every applicable target. | +| 41514 | `ORCH_FVT_PXEBOOT_V514` | `test_dcgm_cuda_version_compatibility` | `slurm_dcgm` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit and DCGM daemon version compatibility. | All stated checks pass for every applicable target. | +| 41515 | `ORCH_FVT_PXEBOOT_V515` | `test_dcgm_neg_cuda_prerequisite` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Verify DCGM deployment requires CUDA driver as a prerequisite. | All stated checks pass for every applicable target. | +| 41516 | `ORCH_FVT_PXEBOOT_V516` | `test_dcgm_neg_daemon_recovery` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Simulate DCGM daemon crash via SIGKILL and verify systemd restarts it. | The node returns within the bounded wait and every stated postcondition check passes. | +| 41517 | `ORCH_FVT_PXEBOOT_V517` | `test_dcgm_neg_socket_inaccessible` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Remove DCGM Unix socket and verify dcgmi returns a clear error. | The expected rejection occurs and no prohibited state is accepted. | +| 41518 | `ORCH_FVT_PXEBOOT_V518` | `test_dcgm_neg_package_install_failure` | `slurm_dcgm` | `buildstream`, `functional`, `slurm` | Verify error handling when datacenter-gpu-manager package is unavailable. | The expected rejection occurs and no prohibited state is accepted. | + +DCGM cases skip when no GPU GRES nodes are found in the Slurm inventory. + +### Slurm OpenMPI (`slurm_openmpi`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41601 | `ORCH_FVT_PXEBOOT_V057` | `test_slurm_openmpi_installation` | `slurm_openmpi` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify OpenMPI discovery and version on every compute node. | All stated checks pass for every applicable target. | +| 41602 | `ORCH_FVT_PXEBOOT_V058` | `test_slurm_openmpi_job` | `slurm_openmpi` | `buildstream`, `functional`, `non_disruptive`, `sanity`, `slurm` | Run an OpenMPI-backed job when OpenMPI is configured. | The operation completes successfully and returns the expected result. | + +### Slurm UCX (`slurm_ucx`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41701 | `ORCH_FVT_PXEBOOT_V059` | `test_slurm_ucx_transport` | `slurm_ucx` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify UCX exposes an InfiniBand-capable transport. | All stated checks pass for every applicable target. | + +### Slurm InfiniBand (`slurm_infiniband`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41801 | `ORCH_FVT_PXEBOOT_V060` | `test_slurm_infiniband_configuration` | `slurm_infiniband` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify mapped IB interface, address, prefix, link, MTU, and OFED. | All stated checks pass for every applicable target. | +| 41802 | `ORCH_FVT_PXEBOOT_V061` | `test_slurm_infiniband_connectivity` | `slurm_infiniband` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | Verify every mapped IB endpoint can reach every mapped peer. | All stated checks pass for every applicable target. | + +### Slurm Apptainer (`slurm_apptainer`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 41901 | `ORCH_FVT_PXEBOOT_V063` | `test_apptainer_runtime` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify the Apptainer executable and version on every compute node. | All stated checks pass for every applicable target. | +| 41902 | `ORCH_FVT_PXEBOOT_V064` | `test_apptainer_shared_artifacts` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify shared image directories and downloader artifacts. | All stated checks pass for every applicable target. | +| 41903 | `ORCH_FVT_PXEBOOT_V065` | `test_apptainer_pulp_policy` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify the generated downloader uses only the configured Pulp source. | All stated checks pass for every applicable target. | +| 41904 | `ORCH_FVT_PXEBOOT_V066` | `test_apptainer_shared_storage` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify /hpc_tools is a shared mounted filesystem on every compute. | All stated checks pass for every applicable target. | +| 41905 | `ORCH_FVT_PXEBOOT_V067` | `test_apptainer_download` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the deployed downloader and require at least one usable SIF. | The operation completes successfully and returns the expected result. | +| 41906 | `ORCH_FVT_PXEBOOT_V068` | `test_apptainer_download_idempotency` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `image_download`, `non_disruptive`, `sanity` | Rerun the downloader and verify existing image metadata is unchanged. | The repeated operation succeeds without changing protected state. | +| 41907 | `ORCH_FVT_PXEBOOT_V069` | `test_apptainer_download_memory` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `image_download`, `non_disruptive`, `sanity` | Run the downloader and enforce a bounded peak resident-memory use. | The operation completes successfully and returns the expected result. | +| 41908 | `ORCH_FVT_PXEBOOT_V070` | `test_apptainer_image_inventory` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify every compute sees one consistent non-empty SIF inventory. | All stated checks pass for every applicable target. | +| 41909 | `ORCH_FVT_PXEBOOT_V071` | `test_apptainer_sif_format` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify each discovered image is a valid inspectable SIF. | All stated checks pass for every applicable target. | +| 41910 | `ORCH_FVT_PXEBOOT_V072` | `test_apptainer_sif_permissions` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify shared SIF files are non-empty and world-readable. | All stated checks pass for every applicable target. | +| 41911 | `ORCH_FVT_PXEBOOT_V073` | `test_apptainer_sif_integrity` | `slurm_apptainer` | `apptainer`, `buildstream`, `non_disruptive`, `sanity` | Verify the selected SIF has the same checksum on every compute. | All stated checks pass for every applicable target. | +| 41912 | `ORCH_FVT_PXEBOOT_V074` | `test_apptainer_ldap_readability` | `slurm_apptainer` | `apptainer`, `non_disruptive`, `sanity` | Verify the LDAP test identity can read a shared SIF when enabled. | All stated checks pass for every applicable target. | +| 41913 | `ORCH_FVT_PXEBOOT_V075` | `test_apptainer_non_root_execution` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Verify an unprivileged local identity can execute a shared SIF. | All stated checks pass for every applicable target. | +| 41914 | `ORCH_FVT_PXEBOOT_V076` | `test_apptainer_missing_image_contract` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify the downloader records pull failures and exits non-zero. | The expected rejection occurs and no prohibited state is accepted. | +| 41915 | `ORCH_FVT_PXEBOOT_V077` | `test_apptainer_single_node_job` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Run one exact-node container job on every mapped compute. | The operation completes successfully and returns the expected result. | +| 41916 | `ORCH_FVT_PXEBOOT_V078` | `test_apptainer_multi_node_job` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Run one container allocation spanning all mapped computes. | The operation completes successfully and returns the expected result. | +| 41917 | `ORCH_FVT_PXEBOOT_V079` | `test_apptainer_ldap_job` | `slurm_apptainer` | `apptainer`, `functional`, `non_disruptive`, `sanity` | Run targeted container jobs as the configured LDAP test identity. | The operation completes successfully and returns the expected result. | +| 41918 | `ORCH_FVT_PXEBOOT_V080` | `test_apptainer_concurrent_jobs` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Run bounded concurrent container jobs on distinct computes. | The operation completes successfully and returns the expected result. | +| 41919 | `ORCH_FVT_PXEBOOT_V081` | `test_apptainer_invalid_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a nonexistent SIF fails through the Slurm execution path. | The expected rejection occurs and no prohibited state is accepted. | +| 41920 | `ORCH_FVT_PXEBOOT_V082` | `test_apptainer_restricted_sif` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify an unprivileged identity cannot execute a mode-0600 SIF. | The expected rejection occurs and no prohibited state is accepted. | +| 41921 | `ORCH_FVT_PXEBOOT_V083` | `test_apptainer_nfs_visibility` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify containers can read the shared image path on every compute. | All stated checks pass for every applicable target. | +| 41922 | `ORCH_FVT_PXEBOOT_V084` | `test_apptainer_slurm_environment` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify Slurm allocation variables propagate into containers. | All stated checks pass for every applicable target. | +| 41923 | `ORCH_FVT_PXEBOOT_V085` | `test_apptainer_job_array` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Submit and wait for a bounded Apptainer Slurm job array. | The submission completes with the expected final state and output. | +| 41924 | `ORCH_FVT_PXEBOOT_V086` | `test_apptainer_failure_cleanup` | `slurm_apptainer` | `apptainer`, `functional`, `negative`, `non_disruptive` | Verify a failed image launch leaves no matching runtime process. | The expected rejection occurs and no prohibited state is accepted. | +| 41925 | `ORCH_FVT_PXEBOOT_V087` | `test_apptainer_gpu_access` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify scheduler-declared GPU nodes expose GPUs in the container. | All stated checks pass for every applicable target. | +| 41926 | `ORCH_FVT_PXEBOOT_V088` | `test_apptainer_gpu_count` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify each container sees the same GPU count as its host. | All stated checks pass for every applicable target. | +| 41927 | `ORCH_FVT_PXEBOOT_V089` | `test_apptainer_cuda_workload` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Execute a bounded NVIDIA device query in each GPU container. | The operation completes successfully and returns the expected result. | +| 41928 | `ORCH_FVT_PXEBOOT_V090` | `test_apptainer_gpu_memory` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify the GPU query leaves no material device-memory allocation. | All stated checks pass for every applicable target. | +| 41929 | `ORCH_FVT_PXEBOOT_V091` | `test_apptainer_infiniband` | `slurm_apptainer` | `apptainer`, `buildstream`, `functional`, `non_disruptive`, `sanity` | Verify mapped compute nodes expose InfiniBand devices in containers. | All stated checks pass for every applicable target. | +| 41930 | `ORCH_FVT_PXEBOOT_V092` | `test_apptainer_reboot_storage` | `slurm_apptainer` | `apptainer`, `reboot` | Reboot one compute and verify the shared mount and SIF checksum. | The node returns within the bounded wait and every stated post-reboot check passes. | +| 41931 | `ORCH_FVT_PXEBOOT_V093` | `test_apptainer_reboot_job` | `slurm_apptainer` | `apptainer`, `reboot` | Run an exact-node container job after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | +| 41932 | `ORCH_FVT_PXEBOOT_V094` | `test_apptainer_reboot_artifacts` | `slurm_apptainer` | `apptainer`, `reboot` | Verify downloader artifacts and policy after the authorized reboot. | The node returns within the bounded wait and every stated post-reboot check passes. | Image download has a 20-minute ceiling with polling progress every 20 -seconds. The reboot cases share one reboot state instead of rebooting the -same compute node independently for each postcondition. - -### Slurm HPC Benchmarks - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 295 | `ORCH_FVT_PXEBOOT_V095` | `test_hpc_benchmarks_prerequisites` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `functional`, `benchmark`, `slurm` | Verify script presence/executability, platform helper, tool list, and /hpc_tools mount (NFS export check is informational). | Every mapped compute passes before downloads begin. | -| 296 | `ORCH_FVT_PXEBOOT_V096` | `test_hpc_benchmarks_staging_fingerprint_idempotency` | `slurm_hpc_benchmarks` | `non_disruptive`, `functional`, `benchmark`, `slurm` | Pull all applicable tools per live platform; validate tarballs and repeat. | Initial downloads succeed; the rerun performs zero downloads and preserves SHA-256, size, and mtime. | -| 297 | `ORCH_FVT_PXEBOOT_V097` | `test_hpc_benchmarks_concurrent_staging` | `slurm_hpc_benchmarks` | `non_disruptive`, `functional`, `benchmark`, `slurm` | Start two same-platform compute pulls together on shared storage. | One process downloads each tool once; both see the same archives. | -| 330 | `ORCH_FVT_PXEBOOT_V223` | `test_hpc_benchmarks_run_staging` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Execute pull_benchmarks.sh to populate staged artifacts before verification TCs. | All stated checks pass for every applicable target. | -| 331 | `ORCH_FVT_PXEBOOT_V200` | `test_hpc_benchmarks_json_declaration` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify benchmark_tools.list JSON syntax and required fields. | All stated checks pass for every applicable target. | -| 332 | `ORCH_FVT_PXEBOOT_V201` | `test_hpc_benchmarks_local_repo_sync` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify offline repo sync status on the OIM. | All stated checks pass for every applicable target. | -| 333 | `ORCH_FVT_PXEBOOT_V202` | `test_hpc_benchmarks_tools_dir_creation` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify /hpc_tools directory layout and permissions. | All stated checks pass for every applicable target. | -| 334 | `ORCH_FVT_PXEBOOT_V203` | `test_hpc_benchmarks_artifact_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify declared benchmark artifacts are staged per tool. | All stated checks pass for every applicable target. | -| 335 | `ORCH_FVT_PXEBOOT_V219` | `test_hpc_benchmarks_platform_script` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify omnia_platform.sh deployment and platform detection. | All stated checks pass for every applicable target. | -| 336 | `ORCH_FVT_PXEBOOT_V220` | `test_hpc_benchmarks_container_image_list` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify container_image.list deployment and content validation. | All stated checks pass for every applicable target. | -| 337 | `ORCH_FVT_PXEBOOT_V221` | `test_hpc_benchmarks_platform_directory_structure` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify platform-specific directory structure exists per architecture. | All stated checks pass for every applicable target. | -| 338 | `ORCH_FVT_PXEBOOT_V222` | `test_hpc_benchmarks_offline_package_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify offline packages are copied to slurm_config_path/packages/{arch}/. | All stated checks pass for every applicable target. | -| 339 | `ORCH_FVT_PXEBOOT_V204` | `test_hpc_benchmarks_msr_safe_arch_boundary` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify msr-safe is staged only for x86_64. | All stated checks pass for every applicable target. | -| 340 | `ORCH_FVT_PXEBOOT_V211` | `test_hpc_benchmarks_post_staging_validation` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify post-staging validation of benchmark tool directories. | All stated checks pass for every applicable target. | -| 341 | `ORCH_FVT_PXEBOOT_V212` | `test_hpc_benchmarks_rhel_compatibility` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify benchmark staging on RHEL 10.x. | All stated checks pass for every applicable target. | -| 342 | `ORCH_FVT_PXEBOOT_V205` | `test_hpc_benchmarks_container_first_guidance` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify pull_benchmarks.sh and benchmark_tools.list are deployed. | All stated checks pass for every applicable target. | -| 343 | `ORCH_FVT_PXEBOOT_V206` | `test_hpc_benchmarks_source_only_delivery` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify no compile or build commands are staged. | All stated checks pass for every applicable target. | -| 344 | `ORCH_FVT_PXEBOOT_V209` | `test_hpc_benchmarks_nfs_accessibility` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify /hpc_tools NFS is mounted and readable on compute nodes. | All stated checks pass for every applicable target. | -| 345 | `ORCH_FVT_PXEBOOT_V208` | `test_hpc_benchmarks_e2e_provisioning` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Verify end-to-end benchmark provisioning pipeline. | All stated checks pass for every applicable target. | -| 346 | `ORCH_FVT_PXEBOOT_V224` | `test_hpc_benchmarks_cleanup_staging` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | Remove staged benchmark tool directories after verification (controlled by cleanup_benchmark_tools in test_config.yml). | All stated checks pass for every applicable target. | -| 347 | `ORCH_FVT_PXEBOOT_V213` | `test_hpc_benchmarks_cuda_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify CUDA flow is unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 348 | `ORCH_FVT_PXEBOOT_V214` | `test_hpc_benchmarks_nvhpc_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify NVIDIA HPC SDK flow is unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 349 | `ORCH_FVT_PXEBOOT_V215` | `test_hpc_benchmarks_container_image_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify container image flow is unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 350 | `ORCH_FVT_PXEBOOT_V216` | `test_hpc_benchmarks_openmpi_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify OpenMPI/UCX are unaffected by benchmark staging. | The operation completes successfully and returns the expected result. | -| 351 | `ORCH_FVT_PXEBOOT_V210` | `test_hpc_benchmarks_airgapped_staging` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify benchmark staging completes without external egress. | The operation completes successfully and returns the expected result. | -| 352 | `ORCH_FVT_PXEBOOT_V217` | `test_hpc_benchmarks_existing_dirs_preserved` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify pre-existing /hpc_tools directories are preserved. | The operation completes successfully and returns the expected result. | -| 353 | `ORCH_FVT_PXEBOOT_V218` | `test_hpc_benchmarks_staging_idempotency` | `slurm_hpc_benchmarks` | `destructive`, `sanity`, `slurm` | Verify benchmark staging is idempotent. | The operation completes successfully and returns the expected result. | - -### CoreDNS/CoreDHCP - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 350 | `ORCH_FVT_PXEBOOT_V300` | `test_coredns_container_state` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify coresmd containers run with expected image; observe dns_enabled. | All stated checks pass for every applicable target. | -| 351 | `ORCH_FVT_PXEBOOT_V301` | `test_coredns_forward_resolution` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Query CoreDNS on the OIM for every SMD-mapped node. | All stated checks pass for every applicable target. | -| 352 | `ORCH_FVT_PXEBOOT_V302` | `test_coredns_reverse_resolution` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Query CoreDNS on the OIM for PTR records. | All stated checks pass for every applicable target. | -| 353 | `ORCH_FVT_PXEBOOT_V303` | `test_coredns_dns_forwarders` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify DNS forwarders are configured and can resolve external domains. | All stated checks pass for every applicable target. | -| 354 | `ORCH_FVT_PXEBOOT_V304` | `test_coredns_corefile_config` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify Corefile configuration is correctly rendered. | All stated checks pass for every applicable target. | -| 355 | `ORCH_FVT_PXEBOOT_V305` | `test_coredhcp_config_file` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify coredhcp.yaml configuration file is correctly rendered. | All stated checks pass for every applicable target. | -| 356 | `ORCH_FVT_PXEBOOT_V306` | `test_coredhcp_multisubnet_running_image` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify the running coresmd-coredhcp image on multi-subnet datasets. | All stated checks pass for every applicable target. | -| 357 | `ORCH_FVT_PXEBOOT_V307` | `test_dns_compute_resolv_conf` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify every mapped compute node has CoreDNS as primary nameserver. | All stated checks pass for every applicable target. | -| 358 | `ORCH_FVT_PXEBOOT_V308` | `test_dns_compute_forward_getent` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Run `getent hosts` on every compute for SMD-derived candidate FQDNs. | All stated checks pass for every applicable target. | -| 359 | `ORCH_FVT_PXEBOOT_V309` | `test_coredns_idempotency` | `coredns_coredhcp` | `non_disruptive`, `sanity` | Verify CoreDNS/CoreDHCP state stability (no-drift). | State is identical across the settle window with no spontaneous changes. | -| 360 | `ORCH_FVT_PXEBOOT_V310` | `test_dns_node_addition_pipeline` | `coredns_coredhcp` | `destructive`, `sanity` | Prove the SMD-to-CoreDNS pipeline resolves every SMD-registered mapped node. | The operation completes successfully and returns the expected result. | -| 361 | `ORCH_FVT_PXEBOOT_V311` | `test_dns_smd_unreachable_cached_resolution` | `coredns_coredhcp` | `destructive`, `sanity` | Pause the SMD container briefly; require CoreDNS to keep serving cached. | The node returns within the bounded wait and every stated postcondition check passes. | - -### PowerVault iSCSI Storage - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 301 | `ORCH_FVT_PXEBOOT_V400` | `test_powervault_iscsi_service` | `powervault` | `non_disruptive`, `sanity` | Verify iscsid is active and enabled on all PowerVault target nodes. | All stated checks pass for every applicable target. | -| 302 | `ORCH_FVT_PXEBOOT_V401` | `test_powervault_iscsi_initiator_name` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI initiator name matches config on all target nodes. | All stated checks pass for every applicable target. | -| 303 | `ORCH_FVT_PXEBOOT_V402` | `test_powervault_iscsi_discovery` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI target discovery succeeds from all portal IPs. | All stated checks pass for every applicable target. | -| 304 | `ORCH_FVT_PXEBOOT_V403` | `test_powervault_iscsi_sessions` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI sessions are active on all target nodes. | All stated checks pass for every applicable target. | -| 305 | `ORCH_FVT_PXEBOOT_V404` | `test_powervault_iscsi_startup_automatic` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI node startup is automatic on all target nodes. | All stated checks pass for every applicable target. | -| 306 | `ORCH_FVT_PXEBOOT_V405` | `test_powervault_portal_reachability` | `powervault` | `non_disruptive`, `sanity` | Verify iSCSI portal ports are reachable and sessions healthy. | All stated checks pass for every applicable target. | -| 307 | `ORCH_FVT_PXEBOOT_V406` | `test_powervault_multipath_service` | `powervault` | `non_disruptive`, `sanity` | Verify multipathd is active and enabled on all target nodes. | All stated checks pass for every applicable target. | -| 308 | `ORCH_FVT_PXEBOOT_V407` | `test_powervault_multipath_device` | `powervault` | `non_disruptive`, `sanity` | Verify multipath device exists and matches volume_id. | All stated checks pass for every applicable target. | -| 309 | `ORCH_FVT_PXEBOOT_V408` | `test_powervault_multipath_redundancy` | `powervault` | `non_disruptive`, `sanity` | Verify multipath device has multiple paths for redundancy. | All stated checks pass for every applicable target. | -| 310 | `ORCH_FVT_PXEBOOT_V409` | `test_powervault_gpt_partition` | `powervault` | `non_disruptive`, `sanity` | Verify GPT partition exists on multipath device. | All stated checks pass for every applicable target. | -| 311 | `ORCH_FVT_PXEBOOT_V410` | `test_powervault_filesystem_type` | `powervault` | `non_disruptive`, `sanity` | Verify filesystem formatted with correct type. | All stated checks pass for every applicable target. | -| 312 | `ORCH_FVT_PXEBOOT_V411` | `test_powervault_mount_point_directory` | `powervault` | `non_disruptive`, `sanity` | Verify mount point directory exists on all target nodes. | All stated checks pass for every applicable target. | -| 313 | `ORCH_FVT_PXEBOOT_V412` | `test_powervault_volume_mounted` | `powervault` | `non_disruptive`, `sanity` | Verify PowerVault volume is actively mounted on all target nodes. | All stated checks pass for every applicable target. | -| 314 | `ORCH_FVT_PXEBOOT_V413` | `test_powervault_mount_options` | `powervault` | `non_disruptive`, `sanity` | Verify mount options applied correctly on all target nodes. | All stated checks pass for every applicable target. | -| 315 | `ORCH_FVT_PXEBOOT_V414` | `test_powervault_fstab_entry` | `powervault` | `non_disruptive`, `sanity` | Verify persistent fstab entry created on all target nodes. | All stated checks pass for every applicable target. | -| 316 | `ORCH_FVT_PXEBOOT_V415` | `test_powervault_node_subdirectory` | `powervault` | `non_disruptive`, `sanity` | Verify per-node subdirectory exists under mount point. | All stated checks pass for every applicable target. | -| 317 | `ORCH_FVT_PXEBOOT_V416` | `test_powervault_bind_mounts` | `powervault` | `non_disruptive`, `sanity` | Verify bind mount targets are active on all target nodes. | All stated checks pass for every applicable target. | -| 318 | `ORCH_FVT_PXEBOOT_V417` | `test_powervault_bind_fstab_entries` | `powervault` | `non_disruptive`, `sanity` | Verify bind mount fstab entries are persistent on all target nodes. | All stated checks pass for every applicable target. | -| 319 | `ORCH_FVT_PXEBOOT_V418` | `test_powervault_bind_isolation` | `powervault` | `non_disruptive`, `sanity` | Verify per-node data separation via bind mounts. | All stated checks pass for every applicable target. | -| 320 | `ORCH_FVT_PXEBOOT_V419` | `test_powervault_functional_group_targeting` | `powervault` | `non_disruptive`, `sanity` | Verify PV mount only on correct functional groups. | All stated checks pass for every applicable target. | -| 321 | `ORCH_FVT_PXEBOOT_V420` | `test_powervault_multiple_prefix_targeting` | `powervault` | `non_disruptive`, `sanity` | Verify multiple prefixes target all groups correctly. | All stated checks pass for every applicable target. | -| 322 | `ORCH_FVT_PXEBOOT_V421` | `test_powervault_setup_log` | `powervault` | `non_disruptive`, `sanity` | Verify cloud-init runcmd log exists and shows completion. | All stated checks pass for every applicable target. | -| 323 | `ORCH_FVT_PXEBOOT_V422` | `test_powervault_cloud_init_groups_dict` | `powervault` | `non_disruptive`, `sanity` | Verify rendered iSCSI setup scripts deployed on target nodes. | All stated checks pass for every applicable target. | -| 324 | `ORCH_FVT_PXEBOOT_V423` | `test_powervault_no_duplicate_fstab` | `powervault` | `non_disruptive`, `sanity` | Verify no duplicate fstab entries on all target nodes. | All stated checks pass for every applicable target. | -| 325 | `ORCH_FVT_PXEBOOT_V424` | `test_powervault_all_mounts_writable` | `powervault` | `non_disruptive`, `sanity` | Verify all PV mounts (main + bind) are writable. | All stated checks pass for every applicable target. | -| 326 | `ORCH_FVT_PXEBOOT_V425` | `test_powervault_permissions` | `powervault` | `non_disruptive`, `sanity` | Verify permissions on mount point match config. | All stated checks pass for every applicable target. | -| 327 | `ORCH_FVT_PXEBOOT_V426` | `test_powervault_io_write_read` | `powervault` | `non_disruptive`, `sanity` | Verify write-read I/O on PV mount points. | All stated checks pass for every applicable target. | -| 328 | `ORCH_FVT_PXEBOOT_V427` | `test_powervault_bind_io` | `powervault` | `non_disruptive`, `sanity` | Verify bind-mount I/O reaches PV backing store. | All stated checks pass for every applicable target. | -| 329 | `ORCH_FVT_PXEBOOT_V428` | `test_powervault_slurm_mandatory_bind_mounts` | `powervault` | `non_disruptive`, `sanity` | Verify /var/lib/mysql and /var/spool/slurm configured as bind targets. | All stated checks pass for every applicable target. | -| 330 | `ORCH_FVT_PXEBOOT_V429` | `test_powervault_mysql_data_on_mount` | `powervault` | `non_disruptive`, `sanity` | Verify MySQL datadir is on PowerVault mount. | All stated checks pass for every applicable target. | - -PowerVault tests skip when `powervault_config` is absent or empty in `storage_config.yml`. The suite validates the iSCSI/multipath/bind-mount contract deployed by `setup_iscsi_storage.sh.j2` and functional_group_prefix targeting. -### Additional cloud-init - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 300 | `ORCH_FVT_PXEBOOT_V095` | `test_additional_cloud_init_smd_groups` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify SMD groups exist for additional cloud-init configuration. | All stated checks pass for every applicable target. | -| 301 | `ORCH_FVT_PXEBOOT_V096` | `test_additional_cloud_init_metadata_groups` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify metadata-service groups and templates for additional cloud-init. | All stated checks pass for every applicable target. | -| 302 | `ORCH_FVT_PXEBOOT_V099` | `test_additional_cloud_init_write_files` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify write_files entries were applied on provisioned nodes. | All stated checks pass for every applicable target. | -| 303 | `ORCH_FVT_PXEBOOT_V100` | `test_additional_cloud_init_runcmd` | `additional_cloud_init` | `additional_cloud_init`, `non_disruptive`, `sanity` | Verify runcmd entries executed during cloud-init on provisioned nodes. | All stated checks pass for every applicable target. | - -Additional cloud-init tests skip automatically when -`additional_cloud_init_config_file` is empty or not configured in -`orchestrator_config.yml`. When enabled, V095 and V096 verify controller-side -state (SMD groups and metadata-service templates); V099 and V100 verify -node-side artifacts (files created by `write_files` and cloud-init completion -confirming `runcmd` execution). - -### DCGM / CUDA - -| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | -|---:|---|---|---|---|---|---| -| 501 | `ORCH_FVT_PXEBOOT_V501` | `test_dcgm_cuda_validation` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify NVIDIA driver and CUDA toolkit on GPU nodes. | All stated checks pass for every applicable target. | -| 502 | `ORCH_FVT_PXEBOOT_V502` | `test_dcgm_cuda_atomic_lock` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit installed via atomic lock. | All stated checks pass for every applicable target. | -| 503 | `ORCH_FVT_PXEBOOT_V503` | `test_dcgm_package_installed` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify datacenter-gpu-manager RPM and DCGM binaries. | All stated checks pass for every applicable target. | -| 504 | `ORCH_FVT_PXEBOOT_V504` | `test_dcgm_daemon_running` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify nvidia-dcgm service is active and enabled. | All stated checks pass for every applicable target. | -| 505 | `ORCH_FVT_PXEBOOT_V505` | `test_dcgm_gpu_discovery` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery enumerates GPUs with unique UUIDs. | All stated checks pass for every applicable target. | -| 506 | `ORCH_FVT_PXEBOOT_V506` | `test_dcgm_gpu_metrics` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify dcgmi dmon returns metric samples for each GPU. | All stated checks pass for every applicable target. | -| 507 | `ORCH_FVT_PXEBOOT_V507` | `test_dcgm_cuda_login_compiler` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit accessible on login_compiler nodes. | All stated checks pass for every applicable target. | -| 508 | `ORCH_FVT_PXEBOOT_V508` | `test_dcgm_cuda_compute_node` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit and driver on compute nodes. | All stated checks pass for every applicable target. | -| 509 | `ORCH_FVT_PXEBOOT_V509` | `test_dcgm_multi_gpu_discovery` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify dcgmi discovery on multi-GPU nodes. | All stated checks pass for every applicable target. | -| 510 | `ORCH_FVT_PXEBOOT_V510` | `test_dcgm_multi_gpu_no_login_compiler` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify GPU nodes work without login_compiler present. | All stated checks pass for every applicable target. | -| 511 | `ORCH_FVT_PXEBOOT_V511` | `test_dcgm_multi_login_compiler_lock` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit install uses atomic lock with multiple login_compilers. | All stated checks pass for every applicable target. | -| 512 | `ORCH_FVT_PXEBOOT_V512` | `test_dcgm_toolkit_nfs_storage` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit is NFS-mounted and accessible. | All stated checks pass for every applicable target. | -| 513 | `ORCH_FVT_PXEBOOT_V513` | `test_dcgm_rhel_compatibility` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify GPU node OS is a supported RHEL version. | All stated checks pass for every applicable target. | -| 514 | `ORCH_FVT_PXEBOOT_V514` | `test_dcgm_cuda_version_compatibility` | `slurm_dcgm` | `non_disruptive`, `sanity`, `slurm` | Verify CUDA toolkit and DCGM daemon version compatibility. | All stated checks pass for every applicable target. | -| 515 | `ORCH_FVT_PXEBOOT_V515` | `test_dcgm_neg_cuda_prerequisite` | `slurm_dcgm` | `negative`, `non_disruptive`, `sanity`, `slurm` | Verify DCGM deployment requires CUDA prerequisites. | All stated checks pass for every applicable target. | -| 516 | `ORCH_FVT_PXEBOOT_V516` | `test_dcgm_neg_daemon_recovery` | `slurm_dcgm` | `destructive`, `sanity`, `slurm` | Verify DCGM daemon auto-recovery after SIGKILL. | The node returns within the bounded wait and every stated postcondition check passes. | -| 517 | `ORCH_FVT_PXEBOOT_V517` | `test_dcgm_neg_socket_inaccessible` | `slurm_dcgm` | `destructive`, `sanity`, `slurm` | Verify dcgmi returns clear error when socket is removed. | The expected rejection occurs and no prohibited state is accepted. | -| 518 | `ORCH_FVT_PXEBOOT_V518` | `test_dcgm_neg_package_install_failure` | `slurm_dcgm` | `negative`, `non_disruptive`, `sanity`, `slurm` | Verify error handling when DCGM package is unavailable. | The expected rejection occurs and no prohibited state is accepted. | - -DCGM tests use platform-aware CUDA toolkit paths resolved via -`omnia_platform.sh`. Compute nodes access the toolkit through the -`/usr/local/cuda` bind mount; the login_compiler accesses it directly -from the NFS-shared platform path. Tests skip when no GPU GRES nodes -are found in the Slurm inventory. +seconds. The reboot cases share one reboot state and run last in the suite. + +### Slurm HPC benchmarks (`slurm_hpc_benchmarks`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42001 | `ORCH_FVT_PXEBOOT_V225` | `test_hpc_benchmarks_prerequisites` | `slurm_hpc_benchmarks` | `benchmark`, `functional`, `non_disruptive`, `sanity`, `slurm` | V225: Verify script presence, platform helper, tool list, and /hpc_tools mount (NFS export check is informational). | Every mapped compute passes before downloads begin. | +| 42002 | `ORCH_FVT_PXEBOOT_V226` | `test_hpc_benchmarks_staging_fingerprint_idempotency` | `slurm_hpc_benchmarks` | `benchmark`, `functional`, `non_disruptive`, `slurm` | V226: Pull all applicable tools per live platform; validate tarballs and repeat. | Initial downloads succeed; the rerun performs zero downloads and preserves SHA-256, size, and mtime. | +| 42003 | `ORCH_FVT_PXEBOOT_V227` | `test_hpc_benchmarks_concurrent_staging` | `slurm_hpc_benchmarks` | `benchmark`, `functional`, `non_disruptive`, `slurm` | V227: Start two same-platform compute pulls together on shared storage. | One process downloads each tool once; both see the same archives. | +| 42004 | `ORCH_FVT_PXEBOOT_V223` | `test_hpc_benchmarks_run_staging` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-00: Execute pull_benchmarks.sh to populate staged artifacts before the verification cases. | All stated checks pass for every applicable target. | +| 42005 | `ORCH_FVT_PXEBOOT_V200` | `test_hpc_benchmarks_json_declaration` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-01: Verify benchmark_tools.list is deployed and non-empty per arch. | All stated checks pass for every applicable target. | +| 42006 | `ORCH_FVT_PXEBOOT_V201` | `test_hpc_benchmarks_local_repo_sync` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-02: Verify each declared tool has files under the Pulp offline URL. | All stated checks pass for every applicable target. | +| 42007 | `ORCH_FVT_PXEBOOT_V202` | `test_hpc_benchmarks_tools_dir_creation` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-03: Verify /hpc_tools directory layout and 0755 permissions. | All stated checks pass for every applicable target. | +| 42008 | `ORCH_FVT_PXEBOOT_V203` | `test_hpc_benchmarks_artifact_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-04: Verify declared benchmark artifacts are staged per tool. | The operation completes successfully and returns the expected result. | +| 42009 | `ORCH_FVT_PXEBOOT_V204` | `test_hpc_benchmarks_msr_safe_arch_boundary` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-05: Verify msr-safe is staged only for x86_64 nodes. | All stated checks pass for every applicable target. | +| 42010 | `ORCH_FVT_PXEBOOT_V211` | `test_hpc_benchmarks_post_staging_validation` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-12: Verify post-staging validation of benchmark tool directories. | All stated checks pass for every applicable target. | +| 42011 | `ORCH_FVT_PXEBOOT_V212` | `test_hpc_benchmarks_rhel_compatibility` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-13: Verify every compute node runs the targeted RHEL major. | All stated checks pass for every applicable target. | +| 42012 | `ORCH_FVT_PXEBOOT_V219` | `test_hpc_benchmarks_platform_script` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-06a: Verify omnia_platform.sh deployment and platform detection. | All stated checks pass for every applicable target. | +| 42013 | `ORCH_FVT_PXEBOOT_V220` | `test_hpc_benchmarks_container_image_list` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06b: Verify container_image.list deployment and content validation. | All stated checks pass for every applicable target. | +| 42014 | `ORCH_FVT_PXEBOOT_V221` | `test_hpc_benchmarks_platform_directory_structure` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06c: Verify platform-specific directory structure exists per architecture. | All stated checks pass for every applicable target. | +| 42015 | `ORCH_FVT_PXEBOOT_V222` | `test_hpc_benchmarks_offline_package_copy` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06d: Verify offline packages are copied to slurm_config_path/packages/{arch}/. | All stated checks pass for every applicable target. | +| 42016 | `ORCH_FVT_PXEBOOT_V205` | `test_hpc_benchmarks_container_first_guidance` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-06: Verify pull_benchmarks.sh and benchmark_tools.list are deployed. | All stated checks pass for every applicable target. | +| 42017 | `ORCH_FVT_PXEBOOT_V206` | `test_hpc_benchmarks_source_only_delivery` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-07: Verify no compile/build commands are staged with the artifacts. | All stated checks pass for every applicable target. | +| 42018 | `ORCH_FVT_PXEBOOT_V209` | `test_hpc_benchmarks_nfs_accessibility` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-10: Verify /hpc_tools NFS is mounted and readable on compute nodes. | All stated checks pass for every applicable target. | +| 42019 | `ORCH_FVT_PXEBOOT_V208` | `test_hpc_benchmarks_e2e_provisioning` | `slurm_hpc_benchmarks` | `non_disruptive`, `sanity`, `slurm` | TC-09: Verify the end-to-end benchmark provisioning pipeline. | All stated checks pass for every applicable target. | +| 42020 | `ORCH_FVT_PXEBOOT_V224` | `test_hpc_benchmarks_cleanup_staging` | `slurm_hpc_benchmarks` | `buildstream`, `non_disruptive`, `sanity`, `slurm` | TC-00b: Remove staged benchmark tool directories after verification (controlled by `cleanup_benchmark_tools` in `test_config.yml`). | All stated checks pass for every applicable target. | +| 42021 | `ORCH_FVT_PXEBOOT_V213` | `test_hpc_benchmarks_cuda_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-14: Verify /hpc_tools/cuda is unchanged after benchmark staging. | All stated checks pass for every applicable target. | +| 42022 | `ORCH_FVT_PXEBOOT_V214` | `test_hpc_benchmarks_nvhpc_flow_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-15: Verify /hpc_tools/nvidia_sdk is unchanged after staging. | All stated checks pass for every applicable target. | +| 42023 | `ORCH_FVT_PXEBOOT_V215` | `test_hpc_benchmarks_container_image_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-16: Verify /hpc_tools/container_images is unchanged after staging. | All stated checks pass for every applicable target. | +| 42024 | `ORCH_FVT_PXEBOOT_V216` | `test_hpc_benchmarks_openmpi_unaffected` | `slurm_hpc_benchmarks` | `destructive`, `functional`, `slurm` | TC-17: Verify OpenMPI/UCX discovery is stable across a staging run. | All stated checks pass for every applicable target. | +| 42025 | `ORCH_FVT_PXEBOOT_V210` | `test_hpc_benchmarks_airgapped_staging` | `slurm_hpc_benchmarks` | `destructive`, `slurm` | TC-11: Verify staging succeeds while external egress is unavailable. | All stated checks pass for every applicable target. | +| 42026 | `ORCH_FVT_PXEBOOT_V217` | `test_hpc_benchmarks_existing_dirs_preserved` | `slurm_hpc_benchmarks` | `destructive`, `slurm` | TC-18: Verify pre-existing /hpc_tools subdirs survive a staging run. | The operation completes successfully and returns the expected result. | +| 42027 | `ORCH_FVT_PXEBOOT_V218` | `test_hpc_benchmarks_staging_idempotency` | `slurm_hpc_benchmarks` | `destructive`, `slurm` | TC-19: Verify a second staging run keeps the /hpc_tools snapshot stable. | The repeated operation succeeds without changing protected state. | + +### VAST storage (`vast_storage`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42101 | `ORCH_FVT_PXEBOOT_V700` | `test_vast_vastnfs_installation` | `vast_storage` | `sanity`, `vast_installation` | Verify vastnfs-ctl status on compute nodes. | All stated checks pass for every applicable target. | +| 42102 | `ORCH_FVT_PXEBOOT_V701` | `test_vast_mount_points` | `vast_storage` | `sanity`, `vast_mounts` | Verify /scratch, /home, /apps, /projects directories. | All stated checks pass for every applicable target. | +| 42103 | `ORCH_FVT_PXEBOOT_V702` | `test_vast_scratch_hostname_isolation` | `vast_storage` | `sanity`, `vast_scratch` | Verify /scratch// per node; file isolation. | All stated checks pass for every applicable target. | +| 42104 | `ORCH_FVT_PXEBOOT_V703` | `test_vast_mount_options` | `vast_storage` | `sanity`, `vast_mounts` | Verify proto=rdma and port=20049 in /proc/mounts. | All stated checks pass for every applicable target. | +| 42105 | `ORCH_FVT_PXEBOOT_V704` | `test_vast_ldapuser_scratch_directory` | `vast_storage` | `sanity`, `vast_scratch` | Verify /scratch// on login_compiler nodes. | All stated checks pass for every applicable target. | +| 42106 | `ORCH_FVT_PXEBOOT_V705` | `test_vast_ldapuser_subdirectories` | `vast_storage` | `sanity`, `vast_scratch` | Verify data/, jobs/, results/, tmp/ subdirectories. | All stated checks pass for every applicable target. | +| 42107 | `ORCH_FVT_PXEBOOT_V706` | `test_vast_ldapuser_permissions` | `vast_storage` | `sanity`, `vast_scratch` | Verify cross-user permission isolation. | All stated checks pass for every applicable target. | +| 42108 | `ORCH_FVT_PXEBOOT_V707` | `test_vast_scratch_isolation` | `vast_storage` | `sanity`, `vast_scratch` | Verify file isolation between scratch subdirectories. | All stated checks pass for every applicable target. | +| 42109 | `ORCH_FVT_PXEBOOT_V708` | `test_vast_control_node_no_vast` | `vast_storage` | `sanity`, `vast_targeting` | Verify Slurm control node has no VAST storage. | All stated checks pass for every applicable target. | +| 42110 | `ORCH_FVT_PXEBOOT_V709` | `test_vast_vastnfs_rpm_and_module` | `vast_storage` | `sanity`, `vast_installation` | Verify vastnfs RPM, kernel module, and service. | All stated checks pass for every applicable target. | +| 42111 | `ORCH_FVT_PXEBOOT_V710` | `test_vast_fstab_entries` | `vast_storage` | `sanity`, `vast_mounts` | Verify /etc/fstab entries with proto=rdma. | All stated checks pass for every applicable target. | +| 42112 | `ORCH_FVT_PXEBOOT_V711` | `test_vast_rdma_mount` | `vast_storage` | `sanity`, `vast_mounts` | Verify RDMA transport and 1 GB I/O checksum. | All stated checks pass for every applicable target. | +| 42113 | `ORCH_FVT_PXEBOOT_V712` | `test_vast_qss_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify VAST on compute/login only; controller has none. | All stated checks pass for every applicable target. | +| 42114 | `ORCH_FVT_PXEBOOT_V713` | `test_vast_slurm_logs_persistence` | `vast_storage` | `sanity`, `vast_persistence` | Verify Slurm logs on persistent storage; sacct accessible. | All stated checks pass for every applicable target. | +| 42115 | `ORCH_FVT_PXEBOOT_V714` | `test_vast_control_node_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify controller mount table (NFS/PV only, no VAST). | All stated checks pass for every applicable target. | +| 42116 | `ORCH_FVT_PXEBOOT_V715` | `test_vast_compute_node_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify compute node mount table (NFS + VAST). | All stated checks pass for every applicable target. | +| 42117 | `ORCH_FVT_PXEBOOT_V716` | `test_vast_login_node_mounts` | `vast_storage` | `sanity`, `vast_targeting` | Verify login/compiler node mount table. | All stated checks pass for every applicable target. | + +VAST checks apply only when `slurm_cluster[0].vast_storage_name` selects a VAST +storage entry. LDAP scratch checks use the `ldap_username` test credential. + +### Kubernetes recovery (`kubernetes_recovery`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42201 | `ORCH_FVT_PXEBOOT_V021` | `test_kubernetes_local_etcd_recovery` | `kubernetes_recovery` | `kubernetes`, `reboot` | Reboot a control plane and prove its local-etcd UUID is preserved. | The node returns within the bounded wait and every stated post-reboot check passes. | +| 42202 | `ORCH_FVT_PXEBOOT_V022` | `test_kubernetes_control_plane_recovery` | `kubernetes_recovery` | `kubernetes`, `reboot` | Reboot the VIP owner and verify control-plane recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | + +Both cases require two control planes and observe readiness from a node +that is not rebooted. After the reboot they wait, within bounded limits, +for a new boot ID, cloud-init, Kubernetes Ready, and healthy etcd +endpoints. The local-etcd case also requires `etcd-fstab-update.sh` to +finish on the new boot, `etcd-disk-setup.sh` not to run again, and the +mount source and UUID to be unchanged. + +### Slurm recovery (`slurm_recovery`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42301 | `ORCH_FVT_PXEBOOT_V062` | `test_slurm_cluster_recovery` | `slurm_recovery` | `functional`, `reboot`, `slurm` | Reboot mapped Slurm nodes and verify scheduler and workload recovery. | The node returns within the bounded wait and every stated post-reboot check passes. | + +### Slurm node lifecycle (`slurm_lifecycle`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 42401 | `ORCH_FVT_PXEBOOT_V500` | `test_slurm_node_remove` | `slurm_lifecycle` | `functional`, `node_lifecycle`, `slurm` | Remove Slurm compute node(s) from PXE mapping, provision, verify. | The operation completes successfully and returns the expected result. | +| 42402 | `ORCH_FVT_PXEBOOT_V501` | `test_slurm_node_add` | `slurm_lifecycle` | `functional`, `node_lifecycle`, `slurm` | Restore removed node(s) to PXE mapping, provision, verify re-addition. | The operation completes successfully and returns the expected result. | + +Node removal and re-addition use `slurm_lifecycle_remove_add_nodes` from +`test_config.yml`; they run only when `--marker` selects `node_lifecycle`. +Removal skips when the PXE mapping has fewer than two `slurm_node_*` entries, +and re-addition then skips because no backup is left. ## Cleanup test cases -These cases execute the explicitly selected full cleanup and verify each -component against the configured deletion or preservation policy. +These cases execute the explicitly selected full cleanup and verify each component against the configured deletion or preservation policy. + +### Lifecycle execution + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50000 | `ORCH_FVT_CLEANUP_E001` | `test_deploy_cleanup` | `root` | `deploy`, `destructive`, `sanity` | Run ``orchestrator.yml --tags cleanup`` exactly once. | The selected Orchestrator lifecycle exits successfully. | + +### OpenCHAMI (`openchami`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50101 | `ORCH_FVT_CLEANUP_V001` | `test_openchami_removed` | `openchami` | `destructive`, `sanity` | Verify OpenCHAMI runtime, volumes, packages and state are removed. | All stated checks pass for every applicable target. | + +### OpenLDAP (`openldap`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50201 | `ORCH_FVT_CLEANUP_V002` | `test_openldap_removed` | `openldap` | `destructive`, `sanity` | Verify the OpenLDAP proxy service, container and state are removed. | All stated checks pass for every applicable target. | + +### Slurm (`slurm`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50301 | `ORCH_FVT_CLEANUP_V003` | `test_slurm_cleanup` | `slurm` | `destructive`, `sanity` | Verify Slurm's selected data policy and configured storage detachment. | All stated checks pass for every applicable target. | + +### Kubernetes (`kubernetes`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50401 | `ORCH_FVT_CLEANUP_V004` | `test_kubernetes_cleanup` | `kubernetes` | `destructive`, `sanity` | Verify Kubernetes's selected data policy and storage detachment. | All stated checks pass for every applicable target. | + +### Artifacts (`artifacts`) + +| Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | +|---:|---|---|---|---|---|---| +| 50501 | `ORCH_FVT_CLEANUP_V005` | `test_artifacts_removed_and_inputs_preserved` | `artifacts` | `destructive`, `sanity` | Verify generated state is removed without deleting required inputs. | All stated checks pass for every applicable target. | + +### Credentials (`credentials`) | Order | TC ID | Test | Suite | Markers | Validation | Pass criteria | |---:|---|---|---|---|---|---| -| 0 | `ORCH_FVT_CLEANUP_E001` | `test_deploy_cleanup` | `root` | `deploy`, `destructive`, `sanity` | Run ``orchestrator.yml --tags cleanup`` exactly once. | The selected Orchestrator lifecycle exits successfully. | -| 1 | `ORCH_FVT_CLEANUP_V001` | `test_openchami_removed` | `openchami` | `destructive`, `sanity` | Verify OpenCHAMI runtime, volumes, packages and state are removed. | All stated checks pass for every applicable target. | -| 2 | `ORCH_FVT_CLEANUP_V002` | `test_openldap_removed` | `openldap` | `destructive`, `sanity` | Verify the OpenLDAP proxy service, container and state are removed. | All stated checks pass for every applicable target. | -| 3 | `ORCH_FVT_CLEANUP_V003` | `test_slurm_cleanup` | `slurm` | `destructive`, `sanity` | Verify Slurm's selected data policy and configured storage detachment. | All stated checks pass for every applicable target. | -| 4 | `ORCH_FVT_CLEANUP_V004` | `test_kubernetes_cleanup` | `kubernetes` | `destructive`, `sanity` | Verify Kubernetes's selected data policy and storage detachment. | All stated checks pass for every applicable target. | -| 5 | `ORCH_FVT_CLEANUP_V005` | `test_artifacts_removed_and_inputs_preserved` | `artifacts` | `destructive`, `sanity` | Verify generated state is removed without deleting required inputs. | All stated checks pass for every applicable target. | -| 6 | `ORCH_FVT_CLEANUP_V006` | `test_credentials_follow_selected_policy` | `credentials` | `destructive`, `sanity` | Verify credentials are removed or preserved as selected. | All stated checks pass for every applicable target. | +| 50601 | `ORCH_FVT_CLEANUP_V006` | `test_credentials_follow_selected_policy` | `credentials` | `destructive`, `sanity` | Verify credentials are removed or preserved as selected. | All stated checks pass for every applicable target. | Cleanup verification follows the configured policy; it does not assume every input file, credential, or externally managed data path must always be deleted. @@ -469,18 +722,21 @@ input file, credential, or externally managed data path must always be deleted. | Marker | Purpose | |---|---| -| `sanity` | Default positive PXE and lifecycle coverage | +| `sanity` | Baseline positive PXE and lifecycle coverage | +| `buildstream` | BuildStream validation subset | | `functional` | Temporary workload or job behavior | -| `openldap`, `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer`, `additional_cloud_init` | Capability selectors | +| `openldap`, `connectivity`, `cloudinit`, `kubernetes`, `slurm`, `apptainer`, `benchmark`, `additional_cloud_init`, `mount_config`, `minimal_os`, `boot_image` | Capability selectors | +| `powervault_infrastructure`, `powervault_mounts`, `powervault_binds`, `powervault_cloudinit` | PowerVault subset selectors | +| `vast_installation`, `vast_mounts`, `vast_scratch`, `vast_targeting`, `vast_persistence` | VAST storage subset selectors | | `image_download` | Explicit authorization to modify shared Apptainer image storage | | `negative` | Expected rejection and error-path behavior | | `non_disruptive` | Work that does not reboot or drain cluster nodes | -| `disruptive` | Maintenance-window recovery behavior | -| `reboot` | Reboot subset of disruptive cases | -| `scheduler_state` | Scheduler drain/resume subset of disruptive cases | +| `reboot` | Node reboot cases; they run only when `--marker` selects `reboot` | +| `node_lifecycle` | Node remove/add cases; they run only when `--marker` selects `node_lifecycle` | +| `scheduler_state` | Scheduler drain/resume case | | `destructive` | Destructive cleanup selector | -A comma is OR; a plus is AND. For example, `sanity,functional` selects either +With no marker every test runs except reboot and node_lifecycle cases, which are deselected. A comma is OR; a plus is AND. For example, `sanity,functional` selects either class, while `slurm+non_disruptive` selects tests carrying both markers. ## Commands @@ -516,7 +772,7 @@ class, while `slurm+non_disruptive` selects tests carrying both markers. ./run_validation.sh fvt_orchestrator pxeboot verify \ --suite slurm_apptainer --marker functional+image_download ./run_validation.sh fvt_orchestrator pxeboot verify \ - --marker disruptive+reboot + --marker reboot,node_lifecycle # Explicit full cleanup. ./run_validation.sh fvt_orchestrator cleanup test --marker sanity diff --git a/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py b/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py index 20fc280b5a..0ceae9c489 100644 --- a/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py +++ b/test/orchestrator/fvt/cleanup/artifacts/test_artifacts.py @@ -15,16 +15,20 @@ """Artifact postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_artifacts +from library.functions import ( + TestLogger, + check_cleanup_artifacts, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(5) +@pytest.mark.order(50501) def test_artifacts_removed_and_inputs_preserved(host): """Verify generated state is removed without deleting required inputs.""" - verify_cleanup( - host, "cleanup_artifacts", "Artifact cleanup", check_cleanup_artifacts - ) + tc = TC["cleanup_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Artifact cleanup", check_cleanup_artifacts) diff --git a/test/orchestrator/fvt/cleanup/credentials/test_credentials.py b/test/orchestrator/fvt/cleanup/credentials/test_credentials.py index 87abdf845b..1298c99dad 100644 --- a/test/orchestrator/fvt/cleanup/credentials/test_credentials.py +++ b/test/orchestrator/fvt/cleanup/credentials/test_credentials.py @@ -15,19 +15,20 @@ """Credential postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_credentials +from library.functions import ( + TestLogger, + check_cleanup_credentials, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(6) +@pytest.mark.order(50601) def test_credentials_follow_selected_policy(host): """Verify credentials are removed or preserved as selected.""" - verify_cleanup( - host, - "cleanup_credentials", - "Credential cleanup", - check_cleanup_credentials, - ) + tc = TC["cleanup_credentials"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Credential cleanup", check_cleanup_credentials) diff --git a/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py b/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py index 5ccdf9150b..bb38a12240 100644 --- a/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py +++ b/test/orchestrator/fvt/cleanup/kubernetes/test_kubernetes.py @@ -15,19 +15,20 @@ """Kubernetes storage postconditions produced by full cleanup.""" import pytest -from library.functions import check_cleanup_kubernetes +from library.functions import ( + TestLogger, + check_cleanup_kubernetes, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(4) +@pytest.mark.order(50401) def test_kubernetes_cleanup(host): """Verify Kubernetes's selected data policy and storage detachment.""" - verify_cleanup( - host, - "cleanup_kubernetes", - "Kubernetes cleanup", - check_cleanup_kubernetes, - ) + tc = TC["cleanup_kubernetes"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Kubernetes cleanup", check_cleanup_kubernetes) diff --git a/test/orchestrator/fvt/cleanup/openchami/test_openchami.py b/test/orchestrator/fvt/cleanup/openchami/test_openchami.py index 8d67a31aaa..41ef0e0f26 100644 --- a/test/orchestrator/fvt/cleanup/openchami/test_openchami.py +++ b/test/orchestrator/fvt/cleanup/openchami/test_openchami.py @@ -15,16 +15,20 @@ """OpenCHAMI postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_openchami +from library.functions import ( + TestLogger, + check_cleanup_openchami, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(1) +@pytest.mark.order(50101) def test_openchami_removed(host): """Verify OpenCHAMI runtime, volumes, packages and state are removed.""" - verify_cleanup( - host, "cleanup_openchami", "OpenCHAMI cleanup", check_cleanup_openchami - ) + tc = TC["cleanup_openchami"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "OpenCHAMI cleanup", check_cleanup_openchami) diff --git a/test/orchestrator/fvt/cleanup/openldap/test_openldap.py b/test/orchestrator/fvt/cleanup/openldap/test_openldap.py index ff025b9cb2..a95e875cad 100644 --- a/test/orchestrator/fvt/cleanup/openldap/test_openldap.py +++ b/test/orchestrator/fvt/cleanup/openldap/test_openldap.py @@ -15,14 +15,20 @@ """OpenLDAP postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_openldap +from library.functions import ( + TestLogger, + check_cleanup_openldap, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(2) +@pytest.mark.order(50201) def test_openldap_removed(host): """Verify the OpenLDAP proxy service, container and state are removed.""" - verify_cleanup(host, "cleanup_openldap", "OpenLDAP cleanup", check_cleanup_openldap) + tc = TC["cleanup_openldap"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "OpenLDAP cleanup", check_cleanup_openldap) diff --git a/test/orchestrator/fvt/cleanup/slurm/test_slurm.py b/test/orchestrator/fvt/cleanup/slurm/test_slurm.py index 5e9d104e88..f2a0d03b9f 100644 --- a/test/orchestrator/fvt/cleanup/slurm/test_slurm.py +++ b/test/orchestrator/fvt/cleanup/slurm/test_slurm.py @@ -15,14 +15,20 @@ """Slurm storage postconditions produced by full Orchestrator cleanup.""" import pytest -from library.functions import check_cleanup_slurm +from library.functions import ( + TestLogger, + check_cleanup_slurm, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_cleanup @pytest.mark.sanity @pytest.mark.destructive -@pytest.mark.order(3) +@pytest.mark.order(50301) def test_slurm_cleanup(host): """Verify Slurm's selected data policy and configured storage detachment.""" - verify_cleanup(host, "cleanup_slurm", "Slurm cleanup", check_cleanup_slurm) + tc = TC["cleanup_slurm"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_cleanup(test_log, host, "Slurm cleanup", check_cleanup_slurm) diff --git a/test/orchestrator/fvt/cleanup/test_playbook.py b/test/orchestrator/fvt/cleanup/test_playbook.py index 073d1041c5..d2141e801c 100644 --- a/test/orchestrator/fvt/cleanup/test_playbook.py +++ b/test/orchestrator/fvt/cleanup/test_playbook.py @@ -29,7 +29,7 @@ @pytest.mark.destructive @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(0) +@pytest.mark.order(50000) def test_deploy_cleanup(host): """Run ``orchestrator.yml --tags cleanup`` exactly once.""" tc = TC["deploy_cleanup"] diff --git a/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py b/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py index d77cb9b976..0fe43d1ec6 100644 --- a/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py +++ b/test/orchestrator/fvt/precheck/dependencies/test_dependencies.py @@ -16,30 +16,38 @@ import pytest from library.functions import ( + TestLogger, check_precheck_dependencies, check_precheck_repositories, check_precheck_s3_artifacts, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(4) +@pytest.mark.order(10401) def test_precheck_dependencies(host): """Require both configured/default dependency outputs to be usable.""" - verify_precheck(host, "precheck_dependencies", check_precheck_dependencies) + tc = TC["precheck_dependencies"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_dependencies) @pytest.mark.sanity -@pytest.mark.order(6) +@pytest.mark.order(10402) def test_precheck_s3_artifacts(host): """Require every kernel, initrd, and rootfs artifact to be reachable.""" - verify_precheck(host, "precheck_s3_artifacts", check_precheck_s3_artifacts) + tc = TC["precheck_s3_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_s3_artifacts) @pytest.mark.sanity -@pytest.mark.order(7) +@pytest.mark.order(10403) def test_precheck_repositories(host): """Require every published RPM and file repository to be reachable.""" - verify_precheck(host, "precheck_repositories", check_precheck_repositories) + tc = TC["precheck_repositories"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_repositories) diff --git a/test/orchestrator/fvt/precheck/environment/test_environment.py b/test/orchestrator/fvt/precheck/environment/test_environment.py index df54bf8134..d38079b921 100644 --- a/test/orchestrator/fvt/precheck/environment/test_environment.py +++ b/test/orchestrator/fvt/precheck/environment/test_environment.py @@ -16,22 +16,28 @@ import pytest from library.functions import ( + TestLogger, check_precheck_admin_ipv4, check_precheck_hostname_domain, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(1) +@pytest.mark.order(10101) def test_precheck_hostname_domain(host): """Require the host identity to match omnia.env exactly.""" - verify_precheck(host, "precheck_hostname_domain", check_precheck_hostname_domain) + tc = TC["precheck_hostname_domain"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_hostname_domain) @pytest.mark.sanity -@pytest.mark.order(2) +@pytest.mark.order(10102) def test_precheck_admin_ipv4(host): """Require the configured administrative IPv4 on a global interface.""" - verify_precheck(host, "precheck_admin_ipv4", check_precheck_admin_ipv4) + tc = TC["precheck_admin_ipv4"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_admin_ipv4) diff --git a/test/orchestrator/fvt/precheck/inputs/test_inputs.py b/test/orchestrator/fvt/precheck/inputs/test_inputs.py index b48e55a298..bc9cfc117d 100644 --- a/test/orchestrator/fvt/precheck/inputs/test_inputs.py +++ b/test/orchestrator/fvt/precheck/inputs/test_inputs.py @@ -15,13 +15,19 @@ """Required and conditionally selected project-input contracts.""" import pytest -from library.functions import check_precheck_inputs +from library.functions import ( + TestLogger, + check_precheck_inputs, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(5) +@pytest.mark.order(10501) def test_precheck_inputs(host): """Require every source-selected Orchestrator input to be non-empty.""" - verify_precheck(host, "precheck_inputs", check_precheck_inputs) + tc = TC["precheck_inputs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_inputs) diff --git a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py index 0bbb68b774..9c193caae2 100644 --- a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py +++ b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_hardware.py @@ -14,70 +14,87 @@ """OIM CPU, memory, and disk-space readiness contracts.""" +from functools import partial + import pytest from library.functions import ( + TestLogger, check_oim_cpu_threshold, check_oim_disk_threshold, check_oim_memory_threshold, ) +from library.vars import OIM_NEGATIVE_INPUTS as NEG +from library.vars import TEST_CASES as TC -from fvt.result import verify_precheck - -pytestmark = [pytest.mark.sanity] - +from fvt.result import verify_precheck, verify_precheck_rejection # ── positive tests ────────────────────────────────────────────────── -@pytest.mark.order(10) +@pytest.mark.sanity +@pytest.mark.order(10201) def test_oim_cpu_threshold(host): """Require OIM CPU core count to meet the configured minimum.""" - verify_precheck(host, "oim_cpu_threshold", check_oim_cpu_threshold) + tc = TC["oim_cpu_threshold"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_cpu_threshold) -@pytest.mark.order(11) +@pytest.mark.sanity +@pytest.mark.order(10202) def test_oim_memory_threshold(host): """Require OIM memory to meet the configured minimum.""" - verify_precheck(host, "oim_memory_threshold", check_oim_memory_threshold) + tc = TC["oim_memory_threshold"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_memory_threshold) -@pytest.mark.order(12) +@pytest.mark.sanity +@pytest.mark.order(10203) def test_oim_disk_threshold(host): """Require OIM root filesystem to meet the configured minimum.""" - verify_precheck(host, "oim_disk_threshold", check_oim_disk_threshold) - - -# ── negative tests ────────────────────────────────────────────────── + tc = TC["oim_disk_threshold"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_disk_threshold) @pytest.mark.negative -@pytest.mark.order(20) +@pytest.mark.order(10210) def test_neg_cpu_below_threshold(host): """Detect failure when CPU threshold exceeds actual cores.""" - result = check_oim_cpu_threshold(host, min_cores=99999) - assert not result["success"], ( - "CPU check should fail when threshold exceeds actual cores" + tc = TC["oim_cpu_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_cpu_threshold, min_cores=NEG["min_cores"]), ) - assert result["error"], "Failure must include an actionable message" @pytest.mark.negative -@pytest.mark.order(21) +@pytest.mark.order(10211) def test_neg_memory_below_threshold(host): """Detect failure when memory threshold exceeds actual RAM.""" - result = check_oim_memory_threshold(host, min_memory_gb=99999) - assert not result["success"], ( - "Memory check should fail when threshold exceeds actual RAM" + tc = TC["oim_memory_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_memory_threshold, min_memory_gb=NEG["min_memory_gb"]), ) - assert result["error"], "Failure must include an actionable message" @pytest.mark.negative -@pytest.mark.order(22) +@pytest.mark.order(10212) def test_neg_disk_below_threshold(host): """Detect failure when disk threshold exceeds actual capacity.""" - result = check_oim_disk_threshold(host, min_disk_gb=99999) - assert not result["success"], ( - "Disk check should fail when threshold exceeds actual capacity" + tc = TC["oim_disk_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_disk_threshold, min_disk_gb=NEG["min_disk_gb"]), ) - assert result["error"], "Failure must include an actionable message" diff --git a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py index e77bde4a5a..932b5322d7 100644 --- a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py +++ b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_network.py @@ -14,95 +14,71 @@ """OIM admin/public NIC presence, IPv4, and overlap contracts.""" +from functools import partial + import pytest from library.functions import ( + TestLogger, + check_oim_forced_pxe_public_overlap, check_oim_pxe_nic_ipv4, check_oim_pxe_nic_present, - check_oim_pxe_public_overlap, check_oim_public_nic_present, ) +from library.vars import OIM_NEGATIVE_INPUTS as NEG +from library.vars import TEST_CASES as TC -from fvt.result import verify_precheck - -pytestmark = [pytest.mark.sanity] - +from fvt.result import verify_precheck, verify_precheck_rejection # ── positive tests ────────────────────────────────────────────────── -@pytest.mark.order(13) +@pytest.mark.sanity +@pytest.mark.order(10204) def test_oim_pxe_nic_present(host): """Require the configured admin NIC to exist and be UP.""" - verify_precheck(host, "oim_pxe_nic_present", check_oim_pxe_nic_present) + tc = TC["oim_pxe_nic_present"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_pxe_nic_present) -@pytest.mark.order(14) +@pytest.mark.sanity +@pytest.mark.order(10205) def test_oim_public_nic_present(host): """Require the public/default-route NIC to exist and be UP.""" - verify_precheck( - host, "oim_public_nic_present", check_oim_public_nic_present - ) + tc = TC["oim_public_nic_present"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_public_nic_present) -@pytest.mark.order(15) +@pytest.mark.sanity +@pytest.mark.order(10206) def test_oim_pxe_nic_ipv4(host): """Require the admin NIC to carry the configured IPv4 address.""" - verify_precheck(host, "oim_pxe_nic_ipv4", check_oim_pxe_nic_ipv4) - - -# ── negative tests ────────────────────────────────────────────────── + tc = TC["oim_pxe_nic_ipv4"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_pxe_nic_ipv4) @pytest.mark.negative -@pytest.mark.order(23) +@pytest.mark.order(10213) def test_neg_pxe_nic_missing(host): """Detect failure when a nonexistent NIC name is checked.""" - result = check_oim_pxe_nic_present.__wrapped__(host) if hasattr( - check_oim_pxe_nic_present, "__wrapped__" - ) else None - # Directly call the underlying function with a bogus NIC name - from library.functions.oim_readiness_precheck_func import ( - OIM_READINESS_COMMANDS, - prepare_result, - run_on_host, - ) - - nic_name = "nonexistent_nic_000" - cmd_result = run_on_host( - host, OIM_READINESS_COMMANDS["nic_operstate"], nic_name - ) - state = cmd_result.stdout.strip() if cmd_result.rc == 0 else "" - assert not (cmd_result.rc == 0 and state), ( - f"Interface {nic_name} should not exist" + tc = TC["oim_missing_nic_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_pxe_nic_present, nic_name=NEG["missing_nic"]), ) @pytest.mark.negative -@pytest.mark.order(24) +@pytest.mark.order(10214) def test_neg_pxe_public_overlap(host): """Detect overlap when PXE NIC is forced to match the public NIC.""" - # Find the actual public NIC from default route - from omnia_auto import run_on_host as _run - - route = _run(host, "ip -4 route show default 2>/dev/null | head -1") - route_line = route.stdout.strip() - if not route_line: - pytest.skip("No default route; overlap test not applicable") - - parts = route_line.split() - public_nic = "" - for i, token in enumerate(parts): - if token == "dev" and i + 1 < len(parts): - public_nic = parts[i + 1] - break - - if not public_nic: - pytest.skip("Cannot parse public NIC from default route") - - result = check_oim_pxe_public_overlap(host, nic_name=public_nic) - assert not result["success"], ( - "Overlap check should fail when PXE NIC equals public NIC" - ) - assert "overlap" in result["error"].lower() or "share" in result["error"].lower(), ( - "Error message must mention the overlap" + tc = TC["oim_nic_overlap_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, tc, host, check_oim_forced_pxe_public_overlap ) diff --git a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py index 5fe1cbeecb..280e6d2c73 100644 --- a/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py +++ b/test/orchestrator/fvt/precheck/oim_readiness/test_oim_prerequisites.py @@ -14,80 +14,91 @@ """OIM SSH, internet, and OS-version readiness contracts.""" +from functools import partial + import pytest from library.functions import ( + TestLogger, check_oim_internet_reachability, check_oim_os_version, check_oim_ssh_preflight, ) +from library.vars import OIM_NEGATIVE_INPUTS as NEG +from library.vars import TEST_CASES as TC -from fvt.result import verify_precheck - -pytestmark = [pytest.mark.sanity] - +from fvt.result import verify_precheck, verify_precheck_rejection # ── positive tests ────────────────────────────────────────────────── -@pytest.mark.order(16) +@pytest.mark.sanity +@pytest.mark.order(10207) def test_oim_ssh_preflight(host): """Require passwordless SSH from OIM to a mapped target node.""" - verify_precheck(host, "oim_ssh_preflight", check_oim_ssh_preflight) + tc = TC["oim_ssh_preflight"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_ssh_preflight) -@pytest.mark.order(18) +@pytest.mark.sanity +@pytest.mark.order(10208) def test_oim_internet_reachability(host): """Require internet reachability when not in air-gapped mode.""" - verify_precheck( - host, "oim_internet_reachability", check_oim_internet_reachability - ) + tc = TC["oim_internet_reachability"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_internet_reachability) -@pytest.mark.order(19) +@pytest.mark.sanity +@pytest.mark.order(10209) def test_oim_os_version(host): """Require the OIM OS to match the expected distribution and version.""" - verify_precheck(host, "oim_os_version", check_oim_os_version) - - -# ── negative tests ────────────────────────────────────────────────── + tc = TC["oim_os_version"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_oim_os_version) @pytest.mark.negative -@pytest.mark.order(25) +@pytest.mark.order(10215) def test_neg_internet_airgapped(host): """Verify air-gapped mode passes even without internet.""" - result = check_oim_internet_reachability(host, require_internet=False) - assert result["success"], ( - "Air-gapped mode must pass regardless of internet availability" + tc = TC["oim_airgapped_internet"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck( + test_log, + tc, + host, + partial(check_oim_internet_reachability, require_internet=False), ) @pytest.mark.negative -@pytest.mark.order(27) +@pytest.mark.order(10216) def test_neg_ssh_unreachable_target(host): """Detect SSH failure to a bogus target address.""" - from library.functions.oim_readiness_precheck_func import ( - OIM_READINESS_COMMANDS, - run_on_host, - ) - - bogus_target = "192.0.2.1" # RFC 5737 TEST-NET, guaranteed unreachable - result = run_on_host( - host, OIM_READINESS_COMMANDS["ssh_check"], bogus_target - ) - assert result.rc != 0, ( - f"SSH to {bogus_target} should fail (unreachable TEST-NET address)" + tc = TC["oim_unreachable_ssh_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial(check_oim_ssh_preflight, target=NEG["unreachable_target"]), ) @pytest.mark.negative -@pytest.mark.order(28) +@pytest.mark.order(10217) def test_neg_os_version_mismatch(host): """Detect failure when expected OS version does not match actual.""" - result = check_oim_os_version( - host, expected_id="nonexistent_os", expected_version="0.0" - ) - assert not result["success"], ( - "OS check should fail when expected ID/version do not match" + tc = TC["oim_os_mismatch_rejection"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck_rejection( + test_log, + tc, + host, + partial( + check_oim_os_version, + expected_id=NEG["os_id"], + expected_version=NEG["os_version"], + ), ) - assert result["error"], "Failure must include an actionable message" diff --git a/test/orchestrator/fvt/precheck/storage/test_storage.py b/test/orchestrator/fvt/precheck/storage/test_storage.py index fe37cfecdf..fd5f2c8bd5 100644 --- a/test/orchestrator/fvt/precheck/storage/test_storage.py +++ b/test/orchestrator/fvt/precheck/storage/test_storage.py @@ -15,13 +15,19 @@ """Configured NFS endpoint reachability contract.""" import pytest -from library.functions import check_precheck_nfs_servers +from library.functions import ( + TestLogger, + check_precheck_nfs_servers, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck @pytest.mark.sanity -@pytest.mark.order(3) +@pytest.mark.order(10301) def test_precheck_nfs_servers(host): """Require every mapping-selected NFS server to answer ICMP from the OIM.""" - verify_precheck(host, "precheck_nfs_servers", check_precheck_nfs_servers) + tc = TC["precheck_nfs_servers"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_nfs_servers) diff --git a/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py b/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py index b0ca1e814c..e51c58f41c 100644 --- a/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py +++ b/test/orchestrator/fvt/precheck/storage/test_storage_config_validation.py @@ -21,63 +21,67 @@ import pytest from library.functions import ( + TestLogger, check_precheck_mount_invalid_mount_params, check_precheck_mount_missing_mount_point, check_precheck_mount_missing_source, check_precheck_mount_missing_targeting, check_precheck_mount_node_key_without_mount_point, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_precheck -pytestmark = [pytest.mark.sanity, pytest.mark.negative, pytest.mark.mount_config] - - -@pytest.mark.order(30) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10302) def test_mount_missing_mount_point(host): """TC-CI-NEG-001: Every mount entry must have a valid absolute mount_point.""" - verify_precheck( - host, - "precheck_mount_missing_mount_point", - check_precheck_mount_missing_mount_point, - ) + tc = TC["precheck_mount_missing_mount_point"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_missing_mount_point) -@pytest.mark.order(31) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10303) def test_mount_missing_targeting(host): """TC-CI-NEG-002: Every mount entry must have targeting (prefix or groups).""" - verify_precheck( - host, - "precheck_mount_missing_targeting", - check_precheck_mount_missing_targeting, - ) + tc = TC["precheck_mount_missing_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_missing_targeting) -@pytest.mark.order(32) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10304) def test_mount_invalid_mount_params(host): """TC-CI-NEG-003: mount_params must reference an existing profile.""" - verify_precheck( - host, - "precheck_mount_invalid_mount_params", - check_precheck_mount_invalid_mount_params, - ) + tc = TC["precheck_mount_invalid_mount_params"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_invalid_mount_params) -@pytest.mark.order(33) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10305) def test_mount_missing_source(host): """TC-CI-NEG-004: Every mount entry must have a non-empty source.""" - verify_precheck( - host, - "precheck_mount_missing_source", - check_precheck_mount_missing_source, - ) + tc = TC["precheck_mount_missing_source"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_precheck(test_log, tc, host, check_precheck_mount_missing_source) -@pytest.mark.order(34) +@pytest.mark.negative +@pytest.mark.mount_config +@pytest.mark.order(10306) def test_mount_node_key_without_mount_point(host): """TC-CI-NEG-005: node_mount_point is mandatory when node_key is set.""" + tc = TC["precheck_mount_node_key_without_mount_point"] + test_log = TestLogger(tc["title"], tc["id"]) verify_precheck( + test_log, + tc, host, - "precheck_mount_node_key_without_mount_point", check_precheck_mount_node_key_without_mount_point, ) diff --git a/test/orchestrator/fvt/precheck/test_playbook.py b/test/orchestrator/fvt/precheck/test_playbook.py index 5ac7ec653d..550a127155 100644 --- a/test/orchestrator/fvt/precheck/test_playbook.py +++ b/test/orchestrator/fvt/precheck/test_playbook.py @@ -23,7 +23,7 @@ @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(0) +@pytest.mark.order(10000) def test_deploy_precheck(host): """Run ``orchestrator.yml --tags precheck`` exactly once.""" tc = TC["deploy_precheck"] diff --git a/test/orchestrator/fvt/prepare/network/test_network.py b/test/orchestrator/fvt/prepare/network/test_network.py index 745cdf7efc..a8f4afaf09 100644 --- a/test/orchestrator/fvt/prepare/network/test_network.py +++ b/test/orchestrator/fvt/prepare/network/test_network.py @@ -39,7 +39,7 @@ def _assert_result(test_log, component, result): @pytest.mark.sanity -@pytest.mark.order(6) +@pytest.mark.order(20201) def test_firewall_and_podman_network_policy(host): """Verify OpenCHAMI ports and trusted Podman interfaces.""" tc = TC["firewall_network"] @@ -53,7 +53,7 @@ def test_firewall_and_podman_network_policy(host): @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(7) +@pytest.mark.order(20202) def test_coredhcp_and_coredns_configuration(host): """Verify rendered DHCP/DNS configuration and additional routes.""" tc = TC["coredhcp_network"] diff --git a/test/orchestrator/fvt/prepare/openchami/test_openchami.py b/test/orchestrator/fvt/prepare/openchami/test_openchami.py index aff419b465..5db5a4cff8 100644 --- a/test/orchestrator/fvt/prepare/openchami/test_openchami.py +++ b/test/orchestrator/fvt/prepare/openchami/test_openchami.py @@ -45,7 +45,7 @@ def _assert_result(test_log, component, result): @pytest.mark.sanity -@pytest.mark.order(1) +@pytest.mark.order(20101) def test_openchami_containers_running(host): """Verify all long-running OpenCHAMI containers.""" tc = TC["openchami_containers"] @@ -58,7 +58,7 @@ def test_openchami_containers_running(host): @pytest.mark.sanity -@pytest.mark.order(2) +@pytest.mark.order(20102) def test_openchami_services_ready(host): """Verify OpenCHAMI units and successful SMD initialization.""" tc = TC["openchami_services"] @@ -72,7 +72,7 @@ def test_openchami_services_ready(host): @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(3) +@pytest.mark.order(20103) def test_openchami_apis_ready(host): """Verify the authenticated SMD, Boot and Metadata APIs.""" tc = TC["openchami_apis"] @@ -85,7 +85,7 @@ def test_openchami_apis_ready(host): @pytest.mark.sanity -@pytest.mark.order(4) +@pytest.mark.order(20104) def test_openchami_persistent_storage_and_tls(host): """Verify persistent data volumes and HAProxy certificates.""" tc = TC["openchami_storage"] @@ -98,7 +98,7 @@ def test_openchami_persistent_storage_and_tls(host): @pytest.mark.sanity -@pytest.mark.order(5) +@pytest.mark.order(20105) def test_openchami_packages_and_artifacts(host): """Verify installed packages and generated configuration files.""" tc = TC["openchami_artifacts"] @@ -112,7 +112,7 @@ def test_openchami_packages_and_artifacts(host): @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(13) +@pytest.mark.order(20106) def test_postgresql_readiness(host): """Verify PostgreSQL and its required SMD database contract.""" tc = TC["postgresql_readiness"] diff --git a/test/orchestrator/fvt/prepare/openldap/test_openldap.py b/test/orchestrator/fvt/prepare/openldap/test_openldap.py index 3968905e2e..73153efbc2 100644 --- a/test/orchestrator/fvt/prepare/openldap/test_openldap.py +++ b/test/orchestrator/fvt/prepare/openldap/test_openldap.py @@ -3,16 +3,17 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -"""OpenLDAP postconditions produced by Orchestrator prepare.""" +"""Local OpenLDAP postconditions produced by Orchestrator prepare. + +External LDAP proxy cases run in the pxeboot slurm_ldap suite. +""" import pytest from library.functions import TestLogger from library.functions.openldap_prepare_func import ( - check_prepare_external_ldap_backend, check_prepare_openldap_artifacts, check_prepare_openldap_endpoint, check_prepare_openldap_runtime, - reconcile_prepare_external_ldap_proxy, ) from library.messages import ( PREPARE_TEST_ASSERT_MSGS as ASSERT, @@ -49,21 +50,7 @@ def _assert_result(test_log, component, result): @pytest.mark.openldap @pytest.mark.sanity -@pytest.mark.order(8) -def test_external_ldap_proxy(host): - """Reconcile and verify the explicitly enabled LDAP meta-proxy.""" - tc = TC["external_ldap_proxy"] - test_log = TestLogger(tc["title"], tc["id"]) - _assert_result( - test_log, - "External LDAP proxy", - reconcile_prepare_external_ldap_proxy(host), - ) - - -@pytest.mark.openldap -@pytest.mark.sanity -@pytest.mark.order(9) +@pytest.mark.order(20301) def test_openldap_runtime(host): """Verify the enabled service and container are healthy.""" tc = TC["openldap_runtime"] @@ -77,7 +64,7 @@ def test_openldap_runtime(host): @pytest.mark.openldap @pytest.mark.sanity -@pytest.mark.order(10) +@pytest.mark.order(20302) def test_openldap_artifacts(host): """Verify configuration modes, syntax and TLS lifetime.""" tc = TC["openldap_artifacts"] @@ -92,7 +79,7 @@ def test_openldap_artifacts(host): @pytest.mark.openldap @pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(11) +@pytest.mark.order(20303) def test_openldap_endpoint(host): """Verify the local LDAP endpoint and published listeners.""" tc = TC["openldap_endpoint"] @@ -102,18 +89,3 @@ def test_openldap_endpoint(host): "OpenLDAP endpoint", check_prepare_openldap_endpoint(host), ) - - -@pytest.mark.openldap -@pytest.mark.functional -@pytest.mark.sanity -@pytest.mark.order(12) -def test_external_ldap_backend(host): - """Verify external LDAP reachability from the omnia_auth container.""" - tc = TC["external_ldap_backend"] - test_log = TestLogger(tc["title"], tc["id"]) - _assert_result( - test_log, - "External LDAP backend", - check_prepare_external_ldap_backend(host), - ) diff --git a/test/orchestrator/fvt/prepare/test_playbook.py b/test/orchestrator/fvt/prepare/test_playbook.py index 5270115559..a31fe85782 100644 --- a/test/orchestrator/fvt/prepare/test_playbook.py +++ b/test/orchestrator/fvt/prepare/test_playbook.py @@ -27,7 +27,7 @@ @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(0) +@pytest.mark.order(20000) def test_deploy_prepare(host): """Run ``orchestrator.yml --tags prepare``.""" tc = TC["deploy_prepare"] diff --git a/test/orchestrator/fvt/provision/openchami/test_provision_state.py b/test/orchestrator/fvt/provision/openchami/test_provision_state.py index c6737ec265..e5dc028f29 100644 --- a/test/orchestrator/fvt/provision/openchami/test_provision_state.py +++ b/test/orchestrator/fvt/provision/openchami/test_provision_state.py @@ -52,83 +52,91 @@ def _assert_result(test_log, component, result): ) -def _run_check(host, registry_key, check): - """Create the registered logger and run one provision check.""" - tc = TC[registry_key] - _assert_result(TestLogger(tc["title"], tc["id"]), tc["component"], check(host)) - - @pytest.mark.sanity -@pytest.mark.order(101) +@pytest.mark.order(30101) def test_provision_reports(host): """Verify the provision report and generated inventory contracts.""" - _run_check(host, "provision_reports", check_provision_reports) + tc = TC["provision_reports"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_provision_reports(host)) @pytest.mark.sanity -@pytest.mark.order(102) +@pytest.mark.order(30102) def test_smd_identity(host): """Verify XNAME, administrative MAC, and IP identity in SMD.""" - _run_check(host, "smd_identity", check_smd_identity) + tc = TC["smd_identity"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_smd_identity(host)) @pytest.mark.sanity -@pytest.mark.order(103) +@pytest.mark.order(30103) def test_smd_group_membership(host): """Verify expected groups and reject competing cloud-init groups.""" - _run_check(host, "smd_groups", check_smd_groups) + tc = TC["smd_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_smd_groups(host)) @pytest.mark.sanity -@pytest.mark.order(104) +@pytest.mark.order(30104) def test_boot_service_configurations(host): """Verify BootConfigurations and their mapped administrative MACs.""" - _run_check( - host, - "boot_configurations", - check_boot_configurations, - ) + tc = TC["boot_configurations"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_configurations(host)) @pytest.mark.sanity -@pytest.mark.order(105) +@pytest.mark.order(30105) def test_boot_service_node_identity(host): """Verify synchronized XNAME-to-bootMac records.""" - _run_check(host, "boot_nodes", check_boot_nodes) + tc = TC["boot_nodes"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_nodes(host)) @pytest.mark.sanity -@pytest.mark.order(106) +@pytest.mark.order(30106) def test_metadata_service_groups(host): """Verify one usable cloud-init template per functional group.""" - _run_check(host, "metadata_groups", check_metadata_groups) + tc = TC["metadata_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_metadata_groups(host)) @pytest.mark.sanity -@pytest.mark.order(107) +@pytest.mark.order(30107) def test_metadata_service_instances(host): """Verify unique per-node hostname metadata.""" - _run_check( - host, - "metadata_instances", - check_metadata_instances, - ) + tc = TC["metadata_instances"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_metadata_instances(host)) @pytest.mark.sanity -@pytest.mark.order(108) +@pytest.mark.order(30108) def test_coredhcp_and_coredns_inventory(host): """Verify the SMD identity records consumed by CoreDHCP/CoreDNS.""" - _run_check(host, "network_inventory", check_network_inventory) + tc = TC["network_inventory"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_network_inventory(host)) -@pytest.mark.order(109) +@pytest.mark.boot_image +@pytest.mark.order(30109) def test_boot_image_identity(host): """Verify Boot Service kernel/initrd paths match build_status.yml.""" - _run_check(host, "boot_image_identity", check_boot_image_identity) + tc = TC["boot_image_identity"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_image_identity(host)) -@pytest.mark.order(110) +@pytest.mark.boot_image +@pytest.mark.order(30110) def test_boot_image_architecture(host): """Verify build_status.yml architecture keys match functional group names.""" - _run_check(host, "boot_image_architecture", check_boot_image_architecture) + tc = TC["boot_image_architecture"] + test_log = TestLogger(tc["title"], tc["id"]) + _assert_result(test_log, tc["component"], check_boot_image_architecture(host)) diff --git a/test/orchestrator/fvt/provision/test_playbook.py b/test/orchestrator/fvt/provision/test_playbook.py index 158579e1a4..4727826fd6 100644 --- a/test/orchestrator/fvt/provision/test_playbook.py +++ b/test/orchestrator/fvt/provision/test_playbook.py @@ -23,7 +23,7 @@ @pytest.mark.deploy @pytest.mark.sanity -@pytest.mark.order(100) +@pytest.mark.order(30000) def test_deploy_provision(host): """Run ``orchestrator.yml --tags provision``.""" tc = TC["deploy_provision"] diff --git a/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py b/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py index de65f5e676..e33a17e653 100644 --- a/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py +++ b/test/orchestrator/fvt/pxeboot/additional_cloud_init/test_additional_cloud_init.py @@ -27,11 +27,13 @@ import pytest from library.functions import ( + TestLogger, check_additional_cloud_init_metadata_groups, check_additional_cloud_init_runcmd, check_additional_cloud_init_smd_groups, check_additional_cloud_init_write_files, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -40,53 +42,45 @@ @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(300) +@pytest.mark.order(40301) def test_additional_cloud_init_smd_groups(host): """Verify SMD groups exist for additional cloud-init configuration.""" - verify_pxeboot( - host, - "additional_cloud_init_smd_groups", - check_additional_cloud_init_smd_groups, - ) + tc = TC["additional_cloud_init_smd_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_smd_groups) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(301) +@pytest.mark.order(40302) def test_additional_cloud_init_metadata_groups(host): """Verify metadata-service groups and templates for additional cloud-init.""" - verify_pxeboot( - host, - "additional_cloud_init_metadata_groups", - check_additional_cloud_init_metadata_groups, - ) + tc = TC["additional_cloud_init_metadata_groups"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_metadata_groups) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(302) +@pytest.mark.order(40303) def test_additional_cloud_init_write_files(host): """Verify write_files entries were applied on provisioned nodes.""" - verify_pxeboot( - host, - "additional_cloud_init_write_files", - check_additional_cloud_init_write_files, - ) + tc = TC["additional_cloud_init_write_files"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_write_files) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.additional_cloud_init @pytest.mark.non_disruptive -@pytest.mark.order(303) +@pytest.mark.order(40304) def test_additional_cloud_init_runcmd(host): """Verify runcmd entries executed during cloud-init on provisioned nodes.""" - verify_pxeboot( - host, - "additional_cloud_init_runcmd", - check_additional_cloud_init_runcmd, - ) + tc = TC["additional_cloud_init_runcmd"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_additional_cloud_init_runcmd) diff --git a/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py b/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py index 9c773a38f0..6a4a8c218a 100644 --- a/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py +++ b/test/orchestrator/fvt/pxeboot/cloudinit/test_cloudinit.py @@ -24,7 +24,7 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.cloudinit -@pytest.mark.order(204) +@pytest.mark.order(40201) def test_node_cloud_init(host): """Verify PXE report freshness and direct cloud-init JSON state.""" tc = TC["node_cloud_init"] diff --git a/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py b/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py index b5580906d3..b0bb106c30 100644 --- a/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py +++ b/test/orchestrator/fvt/pxeboot/connectivity/test_connectivity.py @@ -23,75 +23,61 @@ check_node_ping, check_node_ssh, ) -from library.messages import PXEBOOT_TEST_ASSERT_MSGS as ASSERT -from library.messages import PXEBOOT_TEST_LOG_MSGS as LOG from library.vars import TEST_CASES as TC - -def _check(host, key, callback): - tc = TC[key] - test_log = TestLogger(tc["title"], tc["id"]) - result = callback(host) - fields = result["details"]["fields"] - if result.get("skipped"): - test_log.passed_fields( - LOG["check_passed"].format(component=tc["component"]), fields - ) - pytest.skip(result.get("error") or "Not applicable to this configuration") - if result["success"]: - test_log.passed_fields( - LOG["check_passed"].format(component=tc["component"]), fields - ) - else: - test_log.failed_fields( - LOG["check_failed"].format(component=tc["component"]), - [*fields, ("Error", result["error"])], - ) - assert result["success"], ASSERT["verification_failed"].format( - component=tc["component"], error=result["error"] - ) +from fvt.result import verify_pxeboot @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(201) +@pytest.mark.order(40101) def test_node_ping(host): """Verify ping from the OIM to every mapped administrative IP.""" - _check(host, "node_ping", check_node_ping) + tc = TC["node_ping"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_ping) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(202) +@pytest.mark.order(40102) def test_node_ssh(host): """Verify passwordless root SSH from the OIM to every mapped node.""" - _check(host, "node_ssh", check_node_ssh) + tc = TC["node_ssh"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_ssh) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(203) +@pytest.mark.order(40103) def test_node_hostname_ssh(host): """Verify mapped hostnames resolve and support passwordless root SSH.""" - _check(host, "node_hostname_ssh", check_node_hostname_ssh) + tc = TC["node_hostname_ssh"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_hostname_ssh) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(298) +@pytest.mark.order(40104) def test_node_architecture(host): """Verify each node's live architecture matches its functional group.""" - _check(host, "node_architecture", check_node_architecture) + tc = TC["node_architecture"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_architecture) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.connectivity -@pytest.mark.order(299) +@pytest.mark.order(40105) def test_node_os_version(host): """Verify each node's live OS version matches its functional group.""" - _check(host, "node_os_version", check_node_os_version) + tc = TC["node_os_version"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_node_os_version) diff --git a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py index 662b863d07..b19d27e75a 100644 --- a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py +++ b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_destructive.py @@ -21,30 +21,29 @@ import pytest from library.functions import ( + TestLogger, check_dns_node_addition_pipeline, check_dns_smd_unreachable_cached_resolution, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.destructive] - - @pytest.mark.functional -@pytest.mark.order(360) +@pytest.mark.destructive +@pytest.mark.order(40711) def test_dns_node_addition_pipeline(host): """TC-11: prove SMD-to-CoreDNS pipeline resolves every mapped node.""" - verify_pxeboot( - host, "dns_node_addition_pipeline", check_dns_node_addition_pipeline - ) + tc = TC["dns_node_addition_pipeline"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_node_addition_pipeline) @pytest.mark.functional -@pytest.mark.order(361) +@pytest.mark.destructive +@pytest.mark.order(40712) def test_dns_smd_unreachable_cached_resolution(host): """TC-12: pause SMD briefly; CoreDNS must keep serving cached records.""" - verify_pxeboot( - host, - "dns_smd_unreachable_cached_resolution", - check_dns_smd_unreachable_cached_resolution, - ) + tc = TC["dns_smd_unreachable_cached_resolution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_smd_unreachable_cached_resolution) diff --git a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py index f52d5ad703..24fd0d0d8e 100644 --- a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py +++ b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_nodes.py @@ -22,29 +22,34 @@ import pytest from library.functions import ( + TestLogger, check_dns_compute_forward_getent, check_dns_compute_resolv_conf, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(358) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(40709) def test_dns_compute_resolv_conf(host): """TC-09: /etc/resolv.conf on every compute has CoreDNS as primary.""" - verify_pxeboot(host, "dns_compute_resolv_conf", check_dns_compute_resolv_conf) + tc = TC["dns_compute_resolv_conf"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_compute_resolv_conf) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(359) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(40710) def test_dns_compute_forward_getent(host): """TC-10: getent hosts on every compute resolves every mapped peer.""" - verify_pxeboot( - host, "dns_compute_forward_getent", check_dns_compute_forward_getent - ) + tc = TC["dns_compute_forward_getent"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dns_compute_forward_getent) diff --git a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py index 53dc430a88..6298bed164 100644 --- a/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py +++ b/test/orchestrator/fvt/pxeboot/coredns_coredhcp/test_runtime.py @@ -22,6 +22,7 @@ import pytest from library.functions import ( + TestLogger, check_coredhcp_config_file, check_coredhcp_multisubnet_running_image, check_coredns_container_state, @@ -31,91 +32,98 @@ check_coredns_idempotency, check_coredns_reverse_resolution, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.non_disruptive] - - @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(350) +@pytest.mark.non_disruptive +@pytest.mark.order(40701) def test_coredns_container_state(host): """TC-01: coresmd containers match the dns_enabled dataset (enabled+disabled).""" - verify_pxeboot(host, "coredns_container_state", check_coredns_container_state) + tc = TC["coredns_container_state"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_container_state) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(351) +@pytest.mark.non_disruptive +@pytest.mark.order(40702) def test_coredns_forward_resolution(host): """TC-02: dig FQDN from OIM for every mapped node; compare to ADMIN_IP.""" - verify_pxeboot( - host, "coredns_forward_resolution", check_coredns_forward_resolution - ) + tc = TC["coredns_forward_resolution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_forward_resolution) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(352) +@pytest.mark.non_disruptive +@pytest.mark.order(40703) def test_coredns_reverse_resolution(host): """TC-03: dig -x from OIM for every mapped ADMIN_IP; compare to FQDN.""" - verify_pxeboot( - host, "coredns_reverse_resolution", check_coredns_reverse_resolution - ) - + tc = TC["coredns_reverse_resolution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_reverse_resolution) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(353) +@pytest.mark.non_disruptive +@pytest.mark.order(40704) def test_coredns_dns_forwarders(host): """TC-04: verify DNS forwarders are configured and can resolve external domains.""" - verify_pxeboot( - host, "coredns_dns_forwarders", check_coredns_dns_forwarders - ) + tc = TC["coredns_dns_forwarders"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_dns_forwarders) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(354) +@pytest.mark.non_disruptive +@pytest.mark.order(40705) def test_coredns_corefile_config(host): """TC-05: verify Corefile configuration is correctly rendered.""" - verify_pxeboot( - host, "coredns_corefile_config", check_coredns_corefile_config - ) + tc = TC["coredns_corefile_config"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_corefile_config) -@pytest.mark.functional @pytest.mark.buildstream +@pytest.mark.functional @pytest.mark.sanity -@pytest.mark.order(355) +@pytest.mark.non_disruptive +@pytest.mark.order(40706) def test_coredhcp_config_file(host): """TC-06: verify coredhcp.yaml configuration file is correctly rendered.""" - verify_pxeboot( - host, "coredhcp_config_file", check_coredhcp_config_file - ) + tc = TC["coredhcp_config_file"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredhcp_config_file) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(356) +@pytest.mark.non_disruptive +@pytest.mark.order(40707) def test_coredhcp_multisubnet_running_image(host): """TC-07: multi-subnet dataset -> coresmd containers use expected image.""" - verify_pxeboot( - host, - "coredhcp_multisubnet_running_image", - check_coredhcp_multisubnet_running_image, - ) + tc = TC["coredhcp_multisubnet_running_image"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredhcp_multisubnet_running_image) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(357) +@pytest.mark.non_disruptive +@pytest.mark.order(40708) def test_coredns_idempotency(host): """TC-08: coresmd state stability (no-drift) across a settle window.""" - verify_pxeboot(host, "coredns_idempotency", check_coredns_idempotency) + tc = TC["coredns_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_coredns_idempotency) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py index bed06f31f9..ff15c1018c 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_cluster_health.py @@ -16,12 +16,14 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_control_plane, check_kubernetes_node_services, check_kubernetes_nodes, check_kubernetes_system_pods, check_kubernetes_virtual_ip, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -29,43 +31,53 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(205) +@pytest.mark.order(40801) def test_kubernetes_nodes(host): """Verify mapped Kubernetes membership and Ready state.""" - verify_pxeboot(host, "kubernetes_nodes", check_kubernetes_nodes) + tc = TC["kubernetes_nodes"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_nodes) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(206) +@pytest.mark.order(40802) def test_kubernetes_node_services(host): """Verify required services on every Kubernetes role.""" - verify_pxeboot(host, "kubernetes_services", check_kubernetes_node_services) + tc = TC["kubernetes_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_node_services) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(209) +@pytest.mark.order(40805) def test_kubernetes_control_plane(host): """Verify API readiness and the configured control plane.""" - verify_pxeboot(host, "kubernetes_control_plane", check_kubernetes_control_plane) + tc = TC["kubernetes_control_plane"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_control_plane) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(210) +@pytest.mark.order(40806) def test_kubernetes_system_pods(host): """Verify required system, CNI, storage, and HA workloads.""" - verify_pxeboot(host, "kubernetes_system_pods", check_kubernetes_system_pods) + tc = TC["kubernetes_system_pods"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_system_pods) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(211) +@pytest.mark.order(40807) def test_kubernetes_virtual_ip(host): """Verify exactly one owner for the configured Kubernetes VIP.""" - verify_pxeboot(host, "kubernetes_virtual_ip", check_kubernetes_virtual_ip) + tc = TC["kubernetes_virtual_ip"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_virtual_ip) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py index 6aa8e94107..80661e8a62 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_versions.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_configured_versions, check_kubernetes_version_compatibility, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -26,20 +28,20 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(207) +@pytest.mark.order(40803) def test_kubernetes_version_compatibility(host): """Verify Kubernetes, kubeadm, and CRI-O version alignment.""" - verify_pxeboot(host, "kubernetes_versions", check_kubernetes_version_compatibility) + tc = TC["kubernetes_versions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_version_compatibility) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(208) +@pytest.mark.order(40804) def test_kubernetes_configured_versions(host): """Verify deployed component versions against the selected catalog.""" - verify_pxeboot( - host, - "kubernetes_configured_versions", - check_kubernetes_configured_versions, - ) + tc = TC["kubernetes_configured_versions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_configured_versions) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py index 2b2762dee7..3fbd7653b4 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_cluster/test_workloads.py @@ -15,7 +15,11 @@ """Kubernetes workload scheduling contract.""" import pytest -from library.functions import check_kubernetes_workload_scheduling +from library.functions import ( + TestLogger, + check_kubernetes_workload_scheduling, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -24,7 +28,9 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(212) +@pytest.mark.order(40808) def test_kubernetes_workload_scheduling(host): """Create, verify, and remove an isolated scheduling probe.""" - verify_pxeboot(host, "kubernetes_workload", check_kubernetes_workload_scheduling) + tc = TC["kubernetes_workload"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_workload_scheduling) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py index 80e5aeddf8..40d7b9ca3e 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_etcd_health.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_etcd_health, check_kubernetes_etcd_topology, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -26,16 +28,20 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(213) +@pytest.mark.order(40901) def test_kubernetes_etcd_health(host): """Verify health for all etcd endpoints.""" - verify_pxeboot(host, "kubernetes_etcd_health", check_kubernetes_etcd_health) + tc = TC["kubernetes_etcd_health"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_etcd_health) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(214) +@pytest.mark.order(40902) def test_kubernetes_etcd_topology(host): """Verify etcd membership, leader election, and raft consistency.""" - verify_pxeboot(host, "kubernetes_etcd_topology", check_kubernetes_etcd_topology) + tc = TC["kubernetes_etcd_topology"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_etcd_topology) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py index cfc8153779..4b9d7f6f57 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_etcd/test_local_etcd.py @@ -12,13 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Kubernetes local-etcd mount and disk safety contracts.""" +"""Kubernetes local-etcd mount, provisioning, and disk-media contracts.""" import pytest from library.functions import ( + TestLogger, check_kubernetes_local_etcd, check_kubernetes_local_etcd_integrity, + check_kubernetes_local_etcd_media, + check_kubernetes_local_etcd_provisioning, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -26,20 +30,38 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(215) +@pytest.mark.order(40903) def test_kubernetes_local_etcd(host): """Verify each control plane has the configured etcd mount.""" - verify_pxeboot(host, "kubernetes_local_etcd", check_kubernetes_local_etcd) + tc = TC["kubernetes_local_etcd"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(216) +@pytest.mark.order(40904) def test_kubernetes_local_etcd_integrity(host): """Verify disk selection, ext4 label, UUID fstab, and boot persistence.""" - verify_pxeboot( - host, - "kubernetes_local_etcd_integrity", - check_kubernetes_local_etcd_integrity, - ) + tc = TC["kubernetes_local_etcd_integrity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_integrity) + + +@pytest.mark.kubernetes +@pytest.mark.order(40905) +def test_kubernetes_local_etcd_provisioning(host): + """Verify GPT selection and local-etcd scripts, logs, and selected disk.""" + tc = TC["kubernetes_local_etcd_provisioning"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_provisioning) + + +@pytest.mark.kubernetes +@pytest.mark.order(40906) +def test_kubernetes_local_etcd_media(host): + """Verify every control plane places etcd on the configured disk media.""" + tc = TC["kubernetes_local_etcd_media"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_media) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py b/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py index 21b7d9e62b..ccac59568a 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_recovery/test_reboot.py @@ -16,30 +16,30 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_control_plane_recovery, check_kubernetes_local_etcd_recovery, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -@pytest.mark.disruptive @pytest.mark.reboot @pytest.mark.kubernetes -@pytest.mark.order(224) +@pytest.mark.order(42201) def test_kubernetes_local_etcd_recovery(host): """Reboot a control plane and prove its local-etcd UUID is preserved.""" - verify_pxeboot( - host, - "kubernetes_local_etcd_recovery", - check_kubernetes_local_etcd_recovery, - ) + tc = TC["kubernetes_local_etcd_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_local_etcd_recovery) -@pytest.mark.disruptive @pytest.mark.reboot @pytest.mark.kubernetes -@pytest.mark.order(225) +@pytest.mark.order(42202) def test_kubernetes_control_plane_recovery(host): """Reboot the VIP owner and verify control-plane recovery.""" - verify_pxeboot(host, "kubernetes_recovery", check_kubernetes_control_plane_recovery) + tc = TC["kubernetes_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_control_plane_recovery) diff --git a/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py b/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py index ba64d37594..da01275806 100644 --- a/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py +++ b/test/orchestrator/fvt/pxeboot/kubernetes_storage/test_storage.py @@ -16,12 +16,14 @@ import pytest from library.functions import ( + TestLogger, check_kubernetes_csi_dynamic_provisioning, check_kubernetes_default_storage_class, check_kubernetes_nfs_dynamic_provisioning, check_kubernetes_snapshot_controller, check_kubernetes_storage, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -29,53 +31,55 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(218) +@pytest.mark.order(41001) def test_kubernetes_storage(host): """Verify configured NFS and PowerScale storage objects.""" - verify_pxeboot(host, "kubernetes_storage", check_kubernetes_storage) + tc = TC["kubernetes_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_storage) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(219) +@pytest.mark.order(41002) def test_kubernetes_default_storage_class(host): """Verify exactly one expected default StorageClass.""" - verify_pxeboot( - host, "kubernetes_default_storage", check_kubernetes_default_storage_class - ) + tc = TC["kubernetes_default_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_default_storage_class) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(220) +@pytest.mark.order(41003) def test_kubernetes_snapshot_controller(host): """Verify PowerScale snapshot components when configured.""" - verify_pxeboot( - host, "kubernetes_snapshot_controller", check_kubernetes_snapshot_controller - ) + tc = TC["kubernetes_snapshot_controller"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_snapshot_controller) @pytest.mark.functional @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(222) +@pytest.mark.order(41004) def test_kubernetes_nfs_dynamic_provisioning(host): """Create and remove an isolated NFS-backed workload.""" - verify_pxeboot( - host, "kubernetes_nfs_dynamic", check_kubernetes_nfs_dynamic_provisioning - ) + tc = TC["kubernetes_nfs_dynamic"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_nfs_dynamic_provisioning) @pytest.mark.functional @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.kubernetes -@pytest.mark.order(223) +@pytest.mark.order(41005) def test_kubernetes_csi_dynamic_provisioning(host): """Create and remove an isolated PowerScale-backed workload.""" - verify_pxeboot( - host, "kubernetes_csi_dynamic", check_kubernetes_csi_dynamic_provisioning - ) + tc = TC["kubernetes_csi_dynamic"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_kubernetes_csi_dynamic_provisioning) diff --git a/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py b/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py index 6714a60063..8f3990d0b0 100644 --- a/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py +++ b/test/orchestrator/fvt/pxeboot/minimal_os/test_minimal_os.py @@ -24,6 +24,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_minimal_os_additional_packages, check_minimal_os_additional_packages_fallback, check_minimal_os_base_packages, @@ -40,143 +41,169 @@ check_minimal_os_required_services, check_minimal_os_ssh_key_access, ) +from library.vars import TEST_CASES as TC -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.minimal_os, -] - -@pytest.mark.order(600) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40501) def test_minimal_os_base_packages(host): """Verify base OS packages are installed on all OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_base_packages", check_minimal_os_base_packages - ) + tc = TC["minimal_os_base_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_base_packages) -@pytest.mark.order(601) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40502) def test_minimal_os_ldms_packages(host): """Verify LDMS monitoring packages and binary on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_ldms_packages", check_minimal_os_ldms_packages - ) + tc = TC["minimal_os_ldms_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_ldms_packages) -@pytest.mark.order(602) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40503) def test_minimal_os_required_services(host): """Verify required services are active on all OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_required_services", check_minimal_os_required_services - ) + tc = TC["minimal_os_required_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_required_services) -@pytest.mark.order(604) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40504) def test_minimal_os_excluded_packages(host): """Verify workload-specific packages are NOT installed on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_excluded_packages", check_minimal_os_excluded_packages - ) + tc = TC["minimal_os_excluded_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_excluded_packages) -@pytest.mark.order(605) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40505) def test_minimal_os_excluded_services(host): """Verify workload-specific services are NOT active on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_excluded_services", check_minimal_os_excluded_services - ) + tc = TC["minimal_os_excluded_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_excluded_services) -@pytest.mark.order(606) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40506) def test_minimal_os_package_manager(host): """Verify package manager is functional on OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_package_manager", check_minimal_os_package_manager - ) + tc = TC["minimal_os_package_manager"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_package_manager) -@pytest.mark.order(607) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40507) def test_minimal_os_kernel_version(host): """Verify kernel version is consistent across OS-only nodes per FG.""" - verify_pxeboot( - host, "minimal_os_kernel_version", check_minimal_os_kernel_version - ) + tc = TC["minimal_os_kernel_version"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_kernel_version) -@pytest.mark.order(608) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40508) def test_minimal_os_network_identity(host): """Verify admin IP is configured on all OS-only nodes.""" - verify_pxeboot( - host, "minimal_os_network_identity", check_minimal_os_network_identity - ) + tc = TC["minimal_os_network_identity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_network_identity) -@pytest.mark.order(609) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40509) def test_minimal_os_functional_group_schema(host): """Verify functional-group definitions are valid on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_functional_group_schema", - check_minimal_os_functional_group_schema, - ) + tc = TC["minimal_os_functional_group_schema"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_functional_group_schema) -@pytest.mark.order(610) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40510) def test_minimal_os_additional_packages(host): """Verify configured extra packages are installed on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_additional_packages", - check_minimal_os_additional_packages, - ) + tc = TC["minimal_os_additional_packages"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_additional_packages) -@pytest.mark.order(611) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40511) def test_minimal_os_additional_packages_fallback(host): """Verify absent additional_packages config is handled on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_additional_packages_fallback", - check_minimal_os_additional_packages_fallback, - ) + tc = TC["minimal_os_additional_packages_fallback"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_additional_packages_fallback) -@pytest.mark.order(612) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40512) def test_minimal_os_ldms_service_state(host): """Verify LDMS service is installed but inactive at handoff.""" - verify_pxeboot( - host, - "minimal_os_ldms_service_state", - check_minimal_os_ldms_service_state, - ) + tc = TC["minimal_os_ldms_service_state"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_ldms_service_state) -@pytest.mark.order(613) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40513) def test_minimal_os_ssh_key_access(host): """Verify SSH key authentication is proven on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_ssh_key_access", - check_minimal_os_ssh_key_access, - ) + tc = TC["minimal_os_ssh_key_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_ssh_key_access) -@pytest.mark.order(614) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40514) def test_minimal_os_network_isolation(host): """Verify default route is on management network on OS-only nodes.""" - verify_pxeboot( - host, - "minimal_os_network_isolation", - check_minimal_os_network_isolation, - ) + tc = TC["minimal_os_network_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_network_isolation) -@pytest.mark.order(615) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.minimal_os +@pytest.mark.order(40515) def test_minimal_os_no_embedded_credentials(host): """Verify no plaintext secrets are present in the OS image.""" - verify_pxeboot( - host, - "minimal_os_no_embedded_credentials", - check_minimal_os_no_embedded_credentials, - ) + tc = TC["minimal_os_no_embedded_credentials"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_minimal_os_no_embedded_credentials) diff --git a/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py b/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py index 89dedd9bec..627af85537 100644 --- a/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py +++ b/test/orchestrator/fvt/pxeboot/mount_config/test_mount_config.py @@ -25,6 +25,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_mount_config_bind_fstab, check_mount_config_bind_mounts, check_mount_config_fg_targeting, @@ -38,93 +39,135 @@ check_mount_config_volume_mounted, check_mount_config_writable, ) +from library.vars import TEST_CASES as TC -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.mount_config, -] - - -@pytest.mark.order(500) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40401) def test_mount_config_mount_point(host): """Verify NFS mount point directories exist on all target nodes.""" - verify_pxeboot(host, "mount_config_mount_point", check_mount_config_mount_point) + tc = TC["mount_config_mount_point"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_mount_point) -@pytest.mark.order(501) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40402) def test_mount_config_volume_mounted(host): """Verify NFS volumes are actively mounted on all target nodes.""" - verify_pxeboot( - host, "mount_config_volume_mounted", check_mount_config_volume_mounted - ) + tc = TC["mount_config_volume_mounted"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_volume_mounted) -@pytest.mark.order(502) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40403) def test_mount_config_mount_options(host): """Verify NFS mount options match storage_config.yml on target nodes.""" - verify_pxeboot( - host, "mount_config_mount_options", check_mount_config_mount_options - ) + tc = TC["mount_config_mount_options"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_mount_options) -@pytest.mark.order(503) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40404) def test_mount_config_fstab(host): """Verify NFS fstab entries are persistent on all target nodes.""" - verify_pxeboot(host, "mount_config_fstab", check_mount_config_fstab) + tc = TC["mount_config_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_fstab) -@pytest.mark.order(504) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40405) def test_mount_config_bind_mounts(host): """Verify NFS bind mount targets are active on all target nodes.""" - verify_pxeboot(host, "mount_config_bind_mounts", check_mount_config_bind_mounts) + tc = TC["mount_config_bind_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_bind_mounts) -@pytest.mark.order(505) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40406) def test_mount_config_bind_fstab(host): """Verify NFS bind mount fstab entries are persistent on target nodes.""" - verify_pxeboot(host, "mount_config_bind_fstab", check_mount_config_bind_fstab) + tc = TC["mount_config_bind_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_bind_fstab) -@pytest.mark.order(506) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40407) def test_mount_config_node_subdirectory(host): """Verify per-node subdirectory exists under NFS mount point.""" - verify_pxeboot( - host, - "mount_config_node_subdirectory", - check_mount_config_node_subdirectory, - ) + tc = TC["mount_config_node_subdirectory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_node_subdirectory) -@pytest.mark.order(507) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40408) def test_mount_config_permissions(host): """Verify NFS mount permissions match storage_config.yml on nodes.""" - verify_pxeboot(host, "mount_config_permissions", check_mount_config_permissions) + tc = TC["mount_config_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_permissions) -@pytest.mark.order(508) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40409) def test_mount_config_fg_targeting(host): """Verify NFS mounts are present on target FGs and absent on others.""" - verify_pxeboot(host, "mount_config_fg_targeting", check_mount_config_fg_targeting) + tc = TC["mount_config_fg_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_fg_targeting) -@pytest.mark.order(509) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40410) def test_mount_config_no_duplicate_fstab(host): """Verify no duplicate NFS fstab entries on target nodes.""" - verify_pxeboot( - host, - "mount_config_no_duplicate_fstab", - check_mount_config_no_duplicate_fstab, - ) + tc = TC["mount_config_no_duplicate_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_no_duplicate_fstab) -@pytest.mark.order(510) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40411) def test_mount_config_writable(host): """Verify all configured NFS mounts are writable on target nodes.""" - verify_pxeboot(host, "mount_config_writable", check_mount_config_writable) + tc = TC["mount_config_writable"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_writable) -@pytest.mark.order(511) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.mount_config +@pytest.mark.order(40412) def test_mount_config_oim_mount(host): """Verify NFS storage is mounted on the OIM when mount_on_oim is true.""" - verify_pxeboot(host, "mount_config_oim_mount", check_mount_config_oim_mount) + tc = TC["mount_config_oim_mount"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_mount_config_oim_mount) diff --git a/test/orchestrator/fvt/pxeboot/powervault/conftest.py b/test/orchestrator/fvt/pxeboot/powervault/conftest.py deleted file mode 100644 index efccdb72a2..0000000000 --- a/test/orchestrator/fvt/pxeboot/powervault/conftest.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Pytest fixtures for PowerVault iSCSI storage verification.""" - -import pytest - - -def pytest_configure(config): - """Register custom markers for PowerVault tests.""" - config.addinivalue_line( - "markers", "powervault_infrastructure: iSCSI and multipath infrastructure tests" - ) - config.addinivalue_line( - "markers", "powervault_mounts: partition, filesystem, and mount validation" - ) - config.addinivalue_line( - "markers", "powervault_binds: bind mount and targeting validation" - ) - config.addinivalue_line( - "markers", "powervault_cloudinit: cloud-init and log validation" - ) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_binds.py b/test/orchestrator/fvt/pxeboot/powervault/test_binds.py index 9db9a2cbae..40784e4404 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_binds.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_binds.py @@ -18,6 +18,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_node_subdirectory, check_powervault_bind_mounts, check_powervault_bind_fstab_entries, @@ -28,88 +29,103 @@ check_powervault_slurm_mandatory_bind_mounts, check_powervault_mysql_data_on_mount, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(316) +@pytest.mark.order(40616) @pytest.mark.powervault_binds def test_powervault_node_subdirectory(host): """Verify per-node subdirectory exists under mount point.""" - verify_pxeboot(host, "powervault_node_subdirectory", check_powervault_node_subdirectory) + tc = TC["powervault_node_subdirectory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_node_subdirectory) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(317) +@pytest.mark.order(40617) @pytest.mark.powervault_binds def test_powervault_bind_mounts(host): """Verify bind mount targets are active on all target nodes.""" - verify_pxeboot(host, "powervault_bind_mounts", check_powervault_bind_mounts) + tc = TC["powervault_bind_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_mounts) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(318) +@pytest.mark.order(40618) @pytest.mark.powervault_binds def test_powervault_bind_fstab_entries(host): """Verify bind mount fstab entries are persistent on all target nodes.""" - verify_pxeboot(host, "powervault_bind_fstab_entries", check_powervault_bind_fstab_entries) + tc = TC["powervault_bind_fstab_entries"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_fstab_entries) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(319) +@pytest.mark.order(40619) @pytest.mark.powervault_binds def test_powervault_bind_isolation(host): """Verify per-node data separation via bind mounts.""" - verify_pxeboot(host, "powervault_bind_isolation", check_powervault_bind_isolation) + tc = TC["powervault_bind_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_isolation) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(320) +@pytest.mark.order(40620) @pytest.mark.powervault_binds def test_powervault_functional_group_targeting(host): """Verify PV mount only on correct functional groups.""" - verify_pxeboot(host, "powervault_functional_group_targeting", check_powervault_functional_group_targeting) + tc = TC["powervault_functional_group_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_functional_group_targeting) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(321) +@pytest.mark.order(40621) @pytest.mark.powervault_binds def test_powervault_multiple_prefix_targeting(host): """Verify multiple prefixes target all groups correctly.""" - verify_pxeboot(host, "powervault_multiple_prefix_targeting", check_powervault_multiple_prefix_targeting) + tc = TC["powervault_multiple_prefix_targeting"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multiple_prefix_targeting) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(328) +@pytest.mark.order(40628) @pytest.mark.powervault_binds def test_powervault_bind_io(host): """Verify bind-mount I/O reaches the PV backing store.""" - verify_pxeboot(host, "powervault_bind_io", check_powervault_bind_io) + tc = TC["powervault_bind_io"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_bind_io) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(329) +@pytest.mark.order(40629) @pytest.mark.powervault_binds def test_powervault_slurm_mandatory_bind_mounts(host): """Verify /var/lib/mysql and /var/spool/slurm configured as bind targets.""" - verify_pxeboot( - host, - "powervault_slurm_mandatory_bind_mounts", - check_powervault_slurm_mandatory_bind_mounts, - ) + tc = TC["powervault_slurm_mandatory_bind_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_slurm_mandatory_bind_mounts) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(330) +@pytest.mark.order(40630) @pytest.mark.powervault_binds def test_powervault_mysql_data_on_mount(host): """Verify MySQL/MariaDB datadir is on a PowerVault mount.""" - verify_pxeboot(host, "powervault_mysql_data_on_mount", check_powervault_mysql_data_on_mount) + tc = TC["powervault_mysql_data_on_mount"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_mysql_data_on_mount) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py b/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py index f4188acfce..7c90c7e616 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_cloudinit.py @@ -18,54 +18,66 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_setup_log, check_powervault_cloud_init_groups_dict, check_powervault_no_duplicate_fstab, check_powervault_all_mounts_writable, check_powervault_permissions, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(322) +@pytest.mark.order(40622) @pytest.mark.powervault_cloudinit def test_powervault_setup_log(host): """Verify cloud-init runcmd log exists and shows completion.""" - verify_pxeboot(host, "powervault_setup_log", check_powervault_setup_log) + tc = TC["powervault_setup_log"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_setup_log) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(323) +@pytest.mark.order(40623) @pytest.mark.powervault_cloudinit def test_powervault_cloud_init_groups_dict(host): """Verify rendered iSCSI setup scripts deployed on target nodes.""" - verify_pxeboot(host, "powervault_cloud_init_groups_dict", check_powervault_cloud_init_groups_dict) + tc = TC["powervault_cloud_init_groups_dict"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_cloud_init_groups_dict) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(324) +@pytest.mark.order(40624) @pytest.mark.powervault_cloudinit def test_powervault_no_duplicate_fstab(host): """Verify no duplicate fstab entries on all target nodes.""" - verify_pxeboot(host, "powervault_no_duplicate_fstab", check_powervault_no_duplicate_fstab) + tc = TC["powervault_no_duplicate_fstab"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_no_duplicate_fstab) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(325) +@pytest.mark.order(40625) @pytest.mark.powervault_cloudinit def test_powervault_all_mounts_writable(host): """Verify all PV mounts (main + bind) are writable.""" - verify_pxeboot(host, "powervault_all_mounts_writable", check_powervault_all_mounts_writable) + tc = TC["powervault_all_mounts_writable"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_all_mounts_writable) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(326) +@pytest.mark.order(40626) @pytest.mark.powervault_cloudinit def test_powervault_permissions(host): """Verify permissions on mount point match config.""" - verify_pxeboot(host, "powervault_permissions", check_powervault_permissions) + tc = TC["powervault_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_permissions) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py b/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py index d695219703..08ba7f567d 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_infrastructure.py @@ -18,6 +18,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_iscsi_service, check_powervault_iscsi_initiator_name, check_powervault_iscsi_discovery, @@ -28,84 +29,103 @@ check_powervault_multipath_device, check_powervault_multipath_redundancy, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(301) +@pytest.mark.order(40601) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_service(host): """Verify iscsid is active and enabled on all PowerVault target nodes.""" - verify_pxeboot(host, "powervault_iscsi_service", check_powervault_iscsi_service) + tc = TC["powervault_iscsi_service"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_service) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(302) +@pytest.mark.order(40602) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_initiator_name(host): """Verify iSCSI initiator name matches config on all target nodes.""" - verify_pxeboot(host, "powervault_iscsi_initiator_name", check_powervault_iscsi_initiator_name) + tc = TC["powervault_iscsi_initiator_name"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_initiator_name) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(303) +@pytest.mark.order(40603) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_discovery(host): """Verify iSCSI target discovery succeeds from all portal IPs.""" - verify_pxeboot(host, "powervault_iscsi_discovery", check_powervault_iscsi_discovery) + tc = TC["powervault_iscsi_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_discovery) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(304) +@pytest.mark.order(40604) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_sessions(host): """Verify iSCSI sessions are active on all target nodes.""" - verify_pxeboot(host, "powervault_iscsi_sessions", check_powervault_iscsi_sessions) + tc = TC["powervault_iscsi_sessions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_sessions) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(305) +@pytest.mark.order(40605) @pytest.mark.powervault_infrastructure def test_powervault_iscsi_startup_automatic(host): """Verify iSCSI node startup is automatic on all target nodes.""" - verify_pxeboot(host, "powervault_iscsi_startup_automatic", check_powervault_iscsi_startup_automatic) + tc = TC["powervault_iscsi_startup_automatic"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_iscsi_startup_automatic) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(306) +@pytest.mark.order(40606) @pytest.mark.powervault_infrastructure def test_powervault_portal_reachability(host): """Verify iSCSI portal ports are reachable and sessions healthy.""" - verify_pxeboot(host, "powervault_portal_reachability", check_powervault_portal_reachability) + tc = TC["powervault_portal_reachability"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_portal_reachability) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(307) +@pytest.mark.order(40607) @pytest.mark.powervault_infrastructure def test_powervault_multipath_service(host): """Verify multipathd is active and enabled on all target nodes.""" - verify_pxeboot(host, "powervault_multipath_service", check_powervault_multipath_service) + tc = TC["powervault_multipath_service"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multipath_service) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(308) +@pytest.mark.order(40608) @pytest.mark.powervault_infrastructure def test_powervault_multipath_device(host): """Verify multipath device exists and matches volume_id.""" - verify_pxeboot(host, "powervault_multipath_device", check_powervault_multipath_device) + tc = TC["powervault_multipath_device"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multipath_device) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(309) +@pytest.mark.order(40609) @pytest.mark.powervault_infrastructure def test_powervault_multipath_redundancy(host): """Verify multipath device has multiple paths for redundancy.""" - verify_pxeboot(host, "powervault_multipath_redundancy", check_powervault_multipath_redundancy) + tc = TC["powervault_multipath_redundancy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_multipath_redundancy) diff --git a/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py b/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py index 76ad20834c..d3c25b8ac5 100644 --- a/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py +++ b/test/orchestrator/fvt/pxeboot/powervault/test_mounts.py @@ -18,6 +18,7 @@ from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_powervault_gpt_partition, check_powervault_filesystem_type, check_powervault_mount_point_directory, @@ -26,66 +27,81 @@ check_powervault_fstab_entry, check_powervault_io_write_read, ) +from library.vars import TEST_CASES as TC @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(310) +@pytest.mark.order(40610) @pytest.mark.powervault_mounts def test_powervault_gpt_partition(host): """Verify GPT partition exists on multipath device.""" - verify_pxeboot(host, "powervault_gpt_partition", check_powervault_gpt_partition) + tc = TC["powervault_gpt_partition"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_gpt_partition) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(311) +@pytest.mark.order(40611) @pytest.mark.powervault_mounts def test_powervault_filesystem_type(host): """Verify filesystem formatted with correct type.""" - verify_pxeboot(host, "powervault_filesystem_type", check_powervault_filesystem_type) + tc = TC["powervault_filesystem_type"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_filesystem_type) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(312) +@pytest.mark.order(40612) @pytest.mark.powervault_mounts def test_powervault_mount_point_directory(host): """Verify mount point directory exists on all target nodes.""" - verify_pxeboot(host, "powervault_mount_point_directory", check_powervault_mount_point_directory) + tc = TC["powervault_mount_point_directory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_mount_point_directory) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(313) +@pytest.mark.order(40613) @pytest.mark.powervault_mounts def test_powervault_volume_mounted(host): """Verify PowerVault volume is actively mounted on all target nodes.""" - verify_pxeboot(host, "powervault_volume_mounted", check_powervault_volume_mounted) + tc = TC["powervault_volume_mounted"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_volume_mounted) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(314) +@pytest.mark.order(40614) @pytest.mark.powervault_mounts def test_powervault_mount_options(host): """Verify mount options applied correctly on all target nodes.""" - verify_pxeboot(host, "powervault_mount_options", check_powervault_mount_options) + tc = TC["powervault_mount_options"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_mount_options) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(315) +@pytest.mark.order(40615) @pytest.mark.powervault_mounts def test_powervault_fstab_entry(host): """Verify persistent fstab entry created on all target nodes.""" - verify_pxeboot(host, "powervault_fstab_entry", check_powervault_fstab_entry) + tc = TC["powervault_fstab_entry"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_fstab_entry) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(327) +@pytest.mark.order(40627) @pytest.mark.powervault_mounts def test_powervault_io_write_read(host): """Verify write-read I/O succeeds on every PV mount point.""" - verify_pxeboot(host, "powervault_io_write_read", check_powervault_io_write_read) + tc = TC["powervault_io_write_read"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_powervault_io_write_read) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py index 433607a5a8..93f02d21ce 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_accelerators.py @@ -16,12 +16,14 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_cuda_workload, check_apptainer_gpu_access, check_apptainer_gpu_count, check_apptainer_gpu_memory, check_apptainer_infiniband, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -31,10 +33,12 @@ @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(290) +@pytest.mark.order(41925) def test_apptainer_gpu_access(host): """Verify scheduler-declared GPU nodes expose GPUs in the container.""" - verify_pxeboot(host, "apptainer_gpu_access", check_apptainer_gpu_access) + tc = TC["apptainer_gpu_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_gpu_access) @pytest.mark.apptainer @@ -42,10 +46,12 @@ def test_apptainer_gpu_access(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(291) +@pytest.mark.order(41926) def test_apptainer_gpu_count(host): """Verify each container sees the same GPU count as its host.""" - verify_pxeboot(host, "apptainer_gpu_count", check_apptainer_gpu_count) + tc = TC["apptainer_gpu_count"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_gpu_count) @pytest.mark.apptainer @@ -53,10 +59,12 @@ def test_apptainer_gpu_count(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(292) +@pytest.mark.order(41927) def test_apptainer_cuda_workload(host): """Execute a bounded NVIDIA device query in each GPU container.""" - verify_pxeboot(host, "apptainer_cuda_workload", check_apptainer_cuda_workload) + tc = TC["apptainer_cuda_workload"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_cuda_workload) @pytest.mark.apptainer @@ -64,10 +72,12 @@ def test_apptainer_cuda_workload(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(293) +@pytest.mark.order(41928) def test_apptainer_gpu_memory(host): """Verify the GPU query leaves no material device-memory allocation.""" - verify_pxeboot(host, "apptainer_gpu_memory", check_apptainer_gpu_memory) + tc = TC["apptainer_gpu_memory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_gpu_memory) @pytest.mark.apptainer @@ -75,7 +85,9 @@ def test_apptainer_gpu_memory(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(294) +@pytest.mark.order(41929) def test_apptainer_infiniband(host): """Verify mapped compute nodes expose InfiniBand devices in containers.""" - verify_pxeboot(host, "apptainer_infiniband", check_apptainer_infiniband) + tc = TC["apptainer_infiniband"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_infiniband) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py index 6dd4267557..8453ebd05c 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_images.py @@ -16,10 +16,12 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_download, check_apptainer_download_idempotency, check_apptainer_download_memory, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -30,10 +32,12 @@ @pytest.mark.functional @pytest.mark.image_download @pytest.mark.non_disruptive -@pytest.mark.order(270) +@pytest.mark.order(41905) def test_apptainer_download(host): """Run the deployed downloader and require at least one usable SIF.""" - verify_pxeboot(host, "apptainer_download", check_apptainer_download) + tc = TC["apptainer_download"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_download) @pytest.mark.apptainer @@ -42,14 +46,12 @@ def test_apptainer_download(host): @pytest.mark.functional @pytest.mark.image_download @pytest.mark.non_disruptive -@pytest.mark.order(271) +@pytest.mark.order(41906) def test_apptainer_download_idempotency(host): """Rerun the downloader and verify existing image metadata is unchanged.""" - verify_pxeboot( - host, - "apptainer_download_idempotency", - check_apptainer_download_idempotency, - ) + tc = TC["apptainer_download_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_download_idempotency) @pytest.mark.apptainer @@ -58,7 +60,9 @@ def test_apptainer_download_idempotency(host): @pytest.mark.functional @pytest.mark.image_download @pytest.mark.non_disruptive -@pytest.mark.order(272) +@pytest.mark.order(41907) def test_apptainer_download_memory(host): """Run the downloader and enforce a bounded peak resident-memory use.""" - verify_pxeboot(host, "apptainer_download_memory", check_apptainer_download_memory) + tc = TC["apptainer_download_memory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_download_memory) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py index 8449e9a287..ba005df083 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_jobs.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_concurrent_jobs, check_apptainer_job_array, check_apptainer_ldap_job, @@ -24,6 +25,7 @@ check_apptainer_single_node_job, check_apptainer_slurm_environment, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -33,10 +35,12 @@ @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(280) +@pytest.mark.order(41915) def test_apptainer_single_node_job(host): """Run one exact-node container job on every mapped compute.""" - verify_pxeboot(host, "apptainer_single_node_job", check_apptainer_single_node_job) + tc = TC["apptainer_single_node_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_single_node_job) @pytest.mark.apptainer @@ -44,20 +48,24 @@ def test_apptainer_single_node_job(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(281) +@pytest.mark.order(41916) def test_apptainer_multi_node_job(host): """Run one container allocation spanning all mapped computes.""" - verify_pxeboot(host, "apptainer_multi_node_job", check_apptainer_multi_node_job) + tc = TC["apptainer_multi_node_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_multi_node_job) @pytest.mark.apptainer @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(282) +@pytest.mark.order(41917) def test_apptainer_ldap_job(host): """Run targeted container jobs as the configured LDAP test identity.""" - verify_pxeboot(host, "apptainer_ldap_job", check_apptainer_ldap_job) + tc = TC["apptainer_ldap_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_ldap_job) @pytest.mark.apptainer @@ -65,10 +73,12 @@ def test_apptainer_ldap_job(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(283) +@pytest.mark.order(41918) def test_apptainer_concurrent_jobs(host): """Run bounded concurrent container jobs on distinct computes.""" - verify_pxeboot(host, "apptainer_concurrent_jobs", check_apptainer_concurrent_jobs) + tc = TC["apptainer_concurrent_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_concurrent_jobs) @pytest.mark.apptainer @@ -76,10 +86,12 @@ def test_apptainer_concurrent_jobs(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(286) +@pytest.mark.order(41921) def test_apptainer_nfs_visibility(host): """Verify containers can read the shared image path on every compute.""" - verify_pxeboot(host, "apptainer_nfs_visibility", check_apptainer_nfs_visibility) + tc = TC["apptainer_nfs_visibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_nfs_visibility) @pytest.mark.apptainer @@ -87,12 +99,12 @@ def test_apptainer_nfs_visibility(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(287) +@pytest.mark.order(41922) def test_apptainer_slurm_environment(host): """Verify Slurm allocation variables propagate into containers.""" - verify_pxeboot( - host, "apptainer_slurm_environment", check_apptainer_slurm_environment - ) + tc = TC["apptainer_slurm_environment"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_slurm_environment) @pytest.mark.apptainer @@ -100,7 +112,9 @@ def test_apptainer_slurm_environment(host): @pytest.mark.sanity @pytest.mark.functional @pytest.mark.non_disruptive -@pytest.mark.order(288) +@pytest.mark.order(41923) def test_apptainer_job_array(host): """Submit and wait for a bounded Apptainer Slurm job array.""" - verify_pxeboot(host, "apptainer_job_array", check_apptainer_job_array) + tc = TC["apptainer_job_array"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_job_array) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py index 0a59521ef4..8ffabd8bac 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_negative.py @@ -16,11 +16,13 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_failure_cleanup, check_apptainer_invalid_sif, check_apptainer_missing_image_contract, check_apptainer_restricted_sif, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -29,41 +31,45 @@ @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(279) +@pytest.mark.order(41914) def test_apptainer_missing_image_contract(host): """Verify the downloader records pull failures and exits non-zero.""" - verify_pxeboot( - host, - "apptainer_missing_image_contract", - check_apptainer_missing_image_contract, - ) + tc = TC["apptainer_missing_image_contract"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_missing_image_contract) @pytest.mark.apptainer @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(284) +@pytest.mark.order(41919) def test_apptainer_invalid_sif(host): """Verify a nonexistent SIF fails through the Slurm execution path.""" - verify_pxeboot(host, "apptainer_invalid_sif", check_apptainer_invalid_sif) + tc = TC["apptainer_invalid_sif"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_invalid_sif) @pytest.mark.apptainer @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(285) +@pytest.mark.order(41920) def test_apptainer_restricted_sif(host): """Verify an unprivileged identity cannot execute a mode-0600 SIF.""" - verify_pxeboot(host, "apptainer_restricted_sif", check_apptainer_restricted_sif) + tc = TC["apptainer_restricted_sif"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_restricted_sif) @pytest.mark.apptainer @pytest.mark.functional @pytest.mark.negative @pytest.mark.non_disruptive -@pytest.mark.order(289) +@pytest.mark.order(41924) def test_apptainer_failure_cleanup(host): """Verify a failed image launch leaves no matching runtime process.""" - verify_pxeboot(host, "apptainer_failure_cleanup", check_apptainer_failure_cleanup) + tc = TC["apptainer_failure_cleanup"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_failure_cleanup) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py index 5d9c1faa32..ae4a00bcac 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_reboot.py @@ -16,36 +16,41 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_reboot_artifacts, check_apptainer_reboot_job, check_apptainer_reboot_storage, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @pytest.mark.apptainer -@pytest.mark.disruptive @pytest.mark.reboot -@pytest.mark.order(295) +@pytest.mark.order(41930) def test_apptainer_reboot_storage(host): """Reboot one compute and verify the shared mount and SIF checksum.""" - verify_pxeboot(host, "apptainer_reboot_storage", check_apptainer_reboot_storage) + tc = TC["apptainer_reboot_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_reboot_storage) @pytest.mark.apptainer -@pytest.mark.disruptive @pytest.mark.reboot -@pytest.mark.order(296) +@pytest.mark.order(41931) def test_apptainer_reboot_job(host): """Run an exact-node container job after the authorized reboot.""" - verify_pxeboot(host, "apptainer_reboot_job", check_apptainer_reboot_job) + tc = TC["apptainer_reboot_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_reboot_job) @pytest.mark.apptainer -@pytest.mark.disruptive @pytest.mark.reboot -@pytest.mark.order(297) +@pytest.mark.order(41932) def test_apptainer_reboot_artifacts(host): """Verify downloader artifacts and policy after the authorized reboot.""" - verify_pxeboot(host, "apptainer_reboot_artifacts", check_apptainer_reboot_artifacts) + tc = TC["apptainer_reboot_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_reboot_artifacts) diff --git a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py index 425c9931e5..cd8ba8f4fc 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py +++ b/test/orchestrator/fvt/pxeboot/slurm_apptainer/test_runtime.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_apptainer_image_inventory, check_apptainer_ldap_readability, check_apptainer_non_root_execution, @@ -27,6 +28,7 @@ check_apptainer_sif_integrity, check_apptainer_sif_permissions, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -35,98 +37,116 @@ @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(266) +@pytest.mark.order(41901) def test_apptainer_runtime(host): """Verify the Apptainer executable and version on every compute node.""" - verify_pxeboot(host, "apptainer_runtime", check_apptainer_runtime) + tc = TC["apptainer_runtime"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_runtime) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(267) +@pytest.mark.order(41902) def test_apptainer_shared_artifacts(host): """Verify shared image directories and downloader artifacts.""" - verify_pxeboot(host, "apptainer_shared_artifacts", check_apptainer_shared_artifacts) + tc = TC["apptainer_shared_artifacts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_shared_artifacts) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(268) +@pytest.mark.order(41903) def test_apptainer_pulp_policy(host): """Verify the generated downloader uses only the configured Pulp source.""" - verify_pxeboot(host, "apptainer_pulp_policy", check_apptainer_pulp_policy) + tc = TC["apptainer_pulp_policy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_pulp_policy) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(269) +@pytest.mark.order(41904) def test_apptainer_shared_storage(host): """Verify /hpc_tools is a shared mounted filesystem on every compute.""" - verify_pxeboot(host, "apptainer_shared_storage", check_apptainer_shared_storage) + tc = TC["apptainer_shared_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_shared_storage) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(273) +@pytest.mark.order(41908) def test_apptainer_image_inventory(host): """Verify every compute sees one consistent non-empty SIF inventory.""" - verify_pxeboot(host, "apptainer_image_inventory", check_apptainer_image_inventory) + tc = TC["apptainer_image_inventory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_image_inventory) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(274) +@pytest.mark.order(41909) def test_apptainer_sif_format(host): """Verify each discovered image is a valid inspectable SIF.""" - verify_pxeboot(host, "apptainer_sif_format", check_apptainer_sif_format) + tc = TC["apptainer_sif_format"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_sif_format) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(275) +@pytest.mark.order(41910) def test_apptainer_sif_permissions(host): """Verify shared SIF files are non-empty and world-readable.""" - verify_pxeboot(host, "apptainer_sif_permissions", check_apptainer_sif_permissions) + tc = TC["apptainer_sif_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_sif_permissions) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(276) +@pytest.mark.order(41911) def test_apptainer_sif_integrity(host): """Verify the selected SIF has the same checksum on every compute.""" - verify_pxeboot(host, "apptainer_sif_integrity", check_apptainer_sif_integrity) + tc = TC["apptainer_sif_integrity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_sif_integrity) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.sanity -@pytest.mark.order(277) +@pytest.mark.order(41912) def test_apptainer_ldap_readability(host): """Verify the LDAP test identity can read a shared SIF when enabled.""" - verify_pxeboot(host, "apptainer_ldap_readability", check_apptainer_ldap_readability) + tc = TC["apptainer_ldap_readability"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_ldap_readability) @pytest.mark.apptainer @pytest.mark.non_disruptive @pytest.mark.sanity @pytest.mark.functional -@pytest.mark.order(278) +@pytest.mark.order(41913) def test_apptainer_non_root_execution(host): """Verify an unprivileged local identity can execute a shared SIF.""" - verify_pxeboot( - host, "apptainer_non_root_execution", check_apptainer_non_root_execution - ) + tc = TC["apptainer_non_root_execution"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_apptainer_non_root_execution) diff --git a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py index 982b4858dd..13ea104268 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py +++ b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_cluster_health.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_configless_mode, check_slurm_configuration_consistency, check_slurm_cross_node_ssh, @@ -23,6 +24,7 @@ check_slurm_scheduler, check_slurm_services, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -31,59 +33,69 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(226) +@pytest.mark.order(41101) def test_slurm_membership(host): """Verify mapped membership, healthy state, and basic hardware fields.""" - verify_pxeboot(host, "slurm_membership", check_slurm_membership) + tc = TC["slurm_membership"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_membership) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(227) +@pytest.mark.order(41102) def test_slurm_scheduler(host): """Verify mapped compute nodes have healthy, available partitions.""" - verify_pxeboot(host, "slurm_scheduler", check_slurm_scheduler) + tc = TC["slurm_scheduler"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_scheduler) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(228) +@pytest.mark.order(41103) def test_slurm_services(host): """Verify role and feature-specific Slurm services.""" - verify_pxeboot(host, "slurm_services", check_slurm_services) + tc = TC["slurm_services"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_services) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(229) +@pytest.mark.order(41104) def test_slurm_cross_node_ssh(host): """Verify every mapped Slurm role can reach every peer over root SSH.""" - verify_pxeboot(host, "slurm_cross_ssh", check_slurm_cross_node_ssh) + tc = TC["slurm_cross_ssh"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_cross_node_ssh) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(230) +@pytest.mark.order(41105) def test_slurm_configless_mode(host): """Verify configless controller access and expected cluster identity.""" - verify_pxeboot(host, "slurm_configless", check_slurm_configless_mode) + tc = TC["slurm_configless"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_configless_mode) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(231) +@pytest.mark.order(41106) def test_slurm_configuration_consistency(host): """Compare authoritative Slurm files with every configless client cache.""" - verify_pxeboot( - host, "slurm_config_consistency", check_slurm_configuration_consistency - ) + tc = TC["slurm_config_consistency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_configuration_consistency) diff --git a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py index 36c2a2783d..a42761bae6 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py +++ b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_configuration.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_slurm_custom_configuration, check_slurm_reconfigure, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -28,17 +30,21 @@ @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm -@pytest.mark.order(232) +@pytest.mark.order(41107) def test_slurm_reconfigure(host): """Reconfigure Slurm and verify membership remains healthy.""" - verify_pxeboot(host, "slurm_reconfigure", check_slurm_reconfigure) + tc = TC["slurm_reconfigure"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_reconfigure) @pytest.mark.non_disruptive @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm -@pytest.mark.order(234) +@pytest.mark.order(41109) def test_slurm_custom_configuration(host): """Verify custom values, NFS delivery, and effective visibility.""" - verify_pxeboot(host, "slurm_custom_configuration", check_slurm_custom_configuration) + tc = TC["slurm_custom_configuration"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_custom_configuration) diff --git a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py index 803ab6bc19..6c3a63bfde 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py +++ b/test/orchestrator/fvt/pxeboot/slurm_cluster/test_discovery.py @@ -15,7 +15,11 @@ """Slurm homogeneous and heterogeneous hardware-discovery contracts.""" import pytest -from library.functions import check_slurm_hardware_discovery +from library.functions import ( + TestLogger, + check_slurm_hardware_discovery, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -24,7 +28,9 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(233) +@pytest.mark.order(41108) def test_slurm_hardware_discovery(host): """Verify runtime hardware matches the configured discovery strategy.""" - verify_pxeboot(host, "slurm_hardware_discovery", check_slurm_hardware_discovery) + tc = TC["slurm_hardware_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_hardware_discovery) diff --git a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py index a8b6d2ef27..6e776a2573 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py +++ b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm.py @@ -22,6 +22,7 @@ import pytest from library.functions import ( + TestLogger, check_dcgm_cuda_atomic_lock, check_dcgm_cuda_compute_node, check_dcgm_cuda_login_compiler, @@ -37,117 +38,182 @@ check_dcgm_rhel_compatibility, check_dcgm_toolkit_nfs_storage, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.slurm, - pytest.mark.non_disruptive, -] - - # -- CUDA installation checks ------------------------------------------------ -@pytest.mark.order(300) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41501) def test_dcgm_cuda_validation(host): """Verify NVIDIA driver and CUDA toolkit are installed on GPU nodes.""" - verify_pxeboot(host, "dcgm_cuda_validation", check_dcgm_cuda_validation) + tc = TC["dcgm_cuda_validation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_validation) -@pytest.mark.order(301) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41502) def test_dcgm_cuda_atomic_lock(host): """Verify CUDA toolkit is installed to /hpc_tools/cuda via atomic lock.""" - verify_pxeboot(host, "dcgm_cuda_atomic_lock", check_dcgm_cuda_atomic_lock) + tc = TC["dcgm_cuda_atomic_lock"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_atomic_lock) -@pytest.mark.order(302) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41503) def test_dcgm_cuda_compute_node(host): """Verify both CUDA toolkit and CUDA driver on compute nodes.""" - verify_pxeboot(host, "dcgm_cuda_compute_node", check_dcgm_cuda_compute_node) + tc = TC["dcgm_cuda_compute_node"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_compute_node) -@pytest.mark.order(303) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41504) def test_dcgm_cuda_login_compiler(host): """Verify CUDA toolkit accessible on login_compiler nodes.""" - verify_pxeboot(host, "dcgm_cuda_login_compiler", check_dcgm_cuda_login_compiler) + tc = TC["dcgm_cuda_login_compiler"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_login_compiler) # -- DCGM service and package checks ----------------------------------------- -@pytest.mark.order(304) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41505) def test_dcgm_package_installed(host): """Verify datacenter-gpu-manager RPM and dcgmi binary on GPU nodes.""" - verify_pxeboot(host, "dcgm_package_installed", check_dcgm_package_installed) + tc = TC["dcgm_package_installed"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_package_installed) -@pytest.mark.order(305) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41506) def test_dcgm_daemon_running(host): """Verify nvidia-dcgm service is active and enabled on GPU nodes.""" - verify_pxeboot(host, "dcgm_daemon_running", check_dcgm_daemon_running) + tc = TC["dcgm_daemon_running"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_daemon_running) # -- GPU discovery and monitoring checks -------------------------------------- -@pytest.mark.order(306) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41507) def test_dcgm_gpu_discovery(host): """Verify dcgmi discovery enumerates GPUs with unique UUIDs.""" - verify_pxeboot(host, "dcgm_gpu_discovery", check_dcgm_gpu_discovery) + tc = TC["dcgm_gpu_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_gpu_discovery) -@pytest.mark.order(307) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41508) def test_dcgm_gpu_metrics(host): """Verify dcgmi dmon returns metric samples for each GPU node.""" - verify_pxeboot(host, "dcgm_gpu_metrics", check_dcgm_gpu_metrics) + tc = TC["dcgm_gpu_metrics"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_gpu_metrics) -@pytest.mark.order(308) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41509) def test_dcgm_multi_gpu_discovery(host): """Verify dcgmi discovery on multi-GPU nodes.""" - verify_pxeboot(host, "dcgm_multi_gpu_discovery", check_dcgm_multi_gpu_discovery) + tc = TC["dcgm_multi_gpu_discovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_multi_gpu_discovery) # -- Multi-node configuration checks ----------------------------------------- -@pytest.mark.order(309) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41510) def test_dcgm_multi_gpu_no_login_compiler(host): """Verify GPU nodes work without login_compiler present.""" - verify_pxeboot( - host, - "dcgm_multi_gpu_no_login_compiler", - check_dcgm_multi_gpu_no_login_compiler, - ) + tc = TC["dcgm_multi_gpu_no_login_compiler"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_multi_gpu_no_login_compiler) -@pytest.mark.order(310) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41511) def test_dcgm_multi_login_compiler_lock(host): """Verify CUDA toolkit install uses atomic lock with multiple login_compilers.""" - verify_pxeboot( - host, - "dcgm_multi_login_compiler_lock", - check_dcgm_multi_login_compiler_lock, - ) + tc = TC["dcgm_multi_login_compiler_lock"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_multi_login_compiler_lock) # -- Platform compatibility checks ------------------------------------------- -@pytest.mark.order(311) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41512) def test_dcgm_toolkit_nfs_storage(host): """Verify /hpc_tools is NFS-mounted and CUDA toolkit accessible.""" - verify_pxeboot(host, "dcgm_toolkit_nfs_storage", check_dcgm_toolkit_nfs_storage) + tc = TC["dcgm_toolkit_nfs_storage"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_toolkit_nfs_storage) -@pytest.mark.order(312) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41513) def test_dcgm_rhel_compatibility(host): """Verify GPU node OS is a supported RHEL version.""" - verify_pxeboot(host, "dcgm_rhel_compatibility", check_dcgm_rhel_compatibility) + tc = TC["dcgm_rhel_compatibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_rhel_compatibility) -@pytest.mark.order(313) +@pytest.mark.buildstream +@pytest.mark.sanity +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41514) def test_dcgm_cuda_version_compatibility(host): """Verify CUDA toolkit and DCGM daemon version compatibility.""" - verify_pxeboot( - host, - "dcgm_cuda_version_compatibility", - check_dcgm_cuda_version_compatibility, - ) + tc = TC["dcgm_cuda_version_compatibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_cuda_version_compatibility) diff --git a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py index 50ef9fdcc5..adb74bcf5e 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py +++ b/test/orchestrator/fvt/pxeboot/slurm_dcgm/test_dcgm_negative.py @@ -25,50 +25,54 @@ import pytest from library.functions import ( + TestLogger, check_dcgm_neg_cuda_prerequisite, check_dcgm_neg_daemon_recovery, check_dcgm_neg_package_install_failure, check_dcgm_neg_socket_inaccessible, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [ - pytest.mark.buildstream, - pytest.mark.sanity, - pytest.mark.slurm, - pytest.mark.functional, -] - - -@pytest.mark.order(320) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41515) def test_dcgm_neg_cuda_prerequisite(host): """Verify DCGM deployment requires CUDA driver as a prerequisite.""" - verify_pxeboot( - host, "dcgm_neg_cuda_prerequisite", check_dcgm_neg_cuda_prerequisite, - ) + tc = TC["dcgm_neg_cuda_prerequisite"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_cuda_prerequisite) -@pytest.mark.order(321) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41516) def test_dcgm_neg_daemon_recovery(host): """Simulate DCGM daemon crash via SIGKILL and verify systemd restarts it.""" - verify_pxeboot( - host, "dcgm_neg_daemon_recovery", check_dcgm_neg_daemon_recovery, - ) + tc = TC["dcgm_neg_daemon_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_daemon_recovery) -@pytest.mark.order(322) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41517) def test_dcgm_neg_socket_inaccessible(host): """Remove DCGM Unix socket and verify dcgmi returns a clear error.""" - verify_pxeboot( - host, "dcgm_neg_socket_inaccessible", check_dcgm_neg_socket_inaccessible, - ) + tc = TC["dcgm_neg_socket_inaccessible"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_socket_inaccessible) -@pytest.mark.order(323) +@pytest.mark.buildstream +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.order(41518) def test_dcgm_neg_package_install_failure(host): """Verify error handling when datacenter-gpu-manager package is unavailable.""" - verify_pxeboot( - host, - "dcgm_neg_package_install_failure", - check_dcgm_neg_package_install_failure, - ) + tc = TC["dcgm_neg_package_install_failure"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_dcgm_neg_package_install_failure) diff --git a/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py b/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py index af3064298a..78a2372a49 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py +++ b/test/orchestrator/fvt/pxeboot/slurm_gpu/test_gpu.py @@ -16,10 +16,12 @@ import pytest from library.functions import ( + TestLogger, check_slurm_gpu_inventory, check_slurm_gpu_job, check_slurm_gpu_memory_stress, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -28,10 +30,12 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(257) +@pytest.mark.order(41401) def test_slurm_gpu_inventory(host): """Verify NVIDIA runtime state on scheduler-declared GPU nodes.""" - verify_pxeboot(host, "slurm_gpu_inventory", check_slurm_gpu_inventory) + tc = TC["slurm_gpu_inventory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_gpu_inventory) @pytest.mark.buildstream @@ -39,10 +43,12 @@ def test_slurm_gpu_inventory(host): @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.functional -@pytest.mark.order(258) +@pytest.mark.order(41402) def test_slurm_gpu_job(host): """Allocate a GPU through Slurm and query the device.""" - verify_pxeboot(host, "slurm_gpu_job", check_slurm_gpu_job) + tc = TC["slurm_gpu_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_gpu_job) @pytest.mark.buildstream @@ -50,7 +56,9 @@ def test_slurm_gpu_job(host): @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.functional -@pytest.mark.order(259) +@pytest.mark.order(41403) def test_slurm_gpu_memory_stress(host): """Compile and run a bounded GPU memory workload through Slurm.""" - verify_pxeboot(host, "slurm_gpu_memory", check_slurm_gpu_memory_stress) + tc = TC["slurm_gpu_memory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_gpu_memory_stress) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py index affd69de51..061af00cef 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/conftest.py @@ -41,13 +41,6 @@ def pytest_addoption(parser): ) -def pytest_configure(config): - """Register custom markers for the HPC benchmarks suite.""" - config.addinivalue_line( - "markers", "benchmark: HPC benchmark staging and execution tests" - ) - - def _destructive_authorized(config) -> bool: """Return True when destructive HPC benchmarks tests may run.""" if config.getoption("--run-destructive", default=False): diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py index e7ab8af5af..a9eb3d28b4 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_contracts.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_container_first_guidance, check_hpc_benchmarks_container_image_list, check_hpc_benchmarks_nfs_accessibility, @@ -24,59 +25,59 @@ check_hpc_benchmarks_platform_script, check_hpc_benchmarks_source_only_delivery, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(335) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42012) def test_hpc_benchmarks_platform_script(host): """TC-06a: Verify omnia_platform.sh deployment and platform detection.""" - verify_pxeboot( - host, - "hpc_benchmarks_platform_script", - check_hpc_benchmarks_platform_script, - ) + tc = TC["hpc_benchmarks_platform_script"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_platform_script) @pytest.mark.sanity -@pytest.mark.order(336) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42013) def test_hpc_benchmarks_container_image_list(host): """TC-06b: Verify container_image.list deployment and content validation.""" - verify_pxeboot( - host, - "hpc_benchmarks_container_image_list", - check_hpc_benchmarks_container_image_list, - ) + tc = TC["hpc_benchmarks_container_image_list"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_container_image_list) @pytest.mark.sanity -@pytest.mark.order(337) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42014) def test_hpc_benchmarks_platform_directory_structure(host): """TC-06c: Verify platform-specific directory structure exists per architecture.""" - verify_pxeboot( - host, - "hpc_benchmarks_platform_directory_structure", - check_hpc_benchmarks_platform_directory_structure, - ) + tc = TC["hpc_benchmarks_platform_directory_structure"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_platform_directory_structure) @pytest.mark.sanity -@pytest.mark.order(338) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42015) def test_hpc_benchmarks_offline_package_copy(host): """TC-06d: Verify offline packages are copied to slurm_config_path/packages/{arch}/.""" - verify_pxeboot( - host, - "hpc_benchmarks_offline_package_copy", - check_hpc_benchmarks_offline_package_copy, - ) + tc = TC["hpc_benchmarks_offline_package_copy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_offline_package_copy) @pytest.mark.sanity -@pytest.mark.order(342) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42016) def test_hpc_benchmarks_container_first_guidance(host): """TC-06: Verify pull_benchmarks.sh and benchmark_tools.list are deployed. @@ -84,30 +85,30 @@ def test_hpc_benchmarks_container_first_guidance(host): deployment contract actually enforced by hpc_tools.yml. See the check function's docstring for the rationale. """ - verify_pxeboot( - host, - "hpc_benchmarks_container_first_guidance", - check_hpc_benchmarks_container_first_guidance, - ) + tc = TC["hpc_benchmarks_container_first_guidance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_container_first_guidance) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(343) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42017) def test_hpc_benchmarks_source_only_delivery(host): """TC-07: Verify no compile/build commands are staged with the artifacts.""" - verify_pxeboot( - host, - "hpc_benchmarks_source_only_delivery", - check_hpc_benchmarks_source_only_delivery, - ) + tc = TC["hpc_benchmarks_source_only_delivery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_source_only_delivery) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(344) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42018) def test_hpc_benchmarks_nfs_accessibility(host): """TC-10: Verify /hpc_tools NFS is mounted and readable on compute nodes.""" - verify_pxeboot( - host, "hpc_benchmarks_nfs_accessibility", check_hpc_benchmarks_nfs_accessibility - ) + tc = TC["hpc_benchmarks_nfs_accessibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_nfs_accessibility) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py index 72446aa49f..b253e147e6 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_destructive.py @@ -31,6 +31,7 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_airgapped_staging, check_hpc_benchmarks_container_image_unaffected, check_hpc_benchmarks_cuda_flow_unaffected, @@ -39,87 +40,85 @@ check_hpc_benchmarks_openmpi_unaffected, check_hpc_benchmarks_staging_idempotency, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.destructive] - - # --- Invariance checks (staging run + snapshot diff) --- @pytest.mark.functional -@pytest.mark.order(347) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42021) def test_hpc_benchmarks_cuda_flow_unaffected(host): """TC-14: Verify /hpc_tools/cuda is unchanged after benchmark staging.""" - verify_pxeboot( - host, - "hpc_benchmarks_cuda_flow_unaffected", - check_hpc_benchmarks_cuda_flow_unaffected, - ) + tc = TC["hpc_benchmarks_cuda_flow_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_cuda_flow_unaffected) @pytest.mark.functional -@pytest.mark.order(348) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42022) def test_hpc_benchmarks_nvhpc_flow_unaffected(host): """TC-15: Verify /hpc_tools/nvidia_sdk is unchanged after staging.""" - verify_pxeboot( - host, - "hpc_benchmarks_nvhpc_flow_unaffected", - check_hpc_benchmarks_nvhpc_flow_unaffected, - ) + tc = TC["hpc_benchmarks_nvhpc_flow_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_nvhpc_flow_unaffected) @pytest.mark.functional -@pytest.mark.order(349) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42023) def test_hpc_benchmarks_container_image_unaffected(host): """TC-16: Verify /hpc_tools/container_images is unchanged after staging.""" - verify_pxeboot( - host, - "hpc_benchmarks_container_image_unaffected", - check_hpc_benchmarks_container_image_unaffected, - ) + tc = TC["hpc_benchmarks_container_image_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_container_image_unaffected) @pytest.mark.functional -@pytest.mark.order(350) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42024) def test_hpc_benchmarks_openmpi_unaffected(host): """TC-17: Verify OpenMPI/UCX discovery is stable across a staging run.""" - verify_pxeboot( - host, - "hpc_benchmarks_openmpi_unaffected", - check_hpc_benchmarks_openmpi_unaffected, - ) + tc = TC["hpc_benchmarks_openmpi_unaffected"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_openmpi_unaffected) # --- Direct staging execution checks --- -@pytest.mark.order(351) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42025) def test_hpc_benchmarks_airgapped_staging(host): """TC-11: Verify staging succeeds while external egress is unavailable.""" - verify_pxeboot( - host, - "hpc_benchmarks_airgapped_staging", - check_hpc_benchmarks_airgapped_staging, - ) + tc = TC["hpc_benchmarks_airgapped_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_airgapped_staging) -@pytest.mark.order(352) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42026) def test_hpc_benchmarks_existing_dirs_preserved(host): """TC-18: Verify pre-existing /hpc_tools subdirs survive a staging run.""" - verify_pxeboot( - host, - "hpc_benchmarks_existing_dirs_preserved", - check_hpc_benchmarks_existing_dirs_preserved, - ) + tc = TC["hpc_benchmarks_existing_dirs_preserved"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_existing_dirs_preserved) -@pytest.mark.order(353) +@pytest.mark.slurm +@pytest.mark.destructive +@pytest.mark.order(42027) def test_hpc_benchmarks_staging_idempotency(host): """TC-19: Verify a second staging run keeps the /hpc_tools snapshot stable.""" - verify_pxeboot( - host, - "hpc_benchmarks_staging_idempotency", - check_hpc_benchmarks_staging_idempotency, - ) + tc = TC["hpc_benchmarks_staging_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_staging_idempotency) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py index 2171975967..71736170db 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_pipeline.py @@ -21,44 +21,47 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_concurrent_staging, check_hpc_benchmarks_e2e_provisioning, check_hpc_benchmarks_staging_fingerprint_idempotency, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.functional @pytest.mark.benchmark -@pytest.mark.order(296) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42002) def test_hpc_benchmarks_staging_fingerprint_idempotency(host): - """V096: Pull tools, validate tarballs, repeat — SHA-256, size, mtime must hold.""" + """V226: Pull tools, validate tarballs, repeat — SHA-256, size, mtime must hold.""" + tc = TC["hpc_benchmarks_staging_fingerprint_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) verify_pxeboot( - host, - "hpc_benchmarks_staging_fingerprint_idempotency", - check_hpc_benchmarks_staging_fingerprint_idempotency, + test_log, tc, host, check_hpc_benchmarks_staging_fingerprint_idempotency ) @pytest.mark.functional @pytest.mark.benchmark -@pytest.mark.order(297) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42003) def test_hpc_benchmarks_concurrent_staging(host): - """V097: Start two same-platform pulls together; both see identical archives.""" - verify_pxeboot( - host, - "hpc_benchmarks_concurrent_staging", - check_hpc_benchmarks_concurrent_staging, - ) + """V227: Start two same-platform pulls together; both see identical archives.""" + tc = TC["hpc_benchmarks_concurrent_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_concurrent_staging) @pytest.mark.sanity -@pytest.mark.order(345) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42019) def test_hpc_benchmarks_e2e_provisioning(host): """TC-09: Verify the end-to-end benchmark provisioning pipeline.""" - verify_pxeboot( - host, "hpc_benchmarks_e2e_provisioning", check_hpc_benchmarks_e2e_provisioning - ) + tc = TC["hpc_benchmarks_e2e_provisioning"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_e2e_provisioning) diff --git a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py index fce2568dd5..77c20cd048 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py +++ b/test/orchestrator/fvt/pxeboot/slurm_hpc_benchmarks/test_staging.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_hpc_benchmarks_artifact_copy, check_hpc_benchmarks_cleanup_staging, check_hpc_benchmarks_json_declaration, @@ -27,26 +28,28 @@ check_hpc_benchmarks_run_staging, check_hpc_benchmarks_tools_dir_creation, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [pytest.mark.slurm, pytest.mark.non_disruptive] - - @pytest.mark.sanity @pytest.mark.functional @pytest.mark.benchmark -@pytest.mark.order(295) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42001) def test_hpc_benchmarks_prerequisites(host): - """V095: Verify script presence, platform helper, tool list, mount (NFS export informational).""" - verify_pxeboot( - host, "hpc_benchmarks_prerequisites", check_hpc_benchmarks_prerequisites - ) + """V225: Verify script presence, platform helper, tool list, mount (NFS export informational).""" + tc = TC["hpc_benchmarks_prerequisites"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_prerequisites) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(330) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42004) def test_hpc_benchmarks_run_staging(host): """TC-00: Execute pull_benchmarks.sh to populate staged artifacts. @@ -54,94 +57,102 @@ def test_hpc_benchmarks_run_staging(host): not skip with "staging not yet executed". Skips gracefully when pull_benchmarks.sh is not deployed. """ - verify_pxeboot( - host, "hpc_benchmarks_run_staging", check_hpc_benchmarks_run_staging - ) + tc = TC["hpc_benchmarks_run_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_run_staging) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(331) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42005) def test_hpc_benchmarks_json_declaration(host): """TC-01: Verify benchmark_tools.list is deployed and non-empty per arch.""" - verify_pxeboot( - host, "hpc_benchmarks_json_declaration", check_hpc_benchmarks_json_declaration - ) + tc = TC["hpc_benchmarks_json_declaration"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_json_declaration) @pytest.mark.sanity -@pytest.mark.order(332) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42006) def test_hpc_benchmarks_local_repo_sync(host): """TC-02: Verify each declared tool has files under the Pulp offline URL.""" - verify_pxeboot( - host, "hpc_benchmarks_local_repo_sync", check_hpc_benchmarks_local_repo_sync - ) + tc = TC["hpc_benchmarks_local_repo_sync"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_local_repo_sync) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(333) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42007) def test_hpc_benchmarks_tools_dir_creation(host): """TC-03: Verify /hpc_tools directory layout and 0755 permissions.""" - verify_pxeboot( - host, - "hpc_benchmarks_tools_dir_creation", - check_hpc_benchmarks_tools_dir_creation, - ) + tc = TC["hpc_benchmarks_tools_dir_creation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_tools_dir_creation) @pytest.mark.sanity -@pytest.mark.order(334) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42008) def test_hpc_benchmarks_artifact_copy(host): """TC-04: Verify declared benchmark artifacts are staged per tool.""" - verify_pxeboot( - host, "hpc_benchmarks_artifact_copy", check_hpc_benchmarks_artifact_copy - ) + tc = TC["hpc_benchmarks_artifact_copy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_artifact_copy) @pytest.mark.sanity -@pytest.mark.order(337) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42009) def test_hpc_benchmarks_msr_safe_arch_boundary(host): """TC-05: Verify msr-safe is staged only for x86_64 nodes.""" - verify_pxeboot( - host, - "hpc_benchmarks_msr_safe_arch_boundary", - check_hpc_benchmarks_msr_safe_arch_boundary, - ) + tc = TC["hpc_benchmarks_msr_safe_arch_boundary"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_msr_safe_arch_boundary) @pytest.mark.sanity -@pytest.mark.order(338) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42010) def test_hpc_benchmarks_post_staging_validation(host): """TC-12: Verify post-staging validation of benchmark tool directories.""" - verify_pxeboot( - host, - "hpc_benchmarks_post_staging_validation", - check_hpc_benchmarks_post_staging_validation, - ) + tc = TC["hpc_benchmarks_post_staging_validation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_post_staging_validation) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(339) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42011) def test_hpc_benchmarks_rhel_compatibility(host): """TC-13: Verify every compute node runs the targeted RHEL major.""" - verify_pxeboot( - host, "hpc_benchmarks_rhel_compatibility", check_hpc_benchmarks_rhel_compatibility - ) + tc = TC["hpc_benchmarks_rhel_compatibility"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_rhel_compatibility) @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(346) +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(42020) def test_hpc_benchmarks_cleanup_staging(host): """TC-00b: Remove staged benchmark tool directories after verification. Controlled by ``cleanup_benchmark_tools`` in ``test_config.yml``. When false, staged artifacts are retained for manual inspection. """ - verify_pxeboot( - host, - "hpc_benchmarks_cleanup_staging", - check_hpc_benchmarks_cleanup_staging, - ) + tc = TC["hpc_benchmarks_cleanup_staging"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_hpc_benchmarks_cleanup_staging) diff --git a/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py b/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py index c6833e5866..643e47d9aa 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py +++ b/test/orchestrator/fvt/pxeboot/slurm_infiniband/test_infiniband.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_slurm_infiniband_configuration, check_slurm_infiniband_connectivity, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -27,17 +29,21 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(263) +@pytest.mark.order(41801) def test_slurm_infiniband_configuration(host): """Verify mapped IB interface, address, prefix, link, MTU, and OFED.""" - verify_pxeboot(host, "slurm_ib_configuration", check_slurm_infiniband_configuration) + tc = TC["slurm_ib_configuration"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_infiniband_configuration) @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(264) +@pytest.mark.order(41802) def test_slurm_infiniband_connectivity(host): """Verify every mapped IB endpoint can reach every mapped peer.""" - verify_pxeboot(host, "slurm_ib_connectivity", check_slurm_infiniband_connectivity) + tc = TC["slurm_ib_connectivity"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_infiniband_connectivity) diff --git a/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py b/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py index 7434de7361..645e13535d 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py +++ b/test/orchestrator/fvt/pxeboot/slurm_jobs/test_root_jobs.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_compiler_node_jobs, check_slurm_concurrent_jobs, check_slurm_control_node_jobs, @@ -24,6 +25,7 @@ check_slurm_job_queueing, check_slurm_login_node_jobs, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -33,10 +35,12 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(235) +@pytest.mark.order(41201) def test_slurm_control_node_jobs(host): """Run one targeted job per compute from every Slurm control node.""" - verify_pxeboot(host, "slurm_basic_jobs", check_slurm_control_node_jobs) + tc = TC["slurm_basic_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_node_jobs) @pytest.mark.functional @@ -44,10 +48,12 @@ def test_slurm_control_node_jobs(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(236) +@pytest.mark.order(41202) def test_slurm_login_node_jobs(host): """Run one targeted job per compute from every mapped login node.""" - verify_pxeboot(host, "slurm_login_jobs", check_slurm_login_node_jobs) + tc = TC["slurm_login_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_node_jobs) @pytest.mark.functional @@ -55,10 +61,12 @@ def test_slurm_login_node_jobs(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(237) +@pytest.mark.order(41203) def test_slurm_compiler_node_jobs(host): """Run one targeted job per compute from every login compiler node.""" - verify_pxeboot(host, "slurm_compiler_jobs", check_slurm_compiler_node_jobs) + tc = TC["slurm_compiler_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_node_jobs) @pytest.mark.functional @@ -66,24 +74,25 @@ def test_slurm_compiler_node_jobs(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(238) +@pytest.mark.order(41204) def test_slurm_concurrent_jobs(host): """Submit concurrent jobs and verify final accounting state.""" - verify_pxeboot(host, "slurm_concurrent_jobs", check_slurm_concurrent_jobs) + tc = TC["slurm_concurrent_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_concurrent_jobs) @pytest.mark.functional @pytest.mark.negative @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(239) +@pytest.mark.order(41205) def test_slurm_insufficient_resources(host): """Verify an impossible immediate allocation is rejected.""" - verify_pxeboot( - host, "slurm_insufficient_resources", check_slurm_insufficient_resources - ) + tc = TC["slurm_insufficient_resources"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_insufficient_resources) @pytest.mark.functional @@ -91,18 +100,21 @@ def test_slurm_insufficient_resources(host): @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(240) +@pytest.mark.order(41206) def test_slurm_job_queueing(host): """Saturate idle computes and verify one follower queues then completes.""" - verify_pxeboot(host, "slurm_job_queueing", check_slurm_job_queueing) + tc = TC["slurm_job_queueing"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_job_queueing) -@pytest.mark.disruptive @pytest.mark.buildstream @pytest.mark.sanity @pytest.mark.scheduler_state @pytest.mark.slurm -@pytest.mark.order(241) +@pytest.mark.order(41207) def test_slurm_drain_queue_recovery(host): """Drain one compute node, verify queuing, and restore it.""" - verify_pxeboot(host, "slurm_drain_queue", check_slurm_drain_queue_recovery) + tc = TC["slurm_drain_queue"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_drain_queue_recovery) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py index 874407f297..2c31e08b31 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py +++ b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_authentication.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_compiler_ldap_authentication, check_slurm_compiler_ldap_invalid_password, check_slurm_control_ldap_authentication, @@ -26,6 +27,7 @@ check_slurm_pam_no_job_access, check_slurm_pam_policy, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -35,39 +37,36 @@ @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(242) +@pytest.mark.order(41303) def test_slurm_pam_policy(host): """Verify SSHD, the PAM module, and pam_slurm_adopt account policy.""" - verify_pxeboot(host, "slurm_pam", check_slurm_pam_policy) + tc = TC["slurm_pam"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_pam_policy) @pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(243) +@pytest.mark.order(41304) def test_slurm_control_ldap_authentication(host): """Verify a valid LDAP password on the Slurm control node.""" - verify_pxeboot( - host, - "slurm_control_ldap_auth", - check_slurm_control_ldap_authentication, - ) + tc = TC["slurm_control_ldap_auth"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_ldap_authentication) -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(244) +@pytest.mark.order(41305) def test_slurm_control_ldap_invalid_password(host): """Verify an invalid LDAP password is rejected on the control node.""" - verify_pxeboot( - host, - "slurm_control_ldap_invalid_password", - check_slurm_control_ldap_invalid_password, - ) + tc = TC["slurm_control_ldap_invalid_password"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_ldap_invalid_password) @pytest.mark.buildstream @@ -75,30 +74,25 @@ def test_slurm_control_ldap_invalid_password(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(245) +@pytest.mark.order(41306) def test_slurm_login_ldap_authentication(host): """Verify a valid LDAP password on every mapped login node.""" - verify_pxeboot( - host, - "slurm_login_ldap_auth", - check_slurm_login_ldap_authentication, - ) + tc = TC["slurm_login_ldap_auth"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_ldap_authentication) @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(246) +@pytest.mark.order(41307) def test_slurm_login_ldap_invalid_password(host): """Verify an invalid LDAP password is rejected on every login node.""" - verify_pxeboot( - host, - "slurm_login_ldap_invalid_password", - check_slurm_login_ldap_invalid_password, - ) + tc = TC["slurm_login_ldap_invalid_password"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_ldap_invalid_password) @pytest.mark.buildstream @@ -106,50 +100,47 @@ def test_slurm_login_ldap_invalid_password(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(247) +@pytest.mark.order(41308) def test_slurm_compiler_ldap_authentication(host): """Verify a valid LDAP password on every login-compiler node.""" - verify_pxeboot( - host, - "slurm_compiler_ldap_auth", - check_slurm_compiler_ldap_authentication, - ) + tc = TC["slurm_compiler_ldap_auth"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_ldap_authentication) @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(248) +@pytest.mark.order(41309) def test_slurm_compiler_ldap_invalid_password(host): """Verify invalid LDAP passwords are rejected on login-compiler nodes.""" - verify_pxeboot( - host, - "slurm_compiler_ldap_invalid_password", - check_slurm_compiler_ldap_invalid_password, - ) + tc = TC["slurm_compiler_ldap_invalid_password"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_ldap_invalid_password) -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(249) +@pytest.mark.order(41310) def test_slurm_pam_no_job_access(host): """Verify LDAP compute login is denied without an active job.""" - verify_pxeboot(host, "slurm_pam_no_job", check_slurm_pam_no_job_access) + tc = TC["slurm_pam_no_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_pam_no_job_access) @pytest.mark.buildstream -@pytest.mark.sanity @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.negative -@pytest.mark.order(250) +@pytest.mark.order(41311) def test_slurm_invalid_ldap_identity(host): """Verify a generated missing directory identity is rejected.""" - verify_pxeboot(host, "slurm_invalid_ldap", check_slurm_invalid_ldap_identity) + tc = TC["slurm_invalid_ldap"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_invalid_ldap_identity) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_external_ldap.py b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_external_ldap.py new file mode 100644 index 0000000000..7c775a88a1 --- /dev/null +++ b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_external_ldap.py @@ -0,0 +1,55 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""External LDAP proxy setup that LDAP login checks depend on. + +These cases run first in the slurm_ldap suite. Prepare only starts the local +omnia_auth container, so pointing it at the external directory belongs here, +immediately before the LDAP authentication and job cases. +""" + +import pytest +from library.functions import ( + TestLogger, + check_external_ldap_backend, + reconcile_external_ldap_proxy, +) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot + + +@pytest.mark.sanity +@pytest.mark.openldap +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41301) +def test_external_ldap_proxy(host): + """Reconcile and verify the explicitly enabled LDAP meta-proxy.""" + tc = TC["external_ldap_proxy"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, reconcile_external_ldap_proxy) + + +@pytest.mark.functional +@pytest.mark.sanity +@pytest.mark.openldap +@pytest.mark.slurm +@pytest.mark.non_disruptive +@pytest.mark.order(41302) +def test_external_ldap_backend(host): + """Verify external LDAP reachability from the omnia_auth container.""" + tc = TC["external_ldap_backend"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_external_ldap_backend) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py index 9dac46530c..84a9db3601 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py +++ b/test/orchestrator/fvt/pxeboot/slurm_ldap/test_jobs_and_pam.py @@ -16,6 +16,7 @@ import pytest from library.functions import ( + TestLogger, check_slurm_compiler_ldap_jobs, check_slurm_compiler_pam_job_access, check_slurm_control_ldap_jobs, @@ -23,6 +24,7 @@ check_slurm_login_ldap_jobs, check_slurm_login_pam_job_access, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -32,10 +34,12 @@ @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(251) +@pytest.mark.order(41312) def test_slurm_control_ldap_jobs(host): """Submit and complete an LDAP-owned job from the control node.""" - verify_pxeboot(host, "slurm_control_ldap_jobs", check_slurm_control_ldap_jobs) + tc = TC["slurm_control_ldap_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_ldap_jobs) @pytest.mark.functional @@ -43,14 +47,12 @@ def test_slurm_control_ldap_jobs(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(252) +@pytest.mark.order(41313) def test_slurm_control_pam_job_access(host): """Verify control-submitted PAM access during and after a job.""" - verify_pxeboot( - host, - "slurm_control_pam_job_access", - check_slurm_control_pam_job_access, - ) + tc = TC["slurm_control_pam_job_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_control_pam_job_access) @pytest.mark.functional @@ -59,10 +61,12 @@ def test_slurm_control_pam_job_access(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(253) +@pytest.mark.order(41314) def test_slurm_login_ldap_jobs(host): """Submit and complete an LDAP-owned job from every login node.""" - verify_pxeboot(host, "slurm_login_ldap_jobs", check_slurm_login_ldap_jobs) + tc = TC["slurm_login_ldap_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_ldap_jobs) @pytest.mark.functional @@ -71,14 +75,12 @@ def test_slurm_login_ldap_jobs(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(254) +@pytest.mark.order(41315) def test_slurm_login_pam_job_access(host): """Verify login-node PAM access during and after a job.""" - verify_pxeboot( - host, - "slurm_login_pam_job_access", - check_slurm_login_pam_job_access, - ) + tc = TC["slurm_login_pam_job_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_login_pam_job_access) @pytest.mark.functional @@ -87,10 +89,12 @@ def test_slurm_login_pam_job_access(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(255) +@pytest.mark.order(41316) def test_slurm_compiler_ldap_jobs(host): """Submit an LDAP-owned job from every login-compiler node.""" - verify_pxeboot(host, "slurm_compiler_ldap_jobs", check_slurm_compiler_ldap_jobs) + tc = TC["slurm_compiler_ldap_jobs"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_ldap_jobs) @pytest.mark.functional @@ -99,11 +103,9 @@ def test_slurm_compiler_ldap_jobs(host): @pytest.mark.openldap @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(256) +@pytest.mark.order(41317) def test_slurm_compiler_pam_job_access(host): """Verify login-compiler PAM access during and after a job.""" - verify_pxeboot( - host, - "slurm_compiler_pam_job_access", - check_slurm_compiler_pam_job_access, - ) + tc = TC["slurm_compiler_pam_job_access"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_compiler_pam_job_access) diff --git a/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py b/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py index c348a4c57c..a9705a8854 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py +++ b/test/orchestrator/fvt/pxeboot/slurm_lifecycle/test_slurm_lifecycle.py @@ -23,26 +23,31 @@ import pytest from library.functions import ( + TestLogger, check_slurm_node_add, check_slurm_node_remove, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -pytestmark = [ - pytest.mark.slurm, - pytest.mark.functional, - pytest.mark.disruptive, -] - - -@pytest.mark.order(290) +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.node_lifecycle +@pytest.mark.order(42401) def test_slurm_node_remove(host): """Remove Slurm compute node(s) from PXE mapping, provision, verify.""" - verify_pxeboot(host, "slurm_node_remove", check_slurm_node_remove) + tc = TC["slurm_node_remove"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_node_remove) -@pytest.mark.order(291) +@pytest.mark.slurm +@pytest.mark.functional +@pytest.mark.node_lifecycle +@pytest.mark.order(42402) def test_slurm_node_add(host): """Restore removed node(s) to PXE mapping, provision, verify re-addition.""" - verify_pxeboot(host, "slurm_node_add", check_slurm_node_add) + tc = TC["slurm_node_add"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_node_add) diff --git a/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py b/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py index fa55a5c484..822e12cde7 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py +++ b/test/orchestrator/fvt/pxeboot/slurm_openmpi/test_openmpi.py @@ -16,9 +16,11 @@ import pytest from library.functions import ( + TestLogger, check_slurm_openmpi_installation, check_slurm_openmpi_job, ) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -27,10 +29,12 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(260) +@pytest.mark.order(41601) def test_slurm_openmpi_installation(host): """Verify OpenMPI discovery and version on every compute node.""" - verify_pxeboot(host, "slurm_openmpi_installation", check_slurm_openmpi_installation) + tc = TC["slurm_openmpi_installation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_openmpi_installation) @pytest.mark.buildstream @@ -38,7 +42,9 @@ def test_slurm_openmpi_installation(host): @pytest.mark.slurm @pytest.mark.non_disruptive @pytest.mark.functional -@pytest.mark.order(261) +@pytest.mark.order(41602) def test_slurm_openmpi_job(host): """Run an OpenMPI-backed job when OpenMPI is configured.""" - verify_pxeboot(host, "slurm_openmpi_job", check_slurm_openmpi_job) + tc = TC["slurm_openmpi_job"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_openmpi_job) diff --git a/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py b/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py index 79ca8ed3e8..9f1bf98a6d 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py +++ b/test/orchestrator/fvt/pxeboot/slurm_recovery/test_reboot.py @@ -15,16 +15,21 @@ """Explicitly authorized full Slurm cluster reboot recovery.""" import pytest -from library.functions import check_slurm_cluster_recovery +from library.functions import ( + TestLogger, + check_slurm_cluster_recovery, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot -@pytest.mark.disruptive @pytest.mark.functional @pytest.mark.reboot @pytest.mark.slurm -@pytest.mark.order(265) +@pytest.mark.order(42301) def test_slurm_cluster_recovery(host): """Reboot mapped Slurm nodes and verify scheduler and workload recovery.""" - verify_pxeboot(host, "slurm_recovery", check_slurm_cluster_recovery) + tc = TC["slurm_recovery"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_cluster_recovery) diff --git a/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py b/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py index 52734597aa..3c6ce002f7 100644 --- a/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py +++ b/test/orchestrator/fvt/pxeboot/slurm_ucx/test_ucx.py @@ -15,7 +15,11 @@ """Slurm UCX transport contract.""" import pytest -from library.functions import check_slurm_ucx_transport +from library.functions import ( + TestLogger, + check_slurm_ucx_transport, +) +from library.vars import TEST_CASES as TC from fvt.result import verify_pxeboot @@ -24,7 +28,9 @@ @pytest.mark.sanity @pytest.mark.slurm @pytest.mark.non_disruptive -@pytest.mark.order(262) +@pytest.mark.order(41701) def test_slurm_ucx_transport(host): """Verify UCX exposes an InfiniBand-capable transport.""" - verify_pxeboot(host, "slurm_ucx_transport", check_slurm_ucx_transport) + tc = TC["slurm_ucx_transport"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_slurm_ucx_transport) diff --git a/test/orchestrator/fvt/pxeboot/test_playbook.py b/test/orchestrator/fvt/pxeboot/test_playbook.py index 838111a4f4..5527ccfc78 100644 --- a/test/orchestrator/fvt/pxeboot/test_playbook.py +++ b/test/orchestrator/fvt/pxeboot/test_playbook.py @@ -24,7 +24,7 @@ @pytest.mark.deploy @pytest.mark.buildstream @pytest.mark.sanity -@pytest.mark.order(200) +@pytest.mark.order(40000) def test_deploy_pxeboot(host): """Run ``orchestrator.yml --tags pxeboot``.""" tc = TC["deploy_pxeboot"] diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py index f706fabc9d..776cd1ed8d 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_installation.py @@ -16,24 +16,31 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( + TestLogger, check_vast_vastnfs_installation, check_vast_vastnfs_rpm_and_module, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(700) @pytest.mark.vast_installation +@pytest.mark.order(42101) def test_vast_vastnfs_installation(host): """Verify vastnfs-ctl status on compute nodes.""" - verify_pxeboot(host, "vast_vastnfs_installation", check_vast_vastnfs_installation) + tc = TC["vast_vastnfs_installation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_vastnfs_installation) @pytest.mark.sanity -@pytest.mark.order(709) @pytest.mark.vast_installation +@pytest.mark.order(42110) def test_vast_vastnfs_rpm_and_module(host): """Verify vastnfs RPM, kernel module, and service.""" - verify_pxeboot(host, "vast_vastnfs_rpm_and_module", check_vast_vastnfs_rpm_and_module) + tc = TC["vast_vastnfs_rpm_and_module"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_vastnfs_rpm_and_module) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py index 023781ba9f..748010113e 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_mounts.py @@ -16,42 +16,53 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( - check_vast_mount_points, - check_vast_mount_options, + TestLogger, check_vast_fstab_entries, + check_vast_mount_options, + check_vast_mount_points, check_vast_rdma_mount, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(701) @pytest.mark.vast_mounts +@pytest.mark.order(42102) def test_vast_mount_points(host): """Verify /scratch, /home, /apps, /projects directories.""" - verify_pxeboot(host, "vast_mount_points", check_vast_mount_points) + tc = TC["vast_mount_points"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_mount_points) @pytest.mark.sanity -@pytest.mark.order(703) @pytest.mark.vast_mounts +@pytest.mark.order(42104) def test_vast_mount_options(host): """Verify proto=rdma and port=20049 in /proc/mounts.""" - verify_pxeboot(host, "vast_mount_options", check_vast_mount_options) + tc = TC["vast_mount_options"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_mount_options) @pytest.mark.sanity -@pytest.mark.order(710) @pytest.mark.vast_mounts +@pytest.mark.order(42111) def test_vast_fstab_entries(host): """Verify /etc/fstab entries with proto=rdma.""" - verify_pxeboot(host, "vast_fstab_entries", check_vast_fstab_entries) + tc = TC["vast_fstab_entries"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_fstab_entries) @pytest.mark.sanity -@pytest.mark.order(711) @pytest.mark.vast_mounts +@pytest.mark.order(42112) def test_vast_rdma_mount(host): """Verify RDMA transport and 1 GB I/O checksum.""" - verify_pxeboot(host, "vast_rdma_mount", check_vast_rdma_mount) + tc = TC["vast_rdma_mount"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_rdma_mount) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py index 57aec1d8dd..5b6c6e7d3f 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_persistence.py @@ -16,16 +16,20 @@ import pytest +from library.functions import ( + TestLogger, + check_vast_slurm_logs_persistence, +) +from library.vars import TEST_CASES as TC + from fvt.result import verify_pxeboot -from library.functions import check_vast_slurm_logs_persistence @pytest.mark.sanity -@pytest.mark.order(713) @pytest.mark.vast_persistence +@pytest.mark.order(42114) def test_vast_slurm_logs_persistence(host): """Verify Slurm logs on persistent storage; sacct accessible.""" - verify_pxeboot( - host, "vast_slurm_logs_persistence", - check_vast_slurm_logs_persistence, - ) + tc = TC["vast_slurm_logs_persistence"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_slurm_logs_persistence) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py index cafd2504a9..e51adff42d 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_scratch.py @@ -16,63 +16,64 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( - check_vast_scratch_hostname_isolation, + TestLogger, + check_vast_ldapuser_permissions, check_vast_ldapuser_scratch_directory, check_vast_ldapuser_subdirectories, - check_vast_ldapuser_permissions, + check_vast_scratch_hostname_isolation, check_vast_scratch_isolation, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(702) @pytest.mark.vast_scratch +@pytest.mark.order(42103) def test_vast_scratch_hostname_isolation(host): """Verify /scratch// per node; file isolation.""" - verify_pxeboot( - host, "vast_scratch_hostname_isolation", - check_vast_scratch_hostname_isolation, - ) + tc = TC["vast_scratch_hostname_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_scratch_hostname_isolation) @pytest.mark.sanity -@pytest.mark.order(704) @pytest.mark.vast_scratch +@pytest.mark.order(42105) def test_vast_ldapuser_scratch_directory(host): """Verify /scratch// on login_compiler nodes.""" - verify_pxeboot( - host, "vast_ldapuser_scratch_directory", - check_vast_ldapuser_scratch_directory, - ) + tc = TC["vast_ldapuser_scratch_directory"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_ldapuser_scratch_directory) @pytest.mark.sanity -@pytest.mark.order(705) @pytest.mark.vast_scratch +@pytest.mark.order(42106) def test_vast_ldapuser_subdirectories(host): """Verify data/, jobs/, results/, tmp/ subdirectories.""" - verify_pxeboot( - host, "vast_ldapuser_subdirectories", - check_vast_ldapuser_subdirectories, - ) + tc = TC["vast_ldapuser_subdirectories"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_ldapuser_subdirectories) @pytest.mark.sanity -@pytest.mark.order(706) @pytest.mark.vast_scratch +@pytest.mark.order(42107) def test_vast_ldapuser_permissions(host): """Verify cross-user permission isolation.""" - verify_pxeboot( - host, "vast_ldapuser_permissions", - check_vast_ldapuser_permissions, - ) + tc = TC["vast_ldapuser_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_ldapuser_permissions) @pytest.mark.sanity -@pytest.mark.order(707) @pytest.mark.vast_scratch +@pytest.mark.order(42108) def test_vast_scratch_isolation(host): """Verify file isolation between scratch subdirectories.""" - verify_pxeboot(host, "vast_scratch_isolation", check_vast_scratch_isolation) + tc = TC["vast_scratch_isolation"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_scratch_isolation) diff --git a/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py b/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py index c2d216d1d3..d8ac1e4a41 100644 --- a/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py +++ b/test/orchestrator/fvt/pxeboot/vast_storage/test_targeting.py @@ -16,51 +16,64 @@ import pytest -from fvt.result import verify_pxeboot from library.functions import ( - check_vast_control_node_no_vast, - check_vast_qss_mounts, - check_vast_control_node_mounts, + TestLogger, check_vast_compute_node_mounts, + check_vast_control_node_mounts, + check_vast_control_node_no_vast, check_vast_login_node_mounts, + check_vast_qss_mounts, ) +from library.vars import TEST_CASES as TC + +from fvt.result import verify_pxeboot @pytest.mark.sanity -@pytest.mark.order(708) @pytest.mark.vast_targeting +@pytest.mark.order(42109) def test_vast_control_node_no_vast(host): """Verify Slurm control node has no VAST storage.""" - verify_pxeboot(host, "vast_control_node_no_vast", check_vast_control_node_no_vast) + tc = TC["vast_control_node_no_vast"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_control_node_no_vast) @pytest.mark.sanity -@pytest.mark.order(712) @pytest.mark.vast_targeting +@pytest.mark.order(42113) def test_vast_qss_mounts(host): """Verify VAST on compute/login only; controller has none.""" - verify_pxeboot(host, "vast_qss_mounts", check_vast_qss_mounts) + tc = TC["vast_qss_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_qss_mounts) @pytest.mark.sanity -@pytest.mark.order(714) @pytest.mark.vast_targeting +@pytest.mark.order(42115) def test_vast_control_node_mounts(host): """Verify controller mount table (NFS/PV only, no VAST).""" - verify_pxeboot(host, "vast_control_node_mounts", check_vast_control_node_mounts) + tc = TC["vast_control_node_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_control_node_mounts) @pytest.mark.sanity -@pytest.mark.order(715) @pytest.mark.vast_targeting +@pytest.mark.order(42116) def test_vast_compute_node_mounts(host): """Verify compute node mount table (NFS + VAST).""" - verify_pxeboot(host, "vast_compute_node_mounts", check_vast_compute_node_mounts) + tc = TC["vast_compute_node_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_compute_node_mounts) @pytest.mark.sanity -@pytest.mark.order(716) @pytest.mark.vast_targeting +@pytest.mark.order(42117) def test_vast_login_node_mounts(host): """Verify login/compiler node mount table.""" - verify_pxeboot(host, "vast_login_node_mounts", check_vast_login_node_mounts) + tc = TC["vast_login_node_mounts"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_pxeboot(test_log, tc, host, check_vast_login_node_mounts) diff --git a/test/orchestrator/fvt/result.py b/test/orchestrator/fvt/result.py index d8482341cb..16e707d109 100644 --- a/test/orchestrator/fvt/result.py +++ b/test/orchestrator/fvt/result.py @@ -12,23 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test-layer rendering and assertion policy for structured FVT results.""" +"""Test-layer rendering and assertion policy for structured FVT results. + +Each test resolves its registry entry, creates its own ``TestLogger`` start +record, and passes both here. These helpers only render the final record and +enforce the result contract. +""" import pytest -from library.functions import TestLogger from library.messages import CLEANUP_TEST_ASSERT_MSGS as CLEANUP_ASSERT from library.messages import CLEANUP_TEST_LOG_MSGS as CLEANUP_LOG from library.messages import PRECHECK_TEST_ASSERT_MSGS as PRECHECK_ASSERT from library.messages import PRECHECK_TEST_LOG_MSGS as PRECHECK_LOG from library.messages import PXEBOOT_TEST_ASSERT_MSGS as PXEBOOT_ASSERT from library.messages import PXEBOOT_TEST_LOG_MSGS as PXEBOOT_LOG -from library.vars import TEST_CASES -def _report(case, result, fields, component, messages): +def _report(test_log, result, fields, component, messages): """Render one final result and enforce its success contract.""" log_messages, assert_messages = messages - test_log = TestLogger(case["title"], case["id"]) if result["success"]: test_log.passed_fields( log_messages["check_passed"].format(component=component), @@ -45,12 +47,24 @@ def _report(case, result, fields, component, messages): ) -def verify_precheck(host, key, checker): +def _skip_if_requested(test_log, case, result, fields, log_messages): + """Record and raise a documented skip for an optional capability.""" + if not result.get("skipped"): + return + test_log.skipped_fields( + log_messages["check_skipped"].format(component=case["component"]), + fields, + ) + reason = dict(fields).get("Reason") or (fields[0][1] if fields else "") + pytest.skip(str(reason or f"{case['component']} is not enabled")) + + +def verify_precheck(test_log, case, host, checker): """Execute and enforce one precheck verification result.""" - case = TEST_CASES[key] result = checker(host) + _skip_if_requested(test_log, case, result, result["fields"], PRECHECK_LOG) _report( - case, + test_log, result, result["fields"], case["component"], @@ -58,12 +72,33 @@ def verify_precheck(host, key, checker): ) -def verify_cleanup(host, key, component, checker): +def verify_precheck_rejection(test_log, case, host, checker): + """Require one precheck to reject an invalid condition with an error.""" + result = checker(host) + fields = result["fields"] + _skip_if_requested(test_log, case, result, fields, PRECHECK_LOG) + component = case["component"] + rejected = not result["success"] and bool(result["error"]) + if rejected: + test_log.passed_fields( + PRECHECK_LOG["rejection_passed"].format(component=component), + [*fields, ("Rejection", result["error"])], + ) + else: + test_log.failed_fields( + PRECHECK_LOG["rejection_failed"].format(component=component), + fields, + ) + assert rejected, PRECHECK_ASSERT["rejection_missing"].format( + component=component + ) + + +def verify_cleanup(test_log, host, component, checker): """Execute and enforce one cleanup verification result.""" - case = TEST_CASES[key] result = checker(host) _report( - case, + test_log, result, result["fields"], component, @@ -71,20 +106,13 @@ def verify_cleanup(host, key, component, checker): ) -def verify_pxeboot(host, key, checker): +def verify_pxeboot(test_log, case, host, checker): """Execute and enforce one PXE post-boot verification result.""" - case = TEST_CASES[key] result = checker(host) fields = result["details"]["fields"] - if result.get("skipped"): - TestLogger(case["title"], case["id"]).skipped_fields( - PXEBOOT_LOG["check_skipped"].format(component=case["component"]), - fields, - ) - reason = fields[0][1] if fields else f"{case['component']} is not enabled" - pytest.skip(str(reason)) + _skip_if_requested(test_log, case, result, fields, PXEBOOT_LOG) _report( - case, + test_log, result, fields, case["component"], diff --git a/test/orchestrator/library/functions/__init__.py b/test/orchestrator/library/functions/__init__.py index bd3b834753..5412fd2031 100644 --- a/test/orchestrator/library/functions/__init__.py +++ b/test/orchestrator/library/functions/__init__.py @@ -174,6 +174,11 @@ cleanup_extra_vars, cleanup_selection_fields, ) +from .external_ldap_pxeboot_func import ( + check_external_ldap_backend, + check_external_ldap_proxy, + reconcile_external_ldap_proxy, +) from .kubernetes_etcd_pxeboot_func import ( check_kubernetes_etcd_health, check_kubernetes_etcd_topology, @@ -194,6 +199,7 @@ from .kubernetes_runtime_pxeboot_func import ( check_kubernetes_configured_versions, check_kubernetes_local_etcd_integrity, + check_kubernetes_local_etcd_media, check_kubernetes_local_etcd_provisioning, check_kubernetes_version_compatibility, ) @@ -267,6 +273,7 @@ from .oim_readiness_precheck_func import ( check_oim_cpu_threshold, check_oim_disk_threshold, + check_oim_forced_pxe_public_overlap, check_oim_internet_reachability, check_oim_memory_threshold, check_oim_os_version, @@ -516,6 +523,8 @@ def run_playbook(tag: str | None = None, **kwargs): "check_hpc_benchmarks_tools_dir_creation", "check_kubernetes_configured_versions", "check_kubernetes_control_plane", + "check_external_ldap_backend", + "check_external_ldap_proxy", "check_kubernetes_control_plane_recovery", "check_kubernetes_csi_dynamic_provisioning", "check_kubernetes_default_storage_class", @@ -523,6 +532,7 @@ def run_playbook(tag: str | None = None, **kwargs): "check_kubernetes_etcd_topology", "check_kubernetes_local_etcd", "check_kubernetes_local_etcd_integrity", + "check_kubernetes_local_etcd_media", "check_kubernetes_local_etcd_provisioning", "check_kubernetes_local_etcd_recovery", "check_kubernetes_nfs_dynamic_provisioning", @@ -575,6 +585,17 @@ def run_playbook(tag: str | None = None, **kwargs): "check_node_os_version", "check_node_ping", "check_node_ssh", + "check_oim_cpu_threshold", + "check_oim_disk_threshold", + "check_oim_forced_pxe_public_overlap", + "check_oim_internet_reachability", + "check_oim_memory_threshold", + "check_oim_os_version", + "check_oim_public_nic_present", + "check_oim_pxe_nic_ipv4", + "check_oim_pxe_nic_present", + "check_oim_pxe_public_overlap", + "check_oim_ssh_preflight", "check_precheck_admin_ipv4", "check_precheck_dependencies", "check_precheck_hostname_domain", @@ -658,5 +679,6 @@ def run_playbook(tag: str | None = None, **kwargs): "cleanup_selection_fields", "persistent_changed_count", "resolve_nft_thresholds", + "reconcile_external_ldap_proxy", "run_playbook", ] diff --git a/test/orchestrator/library/functions/_powervault_helpers.py b/test/orchestrator/library/functions/_powervault_helpers.py index 37267ba31f..9347ba2c93 100644 --- a/test/orchestrator/library/functions/_powervault_helpers.py +++ b/test/orchestrator/library/functions/_powervault_helpers.py @@ -38,9 +38,7 @@ from ..vars.pxeboot_vars import ( POWERVAULT_DEFAULT_FS_TYPE, - POWERVAULT_DEFAULT_ISCSI_PORT, POWERVAULT_DEFAULT_MOUNT_OPTS, - POWERVAULT_DEFAULT_NODE_KEY, POWERVAULT_ISCSI_INITIATOR_PATH, POWERVAULT_LOG_COMPLETE_MSG, POWERVAULT_LOG_TEMPLATE, diff --git a/test/orchestrator/library/functions/_pxeboot_helpers.py b/test/orchestrator/library/functions/_pxeboot_helpers.py index d0a4652596..6bec2ea6ff 100644 --- a/test/orchestrator/library/functions/_pxeboot_helpers.py +++ b/test/orchestrator/library/functions/_pxeboot_helpers.py @@ -23,7 +23,7 @@ from concurrent.futures import TimeoutError as FutureTimeoutError from typing import Any -from omnia_auto import read_remote_env, run_on_host, run_ssh_command +from omnia_auto import log, read_remote_env, run_on_host, run_ssh_command from ..vars.pxeboot_vars import ( CATALOG_ROLE_PREFIXES, @@ -443,10 +443,8 @@ def report_poll_progress( f" ↻ {label}: still in progress; retrying status check " f"(attempt {attempt}, elapsed {elapsed}s, remaining {remaining}s){suffix}" ) - # INFO logging is intentionally hidden in normal validation runs. Long-running - # pull/reboot polls must remain visible so the operator can distinguish active - # progress from a hung test. - print(message, flush=True) + # Poll progress is available in verbose output without bypassing structured logging. + log(message, "INFO") def run_remote_with_progress( @@ -471,7 +469,7 @@ def run_remote_with_progress( try: result = future.result(timeout=min(poll_seconds, remaining)) elapsed = int(time.monotonic() - started) - print(f" ✓ {label}: completed after {elapsed}s", flush=True) + log(f"{label}: completed after {elapsed}s", "OK") return result except FutureTimeoutError: attempt += 1 diff --git a/test/orchestrator/library/functions/_workload_helpers.py b/test/orchestrator/library/functions/_workload_helpers.py index 88649ba787..e9893f4c43 100644 --- a/test/orchestrator/library/functions/_workload_helpers.py +++ b/test/orchestrator/library/functions/_workload_helpers.py @@ -29,6 +29,7 @@ SLURM_SUBMISSION_PREFIXES, ) from ._provision_helpers import load_context +from .external_ldap_func import load_external_ldap_settings from ._pxeboot_helpers import ( first_row, load_workload_context, @@ -159,6 +160,19 @@ def slurm_submission_rows(rows): ] +def ldap_identity_skip(context) -> str: + """Return why LDAP-identity checks must skip, or an empty string. + + The LDAP test identity lives in the external directory proxied by + omnia_auth, so these checks also require validate_external_ldap. + """ + if not context["features"].get("openldap", False): + return "OpenLDAP is not enabled" + if not load_external_ldap_settings()["validation_enabled"]: + return "validate_external_ldap is false; LDAP identity checks are not requested" + return "" + + def ldap_test_username() -> str: """Return the validated non-secret LDAP test identity.""" username = str(load_test_credentials().get("ldap_username") or "") diff --git a/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py b/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py index 50280c7322..ad3d21a7b0 100644 --- a/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py +++ b/test/orchestrator/library/functions/apptainer_jobs_pxeboot_func.py @@ -18,6 +18,8 @@ import re import shlex +from omnia_auto import log + from ..vars.pxeboot_vars import ( APPTAINER_ARRAY_SIZE, APPTAINER_CONCURRENT_JOB_COUNT, @@ -27,7 +29,6 @@ ) from ._apptainer_helpers import ( _wait_for_array_accounting, - _wait_for_job_accounting, apptainer_context, command_error, grouped_node_fields, @@ -37,7 +38,12 @@ safe_node_name, ) from ._pxeboot_helpers import remote_command, runtime_exception, runtime_result -from ._workload_helpers import ldap_test_username, optional_skip, require_functional +from ._workload_helpers import ( + ldap_identity_skip, + ldap_test_username, + optional_skip, + require_functional, +) def _functional_context(host, summary): @@ -179,10 +185,9 @@ def check_apptainer_ldap_job(host): ) if isinstance(_result, dict) and "skipped" in _result: return _result - if not context["features"].get("openldap", False): - return optional_skip( - summary, "OpenLDAP is not selected for the mapped Slurm roles" - ) + reason = ldap_identity_skip(context) + if reason: + return optional_skip(summary, reason) return _targeted_job_check(host, summary, username=ldap_test_username()) except (OSError, RuntimeError, TypeError, ValueError) as exc: return runtime_exception(summary, exc) @@ -354,10 +359,9 @@ def check_apptainer_restricted_sif(host): % shlex.quote(restricted), ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: restricted SIF removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: restricted SIF removal failed: {str(exc)[:100]}", + "WARN", ) diff --git a/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py b/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py index 481069f3ce..e6b4be426d 100644 --- a/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py +++ b/test/orchestrator/library/functions/apptainer_recovery_pxeboot_func.py @@ -103,8 +103,8 @@ def _prepare_recovery(host): _RECOVERY_HOST = host_id gated = require_marker( "Apptainer compute-node reboot", - "disruptive", - "Select reboot or disruptive to authorize a compute-node reboot", + "reboot", + "Select the reboot marker to authorize a compute-node reboot", ) if gated: _RECOVERY_STATE.update({"skipped_result": gated}) diff --git a/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py b/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py index 2dadc262a3..71d9868bd0 100644 --- a/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py +++ b/test/orchestrator/library/functions/apptainer_runtime_pxeboot_func.py @@ -37,7 +37,12 @@ runtime_exception, runtime_result, ) -from ._workload_helpers import ldap_test_username, optional_skip, require_functional +from ._workload_helpers import ( + ldap_identity_skip, + ldap_test_username, + optional_skip, + require_functional, +) def _computes_or_skip(host, summary): @@ -285,10 +290,9 @@ def check_apptainer_ldap_readability(host): context, _control, computes, skipped = _computes_or_skip(host, summary) if skipped: return skipped - if not context["features"].get("openldap", False): - return optional_skip( - summary, "OpenLDAP is not selected for the mapped Slurm roles" - ) + reason = ldap_identity_skip(context) + if reason: + return optional_skip(summary, reason) username = ldap_test_username() outcomes = {} for row in computes: @@ -325,9 +329,12 @@ def check_apptainer_non_root_execution(host): gated = require_functional(summary) if gated: return gated - _context, _control, computes, skipped = _computes_or_skip(host, summary) + context, _control, computes, skipped = _computes_or_skip(host, summary) if skipped: return skipped + reason = ldap_identity_skip(context) + if reason: + return optional_skip(summary, reason) username = ldap_test_username() outcomes = {} for row in computes: diff --git a/test/orchestrator/library/functions/dcgm_pxeboot_func.py b/test/orchestrator/library/functions/dcgm_pxeboot_func.py index d3d5f2c1f3..cccaca7abf 100644 --- a/test/orchestrator/library/functions/dcgm_pxeboot_func.py +++ b/test/orchestrator/library/functions/dcgm_pxeboot_func.py @@ -21,11 +21,7 @@ import re import time -from ..vars.pxeboot_vars import ( - PXEBOOT_COMMANDS, - SLURM_COMPILER_PREFIX, - SLURM_COMPUTE_PREFIX, -) +from ..vars.pxeboot_vars import PXEBOOT_COMMANDS, SLURM_COMPILER_PREFIX from ._pxeboot_helpers import ( first_row, remote_command, diff --git a/test/orchestrator/library/functions/external_ldap_pxeboot_func.py b/test/orchestrator/library/functions/external_ldap_pxeboot_func.py new file mode 100644 index 0000000000..8546512060 --- /dev/null +++ b/test/orchestrator/library/functions/external_ldap_pxeboot_func.py @@ -0,0 +1,201 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""External LDAP meta-proxy reconciliation and verification after PXE boot. + +The prepare lifecycle only starts the local omnia_auth container. Pointing it +at an external directory is a test-side configuration step, so it runs in the +pxeboot slurm_ldap suite immediately before the LDAP login checks. +""" + +import ipaddress +import re +from typing import Any + +from ._pxeboot_helpers import runtime_result +from .external_ldap_func import ( + configure_external_ldap_proxy, + domain_to_dn, + load_external_ldap_settings, + resolve_proxy_config_path, + verify_external_ldap_backend, +) +from .openldap_prepare_func import openldap_enabled + + +def _openldap_skip(component: str) -> dict[str, Any]: + """Return a consistent expected skip for catalog-disabled OpenLDAP.""" + return runtime_result( + True, + "OpenLDAP is disabled by the active catalog", + [ + ("Reason", "OpenLDAP is disabled by the active catalog"), + ("Component", component), + ("openldap_support", False), + ], + skipped=True, + ) + + +def reconcile_external_ldap_proxy(host) -> dict[str, Any]: + """Reconcile and verify the opt-in external LDAP proxy configuration.""" + try: + if not openldap_enabled(host): + return _openldap_skip("External LDAP proxy") + except (TypeError, ValueError) as exc: + return runtime_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) + + try: + settings = load_external_ldap_settings() + if not settings["validation_enabled"]: + return runtime_result( + True, + "External LDAP validation is not requested", + [ + ("Reason", "validate_external_ldap is false"), + ("validate_external_ldap", False), + ], + skipped=True, + ) + except (TypeError, ValueError) as exc: + return runtime_result(False, "External LDAP settings are invalid", [], str(exc)) + + reconciliation = configure_external_ldap_proxy(host) + fields = [ + ("Configuration changed", reconciliation.get("changed", False)), + ("Reconciliation", reconciliation["details"]), + ] + if not reconciliation["success"]: + return runtime_result( + False, + reconciliation["details"], + fields, + reconciliation["error"], + ) + + verification = check_external_ldap_proxy(host) + return runtime_result( + verification["success"], + "External LDAP proxy reconciled and verified", + [*fields, *verification["details"]["fields"]], + verification["error"], + ) + + +def check_external_ldap_proxy(host) -> dict[str, Any]: + """Verify the deployed external LDAP meta-proxy configuration.""" + try: + if not openldap_enabled(host): + return _openldap_skip("External LDAP proxy") + settings = load_external_ldap_settings() + if not settings["validation_enabled"]: + return runtime_result( + True, + "External LDAP validation is not requested", + [ + ("Reason", "validate_external_ldap is false"), + ("validate_external_ldap", False), + ], + skipped=True, + ) + path = resolve_proxy_config_path(host, settings) + deployed = host.file(path) + if not deployed.is_file: + return runtime_result( + False, + "External LDAP proxy config missing", + [("Path", path)], + path, + ) + content = deployed.content_string + external_dn = domain_to_dn(str(settings["domain"])) + endpoint_host = str(settings["server_ip"]) + try: + if ipaddress.ip_address(endpoint_host).version == 6: + endpoint_host = f"[{endpoint_host}]" + except ValueError: + pass + expected = { + "database meta": bool( + re.search(r"^database\s+meta\s*$", content, re.MULTILINE) + ), + "back_ldap module": "back_ldap" in content, + "back_meta module": "back_meta" in content, + "external endpoint": ( + f"ldap://{endpoint_host}:{settings['server_port']}/" in content + ), + "suffix massage": external_dn in content, + "idassert bind": bool( + re.search(r"^idassert-bind\s*$", content, re.MULTILINE) + ), + } + except (OSError, TypeError, ValueError) as exc: + return runtime_result( + False, "External LDAP proxy verification failed", [], str(exc) + ) + + fields = [ + (name, "present" if value else "missing") for name, value in expected.items() + ] + failures = [name for name, value in expected.items() if not value] + fields.extend( + [ + ("Config owner", f"{deployed.user}:{deployed.group}"), + ("Config mode", oct(deployed.mode)), + ] + ) + if deployed.user != "root" or deployed.group != "root": + failures.append("config owner") + if deployed.mode != 0o600: + failures.append("config mode") + return runtime_result( + not failures, + "External LDAP meta-proxy configuration checked", + fields, + "; ".join(failures), + ) + + +def check_external_ldap_backend(host) -> dict[str, Any]: + """Verify external LDAP is reachable from the omnia_auth container.""" + try: + if not openldap_enabled(host): + return _openldap_skip("External LDAP backend") + settings = load_external_ldap_settings() + if not settings["validation_enabled"]: + return runtime_result( + True, + "External LDAP validation is not requested", + [ + ("Reason", "validate_external_ldap is false"), + ("validate_external_ldap", False), + ], + skipped=True, + ) + except (TypeError, ValueError) as exc: + return runtime_result(False, "External LDAP settings are invalid", [], str(exc)) + + probe = verify_external_ldap_backend(host) + return runtime_result( + probe["success"], + probe["details"], + [ + ("Endpoint", probe.get("endpoint", "unavailable")), + ( + "LDAP protocol reachability", + "passed" if probe["success"] else "failed", + ), + ], + probe["error"], + ) diff --git a/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py b/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py index a6a0332e56..3d106ba320 100644 --- a/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py +++ b/test/orchestrator/library/functions/hpc_benchmarks_pxeboot_func.py @@ -38,7 +38,6 @@ HPC_TOOLS_DIRECTORY_MODE, HPC_TOOLS_NVIDIA_SDK_DIRECTORY, HPC_TOOLS_PLATFORMS_DIRECTORY, - HPC_TOOLS_SCRIPTS_DIRECTORY, PXEBOOT_COMMANDS, ) from ._hpc_benchmarks_helpers import ( @@ -71,7 +70,6 @@ snapshot_directories, snapshot_tool_fingerprints, source_only_scan_paths, - staged_tool_directories, tools_list_deployed, wait_for_pid, ) diff --git a/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py b/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py index 4f28749af7..26b654269f 100644 --- a/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py +++ b/test/orchestrator/library/functions/kubernetes_recovery_pxeboot_func.py @@ -18,6 +18,9 @@ import time from ..vars.pxeboot_vars import ( + ETCD_HEALTH_WAIT_TIMEOUT_SECONDS, + ETCD_SETUP_LOG, + ETCD_UPDATE_LOG, PXEBOOT_COMMANDS, RECOVERY_POLL_SECONDS, RECOVERY_WAIT_TIMEOUT_SECONDS, @@ -36,7 +39,10 @@ from ._workload_helpers import kubernetes_context as _context from ._workload_helpers import optional_skip as _skip from .kubernetes_etcd_pxeboot_func import check_kubernetes_etcd_health -from .kubernetes_runtime_pxeboot_func import check_kubernetes_local_etcd_integrity +from .kubernetes_runtime_pxeboot_func import ( + check_kubernetes_local_etcd_integrity, + etcd_boot_script_state, +) def _vip_configuration(context, config) -> tuple[bool, str]: @@ -103,6 +109,36 @@ def _wait_for_new_boot(host, row, previous_boot_id: str) -> bool: return False +def _wait_for_etcd_health(host) -> dict: + """Poll etcd endpoint health until every member reports healthy.""" + started = time.monotonic() + deadline = started + ETCD_HEALTH_WAIT_TIMEOUT_SECONDS + attempt = 0 + result = check_kubernetes_etcd_health(host) + while not result["success"] and time.monotonic() < deadline: + attempt += 1 + report_poll_progress( + "etcd endpoint health", + attempt, + started, + ETCD_HEALTH_WAIT_TIMEOUT_SECONDS, + result.get("error", ""), + ) + time.sleep(RECOVERY_POLL_SECONDS) + result = check_kubernetes_etcd_health(host) + return result + + +def _post_reboot_boot_scripts(host, row) -> tuple[bool, bool]: + """Return whether fstab-update completed and disk setup did not rerun.""" + states, boot_time = etcd_boot_script_state(host, row) + update_time, update_done = states.get(ETCD_UPDATE_LOG, (None, False)) + setup_time, _setup_done = states.get(ETCD_SETUP_LOG, (None, False)) + update_ok = update_done and update_time is not None and update_time >= boot_time + setup_not_rerun = setup_time is None or setup_time < boot_time + return update_ok, setup_not_rerun + + def _cluster_node_name(host, control, row) -> str: """Return the node name used by the Kubernetes API.""" payload = remote_json(host, control, PXEBOOT_COMMANDS["kubernetes_nodes"]) @@ -123,10 +159,10 @@ def check_kubernetes_control_plane_recovery(host): """Reboot the VIP owner and verify failover, cloud-init, and node recovery.""" summary = "Kubernetes control-plane recovery" try: - if not marker_is_authorized("disruptive"): + if not marker_is_authorized("reboot"): return _skip( summary, - "Select the disruptive marker to authorize a node reboot", + "Select the reboot marker to authorize a node reboot", ) context, rows, _control, config = _context(host) if not rows: @@ -197,6 +233,7 @@ def check_kubernetes_control_plane_recovery(host): ) final_owners = _vip_owners(host, rows, vip) final_vip_ok = len(final_owners) == 1 + etcd = _wait_for_etcd_health(host) if ready_ok else {"success": False} local_etcd = check_kubernetes_local_etcd_integrity(host) local_etcd_required = bool(config.get("etcd_on_local_disk", False)) local_etcd_ok = ( @@ -208,6 +245,7 @@ def check_kubernetes_control_plane_recovery(host): and cloud_init_ok and ready_ok and final_vip_ok + and etcd["success"] and local_etcd_ok ) error_parts = [] @@ -223,6 +261,8 @@ def check_kubernetes_control_plane_recovery(host): error_parts.append( f"final VIP owners: {', '.join(final_owners) or 'none'}" ) + if not etcd["success"]: + error_parts.append("etcd endpoint health check failed") if not local_etcd_ok: error_parts.append("local-etcd integrity failed") return runtime_result( @@ -236,6 +276,7 @@ def check_kubernetes_control_plane_recovery(host): ("VIP failover", "passed" if failover_ok else "FAILED"), ("Cloud-init state", "passed" if cloud_init_ok else "FAILED"), ("Kubernetes Ready", "passed" if ready_ok else "FAILED"), + ("etcd endpoint health", "passed" if etcd["success"] else "FAILED"), ( "Local etcd persistence", ( @@ -256,16 +297,24 @@ def check_kubernetes_local_etcd_recovery(host): """Reboot one control plane and prove local-etcd identity is preserved.""" summary = "Kubernetes local-etcd reboot persistence" try: - if not marker_is_authorized("disruptive"): + if not marker_is_authorized("reboot"): return _skip( summary, - "Select the disruptive marker to authorize a node reboot", + "Select the reboot marker to authorize a node reboot", ) _context_data, rows, control, config = _context(host) if not rows or not bool(config.get("etcd_on_local_disk", False)): return _skip(summary, "etcd_on_local_disk is disabled") + control_rows = [ + row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] + ] + if len(control_rows) < 2: + return _skip(summary, "At least two control-plane nodes are required") + watcher = next( + row for row in control_rows if row["HOSTNAME"] != control["HOSTNAME"] + ) before_integrity = check_kubernetes_local_etcd_integrity(host) - cluster_node_name = _cluster_node_name(host, control, control) + cluster_node_name = _cluster_node_name(host, watcher, control) before = remote_command( host, control, @@ -294,11 +343,14 @@ def check_kubernetes_local_etcd_recovery(host): if ssh_ok: ready_ok, _detail = wait_for_remote_command( host, - control, + watcher, PXEBOOT_COMMANDS["kubernetes_node_ready"] % cluster_node_name, RECOVERY_WAIT_TIMEOUT_SECONDS, RECOVERY_POLL_SECONDS, ) + update_ok, setup_not_rerun = ( + _post_reboot_boot_scripts(host, control) if ssh_ok else (False, False) + ) after_integrity = check_kubernetes_local_etcd_integrity(host) after = remote_command( host, @@ -310,11 +362,13 @@ def check_kubernetes_local_etcd_recovery(host): and bool(after.stdout.strip()) and after.stdout.strip() == before_identity ) - etcd = check_kubernetes_etcd_health(host) + etcd = _wait_for_etcd_health(host) if ready_ok else {"success": False} ok = ( ssh_ok and cloud_ok and ready_ok + and update_ok + and setup_not_rerun and after_integrity["success"] and identity_preserved and etcd["success"] @@ -326,6 +380,10 @@ def check_kubernetes_local_etcd_recovery(host): error_parts.append("cloud-init did not complete") if not ready_ok: error_parts.append("Kubernetes node not Ready") + if not update_ok: + error_parts.append("etcd-fstab-update.sh did not complete on this boot") + if not setup_not_rerun: + error_parts.append("etcd-disk-setup.sh ran again after reboot") if not identity_preserved: error_parts.append("mount source/UUID changed across reboot") if not after_integrity["success"]: @@ -338,9 +396,18 @@ def check_kubernetes_local_etcd_recovery(host): [ ("Rebooted node", control["HOSTNAME"]), ("Kubernetes identity", cluster_node_name), + ("Readiness observer", watcher["HOSTNAME"]), ("New boot observed", "passed" if ssh_ok else "FAILED"), ("Cloud-init completion", "passed" if cloud_ok else "FAILED"), ("Kubernetes Ready", "passed" if ready_ok else "FAILED"), + ( + "etcd-fstab-update.sh this boot", + "passed" if update_ok else "FAILED", + ), + ( + "etcd-disk-setup.sh not rerun", + "passed" if setup_not_rerun else "FAILED", + ), ( "Mount source and UUID preserved", "passed" if identity_preserved else "FAILED", diff --git a/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py b/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py index 7b22735f9b..fb61fb76b7 100644 --- a/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py +++ b/test/orchestrator/library/functions/kubernetes_runtime_pxeboot_func.py @@ -20,7 +20,14 @@ from collections.abc import Mapping from typing import Any -from ..vars.pxeboot_vars import PXEBOOT_COMMANDS +from omnia_auto import load_test_config + +from ..vars.pxeboot_vars import ( + ETCD_MEDIA_TYPES, + ETCD_SETUP_LOG, + ETCD_UPDATE_LOG, + PXEBOOT_COMMANDS, +) from ._pxeboot_helpers import ( group_fields, remote_command, @@ -349,17 +356,8 @@ def _root_disk_name(root_source: str, by_name: Mapping[str, Mapping[str, Any]]) return name -def _local_etcd_node_state(host, row) -> tuple[bool, str]: - mount_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_mount"]) - filesystems = ( - mount_payload.get("filesystems", []) if isinstance(mount_payload, dict) else [] - ) - if len(filesystems) != 1: - return False, f"mount records={len(filesystems)}" - mount = filesystems[0] - mount_source = str(mount.get("source") or "") - mount_type = str(mount.get("fstype") or "").lower() - +def _etcd_disk_layout(host, row) -> tuple[dict[str, Any], dict[str, Any], dict, str]: + """Return the etcd partition, its parent disk, all devices, and the root disk.""" block_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_block_devices"]) devices = _flatten_devices(block_payload.get("blockdevices", [])) by_name = { @@ -371,17 +369,86 @@ def _local_etcd_node_state(host, row) -> tuple[bool, str]: (device for device in devices if _mountpoint_matches(device, "/var/lib/etcd")), {}, ) - selected_disk = _parent_disk(selected, by_name) if selected else {} - selected_disk_name = str(selected_disk.get("name") or "") - + selected_disk = dict(_parent_disk(selected, by_name)) if selected else {} root = remote_command(host, row, PXEBOOT_COMMANDS["etcd_root_source"]) root_disk_name = _root_disk_name(root.stdout, by_name) if root.rc == 0 else "" + return selected, selected_disk, by_name, root_disk_name + + +def _disk_media(disk: Mapping[str, Any]) -> str: + """Classify a whole disk as nvme, hdd, ssd, or unknown.""" + name = str(disk.get("name") or "") + if str(disk.get("tran") or "").lower() == "nvme" or name.startswith("nvme"): + return "nvme" + rotational = str(disk.get("rota")) + if rotational in {"1", "True"}: + return "hdd" + if rotational in {"0", "False"}: + return "ssd" + return "unknown" + + +def _parse_boot_scripts(text: str) -> dict[str, tuple[int | None, bool]]: + """Parse ``||`` lines into per-log state.""" + states: dict[str, tuple[int | None, bool]] = {} + for line in text.splitlines(): + parts = line.strip().split("|") + if len(parts) != 3: + continue + path, mtime, done = parts + states[path] = ( + int(mtime) if mtime.isdigit() else None, + done.isdigit() and int(done) > 0, + ) + return states + + +def _completed_this_boot( + states: Mapping[str, tuple[int | None, bool]], + boot_time: int, + paths: tuple[str, ...] = (ETCD_UPDATE_LOG, ETCD_SETUP_LOG), +) -> str: + """Return the first log in ``paths`` that finished with DONE on this boot.""" + for path in paths: + mtime, done = states.get(path, (None, False)) + if done and mtime is not None and mtime >= boot_time: + return path + return "" + + +def _boot_script_label(completed: str) -> str: + """Render which boot script completed on this boot.""" + return f"{os.path.basename(completed)} DONE" if completed else "stale/incomplete" + + +def etcd_boot_script_state(host, row) -> tuple[dict[str, tuple[int | None, bool]], int]: + """Return etcd boot-script log state and the current boot time for one node.""" + scripts = remote_command(host, row, PXEBOOT_COMMANDS["etcd_boot_scripts"]) + boot_time = remote_command(host, row, PXEBOOT_COMMANDS["node_boot_time"]) + if scripts.rc != 0 or boot_time.rc != 0 or not boot_time.stdout.strip().isdigit(): + raise RuntimeError(f"{row['HOSTNAME']} etcd boot-script state is unavailable") + return _parse_boot_scripts(scripts.stdout), int(boot_time.stdout.strip()) + + +def _local_etcd_node_state(host, row) -> tuple[bool, str]: + mount_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_mount"]) + filesystems = ( + mount_payload.get("filesystems", []) if isinstance(mount_payload, dict) else [] + ) + if len(filesystems) != 1: + return False, f"mount records={len(filesystems)}" + mount = filesystems[0] + mount_source = str(mount.get("source") or "") + mount_type = str(mount.get("fstype") or "").lower() + + selected, selected_disk, by_name, root_disk_name = _etcd_disk_layout(host, row) + selected_disk_name = str(selected_disk.get("name") or "") fstab = remote_command(host, row, PXEBOOT_COMMANDS["etcd_fstab"]) fstab_lines = [line.strip() for line in fstab.stdout.splitlines() if line.strip()] permissions = remote_command(host, row, PXEBOOT_COMMANDS["etcd_permissions"]) manifest = remote_command(host, row, PXEBOOT_COMMANDS["etcd_manifest_data_dir"]) - log_time = remote_command(host, row, PXEBOOT_COMMANDS["etcd_boot_log"]) - boot_time = remote_command(host, row, PXEBOOT_COMMANDS["node_boot_time"]) + states, boot_time = etcd_boot_script_state(host, row) + completed = _completed_this_boot(states, boot_time) uuid = str(selected.get("uuid") or "") label = str(selected.get("label") or "") @@ -395,22 +462,14 @@ def _local_etcd_node_state(host, row) -> tuple[bool, str]: ) boss_disks = [ device - for device in devices + for device in by_name.values() if str(device.get("type")) == "disk" and str(device.get("name") or "") != root_disk_name and "boss" in str(device.get("model") or "").lower() ] boss_ok = not boss_disks or "boss" in str(selected_disk.get("model") or "").lower() - media_ok = str(selected_disk.get("type") or "") == "disk" and str( - selected_disk.get("rota") - ) in {"0", "1", "False", "True"} - log_current = ( - log_time.rc == 0 - and boot_time.rc == 0 - and log_time.stdout.strip().isdigit() - and boot_time.stdout.strip().isdigit() - and int(log_time.stdout.strip()) >= int(boot_time.stdout.strip()) - ) + media = _disk_media(selected_disk) + media_ok = str(selected_disk.get("type") or "") == "disk" and media != "unknown" ok = all( ( bool(mount_source), @@ -425,51 +484,53 @@ def _local_etcd_node_state(host, row) -> tuple[bool, str]: manifest.rc == 0, boss_ok, media_ok, - log_current, + bool(completed), ) ) - media = ( - "NVMe" - if str(selected_disk.get("tran") or "") == "nvme" - else ("HDD" if str(selected_disk.get("rota")) in {"1", "True"} else "SSD") - ) detail = ( f"source={mount_source or 'missing'} | disk={selected_disk_name or 'missing'} " - f"({media}) | fs={fstype or 'missing'} | label={label or 'missing'} | " + f"({media.upper()}) | fs={fstype or 'missing'} | label={label or 'missing'} | " f"UUID={'present' if uuid else 'missing'} | fstab={'valid' if fstab_ok else 'invalid'} | " f"permissions={permissions.stdout.strip() or 'invalid'} | " - f"boot-script={'current' if log_current else 'stale/missing'}" + f"boot-script={_boot_script_label(completed)}" ) return ok, detail +def _control_plane_check(host, summary: str, node_state, failure: str): + """Apply one per-node local-etcd contract to every mapped control plane.""" + _runtime, rows, _control, config = _context(host) + if not rows: + return _skip(summary, "No Kubernetes nodes are mapped") + if not bool(config.get("etcd_on_local_disk", False)): + return _skip(summary, "etcd_on_local_disk is disabled") + control_rows = [ + row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] + ] + outcomes = {row["HOSTNAME"]: node_state(host, row) for row in control_rows} + failures = [name for name, outcome in outcomes.items() if not outcome[0]] + return runtime_result( + bool(control_rows) and not failures, + summary, + group_fields(control_rows, outcomes), + ( + f"{failure}: " + ", ".join(failures) + if failures + else ( + "No Kubernetes control-plane nodes are mapped" + if not control_rows + else "" + ) + ), + ) + + def check_kubernetes_local_etcd_integrity(host): """Verify local-etcd disk selection, filesystem, mount, and boot persistence.""" summary = "Kubernetes local-etcd storage integrity" try: - _runtime, rows, _control, config = _context(host) - if not rows or not bool(config.get("etcd_on_local_disk", False)): - return _skip(summary, "etcd_on_local_disk is disabled") - control_rows = [ - row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] - ] - outcomes = { - row["HOSTNAME"]: _local_etcd_node_state(host, row) for row in control_rows - } - failures = [name for name, outcome in outcomes.items() if not outcome[0]] - return runtime_result( - bool(control_rows) and not failures, - summary, - group_fields(control_rows, outcomes), - ( - "Invalid local-etcd storage on: " + ", ".join(failures) - if failures - else ( - "No Kubernetes control-plane nodes are mapped" - if not control_rows - else "" - ) - ), + return _control_plane_check( + host, summary, _local_etcd_node_state, "Invalid local-etcd storage on" ) except (OSError, RuntimeError, TypeError, ValueError) as exc: return runtime_exception(summary, exc) @@ -477,25 +538,12 @@ def check_kubernetes_local_etcd_integrity(host): def _local_etcd_provisioning_state(host, row) -> tuple[bool, str]: """Return the local-etcd disk-layout and provisioning-artifact state.""" - block_payload = remote_json(host, row, PXEBOOT_COMMANDS["etcd_block_devices"]) - devices = _flatten_devices(block_payload.get("blockdevices", [])) - by_name = { - str(device.get("name")): device - for device in devices - if str(device.get("name") or "") - } - selected = next( - (device for device in devices if _mountpoint_matches(device, "/var/lib/etcd")), - {}, - ) - selected_disk = _parent_disk(selected, by_name) if selected else {} + selected, selected_disk, by_name, root_name = _etcd_disk_layout(host, row) selected_name = str(selected_disk.get("name") or "") pttype = str(selected_disk.get("pttype") or "").lower() - root = remote_command(host, row, PXEBOOT_COMMANDS["etcd_root_source"]) - root_name = _root_disk_name(root.stdout, by_name) if root.rc == 0 else "" boss_disks = [ device - for device in devices + for device in by_name.values() if str(device.get("type")) == "disk" and "boss" in str(device.get("model") or "").lower() ] @@ -507,29 +555,28 @@ def _local_etcd_provisioning_state(host, row) -> tuple[bool, str]: ) setup_script = remote_command(host, row, PXEBOOT_COMMANDS["etcd_setup_script"]) update_script = remote_command(host, row, PXEBOOT_COMMANDS["etcd_update_script"]) - setup_log = remote_command(host, row, PXEBOOT_COMMANDS["etcd_setup_log"]) - update_log = remote_command(host, row, PXEBOOT_COMMANDS["etcd_update_log"]) selection_log = remote_command(host, row, PXEBOOT_COMMANDS["etcd_selection_log"]) + states, boot_time = etcd_boot_script_state(host, row) + completed = _completed_this_boot(states, boot_time) scripts_ok = setup_script.rc == 0 and update_script.rc == 0 - logs_ok = setup_log.rc == 0 and update_log.rc == 0 logged_disks = re.findall( r"(?:found|candidate|disk):\s*/dev/([A-Za-z0-9._-]+)", selection_log.stdout, ) - selection_logged = selection_log.rc == 0 and selected_name in logged_disks + selection_logged = selected_name in logged_disks ok = ( bool(selected.get("pkname")) and selection_ok and pttype == "gpt" and scripts_ok - and logs_ok + and bool(completed) and selection_logged ) detail = ( f"disk={selected_name or 'missing'} | selection={selection_mode} | " f"partition-table={pttype or 'missing'} | " f"scripts={'valid' if scripts_ok else 'missing/not executable'} | " - f"logs={'complete' if logs_ok else 'missing/incomplete'} | " + f"boot-script={_boot_script_label(completed)} | " f"logged-selection={'matched' if selection_logged else 'mismatch'}" ) return ok, detail @@ -539,32 +586,43 @@ def check_kubernetes_local_etcd_provisioning(host): """Verify GPT selection and local-etcd provisioning artifacts.""" summary = "Kubernetes local-etcd provisioning contract" try: - _runtime, rows, _control, config = _context(host) - if not rows: - return _skip(summary, "No Kubernetes nodes are mapped") - if not bool(config.get("etcd_on_local_disk", False)): - return _skip(summary, "etcd_on_local_disk is disabled") - control_rows = [ - row for row in rows if "control_plane" in row["EXPECTED_FUNCTIONAL_GROUP"] - ] - outcomes = { - row["HOSTNAME"]: _local_etcd_provisioning_state(host, row) - for row in control_rows - } - failures = [name for name, outcome in outcomes.items() if not outcome[0]] - return runtime_result( - bool(control_rows) and not failures, + return _control_plane_check( + host, summary, - group_fields(control_rows, outcomes), - ( - "Invalid local-etcd provisioning on: " + ", ".join(failures) - if failures - else ( - "No Kubernetes control-plane nodes are mapped" - if not control_rows - else "" - ) - ), + _local_etcd_provisioning_state, + "Invalid local-etcd provisioning on", + ) + except (OSError, RuntimeError, TypeError, ValueError) as exc: + return runtime_exception(summary, exc) + + +def check_kubernetes_local_etcd_media(host): + """Verify every control plane places etcd on the configured disk media.""" + summary = "Kubernetes local-etcd disk media" + try: + expected = str(load_test_config().get("expected_etcd_disk_media") or "") + expected = expected.strip().lower() + if not expected: + return _skip(summary, "expected_etcd_disk_media is not configured") + if expected not in ETCD_MEDIA_TYPES: + raise ValueError( + "expected_etcd_disk_media must be one of: " + + ", ".join(ETCD_MEDIA_TYPES) + ) + + def node_state(host, row): + _selected, disk, _by_name, _root = _etcd_disk_layout(host, row) + media = _disk_media(disk) + return media == expected, ( + f"disk={disk.get('name') or 'missing'} | " + f"media={media.upper()} | expected={expected.upper()}" + ) + + return _control_plane_check( + host, + summary, + node_state, + f"etcd is not on {expected.upper()} media on", ) except (OSError, RuntimeError, TypeError, ValueError) as exc: return runtime_exception(summary, exc) diff --git a/test/orchestrator/library/functions/minimal_os_pxeboot_func.py b/test/orchestrator/library/functions/minimal_os_pxeboot_func.py index 44e92eecf2..68275dc973 100644 --- a/test/orchestrator/library/functions/minimal_os_pxeboot_func.py +++ b/test/orchestrator/library/functions/minimal_os_pxeboot_func.py @@ -19,7 +19,6 @@ software (Slurm, Kubernetes, container runtimes, GPU drivers, MPI stacks). """ -import re from typing import Any from ..vars.pxeboot_vars import PXEBOOT_COMMANDS diff --git a/test/orchestrator/library/functions/mount_config_precheck_func.py b/test/orchestrator/library/functions/mount_config_precheck_func.py index 93918b9a9a..514072801a 100644 --- a/test/orchestrator/library/functions/mount_config_precheck_func.py +++ b/test/orchestrator/library/functions/mount_config_precheck_func.py @@ -32,7 +32,6 @@ """ import os -import re from typing import Any from ..vars.pxeboot_vars import STORAGE_CONFIG diff --git a/test/orchestrator/library/functions/mount_config_pxeboot_func.py b/test/orchestrator/library/functions/mount_config_pxeboot_func.py index b7e77f6a98..1738e72d12 100644 --- a/test/orchestrator/library/functions/mount_config_pxeboot_func.py +++ b/test/orchestrator/library/functions/mount_config_pxeboot_func.py @@ -22,7 +22,6 @@ """ import os -import re from typing import Any from ..vars.pxeboot_vars import ( @@ -31,9 +30,7 @@ STORAGE_CONFIG, ) from ._pxeboot_helpers import ( - group_fields, load_runtime_context, - load_workload_context, remote_command, rows_matching, runtime_exception, diff --git a/test/orchestrator/library/functions/oim_readiness_precheck_func.py b/test/orchestrator/library/functions/oim_readiness_precheck_func.py index 2a10a52d02..f4c8347d89 100644 --- a/test/orchestrator/library/functions/oim_readiness_precheck_func.py +++ b/test/orchestrator/library/functions/oim_readiness_precheck_func.py @@ -19,6 +19,8 @@ Every function is read-only and never mutates the target environment. """ +import csv +import io import posixpath import re from typing import Any @@ -58,6 +60,17 @@ def _admin_network(host) -> dict[str, Any]: raise ValueError("network_spec.yml does not define admin_network") +def _default_route_nic(host) -> tuple[str, str]: + """Return the default IPv4 route line and its device, or empty strings.""" + result = run_on_host(host, OIM_READINESS_COMMANDS["default_route"]) + line = result.stdout.strip() if result.rc == 0 else "" + parts = line.split() + for index, token in enumerate(parts): + if token == "dev" and index + 1 < len(parts): + return line, parts[index + 1] + return line, "" + + # ── hardware checks ──────────────────────────────────────────────── @@ -143,11 +156,12 @@ def check_oim_disk_threshold( # ── network checks ───────────────────────────────────────────────── -def check_oim_pxe_nic_present(host) -> dict[str, Any]: - """Verify the configured admin NIC exists and is operationally UP.""" +def check_oim_pxe_nic_present(host, *, nic_name: str = "") -> dict[str, Any]: + """Verify the configured (or explicitly named) admin NIC exists and is UP.""" try: - admin = _admin_network(host) - nic_name = str(admin.get("oim_nic_name") or "").strip() + if not nic_name: + admin = _admin_network(host) + nic_name = str(admin.get("oim_nic_name") or "").strip() except (OSError, TypeError, ValueError) as exc: return prepare_result( False, "Unable to resolve admin NIC", [], str(exc) @@ -199,15 +213,12 @@ def check_oim_public_nic_present(host) -> dict[str, Any]: or PXE-only environments do not require a public interface). """ try: - result = run_on_host( - host, "ip -4 route show default 2>/dev/null | head -1" - ) + line, nic_name = _default_route_nic(host) except (OSError, RuntimeError) as exc: return prepare_result( False, "Unable to query default route", [], str(exc) ) - line = result.stdout.strip() if not line: return prepare_result( True, @@ -215,13 +226,6 @@ def check_oim_public_nic_present(host) -> dict[str, Any]: [("Default route", "none"), ("Public NIC check", "skipped")], ) - parts = line.split() - nic_name = "" - for i, token in enumerate(parts): - if token == "dev" and i + 1 < len(parts): - nic_name = parts[i + 1] - break - if not nic_name: return prepare_result( False, @@ -314,18 +318,7 @@ def check_oim_pxe_public_overlap( [("PXE NIC", "not configured")], ) - route_result = run_on_host( - host, "ip -4 route show default 2>/dev/null | head -1" - ) - route_line = route_result.stdout.strip() - public_nic = "" - if route_line: - parts = route_line.split() - for i, token in enumerate(parts): - if token == "dev" and i + 1 < len(parts): - public_nic = parts[i + 1] - break - + _route_line, public_nic = _default_route_nic(host) if not public_nic: return prepare_result( True, @@ -353,15 +346,34 @@ def check_oim_pxe_public_overlap( ) +def check_oim_forced_pxe_public_overlap(host) -> dict[str, Any]: + """Run the overlap check with the PXE NIC forced to the public NIC.""" + try: + _route_line, public_nic = _default_route_nic(host) + except (OSError, RuntimeError) as exc: + return prepare_result(False, "Unable to query default route", [], str(exc)) + if not public_nic: + return prepare_result( + True, + "No default route; forced overlap is not applicable", + [("Reason", "No default IPv4 route on the OIM")], + skipped=True, + ) + return check_oim_pxe_public_overlap(host, nic_name=public_nic) + + # ── prerequisite checks ──────────────────────────────────────────── -def check_oim_ssh_preflight(host) -> dict[str, Any]: +def check_oim_ssh_preflight(host, *, target: str = "") -> dict[str, Any]: """Verify passwordless SSH from OIM to a configured target node. - Uses the first mapped node from the PXE mapping CSV as the target. - Skips if no mapping is configured or no nodes are present. + Uses the first mapped node from the PXE mapping CSV as the target unless + ``target`` is given. Skips if no mapping is configured or no nodes are + present. """ + if target: + return _ssh_preflight_result(host, target) try: input_dir = resolve_target_input_project_path(host) config = read_yaml_mapping( @@ -384,7 +396,7 @@ def check_oim_ssh_preflight(host) -> dict[str, Any]: ) # Read the first ADMIN_IP from the mapping - raw = run_on_host(host, "head -2 %s", mapping_path) + raw = run_on_host(host, OIM_READINESS_COMMANDS["mapping_head"], mapping_path) lines = raw.stdout.strip().split("\n") if raw.rc == 0 else [] if len(lines) < 2: return prepare_result( @@ -393,9 +405,6 @@ def check_oim_ssh_preflight(host) -> dict[str, Any]: [("PXE mapping", mapping_path), ("SSH preflight", "skipped")], ) - import csv - import io - reader = csv.DictReader(io.StringIO(raw.stdout)) row = next(reader, None) if not row or not row.get("ADMIN_IP", "").strip(): @@ -405,7 +414,11 @@ def check_oim_ssh_preflight(host) -> dict[str, Any]: [("PXE mapping", mapping_path), ("SSH preflight", "skipped")], ) - target = row["ADMIN_IP"].strip() + return _ssh_preflight_result(host, row["ADMIN_IP"].strip()) + + +def _ssh_preflight_result(host, target: str) -> dict[str, Any]: + """Return the structured passwordless-SSH result for one target.""" ssh_result = run_on_host( host, OIM_READINESS_COMMANDS["ssh_check"], target ) diff --git a/test/orchestrator/library/functions/openldap_prepare_func.py b/test/orchestrator/library/functions/openldap_prepare_func.py index d2e0197bbc..e03108114a 100644 --- a/test/orchestrator/library/functions/openldap_prepare_func.py +++ b/test/orchestrator/library/functions/openldap_prepare_func.py @@ -14,9 +14,7 @@ """OpenLDAP postcondition helpers for the prepare lifecycle.""" -import ipaddress import posixpath -import re import time from typing import Any @@ -37,20 +35,13 @@ PREPARE_COMMANDS, ) from ._prepare_helpers import prepare_result, read_yaml_mapping -from .external_ldap_func import ( - configure_external_ldap_proxy, - domain_to_dn, - load_external_ldap_settings, - resolve_proxy_config_path, - verify_external_ldap_backend, -) from .project_func import ( resolve_target_omnia_data_path, resolve_target_output_project_path, ) -def _openldap_enabled(host) -> bool: +def openldap_enabled(host) -> bool: """Read the authoritative catalog decision from Orchestrator state.""" state_path = posixpath.join( resolve_target_output_project_path(host), "orchestrator_state.yml" @@ -72,52 +63,10 @@ def _openldap_skip(component: str) -> dict[str, Any]: ) -def reconcile_prepare_external_ldap_proxy(host) -> dict[str, Any]: - """Reconcile and verify the opt-in external LDAP proxy configuration.""" - try: - if not _openldap_enabled(host): - return _openldap_skip("External LDAP proxy") - except (TypeError, ValueError) as exc: - return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) - - try: - settings = load_external_ldap_settings() - if not settings["validation_enabled"]: - return prepare_result( - True, - "External LDAP validation is not requested", - [("validate_external_ldap", False)], - skipped=True, - ) - except (TypeError, ValueError) as exc: - return prepare_result(False, "External LDAP settings are invalid", [], str(exc)) - - reconciliation = configure_external_ldap_proxy(host) - fields = [ - ("Configuration changed", reconciliation.get("changed", False)), - ("Reconciliation", reconciliation["details"]), - ] - if not reconciliation["success"]: - return prepare_result( - False, - reconciliation["details"], - fields, - reconciliation["error"], - ) - - verification = check_prepare_external_ldap_proxy(host) - return prepare_result( - verification["success"], - "External LDAP proxy reconciled and verified", - [*fields, *verification["fields"]], - verification["error"], - ) - - def check_prepare_openldap_runtime(host) -> dict[str, Any]: """Verify enabled OpenLDAP has a healthy service and container.""" try: - if not _openldap_enabled(host): + if not openldap_enabled(host): return _openldap_skip("OpenLDAP runtime") except (TypeError, ValueError) as exc: return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) @@ -165,7 +114,7 @@ def check_prepare_openldap_runtime(host) -> dict[str, Any]: def check_prepare_openldap_artifacts(host) -> dict[str, Any]: """Verify OpenLDAP secret modes, certificate lifetime and slapd syntax.""" try: - if not _openldap_enabled(host): + if not openldap_enabled(host): return _openldap_skip("OpenLDAP artifacts") except (TypeError, ValueError) as exc: return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) @@ -218,7 +167,7 @@ def check_prepare_openldap_artifacts(host) -> dict[str, Any]: def check_prepare_openldap_endpoint(host) -> dict[str, Any]: """Verify local LDAP health and both published listeners.""" try: - if not _openldap_enabled(host): + if not openldap_enabled(host): return _openldap_skip("OpenLDAP endpoint") except (TypeError, ValueError) as exc: return prepare_result(False, "Unable to resolve OpenLDAP state", [], str(exc)) @@ -245,105 +194,3 @@ def check_prepare_openldap_endpoint(host) -> dict[str, Any]: fields, "; ".join(failures), ) - - -def check_prepare_external_ldap_proxy(host) -> dict[str, Any]: - """Verify the deployed external LDAP meta-proxy configuration.""" - try: - if not _openldap_enabled(host): - return _openldap_skip("External LDAP proxy") - settings = load_external_ldap_settings() - if not settings["validation_enabled"]: - return prepare_result( - True, - "External LDAP validation is not requested", - [("validate_external_ldap", False)], - skipped=True, - ) - path = resolve_proxy_config_path(host, settings) - deployed = host.file(path) - if not deployed.is_file: - return prepare_result( - False, - "External LDAP proxy config missing", - [("Path", path)], - path, - ) - content = deployed.content_string - external_dn = domain_to_dn(str(settings["domain"])) - endpoint_host = str(settings["server_ip"]) - try: - if ipaddress.ip_address(endpoint_host).version == 6: - endpoint_host = f"[{endpoint_host}]" - except ValueError: - pass - expected = { - "database meta": bool( - re.search(r"^database\s+meta\s*$", content, re.MULTILINE) - ), - "back_ldap module": "back_ldap" in content, - "back_meta module": "back_meta" in content, - "external endpoint": ( - f"ldap://{endpoint_host}:{settings['server_port']}/" in content - ), - "suffix massage": external_dn in content, - "idassert bind": bool( - re.search(r"^idassert-bind\s*$", content, re.MULTILINE) - ), - } - except (OSError, TypeError, ValueError) as exc: - return prepare_result( - False, "External LDAP proxy verification failed", [], str(exc) - ) - - fields = [ - (name, "present" if value else "missing") for name, value in expected.items() - ] - failures = [name for name, value in expected.items() if not value] - fields.extend( - [ - ("Config owner", f"{deployed.user}:{deployed.group}"), - ("Config mode", oct(deployed.mode)), - ] - ) - if deployed.user != "root" or deployed.group != "root": - failures.append("config owner") - if deployed.mode != 0o600: - failures.append("config mode") - return prepare_result( - not failures, - "External LDAP meta-proxy configuration checked", - fields, - "; ".join(failures), - ) - - -def check_prepare_external_ldap_backend(host) -> dict[str, Any]: - """Verify external LDAP is reachable from the omnia_auth container.""" - try: - if not _openldap_enabled(host): - return _openldap_skip("External LDAP backend") - settings = load_external_ldap_settings() - if not settings["validation_enabled"]: - return prepare_result( - True, - "External LDAP validation is not requested", - [("validate_external_ldap", False)], - skipped=True, - ) - except (TypeError, ValueError) as exc: - return prepare_result(False, "External LDAP settings are invalid", [], str(exc)) - - probe = verify_external_ldap_backend(host) - return prepare_result( - probe["success"], - probe["details"], - [ - ("Endpoint", probe.get("endpoint", "unavailable")), - ( - "LDAP protocol reachability", - "passed" if probe["success"] else "failed", - ), - ], - probe["error"], - ) diff --git a/test/orchestrator/library/functions/powervault_pxeboot_func.py b/test/orchestrator/library/functions/powervault_pxeboot_func.py index 7c50a0dab1..45c079b4d2 100644 --- a/test/orchestrator/library/functions/powervault_pxeboot_func.py +++ b/test/orchestrator/library/functions/powervault_pxeboot_func.py @@ -35,11 +35,7 @@ from typing import Any -from ..vars.pxeboot_vars import ( - POWERVAULT_DEFAULT_ISCSI_PORT, - POWERVAULT_DEFAULT_NODE_KEY, - PXEBOOT_COMMANDS, -) +from ..vars.pxeboot_vars import POWERVAULT_DEFAULT_ISCSI_PORT from ._powervault_helpers import ( SLURM_MANDATORY_BIND_TARGETS, error_result, @@ -48,7 +44,6 @@ get_powervault_entries, get_target_nodes, optional_skip, - resolve_node_key_value, resolve_pv_fs_type, resolve_pv_mount_opts, skip_if_no_powervault, diff --git a/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py b/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py index b1ee5afc26..ef1a2aebb0 100644 --- a/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_auth_pxeboot_func.py @@ -20,7 +20,6 @@ import secrets import shlex import socket -import subprocess import time from pathlib import Path @@ -28,6 +27,8 @@ from omnia_auto import ( connection_params, load_test_credentials, + log, + run_on_host, ) from ..vars.pxeboot_vars import ( @@ -47,6 +48,7 @@ runtime_exception, runtime_result, ) +from ._workload_helpers import ldap_identity_skip as _ldap_identity_skip from ._workload_helpers import ldap_test_username as _ldap_username from ._workload_helpers import optional_skip as _skip from ._workload_helpers import require_functional as _require_functional @@ -114,6 +116,7 @@ def _mapped_host_key_matches(exc: paramiko.BadHostKeyException, target: str) -> def _remove_stale_mapped_host_key( + host, target: str, replacement_key: paramiko.PKey, ) -> tuple[bool, str]: @@ -124,16 +127,15 @@ def _remove_stale_mapped_host_key( if not os.path.isfile(known_hosts): return False, f"known-hosts file does not exist: {known_hosts}" try: - result = subprocess.run( - ["ssh-keygen", "-R", target, "-f", known_hosts], - capture_output=True, - text=True, - timeout=10, - check=False, + result = run_on_host( + host, + PXEBOOT_COMMANDS["ssh_remove_host_key"], + shlex.quote(target), + shlex.quote(known_hosts), ) - except (OSError, subprocess.TimeoutExpired) as exc: + except (OSError, RuntimeError, TypeError, ValueError) as exc: return False, f"unable to remove the stale SSH host key: {exc}" - if result.returncode != 0: + if result.rc != 0: detail = (result.stderr or result.stdout).strip() return False, detail or "ssh-keygen could not remove the stale host key" key_type = replacement_key.get_name() @@ -154,7 +156,7 @@ def _remove_stale_mapped_host_key( def _ldap_ssh_identity( - target: str, username: str, auth_secret: str + host, target: str, username: str, auth_secret: str ) -> tuple[bool, str]: if not _HOST_RE.fullmatch(target) or not _IDENTITY_RE.fullmatch(username): raise ValueError("LDAP SSH target or identity is invalid") @@ -222,7 +224,7 @@ def _ldap_ssh_identity( return True, detail except paramiko.BadHostKeyException as exc: if attempt == 0 and _mapped_host_key_matches(exc, target): - removed, detail = _remove_stale_mapped_host_key(target, exc.key) + removed, detail = _remove_stale_mapped_host_key(host, target, exc.key) if removed: host_key_reconciled = True continue @@ -242,7 +244,9 @@ def _ldap_ssh_identity( return False, "SSH host-key reconciliation did not establish a session" -def _open_ldap_persistent_session(target: str, username: str, auth_secret: str): +def _open_ldap_persistent_session( + host, target: str, username: str, auth_secret: str +): """Open an LDAP SSH command that must be killed when its job ends.""" if not _HOST_RE.fullmatch(target) or not _IDENTITY_RE.fullmatch(username): raise ValueError("LDAP SSH target or identity is invalid") @@ -315,7 +319,7 @@ def _open_ldap_persistent_session(target: str, username: str, auth_secret: str): return session, detail except paramiko.BadHostKeyException as exc: if attempt == 0 and _mapped_host_key_matches(exc, target): - removed, detail = _remove_stale_mapped_host_key(target, exc.key) + removed, detail = _remove_stale_mapped_host_key(host, target, exc.key) if removed: host_key_reconciled = True continue @@ -607,17 +611,24 @@ def _check_slurm_role_ldap_authentication( try: context, rows, _control, _config = _context(host) if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + return _skip( + summary, "OpenLDAP is not enabled" if rows else "No Slurm nodes are mapped" + ) role_rows = _role_rows(rows, role_prefix) if not role_rows: return _skip(summary, f"No {missing_role} nodes are mapped") + reason = _ldap_identity_skip(context) + if reason: + return _skip(summary, reason) username, auth_secret = _ldap_credentials() credential = ( "invalid-" + secrets.token_urlsafe(24) if invalid_password else auth_secret ) outcomes = {} for row in role_rows: - accepted, detail = _ldap_ssh_identity(row["ADMIN_IP"], username, credential) + accepted, detail = _ldap_ssh_identity( + host, row["ADMIN_IP"], username, credential + ) success = not accepted if invalid_password else accepted outcomes[row["HOSTNAME"]] = { "success": success, @@ -750,8 +761,9 @@ def check_slurm_pam_no_job_access(host): summary = "Slurm PAM no-job access policy" try: context, rows, control, _config = _context(host) - if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + reason = _ldap_identity_skip(context) if rows else "No Slurm nodes are mapped" + if reason: + return _skip(summary, reason) username, auth_secret = _ldap_credentials() computes = _compute_rows(rows) if not computes: @@ -782,7 +794,7 @@ def check_slurm_pam_no_job_access(host): outcomes = {} for row in computes: allowed, _detail = _ldap_ssh_identity( - row["ADMIN_IP"], username, auth_secret + host, row["ADMIN_IP"], username, auth_secret ) outcomes[row["HOSTNAME"]] = ( not allowed, @@ -840,13 +852,18 @@ def _check_slurm_role_pam_job_access( return _skip(summary, "Select the functional marker to authorize test jobs") context, rows, control, slurm_config = _context(host) if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + return _skip( + summary, "OpenLDAP is not enabled" if rows else "No Slurm nodes are mapped" + ) computes = _compute_rows(rows) submit_rows = _role_rows(rows, role_prefix) if not submit_rows: return _skip(summary, f"No {missing_role} nodes are mapped") if not computes: return _skip(summary, "At least one Slurm compute node is required") + reason = _ldap_identity_skip(context) + if reason: + return _skip(summary, reason) username, auth_secret = _ldap_credentials() workspace_info = _prepare_ldap_workspace( host, @@ -904,6 +921,7 @@ def _check_slurm_role_pam_job_access( ) if running: session, login_detail = _open_ldap_persistent_session( + host, compute["ADMIN_IP"], username, auth_secret, @@ -927,7 +945,7 @@ def _check_slurm_role_pam_job_access( persistent_sessions.remove(session) time.sleep(PAM_ACCESS_SETTLE_SECONDS) access_after, post_login_detail = _ldap_ssh_identity( - compute["ADMIN_IP"], username, auth_secret + host, compute["ADMIN_IP"], username, auth_secret ) stdout, stderr = _job_artifacts(host, control, workspace, job_id) output_lines = [ @@ -1110,10 +1128,10 @@ def _check_slurm_role_pam_job_access( PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel PAM job {job_id}: " + log( + f"cleanup: failed to cancel PAM job {job_id}: " f"{str(exc)[:100]}", - flush=True, + "WARN", ) if control is not None and workspace_info is not None and username: try: @@ -1124,10 +1142,9 @@ def _check_slurm_role_pam_job_access( username, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: LDAP workspace removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: LDAP workspace removal failed: {str(exc)[:100]}", + "WARN", ) @@ -1169,8 +1186,9 @@ def check_slurm_invalid_ldap_identity(host): summary = "Slurm invalid LDAP identity rejection" try: context, rows, _control, _config = _context(host) - if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + reason = _ldap_identity_skip(context) if rows else "No Slurm nodes are mapped" + if reason: + return _skip(summary, reason) invalid_user = "omnia_missing_" + secrets.token_hex(6) allowed = _allowed_rows(rows) outcomes = {} @@ -1215,13 +1233,18 @@ def _check_slurm_role_ldap_jobs( return gated context, rows, control, slurm_config = _context(host) if not rows or not context["features"].get("openldap", False): - return _skip(summary, "OpenLDAP is not enabled") + return _skip( + summary, "OpenLDAP is not enabled" if rows else "No Slurm nodes are mapped" + ) submit_rows = _role_rows(rows, role_prefix) if not submit_rows: return _skip(summary, f"No {missing_role} nodes are mapped") computes = _compute_rows(rows) if not computes: return _skip(summary, "At least one Slurm compute node is required") + reason = _ldap_identity_skip(context) + if reason: + return _skip(summary, reason) username = _ldap_username() workspace_info = _prepare_ldap_workspace( host, @@ -1409,10 +1432,10 @@ def _check_slurm_role_ldap_jobs( PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel LDAP job {job_id}: " + log( + f"cleanup: failed to cancel LDAP job {job_id}: " f"{str(exc)[:100]}", - flush=True, + "WARN", ) if control is not None and workspace_info is not None and username: try: @@ -1423,10 +1446,9 @@ def _check_slurm_role_ldap_jobs( username, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: LDAP workspace removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: LDAP workspace removal failed: {str(exc)[:100]}", + "WARN", ) diff --git a/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py b/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py index c66f8666ff..17701122ae 100644 --- a/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_jobs_pxeboot_func.py @@ -21,6 +21,8 @@ import time from pathlib import Path +from omnia_auto import log + from ..vars.pxeboot_vars import ( PXEBOOT_COMMANDS, RECOVERY_POLL_SECONDS, @@ -36,7 +38,6 @@ ) from ._pxeboot_helpers import ( first_row, - marker_is_authorized, remote_command, runtime_exception, runtime_result, @@ -574,10 +575,9 @@ def check_slurm_concurrent_jobs(host): PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel job {job_id}: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: failed to cancel job {job_id}: {str(exc)[:100]}", + "WARN", ) try: remote_command( @@ -1093,10 +1093,9 @@ def check_slurm_job_queueing(host): PXEBOOT_COMMANDS["slurm_cancel_job"] % job_id, ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: failed to cancel job {job_id}: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: failed to cancel job {job_id}: {str(exc)[:100]}", + "WARN", ) for row in computes: for job_id in jobs: @@ -1129,11 +1128,6 @@ def check_slurm_drain_queue_recovery(host): control = None compute = None try: - if not marker_is_authorized("disruptive"): - return _skip( - summary, - "Select the disruptive marker to authorize node draining", - ) _runtime, rows, control, _config = _context(host) computes = _compute_rows(rows) if not computes: @@ -1501,16 +1495,15 @@ def check_slurm_openmpi_job(host): f"rmdir {shlex.quote(workspace)} 2>/dev/null || true", ) if cleanup.rc != 0: - print( - f" ! cleanup: OpenMPI workspace {workspace} removal " + log( + f"cleanup: OpenMPI workspace {workspace} removal " f"returned rc={cleanup.rc}", - flush=True, + "WARN", ) except (OSError, RuntimeError, TypeError, ValueError) as exc: - print( - f" ! cleanup: OpenMPI workspace removal failed: " - f"{str(exc)[:100]}", - flush=True, + log( + f"cleanup: OpenMPI workspace removal failed: {str(exc)[:100]}", + "WARN", ) diff --git a/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py b/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py index 9bb3617a01..210f6fdb16 100644 --- a/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_lifecycle_pxeboot_func.py @@ -33,17 +33,11 @@ from omnia_auto import load_test_config, run_on_host -from ..vars.pxeboot_vars import PXEBOOT_COMMANDS, SLURM_COMPUTE_PREFIX -from ._pxeboot_helpers import ( - remote_command, - runtime_exception, - runtime_result, -) +from ..vars.pxeboot_vars import PXEBOOT_COMMANDS +from ._pxeboot_helpers import remote_command, runtime_exception from ._provision_helpers import ( api_json, - group_members, interface_ips, - load_context, normalise_mac, resource_list, ) @@ -56,6 +50,9 @@ # Functional group prefix for Slurm compute nodes. _SLURM_NODE_PREFIX = "slurm_node_" +# Removal needs at least this many slurm_node rows in the PXE mapping. +_MIN_SLURM_NODES = 2 + def _result( success: bool, @@ -140,8 +137,8 @@ def _resolve_target_nodes( Returns: ``(admin_ips, all_rows, header_line)`` when target nodes are identified, or ``None`` when the test should be skipped because - no ``slurm_node_*`` rows exist and the user did not provide - explicit IPs. + the mapping has fewer than ``_MIN_SLURM_NODES`` ``slurm_node_*`` + rows. Raises: ValueError: When the user supplied explicit IPs that are invalid @@ -160,6 +157,8 @@ def _resolve_target_nodes( row for row in all_rows if row.get("FUNCTIONAL_GROUP_NAME", "").startswith(_SLURM_NODE_PREFIX) ] + if len(slurm_node_rows) < _MIN_SLURM_NODES: + return None if configured: # User provided specific Admin IPs — validation failures are errors. @@ -184,14 +183,18 @@ def _resolve_target_nodes( ) return requested_ips, all_rows, header - # Default: remove the last slurm_node entry. Skip when none exist. - if not slurm_node_rows: - return None - + # Default: remove the last slurm_node entry. last_node = slurm_node_rows[-1] return [last_node["ADMIN_IP"]], all_rows, header +def _write_remote_file(host, path: str, content: str) -> Any: + """Write content through a heredoc; the delimiter must be on its own line.""" + if not content.endswith("\n"): + content += "\n" + return run_on_host(host, f"cat > {path} << 'OMNIA_EOF'\n{content}OMNIA_EOF") + + def _write_mapping(host, path: str, header: str, rows: list[dict[str, str]]) -> None: """Write the mapping file back with the given rows.""" if not rows: @@ -203,10 +206,8 @@ def _write_mapping(host, path: str, header: str, rows: list[dict[str, str]]) -> ) writer.writeheader() writer.writerows(rows) - content = output.getvalue() # Write via shell to handle remote/local transparently. - escaped = content.replace("'", "'\\''") - result = run_on_host(host, f"cat > {path} << 'OMNIA_EOF'\n{escaped}OMNIA_EOF") + result = _write_remote_file(host, path, output.getvalue()) if result.rc != 0: raise RuntimeError(f"Failed to write PXE mapping: {result.stderr}") @@ -231,8 +232,7 @@ def restore_pxe_mapping(host) -> None: raise FileNotFoundError(f"PXE mapping backup not found: {source}") with open(source, encoding="utf-8") as fh: content = fh.read() - escaped = content.replace("'", "'\\''") - result = run_on_host(host, f"cat > {dest} << 'OMNIA_EOF'\n{escaped}OMNIA_EOF") + result = _write_remote_file(host, dest, content) if result.rc != 0: raise RuntimeError(f"Failed to restore PXE mapping: {result.stderr}") @@ -437,7 +437,14 @@ def check_slurm_node_remove(host) -> dict[str, Any]: mapping_path = _mapping_path(host) resolved = _resolve_target_nodes(host, mapping_path) if resolved is None: - return _skip(summary, "No slurm_node entries in PXE mapping") + # A stale backup would make the add test restore an old mapping. + if os.path.isfile(_backup_path()): + os.remove(_backup_path()) + return _skip( + summary, + f"At least {_MIN_SLURM_NODES} slurm_node entries are required " + "in the PXE mapping", + ) remove_ips, all_rows, header = resolved removed_rows = [row for row in all_rows if row["ADMIN_IP"] in remove_ips] diff --git a/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py b/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py index 6315c4eb92..bbbae942ff 100644 --- a/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py +++ b/test/orchestrator/library/functions/slurm_recovery_pxeboot_func.py @@ -33,6 +33,7 @@ runtime_result, wait_for_cloud_init, ) +from ._workload_helpers import ldap_identity_skip as _ldap_identity_skip from ._workload_helpers import ldap_test_username as _ldap_username from ._workload_helpers import optional_skip as _skip from ._workload_helpers import slurm_context as _context @@ -108,10 +109,10 @@ def check_slurm_cluster_recovery(host): """ summary = "Slurm cluster reboot recovery" try: - if not marker_is_authorized("disruptive"): + if not marker_is_authorized("reboot"): return _skip( summary, - "Select the disruptive marker to authorize a cluster reboot", + "Select the reboot marker to authorize a cluster reboot", ) context, rows, control, _config = _context(host) if not rows: @@ -188,8 +189,9 @@ def check_slurm_cluster_recovery(host): accounting_ok = accounting_state.startswith("COMPLETED") ldap_ok = True - ldap_detail = "not applicable" - if context["features"].get("openldap", False): + ldap_skip = _ldap_identity_skip(context) + ldap_detail = f"not applicable ({ldap_skip})" if ldap_skip else "not applicable" + if not ldap_skip: username = _ldap_username() submit_rows = _submit_rows(rows) if submit_rows: diff --git a/test/orchestrator/library/functions/vast_storage_pxeboot_func.py b/test/orchestrator/library/functions/vast_storage_pxeboot_func.py index 7066dcc349..f3204e797b 100644 --- a/test/orchestrator/library/functions/vast_storage_pxeboot_func.py +++ b/test/orchestrator/library/functions/vast_storage_pxeboot_func.py @@ -40,7 +40,6 @@ """ import os -import re import time from typing import Any @@ -48,7 +47,6 @@ from ..vars.pxeboot_vars import ( OMNIA_CONFIG, - PXEBOOT_COMMANDS, SLURM_COMPILER_PREFIX, SLURM_COMPUTE_PREFIX, SLURM_CONTROL_PREFIX, @@ -59,7 +57,6 @@ from ._provision_helpers import load_context from ._pxeboot_helpers import ( first_row, - remote_command, rows_matching, runtime_exception, runtime_result, diff --git a/test/orchestrator/library/messages/precheck_msgs.py b/test/orchestrator/library/messages/precheck_msgs.py index b366c9f589..2ba4bc7bf9 100644 --- a/test/orchestrator/library/messages/precheck_msgs.py +++ b/test/orchestrator/library/messages/precheck_msgs.py @@ -19,6 +19,9 @@ "playbook_failed": "Orchestrator precheck failed", "check_passed": "{component} verification passed", "check_failed": "{component} verification failed", + "check_skipped": "{component} verification skipped", + "rejection_passed": "{component} rejected the invalid condition", + "rejection_failed": "{component} accepted an invalid condition", } TEST_ASSERT_MSGS = { @@ -30,4 +33,9 @@ "{component} verification failed: {error}. Correct the reported " "environment, input, storage, or dependency state and rerun precheck." ), + "rejection_missing": ( + "{component} did not reject the invalid condition or returned no " + "actionable error. HOW TO FIX: review the checker failure path and " + "rerun the precheck negative suite." + ), } diff --git a/test/orchestrator/library/vars/__init__.py b/test/orchestrator/library/vars/__init__.py index e5ffa0ef06..56b3b23178 100644 --- a/test/orchestrator/library/vars/__init__.py +++ b/test/orchestrator/library/vars/__init__.py @@ -21,14 +21,17 @@ LDAP_READY_DELAY_SECONDS, LDAP_READY_RETRIES, ) +from .precheck_vars import OIM_NEGATIVE_INPUTS from .test_case_vars import ( CLEANUP_TEST_CASES, NFT_TEST_CASES, + ORDER_BLOCK_SIZE, PRECHECK_TEST_CASES, PREPARE_TEST_CASES, PROVISION_TEST_CASES, PXEBOOT_TEST_CASES, TEST_CASES, + TEST_ORDER_BLOCKS, ) __all__ = [ @@ -39,9 +42,12 @@ "LDAP_READY_DELAY_SECONDS", "LDAP_READY_RETRIES", "NFT_TEST_CASES", + "OIM_NEGATIVE_INPUTS", + "ORDER_BLOCK_SIZE", "PRECHECK_TEST_CASES", "PREPARE_TEST_CASES", "PROVISION_TEST_CASES", "PXEBOOT_TEST_CASES", "TEST_CASES", + "TEST_ORDER_BLOCKS", ] diff --git a/test/orchestrator/library/vars/domain_vars.py b/test/orchestrator/library/vars/domain_vars.py index 67567babba..2b1b95eb30 100644 --- a/test/orchestrator/library/vars/domain_vars.py +++ b/test/orchestrator/library/vars/domain_vars.py @@ -48,14 +48,25 @@ "kubernetes", "slurm", "apptainer", + "benchmark", "additional_cloud_init", "mount_config", "minimal_os", + "boot_image", + "powervault_infrastructure", + "powervault_mounts", + "powervault_binds", + "powervault_cloudinit", + "vast_installation", + "vast_mounts", + "vast_targeting", + "vast_scratch", + "vast_persistence", "image_download", "negative", "non_disruptive", - "disruptive", "reboot", + "node_lifecycle", "scheduler_state", "destructive", "nft", @@ -70,16 +81,32 @@ # ===================================================================== SUITES: dict[str, list[str]] = { - "precheck": ["environment", "storage", "dependencies", "inputs"], - "prepare": ["openchami", "network", "openldap"], - "provision": ["openchami"], + "precheck": [ + "environment", + "oim_readiness", + "storage", + "dependencies", + "inputs", + ], + "prepare": [ + "openchami", + "network", + "openldap", + ], + "provision": [ + "openchami", + ], "pxeboot": [ "connectivity", "cloudinit", + "additional_cloud_init", + "mount_config", + "minimal_os", + "powervault", + "coredns_coredhcp", "kubernetes_cluster", "kubernetes_etcd", "kubernetes_storage", - "kubernetes_recovery", "slurm_cluster", "slurm_jobs", "slurm_ldap", @@ -88,16 +115,12 @@ "slurm_openmpi", "slurm_ucx", "slurm_infiniband", - "slurm_recovery", "slurm_apptainer", "slurm_hpc_benchmarks", - "coredns_coredhcp", - "powervault", "vast_storage", + "kubernetes_recovery", + "slurm_recovery", "slurm_lifecycle", - "additional_cloud_init", - "mount_config", - "minimal_os", ], "cleanup": [ "openchami", @@ -111,8 +134,8 @@ # Each lifecycle owns one playbook execution followed by independent checks. ALL_EXEC_TAGS: list[str] = ["precheck", "prepare", "provision", "pxeboot"] -ALL_EXEC_MARKER: str = "sanity" -ALL_VERIFY_EXCLUDE_MARKERS: list[str] = ["disruptive", "negative"] +ALL_EXEC_MARKER: str = "" +ALL_VERIFY_EXCLUDE_MARKERS: list[str] = [] VERIFY_ONLY_TAGS: list[str] = [] diff --git a/test/orchestrator/library/vars/precheck_vars.py b/test/orchestrator/library/vars/precheck_vars.py index 5776020d85..95474b7b64 100644 --- a/test/orchestrator/library/vars/precheck_vars.py +++ b/test/orchestrator/library/vars/precheck_vars.py @@ -59,6 +59,8 @@ "disk_gb": "df -BG / | tail -1 | awk '{print $2}' | tr -d 'G'", "nic_operstate": "cat /sys/class/net/%s/operstate 2>/dev/null", "nic_ipv4": "ip -4 -o addr show %s scope global", + "default_route": "ip -4 route show default 2>/dev/null | head -1", + "mapping_head": "head -2 %s", "ssh_check": ( "ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 " "-o BatchMode=yes %s whoami 2>/dev/null" @@ -68,3 +70,15 @@ } INTERNET_CHECK_HOSTS: tuple[str, ...] = ("8.8.8.8", "1.1.1.1", "208.67.222.222") + +# Deliberately unattainable inputs used by the read-only OIM negative cases. +# 192.0.2.1 is RFC 5737 TEST-NET-1 and is never routable. +OIM_NEGATIVE_INPUTS: dict[str, object] = { + "min_cores": 99999, + "min_memory_gb": 99999, + "min_disk_gb": 99999, + "missing_nic": "omnia_fvt_missing0", + "unreachable_target": "192.0.2.1", + "os_id": "omnia_fvt_missing_os", + "os_version": "0.0", +} diff --git a/test/orchestrator/library/vars/pxeboot_vars.py b/test/orchestrator/library/vars/pxeboot_vars.py index 4d28dc3568..46886a7d46 100644 --- a/test/orchestrator/library/vars/pxeboot_vars.py +++ b/test/orchestrator/library/vars/pxeboot_vars.py @@ -39,6 +39,12 @@ PAM_SESSION_TERMINATION_TIMEOUT_SECONDS = 30 SLURM_DRAIN_REASON = "omnia_fvt_validation" ETCD_RAFT_INDEX_DELTA_MAX = 100 +# A rebooted etcd member must resynchronize its raft log before every +# endpoint reports healthy; five minutes covers a full snapshot transfer. +ETCD_HEALTH_WAIT_TIMEOUT_SECONDS = 300 +ETCD_MEDIA_TYPES = ("ssd", "hdd", "nvme") +ETCD_SETUP_LOG = "/var/log/etcd-disk-setup.log" +ETCD_UPDATE_LOG = "/var/log/diskless-etcd-mount.log" APPTAINER_IMAGE_DIRECTORY = "/hpc_tools/container_images" APPTAINER_SCRIPT_DIRECTORY = "/hpc_tools/scripts" APPTAINER_DOWNLOAD_SCRIPT = "/hpc_tools/scripts/download_container_image.sh" @@ -398,7 +404,7 @@ "etcd_mount": "findmnt -J /var/lib/etcd", "etcd_mount_identity": "findmnt -n -o SOURCE,UUID,FSTYPE /var/lib/etcd", "etcd_block_devices": ( - "lsblk -J -b -o NAME,PATH,PKNAME,TYPE,FSTYPE,LABEL,UUID," + "lsblk -J -b -o NAME,PATH,PKNAME,TYPE,PTTYPE,FSTYPE,LABEL,UUID," "MOUNTPOINTS,MODEL,ROTA,TRAN" ), "etcd_root_source": "findmnt -n -o SOURCE /", @@ -411,17 +417,19 @@ "grep -Eq -- '--data-dir(=|[[:space:]]+)/var/lib/etcd([[:space:]]|$)' " "/etc/kubernetes/manifests/etcd.yaml" ), - "etcd_boot_log": ( - "stat -c '%Y' /var/log/diskless-etcd-mount.log 2>/dev/null || " - "stat -c '%Y' /var/log/etcd-disk-setup.log" + # One line per boot script log: ||. + "etcd_boot_scripts": ( + "for log in /var/log/etcd-disk-setup.log /var/log/diskless-etcd-mount.log; " + "do if [ -s \"$log\" ]; then printf '%s|%s|%s\\n' \"$log\" " + "\"$(stat -c %Y \"$log\")\" " + "\"$(tail -n 5 \"$log\" | grep -c '===== DONE =====')\"; " + "else printf '%s|missing|0\\n' \"$log\"; fi; done" ), "etcd_setup_script": "test -x /usr/local/bin/etcd-disk-setup.sh", "etcd_update_script": "test -x /usr/local/bin/etcd-fstab-update.sh", - "etcd_setup_log": "test -s /var/log/etcd-disk-setup.log", - "etcd_update_log": "test -s /var/log/diskless-etcd-mount.log", "etcd_selection_log": ( "cat /var/log/etcd-disk-setup.log /var/log/diskless-etcd-mount.log " - "2>/dev/null" + "2>/dev/null || true" ), "node_boot_time": 'date -d "$(uptime -s)" +%s', "node_boot_id": "cat /proc/sys/kernel/random/boot_id", @@ -482,7 +490,7 @@ ), "slurm_drain_node": ("scontrol update NodeName=%s State=DRAIN Reason=%s"), "slurm_drain_state": ( - "sinfo --noheader --nodes=%s --format='%T' | head -1" + "sinfo --noheader --nodes=%s --format='%%T' | head -1" ), "slurm_resume_node": "scontrol update NodeName=%s State=RESUME", "slurm_cancel_job": "scancel %s", @@ -523,6 +531,7 @@ "slurm_hardware": "scontrol show nodes --oneliner", "openmpi": _OPENMPI_DISCOVERY_COMMAND + '"$mpirun_path" --version', "slurm_mpi_plugins": "srun --mpi=list 2>&1", + "ssh_remove_host_key": "ssh-keygen -R %s -f %s", "openmpi_compile": ("srun --nodes=1 --ntasks=1 --nodelist=%s bash -lc %s"), "openmpi_job": ( "srun --nodes=%s --ntasks=%s --ntasks-per-node=1 " diff --git a/test/orchestrator/library/vars/test_case_vars.py b/test/orchestrator/library/vars/test_case_vars.py index 7a6b607cf6..b74355ad18 100644 --- a/test/orchestrator/library/vars/test_case_vars.py +++ b/test/orchestrator/library/vars/test_case_vars.py @@ -129,6 +129,46 @@ "title": "Verify OIM OS matches expected distribution and version", "component": "OIM OS version", }, + "oim_cpu_rejection": { + "id": "ORCH_FVT_PRECHECK_V110", + "title": "Reject an unattainable OIM CPU threshold", + "component": "OIM CPU threshold", + }, + "oim_memory_rejection": { + "id": "ORCH_FVT_PRECHECK_V111", + "title": "Reject an unattainable OIM memory threshold", + "component": "OIM memory threshold", + }, + "oim_disk_rejection": { + "id": "ORCH_FVT_PRECHECK_V112", + "title": "Reject an unattainable OIM disk threshold", + "component": "OIM disk threshold", + }, + "oim_missing_nic_rejection": { + "id": "ORCH_FVT_PRECHECK_V113", + "title": "Reject a nonexistent admin NIC", + "component": "OIM admin NIC", + }, + "oim_nic_overlap_rejection": { + "id": "ORCH_FVT_PRECHECK_V114", + "title": "Reject a PXE NIC that equals the public NIC", + "component": "PXE/public NIC overlap", + }, + "oim_airgapped_internet": { + "id": "ORCH_FVT_PRECHECK_V115", + "title": "Accept missing internet access in air-gapped mode", + "component": "Internet reachability", + }, + "oim_unreachable_ssh_rejection": { + "id": "ORCH_FVT_PRECHECK_V116", + "title": "Reject SSH to an unreachable target", + "component": "OIM SSH preflight", + }, + "oim_os_mismatch_rejection": { + "id": "ORCH_FVT_PRECHECK_V117", + "title": "Reject a mismatched OIM OS identity", + "component": "OIM OS version", + }, } PREPARE_TEST_CASES: dict[str, dict[str, str]] = { @@ -164,10 +204,6 @@ "id": "ORCH_FVT_PREPARE_V007", "title": "Verify CoreDHCP and CoreDNS match network_spec.yml", }, - "external_ldap_proxy": { - "id": "ORCH_FVT_PREPARE_V008", - "title": "Reconcile and verify the external LDAP proxy", - }, "openldap_runtime": { "id": "ORCH_FVT_PREPARE_V009", "title": "Verify OpenLDAP service and container health", @@ -180,10 +216,6 @@ "id": "ORCH_FVT_PREPARE_V011", "title": "Verify OpenLDAP listeners and local endpoint", }, - "external_ldap_backend": { - "id": "ORCH_FVT_PREPARE_V012", - "title": "Verify external LDAP backend reachability", - }, "postgresql_readiness": { "id": "ORCH_FVT_PREPARE_V013", "title": "Verify PostgreSQL and SMD database readiness", @@ -288,7 +320,7 @@ "component": "Kubernetes version compatibility", }, "kubernetes_configured_versions": { - "id": "ORCH_FVT_PXEBOOT_V101", + "id": "ORCH_FVT_PXEBOOT_V519", "title": "Verify configured Kubernetes component versions", "component": "Kubernetes configured versions", }, @@ -333,10 +365,15 @@ "component": "Kubernetes local-etcd disk integrity", }, "kubernetes_local_etcd_provisioning": { - "id": "ORCH_FVT_PXEBOOT_V102", + "id": "ORCH_FVT_PXEBOOT_V520", "title": "Verify Kubernetes local-etcd provisioning contract", "component": "Kubernetes local-etcd provisioning", }, + "kubernetes_local_etcd_media": { + "id": "ORCH_FVT_PXEBOOT_V523", + "title": "Verify Kubernetes local-etcd disk media", + "component": "Kubernetes local-etcd disk media", + }, "kubernetes_storage": { "id": "ORCH_FVT_PXEBOOT_V016", "title": "Verify Kubernetes NFS and CSI storage", @@ -353,7 +390,7 @@ "component": "Kubernetes snapshot components", }, "kubernetes_nfs_contract": { - "id": "ORCH_FVT_PXEBOOT_V103", + "id": "ORCH_FVT_PXEBOOT_V521", "title": "Verify Kubernetes NFS provisioner and backend contract", "component": "Kubernetes NFS provisioner", }, @@ -457,6 +494,16 @@ "title": "Verify Slurm drain queue and resume behavior", "component": "Slurm drain and queue", }, + "external_ldap_proxy": { + "id": "ORCH_FVT_PXEBOOT_V524", + "title": "Reconcile and verify the external LDAP proxy", + "component": "External LDAP proxy", + }, + "external_ldap_backend": { + "id": "ORCH_FVT_PXEBOOT_V525", + "title": "Verify external LDAP backend reachability", + "component": "External LDAP backend", + }, "slurm_pam": { "id": "ORCH_FVT_PXEBOOT_V039", "title": "Verify Slurm pam_slurm_adopt integration", @@ -748,17 +795,17 @@ "component": "HPC benchmarks staging cleanup", }, "hpc_benchmarks_prerequisites": { - "id": "ORCH_FVT_PXEBOOT_V095", + "id": "ORCH_FVT_PXEBOOT_V225", "title": "Verify benchmark prerequisites: script, platform, tools list, mount (NFS export informational)", "component": "HPC benchmarks prerequisites", }, "hpc_benchmarks_staging_fingerprint_idempotency": { - "id": "ORCH_FVT_PXEBOOT_V096", + "id": "ORCH_FVT_PXEBOOT_V226", "title": "Verify staging idempotency with SHA-256, size, and mtime fingerprints", "component": "HPC benchmarks staging fingerprint idempotency", }, "hpc_benchmarks_concurrent_staging": { - "id": "ORCH_FVT_PXEBOOT_V097", + "id": "ORCH_FVT_PXEBOOT_V227", "title": "Verify concurrent staging from two compute nodes on shared storage", "component": "HPC benchmarks concurrent staging", }, @@ -817,7 +864,6 @@ "title": "Verify no compile or build commands are staged", "component": "HPC benchmarks source-only delivery", }, - "hpc_benchmarks_e2e_provisioning": { "id": "ORCH_FVT_PXEBOOT_V208", "title": "Verify end-to-end benchmark provisioning pipeline", @@ -1104,7 +1150,7 @@ }, # ── DCGM / CUDA verification (ORCH_FVT_PXEBOOT_V501-V518) ───────── "dcgm_cuda_validation": { - "id": "ORCH_FVT_PXEBOOT_V501", + "id": "ORCH_FVT_PXEBOOT_V522", "title": "Verify NVIDIA driver and CUDA toolkit on GPU nodes", "component": "CUDA driver and toolkit validation", }, @@ -1547,3 +1593,81 @@ **CLEANUP_TEST_CASES, **NFT_TEST_CASES, } + + +# ===================================================================== +# Pytest execution order blocks +# ===================================================================== +# Every test's @pytest.mark.order value lies inside its suite's block: +# +# block start = lifecycle base + suite slot * ORDER_BLOCK_SIZE +# order = block start + position in suite (1..99) +# +# Lifecycle bases are 10000 apart, so values are unique across lifecycles. +# "root" holds the lifecycle execution case at the block start itself. Suites +# are listed in dependency order: reachability and node state first, then +# cluster health, temporary workloads, and finally reboot or node-removal +# suites. Within a suite, reboot and destructive cases come last. Add a +# test by taking the next free number in its suite block; add a suite by +# appending a new block. +ORDER_BLOCK_SIZE = 100 + +TEST_ORDER_BLOCKS: dict[str, dict[str, int]] = { + "precheck": { + "root": 10000, + "environment": 10100, + "oim_readiness": 10200, + "storage": 10300, + "dependencies": 10400, + "inputs": 10500, + }, + "prepare": { + "root": 20000, + "openchami": 20100, + "network": 20200, + "openldap": 20300, + }, + "provision": { + "root": 30000, + "openchami": 30100, + }, + "pxeboot": { + "root": 40000, + "connectivity": 40100, + "cloudinit": 40200, + "additional_cloud_init": 40300, + "mount_config": 40400, + "minimal_os": 40500, + "powervault": 40600, + "coredns_coredhcp": 40700, + "kubernetes_cluster": 40800, + "kubernetes_etcd": 40900, + "kubernetes_storage": 41000, + "slurm_cluster": 41100, + "slurm_jobs": 41200, + "slurm_ldap": 41300, + "slurm_gpu": 41400, + "slurm_dcgm": 41500, + "slurm_openmpi": 41600, + "slurm_ucx": 41700, + "slurm_infiniband": 41800, + "slurm_apptainer": 41900, + "slurm_hpc_benchmarks": 42000, + "vast_storage": 42100, + "kubernetes_recovery": 42200, + "slurm_recovery": 42300, + "slurm_lifecycle": 42400, + }, + "cleanup": { + "root": 50000, + "openchami": 50100, + "openldap": 50200, + "slurm": 50300, + "kubernetes": 50400, + "artifacts": 50500, + "credentials": 50600, + }, + "nft": { + "root": 60000, + }, +} diff --git a/test/orchestrator/nft/README.md b/test/orchestrator/nft/README.md index 82275c9be4..6ac87633df 100644 --- a/test/orchestrator/nft/README.md +++ b/test/orchestrator/nft/README.md @@ -9,22 +9,28 @@ tests. | Order | TC ID | Test | Marker | Contract | |---:|---|---|---|---| -| 10 | `ORCH_NFT_001` | `test_precheck_performance` | `performance` | Precheck completes within its configured threshold. | -| 20 | `ORCH_NFT_002` | `test_prepare_performance` | `performance`, `destructive` | Prepare completes within its configured threshold. | -| 30 | `ORCH_NFT_005` | `test_prepare_idempotency` | `idempotency`, `destructive` | The second prepare has no persistent changes, preserves container identity, and leaves all required OpenCHAMI services ready. | -| 40 | `ORCH_NFT_003` | `test_provision_performance` | `performance`, `destructive` | Provision completes within its configured threshold. | -| 50 | `ORCH_NFT_006` | `test_precheck_idempotency` | `idempotency` | The second precheck succeeds without persistent changes. | -| 60 | `ORCH_NFT_008` | `test_credential_file_permissions` | `security` | The encrypted credential file is root-owned and mode `0600` or `0640`. | -| 61 | `ORCH_NFT_009` | `test_ssh_private_key_permissions` | `security` | `/root/.ssh/oim_rsa` is root-owned and mode `0600`. | -| 62 | `ORCH_NFT_010` | `test_log_file_permissions` | `security` | Orchestrator log files are root-owned and have no permissions for other users. | -| 63 | `ORCH_NFT_011` | `test_vault_encryption` | `security` | Product credentials have a supported Ansible Vault header and a root-owned mode-`0600` key. | -| 90 | `ORCH_NFT_004` | `test_cleanup_performance` | `performance`, `destructive` | Full cleanup completes within its configured threshold. | -| 91 | `ORCH_NFT_007` | `test_cleanup_idempotency` | `idempotency`, `destructive` | Two cleanup executions succeed, the second has no persistent changes, and every cleanup FVT postcondition passes. | +| 60001 | `ORCH_NFT_001` | `test_precheck_performance` | `performance` | Precheck completes within its configured threshold. | +| 60002 | `ORCH_NFT_002` | `test_prepare_performance` | `performance`, `destructive` | Prepare completes within its configured threshold. | +| 60003 | `ORCH_NFT_005` | `test_prepare_idempotency` | `idempotency`, `destructive` | The second prepare has no persistent changes, preserves container identity, and leaves all required OpenCHAMI services ready. | +| 60004 | `ORCH_NFT_003` | `test_provision_performance` | `performance`, `destructive` | Provision completes within its configured threshold. | +| 60005 | `ORCH_NFT_006` | `test_precheck_idempotency` | `idempotency` | The second precheck succeeds without persistent changes. | +| 60006 | `ORCH_NFT_008` | `test_credential_file_permissions` | `security` | The encrypted credential file is root-owned and mode `0600` or `0640`. | +| 60007 | `ORCH_NFT_009` | `test_ssh_private_key_permissions` | `security` | `/root/.ssh/oim_rsa` is root-owned and mode `0600`. | +| 60008 | `ORCH_NFT_010` | `test_log_file_permissions` | `security` | Orchestrator log files are root-owned and have no permissions for other users. | +| 60009 | `ORCH_NFT_011` | `test_vault_encryption` | `security` | Product credentials have a supported Ansible Vault header and a root-owned mode-`0600` key. | +| 60010 | `ORCH_NFT_004` | `test_cleanup_performance` | `performance`, `destructive` | Full cleanup completes within its configured threshold. | +| 60011 | `ORCH_NFT_007` | `test_cleanup_idempotency` | `idempotency`, `destructive` | Two cleanup executions succeed, the second has no persistent changes, and every cleanup FVT postcondition passes. | +| 60012 | `ORCH_NFT_012` | `test_clean_baseline` | `lifecycle`, `destructive` | The OIM starts from a clean baseline before a fresh install. | +| 60013 | `ORCH_NFT_013` | `test_lifecycle_fresh_install` | `lifecycle`, `destructive` | The complete fresh-install lifecycle succeeds from the clean baseline. | +| 60014 | `ORCH_NFT_014` | `test_lifecycle_provision_verify` | `lifecycle` | Provision and node state are valid after the fresh-install lifecycle. | `ORCH_NFT_001` and `ORCH_NFT_006` use the current `precheck` lifecycle. They retain the stable IDs formerly associated with the retired standalone validation operation. +Orders use the NFT block `60000` in `TEST_ORDER_BLOCKS`; the sequence follows +the lifecycle phase each contract exercises. + ## Performance thresholds `test_config.yml` defines the accepted wall-clock limits in seconds: diff --git a/test/orchestrator/nft/result.py b/test/orchestrator/nft/result.py index 2af6926ccf..e1a723bf14 100644 --- a/test/orchestrator/nft/result.py +++ b/test/orchestrator/nft/result.py @@ -14,15 +14,10 @@ """Render and enforce structured Orchestrator NFT results.""" -from library.functions import TestLogger -from library.vars import TEST_CASES - -def verify_nft(host, key, checker, *args): +def verify_nft(test_log, case, host, checker, *args): """Execute one NFT contract, report safe fields, and enforce success.""" - case = TEST_CASES[key] result = checker(host, *args) - test_log = TestLogger(case["title"], case["id"]) if result["success"]: test_log.passed_fields(result["details"], result["fields"]) else: diff --git a/test/orchestrator/nft/test_idempotency.py b/test/orchestrator/nft/test_idempotency.py index 78eab1ab2b..21e9a46d02 100644 --- a/test/orchestrator/nft/test_idempotency.py +++ b/test/orchestrator/nft/test_idempotency.py @@ -16,10 +16,12 @@ import pytest from library.functions import ( + TestLogger, check_cleanup_idempotency, check_precheck_idempotency, check_prepare_idempotency, ) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @@ -27,24 +29,30 @@ @pytest.mark.nft @pytest.mark.idempotency @pytest.mark.destructive -@pytest.mark.order(30) +@pytest.mark.order(60003) def test_prepare_idempotency(host): """Require repeated prepare to preserve runtime identity and readiness.""" - verify_nft(host, "prepare_idempotency", check_prepare_idempotency) + tc = TC["prepare_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_prepare_idempotency) @pytest.mark.nft @pytest.mark.idempotency -@pytest.mark.order(50) +@pytest.mark.order(60005) def test_precheck_idempotency(host): """Require repeated precheck to remain read-only.""" - verify_nft(host, "precheck_idempotency", check_precheck_idempotency) + tc = TC["precheck_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_precheck_idempotency) @pytest.mark.nft @pytest.mark.idempotency @pytest.mark.destructive -@pytest.mark.order(91) +@pytest.mark.order(60011) def test_cleanup_idempotency(host): """Require repeated full cleanup to remain successful and unchanged.""" - verify_nft(host, "cleanup_idempotency", check_cleanup_idempotency) + tc = TC["cleanup_idempotency"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_cleanup_idempotency) diff --git a/test/orchestrator/nft/test_lifecycle.py b/test/orchestrator/nft/test_lifecycle.py index b0f3e85372..078fb0119c 100644 --- a/test/orchestrator/nft/test_lifecycle.py +++ b/test/orchestrator/nft/test_lifecycle.py @@ -23,10 +23,12 @@ import pytest from library.functions import ( + TestLogger, check_clean_baseline, check_lifecycle_fresh_install, check_lifecycle_provision_verify, ) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @@ -34,24 +36,30 @@ @pytest.mark.nft @pytest.mark.lifecycle @pytest.mark.destructive -@pytest.mark.order(100) +@pytest.mark.order(60012) def test_clean_baseline(host): """Require a provably clean OIM state before the fresh-install lifecycle.""" - verify_nft(host, "clean_baseline", check_clean_baseline) + tc = TC["clean_baseline"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_clean_baseline) @pytest.mark.nft @pytest.mark.lifecycle @pytest.mark.destructive -@pytest.mark.order(101) +@pytest.mark.order(60013) def test_lifecycle_fresh_install(host): """Require the complete fresh-install lifecycle to succeed from clean baseline.""" - verify_nft(host, "lifecycle_fresh_install", check_lifecycle_fresh_install) + tc = TC["lifecycle_fresh_install"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_fresh_install) @pytest.mark.nft @pytest.mark.lifecycle -@pytest.mark.order(102) +@pytest.mark.order(60014) def test_lifecycle_provision_verify(host): """Verify provision state and node connectivity after the fresh-install lifecycle.""" - verify_nft(host, "lifecycle_provision_verify", check_lifecycle_provision_verify) + tc = TC["lifecycle_provision_verify"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_provision_verify) diff --git a/test/orchestrator/nft/test_performance.py b/test/orchestrator/nft/test_performance.py index c852978b54..9f3a176c79 100644 --- a/test/orchestrator/nft/test_performance.py +++ b/test/orchestrator/nft/test_performance.py @@ -15,41 +15,53 @@ """Orchestrator lifecycle-duration contracts.""" import pytest -from library.functions import check_lifecycle_performance +from library.functions import ( + TestLogger, + check_lifecycle_performance, +) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @pytest.mark.nft @pytest.mark.performance -@pytest.mark.order(10) +@pytest.mark.order(60001) def test_precheck_performance(host): """Require precheck to complete within its configured threshold.""" - verify_nft(host, "precheck_performance", check_lifecycle_performance, "precheck") + tc = TC["precheck_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "precheck") @pytest.mark.nft @pytest.mark.performance @pytest.mark.destructive -@pytest.mark.order(20) +@pytest.mark.order(60002) def test_prepare_performance(host): """Require prepare to complete within its configured threshold.""" - verify_nft(host, "prepare_performance", check_lifecycle_performance, "prepare") + tc = TC["prepare_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "prepare") @pytest.mark.nft @pytest.mark.performance @pytest.mark.destructive -@pytest.mark.order(40) +@pytest.mark.order(60004) def test_provision_performance(host): """Require provision to complete within its configured threshold.""" - verify_nft(host, "provision_performance", check_lifecycle_performance, "provision") + tc = TC["provision_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "provision") @pytest.mark.nft @pytest.mark.performance @pytest.mark.destructive -@pytest.mark.order(90) +@pytest.mark.order(60010) def test_cleanup_performance(host): """Require full cleanup to complete within its configured threshold.""" - verify_nft(host, "cleanup_performance", check_lifecycle_performance, "cleanup") + tc = TC["cleanup_performance"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_lifecycle_performance, "cleanup") diff --git a/test/orchestrator/nft/test_security.py b/test/orchestrator/nft/test_security.py index a9e73532a1..2731e034bf 100644 --- a/test/orchestrator/nft/test_security.py +++ b/test/orchestrator/nft/test_security.py @@ -16,42 +16,52 @@ import pytest from library.functions import ( + TestLogger, check_credential_file_permissions, check_log_file_permissions, check_ssh_private_key_permissions, check_vault_encryption, ) +from library.vars import TEST_CASES as TC from nft.result import verify_nft @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(60) +@pytest.mark.order(60006) def test_credential_file_permissions(host): """Require restricted permissions on encrypted product credentials.""" - verify_nft(host, "credential_file_permissions", check_credential_file_permissions) + tc = TC["credential_file_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_credential_file_permissions) @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(61) +@pytest.mark.order(60007) def test_ssh_private_key_permissions(host): """Require root-only access to the OIM private key.""" - verify_nft(host, "ssh_private_key_permissions", check_ssh_private_key_permissions) + tc = TC["ssh_private_key_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_ssh_private_key_permissions) @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(62) +@pytest.mark.order(60008) def test_log_file_permissions(host): """Require Orchestrator logs to remain unavailable to other users.""" - verify_nft(host, "log_file_permissions", check_log_file_permissions) + tc = TC["log_file_permissions"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_log_file_permissions) @pytest.mark.nft @pytest.mark.security -@pytest.mark.order(63) +@pytest.mark.order(60009) def test_vault_encryption(host): """Require supported vault encryption and a protected vault key.""" - verify_nft(host, "vault_encryption", check_vault_encryption) + tc = TC["vault_encryption"] + test_log = TestLogger(tc["title"], tc["id"]) + verify_nft(test_log, tc, host, check_vault_encryption) diff --git a/test/orchestrator/test_config.yml b/test/orchestrator/test_config.yml index 4fbcbfe02c..3c25898d7e 100644 --- a/test/orchestrator/test_config.yml +++ b/test/orchestrator/test_config.yml @@ -265,7 +265,7 @@ cleanup_benchmark_tools: true # The test automatically removes the LAST slurm_node entry from the PXE # mapping file, runs provision, verifies removal, then restores the # original mapping, runs provision again, and verifies re-addition. -# If no slurm_node is present in the mapping, the test is skipped. +# If the mapping has fewer than two slurm_node entries, the test is skipped. # # When one or more Admin IPs are provided (comma-separated): # The test validates that every IP exists in the PXE mapping AND belongs @@ -275,6 +275,19 @@ cleanup_benchmark_tools: true # Example: "182.10.0.50,182.10.0.52" slurm_lifecycle_remove_add_nodes: "" +# ============================================================================= +# KUBERNETES LOCAL-ETCD DISK MEDIA +# ============================================================================= +# Disk media expected under /var/lib/etcd on every Kubernetes control plane +# when etcd_on_local_disk is enabled. The pxeboot kubernetes_etcd suite +# classifies the selected disk from lsblk (TRAN=nvme -> nvme, ROTA=1 -> hdd, +# ROTA=0 -> ssd) and fails when any control plane differs. +# +# Supported values: ssd, hdd, nvme +# When empty (default), the media check is skipped and the detected media is +# still reported by the local-etcd integrity check. +expected_etcd_disk_media: "" + # ============================================================================= # NON-FUNCTIONAL PERFORMANCE CONTRACTS # ============================================================================= diff --git a/test/orchestrator/ut/README.md b/test/orchestrator/ut/README.md index 5d8fbae901..b25c9b6c7c 100644 --- a/test/orchestrator/ut/README.md +++ b/test/orchestrator/ut/README.md @@ -11,6 +11,10 @@ an OIM, calling OpenCHAMI, or executing an Orchestrator playbook. | `test_pxeboot_contracts.py` | Kubernetes version parsing/skew, Slurm parsers and shared-storage selection, catalog feature resolution, mutation authorization, cloud-init interpretation, and Apptainer image-path validation | | `test_cleanup_contracts.py` | Cleanup after Kubernetes apply failure and Slurm drain-test restoration | | `test_nft_contracts.py` | NFT threshold validation and Ansible idempotency-recap interpretation | +| `test_additional_cloud_init_contracts.py` | Additional cloud-init input validation: disabled paths, allowed sections, prohibited keys, list types, and functional-group names | +| `test_hpc_platform_scripts.py` | Per-node RHEL HPC storage and Pulp paths, OS-dependent Slurm scripts, and offline benchmark downloads | +| `test_orchestrator_status_reconcile.py` | Lifecycle status reconciliation for external and platform-managed PXE boot results | +| `test_kubernetes_etcd_contracts.py` | Local-etcd boot-script DONE detection, disk-media classification, root-disk exclusion, post-reboot setup-rerun detection, media expectation, and the external-LDAP identity gate | Parameterized cases count as separate pytest results. Function names and docstrings carry `ORCH_UT_*` identifiers where a stable unit contract has been @@ -44,6 +48,10 @@ python3 -m pytest --confcutdir=ut ut/test_prepare_contracts.py -q python3 -m pytest --confcutdir=ut ut/test_pxeboot_contracts.py -q python3 -m pytest --confcutdir=ut ut/test_cleanup_contracts.py -q python3 -m pytest --confcutdir=ut ut/test_nft_contracts.py -q +python3 -m pytest --confcutdir=ut ut/test_additional_cloud_init_contracts.py -q +python3 -m pytest --confcutdir=ut ut/test_hpc_platform_scripts.py -q +python3 -m pytest --confcutdir=ut ut/test_orchestrator_status_reconcile.py -q +python3 -m pytest --confcutdir=ut ut/test_kubernetes_etcd_contracts.py -q ``` Unit tests must remain isolated: mock external commands at the helper boundary diff --git a/test/orchestrator/ut/test_additional_cloud_init_contracts.py b/test/orchestrator/ut/test_additional_cloud_init_contracts.py index 9d858d6235..38da00fd6c 100644 --- a/test/orchestrator/ut/test_additional_cloud_init_contracts.py +++ b/test/orchestrator/ut/test_additional_cloud_init_contracts.py @@ -17,8 +17,6 @@ import sys from pathlib import Path -import pytest - # The validator lives in the src orchestrator plugins tree. The UT conftest # already adds the test plugin root; we also need the *src* plugin root so # that ``module_utils.orchestrator_validation`` resolves. @@ -71,7 +69,7 @@ def _write_mapping(tmp_path: Path, groups: list[str]) -> str: # --------------------------------------------------------------------------- def test_empty_path_disables_validation(tmp_path): - """ORCH_UT_040: Empty config path disables validation with no errors.""" + """ORCH_UT_085: Empty config path disables validation with no errors.""" errors = validator.validate( {"additional_cloud_init_config_file": ""}, str(tmp_path) ) @@ -79,20 +77,20 @@ def test_empty_path_disables_validation(tmp_path): def test_missing_key_disables_validation(tmp_path): - """ORCH_UT_041: Absent config key disables validation with no errors.""" + """ORCH_UT_086: Absent config key disables validation with no errors.""" errors = validator.validate({}, str(tmp_path)) assert errors == [] def test_null_yaml_file_passes(tmp_path): - """ORCH_UT_042: An empty/null YAML file passes (feature disabled).""" + """ORCH_UT_087: An empty/null YAML file passes (feature disabled).""" path = _write_yaml(tmp_path, "---\n") errors = validator.validate(_config(path), str(tmp_path)) assert errors == [] def test_valid_common_section(tmp_path): - """ORCH_UT_043: Valid common section with write_files and runcmd passes.""" + """ORCH_UT_088: Valid common section with write_files and runcmd passes.""" content = ( "common:\n" " write_files:\n" @@ -108,7 +106,7 @@ def test_valid_common_section(tmp_path): def test_valid_groups_section(tmp_path): - """ORCH_UT_044: Valid per-FG groups section passes when FG exists.""" + """ORCH_UT_089: Valid per-FG groups section passes when FG exists.""" _write_mapping(tmp_path, ["slurm_node_rhel_10_0_x86_64"]) content = ( "groups:\n" diff --git a/test/orchestrator/ut/test_cleanup_contracts.py b/test/orchestrator/ut/test_cleanup_contracts.py index 100efbe436..d1c3c9b48d 100644 --- a/test/orchestrator/ut/test_cleanup_contracts.py +++ b/test/orchestrator/ut/test_cleanup_contracts.py @@ -67,18 +67,23 @@ def test_slurm_drain_probe_always_cancels_job_and_resumes_node(monkeypatch): "ADMIN_IP": "192.0.2.21", "EXPECTED_FUNCTIONAL_GROUP": "slurm_node_rhel_10_0_x86_64", } - monkeypatch.setattr( - slurm_jobs, "marker_is_authorized", lambda marker: marker == "disruptive" - ) monkeypatch.setattr( slurm_jobs, "_context", lambda _host: ({}, [control, compute], control, {}), ) + node_state = ["idle"] + def remote_command(_host, _row, command): commands.append(command) - if "sbatch" in command: + if "State=DRAIN" in command: + node_state[0] = "drained" + elif "State=RESUME" in command: + node_state[0] = "idle" + elif "sinfo" in command: + return _result(stdout=node_state[0]) + elif "sbatch" in command: return _result(stdout="12345|PENDING") return _result() diff --git a/test/orchestrator/ut/test_kubernetes_etcd_contracts.py b/test/orchestrator/ut/test_kubernetes_etcd_contracts.py new file mode 100644 index 0000000000..5424d01ed7 --- /dev/null +++ b/test/orchestrator/ut/test_kubernetes_etcd_contracts.py @@ -0,0 +1,213 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Local-etcd boot-script, disk-media, recovery, and LDAP-gate contracts.""" + +import subprocess + +import pytest +from library.functions import _workload_helpers as workload +from library.functions import kubernetes_recovery_pxeboot_func as recovery +from library.functions import kubernetes_runtime_pxeboot_func as runtime +from library.vars.pxeboot_vars import ( + ETCD_SETUP_LOG, + ETCD_UPDATE_LOG, + PXEBOOT_COMMANDS, +) + +BOOT = 1_700_000_000 + + +def test_boot_script_lines_are_parsed(): + """ORCH_UT_075: Boot-script state lines parse into mtime and DONE flags.""" + states = runtime._parse_boot_scripts( + f"{ETCD_SETUP_LOG}|missing|0\n{ETCD_UPDATE_LOG}|{BOOT}|2\nnoise\n" + ) + + assert states[ETCD_SETUP_LOG] == (None, False) + assert states[ETCD_UPDATE_LOG] == (BOOT, True) + + +@pytest.mark.parametrize( + ("states", "expected"), + [ + # First boot: the early fstab-update run fails, disk setup completes. + ( + {ETCD_SETUP_LOG: (BOOT + 60, True), ETCD_UPDATE_LOG: (BOOT + 30, False)}, + ETCD_SETUP_LOG, + ), + # Later boot: setup log is gone, fstab-update completes. + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT + 30, True)}, ETCD_UPDATE_LOG), + # A DONE marker left by an earlier boot is stale. + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT - 5, True)}, ""), + # A current log without DONE means the script failed. + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT + 30, False)}, ""), + ], +) +def test_completed_boot_script_requires_current_done(states, expected): + """ORCH_UT_076: Only a DONE marker written on this boot counts.""" + assert runtime._completed_this_boot(states, BOOT) == expected + + +@pytest.mark.parametrize( + ("disk", "media"), + [ + ({"name": "nvme0n1", "tran": "nvme", "rota": False}, "nvme"), + ({"name": "nvme1n1", "tran": None, "rota": "0"}, "nvme"), + ({"name": "sdb", "tran": "sata", "rota": True}, "hdd"), + ({"name": "sdb", "tran": "sas", "rota": "1"}, "hdd"), + ({"name": "sdc", "tran": "sata", "rota": False}, "ssd"), + ({"name": "sdc", "tran": "sata", "rota": None}, "unknown"), + ], +) +def test_disk_media_classification(disk, media): + """ORCH_UT_077: Disk media follows lsblk TRAN and ROTA like the 2.2 suite.""" + assert runtime._disk_media(disk) == media + + +def test_partition_resolves_to_parent_and_root_disk(): + """ORCH_UT_078: Partitions resolve to whole disks for root exclusion.""" + devices = runtime._flatten_devices( + [ + {"name": "sda", "type": "disk", "children": [{"name": "sda2", "pkname": "sda"}]}, + { + "name": "nvme0n1", + "type": "disk", + "pttype": "gpt", + "children": [ + { + "name": "nvme0n1p1", + "pkname": "nvme0n1", + "mountpoints": ["/var/lib/etcd"], + } + ], + }, + ] + ) + by_name = {device["name"]: device for device in devices} + selected = by_name["nvme0n1p1"] + + assert runtime._mountpoint_matches(selected, "/var/lib/etcd") + assert runtime._parent_disk(selected, by_name)["name"] == "nvme0n1" + assert runtime._root_disk_name("/dev/sda2", by_name) == "sda" + + +@pytest.mark.parametrize( + ("states", "expected"), + [ + ({ETCD_SETUP_LOG: (None, False), ETCD_UPDATE_LOG: (BOOT + 30, True)}, (True, True)), + ({ETCD_SETUP_LOG: (BOOT + 40, True), ETCD_UPDATE_LOG: (BOOT + 30, True)}, (True, False)), + ({ETCD_SETUP_LOG: (BOOT - 900, True), ETCD_UPDATE_LOG: (BOOT + 30, False)}, (False, True)), + ], +) +def test_post_reboot_detects_disk_setup_rerun(monkeypatch, states, expected): + """ORCH_UT_079: After reboot, fstab-update must finish and setup must not rerun.""" + monkeypatch.setattr(recovery, "etcd_boot_script_state", lambda _host, _row: (states, BOOT)) + + assert recovery._post_reboot_boot_scripts(object(), {"HOSTNAME": "cp1"}) == expected + + +def _media_context(monkeypatch, expected, media_by_host): + rows = [ + { + "HOSTNAME": name, + "ADMIN_IP": f"192.0.2.{index}", + "EXPECTED_FUNCTIONAL_GROUP": "service_kube_control_plane_x86_64", + } + for index, name in enumerate(media_by_host, start=10) + ] + monkeypatch.setattr(runtime, "load_test_config", lambda: {"expected_etcd_disk_media": expected}) + monkeypatch.setattr( + runtime, "_context", lambda _host: ({}, rows, rows[0], {"etcd_on_local_disk": True}) + ) + monkeypatch.setattr( + runtime, + "_etcd_disk_layout", + lambda _host, row: ({}, media_by_host[row["HOSTNAME"]], {}, "sda"), + ) + + +def test_media_check_skips_when_not_configured(monkeypatch): + """ORCH_UT_080: An empty expected media is a documented skip.""" + _media_context(monkeypatch, "", {"cp1": {"name": "sdb", "rota": "0"}}) + + result = runtime.check_kubernetes_local_etcd_media(object()) + + assert result["success"] and result["skipped"] + + +def test_media_check_rejects_unknown_media_value(monkeypatch): + """ORCH_UT_081: An unsupported expected media fails closed.""" + _media_context(monkeypatch, "tape", {"cp1": {"name": "sdb", "rota": "0"}}) + + result = runtime.check_kubernetes_local_etcd_media(object()) + + assert not result["success"] and not result["skipped"] + assert "expected_etcd_disk_media" in result["error"] + + +def test_media_check_requires_every_control_plane(monkeypatch): + """ORCH_UT_082: One control plane on other media fails the check.""" + _media_context( + monkeypatch, + "nvme", + {"cp1": {"name": "nvme0n1", "tran": "nvme"}, "cp2": {"name": "sdb", "rota": "0"}}, + ) + + result = runtime.check_kubernetes_local_etcd_media(object()) + + assert not result["success"] + assert "cp2" in result["error"] and "cp1" not in result["error"] + + +@pytest.mark.parametrize( + ("openldap", "validate", "reason"), + [ + (False, True, "OpenLDAP is not enabled"), + (True, False, "validate_external_ldap is false"), + (True, True, ""), + ], +) +def test_ldap_identity_checks_require_external_validation(monkeypatch, openldap, validate, reason): + """ORCH_UT_083: LDAP-identity checks skip unless external LDAP is validated.""" + monkeypatch.setattr( + workload, "load_external_ldap_settings", lambda: {"validation_enabled": validate} + ) + + result = workload.ldap_identity_skip({"features": {"openldap": openldap}}) + + assert result.startswith(reason) if reason else result == "" + + +def test_boot_script_command_output_is_parseable(tmp_path): + """ORCH_UT_084: The remote boot-script probe emits the parsed line format.""" + setup_log = tmp_path / "etcd-disk-setup.log" + update_log = tmp_path / "diskless-etcd-mount.log" + update_log.write_text("[INFO] start\n2026-01-01_00:00:00 [OK] ===== DONE =====\n") + command = ( + PXEBOOT_COMMANDS["etcd_boot_scripts"] + .replace(ETCD_SETUP_LOG, str(setup_log)) + .replace(ETCD_UPDATE_LOG, str(update_log)) + ) + + output = subprocess.run( + ["bash", "-c", command], capture_output=True, text=True, check=True, timeout=10 + ).stdout + states = runtime._parse_boot_scripts( + output.replace(str(setup_log), ETCD_SETUP_LOG).replace(str(update_log), ETCD_UPDATE_LOG) + ) + + assert states[ETCD_SETUP_LOG] == (None, False) + assert states[ETCD_UPDATE_LOG][1] is True + assert isinstance(states[ETCD_UPDATE_LOG][0], int) diff --git a/test/orchestrator/ut/test_prepare_contracts.py b/test/orchestrator/ut/test_prepare_contracts.py index 03d0c9b730..d45291bc69 100644 --- a/test/orchestrator/ut/test_prepare_contracts.py +++ b/test/orchestrator/ut/test_prepare_contracts.py @@ -18,6 +18,7 @@ import pytest from library.functions import external_ldap_func as external_ldap +from library.functions import external_ldap_pxeboot_func as external_ldap_pxeboot from library.functions import openchami_prepare_func as openchami from library.functions import openldap_prepare_func as openldap from library.functions import postgres_prepare_func as postgres @@ -174,7 +175,7 @@ def test_postgresql_readiness_accepts_complete_read_only_probe(monkeypatch): def test_explicitly_disabled_optional_openldap_skips(monkeypatch): """ORCH_UT_021: Catalog-disabled OpenLDAP is an intentional skip.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: False) + monkeypatch.setattr(openldap, "openldap_enabled", lambda _host: False) verification = openldap.check_prepare_openldap_runtime(object()) @@ -184,7 +185,7 @@ def test_explicitly_disabled_optional_openldap_skips(monkeypatch): def test_enabled_optional_openldap_missing_runtime_fails(monkeypatch): """ORCH_UT_022: Enabled OpenLDAP cannot skip a missing runtime.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: True) + monkeypatch.setattr(openldap, "openldap_enabled", lambda _host: True) monkeypatch.setattr( openldap, "run_on_host", @@ -246,9 +247,9 @@ def test_external_ldap_validation_requires_complete_endpoint(): def test_external_ldap_validation_opt_out_skips_before_mutation(monkeypatch): """ORCH_UT_028: Disabled validation cannot reconcile the LDAP proxy.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: True) + monkeypatch.setattr(external_ldap_pxeboot, "openldap_enabled", lambda _host: True) monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "load_external_ldap_settings", lambda: { "validation_enabled": False, @@ -260,23 +261,23 @@ def unexpected_reconciliation(_host): raise AssertionError("LDAP reconciliation must not run") monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "configure_external_ldap_proxy", unexpected_reconciliation, ) - verification = openldap.reconcile_prepare_external_ldap_proxy(object()) + verification = external_ldap_pxeboot.reconcile_external_ldap_proxy(object()) assert verification["success"] assert verification["skipped"] - assert dict(verification["fields"])["validate_external_ldap"] is False + assert dict(verification["details"]["fields"])["validate_external_ldap"] is False def test_external_ldap_validation_does_not_hide_existing_proxy_failure(monkeypatch): """ORCH_UT_029: Validation-only mode reports a deployed proxy failure.""" - monkeypatch.setattr(openldap, "_openldap_enabled", lambda _host: True) + monkeypatch.setattr(external_ldap_pxeboot, "openldap_enabled", lambda _host: True) monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "load_external_ldap_settings", lambda: { "validation_enabled": True, @@ -284,7 +285,7 @@ def test_external_ldap_validation_does_not_hide_existing_proxy_failure(monkeypat }, ) monkeypatch.setattr( - openldap, + external_ldap_pxeboot, "configure_external_ldap_proxy", lambda _host: { "success": True, @@ -295,18 +296,20 @@ def test_external_ldap_validation_does_not_hide_existing_proxy_failure(monkeypat }, ) monkeypatch.setattr( - openldap, - "check_prepare_external_ldap_proxy", + external_ldap_pxeboot, + "check_external_ldap_proxy", lambda _host: { "success": False, "skipped": False, - "details": "External LDAP proxy verification failed", - "fields": [], + "details": { + "summary": "External LDAP proxy verification failed", + "fields": [], + }, "error": "deployed proxy does not match", }, ) - verification = openldap.reconcile_prepare_external_ldap_proxy(object()) + verification = external_ldap_pxeboot.reconcile_external_ldap_proxy(object()) assert not verification["success"] assert not verification["skipped"] diff --git a/test/orchestrator/ut/test_pxeboot_contracts.py b/test/orchestrator/ut/test_pxeboot_contracts.py index e8c1146be3..69dee14b49 100644 --- a/test/orchestrator/ut/test_pxeboot_contracts.py +++ b/test/orchestrator/ut/test_pxeboot_contracts.py @@ -15,6 +15,7 @@ """Focused unit contracts for PXE parsers, features, and safety gates.""" import json +import subprocess from types import SimpleNamespace import pytest @@ -27,6 +28,7 @@ from library.functions import kubernetes_storage_pxeboot_func as kubernetes_storage from library.functions import slurm_auth_pxeboot_func as slurm_auth from library.functions import slurm_configuration_pxeboot_func as slurm_config +from library.functions import slurm_lifecycle_pxeboot_func as slurm_lifecycle from library.functions import slurm_pxeboot_func as slurm @@ -201,11 +203,11 @@ def test_marker_authorization_requires_exact_explicit_selection(monkeypatch): """ORCH_UT_006: Mutation helpers trust only exact authorized markers.""" monkeypatch.setenv( "OMNIA_FVT_AUTHORIZED_MARKERS", - "functional,disruptive", + "functional,reboot", ) assert helpers.marker_is_authorized("functional") - assert helpers.marker_is_authorized("disruptive") - assert not helpers.marker_is_authorized("reboot") + assert helpers.marker_is_authorized("reboot") + assert not helpers.marker_is_authorized("image_download") monkeypatch.delenv("OMNIA_FVT_AUTHORIZED_MARKERS") assert not helpers.marker_is_authorized("functional") @@ -703,3 +705,132 @@ def unexpected_remote_command(*_args, **_kwargs): assert dict(result["details"]["fields"])["Reason"] == ( "No Slurm compute nodes are mapped" ) + + +@pytest.mark.parametrize("content", ["H,A\nr1,x\n", "H,A\nr1,x", "H,A\nr1,'q'"]) +def test_slurm_lifecycle_mapping_write_keeps_content_exact( + monkeypatch, tmp_path, content +): + """ORCH_UT_090: Heredoc writes never append the delimiter to the data.""" + def run_locally(_host, command): + proc = subprocess.run( + ["bash", "-c", command], capture_output=True, text=True, check=False + ) + return SimpleNamespace(rc=proc.returncode, stdout=proc.stdout, + stderr=proc.stderr) + + monkeypatch.setattr(slurm_lifecycle, "run_on_host", run_locally) + target = tmp_path / "pxe_mapping_file.csv" + result = slurm_lifecycle._write_remote_file(None, str(target), content) + + assert result.rc == 0 + written = target.read_text(encoding="utf-8") + assert "OMNIA_EOF" not in written + assert written.rstrip("\n") == content.rstrip("\n") + + +_LIFECYCLE_HEADER = "FUNCTIONAL_GROUP_NAME,GROUP_NAME,HOSTNAME,ADMIN_MAC,ADMIN_IP" + + +@pytest.mark.parametrize( + ("slurm_nodes", "configured", "expected"), + [ + (1, "", None), + (1, "192.0.2.11", None), + (2, "", ["192.0.2.12"]), + (2, "192.0.2.11", ["192.0.2.11"]), + ], +) +def test_slurm_node_remove_requires_two_slurm_nodes( + monkeypatch, slurm_nodes, configured, expected +): + """ORCH_UT_091: Node removal skips with fewer than two slurm_node rows.""" + rows = ["slurm_control_node_x86_64,grp2,ctl,aa:bb:cc:dd:ee:00,192.0.2.10"] + rows += [ + f"slurm_node_x86_64,grp3,node{i},aa:bb:cc:dd:ee:0{i},192.0.2.1{i}" + for i in range(1, slurm_nodes + 1) + ] + content = "\n".join([_LIFECYCLE_HEADER, *rows]) + "\n" + host = SimpleNamespace( + file=lambda _path: SimpleNamespace(is_file=True, content_string=content) + ) + monkeypatch.setattr( + slurm_lifecycle, + "load_test_config", + lambda: {"slurm_lifecycle_remove_add_nodes": configured}, + ) + + resolved = slurm_lifecycle._resolve_target_nodes(host, "mapping.csv") + + assert (resolved[0] if resolved else None) == expected + + +def test_slurm_node_remove_skip_discards_stale_backup(monkeypatch, tmp_path): + """ORCH_UT_092: A skipped removal leaves no backup for the add test.""" + backup = tmp_path / "pxe_mapping_file.csv.backup" + backup.write_text("stale", encoding="utf-8") + monkeypatch.setattr(slurm_lifecycle, "_backup_path", lambda: str(backup)) + monkeypatch.setattr(slurm_lifecycle, "_mapping_path", lambda _host: "m.csv") + monkeypatch.setattr( + slurm_lifecycle, "_resolve_target_nodes", lambda _host, _path: None + ) + + result = slurm_lifecycle.check_slurm_node_remove(None) + + assert result["skipped"] and result["success"] + assert "At least 2 slurm_node" in result["details"]["fields"][0][1] + assert not backup.exists() + + +def test_stale_host_key_removal_uses_central_runner(monkeypatch, tmp_path): + """ORCH_UT_093: Host-key reconciliation uses the centralized local runner.""" + known_hosts = tmp_path / "known_hosts" + known_hosts.write_text("old key\n", encoding="utf-8") + monkeypatch.setenv("OMNIA_SSH_KNOWN_HOSTS", str(known_hosts)) + calls = [] + + def run_locally(host, command, *args): + calls.append((host, command, args)) + return SimpleNamespace(rc=0, stdout="", stderr="") + + monkeypatch.setattr(slurm_auth, "run_on_host", run_locally) + key = SimpleNamespace( + get_name=lambda: "ssh-ed25519", + get_base64=lambda: "A" * 68, + ) + host = object() + + success, _detail = slurm_auth._remove_stale_mapped_host_key( + host, "192.0.2.10", key + ) + + assert success + assert calls == [ + ( + host, + slurm_auth.PXEBOOT_COMMANDS["ssh_remove_host_key"], + ("192.0.2.10", str(known_hosts)), + ) + ] + assert known_hosts.read_text(encoding="utf-8").endswith( + f"192.0.2.10 ssh-ed25519 {'A' * 68}\n" + ) + + +def test_poll_progress_uses_structured_logging(monkeypatch): + """ORCH_UT_094: Poll progress is emitted through structured logging.""" + messages = [] + monkeypatch.setattr( + helpers, "log", lambda message, level: messages.append((message, level)) + ) + monkeypatch.setattr(helpers.time, "monotonic", lambda: 12.0) + + helpers.report_poll_progress("node reboot", 2, 10.0, 30, "booting") + + assert messages == [ + ( + " ↻ node reboot: still in progress; retrying status check " + "(attempt 2, elapsed 2s, remaining 28s) | last state: booting", + "INFO", + ) + ] diff --git a/test/plugins/dist/omnia_auto-1.0.0-py3-none-any.whl b/test/plugins/dist/omnia_auto-1.0.0-py3-none-any.whl index 59ca5d09b9..c30291e0c9 100644 Binary files a/test/plugins/dist/omnia_auto-1.0.0-py3-none-any.whl and b/test/plugins/dist/omnia_auto-1.0.0-py3-none-any.whl differ diff --git a/test/plugins/docs/06_report.md b/test/plugins/docs/06_report.md index ba1ae15f00..4790c81245 100644 --- a/test/plugins/docs/06_report.md +++ b/test/plugins/docs/06_report.md @@ -13,7 +13,6 @@ report files. The HTML report includes: - CSS conic-gradient donut chart - Suite breakdown table with progress bars - Detailed test results with search/filter/collapse by run_id -- Skip classification badges (expected / unexpected / framework) - Sensitive data redaction (IPs, passwords, secrets, paths) - Pipeline-aware report naming (pipelineId_reportName for CI, reportName for CLI) - Multiple test run segregation by run_id (collapsible sections) diff --git a/test/plugins/omnia_auto/functions/formatting_func.py b/test/plugins/omnia_auto/functions/formatting_func.py index 2badce00c8..fc63c8ae00 100644 --- a/test/plugins/omnia_auto/functions/formatting_func.py +++ b/test/plugins/omnia_auto/functions/formatting_func.py @@ -346,6 +346,7 @@ def add_session_result( status: str, duration: float, tc_id: str = "", + reason: str = "", ) -> None: """Append a test result for the session summary table. @@ -354,6 +355,7 @@ def add_session_result( status: ``PASSED``, ``FAILED``, or ``SKIPPED``. duration: Duration in seconds. tc_id: Test case ID (e.g. ``IMGBM_FVT_PREPARE_E001``). + reason: Skip reason shown below the summary table. """ result = { "test_name": test_name, @@ -361,6 +363,8 @@ def add_session_result( "status": status, "duration": duration, } + if reason: + result["reason"] = " ".join(str(reason).split()) _SESSION_RESULTS.set((*_SESSION_RESULTS.get(), result)) @@ -473,5 +477,14 @@ def _render_summary(results) -> None: f"/ {total} total " f"({total_dur:.2f}s)" ) + skip_reasons = [r for r in skipped if r.get("reason")] + if skip_reasons: + print(f"\n {Colors.YELLOW}Skip reasons{Colors.RESET}") + for r in skip_reasons: + label = r.get("tc_id") or r["test_name"] + print( + f" {Colors.CYAN}{label:<{tc_id_width}}{Colors.RESET} " + f"{TestLogger._truncate(r['reason'], MAX_LINE_WIDTH)}" + ) print(sep) print() diff --git a/test/plugins/omnia_auto/functions/report_func.py b/test/plugins/omnia_auto/functions/report_func.py index 9c64d0e113..5648ced117 100644 --- a/test/plugins/omnia_auto/functions/report_func.py +++ b/test/plugins/omnia_auto/functions/report_func.py @@ -22,7 +22,6 @@ - Suite breakdown tables with progress bars - Collapsible scenario sections with search/filter - Playbook execution logs accordion with CRITICAL/WARNING highlighting -- Skip classification (expected / unexpected / framework) - Sensitive data redaction (IPs, passwords, secrets, paths) - Playbook failure capture (record_playbook_failure) - Separate test runs for repeated domain execution @@ -118,37 +117,6 @@ def _redact_sensitive(text: str) -> str: return text -# ── Skip Classification ───────────────────────────────────────────────────── - -def _classify_skip(details: str = "", error: str = "") -> str: - """Classify a skipped test into expected / unexpected / framework. - - Returns one of: ``'expected'``, ``'unexpected'``, ``'framework'``. - """ - text = ((details or "") + " " + (error or "")).lower() - if not text.strip(): - return "unexpected" - - framework_kw = [ - "fixture", "setup failed", "import error", "conftest", - "collection error", "module not found", "no module named", - "parametrize", "setup error", "teardown error", - ] - if any(kw in text for kw in framework_kw): - return "framework" - - expected_kw = [ - "not enabled", "not configured", "disabled", "not supported", - "not available", "not applicable", "not installed", "feature not", - "requires", "only applies", "not present", "skipping", - "condition", "marker", "prerequisite", - ] - if any(kw in text for kw in expected_kw): - return "expected" - - return "unexpected" - - # ── Playbook / Molecule Log Analysis ──────────────────────────────────────── def _count_log_issues(logs: str) -> dict: @@ -350,12 +318,6 @@ def add_result( if error: result["error"] = error - # Add skip classification for skipped tests - if normalized_status == "SKIPPED": - result["skip_type"] = _classify_skip( - details or "", error or "" - ) - self.results.append(result) def _add_dict_result(self, payload: dict): @@ -412,13 +374,6 @@ def _add_dict_result(self, payload: dict): if payload.get("markers"): result["markers"] = payload.get("markers") - # Add skip classification for skipped tests - if normalized_status == "SKIPPED": - result["skip_type"] = _classify_skip( - payload.get("details", ""), - payload.get("error", ""), - ) - self.results.append(result) # ── Save & Persist ─────────────────────────────────────────────── diff --git a/test/plugins/omnia_auto/functions/report_html.py b/test/plugins/omnia_auto/functions/report_html.py index c018e250fe..fe3c7a3c66 100644 --- a/test/plugins/omnia_auto/functions/report_html.py +++ b/test/plugins/omnia_auto/functions/report_html.py @@ -100,31 +100,6 @@ def _redact_sensitive(text: str) -> str: return text -# ── Skip Classification ───────────────────────────────────────────────────── - -def _classify_skip(details: str = "", error: str = "") -> str: - """Classify a skipped test.""" - text = ((details or "") + " " + (error or "")).lower() - if not text.strip(): - return "unexpected" - framework_kw = [ - "fixture", "setup failed", "import error", "conftest", - "collection error", "module not found", "no module named", - "parametrize", "setup error", "teardown error", - ] - if any(kw in text for kw in framework_kw): - return "framework" - expected_kw = [ - "not enabled", "not configured", "disabled", "not supported", - "not available", "not applicable", "not installed", "feature not", - "requires", "only applies", "not present", "skipping", - "condition", "marker", "prerequisite", - ] - if any(kw in text for kw in expected_kw): - return "expected" - return "unexpected" - - # ── Log Issue Counter ──────────────────────────────────────────────────────── def _count_log_issues(logs: str) -> dict: @@ -208,7 +183,6 @@ def generate_html(data: Dict[str, Any]) -> str: "details": _redact_sensitive(result.get("details", "")), "error": _redact_sensitive(result.get("error", "")), "server": server_ip, - "skip_type": result.get("skip_type", ""), "tc_id": result.get("tc_id", ""), "run_id": run_id, }) @@ -313,16 +287,6 @@ def generate_html(data: Dict[str, Any]) -> str: has_log = bool(t.get("details") or t.get("error")) log_icon = 'View' if has_log else '—' - # Skip type badge - skip_badge = "" - if status_raw == "SKIPPED" and t.get("skip_type"): - skip_cls = { - "expected": "badge-skip", - "unexpected": "badge-warn", - "framework": "badge-fail", - }.get(t["skip_type"], "badge-skip") - skip_badge = f' {_escape_html(t["skip_type"])}' - # TC ID display tc_prefix = f"[{_escape_html(t['tc_id'])}] " if t.get("tc_id") else "" @@ -347,7 +311,7 @@ def generate_html(data: Dict[str, Any]) -> str: {tc_prefix}{_escape_html(t["test_name"])} - {sl}{skip_badge} + {sl} {t["duration"]:.2f}s {log_icon} {log_section}'''