test: unify architecture-specific model contracts

CMakeLists.txt

@@ -182,12 +182,21 @@ if(BUILD_TESTING)
     add_executable(feature-input-test tests/models/feature-input-test.cpp)
     target_link_libraries(feature-input-test PRIVATE clap nlohmann_json::nlohmann_json)
     target_include_directories(feature-input-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src")
+    # One architecture-parameterized runner, registered per fixture like the
+    # common llama.cpp tokenizer tests. Keep native numerical oracles separate.
+    add_test(NAME model_contract_framework
+        COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/model_contract_test.py")
+    set_tests_properties(model_contract_framework PROPERTIES TIMEOUT 30)
+    function(clap_add_model_contract name architecture suite)
+        add_test(NAME ${name}
+            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/model_tests.py"
+                    --arch ${architecture} --suite ${suite} --cli "$<TARGET_FILE:clap-cli>" ${ARGN})
+    endfunction()
     if(CLAP_BUILD_CLI)
         add_executable(clap2-model-test tests/models/clap2-model-test.cpp)
         target_link_libraries(clap2-model-test PRIVATE clap nlohmann_json::nlohmann_json)
-        add_test(NAME clap2_contract
-            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clap2_invalid.py"
-                    "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/clap2-smoke")
+        clap_add_model_contract(clap2_contract clap2 invalid
+            --build "${CMAKE_CURRENT_BINARY_DIR}" --fixture "${CMAKE_CURRENT_BINARY_DIR}/clap2-smoke")
         set_tests_properties(clap2_contract PROPERTIES TIMEOUT 180 FIXTURES_SETUP clap2_smoke)
         add_executable(clap2-rope-test tests/models/clap2-rope-test.cpp)
         target_include_directories(clap2-rope-test PRIVATE src)
@@ -279,9 +288,8 @@ if(BUILD_TESTING)
                 COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/parity.py"
                         "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/model-fixture")
             set_tests_properties(clap_model_parity PROPERTIES TIMEOUT 600)
-            add_test(NAME clap_model_cli
-                COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/cli.py"
-                        "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/model-fixture")
+            clap_add_model_contract(clap_model_cli clap cli
+                --fixture "${CMAKE_CURRENT_BINARY_DIR}/model-fixture")
             set_tests_properties(clap_model_cli PROPERTIES DEPENDS clap_model_parity TIMEOUT 60)
         endif()
         option(CLAMP3_MODEL_PARITY "Run optional CLaMP3 Transformers oracle tests (torch/transformers/numpy/tokenizers required)" OFF)
@@ -310,19 +318,15 @@ if(BUILD_TESTING)
                         "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture/model-f16.gguf"
                         "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture/fixture-f16.json")
             set_tests_properties(clamp3_model_native_f16 PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120)
-            add_test(NAME clamp3_model_cli
-                COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_cli.py"
-                        "$<TARGET_FILE:clap-cli>" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture")
+            clap_add_model_contract(clamp3_model_cli clamp3 cli
+                --fixture "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture")
             set_tests_properties(clamp3_model_cli PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120)
-            add_test(NAME clamp3_invalid_metadata
-                COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_invalid.py"
-                        "$<TARGET_FILE:clap-cli>" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture")
+            clap_add_model_contract(clamp3_invalid_metadata clamp3 invalid
+                --fixture "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture")
             set_tests_properties(clamp3_invalid_metadata PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120)
         endif()
         option(MERT2_MODEL_PARITY "Run optional official MERT-v2-FullSong oracle tests (torch/torchaudio/transformers/safetensors required)" OFF)
-        add_test(NAME mert_invalid_metadata
-            COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/mert_invalid.py"
-                    "$<TARGET_FILE:clap-cli>")
+        clap_add_model_contract(mert_invalid_metadata mert invalid)
         set_tests_properties(mert_invalid_metadata PROPERTIES TIMEOUT 30)
         option(MERT_MODEL_PARITY "Run optional reviewed official MERT-v1 numerical oracles" OFF)
         set(MERT_REFERENCE_SOURCE "" CACHE PATH "Reviewed pinned MERT-v1 source (modeling_MERT.py/configuration_MERT.py)")
@@ -371,9 +375,8 @@ if(BUILD_TESTING)
                         "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture"
                         "${MERT2_REFERENCE_SOURCE}")
             set_tests_properties(mert2_model_parity PROPERTIES TIMEOUT 600)
-            add_test(NAME mert2_model_cli
-                COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/mert2_cli.py"
-                        "$<TARGET_FILE:clap-cli>" "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture")
+            clap_add_model_contract(mert2_model_cli mert2 cli
+                --fixture "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture")
             set_tests_properties(mert2_model_cli PROPERTIES DEPENDS mert2_model_parity TIMEOUT 180)
         endif()
         add_test(NAME muq_conversion_contract

tests/models/MODEL_TESTS.md

@@ -0,0 +1,65 @@
+# Unified architecture model contracts
+
+`model_tests.py` is the common stdlib-only entry point for model/CLI contracts.
+Following llama.cpp's architecture test and per-model tokenizer registration
+pattern, CTest selects an architecture and passes its generated fixture to one
+runner. This is not a dispatcher to the former standalone scripts: command
+execution, exit/diagnostic checks, JSON vector/frame validation, waveform
+writing and string-metadata GGUF rewriting live in `model_contract.py`.
+Architecture modules supply fixtures, mutations and expected results.
+
+```sh
+python tests/models/model_tests.py --list
+python tests/models/model_contract_test.py
+python tests/models/model_tests.py --arch clap2 --suite invalid \
+    --build "$BUILD" --fixture "$BUILD/clap2-smoke"
+python tests/models/model_tests.py --arch mert --suite invalid --cli "$BUILD/clap"
+python tests/models/model_tests.py --arch mert2 --suite cli \
+    --cli "$BUILD/clap" --fixture "$BUILD/mert2-fixture" --case full-song-frames
+```
+
+`--case` selects case names by substring. Unknown architectures, unsupported
+suites, missing required arguments and filters selecting no cases fail rather
+than silently skipping coverage. `--list` requires neither native binaries nor
+fixtures nor optional Python packages. Failures identify architecture, suite,
+case, native argv, exit status and captured stdout/stderr. Preparation failures
+identify architecture/suite and the underlying error.
+
+## Coverage retained from the standalone runners
+
+| Architecture/suite | Fixture source | Contract cases |
+| --- | --- | --- |
+| `clap/cli` | optional `parity.py` | All text cases, exact dimensions and max absolute error **< 3e-5**; all three invalid argument combinations; 16 kHz resampled versus 48 kHz native silence, including the explicit 32-mel frontend rejection contract. |
+| `clap2/invalid` | stdlib `clap2_fixture.py` | Converter schema/tensor shapes and metadata discovery; complete analytical F32/F16 native public API smoke; all 12 malformed config/processor/tokenizer cases rejected by both converter and native loader; missing and wrong-shape QKV tensors and original diagnostic alternatives. |
+| `clamp3/cli` | optional `clamp3_parity.py` | 48 kHz mono waveform and dimension 6; all three obsolete options rejected with diagnostics; bad-architecture model rejection. |
+| `clamp3/invalid` | optional `clamp3_parity.py` | Negative pad token, negative epsilon and causal decoder mutations, preserving tensor infos/payload and exact expected diagnostics. |
+| `mert/invalid` | stdlib metadata fixture | All 31 native metadata mutations, requiring a **positive** rejection exit and original diagnostic substring; all 44 converter metadata/processor/revision rejections; all seven invalid positional-weight representations; post/pre-norm identity, exact config and source revision preservation. |
+| `mert2/cli` | optional `mert2_parity.py` | Same full-song case and IEEE F32 waveform; pooled dimension and max error **<= fixture tolerance**; frame count, dimension, full value count and max error **<= fixture tolerance**; each native invocation retains its 120-second timeout. |
+
+CTest names, parity-option gates, dependencies, fixture setup and existing
+60/120/180/30-second CTest deadlines remain unchanged. `model_contract_framework`
+adds dependency-free selection/diagnostic, tolerance, GGUF-rewrite and adapter
+inventory regressions. Native public C API calls remain explicit in the
+unchanged C++ oracle executables; this framework does not wrap that API.
+
+## Extending coverage
+
+Register an `Architecture` in `model_architectures/__init__.py`, explicitly
+listing its suites and required fixture/build directory. Its `cli(context)` or
+`invalid(context)` generator yields named `Command` records. Use `Vector` and
+`Frames` for shared output checks, `Diagnostic` when alternative messages have
+different case-sensitivity, and `Compare` for paired native outputs. Reserve
+`Check` for genuinely architecture-specific conversion checks. Keep command
+execution in the shared runner, not in adapters. Register each fixture through
+`clap_add_model_contract` and attach its existing CTest properties explicitly.
+
+The string-only GGUF helper intentionally supports the generated test fixtures,
+not arbitrary GGUF metadata types. No production loader/frontend code changes
+are needed to add an architecture contract.
+
+Official numerical oracles, tokenizer internals, FFT/RoPE tests, bundled-model
+pipeline tests and MuQ conversion/quantization checks remain separate because
+they have distinct algorithms, dependencies or contracts. The unified runner
+does not replace or weaken them. Optional fixtures are still generated by their
+existing parity tests; CTest `DEPENDS` retains ordering semantics, not automatic
+fixture generation when a user selects only an optional CLI test.

tests/models/clap2_SOURCE.md

@@ -66,7 +66,8 @@ little-endian IEEE F32; the runner handles host byte order.
 ## Default stdlib-only regression
 
 ```
-python tests/models/clap2_invalid.py "$BUILD" "$BUILD/clap2-smoke"
+python tests/models/model_tests.py --arch clap2 --suite invalid \
+    --build "$BUILD" --fixture "$BUILD/clap2-smoke"
 ```
 
 This generates complete analytical F32/F16 GGUFs, checks tokenizer/text/token/audio