diff --git a/CMakeLists.txt b/CMakeLists.txt index 1aeaf8e..0301906 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -182,12 +182,21 @@ if(BUILD_TESTING) add_executable(feature-input-test tests/models/feature-input-test.cpp) target_link_libraries(feature-input-test PRIVATE clap nlohmann_json::nlohmann_json) target_include_directories(feature-input-test PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + # One architecture-parameterized runner, registered per fixture like the + # common llama.cpp tokenizer tests. Keep native numerical oracles separate. + add_test(NAME model_contract_framework + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/model_contract_test.py") + set_tests_properties(model_contract_framework PROPERTIES TIMEOUT 30) + function(clap_add_model_contract name architecture suite) + add_test(NAME ${name} + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/model_tests.py" + --arch ${architecture} --suite ${suite} --cli "$" ${ARGN}) + endfunction() if(CLAP_BUILD_CLI) add_executable(clap2-model-test tests/models/clap2-model-test.cpp) target_link_libraries(clap2-model-test PRIVATE clap nlohmann_json::nlohmann_json) - add_test(NAME clap2_contract - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clap2_invalid.py" - "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/clap2-smoke") + clap_add_model_contract(clap2_contract clap2 invalid + --build "${CMAKE_CURRENT_BINARY_DIR}" --fixture "${CMAKE_CURRENT_BINARY_DIR}/clap2-smoke") set_tests_properties(clap2_contract PROPERTIES TIMEOUT 180 FIXTURES_SETUP clap2_smoke) add_executable(clap2-rope-test tests/models/clap2-rope-test.cpp) target_include_directories(clap2-rope-test PRIVATE src) @@ -279,9 +288,8 @@ if(BUILD_TESTING) COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/parity.py" "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/model-fixture") set_tests_properties(clap_model_parity PROPERTIES TIMEOUT 600) - add_test(NAME clap_model_cli - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/cli.py" - "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/model-fixture") + clap_add_model_contract(clap_model_cli clap cli + --fixture "${CMAKE_CURRENT_BINARY_DIR}/model-fixture") set_tests_properties(clap_model_cli PROPERTIES DEPENDS clap_model_parity TIMEOUT 60) endif() option(CLAMP3_MODEL_PARITY "Run optional CLaMP3 Transformers oracle tests (torch/transformers/numpy/tokenizers required)" OFF) @@ -310,19 +318,15 @@ if(BUILD_TESTING) "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture/model-f16.gguf" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture/fixture-f16.json") set_tests_properties(clamp3_model_native_f16 PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120) - add_test(NAME clamp3_model_cli - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_cli.py" - "$" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") + clap_add_model_contract(clamp3_model_cli clamp3 cli + --fixture "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") set_tests_properties(clamp3_model_cli PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120) - add_test(NAME clamp3_invalid_metadata - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_invalid.py" - "$" "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") + clap_add_model_contract(clamp3_invalid_metadata clamp3 invalid + --fixture "${CMAKE_CURRENT_BINARY_DIR}/clamp3-fixture") set_tests_properties(clamp3_invalid_metadata PROPERTIES DEPENDS clamp3_model_parity TIMEOUT 120) endif() option(MERT2_MODEL_PARITY "Run optional official MERT-v2-FullSong oracle tests (torch/torchaudio/transformers/safetensors required)" OFF) - add_test(NAME mert_invalid_metadata - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/mert_invalid.py" - "$") + clap_add_model_contract(mert_invalid_metadata mert invalid) set_tests_properties(mert_invalid_metadata PROPERTIES TIMEOUT 30) option(MERT_MODEL_PARITY "Run optional reviewed official MERT-v1 numerical oracles" OFF) set(MERT_REFERENCE_SOURCE "" CACHE PATH "Reviewed pinned MERT-v1 source (modeling_MERT.py/configuration_MERT.py)") @@ -371,9 +375,8 @@ if(BUILD_TESTING) "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture" "${MERT2_REFERENCE_SOURCE}") set_tests_properties(mert2_model_parity PROPERTIES TIMEOUT 600) - add_test(NAME mert2_model_cli - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/mert2_cli.py" - "$" "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture") + clap_add_model_contract(mert2_model_cli mert2 cli + --fixture "${CMAKE_CURRENT_BINARY_DIR}/mert2-fixture") set_tests_properties(mert2_model_cli PROPERTIES DEPENDS mert2_model_parity TIMEOUT 180) endif() add_test(NAME muq_conversion_contract diff --git a/tests/models/MODEL_TESTS.md b/tests/models/MODEL_TESTS.md new file mode 100644 index 0000000..19f3125 --- /dev/null +++ b/tests/models/MODEL_TESTS.md @@ -0,0 +1,65 @@ +# Unified architecture model contracts + +`model_tests.py` is the common stdlib-only entry point for model/CLI contracts. +Following llama.cpp's architecture test and per-model tokenizer registration +pattern, CTest selects an architecture and passes its generated fixture to one +runner. This is not a dispatcher to the former standalone scripts: command +execution, exit/diagnostic checks, JSON vector/frame validation, waveform +writing and string-metadata GGUF rewriting live in `model_contract.py`. +Architecture modules supply fixtures, mutations and expected results. + +```sh +python tests/models/model_tests.py --list +python tests/models/model_contract_test.py +python tests/models/model_tests.py --arch clap2 --suite invalid \ + --build "$BUILD" --fixture "$BUILD/clap2-smoke" +python tests/models/model_tests.py --arch mert --suite invalid --cli "$BUILD/clap" +python tests/models/model_tests.py --arch mert2 --suite cli \ + --cli "$BUILD/clap" --fixture "$BUILD/mert2-fixture" --case full-song-frames +``` + +`--case` selects case names by substring. Unknown architectures, unsupported +suites, missing required arguments and filters selecting no cases fail rather +than silently skipping coverage. `--list` requires neither native binaries nor +fixtures nor optional Python packages. Failures identify architecture, suite, +case, native argv, exit status and captured stdout/stderr. Preparation failures +identify architecture/suite and the underlying error. + +## Coverage retained from the standalone runners + +| Architecture/suite | Fixture source | Contract cases | +| --- | --- | --- | +| `clap/cli` | optional `parity.py` | All text cases, exact dimensions and max absolute error **< 3e-5**; all three invalid argument combinations; 16 kHz resampled versus 48 kHz native silence, including the explicit 32-mel frontend rejection contract. | +| `clap2/invalid` | stdlib `clap2_fixture.py` | Converter schema/tensor shapes and metadata discovery; complete analytical F32/F16 native public API smoke; all 12 malformed config/processor/tokenizer cases rejected by both converter and native loader; missing and wrong-shape QKV tensors and original diagnostic alternatives. | +| `clamp3/cli` | optional `clamp3_parity.py` | 48 kHz mono waveform and dimension 6; all three obsolete options rejected with diagnostics; bad-architecture model rejection. | +| `clamp3/invalid` | optional `clamp3_parity.py` | Negative pad token, negative epsilon and causal decoder mutations, preserving tensor infos/payload and exact expected diagnostics. | +| `mert/invalid` | stdlib metadata fixture | All 31 native metadata mutations, requiring a **positive** rejection exit and original diagnostic substring; all 44 converter metadata/processor/revision rejections; all seven invalid positional-weight representations; post/pre-norm identity, exact config and source revision preservation. | +| `mert2/cli` | optional `mert2_parity.py` | Same full-song case and IEEE F32 waveform; pooled dimension and max error **<= fixture tolerance**; frame count, dimension, full value count and max error **<= fixture tolerance**; each native invocation retains its 120-second timeout. | + +CTest names, parity-option gates, dependencies, fixture setup and existing +60/120/180/30-second CTest deadlines remain unchanged. `model_contract_framework` +adds dependency-free selection/diagnostic, tolerance, GGUF-rewrite and adapter +inventory regressions. Native public C API calls remain explicit in the +unchanged C++ oracle executables; this framework does not wrap that API. + +## Extending coverage + +Register an `Architecture` in `model_architectures/__init__.py`, explicitly +listing its suites and required fixture/build directory. Its `cli(context)` or +`invalid(context)` generator yields named `Command` records. Use `Vector` and +`Frames` for shared output checks, `Diagnostic` when alternative messages have +different case-sensitivity, and `Compare` for paired native outputs. Reserve +`Check` for genuinely architecture-specific conversion checks. Keep command +execution in the shared runner, not in adapters. Register each fixture through +`clap_add_model_contract` and attach its existing CTest properties explicitly. + +The string-only GGUF helper intentionally supports the generated test fixtures, +not arbitrary GGUF metadata types. No production loader/frontend code changes +are needed to add an architecture contract. + +Official numerical oracles, tokenizer internals, FFT/RoPE tests, bundled-model +pipeline tests and MuQ conversion/quantization checks remain separate because +they have distinct algorithms, dependencies or contracts. The unified runner +does not replace or weaken them. Optional fixtures are still generated by their +existing parity tests; CTest `DEPENDS` retains ordering semantics, not automatic +fixture generation when a user selects only an optional CLI test. diff --git a/tests/models/clamp3_cli.py b/tests/models/clamp3_cli.py deleted file mode 100644 index 02bcb62..0000000 --- a/tests/models/clamp3_cli.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -"""CLaMP3 waveform CLI; feature/source fallbacks are deliberately removed.""" -import json -from pathlib import Path -import subprocess -import sys -import wave -binary,fixture=map(Path,sys.argv[1:3]) -path=fixture/'input.wav' -with wave.open(str(path),'wb') as output: - output.setnchannels(1);output.setsampwidth(2);output.setframerate(48000) - output.writeframes(b'\0\0'*48000) -result=subprocess.run([str(binary), 'embedding','-m',str(fixture/'model.gguf'),'-a',str(path),'-t','2'],capture_output=True,text=True) -assert result.returncode==0,result.stderr -assert len(json.loads(result.stdout))==6 -for args in [['--audio-features','obsolete.npy'],['--mert-model',str(fixture/'model.gguf')], - ['-a',str(path),'--crop-offset','0']]: - result=subprocess.run([str(binary), 'embedding','-m',str(fixture/'model.gguf'),*args],capture_output=True,text=True) - assert result.returncode!=0 and result.stderr,args -result=subprocess.run([str(binary), 'embedding','-m',str(fixture/'bad-architecture.gguf'),'--text','a'],capture_output=True,text=True) -assert result.returncode!=0 and 'architecture' in result.stderr -print('CLaMP3 bundled waveform CLI and obsolete-option rejection passed') diff --git a/tests/models/clamp3_invalid.py b/tests/models/clamp3_invalid.py deleted file mode 100644 index c91279e..0000000 --- a/tests/models/clamp3_invalid.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -"""Reject malformed architecture metadata before graph execution.""" -import json -from pathlib import Path -import struct -import subprocess -import sys - -binary, fixture = Path(sys.argv[1]), Path(sys.argv[2]) -source = (fixture / 'model.gguf').read_bytes() -_, version, tensors, count = struct.unpack_from('<4sIQQ', source) -assert version == 3 -position = 24 - -def read_string(): - global position - size, = struct.unpack_from('0 and message in result.stderr, (key,result.returncode,result.stderr) - print(f'{len(cases)} invalid MERT metadata cases rejected') - -# Converter validation must also run without torch and before checkpoint loading. -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from conversion import mert as converter -preprocessor = dict(feature_extractor_type='Wav2Vec2FeatureExtractor', feature_size=1, - sampling_rate=24000, do_normalize=True, padding_value=0, - padding_side='right', return_attention_mask=True) -revision = '12af15fef9d0ac838c3f475bfbbf26d2060dd4f5' -count = 0 -with tempfile.TemporaryDirectory(prefix='mert-converter-invalid-') as temp: - source = Path(temp) - def reject(config, processor, sha): - global count - (source/'config.json').write_text(json.dumps(config)) - (source/'preprocessor_config.json').write_text(json.dumps(processor)) - try: - converter.mert_inputs(source, sha, object()) - except ValueError: - count += 1 - else: - raise AssertionError('converter accepted invalid metadata') - for key, value, _ in cases: - config = copy.deepcopy(base); config[key] = value - reject(config, preprocessor, revision) - for key, value in [('sampling_rate',48000), ('do_normalize',False), - ('feature_size',2), ('return_attention_mask',False), - ('feature_size',True), ('return_attention_mask',1), - ('sampling_rate',24000.0), ('do_normalize',1), ('padding_value',False)]: - processor = dict(preprocessor); processor[key] = value - reject(base, processor, revision) - for sha in [None, '', 'a'*39, 'x'*40]: - reject(base, preprocessor, sha) -print(f'{count} invalid MERT converter cases rejected before checkpoint loading') - -# Only the checkpoint-loading boundary is stubbed: test the real converter's -# positional-weight representation validation without making Torch a default dependency. -class Checkpoint: - def __init__(self, names): - self.weights = {'encoder.pos_conv_embed.conv.'+name: object() for name in names} - def load(self, *args, **kwargs): - assert kwargs['weights_only'] and kwargs['mmap'] - return dict(self.weights) - def _weight_norm(self, *args): - raise AssertionError('invalid weight-norm pair reached materialization') - -representations = [ - ['weight_g'], ['weight_v'], ['parametrizations.weight.original0'], - ['parametrizations.weight.original1'], ['weight','weight_g','weight_v'], - ['weight_g','weight_v','parametrizations.weight.original0','parametrizations.weight.original1'], - [], -] -with tempfile.TemporaryDirectory(prefix='mert-converter-weights-') as temp: - source = Path(temp) - (source/'config.json').write_text(json.dumps(base)) - (source/'preprocessor_config.json').write_text(json.dumps(preprocessor)) - for names in representations: - try: - converter.mert_inputs(source, revision, Checkpoint(names)) - except ValueError: - pass - else: - raise AssertionError(f'accepted invalid positional weight representation: {names}') -print(f'{len(representations)} invalid MERT positional weight representations rejected') - -with tempfile.TemporaryDirectory(prefix='mert-converter-identity-') as temp: - source = Path(temp) - (source/'preprocessor_config.json').write_text(json.dumps(preprocessor)) - for stable, identity in [(False, 'm-a-p/MERT-v1-95M'), (True, 'm-a-p/MERT-v1-330M'), - (True, None)]: - config = dict(base, do_stable_layer_norm=stable) - if identity is not None: - config['_name_or_path'] = identity - (source/'config.json').write_text(json.dumps(config)) - _, metadata = converter.mert_inputs(source, revision, Checkpoint(['weight'])) - assert metadata['general.name'] == (identity or 'MERT-v1') - assert json.loads(metadata['mert.config']) == config - assert metadata['mert.source_revision'] == revision -print('MERT post/pre-norm identity, exact config and revision preserved') diff --git a/tests/models/model_architectures/__init__.py b/tests/models/model_architectures/__init__.py new file mode 100644 index 0000000..4f85cf2 --- /dev/null +++ b/tests/models/model_architectures/__init__.py @@ -0,0 +1,27 @@ +"""Architecture fixtures/adapters for the common model contract runner. + +Keep optional Torch/Transformers oracle imports out of this registry. Adding an +architecture means supplying case data here, not another CLI test entry point. +""" +from dataclasses import dataclass +from importlib import import_module + + +@dataclass(frozen=True) +class Architecture: + module: str + suites: tuple + fixture_suites: tuple = () + build_suites: tuple = () + + def cases(self, suite, context): + return getattr(import_module(f"model_architectures.{self.module}"), suite)(context) + + +ARCHITECTURES = { + "clap": Architecture("clap", ("cli",), ("cli",)), + "clap2": Architecture("clap2", ("invalid",), ("invalid",), ("invalid",)), + "clamp3": Architecture("clamp3", ("cli", "invalid"), ("cli", "invalid")), + "mert": Architecture("mert", ("invalid",)), + "mert2": Architecture("mert2", ("cli",), ("cli",)), +} diff --git a/tests/models/model_architectures/clamp3.py b/tests/models/model_architectures/clamp3.py new file mode 100644 index 0000000..eb6c5a7 --- /dev/null +++ b/tests/models/model_architectures/clamp3.py @@ -0,0 +1,32 @@ +"""CLaMP3 fixture data; waveform-only CLI and malformed config contracts.""" +import json + +from model_contract import Command, Vector, metadata_model, pcm_silence, read_metadata_model + + +def cli(ctx): + path = ctx.fixture / "input.wav" + pcm_silence(path, 48000, 48000) + model = ctx.fixture / "model.gguf" + yield Command("waveform", ctx.embedding(model, "-a", path, "-t", "2"), output=Vector(6)) + for name, args in ( + ("obsolete-audio-features", ["--audio-features", "obsolete.npy"]), + ("obsolete-mert-model", ["--mert-model", model]), + ("obsolete-crop-offset", ["-a", path, "--crop-offset", "0"])): + yield Command(name, ctx.embedding(model, *args), success=False, nonempty_stderr=True) + yield Command("bad-architecture", ctx.embedding(ctx.fixture / "bad-architecture.gguf", "--text", "a"), + success=False, diagnostics=("architecture",)) + + +def invalid(ctx): + metadata, tensors, infos, payload = read_metadata_model(ctx.fixture / "model.gguf") + for name, branch, key, value, expected in ( + ("negative-pad", "text_config", "pad_token_id", -1, "padding token"), + ("negative-epsilon", "audio_config", "layer_norm_eps", -1, "layer norm epsilon"), + ("causal-decoder", "text_config", "is_decoder", True, "decoder")): + config = json.loads(metadata["clamp3.config"]) + config[branch][key] = value + changed = dict(metadata, **{"clamp3.config": json.dumps(config)}) + path = ctx.fixture / (name + ".gguf") + metadata_model(path, changed, tensors, infos, payload) + yield Command(name, ctx.embedding(path, "--text", "ab"), success=False, diagnostics=(expected,)) diff --git a/tests/models/model_architectures/clap.py b/tests/models/model_architectures/clap.py new file mode 100644 index 0000000..5dd0309 --- /dev/null +++ b/tests/models/model_architectures/clap.py @@ -0,0 +1,36 @@ +"""LAION CLAP text parity and waveform resampling contract.""" +import json + +from model_contract import Command, Compare, Vector, pcm_silence + + +def cli(ctx): + reference = json.loads((ctx.fixture / "fixture.json").read_text()) + model = ctx.fixture / "model.gguf" + for index, case in enumerate(reference["text_cases"]): + yield Command(f"text-{index}", ctx.embedding(model, "--text", case["text"]), + output=Vector(len(case["expected"]), case["expected"], 3e-5)) + for index, args in enumerate(([], ["--model", model], + ["--model", model, "--text", "x", "--audio", "x.wav"])): + yield Command(f"invalid-arguments-{index}", [ctx.cli, "embedding", *args], success=False) + + wav = ctx.fixture / "wrong-rate.wav" + control_wav = ctx.fixture / "native-rate.wav" + pcm_silence(wav, 16000, 100) + pcm_silence(control_wav, 48000, 300) + + def resampling(results): + # Tiny 32-mel models intentionally lack the waveform frontend. Compare + # exit/stdout AND the stable error contract, never backend timing logs. + actual, control = results + assert (actual.returncode, actual.stdout) == (control.returncode, control.stdout), ( + f"resampled/native mismatch: {actual!r}\n{control!r}") + if actual.returncode: + for result in results: + assert "waveform frontend requires 64 mel bins" in result.stderr, result.stderr + else: + Vector(len(reference["text_cases"][0]["expected"]))(json.loads(actual.stdout)) + + yield Compare("waveform-resampling", [ + Command(str(path), ctx.embedding(model, "--audio", path), success=None) + for path in (wav, control_wav)], resampling) diff --git a/tests/models/model_architectures/clap2.py b/tests/models/model_architectures/clap2.py new file mode 100644 index 0000000..e80da88 --- /dev/null +++ b/tests/models/model_architectures/clap2.py @@ -0,0 +1,98 @@ +"""Analytical CLAP2 fixtures and conversion/native rejection case data.""" +import copy +import json +from pathlib import Path +import sys + +from clap2_fixture import generate, model_data, write_gguf +from model_contract import Check, Command, Diagnostic + + +def invalid(ctx): + sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + from conversion.clap2 import clap2_inputs, expected_tensor_shapes, validate_config, validate_tokenizer + + output = ctx.fixture + metadata, tensors = generate(output) + config, processor, tokenizer, _ = model_data() + + def conversion_contract(): + validate_config(config, processor) + validate_tokenizer(tokenizer, config) + assert expected_tensor_shapes(config) == {name: shape for name, (shape, _) in tensors.items()} + for name, data in (("config.json", config), ("processor_config.json", processor), ("tokenizer.json", tokenizer)): + (output / name).write_text(json.dumps(data)) + # Metadata discovery is independent of safetensors payload parsing. The + # native smoke below uses real complete GGUFs, not these placeholder bytes. + (output / "model.safetensors").write_bytes(b"") + _, converted_metadata, _ = clap2_inputs(output) + for field in ("clap2.config", "clap2.processor_config", "clap2.tokenizer"): + assert json.loads(metadata[field]) == json.loads(converted_metadata[field]) + + yield Check("conversion-metadata", conversion_contract) + # C API calls remain visible in clap2-model-test.cpp. No method facade. + executable = ctx.build / ("clap2-model-test.exe" if sys.platform == "win32" else "clap2-model-test") + + def native(path): + return [executable, path, output / "fixture.json"] + + for dtype in ("f32", "f16"): + yield Command(f"analytical-{dtype}", native(output / f"model-{dtype}.gguf")) + + cases = [] + + def config_case(label, field, value, section="audio_config"): + changed = copy.deepcopy(config) + changed[section][field] = value + cases.append((label, changed, processor, tokenizer)) + + config_case("zero heads", "num_attention_heads", 0) + config_case("fractional width", "hidden_size", 8.5) + config_case("unsupported activation", "hidden_act", "gelu") + config_case("unsupported audio rope", "rope_parameters", dict(rope_type="default", rope_theta=10000.0)) + config_case("unsupported text rope", "rope_parameters", dict(rope_type="axial", rope_theta=10000.0), "text_config") + config_case("audio QK normalization", "use_qk_norm", True) + config_case("text QK normalization", "use_qk_norm", False, "text_config") + config_case("out-of-range special id", "bos_token_id", 260, "text_config") + changed = copy.deepcopy(processor) + changed["feature_extractor"]["feature_size"] = 64 + cases.append(("wrong mel height", config, changed, tokenizer)) + changed = copy.deepcopy(processor) + changed["feature_extractor"]["max_num_patches"] = 1504 + cases.append(("training token budget is not a crop policy", config, changed, tokenizer)) + changed = copy.deepcopy(tokenizer) + changed["model"]["type"] = "WordPiece" + cases.append(("wrong tokenizer model", config, processor, changed)) + changed = copy.deepcopy(tokenizer) + changed["post_processor"]["single"][0]["SpecialToken"]["id"] = "<|eos|>" + cases.append(("wrong special template", config, processor, changed)) + + for index, (label, c, p, t) in enumerate(cases): + def reject(c=c, p=p, t=t, label=label): + try: + validate_config(c, p) + validate_tokenizer(t, c) + except (ValueError, KeyError, TypeError): + return + raise AssertionError(f"converter accepted {label}") + + yield Check(f"converter-{label}", reject) + bad_metadata = dict(metadata, **{"clap2.config": json.dumps(c), + "clap2.processor_config": json.dumps(p), + "clap2.tokenizer": json.dumps(t)}) + path = output / f"invalid-{index}.gguf" + write_gguf(path, bad_metadata, tensors) + yield Command(f"native-{label}", native(path), success=False, + diagnostics=("clap2", "token", "bpe", "template"), + diagnostic_any=True, diagnostic_lower=True) + for label in ("missing", "wrong-shape"): + bad_tensors = dict(tensors) + target = "text_model.encoder.layers.0.self_attn.qkv_proj.weight" + if label == "missing": + del bad_tensors[target] + else: + bad_tensors[target] = ((23, 8), [0.0] * (23 * 8)) + path = output / f"invalid-tensor-{label}.gguf" + write_gguf(path, metadata, bad_tensors) + yield Command(f"tensor-{label}", native(path), success=False, + diagnostics=(Diagnostic("tensor", ignore_case=True), "qkv_proj"), diagnostic_any=True) diff --git a/tests/models/model_architectures/mert.py b/tests/models/model_architectures/mert.py new file mode 100644 index 0000000..35e3ca6 --- /dev/null +++ b/tests/models/model_architectures/mert.py @@ -0,0 +1,128 @@ +"""MERT metadata case table plus unique converter checkpoint-boundary checks.""" +import copy +import json +from pathlib import Path +import sys +import tempfile + +from model_contract import Check, Command, metadata_model + +base=dict(model_type='mert_model',sample_rate=24000,feature_extractor_cqt=False, + deepnorm=False,do_stable_layer_norm=False,feat_extract_norm='group',feat_proj_layer_norm=True, + hidden_act='gelu',feat_extract_activation='gelu',hidden_size=16,intermediate_size=32, + num_hidden_layers=2,num_attention_heads=2,num_conv_pos_embeddings=8, + num_conv_pos_embedding_groups=2,conv_dim=[8]*7,conv_kernel=[10,3,3,3,3,2,2], + conv_stride=[5,2,2,2,2,2,2],layer_norm_eps=1e-5,conv_bias=False) + +cases=[('sample_rate',48000,'24 kHz'),('hidden_size',0,'invalid MERT'), + ('num_attention_heads',3,'groups'),('num_conv_pos_embedding_groups',3,'groups'), + ('num_conv_pos_embeddings',0,'invalid MERT'),('intermediate_size',2.5,'invalid MERT'), + ('num_hidden_layers',True,'invalid MERT'),('conv_dim',[],'geometry'), + ('conv_kernel',[-1]*7,'dimension'),('conv_stride',[0]*7,'dimension'), + ('conv_dim',[8.5]*7,'dimension'),('conv_stride',[100000]*7,'receptive'), + ('layer_norm_eps',0,'layer_norm_eps'),('feature_extractor_cqt',True,'variant'), + ('deepnorm',True,'variant'),('do_stable_layer_norm',1,'invalid MERT boolean'), + ('do_stable_layer_norm','true','invalid MERT boolean'), + ('feat_extract_norm','layer','variant'),('hidden_act','relu','variant'), + ('conv_pos_batch_norm',True,'variant'),('feat_proj_layer_norm',False,'variant'), + ('num_feat_extract_layers',6,'convolution geometry'), + ('num_feat_extract_layers',7.5,'convolution geometry'), + ('hidden_size',2**32+16,'invalid MERT'), + ('conv_dim',[2**32+8]*7,'dimension'), + ('conv_bias',0,'invalid MERT boolean'), + ('feat_proj_layer_norm',1,'invalid MERT boolean'), + ('feature_extractor_cqt',0,'invalid MERT boolean'), + ('layer_norm_eps',True,'layer_norm_eps'), + ('layer_norm_eps',1e100,'layer_norm_eps'), + ('layer_norm_eps',1e-100,'layer_norm_eps')] + + +def invalid(ctx): + for index, (key, value, message) in enumerate(cases): + config = copy.deepcopy(base) + config[key] = value + target = ctx.work / f"invalid-{index}.gguf" + metadata_model(target, {"general.architecture": "mert", "mert.config": json.dumps(config)}) + yield Command(f"metadata-{index}-{key}", ctx.embedding(target, "--text", "invalid"), + success=False, positive_exit=True, diagnostics=(message,)) + yield Check("converter-contract", conversion_contract) + + +def conversion_contract(): + # Converter validation must also run without torch and before checkpoint loading. + sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + from conversion import mert as converter + preprocessor = dict(feature_extractor_type='Wav2Vec2FeatureExtractor', feature_size=1, + sampling_rate=24000, do_normalize=True, padding_value=0, + padding_side='right', return_attention_mask=True) + revision = '12af15fef9d0ac838c3f475bfbbf26d2060dd4f5' + count = 0 + with tempfile.TemporaryDirectory(prefix='mert-converter-invalid-') as temp: + source = Path(temp) + def reject(config, processor, sha): + nonlocal count + (source/'config.json').write_text(json.dumps(config)) + (source/'preprocessor_config.json').write_text(json.dumps(processor)) + try: + converter.mert_inputs(source, sha, object()) + except ValueError: + count += 1 + else: + raise AssertionError('converter accepted invalid metadata') + for key, value, _ in cases: + config = copy.deepcopy(base); config[key] = value + reject(config, preprocessor, revision) + for key, value in [('sampling_rate',48000), ('do_normalize',False), + ('feature_size',2), ('return_attention_mask',False), + ('feature_size',True), ('return_attention_mask',1), + ('sampling_rate',24000.0), ('do_normalize',1), ('padding_value',False)]: + processor = dict(preprocessor); processor[key] = value + reject(base, processor, revision) + for sha in [None, '', 'a'*39, 'x'*40]: + reject(base, preprocessor, sha) + print(f'{count} invalid MERT converter cases rejected before checkpoint loading') + + # Only the checkpoint-loading boundary is stubbed: test the real converter's + # positional-weight representation validation without making Torch a default dependency. + class Checkpoint: + def __init__(self, names): + self.weights = {'encoder.pos_conv_embed.conv.'+name: object() for name in names} + def load(self, *args, **kwargs): + assert kwargs['weights_only'] and kwargs['mmap'] + return dict(self.weights) + def _weight_norm(self, *args): + raise AssertionError('invalid weight-norm pair reached materialization') + + representations = [ + ['weight_g'], ['weight_v'], ['parametrizations.weight.original0'], + ['parametrizations.weight.original1'], ['weight','weight_g','weight_v'], + ['weight_g','weight_v','parametrizations.weight.original0','parametrizations.weight.original1'], + [], + ] + with tempfile.TemporaryDirectory(prefix='mert-converter-weights-') as temp: + source = Path(temp) + (source/'config.json').write_text(json.dumps(base)) + (source/'preprocessor_config.json').write_text(json.dumps(preprocessor)) + for names in representations: + try: + converter.mert_inputs(source, revision, Checkpoint(names)) + except ValueError: + pass + else: + raise AssertionError(f'accepted invalid positional weight representation: {names}') + print(f'{len(representations)} invalid MERT positional weight representations rejected') + + with tempfile.TemporaryDirectory(prefix='mert-converter-identity-') as temp: + source = Path(temp) + (source/'preprocessor_config.json').write_text(json.dumps(preprocessor)) + for stable, identity in [(False, 'm-a-p/MERT-v1-95M'), (True, 'm-a-p/MERT-v1-330M'), + (True, None)]: + config = dict(base, do_stable_layer_norm=stable) + if identity is not None: + config['_name_or_path'] = identity + (source/'config.json').write_text(json.dumps(config)) + _, metadata = converter.mert_inputs(source, revision, Checkpoint(['weight'])) + assert metadata['general.name'] == (identity or 'MERT-v1') + assert json.loads(metadata['mert.config']) == config + assert metadata['mert.source_revision'] == revision + print('MERT post/pre-norm identity, exact config and revision preserved') diff --git a/tests/models/model_architectures/mert2.py b/tests/models/model_architectures/mert2.py new file mode 100644 index 0000000..1ac30e8 --- /dev/null +++ b/tests/models/model_architectures/mert2.py @@ -0,0 +1,17 @@ +"""MERT2 full-song fixture adapter; pooled and frame numerical checks.""" +import json + +from model_contract import Command, Frames, Vector, float_wave + + +def cli(ctx): + fixture = json.loads((ctx.fixture / "fixture.json").read_text()) + case = next(value for value in fixture["waveform_cases"] if value["full_song"]) + wav = ctx.fixture / "full-song.wav" + float_wave(wav, fixture["sample_rate"], case["waveform"]) + args = ctx.embedding(ctx.fixture / "model.gguf", "--threads", "2", "--audio", wav) + yield Command("full-song-pooled", args, timeout=120, + output=Vector(fixture["hidden_size"], case["expected_pooled"], + fixture["tolerance"], inclusive=True)) + yield Command("full-song-frames", [*args, "--frames"], timeout=120, + output=Frames(case["frames"], fixture["hidden_size"], case["expected_frames"], fixture["tolerance"])) diff --git a/tests/models/model_contract.py b/tests/models/model_contract.py new file mode 100644 index 0000000..4ed7dd1 --- /dev/null +++ b/tests/models/model_contract.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Shared architecture contract execution; no model/converter dependencies. + +Adapters describe real native commands and expected outputs, not subprocess +wrappers. Numerical oracle executables keep calling the public C API directly. +""" +from dataclasses import dataclass +import json +from pathlib import Path +import struct +import subprocess +from typing import Callable, Optional, Sequence, Union +import wave + + +@dataclass +class Vector: + dimension: int + expected: Optional[Sequence[float]] = None + tolerance: float = 0.0 + inclusive: bool = False + + def __call__(self, actual): + assert isinstance(actual, list), f"expected vector, got {type(actual).__name__}" + assert len(actual) == self.dimension, (len(actual), self.dimension) + if self.expected is not None: + assert len(self.expected) == self.dimension, "reference vector dimension mismatch" + error = max(abs(a - b) for a, b in zip(actual, self.expected)) + assert (error <= self.tolerance if self.inclusive else error < self.tolerance), ( + f"max error {error}, tolerance {'<=' if self.inclusive else '<'} {self.tolerance}") + + +@dataclass +class Frames: + frames: int + dimension: int + expected: Sequence[float] + tolerance: float + + def __call__(self, actual): + assert actual["frames"] == self.frames, (actual["frames"], self.frames) + assert actual["dimension"] == self.dimension, (actual["dimension"], self.dimension) + Vector(self.frames * self.dimension, self.expected, self.tolerance, inclusive=True)(actual["values"]) + + +@dataclass +class Diagnostic: + text: str + ignore_case: bool = False + + def matches(self, stderr): + return self.text in (stderr.lower() if self.ignore_case else stderr) + + +@dataclass +class Command: + name: str + args: Sequence[object] + # None is reserved for paired contracts where both outcomes are valid. + success: Optional[bool] = True + diagnostics: Sequence[Union[str, Diagnostic]] = () + # Some pre-existing loader contracts accept any of several diagnostics. + diagnostic_any: bool = False + diagnostic_lower: bool = False + positive_exit: bool = False + nonempty_stderr: bool = False + output: Optional[Callable] = None + timeout: Optional[float] = None + + def run(self): + argv = [str(arg) for arg in self.args] + try: + result = subprocess.run(argv, capture_output=True, text=True, timeout=self.timeout) + except subprocess.TimeoutExpired as error: + raise AssertionError(f"command timed out after {self.timeout}s: {argv!r}\n" + f"stdout: {error.stdout!r}\nstderr: {error.stderr!r}") from error + detail = (f"command: {argv!r}\nexit: {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}") + try: + if self.success is True: + assert result.returncode == 0, "expected success" + elif self.success is False: + if self.positive_exit: + assert result.returncode > 0, "expected positive rejection exit (not a signal)" + else: + assert result.returncode != 0, "expected rejection" + if self.nonempty_stderr: + assert result.stderr, "expected stderr diagnostic" + if self.diagnostics: + text = result.stderr.lower() if self.diagnostic_lower else result.stderr + matches = [word.matches(result.stderr) if isinstance(word, Diagnostic) else word in text + for word in self.diagnostics] + assert (any(matches) if self.diagnostic_any else all(matches)), ( + f"expected {'any' if self.diagnostic_any else 'all'} diagnostics: {self.diagnostics!r}") + if self.output is not None: + self.output(json.loads(result.stdout)) + except (AssertionError, ValueError, KeyError, TypeError) as error: + raise AssertionError(f"{error}\n{detail}") from error + return result + + +@dataclass +class Compare: + name: str + commands: Sequence[Command] + check: Callable + + def run(self): + results = [command.run() for command in self.commands] + try: + self.check(results) + except Exception as error: + raise AssertionError(f"{error}\npaired command results: {results!r}") from error + + +@dataclass +class Check: + """Architecture-specific converter/paired-output checks, never a C API facade.""" + name: str + check: Callable + + def run(self): + self.check() + + +@dataclass +class Context: + cli: Optional[Path] = None + build: Optional[Path] = None + fixture: Optional[Path] = None + work: Optional[Path] = None + + def embedding(self, model, *args): + return [self.cli, "embedding", "--model", model, *args] + + +def run_cases(architecture, suite, cases, case_filter=None): + selected = 0 + for case in cases: + if case_filter is not None and case_filter not in case.name: + continue + selected += 1 + label = f"{architecture}/{suite}/{case.name}" + try: + case.run() + except Exception as error: + raise AssertionError(f"{label}: {error}") from error + print(f"PASS {label}", flush=True) + if not selected: + raise ValueError(f"no cases matched {architecture}/{suite}: {case_filter!r}") + print(f"{architecture}/{suite}: {selected} cases passed", flush=True) + return selected + + +def pcm_silence(path, rate, samples): + with wave.open(str(path), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(rate) + output.writeframes(b"\0\0" * samples) + + +def float_wave(path, rate, waveform): + samples = struct.pack("<" + "f" * len(waveform), *waveform) + path.write_bytes(b"RIFF" + struct.pack("