diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c9af8f..a93542a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -188,7 +188,18 @@ if(BUILD_TESTING) add_test(NAME clap2_contract COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clap2_invalid.py" "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/clap2-smoke") - set_tests_properties(clap2_contract PROPERTIES TIMEOUT 180) + set_tests_properties(clap2_contract PROPERTIES TIMEOUT 180 FIXTURES_SETUP clap2_smoke) + add_executable(clap2-rope-test tests/models/clap2-rope-test.cpp) + target_include_directories(clap2-rope-test PRIVATE src) + target_link_libraries(clap2-rope-test PRIVATE clap ggml::ggml nlohmann_json::nlohmann_json) + add_test(NAME clap2_rope + COMMAND clap2-rope-test "${CMAKE_CURRENT_BINARY_DIR}/clap2-smoke/model-f32.gguf") + set_tests_properties(clap2_rope PROPERTIES + FIXTURES_REQUIRED clap2_smoke TIMEOUT 60 ENVIRONMENT "OMP_NUM_THREADS=2") + add_test(NAME clap2_oracle_config + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clap2_oracle_test.py") + add_test(NAME clap2_sizes + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clap2_sizes.py") option(CLAP2_MODEL_PARITY "Run optional pinned open_clip CLAP2 numerical oracles" OFF) set(CLAP2_REFERENCE_SOURCE "" CACHE PATH "Pinned official open_clip source directory") set(CLAP2_REFERENCE_ASSETS "" CACHE PATH "Pinned CLAP2 config/tokenizer/checkpoint-header assets") @@ -201,7 +212,15 @@ if(BUILD_TESTING) COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clap2_parity.py" "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/clap2-fixture" "${CLAP2_REFERENCE_SOURCE}" "${CLAP2_REFERENCE_ASSETS}") - set_tests_properties(clap2_model_parity PROPERTIES TIMEOUT 600) + set_tests_properties(clap2_model_parity PROPERTIES TIMEOUT 600 RESOURCE_LOCK clap2_oracles) + foreach(profile IN ITEMS xxs-head56 asymmetric-head64) + add_test(NAME clap2_model_parity_${profile} + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clap2_parity.py" + "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/clap2-fixture-${profile}" + "${CLAP2_REFERENCE_SOURCE}" "${CLAP2_REFERENCE_ASSETS}" --profile "${profile}") + set_tests_properties(clap2_model_parity_${profile} PROPERTIES + TIMEOUT 600 RESOURCE_LOCK clap2_oracles) + endforeach() endif() add_test(NAME clamp3_bundle_contract COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/models/clamp3_bundle_contract.py" "$") diff --git a/README.md b/README.md index d71de43..a5155dd 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,11 @@ C++20/ggml implementations of audio embedding inference features: | Model | Text | Audio | Embedding dimensions | Sample rate | | --- | --- | --- | --- | --- | | [CLAP](https://huggingface.co/laion/larger_clap_music) | ✅ | ✅ | 512 | 48 kHz | -| [CLAP2](https://huggingface.co/laion/clap2-large-audioxl-s1B) | ✅ | ✅ | 960 | 32 kHz | +| [CLAP2 Large](https://huggingface.co/laion/clap2-large-audioxl-s1B) | ✅ | ✅ | 960 | 32 kHz | +| [CLAP2 Medium](https://huggingface.co/laion/clap2-medium-audioxl-s1B) | ✅ | ✅ | 640 | 32 kHz | +| [CLAP2 Small](https://huggingface.co/laion/clap2-small-audioxl-s1B) | ✅ | ✅ | 512 | 32 kHz | +| [CLAP2 XS](https://huggingface.co/laion/clap2-xs-audioxl-s1B) | ✅ | ✅ | 320 | 32 kHz | +| [CLAP2 XXS](https://huggingface.co/laion/clap2-xxs-audioxl-s1B) | ✅ | ✅ | 224 | 32 kHz | | [CLaMP3](https://github.com/sanderwood/clamp3) | ✅ | ✅ | 768 | 24 kHz | | [MERT v1 95M](https://huggingface.co/m-a-p/MERT-v1-95M) | ❌ | ✅ | 768 | 24 kHz | | [MERT v1 330M](https://huggingface.co/m-a-p/MERT-v1-330M) | ❌ | ✅ | 1,024 | 24 kHz | diff --git a/conversion/clap2.py b/conversion/clap2.py index 44f102f..b3c0b11 100644 --- a/conversion/clap2.py +++ b/conversion/clap2.py @@ -15,8 +15,20 @@ from .base import ModelBase, string -SOURCE_REPO = 'laion/clap2-large-audioxl-s1B' -SOURCE_REVISION = 'e836feaa79cd1f7de939fe77065819f48fc44e4c' +# Published tower dimensions (width, FFN width, heads, depth), projection and +# immutable HF revisions. In particular XXS uses 56-dimensional attention heads. +PUBLISHED_MODELS = { + 'large': ((1024, 2816, 16, 12), (960, 2560, 15, 11), 960, + 'e836feaa79cd1f7de939fe77065819f48fc44e4c'), + 'medium': ((640, 1728, 10, 12), (640, 1728, 10, 11), 640, + '3a8d9d79cb98e4d409fce24092a69718a821a7ad'), + 'small': ((512, 1408, 8, 12), (512, 1408, 8, 11), 512, + '9891ee368dbec866dab1919f163b7e091ab0ed37'), + 'xs': ((320, 896, 5, 12), (320, 896, 5, 12), 320, + 'adc510748f0b759918ba4aaee94fc864e05c48d9'), + 'xxs': ((224, 640, 4, 12), (224, 640, 4, 12), 224, + 'adaca9be91f5045d06e10569de3075e1ef37f823'), +} OPEN_CLIP_REVISION = 'f9a0948d573009f8fa3e530e55814cb11fb8e8bd' @@ -194,8 +206,7 @@ def clap2_inputs(source, revision=None, tokenizer_path=None): source = Path(source) if not source.is_dir(): raise ValueError('CLAP2 source must be a checkpoint directory') - revision = SOURCE_REVISION if revision is None else revision - if not isinstance(revision, str) or re.fullmatch(r'[0-9a-fA-F]{40}', revision) is None: + if revision is not None and (not isinstance(revision, str) or re.fullmatch(r'[0-9a-fA-F]{40}', revision) is None): raise ValueError('CLAP2 revision must be a full 40-character commit SHA') config = _read_json(source / 'config.json') processor = _read_json(source / 'processor_config.json') @@ -205,12 +216,22 @@ def clap2_inputs(source, revision=None, tokenizer_path=None): checkpoint = source / 'model.safetensors' if not checkpoint.is_file(): raise ValueError('CLAP2 source is missing HF model.safetensors (open_clip weights are not interchangeable)') - metadata = {'general.architecture': 'clap2', 'general.name': SOURCE_REPO, - 'clap2.source_repo': SOURCE_REPO, 'clap2.source_revision': revision.lower(), + metadata = {'general.architecture': 'clap2', 'general.name': 'CLAP2', 'clap2.open_clip_revision': OPEN_CLIP_REVISION, 'clap2.config': json.dumps(config, separators=(',', ':')), 'clap2.processor_config': json.dumps(processor, separators=(',', ':')), 'clap2.tokenizer': json.dumps(tokenizer, separators=(',', ':'), ensure_ascii=False)} + fields = ('hidden_size', 'intermediate_size', 'num_attention_heads', 'num_hidden_layers') + signature = tuple(tuple(config[branch + '_config'][key] for key in fields) + for branch in ('audio', 'text')) + (config['projection_dim'],) + for size, (*dimensions, pinned_revision) in PUBLISHED_MODELS.items(): + if signature == tuple(dimensions): + repo = f'laion/clap2-{size}-audioxl-s1B' + metadata.update({'general.name': repo, 'clap2.source_repo': repo}) + revision = pinned_revision if revision is None else revision + break + if revision is not None: + metadata['clap2.source_revision'] = revision.lower() return checkpoint, metadata, config diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 2f9cb46..bcb748a 100644 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -33,7 +33,7 @@ def parse_args(): parser.add_argument('--clamp3-config', type=Path, help='CLaMP3 combined config JSON (otherwise infer from sibling XLM-R config)') parser.add_argument('--tokenizer', type=Path, help='CLAP2/CLaMP3/MuLan local tokenizer.json (default: beside checkpoint)') - parser.add_argument('--clap2-revision', help='full HF revision SHA for CLAP2 metadata (default: pinned laion checkpoint)') + parser.add_argument('--clap2-revision', help='full HF revision SHA for CLAP2 metadata (default: pinned revision for the detected model size)') parser.add_argument('--mert2-revision', help='full pinned Hugging Face revision SHA for MERT2 metadata') parser.add_argument('--muq-revision', help='full pinned Hugging Face revision SHA for MuQ metadata') parser.add_argument('--mert-revision', help='full pinned Hugging Face revision SHA for MERT-v1 metadata') diff --git a/tests/models/clap2-model-test.cpp b/tests/models/clap2-model-test.cpp index cb5a876..f4f1f1a 100644 --- a/tests/models/clap2-model-test.cpp +++ b/tests/models/clap2-model-test.cpp @@ -102,7 +102,7 @@ int main(int argc, char **argv) { return clap_encode_audio(model.get(), wave.data(), wave.size(), fixture.at("sample_rate"), 0, out, count); }), - test.at("expected").get>(), tolerance, test.at("name")); + test.at("expected").get>(), test.value("tolerance", tolerance), test.at("name")); float *out = nullptr; size_t count = 0; const auto status = diff --git a/tests/models/clap2-rope-test.cpp b/tests/models/clap2-rope-test.cpp new file mode 100644 index 0000000..f8466a8 --- /dev/null +++ b/tests/models/clap2-rope-test.cpp @@ -0,0 +1,99 @@ +// CLAP2 size-geometry regression. Uses only the tiny analytical GGUF's +// constructor metadata; rotary inputs have the exact published tower widths. +#include "model.h" + +#include +#include +#include +#include +#include + +namespace clap::detail { +struct FeatureTestAccess { + static ggml_tensor *rotate(const Clap2Model &model, Graph &graph, ggml_tensor *input, int heads, int time, + int prefix) { + return model.rotate(graph, input, heads, 10000.f, time, prefix); + } +}; +} // namespace clap::detail + +using namespace clap::detail; + +static void rotary(const Clap2Model &model, const GraphRuntime &runtime, const char *name, int width, int heads, + bool audio) { + // A non-square frequency/time grid detects swapped axes and flattening. + // Cross both axes and exercise every head, including XXS's 56-wide heads. + const int time = 7, frequency = 8, prefix = audio ? 1 : 0; + const int tokens = audio ? time * frequency + prefix : 19; + const int head_dim = width / heads; + HostMatrix input(tokens, width); + for (size_t i = 0; i < input.data.size(); ++i) + input.data[i] = std::sin(float(i) * 0.17f) + 0.3f * std::cos(float(i) * 0.11f); + Graph graph(runtime); + auto *output = FeatureTestAccess::rotate(model, graph, graph.input(input), heads, audio ? time : 0, prefix); + const auto actual = graph.run(output); + if (actual.size() != input.data.size()) + throw std::runtime_error("CLAP2 rotary shape mismatch"); + float maximum = 0; + // Scalar expression of upstream timm build_rotary_pos_embed: dim/4 + // inverse-frequency bands per axis, repeated interleaved. ModernText + // uses dim/2 bands and ordinary positions. No power-of-two assumption. + for (int token = 0; token < tokens; ++token) + for (int head = 0; head < heads; ++head) + for (int channel = 0; channel < head_dim; channel += 2) { + const int axis_dim = audio ? head_dim / 2 : head_dim; + const int within_axis = channel % axis_dim; + const int position = audio ? (channel < axis_dim ? std::max(0, token - prefix) / time + : std::max(0, token - prefix) % time) + : token; + const double inverse_frequency = std::exp(-std::log(10000.0) * within_axis / axis_dim); + const double angle = token < prefix ? 0.0 : position * inverse_frequency; + const int column = head * head_dim + channel; + const double even = input[token][column], odd = input[token][column + 1]; + const double expected[] = {even * std::cos(angle) - odd * std::sin(angle), + odd * std::cos(angle) + even * std::sin(angle)}; + for (int component = 0; component < 2; ++component) { + const float value = actual[size_t(token) * width + column + component]; + if (!std::isfinite(value)) + throw std::runtime_error("CLAP2 rotary produced non-finite output"); + maximum = std::max(maximum, float(std::abs(value - expected[component]))); + if (token < prefix && value != input[token][column + component]) + throw std::runtime_error("CLAP2 audio register token was rotated"); + } + } + std::cout << name << (audio ? " audio" : " text") << " width=" << width << " heads=" << heads + << " head_dim=" << head_dim << " max_abs_error=" << maximum << '\n'; + if (maximum > 3e-6f) + throw std::runtime_error("CLAP2 rotary differs from independent scalar oracle"); +} + +int main(int argc, char **argv) { + try { + if (argc != 2) + throw std::runtime_error("usage: clap2-rope-test TINY-CLAP2.gguf"); + clap::RuntimeOptions options; + options.device = "cpu"; + ModelLoader loader(options); + loader.load(argv[1]); + Clap2Model model(loader, 2); + GraphRuntime runtime(loader, 2, true); + // Published config.json dimensions from pinned laion/clap2-*-audioxl-s1B. + struct Size { + const char *name; + int audio_width, audio_heads, text_width, text_heads; + }; + const Size sizes[] = {{"large", 1024, 16, 960, 15}, + {"medium", 640, 10, 640, 10}, + {"small", 512, 8, 512, 8}, + {"xs", 320, 5, 320, 5}, + {"xxs", 224, 4, 224, 4}}; + for (const auto &size : sizes) { + rotary(model, runtime, size.name, size.audio_width, size.audio_heads, true); + rotary(model, runtime, size.name, size.text_width, size.text_heads, false); + } + return 0; + } catch (const std::exception &error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/tests/models/clap2_SOURCE.md b/tests/models/clap2_SOURCE.md index 47d3c40..d76e56e 100644 --- a/tests/models/clap2_SOURCE.md +++ b/tests/models/clap2_SOURCE.md @@ -15,10 +15,23 @@ Relevant upstream files: - `src/open_clip/transformer.py`: ModernTextTransformer and masked MAP pool - `src/open_clip/tokenizer.py`: TikTokenTokenizer, `encode_ordinary` semantics -Published HF assets/checkpoint are `laion/clap2-large-audioxl-s1B` revision -`e836feaa79cd1f7de939fe77065819f48fc44e4c`. The fixture generator validates its -complete reduced-depth HF tensor-name set against the downloaded checkpoint -header. Production conversion keeps those HF tensor names unchanged. +Published HF assets/checkpoints are the five `laion/clap2-*-audioxl-s1B` releases: + +| Size | HF revision | Audio / text width | Audio / text layers | Projection | Head width | +| --- | --- | --- | --- | --- | --- | +| large | `e836feaa79cd1f7de939fe77065819f48fc44e4c` | 1024 / 960 | 12 / 11 | 960 | 64 | +| medium | `3a8d9d79cb98e4d409fce24092a69718a821a7ad` | 640 / 640 | 12 / 11 | 640 | 64 | +| small | `9891ee368dbec866dab1919f163b7e091ab0ed37` | 512 / 512 | 12 / 11 | 512 | 64 | +| xs | `adc510748f0b759918ba4aaee94fc864e05c48d9` | 320 / 320 | 12 / 12 | 320 | 64 | +| xxs | `adaca9be91f5045d06e10569de3075e1ef37f823` | 224 / 224 | 12 / 12 | 224 | 56 | + +`clap2_sizes.json` retains the pinned HF configurations and independently read +published tensor-schema digests for offline regressions. +Native inference derives tensor dimensions, layer counts and RoPE bands from +configuration; XXS has 14 axial frequency bands per axis, not a rounded 64-wide +head. `clap2-rope-test` checks every published tower geometry against an independent +scalar oracle on an asymmetric frequency/time grid. Production conversion keeps +the HF tensor names unchanged. ### Important text MLP layout difference @@ -81,10 +94,21 @@ loader/API smoke coverage, NOT numerical architecture/frontend parity. ``` `--generate-only` prepares both GGUFs and fixture JSONs without native execution. +Use `--profile xxs-head56` to test the released XXS head geometry with small +nondegenerate towers, or `--profile asymmetric-head64` for unequal tower widths, +depths and MLP widths. The default `tiny` profile preserves the original reduced +fixture. These profiles run the same tokenizer, padding, waveform and crop-sensitive +long-audio cases in F32/F16. `--tokenizer-assets COMMON_ASSETS` can share identical +tokenizer/header assets across sizes. + `OFFICIAL_OPEN_CLIP` contains the reviewed checkout, `src/open_clip`, and `PINNED_SHA` containing the exact source SHA above. `HF_ASSETS` contains pinned `config.json`, `processor_config.json`, `tokenizer.json`, `tokenizer_config.json`, -and `model.safetensors.header.json`; no full weights are needed. `OUTPUT` must +`open_clip_config.json`, `identity.json` (`repo` and full `revision` SHA), +and `model.safetensors.header.json`; no full weights are needed. A full HF API +`hf-manifest.json` can supply `id` and `sha` instead of `identity.json`. The oracle +cross-checks the HF and open_clip tower dimensions before constructing a model. +`OUTPUT` must be separate from downloaded assets and checkpoints. It is updated in place, not recursively deleted. tiktoken may fetch/cache its standard r50k_base ranks on first use; no model checkpoint is downloaded by these scripts. @@ -109,7 +133,26 @@ Coverage: counted as user tokens. The oracle verifies compact/padded inference equality and independently tests interior/suffix padding and masked MAP pooling. - Upstream invalid audio-patch padding invariance. -- F32/F16 storage-rounded parity under a 5e-5 maximum-absolute embedding tolerance. +- F32/F16 storage-rounded parity under a 5e-5 maximum-absolute embedding tolerance, + except the documented frontend-sensitive case below. + +### FFT precision boundary + +The synthetic `asymmetric-head64` profile's `multiple` pure-tone waveform spans +more than 130 dB, including mel bins close to the −100 dB clamp. The pinned CPU +Torch frontend uses an F32 MKL FFT; native inference uses the shared double FFT. +Roundoff in near-zero spectral bins can therefore change the log-mel floor and +produce an embedding difference around 1e-4 despite the same mathematical +frontend. This single waveform case carries an explicit **2e-4** comparison +bound and precision annotation in both storage modes. Other waveform cases, +all text/token cases, and published-checkpoint tolerances are unchanged. + +To diagnose this boundary, compare native and official log-mel first, then inject +the same official mel into the native patchify/audio tower through a private +`FeatureTestAccess` probe. Cross-combine native/Torch powers and mel filters, and +compare official F32 versus double FFT before attributing a difference to model +geometry. The asymmetric tower agrees within 3e-7 with identical official mel; +a filterbank-only change does not resolve this FFT-specific difference. Tiny random-weight parity is not published-checkpoint parity. diff --git a/tests/models/clap2_checkpoint_parity.py b/tests/models/clap2_checkpoint_parity.py index 5b5267d..a168f82 100644 --- a/tests/models/clap2_checkpoint_parity.py +++ b/tests/models/clap2_checkpoint_parity.py @@ -4,6 +4,9 @@ python clap2_checkpoint_parity.py BRANCH CHECKPOINT_DIR OUTPUT OFFICIAL_OPEN_CLIP HF_ASSETS BRANCH is text or audio. No downloads. Creates fixture-BRANCH-f32.json for clap2-model-test; execute native separately after this process releases torch. +HF_ASSETS supplies config.json, open_clip_config.json and identity.json (or the +legacy hf-manifest.json). The official tower shape and revision come from those +assets, not the large model's source configuration. """ import argparse import importlib.metadata @@ -11,7 +14,7 @@ from pathlib import Path import sys -from clap2_parity import HF_REVISION, REVISION, hf_name, hf_value +from clap2_parity import REVISION, hf_name, hf_value, load_oracle_assets def main(): @@ -24,6 +27,7 @@ def main(): parser.add_argument("--dtype", choices=("f32", "f16"), default="f32", help="storage-rounded oracle, always F32 activations") args = parser.parse_args() + _, cfg, identity = load_oracle_assets(args.assets) assert (args.official / "PINNED_SHA").read_text().strip() == REVISION assert importlib.metadata.version("timm") == "1.0.30" sys.path.insert(0, str(args.official / "src")) @@ -36,7 +40,6 @@ def main(): from open_clip.audio.naflex_audio import AudioNaFlexCfg, AudioNaFlexPatchify from open_clip.tokenizer import TikTokenTokenizer torch.set_num_threads(2) - cfg = json.loads((args.official / "src/open_clip/model_configs/naflexclap_large.json").read_text()) args.output.mkdir(parents=True, exist_ok=True) # Instantiate only one tower; assign=True replaces initialized parameters # with mmap-backed safetensors views rather than cloning the full checkpoint. @@ -51,12 +54,12 @@ def main(): state[name] = value.half().float() if args.dtype == "f16" and value.ndim == 2 else value tower.load_state_dict(state, strict=True, assign=True) tower.eval() - fixture = dict(source_revision=REVISION, hf_revision=HF_REVISION, sample_rate=32000, + fixture = dict(source_revision=REVISION, hf_revision=identity["revision"], hf_repo=identity["repo"], sample_rate=32000, projection_dim=cfg["embed_dim"], context_length=cfg["text_cfg"]["context_length"], tolerance=8e-5, text_cases=[], token_cases=[], waveform_cases=[]) with torch.no_grad(): if args.branch == "text": - tok = TikTokenTokenizer("r50k_base", context_length=256) + tok = TikTokenTokenizer("r50k_base", context_length=cfg["text_cfg"]["context_length"]) for text in ("", "a dog barking", "We're I'M can't CAN'T", "café 中文 🙂", "a b\t\nnext ", "<|endoftext|> <|bos|> <|pad|>"): ids = tok(text)[0] diff --git a/tests/models/clap2_oracle_test.py b/tests/models/clap2_oracle_test.py new file mode 100644 index 0000000..a20c3f6 --- /dev/null +++ b/tests/models/clap2_oracle_test.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Optional oracle-tooling unit tests; stdlib only, no model imports/downloads.""" +import copy +import json +from pathlib import Path +import tempfile +import unittest + +import clap2_parity as oracle + + +class OracleConfigTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.assets = Path(self.directory.name) + self.hf = { + "projection_dim": 224, + "audio_config": dict(hidden_size=224, intermediate_size=640, num_attention_heads=4, + num_hidden_layers=12, layerscale_value=1e-5), + "text_config": dict(hidden_size=224, intermediate_size=640, num_attention_heads=4, + num_hidden_layers=12, max_position_embeddings=256), + } + self.upstream = { + "embed_dim": 224, + "audio_cfg": {"naflexvit_cfg": dict(embed_dim=224, depth=12, num_heads=4, + mlp_ratio=640 / 224)}, + "text_cfg": dict(width=224, layers=12, heads=4, mlp_ratio=640 / 224, context_length=256), + } + self.identity = {"repo": "laion/clap2-xxs-audioxl-s1B", "revision": "a" * 40} + self.write("config.json", self.hf) + self.write("open_clip_config.json", {"model_cfg": self.upstream}) + self.write("identity.json", self.identity) + + def write(self, name, value): + (self.assets / name).write_text(json.dumps(value)) + + def test_published_config_and_identity_are_asset_specific(self): + config, upstream, identity = oracle.load_oracle_assets(self.assets) + self.assertEqual(config, self.hf) + self.assertEqual(upstream, self.upstream) + self.assertEqual(identity, self.identity) + self.assertEqual(upstream["audio_cfg"]["naflexvit_cfg"]["embed_dim"], 224) + + def test_legacy_manifest_supplies_exact_identity(self): + (self.assets / "identity.json").unlink() + self.write("hf-manifest.json", {"id": self.identity["repo"], "sha": self.identity["revision"]}) + _, _, identity = oracle.load_oracle_assets(self.assets) + self.assertEqual(identity, self.identity) + + def test_missing_identity_is_not_silently_labeled_large(self): + (self.assets / "identity.json").unlink() + with self.assertRaises(FileNotFoundError): + oracle.load_oracle_assets(self.assets) + + def test_invalid_revision_is_rejected(self): + self.write("identity.json", dict(self.identity, revision="main")) + with self.assertRaisesRegex(ValueError, "revision"): + oracle.load_oracle_assets(self.assets) + + def test_mismatched_official_config_is_rejected(self): + self.upstream["text_cfg"]["layers"] = 11 + self.write("open_clip_config.json", {"model_cfg": self.upstream}) + with self.assertRaisesRegex(ValueError, "text.*layers"): + oracle.load_oracle_assets(self.assets) + + def test_xxs_profile_preserves_head56_and_noninteger_mlp(self): + original = copy.deepcopy((self.hf, self.upstream)) + config, upstream = oracle.tiny_configs(self.hf, self.upstream, "xxs-head56") + for branch in ("audio", "text"): + cfg = config[branch + "_config"] + self.assertEqual(cfg["hidden_size"] // cfg["num_attention_heads"], 56) + self.assertEqual(cfg["intermediate_size"], 160) + self.assertEqual(cfg["num_hidden_layers"], 2) + self.assertEqual(int(upstream["text_cfg"]["width"] * upstream["text_cfg"]["mlp_ratio"]), 160) + self.assertEqual(config["projection_dim"], 56) + self.assertEqual((self.hf, self.upstream), original) + + def test_asymmetric_profile_exercises_distinct_widths_and_depths(self): + config, upstream = oracle.tiny_configs(self.hf, self.upstream, "asymmetric-head64") + audio, text = config["audio_config"], config["text_config"] + self.assertNotEqual(audio["hidden_size"], text["hidden_size"]) + self.assertNotEqual(audio["num_hidden_layers"], text["num_hidden_layers"]) + self.assertEqual(audio["hidden_size"] // audio["num_attention_heads"], 64) + self.assertEqual(text["hidden_size"] // text["num_attention_heads"], 64) + self.assertEqual(upstream["text_cfg"]["layers"], text["num_hidden_layers"]) + self.assertEqual(upstream["audio_cfg"]["naflexvit_cfg"]["depth"], audio["num_hidden_layers"]) + + def test_default_profile_preserves_original_tiny_dimensions(self): + config, upstream = oracle.tiny_configs(self.hf, self.upstream, "tiny") + self.assertEqual(config["projection_dim"], 12) + for branch in ("audio", "text"): + cfg = config[branch + "_config"] + self.assertEqual((cfg["hidden_size"], cfg["intermediate_size"], cfg["num_attention_heads"], + cfg["num_hidden_layers"]), (16, 32, 2, 2)) + self.assertEqual(upstream["text_cfg"]["context_length"], 32) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/clap2_parity.py b/tests/models/clap2_parity.py index 1e4d76e..317b95c 100644 --- a/tests/models/clap2_parity.py +++ b/tests/models/clap2_parity.py @@ -4,6 +4,10 @@ Usage: python clap2_parity.py BUILD OUTPUT OFFICIAL_OPEN_CLIP HF_ASSETS Use --generate-only to prepare fixtures before the native target is built. No network/model downloads; only generated tiny weights (not checkpoint parity). +--profile xxs-head56 preserves the released XXS head dimension at reduced width. +--profile asymmetric-head64 exercises unequal tower widths and depths. +HF_ASSETS supplies the release config, open_clip_config.json and identity.json +(or legacy hf-manifest.json); --tokenizer-assets can reuse common tokenizer/header files. """ import argparse import copy @@ -16,7 +20,62 @@ import sys REVISION = "f9a0948d573009f8fa3e530e55814cb11fb8e8bd" -HF_REVISION = "e836feaa79cd1f7de939fe77065819f48fc44e4c" + + +def load_oracle_assets(assets): + """Read the chosen release's official configuration and immutable identity. + + Kept stdlib-only so configuration/provenance checks need no Torch imports. + The legacy full HF API manifest is accepted for pre-existing large assets. + """ + config = json.loads((assets / "config.json").read_text()) + upstream = json.loads((assets / "open_clip_config.json").read_text())["model_cfg"] + if (assets / "identity.json").exists(): + identity = json.loads((assets / "identity.json").read_text()) + else: + manifest = json.loads((assets / "hf-manifest.json").read_text()) + identity = dict(repo=manifest["id"], revision=manifest["sha"]) + if not isinstance(identity.get("repo"), str) or not identity["repo"]: + raise ValueError("oracle assets require a source repo") + if not isinstance(identity.get("revision"), str) or not re.fullmatch(r"[0-9a-f]{40}", identity["revision"]): + raise ValueError("oracle assets require a full immutable HF revision") + comparisons = [("projection_dim", config["projection_dim"], upstream["embed_dim"])] + for branch in ("audio", "text"): + hf = config[branch + "_config"] + official = (upstream["audio_cfg"]["naflexvit_cfg"] if branch == "audio" else upstream["text_cfg"]) + width, layers, heads = (("embed_dim", "depth", "num_heads") if branch == "audio" else + ("width", "layers", "heads")) + for field, key in (("hidden_size", width), ("num_hidden_layers", layers), ("num_attention_heads", heads)): + comparisons.append((f"{branch}.{key}", hf[field], official[key])) + comparisons.append((f"{branch}.intermediate_size", hf["intermediate_size"], + int(official[width] * official["mlp_ratio"]))) + for label, hf_value, official_value in comparisons: + if hf_value != official_value: + raise ValueError(f"HF/open_clip configuration mismatch {label}: {hf_value} != {official_value}") + return config, upstream, identity + + +def tiny_configs(config, upstream, profile): + """Shrink official towers, retaining the selected head/layout edge cases.""" + # (audio width, text width, audio hidden, text hidden, audio heads, text heads, + # audio depth, text depth, projection width) + profiles = { + "tiny": (16, 16, 32, 32, 2, 2, 2, 2, 12), + "asymmetric-head64": (128, 64, 352, 192, 2, 1, 2, 1, 48), + "xxs-head56": (56, 56, 160, 160, 1, 1, 2, 2, 56), + } + aw, tw, ah, th, an, tn, al, tl, projection = profiles[profile] + config, upstream = copy.deepcopy(config), copy.deepcopy(upstream) + config["projection_dim"] = upstream["embed_dim"] = projection + for branch, width, hidden, heads, layers in (("audio", aw, ah, an, al), ("text", tw, th, tn, tl)): + config[branch + "_config"].update(hidden_size=width, intermediate_size=hidden, + num_attention_heads=heads, num_hidden_layers=layers) + config["audio_config"]["layerscale_value"] = 0.3 + config["text_config"]["max_position_embeddings"] = 32 + upstream["audio_cfg"]["naflexvit_cfg"].update( + embed_dim=aw, depth=al, num_heads=an, mlp_ratio=ah / aw, init_values=0.3) + upstream["text_cfg"].update(width=tw, heads=tn, layers=tl, mlp_ratio=th / tw, context_length=32) + return config, upstream def hf_name(name): @@ -86,7 +145,14 @@ def main(): parser.add_argument("official", type=Path) parser.add_argument("assets", type=Path) parser.add_argument("--generate-only", action="store_true") + parser.add_argument("--profile", choices=("tiny", "asymmetric-head64", "xxs-head56"), default="tiny", + help="tiny tower shape contract; xxs preserves the released 56-wide RoPE heads") + parser.add_argument("--tokenizer-assets", type=Path, + help="shared tokenizer files and HF tensor-name header (default: ASSETS)") args = parser.parse_args() + tokenizer_assets = args.tokenizer_assets or args.assets + source_config, source_upstream, identity = load_oracle_assets(args.assets) + config, upstream_config = tiny_configs(source_config, source_upstream, args.profile) root = Path(__file__).resolve().parents[2] assert (args.official / "PINNED_SHA").read_text().strip() == REVISION sys.path.insert(0, str(args.official / "src")) @@ -105,20 +171,11 @@ def main(): torch.set_num_threads(2) torch.manual_seed(20261107) args.output.mkdir(parents=True, exist_ok=True) - config = json.loads((args.assets / "config.json").read_text()) - config["projection_dim"] = 12 - for tower in ("audio_config", "text_config"): - config[tower].update(hidden_size=16, intermediate_size=32, num_attention_heads=2, num_hidden_layers=2) - config["audio_config"]["layerscale_value"] = 0.3 - config["text_config"]["max_position_embeddings"] = 32 (args.output / "config.json").write_text(json.dumps(config)) + (args.output / "identity.json").write_text(json.dumps(identity)) for name in ("processor_config.json", "tokenizer.json", "tokenizer_config.json"): - shutil.copyfile(args.assets / name, args.output / name) - upstream_config = json.loads((args.official / "src/open_clip/model_configs/naflexclap_large.json").read_text()) - upstream_config["embed_dim"] = config["projection_dim"] - upstream_config["audio_cfg"]["naflexvit_cfg"].update( - embed_dim=16, depth=2, num_heads=2, mlp_ratio=2.0, init_values=0.3) - upstream_config["text_cfg"].update(width=16, heads=2, layers=2, mlp_ratio=2.0, context_length=32) + source = args.assets if name != "tokenizer.json" else tokenizer_assets + shutil.copyfile(source / name, args.output / name) model = open_clip.CLAP(**upstream_config).eval() # Nontrivial biases/scales prevent zero/default initialization hiding bugs. with torch.no_grad(): @@ -131,15 +188,16 @@ def main(): value.normal_(std=0.12 if value.ndim > 1 else 0.05) state = {hf_name(name): hf_value(name, value).detach().clone().contiguous() for name, value in model.state_dict().items()} - header = json.loads((args.assets / "model.safetensors.header.json").read_text()) + header = json.loads((tokenizer_assets / "model.safetensors.header.json").read_text()) # Compare the complete reduced-depth HF key set (no renamed checkpoint tensors). expected_names = {name for name in header if name != "__metadata__" and - ((m := re.search(r"\.layers\.(\d+)\.", name)) is None or int(m[1]) < 2)} + ((m := re.search(r"(audio|text)_model\.encoder\.layers\.(\d+)\.", name)) is None or + int(m[2]) < config[m[1] + "_config"]["num_hidden_layers"])} assert set(state) == expected_names, (set(state) - expected_names, expected_names - set(state)) save_file(state, args.output / "model.safetensors") tokenizer = TikTokenTokenizer(encoding_name="r50k_base", context_length=32) - hf_tokenizer = Tokenizer.from_file(str(args.assets / "tokenizer.json")) + hf_tokenizer = Tokenizer.from_file(str(tokenizer_assets / "tokenizer.json")) hf_tokenizer.encode_special_tokens = True # split_special_tokens / encode_ordinary contract texts = ["", "a dog barking", "We're I'M can't CAN'T I'd i'd", "a b c\t\nend ", "café CAFÉ e\u0301 中文 🙂", "1234567890 ² Ⅷ", "line\r\n\t next\n\n", "word " * 50, @@ -184,8 +242,9 @@ def main(): @torch.no_grad() def fixtures(current, tolerance): - result = dict(source_revision=REVISION, hf_revision=HF_REVISION, sample_rate=32000, - projection_dim=12, context_length=32, tolerance=tolerance, + result = dict(source_revision=REVISION, hf_revision=identity["revision"], hf_repo=identity["repo"], + oracle_profile=args.profile, sample_rate=32000, + projection_dim=config["projection_dim"], context_length=32, tolerance=tolerance, text_cases=[], token_cases=[], waveform_cases=[]) for text, ids in text_inputs: output = current.encode_text(ids[None], normalize=True)[0] @@ -218,8 +277,17 @@ def fixtures(current, tolerance): for key, value in batch.items()} padded_batch["patches"][:, -3:] = 999 torch.testing.assert_close(current.encode_audio(batch), current.encode_audio(padded_batch), atol=2e-6, rtol=2e-6) - result["waveform_cases"].append(dict(name=label, waveform_file=f"{label}.f32", samples=len(wave), - mel_frames=mel.shape[0], patches=patches["patches"].shape[0], expected=output.tolist())) + case = dict(name=label, waveform_file=f"{label}.f32", samples=len(wave), + mel_frames=mel.shape[0], patches=patches["patches"].shape[0], expected=output.tolist()) + if args.profile == "asymmetric-head64" and label == "multiple": + # This tonal fixture spans >130 dB: F32 MKL FFT residuals near + # the -100 dB log floor differ from a double FFT. Injecting the + # same official mel gives <3e-7 tower error; using official + # double FFT reproduces native results. Bound this frontend- + # sensitive case separately, not text, other waves or weights. + case["tolerance"] = 2e-4 + case["precision_note"] = "F32 reference FFT versus native double FFT near the log-mel floor" + result["waveform_cases"].append(case) return result for dtype in ("f32", "f16"): @@ -233,7 +301,7 @@ def fixtures(current, tolerance): fixture.write_text(json.dumps(fixtures(current, 5e-5))) target = args.output / f"model-{dtype}.gguf" subprocess.run([sys.executable, str(root / "convert_hf_to_gguf.py"), str(args.output), - str(target), "--dtype", dtype], check=True) + str(target), "--dtype", dtype, "--clap2-revision", identity["revision"]], check=True) if not args.generate_only: subprocess.run([str(args.build / "clap2-model-test"), str(target), str(fixture)], check=True) print(f"Official open_clip/timm CLAP2 {dtype} fixture passed; native={'not run' if args.generate_only else 'passed'}", flush=True) diff --git a/tests/models/clap2_sizes.json b/tests/models/clap2_sizes.json new file mode 100644 index 0000000..b3d62f8 --- /dev/null +++ b/tests/models/clap2_sizes.json @@ -0,0 +1,351 @@ +{ + "processor": { + "feature_extractor": { + "f_max": 16000.0, + "f_min": 50.0, + "feature_extractor_type": "Clap2FeatureExtractor", + "feature_size": 128, + "hop_length": 320, + "max_num_patches": null, + "n_fft": 1024, + "padding_side": "right", + "padding_value": 0.0, + "patch_size": 16, + "return_attention_mask": true, + "sampling_rate": 32000 + }, + "processor_class": "Clap2Processor" + }, + "models": { + "large": { + "identity": { + "repo": "laion/clap2-large-audioxl-s1B", + "revision": "e836feaa79cd1f7de939fe77065819f48fc44e4c" + }, + "config": { + "architectures": [ + "Clap2Model" + ], + "audio_config": { + "attention_bias": true, + "attention_dropout": 0.0, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 2816, + "layerscale_value": 1e-05, + "mlp_bias": true, + "model_type": "clap2_audio_model", + "num_attention_heads": 16, + "num_hidden_layers": 12, + "num_mel_bins": 128, + "num_register_tokens": 1, + "patch_size": 16, + "pooler_hidden_act": "gelu", + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "axial" + }, + "use_qk_norm": false + }, + "dtype": "float32", + "logit_scale_init_value": 2.6592, + "model_type": "clap2", + "projection_dim": 960, + "projection_hidden_act": "gelu", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 50259, + "eos_token_id": 50257, + "hidden_act": "silu", + "hidden_size": 960, + "initializer_range": 0.02, + "intermediate_size": 2560, + "layerscale_value": null, + "max_position_embeddings": 256, + "mlp_bias": false, + "model_type": "clap2_text_model", + "num_attention_heads": 15, + "num_hidden_layers": 11, + "pad_token_id": 50258, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "use_qk_norm": true, + "vocab_size": 50260 + }, + "transformers_version": "5.23.0.dev0" + }, + "tensor_count": 261, + "tensor_shapes_sha256": "642dc0e223fca8751b03d3b91c0f5875556a2b66b8f1b8e4cce67ae1903a59d9" + }, + "medium": { + "identity": { + "repo": "laion/clap2-medium-audioxl-s1B", + "revision": "3a8d9d79cb98e4d409fce24092a69718a821a7ad" + }, + "config": { + "architectures": [ + "Clap2Model" + ], + "audio_config": { + "attention_bias": true, + "attention_dropout": 0.0, + "hidden_act": "silu", + "hidden_size": 640, + "initializer_range": 0.02, + "intermediate_size": 1728, + "layerscale_value": 1e-05, + "mlp_bias": true, + "model_type": "clap2_audio_model", + "num_attention_heads": 10, + "num_hidden_layers": 12, + "num_mel_bins": 128, + "num_register_tokens": 1, + "patch_size": 16, + "pooler_hidden_act": "gelu", + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "axial" + }, + "use_qk_norm": false + }, + "dtype": "float32", + "logit_scale_init_value": 2.6592, + "model_type": "clap2", + "projection_dim": 640, + "projection_hidden_act": "gelu", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 50259, + "eos_token_id": 50257, + "hidden_act": "silu", + "hidden_size": 640, + "initializer_range": 0.02, + "intermediate_size": 1728, + "layerscale_value": null, + "max_position_embeddings": 256, + "mlp_bias": false, + "model_type": "clap2_text_model", + "num_attention_heads": 10, + "num_hidden_layers": 11, + "pad_token_id": 50258, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "use_qk_norm": true, + "vocab_size": 50260 + }, + "transformers_version": "5.23.0.dev0" + }, + "tensor_count": 261, + "tensor_shapes_sha256": "9f697c858ef3e61bac1ece6b2bf927ffcbc5958dbc50d1ef5bef9c4715ca287e" + }, + "small": { + "identity": { + "repo": "laion/clap2-small-audioxl-s1B", + "revision": "9891ee368dbec866dab1919f163b7e091ab0ed37" + }, + "config": { + "architectures": [ + "Clap2Model" + ], + "audio_config": { + "attention_bias": true, + "attention_dropout": 0.0, + "hidden_act": "silu", + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 1408, + "layerscale_value": 1e-05, + "mlp_bias": true, + "model_type": "clap2_audio_model", + "num_attention_heads": 8, + "num_hidden_layers": 12, + "num_mel_bins": 128, + "num_register_tokens": 1, + "patch_size": 16, + "pooler_hidden_act": "gelu", + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "axial" + }, + "use_qk_norm": false + }, + "dtype": "float32", + "logit_scale_init_value": 2.6592, + "model_type": "clap2", + "projection_dim": 512, + "projection_hidden_act": "gelu", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 50259, + "eos_token_id": 50257, + "hidden_act": "silu", + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 1408, + "layerscale_value": null, + "max_position_embeddings": 256, + "mlp_bias": false, + "model_type": "clap2_text_model", + "num_attention_heads": 8, + "num_hidden_layers": 11, + "pad_token_id": 50258, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "use_qk_norm": true, + "vocab_size": 50260 + }, + "transformers_version": "5.23.0.dev0" + }, + "tensor_count": 261, + "tensor_shapes_sha256": "d264877be7061d23474dd21d5929c88c8cc459bbfc0ca06e78c98122dcb88ea1" + }, + "xs": { + "identity": { + "repo": "laion/clap2-xs-audioxl-s1B", + "revision": "adc510748f0b759918ba4aaee94fc864e05c48d9" + }, + "config": { + "architectures": [ + "Clap2Model" + ], + "audio_config": { + "attention_bias": true, + "attention_dropout": 0.0, + "hidden_act": "silu", + "hidden_size": 320, + "initializer_range": 0.02, + "intermediate_size": 896, + "layerscale_value": 1e-05, + "mlp_bias": true, + "model_type": "clap2_audio_model", + "num_attention_heads": 5, + "num_hidden_layers": 12, + "num_mel_bins": 128, + "num_register_tokens": 1, + "patch_size": 16, + "pooler_hidden_act": "gelu", + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "axial" + }, + "use_qk_norm": false + }, + "dtype": "float32", + "logit_scale_init_value": 2.6592, + "model_type": "clap2", + "projection_dim": 320, + "projection_hidden_act": "gelu", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 50259, + "eos_token_id": 50257, + "hidden_act": "silu", + "hidden_size": 320, + "initializer_range": 0.02, + "intermediate_size": 896, + "layerscale_value": null, + "max_position_embeddings": 256, + "mlp_bias": false, + "model_type": "clap2_text_model", + "num_attention_heads": 5, + "num_hidden_layers": 12, + "pad_token_id": 50258, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "use_qk_norm": true, + "vocab_size": 50260 + }, + "transformers_version": "5.23.0.dev0" + }, + "tensor_count": 269, + "tensor_shapes_sha256": "feead1b07ace75040b3300961f724a187aeac0b2215506c87cc6e3c20ff97d60" + }, + "xxs": { + "identity": { + "repo": "laion/clap2-xxs-audioxl-s1B", + "revision": "adaca9be91f5045d06e10569de3075e1ef37f823" + }, + "config": { + "architectures": [ + "Clap2Model" + ], + "audio_config": { + "attention_bias": true, + "attention_dropout": 0.0, + "hidden_act": "silu", + "hidden_size": 224, + "initializer_range": 0.02, + "intermediate_size": 640, + "layerscale_value": 1e-05, + "mlp_bias": true, + "model_type": "clap2_audio_model", + "num_attention_heads": 4, + "num_hidden_layers": 12, + "num_mel_bins": 128, + "num_register_tokens": 1, + "patch_size": 16, + "pooler_hidden_act": "gelu", + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "axial" + }, + "use_qk_norm": false + }, + "dtype": "float32", + "logit_scale_init_value": 2.6592, + "model_type": "clap2", + "projection_dim": 224, + "projection_hidden_act": "gelu", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 50259, + "eos_token_id": 50257, + "hidden_act": "silu", + "hidden_size": 224, + "initializer_range": 0.02, + "intermediate_size": 640, + "layerscale_value": null, + "max_position_embeddings": 256, + "mlp_bias": false, + "model_type": "clap2_text_model", + "num_attention_heads": 4, + "num_hidden_layers": 12, + "pad_token_id": 50258, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "use_qk_norm": true, + "vocab_size": 50260 + }, + "transformers_version": "5.23.0.dev0" + }, + "tensor_count": 269, + "tensor_shapes_sha256": "3035d5225765f9adfff00b030af2396433a6da7d9eb3a9ab9492757c11b7e442" + } + } +} diff --git a/tests/models/clap2_sizes.py b/tests/models/clap2_sizes.py new file mode 100644 index 0000000..c5426f2 --- /dev/null +++ b/tests/models/clap2_sizes.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Offline published-size configuration and source-identity regressions.""" +import copy +import hashlib +import json +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +from conversion.clap2 import clap2_inputs, expected_tensor_shapes, validate_config +from clap2_fixture import model_data + + +class PublishedSizes(unittest.TestCase): + def test_published_configs_and_identity(self): + fixtures = json.loads(Path(__file__).with_suffix('.json').read_text()) + for size, model in fixtures['models'].items(): + with self.subTest(size=size), tempfile.TemporaryDirectory() as temp: + source = Path(temp) + config = model['config'] + validate_config(config, fixtures['processor']) + shapes = expected_tensor_shapes(config) + self.assertEqual(len(shapes), model['tensor_count']) + schema = {name: list(shape) for name, shape in shapes.items()} + digest = hashlib.sha256(json.dumps(schema, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + # Digest of every tensor name/shape from the pinned published + # safetensors header, independent of the converter's schema. + self.assertEqual(digest, model['tensor_shapes_sha256']) + self.assertEqual(shapes['text_projection.weight'], + (config['projection_dim'], config['text_config']['hidden_size'])) + # Use a complete synthetic byte-BPE vocabulary to keep this + # discovery test stdlib-only; published tokenizer parity is separate. + tokenizer = model_data()[2] + config = copy.deepcopy(config) + for key in ('vocab_size', 'bos_token_id', 'eos_token_id', 'pad_token_id'): + config['text_config'][key] = model_data()[0]['text_config'][key] + for name, data in (('config.json', config), ('processor_config.json', fixtures['processor']), + ('tokenizer.json', tokenizer)): + (source / name).write_text(json.dumps(data)) + (source / 'model.safetensors').touch() + _, metadata, _ = clap2_inputs(source) + self.assertEqual(metadata['general.name'], model['identity']['repo']) + self.assertEqual(metadata['clap2.source_repo'], model['identity']['repo']) + self.assertEqual(metadata['clap2.source_revision'], model['identity']['revision']) + override = 'a' * 40 + self.assertEqual(clap2_inputs(source, override)[1]['clap2.source_revision'], override) + + def test_custom_model_does_not_claim_large_checkpoint(self): + config, processor, tokenizer, _ = model_data() + with tempfile.TemporaryDirectory() as temp: + source = Path(temp) + for name, data in (('config.json', config), ('processor_config.json', processor), + ('tokenizer.json', tokenizer)): + (source / name).write_text(json.dumps(data)) + (source / 'model.safetensors').touch() + _, metadata, _ = clap2_inputs(source) + self.assertEqual(metadata['general.name'], 'CLAP2') + self.assertNotIn('clap2.source_repo', metadata) + self.assertNotIn('clap2.source_revision', metadata) + self.assertEqual(clap2_inputs(source, 'b' * 40)[1]['clap2.source_revision'], 'b' * 40) + + +if __name__ == '__main__': + unittest.main()