Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
362 changes: 181 additions & 181 deletions CMakeLists.txt

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion tests/models/checkpoint_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
root = Path(__file__).resolve().parents[2]
gguf = out / 'larger-clap-music-f32.gguf'
subprocess.run([sys.executable, str(root / 'convert_hf_to_gguf.py'), str(source), str(gguf)], check=True)
subprocess.run([str(build / 'clap-model-test'), str(gguf), str(fixture_path)], check=True)
subprocess.run([str(build / 'test-model'), str(gguf), str(fixture_path)], check=True)
# Exercise the public native executable, including its float32 WAV decoder.
raw = waveform.astype('<f4').tobytes()
wav = out / 'oracle.wav'
Expand Down
4 changes: 2 additions & 2 deletions tests/models/clamp3_bundled.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def run(model, *args, ok=True):
bundled_args = base_args.copy(); bundled_args[3] = str(bundled)
subprocess.run([*bundled_args, '--mert-revision', revision], check=True)
# Native API verifies waveform dispatch, resampling, text and feature compatibility.
subprocess.run([str(build/'clamp3-bundled-test'), str(bundled), str(standalone),
subprocess.run([str(build/'test-clamp3-bundled'), str(bundled), str(standalone),
str(out/f'model-{dtype}.gguf')], check=True)
for case in ['broadband', 'silence', 'tone', 'two_chunks', 'discard_tail', 'resampled-input']:
wave = out / f'{case}.wav'
Expand All @@ -58,7 +58,7 @@ def run(model, *args, ok=True):
if quantized.exists(): quantized.unlink()
subprocess.run([str(build/'clap'), 'quantize', '--type', preset,
str(bundled), str(quantized)], check=True)
rounded = {n:(t,v) for n,t,v in stored_tensors(quantized, build/'clap-q8-test' if preset=='Q4_K_M' else None)
rounded = {n:(t,v) for n,t,v in stored_tensors(quantized, build/'test-q8' if preset=='Q4_K_M' else None)
if n.startswith('mert.')}
assert original.keys()==rounded.keys()
for name,(typ,values) in original.items():
Expand Down
2 changes: 1 addition & 1 deletion tests/models/clamp3_checkpoint_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,5 +88,5 @@ def pooled(model,data,is_audio):
gguf=args.gguf or out/('model-'+args.dtype+'.gguf')
if args.gguf is None:
subprocess.run([sys.executable,str(converter),str(checkpoint_path),str(gguf),'--dtype',args.dtype],check=True)
subprocess.run([str(args.build/'clamp3-model-test'),str(gguf),str(out/'fixture.json')],check=True)
subprocess.run([str(args.build/'test-clamp3-model'),str(gguf),str(out/'fixture.json')],check=True)
print('Published CLaMP3 checkpoint parity passed:',args.dtype)
8 changes: 4 additions & 4 deletions tests/models/clamp3_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ def audio_oracle(features):
target = out / f'model-{suffix}.gguf'
subprocess.run([str(build / 'clap'), 'quantize', '--type',
'Q4_K_M' if 'q4' in suffix else 'Q8_0', str(out / source), str(target)], check=True)
count = load_storage(model, target, clamp3_tensor_name, build / 'clap-q8-test' if 'q4' in suffix else None)
count = load_storage(model, target, clamp3_tensor_name, build / 'test-q8' if 'q4' in suffix else None)
quantized_fixture = json.loads(json.dumps(fixture))
quantized_fixture['embedding_tolerance'] = 2e-3
for case in quantized_fixture['text_cases']:
Expand All @@ -206,11 +206,11 @@ def audio_oracle(features):
fixture_path = out / f'fixture-{suffix}.json'
fixture_path.write_text(json.dumps(quantized_fixture))
# This oracle explicitly models the CPU's activation packing.
subprocess.run([str(build / 'clamp3-model-test'), str(target), str(fixture_path)],
subprocess.run([str(build / 'test-clamp3-model'), str(target), str(fixture_path)],
check=True, env={**os.environ, 'CLAP_DEVICE': 'cpu'})
# Separately verify portable F32 activations on the selected device. No
# activation packing hooks: only the stored weights are rounded/quantized.
load_storage(model, target, clamp3_tensor_name, build / 'clap-q8-test' if 'q4' in suffix else None,
load_storage(model, target, clamp3_tensor_name, build / 'test-q8' if 'q4' in suffix else None,
pack_activations=False)
for case in quantized_fixture['text_cases']:
case['expected'] = text_oracle(case['ids']).tolist()
Expand All @@ -219,7 +219,7 @@ def audio_oracle(features):
case['expected'] = audio_oracle(features).tolist()
float_fixture = out / f'fixture-{suffix}-f32-activations.json'
float_fixture.write_text(json.dumps(quantized_fixture))
subprocess.run([str(build / 'clamp3-model-test'), str(target), str(float_fixture), '--f32'], check=True)
subprocess.run([str(build / 'test-clamp3-model'), str(target), str(float_fixture), '--f32'], check=True)
print(f'CLaMP3 {suffix}: {count} quantized parameters, text/audio storage-rounded parity passed')

# Corrupt the architecture string in-place without altering GGUF offsets.
Expand Down
14 changes: 7 additions & 7 deletions tests/models/clap2_SOURCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ Published HF assets/checkpoints are the five `laion/clap2-*-audioxl-s1B` release
published tensor-schema digests for offline regressions.
Native inference derives tensor dimensions, layer counts and RoPE bands from
configuration; XXS has 14 axial frequency bands per axis, not a rounded 64-wide
head. `clap2-rope-test` checks every published tower geometry against an independent
head. `test-clap2-rope` checks every published tower geometry against an independent
scalar oracle on an asymmetric frequency/time grid. Production conversion keeps
the HF tensor names unchanged.

Expand All @@ -51,12 +51,12 @@ to their corresponding HF tensors.

## Native runner

Build `clap2-model-test` from `tests/models/clap2-model-test.cpp`, linked with
Build `test-clap2-model` from `tests/models/test-clap2-model.cpp`, linked with
`clap` and `nlohmann_json::nlohmann_json`. It uses the public C API and existing
`tests/c-api-helpers.h`; no private include directories are needed.

```
"$BUILD/clap2-model-test" MODEL.gguf FIXTURE.json
"$BUILD/test-clap2-model" MODEL.gguf FIXTURE.json
```

It forces CPU, two threads and F32 activation compute. F16 below means rounded
Expand All @@ -80,8 +80,8 @@ conversion is covered by the optional numerical oracle below.

```
python tests/models/clap2_fixture.py "$BUILD/clap2-smoke"
"$BUILD/clap2-model-test" "$BUILD/clap2-smoke/model-f32.gguf" "$BUILD/clap2-smoke/fixture.json"
"$BUILD/clap2-model-test" "$BUILD/clap2-smoke/model-f16.gguf" "$BUILD/clap2-smoke/fixture.json"
"$BUILD/test-clap2-model" "$BUILD/clap2-smoke/model-f32.gguf" "$BUILD/clap2-smoke/fixture.json"
"$BUILD/test-clap2-model" "$BUILD/clap2-smoke/model-f16.gguf" "$BUILD/clap2-smoke/fixture.json"
```

The degenerate analytical model has constant unit outputs. It establishes valid
Expand Down Expand Up @@ -161,8 +161,8 @@ Tiny random-weight parity is not published-checkpoint parity.
```
"$PYTHON" tests/models/clap2_checkpoint_parity.py text "$CHECKPOINT_DIR" "$OUTPUT" "$OFFICIAL_OPEN_CLIP" "$HF_ASSETS" --dtype f32
"$PYTHON" tests/models/clap2_checkpoint_parity.py audio "$CHECKPOINT_DIR" "$OUTPUT" "$OFFICIAL_OPEN_CLIP" "$HF_ASSETS" --dtype f32
"$BUILD/clap2-model-test" MODEL-F32.gguf "$OUTPUT/fixture-text-f32.json"
"$BUILD/clap2-model-test" MODEL-F32.gguf "$OUTPUT/fixture-audio-f32.json"
"$BUILD/test-clap2-model" MODEL-F32.gguf "$OUTPUT/fixture-text-f32.json"
"$BUILD/test-clap2-model" MODEL-F32.gguf "$OUTPUT/fixture-audio-f32.json"
```

`CHECKPOINT_DIR/model.safetensors` must already exist. Use `--dtype f16` with an
Expand Down
2 changes: 1 addition & 1 deletion tests/models/clap2_checkpoint_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

python clap2_checkpoint_parity.py BRANCH CHECKPOINT_DIR OUTPUT OFFICIAL_OPEN_CLIP HF_ASSETS
BRANCH is text or audio. No downloads. Creates fixture-BRANCH-f32.json for
clap2-model-test; execute native separately after this process releases torch.
test-clap2-model; execute native separately after this process releases torch.
HF_ASSETS supplies config.json, open_clip_config.json and identity.json (or the
legacy hf-manifest.json). The official tower shape and revision come from those
assets, not the large model's source configuration.
Expand Down
6 changes: 3 additions & 3 deletions tests/models/clap2_invalid.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def main():
assert json.loads(metadata[field]) == json.loads(converted_metadata[field])

for dtype in ("f32", "f16"):
subprocess.run([str(build / "clap2-model-test"), str(output / f"model-{dtype}.gguf"),
subprocess.run([str(build / "test-clap2-model"), str(output / f"model-{dtype}.gguf"),
str(output / "fixture.json")], check=True)

invalid = []
Expand Down Expand Up @@ -76,7 +76,7 @@ def config_case(label, field, value, section="audio_config"):
"clap2.tokenizer": json.dumps(t)})
path = output / f"invalid-{index}.gguf"
write_gguf(path, bad_metadata, tensors)
result = subprocess.run([str(build / "clap2-model-test"), str(path), str(output / "fixture.json")],
result = subprocess.run([str(build / "test-clap2-model"), str(path), str(output / "fixture.json")],
text=True, capture_output=True)
if result.returncode == 0:
failures.append(f"native accepted {label}")
Expand All @@ -92,7 +92,7 @@ def config_case(label, field, value, section="audio_config"):
bad_tensors[target] = ((23, 8), [0.0] * (23 * 8))
path = output / f"invalid-tensor-{label}.gguf"
write_gguf(path, metadata, bad_tensors)
result = subprocess.run([str(build / "clap2-model-test"), str(path), str(output / "fixture.json")],
result = subprocess.run([str(build / "test-clap2-model"), str(path), str(output / "fixture.json")],
text=True, capture_output=True)
assert result.returncode != 0 and ("tensor" in result.stderr.lower() or "qkv_proj" in result.stderr), result.stderr
print(f"Rejected {label} tensor: {result.stderr.strip()}")
Expand Down
2 changes: 1 addition & 1 deletion tests/models/clap2_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ def fixtures(current, tolerance):
subprocess.run([sys.executable, str(root / "convert_hf_to_gguf.py"), str(args.output),
str(target), "--dtype", dtype, "--clap2-revision", identity["revision"]], check=True)
if not args.generate_only:
subprocess.run([str(args.build / "clap2-model-test"), str(target), str(fixture)], check=True)
subprocess.run([str(args.build / "test-clap2-model"), str(target), str(fixture)], check=True)
print(f"Official open_clip/timm CLAP2 {dtype} fixture passed; native={'not run' if args.generate_only else 'passed'}", flush=True)
versions = {name: importlib.metadata.version(name) for name in
("torch", "timm", "tiktoken", "safetensors", "tokenizers", "numpy")}
Expand Down
4 changes: 2 additions & 2 deletions tests/models/encoder_quantized.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def main(args):
for preset in ['Q8_0','Q4_K_M']:
target=out/f'{dtype}-{preset}.gguf'; target.unlink(missing_ok=True)
run(args.build/'clap','quantize','--type',preset,source,target)
stored={name:(typ,data) for name,typ,data in tensors(target,args.build/'clap-q8-test' if preset=='Q4_K_M' else None)}
stored={name:(typ,data) for name,typ,data in tensors(target,args.build/'test-q8' if preset=='Q4_K_M' else None)}
kinds={typ for typ,_ in stored.values()}; assert (8 in kinds if preset=='Q8_0' else {12,14}<=kinds), kinds
installed=set()
with torch.no_grad():
Expand All @@ -159,7 +159,7 @@ def main(args):
assert quantized<=installed,quantized-installed
assert all(np.any(stored[n][1]!=0) for n in quantized), 'zero quantized fixture'
roundtrip=out/f'{dtype}-{preset}-decoded.gguf'
run(args.build/'clap-q8-test','decode',target,roundtrip)
run(args.build/'test-q8','decode',target,roundtrip)
for label,wave in waves:
audio=out/f'{label}.wav'; wav(audio,wave.numpy(),rate)
command=[str(args.build/'clap'), 'embedding','-m',str(target),'-a',str(audio),'-t','2']
Expand Down
6 changes: 3 additions & 3 deletions tests/models/feature_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@
def extract(build, model, audio):
"""Decode/resample audio and run private MERT all-layer CLaMP3 pooling.

feature-extract-test MERT.gguf AUDIO emits {values, frames, dimension} JSON;
test-feature-extract MERT.gguf AUDIO emits {values, frames, dimension} JSON;
two inference threads are requested by FeatureTestAccess, with normal runtime device selection.
Build with BUILD_TESTING=ON and MERT_MODEL_PARITY=ON. Never calls clap embedding.
"""
return json.loads(subprocess.check_output([
str(Path(build)/'feature-extract-test'), str(model), str(audio)]))
str(Path(build)/'test-feature-extract'), str(model), str(audio)]))


def encode(build, model, features, frames, output):
path = Path(output)/'private-features.json'
path.write_text(json.dumps({'values': list(map(float, features)), 'frames': int(frames)}))
return json.loads(subprocess.check_output([str(Path(build)/'feature-input-test'),str(model),str(path)]))
return json.loads(subprocess.check_output([str(Path(build)/'test-feature-input'),str(model),str(path)]))
2 changes: 1 addition & 1 deletion tests/models/mert2_checkpoint_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@
if args.gguf is None:
subprocess.run([sys.executable, str(converter), str(args.checkpoint), str(gguf),
"--mert2-revision", revision], check=True)
subprocess.run([str(args.build / "mert2-model-test"), str(gguf), str(fixture_path)], check=True)
subprocess.run([str(args.build / "test-mert2-model"), str(gguf), str(fixture_path)], check=True)

# Exercise the public CLI once with the actual checkpoint.
raw_wave = struct.pack("<" + "f" * len(waveform), *waveform.tolist())
Expand Down
4 changes: 2 additions & 2 deletions tests/models/mert2_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def fixture_for(current, tolerance):
assert metadata["general.name"] == model_name, metadata
assert json.loads(metadata["mert2.config"]) == json.loads((current / "config.json").read_text())
assert metadata["mert2.source_revision"] == revision
subprocess.run([str(build / "mert2-model-test"), str(target),
subprocess.run([str(build / "test-mert2-model"), str(target),
str(current / f"fixture{suffix}.json")], check=True)
print(f"Synthetic official MERT2 {variant} F32/F16 and >30-second parity passed", flush=True)

Expand All @@ -148,7 +148,7 @@ def fixture_for(current, tolerance):
metadata["mert2.config"] = json.dumps(raw, separators=(",", ":"))
invalid = out / "invalid-variant.gguf"
write_safetensors_gguf(invalid, checkpoint, metadata, "f32", np, torch)
failed = subprocess.run([str(build / "mert2-model-test"), str(invalid), str(out / "fixture.json")],
failed = subprocess.run([str(build / "test-mert2-model"), str(invalid), str(out / "fixture.json")],
text=True, capture_output=True)
assert failed.returncode and "variant" in failed.stderr, failed.stderr
print("Unsupported MERT2 variant rejected by converter and runtime")
2 changes: 1 addition & 1 deletion tests/models/mert_checkpoint_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def run(args):
audio.embeddings.register_buffer('token_type_ids', torch.zeros(1,128,dtype=torch.long), persistent=False)
audio.eval()
cli = str(args.build/'clap')
subprocess.run([str(args.build/'mert-model-test'), str(args.mert_gguf), str(args.output)], check=True)
subprocess.run([str(args.build/'test-mert-model'), str(args.mert_gguf), str(args.output)], check=True)
rng = np.random.default_rng(719)
cases = {'broadband': rng.normal(.1,.12,24000).astype('float32'),
# Unequal chunk statistics expose per-window rather than recording normalization.
Expand Down
4 changes: 2 additions & 2 deletions tests/models/mert_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def run(args):
subprocess.run([sys.executable, str(root/'convert_hf_to_gguf.py'),str(source),str(target),
'--dtype', dtype,'--mert-revision', revision], check=True)
check_metadata(target, source, revision)
subprocess.run([str(args.build/'mert-model-test'), str(target), str(args.out)], check=True)
subprocess.run([str(args.build/'test-mert-model'), str(target), str(args.out)], check=True)
cases['resampled'] = np.frombuffer((args.out/'resampled-control.wav').read_bytes()[44:], dtype='<f4').copy()
# Compare identical storage-rounded weights, not a half-activation oracle.
if dtype == 'f16':
Expand Down Expand Up @@ -150,7 +150,7 @@ def run(args):
print(f'{dtype}/48kHz private helper -> 24kHz MERT matches native-rate control', flush=True)
for n in [0,399,23999]:
path=args.out/f'short-{n}.wav';wav(path,np.zeros(n,dtype='float32'))
result=subprocess.run([str(args.build/'feature-extract-test'), str(target), str(path)],
result=subprocess.run([str(args.build/'test-feature-extract'), str(target), str(path)],
capture_output=True,text=True)
assert result.returncode != 0, (n,result.stdout)
print('MERT-v1 numerical parity passed',flush=True)
Expand Down
2 changes: 1 addition & 1 deletion tests/models/muq_mulan_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def wave(n):
'--muq-mulan-revision','2e01c796b71dca71b45251384c04cd7b237c9020',
'--muq-config',str(out/'muq-config.json'),'--text-config',str(out/'text-config.json')]
subprocess.run(command,check=True)
subprocess.run([str(build/'muq-mulan-model-test'),str(target),str(path)],check=True)
subprocess.run([str(build/'test-muq-mulan-model'),str(target),str(path)],check=True)
# Native CLI must return joint vectors and reject every unsupported input mode.
x=waves[-1][1]; data=x.numpy().astype('<f4').tobytes(); wav=out/'input.wav'
wav.write_bytes(b'RIFF'+struct.pack('<I',36+len(data))+b'WAVEfmt '+struct.pack('<IHHIIHH',16,3,1,24000,96000,4,32)+b'data'+struct.pack('<I',len(data))+data)
Expand Down
2 changes: 1 addition & 1 deletion tests/models/muq_parity.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def waveform(samples):
gguf = out / ('model-' + dtype + '.gguf')
subprocess.run([sys.executable, str(root / 'convert_hf_to_gguf.py'), str(out), str(gguf),
'--dtype', dtype, '--muq-revision', MODEL_REVISION], check=True)
subprocess.run([str(build / 'muq-model-test'), str(gguf), str(fixture_path)], check=True)
subprocess.run([str(build / 'test-muq-model'), str(gguf), str(fixture_path)], check=True)
case = cases[2]
data = struct.pack('<' + 'f' * len(case['waveform']), *case['waveform'])
wav = out / 'input.wav'
Expand Down
Loading
Loading