perf: use native Metal matrix multiplication by default

README.md

@@ -71,12 +71,41 @@ curl http://127.0.0.1:8080/v1/embeddings \
 Audio inputs use one `input_audio` content block per item. Encode the complete
 WAV, MP3 or FLAC file as standard base64 (no whitespace or data-URL prefix):
 
-```sh
-python3 -c 'import base64,json,sys; print(json.dumps({"input":[{"content":[{"type":"input_audio","input_audio":{"data":base64.b64encode(open(sys.argv[1],"rb").read()).decode()}}]}]}))' dog.wav |
+```bash
+audio_base64=$(base64 < dog.wav | tr -d '\r\n')
+printf '{"input":[{"content":[{"type":"input_audio","input_audio":{"data":"%s"}}]}]}' "$audio_base64" |
   curl http://127.0.0.1:8080/v1/embeddings \
     -H 'Content-Type: application/json' --data-binary @-
 ```
 
+### Audio search (brute force)
+
+Search the `esc5` audio directory recursively for `dog` and return the ten most related audio files:
+
+```sh
+clap search esc5 --model larger_clap_general-q4_k_m.gguf --text 'dog' --top-k 10
+```
+
+<details>
+<summary>Show example search results</summary>
+
+```json
+[
+  {"path": "esc5/2-118072-A-0-dog.wav", "score": 0.3736357720143104},
+  {"path": "esc5/1-110389-A-0-dog.wav", "score": 0.36165991257288893},
+  {"path": "esc5/4-183992-A-0-dog.wav", "score": 0.3574270305924374},
+  {"path": "esc5/1-30226-A-0-dog.wav", "score": 0.34910229736736315},
+  {"path": "esc5/3-180977-A-0-dog.wav", "score": 0.34359234068457495},
+  {"path": "esc5/4-194754-A-0-dog.wav", "score": 0.3425423428503198},
+  {"path": "esc5/2-118964-A-0-dog.wav", "score": 0.3389233902904601},
+  {"path": "esc5/1-100032-A-0-dog.wav", "score": 0.33009392183878733},
+  {"path": "esc5/5-9032-A-0-dog.wav", "score": 0.3280081351893563},
+  {"path": "esc5/4-182395-A-0-dog.wav", "score": 0.32210240115509925}
+]
+```
+
+</details>
+
 ## License
 
 This project is licensed under the [Apache License 2.0](LICENSE).

src/graph.cpp

@@ -288,11 +288,10 @@ ggml_tensor *Graph::weight(ggml_tensor *source, bool f32) {
     return converted.emplace(source, ggml_cast(ctx(), w, GGML_TYPE_F32)).first->second;
 }
 ggml_tensor *Graph::matmul(ggml_tensor *weight, ggml_tensor *input, bool offload) {
-    // The pinned Metal MM/MMA kernels round operands to F16. Express the
-    // same product as a batch of native GGML matrix-vector products instead:
-    // [K,M,B] x [K,N,B] -> [K,M,1,B] x [K,1,N,B]. This keeps F32 operands
-    // intact and still executes on the selected GPU, without custom kernels.
-    if (runtime.metal && weight->ne[0] >= 64 && input->ne[1] > 1) {
+    // Let ggml select native MM/MMA or MV kernels, as in llama.cpp. Only
+    // explicit/model-required F32 activations need the vector-product fallback:
+    // Metal's matrix kernels can round operands to F16 despite F32 accumulation.
+    if (runtime.metal && runtime.f32_activations && weight->ne[0] >= 64 && input->ne[1] > 1) {
         const int64_t rows = input->ne[1], batches = input->ne[2], groups = input->ne[3];
         // Retain general GGML broadcasting when the two batch axes differ.
         if ((weight->ne[2] == batches && weight->ne[3] == groups) || (weight->ne[2] == 1 && weight->ne[3] == 1)) {