Let capable hardware run speech on the device

.github/scripts/assert_intent_roundtrip.py

@@ -118,7 +118,12 @@ def send(self, msg_type, data=None, context=None):
                 self._dropped(error)
 
     def wait_for(self, reply_type, timeout, match=None):
-        """Read until `reply_type` arrives, or the deadline passes. None on timeout."""
+        """Read until `reply_type` arrives, or the deadline passes. None on timeout.
+
+        `reply_type` may be a tuple of types, for an answer that arrives under a spec
+        topic on newer services and a legacy one on older ones.
+        """
+        reply_types = reply_type if isinstance(reply_type, tuple) else (reply_type,)
         deadline = time.monotonic() + timeout
         while True:
             remaining = deadline - time.monotonic()
@@ -140,7 +145,7 @@ def wait_for(self, reply_type, timeout, match=None):
                 message = json.loads(raw)
             except (TypeError, ValueError):
                 continue
-            if message.get("type") != reply_type:
+            if message.get("type") not in reply_types:
                 continue
             if match and not match(message):
                 continue

.github/scripts/assert_local_speech.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+# Assert that a local speech install really runs speech on the device.
+#
+#   1. mycroft.conf names the onnx-asr and phoonnx plugins, with the public server
+#      only as the STT fallback, and the STT model is on disk already: the installer
+#      fetches it rather than leaving it to the first utterance;
+#   2. the running listener loaded the onnx-asr recognizer - its own log says so;
+#   3. the running audio service speaks with phoonnx (assert_local_speech_bus.py
+#      asks it over the bus, and it names the plugin it used);
+#   4. what that voice said, the running listener hears back. A listener with no
+#      microphone never starts answering the bus, which is every CI runner, and then
+#      the recognizer hears it in a fresh process instead.
+#
+# 1 alone was all this used to check, and it passes with services that never loaded
+# what the file names - they only reload a mycroft.conf rewritten in place.
+#
+#   assert_local_speech.sh [venv-python]   a virtualenv install
+#   assert_local_speech.sh --containers    a containers install (ovos_listener and
+#                                          ovos_audio)
+#
+# Never run this under sudo. HOME decides which mycroft.conf is read and where the
+# models and logs are, and all of them belong to the install user.
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+stt_config=$(
+    cat <<'PY'
+import os
+from ovos_config import Configuration
+
+stt = Configuration()["stt"]
+if stt.get("module") != "ovos-stt-plugin-onnx-asr":
+    raise SystemExit(f"STT is {stt.get('module')!r}, not the local onnx-asr plugin")
+if stt.get("fallback_module") != "ovos-stt-plugin-server":
+    raise SystemExit(f"STT fallback is {stt.get('fallback_module')!r}, not the public server")
+models = os.path.expanduser("~/.local/share/ovos_stt_plugin_onnxasr")
+if not os.path.isdir(models) or not os.listdir(models):
+    raise SystemExit(f"no STT model in {models}: the install did not fetch it")
+print(f"config    : STT {(stt.get(stt['module']) or {}).get('model')}, public fallback")
+PY
+)
+
+tts_config=$(
+    cat <<'PY'
+from ovos_config import Configuration
+
+tts = Configuration()["tts"]
+if tts.get("module") != "ovos-tts-plugin-phoonnx":
+    raise SystemExit(f"TTS is {tts.get('module')!r}, not the local phoonnx plugin")
+print(f"config    : TTS {(tts.get(tts['module']) or {}).get('voice')}")
+PY
+)
+
+# What the voice said, heard in a process of its own. The path is "-" to read the
+# audio from stdin, which is how it reaches a container.
+hear=$(
+    cat <<'PY'
+import sys
+import tempfile
+from ovos_config import Configuration
+from ovos_plugin_manager.stt import load_stt_plugin
+from ovos_plugin_manager.utils.audio import AudioData
+
+path = sys.argv[1]
+if path == "-":
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as wav:
+        wav.write(sys.stdin.buffer.read())
+    path = wav.name
+config = Configuration()
+stt, lang = config["stt"], config["lang"]
+recognizer = dict(stt.get(stt["module"]) or {}, lang=lang)
+heard = load_stt_plugin(stt["module"])(config=recognizer).execute(
+    AudioData.from_file(path), language=lang) or ""
+if "time" not in heard.lower():
+    raise SystemExit(f"the voice said 'what time is it' and the recognizer heard {heard!r}")
+print(f"recognizer: heard {heard!r} in a fresh process")
+PY
+)
+
+wav="$(mktemp --suffix=.wav)"
+websocket_dir=""
+cleanup() {
+    rm -f "$wav"
+    if [ -n "$websocket_dir" ]; then
+        rm -rf "$websocket_dir"
+    fi
+}
+trap cleanup EXIT
+
+if [ "${1:-}" = "--containers" ]; then
+    docker exec ovos_listener python3 -c "$stt_config"
+    docker exec ovos_audio python3 -c "$tts_config"
+    listener_log() { docker logs ovos_listener 2>&1; }
+    hear_fresh() { docker exec -i ovos_listener python3 -c "$hear" - <"$wav"; }
+    # No host venv: the bus check needs websocket-client and nothing else, so it goes
+    # into a throwaway directory rather than ~/.local (see assert_intent_roundtrip.sh).
+    bus_python=python3
+    if ! python3 -c "import websocket" 2>/dev/null; then
+        websocket_dir="$(mktemp -d)"
+        python3 -m pip install --quiet --disable-pip-version-check --no-cache-dir \
+            --target "$websocket_dir" websocket-client
+        export PYTHONPATH="${websocket_dir}${PYTHONPATH:+:${PYTHONPATH}}"
+    fi
+else
+    venv_python="${1:-${HOME}/.venvs/ovos/bin/python}"
+    if [ ! -x "${venv_python}" ]; then
+        echo "no virtualenv python at ${venv_python}" >&2
+        exit 1
+    fi
+    "${venv_python}" -c "$stt_config"
+    "${venv_python}" -c "$tts_config"
+    listener_log() { cat "${XDG_STATE_HOME:-${HOME}/.local/state}/mycroft/voice.log"; }
+    hear_fresh() { "${venv_python}" -c "$hear" "$wav"; }
+    bus_python="${venv_python}"
+fi
+
+# The plugin logs as it loads its model, and every line names its module. Not grep
+# -q: it stops reading at the first match, the log writer dies of SIGPIPE, and
+# pipefail turns a found line into a failure.
+if ! listener_log | grep -F "ovos_stt_plugin_onnxasr" >/dev/null; then
+    echo "the running listener never loaded the onnx-asr recognizer" >&2
+    exit 1
+fi
+echo "listener  : loaded the onnx-asr recognizer"
+
+status=0
+"${bus_python}" "${here}/assert_local_speech_bus.py" --wav "$wav" || status=$?
+case "$status" in
+0) ;;
+3) hear_fresh ;;
+*) exit "$status" ;;
+esac

.github/scripts/assert_local_speech_bus.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""Have the running services speak and listen, over the messagebus.
+
+assert_local_speech.sh checks mycroft.conf and the plugins it names. That is not the
+same as the services using them: they load their plugins when they start and reload
+only a configuration file rewritten in place, so a mycroft.conf they never picked up
+passes every check made on the file while the device keeps talking to the public
+servers. This asks the services themselves:
+
+  1. the audio service synthesises a sentence for the bus (speak:b64_audio) and names
+     the plugin that did it, which must be the local one;
+  2. the listener transcribes that audio with its own recognizer
+     (recognizer_loop:b64_transcribe), never its fallback, and must hear the sentence.
+
+The listener only starts answering the bus once its microphone has started, and a CI
+runner has no microphone. When it never reports ready, the audio is still written to
+--wav and this exits 3, so the caller can have the recognizer hear it in a fresh
+process instead.
+
+Like assert_intent_roundtrip.py, which this borrows its bus client from, it speaks the
+bus protocol over a bare websocket, so it runs the same on a host with no venv.
+"""
+import argparse
+import base64
+import io
+import secrets
+import sys
+import time
+import wave
+
+from assert_intent_roundtrip import Bus, Failure, wait_ready
+
+LISTENER_UNAVAILABLE = 3
+LOCAL_TTS = "ovos-tts-plugin-phoonnx"
+
+
+def speak(bus, utterance, timeout):
+    """The audio service's own TTS, as bytes of a WAV file, and the plugin it used."""
+    marker = secrets.token_hex(8)
+    bus.send("speak:b64_audio", {"utterance": utterance},
+             {"source": ["ci"], "ci_probe": marker})
+    # Newer services answer on the spec topic, older ones on the legacy one, and the
+    # bus client may mirror one onto the other: whichever comes first will do.
+    answer = bus.wait_for(("ovos.audio.speech", "speak:b64_audio.response"), timeout,
+                          match=lambda m: (m.get("context") or {}).get("ci_probe") == marker)
+    if answer is None:
+        raise Failure(
+            f"the audio service did not synthesise {utterance!r} for the bus within "
+            f"{timeout:.0f}s, having reported itself ready."
+        )
+    data = answer.get("data") or {}
+    try:
+        return base64.b64decode(data["audio"]), data.get("tts_id")
+    except (KeyError, TypeError, ValueError) as error:
+        raise Failure(f"the audio service answered without usable audio: {error!r}")
+
+
+def pcm(wav_bytes):
+    """The frames of a mono WAV, with the rate and width the listener needs to read them."""
+    try:
+        with wave.open(io.BytesIO(wav_bytes)) as wav:
+            if wav.getnchannels() != 1:
+                raise Failure(f"the voice produced {wav.getnchannels()} channels, not one")
+            return wav.readframes(wav.getnframes()), wav.getframerate(), wav.getsampwidth()
+    except (wave.Error, EOFError) as error:
+        raise Failure(f"the audio service's answer is not a WAV file: {error}")
+
+
+def transcribe(bus, wav_bytes, timeout):
+    frames, rate, width = pcm(wav_bytes)
+    marker = secrets.token_hex(8)
+    bus.send("recognizer_loop:b64_transcribe",
+             {"audio": base64.b64encode(frames).decode("ascii"),
+              "sample_rate": rate, "sample_width": width},
+             {"source": ["ci"], "ci_probe": marker})
+    answer = bus.wait_for("recognizer_loop:b64_transcribe.response", timeout,
+                          match=lambda m: (m.get("context") or {}).get("ci_probe") == marker)
+    if answer is None:
+        raise Failure(
+            f"the listener did not transcribe the audio within {timeout:.0f}s, having "
+            f"reported itself ready."
+        )
+    heard = []
+    for transcription in (answer.get("data") or {}).get("transcriptions") or []:
+        # (text, confidence) pairs, which JSON turns into lists; plain strings from
+        # older listeners.
+        heard.append(transcription[0] if isinstance(transcription, (list, tuple))
+                     else str(transcription))
+    return heard[0] if heard else ""
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--url", default="ws://127.0.0.1:8181/core")
+    parser.add_argument("--utterance", default="what time is it")
+    parser.add_argument("--expect-word", default="time",
+                        help="a word the recognizer has to hear in --utterance")
+    parser.add_argument("--expect-tts", default=LOCAL_TTS,
+                        help="the plugin the audio service has to name")
+    parser.add_argument("--wav", required=True,
+                        help="where to write what the voice said")
+    parser.add_argument("--connect-timeout", type=float, default=30)
+    parser.add_argument("--ready-timeout", type=float, default=300,
+                        help="the services restart onto local speech, and the "
+                             "listener loads its model before it answers")
+    parser.add_argument("--listener-ready-timeout", type=float, default=60,
+                        help="how long to give a listener that may have no "
+                             "microphone to start with")
+    parser.add_argument("--reply-timeout", type=float, default=120)
+    args = parser.parse_args(argv)
+
+    bus = None
+    try:
+        bus = Bus(args.url, args.connect_timeout)
+        wait_ready(bus, "audio", args.ready_timeout)
+
+        started = time.monotonic()
+        wav_bytes, tts_id = speak(bus, args.utterance, args.reply_timeout)
+        print(f"voice     : the audio service said {args.utterance!r} with {tts_id} "
+              f"in {time.monotonic() - started:.1f}s")
+        with open(args.wav, "wb") as out:
+            out.write(wav_bytes)
+        if tts_id != args.expect_tts:
+            raise Failure(
+                f"the running audio service speaks with {tts_id!r}, not {args.expect_tts!r}. "
+                f"mycroft.conf may name the local voice, but the service never loaded it."
+            )
+
+        try:
+            wait_ready(bus, "voice", args.listener_ready_timeout)
+        except Failure:
+            print("listener  : not ready, so it is not answering the bus here (no "
+                  "microphone to start with?)")
+            return LISTENER_UNAVAILABLE
+
+        started = time.monotonic()
+        heard = transcribe(bus, wav_bytes, args.reply_timeout)
+        print(f"listener  : heard {heard!r} in {time.monotonic() - started:.1f}s")
+        if args.expect_word not in heard.lower():
+            raise Failure(
+                f"the voice said {args.utterance!r} and the running listener heard "
+                f"{heard!r}."
+            )
+        print("the running services speak and hear on this device")
+        return 0
+    except Failure as failure:
+        print(f"\nlocal speech check failed: {failure}", file=sys.stderr)
+        return 1
+    finally:
+        if bus is not None:
+            bus.close()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/scenarios-ubuntu2404.yml

@@ -245,6 +245,32 @@ jobs:
         feature_set:
           - minimal
           - all
+        speech_engine:
+          - public
+        # Local speech is alpha only. Both runners have the 16 GB and the AVX2/NEON the
+        # installer's hardware check asks for, so a run that falls back to public here
+        # is a regression, not a runner limit.
+        include:
+          - runner: ubuntu-24.04
+            method: virtualenv
+            channel: alpha
+            feature_set: all
+            speech_engine: local
+          - runner: ubuntu-24.04-arm
+            method: virtualenv
+            channel: alpha
+            feature_set: all
+            speech_engine: local
+          - runner: ubuntu-24.04
+            method: containers
+            channel: alpha
+            feature_set: all
+            speech_engine: local
+          - runner: ubuntu-24.04-arm
+            method: containers
+            channel: alpha
+            feature_set: all
+            speech_engine: local
 
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 120
@@ -345,6 +371,7 @@ jobs:
           raspberry_pi_tuning: false
           share_telemetry: false
           share_usage_telemetry: false
+          speech_engine: ${{ matrix.speech_engine }}
           EOF
 
       - name: Run installer scenario exhaustive
@@ -410,6 +437,15 @@ jobs:
             --utterance "what time is it" --expect-speak \
             --ready-timeout 420 --reply-timeout 120 --speak-timeout 120
 
+      - name: Validate speech runs on the device
+        if: matrix.speech_engine == 'local'
+        run: |
+          if [ "${{ matrix.method }}" = "containers" ]; then
+            .github/scripts/assert_local_speech.sh --containers
+          else
+            .github/scripts/assert_local_speech.sh "$HOME/.venvs/ovos/bin/python"
+          fi
+
       - name: Create uninstall scenario file
         shell: bash
         run: |

AUDIT.md

@@ -0,0 +1,28 @@
+Last Edit: Codex (GPT-6) - 2026-10-08 - Motive: Record PR 648 findings, regression evidence and remaining validation limits.
+
+# Audit
+
+## 2026-10-08 — PR 648 follow-up
+
+- **Fixed: rejected STT downloads survived failed probes.** Timeout and unsuccessful
+  process exits now call `forget_model()` in
+  [`first_working()`](ansible/roles/ovos_config/files/speech_setup.py).
+  [`SpeechSetupTest.test_failed_probes_remove_only_new_downloads()`](scripts/test_speech_setup.py)
+  reproduces both failures, checks that existing caches survive and verifies the
+  successful fallback's files remain. The new assertions failed before the fix.
+- **Fixed: local-speech privacy wording.** The selection and summary disclose
+  public fallback. The speech screen and [automation guide](docs/automation.md)
+  describe failed/empty recognition uploads and public speech when a local model
+  cannot run; they make no fully offline promise.
+- **Verification:** `pytest -q scripts/test_speech_setup.py scripts/test_local_speech_bus.py`
+  passes **28 tests and 4 subtests**. **99 BATS tests pass with no skips**, including
+  all locales, navigation, real whiptail and speech Ansible tasks. Ruff, ShellCheck,
+  the offline contract checker and `git diff --check` pass. Display verification
+  is recorded in [the maintenance report](MAINTENANCE_REPORT.md).
+- **Outstanding integration evidence:** the full matrix for reviewed head
+  `f10c2d5f` failed two container scenarios with skill-ID errors before completing
+  all speech checks. See [run 37778575537](https://github.com/OpenVoiceOS/ovos-installer/actions/runs/37778575537).
+  The affected launch IDs are not changed by this patch. A new matrix run and
+  Raspberry Pi 5 timing are required before claiming complete device coverage.
+
+This is a focused review of the changed paths, not a whole-repository security audit.