fix: a skill with huge dialog files can no longer crash classifier training

docs/configuration.md

@@ -110,7 +110,8 @@ override the shared `mode`.
 | classifier | `threshold` | 0.5 | confidence needed to dispatch |
 | classifier | `route_decline_class` | `active` | what "not for me" does: `active` passes it to the LLM only if the assistant spoke in the last `active_s`; `always` always passes; `never` declines |
 | classifier | `active_s` | 120 | the "assistant just spoke" window, in seconds |
-| classifier | `dialog_negatives` | true | train "not for me" on the skills' own spoken lines |
+| classifier | `dialog_negatives` | true | train "not for me" on the skills' own spoken lines, at most 2,000 per skill |
+| classifier | `dialog_negatives_exclude` | `["ovos-skill-days-in-history.openvoiceos"]` | skills whose spoken lines are left out; days-in-history ships 136,639 lines of historical events, which only slow training down |
 | llm | `url`, `model`, `api_key` | none | an OpenAI-compatible endpoint; without `url` the step never matches |
 | llm | `timeout_s` | 4 | per request; there are two per utterance |
 | llm | `route_max_tokens`, `cache_size` | 200, 1024 | room for one tool call; cached responses |

ovos_switchboard/adapters/dialog_files.py

@@ -9,13 +9,19 @@
 
 LOG = logging.getLogger(__name__)
 ENTRY_POINT_GROUPS = ("ovos.plugin.skill", "opm.pipeline")
+# Some skills ship data as dialog files (days-in-history: 136,639 lines, one per historical event). Every line becomes
+# a training row, and at that size the classifier's fit overflows SciPy's L-BFGS-B and crashes. The largest ordinary
+# skill measured had 1,643 lines (dad jokes), so the cap leaves those as they were.
+MAX_LINES_PER_PACKAGE = 2000
 
 
-def package_dirs():
-    """Package directories of every installed skill and pipeline plugin."""
+def package_dirs(exclude=()):
+    """Package directories of every installed skill and pipeline plugin, except the skill ids in `exclude`."""
     dirs = set()
     for group in ENTRY_POINT_GROUPS:
         for ep in entry_points(group=group):
+            if ep.name in exclude:
+                continue
             top = ep.value.split(":", 1)[0].split(".", 1)[0]
             try:
                 spec = importlib.util.find_spec(top)
@@ -32,14 +38,30 @@ def dialog_files(root, lang):
     return sorted({f for p in patterns for f in glob.glob(os.path.join(root, p), recursive=True)})
 
 
-def installed_dialog_lines(lang="en-us", roots=None):
-    """Every line of every installed .dialog file for `lang`."""
+def installed_dialog_lines(lang="en-us", roots=None, max_per_package=MAX_LINES_PER_PACKAGE, exclude=()):
+    """Every line of every installed .dialog file for `lang`, at most `max_per_package` spoken lines per package,
+    skipping the skill ids in `exclude`."""
     lines = []
-    for root in roots if roots is not None else package_dirs():
-        for path in dialog_files(root, lang):
-            try:
-                with open(path, encoding="utf-8", errors="ignore") as f:
-                    lines += f.read().splitlines()
-            except OSError as e:
-                LOG.warning("skipping unreadable dialog file %s: %s", path, e)
+    for root in roots if roots is not None else package_dirs(exclude):
+        lines += _capped(root, _package_lines(root, lang), max_per_package)
+    return lines
+
+
+def _package_lines(root, lang):
+    lines = []
+    for path in dialog_files(root, lang):
+        try:
+            with open(path, encoding="utf-8", errors="ignore") as f:
+                lines += f.read().splitlines()
+        except OSError as e:
+            LOG.warning("skipping unreadable dialog file %s: %s", path, e)
     return lines
+
+
+def _capped(root, lines, limit):
+    """The package's lines, or an evenly spaced, deterministic sample of its spoken lines when it has too many."""
+    spoken = [line for line in lines if line.strip() and not line.lstrip().startswith("#")]
+    if len(spoken) <= limit:
+        return lines
+    LOG.info("%s has %d dialog lines; using %d of them as decline negatives", root, len(spoken), limit)
+    return [spoken[i * len(spoken) // limit] for i in range(limit)]

ovos_switchboard/core/options.py

@@ -10,15 +10,16 @@
 
 
 def options_from(cls, config):
-    """An options dataclass from the config keys it declares. Unknown keys are ignored; numbers are coerced."""
+    """An options dataclass from the config keys it declares. Unknown keys are ignored; numbers and lists take the
+    field's type."""
     config = config or {}
     return cls(**{f.name: _coerce(f.default, config[f.name]) for f in fields(cls) if f.name in config})
 
 
 def _coerce(default, value):
     if default is MISSING or default is None or isinstance(default, bool):
         return value
-    return type(default)(value) if isinstance(default, (int, float)) else value
+    return type(default)(value) if isinstance(default, (int, float, tuple)) else value
 
 
 @dataclass(frozen=True)
@@ -93,6 +94,8 @@ class ClassifierOptions:
     route_decline_class: str = "active"
     active_s: float = 120.0         # the "assistant active" window for route_decline_class="active"
     dialog_negatives: bool = True   # train the decline class on installed skills' .dialog lines
+    # skills whose dialog files are data, not phrasing: days-in-history reads out 136,639 historical events
+    dialog_negatives_exclude: tuple = ("ovos-skill-days-in-history.openvoiceos",)
 
     def __post_init__(self):
         if self.route_decline_class not in DECLINE_POLICIES:

ovos_switchboard/ovos/steps.py

@@ -241,7 +241,8 @@ def build(self, b):
         """The classifier stage for the current registrations, trained or loaded from the model cache."""
         synthetic = list(self.synthetic)
         if self.options.dialog_negatives:
-            synthetic += harvest_dialog_negatives(self.runtime.dialog_lines(b.lang), b.definitions, b.lang)
+            lines = self.runtime.dialog_lines(b.lang, exclude=self.options.dialog_negatives_exclude)
+            synthetic += harvest_dialog_negatives(lines, b.definitions, b.lang)
         rows = training_data(b.device, b.schema, synthetic)
         classifier, cached = self.runtime.models.get_or_build("classifier", [model_fingerprint(), rows],
                                                               lambda: train_isolated(rows), Classifier)

test/test_dialogs.py

@@ -13,6 +13,22 @@ def test_dialog_lines_from_skill_locale_folders(tmp_path):
     assert sorted(lines) == ["# comment", "(The|Your) {device} is off", "All done."]
 
 
+def test_a_package_with_too_many_lines_is_sampled_evenly(tmp_path):
+    folder = tmp_path / "skill_history" / "locale" / "en-us"
+    folder.mkdir(parents=True)
+    (folder / "events.dialog").write_text("# events\n" + "".join(f"Event number {i}.\n" for i in range(100)))
+    lines = installed_dialog_lines("en-us", roots=[str(tmp_path / "skill_history")], max_per_package=10)
+    assert lines == [f"Event number {i}." for i in range(0, 100, 10)]
+
+
+def test_a_package_under_the_cap_keeps_every_line(tmp_path):
+    folder = tmp_path / "skill_small" / "locale" / "en-us"
+    folder.mkdir(parents=True)
+    (folder / "done.dialog").write_text("# comment\nAll done.\nFinished.\n")
+    assert installed_dialog_lines("en-us", roots=[str(tmp_path / "skill_small")], max_per_package=2) == [
+        "# comment", "All done.", "Finished."]
+
+
 def test_negatives_skip_comments_short_lines_and_real_requests():
     definitions = {"s": [{"skill_id": "s", "intent_name": "time", "lang": "en-us", "method": "template",
                           "definition": {"samples": ["what time is it"]}}]}

test/test_model_cache.py

@@ -86,7 +86,7 @@ def test_classifier_step_reuses_its_model_across_restarts(bus, tmp_path):
     def runtime():
         return SwitchboardRuntime(bus, Settings(retrain_debounce_s=3600, decision_log=False,
                                                 model_cache_path=str(tmp_path)),
-                                  dialog_lines=lambda lang: DIALOG_LINES, start=False)
+                                  dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False)
     first = runtime()
     ClassifierStep(bus, {}, runtime=first)
     register_all(bus)

test/test_steps.py

@@ -24,7 +24,7 @@ def utterance_message():
 def runtime_fixture(bus, tmp_path):
     rt = SwitchboardRuntime(bus, Settings(mode="active", retrain_debounce_s=3600,
                                           decision_log_path=str(tmp_path / "decisions.jsonl")),
-                            dialog_lines=lambda lang: DIALOG_LINES, start=False)
+                            dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False)
     register_all(bus)
     yield rt
     rt.shutdown()
@@ -130,7 +130,7 @@ def test_high_risk_hold_consumes_the_utterance(runtime, tmp_path):
 def confirming_fixture(bus, tmp_path):
     rt = SwitchboardRuntime(bus, Settings(mode="active", high_risk="confirm", retrain_debounce_s=3600,
                                           decision_log_path=str(tmp_path / "decisions.jsonl")),
-                            dialog_lines=lambda lang: DIALOG_LINES, start=False)
+                            dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False)
     register_all(bus)
     yield rt
     rt.shutdown()
@@ -194,7 +194,7 @@ def test_first_build_skips_the_debounce_and_answers_readiness(bus, tmp_path):
     """Live: "I am ready" came 8 s before Switchboard was; 5 s of it was the debounce after skills had loaded."""
     rt = SwitchboardRuntime(bus, Settings(mode="active", retrain_debounce_s=3600, bus_timeout_s=0.1,
                                           decision_log_path=str(tmp_path / "decisions.jsonl")),
-                            dialog_lines=lambda lang: DIALOG_LINES, start=False)
+                            dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False)
     register_all(bus)
     ExactStep(bus, {}, runtime=rt)
     replies = []
@@ -279,7 +279,7 @@ def test_decider_step_dispatches_and_declines(runtime, monkeypatch):
 def test_shadow_mode_never_matches_and_later_steps_skip_settled_utterances(bus, tmp_path):
     rt = SwitchboardRuntime(bus, Settings(mode="shadow", retrain_debounce_s=3600,
                                           decision_log_path=str(tmp_path / "decisions.jsonl")),
-                            dialog_lines=lambda lang: DIALOG_LINES, start=False)
+                            dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False)
     register_all(bus)
     steps = [ExactStep(bus, {}, runtime=rt), ClassifierStep(bus, {}, runtime=rt)]
     rt.rebuild()
@@ -345,7 +345,7 @@ def wait_for(condition, timeout=5.0):
 
 def test_rebuilds_wait_until_core_has_loaded_skills(bus):
     rt = SwitchboardRuntime(bus, Settings(retrain_debounce_s=0.01, bus_timeout_s=0.1, decision_log=False),
-                            dialog_lines=lambda lang: DIALOG_LINES, start=False)
+                            dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False)
     ExactStep(bus, {}, runtime=rt)
     register_all(bus)
     time.sleep(0.2)
@@ -358,7 +358,7 @@ def test_rebuilds_wait_until_core_has_loaded_skills(bus):
 def test_a_reloaded_pipeline_builds_when_core_is_already_ready(bus):
     bus.on("mycroft.skills.is_ready", lambda m: bus.emit(m.response({"status": True})))
     rt = SwitchboardRuntime(bus, Settings(retrain_debounce_s=0.01, bus_timeout_s=0.1, decision_log=False),
-                            dialog_lines=lambda lang: DIALOG_LINES)
+                            dialog_lines=lambda lang, exclude=(): DIALOG_LINES)
     ExactStep(bus, {}, runtime=rt)
     register_all(bus)
     assert wait_for(lambda: rt.stage("exact") is not None)
@@ -375,3 +375,22 @@ def test_wake_phrases_reach_the_exact_and_prefix_steps(bus):
     name, match = run_pipeline([exact, prefix], "hey mycroft tell me the time oh and")
     assert (name, match.match_type) == ("prefix", "ovos-skill-date-time.openvoiceos:what_time_is_it")
     rt.shutdown()
+
+
+def test_the_classifier_skips_dialog_lines_from_excluded_skills(bus, tmp_path):
+    asked = []
+    rt = SwitchboardRuntime(bus, Settings(mode="active", retrain_debounce_s=3600,
+                                          decision_log_path=str(tmp_path / "decisions.jsonl")),
+                            dialog_lines=lambda lang, exclude=(): asked.append(exclude) or DIALOG_LINES, start=False)
+    register_all(bus)
+    try:
+        ClassifierStep(rt.bus, {"dialog_negatives_exclude": ["skill-trivia.example"]}, runtime=rt)
+        rt.rebuild()
+        assert asked == [("skill-trivia.example",)]
+    finally:
+        rt.shutdown()
+
+
+def test_days_in_history_dialogs_are_excluded_by_default(runtime):
+    assert "ovos-skill-days-in-history.openvoiceos" in ClassifierStep(
+        runtime.bus, {}, runtime=runtime).options.dialog_negatives_exclude