diff --git a/docs/configuration.md b/docs/configuration.md index a71c72d..d8b797b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -110,7 +110,8 @@ override the shared `mode`. | classifier | `threshold` | 0.5 | confidence needed to dispatch | | classifier | `route_decline_class` | `active` | what "not for me" does: `active` passes it to the LLM only if the assistant spoke in the last `active_s`; `always` always passes; `never` declines | | classifier | `active_s` | 120 | the "assistant just spoke" window, in seconds | -| classifier | `dialog_negatives` | true | train "not for me" on the skills' own spoken lines | +| classifier | `dialog_negatives` | true | train "not for me" on the skills' own spoken lines, at most 2,000 per skill | +| classifier | `dialog_negatives_exclude` | `["ovos-skill-days-in-history.openvoiceos"]` | skills whose spoken lines are left out; days-in-history ships 136,639 lines of historical events, which only slow training down | | llm | `url`, `model`, `api_key` | none | an OpenAI-compatible endpoint; without `url` the step never matches | | llm | `timeout_s` | 4 | per request; there are two per utterance | | llm | `route_max_tokens`, `cache_size` | 200, 1024 | room for one tool call; cached responses | diff --git a/ovos_switchboard/adapters/dialog_files.py b/ovos_switchboard/adapters/dialog_files.py index 3f8e380..a6be431 100644 --- a/ovos_switchboard/adapters/dialog_files.py +++ b/ovos_switchboard/adapters/dialog_files.py @@ -9,13 +9,19 @@ LOG = logging.getLogger(__name__) ENTRY_POINT_GROUPS = ("ovos.plugin.skill", "opm.pipeline") +# Some skills ship data as dialog files (days-in-history: 136,639 lines, one per historical event). Every line becomes +# a training row, and at that size the classifier's fit overflows SciPy's L-BFGS-B and crashes. The largest ordinary +# skill measured had 1,643 lines (dad jokes), so the cap leaves those as they were. +MAX_LINES_PER_PACKAGE = 2000 -def package_dirs(): - """Package directories of every installed skill and pipeline plugin.""" +def package_dirs(exclude=()): + """Package directories of every installed skill and pipeline plugin, except the skill ids in `exclude`.""" dirs = set() for group in ENTRY_POINT_GROUPS: for ep in entry_points(group=group): + if ep.name in exclude: + continue top = ep.value.split(":", 1)[0].split(".", 1)[0] try: spec = importlib.util.find_spec(top) @@ -32,14 +38,30 @@ def dialog_files(root, lang): return sorted({f for p in patterns for f in glob.glob(os.path.join(root, p), recursive=True)}) -def installed_dialog_lines(lang="en-us", roots=None): - """Every line of every installed .dialog file for `lang`.""" +def installed_dialog_lines(lang="en-us", roots=None, max_per_package=MAX_LINES_PER_PACKAGE, exclude=()): + """Every line of every installed .dialog file for `lang`, at most `max_per_package` spoken lines per package, + skipping the skill ids in `exclude`.""" lines = [] - for root in roots if roots is not None else package_dirs(): - for path in dialog_files(root, lang): - try: - with open(path, encoding="utf-8", errors="ignore") as f: - lines += f.read().splitlines() - except OSError as e: - LOG.warning("skipping unreadable dialog file %s: %s", path, e) + for root in roots if roots is not None else package_dirs(exclude): + lines += _capped(root, _package_lines(root, lang), max_per_package) + return lines + + +def _package_lines(root, lang): + lines = [] + for path in dialog_files(root, lang): + try: + with open(path, encoding="utf-8", errors="ignore") as f: + lines += f.read().splitlines() + except OSError as e: + LOG.warning("skipping unreadable dialog file %s: %s", path, e) return lines + + +def _capped(root, lines, limit): + """The package's lines, or an evenly spaced, deterministic sample of its spoken lines when it has too many.""" + spoken = [line for line in lines if line.strip() and not line.lstrip().startswith("#")] + if len(spoken) <= limit: + return lines + LOG.info("%s has %d dialog lines; using %d of them as decline negatives", root, len(spoken), limit) + return [spoken[i * len(spoken) // limit] for i in range(limit)] diff --git a/ovos_switchboard/core/options.py b/ovos_switchboard/core/options.py index 463aea4..9c155f6 100644 --- a/ovos_switchboard/core/options.py +++ b/ovos_switchboard/core/options.py @@ -10,7 +10,8 @@ def options_from(cls, config): - """An options dataclass from the config keys it declares. Unknown keys are ignored; numbers are coerced.""" + """An options dataclass from the config keys it declares. Unknown keys are ignored; numbers and lists take the + field's type.""" config = config or {} return cls(**{f.name: _coerce(f.default, config[f.name]) for f in fields(cls) if f.name in config}) @@ -18,7 +19,7 @@ def options_from(cls, config): def _coerce(default, value): if default is MISSING or default is None or isinstance(default, bool): return value - return type(default)(value) if isinstance(default, (int, float)) else value + return type(default)(value) if isinstance(default, (int, float, tuple)) else value @dataclass(frozen=True) @@ -93,6 +94,8 @@ class ClassifierOptions: route_decline_class: str = "active" active_s: float = 120.0 # the "assistant active" window for route_decline_class="active" dialog_negatives: bool = True # train the decline class on installed skills' .dialog lines + # skills whose dialog files are data, not phrasing: days-in-history reads out 136,639 historical events + dialog_negatives_exclude: tuple = ("ovos-skill-days-in-history.openvoiceos",) def __post_init__(self): if self.route_decline_class not in DECLINE_POLICIES: diff --git a/ovos_switchboard/ovos/steps.py b/ovos_switchboard/ovos/steps.py index 5ab2cb2..40c448b 100644 --- a/ovos_switchboard/ovos/steps.py +++ b/ovos_switchboard/ovos/steps.py @@ -241,7 +241,8 @@ def build(self, b): """The classifier stage for the current registrations, trained or loaded from the model cache.""" synthetic = list(self.synthetic) if self.options.dialog_negatives: - synthetic += harvest_dialog_negatives(self.runtime.dialog_lines(b.lang), b.definitions, b.lang) + lines = self.runtime.dialog_lines(b.lang, exclude=self.options.dialog_negatives_exclude) + synthetic += harvest_dialog_negatives(lines, b.definitions, b.lang) rows = training_data(b.device, b.schema, synthetic) classifier, cached = self.runtime.models.get_or_build("classifier", [model_fingerprint(), rows], lambda: train_isolated(rows), Classifier) diff --git a/test/test_dialogs.py b/test/test_dialogs.py index 3600e45..aa52b2b 100644 --- a/test/test_dialogs.py +++ b/test/test_dialogs.py @@ -13,6 +13,22 @@ def test_dialog_lines_from_skill_locale_folders(tmp_path): assert sorted(lines) == ["# comment", "(The|Your) {device} is off", "All done."] +def test_a_package_with_too_many_lines_is_sampled_evenly(tmp_path): + folder = tmp_path / "skill_history" / "locale" / "en-us" + folder.mkdir(parents=True) + (folder / "events.dialog").write_text("# events\n" + "".join(f"Event number {i}.\n" for i in range(100))) + lines = installed_dialog_lines("en-us", roots=[str(tmp_path / "skill_history")], max_per_package=10) + assert lines == [f"Event number {i}." for i in range(0, 100, 10)] + + +def test_a_package_under_the_cap_keeps_every_line(tmp_path): + folder = tmp_path / "skill_small" / "locale" / "en-us" + folder.mkdir(parents=True) + (folder / "done.dialog").write_text("# comment\nAll done.\nFinished.\n") + assert installed_dialog_lines("en-us", roots=[str(tmp_path / "skill_small")], max_per_package=2) == [ + "# comment", "All done.", "Finished."] + + def test_negatives_skip_comments_short_lines_and_real_requests(): definitions = {"s": [{"skill_id": "s", "intent_name": "time", "lang": "en-us", "method": "template", "definition": {"samples": ["what time is it"]}}]} diff --git a/test/test_model_cache.py b/test/test_model_cache.py index b273428..b09a08a 100644 --- a/test/test_model_cache.py +++ b/test/test_model_cache.py @@ -86,7 +86,7 @@ def test_classifier_step_reuses_its_model_across_restarts(bus, tmp_path): def runtime(): return SwitchboardRuntime(bus, Settings(retrain_debounce_s=3600, decision_log=False, model_cache_path=str(tmp_path)), - dialog_lines=lambda lang: DIALOG_LINES, start=False) + dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False) first = runtime() ClassifierStep(bus, {}, runtime=first) register_all(bus) diff --git a/test/test_steps.py b/test/test_steps.py index faf71f3..67da394 100644 --- a/test/test_steps.py +++ b/test/test_steps.py @@ -24,7 +24,7 @@ def utterance_message(): def runtime_fixture(bus, tmp_path): rt = SwitchboardRuntime(bus, Settings(mode="active", retrain_debounce_s=3600, decision_log_path=str(tmp_path / "decisions.jsonl")), - dialog_lines=lambda lang: DIALOG_LINES, start=False) + dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False) register_all(bus) yield rt rt.shutdown() @@ -130,7 +130,7 @@ def test_high_risk_hold_consumes_the_utterance(runtime, tmp_path): def confirming_fixture(bus, tmp_path): rt = SwitchboardRuntime(bus, Settings(mode="active", high_risk="confirm", retrain_debounce_s=3600, decision_log_path=str(tmp_path / "decisions.jsonl")), - dialog_lines=lambda lang: DIALOG_LINES, start=False) + dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False) register_all(bus) yield rt rt.shutdown() @@ -194,7 +194,7 @@ def test_first_build_skips_the_debounce_and_answers_readiness(bus, tmp_path): """Live: "I am ready" came 8 s before Switchboard was; 5 s of it was the debounce after skills had loaded.""" rt = SwitchboardRuntime(bus, Settings(mode="active", retrain_debounce_s=3600, bus_timeout_s=0.1, decision_log_path=str(tmp_path / "decisions.jsonl")), - dialog_lines=lambda lang: DIALOG_LINES, start=False) + dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False) register_all(bus) ExactStep(bus, {}, runtime=rt) replies = [] @@ -279,7 +279,7 @@ def test_decider_step_dispatches_and_declines(runtime, monkeypatch): def test_shadow_mode_never_matches_and_later_steps_skip_settled_utterances(bus, tmp_path): rt = SwitchboardRuntime(bus, Settings(mode="shadow", retrain_debounce_s=3600, decision_log_path=str(tmp_path / "decisions.jsonl")), - dialog_lines=lambda lang: DIALOG_LINES, start=False) + dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False) register_all(bus) steps = [ExactStep(bus, {}, runtime=rt), ClassifierStep(bus, {}, runtime=rt)] rt.rebuild() @@ -345,7 +345,7 @@ def wait_for(condition, timeout=5.0): def test_rebuilds_wait_until_core_has_loaded_skills(bus): rt = SwitchboardRuntime(bus, Settings(retrain_debounce_s=0.01, bus_timeout_s=0.1, decision_log=False), - dialog_lines=lambda lang: DIALOG_LINES, start=False) + dialog_lines=lambda lang, exclude=(): DIALOG_LINES, start=False) ExactStep(bus, {}, runtime=rt) register_all(bus) time.sleep(0.2) @@ -358,7 +358,7 @@ def test_rebuilds_wait_until_core_has_loaded_skills(bus): def test_a_reloaded_pipeline_builds_when_core_is_already_ready(bus): bus.on("mycroft.skills.is_ready", lambda m: bus.emit(m.response({"status": True}))) rt = SwitchboardRuntime(bus, Settings(retrain_debounce_s=0.01, bus_timeout_s=0.1, decision_log=False), - dialog_lines=lambda lang: DIALOG_LINES) + dialog_lines=lambda lang, exclude=(): DIALOG_LINES) ExactStep(bus, {}, runtime=rt) register_all(bus) assert wait_for(lambda: rt.stage("exact") is not None) @@ -375,3 +375,22 @@ def test_wake_phrases_reach_the_exact_and_prefix_steps(bus): name, match = run_pipeline([exact, prefix], "hey mycroft tell me the time oh and") assert (name, match.match_type) == ("prefix", "ovos-skill-date-time.openvoiceos:what_time_is_it") rt.shutdown() + + +def test_the_classifier_skips_dialog_lines_from_excluded_skills(bus, tmp_path): + asked = [] + rt = SwitchboardRuntime(bus, Settings(mode="active", retrain_debounce_s=3600, + decision_log_path=str(tmp_path / "decisions.jsonl")), + dialog_lines=lambda lang, exclude=(): asked.append(exclude) or DIALOG_LINES, start=False) + register_all(bus) + try: + ClassifierStep(rt.bus, {"dialog_negatives_exclude": ["skill-trivia.example"]}, runtime=rt) + rt.rebuild() + assert asked == [("skill-trivia.example",)] + finally: + rt.shutdown() + + +def test_days_in_history_dialogs_are_excluded_by_default(runtime): + assert "ovos-skill-days-in-history.openvoiceos" in ClassifierStep( + runtime.bus, {}, runtime=runtime).options.dialog_negatives_exclude