From 4d7e4eba84bb5ccab0b1443995d3ca8cba607204 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 25 Sep 2026 17:40:56 +0200 Subject: [PATCH] fix(vendor-neutrality-score): never count partial-coverage tools as backends The scorer counted every `**Kind:** implementation` tool as a backend vendor and ignored the `**Coverage:**` qualifier, although docs/labels-and-capabilities.md says a partial tool "must not be counted as a complete/selectable backend". A read-only foundation for a new forge would therefore raise the contract counts in docs/vendor-neutrality.md as if it were a full backend. The scorer now reads `**Coverage:**` (`partial` or `partial-read-only`, backticks allowed), keeps partial tools out of the vendor count, lists them in the Basis column and the JSON output, and rejects unknown values. No score changes today: Atlassian stays counted through jira / jira-patch, and bitbucket is now shown as not counted. The coverage-qualifiers section also documents `partial`, which the bitbucket README already uses, and the scorer's behaviour. Generated-by: Claude Opus 5 --- docs/labels-and-capabilities.md | 8 ++++ docs/vendor-neutrality.md | 4 +- .../specs/meta-and-quality-tooling.md | 4 +- tools/vendor-neutrality-score/README.md | 4 ++ .../src/vendor_neutrality_score/__init__.py | 46 +++++++++++++++++-- .../tests/test_vendor_neutrality_score.py | 45 ++++++++++++++++++ 6 files changed, 105 insertions(+), 6 deletions(-) diff --git a/docs/labels-and-capabilities.md b/docs/labels-and-capabilities.md index 9faa053ac..0e7cabc43 100644 --- a/docs/labels-and-capabilities.md +++ b/docs/labels-and-capabilities.md @@ -157,6 +157,14 @@ when the tool intentionally implements only part of a contract. `partial-read-only` means the tool implements a read-only subset of named contract operations, but does not satisfy the complete contract and must not be advertised as a complete/selectable backend. +`partial` means the same without the read-only restriction: some named +operations, including writes, are implemented, but not the complete contract. + +The README form is a line `**Coverage:** partial` (or `partial-read-only`). +[`tools/vendor-neutrality-score`](../tools/vendor-neutrality-score/) reads it +and never counts a partial tool as a backend vendor, so a partial foundation +cannot turn a contract green in +[`docs/vendor-neutrality.md`](vendor-neutrality.md). Both capability axes are **orthogonal** to `family:*`. A single query can answer "how is our triage stack doing across PR + issue + diff --git a/docs/vendor-neutrality.md b/docs/vendor-neutrality.md index 47d7e3003..2ccac4d5d 100644 --- a/docs/vendor-neutrality.md +++ b/docs/vendor-neutrality.md @@ -568,9 +568,9 @@ generated block below. | Capability contract | Neutral? | Class | Backends today | Basis | |---|---|---|---|---| -| `contract:tracker` | ✅ | vendor-backed | Atlassian, Fossil, GitHub, SourceHut | 4 backend vendors: Atlassian, Fossil, GitHub, SourceHut | +| `contract:tracker` | ✅ | vendor-backed | Atlassian, Fossil, GitHub, SourceHut | 4 backend vendors: Atlassian, Fossil, GitHub, SourceHut; partial foundation, not counted: bitbucket | | `contract:source-control` | ✅ | vendor-backed | Fossil, Git, GitHub, SourceHut, Subversion | 5 backend vendors: Fossil, Git, GitHub, SourceHut, Subversion | -| `contract:change-request` | ✅ | vendor-backed | Atlassian, GitHub, email | 3 backend vendors: Atlassian, GitHub, email | +| `contract:change-request` | ✅ | vendor-backed | Atlassian, GitHub, email | 3 backend vendors: Atlassian, GitHub, email; partial foundation, not counted: bitbucket | | `contract:mail-archive` | ✅ | vendor-backed | ASF, Google, SourceHut | 3 backend vendors: ASF, Google, SourceHut | | `contract:mail-source` | ✅ | vendor-backed | ASF, Google, Maildir | 3 backend vendors: ASF, Google, Maildir | | `contract:mail-create` | ✅ | vendor-backed | Google, Maildir | 2 backend vendors: Google, Maildir | diff --git a/tools/spec-loop/specs/meta-and-quality-tooling.md b/tools/spec-loop/specs/meta-and-quality-tooling.md index 369986367..27e04933d 100644 --- a/tools/spec-loop/specs/meta-and-quality-tooling.md +++ b/tools/spec-loop/specs/meta-and-quality-tooling.md @@ -79,7 +79,9 @@ trustworthy as it grows. no model calls). For each capability contract, answers: does Magpie already work across more than one vendor, and is any skill locked to a vendor with no alternative? Reads `tools/*/README.md` and - `skills/*/SKILL.md` locally. Capability: `substrate:framework-dev + + `skills/*/SKILL.md` locally. A tool declaring `**Coverage:** partial` + (or `partial-read-only`) is listed but never counted as a backend + vendor. Capability: `substrate:framework-dev + substrate:analytics`. - `tools/preflight-audit/` — dry-runs the bulk-mode pre-flight classifier in live mode (via `gh api graphql`) or replay mode (offline, from a diff --git a/tools/vendor-neutrality-score/README.md b/tools/vendor-neutrality-score/README.md index 7f1abc3c2..beb34aba0 100644 --- a/tools/vendor-neutrality-score/README.md +++ b/tools/vendor-neutrality-score/README.md @@ -49,6 +49,10 @@ score: (the `contract:` it fulfils), `**Kind:**` (`interface` for a pure spec, `implementation` for a concrete backend), and `**Vendor:**` (the backend identity, or `agnostic` for an interface). + An optional `**Coverage:**` of `partial` or `partial-read-only` + ([coverage qualifiers](../../docs/labels-and-capabilities.md#coverage-qualifiers)) + marks a foundation that implements only part of its contracts: + it is listed in the result but never counted as a backend vendor. 2. **`skills/*/SKILL.md`** — the `organization:` frontmatter field plus the skill body, scanned for the concrete backends it names. 3. **The policy** in `src/vendor_neutrality_score/__init__.py` diff --git a/tools/vendor-neutrality-score/src/vendor_neutrality_score/__init__.py b/tools/vendor-neutrality-score/src/vendor_neutrality_score/__init__.py index 84a02f597..8185f4461 100644 --- a/tools/vendor-neutrality-score/src/vendor_neutrality_score/__init__.py +++ b/tools/vendor-neutrality-score/src/vendor_neutrality_score/__init__.py @@ -28,7 +28,10 @@ ``**Capability:**`` (the ``contract:`` it fulfils), ``**Kind:**`` (``interface`` for a pure spec, ``implementation`` for a concrete backend), and ``**Vendor:**`` (the backend identity, - or ``agnostic`` for an interface). + or ``agnostic`` for an interface). An optional ``**Coverage:**`` + of ``partial`` / ``partial-read-only`` marks a foundation that + implements only part of its contracts; it is reported but never + counted as a backend vendor. 2. ``skills/*/SKILL.md`` — the ``organization:`` frontmatter field (declared org scope) plus the skill body (scanned for the concrete backends it names). @@ -144,11 +147,18 @@ _CAP_RE = re.compile(r"^\*\*Capability:\*\*[ \t]+(.+)$", re.MULTILINE) _KIND_RE = re.compile(r"^\*\*Kind:\*\*[ \t]+(.+?)[ \t]*$", re.MULTILINE) _VENDOR_RE = re.compile(r"^\*\*Vendor:\*\*[ \t]+(.+?)[ \t]*$", re.MULTILINE) +_COVERAGE_RE = re.compile(r"^\*\*Coverage:\*\*[ \t]+(.+?)[ \t]*$", re.MULTILINE) _ORG_RE = re.compile(r"^organization:[ \t]*(.+?)[ \t]*$", re.MULTILINE) INTERFACE = "interface" IMPLEMENTATION = "implementation" +# Coverage qualifiers (docs/labels-and-capabilities.md#coverage-qualifiers). +# A partial implementation "must not be counted as a complete/selectable +# backend", so it never adds a vendor to a contract's count. +COMPLETE = "complete" +PARTIAL_COVERAGE = frozenset({"partial", "partial-read-only"}) + # --------------------------------------------------------------------------- # LLM / agent-integration axis # --------------------------------------------------------------------------- @@ -193,6 +203,11 @@ class ToolMeta: contracts: tuple[str, ...] kind: str vendor: str + coverage: str = COMPLETE + + @property + def partial(self) -> bool: + return self.coverage in PARTIAL_COVERAGE @dataclass @@ -204,6 +219,7 @@ class ContractResult: basis: str interfaces: list[str] = field(default_factory=list) implementations: list[ToolMeta] = field(default_factory=list) + partial_implementations: list[ToolMeta] = field(default_factory=list) @property def vendors(self) -> list[str]: @@ -285,7 +301,22 @@ def load_tools(repo_root: Path) -> list[ToolMeta]: raise ValueError( f"tools/{name}: **Kind:** must be '{INTERFACE}' or '{IMPLEMENTATION}', got '{kind}'" ) - tools.append(ToolMeta(name=name, contracts=contracts, kind=kind, vendor=vendor_m.group(1).strip())) + coverage_m = _COVERAGE_RE.search(text) + coverage = coverage_m.group(1).strip().strip("`").strip() if coverage_m else COMPLETE + if coverage != COMPLETE and coverage not in PARTIAL_COVERAGE: + raise ValueError( + f"tools/{name}: **Coverage:** must be one of " + f"{sorted({COMPLETE, *PARTIAL_COVERAGE})}, got '{coverage}'" + ) + tools.append( + ToolMeta( + name=name, + contracts=contracts, + kind=kind, + vendor=vendor_m.group(1).strip(), + coverage=coverage, + ) + ) return tools @@ -399,7 +430,8 @@ def score_contracts(tools: list[ToolMeta]) -> list[ContractResult]: for contract, (klass, summary) in CONTRACT_POLICY.items(): providers = [t for t in tools if contract in t.contracts] interfaces = sorted(t.name for t in providers if t.kind == INTERFACE) - impls = [t for t in providers if t.kind == IMPLEMENTATION] + impls = [t for t in providers if t.kind == IMPLEMENTATION and not t.partial] + partials = [t for t in providers if t.kind == IMPLEMENTATION and t.partial] res = ContractResult( contract=contract, klass=klass, @@ -408,6 +440,7 @@ def score_contracts(tools: list[ToolMeta]) -> list[ContractResult]: basis="", interfaces=interfaces, implementations=sorted(impls, key=lambda t: t.vendor), + partial_implementations=sorted(partials, key=lambda t: t.name), ) if klass == AGNOSTIC: res.green = True @@ -427,6 +460,9 @@ def score_contracts(tools: list[ToolMeta]) -> list[ContractResult]: res.basis = ( f"only {n} backend vendor ({', '.join(res.vendors)}); needs {MIN_VENDORS - n} more" ) + if res.partial_implementations: + names = ", ".join(t.name for t in res.partial_implementations) + res.basis += f"; partial foundation, not counted: {names}" results.append(res) return results @@ -539,6 +575,10 @@ def render_json( "vendors": r.vendors, "interfaces": r.interfaces, "implementations": [{"tool": t.name, "vendor": t.vendor} for t in r.implementations], + "partial_implementations": [ + {"tool": t.name, "vendor": t.vendor, "coverage": t.coverage} + for t in r.partial_implementations + ], } for r in contract_results ], diff --git a/tools/vendor-neutrality-score/tests/test_vendor_neutrality_score.py b/tools/vendor-neutrality-score/tests/test_vendor_neutrality_score.py index a7c515759..f6d3c5709 100644 --- a/tools/vendor-neutrality-score/tests/test_vendor_neutrality_score.py +++ b/tools/vendor-neutrality-score/tests/test_vendor_neutrality_score.py @@ -80,6 +80,51 @@ def test_interface_tools_do_not_count_as_a_backend() -> None: assert r.vendors == ["CVE.org"] +def test_partial_implementation_does_not_count_as_a_backend() -> None: + tools = [ + _tool("github", "contract:tracker", vns.IMPLEMENTATION, "GitHub"), + vns.ToolMeta( + name="gitlab", + contracts=("contract:tracker",), + kind=vns.IMPLEMENTATION, + vendor="GitLab", + coverage="partial", + ), + ] + r = _result(vns.score_contracts(tools), "contract:tracker") + assert r.green is False # one complete backend + one partial = one vendor + assert r.vendors == ["GitHub"] + assert [t.name for t in r.partial_implementations] == ["gitlab"] + assert "partial foundation, not counted: gitlab" in r.basis + + +def _write_contract_tool(root, name: str, extra: str = "") -> None: + d = root / "tools" / name + d.mkdir(parents=True) + (d / "README.md").write_text( + f"# {name}\n\n**Capability:** contract:tracker\n{extra}" + "**Kind:** implementation\n**Vendor:** Example\n\nProse.\n", + encoding="utf-8", + ) + + +def test_load_tools_reads_coverage(tmp_path) -> None: + _write_contract_tool(tmp_path, "full") + _write_contract_tool(tmp_path, "ticked", "**Coverage:** `partial`\n") + _write_contract_tool(tmp_path, "readonly", "**Coverage:** partial-read-only\n") + by_name = {t.name: t for t in vns.load_tools(tmp_path)} + assert by_name["full"].coverage == vns.COMPLETE + assert not by_name["full"].partial + assert by_name["ticked"].coverage == "partial" + assert by_name["readonly"].partial + + +def test_load_tools_unknown_coverage_raises(tmp_path) -> None: + _write_contract_tool(tmp_path, "odd", "**Coverage:** mostly\n") + with pytest.raises(ValueError, match="Coverage"): + vns.load_tools(tmp_path) + + def test_agnostic_contract_is_green_with_only_an_interface() -> None: tools = [_tool("scan-format", "contract:scan-format", vns.INTERFACE, "agnostic")] r = _result(vns.score_contracts(tools), "contract:scan-format")