datasets: pin down the agentdataflow break-test fixtures in tests/

.github/workflows/dataset-agentdataflow-validate.yml

@@ -32,8 +32,8 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Install jsonschema
-        run: python -m pip install --quiet jsonschema
+      - name: Install jsonschema and pytest
+        run: python -m pip install --quiet jsonschema pytest
 
       - name: Validate entries and example against schema.json
         run: python validate.py
@@ -44,3 +44,7 @@ jobs:
           python build_index.py
           git diff --exit-code index.csv
         working-directory: datasets/agentdataflow_toolexchange_traces
+
+      - name: Known-bad fixtures still fail the way they're supposed to
+        run: python -m pytest tests/ -v
+        working-directory: datasets/agentdataflow_toolexchange_traces

datasets/agentdataflow_toolexchange_traces/tests/fixtures/date-added-not-iso.json

@@ -0,0 +1,42 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-date-added-not-iso",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI06"
+  ],
+  "provenance": {
+    "tier": "hypothetical",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing."
+  },
+  "sanitization": {
+    "attestation": true,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "15-09-2026"
+}

datasets/agentdataflow_toolexchange_traces/tests/fixtures/dsgai-mapping-off-enum.json

@@ -0,0 +1,42 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-dsgai-mapping-off-enum",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI99"
+  ],
+  "provenance": {
+    "tier": "hypothetical",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing."
+  },
+  "sanitization": {
+    "attestation": true,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "2026-09-15"
+}

datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-cve-form.json

@@ -0,0 +1,49 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-evidence-citation-bad-cve-form",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI06"
+  ],
+  "provenance": {
+    "tier": "hypothetical",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing.",
+    "evidence": [
+      {
+        "type": "cve",
+        "citation": "CVE-25-1234",
+        "supports": "A malformed CVE id, for the citation-form check."
+      }
+    ]
+  },
+  "sanitization": {
+    "attestation": true,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "2026-09-15"
+}

datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-citation-bad-url-form.json

@@ -0,0 +1,49 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-evidence-citation-bad-url-form",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI06"
+  ],
+  "provenance": {
+    "tier": "hypothetical",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing.",
+    "evidence": [
+      {
+        "type": "url",
+        "citation": "example.com/not-a-scheme",
+        "supports": "A bare hostname typed as a url citation."
+      }
+    ]
+  },
+  "sanitization": {
+    "attestation": true,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "2026-09-15"
+}

datasets/agentdataflow_toolexchange_traces/tests/fixtures/evidence-missing-supports.json

@@ -0,0 +1,48 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-evidence-missing-supports",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI06"
+  ],
+  "provenance": {
+    "tier": "hypothetical",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing.",
+    "evidence": [
+      {
+        "type": "url",
+        "citation": "https://example.com/spec"
+      }
+    ]
+  },
+  "sanitization": {
+    "attestation": true,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "2026-09-15"
+}

datasets/agentdataflow_toolexchange_traces/tests/fixtures/filename-trace-id-mismatch.json

@@ -0,0 +1,42 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-does-not-match-filename",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI06"
+  ],
+  "provenance": {
+    "tier": "hypothetical",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing."
+  },
+  "sanitization": {
+    "attestation": true,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "2026-09-15"
+}

datasets/agentdataflow_toolexchange_traces/tests/fixtures/provenance-missing-evidence.json

@@ -0,0 +1,42 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-provenance-missing-evidence",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI06"
+  ],
+  "provenance": {
+    "tier": "observed_lab",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing."
+  },
+  "sanitization": {
+    "attestation": true,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "2026-09-15"
+}

datasets/agentdataflow_toolexchange_traces/tests/fixtures/sanitization-attestation-false.json

@@ -0,0 +1,42 @@
+{
+  "trace_id": "DSGAI-TRACE-fixture-sanitization-attestation-false",
+  "title": "Minimal valid trace, used as the base every broken fixture mutates",
+  "description": "Not a real security finding. Exists only so each broken fixture in this directory differs from a passing trace by exactly one thing, and so a future contributor changing the schema has something small to update instead of hand-writing a new minimal trace from scratch.",
+  "category": "tool_call",
+  "disposition": "benign",
+  "dsgai_mapping": [
+    "DSGAI06"
+  ],
+  "provenance": {
+    "tier": "hypothetical",
+    "collection_method": "Constructed by hand as a schema-validity fixture; models nothing."
+  },
+  "sanitization": {
+    "attestation": false,
+    "techniques": [
+      "not_applicable_fully_synthetic"
+    ]
+  },
+  "spans": [
+    {
+      "span_id": "s1",
+      "parent_span_id": null,
+      "t_offset_ms": 0,
+      "actor": "agent",
+      "operation": "tool.call",
+      "summary": "Placeholder span with no finding."
+    },
+    {
+      "span_id": "s2",
+      "parent_span_id": "s1",
+      "t_offset_ms": 10,
+      "actor": "tool",
+      "operation": "tool.result",
+      "summary": "Placeholder span with no finding."
+    }
+  ],
+  "security_observations": [
+    "This fixture demonstrates nothing; it exists to validate cleanly."
+  ],
+  "date_added": "2026-09-15"
+}