diff --git a/fintick/dashboard.py b/fintick/dashboard.py index 605d8ee..da8ba8b 100644 --- a/fintick/dashboard.py +++ b/fintick/dashboard.py @@ -148,13 +148,16 @@ def read_feed(database: str | Path, *, limit: int = DEFAULT_LIMIT) -> dict[str, SITE_ORIGIN = os.environ.get("FINTICK_SITE_ORIGIN", "https://fintick.fyi").rstrip("/") -# Crawlable: the board itself. Not crawlable: the JSON API (no prose to index) and the -# ?ops operator view, which is the same page plus telemetry and would read as duplicate -# content. The canonical link handles ?ops for engines that ignore the query rule. +# Crawlable: the board itself. Not crawlable: the JSON API, which carries no prose to +# index. +# +# The operator view is deliberately NOT named here. robots.txt is a public file that +# scanners fetch first, so a Disallow line advertises a path rather than protecting it. +# Consolidating the operator view onto the board is the canonical link's job, and it +# does that without publishing anything. ROBOTS_TXT = """User-agent: * Allow: /$ Disallow: /api/ -Disallow: /*?ops Sitemap: {origin}/sitemap.xml """ diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 52eafb8..d2a8924 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -264,15 +264,21 @@ def test_structured_data_parses(self) -> None: graph = json.loads(block)["@graph"] self.assertEqual({node["@type"] for node in graph}, {"WebSite", "WebApplication"}) - def test_robots_points_at_sitemap_and_shields_api_and_ops(self) -> None: + def test_robots_points_at_sitemap_and_disallows_the_api(self) -> None: status, headers, body = self._get("/robots.txt") text = body.decode() self.assertEqual(status, 200) self.assertTrue(headers["Content-Type"].startswith("text/plain")) self.assertIn("Disallow: /api/", text) - self.assertIn("Disallow: /*?ops", text) self.assertIn("Sitemap: https://fintick.fyi/sitemap.xml", text) + def test_public_documents_never_name_the_operator_view(self) -> None: + # robots.txt and llms.txt are fetched by scanners. Naming the operator view in + # either advertises it; a Disallow line is a signpost, not a shield. + for path in ("/robots.txt", "/llms.txt", "/sitemap.xml"): + _, _, body = self._get(path) + self.assertNotIn("ops", body.decode().lower(), f"{path} names the operator view") + def test_sitemap_is_well_formed_xml(self) -> None: import xml.etree.ElementTree as ET