From e4c383e524bca3eddf2d3c86d8f10c6c9b3e2da8 Mon Sep 17 00:00:00 2001 From: Michael Sitarzewski Date: Thu, 27 Aug 2026 10:44:18 -0500 Subject: [PATCH] fix: stop naming the operator view in robots.txt robots.txt is a public file that scanners fetch first, so a Disallow line advertises a path rather than protecting it. Naming the operator view there published the exact thing the line was meant to keep quiet. The canonical link already consolidates the operator view onto the board for any engine that honors it, and discloses nothing to do so. Nothing links to the view, so crawlers have no way to reach it in the first place. Adds a test asserting no public document names it, so this cannot regress. Note this changes what is advertised, not what is reachable: the view is toggled by client-side JavaScript that ships in the page, so anyone reading the source can still find it. Making it genuinely private needs a real mechanism, not a robots directive. Co-Authored-By: Claude Opus 5 (1M context) --- fintick/dashboard.py | 11 +++++++---- tests/test_dashboard.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/fintick/dashboard.py b/fintick/dashboard.py index 605d8ee..da8ba8b 100644 --- a/fintick/dashboard.py +++ b/fintick/dashboard.py @@ -148,13 +148,16 @@ def read_feed(database: str | Path, *, limit: int = DEFAULT_LIMIT) -> dict[str, SITE_ORIGIN = os.environ.get("FINTICK_SITE_ORIGIN", "https://fintick.fyi").rstrip("/") -# Crawlable: the board itself. Not crawlable: the JSON API (no prose to index) and the -# ?ops operator view, which is the same page plus telemetry and would read as duplicate -# content. The canonical link handles ?ops for engines that ignore the query rule. +# Crawlable: the board itself. Not crawlable: the JSON API, which carries no prose to +# index. +# +# The operator view is deliberately NOT named here. robots.txt is a public file that +# scanners fetch first, so a Disallow line advertises a path rather than protecting it. +# Consolidating the operator view onto the board is the canonical link's job, and it +# does that without publishing anything. ROBOTS_TXT = """User-agent: * Allow: /$ Disallow: /api/ -Disallow: /*?ops Sitemap: {origin}/sitemap.xml """ diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 52eafb8..d2a8924 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -264,15 +264,21 @@ def test_structured_data_parses(self) -> None: graph = json.loads(block)["@graph"] self.assertEqual({node["@type"] for node in graph}, {"WebSite", "WebApplication"}) - def test_robots_points_at_sitemap_and_shields_api_and_ops(self) -> None: + def test_robots_points_at_sitemap_and_disallows_the_api(self) -> None: status, headers, body = self._get("/robots.txt") text = body.decode() self.assertEqual(status, 200) self.assertTrue(headers["Content-Type"].startswith("text/plain")) self.assertIn("Disallow: /api/", text) - self.assertIn("Disallow: /*?ops", text) self.assertIn("Sitemap: https://fintick.fyi/sitemap.xml", text) + def test_public_documents_never_name_the_operator_view(self) -> None: + # robots.txt and llms.txt are fetched by scanners. Naming the operator view in + # either advertises it; a Disallow line is a signpost, not a shield. + for path in ("/robots.txt", "/llms.txt", "/sitemap.xml"): + _, _, body = self._get(path) + self.assertNotIn("ops", body.decode().lower(), f"{path} names the operator view") + def test_sitemap_is_well_formed_xml(self) -> None: import xml.etree.ElementTree as ET