Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions .github/workflows/link-freshness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ on:
schedule:
- cron: "17 3 * * 1"
workflow_dispatch:
pull_request:
paths:
- ".github/workflows/link-freshness.yml"
- "scripts/check_links.py"
- "tests/test_check_links.py"

permissions:
contents: read
Expand All @@ -21,5 +26,7 @@ jobs:
- name: Install runtime dependency
run: python -m pip install -r requirements.txt
- name: Check official and evidence links
continue-on-error: true
run: python scripts/check_links.py --strict --max-urls 150 --output link-check.json
# Network findings are advisory; unexpected checker errors must still fail the job.
run: >-
python scripts/check_links.py --max-urls 150 --output link-check.json
--summary "$GITHUB_STEP_SUMMARY"
2 changes: 1 addition & 1 deletion datasets/china-rural-economic-statistics-1949-1986.md
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ provenance:
added: '2026-08-11'
confidence: high
verified: true
- source: https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ
- source: https://books.google.com/books?id=gS0yAAAAMAAJ
field_scope:
- corroborating bibliographic metadata
- no eBook availability statement
Expand Down
2 changes: 1 addition & 1 deletion dist/catalog.json
Original file line number Diff line number Diff line change
Expand Up @@ -31149,7 +31149,7 @@
"verified": true
},
{
"source": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ",
"source": "https://books.google.com/books?id=gS0yAAAAMAAJ",
"field_scope": [
"corroborating bibliographic metadata",
"no eBook availability statement"
Expand Down
2 changes: 1 addition & 1 deletion dist/catalog.jsonl

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions dist/checksums.sha256
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
f2fd0a1451ef6065c67dea32b682da7287a7b1645e3de3e0e0e2db2d9b7375e4 catalog.csv
ac8bd8aa531ffcf5b1703d28af0fb72ad1af8a10f561e259fe293b38a9fc6e35 catalog.json
d17d59eff8e8726f1c0eb29a2e09df7e8d476cb0a513b679da1731b319632bb7 catalog.jsonl
261e139cf6aa7c8a10d17b38887979a12f2403c7200d9595088d5db13a489617 catalog.json
0411893c36a16f3548dfec72e24c5ad6bad2794a637158900c0906e54b531b88 catalog.jsonl
a74962644744f55bcbaacb6eee9a761581e33f7c57d75f1161496fe385f5ae6a catalog.jsonld
dcf20c293a38e2befeb08e1fac2acf6acfa779e49d9609af2ac9c09eb32814f1 joins.json
dd8dad60a069b0a50d6c30c617f89d8c2daac2c3c2834e75bb3e68e61d5ba5b1 router_index.json
2 changes: 1 addition & 1 deletion docs/catalog.json
Original file line number Diff line number Diff line change
Expand Up @@ -31149,7 +31149,7 @@
"verified": true
},
{
"source": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ",
"source": "https://books.google.com/books?id=gS0yAAAAMAAJ",
"field_scope": [
"corroborating bibliographic metadata",
"no eBook availability statement"
Expand Down
2 changes: 1 addition & 1 deletion docs/datasets/china-rural-economic-statistics-1949-1986.md
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ provenance:
added: '2026-08-11'
confidence: high
verified: true
- source: https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ
- source: https://books.google.com/books?id=gS0yAAAAMAAJ
field_scope:
- corroborating bibliographic metadata
- no eBook availability statement
Expand Down
2 changes: 2 additions & 0 deletions guides/operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,8 @@ Generated files are outputs, not canonical sources: `DATASET_INDEX.md`, `ledgers

The public health report describes durable catalog inputs so the same report works in a fresh clone. Local fetch-cache diagnostics are printed separately by the validator and still fail the command when they expose an unrecorded challenge page; the raw cache is not a publication dependency. The static site serves copied canonical Markdown unchanged using `.nojekyll`. Its builder preserves existing files, and the freshness check reports unexpected pages for review rather than silently deleting them.

The weekly **Check external link freshness** workflow is a separate network diagnostic. Its run summary shows how many URLs were sampled and lists the ones needing review; a successful job means the scan finished, not that every link was reachable. Read a failed URL in context before changing a record: a login wall, denied automated request or timeout does not establish that the data are gone. Unexpected script errors fail the job. For a local report, run `python scripts/check_links.py --summary link-summary.md`; add `--strict` when you explicitly want any unsuccessful URL to produce a nonzero exit code.

## Queue and provenance

The task lifecycle is:
Expand Down
1 change: 1 addition & 0 deletions ledgers/changes.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -559,3 +559,4 @@
{"timestamp":"2026-09-28T19:33:43+08:00","action":"acquisition-grounding","dataset":"nbs-time-use-survey","agent":"codex","detail":"Primary NBS replies and laboratory documentation establish a conditional 2018 diary-microdata application route: institution registration, approved project, onsite analysis and reviewed outputs. Replaced the prior wholly unverified-access description without promising raw downloads or all-round access. The official 2018 bulletin was read by ordinary GET: named 11-province design, 20,226 households/48,580 people, and explicit Shanghai exclusion for published results; these are not verified laboratory inventory counts. The 2024 opening statement remains an announcement. Sanitized fields/geography, fees, current appointments and fully read economics-paper use remain open; status stays grounding. A 2023 Statistical Research PDF use lead returned 502 and is not used_by evidence.","files_changed":["datasets/nbs-time-use-survey.md","ledgers/changes.jsonl","ledgers/completed_tasks.jsonl"],"no_files_deleted_or_moved":true}
{"timestamp":"2026-09-28T23:39:14+08:00","action":"product-entry-and-evidence-consistency","agent":"codex","datasets":["cfps","china-land-transaction"],"detail":"Approved bounded public-product polish: shortened README around research-question use with Chinese and English prompts; routed answering and maintenance through one shared AGENTS/mental-model entry and scoped personal Kimi/DeepSeek notes. Aligned land prose with its existing grounding boundary and preserved former historical-price/completeness concerns as unverified; corrected CFPS prose on annual interpolation, local inference and the separate CHARLS provider/application identity using existing canonical records. No new source verification, status promotion, paper-entry removal or provenance removal. Fixed project-relative static record links, made readiness conditional in the UI/quality card, kept copied Markdown intact, and preserved existing generated pages for explicit freshness review. Public health now depends on durable inputs while local cache diagnostics still fail unrecorded challenges. Known scratch files are ignored, not deleted. Retained remote Ruff 0.15 and Actions upgrades without replacing newer local checks. Public-copy validator, Ruff 0.15.21, credential scan (403 files, zero findings), 65 tests and actual router-script checks passed. All 157 canonical records retained; only two canonical files edited. Three maintainer decision walkthroughs covered CFPS/CHARLS choice, conditional land-household construction and rejection of a national unemployment series for city-panel work; these were not blind model tests. Existing duplicate-task-title warning and legacy evidence debt remain. No commit or push.","files_changed":["README.md","AGENTS.md","guides/mental-model.md","guides/operations.md","guides/usage.md","guides/kimi-k3-handoff.zh-CN.md","CONTRIBUTING.md","datasets/cfps.md","datasets/china-land-transaction.md","scripts/build_site.py","scripts/build_quality_card.py","scripts/check_generated_views.py","scripts/validate_kb.py","tests/test_exports.py",".gitignore","requirements-dev.txt","pyproject.toml",".github/workflows/validate.yml",".github/workflows/link-freshness.yml"],"no_files_deleted_or_moved":true}
{"timestamp":"2026-09-29T01:24:14+08:00","action":"user-review-evidence-repair","agent":"codex","datasets":["chfs","china-pku-digital-financial-inclusion-index","cnopendata-local-government-work-report-corpus","china-5a-scenic-area-list"],"detail":"Focused correction of the independent user-role review findings. Preserved concrete variables, comparative research fit, acquisition steps, identities, paper leads and catalog statuses. CHFS wording now separates measured financial behavior from causal identification and approved access from all-wave access. PKU research-fit and prose no longer promote an abstract-only JRS lead to a verified use. Re-read the CnOpenData provider page: its TXT/PDF counts do not reconcile with the total, and its detailed periods differ by government level; retained the figures and the disagreement instead of inferring a city-only subset. Re-read the official 2024-02-06 5A announcement and counted 21 distinct provincial-level units. Public README corpus row reflects the period and count scope; its agent note retains product advantages without a star-count appeal. No new functionality, dataset deletion, status promotion, extra constraint system or full-catalog audit claim. CHFS access and JRS full-text were not newly verified; provider format reconciliation still requires a file manifest or reply."}
{"timestamp":"2026-10-26T18:25:00+08:00","action":"repair-link-freshness-check","agent":"codex","datasets":["china-rural-economic-statistics-1949-1986"],"detail":"GitHub run 40295619737 crashed with UnicodeEncodeError on a mixed Chinese/percent-encoded Google Books URL; continue-on-error concealed the incomplete scan as a successful job. Encode request path/query for HEAD and GET while preserving original report URLs and percent escapes; retain expected network failures as findings and let unexpected checker errors fail. The weekly job now publishes a readable bounded-scan summary without strict mode or blanket error suppression. Corrected only the book source URL to https://books.google.com/books?id=gS0yAAAAMAAJ; a fresh GET returned HTTP 200 and the title 中国农村经济统计大全, 1949-1986. Product identity, evidence scope and catalog status are unchanged. Local full network results are not link-death evidence because the local DNS environment resolves targets to non-public addresses and existing guards block them; use GitHub-hosted results for network follow-up. No records or files deleted."}
6 changes: 3 additions & 3 deletions ledgers/health.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"generated_at": "2026-09-30T11:34:44+08:00",
"generated_at": "2026-10-26T18:28:33+08:00",
"status": "ok",
"errors": [],
"warnings": [
Expand All @@ -17,7 +17,7 @@
"done": 434,
"failed": 157,
"candidates": 49,
"changes": 561
"changes": 562
},
"idea_benchmark_cases": 10,
"blind_benchmark_v2_cases": 27,
Expand Down Expand Up @@ -66,7 +66,7 @@
},
"input_fingerprint": {
"algorithm": "sha256",
"digest": "692ae6d0b526d9362054fe05ef3c23e89f05355dc7da8ec5d2d4dc24cf0d4f94",
"digest": "266d7ed7add3f294c25999131895da67517c361c59a298e84b6c484d5d413484",
"file_count": 168
}
}
49 changes: 42 additions & 7 deletions scripts/check_links.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from datetime import datetime
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urlsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener

sys.dont_write_bytecode = True
Expand Down Expand Up @@ -53,43 +54,77 @@ def check(url: str, timeout: float) -> dict[str, object]:
issues = public_http_url_issues(url, resolve_dns=False)
if issues:
return {"url_fingerprint": url_fingerprint(url), "result": "blocked", "issues": issues}
request = Request(url, method="HEAD", headers={"User-Agent": "econ-dataknowhow-link-check/1.0"})
# urllib needs an ASCII request target, even when the recorded URL contains Chinese text.
# Preserve URL separators and existing percent escapes on both request paths.
parts = urlsplit(url)
request_url = parts._replace(
path=quote(parts.path, safe="/%:@!$&'()*+,;="),
query=quote(parts.query, safe="/%?:@!$&'()*+,;="),
).geturl()
request = Request(request_url, method="HEAD", headers={"User-Agent": "econ-dataknowhow-link-check/1.0"})
try:
with safe_open(request, timeout=timeout) as response:
return {"url": url, "status": response.status, "final_url": response.geturl(), "result": "ok"}
except HTTPError as exc:
if exc.code != 405:
return {"url": url, "status": exc.code, "result": "http-error", "error": str(exc.reason)}
fallback = Request(
url,
request_url,
method="GET",
headers={"User-Agent": "econ-dataknowhow-link-check/1.0", "Range": "bytes=0-1024"},
)
try:
with safe_open(fallback, timeout=timeout) as response:
return {"url": url, "status": response.status, "final_url": response.geturl(), "result": "ok"}
except Exception as fallback_exc: # network diagnostics are deliberately best effort
return {"url": url, "status": exc.code, "result": "error", "error_type": type(fallback_exc).__name__}
except HTTPError as fallback_exc:
return {"url": url, "status": fallback_exc.code, "result": "http-error", "error": str(fallback_exc.reason)}
except (URLError, TimeoutError, OSError) as fallback_exc:
return {"url": url, "result": "error", "error_type": type(fallback_exc).__name__, "error": str(fallback_exc)}
except (URLError, TimeoutError, OSError) as exc:
return {"url": url, "result": "error", "error_type": type(exc).__name__}
return {"url": url, "result": "error", "error_type": type(exc).__name__, "error": str(exc)}


def build_summary(results: list[dict[str, object]], available_count: int) -> str:
failures = [item for item in results if item.get("result") != "ok"]
lines = [
"## External link freshness",
"",
f"Checked **{len(results)} of {available_count}** unique URLs; **{len(failures)}** need review.",
"",
"This is a bounded network diagnostic. Access restrictions and temporary connection errors need review "
"before changing a data record.",
]
if failures:
lines.extend(["", "| URL | HTTP status | Result | Detail |", "| --- | --- | --- | --- |"])
for item in failures:
url = str(item.get("url", "")).replace("|", "%7C")
label = f"<{url}>" if url else f"Blocked URL ({item.get('url_fingerprint', '')})"
detail = item.get("error") or item.get("error_type") or ", ".join(item.get("issues", []))
detail = str(detail).replace("|", "\\|").replace("\n", " ")
lines.append(f"| {label} | {item.get('status', '—')} | {item['result']} | {detail} |")
return "\n".join(lines) + "\n"


def main() -> int:
parser = argparse.ArgumentParser(description="Best-effort external link freshness check; never edits canonical records.")
parser.add_argument("--timeout", type=float, default=10.0)
parser.add_argument("--max-urls", type=int, default=150)
parser.add_argument("--output", type=Path, default=ROOT / "ledgers" / "link-check.json")
parser.add_argument("--summary", type=Path, help="also write a readable Markdown report, e.g. GITHUB_STEP_SUMMARY")
parser.add_argument("--strict", action="store_true", help="return non-zero when any checked URL is not successful")
args = parser.parse_args()
urls = collect_urls()[: max(args.max_urls, 0)]
results = [check(url, args.timeout) for url in urls]
urls = collect_urls()
results = [check(url, args.timeout) for url in urls[: max(args.max_urls, 0)]]
payload = {
"schema_version": 1,
"checked_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"available_count": len(urls),
"checked_count": len(results),
"results": results,
}
atomic_write_text(args.output, json.dumps(payload, ensure_ascii=False, indent=2) + "\n")
if args.summary:
atomic_write_text(args.summary, build_summary(results, len(urls)))
failures = sum(1 for item in results if item.get("result") != "ok")
print(f"checked={len(results)} failures={failures} output={args.output}")
return 1 if args.strict and failures else 0
Expand Down
88 changes: 88 additions & 0 deletions tests/test_check_links.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
from __future__ import annotations

import json
import sys
from contextlib import nullcontext
from types import SimpleNamespace
from urllib.error import HTTPError, URLError

import pytest

import check_links
from check_links import check


Expand All @@ -12,3 +21,82 @@ def test_link_checker_blocks_private_and_script_targets_without_network() -> Non
assert "non-http-scheme" in script["issues"]
assert "url" not in private
assert "url" not in script


@pytest.mark.parametrize("fallback", [False, True])
def test_unicode_url_is_encoded_for_head_and_get(monkeypatch, fallback) -> None:
url = "https://books.google.com/books/about/%E4%B8%AD国农村经济.html?id=123&title=统计"
calls = []

def fake_open(request, timeout):
calls.append(request)
request.full_url.encode("ascii") # Matches urllib's HTTP request-target requirement.
if fallback and request.get_method() == "HEAD":
raise HTTPError(request.full_url, 405, "Method Not Allowed", {}, None)
return nullcontext(SimpleNamespace(status=200, geturl=lambda: request.full_url))

monkeypatch.setattr(check_links, "safe_open", fake_open)
result = check(url, timeout=0.01)

assert result["result"] == "ok"
assert result["url"] == url
assert "%E4%B8%AD%E5%9B%BD" in result["final_url"]
assert "title=%E7%BB%9F%E8%AE%A1" in result["final_url"]
assert "%25E4" not in result["final_url"]
assert [request.get_method() for request in calls] == (["HEAD", "GET"] if fallback else ["HEAD"])


def test_get_fallback_reports_its_own_http_status(monkeypatch) -> None:
def fake_open(request, timeout):
status = 405 if request.get_method() == "HEAD" else 403
raise HTTPError(request.full_url, status, "request denied", {}, None)

monkeypatch.setattr(check_links, "safe_open", fake_open)
result = check("https://example.org/", timeout=0.01)
assert result["result"] == "http-error"
assert result["status"] == 403


@pytest.mark.parametrize("strict, expected_exit", [(False, 0), (True, 1)])
def test_cli_finishes_and_reports_network_findings(monkeypatch, tmp_path, strict, expected_exit) -> None:
urls = ["https://example.org/ok", "https://example.org/denied", "https://example.org/timeout", "https://example.org/later"]
output = tmp_path / "links.json"
summary = tmp_path / "summary.md"
arguments = ["check_links.py", "--max-urls", "3", "--output", str(output), "--summary", str(summary)]
if strict:
arguments.append("--strict")

def fake_open(request, timeout):
if request.full_url.endswith("/denied"):
raise HTTPError(request.full_url, 403, "Forbidden", {}, None)
if request.full_url.endswith("/timeout"):
raise URLError("timed out")
return nullcontext(SimpleNamespace(status=200, geturl=lambda: request.full_url))

monkeypatch.setattr(sys, "argv", arguments)
monkeypatch.setattr(check_links, "collect_urls", lambda: urls)
monkeypatch.setattr(check_links, "safe_open", fake_open)

assert check_links.main() == expected_exit
report = json.loads(output.read_text(encoding="utf-8"))
assert report["available_count"] == 4
assert report["checked_count"] == 3
assert [item["result"] for item in report["results"]] == ["ok", "http-error", "error"]
markdown = summary.read_text(encoding="utf-8")
assert "**3 of 4**" in markdown
assert "**2** need review" in markdown
assert "https://example.org/denied" in markdown
assert "403" in markdown
assert "https://example.org/timeout" in markdown
assert "timed out" in markdown


def test_unexpected_checker_error_is_not_hidden(monkeypatch, tmp_path) -> None:
def broken_check(url, timeout):
raise RuntimeError("unexpected checker error")

monkeypatch.setattr(sys, "argv", ["check_links.py", "--output", str(tmp_path / "links.json")])
monkeypatch.setattr(check_links, "collect_urls", lambda: ["https://example.org/"])
monkeypatch.setattr(check_links, "check", broken_check)
with pytest.raises(RuntimeError, match="unexpected checker error"):
check_links.main()
Loading