From c903332d0036903ebe30cabfca0878435b1b61ed Mon Sep 17 00:00:00 2001 From: sheeesocute <103700059@qq.com> Date: Mon, 26 Oct 2026 18:38:07 +0800 Subject: [PATCH] Fix Unicode URL crashes and report link-check findings clearly --- .github/workflows/link-freshness.yml | 11 ++- ...ina-rural-economic-statistics-1949-1986.md | 2 +- dist/catalog.json | 2 +- dist/catalog.jsonl | 2 +- dist/checksums.sha256 | 4 +- docs/catalog.json | 2 +- ...ina-rural-economic-statistics-1949-1986.md | 2 +- guides/operations.md | 2 + ledgers/changes.jsonl | 1 + ledgers/health.json | 6 +- scripts/check_links.py | 49 +++++++++-- tests/test_check_links.py | 88 +++++++++++++++++++ 12 files changed, 152 insertions(+), 19 deletions(-) diff --git a/.github/workflows/link-freshness.yml b/.github/workflows/link-freshness.yml index 42d3210..5a005d9 100644 --- a/.github/workflows/link-freshness.yml +++ b/.github/workflows/link-freshness.yml @@ -4,6 +4,11 @@ on: schedule: - cron: "17 3 * * 1" workflow_dispatch: + pull_request: + paths: + - ".github/workflows/link-freshness.yml" + - "scripts/check_links.py" + - "tests/test_check_links.py" permissions: contents: read @@ -21,5 +26,7 @@ jobs: - name: Install runtime dependency run: python -m pip install -r requirements.txt - name: Check official and evidence links - continue-on-error: true - run: python scripts/check_links.py --strict --max-urls 150 --output link-check.json + # Network findings are advisory; unexpected checker errors must still fail the job. + run: >- + python scripts/check_links.py --max-urls 150 --output link-check.json + --summary "$GITHUB_STEP_SUMMARY" diff --git a/datasets/china-rural-economic-statistics-1949-1986.md b/datasets/china-rural-economic-statistics-1949-1986.md index b096358..85dc4de 100644 --- a/datasets/china-rural-economic-statistics-1949-1986.md +++ b/datasets/china-rural-economic-statistics-1949-1986.md @@ -197,7 +197,7 @@ provenance: added: '2026-08-11' confidence: high verified: true - - source: https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ + - source: https://books.google.com/books?id=gS0yAAAAMAAJ field_scope: - corroborating bibliographic metadata - no eBook availability statement diff --git a/dist/catalog.json b/dist/catalog.json index 49d53f9..fe9d78e 100644 --- a/dist/catalog.json +++ b/dist/catalog.json @@ -31149,7 +31149,7 @@ "verified": true }, { - "source": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ", + "source": "https://books.google.com/books?id=gS0yAAAAMAAJ", "field_scope": [ "corroborating bibliographic metadata", "no eBook availability statement" diff --git a/dist/catalog.jsonl b/dist/catalog.jsonl index 631ffd2..bd27087 100644 --- a/dist/catalog.jsonl +++ b/dist/catalog.jsonl @@ -87,7 +87,7 @@ {"schema_version": 3, "catalog_status": "ready", "id": "china-qss-chinawomen-derived-2017", "name": "QSS chinawomen public county-cohort and crop derivative", "aka": ["chinawomen.csv", "QSS Table 7.4 China sex-ratio and agricultural-crop data", "QSS chinawomen R dataset", "中国县—出生年份性别比与农作物种植派生数据"], "provider": "Kosuke Imai, Jeffrey Arnold and Tyler Simko; QSS supplementary repository and qss R package", "china_related": true, "domains": ["regional", "rural", "agriculture", "demographics", "gender", "education"], "data_pathway": {"mode": "direct", "origin": "researcher-constructed", "target_artifact": "Public QSS supplementary file UNCERTAINTY/chinawomen.csv and the same table packaged as qss::chinawomen (chinawomen.rda). It is a compact, paper-related derivative for QSS Table 7.4, not the original Qian (2008 QJE) census extracts or a complete replication package.", "availability": "reproducible", "ordinary_researcher_feasible": true, "summary": "The CSV is directly downloadable from the public QSS GitHub repository, and the qss-package build script copies that CSV and reads it into an R data object using the documented column specification. The package documentation reports 51,766 county-birth-year rows and nine variables, so a researcher can use the released derivative immediately for teaching, descriptive checks, or a carefully labelled extension of the Qian-related exercise.", "barrier": "The public route does not establish the exact upstream census files, sample weights, Michigan GIS concordance, or all restrictions used in Qian's final QJE analysis. The numeric admin code has no public codebook or province/boundary crosswalk in the checked package, and the data license of the underlying source material is not separately documented beyond the GPL-2 package license."}, "unit_of_observation": "County identifier by birth-cohort year row", "structure": "County-by-cohort cross-section with county crop measures repeated across cohorts", "geo_granularity": ["county identifier"], "geography": "Chinese counties represented by the package's numeric admin codes. The checked CSV has 1,648 unique admin values; the package does not publish a county-name, province, boundary-vintage, or complete-universe codebook.", "time_span": {"start": "1962", "end": "1990", "last_confirmed_release": "2022-04-20", "coverage_note": "biryr is the birth/cohort year and ranges from 1962 through 1990 in the checked CSV. This is not an annual agricultural panel: teasown, orch, and cashcrop are county crop fields repeated or aligned to cohort rows, while post is a supplied reform-period indicator.", "last_checked": "2026-09-28"}, "frequency": ["cohort-year"], "sample_size": "51,766 rows, 1,648 unique admin codes, 1962-1990 birth years; the checked CSV has 2,684 rows with birpop recorded as NA. These counts are for the public derivative, not proof of Qian's original 1% census sample or its final estimation sample.", "key_variables": [{"admin": "numeric county identifier with no accompanying public crosswalk"}, {"biryr": "birth/cohort year"}, {"birpop": "birth population in the cohort row"}, {"han": "Han share or Han indicator as represented in the derivative"}, {"sex": "proportion male in the birth cohort"}, {"teasown": "quantity of tea sown in the county"}, {"orch": "quantity of orchard-type crops planted"}, {"cashcrop": "quantity of cash crops planted"}, {"post": "indicator for introduction of the price-reform period"}], "research_fit": {"best_for": ["Reproducing the public QSS Table 7.4 exercise and inspecting the relationship between county crop measures, cohorts, and sex ratios with a small R/Python/Stata conversion step", "A transparent, public starting table when the research question only needs the released county/cohort variables and can tolerate an undocumented geography key", "Teaching or robustness demonstrations that explicitly label the file as a Qian- related derivative rather than an exact QJE replication"], "choose_over": ["Choose this public derivative over the unresolved Qian census candidate when an immediately downloadable demonstration table is more important than exact sample recovery.", "Choose the China county population/agriculture GIS record when a documented historical boundary layer or broader 1990 county aggregates are needed; this file has no public GIS geometry or county-name crosswalk.", "Choose the China Census record or an approved census product when individual, household, later-wave, or raw census variables are essential."], "not_good_for": ["Exact reproduction of Qian's 2008 QJE estimates, weights, county restrictions, or matched 1990/1997 census/GIS pipeline", "A national county panel, current regional data, agricultural output or price series, or a file with stable cross-study county identifiers", "Treating post as a standalone policy or causal-variation record"], "needs_join_for": ["County names, province labels, boundaries, or GIS coordinates via a separately verified historical concordance", "Original population-census education, occupation, household, or migration fields", "RCRE/NFS household labor inputs and external crop prices if the research needs the mechanisms documented in Qian's paper"], "variation_available": ["post is an included data field marking a reform period; it is not a classification of an exogenous shock and no variation record is created here."], "topics": ["rural China", "county demography", "sex ratios", "agricultural specialization", "crop geography", "historical regional development"]}, "good_for": ["Public county-cohort descriptive analysis using the released nine-field table", "Reproducing the QSS teaching exercise with the original column names", "A clearly bounded input for a historical rural-China data demonstration"], "identification": ["No standalone causal-identification design is documented for this public derivative; use it as the released county-cohort table rather than as evidence of an encoded treatment assignment.", "post is a supplied reform-period field, not a verified policy-assignment, treatment, or comparison definition."], "linkable_keys": ["admin (within-file only unless a documented crosswalk is obtained)", "biryr"], "joins": [{"target": "china-county-population-agriculture-gis-1990", "relation": "complement", "keys": ["Historical county identifier or documented name crosswalk", "1990 reference year"], "method": "Use the GIS product only after obtaining and checking a concordance between its county key and admin. Do not assume that a six-digit-looking admin code has the same meaning or boundary vintage as the GIS product.", "evidence_status": "plausible"}, {"target": "china-census", "relation": "complement", "keys": ["County code after a verified historical concordance", "Birth/cohort year where the approved census file supports it"], "method": "Treat the QSS table as a derived outcome/crop table and the census record as a separate source for additional fields or later waves. A shared topic or county label is not evidence of a lossless match.", "evidence_status": "plausible"}, {"target": "china-agricultural-yearbook", "relation": "complement", "keys": ["County/province crosswalk", "Reference year"], "method": "Use yearbook values only for independently documented agricultural controls and record any change in units or geography; they do not reveal the QSS file's source fields or turn it into an annual panel.", "evidence_status": "plausible"}], "access_routes": [{"route": "QSS original repository CSV", "access_status": "available", "direct_url": "https://github.com/kosukeimai/qss/blob/master/UNCERTAINTY/chinawomen.csv", "requirements": ["A browser, Git, or direct raw-file download", "Acceptance of the repository's GPL-2.0 terms for the QSS materials"], "steps": ["Open the QSS repository file and use its Raw/download route.", "Preserve the header and the nine-column layout; the public file is about 2.5 MB.", "Record the retrieval date and inspect missing values before analysis.", "Keep the file labelled as a QSS derivative and do not claim that it is the exact Qian census extract."], "deliverable": "51,766-row CSV with nine columns: admin, biryr, birpop, han, sex, teasown, orch, cashcrop, post.", "cost": "free", "last_checked": "2026-09-28", "caveat": "The repository and package license the QSS materials under GPL-2/GPL-2.0, but the underlying census/agricultural source rights and redistribution conditions are not separately documented."}, {"route": "qss R package on GitHub", "access_status": "available", "direct_url": "https://github.com/kosukeimai/qss-package", "requirements": ["R and the remotes/devtools installation path, or a prebuilt qss package copy", "Ability to use the package under GPL-2"], "steps": ["Install with remotes::install_github(\"kosukeimai/qss-package\") or inspect the repository's data/chinawomen.rda member.", "Load data(chinawomen, package = \"qss\") and compare the column names with the CSV.", "Use the package only as a convenience wrapper; the build script shows that it copies the original QSS CSV and parses it with a column specification."], "deliverable": "qss::chinawomen R data frame with the same nine fields and 51,766 rows.", "cost": "free", "last_checked": "2026-09-28", "caveat": "The R package route does not add the missing county crosswalk or prove access to Qian's upstream sources."}], "access": {"url": "https://github.com/kosukeimai/qss/blob/master/UNCERTAINTY/chinawomen.csv", "cost": "free", "license": "QSS repository and qss-package metadata display GPL-2/GPL-2.0; verify upstream-data rights before redistributing a derived product", "format": ["csv", "rda"], "api": false, "how_to_get": "Download the QSS CSV directly, or install qss-package and load qss::chinawomen; retain the source version and the derivative label."}, "caveats": ["QSS documentation cites Qian's QJE paper and labels this as the QSS Table 7.4 exercise, but it does not say that the CSV is the exact paper-delivered analysis file.", "Qian's final paper describes a matched 1% 1997 Agricultural Census, 1% 1990 Population Census, and Michigan GIS dataset at the county/birth-year level, with 1,621 counties in 15 southern provinces and rural residents born 1962-1990. The public derivative has 1,648 admin codes and does not expose those restrictions or the GIS/census fields; preserve this difference.", "The file is cohort-oriented, not a repeated annual agriculture panel. Crop fields, post, and cohort rows should not be interpreted as a complete time series.", "The package has no public county-name/province/boundary codebook in the checked files. Do not infer geography from the numeric admin code.", "The source CSV contains missing birpop values; decide how to handle them and record any exclusions rather than silently dropping rows."], "production": {"raw_sources": [{"name": "QSS UNCERTAINTY/chinawomen.csv", "source_type": "dataset", "role": "Public derivative table used in the QSS Table 7.4 exercise", "access_route": "Public QSS GitHub repository", "url": "https://github.com/kosukeimai/qss/blob/master/UNCERTAINTY/chinawomen.csv", "coverage": "51,766 county-birth-year rows, 1962-1990 cohort years, nine fields", "last_checked": "2026-08-13"}, {"name": "Qian (2008) paper-described census/GIS inputs", "source_type": "document", "role": "Documents the upstream lineage and exact-paper boundary; it is not a file delivered by QSS", "access_route": "Author-hosted QJE paper PDF", "url": "https://www.kellogg.northwestern.edu/faculty/qian/resources/missing-women_qje_20080407_all.pdf", "coverage": "1% 1997 Agricultural Census, 1% 1990 Population Census, Michigan GIS, county/birth-year matching, 1,621 southern counties as described in the paper", "last_checked": "2026-08-13"}], "acquisition_methods": ["repository download", "R package installation", "CSV-to-R-data serialization"], "sample_construction": "The QSS package build script copies CSV files from the supplementary qss repository, applies the documented chinawomen column specification, and saves the resulting data frame as chinawomen.rda. The checked materials do not document the upstream census extraction, county matching, weighting, or GIS construction that produced the derivative CSV.", "pipeline_stages": [{"stage": "collect", "inputs": ["QSS UNCERTAINTY/chinawomen.csv"], "method": "Download the public CSV from the QSS supplementary repository.", "tools": ["GitHub or raw-file download"], "output": "Public nine-column CSV", "evidence": "QSS repository file and repository README"}, {"stage": "parse", "inputs": ["chinawomen.csv"], "method": "Read the header using the explicit column specification and parse integer/double fields; the build script skips the header and writes the R data object.", "tools": ["R"], "output": "qss::chinawomen data frame and chinawomen.rda", "evidence": "qss-package data-raw/build.R and data-raw/spec/chinawomen.R"}, {"stage": "validate", "inputs": ["Public CSV", "Package documentation"], "method": "Compare row count, nine column names, numeric types, cohort-year range, and missing-value behavior. Do not use this check as proof of the exact Qian census sample.", "tools": ["R or Python/pandas"], "output": "A reproducible derivative table with an explicit provenance warning", "evidence": "qss-package documentation and direct CSV inspection; no reconstruction of the Qian upstream data was performed"}], "constructed_variables": [], "output": {"unit_of_observation": "County identifier by cohort/birth year", "structure": "Cross-sectional cohort table", "geography": "Numeric county admin codes; names and boundaries unresolved", "time_span": "1962-1990 cohort years", "key_variables": ["admin", "biryr", "birpop", "han", "sex", "teasown", "orch", "cashcrop", "post"], "formats": ["csv", "rda"]}, "reproducibility": {"level": "medium", "starting_point": "https://github.com/kosukeimai/qss/blob/master/UNCERTAINTY/chinawomen.csv", "code_available": true, "code_url": "https://raw.githubusercontent.com/kosukeimai/qss-package/master/data-raw/build.R", "requirements": ["Public GitHub access or a local clone", "R only if the qss-package serialization route is desired", "A separately documented county crosswalk for spatial joins"], "blockers": ["Exact upstream census/agricultural files and GIS concordance are not included", "County codebook and boundary vintage are not identified", "Exact match to Qian's final estimation sample and weights is not established"]}, "compliance": {"terms_or_license": "GPL-2/GPL-2.0 for QSS materials; upstream data rights require separate verification", "robots_or_rate_limits": "Use GitHub's public file/repository route; do not scrape provider systems to infer missing upstream data", "personal_or_sensitive_data": "Aggregate/cohort fields as documented; no individual-level records are identified in the public CSV", "redistribution": "Follow GPL terms for the package and check source-data rights before redistributing a repackaged derivative", "review_needed": true}}, "quality": {"profile_status": "verified", "access_status": "verified", "paper_use_status": "partial", "last_audited": "2026-09-28"}, "used_by": [{"cite": "Qian (2008), Missing Women and the Price of Tea in China: The Effect of Sex-Specific Earnings on Sex Imbalance", "doi": "https://doi.org/10.1162/qjec.2008.123.3.1251", "journal": "QJE", "year": 2008, "dataset_role": "Related public derivative referenced by the QSS exercise; not certified as the exact QJE analysis file", "evidence_type": "teaching-derivative", "evidence_url": "https://github.com/kosukeimai/qss/blob/master/UNCERTAINTY/chinawomen.csv", "data_note": "QSS documentation explicitly references Qian's paper and describes chinawomen as demographic and agricultural crop data for individual Chinese counties. The Qian paper itself documents a larger/more restricted matched census-GIS construction; the QSS file is therefore a usable public derivative, not evidence that the original 1% census extract or the complete QJE replication package is public."}], "provenance": [{"source": "https://www.kellogg.northwestern.edu/faculty/qian/resources/missing-women_qje_20080407_all.pdf", "field_scope": ["QJE paper identity and DOI context", "1% 1997 Agricultural Census, 1% 1990 Population Census and Michigan GIS inputs", "county/birth-year matching, 1,621 counties in 15 southern provinces, rural 1962-1990 sample boundary"], "added": "2026-08-13", "confidence": "high", "verified": true}, {"source": "https://github.com/kosukeimai/qss/blob/master/UNCERTAINTY/chinawomen.csv", "field_scope": ["Public CSV identity and download route", "header, nine columns, and repository file boundary"], "added": "2026-08-13", "confidence": "high", "verified": true}, {"source": "https://raw.githubusercontent.com/kosukeimai/qss/master/UNCERTAINTY/chinawomen.csv", "field_scope": ["Current direct-download route returned HTTP 200 on 2026-09-28", "Current CSV content length was 2,618,228 bytes and retains the documented nine-field header"], "added": "2026-09-28", "confidence": "high", "verified": true}, {"source": "https://raw.githubusercontent.com/kosukeimai/qss-package/master/DESCRIPTION", "field_scope": ["Current package metadata identifies qss as packaged QSS supplementary material and states GPL-2"], "added": "2026-09-28", "confidence": "high", "verified": true}, {"source": "https://raw.githubusercontent.com/kosukeimai/qss-package/master/data-raw/build.R", "field_scope": ["Current build script copies QSS CSV files and serializes them as package data objects", "The chinawomen column specification remains available in the same public repository"], "added": "2026-09-28", "confidence": "high", "verified": true}, {"source": "https://rdrr.io/github/kosukeimai/qss-data/man/chinawomen.html", "field_scope": ["51,766-row, nine-variable documentation", "variable descriptions and explicit Qian/QSS reference"], "added": "2026-08-13", "confidence": "high", "verified": true}, {"source": "https://raw.githubusercontent.com/kosukeimai/qss-package/master/data-raw/build.R", "field_scope": ["CSV-copy and CSV-to-RDA packaging steps", "use of the column specification and public package output"], "added": "2026-08-13", "confidence": "high", "verified": true}, {"source": "https://raw.githubusercontent.com/kosukeimai/qss-package/master/DESCRIPTION", "field_scope": ["qss package authors, supplementary-material identity and GPL-2 license"], "added": "2026-08-13", "confidence": "high", "verified": true}, {"source": "https://github.com/kosukeimai/qss/blob/master/UNCERTAINTY/chinawomen.csv", "field_scope": ["local read-only inspection of 51,766 rows, 1,648 unique admin values, 1962-1990 cohort range, 2,684 missing birpop values, and file size about 2.5 MB"], "added": "2026-08-13", "confidence": "high", "verified": true}], "related_datasets": [{"id": "china-county-population-agriculture-gis-1990", "relation": "complement"}, {"id": "china-census", "relation": "complement"}, {"id": "china-agricultural-yearbook", "relation": "complement"}], "source_path": "datasets/china-qss-chinawomen-derived-2017.md", "description_markdown": "## Positioning in one sentence\n\nThis is a small, public, QSS-packaged China county-by-cohort table related to Qian's QJE study: it is immediately usable for the nine released variables through a current direct CSV route, but it is not the original 1% census/GIS match and cannot be used as an exact QJE replication without additional evidence.\n\n## Select rules\n\n- Use it when the question needs the public QSS exercise table and can work with numeric county IDs, 1962-1990 cohort rows, and the nine documented fields.\n- Use the unresolved Qian agricultural-census candidate when exact sample, weights, GIS linkage, or the original paper's full analysis is the goal; use the 1990 county/GIS product when a public historical spatial key is more important.\n- Never silently upgrade this derivative into the 1997 Agricultural Census, 1990 Population Census, or Qian's complete matched dataset.\n\n## Get recipe\n\nDownload `UNCERTAINTY/chinawomen.csv` from the QSS repository, retain the header, inspect missing `birpop`, and keep the file labelled as a derived teaching asset. If an R object is more convenient, install `qss-package` and load `qss::chinawomen`; the package build script simply copies and parses the CSV. Obtain a separate, documented county crosswalk before any spatial join, and do not claim exact QJE replication from the public file.\n\n## Connections and limitations\n\nThe only safe internal key is `admin` together with `biryr`; `admin` has no verified crosswalk in the package. Joining to the 1990 county/GIS record, census waves, or yearbooks requires a historical concordance and a check of county boundaries. The data are useful precisely because the derivative is public and compact, while the unknown source lineage, weighting, and paper-sample restrictions remain visible instead of being filled by assumption.", "research_position": "This is a small, public, QSS-packaged China county-by-cohort table related to Qian's QJE study: it is immediately usable for the nine released variables through a current direct CSV route, but it is not the original 1% census/GIS match and cannot be used as an exact QJE replication without additional evidence."} {"schema_version": 3, "catalog_status": "ready", "id": "china-restud-melon-market-labels-2025", "name": "Shijiazhuang watermelon retail-market experiment data (sellers, transactions, household purchases; Bai 2025 ReStud)", "aka": ["Melons as Lemons replication package", "Bai watermelon branding experiment data", "Zenodo 10.5281/zenodo.13909671", "石家庄西瓜市场品牌标签实验数据"], "provider": "Jie Bai (author); field experiment conducted in Shijiazhuang, Hebei, China (2014); replication package on Zenodo (CC-BY-4.0)", "china_related": true, "domains": ["development", "agriculture", "retail-markets", "information-economics", "field-experiment"], "data_pathway": {"mode": "direct", "origin": "researcher-collected", "target_artifact": "The author's field-experiment dataset: 60 sellers in 60 Shijiazhuang watermelon markets randomized into 3 branding treatments (laser-cut label, sticker label, no label) crossed with an incentive treatment (6 groups); daily seller sales records (60,806 transactions, 19 Jul - 6 Sep 2014), biweekly sweetness quality checks, daily pricing/wholesale-price observations, and household purchasing diaries (675 households in 27 communities; 15,292 records; final analysis sample 4,309 watermelon purchases from 573 households in 26 communities). Published as a Zenodo replication package (10.5281/zenodo.13909671) under CC-BY-4.0 explicitly containing \"all the STATA and Matlab codes as well as the original and intermediary data inputs\".", "availability": "reproducible", "ordinary_researcher_feasible": true, "summary": "The paper's full text (working-paper version) documents the experiment and data collection in detail, and the Zenodo DataCite record states the package contains the STATA/Matlab code and the original and intermediary data inputs for all tables and figures, under CC-BY-4.0. This is the strongest acquisition route among the current batch: a direct, free, license-clear download. The public record currently exposes a separate _readme.docx and two downloadable ZIP archives, including the later \"Replication Package REStud 25751_v2.zip\" (9,728,123 bytes). Its complete 270-member archive manifest was inspected on 2026-09-28: it contains Stata seller transactions, quality sampling, surveys, household diaries and baseline/follow-up files; Matlab inputs; survey forms; Stata and Matlab code; and rendered results. Column-level content remains a separate inspection step.", "barrier": "The archive is directly usable, but its public manifest alone does not document every variable label or confirm any difference between the cached working-paper text and the final ReStud version."}, "unit_of_observation": "Seller-day records (sales transactions); seller-week (quality checks); household purchase records; households (panel)", "structure": "Multi-component: transaction-level seller sales (60,806 records), biweekly seller quality checks, daily pricing observations, household purchase diaries (panel), baseline/endline/follow-up surveys", "geo_granularity": ["market (60 local markets; average distance between markets ~3 km)", "community (27 communities for household sample)"], "geography": "Shijiazhuang, Hebei province, China (urban; over 800 gated communities and 200+ local markets in the city)", "time_span": {"start": "2014-07", "end": "2015", "last_confirmed_release": "2024-11-25", "coverage_note": "Intervention 13-19 July 2014 (rolling-in excluded; day 1 = 19 July); phasing out 6-12 September 2014; sales records 19 July - 6 September 2014 (8 weeks); endline survey at final visit; two follow-up surveys after the intervention (including one year later). Replication package issued 2024-11-25 (DataCite).", "last_checked": "2026-09-27"}, "frequency": ["daily (seller sales, pricing observations)", "biweekly (quality checks, twice per week)", "purchase-record level (household diaries)", "survey wave (baseline/endline/follow-ups)"], "sample_size": "60 sellers in 60 markets; 60,806 transaction records (81% watermelon, 19% peach); 675 households in 27 communities, 15,292 purchase records; final household analysis sample 4,309 watermelon purchases, 573 households, 26 communities", "key_variables": [{"Seller": "daily sales records (fruit type, quantity in Jin, sales value in RMB, quality category premium/normal), daily wholesale price, branding treatment (laser/sticker/label-less), incentive treatment, quality-check sweetness (center and side, averaged), pricing and differentiation behavior"}, {"Household": "purchase date, place, quantity, amount paid, satisfaction rating 1-5, whether bought from sample seller, whether fruit branded"}, "Baseline/endline/follow-up survey responses (sellers' future differentiation plans, consumer perceptions)", {"Stratification": "baseline average housing price in surrounding gated communities (above/below median)"}], "research_fit": {"best_for": ["Field-experimental evidence on quality signaling, seller reputation, and consumer learning in developing-country retail markets", "Estimating how branding technologies (costly signaling) affect seller quality provision, prices, sales, and profits", "Structural estimation of consumer learning and seller reputation (the paper estimates an empirical model on the experimental variation)", "Welfare analysis of information frictions and market fragmentation in agricultural retail"], "choose_over": ["Choose this over china-weekly-provincial-hog-prices when the question is quality signaling and seller behavior at transaction level, not a price panel.", "Choose this over survey-based market studies when randomized branding treatments and both seller-side and household-side records are needed.", "For other crops or other cities, the experiment must be re-run; the data are specific to Shijiazhuang watermelons, summer 2014."], "not_good_for": ["National or provincial agricultural market statistics (60 markets in one city, one season)", "Post-2015 market behavior (follow-ups cover longer-term seller behavior, but the main panel is 2014)", "Questions needing large-N seller or market samples (60 sellers; one per market by design)", "Reconstructing the experiment's raw context (e.g., city-wide sales) beyond the recorded fields"], "needs_join_for": ["City-level context or controls (e.g., weather during the season) if needed for robustness", "Other retail-market data for generalizing beyond watermelons"], "variation_available": ["Market-level randomization into 6 treatment cells (3 branding x 2 incentive)", {"Temporal variation": "mandatory differentiation (first 2 weeks) vs free choice; incentive removal at week 6 (unanticipated)"}, "Data-side variation dimensions only; the identification design belongs to the Econ-Variation repository."], "topics": ["quality signaling", "seller reputation", "consumer learning", "field experiment", "agricultural markets", "branding"]}, "good_for": ["Signaling and reputation experiments in retail markets", "Structural models of consumer learning estimated on experimental data", "Household purchase-diary analysis of demand responses to quality signals"], "identification": [{"This is a released experimental-data asset": "it supports reproducing the paper's randomized-branding analysis, but the public files themselves do not establish external validity beyond the 60 Shijiazhuang markets and the documented 2014 study window."}], "linkable_keys": ["Seller identifier (60 sample sellers)", "Market identifier (60 markets) and community identifier (27 communities)", "Date (day-level records)", "Household identifier (household diaries)"], "joins": [{"target": "china-weekly-provincial-hog-prices", "relation": "often-confused-with", "keys": ["None (different commodity, unit, and geography)"], "method": "Keep the Shijiazhuang experiment separate from the provincial hog-price panel; they answer different questions.", "evidence_status": "verified"}], "access_routes": [{"route": "Zenodo replication package (ReStud)", "access_status": "available", "direct_url": "https://zenodo.org/doi/10.5281/zenodo.13909671", "requirements": ["None; CC-BY-4.0 license"], "steps": ["Open the Zenodo record 10.5281/zenodo.13909671 (latest version) or the concept DOI 10.5281/zenodo.13909670.", "Download _readme.docx and the latest named archive, \"Replication Package REStud 25751_v2.zip\" (9,728,123 bytes in the record checked 2026-09-27).", {"Read the README and select the needed data path": "data/stata/ contains seller_transactions.dta, seller_transactions_collapsed.dta, qualitysampling.dta, household_diary.dta, baseline/follow-up and surveyor files; data/matlab/ contains the structural-estimation inputs."}, "Use code/stata do files/ and code/matlab m files/ for replication, then check the paper's final ReStud version for any data-section differences."], "deliverable": "A directly downloadable v2 archive with named Stata data, Matlab inputs, survey instruments, Stata/Matlab code, and rendered tables/figures; table columns and the published-version comparison remain to be checked for a new analysis.", "cost": "free", "last_checked": "2026-09-27", "caveat": "Zenodo API and the complete v2 ZIP manifest confirm the practical file-level route. The archive listing does not itself establish every variable label or all researcher-side privacy decisions."}, {"route": "Paper full text (working-paper version)", "access_status": "available", "direct_url": "needs-verification", "requirements": "None", "steps": ["Read the working-paper full text (cached copy) sections 4-5 for the experimental design and data-collection details.", {"Cross-check with the published ReStud version (92(6)": "3574-3610) for any changes."}], "deliverable": "Design and data documentation; not the data.", "cost": "free", "last_checked": "2026-08-15", "caveat": "The cached text is the job-market-paper version; the published version's data section was not read this round."}], "access": {"url": "https://zenodo.org/doi/10.5281/zenodo.13909671", "cost": "free", "license": "CC-BY-4.0", "format": ["STATA files", "Matlab files", "CSV and MAT inputs", "DOCX survey instruments and README"], "api": true, "how_to_get": "Download the public v2 Zenodo ZIP, read its README, select the Stata or Matlab data path, and run the supplied code after checking local dependencies."}, "caveats": [{"The complete public v2 ZIP manifest is verified": "it includes Stata seller transactions, quality sampling, household diary/baseline/follow-up and surveyor files; Matlab inputs; survey forms; Stata/Matlab code; and results. Inspect table labels before using any field beyond its filename-level description."}, "The cached full text is the working-paper version; the ReStud-published version (2025, vol 92) may differ in data details.", "Sample is one city (Shijiazhuang), one season (summer 2014), 60 sellers (one per market by design) - no national generalization.", "Only the \"Jingxin\" watermelon breed is used in the analysis (other breeds <2% of recorded sales).", "Sellers' self-recorded sales have known recording noise; the paper validates with surveyor-counted branded-watermelon records and self-recalled totals.", {"Household analysis sample drops households with 3+ missing weekly records (final": "573 households, 26 communities, 4,309 watermelon purchases)."}], "production": {"raw_sources": [{"name": "Author field experiment (Shijiazhuang, 2014)", "source_type": "dataset", "role": "All experimental data (sales, quality, pricing, household diaries, surveys)", "access_route": "Released as the Zenodo replication package", "url": "https://zenodo.org/doi/10.5281/zenodo.13909671", "coverage": "60 sellers, 60 markets; 19 Jul - 6 Sep 2014; 675 households, 27 communities", "last_checked": "2026-08-15"}], "acquisition_methods": ["in-person survey (baseline/endline/follow-ups)", "seller daily record sheets (sales)", "surveyor daily market visits (pricing, branding service, counts)", "biweekly unannounced quality checks (sweetness meter)", "household purchase diaries"], "sample_construction": "60 sellers recruited from 60 different markets (one per market; 3-5 fruit sellers per market typically) following initial screening and sequential selection (Appendix C.1 of the paper); sellers randomized into 6 groups (3 branding x 2 incentive), stratified on baseline housing prices of surrounding gated communities; 675 households in 27 communities recruited for purchase diaries, evenly distributed across treatment groups; households with 3+ missing weekly records dropped (final 573 households, 26 communities).", "pipeline_stages": [{"stage": "collect", "inputs": ["Seller daily sales records", "Quality checks (twice weekly, sweetness meter)", "Surveyor daily pricing visits and branding service", "Household diaries", "Surveys"], "method": "Field experiment with daily/biweekly data collection over 8 weeks", "output": "Transaction records (60,806), quality checks, pricing series, household panel", "evidence": "Paper sections 4.1 and 4.3"}, {"stage": "clean", "inputs": ["Seller records (omissions, lumped sales)", "Household diaries (missing place/branding/satisfaction)"], "method": "Exclude rolling-in period (pre-19 July); validate with surveyor counts and recalled totals; match household-seller data for missing info; drop households with 3+ missing weekly records", "output": "Analysis samples (sellers full; households 573/26 communities; 4,309 watermelon records)", "evidence": "Paper section 4.3 and Appendix C.2"}], "constructed_variables": [{"name": "Sales profit", "concept": "Daily sales profit computed from sales quantity and prices", "source_fields": ["Seller sales records", "Prices"], "method": "Sales quantity x price-based computation (paper section 4.3 footnote 34)", "validation": "Paper text", "limitations": "Recording noise; validated against surveyor counts"}, {"name": "Watermelon quality (sweetness)", "concept": "Average sweetness measured at center and side", "source_fields": ["Biweekly quality checks"], "method": "Sweetness meter; average of center and side readings", "validation": "Maps to consumer ratings (sweetness >10.5 roughly maps to subjective ratings)", "limitations": "Quality checks are periodic, not continuous"}], "validation": [{"Recording-noise checks": "self-recalled vs recorded totals; surveyor-counted branded melons sold vs seller records"}, "Final sample characteristics compared to full sample (Appendix C.2)"], "output": {"unit_of_observation": "Seller transaction; seller-week (quality); household purchase record", "structure": "Multi-component experimental dataset", "geography": "Shijiazhuang urban markets and communities", "time_span": "July-September 2014 (main); follow-ups to ~1 year after", "key_variables": ["Sales (quantity Jin, value RMB, quality category)", "Quality (sweetness)", "Prices (retail and wholesale)", "Treatment (branding x incentive)", "Household purchases (date, place, quantity, amount, satisfaction, branding)"], "formats": ["STATA", "Matlab"]}, "reproducibility": {"level": "high", "starting_point": "https://zenodo.org/doi/10.5281/zenodo.13909671", "code_available": true, "code_url": "https://zenodo.org/doi/10.5281/zenodo.13909671", "requirements": ["STATA and Matlab (versions per README)", "The downloaded package (code + original and intermediary data inputs)"], "blockers": ["Published ReStud version's data section not read (working-paper version cached)"]}, "compliance": {"terms_or_license": "CC-BY-4.0 (replication package)", "robots_or_rate_limits": "Zenodo public download; no scraping needed", "personal_or_sensitive_data": "Household diaries may contain personal consumption records; follow CC-BY-4.0 attribution and reasonable privacy handling", "redistribution": "CC-BY-4.0 permits sharing with attribution", "review_needed": false}}, "quality": {"profile_status": "verified", "access_status": "verified", "paper_use_status": "verified", "last_audited": "2026-09-28"}, "used_by": [{"cite": "Bai, Jie (2025), Melons as Lemons: Asymmetric Information, Consumer Learning and Seller Reputation, Review of Economic Studies 92(6): 3574-3610", "doi": "https://doi.org/10.1093/restud/rdaf006", "journal": "Review of Economic Studies", "year": 2025, "dataset_role": "Main dataset (field experiment on 60 sellers and 675 households) for signaling, reputation, and consumer-learning analyses", "evidence_type": "data-section", "evidence_url": "https://zenodo.org/doi/10.5281/zenodo.13909671", "data_note": "Paper (working-paper version, cached) sections 4.1-4.3 document the design and data: 60 sellers in 60 Shijiazhuang markets; 6 treatment cells (3 branding x 2 incentive); 60,806 transaction records (19 Jul - 6 Sep 2014); biweekly sweetness checks; daily pricing visits; 675 households / 15,292 purchase records (final analysis sample 4,309 watermelon purchases from 573 households in 26 communities). Zenodo replication package 10.5281/zenodo.13909671 (CC-BY-4.0) contains STATA/Matlab code and original and intermediary data inputs per the DataCite description."}], "provenance": [{"source": "Working-paper full text (cached melon_nfe_pdf.txt), sections 1, 4.1-4.3, and appendix notes", "field_scope": ["experiment design (treatments, timeline, stratification)", "data collection (sales records, quality checks, pricing, household diaries, surveys)", "sample construction and cleaning (rolling-in exclusion, household drops)", "key figures (60,806 transactions; 675 households; final 4,309 records)"], "added": "2026-08-15", "confidence": "high", "verified": true}, {"source": "DataCite records 10.5281/zenodo.13909670 (concept) and 10.5281/zenodo.13909671 (version)", "field_scope": ["package existence, title, creator, year (2024-11-25), license (CC-BY-4.0)", {"description": "all the STATA and Matlab codes as well as the original and intermediary data inputs"}], "added": "2026-08-15", "confidence": "high", "verified": true}, {"source": "Zenodo record API https://zenodo.org/api/records/13909671 and public v2 archive response", "field_scope": ["current_public_access", "top_level_file_inventory", "archive_names_and_sizes", "open_access_and_license"], "added": "2026-09-27", "confidence": "high", "verified": true}, {"source": "Complete public v2 ZIP archive manifest (downloaded and inspected 2026-09-28)", "field_scope": ["exact code, results, Stata-data, Matlab-input and survey-instrument paths", "practical archive structure and uncompressed size (18,911,698 bytes across 270 members)"], "added": "2026-09-28", "confidence": "high", "verified": true}, {"source": "ReStud page (via cached Crossref/abstract evidence)", "field_scope": ["publication identity (ReStud 92(6), DOI 10.1093/restud/rdaf006)"], "added": "2026-08-15", "confidence": "high", "verified": true}], "related_datasets": [{"id": "china-weekly-provincial-hog-prices", "relation": "often-confused-with"}], "source_path": "datasets/china-restud-melon-market-labels-2025.md", "description_markdown": "## Positioning in one sentence\n\nA complete field-experiment asset: 60 Shijiazhuang watermelon sellers (6 randomized branding x incentive cells) with 60,806 transaction records, biweekly quality checks, daily pricing, and household purchase diaries (675 households), released as a CC-BY-4.0 Zenodo replication package (10.5281/zenodo.13909671) that per its metadata includes the original and intermediary data inputs - the most directly reproducible asset in this batch.\n\n## Select rules\n\n- Prioritize it when the question is quality signaling, seller reputation, or consumer learning in a developing-country retail market with randomized branding treatments.\n- Switch to china-weekly-provincial-hog-prices for a long price panel; switch to other survey assets for household welfare questions beyond the purchase diaries.\n- It cannot support national agricultural statistics, other crops/cities, or post-2015 market behavior beyond the follow-up surveys.\n\n## Get recipe\n\n1. Download the Zenodo package (10.5281/zenodo.13909671; CC-BY-4.0, no account needed for public files).\n2. Read the README and select data/stata/ for the seller, quality, household and survey layers, or data/matlab/ for structural-estimation inputs; use code/stata do files/ and code/matlab m files/ for replication.\n3. Cross-check the data section of the published ReStud version (92(6)) against the cached working-paper text for any differences.\n4. Run the code to regenerate tables/figures; extend only with the paper's cleaning rules (rolling-in exclusion, Jingxin breed, household completeness filters).\n\n## Connections and Limitations\n\nThe dataset is self-contained (seller-market-household links with daily dates); joins to outside sources are limited by the single-city, single-season scope. The public v2 archive contains the named Stata, Matlab, survey, code and result paths, while column-level interpretation and any difference between the working-paper and published data sections remain separate checks. Quality (sweetness) is measured periodically, and seller records have known recording noise validated by surveyor counts.", "research_position": "A complete field-experiment asset: 60 Shijiazhuang watermelon sellers (6 randomized branding x incentive cells) with 60,806 transaction records, biweekly quality checks, daily pricing, and household purchase diaries (675 households), released as a CC-BY-4.0 Zenodo replication package (10.5281/zenodo.13909671) that per its metadata includes the original and intermediary data inputs - the most directly reproducible asset in this batch."} {"schema_version": 3, "catalog_status": "ready", "id": "china-rural-ecommerce-couture-2021-replication", "name": "Data and Code for: Connecting the Countryside via E-Commerce (Couture, Faber, Gu & Liu 2021 AER:Insights replication package, openICPSR 117506 V2)", "aka": ["openICPSR project 117506 V2", "10.3886/E117506V2", "Connecting the Countryside via E-Commerce replication data", "Rural Taobao RCT survey and price data", "农村电商入户随机实验复现数据"], "provider": "American Economic Association [publisher] / ICPSR [distributor]; deposited by Victor Couture (UC Berkeley Haas), Benjamin Faber (UC Berkeley), Lizhi Liu (Georgetown), Yizhen Gu (Jinan University)", "china_related": true, "domains": ["rural", "e-commerce", "regional", "digital-economy", "consumption", "trade-costs"], "data_pathway": {"mode": "direct", "origin": "researcher-collected", "target_artifact": "openICPSR project 117506 version V2: README.pdf (52 KB) plus Data_and_Code/Code, Data, Output, and Survey_Material. The current public tree lists three Stata scripts and nine named deidentified survey/price data files; it does not show the internal Alibaba transaction universe.", "availability": "partially-reproducible", "ordinary_researcher_feasible": true, "summary": "The AEA/openICPSR deposit supplies deidentified household-survey and retail-price files, survey material, code, and output for the paper's 100-village Rural Taobao evaluation. The current public listing names three Stata scripts and nine deidentified survey/price files; downloading them follows ICPSR login. The paper's second layer - the firm's internal transaction database covering about 12,000 villages in 5 provinces (27.3M purchase records Nov 2015-Apr 2017) - is described in the paper but is not shown in the public package tree.", "barrier": "The public listing does not state all nested content, exact download terms, or whether any part of the internal Alibaba transaction database is released. Login is required for download, and deidentification limits external linkage even for released microdata."}, "unit_of_observation": "Household (survey waves), retail price quote (store price survey), village (RCT unit and village-level aggregates), transaction (deidentified firm-data extracts if present in the package)", "structure": "Two-wave household panel (baseline end-2015/early-2016, endline ~12 months later) plus a two-round retail price survey plus village-level files", "geo_granularity": ["village", "county"], "geography": "RCT: 8 counties in Anhui, Henan and Guizhou provinces (100 villages: 40 control + 60 treatment drawn from 432 candidates). Firm admin database: ~12,000 program villages in 5 provinces (Anhui, Guangxi, Guizhou, Henan, Yunnan)", "time_span": {"start": "2015-12", "end": "2017-04", "last_confirmed_release": "2021-02-18", "coverage_note": "openICPSR project metadata (DataCite): Time Period 12/1/2015 - 5/31/2017, Geographic Unit village, Data Type survey data, Geographic Coverage China. Paper data section (NBER WP w24384 and AER:Insights supplemental appendix, read 2026-08-14): baseline household survey end-2015/early-2016; endline one year later; purchase transaction DB Nov 2015 - Apr 2017 (~27.3M records, ~12,000 villages, 5 provinces); sales/out-shipment DB Jan 2016 - Apr 2017 (~500,000 shipments).", "last_checked": "2026-09-27"}, "frequency": ["two survey waves", "monthly transaction series (firm DB, not confirmed in package)"], "sample_size": "100 RCT villages (60 treatment / 40 control) in 8 counties of 3 provinces, sampled from 432 candidates; ~28 households per village at baseline (14 inner-zone + 14 outer-zone) plus 10 added inner-zone households at endline; ~115 price quotes per village; ~27.3M purchase records and ~500K out-shipments in the firm DB (paper-level, not confirmed as released files)", "key_variables": ["Household retail consumption expenditures split across 9 categories, production inputs", "Household incomes, hours worked, occupations and sectors of members, asset ownership, financial accounts, internet use, migration", "Retail price quotes at barcode-equivalent product level (baseline and endline)", "Village treatment assignment (randomized) and actual treatment status", "Deidentified price file (price_raw_deidentified_round2.dta per prior repository-metadata search)", "Firm purchase DB fields (paper): product category, number of units, amount paid, unique buyer identifier", "Firm sales DB fields (paper): village of origin, out-shipment weight in kg"], "research_fit": {"best_for": ["Replicating or extending the paper's RCT estimates of rural e-commerce (Rural Taobao terminal) effects on household consumption, cost of living, and local retail prices in 100 Chinese villages", "Studying the 2015-2017 window of the Alibaba Rural Taobao rollout with the authors' deidentified survey/price files", "Using the paper's survey instruments and sampling design (inner/outer zone, price-quote protocol) as a template"], "choose_over": ["Choose this package over the MOFCOM demonstration-county lists when the research needs household-level microdata from a randomized e-commerce rollout rather than a county program roster", "Choose this package over general surveys (CFPS etc.) when the exact RCT treatment contrast and price data are required; use CFPS for nationally representative household measures (the paper itself uses CFPS 2012 consumption shares for survey weights)"], "not_good_for": ["A nationally representative picture of rural China (the RCT covers 100 villages in 3 provinces)", "The full Alibaba transaction universe (27.3M records): the paper describes the firm's internal database; the public package is not confirmed to contain it", "Effects after April 2017 or outside the 5 program provinces", "The MOFCOM demonstration program counties (a different program from Alibaba's Rural Taobao)"], "needs_join_for": ["County/population context (the paper uses 2010 census township-level data for market-access measures and the 2008 economic census for village establishment counts)", "CFPS 2012 microdata for expenditure weights", "GIS/distance measures (village locations, distances to terminals)"], "variation_available": ["Village-level randomized treatment assignment (ITT/TOT) and village exposure spillovers within 3/10 km are data dimensions of the released files; causal claims and design threats belong to the Econ-Variation repository"], "topics": ["rural e-commerce", "e-commerce RCT", "consumption and cost of living", "retail prices", "trade costs", "digital economy"]}, "good_for": ["RCT-based evaluation of rural e-commerce access effects on households and local retailers", "Reuse of the authors' deidentified survey and price data for the 2015-2017 window"], "identification": ["The released files support replication of the paper's documented deidentified survey and price analyses; they do not establish availability of Alibaba's internal transaction universe or independently settle causal interpretation."], "linkable_keys": ["Village identifier (within the released deidentified files)", "Household identifier (survey rounds)", "County (8 RCT counties; exact identifiers per released files)"], "joins": [{"target": "cfps", "relation": "complement", "keys": [{"None required": "CFPS enters only as expenditure shares for survey design"}], "method": "Use CFPS only for nationally representative consumption weights or broader household context; do not merge individual records", "evidence_status": "literature-used"}, {"target": "china-census", "relation": "complement", "keys": ["Township-level units"], "method": "The paper's market-access measures use 2010 census township populations; replicate the authors' township match if extending their analysis", "evidence_status": "literature-used"}, {"target": "china-mofcom-rural-ecommerce-demo-counties", "relation": "often-confused-with", "keys": ["County name"], "method": "Keep the two programs separate: Alibaba Rural Taobao (this package) vs the MOFCOM demonstration program (that record)", "evidence_status": "verified"}], "access_routes": [{"route": "openICPSR AEA replication project 117506 V2", "access_status": "available-with-login", "direct_url": "https://doi.org/10.3886/E117506V2", "requirements": ["ICPSR login and acceptance of the current download terms; the current download action redirects to the ICPSR login flow."], "steps": ["Open the DOI https://doi.org/10.3886/E117506V2 (resolves to https://www.openicpsr.org/openicpsr/project/117506/version/V2/view).", "Read the project description and version history (V2 published 2021-02-18 by AEA/ICPSR).", "Read README.pdf before downloading; inspect Data_and_Code/Code, Data, Output, and Survey_Material.", "Download the files needed after login. The visible Code folder lists do_file_analysis.do, retail_data_processing.do, and survey_data_processing.do.", "The visible Data folder lists Merge_Village_Vars.dta, deidentified round-1/round-2 price and store files, retail_data_analysis.dta, survey_data_analysis.dta, and deidentified round-1/round-2 survey files.", "Record the version and file citation (Couture, Faber, Liu & Gu 2021, 10.3886/E117506V2)."], "deliverable": "Login-mediated public V2 materials: README.pdf (52 KB), Code, Data, Output, and Survey_Material. The current public directory enumerates three Stata scripts and nine named data files, including deidentified survey and price rounds; the full internal transaction database is not shown.", "cost": "registration", "last_checked": "2026-09-27", "caveat": "The current public tree confirms the listed folders and files, but the download flow is login-mediated and it does not establish that Alibaba's internal transaction database or every paper input is included."}, {"route": "AEA article page and materials", "access_status": "available", "direct_url": "https://www.aeaweb.org/articles?id=10.1257/aeri.20190382", "requirements": ["Public web access"], "steps": ["Confirm article identity, abstract, citation, and the Replication Package link (DOI 10.3886/E117506V2) and Supplemental Appendix (https://www.aeaweb.org/articles/materials/14082, read 2026-08-14).", "Use the published PDF (complimentary download per AEA page; automated fetch returned 403 in this environment) and the supplemental appendix for the data construction details."], "deliverable": "Article text, supplemental appendix, disclosure statement", "cost": "free", "last_checked": "2026-08-14", "caveat": "The complimentary PDF link returned 403 to an automated client (2026-08-14); the NBER working paper w24384 PDF (fetched and read 2026-08-14) contains the same data section for the pre-publication version."}, {"route": "NBER working paper w24384 (open full text)", "access_status": "available", "direct_url": "https://www.nber.org/system/files/working_papers/w24384/w24384.pdf", "requirements": ["Public web access"], "steps": ["Download the PDF and read the Experimental Design and Data section (fetched and read in full 2026-08-14)."], "deliverable": "Full working-paper text with the data section", "cost": "free", "last_checked": "2026-08-14", "caveat": "Working-paper version (2018); the published AER:Insights version may differ in detail."}], "access": {"url": "https://doi.org/10.3886/E117506V2", "cost": "registration", "license": "openICPSR standard terms for public AEA replication projects; material distributed as received from the depositor; original data sources (Alibaba transaction DB, survey fieldwork) remain private", "format": ["dta", "pdf", "do", "xlsx"], "api": false, "how_to_get": "Open project 117506 V2 through the DOI, inspect the current public folder tree, use the ICPSR login-mediated download flow, read README.pdf, and download only the needed deidentified survey, price, code, output, or survey-material files."}, "caveats": ["The paper's firm-admin transaction database (27.3M records across ~12,000 villages) is described in the paper as internal firm data; the public package is not confirmed to include it. Do not promise the full transaction universe to a researcher.", "The current public V2 tree lists the Code/Data/Output/Survey_Material structure, three Stata scripts, and nine named survey/price data files. It does not prove every nested item, exact download terms, or inclusion of any internal firm database.", "Download actions redirect to ICPSR login; use the browser flow and inspect README.pdf before reuse.", "The survey covers 100 villages in 3 provinces; representativeness for China's countryside is limited (the paper itself compares RCT villages with the 5-province program population in the appendix)."], "production": {"raw_sources": [{"name": "Alibaba Rural Taobao program village candidate lists (8 county operations teams, 432 candidate villages)", "source_type": "other", "role": "Sampling frame for the RCT (village candidates extended by 5 per county)", "access_route": "Internal program lists obtained by the authors; not publicly released", "url": "https://www.nber.org/system/files/working_papers/w24384/w24384.pdf", "coverage": "432 villages in 8 counties (Anhui, Henan, Guizhou)", "last_checked": "2026-08-14"}, {"name": "Household survey fieldwork (baseline end-2015/early-2016; endline ~12 months later; RCCC-trained teams)", "source_type": "dataset", "role": "Main survey microdata in the deposit (consumption, income, labor, assets, internet, migration)", "access_route": "Released deidentified in the openICPSR package", "url": "https://doi.org/10.3886/E117506V2", "coverage": "100 villages; ~28 households/village baseline, +10 inner-zone households at endline", "last_checked": "2026-08-14"}, {"name": "Local retail price survey (two rounds)", "source_type": "dataset", "role": "Price-quote data (target 115 quotes per village) in the deposit; deidentified price file price_raw_deidentified_round2.dta per prior metadata search", "access_route": "Released deidentified in the openICPSR package", "url": "https://doi.org/10.3886/E117506V2", "coverage": "Stores in/within 15-minute walk of sample villages; two rounds", "last_checked": "2026-08-14"}, {"name": "Anonymous Firm (Alibaba) internal transaction database", "source_type": "dataset", "role": "Paper's second analysis layer - purchase DB (Nov 2015-Apr 2017, ~27.3M records, ~12,000 villages, 5 provinces) and sales/out-shipment DB (Jan 2016-Apr 2017, ~500K shipments)", "access_route": "Internal firm data described in the paper; release status in the public package NOT confirmed", "url": "https://www.nber.org/system/files/working_papers/w24384/w24384.pdf", "coverage": "5 provinces (Anhui, Guangxi, Guizhou, Henan, Yunnan), Nov 2015-Apr 2017", "last_checked": "2026-08-14"}, {"name": "CFPS 2012 microdata and 2008 economic census (auxiliary)", "source_type": "dataset", "role": "CFPS 2012 used for consumption-category weights in Anhui/Henan; 2008 economic census establishment data used for village-level stratification variables", "access_route": "Separate provider routes (see cfps and china-economic-census records)", "url": "https://www.isss.pku.edu.cn/cfps/en/", "coverage": "As documented in the paper's appendix", "last_checked": "2026-08-14"}], "acquisition_methods": ["repository download", "field survey", "price survey"], "sample_construction": "The RCT sample is 40 control + 60 treatment villages randomly selected per county from extended candidate lists (432 candidates, average 54 per county), stratified on delivery-service status, applicant test score, village population, and economic-census establishment counts, with a 2.5 km minimum distance between list villages where possible. Households: 14 inner-zone (within 300 m of planned terminal) + 14 outer-zone per village at baseline, 10 additional inner-zone at endline, selected by random walk from mapped residences.", "pipeline_stages": [{"stage": "collect", "inputs": ["Village candidate lists", "Household and price survey fieldwork"], "method": "Field surveys conducted by trained teams (RCCC supervisors and local university students); paper-based questionnaires with transaction-level recall spreadsheets aggregated into final questionnaires; price data per IMF/ILO-style protocol with product pictures.", "tools": ["Field survey teams"], "output": "Raw household and price survey files", "evidence": "Supplemental appendix F (read 2026-08-14)"}, {"stage": "clean", "inputs": ["Raw survey files", "Firm transaction extracts"], "method": "Clean the household survey (exclude 0.25% unreliable households per surveyor ratings), winsorize tails, build expenditure/income aggregates, deidentify firm-price data.", "tools": ["Stata"], "output": "Cleaned household dataset, price dataset (e.g. price_raw_deidentified_round2.dta per prior metadata search), village files", "evidence": "Supplemental appendix F; openICPSR file names (prior metadata search)"}, {"stage": "aggregate", "inputs": ["Firm transaction database"], "method": "Aggregate the purchase/sales transaction DB to village-month outcomes (number of buyers, transactions, terminal sales, out-shipments) for the paper's appendix analyses.", "tools": ["Stata"], "output": "Village-month outcomes used in the paper's appendix tables", "evidence": "Paper and appendix D (read 2026-08-14); release status of these aggregates in the package not confirmed"}, {"stage": "validate", "inputs": ["Deposited files", "Paper tables"], "method": "Run the released code against the deposited data and compare with the published tables.", "tools": ["Stata"], "output": "Reproduced tables", "evidence": "Replication package; not executed this pass"}], "constructed_variables": [{"name": "treatment indicators (ITT/TOT)", "concept": "Randomized village assignment and actual terminal rollout status", "source_fields": ["Village assignment records"], "method": "ITT = assigned treatment (60 villages); TOT = actual rollout (38 of 60 treatment, 5 of 40 control) instrumented by assignment", "validation": "Compliance pattern documented in the paper (2026-08-14 read)", "limitations": "Compliance incomplete; one county's endline data collection was suspended (4 of 100 villages without endline)"}, {"name": "e-commerce uptake and expenditure shares", "concept": "Household use of and spending through the Rural Taobao option", "source_fields": ["Household survey consumption module"], "method": "Constructed from the survey's modality-coded purchase transactions (online vs offline, in/out of village)", "validation": "Paper tables 1-3", "limitations": "Self-reported; survey months differ across villages (seasonality addressed in appendix with the firm DB)"}], "output": {"unit_of_observation": "Household, price quote, village", "structure": "Two-wave household panel + two-round price survey + village files", "geography": "8 counties in Anhui, Henan, Guizhou (RCT); 5-province program universe in paper-level appendix analyses", "time_span": "Survey 2015-12 to 2017 (two rounds); paper-level transaction window Nov 2015-Apr 2017", "key_variables": ["Consumption by 9 categories", "Income and labor outcomes", "Retail price quotes", "Treatment/uptake variables", "Deidentified price file"], "formats": ["dta"]}, "reproducibility": {"level": "medium", "starting_point": "https://doi.org/10.3886/E117506V2", "code_available": true, "code_url": "https://www.openicpsr.org/openicpsr/project/117506/version/V2/view", "requirements": ["ICPSR login and browser access", "Stata (authors' code)", "Time to read README.pdf and confirm any nested file or licensing detail not visible in the public tree"], "blockers": ["The paper's full transaction database is internal firm data; any released extract is deidentified and partial", "Survey instruments and deidentification choices must be read from the deposit"]}, "compliance": {"terms_or_license": "openICPSR terms for AEA replication projects; material distributed as received; deidentified data only", "robots_or_rate_limits": "Public file-tree browsing is available; download uses the repository's login flow. Do not automate authenticated retrieval.", "personal_or_sensitive_data": "Household survey data are deidentified in the deposit; do not attempt re-identification", "redistribution": "Follow the openICPSR terms and the AEA data/code policy for redistribution", "review_needed": false}}, "quality": {"profile_status": "verified", "access_status": "verified", "paper_use_status": "verified", "last_audited": "2026-09-27"}, "used_by": [{"cite": "Couture, Victor, Benjamin Faber, Yizhen Gu, and Lizhi Liu (2021), Connecting the Countryside via E-Commerce: Evidence from China, AER: Insights 3(1): 35-50", "doi": "https://doi.org/10.1257/aeri.20190382", "journal": "AER: Insights", "year": 2021, "dataset_role": "Main data for the RCT evaluation - household survey, retail price survey, and firm transaction database used for consumption, income, price, and take-up outcomes", "evidence_type": "replication", "evidence_url": "https://doi.org/10.3886/E117506V2", "data_note": "AEA article page (read 2026-08-14) lists the Replication Package (DOI 10.3886/E117506V2) and Supplemental Appendix. DataCite (read 2026-08-14) confirms title \"Data and Code for: Connecting the Countryside via E-Commerce\", version 2, ICPSR publisher, collected 2015-12-01/2017-05-31, issued 2021, creators Couture/Faber/Liu/Gu. NBER WP w24384 and the supplemental appendix (read in full 2026-08-14) document the RCT design (100 villages, 8 counties, 3 provinces), the household and price surveys, and the firm's internal purchase/sales databases (5 provinces, Nov 2015-Apr 2017). The current V2 public tree confirms README.pdf plus Code/Data/Output/Survey_Material and lists three Stata scripts and nine named deidentified survey/price files. The public tree does not show the full internal Alibaba transaction database."}], "provenance": [{"source": "AEA article page https://www.aeaweb.org/articles?id=10.1257/aeri.20190382 (read 2026-08-14)", "field_scope": ["paper identity and citation", "replication package DOI", "supplemental appendix link", "abstract"], "added": "2026-08-14", "confidence": "high", "verified": true}, {"source": "DataCite API https://api.datacite.org/dois/10.3886/E117506V2 (read 2026-08-14)", "field_scope": ["package title, version, publisher, creators", "collection dates and issue year", "related identifier (isVersionOf 10.3886/e117506)"], "added": "2026-08-14", "confidence": "high", "verified": true}, {"source": "NBER working paper w24384 PDF https://www.nber.org/system/files/working_papers/w24384/w24384.pdf (read in full 2026-08-14)", "field_scope": ["RCT design and sample", "survey content", "firm transaction database scope"], "added": "2026-08-14", "confidence": "high", "verified": true}, {"source": "AER:Insights supplemental appendix https://www.aeaweb.org/articles/materials/14082 (read in full 2026-08-14)", "field_scope": ["sampling and field protocols", "deidentification and cleaning", "appendix estimation details"], "added": "2026-08-14", "confidence": "high", "verified": true}, {"source": "Wayback snapshots of openICPSR project 117506 (V2 view 2022-12-25; V1 Data_and_Code folder 2024-01-16; V2 Data_and_Code folder 2025-03-19; fetched 2026-08-14)", "field_scope": ["V2 top-level manifest (README.pdf 52 KB + Data_and_Code)", "V2 Data_and_Code folder structure (Code/Data/Output/Survey_Material)", "project description fields (village unit, survey data type, time period)"], "added": "2026-08-14", "confidence": "high", "verified": true}, {"source": "Current openICPSR project 117506 V2 public file tree https://www.openicpsr.org/openicpsr/project/117506/version/V2/view", "field_scope": ["current V2 project identity, China village survey scope, and 2015-12 to 2017-05 time period", "README.pdf and Data_and_Code/Code, Data, Output, Survey_Material structure", "three named Stata scripts and nine named deidentified survey/price data files with displayed sizes", "current ICPSR login-mediated download boundary"], "added": "2026-09-27", "confidence": "high", "verified": true}, {"source": "openICPSR direct pages (https://www.openicpsr.org/openicpsr/project/117506/version/V2/view and OAI endpoints)", "field_scope": ["live V2 file manifest", "README contents"], "added": "2026-08-14", "confidence": "low", "verified": false}], "related_datasets": [{"id": "china-mofcom-rural-ecommerce-demo-counties", "relation": "often-confused-with"}, {"id": "china-senddown-county-exposure", "relation": "benchmark"}, {"id": "cfps", "relation": "complement"}, {"id": "china-census", "relation": "complement"}], "source_path": "datasets/china-rural-ecommerce-couture-2021-replication.md", "description_markdown": "## Positioning in one sentence\n\nThis is the login-mediated AEA/ICPSR replication package behind Couture, Faber, Gu & Liu (2021 AER:Insights): it visibly contains deidentified household-survey and retail-price files, Stata code, output, and survey material from a 100-village Rural Taobao RCT (2015-2017). The paper's full 27.3M-record Alibaba transaction universe remains internal firm data and is not shown as part of the public package.\n\n## Select rules\n\n- Prioritize it when the research needs the paper's RCT microdata (household consumption/income, retail prices, take-up) for rural e-commerce access.\n- Switch to the MOFCOM demonstration-county lists when the question is the national policy program's county rollout; the two programs are different and must not be merged.\n- Use CFPS for nationally representative rural household measures; this package is a 100-village experiment, not a national sample.\n\n## Get recipe\n\n1. Open https://doi.org/10.3886/E117506V2 and inspect the public V2 tree: README.pdf plus Data_and_Code/Code, Data, Output, and Survey_Material.\n2. Use the ICPSR login-mediated download flow. The visible Data folder names deidentified round-1/round-2 survey and price files, while the Code folder names three Stata scripts; inspect README.pdf before treating any file as a paper analysis input.\n3. Download the survey/price files, code, output, or survey material needed; record version V2 and the file citation.\n4. Read the paper data section (NBER w24384 or the published PDF) and supplemental appendix for variable construction.\n\n## Connections and Limitations\n\nThe natural units are household, price quote, and village within the 8 RCT counties; village identifiers inside the deidentified files are the link keys, and any extension needs the paper's township-level census market-access construction. The public package is not confirmed to include the full Alibaba transaction database, and the sample covers 100 villages in 3 provinces, so national generalization requires the authors' own representativeness checks rather than assumption. The remaining data-side check is whether a particular nested file and its terms meet an extension's needs.", "research_position": "This is the login-mediated AEA/ICPSR replication package behind Couture, Faber, Gu & Liu (2021 AER:Insights): it visibly contains deidentified household-survey and retail-price files, Stata code, output, and survey material from a 100-village Rural Taobao RCT (2015-2017). The paper's full 27.3M-record Alibaba transaction universe remains internal firm data and is not shown as part of the public package."} -{"schema_version": 3, "catalog_status": "ready", "id": "china-rural-economic-statistics-1949-1986", "name": "Compilation of China's Rural Economic Statistics, 1949-1986 (中国农村经济统计大全:1949-1986)", "aka": ["中国农村经济统计大全:1949-1986", "中国农村经济统计大全: 1949-1986", "China Rural Economic Statistics Encyclopedia, 1949-1986", "Compilation of China's Rural Economic Statistics: 1949-86"], "provider": "Ministry of Agriculture Planning Division (中华人民共和国农业部计划司); Agricultural Press (农业出版社), Beijing, 1989", "china_related": true, "domains": ["regional", "rural", "agriculture", "development", "public"], "data_pathway": {"mode": "direct", "origin": "ready-made", "target_artifact": "A Chinese-language printed statistical compilation of historical rural and agricultural statistics for 1949-1986; the paper-used object is the relevant provincial tables, not a released copy of Li and Yang's cleaned panel", "availability": "partially-reproducible", "ordinary_researcher_feasible": true, "summary": "The title, edition, ISBN, library identifiers, and paper use are documented. The current CiNii catalogue identifies the May 1989 Agricultural Press volume, its Ministry of Agriculture Planning Division compiler, 2 preliminary pages plus 9 pages plus 707 pages, and 38 university-library holdings. A researcher can select a holding or request permitted document delivery, then manually transcribe and validate the needed tables with page provenance. No stable public machine-readable download or open reuse licence was verified.", "barrier": "The route is library-mediated and table extraction is manual. The volume's exact table inventory, scan availability, and copyright/reuse conditions must be checked with the holding institution; a citation does not imply access to the final analysis file."}, "unit_of_observation": "Provincial or other aggregate rural/agricultural statistical entry; the exact grain varies by chapter and table", "structure": "Historical aggregate tables, with annual series where a table reports them", "geo_granularity": ["province", "national or regional aggregate where supplied by a table"], "geography": "Mainland Chinese provinces and aggregate regions represented in the compilation; the presence of a province, indicator, and year must be checked table by table", "time_span": {"start": "1949", "end": "1986", "last_confirmed_release": "1989-05", "coverage_note": "The book title and bibliographic records cover 1949-1986. Li and Yang's final provincial panel runs from 1954 to 1989 because they supplement the book with the China Statistical Yearbook, provincial agricultural statistical yearbooks, and other paper-specific sources. Do not treat the book as evidence for the post-1986 years.", "last_checked": "2026-09-28"}, "frequency": ["annual historical series where reported", "table-dependent aggregate observations"], "sample_size": "Not stated as a single machine-readable count; province and indicator coverage varies by table", "key_variables": ["Provincial agricultural input and output series", "Grain and crop output; Li and Yang define grain as the sum of eight crops in their derived panel", "Sown area and related agricultural production measures", "Draft animals and farm capital inputs, including the paper's equivalent-horsepower construction", "Agricultural population/labor and related regional controls where reported", "Procurement, retention, or grain-allocation tables where reported"], "research_fit": {"best_for": ["Historical provincial agricultural production and input comparisons, especially for the pre-1986 planning-era baseline", "Recovering source tables behind Li and Yang's 1954-1986 provincial agricultural series", "Building a documented historical aggregate panel when the researcher can preserve table/page/unit provenance"], "choose_over": ["Choose it over a modern China Statistical Yearbook when the question requires the 1949-1986 historical compilation or its pre-reform agricultural tables", "Use the China Statistical Yearbook and provincial agricultural yearbooks alongside it when the target period extends beyond 1986 or when missing years/definitions need cross-checking", "Prefer a machine-readable released replication file when the goal is to reproduce Li and Yang's final regression sample rather than inspect the underlying historical source"], "not_good_for": ["Household, farm, village, firm, or individual microdata", "A guaranteed balanced province-year panel, county-level observations, or a current agricultural series", "Treating every indicator in the book as available for every province and year", "Inferring causal treatment, exogenous exposure, or the paper's retrospective survey from the book"], "needs_join_for": ["1987-1989 agricultural outcomes and missing values from the China Statistical Yearbook or provincial Agricultural Statistical Yearbooks", "Population, mortality, and other controls not present or not continuous in the selected tables", "Li and Yang's weather, collective-unit scale, and exit-rights variables, which came from a separate 1999 retrospective province survey"], "variation_available": ["Provincial cross-sectional and historical-year dimensions only; this record makes no causal or exogenous-variation claim"], "topics": ["agricultural production", "rural development", "central planning", "historical regional growth", "provincial input-output comparisons"]}, "good_for": ["Manual reconstruction or cross-checking of historical provincial agricultural aggregates", "Establishing the source boundary for the agricultural component of Li and Yang's Great Leap Forward panel"], "identification": ["This descriptive historical statistical compilation does not itself encode a causal design or a verified treatment-assignment rule.", "Its provincial and historical dimensions require separate design evidence; they are not classified here as exogenous variation."], "linkable_keys": ["Province name and historical province label", "Year", "Table title, indicator label, unit, and page number", "Researcher-built province concordance if administrative labels changed"], "joins": [{"target": "china-stat-yearbook", "relation": "complement", "keys": ["Province name/code", "Year", "Indicator and unit"], "method": "Use the later Yearbook and provincial Agricultural Statistical Yearbooks to extend 1987-1989 and cross-check missing or conflicting entries. Preserve the book edition and source page instead of silently merging revisions.", "evidence_status": "literature-used"}], "access_routes": [{"route": "Library catalogue and interlibrary loan", "access_status": "available-with-conditions", "direct_url": "https://ci.nii.ac.jp/ncid/BN10983973", "requirements": ["Library membership or interlibrary-loan/scan request", "Chinese-language table reading and manual transcription"], "steps": ["Search by the Chinese title, ISBN 7109009300, or NCID BN10983973", "Confirm the title page, edition, publisher imprint, table of contents, and relevant province tables", "Request only the needed pages under the holding library's scan and copyright rules", "Transcribe values with page, table, unit, footnote, and missing-value codes; validate against a later Yearbook where possible"], "deliverable": "Physical consultation, permitted library scan, or researcher-transcribed table; no public machine-readable file was verified", "cost": "mixed", "last_checked": "2026-09-28", "caveat": "CiNii and WorldCat establish catalogue identity and holdings, not that every holding library offers a scan or permits redistribution."}, {"route": "WorldCat holding-library search", "access_status": "available-with-conditions", "direct_url": "https://search.worldcat.org/pt/title/%3A-%281949-1986%29/oclc/22378850", "requirements": ["A participating library or document-delivery service"], "steps": ["Search OCLC 22378850 or ISBN 9787109009301", "Ask the library to verify the 1989 first-edition Chinese volume and the requested chapter/table", "Record the institution, edition, page range, and permission terms in the extraction log"], "deliverable": "Library copy or permitted document delivery, not an open data download", "cost": "mixed", "last_checked": "2026-08-11", "caveat": "WorldCat is a discovery and holding route; access, fees, and scan rights depend on the institution."}], "access": {"url": "https://ci.nii.ac.jp/ncid/BN10983973", "cost": "mixed", "license": "Printed-volume copyright and library scan/document-delivery terms; no open-data licence was identified", "format": ["printed Chinese volume", "permitted library scan", "researcher-transcribed table"], "api": false, "how_to_get": "Locate the CiNii or WorldCat record by title/ISBN/NCID/OCLC, obtain a library copy or permitted scan, verify the title page and table definitions, and transcribe only the needed tables with page-level provenance. Google Books confirms bibliographic metadata and reports no eBook availability; it is not a download route."}, "caveats": ["CiNii records 707 pages while Google Books records 718 pages; retain this discrepancy until a physical title page and pagination are checked.", "The source ends in 1986, whereas Li and Yang's final panel extends to 1989 through other yearbooks and sources.", "Li and Yang's paper-derived variables (for example, eight-crop grain output and equivalent horsepower) are transformations of source tables, not a claim that the book contains those exact final columns.", "The source does not prove that Li and Yang's final cleaned panel, retrospective province survey, or code is publicly reproducible.", "Rojas databank tables and third-party commercial PDF claims may help locate related historical series, but they are not accepted as a direct substitute for this volume or evidence of its licensing.", "Province labels, units, footnotes, and missing-value conventions must be preserved; historical aggregate definitions may change across the 1949-1986 span."], "quality": {"profile_status": "verified", "access_status": "verified", "paper_use_status": "verified", "last_audited": "2026-09-28"}, "used_by": [{"cite": "Li & Yang (2005), The Great Leap Forward: Anatomy of a Central Planning Disaster", "doi": "https://doi.org/10.1086/430804", "journal": "JPE", "year": 2005, "dataset_role": "Main historical source for provincial agricultural input and output series in the 1954-1989 panel", "evidence_type": "data_appendix", "evidence_url": "https://www3.nd.edu/~nmark/ChinaCourse/TheWeeks/Li_Yang_GLF_JPE.pdf", "data_note": "Appendix B says provincial agricultural inputs and outputs came mainly from this Ministry of Agriculture compilation. The paper cross-checked or filled missing observations with the China Statistical Yearbook and provincial Agricultural Statistical Yearbooks; its weather, collective-unit, and exit-rights measures came from a separate retrospective province survey."}], "provenance": [{"source": "https://www3.nd.edu/~nmark/ChinaCourse/TheWeeks/Li_Yang_GLF_JPE.pdf", "field_scope": ["paper actually uses the compilation", "provincial agricultural input/output role", "1954-1989 final-panel boundary and later-yearbook supplementation", "paper-derived variable definitions and separate retrospective survey boundary"], "added": "2026-08-11", "confidence": "high", "verified": true}, {"source": "https://ci.nii.ac.jp/ncid/BN10983973", "field_scope": ["exact Chinese title and editor", "1989 publication metadata", "ISBN, NCID, language, pagination, and catalogue holdings"], "added": "2026-08-11", "confidence": "high", "verified": true}, {"source": "https://ci.nii.ac.jp/ncid/BN10983973", "field_scope": ["Current May 1989 bibliographic metadata, 2 preliminary pages plus 9 pages plus 707-page extent, and 38 university-library holdings", "Current library-mediated acquisition route for the printed volume"], "added": "2026-09-28", "confidence": "high", "verified": true}, {"source": "https://search.worldcat.org/pt/title/%3A-%281949-1986%29/oclc/22378850", "field_scope": ["printed first-edition identity", "publisher, ISBN, OCLC, and holding-library discovery route"], "added": "2026-08-11", "confidence": "high", "verified": true}, {"source": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ", "field_scope": ["corroborating bibliographic metadata", "no eBook availability statement"], "added": "2026-08-11", "confidence": "med", "verified": true}], "related_datasets": [{"id": "china-stat-yearbook", "relation": "complement"}], "source_path": "datasets/china-rural-economic-statistics-1949-1986.md", "description_markdown": "## Positioning in one sentence\n\nThis is a paper-used printed source for historical provincial agricultural statistics, not Li and Yang's final regression file: its irreplaceable value is the 1949-1986 source boundary, while its main cost is finding a library copy and manually preserving table definitions, units, and pages.\n\n## Select rules\n\n- Prioritize it when a question needs the planning-era provincial agricultural tables or when the researcher must audit the source behind Li and Yang's 1954-1986 observations.\n- Pair it with the China Statistical Yearbook and provincial agricultural yearbooks for 1987-1989, missing values, and definition checks; do not silently treat the combined panel as one ready-made release.\n- Switch to a verified replication file if the goal is exact reproduction of Li and Yang's cleaned regressions rather than source-table inspection. Switch to modern yearbooks for current or annual post-1986 coverage.\n- Do not use this record for household/farm microdata or for claims about policy assignment and causal identification.\n\n## Get recipe\n\n1. Search CiNii using `BN10983973`, ISBN `7109009300`, or the Chinese title; use WorldCat OCLC `22378850` if the local catalogue has no result.\n2. Obtain a physical copy, interlibrary-loan scan, or other permitted document delivery. Confirm the 1989 title page and the relevant table's edition.\n3. Define the province-year-indicator cells needed for the research question before transcribing. Record table/page, unit, footnotes, and missing-value codes for every cell.\n4. Cross-check overlaps against the China Statistical Yearbook or provincial Agricultural Statistical Yearbooks, especially when extending beyond 1986 or filling gaps.\n5. Keep any derived panel and transformations (such as eight-crop grain totals or equivalent horsepower) in a separate reproducible project; this record identifies the source asset only.\n\n## Connections and Limitations\n\nThe safest join is province-by-year with an explicit historical label/code concordance and the source table's indicator/unit. A later Yearbook can extend the period and cross-check values, but revisions and changing definitions must remain visible. The book itself does not establish county, household, or farm microdata, and it does not contain the retrospective survey variables that Li and Yang collected separately. Because no open digital release was verified, a future agent should not promise a bulk download or redistribution permission until a specific holding institution confirms it.\n\n## Decision sufficiency check\n\nFor a researcher studying historical provincial agricultural production, this record selects the volume because it is the source actually used for the relevant pre-1986 tables, and its current CiNii route lets the researcher choose among identified holdings for page-provenanced consultation or extraction. It is ready for that bounded source-table use, not for a downloadable panel or exact paper replication: the book ends in 1986, table-specific province/unit coverage and reuse terms still need inspection, and Li and Yang's later years and derived variables remain separate work. If a physical copy is unavailable, if the needed indicator is absent, or if the unit definition cannot be reconciled, the recommendation fails and the researcher should switch to a verified yearbook or replication asset rather than infer the missing values.", "research_position": "This is a paper-used printed source for historical provincial agricultural statistics, not Li and Yang's final regression file: its irreplaceable value is the 1949-1986 source boundary, while its main cost is finding a library copy and manually preserving table definitions, units, and pages."} +{"schema_version": 3, "catalog_status": "ready", "id": "china-rural-economic-statistics-1949-1986", "name": "Compilation of China's Rural Economic Statistics, 1949-1986 (中国农村经济统计大全:1949-1986)", "aka": ["中国农村经济统计大全:1949-1986", "中国农村经济统计大全: 1949-1986", "China Rural Economic Statistics Encyclopedia, 1949-1986", "Compilation of China's Rural Economic Statistics: 1949-86"], "provider": "Ministry of Agriculture Planning Division (中华人民共和国农业部计划司); Agricultural Press (农业出版社), Beijing, 1989", "china_related": true, "domains": ["regional", "rural", "agriculture", "development", "public"], "data_pathway": {"mode": "direct", "origin": "ready-made", "target_artifact": "A Chinese-language printed statistical compilation of historical rural and agricultural statistics for 1949-1986; the paper-used object is the relevant provincial tables, not a released copy of Li and Yang's cleaned panel", "availability": "partially-reproducible", "ordinary_researcher_feasible": true, "summary": "The title, edition, ISBN, library identifiers, and paper use are documented. The current CiNii catalogue identifies the May 1989 Agricultural Press volume, its Ministry of Agriculture Planning Division compiler, 2 preliminary pages plus 9 pages plus 707 pages, and 38 university-library holdings. A researcher can select a holding or request permitted document delivery, then manually transcribe and validate the needed tables with page provenance. No stable public machine-readable download or open reuse licence was verified.", "barrier": "The route is library-mediated and table extraction is manual. The volume's exact table inventory, scan availability, and copyright/reuse conditions must be checked with the holding institution; a citation does not imply access to the final analysis file."}, "unit_of_observation": "Provincial or other aggregate rural/agricultural statistical entry; the exact grain varies by chapter and table", "structure": "Historical aggregate tables, with annual series where a table reports them", "geo_granularity": ["province", "national or regional aggregate where supplied by a table"], "geography": "Mainland Chinese provinces and aggregate regions represented in the compilation; the presence of a province, indicator, and year must be checked table by table", "time_span": {"start": "1949", "end": "1986", "last_confirmed_release": "1989-05", "coverage_note": "The book title and bibliographic records cover 1949-1986. Li and Yang's final provincial panel runs from 1954 to 1989 because they supplement the book with the China Statistical Yearbook, provincial agricultural statistical yearbooks, and other paper-specific sources. Do not treat the book as evidence for the post-1986 years.", "last_checked": "2026-09-28"}, "frequency": ["annual historical series where reported", "table-dependent aggregate observations"], "sample_size": "Not stated as a single machine-readable count; province and indicator coverage varies by table", "key_variables": ["Provincial agricultural input and output series", "Grain and crop output; Li and Yang define grain as the sum of eight crops in their derived panel", "Sown area and related agricultural production measures", "Draft animals and farm capital inputs, including the paper's equivalent-horsepower construction", "Agricultural population/labor and related regional controls where reported", "Procurement, retention, or grain-allocation tables where reported"], "research_fit": {"best_for": ["Historical provincial agricultural production and input comparisons, especially for the pre-1986 planning-era baseline", "Recovering source tables behind Li and Yang's 1954-1986 provincial agricultural series", "Building a documented historical aggregate panel when the researcher can preserve table/page/unit provenance"], "choose_over": ["Choose it over a modern China Statistical Yearbook when the question requires the 1949-1986 historical compilation or its pre-reform agricultural tables", "Use the China Statistical Yearbook and provincial agricultural yearbooks alongside it when the target period extends beyond 1986 or when missing years/definitions need cross-checking", "Prefer a machine-readable released replication file when the goal is to reproduce Li and Yang's final regression sample rather than inspect the underlying historical source"], "not_good_for": ["Household, farm, village, firm, or individual microdata", "A guaranteed balanced province-year panel, county-level observations, or a current agricultural series", "Treating every indicator in the book as available for every province and year", "Inferring causal treatment, exogenous exposure, or the paper's retrospective survey from the book"], "needs_join_for": ["1987-1989 agricultural outcomes and missing values from the China Statistical Yearbook or provincial Agricultural Statistical Yearbooks", "Population, mortality, and other controls not present or not continuous in the selected tables", "Li and Yang's weather, collective-unit scale, and exit-rights variables, which came from a separate 1999 retrospective province survey"], "variation_available": ["Provincial cross-sectional and historical-year dimensions only; this record makes no causal or exogenous-variation claim"], "topics": ["agricultural production", "rural development", "central planning", "historical regional growth", "provincial input-output comparisons"]}, "good_for": ["Manual reconstruction or cross-checking of historical provincial agricultural aggregates", "Establishing the source boundary for the agricultural component of Li and Yang's Great Leap Forward panel"], "identification": ["This descriptive historical statistical compilation does not itself encode a causal design or a verified treatment-assignment rule.", "Its provincial and historical dimensions require separate design evidence; they are not classified here as exogenous variation."], "linkable_keys": ["Province name and historical province label", "Year", "Table title, indicator label, unit, and page number", "Researcher-built province concordance if administrative labels changed"], "joins": [{"target": "china-stat-yearbook", "relation": "complement", "keys": ["Province name/code", "Year", "Indicator and unit"], "method": "Use the later Yearbook and provincial Agricultural Statistical Yearbooks to extend 1987-1989 and cross-check missing or conflicting entries. Preserve the book edition and source page instead of silently merging revisions.", "evidence_status": "literature-used"}], "access_routes": [{"route": "Library catalogue and interlibrary loan", "access_status": "available-with-conditions", "direct_url": "https://ci.nii.ac.jp/ncid/BN10983973", "requirements": ["Library membership or interlibrary-loan/scan request", "Chinese-language table reading and manual transcription"], "steps": ["Search by the Chinese title, ISBN 7109009300, or NCID BN10983973", "Confirm the title page, edition, publisher imprint, table of contents, and relevant province tables", "Request only the needed pages under the holding library's scan and copyright rules", "Transcribe values with page, table, unit, footnote, and missing-value codes; validate against a later Yearbook where possible"], "deliverable": "Physical consultation, permitted library scan, or researcher-transcribed table; no public machine-readable file was verified", "cost": "mixed", "last_checked": "2026-09-28", "caveat": "CiNii and WorldCat establish catalogue identity and holdings, not that every holding library offers a scan or permits redistribution."}, {"route": "WorldCat holding-library search", "access_status": "available-with-conditions", "direct_url": "https://search.worldcat.org/pt/title/%3A-%281949-1986%29/oclc/22378850", "requirements": ["A participating library or document-delivery service"], "steps": ["Search OCLC 22378850 or ISBN 9787109009301", "Ask the library to verify the 1989 first-edition Chinese volume and the requested chapter/table", "Record the institution, edition, page range, and permission terms in the extraction log"], "deliverable": "Library copy or permitted document delivery, not an open data download", "cost": "mixed", "last_checked": "2026-08-11", "caveat": "WorldCat is a discovery and holding route; access, fees, and scan rights depend on the institution."}], "access": {"url": "https://ci.nii.ac.jp/ncid/BN10983973", "cost": "mixed", "license": "Printed-volume copyright and library scan/document-delivery terms; no open-data licence was identified", "format": ["printed Chinese volume", "permitted library scan", "researcher-transcribed table"], "api": false, "how_to_get": "Locate the CiNii or WorldCat record by title/ISBN/NCID/OCLC, obtain a library copy or permitted scan, verify the title page and table definitions, and transcribe only the needed tables with page-level provenance. Google Books confirms bibliographic metadata and reports no eBook availability; it is not a download route."}, "caveats": ["CiNii records 707 pages while Google Books records 718 pages; retain this discrepancy until a physical title page and pagination are checked.", "The source ends in 1986, whereas Li and Yang's final panel extends to 1989 through other yearbooks and sources.", "Li and Yang's paper-derived variables (for example, eight-crop grain output and equivalent horsepower) are transformations of source tables, not a claim that the book contains those exact final columns.", "The source does not prove that Li and Yang's final cleaned panel, retrospective province survey, or code is publicly reproducible.", "Rojas databank tables and third-party commercial PDF claims may help locate related historical series, but they are not accepted as a direct substitute for this volume or evidence of its licensing.", "Province labels, units, footnotes, and missing-value conventions must be preserved; historical aggregate definitions may change across the 1949-1986 span."], "quality": {"profile_status": "verified", "access_status": "verified", "paper_use_status": "verified", "last_audited": "2026-09-28"}, "used_by": [{"cite": "Li & Yang (2005), The Great Leap Forward: Anatomy of a Central Planning Disaster", "doi": "https://doi.org/10.1086/430804", "journal": "JPE", "year": 2005, "dataset_role": "Main historical source for provincial agricultural input and output series in the 1954-1989 panel", "evidence_type": "data_appendix", "evidence_url": "https://www3.nd.edu/~nmark/ChinaCourse/TheWeeks/Li_Yang_GLF_JPE.pdf", "data_note": "Appendix B says provincial agricultural inputs and outputs came mainly from this Ministry of Agriculture compilation. The paper cross-checked or filled missing observations with the China Statistical Yearbook and provincial Agricultural Statistical Yearbooks; its weather, collective-unit, and exit-rights measures came from a separate retrospective province survey."}], "provenance": [{"source": "https://www3.nd.edu/~nmark/ChinaCourse/TheWeeks/Li_Yang_GLF_JPE.pdf", "field_scope": ["paper actually uses the compilation", "provincial agricultural input/output role", "1954-1989 final-panel boundary and later-yearbook supplementation", "paper-derived variable definitions and separate retrospective survey boundary"], "added": "2026-08-11", "confidence": "high", "verified": true}, {"source": "https://ci.nii.ac.jp/ncid/BN10983973", "field_scope": ["exact Chinese title and editor", "1989 publication metadata", "ISBN, NCID, language, pagination, and catalogue holdings"], "added": "2026-08-11", "confidence": "high", "verified": true}, {"source": "https://ci.nii.ac.jp/ncid/BN10983973", "field_scope": ["Current May 1989 bibliographic metadata, 2 preliminary pages plus 9 pages plus 707-page extent, and 38 university-library holdings", "Current library-mediated acquisition route for the printed volume"], "added": "2026-09-28", "confidence": "high", "verified": true}, {"source": "https://search.worldcat.org/pt/title/%3A-%281949-1986%29/oclc/22378850", "field_scope": ["printed first-edition identity", "publisher, ISBN, OCLC, and holding-library discovery route"], "added": "2026-08-11", "confidence": "high", "verified": true}, {"source": "https://books.google.com/books?id=gS0yAAAAMAAJ", "field_scope": ["corroborating bibliographic metadata", "no eBook availability statement"], "added": "2026-08-11", "confidence": "med", "verified": true}], "related_datasets": [{"id": "china-stat-yearbook", "relation": "complement"}], "source_path": "datasets/china-rural-economic-statistics-1949-1986.md", "description_markdown": "## Positioning in one sentence\n\nThis is a paper-used printed source for historical provincial agricultural statistics, not Li and Yang's final regression file: its irreplaceable value is the 1949-1986 source boundary, while its main cost is finding a library copy and manually preserving table definitions, units, and pages.\n\n## Select rules\n\n- Prioritize it when a question needs the planning-era provincial agricultural tables or when the researcher must audit the source behind Li and Yang's 1954-1986 observations.\n- Pair it with the China Statistical Yearbook and provincial agricultural yearbooks for 1987-1989, missing values, and definition checks; do not silently treat the combined panel as one ready-made release.\n- Switch to a verified replication file if the goal is exact reproduction of Li and Yang's cleaned regressions rather than source-table inspection. Switch to modern yearbooks for current or annual post-1986 coverage.\n- Do not use this record for household/farm microdata or for claims about policy assignment and causal identification.\n\n## Get recipe\n\n1. Search CiNii using `BN10983973`, ISBN `7109009300`, or the Chinese title; use WorldCat OCLC `22378850` if the local catalogue has no result.\n2. Obtain a physical copy, interlibrary-loan scan, or other permitted document delivery. Confirm the 1989 title page and the relevant table's edition.\n3. Define the province-year-indicator cells needed for the research question before transcribing. Record table/page, unit, footnotes, and missing-value codes for every cell.\n4. Cross-check overlaps against the China Statistical Yearbook or provincial Agricultural Statistical Yearbooks, especially when extending beyond 1986 or filling gaps.\n5. Keep any derived panel and transformations (such as eight-crop grain totals or equivalent horsepower) in a separate reproducible project; this record identifies the source asset only.\n\n## Connections and Limitations\n\nThe safest join is province-by-year with an explicit historical label/code concordance and the source table's indicator/unit. A later Yearbook can extend the period and cross-check values, but revisions and changing definitions must remain visible. The book itself does not establish county, household, or farm microdata, and it does not contain the retrospective survey variables that Li and Yang collected separately. Because no open digital release was verified, a future agent should not promise a bulk download or redistribution permission until a specific holding institution confirms it.\n\n## Decision sufficiency check\n\nFor a researcher studying historical provincial agricultural production, this record selects the volume because it is the source actually used for the relevant pre-1986 tables, and its current CiNii route lets the researcher choose among identified holdings for page-provenanced consultation or extraction. It is ready for that bounded source-table use, not for a downloadable panel or exact paper replication: the book ends in 1986, table-specific province/unit coverage and reuse terms still need inspection, and Li and Yang's later years and derived variables remain separate work. If a physical copy is unavailable, if the needed indicator is absent, or if the unit definition cannot be reconciled, the recommendation fails and the researcher should switch to a verified yearbook or replication asset rather than infer the missing values.", "research_position": "This is a paper-used printed source for historical provincial agricultural statistics, not Li and Yang's final regression file: its irreplaceable value is the 1949-1986 source boundary, while its main cost is finding a library copy and manually preserving table definitions, units, and pages."} {"schema_version": 3, "catalog_status": "ready", "id": "china-rural-finance-statistical-yearbook", "name": "China Rural Finance Statistical Yearbook, 1989-1994 (中国农村金融统计年鉴)", "aka": ["中国农村金融统计年鉴", "China Rural Finance Statistical Yearbook", "Zhongguo nongcun jinrong tongji nianjian", "CRFSY"], "provider": "Agricultural Bank of China (中国农业银行) as the listed compiler; China Statistical Press (中国统计出版社) as publisher", "china_related": true, "domains": ["rural", "regional", "finance", "development", "firm"], "data_pathway": {"mode": "direct", "origin": "ready-made", "target_artifact": "Printed or licensed annual statistical yearbook volumes covering Agricultural Bank of China and Rural Credit Cooperative rural-finance aggregates; Jin and Qian use the source tables rather than a released copy of their final provincial panel", "availability": "partially-reproducible", "ordinary_researcher_feasible": true, "summary": "The title, compiler, publisher and historical volumes are grounded by current library records, and an institutional Fangzheng yearbook page lists this title for 1979-1996 under authenticated access. A researcher can select the Agricultural Bank/Rural Credit Cooperative source family, request or open a permitted volume, locate the province/reference-year table, and transcribe the needed rural-enterprise-loan or household-savings cells with page provenance. This is a ready route to the source tables, not a release of Jin and Qian's cleaned provincial panel.", "barrier": "Library membership, interlibrary loan, document delivery, or an institutional database entitlement is required. The exact table/page and issue-year versus reference-year mapping must be checked in the authorized copy; the unresolved 1994 extension and the paper's final cleaned panel are not represented as delivered data."}, "unit_of_observation": "National, provincial, autonomous-region, planned-city or institution-level rural-finance statistical entry; exact grain varies by table and volume", "structure": "Annual historical aggregate tables for agricultural-bank and rural-credit-cooperative finance", "geo_granularity": ["national aggregate", "province/autonomous region/municipality where reported", "planned city or branch-level aggregate where reported"], "geography": "Mainland Chinese Agricultural Bank of China and Rural Credit Cooperative system represented in the annual volumes; provincial coverage and table definitions must be checked volume by volume", "time_span": {"start": "1989", "end": "1994", "last_confirmed_release": "1996", "coverage_note": "The QJE bibliography identifies CRFSY annual issues for 1989-1994, while the paper's Appendix Table I uses 1989-1993 for rural-enterprise loans and rural household savings. A catalogued 1995 issue is described as containing 1994 data, so the apparent 1994 gap is a publication-year/reference-year mismatch, not evidence that a 1994-reference-year volume was absent. Table/page contents still require an authorized volume check.", "last_checked": "2026-09-28"}, "frequency": ["annual issue", "table-specific annual series"], "sample_size": "Not a microdata sample; coverage is the set of reporting institutions and regions in each volume/table", "key_variables": ["Rural-enterprise loan balances or loan totals from the Agricultural Bank of China and Rural Credit Cooperatives", "Rural household savings/deposit measures", "Rural-finance branch, institution, personnel, cash and interest tables where present", "National and regional rural-economic benchmark indicators included in the yearbook, which should remain source-labeled"], "research_fit": {"best_for": ["Historical provincial or regional rural-credit supply and rural-enterprise finance measures", "Reconstructing the CRFSY component behind Jin and Qian's 1998 QJE rural-enterprise panel", "Checking whether a rural-finance series is bank/credit-cooperative specific rather than a general macro aggregate"], "choose_over": ["Choose CRFSY over the general China Statistical Yearbook when the research needs Agricultural Bank/Rural Credit Cooperative loans, deposits or branch statistics", "Pair it with the Township Enterprises Statistical Yearbook when the question links rural-enterprise finance to employment/output, keeping both source identities visible", "Use a current banking database or micro loan records only when current identifiers, account-level outcomes or institution-level panels are required"], "not_good_for": ["Household, farm, firm or loan-account microdata and borrower identifiers", "A guaranteed balanced province-year panel or a current rural-finance series", "Treating the yearbook's benchmark macro tables as a substitute for the China Statistical Yearbook or rural survey source", "Assuming a later China Rural Finance Almanac/Yearbook has identical historical definitions without checking the volume"], "needs_join_for": ["Township-enterprise employment/output categories from CTESY", "Population, rural labor and land denominators from CRSY or census/yearbook sources", "General state-enterprise output, CPI and macro controls from the China Statistical Yearbook", "Household or firm outcomes from a separate microdata asset"], "variation_available": ["Annual and regional dimensions only; this record makes no policy-treatment or causal-variation claim"], "topics": ["rural finance", "agricultural banking", "rural credit cooperatives", "township-enterprise finance", "regional development"]}, "good_for": ["Manual reconstruction or validation of historical rural-enterprise loan and household-savings aggregates", "Documenting the finance-source boundary in the Jin and Qian QJE panel", "Comparing rural-credit indicators across provinces while preserving bank/credit-cooperative and yearbook definitions"], "identification": ["Identify the source volume by its Chinese title 中国农村金融统计年鉴, China Agricultural Bank compiler, China Statistical Press publisher, and the intended issue/reference year. Within a permitted volume, identify the needed table by its Agricultural Bank or Rural Credit Cooperative institution label, indicator, unit, province/region headings, table title and page. This source-table route is not the same artifact as Jin and Qian's cleaned provincial analysis panel, a later rural-finance yearbook, or a general macro-finance table with a similar label."], "linkable_keys": ["Province/autonomous-region/municipality or planned-city label", "Year, issue year and reference year", "Institution type (Agricultural Bank or Rural Credit Cooperative)", "Indicator, unit and table/page number", "Historical regional-label concordance where necessary"], "joins": [{"target": "china-township-enterprises-statistical-yearbook", "relation": "complement", "keys": ["Province or region", "Year/reference year", "Rural-enterprise category"], "method": "Join CRFSY loan/savings measures to CTESY employment/output categories only after aligning issue/reference years and preserving each source's enterprise and institution definitions.", "evidence_status": "literature-used"}, {"target": "china-stat-yearbook", "relation": "complement", "keys": ["Province or region", "Year", "Indicator and unit"], "method": "Use the general Statistical Yearbook for population, prices and broad macro controls; do not replace CRFSY bank-specific measures with a similar-sounding aggregate finance table.", "evidence_status": "literature-used"}], "access_routes": [{"route": "CiNii Books library and interlibrary-loan route", "access_status": "available-with-conditions", "direct_url": "https://ci.nii.ac.jp/ncid/BA3743840X", "requirements": ["Library membership or interlibrary-loan/document-delivery access", "Verification of the requested volume and reference year"], "steps": ["Search NCID BA3743840X and ISBNs listed for the 1991-1996 volumes", "Ask a holding library for the 1989-1994 reference-year source family, including the 1995 issue that is described as containing 1994 data", "Confirm title page, compiler, publisher, issue year and tables before requesting pages", "Transcribe only the needed provincial/region cells with table, unit and footnote provenance"], "deliverable": "Physical volume, permitted scan or researcher-transcribed tables; no open machine-readable panel was verified", "cost": "mixed", "last_checked": "2026-08-12", "caveat": "CiNii currently identifies the title/compiler/publisher, lists 1991, 1992, 1993, 1995 and 1996 issues, and shows multiple university holdings. It does not guarantee a scan, interlibrary loan or redistribution permission."}, {"route": "Institutional Fangzheng Yearbook and Reference Library", "access_status": "available-with-conditions", "direct_url": "https://lib.uibe.edu.cn/zy/sjk/dzs/33820.htm", "requirements": ["University or library subscription and permitted on-campus/VPN access", "Apabi/Fangzheng reader or the provider's current access method"], "steps": ["Confirm that the subscribed China Yearbook Full-text Database exposes 中国农村金融统计年鉴 and the 1979-1996 range advertised by the library", "Search the relevant volume and export or transcribe only within the subscription terms", "Record volume, table, page/image and any export restrictions; do not treat the database listing as an open licence"], "deliverable": "On-platform book images or permitted table extraction, not a public bulk download", "cost": "paid", "last_checked": "2026-09-28", "caveat": "The current library page lists 中国农村金融统计年鉴 for 1979-1996 and states that the database is authenticated by campus IP or remote access. It documents a licensed reading route, not unrestricted export or a right to redistribute extracted pages."}, {"route": "Google Books digitized bibliographic/snippet record", "access_status": "needs-verification", "direct_url": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%86%9C%E6%9D%91%E9%87%91%E8%9E%8D%E7%BB%9F%E8%AE%A1%E5%B9%B4%E9%89%B4.html?id=X4gwAAAAMAAJ", "requirements": ["Check whether the needed pages are available in the current snippet view or through a holding library"], "steps": ["Use the 1991 record to corroborate title, Agricultural Bank contributor, publisher and table vocabulary", "Follow the listed 1993/1995 editions or library links where available", "Do not infer full-text access or reuse rights from the digitization metadata"], "deliverable": "Bibliographic metadata and limited snippets; not accepted as the paper's downloadable panel", "cost": "registration", "last_checked": "2026-08-12", "caveat": "Google Books states the item was digitized from the University of Michigan but provides no open-data licence and may show only snippets."}], "access": {"url": "https://ci.nii.ac.jp/ncid/BA3743840X", "cost": "mixed", "license": "Printed-volume copyright, library/document-delivery terms or institutional database terms; no open-data licence for the exact paper-era tables was identified", "format": ["printed Chinese volume", "licensed page images", "researcher-transcribed aggregate table"], "api": false, "how_to_get": "Start with the CiNii record or an institutional yearbook database, verify the exact issue and reference year, and transcribe only the bank/credit-cooperative tables needed with page-level provenance. Keep any reconstructed panel separate from the source volumes."}, "caveats": ["The paper names CRFSY annual issues 1989-1994, but the Appendix Table I reports 1989-1993 for the loan and savings inputs; do not fill the final year from a later publication without recording the source.", "The 1995 issue is described as covering 1994 data; still verify the exact table and whether the QJE bibliography's year denotes reference year or volume label before extraction.", "The yearbook is an aggregate financial source, not borrower-level microdata, a firm registry or a causal treatment file.", "Rural-enterprise loans, rural household savings, Agricultural Bank figures and Rural Credit Cooperative figures must remain distinguishable; similar labels in general financial yearbooks are not automatic substitutes.", "Publication year, reference year, accounting definitions and reporting coverage may change across volumes and require table-note checks."], "quality": {"profile_status": "verified", "access_status": "verified", "paper_use_status": "verified", "last_audited": "2026-09-28"}, "used_by": [{"cite": "Jin & Qian (1998), Public Versus Private Ownership of Firms: Evidence from Rural China", "doi": "https://doi.org/10.1162/003355398555748", "journal": "QJE", "year": 1998, "dataset_role": "Rural-enterprise loan and rural-household-savings component of a provincial rural-enterprise panel", "evidence_type": "data_appendix", "evidence_url": "https://citeseerx.ist.psu.edu/document?doi=39ff6e496517307ca84bf5bb5343192b9278a4ba&repid=rep1&type=pdf", "data_note": "The paper's Appendix Table I and bibliography identify CRFSY 1989-1994 annual issues; the reported loan and savings inputs cover 1989-1993 and distinguish Agricultural Bank/Rural Credit Cooperative rural-enterprise loans from rural household savings. The source volumes are not the paper's final cleaned panel."}], "provenance": [{"source": "https://academic.oup.com/qje/article-abstract/113/3/773/1851137", "field_scope": ["QJE article identity and provincial rural-enterprise study"], "added": "2026-08-12", "confidence": "high", "verified": true}, {"source": "https://citeseerx.ist.psu.edu/document?doi=39ff6e496517307ca84bf5bb5343192b9278a4ba&repid=rep1&type=pdf", "field_scope": ["CRFSY exact source name and 1989-1994 annual issue range", "bibliography publisher and source-family identity"], "added": "2026-08-12", "confidence": "med", "verified": true}, {"source": "https://paperzz.com/doc/7852221/forthcoming--quarterly-journal-of-economics-public-vs.-pr...", "field_scope": ["Appendix Table I rural-enterprise loan and household-savings roles"], "added": "2026-08-12", "confidence": "med", "verified": true}, {"source": "https://ci.nii.ac.jp/ncid/BA3743840X", "field_scope": ["Chinese title, Agricultural Bank compiler, China Statistical Press, historical volumes, ISBNs and library holdings"], "added": "2026-08-12", "confidence": "high", "verified": true}, {"source": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%86%9C%E6%9D%91%E9%87%91%E8%9E%8D%E7%BB%9F%E8%AE%A1%E5%B9%B4%E9%89%B4.html?id=X4gwAAAAMAAJ", "field_scope": ["1991 bibliographic metadata, contributor/publisher and snippet vocabulary for loans, deposits and rural-enterprise finance"], "added": "2026-08-12", "confidence": "high", "verified": true}, {"source": "https://lib.uibe.edu.cn/zy/sjk/dzs/33820.htm", "field_scope": ["Institutional Fangzheng yearbook database route and advertised 1979-1996 collection range"], "added": "2026-08-12", "confidence": "high", "verified": true}, {"source": "https://www.nianjiandata.com/books-zhongguo-nongcunjinrong-tongjinianjian.html", "field_scope": [{"bibliographic/content-year mapping": "1995 issue described as containing 1994 data"}, "1991, 1992, 1993 and 1995 issue labels shown with their corresponding content years", {"boundary": "third-party page is not used as an authorized download, licensing, or table-content source"}], "added": "2026-09-28", "confidence": "medium", "verified": true}, {"source": "CiNii Books NCID BA3743840X and UIBE Fangzheng Yearbook Library page (read 2026-09-28)", "field_scope": ["CiNii title, China Agricultural Bank compiler, China Statistical Press publisher, issue labels and multiple holdings", "authenticated library route and advertised 1979-1996 coverage for this exact title", {"boundary": "this supports source-volume acquisition, not a public machine-readable panel or the paper's final file"}], "added": "2026-09-28", "confidence": "high", "verified": true}], "related_datasets": [{"id": "china-township-enterprises-statistical-yearbook", "relation": "complement"}, {"id": "china-stat-yearbook", "relation": "complement"}], "source_path": "datasets/china-rural-finance-statistical-yearbook.md", "description_markdown": "## Positioning in one sentence\n\nThis is the historical aggregate rural-finance source behind Jin and Qian's loan and savings inputs: its distinctive value is the separation of Agricultural Bank/Rural Credit Cooperative and rural-enterprise finance measures, while the practical barrier is locating the exact annual volumes and preserving their reference-year and reporting definitions.\n\n## Select rules\n\n- Prioritize it when a regional rural-development question needs historical rural-enterprise loans, deposits or credit-cooperative coverage rather than a generic fiscal or macro-finance total.\n- Pair it with CTESY for rural-enterprise employment/output and with the China Statistical Yearbook for population, price and broad macro denominators; do not collapse these products.\n- Use a current banking database or micro survey when the research needs account-level, borrower-level or institution-identifier data.\n- Do not claim exact reproduction until the 1989-1994 issue/table mapping and any 1994 gap are resolved.\n\n## Get recipe\n\n1. Search CiNii by NCID `BA3743840X` and the listed ISBNs, or use an institutional Fangzheng yearbook subscription.\n2. Request the relevant 1989-1994 reference-year volumes and verify title page, compiler, publisher, issue year and reference year; specifically inspect the 1995 issue for the 1994 tables.\n3. Identify the rural-enterprise loan and household-savings tables before extraction, recording bank/cooperative category, province, unit, table and page.\n4. Cross-check overlapping totals with the paper appendix and general yearbooks without silently replacing the source.\n5. Keep the transcribed or reconstructed panel separate from the printed/licensed volumes and document any missing years or definition changes.\n\n## Connections and Limitations\n\nThe safest join is province or region by reference year, with institution type and indicator retained. CTESY supplies the rural-enterprise activity side; the general Statistical Yearbook supplies denominators and macro controls. The record does not provide borrowers, firms, loan accounts or causal treatment dates. If the exact volume/table cannot be obtained or the reference-year definitions cannot be reconciled, the researcher should preserve the gap or switch to a clearly labeled alternative rather than infer the missing series.\n\n## Decision sufficiency check\n\nFor a researcher studying whether historical rural credit tracked township-enterprise expansion, this record selects CRFSY for the bank/cooperative loan and savings measures, CTESY for enterprise activity and the Statistical Yearbook for denominators. It gives a concrete library or authenticated-database start, requires table/page/reference-year provenance, and keeps the 1994 extension and paper-cleaned panel outside the delivered route; this bounded source-table route is ready.", "research_position": "This is the historical aggregate rural-finance source behind Jin and Qian's loan and savings inputs: its distinctive value is the separation of Agricultural Bank/Rural Credit Cooperative and rural-enterprise finance measures, while the practical barrier is locating the exact annual volumes and preserving their reference-year and reporting definitions."} {"schema_version": 3, "catalog_status": "ready", "id": "china-rural-statistical-yearbook-1984-1994", "name": "China Rural Statistical Yearbook, 1984-1994 (中国农村统计年鉴)", "aka": ["中国农村统计年鉴", "China Rural Statistical Yearbook", "Rural Statistical Yearbook of China", "CRSY"], "provider": "National Bureau of Statistics rural-statistics offices (issue-level compiler changed over time); China Statistical Press (中国统计出版社)", "china_related": true, "domains": ["rural", "regional", "development", "labor", "agriculture", "urban"], "data_pathway": {"mode": "direct", "origin": "ready-made", "target_artifact": "Printed annual China Rural Statistical Yearbook volumes and their issue-specific tables for the 1984-1994 source family cited by Jin and Qian; the paper-built provincial panel is not a released copy of the yearbook", "availability": "partially-reproducible", "ordinary_researcher_feasible": true, "summary": "The QJE appendix identifies CRSY as the source for rural gross social product, rural labor, cultivated land, non-farm employment, a 1983 Household Responsibility System measure, and related rural-population denominators. The current National Diet Library catalogue confirms the annual printed series, China Statistical Press publication from 1985 onward, and issue-level compiler changes covering the paper period. A library copy or permitted scan is a concrete starting route for page-provenanced extraction.", "barrier": "The paper's 1984-1994 labels may refer to reference years rather than publication years. Exact issue-to-table mapping, provincial table layout, machine-readable delivery, and reuse rights still require inspection of a physical or licensed copy. A third-party PDF/Excel listing is only an unverified lead and is not evidence of an open or lawful download."}, "unit_of_observation": "National, provincial or other regional rural-statistical entry; the exact grain depends on the issue and table", "structure": "Annual historical aggregate tables with table-dependent rural economic, labor, land and social indicators", "geo_granularity": ["national aggregate where reported", "province or region where reported", "table-specific local aggregates where present"], "geography": "Mainland Chinese rural economy represented in the historical volumes; province coverage and boundary labels must be checked table by table", "time_span": {"start": "1984", "end": "1994", "last_confirmed_release": "1994 volume in the series (issue/reference-year mapping unresolved)", "coverage_note": "Jin and Qian cite annual CRSY issues for 1984-1994. The National Diet Library catalogue records the series from the 1985 volume (published 1986.2) onward and records compiler changes through 1994; Google Books identifies a 1992 volume that reports 1991 rural conditions. Keep publication year, issue label and reference year separate until the title page and table notes are checked.", "last_checked": "2026-09-28"}, "frequency": ["annual issue", "table-specific annual series"], "sample_size": "Not a microdata sample; province, indicator and year coverage vary by table", "key_variables": ["Rural gross social product used as a denominator for private financial assets and product-market development", "Total rural labor force and agricultural labor force for the non-farm-employment share", "Total cultivated land and rural population for per-capita cultivated land", "Rural household adoption of the Dabaogan form of the Household Responsibility System at the end of 1983", "Rural population and other rural basic-condition denominators where reported"], "research_fit": {"best_for": ["Historical provincial or regional rural-economy measures when the question needs rural production, labor, land or household-responsibility-system context in the 1980s and early 1990s", "Reconstructing the CRSY component of Jin and Qian's 1998 QJE provincial rural-enterprise panel", "Building a source-audited aggregate rural panel when table, page, unit and issue provenance can be preserved"], "choose_over": ["Choose CRSY over the general China Statistical Yearbook when the needed measure is specifically rural gross social product, rural labor, cultivated land, non-farm employment or the historical HRS-adoption table", "Use the 1949-1986 rural-economic compilation for earlier historical coverage only; it is a different book and does not silently extend this 1984-1994 source family", "Use a current NBS release for current definitions, not as proof that the paper-era tables or definitions are unchanged"], "not_good_for": ["Household, farm, worker, village or firm microdata and individual identifiers", "A guaranteed balanced province-year panel, county-level data, or a current rural series", "Treating the yearbook as Jin and Qian's final cleaned panel or as a policy-treatment dataset", "Assuming that every table covers every province or that a later digital copy has the same definitions as the paper-era issue"], "needs_join_for": ["Township-enterprise and private-enterprise employment/output from the CTESY and TESM source families", "Rural income, taxes, government revenue and rural price measures from the China Agricultural Yearbook", "Rural-enterprise loans and household savings from the China Rural Finance Statistical Yearbook", "State-industry output and rural CPI inputs from the China Statistical Yearbook", "Urban-population shares from the China Population Yearbook or a compatible census product"], "variation_available": ["Provincial and annual differences in reported rural aggregates; this describes the data dimensions only and is not a policy treatment or causal-variation record"], "topics": ["rural development", "rural industrialization", "non-farm employment", "land and labor allocation", "regional growth", "household responsibility system"]}, "good_for": ["Manual reconstruction or validation of historical provincial rural aggregates", "Documenting the CRSY source boundary behind Jin and Qian (1998 QJE)", "Joining rural labor, land and production denominators to separate enterprise, finance and income yearbooks"], "identification": ["This annual statistical source is descriptive and does not itself encode a causal design or a verified treatment-assignment rule.", "Provincial and annual differences require separate design evidence; they are not classified here as exogenous variation."], "linkable_keys": ["Province or historical region label", "Year, issue label and reference-year convention", "Indicator, unit, table and page number", "Rural population denominator and rural/total labor definition", "Researcher-built province concordance when historical labels or boundaries changed"], "joins": [{"target": "china-township-enterprises-statistical-yearbook", "relation": "complement", "keys": ["Province or region", "Year and reference-year convention", "Rural labor or enterprise indicator"], "method": "Use CRSY for rural labor, land and gross-social-product denominators and CTESY for township/private-enterprise employment and output. Preserve each issue, table, unit and missing-value note instead of copying the paper's derived ratios as raw columns.", "evidence_status": "literature-used"}, {"target": "china-township-enterprises-statistical-material-1978-1985", "relation": "complement", "keys": ["Province or region", "Year", "Rural population or rural-enterprise context"], "method": "Use TESM only for the separate 1980/1986 baseline inputs documented in the QJE appendix; do not use it as a substitute for CRSY's 1987-1994 rural labor and land tables.", "evidence_status": "literature-used"}, {"target": "china-agricultural-yearbook", "relation": "complement", "keys": ["Province or region", "Year and issue/reference-year convention", "Rural income, revenue or price indicator"], "method": "Join only after checking the two books' reference years and province labels. CRSY supplies rural production/labor/land context; CAY supplies the separate income, taxes, revenue and price roles in the QJE construction.", "evidence_status": "literature-used"}, {"target": "china-rural-finance-statistical-yearbook", "relation": "complement", "keys": ["Province or region", "Year", "Rural financial indicator"], "method": "Keep CRSY's rural gross-social-product denominator separate from CRFSY's loan and savings numerators; preserve the previous-year timing used in the paper.", "evidence_status": "literature-used"}, {"target": "china-stat-yearbook", "relation": "complement", "keys": ["Province or region", "Year", "Indicator and unit"], "method": "Use the general Statistical Yearbook for state-industry output or rural CPI inputs and for cross-checking, not as a silent replacement for CRSY rural tables.", "evidence_status": "literature-used"}, {"target": "china-rural-economic-statistics-1949-1986", "relation": "predecessor", "keys": ["Province or region", "Year", "Rural/agricultural indicator and unit"], "method": "Use the 1949-1986 compilation only for earlier historical context and overlapping-definition checks; it is not evidence for the paper's later CRSY tables.", "evidence_status": "plausible"}], "access_routes": [{"route": "National Diet Library catalogue and holding-library/document-delivery route", "access_status": "available-with-conditions", "direct_url": "https://ndlsearch.ndl.go.jp/books/R100000002-Ia0000053636", "requirements": ["Library membership or interlibrary-loan/document-delivery access", "Chinese-language table reading and manual transcription"], "steps": ["Search the NDL record Z41-AC72 and its holdings, beginning with the paper-era 1985-1994 volumes", "Ask the holding library to verify the title page, issue label, publisher, compiler, publication date and table of contents", "Match the requested table and reference year to the QJE Appendix Table I before copying any values", "Request only needed pages under the library's scan and copyright rules, then record table, page, unit, province coverage and missing-value codes"], "deliverable": "Physical volume, permitted scan or researcher-transcribed tables; no open machine-readable CRSY file was verified", "cost": "mixed", "last_checked": "2026-09-28", "caveat": "The NDL record confirms the series and holdings, not that every requested volume is immediately scanable or that a scan may be redistributed."}, {"route": "Google Books bibliographic record for the 1992 volume", "access_status": "discovery-only", "direct_url": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%86%9C%E6%9D%91%E7%BB%9F%E8%AE%A1%E5%B9%B4%E9%89%B4.html?id=hUNauAAACAAJ", "requirements": ["Use the metadata to identify a library or licensed seller; Google Books does not establish a downloadable copy"], "steps": [{"Confirm that the volume is 中国农村统计年鉴": "1992, compiled by the NBS rural social-economic statistics office"}, "Note that the record says it reports 1991 rural conditions and contains rural output, labor, income, finance and social tables", "Locate a lawful physical or licensed copy and compare the title page/table notes before using it"], "deliverable": "Bibliographic identification and a route lead, not the paper's data file", "cost": "mixed", "last_checked": "2026-08-12", "caveat": "Metadata and page count do not prove the exact table used by Jin and Qian or any reuse permission."}, {"route": "Third-party yearbook listing (unverified digital lead)", "access_status": "needs-verification", "direct_url": "https://www.tjnj.net/navibooklist-N2006010438-2.html", "requirements": ["Verify file provenance, exact issue, payment terms and reuse permission", "Compare any delivered file with a library title page and the QJE source citation"], "steps": ["Treat the listing as a discovery lead only", "Do not download, redistribute or cite a file as the paper's source until its edition and rights are checked"], "deliverable": "Possible third-party PDF/Excel subject to provider terms; not accepted as an open or rights-cleared route", "cost": "mixed", "last_checked": "2026-08-12", "caveat": "A listing is not evidence of official provenance, complete coverage or permission to reuse."}], "access": {"url": "https://ndlsearch.ndl.go.jp/books/R100000002-Ia0000053636", "cost": "mixed", "license": "Printed-volume copyright and library/document-delivery terms; no open-data licence for the paper-era issues was identified", "format": ["printed Chinese annual volume", "permitted library scan", "researcher-transcribed table"], "api": false, "how_to_get": "Start with the NDL catalogue and a holding library, verify the issue/reference-year/table boundary against the QJE appendix, and transcribe only the required provincial cells with page, unit, footnote and missing-value provenance. Treat Google Books and third-party listings as metadata or unverified leads, not as automatic downloads."}, "caveats": ["The paper cites CRSY 1984-1994 annual issues, while the NDL series record starts with the 1985 volume published in 1986.2; do not assume the year labels mean the same thing without checking title pages and table notes.", "Compiler responsibility changes across the paper period; preserve the issue-level editor and publisher rather than collapsing all volumes into a timeless provider.", "The QJE variables are ratios or derived measures built from source tables. CRSY does not by itself provide Jin and Qian's final provincial regression panel.", "The source is aggregate and table-dependent; it does not establish county, household, worker or firm identifiers.", "The current NBS series and a later digital yearbook may revise definitions, province names or units; preserve the historical issue and compare before splicing.", "No policy, treatment, causal-variation or identification record is created here."], "quality": {"profile_status": "verified", "access_status": "verified", "paper_use_status": "verified", "last_audited": "2026-09-28"}, "used_by": [{"cite": "Jin & Qian (1998), Public Versus Private Ownership of Firms: Evidence from Rural China", "doi": "https://doi.org/10.1162/003355398555748", "journal": "QJE", "year": 1998, "dataset_role": "Rural production, labor, land and household-responsibility-system inputs and denominators in a provincial rural-enterprise panel", "evidence_type": "data_appendix", "evidence_url": "https://paperzz.com/doc/7852221/forthcoming--quarterly-journal-of-economics-public-vs.-pr...", "data_note": "Appendix Table I identifies CRSY 1985 for the share of rural households not adopting Dabaogan at the end of 1983 and CRSY 1987-1994 for rural gross social product, total/agricultural labor, cultivated land and rural population inputs. The paper uses these source tables to construct ratios and controls; it does not release CRSY or claim that the yearbooks are its final cleaned panel."}], "provenance": [{"source": "https://paperzz.com/doc/7852221/forthcoming--quarterly-journal-of-economics-public-vs.-pr...", "field_scope": ["paper actually uses CRSY", "source-specific variable roles and year labels", "provincial aggregate panel and derived-variable boundary"], "added": "2026-09-28", "confidence": "high", "verified": true}, {"source": "https://ndlsearch.ndl.go.jp/books/R100000002-Ia0000053636", "field_scope": ["Chinese title and annual-series identity", "China Statistical Press publisher", "1985-onward holdings and issue-level compiler changes through the paper period", "library/document-delivery route"], "added": "2026-08-12", "confidence": "high", "verified": true}, {"source": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%86%9C%E6%9D%91%E7%BB%9F%E8%AE%A1%E5%B9%B4%E9%89%B4.html?id=hUNauAAACAAJ", "field_scope": ["1992 volume metadata", "compiler and publisher corroboration", "1991 reference-year description"], "added": "2026-08-12", "confidence": "med", "verified": true}, {"source": "https://www.stats.gov.cn/zs/tjwh/tjkw/tjzl/index_5.html", "field_scope": ["official NBS publication-catalogue context for the yearbook series"], "added": "2026-08-12", "confidence": "med", "verified": true}, {"source": "https://www.tjnj.net/navibooklist-N2006010438-2.html", "field_scope": ["unverified digital-access lead only"], "added": "2026-08-12", "confidence": "low", "verified": false}], "related_datasets": [{"id": "china-township-enterprises-statistical-yearbook", "relation": "complement"}, {"id": "china-township-enterprises-statistical-material-1978-1985", "relation": "complement"}, {"id": "china-agricultural-yearbook", "relation": "complement"}, {"id": "china-rural-finance-statistical-yearbook", "relation": "complement"}, {"id": "china-stat-yearbook", "relation": "complement"}, {"id": "china-rural-economic-statistics-1949-1986", "relation": "predecessor"}], "source_path": "datasets/china-rural-statistical-yearbook-1984-1994.md", "description_markdown": "## Positioning in one sentence\n\nThis is the historical printed rural-statistics source family behind several provincial inputs in Jin and Qian (1998 QJE), not their final regression file: its value is the rural production/labor/land detail for the 1980s and early 1990s, while the main cost is library-mediated access and manual table verification.\n\n## Select rules\n\n- Prioritize it when a research idea needs historical provincial rural gross product, labor, cultivated land, non-farm employment or the 1983 HRS-adoption measure.\n- Pair it with CTESY/TESM for township-enterprise measures, CAY for rural income and government revenue, CRFSY for loans and savings, and CSY for state-industry or CPI inputs; keep the table-level provenance of each source.\n- Use the 1949-1986 rural compilation for earlier historical context, not as an automatic replacement for 1987-1994 CRSY observations. Use current NBS releases for current questions only after checking definition changes.\n- Do not use this record for microdata, firm identifiers, a guaranteed county panel, or policy-treatment claims.\n\n## Get recipe\n\n1. Search the NDL record `Z41-AC72` and the Chinese title for a holding library with the needed 1985-1994 volumes.\n2. Obtain a physical copy, permitted scan or other lawful document delivery. Record the title page, issue label, compiler, publication date and publisher.\n3. Use the QJE Appendix Table I to specify the needed indicators before transcribing. Capture table/page, unit, province labels, reference year and missing-value notes for every cell.\n4. Compare overlapping values and definitions with the companion yearbooks before joining by province and year. Keep publication year and reference year separate.\n5. Store any derived panel or ratios in a separate reproducible project; this record identifies the source tables and their route, not a redistributed copy of the paper's analysis file.\n\n## Connections and Limitations\n\nThe safest join is province-by-reference-year with an explicit historical province concordance, indicator label, unit and table/page provenance. CRSY supplies rural production, labor and land context, while the companion yearbooks supply enterprise, finance, income, price and state-sector components. Because no open machine-readable release was verified, a researcher should not promise a bulk download or redistribute a third-party file until a specific library/provider confirms the issue and terms. A recommendation fails if the needed issue/table is absent, if the issue/reference-year convention cannot be reconciled, or if a changed definition makes the intended join invalid.\n\n## Decision sufficiency check\n\nFor a researcher studying historical regional rural industrialization, this record selects CRSY for rural denominators and labor/land context, tells the researcher to start with a library issue check, and explains why CTESY, CAY and the general Statistical Yearbook must remain separate complements. If the exact table or lawful copy cannot be obtained, the agent should retain the uncertainty and switch to a verified later/earlier source rather than fill the gap from a title, abstract or third-party listing.", "research_position": "This is the historical printed rural-statistics source family behind several provincial inputs in Jin and Qian (1998 QJE), not their final regression file: its value is the rural production/labor/land detail for the 1980s and early 1990s, while the main cost is library-mediated access and manual table verification."} {"schema_version": 2, "catalog_status": "needs-review", "id": "china-satellite-pm25", "name": "China PM2.5 exposure-grid leads (TAP and other non-interchangeable products)", "aka": ["卫星PM2.5", "Satellite-derived PM2.5", "Global Annual PM2.5 Grids", "中国网格PM2.5"], "provider": "Multiple distinct producers. This legacy index names TAP and generic global satellite products as leads, but they are not one canonical data asset and must not inherit one another's coverage, access route or paper use.", "china_related": true, "domains": ["environment", "health", "migration", "urban", "development"], "unit_of_observation": "Grid-day / Grid-year (can be summed to counties, cities, and provinces)", "structure": "gridded-spatiotemporal", "geo_granularity": ["grid", "County", "city", "province"], "geography": "Full coverage in mainland China; resolution and year vary by product", "time_span": {"start": 1998, "end": "ongoing", "coverage_note": "The paper commonly uses global annual satellite PM2.5; TAP also provides about 1km daily fusion products since 2000.", "last_confirmed_release": "ongoing", "last_checked": "2026-07-10"}, "frequency": ["daily", "annual"], "sample_size": "National regular grid; number depends on resolution and study boundaries.", "key_variables": ["PM2.5 concentration", "Grid latitude and longitude", "date/year", "Uncertainty or calibration information", "Satellite AOD", "Meteorological and model auxiliary variables"], "research_fit": {"best_for": ["Long-term, nationwide continuous PM2.5 exposure before and after monitoring station construction", "Study on county and city population migration, health and productivity and medium and long-term pollution"], "choose_over": ["When continuous spatial coverage across the country or PM2.5 before 2013 is required, priority is given to monitoring at official sites."], "not_good_for": ["Total pollution emissions of enterprises", "Studies requiring full equivalence to ground-truth measurements"], "needs_join_for": ["It needs to be superimposed on the grid and administrative boundaries, and then joined with population, survey or enterprise location."], "variation_available": ["Grid and annual pollution changes", "Meteorological instrumental variables such as temperature inversion", "Monitoring information before and after disclosure"], "topics": ["air pollution", "PM2.5", "long term pollution exposure", "population migration", "Meteorological instrumental variables"]}, "good_for": ["Long-term PM2.5 exposure", "pollution and migration", "pollution and health", "National Grid Environment Panel"], "identification": ["IV (thermal inversion/wind direction)", "panel fixed effects", "DID (information disclosure or policy)", "spatial matching"], "linkable_keys": ["Latitude and longitude", "Grid ID", "County/city administrative code", "date/year"], "joins": [{"target": "china-census", "relation": "complement", "keys": ["County/city code", "Year"], "method": "spatial-aggregation", "evidence_status": "literature-used"}, {"target": "cmds", "relation": "complement", "keys": ["city code", "Year"], "method": "spatial-aggregation", "evidence_status": "plausible"}], "access_routes": [{"route": "tap", "access_status": "available-with-registration-or-terms", "direct_url": "http://tapdata.org.cn/", "requirements": "Follow the data application, citation and usage instructions on the TAP website.", "steps": ["Enter the TAP data platform.", "Select contaminants, temporal and spatial resolution.", "Apply according to the platform requirements or download and keep the version description."], "deliverable": "China's high-resolution daily/annual fused pollution grid; the specific version shall be subject to the platform.", "cost": "free", "last_checked": "2026-07-10"}, {"route": "paper-specific-global-grid", "access_status": "needs-verification", "direct_url": "needs-verification", "requirements": "First confirm the global PM2.5 grid version used in the paper data section, and then download it from the original data provider.", "steps": ["Record the product name, version and resolution in the paper.", "Get the same version from the product official website.", "Save version and citation information."], "deliverable": "Annual satellite retrieval PM2.5 grid consistent with the paper.", "cost": "free", "last_checked": "2026-07-10"}], "access": {"url": "http://tapdata.org.cn/", "cost": "free", "license": "Subject to product-specific data usage and citation terms", "format": ["netcdf", "geotiff", "csv"], "api": false, "how_to_get": "First select the specific product and version. TAP can be used as a route for high-resolution fusion data in China; when reproducing existing papers, the same global grid version must be obtained by data section."}, "caveats": "Satellite inversion/model fusion is not ground truth measurement; different products, versions and calibration methods cannot be directly mixed. Aggregation of grids into administrative districts requires area or population weighting.", "quality": {"profile_status": "partial", "access_status": "partial", "paper_use_status": "partial", "last_audited": "2026-09-28"}, "used_by": [{"cite": "Chen, Oliva & Zhang (2022), The Effect of Air Pollution on Migration: Evidence from China", "doi": "https://doi.org/10.1016/j.jdeveco.2022.102833", "journal": "JDE", "year": 2022, "dataset_role": "County-level five-year average PM2.5; joined with census migration and thermal inversion", "evidence_type": "paper_data_section", "evidence_url": "https://www.nber.org/system/files/working_papers/w24036/w24036.pdf", "data_note": "The published paper's data section now resolves a different identity: MERRA-2 M2TMNXAER 5.12.4 is converted and aggregated by the authors. See china-merra2-pm25-chen-oliva-zhang; this citation does not verify TAP or a generic global annual grid as the used product."}, {"cite": "Khanna, Liang, Mobarak & Song (2025), The Productivity Consequences of Pollution-Induced Migration in China", "journal": "AEJ:Applied", "year": 2025, "dataset_role": "City annual PM2.5; combined with migration and spatial model data", "evidence_type": "online_appendix", "evidence_url": "https://assets.aeaweb.org/asset-server/files/22334.pdf", "data_note": "The specific product version should still be further registered in the replication package."}], "provenance": [{"source": "https://www.nber.org/system/files/working_papers/w24036/w24036.pdf", "field_scope": ["paper_use", "unit_of_observation", "identification"], "added": "2026-07-10", "confidence": "high", "verified": true}, {"source": "http://tapdata.org.cn/", "field_scope": ["alternative_access", "coverage"], "added": "2026-07-10", "confidence": "high", "verified": true}], "related_datasets": [{"id": "tap-china-pm25", "relation": "successor"}, {"id": "china-merra2-pm25-chen-oliva-zhang", "relation": "successor"}, {"id": "china-air-quality-monitoring", "relation": "complement"}, {"id": "china-census", "relation": "complement"}, {"id": "china-firm-pollution", "relation": "often-confused-with"}], "source_path": "datasets/china-satellite-pm25.md", "description_markdown": "## Positioning in one sentence\n\nThis legacy lead groups several non-interchangeable PM2.5 products and is not itself a data asset to select. Use it only to discover a specific product, then open that product's canonical record and confirm its producer, version, access route and measurement construction.", "research_position": "This legacy lead groups several non-interchangeable PM2.5 products and is not itself a data asset to select. Use it only to discover a specific product, then open that product's canonical record and confirm its producer, version, access route and measurement construction."} diff --git a/dist/checksums.sha256 b/dist/checksums.sha256 index f1097eb..3765f80 100644 --- a/dist/checksums.sha256 +++ b/dist/checksums.sha256 @@ -1,6 +1,6 @@ f2fd0a1451ef6065c67dea32b682da7287a7b1645e3de3e0e0e2db2d9b7375e4 catalog.csv -ac8bd8aa531ffcf5b1703d28af0fb72ad1af8a10f561e259fe293b38a9fc6e35 catalog.json -d17d59eff8e8726f1c0eb29a2e09df7e8d476cb0a513b679da1731b319632bb7 catalog.jsonl +261e139cf6aa7c8a10d17b38887979a12f2403c7200d9595088d5db13a489617 catalog.json +0411893c36a16f3548dfec72e24c5ad6bad2794a637158900c0906e54b531b88 catalog.jsonl a74962644744f55bcbaacb6eee9a761581e33f7c57d75f1161496fe385f5ae6a catalog.jsonld dcf20c293a38e2befeb08e1fac2acf6acfa779e49d9609af2ac9c09eb32814f1 joins.json dd8dad60a069b0a50d6c30c617f89d8c2daac2c3c2834e75bb3e68e61d5ba5b1 router_index.json diff --git a/docs/catalog.json b/docs/catalog.json index 49d53f9..fe9d78e 100644 --- a/docs/catalog.json +++ b/docs/catalog.json @@ -31149,7 +31149,7 @@ "verified": true }, { - "source": "https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ", + "source": "https://books.google.com/books?id=gS0yAAAAMAAJ", "field_scope": [ "corroborating bibliographic metadata", "no eBook availability statement" diff --git a/docs/datasets/china-rural-economic-statistics-1949-1986.md b/docs/datasets/china-rural-economic-statistics-1949-1986.md index b096358..85dc4de 100644 --- a/docs/datasets/china-rural-economic-statistics-1949-1986.md +++ b/docs/datasets/china-rural-economic-statistics-1949-1986.md @@ -197,7 +197,7 @@ provenance: added: '2026-08-11' confidence: high verified: true - - source: https://books.google.com/books/about/%E4%B8%AD%E5%9B%BD%E5%农村经济统计大全_1949_1986.html?id=gS0yAAAAMAAJ + - source: https://books.google.com/books?id=gS0yAAAAMAAJ field_scope: - corroborating bibliographic metadata - no eBook availability statement diff --git a/guides/operations.md b/guides/operations.md index c94543a..e0638c8 100644 --- a/guides/operations.md +++ b/guides/operations.md @@ -190,6 +190,8 @@ Generated files are outputs, not canonical sources: `DATASET_INDEX.md`, `ledgers The public health report describes durable catalog inputs so the same report works in a fresh clone. Local fetch-cache diagnostics are printed separately by the validator and still fail the command when they expose an unrecorded challenge page; the raw cache is not a publication dependency. The static site serves copied canonical Markdown unchanged using `.nojekyll`. Its builder preserves existing files, and the freshness check reports unexpected pages for review rather than silently deleting them. +The weekly **Check external link freshness** workflow is a separate network diagnostic. Its run summary shows how many URLs were sampled and lists the ones needing review; a successful job means the scan finished, not that every link was reachable. Read a failed URL in context before changing a record: a login wall, denied automated request or timeout does not establish that the data are gone. Unexpected script errors fail the job. For a local report, run `python scripts/check_links.py --summary link-summary.md`; add `--strict` when you explicitly want any unsuccessful URL to produce a nonzero exit code. + ## Queue and provenance The task lifecycle is: diff --git a/ledgers/changes.jsonl b/ledgers/changes.jsonl index fadd247..026e377 100644 --- a/ledgers/changes.jsonl +++ b/ledgers/changes.jsonl @@ -559,3 +559,4 @@ {"timestamp":"2026-09-28T19:33:43+08:00","action":"acquisition-grounding","dataset":"nbs-time-use-survey","agent":"codex","detail":"Primary NBS replies and laboratory documentation establish a conditional 2018 diary-microdata application route: institution registration, approved project, onsite analysis and reviewed outputs. Replaced the prior wholly unverified-access description without promising raw downloads or all-round access. The official 2018 bulletin was read by ordinary GET: named 11-province design, 20,226 households/48,580 people, and explicit Shanghai exclusion for published results; these are not verified laboratory inventory counts. The 2024 opening statement remains an announcement. Sanitized fields/geography, fees, current appointments and fully read economics-paper use remain open; status stays grounding. A 2023 Statistical Research PDF use lead returned 502 and is not used_by evidence.","files_changed":["datasets/nbs-time-use-survey.md","ledgers/changes.jsonl","ledgers/completed_tasks.jsonl"],"no_files_deleted_or_moved":true} {"timestamp":"2026-09-28T23:39:14+08:00","action":"product-entry-and-evidence-consistency","agent":"codex","datasets":["cfps","china-land-transaction"],"detail":"Approved bounded public-product polish: shortened README around research-question use with Chinese and English prompts; routed answering and maintenance through one shared AGENTS/mental-model entry and scoped personal Kimi/DeepSeek notes. Aligned land prose with its existing grounding boundary and preserved former historical-price/completeness concerns as unverified; corrected CFPS prose on annual interpolation, local inference and the separate CHARLS provider/application identity using existing canonical records. No new source verification, status promotion, paper-entry removal or provenance removal. Fixed project-relative static record links, made readiness conditional in the UI/quality card, kept copied Markdown intact, and preserved existing generated pages for explicit freshness review. Public health now depends on durable inputs while local cache diagnostics still fail unrecorded challenges. Known scratch files are ignored, not deleted. Retained remote Ruff 0.15 and Actions upgrades without replacing newer local checks. Public-copy validator, Ruff 0.15.21, credential scan (403 files, zero findings), 65 tests and actual router-script checks passed. All 157 canonical records retained; only two canonical files edited. Three maintainer decision walkthroughs covered CFPS/CHARLS choice, conditional land-household construction and rejection of a national unemployment series for city-panel work; these were not blind model tests. Existing duplicate-task-title warning and legacy evidence debt remain. No commit or push.","files_changed":["README.md","AGENTS.md","guides/mental-model.md","guides/operations.md","guides/usage.md","guides/kimi-k3-handoff.zh-CN.md","CONTRIBUTING.md","datasets/cfps.md","datasets/china-land-transaction.md","scripts/build_site.py","scripts/build_quality_card.py","scripts/check_generated_views.py","scripts/validate_kb.py","tests/test_exports.py",".gitignore","requirements-dev.txt","pyproject.toml",".github/workflows/validate.yml",".github/workflows/link-freshness.yml"],"no_files_deleted_or_moved":true} {"timestamp":"2026-09-29T01:24:14+08:00","action":"user-review-evidence-repair","agent":"codex","datasets":["chfs","china-pku-digital-financial-inclusion-index","cnopendata-local-government-work-report-corpus","china-5a-scenic-area-list"],"detail":"Focused correction of the independent user-role review findings. Preserved concrete variables, comparative research fit, acquisition steps, identities, paper leads and catalog statuses. CHFS wording now separates measured financial behavior from causal identification and approved access from all-wave access. PKU research-fit and prose no longer promote an abstract-only JRS lead to a verified use. Re-read the CnOpenData provider page: its TXT/PDF counts do not reconcile with the total, and its detailed periods differ by government level; retained the figures and the disagreement instead of inferring a city-only subset. Re-read the official 2024-02-06 5A announcement and counted 21 distinct provincial-level units. Public README corpus row reflects the period and count scope; its agent note retains product advantages without a star-count appeal. No new functionality, dataset deletion, status promotion, extra constraint system or full-catalog audit claim. CHFS access and JRS full-text were not newly verified; provider format reconciliation still requires a file manifest or reply."} +{"timestamp":"2026-10-26T18:25:00+08:00","action":"repair-link-freshness-check","agent":"codex","datasets":["china-rural-economic-statistics-1949-1986"],"detail":"GitHub run 40295619737 crashed with UnicodeEncodeError on a mixed Chinese/percent-encoded Google Books URL; continue-on-error concealed the incomplete scan as a successful job. Encode request path/query for HEAD and GET while preserving original report URLs and percent escapes; retain expected network failures as findings and let unexpected checker errors fail. The weekly job now publishes a readable bounded-scan summary without strict mode or blanket error suppression. Corrected only the book source URL to https://books.google.com/books?id=gS0yAAAAMAAJ; a fresh GET returned HTTP 200 and the title 中国农村经济统计大全, 1949-1986. Product identity, evidence scope and catalog status are unchanged. Local full network results are not link-death evidence because the local DNS environment resolves targets to non-public addresses and existing guards block them; use GitHub-hosted results for network follow-up. No records or files deleted."} diff --git a/ledgers/health.json b/ledgers/health.json index 24f3615..0c5b912 100644 --- a/ledgers/health.json +++ b/ledgers/health.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-30T11:34:44+08:00", + "generated_at": "2026-10-26T18:28:33+08:00", "status": "ok", "errors": [], "warnings": [ @@ -17,7 +17,7 @@ "done": 434, "failed": 157, "candidates": 49, - "changes": 561 + "changes": 562 }, "idea_benchmark_cases": 10, "blind_benchmark_v2_cases": 27, @@ -66,7 +66,7 @@ }, "input_fingerprint": { "algorithm": "sha256", - "digest": "692ae6d0b526d9362054fe05ef3c23e89f05355dc7da8ec5d2d4dc24cf0d4f94", + "digest": "266d7ed7add3f294c25999131895da67517c361c59a298e84b6c484d5d413484", "file_count": 168 } } diff --git a/scripts/check_links.py b/scripts/check_links.py index eed55d9..cd84003 100644 --- a/scripts/check_links.py +++ b/scripts/check_links.py @@ -8,6 +8,7 @@ from datetime import datetime from pathlib import Path from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlsplit from urllib.request import HTTPRedirectHandler, Request, build_opener sys.dont_write_bytecode = True @@ -53,7 +54,14 @@ def check(url: str, timeout: float) -> dict[str, object]: issues = public_http_url_issues(url, resolve_dns=False) if issues: return {"url_fingerprint": url_fingerprint(url), "result": "blocked", "issues": issues} - request = Request(url, method="HEAD", headers={"User-Agent": "econ-dataknowhow-link-check/1.0"}) + # urllib needs an ASCII request target, even when the recorded URL contains Chinese text. + # Preserve URL separators and existing percent escapes on both request paths. + parts = urlsplit(url) + request_url = parts._replace( + path=quote(parts.path, safe="/%:@!$&'()*+,;="), + query=quote(parts.query, safe="/%?:@!$&'()*+,;="), + ).geturl() + request = Request(request_url, method="HEAD", headers={"User-Agent": "econ-dataknowhow-link-check/1.0"}) try: with safe_open(request, timeout=timeout) as response: return {"url": url, "status": response.status, "final_url": response.geturl(), "result": "ok"} @@ -61,17 +69,40 @@ def check(url: str, timeout: float) -> dict[str, object]: if exc.code != 405: return {"url": url, "status": exc.code, "result": "http-error", "error": str(exc.reason)} fallback = Request( - url, + request_url, method="GET", headers={"User-Agent": "econ-dataknowhow-link-check/1.0", "Range": "bytes=0-1024"}, ) try: with safe_open(fallback, timeout=timeout) as response: return {"url": url, "status": response.status, "final_url": response.geturl(), "result": "ok"} - except Exception as fallback_exc: # network diagnostics are deliberately best effort - return {"url": url, "status": exc.code, "result": "error", "error_type": type(fallback_exc).__name__} + except HTTPError as fallback_exc: + return {"url": url, "status": fallback_exc.code, "result": "http-error", "error": str(fallback_exc.reason)} + except (URLError, TimeoutError, OSError) as fallback_exc: + return {"url": url, "result": "error", "error_type": type(fallback_exc).__name__, "error": str(fallback_exc)} except (URLError, TimeoutError, OSError) as exc: - return {"url": url, "result": "error", "error_type": type(exc).__name__} + return {"url": url, "result": "error", "error_type": type(exc).__name__, "error": str(exc)} + + +def build_summary(results: list[dict[str, object]], available_count: int) -> str: + failures = [item for item in results if item.get("result") != "ok"] + lines = [ + "## External link freshness", + "", + f"Checked **{len(results)} of {available_count}** unique URLs; **{len(failures)}** need review.", + "", + "This is a bounded network diagnostic. Access restrictions and temporary connection errors need review " + "before changing a data record.", + ] + if failures: + lines.extend(["", "| URL | HTTP status | Result | Detail |", "| --- | --- | --- | --- |"]) + for item in failures: + url = str(item.get("url", "")).replace("|", "%7C") + label = f"<{url}>" if url else f"Blocked URL ({item.get('url_fingerprint', '')})" + detail = item.get("error") or item.get("error_type") or ", ".join(item.get("issues", [])) + detail = str(detail).replace("|", "\\|").replace("\n", " ") + lines.append(f"| {label} | {item.get('status', '—')} | {item['result']} | {detail} |") + return "\n".join(lines) + "\n" def main() -> int: @@ -79,17 +110,21 @@ def main() -> int: parser.add_argument("--timeout", type=float, default=10.0) parser.add_argument("--max-urls", type=int, default=150) parser.add_argument("--output", type=Path, default=ROOT / "ledgers" / "link-check.json") + parser.add_argument("--summary", type=Path, help="also write a readable Markdown report, e.g. GITHUB_STEP_SUMMARY") parser.add_argument("--strict", action="store_true", help="return non-zero when any checked URL is not successful") args = parser.parse_args() - urls = collect_urls()[: max(args.max_urls, 0)] - results = [check(url, args.timeout) for url in urls] + urls = collect_urls() + results = [check(url, args.timeout) for url in urls[: max(args.max_urls, 0)]] payload = { "schema_version": 1, "checked_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "available_count": len(urls), "checked_count": len(results), "results": results, } atomic_write_text(args.output, json.dumps(payload, ensure_ascii=False, indent=2) + "\n") + if args.summary: + atomic_write_text(args.summary, build_summary(results, len(urls))) failures = sum(1 for item in results if item.get("result") != "ok") print(f"checked={len(results)} failures={failures} output={args.output}") return 1 if args.strict and failures else 0 diff --git a/tests/test_check_links.py b/tests/test_check_links.py index 2c241e9..32937f0 100644 --- a/tests/test_check_links.py +++ b/tests/test_check_links.py @@ -1,5 +1,14 @@ from __future__ import annotations +import json +import sys +from contextlib import nullcontext +from types import SimpleNamespace +from urllib.error import HTTPError, URLError + +import pytest + +import check_links from check_links import check @@ -12,3 +21,82 @@ def test_link_checker_blocks_private_and_script_targets_without_network() -> Non assert "non-http-scheme" in script["issues"] assert "url" not in private assert "url" not in script + + +@pytest.mark.parametrize("fallback", [False, True]) +def test_unicode_url_is_encoded_for_head_and_get(monkeypatch, fallback) -> None: + url = "https://books.google.com/books/about/%E4%B8%AD国农村经济.html?id=123&title=统计" + calls = [] + + def fake_open(request, timeout): + calls.append(request) + request.full_url.encode("ascii") # Matches urllib's HTTP request-target requirement. + if fallback and request.get_method() == "HEAD": + raise HTTPError(request.full_url, 405, "Method Not Allowed", {}, None) + return nullcontext(SimpleNamespace(status=200, geturl=lambda: request.full_url)) + + monkeypatch.setattr(check_links, "safe_open", fake_open) + result = check(url, timeout=0.01) + + assert result["result"] == "ok" + assert result["url"] == url + assert "%E4%B8%AD%E5%9B%BD" in result["final_url"] + assert "title=%E7%BB%9F%E8%AE%A1" in result["final_url"] + assert "%25E4" not in result["final_url"] + assert [request.get_method() for request in calls] == (["HEAD", "GET"] if fallback else ["HEAD"]) + + +def test_get_fallback_reports_its_own_http_status(monkeypatch) -> None: + def fake_open(request, timeout): + status = 405 if request.get_method() == "HEAD" else 403 + raise HTTPError(request.full_url, status, "request denied", {}, None) + + monkeypatch.setattr(check_links, "safe_open", fake_open) + result = check("https://example.org/", timeout=0.01) + assert result["result"] == "http-error" + assert result["status"] == 403 + + +@pytest.mark.parametrize("strict, expected_exit", [(False, 0), (True, 1)]) +def test_cli_finishes_and_reports_network_findings(monkeypatch, tmp_path, strict, expected_exit) -> None: + urls = ["https://example.org/ok", "https://example.org/denied", "https://example.org/timeout", "https://example.org/later"] + output = tmp_path / "links.json" + summary = tmp_path / "summary.md" + arguments = ["check_links.py", "--max-urls", "3", "--output", str(output), "--summary", str(summary)] + if strict: + arguments.append("--strict") + + def fake_open(request, timeout): + if request.full_url.endswith("/denied"): + raise HTTPError(request.full_url, 403, "Forbidden", {}, None) + if request.full_url.endswith("/timeout"): + raise URLError("timed out") + return nullcontext(SimpleNamespace(status=200, geturl=lambda: request.full_url)) + + monkeypatch.setattr(sys, "argv", arguments) + monkeypatch.setattr(check_links, "collect_urls", lambda: urls) + monkeypatch.setattr(check_links, "safe_open", fake_open) + + assert check_links.main() == expected_exit + report = json.loads(output.read_text(encoding="utf-8")) + assert report["available_count"] == 4 + assert report["checked_count"] == 3 + assert [item["result"] for item in report["results"]] == ["ok", "http-error", "error"] + markdown = summary.read_text(encoding="utf-8") + assert "**3 of 4**" in markdown + assert "**2** need review" in markdown + assert "https://example.org/denied" in markdown + assert "403" in markdown + assert "https://example.org/timeout" in markdown + assert "timed out" in markdown + + +def test_unexpected_checker_error_is_not_hidden(monkeypatch, tmp_path) -> None: + def broken_check(url, timeout): + raise RuntimeError("unexpected checker error") + + monkeypatch.setattr(sys, "argv", ["check_links.py", "--output", str(tmp_path / "links.json")]) + monkeypatch.setattr(check_links, "collect_urls", lambda: ["https://example.org/"]) + monkeypatch.setattr(check_links, "check", broken_check) + with pytest.raises(RuntimeError, match="unexpected checker error"): + check_links.main()