From b6173ba8b431fbdd6dce74feb8e097222cc34509 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 7 Aug 2026 12:23:57 -0700 Subject: [PATCH] Fold three concept-spelling duplicate lineages; defer entity-drift splits Curation pass over the 10 duplicate-lineage families flagged after the v3 catalog rebuild. Three same-dimension concept-spelling pairs fold cleanly (identical geography and entity, disjoint or continuous observations): - dol.eta.initial_claims.sa -> survivor 830e68bb (us.dol spelling retired binding 330ec66b) - us.census.housing_starts.total_saar -> survivor d0212a1e (retired d542cb8d) - us.frb.industrial_production.total.mom_sa -> survivor d5634fc4 (retired b5a74252) Three noted retire events append to the UUID registry; no surviving UUID changes, no observation is lost (171/171 preserved), and all 83 docket- pinned UUIDs remain live. Catalog: 209 -> 206 series, byte-idempotent under regeneration, suspects=0, ambiguous_aliases=0. The seven entity-drift families (May/June observations disagreeing on entity) cannot fold without moving observations across entity keys, which alias inheritance is dimension-scoped to forbid; a disposable-copy experiment confirmed regeneration restores the split. They are recorded with blockers in docs/catalog-curation-backlog.md, alongside the us.dol.initial_claims.sa weekly/monthly split that awaits upstream cadence metadata correction. Co-Authored-By: Claude Fable 5 --- docs/catalog-curation-backlog.md | 37 +++++++ ledger/series_catalog.json | 154 ++++++----------------------- ledger/series_uuid_registry.jsonl | 3 + tests/test_build_series_catalog.py | 19 ++-- 4 files changed, 82 insertions(+), 131 deletions(-) create mode 100644 docs/catalog-curation-backlog.md diff --git a/docs/catalog-curation-backlog.md b/docs/catalog-curation-backlog.md new file mode 100644 index 0000000..bc2714f --- /dev/null +++ b/docs/catalog-curation-backlog.md @@ -0,0 +1,37 @@ +# Catalog curation backlog + +These series lineages remain split because the catalog builder scopes +concept and alias inheritance to an exact geography and entity slice. +Curating an alias cannot move an observation across entity keys, and the +catalog must not lose observations merely to present one lineage. + +On 2026-08-07, a disposable-copy experiment attempted each entity-drift +fold by deleting the unpinned row and adding its canonical concept to the +docket-pinned row. Regeneration restored both rows and produced a catalog +and UUID registry byte-identical to the originals: all seven pairs still +held one observation on each entity. They must remain split until the +upstream observation metadata is corrected. + +| Concept | Docket-pinned entity and UUID | Other entity and UUID | Blocker | +| --- | --- | --- | --- | +| `bls.cpi.u.headline_mom` | `household/cpi_u_all_items`, `3e796803-a194-4c83-9dc9-27cc870ff08e` | `economy/aggregate`, `70ca2ecf-4323-4453-b42f-23350cb95f22` | May and June observations disagree on entity. | +| `bls.cpi.u.core_mom` | `household/cpi_u_less_food_energy`, `d85c3c7f-874b-4caa-bfad-1ec7a473c293` | `economy/aggregate`, `e5e402d5-1a65-4c5d-a67a-07e3cf03cbf8` | May and June observations disagree on entity. | +| `bls.cps.unemployment_rate` | `person/civilian_labor_force`, `8dbbd54f-4bfd-4735-ad0d-5b55b8bb4ec5` | `economy/aggregate`, `db2f3857-e0fa-48f8-9ab1-6c8989307fd2` | May and June observations disagree on entity. | +| `fed.g17.industrial_production.total_index_mom` | `economy/aggregate`, `dbfd5050-e5df-40af-8922-dc8927d6363a` | `institutional_sector/total_industrial_production`, `d5634fc4-dbe0-4203-a5b7-728708b85a14` | May and June observations disagree on entity. The same-dimension `us.frb` spelling is folded into the latter row separately. | +| `fed.g17.capacity_utilization.total_industry` | `economy/aggregate`, `9668ecdb-5f61-422d-a653-8e8956257ade` | `institutional_sector/total_industry_capacity`, `978252a9-c452-41eb-aece-323340e796fc` | May and June observations disagree on entity. | +| `census.housing_starts.saar` | `economy/aggregate`, `8c999902-84e2-4e90-b49c-325cd535062a` | `dwelling/housing_start`, `d0212a1e-b360-460a-80a8-1cf4ad5ce427` | May and June observations disagree on entity. The same-dimension `us.census` spelling is folded into the latter row separately. | +| `bls.import_price_index.all_imports_mom` | `economy/aggregate`, `1d271205-0e71-4a74-9d5f-47e77577e2b2` | `household/all_imports`, `e8e6e43e-2c46-4620-a070-91d2da004b1c` | May and June observations disagree on entity. | + +## Initial claims cadence metadata + +The `us.dol.initial_claims.sa` rows are one weekly series, not a legitimate +weekly/monthly aggregate split. The two observations declared as monthly +carry week-ending identifiers (`2026-06-06` and `2026-06-13`) and values in +weekly thousands, immediately preceding the weekly observations beginning +`2026-06-20`. After folding their two concept spellings, they remain on +`person/ui_initial_claimant` with monthly cadence under UUID +`830e68bb-9ed6-493c-a8a4-440ba8e53f76`; the docket-pinned weekly rows remain +on `person/ui_claimant` under UUID +`1ad67747-d9d8-4ff9-9ebc-58206e658518`. Correct the two upstream rows' +period and entity metadata before folding these lineages; an alias cannot +bridge either mismatch safely. diff --git a/ledger/series_catalog.json b/ledger/series_catalog.json index b4b6598..497060f 100644 --- a/ledger/series_catalog.json +++ b/ledger/series_catalog.json @@ -5,18 +5,18 @@ "observation_rows": 171, "current_assertion_rows": 171, "docket_seed_sha256": "08266fc9db0b0a3ccadf6095902f8e1b67dd1a5a199cd3841b58b1681434fa13", - "uuid_registry_sha256": "c3db1149d74dedfe63bc903fa088bde00c5a6dc6c02e32665cf093fcb40b7ba8", + "uuid_registry_sha256": "0c4af1f59dd7d65a4335144917075b208124dd8cb3b0f7543372e47653bc94de", "suspect_segments": [], "stripped_segments": { "2026-05": [ "abs.cpi_indicator.allgroups.yoy", + "census.housing_starts.saar", "estat.jp.cpi.core_exfreshfood.yoy", + "fed.g17.industrial_production.total_index_mom", "ons.cpih.annual_rate", "statcan.36-10-0434-01.all_industries.month_to_month_percent_change", "statcan.cpi.allitems.yoy", - "us.bea.core_pce.mom_sa", - "us.census.housing_starts.total_saar", - "us.frb.industrial_production.total.mom_sa" + "us.bea.core_pce.mom_sa" ], "2026-06": [ "abs.cpi.all_groups.yoy", @@ -37,10 +37,10 @@ "eurostat.ea.hicp.flash.yoy" ], "2026_05": [ + "census.housing_starts.saar", "estat.jp.cpi.core_exfreshfood.yoy", - "ons.cpih.annual_rate", - "us.census.housing_starts.total_saar", - "us.frb.industrial_production.total.mom_sa" + "fed.g17.industrial_production.total_index_mom", + "ons.cpih.annual_rate" ], "2026_06": [ "bls.jolts.hires_rate", @@ -185,7 +185,7 @@ "us.dol.initial_claims.sa" ], "week_ending_2026_06_06": [ - "dol.eta.initial_claims.sa" + "us.dol.initial_claims.sa" ] }, "ambiguous_aliases": [], @@ -2105,7 +2105,8 @@ "uuid": "d0212a1e-b360-460a-80a8-1cf4ad5ce427", "concept": "census.housing_starts.saar", "family_patterns": [ - "census.housing_starts.saar.{P}" + "census.housing_starts.saar.{P}", + "us.census.housing_starts.total_saar.{P}" ], "status": "observed", "unit": "millions", @@ -2124,17 +2125,20 @@ "census" ], "aliases": [ - "census.housing_starts.saar.may_2026" + "census.housing_starts.saar.may_2026", + "us.census.housing_starts.total_saar", + "us.census.housing_starts.total_saar.2026_05" ], "source_concepts": [ "census.housing_starts.saar" ], "rid_patterns": [ - "census.housing_starts.saar.{P}.first_print" + "census.housing_starts.saar.{P}.first_print", + "us.census.housing_starts.total_saar.{P}" ], "first_observed_period": "2026-05", "last_observed_period": "2026-05", - "observation_count": 1 + "observation_count": 2 }, { "uuid": "8c999902-84e2-4e90-b49c-325cd535062a", @@ -2588,41 +2592,6 @@ "last_observed_period": "2026-07-18", "observation_count": 4 }, - { - "uuid": "330ec66b-502c-4ddf-9978-02199bab0055", - "concept": "dol.eta.initial_claims.sa", - "family_patterns": [ - "dol.eta.initial_claims.sa.{P}" - ], - "status": "observed", - "unit": "thousands", - "cadence": "month", - "geography": { - "level": "country", - "id": "0100000US", - "vintage": "current", - "name": "United States" - }, - "entity": { - "name": "person", - "role": "ui_initial_claimant" - }, - "sources": [ - "dol" - ], - "aliases": [ - "dol.eta.initial_claims.sa.week_ending_2026_06_06" - ], - "source_concepts": [ - "dol.eta.initial_claims.sa.week_ending_2026_06_06" - ], - "rid_patterns": [ - "dol.eta.initial_claims.sa.{P}" - ], - "first_observed_period": "2026-06", - "last_observed_period": "2026-06", - "observation_count": 1 - }, { "uuid": "32b335ea-b2dd-4fba-8321-e9751df19318", "concept": "ecb.deposit_facility_rate", @@ -3093,7 +3062,8 @@ "uuid": "d5634fc4-dbe0-4203-a5b7-728708b85a14", "concept": "fed.g17.industrial_production.total_index_mom", "family_patterns": [ - "fed.g17.industrial_production.total_index_mom.{P}" + "fed.g17.industrial_production.total_index_mom.{P}", + "us.frb.industrial_production.total.mom_sa.{P}" ], "status": "observed", "unit": "percent_growth", @@ -3112,17 +3082,20 @@ "fed" ], "aliases": [ - "fed.g17.industrial_production.total_index_mom.may_2026" + "fed.g17.industrial_production.total_index_mom.may_2026", + "us.frb.industrial_production.total.mom_sa", + "us.frb.industrial_production.total.mom_sa.2026_05" ], "source_concepts": [ "fed.g17.industrial_production.total_index_mom" ], "rid_patterns": [ - "fed.g17.industrial_production.total_index_mom.{P}.first_print" + "fed.g17.industrial_production.total_index_mom.{P}.first_print", + "us.frb.industrial_production.total.mom_sa.{P}" ], "first_observed_period": "2026-05", "last_observed_period": "2026-05", - "observation_count": 1 + "observation_count": 2 }, { "uuid": "41d26958-709c-44ab-a5ab-b385157db0ac", @@ -6269,41 +6242,6 @@ "last_observed_period": "2026-06", "observation_count": 2 }, - { - "uuid": "d542cb8d-46ef-4af5-b7e0-8d38574120c2", - "concept": "us.census.housing_starts.total_saar", - "family_patterns": [ - "us.census.housing_starts.total_saar.{P}" - ], - "status": "observed", - "unit": "millions", - "cadence": "month", - "geography": { - "level": "country", - "id": "0100000US", - "vintage": "current", - "name": "United States" - }, - "entity": { - "name": "dwelling", - "role": "housing_start" - }, - "sources": [ - "census" - ], - "aliases": [ - "us.census.housing_starts.total_saar.2026_05" - ], - "source_concepts": [ - "census.housing_starts.saar" - ], - "rid_patterns": [ - "us.census.housing_starts.total_saar.{P}" - ], - "first_observed_period": "2026-05", - "last_observed_period": "2026-05", - "observation_count": 1 - }, { "uuid": "1ad67747-d9d8-4ff9-9ebc-58206e658518", "concept": "us.dol.initial_claims.sa", @@ -6341,6 +6279,7 @@ "uuid": "830e68bb-9ed6-493c-a8a4-440ba8e53f76", "concept": "us.dol.initial_claims.sa", "family_patterns": [ + "dol.eta.initial_claims.sa.{P}", "us.dol.initial_claims.sa.{P}" ], "status": "observed", @@ -6360,17 +6299,21 @@ "dol" ], "aliases": [ + "dol.eta.initial_claims.sa", + "dol.eta.initial_claims.sa.week_ending_2026_06_06", "us.dol.initial_claims.sa.week_2026_06_13" ], "source_concepts": [ - "dol.eta.initial_claims.sa" + "dol.eta.initial_claims.sa", + "dol.eta.initial_claims.sa.week_ending_2026_06_06" ], "rid_patterns": [ + "dol.eta.initial_claims.sa.{P}", "us.dol.initial_claims.sa.{P}" ], "first_observed_period": "2026-06", "last_observed_period": "2026-06", - "observation_count": 1 + "observation_count": 2 }, { "uuid": "a4729a07-d9cd-4736-9473-2f845fd71a2d", @@ -6407,41 +6350,6 @@ "last_observed_period": "2026-06", "observation_count": 1 }, - { - "uuid": "b5a74252-bf95-488c-81f6-3deab496c477", - "concept": "us.frb.industrial_production.total.mom_sa", - "family_patterns": [ - "us.frb.industrial_production.total.mom_sa.{P}" - ], - "status": "observed", - "unit": "percent_growth", - "cadence": "month", - "geography": { - "level": "country", - "id": "0100000US", - "vintage": "current", - "name": "United States" - }, - "entity": { - "name": "institutional_sector", - "role": "total_industrial_production" - }, - "sources": [ - "fed" - ], - "aliases": [ - "us.frb.industrial_production.total.mom_sa.2026_05" - ], - "source_concepts": [ - "fed.g17.industrial_production.total_index_mom" - ], - "rid_patterns": [ - "us.frb.industrial_production.total.mom_sa.{P}" - ], - "first_observed_period": "2026-05", - "last_observed_period": "2026-05", - "observation_count": 1 - }, { "uuid": "e64ea5db-b72a-4a46-8948-6142dbc59d91", "concept": "usaspending.dhs.title_vi.award_transaction_obligations", diff --git a/ledger/series_uuid_registry.jsonl b/ledger/series_uuid_registry.jsonl index 10acb4c..d02ab70 100644 --- a/ledger/series_uuid_registry.jsonl +++ b/ledger/series_uuid_registry.jsonl @@ -211,3 +211,6 @@ {"concept": "irs.soi.credit_30d.total_claims", "geography": {"level": "country", "id": "0100000US", "vintage": null}, "entity": null, "uuid": "f5eacf8a-ce09-4721-a92a-9b19766a0d3c"} {"concept": "irs.soi.credit_30d.total_credit_amount", "geography": {"level": "country", "id": "0100000US", "vintage": null}, "entity": null, "uuid": "c70d0bc6-d299-4727-959a-3c7bdb324861"} {"concept": "usaspending.dhs.title_vi.award_transaction_obligations", "geography": {"level": "country", "id": "0100000US", "vintage": null}, "entity": null, "uuid": "e64ea5db-b72a-4a46-8948-6142dbc59d91"} +{"concept": "dol.eta.initial_claims.sa", "geography": {"level": "country", "id": "0100000US", "vintage": "current"}, "entity": {"name": "person", "role": "ui_initial_claimant"}, "uuid": "330ec66b-502c-4ddf-9978-02199bab0055", "retired": true, "note": "curated merge: concept-spelling duplicate lineages folded"} +{"concept": "us.census.housing_starts.total_saar", "geography": {"level": "country", "id": "0100000US", "vintage": "current"}, "entity": {"name": "dwelling", "role": "housing_start"}, "uuid": "d542cb8d-46ef-4af5-b7e0-8d38574120c2", "retired": true, "note": "curated merge: concept-spelling duplicate lineages folded"} +{"concept": "us.frb.industrial_production.total.mom_sa", "geography": {"level": "country", "id": "0100000US", "vintage": "current"}, "entity": {"name": "institutional_sector", "role": "total_industrial_production"}, "uuid": "b5a74252-bf95-488c-81f6-3deab496c477", "retired": true, "note": "curated merge: concept-spelling duplicate lineages folded"} diff --git a/tests/test_build_series_catalog.py b/tests/test_build_series_catalog.py index 3ce80fc..f0fb556 100644 --- a/tests/test_build_series_catalog.py +++ b/tests/test_build_series_catalog.py @@ -792,7 +792,7 @@ def test_committed_catalog_is_current_and_valid() -> None: assert committed["docket_seed_sha256"] is not None assert committed["uuid_registry_sha256"] == registry.sha256() assert bsc.DOCKET_SEED.exists() - assert len(committed["series"]) == 209 + assert len(committed["series"]) == 206 def test_rebuild_without_prior_catalog_is_gated( @@ -1134,9 +1134,9 @@ def test_identity_uuid_map_matches_reviewed_anchor() -> None: """The registry's introduction commit cannot be continuity-checked by the append-only gate (there is no prior registry to extend), so the identity->uuid map verified by the adversarial review of PR #128 is - pinned here. Changing ANY of the 201 bindings — or adding/retiring - one — must edit this constant in the same diff, making wholesale - remints impossible to slip through as regeneration noise. + pinned here. Changing any live binding — or adding or retiring one — + must edit this constant in the same diff, making wholesale remints + impossible to slip through as regeneration noise. Update the constant only alongside registry events that justify it. """ @@ -1164,11 +1164,14 @@ def test_identity_uuid_map_matches_reviewed_anchor() -> None: # Updated 2026-08-06 alongside one registry mint: the Thesis docket # registered census.spm.child_poverty_rate; its placeholder joins the # seed snapshot. All 205 prior bindings unchanged. + # Updated 2026-08-07: +6 docket-only mints for thesis#138 wave-1 + # admission (bea fixed-investment x2, usaspending dhs title-vi, irs soi + # 30D claims+amount, irs actc amount) — see that commit's seed diff. + # Updated 2026-08-07: three same-dimension concept-spelling duplicates + # retired and folded into surviving lineages; 209 live bindings became + # 206 without changing any surviving UUID. assert digest == ( - # 2026-08-07: +6 docket-only mints for thesis#138 wave-1 admission - # (bea fixed-investment x2, usaspending dhs title-vi, irs soi 30D - # claims+amount, irs actc amount) — see the seed diff in this commit. - "396e27996b2494c1defe94d834db4977c36e5b1ec179f71c198a0e20955cbd1e" + "c82492abf083f35f0b9427e7eb9b91a2dceba0ab71fa24bc24ef24a053526056" )