diff --git a/SWEET_python/city_params.py b/SWEET_python/city_params.py index 4845bbe..5694a7d 100644 --- a/SWEET_python/city_params.py +++ b/SWEET_python/city_params.py @@ -1,5 +1,7 @@ import os +import re import sys +import unicodedata from pathlib import Path sys.path.append("/app/SWEET_python/SWEET_python") @@ -241,6 +243,31 @@ def _city_growth(iso3, anchor_year, population, city_historic, city_future): _ISO3_BY_NAME = {name.lower(): iso3 for name, iso3 in defaults_2019.country_to_iso3.items()} +def _fold(text): + """``text`` lower-cased and without accents, as pycountry's fuzzy search compares names.""" + return "".join( + ch for ch in unicodedata.normalize("NFKD", text.lower()) if not unicodedata.combining(ch) + ) + + +def _is_named(country, name): + """Whether ``name`` is whole words of one of ``country``'s own names, or their initials. + + Initials count from two letters ("UK", "UAE"), because a one-word name's are + just its first letter. + """ + wanted = _fold(name) + for own in (country._fields.get(field) for field in ("name", "official_name", "common_name")): + if not own: + continue + initials = "".join(ch for ch in own if ch.isupper()) + if len(initials) > 1 and wanted == _fold(initials): + return True + if re.search(rf"(?, ." where there is + none. The name comes first because SWEET has its own codes for Aruba, Curaçao + and Sint Maarten, which Nominatim codes as the Netherlands. + + Effect: + - **Over the 222 country names Nominatim gave**, 210 resolve as before (the + breakaway regions it names as countries resolve through its code to the + country they're in: Abkhazia and South Ossetia to Georgia, Northern Cyprus + to Cyprus, Somaliland to Somalia). The 11 above now resolve. Sint Maarten + moves from NLD to SXM: an open dump rather than a landfill, food 48.2% + rather than 35.0%, growth 0.63% a year rather than 0.15%. + - **Every code and name in SWEET's own table resolves as before**, so the + city tool's Custom Location, which sends a code, is bit-identical for every + country. + - **Climate TRACE's pipeline is unaffected.** None of its paths calls + `_iso3_for`. + + Tests: `tests/test_country_lookup.py`. + ([#80](https://github.com/RMI/SWEET_python/pull/80), + [RMI/WasteMAP#859](https://github.com/RMI/WasteMAP/pull/859)) - **A Custom Location or site in Kosovo is modelled as Kosovo** (model-output change for a country given by name). `_iso3_for`, which `dst_baseline_blank`, `cityparams_obj_for_blank_site` and `sdst_prepopulate` use to turn the caller's diff --git a/changelog/README.md b/changelog/README.md index 3ec020c..76140da 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by Newest first: -- [2026-10](2026-10.md) — a Custom Location in Kosovo builds: SWEET looks a country up in its own table before asking pycountry, which has no Kosovo, so Kosovo's code XKX no longer raises "Country 'XKX' not found." (WasteMAP answered 500), and the name "Kosovo" no longer resolves to Serbia, which the site tool's prefill for a point in Kosovo returned (model-output change for a country given by name) +- [2026-10](2026-10.md) — a blank or junk country is not found instead of being modelled as another country (`""` was the UK, `"None"` Italy), and the site tool's prefill finds the country for a point in Turkey, both Congos and eight more places, and says so for a point in none, where it crashed (model-output change for a point in Sint Maarten); a Custom Location in Kosovo builds: SWEET looks a country up in its own table before asking pycountry, which has no Kosovo, so Kosovo's code XKX no longer raises "Country 'XKX' not found." (WasteMAP answered 500), and the name "Kosovo" no longer resolves to Serbia, which the site tool's prefill for a point in Kosovo returned (model-output change for a country given by name) - [2026-09](2026-09.md) — WasteMAP's site tool, city tool and Custom Location grow waste with the country's UN population year by year, as Climate TRACE does; big cities keep their own UN city rates (model-output change); model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) diff --git a/tests/test_country_lookup.py b/tests/test_country_lookup.py index 11e12e0..33d4ae9 100644 --- a/tests/test_country_lookup.py +++ b/tests/test_country_lookup.py @@ -45,9 +45,12 @@ def test_a_custom_site_in_kosovo_builds(): assert city.baseline_parameters.population_series.equals(KOSOVO) -def test_the_site_tool_prefills_a_point_in_kosovo_as_kosovo(monkeypatch): - """The prefill names the country with the geocoder's English name, "Kosovo".""" - place = SimpleNamespace(raw={"address": {"country": "Kosovo"}}) +def _prefill(monkeypatch, place, latlon): + """The site tool's prefill for ``latlon``, where the geocoder answers ``place``. + + ``place`` is what geopy's ``reverse`` returns: an object with Nominatim's JSON as + ``raw``, or None where Nominatim can't place the point. + """ monkeypatch.setattr( city_params, "create_geolocator", lambda: SimpleNamespace(reverse=lambda *a, **k: place) ) @@ -62,14 +65,21 @@ async def no_async_database(*args, **kwargs): monkeypatch.setattr(city_params.socket, "create_connection", no_database) monkeypatch.setattr(city_params.asyncpg, "connect", no_async_database) - prefill = asyncio.run( + return asyncio.run( City("latlon_lookup").sdst_prepopulate( "db", 5432, "user", "password", "wastemap", "disable", sites_list=pd.DataFrame(), - latlon=(42.66, 21.17), # Pristina + latlon=latlon, ) ) + +def test_the_site_tool_prefills_a_point_in_kosovo_as_kosovo(monkeypatch): + """The prefill names the country with the geocoder's English name, "Kosovo".""" + place = SimpleNamespace(raw={"address": {"country": "Kosovo"}}) + + prefill = _prefill(monkeypatch, place, (42.66, 21.17)) # Pristina + assert prefill["iso3"] == "XKX" @@ -117,3 +127,120 @@ def test_a_country_that_does_not_exist_is_still_not_found(): assert _resolve("Narnia") is None with pytest.raises(ValueError, match="Country 'ZZZ' not found"): City("*").dst_baseline_blank("ZZZ", 2_000_000, 500.0, 25.0) + + +# --- no country, and a name that isn't a country's ---------------------------- +# +# pycountry's fuzzy search scores every country for an empty name, and scores a +# country for any of its subdivisions that match. So "" was the United Kingdom +# (the most subdivisions), "None" Italy (Frosinone) and "nan" Thailand (Nan +# province), and the caller got that country's model without a word. + + +@pytest.mark.parametrize("given", ["", " ", "\t\n", None, float("nan"), 0, b"GBR"]) +def test_no_country_is_not_found(given): + with pytest.raises(LookupError): + city_params._iso3_for(given) + + +@pytest.mark.parametrize( + "given", + [ + "None", # was Italy: its province Frosinone contains "none" + "nan", # was Thailand: Nan province + "-", # was France, whose subdivisions are full of hyphens + "0", # was Sweden + "England", # was right, but a province's name is not a country's + "Kurdistan", # was Iraq + ], +) +def test_a_name_only_a_subdivision_matches_is_not_a_country(given): + with pytest.raises(LookupError): + city_params._iso3_for(given) + + +@pytest.mark.parametrize( + "given, iso3", + [ + (" Kosovo ", "XKX"), # was Serbia: SWEET's table missed the padded name + ("GBR\n", "GBR"), + ("Brunei", "BRN"), # whole words of "Brunei Darussalam" + ("Vatican", "VAT"), # of "Holy See (Vatican City State)" + ("Sint Maarten", "SXM"), # was the Netherlands, through its subdivision Sint Maarten + ("UK", "GBR"), # initials of "United Kingdom" + ("UAE", "ARE"), + ("826", "GBR"), # ISO 3166-1 numeric, an exact match + ], +) +def test_a_country_is_still_found_by_its_own_name(given, iso3): + assert city_params._iso3_for(given) == iso3 + + +def test_a_custom_location_with_no_country_is_not_the_united_kingdom(): + with pytest.raises(ValueError, match="Country '' not found"): + City("*").dst_baseline_blank("", 2_000_000, 500.0, 25.0) + + +def test_a_custom_site_with_no_country_is_not_the_united_kingdom(): + with pytest.raises(ValueError, match="Country ' ' not found"): + City("custom site").cityparams_obj_for_blank_site( + country=" ", + population=None, + precipitation=500.0, + temperature=10.0, + waste_fractions=Variant(baseline=[0.1] * 10, scenario=[0.1] * 10), + waste_mass_year=Variant[int](baseline=2025, scenario=2025), + ) + + +# --- the site tool's prefill --------------------------------------------------- + + +def test_the_prefill_for_a_point_at_sea_is_not_found(monkeypatch): + """Nominatim can't place it, so geopy returns None. That raised AttributeError.""" + with pytest.raises(ValueError, match="No country at 0.0, 0.0"): + _prefill(monkeypatch, None, (0.0, 0.0)) + + +def test_the_prefill_for_a_point_in_no_country_is_not_found(monkeypatch): + """Nominatim places McMurdo Station in Antarctica with no country. That raised AttributeError.""" + place = SimpleNamespace(raw={"address": {"road": "Scott Base Road"}}) + + with pytest.raises(ValueError, match="No country at -77.85, 166.68"): + _prefill(monkeypatch, place, (-77.85, 166.68)) + + +@pytest.mark.parametrize( + "country, code, iso3", + [ + # Nominatim's English names, which SWEET's table and pycountry spell otherwise. + # Each raised "Country '...' not found.", so the prefill failed. + ("Turkey", "tr", "TUR"), + ("Ivory Coast", "ci", "CIV"), + ("Cape Verde", "cv", "CPV"), + ("Democratic Republic of the Congo", "cd", "COD"), + ("Congo-Brazzaville", "cg", "COG"), + ("East Timor", "tl", "TLS"), + ("Palestinian Territories", "ps", "PSE"), + ("Sahrawi Arab Democratic Republic", "eh", "ESH"), + ("Northern Cyprus", "cy", "CYP"), + ("Somaliland", "so", "SOM"), + ("South Ossetia", "ge", "GEO"), + ("Abkhazia", "ge", "GEO"), # was Georgia through its subdivision; now through the code + # The name wins where SWEET has the country's own code and Nominatim the Netherlands'. + ("Aruba", "nl", "ABW"), + ("Curaçao", "nl", "CUW"), + ("Sint Maarten", "nl", "SXM"), + ], +) +def test_the_prefill_falls_back_to_the_geocoders_country_code(monkeypatch, country, code, iso3): + place = SimpleNamespace(raw={"address": {"country": country, "country_code": code}}) + + assert _prefill(monkeypatch, place, (0.0, 0.0))["iso3"] == iso3 + + +def test_the_prefill_names_a_country_it_cannot_find(monkeypatch): + place = SimpleNamespace(raw={"address": {"country": "Narnia"}}) + + with pytest.raises(ValueError, match="Country 'Narnia' not found"): + _prefill(monkeypatch, place, (0.0, 0.0))