A blank or junk country is not found, and the site tool's prefill finds every country the geocoder names

SWEET_python/city_params.py

@@ -1,5 +1,7 @@
 import os
+import re
 import sys
+import unicodedata
 from pathlib import Path
 
 sys.path.append("/app/SWEET_python/SWEET_python")
@@ -241,6 +243,31 @@ def _city_growth(iso3, anchor_year, population, city_historic, city_future):
 _ISO3_BY_NAME = {name.lower(): iso3 for name, iso3 in defaults_2019.country_to_iso3.items()}
 
 
+def _fold(text):
+    """``text`` lower-cased and without accents, as pycountry's fuzzy search compares names."""
+    return "".join(
+        ch for ch in unicodedata.normalize("NFKD", text.lower()) if not unicodedata.combining(ch)
+    )
+
+
+def _is_named(country, name):
+    """Whether ``name`` is whole words of one of ``country``'s own names, or their initials.
+
+    Initials count from two letters ("UK", "UAE"), because a one-word name's are
+    just its first letter.
+    """
+    wanted = _fold(name)
+    for own in (country._fields.get(field) for field in ("name", "official_name", "common_name")):
+        if not own:
+            continue
+        initials = "".join(ch for ch in own if ch.isupper())
+        if len(initials) > 1 and wanted == _fold(initials):
+            return True
+        if re.search(rf"(?<!\w){re.escape(wanted)}(?!\w)", _fold(own)):
+            return True
+    return False
+
+
 def _iso3_for(country):
     """The ISO3 code for a country code or name. Raises LookupError if there is none.
 
@@ -252,16 +279,53 @@ def _iso3_for(country):
     Then pycountry, an exact match first, because the fuzzy search ranks
     subdivisions: it resolves "MUS" to Turkey, whose province Mus matches, rather
     than Mauritius.
+
+    The fuzzy search's answer counts only when the name is the country's own (see
+    _is_named). Its ranking scores subdivisions too, so it answered with whichever
+    country had a province that matched: "None" was Italy (Frosinone), "nan"
+    Thailand (Nan province), "Sint Maarten" the Netherlands rather than SXM. A
+    province's name is now not found even where the answer was right ("England").
+
+    No country is not found: None, anything that isn't a string, and a blank
+    string raise. The fuzzy search scored every country for an empty name and
+    answered with the one that has the most subdivisions, the United Kingdom.
     """
-    if isinstance(country, str):
-        if country.upper() in defaults_2019.region_lookup_iso3:
-            return country.upper()
-        if country.lower() in _ISO3_BY_NAME:
-            return _ISO3_BY_NAME[country.lower()]
+    if not isinstance(country, str) or not country.strip():
+        raise LookupError(f"No country given: {country!r}")
+    country = country.strip()
+    if country.upper() in defaults_2019.region_lookup_iso3:
+        return country.upper()
+    if country.lower() in _ISO3_BY_NAME:
+        return _ISO3_BY_NAME[country.lower()]
     try:
         return pycountry.countries.lookup(country).alpha_3
     except LookupError:
-        return pycountry.countries.search_fuzzy(country)[0].alpha_3
+        pass
+    for match in pycountry.countries.search_fuzzy(country):
+        if _is_named(match, country):
+            return match.alpha_3
+    raise LookupError(f"No country is named {country!r}")
+
+
+def _iso3_at(address):
+    """The ISO3 code for a geocoded place: Nominatim's ``address``. Raises LookupError.
+
+    By the country's English name first, because SWEET has its own codes for
+    Aruba, Curaçao and Sint Maarten, which Nominatim codes as the Netherlands. Then
+    by Nominatim's ISO 3166-1 code, for the names neither SWEET nor pycountry spell
+    Nominatim's way: Turkey, Ivory Coast, Cape Verde, East Timor, Democratic Republic
+    of the Congo, Congo-Brazzaville, Palestinian Territories, Sahrawi Arab Democratic
+    Republic, and the breakaway regions it names as countries (Abkhazia, South
+    Ossetia, Northern Cyprus, Somaliland), which it codes as the country they're in.
+    """
+    try:
+        return _iso3_for(address.get("country"))
+    except LookupError:
+        code = address.get("country_code")
+        match = pycountry.countries.get(alpha_2=code.upper()) if isinstance(code, str) else None
+        if match is None:
+            raise
+        return match.alpha_3
 
 
 def _country_growth(iso3, anchor_year, fallback_historic, fallback_future):
@@ -9082,10 +9146,15 @@ async def sdst_prepopulate(
                 raise ValueError(f"Region for ISO3 code '{iso3}' not found.")
         else:
             location = geolocator.reverse((latlon[0], latlon[1]), language="en")
-            country = location.raw["address"].get("country")
+            # geopy gives None where Nominatim can't place the point (open sea), and
+            # an address can lack a country (Antarctica).
+            address = location.raw.get("address", {}) if location is not None else {}
+            country = address.get("country")
             try:
-                iso3 = _iso3_for(country)
+                iso3 = _iso3_at(address)
             except LookupError:
+                if country is None:
+                    raise ValueError(f"No country at {latlon[0]}, {latlon[1]}.")
                 raise ValueError(f"Country '{country}' not found.")
             region = defaults_2019.region_lookup_iso3.get(iso3)
             if region is None:

changelog/2026-10.md

@@ -4,10 +4,61 @@
 raised "Country 'XKX' not found." for Kosovo's code, so WasteMAP's city tool
 answered 500, and the site tool modelled a point in Kosovo as Serbia. SWEET now
 looks a country up in its own table before asking pycountry, which has no Kosovo
-(model-output change for a country given by name).
+(model-output change for a country given by name). A blank or junk country is not
+found rather than modelled as the UK or Italy, and the site tool's prefill finds the
+country for a point in Turkey, both Congos and eight more places it used to fail on.
 
 ## Fixed
 
+- **A blank or junk country is not found, instead of being modelled as another
+  country; the site tool's prefill finds every country the geocoder names**
+  (model-output change for the prefill in Sint Maarten). `_iso3_for` ends with
+  pycountry's fuzzy search, which scores every country for an empty name and
+  scores a country for any of its subdivisions that match:
+  - **No country was the United Kingdom.** `""` and `" "` resolved to GBR, the
+    country with the most subdivisions, so a Custom Location or site sent
+    `country=` was modelled as the UK (30,434 t CH4 in 2040 for 2 million
+    people). `None` raised `AttributeError`, not `LookupError`, so the callers
+    answered with a crash rather than "not found".
+  - **A province's name was its country.** "None" was Italy (Frosinone), "nan"
+    Thailand (Nan province), "-" France, "0" Sweden, "Sint Maarten" the
+    Netherlands rather than SXM.
+  - **The prefill crashed for a point in no country.** Nominatim can't place a
+    point at sea, so geopy returns None, and its address for Antarctica has no
+    country. Both raised `AttributeError`.
+  - **The prefill failed for 11 names Nominatim uses.** Measured over 306
+    reverse-geocoded points (one per country and territory): "Turkey", "Ivory
+    Coast", "Cape Verde", "East Timor", "Democratic Republic of the Congo",
+    "Congo-Brazzaville", "Palestinian Territories", "Sahrawi Arab Democratic
+    Republic", "Northern Cyprus", "Somaliland" and "South Ossetia" are spelled
+    neither SWEET's nor pycountry's way, so each raised "Country '...' not
+    found."
+
+  `_iso3_for` now raises `LookupError` for anything that isn't a non-blank
+  string, strips the name, and takes the fuzzy search's answer only when the
+  name is whole words of that country's own name, or its initials ("UK",
+  "UAE"). A province's name is no longer found even where the answer was right
+  ("England"). The prefill (`_iso3_at`) tries the geocoder's English name, then
+  its ISO 3166-1 code, and says "No country at <lat>, <lon>." where there is
+  none. The name comes first because SWEET has its own codes for Aruba, Curaçao
+  and Sint Maarten, which Nominatim codes as the Netherlands.
+
+  Effect:
+  - **Over the 222 country names Nominatim gave**, 210 resolve as before (the
+    breakaway regions it names as countries resolve through its code to the
+    country they're in: Abkhazia and South Ossetia to Georgia, Northern Cyprus
+    to Cyprus, Somaliland to Somalia). The 11 above now resolve. Sint Maarten
+    moves from NLD to SXM: an open dump rather than a landfill, food 48.2%
+    rather than 35.0%, growth 0.63% a year rather than 0.15%.
+  - **Every code and name in SWEET's own table resolves as before**, so the
+    city tool's Custom Location, which sends a code, is bit-identical for every
+    country.
+  - **Climate TRACE's pipeline is unaffected.** None of its paths calls
+    `_iso3_for`.
+
+  Tests: `tests/test_country_lookup.py`.
+  ([#80](https://github.com/RMI/SWEET_python/pull/80),
+  [RMI/WasteMAP#859](https://github.com/RMI/WasteMAP/pull/859))
 - **A Custom Location or site in Kosovo is modelled as Kosovo** (model-output
   change for a country given by name). `_iso3_for`, which `dst_baseline_blank`,
   `cityparams_obj_for_blank_site` and `sdst_prepopulate` use to turn the caller's

changelog/README.md

@@ -15,7 +15,7 @@ The project does not publish semantic version tags, so releases are tracked by
 
 Newest first:
 
-- [2026-10](2026-10.md) — a Custom Location in Kosovo builds: SWEET looks a country up in its own table before asking pycountry, which has no Kosovo, so Kosovo's code XKX no longer raises "Country 'XKX' not found." (WasteMAP answered 500), and the name "Kosovo" no longer resolves to Serbia, which the site tool's prefill for a point in Kosovo returned (model-output change for a country given by name)
+- [2026-10](2026-10.md) — a blank or junk country is not found instead of being modelled as another country (`""` was the UK, `"None"` Italy), and the site tool's prefill finds the country for a point in Turkey, both Congos and eight more places, and says so for a point in none, where it crashed (model-output change for a point in Sint Maarten); a Custom Location in Kosovo builds: SWEET looks a country up in its own table before asking pycountry, which has no Kosovo, so Kosovo's code XKX no longer raises "Country 'XKX' not found." (WasteMAP answered 500), and the name "Kosovo" no longer resolves to Serbia, which the site tool's prefill for a point in Kosovo returned (model-output change for a country given by name)
 - [2026-09](2026-09.md) — WasteMAP's site tool, city tool and Custom Location grow waste with the country's UN population year by year, as Climate TRACE does; big cities keep their own UN city rates (model-output change); model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart
 - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change)
 - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change)

tests/test_country_lookup.py

@@ -45,9 +45,12 @@ def test_a_custom_site_in_kosovo_builds():
     assert city.baseline_parameters.population_series.equals(KOSOVO)
 
 
-def test_the_site_tool_prefills_a_point_in_kosovo_as_kosovo(monkeypatch):
-    """The prefill names the country with the geocoder's English name, "Kosovo"."""
-    place = SimpleNamespace(raw={"address": {"country": "Kosovo"}})
+def _prefill(monkeypatch, place, latlon):
+    """The site tool's prefill for ``latlon``, where the geocoder answers ``place``.
+
+    ``place`` is what geopy's ``reverse`` returns: an object with Nominatim's JSON as
+    ``raw``, or None where Nominatim can't place the point.
+    """
     monkeypatch.setattr(
         city_params, "create_geolocator", lambda: SimpleNamespace(reverse=lambda *a, **k: place)
     )
@@ -62,14 +65,21 @@ async def no_async_database(*args, **kwargs):
     monkeypatch.setattr(city_params.socket, "create_connection", no_database)
     monkeypatch.setattr(city_params.asyncpg, "connect", no_async_database)
 
-    prefill = asyncio.run(
+    return asyncio.run(
         City("latlon_lookup").sdst_prepopulate(
             "db", 5432, "user", "password", "wastemap", "disable",
             sites_list=pd.DataFrame(),
-            latlon=(42.66, 21.17),  # Pristina
+            latlon=latlon,
         )
     )
 
+
+def test_the_site_tool_prefills_a_point_in_kosovo_as_kosovo(monkeypatch):
+    """The prefill names the country with the geocoder's English name, "Kosovo"."""
+    place = SimpleNamespace(raw={"address": {"country": "Kosovo"}})
+
+    prefill = _prefill(monkeypatch, place, (42.66, 21.17))  # Pristina
+
     assert prefill["iso3"] == "XKX"
 
 
@@ -117,3 +127,120 @@ def test_a_country_that_does_not_exist_is_still_not_found():
     assert _resolve("Narnia") is None
     with pytest.raises(ValueError, match="Country 'ZZZ' not found"):
         City("*").dst_baseline_blank("ZZZ", 2_000_000, 500.0, 25.0)
+
+
+# --- no country, and a name that isn't a country's ----------------------------
+#
+# pycountry's fuzzy search scores every country for an empty name, and scores a
+# country for any of its subdivisions that match. So "" was the United Kingdom
+# (the most subdivisions), "None" Italy (Frosinone) and "nan" Thailand (Nan
+# province), and the caller got that country's model without a word.
+
+
+@pytest.mark.parametrize("given", ["", " ", "\t\n", None, float("nan"), 0, b"GBR"])
+def test_no_country_is_not_found(given):
+    with pytest.raises(LookupError):
+        city_params._iso3_for(given)
+
+
+@pytest.mark.parametrize(
+    "given",
+    [
+        "None",  # was Italy: its province Frosinone contains "none"
+        "nan",  # was Thailand: Nan province
+        "-",  # was France, whose subdivisions are full of hyphens
+        "0",  # was Sweden
+        "England",  # was right, but a province's name is not a country's
+        "Kurdistan",  # was Iraq
+    ],
+)
+def test_a_name_only_a_subdivision_matches_is_not_a_country(given):
+    with pytest.raises(LookupError):
+        city_params._iso3_for(given)
+
+
+@pytest.mark.parametrize(
+    "given, iso3",
+    [
+        (" Kosovo ", "XKX"),  # was Serbia: SWEET's table missed the padded name
+        ("GBR\n", "GBR"),
+        ("Brunei", "BRN"),  # whole words of "Brunei Darussalam"
+        ("Vatican", "VAT"),  # of "Holy See (Vatican City State)"
+        ("Sint Maarten", "SXM"),  # was the Netherlands, through its subdivision Sint Maarten
+        ("UK", "GBR"),  # initials of "United Kingdom"
+        ("UAE", "ARE"),
+        ("826", "GBR"),  # ISO 3166-1 numeric, an exact match
+    ],
+)
+def test_a_country_is_still_found_by_its_own_name(given, iso3):
+    assert city_params._iso3_for(given) == iso3
+
+
+def test_a_custom_location_with_no_country_is_not_the_united_kingdom():
+    with pytest.raises(ValueError, match="Country '' not found"):
+        City("*").dst_baseline_blank("", 2_000_000, 500.0, 25.0)
+
+
+def test_a_custom_site_with_no_country_is_not_the_united_kingdom():
+    with pytest.raises(ValueError, match="Country ' ' not found"):
+        City("custom site").cityparams_obj_for_blank_site(
+            country=" ",
+            population=None,
+            precipitation=500.0,
+            temperature=10.0,
+            waste_fractions=Variant(baseline=[0.1] * 10, scenario=[0.1] * 10),
+            waste_mass_year=Variant[int](baseline=2025, scenario=2025),
+        )
+
+
+# --- the site tool's prefill ---------------------------------------------------
+
+
+def test_the_prefill_for_a_point_at_sea_is_not_found(monkeypatch):
+    """Nominatim can't place it, so geopy returns None. That raised AttributeError."""
+    with pytest.raises(ValueError, match="No country at 0.0, 0.0"):
+        _prefill(monkeypatch, None, (0.0, 0.0))
+
+
+def test_the_prefill_for_a_point_in_no_country_is_not_found(monkeypatch):
+    """Nominatim places McMurdo Station in Antarctica with no country. That raised AttributeError."""
+    place = SimpleNamespace(raw={"address": {"road": "Scott Base Road"}})
+
+    with pytest.raises(ValueError, match="No country at -77.85, 166.68"):
+        _prefill(monkeypatch, place, (-77.85, 166.68))
+
+
+@pytest.mark.parametrize(
+    "country, code, iso3",
+    [
+        # Nominatim's English names, which SWEET's table and pycountry spell otherwise.
+        # Each raised "Country '...' not found.", so the prefill failed.
+        ("Turkey", "tr", "TUR"),
+        ("Ivory Coast", "ci", "CIV"),
+        ("Cape Verde", "cv", "CPV"),
+        ("Democratic Republic of the Congo", "cd", "COD"),
+        ("Congo-Brazzaville", "cg", "COG"),
+        ("East Timor", "tl", "TLS"),
+        ("Palestinian Territories", "ps", "PSE"),
+        ("Sahrawi Arab Democratic Republic", "eh", "ESH"),
+        ("Northern Cyprus", "cy", "CYP"),
+        ("Somaliland", "so", "SOM"),
+        ("South Ossetia", "ge", "GEO"),
+        ("Abkhazia", "ge", "GEO"),  # was Georgia through its subdivision; now through the code
+        # The name wins where SWEET has the country's own code and Nominatim the Netherlands'.
+        ("Aruba", "nl", "ABW"),
+        ("Curaçao", "nl", "CUW"),
+        ("Sint Maarten", "nl", "SXM"),
+    ],
+)
+def test_the_prefill_falls_back_to_the_geocoders_country_code(monkeypatch, country, code, iso3):
+    place = SimpleNamespace(raw={"address": {"country": country, "country_code": code}})
+
+    assert _prefill(monkeypatch, place, (0.0, 0.0))["iso3"] == iso3
+
+
+def test_the_prefill_names_a_country_it_cannot_find(monkeypatch):
+    place = SimpleNamespace(raw={"address": {"country": "Narnia"}})
+
+    with pytest.raises(ValueError, match="Country 'Narnia' not found"):
+        _prefill(monkeypatch, place, (0.0, 0.0))