From 5f5d7437c56bb8843cc0719455bcc02016a86cc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebasti=C3=A1n=20Lucas?= <sebalucas@gmail.com>
Date: Fri, 7 Aug 2026 22:03:12 -0300
Subject: [PATCH] Detect and fix KOI8-R mojibake (fixes #231)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The example in #231 is reported as CP437/CP866, but it is KOI8-R: in that
encoding 0xD0 is 'п' and 0xD1 is 'я', the two lead bytes of Cyrillic in
UTF-8, and 0x95/0x81 are '∙'/'│'. No other Cyrillic codepage round-trips
it -- cp866, koi8-u, iso-8859-5 and mac-cyrillic all fail to encode it.

Two changes are needed, and neither one alone is enough:

- add koi8-r to CHARMAP_ENCODINGS, so the text can be re-encoded;
- add lower_common/upper_common to the rule that pairs a character with a
  box-drawing character, so that is_bad() fires. KOI8-R mojibake of
  Cyrillic puts a Cyrillic letter immediately before a box-drawing
  character, and no existing alternative covers that pair.

Adding koi8-r on its own leaves the example unchanged, because the badness
heuristic never flags it.
---
 ftfy/badness.py                   | 2 +-
 ftfy/chardata.py                  | 1 +
 tests/test-cases/in-the-wild.json | 7 +++++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/ftfy/badness.py b/ftfy/badness.py
index 38ec1f4..813ef89 100644
--- a/ftfy/badness.py
+++ b/ftfy/badness.py
@@ -301,7 +301,7 @@
     |
     [{box}] [{kaomoji}]
     |
-    [{lower_accented}{upper_accented}{currency}{numeric}{start_punctuation}{end_punctuation}{law}] [{box}]
+    [{lower_accented}{upper_accented}{lower_common}{upper_common}{currency}{numeric}{start_punctuation}{end_punctuation}{law}] [{box}]
     |
     [{box}] [{end_punctuation}]
     |
diff --git a/ftfy/chardata.py b/ftfy/chardata.py
index 43d117c..f5af37e 100644
--- a/ftfy/chardata.py
+++ b/ftfy/chardata.py
@@ -23,6 +23,7 @@
     "iso-8859-2",
     "macroman",
     "cp437",
+    "koi8-r",
 ]
 
 SINGLE_QUOTE_RE = re.compile("[\u02bc\u2018-\u201b]")
diff --git a/tests/test-cases/in-the-wild.json b/tests/test-cases/in-the-wild.json
index b40c838..c32b11e 100644
--- a/tests/test-cases/in-the-wild.json
+++ b/tests/test-cases/in-the-wild.json
@@ -447,5 +447,12 @@
         "fixed-encoding": "Ningbo，China",
         "fixed": "Ningbo,China",
         "expect": "pass"
+    },
+    {
+        "label": "Russian text in KOI8-R mojibake",
+        "comment": "issue #231. Reported as CP437/CP866, but it is KOI8-R: 0xD0 and 0xD1 decode to 'п' and 'я', which are the UTF-8 lead bytes of Cyrillic.",
+        "original": "п∙я│п╩п╦ п▓я▀ п╫п╣ я▐п╡п╩я▐п╣я┌п╣я│я▄ п╟п╢я─п╣я│п╟я┌п╬п╪",
+        "fixed": "Если Вы не являетесь адресатом",
+        "expect": "pass"
     }
 ]
\ No newline at end of file