Repository navigation
Expand file tree
/
Copy pathtext.py
More file actions
77 lines (64 loc) · 3.02 KB
/
Copy pathtext.py
File metadata and controls
77 lines (64 loc) · 3.02 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
"""`imperfection` for a corpus: what share of texts carries which defect, with examples."""
import re
import unicodedata
from collections import Counter
import pandas as pd
_URL = re.compile(r"(?:https?://|www\.)\S+", re.I)
_HTML = re.compile(r"</?[a-zA-Z][^>]{0,80}>")
_BASE64 = re.compile(r"[A-Za-z0-9+/]{40,}={0,2}")
_REPEAT_CHAR = re.compile(r"(.)\1{4,}", re.S)
_REPEAT_WORD = re.compile(r"\b(\w+)(?:\W+\1\b){2,}", re.I)
_CONTROL = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
_WORD = re.compile(r"\w+")
_CYR = re.compile("[\u0400-\u04ff]")
_LAT = re.compile("[A-Za-z]")
def _letters_frac(s: str) -> float:
return sum(ch.isalpha() for ch in s) / len(s) if s else 0.0
def _mixed_script_word(s: str) -> bool:
for w in _WORD.findall(s):
if _CYR.search(w) and _LAT.search(w):
return True
return False
CHECKS = {
"empty": lambda s: not s.strip(),
"short": None, # filled in from min_chars
"control chars": lambda s: bool(_CONTROL.search(s)),
"not NFKC": lambda s: unicodedata.normalize("NFKC", s) != s,
"url": lambda s: bool(_URL.search(s)),
"html tag": lambda s: bool(_HTML.search(s)),
"base64-like": lambda s: bool(_BASE64.search(s)),
"repeated char": lambda s: bool(_REPEAT_CHAR.search(s)),
"repeated word": lambda s: bool(_REPEAT_WORD.search(s)),
"mixed script": _mixed_script_word,
"few letters": None, # filled in from min_letters
"duplicate": None, # needs the whole corpus
}
def text_imperfection(texts, min_chars: int = 20, min_letters: float = 0.6,
examples: int = 3, no_clean: bool = True) -> pd.DataFrame:
'''
One row per defect: share of texts, count, up to `examples` offending texts (cut to 60 chars).
Defects: empty, shorter than `min_chars`, C0 control characters, not NFKC-normalized, URL,
HTML tag, base64-like run, a character repeated 5+ times, a word repeated 3+ times in a row,
Cyrillic and Latin letters inside one word, letters below `min_letters` of all characters,
exact duplicate (after strip). `no_clean=False` keeps rows with zero hits too.
Non-string items count as "empty". Lengths are not a defect: `pd.Series(map(len, texts)).describe()`.
'''
items = [t if isinstance(t, str) else "" for t in texts]
n = len(items)
checks = dict(CHECKS)
checks["short"] = lambda s: 0 < len(s.strip()) < min_chars
checks["few letters"] = lambda s: bool(s.strip()) and _letters_frac(s) < min_letters
seen = Counter(s.strip() for s in items)
checks["duplicate"] = lambda s: seen[s.strip()] > 1 and bool(s.strip())
rows = []
for name, check in checks.items():
hits = [s for s in items if check(s)]
rows.append({
"Share": len(hits) / n if n else 0.0,
"Count": len(hits),
"Examples": [h[:60] for h in hits[:examples]],
})
table = pd.DataFrame(rows, index=list(checks))
if no_clean:
table = table[table["Count"] > 0]
return table.sort_values("Share", ascending=False)