Repository navigation
Expand file tree
/
Copy pathoutput.py
More file actions
230 lines (198 loc) · 8.02 KB
/
Copy pathoutput.py
File metadata and controls
230 lines (198 loc) · 8.02 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import itertools as it
from math import factorial
try:
from .metrics import harmonic_from_confusion
except ImportError: # zipped drop-in: modules side by side
from metrics import harmonic_from_confusion
# # # # # # # # # # # # # # # #
# # # # Printing functions # # # #
# # # # # # # # # # # # # # # #
def informer(df:pd.DataFrame,target:str,max_classes:int=20) -> dict:
'''
Initial look at a train+test frame (test rows have NaN target):
observations, features, and either class shares or - when the target has more than
`max_classes` distinct values (regression) - its `describe()` under "target".
'''
df_train = df[~df[target].isna()]
tr_n, tr_f = df_train.shape
te_n, _ = df[df[target].isna()].shape
y = df_train[target]
is_regression = pd.api.types.is_numeric_dtype(y) and y.nunique() > max_classes
classes = None if is_regression else y.value_counts()
return {
"classes": classes,
"n_cls": 0 if classes is None else len(classes),
"target": y.describe() if is_regression else None,
"features": tr_f,
"train_obs": tr_n,
"test_obs": te_n,
}
def informer_print(info:dict) -> None:
total = info['train_obs'] + info['test_obs']
print(f"\n\tFeatures:\t{info['features']}")
print(f"\tObservations:\t{info['train_obs']}/{info['test_obs']}")
print(f"\ttrain dataset:\t{info['train_obs']/total:.1%}")
if info["classes"] is not None:
txt_classes = ", ".join([f"{cls}: {n/(info['train_obs']):.1%}"
for cls, n in info["classes"].items()])
print(f"\tclasses:\t{txt_classes}")
else:
t = info["target"]
print(f"\ttarget:\t\tmin {t['min']:.3g}, median {t['50%']:.3g}, mean {t['mean']:.3g}, max {t['max']:.3g}")
def imperfection(df:pd.DataFrame,numeric:bool=False,no_nan:bool=False) -> pd.DataFrame:
'''
Quick evaluation of missing values, types and variety of values in table columns
'''
def form_series(series:pd.Series,name:str) -> pd.Series:
series.name = name
return series
missing_df = form_series(df.isnull().sum() / len(df),"NA Share")
# reduce: equal-length uniques (e.g. a single column) would otherwise expand into a frame
unique_df = form_series(df.apply(pd.unique, result_type="reduce"),"Values")
nunique_df = form_series(unique_df.apply(len),"Num. of unique")
type_df = form_series(df.dtypes,"Type")
full_df = pd.DataFrame([missing_df,nunique_df,type_df,unique_df]).T
if not numeric:
full_df = full_df.where(
(full_df['Type'] != np.dtype('int64'))\
& (full_df['Type'] != np.dtype('float64')))\
.dropna()
if no_nan:
full_df = full_df.where(full_df['NA Share'] > 0).dropna()
return full_df.sort_values(by=['NA Share','Num. of unique'], ascending=False)
def multicolumn(series,cols:int=5) -> pd.DataFrame:
'''
Work with pd.Series and slice of pd.DataFrame:
Function for viewing long data series,
useful with df.sample().T
'''
try:
title = (series.columns.name, series.columns[0])
except AttributeError:
title = (series.dtype, series.name)
n = len(series)
rows= int(n//cols)
if rows*cols < n:
rows += 1
col_sep = pd.Series(["│"] * rows, name="│")
portions = []
for i in range(cols):
portion = series.iloc[rows*i:rows*(i+1)]
if len(portion.index) == 0:
break
portion = portion.reset_index()
portion.columns = [f"col_{i+1}",f"val_{i+1}"]
portions.append(portion)
portions.append(col_sep)
table = pd.concat(portions,axis=1)
return table.where(table.notna(),"").style.set_caption("{}: {}".format(*title))
# # # # # # # # # # # # # # # #
# # # # Plotting functions # # # #
# # # # # # # # # # # # # # # #
# For a visual assessment of the quality of the model, we will use the confusion matrix
yvg = list([(abs(np.cos(x-1)),abs(np.sin(x+1)),abs(np.sin(x*x))) for x in np.arange(-2,0,0.1)])
pal_yvg = sns.color_palette(yvg)
comb = lambda n, k: factorial(n) / (factorial(k) * factorial(n-k)) # n! / (k! * (n – k)!)
comb_2 = lambda n: comb(n,2)
def plot_some_scatters(X,y,name:str=None,s:float=5):
X = np.asarray(X)
n, m = X.shape
ax_combs = tuple(it.combinations(range(m),r=2))
k = comb_2(m)
c = 1
for i in range(2,6):
if k % i == 0:
c = i
r = int(k / c)
fig, axs = plt.subplots(r,c,figsize=(s*c,s*r))
fig.suptitle(name)
for i, ax in enumerate(np.ravel(axs)):
x_idx, y_idx = ax_combs[i]
ax.scatter(X[:,x_idx],X[:,y_idx],c=y)
ax.set(xlabel=f"axis {x_idx}", ylabel=f"axis {y_idx}")
plt.show()
def plot_conf_map(
conf:np.ndarray,
title:str=None,
compact:bool=False,
figsize:tuple=(6.4, 4.8),
blind:bool=False) -> None:
'''
displaying the confusion matrix graphically with additional coefficients
'''
q = len(conf)
conf_m = np.reshape(np.array(conf),(q,q))
conf_share = conf_m/np.sum(conf)
# the metric itself lives in metrics.harmonic_score; here we only draw it
hm, p_list, r_list = harmonic_from_confusion(conf_m)
if not blind:
coef_m = np.hstack((
np.vstack( (conf_share,p_list) ),
np.append(r_list,hm).reshape(-1,1) ))
if compact:
labels = np.asarray([f"{share:.0%}" for share in coef_m.ravel()]).reshape(q+1,q+1)
else:
labels = np.asarray([f"{share:.1%}" for share in coef_m.ravel()]).reshape(q+1,q+1)
labels[-1][-1] = f"H:{hm:.0%}"
fig, ax = plt.subplots(figsize=figsize)
sns.heatmap(coef_m, annot=labels, fmt='', cmap=pal_yvg, vmin=0, vmax=1)
xlabels = [item.get_text() for item in ax.get_xticklabels()]
ylabels = [item.get_text() for item in ax.get_yticklabels()]
xlabels[-1] = 'Rate'
ylabels[-1] = 'Prediction\nvalue'
ax.set_xticklabels(xlabels)
ax.set_yticklabels(ylabels)
if title is not None:
plt.title(title)
plt.show()
return hm, p_list.ravel(), r_list.ravel()
# Images transforms
# This fuctions break our picture
def img_breakdown(img_arr):
return np.vstack((img_arr[:,:,0],img_arr[:,:,1],img_arr[:,:,2],img_arr[:,:,3]))
def img_breakright(img_arr):
return np.hstack((img_arr[:,:,0],img_arr[:,:,1],img_arr[:,:,2],img_arr[:,:,3]))
# and these collect them back
def img_set_up(ldown_arr):
q = ldown_arr.shape[0] // 4
return np.dstack((ldown_arr[q*0:q*1,:],ldown_arr[q*1:q*2,:],ldown_arr[q*2:q*3,:],ldown_arr[q*3:q*4,:]))
def img_set_left(ldown_arr):
q = ldown_arr.shape[1] // 4
return np.dstack((ldown_arr[:,q*0:q*1],ldown_arr[:,q*1:q*2],ldown_arr[:,q*2:q*3],ldown_arr[:,q*3:q*4]))
def img_squeeze(img_arr,trans_class,n_components,conv_pair):
'''
Main process - we break data, transform it,
then inverse_transform and build in original form
'''
conv_break, conv_set = conv_pair
data = conv_break(img_arr)
transform = trans_class(n_components=n_components)
data_trunc = transform.fit_transform(data)
data_re = transform.inverse_transform(data_trunc)
data_recol = conv_set(data_re)
return np.clip(data_recol, 0, 1)
def img_no_ticks(axs):
'''
Options to display pictures
'''
plt.tight_layout(pad=0.0, w_pad=0.0, h_pad=0.0)
for ax in axs:
ax.set_xticks([], [])
ax.set_yticks([], [])
# of course we want to see the result
def img_framaker(img,n,trans_class):
'''
Output function
'''
converting_pairs = ((img_breakdown, img_set_up), (img_breakright, img_set_left))
fig, ax = plt.subplots(figsize=(10,5))
# mixed image to reduce the number of artifacts
new_img = img_squeeze(img,trans_class,n,converting_pairs[0])/2+img_squeeze(img,trans_class,n,converting_pairs[1])/2
ax.imshow(new_img)
ax.text(30,50,str(n))
img_no_ticks([ax])
return fig