diff --git a/type_infer/api.py b/type_infer/api.py index 846541a..079f243 100644 --- a/type_infer/api.py +++ b/type_infer/api.py @@ -17,6 +17,17 @@ def infer_types( ---------- data : pd.DataFrame The input dataset for which we want to infer data type information. + config : Optional[Dict], optional + Configuration dictionary controlling inference behavior. Supported keys include: + - 'engine': Inference engine to use (default: 'rule_based'). + - 'pct_invalid': Percentage threshold of invalid values allowed (default: 2). + - 'seed': Random seed for sampling reproducibility (default: 420). + - 'mp_cutoff': Row threshold to enable multiprocessing (default: 1e4). + + Returns + ------- + TypeInformation + Object containing column types, additional descriptive metadata, and identifiers. """ # Set global defaults if missing if config is None: diff --git a/type_infer/base.py b/type_infer/base.py index 4bba17d..230f557 100644 --- a/type_infer/base.py +++ b/type_infer/base.py @@ -27,13 +27,27 @@ def __init__(self): class BaseEngine: - def __init__(self, stable=True): + """Abstract base class for type inference engines.""" + + def __init__(self, stable: bool = True): + """ + Initialize the base engine. + + :param stable: Whether this engine implementation is production-stable or experimental. + """ self.stable = stable # whether the engine is stable or not (i.e. experimental) def infer(self, df) -> TypeInformation: - """Given a dataframe, infer the types of each column and return a TypeInformation object.""" + """ + Given a dataframe, infer the types of each column and return a TypeInformation object. + + :param df: Input pandas DataFrame to analyze. + :return: TypeInformation containing inferred column data types. + """ raise NotImplementedError class ENGINES: + """Supported type inference engine identifiers.""" + RULE_BASED = 'rule_based' diff --git a/type_infer/helpers.py b/type_infer/helpers.py index 4e2752b..7172cf0 100644 --- a/type_infer/helpers.py +++ b/type_infer/helpers.py @@ -12,6 +12,11 @@ def initialize_log(): + """ + Initialize and configure the colored logger for the type_infer package. + + :return: Configured logging.Logger instance with PID and log level. + """ pid = os.getpid() handler = colorlog.StreamHandler() handler.setFormatter(colorlog.ColoredFormatter()) @@ -27,6 +32,13 @@ def initialize_log(): def get_nr_procs(df=None): + """ + Determine the optimal number of worker processes to use based on available + system memory, CPU count, or environment variables. + + :param df: Optional pandas DataFrame to estimate memory footprint. + :return: Number of parallel worker processes to spawn. + """ if 'MINDSDB_N_WORKERS' in os.environ: try: n = int(os.environ['MINDSDB_N_WORKERS']) @@ -48,6 +60,11 @@ def get_nr_procs(df=None): def seed(seed_nr: int) -> None: + """ + Set random seed for reproducibility across numpy and random modules. + + :param seed_nr: Integer seed value. + """ np.random.seed(seed_nr) random.seed(seed_nr) @@ -93,6 +110,13 @@ def cast_string_to_python_type(string): def clean_float(val): + """ + Sanitize and parse a value into a float, stripping surrounding quotes, + whitespace, and normalizing comma decimal separators. + + :param val: Input value or string representation of a float. + :return: Float value if successfully parsed, or None. + """ if isinstance(val, (int, float)): return float(val) @@ -113,6 +137,13 @@ def clean_float(val): def sample_data(df: pd.DataFrame) -> pd.DataFrame: + """ + Extract a statistically representative random sample of rows from a DataFrame + based on calculated sample size. + + :param df: Input pandas DataFrame to sample from. + :return: Sampled subset of rows as a pandas DataFrame. + """ population_size = len(df) if population_size <= 50: sample_size = population_size diff --git a/type_infer/rule_based/helpers.py b/type_infer/rule_based/helpers.py index fa06fe5..26b878d 100644 --- a/type_infer/rule_based/helpers.py +++ b/type_infer/rule_based/helpers.py @@ -25,11 +25,26 @@ def get_identifier_description_mp(arg_tup): + """ + Multiprocessing wrapper for get_identifier_description. + + :param arg_tup: Tuple containing (data, column_name, data_dtype). + :return: Identifier classification string or None. + """ data, column_name, data_dtype = arg_tup return get_identifier_description(data, column_name, data_dtype) def get_identifier_description(data: Iterable, column_name: str, data_dtype: dtype): + """ + Examine column data and metadata to determine identifier classification + (e.g., 'No Information', 'Hash-like identifier', 'Foreign key', 'UUID', 'Unknown identifier'). + + :param data: Iterable of column values. + :param column_name: Name of the column. + :param data_dtype: Inferred data type of the column. + :return: Classification description string, or None if not an identifier. + """ data = list(data) if isinstance(data[0], list): nr_unique = len(set(tuple(x) for x in data)) @@ -88,6 +103,7 @@ def get_identifier_description(data: Iterable, column_name: str, data_dtype: dty def _is_foreign_key_name(name): + """Check if the column name ends with common foreign key suffixes (e.g. '_id', '-id').""" for endings in ['id', 'ID', 'Id']: for add in ['-', '_', ' ']: if name.endswith(add + endings): @@ -99,6 +115,7 @@ def _is_foreign_key_name(name): def _is_identifier_name(name): + """Check if the column name contains common identifier keywords.""" for keyword in ['account', 'uuid', 'identifier', 'user']: if keyword in name: return True @@ -106,6 +123,12 @@ def _is_identifier_name(name): def get_language_dist(data): + """ + Classify the languages present in the input text data and return their distribution. + + :param data: Iterable of text values. + :return: Dictionary mapping language codes (or 'Unknown') to counts. + """ lang_dist = defaultdict(lambda: 0) lang_dist['Unknown'] = 0 lang_probs_cache = dict() @@ -130,6 +153,12 @@ def get_language_dist(data): def analyze_sentences(data): + """ + Tokenize sentences, count total words, and compute word and sentence length distributions. + + :param data: Iterable of sentence or text items. + :return: Tuple of (total word count, word frequency dict, word count per sentence dict). + """ nr_words = 0 word_dist = defaultdict(int) nr_words_dist = defaultdict(int) @@ -154,6 +183,7 @@ def analyze_sentences(data): def contains_alnum(text): + """Check if the string contains any alphanumeric characters.""" for c in text: if c.isalnum(): return True @@ -161,11 +191,12 @@ def contains_alnum(text): def tokenize_text(text): - """ Generator instead of list comprehension for optimal memory usage & runtime """ + """Generator instead of list comprehension for optimal memory usage & runtime.""" return (t.lower() for t in nltk.word_tokenize(decontracted(text)) if contains_alnum(t)) def decontracted(phrase): + """Expand standard English contractions into full words.""" # specific phrase = re.sub(r"won\'t", "will not", phrase) phrase = re.sub(r"can\'t", "can not", phrase)