Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions type_infer/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,17 @@ def infer_types(
----------
data : pd.DataFrame
The input dataset for which we want to infer data type information.
config : Optional[Dict], optional
Configuration dictionary controlling inference behavior. Supported keys include:
- 'engine': Inference engine to use (default: 'rule_based').
- 'pct_invalid': Percentage threshold of invalid values allowed (default: 2).
- 'seed': Random seed for sampling reproducibility (default: 420).
- 'mp_cutoff': Row threshold to enable multiprocessing (default: 1e4).

Returns
-------
TypeInformation
Object containing column types, additional descriptive metadata, and identifiers.
"""
# Set global defaults if missing
if config is None:
Expand Down
18 changes: 16 additions & 2 deletions type_infer/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,27 @@ def __init__(self):


class BaseEngine:
def __init__(self, stable=True):
"""Abstract base class for type inference engines."""

def __init__(self, stable: bool = True):
"""
Initialize the base engine.

:param stable: Whether this engine implementation is production-stable or experimental.
"""
self.stable = stable # whether the engine is stable or not (i.e. experimental)

def infer(self, df) -> TypeInformation:
"""Given a dataframe, infer the types of each column and return a TypeInformation object."""
"""
Given a dataframe, infer the types of each column and return a TypeInformation object.

:param df: Input pandas DataFrame to analyze.
:return: TypeInformation containing inferred column data types.
"""
raise NotImplementedError


class ENGINES:
"""Supported type inference engine identifiers."""

RULE_BASED = 'rule_based'
31 changes: 31 additions & 0 deletions type_infer/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@


def initialize_log():
"""
Initialize and configure the colored logger for the type_infer package.

:return: Configured logging.Logger instance with PID and log level.
"""
pid = os.getpid()
handler = colorlog.StreamHandler()
handler.setFormatter(colorlog.ColoredFormatter())
Expand All @@ -27,6 +32,13 @@ def initialize_log():


def get_nr_procs(df=None):
"""
Determine the optimal number of worker processes to use based on available
system memory, CPU count, or environment variables.

:param df: Optional pandas DataFrame to estimate memory footprint.
:return: Number of parallel worker processes to spawn.
"""
if 'MINDSDB_N_WORKERS' in os.environ:
try:
n = int(os.environ['MINDSDB_N_WORKERS'])
Expand All @@ -48,6 +60,11 @@ def get_nr_procs(df=None):


def seed(seed_nr: int) -> None:
"""
Set random seed for reproducibility across numpy and random modules.

:param seed_nr: Integer seed value.
"""
np.random.seed(seed_nr)
random.seed(seed_nr)

Expand Down Expand Up @@ -93,6 +110,13 @@ def cast_string_to_python_type(string):


def clean_float(val):
"""
Sanitize and parse a value into a float, stripping surrounding quotes,
whitespace, and normalizing comma decimal separators.

:param val: Input value or string representation of a float.
:return: Float value if successfully parsed, or None.
"""
if isinstance(val, (int, float)):
return float(val)

Expand All @@ -113,6 +137,13 @@ def clean_float(val):


def sample_data(df: pd.DataFrame) -> pd.DataFrame:
"""
Extract a statistically representative random sample of rows from a DataFrame
based on calculated sample size.

:param df: Input pandas DataFrame to sample from.
:return: Sampled subset of rows as a pandas DataFrame.
"""
population_size = len(df)
if population_size <= 50:
sample_size = population_size
Expand Down
33 changes: 32 additions & 1 deletion type_infer/rule_based/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,26 @@


def get_identifier_description_mp(arg_tup):
"""
Multiprocessing wrapper for get_identifier_description.

:param arg_tup: Tuple containing (data, column_name, data_dtype).
:return: Identifier classification string or None.
"""
data, column_name, data_dtype = arg_tup
return get_identifier_description(data, column_name, data_dtype)


def get_identifier_description(data: Iterable, column_name: str, data_dtype: dtype):
"""
Examine column data and metadata to determine identifier classification
(e.g., 'No Information', 'Hash-like identifier', 'Foreign key', 'UUID', 'Unknown identifier').

:param data: Iterable of column values.
:param column_name: Name of the column.
:param data_dtype: Inferred data type of the column.
:return: Classification description string, or None if not an identifier.
"""
data = list(data)
if isinstance(data[0], list):
nr_unique = len(set(tuple(x) for x in data))
Expand Down Expand Up @@ -88,6 +103,7 @@ def get_identifier_description(data: Iterable, column_name: str, data_dtype: dty


def _is_foreign_key_name(name):
"""Check if the column name ends with common foreign key suffixes (e.g. '_id', '-id')."""
for endings in ['id', 'ID', 'Id']:
for add in ['-', '_', ' ']:
if name.endswith(add + endings):
Expand All @@ -99,13 +115,20 @@ def _is_foreign_key_name(name):


def _is_identifier_name(name):
"""Check if the column name contains common identifier keywords."""
for keyword in ['account', 'uuid', 'identifier', 'user']:
if keyword in name:
return True
return False


def get_language_dist(data):
"""
Classify the languages present in the input text data and return their distribution.

:param data: Iterable of text values.
:return: Dictionary mapping language codes (or 'Unknown') to counts.
"""
lang_dist = defaultdict(lambda: 0)
lang_dist['Unknown'] = 0
lang_probs_cache = dict()
Expand All @@ -130,6 +153,12 @@ def get_language_dist(data):


def analyze_sentences(data):
"""
Tokenize sentences, count total words, and compute word and sentence length distributions.

:param data: Iterable of sentence or text items.
:return: Tuple of (total word count, word frequency dict, word count per sentence dict).
"""
nr_words = 0
word_dist = defaultdict(int)
nr_words_dist = defaultdict(int)
Expand All @@ -154,18 +183,20 @@ def analyze_sentences(data):


def contains_alnum(text):
"""Check if the string contains any alphanumeric characters."""
for c in text:
if c.isalnum():
return True
return False


def tokenize_text(text):
""" Generator instead of list comprehension for optimal memory usage & runtime """
"""Generator instead of list comprehension for optimal memory usage & runtime."""
return (t.lower() for t in nltk.word_tokenize(decontracted(text)) if contains_alnum(t))


def decontracted(phrase):
"""Expand standard English contractions into full words."""
# specific
phrase = re.sub(r"won\'t", "will not", phrase)
phrase = re.sub(r"can\'t", "can not", phrase)
Expand Down
Loading