diff --git a/examples/quickstart.ipynb b/examples/quickstart.ipynb new file mode 100644 index 000000000..199ceb703 --- /dev/null +++ b/examples/quickstart.ipynb @@ -0,0 +1,64 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MacroBase Python Quickstart\n", + "\n", + "This notebook demonstrates how to load a dataset, run the high-level `analyze` function, and explore the results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from macrobase_py.api import analyze\n", + "\n", + "result = analyze(\"../core/demo/sample.csv\", config={\n", + " \"target_column\": \"usage\",\n", + " \"pivot\": [\n", + " {\"index\": \"location\", \"values\": [\"usage\"], \"aggfunc\": \"mean\"}\n", + " ],\n", + "})\n", + "\n", + "result.row_count, result.column_count" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "pd.DataFrame([profile.__dict__ for profile in result.column_profiles]).head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result.correlation_matrix" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/streamlit_app.py b/examples/streamlit_app.py new file mode 100644 index 000000000..f15a0925e --- /dev/null +++ b/examples/streamlit_app.py @@ -0,0 +1,108 @@ +"""Simple Streamlit demo for MacroBase Python analytics.""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +import pandas as pd + +try: + import streamlit as st +except ImportError as exc: # pragma: no cover - Streamlit optional + raise SystemExit("Streamlit is required to run this demo. Install it via `pip install streamlit`.") from exc + +from macrobase_py.api import analyze_dataframe + + +SUPPORTED_TYPES = { + ".csv": "csv", + ".tsv": "tsv", + ".txt": "csv", + ".xls": "excel", + ".xlsx": "excel", + ".parquet": "parquet", +} + + +def _load_uploaded_file(path: Path, data) -> pd.DataFrame: + ext = path.suffix.lower() + kind = SUPPORTED_TYPES.get(ext) + if kind == "csv": + sep = "\t" if ext == ".tsv" else "," + return pd.read_csv(data, sep=sep) + if kind == "excel": + return pd.read_excel(data) + if kind == "parquet": + return pd.read_parquet(data) + raise ValueError(f"Unsupported file type: {ext}") + + +def main() -> None: + st.set_page_config(page_title="MacroBase Explorer", layout="wide") + st.title("MacroBase Explorer") + st.write("Upload a tabular dataset to compute quick summaries and anomaly indicators.") + + uploaded_file = st.file_uploader( + "Upload CSV, TSV, Excel, or Parquet files", type=[ext.lstrip(".") for ext in SUPPORTED_TYPES] + ) + + if uploaded_file is None: + st.info("Awaiting file upload…") + return + + dataset_name = uploaded_file.name + try: + dataframe = _load_uploaded_file(Path(dataset_name), uploaded_file) + except Exception as exc: # pragma: no cover - UI feedback + st.error(f"Unable to load file: {exc}") + return + + st.success(f"Loaded `{dataset_name}` with {len(dataframe):,} rows and {dataframe.shape[1]} columns.") + + target_column: Optional[str] = None + if not dataframe.select_dtypes(include=["number"]).columns.empty: + target_column = st.selectbox( + "Select a numeric column to highlight in the correlation matrix", + options=["(none)"] + list(dataframe.select_dtypes(include=["number"]).columns), + ) + if target_column == "(none)": + target_column = None + + pivot_column = st.selectbox("Optional pivot index", options=["(none)"] + list(dataframe.columns)) + pivot_metric = st.selectbox( + "Metric to aggregate", options=["(auto)"] + list(dataframe.select_dtypes(include=["number"]).columns) + ) + + config = {} + if target_column is not None: + config["target_column"] = target_column + if pivot_column != "(none)" and pivot_metric != "(auto)": + config["pivot"] = [ + { + "index": pivot_column, + "values": [pivot_metric], + "aggfunc": "mean", + } + ] + + result = analyze_dataframe(dataframe, config) + + st.subheader("Column Profiles") + st.dataframe(pd.DataFrame([profile.__dict__ for profile in result.column_profiles])) + + st.subheader("Summary Statistics") + st.dataframe(result.summary_statistics) + + if result.correlation_matrix is not None: + st.subheader("Correlation Matrix") + st.dataframe(result.correlation_matrix) + + if result.pivot_views: + st.subheader("Pivot Views") + for pivot in result.pivot_views: + for metric, table in pivot.metrics.items(): + st.write(f"**{metric}**") + st.dataframe(table) + +if __name__ == "__main__": # pragma: no cover - interactive app + main() diff --git a/examples/visualization.ipynb b/examples/visualization.ipynb new file mode 100644 index 000000000..2f5ee9456 --- /dev/null +++ b/examples/visualization.ipynb @@ -0,0 +1,63 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MacroBase Visualization Helpers\n", + "\n", + "This notebook demonstrates how to use the convenience plotting utilities in `macrobase_py.viz`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "from macrobase_py.api import analyze\n", + "from macrobase_py.viz import renderers\n", + "\n", + "result = analyze(\"../core/demo/sample.csv\", config={\n", + " \"target_column\": \"usage\",\n", + " \"pivot\": [\n", + " {\"index\": \"location\", \"columns\": \"version\", \"values\": [\"latency\"], \"aggfunc\": \"mean\"}\n", + " ],\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "renderers.render_summary_table(result);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "renderers.plot_pivot_heatmap(result, metric=\"latency\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/macrobase_py/__init__.py b/macrobase_py/__init__.py index 99d698696..0f30e94d1 100644 --- a/macrobase_py/__init__.py +++ b/macrobase_py/__init__.py @@ -1,3 +1,8 @@ +"""MacroBase Python utilities.""" + +from .api import AnalysisResult, ColumnProfile, PivotView, analyze, analyze_dataframe + +__all__ = ["analyze", "analyze_dataframe", "AnalysisResult", "ColumnProfile", "PivotView"] """Python utilities for working with MacroBase data pipelines.""" from . import metrics, summarizers, utils diff --git a/macrobase_py/api.py b/macrobase_py/api.py new file mode 100644 index 000000000..686ee4a80 --- /dev/null +++ b/macrobase_py/api.py @@ -0,0 +1,247 @@ +"""High-level Python interface for the MacroBase analytical workflow. + +This module exposes convenience wrappers for loading structured data sources, +computing summary statistics, identifying potentially interesting signals, and +returning results in a structured format that can be consumed by notebooks, +applications, and visualization utilities. +""" +from __future__ import annotations + +import pathlib +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, MutableMapping, Optional + +import numpy as np +import pandas as pd + + +@dataclass +class ColumnProfile: + """Summary information about a single column.""" + + name: str + dtype: str + non_nulls: int + nulls: int + unique: int + sample_values: List[Any] = field(default_factory=list) + stats: Dict[str, Any] = field(default_factory=dict) + top_values: List[Dict[str, Any]] = field(default_factory=list) + + +@dataclass +class PivotView: + """Represents a pivoted aggregation over the dataset.""" + + index: List[str] + columns: Optional[List[str]] + metrics: Dict[str, pd.DataFrame] + + def to_dict(self) -> Dict[str, Any]: + return { + "index": self.index, + "columns": self.columns, + "metrics": {name: frame.reset_index().to_dict(orient="records") for name, frame in self.metrics.items()}, + } + + +@dataclass +class AnalysisResult: + """Container returned by :func:`analyze` with rich summary information.""" + + row_count: int + column_count: int + column_profiles: List[ColumnProfile] + summary_statistics: pd.DataFrame + correlation_matrix: Optional[pd.DataFrame] + pivot_views: List[PivotView] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + return { + "row_count": self.row_count, + "column_count": self.column_count, + "column_profiles": [profile.__dict__ for profile in self.column_profiles], + "summary_statistics": self.summary_statistics.reset_index().rename(columns={"index": "metric"}).to_dict(orient="records"), + "correlation_matrix": None + if self.correlation_matrix is None + else self.correlation_matrix.reset_index().rename(columns={"index": "feature"}).to_dict(orient="records"), + "pivot_views": [pivot.to_dict() for pivot in self.pivot_views], + } + + +def analyze(file_path: str | pathlib.Path, config: Optional[MutableMapping[str, Any]] = None) -> AnalysisResult: + """Load a dataset and compute a bundle of summary statistics. + + Parameters + ---------- + file_path: + Path to the input dataset. CSV, TSV, Excel, and Parquet files are + supported out of the box. + config: + Optional configuration dictionary. Recognized keys include: + + ``read_kwargs`` (``dict``) + Additional keyword arguments passed to the pandas reader. + ``target_column`` (``str``) + Column name to treat as a target for correlation analysis. + ``pivot`` (``list[dict]``) + A collection of pivot specifications, each containing ``index``, + optional ``columns``, ``values`` (list of metrics), and ``aggfunc``. + + Returns + ------- + AnalysisResult + Structured analysis output containing summary statistics, column level + profiles, and optional pivot views. + """ + + if config is None: + config = {} + + read_kwargs = dict(config.get("read_kwargs", {})) + frame = _load_dataframe(file_path, read_kwargs) + remaining_config = {key: value for key, value in config.items() if key != "read_kwargs"} + return analyze_dataframe(frame, remaining_config) + + +def analyze_dataframe(frame: pd.DataFrame, config: Optional[MutableMapping[str, Any]] = None) -> AnalysisResult: + if config is None: + config = {} + + column_profiles = _profile_columns(frame) + summary_stats = frame.describe(include="all").transpose() + + correlation_matrix: Optional[pd.DataFrame] = None + target_column = config.get("target_column") + numeric_frame = frame.select_dtypes(include=[np.number]) + if not numeric_frame.empty: + correlation_matrix = numeric_frame.corr() + if target_column is not None and target_column in correlation_matrix.columns: + ordered_columns = [target_column] + [col for col in correlation_matrix.columns if col != target_column] + correlation_matrix = correlation_matrix.loc[ordered_columns, ordered_columns] + + pivot_views: List[PivotView] = [] + for pivot_spec in config.get("pivot", []) or []: + pivot_views.append(_build_pivot(frame, pivot_spec)) + + return AnalysisResult( + row_count=len(frame), + column_count=frame.shape[1], + column_profiles=column_profiles, + summary_statistics=summary_stats, + correlation_matrix=correlation_matrix, + pivot_views=pivot_views, + ) + + +def _load_dataframe(file_path: str | pathlib.Path, read_kwargs: MutableMapping[str, Any]) -> pd.DataFrame: + path = pathlib.Path(file_path) + if not path.exists(): + raise FileNotFoundError(f"Dataset not found: {file_path}") + + ext = path.suffix.lower() + if ext in {".csv", ".tsv", ".txt"}: + sep = read_kwargs.pop("sep", "\t" if ext == ".tsv" else ",") + return pd.read_csv(path, sep=sep, **read_kwargs) + if ext in {".xls", ".xlsx"}: + return pd.read_excel(path, **read_kwargs) + if ext in {".parquet", ".pq"}: + return pd.read_parquet(path, **read_kwargs) + + # Fall back to pandas' inference + return pd.read_csv(path, **read_kwargs) + + +def _profile_columns(frame: pd.DataFrame) -> List[ColumnProfile]: + profiles: List[ColumnProfile] = [] + head = frame.head(5) + for column in frame.columns: + series = frame[column] + stats: Dict[str, Any] = {} + if pd.api.types.is_numeric_dtype(series): + stats = { + "mean": series.mean(), + "std": series.std(), + "min": series.min(), + "max": series.max(), + } + elif pd.api.types.is_datetime64_any_dtype(series): + stats = { + "min": series.min(), + "max": series.max(), + } + + counts = series.value_counts(dropna=True).head(5) + top_values = [ + { + "value": _format_value(idx), + "count": int(count), + "proportion": float(count / series.count()) if series.count() else 0.0, + } + for idx, count in counts.items() + ] + + profile = ColumnProfile( + name=column, + dtype=str(series.dtype), + non_nulls=series.count(), + nulls=series.isna().sum(), + unique=series.nunique(dropna=True), + sample_values=head[column].dropna().tolist(), + stats=stats, + top_values=top_values, + ) + profiles.append(profile) + return profiles + + +def _build_pivot(frame: pd.DataFrame, spec: MutableMapping[str, Any]) -> PivotView: + index = spec.get("index") + if not index: + raise ValueError("Pivot specification requires an 'index' key") + + columns = spec.get("columns") + values = spec.get("values") + aggfunc = spec.get("aggfunc", "mean") + if values is None: + numeric_columns = frame.select_dtypes(include=[np.number]).columns + values = list(numeric_columns) + + pivoted = pd.pivot_table(frame, index=index, columns=columns, values=values, aggfunc=aggfunc) + + metrics: Dict[str, pd.DataFrame] = {} + if isinstance(values, Iterable) and not isinstance(values, str): + for value in values: + subframe = pivoted[value] if value in pivoted else pivoted + if isinstance(subframe, pd.Series): + subframe = subframe.to_frame(name=str(value)) + metrics[str(value)] = subframe + else: + if isinstance(pivoted, pd.Series): + pivoted = pivoted.to_frame(name=str(values)) + metrics[str(values)] = pivoted + + if isinstance(index, (list, tuple)): + index_labels = [str(label) for label in index] + else: + index_labels = [str(index)] + + if columns is None: + column_labels = None + elif isinstance(columns, (list, tuple)): + column_labels = [str(label) for label in columns] + else: + column_labels = [str(columns)] + + return PivotView(index=index_labels, columns=column_labels, metrics=metrics) + + +def _format_value(value: Any) -> Any: + if isinstance(value, (pd.Timestamp, pd.Timedelta)): + return value.isoformat() + if isinstance(value, float): + return round(float(value), 6) + return value + + +__all__ = ["analyze", "analyze_dataframe", "AnalysisResult", "ColumnProfile", "PivotView"] diff --git a/macrobase_py/viz/__init__.py b/macrobase_py/viz/__init__.py index ed520c935..952e34b7f 100644 --- a/macrobase_py/viz/__init__.py +++ b/macrobase_py/viz/__init__.py @@ -1,3 +1,8 @@ +"""Visualization helpers for MacroBase Python.""" + +from .renderers import plot_pivot_heatmap, plot_top_values, plotly_pivot_table, render_summary_table + +__all__ = ["render_summary_table", "plot_top_values", "plot_pivot_heatmap", "plotly_pivot_table"] """Module placeholder for macrobase_py.viz.""" __all__: list[str] = [] diff --git a/macrobase_py/viz/renderers.py b/macrobase_py/viz/renderers.py new file mode 100644 index 000000000..e574f2827 --- /dev/null +++ b/macrobase_py/viz/renderers.py @@ -0,0 +1,148 @@ +"""Visualization helpers for MacroBase Python analyses.""" +from __future__ import annotations + +from typing import Any, Iterable, Optional + +import matplotlib.pyplot as plt +import pandas as pd + +try: # Optional dependency + import plotly.express as px +except Exception: # pragma: no cover - Plotly is optional + px = None # type: ignore + +from macrobase_py.api import AnalysisResult + + +def render_summary_table(result: AnalysisResult, ax: Optional[plt.Axes] = None, *, max_rows: int = 15) -> plt.Axes: + """Render the summary statistics table using Matplotlib.""" + + if ax is None: + _, ax = plt.subplots(figsize=(10, min(max_rows, len(result.summary_statistics)) * 0.4 + 1)) + + table_data = ( + result.summary_statistics.reset_index().rename(columns={"index": "column"}).head(max_rows).fillna("") + ) + ax.axis("off") + table = ax.table(cellText=table_data.values, colLabels=table_data.columns, loc="center") + table.auto_set_font_size(False) + table.set_fontsize(10) + table.scale(1, 1.4) + ax.set_title("Summary Statistics", fontsize=14, pad=12) + return ax + + +def plot_top_values( + result: AnalysisResult, + column: str, + *, + top_n: int = 10, + ax: Optional[plt.Axes] = None, + normalize: bool = True, +) -> plt.Axes: + """Plot the most frequent values for a given column.""" + + if ax is None: + _, ax = plt.subplots(figsize=(8, 4)) + + profile = next((p for p in result.column_profiles if p.name == column), None) + if profile is None: + raise ValueError(f"Column '{column}' is not present in the analysis result") + + if not profile.top_values: + raise ValueError(f"Column '{column}' does not have categorical frequency information") + + value_counts = pd.Series({entry["value"]: entry["count"] for entry in profile.top_values}) + + if normalize and value_counts.sum() != 0: + value_counts = value_counts / value_counts.sum() + + value_counts.sort_values(ascending=False).head(top_n).plot(kind="bar", ax=ax) + ax.set_ylabel("Proportion" if normalize else "Count") + ax.set_title(f"Top {top_n} values for {column}") + return ax + + +def plot_pivot_heatmap(result: AnalysisResult, metric: str, ax: Optional[plt.Axes] = None) -> plt.Axes: + """Render a heatmap for the first pivot view containing ``metric``.""" + + pivot = _find_pivot_metric(result, metric) + if pivot is None: + raise ValueError(f"No pivot view found with metric '{metric}'") + + data = _coerce_pivot_frame(pivot.metrics[metric], metric) + + if ax is None: + _, ax = plt.subplots(figsize=(8, 6)) + + im = ax.imshow(data.values, aspect="auto", cmap="viridis") + ax.set_xticks(range(len(data.columns))) + ax.set_xticklabels([str(col) for col in data.columns], rotation=45, ha="right") + ax.set_yticks(range(len(data.index))) + ax.set_yticklabels([str(idx) for idx in data.index]) + ax.set_title(f"Pivot heatmap ({metric})") + plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04) + return ax + + +def plotly_pivot_table(result: AnalysisResult, metric: str, **kwargs: Any): # pragma: no cover - Visualization only + """Render a pivot table using Plotly (if available).""" + + if px is None: + raise ImportError("plotly is not installed. Install plotly to enable interactive visualizations.") + + pivot = _find_pivot_metric(result, metric) + if pivot is None: + raise ValueError(f"No pivot view found with metric '{metric}'") + + data = _coerce_pivot_frame(pivot.metrics[metric], metric) + fig = px.imshow(data.values, aspect="auto", color_continuous_scale="Viridis", **kwargs) + fig.update_layout( + title=f"Pivot heatmap ({metric})", + xaxis_title=_format_axis_label(pivot.columns, "Columns"), + yaxis_title=_format_axis_label(pivot.index, "Index"), + ) + return fig + + +def _find_pivot_metric(result: AnalysisResult, metric: str): + for pivot in result.pivot_views: + if metric in pivot.metrics: + return pivot + return None + + +def _coerce_pivot_frame(data: pd.DataFrame | pd.Series, metric: str) -> pd.DataFrame: + if isinstance(data, pd.Series): + data = data.to_frame(name=metric) + else: + data = data.copy() + + if isinstance(data.index, pd.MultiIndex): + data.index = [" | ".join(str(part) for part in idx) for idx in data.index] + else: + data.index = data.index.map(str) + + if isinstance(data.columns, pd.MultiIndex): + data.columns = [" | ".join(str(part) for part in col) for col in data.columns] + else: + data.columns = data.columns.map(str) + + data.columns.name = None + return data + + +def _format_axis_label(value: Optional[Iterable[str]] | str, fallback: str) -> str: + if value is None: + return fallback + if isinstance(value, (list, tuple)): + return " | ".join(str(v) for v in value) + return str(value) + + +__all__ = [ + "render_summary_table", + "plot_top_values", + "plot_pivot_heatmap", + "plotly_pivot_table", +]