Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/orcapod/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from .config import OrcapodConfig
from .core.function_pod import (
FunctionPod,
function_pod,
Expand All @@ -14,6 +15,7 @@
from . import types # noqa: F401

__all__ = [
"OrcapodConfig",
"FunctionPod",
"function_pod",
"Pipeline",
Expand Down
16 changes: 8 additions & 8 deletions src/orcapod/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,24 @@


@dataclass(frozen=True)
class Config:
"""Immutable configuration object."""
class OrcapodConfig:
"""Immutable OrcaPod configuration object."""

system_tag_hash_n_char: int = 12
schema_hash_n_char: int = 12
path_hash_n_char: int = 20

def with_updates(self, **kwargs) -> Self:
"""Create a new Config instance with updated values."""
"""Create a new ``OrcapodConfig`` instance with updated values."""
return replace(self, **kwargs)

def merge(self, other: "Config") -> "Config":
def merge(self, other: "OrcapodConfig") -> "OrcapodConfig":
"""Merge with another config, other takes precedence."""
if not isinstance(other, Config):
raise TypeError("Can only merge with another Config instance")
if not isinstance(other, OrcapodConfig):
raise TypeError("Can only merge with another OrcapodConfig instance")

# Get all non-default values from other
defaults = Config()
defaults = OrcapodConfig()
updates = {}
for field_name in self.__dataclass_fields__:
other_value = getattr(other, field_name)
Expand All @@ -33,4 +33,4 @@ def merge(self, other: "Config") -> "Config":


# Module-level default config - created at import time
DEFAULT_CONFIG = Config()
DEFAULT_CONFIG = OrcapodConfig()
10 changes: 5 additions & 5 deletions src/orcapod/core/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import Any

import orcapod.contexts as contexts
from orcapod.config import DEFAULT_CONFIG, Config
from orcapod.config import DEFAULT_CONFIG, OrcapodConfig
from orcapod.types import ContentHash

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -75,7 +75,7 @@ class DataContextMixin:
def __init__(
self,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
**kwargs,
):
super().__init__(**kwargs)
Expand All @@ -85,7 +85,7 @@ def __init__(
self._orcapod_config = config

@property
def orcapod_config(self) -> Config:
def orcapod_config(self) -> OrcapodConfig:
return self._orcapod_config

@property
Expand Down Expand Up @@ -117,7 +117,7 @@ class ContentIdentifiableBase(DataContextMixin, ABC):
def __init__(
self,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
**kwargs: Any,
) -> None:
"""
Expand Down Expand Up @@ -360,7 +360,7 @@ def __init__(
self,
label: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
):
# Init provided here for explicit listing of parmeters
super().__init__(label=label, data_context=data_context, config=config)
Expand Down
6 changes: 3 additions & 3 deletions src/orcapod/core/data_function.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from uuid_utils import uuid7

from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.contexts import DataContext
from orcapod.core.base import TraceableBase
from orcapod.core.datagrams import Datagram, Data
Expand Down Expand Up @@ -135,7 +135,7 @@ def __init__(
version: str = "v0.0",
label: str | None = None,
data_context: str | DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
executor: DataFunctionExecutorProtocol | None = None,
):
super().__init__(label=label, data_context=data_context, config=config)
Expand Down Expand Up @@ -374,7 +374,7 @@ def __init__(
output_schema: SchemaLike | Sequence[type] | None = None,
label: str | None = None,
data_context: str | DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
executor: PythonFunctionExecutorProtocol | None = None,
) -> None:

Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/datagrams/datagram.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from uuid_utils import uuid7

from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.core.base import ContentIdentifiableBase
from orcapod.protocols.semantic_types_protocols import TypeConverterProtocol
from orcapod.semantic_types import infer_python_schema_from_pylist_data
Expand Down Expand Up @@ -68,7 +68,7 @@ def __init__(
meta_info: Mapping[str, DataValue] | None = None,
record_id: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
) -> None:
if isinstance(data, pa.RecordBatch):
data = pa.Table.from_batches([data])
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/function_pod.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from orcapod import contexts
from orcapod.channels import ReadableChannel, WritableChannel
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.core.base import TraceableBase
from orcapod.core.data_function import CachedDataFunction, PythonDataFunction
from orcapod.core.streams.base import StreamBase
Expand Down Expand Up @@ -65,7 +65,7 @@ def __init__(
tracker_manager: TrackerManagerProtocol | None = None,
label: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
) -> None:
super().__init__(
label=label,
Expand Down
8 changes: 4 additions & 4 deletions src/orcapod/core/nodes/function_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

from orcapod import contexts
from orcapod.channels import ReadableChannel, WritableChannel
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.core.cached_function_pod import CachedFunctionPod
from orcapod.core.streams.arrow_table_stream import ArrowTableStream
from orcapod.core.streams.base import StreamBase
Expand Down Expand Up @@ -96,7 +96,7 @@ def __init__(
input_stream: StreamProtocol,
tracker_manager: TrackerManagerProtocol | None = None,
label: str | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash",
):
if tracker_manager is None:
Expand Down Expand Up @@ -450,7 +450,7 @@ def __init__(
input_stream: StreamProtocol,
tracker_manager: TrackerManagerProtocol | None = None,
label: str | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash",
):
super().__init__(
Expand Down Expand Up @@ -664,7 +664,7 @@ def __init__(
input_stream: StreamProtocol,
tracker_manager: TrackerManagerProtocol | None = None,
label: str | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
# Optional DB params for persistent mode:
pipeline_database: ArrowDatabaseProtocol | None = None,
result_database: ArrowDatabaseProtocol | None = None,
Expand Down
6 changes: 3 additions & 3 deletions src/orcapod/core/nodes/operator_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

from orcapod import contexts
from orcapod.channels import Channel, ReadableChannel, WritableChannel
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.core.operators.static_output_pod import StaticOutputOperatorPod
from orcapod.core.streams.arrow_table_stream import ArrowTableStream
from orcapod.core.streams.base import StreamBase
Expand Down Expand Up @@ -74,7 +74,7 @@ def __init__(
input_streams: Collection[StreamProtocol],
tracker_manager: TrackerManagerProtocol | None = None,
label: str | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash",
) -> None:
"""Initialize the shared operator-node state.
Expand Down Expand Up @@ -517,7 +517,7 @@ def __init__(
input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol],
tracker_manager: TrackerManagerProtocol | None = None,
label: str | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
# Optional DB params for persistent mode:
pipeline_database: ArrowDatabaseProtocol | None = None,
cache_mode: CacheMode = CacheMode.OFF,
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/nodes/source_node.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from typing import TYPE_CHECKING, Any

from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.core.base import TraceableBase
from orcapod.errors import SourceSpecMismatchError, UnboundSourceError
from orcapod.protocols.core_protocols import DataProtocol, TagProtocol
Expand Down Expand Up @@ -55,7 +55,7 @@ def __init__(
data_schema: Schema,
data_context: str | contexts.DataContext | None = None,
label: str | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
) -> None:
super().__init__(label=label, data_context=data_context, config=config)
self._name = name
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/operators/static_output_pod.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from typing import TYPE_CHECKING, Any, cast

from orcapod.channels import ReadableChannel, WritableChannel
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.contexts import DataContext
from orcapod.core.base import TraceableBase
from orcapod.core.streams.base import StreamBase
Expand Down Expand Up @@ -269,7 +269,7 @@ def __init__(
upstreams: tuple[StreamProtocol, ...] = (),
label: str | None = None,
data_context: DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
) -> None:
self._pod = pod
self._upstreams = upstreams
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/sources/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from typing import TYPE_CHECKING, Any

from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.core.streams.base import StreamBase
from orcapod.protocols.core_protocols import StreamProtocol
from orcapod.types import ColumnConfig, Schema
Expand Down Expand Up @@ -60,7 +60,7 @@ def __init__(
source_id: str | None = None,
label: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
) -> None:
super().__init__(
label=label,
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/sources/cached_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from typing import TYPE_CHECKING, Any

from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.core.sources.base import RootSource
from orcapod.core.streams.arrow_table_stream import ArrowTableStream
from orcapod.protocols.core_protocols import DataProtocol, SourceProtocol, TagProtocol
Expand Down Expand Up @@ -59,7 +59,7 @@ def __init__(
source_id: str | None = None,
label: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
) -> None:
if data_context is None:
data_context = source.data_context_key
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/sources/db_table_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import pyarrow as pa

from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.protocols.db_connector_protocol import DBConnectorProtocol
else:
pa = LazyModule("pyarrow")
Expand Down Expand Up @@ -69,7 +69,7 @@ def __init__(
source_id: str | None = None,
label: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
*,
_query: str | None = None,
) -> None:
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/sources/postgresql_table_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

if TYPE_CHECKING:
from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig


class PostgreSQLTableSource(DBTableSource):
Expand Down Expand Up @@ -69,7 +69,7 @@ def __init__(
source_id: str | None = None,
label: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: Config | None = None,
config: OrcapodConfig | None = None,
) -> None:
self._dsn = dsn # store before try — needed by to_config even if super() raises
connector = PostgreSQLConnector(dsn) # outside try — if this raises, finally never runs
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/sources/spiraldb_table_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@

if TYPE_CHECKING:
from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig


class SpiralDBTableSource(DBTableSource):
Expand Down Expand Up @@ -98,7 +98,7 @@ def __init__(
source_id: str | None = None,
label: str | None = None,
data_context: "str | contexts.DataContext | None" = None,
config: "Config | None" = None,
config: "OrcapodConfig | None" = None,
overrides: dict[str, str] | None = None,
) -> None:
self._project_id = project_id
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/sources/sqlite_table_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

if TYPE_CHECKING:
from orcapod import contexts
from orcapod.config import Config
from orcapod.config import OrcapodConfig


class SQLiteTableSource(DBTableSource):
Expand Down Expand Up @@ -79,7 +79,7 @@ def __init__(
source_id: str | None = None,
label: str | None = None,
data_context: str | contexts.DataContext | None = None,
config: "Config | None" = None,
config: "OrcapodConfig | None" = None,
) -> None:
self._db_path = db_path
connector = SQLiteConnector(db_path)
Expand Down
4 changes: 2 additions & 2 deletions src/orcapod/core/sources/stream_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
if TYPE_CHECKING:
import pyarrow as pa

from orcapod.config import Config
from orcapod.config import OrcapodConfig
from orcapod.contexts import DataContext
else:
pa = LazyModule("pyarrow")
Expand Down Expand Up @@ -59,7 +59,7 @@ class SourceStreamBuilder:
config: Orcapod config (controls hash character counts).
"""

def __init__(self, data_context: DataContext, config: Config) -> None:
def __init__(self, data_context: DataContext, config: OrcapodConfig) -> None:
self._data_context = data_context
self._config = config

Expand Down
Loading
Loading