From 1844d874eae827c5d68d0956b49b84a0526b3f5e Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 22:06:24 +0000 Subject: [PATCH 01/24] docs(pipeline): add pure-descriptor refactor design spec (ENG-493) Specifies the split of Node/JobNode and Pipeline/PipelineJob hierarchies, SourceSpec elimination, mutating PipelineJob.bind(), and hash stability guarantees for the refactor described in ENG-493. Co-Authored-By: Claude Sonnet 4.6 --- ...ipeline-pure-descriptor-refactor-design.md | 519 ++++++++++++++++++ 1 file changed, 519 insertions(+) create mode 100644 superpowers/specs/2026-05-21-pipeline-pure-descriptor-refactor-design.md diff --git a/superpowers/specs/2026-05-21-pipeline-pure-descriptor-refactor-design.md b/superpowers/specs/2026-05-21-pipeline-pure-descriptor-refactor-design.md new file mode 100644 index 000000000..cc9298112 --- /dev/null +++ b/superpowers/specs/2026-05-21-pipeline-pure-descriptor-refactor-design.md @@ -0,0 +1,519 @@ +# Pipeline Pure-Descriptor Refactor — Design Spec + +**Linear issue:** ENG-493 +**Date:** 2026-05-21 +**Status:** Draft + +--- + +## Overview + +`Pipeline` currently stores `FunctionNode` and `OperatorNode` objects in `_node_lut` and +`_persistent_node_map`. These node objects carry DB attachment logic, executor references, +and table-scope metadata that are purely execution concerns. The goal of this refactor is to +make `Pipeline` a pure computational descriptor — containing only lightweight identity nodes — +while `PipelineJob` becomes the sole stateful, executable form. + +The refactor also eliminates `SourceSpec` (merged into a new lightweight `SourceNode`) and +removes `Pipeline.bind()` in favour of an explicit `PipelineJob` constructor. + +--- + +## Goals & Success Criteria + +- `Pipeline._persistent_node_map` contains **only** `SourceNode | FunctionNode | OperatorNode` + (lightweight, no DB references, no executor). +- `PipelineJob._persistent_node_map` contains **only** `SourceJobNode | FunctionJobNode | + OperatorJobNode`, with live DB references distributed at construction/`bind()` time. +- `SourceSpec` is **eliminated**; `SourceNode` takes its place as the user-facing input-slot + declaration. +- `Pipeline.bind()` is **removed**; the replacement is `PipelineJob.from_pipeline(pipeline, + store=..., sources=...)`. +- `PipelineJob.bind()` is **mutating** — it modifies the job in place and immediately + distributes the new store/sources to all member JobNodes. +- All content hashes and pipeline hashes are **bit-for-bit identical** to the pre-refactor + values (DB path stability preserved). +- All existing external behaviours (operator execution, function-pod execution, partial + execution with unbound slots, serialisation round-trips) are preserved and tested. + +--- + +## Class Hierarchy + +### Node hierarchy + +``` +AbstractNodeBase(TraceableBase, ABC) +│ Shared interface: node_type, label, content_hash, pipeline_hash, +│ output_schema, iter_data — all six concrete types inherit from this. +│ +├── SourceNodeBase(AbstractNodeBase) +│ │ Shared state: _name, _tag_schema, _data_schema +│ │ Shared: identity_structure, pipeline_identity_structure, +│ │ content_hash, pipeline_hash, output_schema, label, name +│ ├── SourceNode ← replaces SourceSpec; iter_data raises UnboundSourceError +│ └── SourceJobNode ← _concrete: StreamProtocol (mutable, see bind()); +│ overrides content_hash → concrete.content_hash(); +│ iter_data delegates to concrete +│ .as_node() → SourceNode +│ +├── FunctionNodeBase(AbstractNodeBase) +│ │ Shared state: _function_pod, _input_stream, _label +│ │ Shared: identity_structure, pipeline_identity_structure, +│ │ content_hash, pipeline_hash, output_schema, upstreams +│ ├── FunctionNode ← iter_data raises PipelineJobRequiredError +│ └── FunctionJobNode ← _pipeline_db, _result_db, _executor, _table_scope; +│ DB-backed iter_data +│ .as_node() → FunctionNode +│ +└── OperatorNodeBase(AbstractNodeBase) + │ Shared state: _operator, _input_streams, _label + │ Shared: identity_structure, pipeline_identity_structure, + │ content_hash, pipeline_hash, output_schema, upstreams + ├── OperatorNode ← iter_data raises PipelineJobRequiredError + └── OperatorJobNode ← _pipeline_db, _cache_mode, _table_scope; + DB-backed iter_data + .as_node() → OperatorNode +``` + +`AbstractNodeBase` is the single typing anchor. All six concrete node types implement the +same interface. `TraceableBase` provides label management, data context, and the +`content_hash` / `pipeline_hash` caching infrastructure that all nodes share. + +### Pipeline hierarchy + +``` +AbstractPipelineBase (recording mechanism, graph state: _node_lut, _upstreams, +│ _graph_edges, _hash_graph, _persistent_node_map, _nodes) +│ +├── Pipeline ← _persistent_node_map: {hash → lightweight Node} +│ no store, no sources, not directly runnable +│ .save() / .load() — blueprint serialisation +│ +└── PipelineJob ← _persistent_node_map: {hash → JobNode with live DB} + _store, _sources, _execution_context (mutable) + .bind() — mutating; distributes DB to all JobNodes + .run() — executes JobNode graph directly + .as_pipeline() — explicit downgrade: creates Pipeline with Nodes + .save() / .load() — full job serialisation +``` + +`Pipeline` and `PipelineJob` are **distinct types** — `isinstance(job, Pipeline)` is `False`. +`AbstractPipelineBase` is the shared anchor for code that needs to accept either. + +--- + +## Detailed Specifications + +### SourceNodeBase + +**Location:** `src/orcapod/core/nodes/source_node.py` + +Shared state: `_name: str`, `_tag_schema: Schema`, `_data_schema: Schema`. + +```python +class SourceNodeBase(TraceableBase, ABC): + node_type = "source" + + def __init__(self, name: str, tag_schema: Schema, data_schema: Schema, ...) -> None: ... + + # identity: (name, tag_schema, data_schema) + def identity_structure(self) -> Any: ... + + # pipeline identity: (tag_schema, data_schema) — matches RootSource base case + def pipeline_identity_structure(self) -> Any: ... + + # Shared: content_hash, pipeline_hash, output_schema, label, name property +``` + +**SourceNode** — the new user-facing input-slot declaration (replaces SourceSpec): + +```python +class SourceNode(SourceNodeBase): + def iter_data(self, ...) -> Iterator: raise UnboundSourceError(...) +``` + +Users write: +```python +slot_a = SourceNode(label="a", tag_schema=..., data_schema=...) +with pipeline: + result = my_pod(slot_a) +``` + +**SourceJobNode** — execution node for a concrete source: + +```python +class SourceJobNode(SourceNodeBase): + def __init__(self, name: str, tag_schema: Schema, data_schema: Schema, + concrete: StreamProtocol | None = None, ...) -> None: ... + + # Override: data-inclusive hash (concrete source's hash) + # When concrete is None (unbound), falls back to schema-based content_hash + # (identical to SourceNode) — consistent with treating the slot as not yet assigned. + def content_hash(self, hasher=None) -> ContentHash: + if self._concrete is None: + return super().content_hash(hasher) # SourceNodeBase schema-based hash + return self._concrete.content_hash(hasher) + + # pipeline_hash() INHERITED — schema-based, same as SourceNode with matching schemas + # This preserves DB path stability. + + def iter_data(self, ...) -> Iterator: + if self._concrete is None: + raise UnboundSourceError( + f"SourceJobNode '{self._name}' has no concrete source bound. " + "Call job.bind(sources={...}) before running." + ) + return self._concrete.iter_data(...) + + def as_node(self) -> SourceNode: + return SourceNode(name=self._name, tag_schema=self._tag_schema, + data_schema=self._data_schema) +``` + +`SourceJobNode._concrete` is a **mutable** field. `bind(sources=...)` updates it in place +(see `_bind_sources()`) so that downstream `FunctionJobNode._input_stream` references — which +point at the same `SourceJobNode` object — automatically see the new concrete without needing +cascading reference updates throughout the graph. + +Hash invariant: `SourceJobNode.pipeline_hash() == SourceNode.pipeline_hash()` for the same +schema, ensuring `FunctionJobNode.pipeline_hash() == FunctionNode.pipeline_hash()` throughout +the chain. `content_hash()` diverges intentionally (data-inclusive vs schema-based). + +**SourceSpec is removed.** All references updated to `SourceNode`. + +--- + +### FunctionNodeBase + +**Location:** `src/orcapod/core/nodes/function_node.py` + +Shared state: `_function_pod: FunctionPodProtocol`, `_input_stream: AbstractNodeBase`, +`_label: str | None`. + +```python +class FunctionNodeBase(TraceableBase, ABC): + node_type = "function" + + def __init__(self, function_pod: FunctionPodProtocol, + input_stream: AbstractNodeBase, + label: str | None = None, ...) -> None: ... + + def identity_structure(self) -> Any: + return (self._function_pod, self._input_stream) + + def pipeline_identity_structure(self) -> Any: + return (self._function_pod, self._input_stream) # pipeline resolver handles routing + + # Shared: content_hash, pipeline_hash, output_schema, upstreams property +``` + +**FunctionNode** — lightweight; no DB: + +```python +class FunctionNode(FunctionNodeBase): + def iter_data(self, ...) -> Iterator: + raise PipelineJobRequiredError( + "FunctionNode cannot iterate data directly. " + "Wrap a Pipeline in a PipelineJob to execute." + ) +``` + +**FunctionJobNode** — execution node: + +```python +class FunctionJobNode(FunctionNodeBase): + def __init__(self, function_pod, input_stream, label=None, + pipeline_database=None, result_database=None, + table_scope="pipeline_hash", executor=None, ...) -> None: ... + + def attach_databases(self, pipeline_database, result_database) -> None: + """Wire live DB references. Called by PipelineJob.bind().""" + self._pipeline_database = pipeline_database + self._result_database = result_database + self._cached_function_pod = CachedFunctionPod(self._function_pod, result_database) + + def iter_data(self, ...) -> Iterator: + # Two-phase: yield cached, then compute missing — identical to current FunctionNode + + def as_node(self) -> FunctionNode: + return FunctionNode(function_pod=self._function_pod, + input_stream=self._input_stream, label=self._label) +``` + +`_table_scope` and `executor` are execution configuration — present on `FunctionJobNode`, +absent from `FunctionNode`. + +--- + +### OperatorNodeBase + +**Location:** `src/orcapod/core/nodes/operator_node.py` + +Mirrors `FunctionNodeBase` but for operators with multiple input streams. + +**OperatorNode** — lightweight, no DB. + +**OperatorJobNode** — adds `_pipeline_database`, `_cache_mode`, `_table_scope`; +`attach_databases()` wires the DB; `as_node()` returns `OperatorNode`. + +--- + +### AbstractPipelineBase + +**Location:** `src/orcapod/pipeline/base.py` (new file) + +```python +class AbstractPipelineBase(AutoRegisteringContextBasedTracker, ABC): + """Shared recording mechanism and graph state for Pipeline and PipelineJob.""" + + def __init__(self, name: str | tuple[str, ...], ...) -> None: + self._name: tuple[str, ...] + self._node_lut: dict[str, AbstractNodeBase] # recording phase + self._upstreams: dict[str, AbstractNodeBase] # leaf nodes + self._graph_edges: list[tuple[str, str]] + self._hash_graph: nx.DiGraph + self._persistent_node_map: dict[str, AbstractNodeBase] # post-compile + self._nodes: dict[str, AbstractNodeBase] # label → node + self._node_graph: nx.DiGraph | None + self._compiled: bool + + # Shared concrete methods: + @property + def name(self) -> tuple[str, ...]: ... + @property + def graph(self) -> nx.DiGraph: ... + @property + def compiled_nodes(self) -> dict[str, AbstractNodeBase]: ... + def reset(self) -> None: ... + def __exit__(self, ...) -> None: ... # calls compile() + def __getattr__(self, item) -> AbstractNodeBase: ... # label lookup + + # Abstract — specialised per subclass: + @abstractmethod + def record_function_pod_invocation(self, pod, input_stream, label=None) -> None: ... + @abstractmethod + def record_operator_pod_invocation(self, pod, upstreams, label=None) -> None: ... + @abstractmethod + def compile(self) -> None: ... +``` + +--- + +### Pipeline + +**Location:** `src/orcapod/pipeline/graph.py` + +Recording creates **lightweight Nodes** in `_node_lut` / `_upstreams`. All leaf inputs must +be `SourceNode` instances — passing a concrete `RootSource` raises `ValueError` with a +message directing users to `PipelineJob`. + +`compile()` walks `_graph_edges` topologically: +- Leaf hashes (in `_upstreams`, not in `_node_lut`) → must be `SourceNode`; stored as-is. +- Non-leaf hashes → `FunctionNode` or `OperatorNode` from `_node_lut`; upstream references + rewired to point at compiled nodes from `_persistent_node_map`. + +`Pipeline` has **no** `bind()` method. To run: `PipelineJob.from_pipeline(pipeline, ...)`. + +`save()` / `load()` serialise the lightweight Node graph (no DB configuration). + +--- + +### PipelineJob + +**Location:** `src/orcapod/pipeline/job.py` + +Additional instance state (beyond `AbstractPipelineBase`): + +```python +self._store: ArrowDatabaseProtocol | None +self._sources: dict[str, StreamProtocol] # SourceNode.name → concrete source +self._execution_context: ExecutionContext | None +self._has_run: bool +self._run_id: str | None +self._unresolved_specs: list[str] +``` + +#### Recording (`with job:`) + +Overrides `record_function_pod_invocation` and `record_operator_pod_invocation`. +Intercepts concrete `RootSource` inputs via `_to_node_stream()`: +- Concrete source → auto-creates `SourceNode(name, schema)` + stores concrete in `_sources`. +- `SourceNode` passed directly → used as-is. +- `DynamicPodStream` → upstreams recursively converted. + +Creates `FunctionJobNode` / `OperatorJobNode` (without DB) in `_node_lut`. + +#### `compile()` + +Walks `_graph_edges` topologically. For each hash: +- **Leaf** → `SourceNode` in `_upstreams`. If `source_node.name in _sources`: create + `SourceJobNode(name, tag_schema, data_schema, concrete=_sources[name])`. Else: create + `SourceJobNode(name, tag_schema, data_schema, concrete=None)` (unbound slot). +- **Non-leaf** → `FunctionJobNode` or `OperatorJobNode` from `_node_lut`; rewire upstream + references to point at compiled JobNodes from `_persistent_node_map`. + +After building `_persistent_node_map`, if `_store` is set, calls `_distribute_databases()`. + +#### `_distribute_databases()` + +```python +def _distribute_databases(self) -> None: + pipeline_db = self._store.at(*self._name) + result_db = pipeline_db.at("_result") + for node in self._persistent_node_map.values(): + if isinstance(node, FunctionJobNode): + node.attach_databases(pipeline_database=pipeline_db, result_database=result_db) + elif isinstance(node, OperatorJobNode): + node.attach_databases(pipeline_database=pipeline_db) +``` + +#### `bind(store=None, sources=None, execution_context=None)` — mutating + +```python +def bind(self, store=None, sources=None, execution_context=None) -> None: + store_changed = store is not None and store is not self._store + if store is not None: + self._store = store + if sources is not None: + # Validate each source against its SourceNode slot schema + # Replace SourceJobNode entries in _persistent_node_map + self._bind_sources(sources) + if execution_context is not None: + self._execution_context = execution_context + if store_changed: + self._distribute_databases() +``` + +`_bind_sources(sources)` validates each new source against the corresponding `SourceJobNode`'s +schema (raises `SourceSchemaMismatchError` on mismatch), then **mutates +`SourceJobNode._concrete` in place** and clears the node's hash cache. Because +`FunctionJobNode._input_stream` holds a reference to the same `SourceJobNode` *object*, +downstream nodes automatically see the updated concrete without any cascading object +replacement. No references throughout the JobNode graph need to be updated. + +#### `run(observer=None)` + +Determines the **runnable subgraph**: all nodes whose transitive upstream `SourceJobNode`s are +all bound (have a concrete source). Unbound branches are collected as `_unresolved_specs`. + +Executes the runnable subgraph in topological order using `SyncPipelineOrchestrator` +(unchanged orchestrator logic). Nodes already have live DB references — no additional wiring +needed. + +Updates `_has_run`, `_run_id`, `_unresolved_specs` in place. Returns `self`. + +#### `as_pipeline() → Pipeline` + +Explicit downgrade. Walks `_persistent_node_map` topologically, calling `.as_node()` on each +JobNode to create the corresponding lightweight Node (with lightweight Node upstreams). Returns +a fresh `Pipeline` object carrying only the Node graph. + +#### `PipelineJob.from_pipeline(pipeline, store=None, sources=None, execution_context=None)` + +Class method. Creates a `PipelineJob` from a compiled `Pipeline`: +1. Copy graph edges, hash graph, name from `pipeline`. +2. Walk `pipeline._persistent_node_map` topologically. For each Node, create the + corresponding JobNode: + - `SourceNode` → `SourceJobNode(name, schemas, concrete=sources.get(name))` + - `FunctionNode` → `FunctionJobNode(function_pod, upstream_job_node, label, table_scope)` + - `OperatorNode` → `OperatorJobNode(operator, upstream_job_nodes, label, table_scope, cache_mode)` +3. Set `_store`, `_sources`, `_execution_context`. +4. If `_store` is set, call `_distribute_databases()`. + +--- + +## Hash Stability Guarantee + +The refactor must produce **identical** `content_hash()` and `pipeline_hash()` values to +pre-refactor for any given pipeline topology and source binding. + +| Object | Identity structure | Hash type | +|--------|--------------------|-----------| +| `SourceNode` | `(name, tag_schema, data_schema)` | Same as old `SourceSpec.content_hash()` | +| `SourceNode.pipeline_hash()` | `(tag_schema, data_schema)` | Same as old `SourceSpec.pipeline_hash()` | +| `SourceJobNode.content_hash()` | delegates to `concrete.content_hash()` | Same as old execution-graph `SourceNode(concrete).content_hash()` | +| `SourceJobNode.pipeline_hash()` | `(tag_schema, data_schema)` — **inherited** | Same as `SourceNode.pipeline_hash()` | +| `FunctionNode.content_hash()` | `(function_pod, input_stream)` via content resolver | Identical to old `FunctionNode.content_hash()` after compile-time rewiring | +| `FunctionNode.pipeline_hash()` | `(function_pod, input_stream)` via pipeline resolver | Identical to old `pipeline_hash()` | +| `FunctionJobNode.content_hash()` | Inherited; `input_stream` is `SourceJobNode` → data-inclusive | Same as old execution-graph `FunctionNode.content_hash()` | +| `FunctionJobNode.pipeline_hash()` | Inherited; pipeline resolver → `SourceJobNode.pipeline_hash()` = schema-based | Same as old `pipeline_hash()` | + +No DB paths change. The pipeline hash chain is invariant. + +--- + +## Serialisation + +### `Pipeline.save()` / `Pipeline.load()` + +Format unchanged except: +- Node type `"source"` entries now reconstruct as `SourceNode` (not `SourceNode(stream=SourceSpec(...))`). +- `source_config.source_type == "node"` (was `"spec"`). + +Format version bump: `"orcapod_pipeline_version": "0.3"`. + +### `PipelineJob.save()` / `PipelineJob.load()` + +Saves the Pipeline blueprint (via `as_pipeline().save()`) plus: +- `bindings.sources`: config per bound source. +- `bindings.store`: store config. +- `run`: `run_id`, `status`, `unresolved_specs`. + +Format version bump: `"orcapod_pipeline_job_version": "0.2"`. + +--- + +## Removed / Changed Public API + +| Before | After | +|--------|-------| +| `SourceSpec(name, tag_schema, data_schema)` | `SourceNode(label, tag_schema, data_schema)` | +| `pipeline.bind(sources, store)` | `PipelineJob.from_pipeline(pipeline, store, sources)` | +| `PipelineJob(name, _pipeline=p, sources=s, store=db)` | `PipelineJob.from_pipeline(p, store=db, sources=s)` | +| `job.bind(sources, store)` → returns new `PipelineJob` | `job.bind(sources, store)` → mutates `job`, returns `None` | +| `pipeline._persistent_node_map` has `FunctionNode` w/ DB | `pipeline._persistent_node_map` has lightweight `FunctionNode` | +| `FunctionNode.attach_databases(...)` on Pipeline nodes | `FunctionJobNode.attach_databases(...)` on PipelineJob nodes only | +| `SourceNode(stream=SourceSpec(...))` in compiled Pipeline | `SourceNode(label, tag_schema, data_schema)` directly | + +--- + +## File Layout Changes + +``` +src/orcapod/ +├── pipeline/ +│ ├── base.py ← NEW: AbstractPipelineBase +│ ├── graph.py ← Pipeline (stripped of bind(), execution Node creation) +│ └── job.py ← PipelineJob (JobNode graph, mutating bind, from_pipeline, as_pipeline) +└── core/ + └── nodes/ + ├── __init__.py ← updated type aliases (AbstractNodeBase, GraphNode, JobNode) + ├── source_node.py ← SourceNodeBase, SourceNode (was SourceSpec too), SourceJobNode + ├── function_node.py ← FunctionNodeBase, FunctionNode, FunctionJobNode + └── operator_node.py ← OperatorNodeBase, OperatorNode, OperatorJobNode + +src/orcapod/core/sources/ + source_spec.py ← DELETED +``` + +--- + +## Tests + +Updated / new test coverage: + +- `Pipeline` with `SourceNode` leaves; verify `SourceSpec` import removed. +- `PipelineJob.from_pipeline(pipeline, store=db, sources={...})` end-to-end. +- `job.bind(store=db)` mutates `job` and immediately wires DB into JobNodes. +- `job.bind(sources={...})` replaces `SourceJobNode` in `_persistent_node_map`. +- `FunctionJobNode.as_node()` returns `FunctionNode` with same `content_hash()`. +- `job.as_pipeline()` returns `Pipeline` whose Nodes match `job`'s topology and hashes. +- Hash stability: `FunctionJobNode.pipeline_hash() == FunctionNode.pipeline_hash()`. +- Partial execution: unbound `SourceJobNode` slots excluded; `unresolved_specs` populated. +- `SourceJobNode(concrete=None).iter_data()` raises `UnboundSourceError`. +- `SourceJobNode(concrete=None).content_hash()` equals `SourceNode(same schemas).content_hash()`. +- `job.bind(sources={name: src})` mutates the existing `SourceJobNode` in place; downstream + `FunctionJobNode._input_stream` reference remains the same object. +- Save / load round-trip for both `Pipeline` and `PipelineJob`. +- Schema mismatch on `bind(sources={...})` raises `SourceSchemaMismatchError`. From 2d436fa1d96ad421a7954ebc4017f72e73758956 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Thu, 21 May 2026 23:15:18 +0000 Subject: [PATCH 02/24] docs(pipeline): add implementation plan for ENG-493 pure-descriptor refactor --- ...-05-21-eng-493-pipeline-pure-descriptor.md | 2683 +++++++++++++++++ 1 file changed, 2683 insertions(+) create mode 100644 superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md diff --git a/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md b/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md new file mode 100644 index 000000000..453c95eb1 --- /dev/null +++ b/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md @@ -0,0 +1,2683 @@ +# Pipeline Pure-Descriptor Refactor Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `Pipeline` a pure computational descriptor (lightweight nodes only, no DB) while `PipelineJob` becomes the sole stateful, executable form — with `FunctionJobNode`/`OperatorJobNode`/`SourceJobNode` carrying all execution state. + +**Architecture:** Create parallel "blueprint" and "execution" node variants sharing base classes. `SourceNode` (schema-only) replaces `SourceSpec` with bit-identical hashes. `FunctionNode`/`OperatorNode` become thin wrappers (no DB); `FunctionJobNode`/`OperatorJobNode` carry all DB logic. `AbstractPipelineBase` extracts shared recording machinery. `Pipeline.bind()` is removed; `PipelineJob.from_pipeline()` is its replacement. `PipelineJob.bind()` becomes mutating (returns `None`). `SourceSpec` is deleted. + +**Tech Stack:** Python, PyArrow, NetworkX; always run `uv run pytest` (never `pytest` directly) + +--- + +## File Layout + +**New files:** +- `src/orcapod/pipeline/base.py` — `AbstractPipelineBase` (shared recording mechanism) +- `tests/test_core/nodes/test_source_node.py` — New unit tests for `SourceNode`/`SourceJobNode` +- `tests/test_core/nodes/test_function_node_split.py` — New unit tests for split hierarchy +- `tests/test_core/nodes/test_operator_node_split.py` — New unit tests for split hierarchy + +**Heavily modified (in execution order):** +1. `src/orcapod/errors.py` — Add `PipelineJobRequiredError` +2. `src/orcapod/core/nodes/source_node.py` — Full rewrite: `SourceNodeBase` + `SourceNode` (schema-only) + `SourceJobNode` +3. `src/orcapod/core/nodes/function_node.py` — Split: `FunctionNodeBase` + thin `FunctionNode` + `FunctionJobNode` +4. `src/orcapod/core/nodes/operator_node.py` — Split: `OperatorNodeBase` + thin `OperatorNode` + `OperatorJobNode` +5. `src/orcapod/core/nodes/__init__.py` — Add `FunctionJobNode`, `OperatorJobNode`, `SourceJobNode` exports; add `JobNode` alias +6. `src/orcapod/pipeline/base.py` ← NEW +7. `src/orcapod/pipeline/graph.py` — `Pipeline`: accept `SourceNode` leaves (not `SourceSpec`), use thin nodes, remove `bind()` +8. `src/orcapod/pipeline/job.py` — `PipelineJob`: use `JobNode` types, `from_pipeline()`, mutating `bind()`, `as_pipeline()` +9. `src/orcapod/pipeline/serialization.py` — New source serialization format, version bumps +10. `src/orcapod/__init__.py` — Remove `SourceSpec`, export `SourceNode` + +**Deleted:** +- `src/orcapod/core/sources/source_spec.py` +- `tests/test_core/sources/test_source_spec.py` + +--- + +## Critical Hash-Stability Invariant + +`SourceNode.identity_structure()` **must** return `("SourceSpec", name, tag_schema, data_schema)` — identical to the old `SourceSpec.identity_structure()`. This ensures: + +``` +SourceNode("x", tag_s, data_s).content_hash() == SourceSpec("x", tag_s, data_s).content_hash() +``` + +DB paths derived from `pipeline_hash()` chains must not change. Every task that touches identity structures must re-verify this invariant with tests before committing. + +--- + +## Task 1: Add PipelineJobRequiredError and rewrite source_node.py + +**Files:** +- Modify: `src/orcapod/errors.py` +- Rewrite: `src/orcapod/core/nodes/source_node.py` +- Create: `tests/test_core/nodes/test_source_node.py` + +- [ ] **Step 1.1: Add PipelineJobRequiredError to errors.py** + +Open `src/orcapod/errors.py` and append after the existing `SourceSpecMismatchError` class: + +```python +class PipelineJobRequiredError(RuntimeError): + """Raised when a lightweight blueprint node is asked to produce data. + + Blueprint nodes (``FunctionNode``, ``OperatorNode``) carry no database + references. Wrap the containing ``Pipeline`` in a ``PipelineJob`` to + obtain executable ``FunctionJobNode`` / ``OperatorJobNode`` variants. + """ +``` + +- [ ] **Step 1.2: Write failing tests for new SourceNode/SourceJobNode interface** + +Create `tests/test_core/nodes/` directory if it doesn't exist, and create +`tests/test_core/nodes/__init__.py` (empty) and +`tests/test_core/nodes/test_source_node.py`: + +```python +"""Tests for SourceNode (schema-only slot) and SourceJobNode (execution variant).""" +from __future__ import annotations + +import pytest + +from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.types import Schema + + +@pytest.fixture +def tag_schema(): + return Schema({"id": int}) + + +@pytest.fixture +def data_schema(): + return Schema({"value": float}) + + +class TestSourceNodeHashStability: + """SourceNode must produce bit-identical hashes to SourceSpec with the same args.""" + + def test_content_hash_matches_source_spec(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.source_spec import SourceSpec + + spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + assert node.content_hash() == spec.content_hash() + + def test_pipeline_hash_matches_source_spec(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.source_spec import SourceSpec + + spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + assert node.pipeline_hash() == spec.pipeline_hash() + + def test_different_names_different_content_hash(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + b = SourceNode(name="slot_b", tag_schema=tag_schema, data_schema=data_schema) + assert a.content_hash() != b.content_hash() + + def test_different_names_same_pipeline_hash(self, tag_schema, data_schema): + """pipeline_hash is schema-only, name-independent.""" + from orcapod.core.nodes.source_node import SourceNode + + a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + b = SourceNode(name="slot_b", tag_schema=tag_schema, data_schema=data_schema) + assert a.pipeline_hash() == b.pipeline_hash() + + +class TestSourceNodeInterface: + def test_iter_data_raises_unbound_error(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + with pytest.raises(UnboundSourceError): + list(node.iter_data()) + + def test_output_schema(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + t, d = node.output_schema() + assert t == tag_schema + assert d == data_schema + + def test_label_resolves_to_name(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="my_slot", tag_schema=tag_schema, data_schema=data_schema) + assert node.label == "my_slot" + + def test_node_type(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert node.node_type == "source" + + def test_name_property(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="my_slot", tag_schema=tag_schema, data_schema=data_schema) + assert node.name == "my_slot" + + def test_validate_compatible_source(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.dict_source import DictSource + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + node.validate(src) # must not raise + + def test_validate_incompatible_raises(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.dict_source import DictSource + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + src = DictSource(records=[{"id": 1, "wrong": 1.0}], tag_columns=["id"]) + with pytest.raises(SourceSpecMismatchError): + node.validate(src) + + +class TestSourceJobNode: + def test_unbound_iter_data_raises(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceJobNode + + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + with pytest.raises(UnboundSourceError): + list(job_node.iter_data()) + + def test_unbound_content_hash_matches_source_node(self, tag_schema, data_schema): + """Unbound SourceJobNode has same content_hash as SourceNode.""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert job_node.content_hash() == node.content_hash() + + def test_pipeline_hash_matches_source_node(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert job_node.pipeline_hash() == node.pipeline_hash() + + def test_bound_content_hash_is_concrete_hash(self, tag_schema, data_schema): + """Bound SourceJobNode content_hash() == concrete.content_hash().""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.core.sources.dict_source import DictSource + + src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + ) + assert job_node.content_hash() == src.content_hash() + + def test_bound_pipeline_hash_still_schema_based(self, tag_schema, data_schema): + """pipeline_hash stays schema-based even when concrete is bound.""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.core.sources.dict_source import DictSource + + src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + ) + assert job_node.pipeline_hash() == node.pipeline_hash() + + def test_as_node_returns_source_node(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + node = job_node.as_node() + assert isinstance(node, SourceNode) + assert node.content_hash() == job_node.content_hash() + + def test_mutable_concrete_updates_in_place(self, tag_schema, data_schema): + """Binding concrete mutates _concrete in-place.""" + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.sources.dict_source import DictSource + + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert job_node._concrete is None + src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + job_node._concrete = src + assert job_node._concrete is src +``` + +- [ ] **Step 1.3: Run tests to confirm failure** + +```bash +uv run pytest tests/test_core/nodes/test_source_node.py -v 2>&1 | tail -20 +``` + +Expected: ImportError or AttributeError (SourceNode doesn't yet have `name=` constructor arg) + +- [ ] **Step 1.4: Rewrite src/orcapod/core/nodes/source_node.py** + +Replace the entire file content: + +```python +"""Source node hierarchy for Pipeline and PipelineJob. + +SourceNode — schema-only input-slot declaration (replaces SourceSpec). +SourceJobNode — execution variant that wraps a concrete StreamProtocol. +Both share SourceNodeBase which provides hash-stable identity. + +Hash-stability guarantee: + SourceNode(name=n, tag_schema=t, data_schema=d).content_hash() + == SourceSpec(name=n, tag_schema=t, data_schema=d).content_hash() + +This is achieved by using identical identity_structure(): + ("SourceSpec", name, tag_schema, data_schema) +""" +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any + +from orcapod import contexts +from orcapod.config import DEFAULT_CONFIG +from orcapod.core.base import TraceableBase +from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.protocols.core_protocols import DataProtocol, TagProtocol +from orcapod.types import ColumnConfig, ContentHash, Schema + +if TYPE_CHECKING: + import pyarrow as pa + + from orcapod.protocols.core_protocols import StreamProtocol + +logger = logging.getLogger(__name__) + + +class SourceNodeBase(TraceableBase, ABC): + """Abstract base for SourceNode and SourceJobNode. + + Provides schema-based identity (content_hash, pipeline_hash) and + shared properties. Both sub-types carry identical schemas so their + pipeline_hash() values always match; content_hash() diverges only + when SourceJobNode has a concrete source bound. + + Args: + name: The input-slot name used as the key in + ``PipelineJob.bind(sources={name: source})``. + tag_schema: Mapping of tag column names to Python types. + data_schema: Mapping of data column names to Python types. + data_context: Optional data context override. + """ + + node_type = "source" + + def __init__( + self, + name: str, + tag_schema: Schema, + data_schema: Schema, + data_context: str | contexts.DataContext | None = None, + ) -> None: + super().__init__(data_context=data_context) + self._name = name + self._tag_schema = tag_schema + self._data_schema = data_schema + + # ------------------------------------------------------------------ + # Identity — hash-stable against old SourceSpec + # ------------------------------------------------------------------ + + def identity_structure(self) -> Any: + """Return the content identity: ``("SourceSpec", name, tag_schema, data_schema)``. + + Deliberately matches ``SourceSpec.identity_structure()`` so that a + ``SourceNode`` constructed with the same arguments as a ``SourceSpec`` + produces an identical ``content_hash()``. This preserves all DB paths + computed from pre-refactor pipelines. + """ + return ("SourceSpec", self._name, self._tag_schema, self._data_schema) + + def pipeline_identity_structure(self) -> Any: + """Return the pipeline identity: ``(tag_schema, data_schema)`` (name-independent). + + Matches ``RootSource.pipeline_identity_structure()`` so that sources + with identical schemas share the same DB table paths regardless of name. + """ + return (self._tag_schema, self._data_schema) + + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ + + @property + def name(self) -> str: + """The input-slot name used as the key in ``PipelineJob.bind(sources={...})``.""" + return self._name + + def computed_label(self) -> str | None: + """Resolve the node label to the slot name. + + Implements ``LabelableMixin.computed_label()`` so that ``self.label`` + resolves to the slot name without an explicit label assignment. + + Returns: + The slot name. + """ + return self._name + + @property + def tag_schema(self) -> Schema: + """Tag schema for this input slot.""" + return self._tag_schema + + @property + def data_schema(self) -> Schema: + """Data schema for this input slot.""" + return self._data_schema + + def output_schema( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """Return ``(tag_schema, data_schema)``. + + Args: + columns: Ignored. + all_info: Ignored. + + Returns: + Tuple of ``(tag_schema, data_schema)``. + """ + return (self._tag_schema, self._data_schema) + + def keys( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Return ``(tag_keys, data_keys)``. + + Args: + columns: Ignored. + all_info: Ignored. + + Returns: + Tuple of ``(tag_column_names, data_column_names)``. + """ + return (tuple(self._tag_schema.keys()), tuple(self._data_schema.keys())) + + # ------------------------------------------------------------------ + # Validation + # ------------------------------------------------------------------ + + def validate(self, source: StreamProtocol) -> None: + """Check that *source* is schema-compatible with this node's declared schema. + + Args: + source: A concrete stream to validate. + + Raises: + SourceSpecMismatchError: If schema columns don't match. + """ + source_tag, source_data = source.output_schema() + + tag_issues: list[str] = [] + data_issues: list[str] = [] + + spec_tag_cols = set(self._tag_schema.keys()) + src_tag_cols = set(source_tag.keys()) + if spec_tag_cols != src_tag_cols: + missing = spec_tag_cols - src_tag_cols + extra = src_tag_cols - spec_tag_cols + if missing: + tag_issues.append(f"missing tag columns: {sorted(missing)}") + if extra: + tag_issues.append(f"unexpected tag columns: {sorted(extra)}") + + spec_data_cols = set(self._data_schema.keys()) + src_data_cols = set(source_data.keys()) + if spec_data_cols != src_data_cols: + missing = spec_data_cols - src_data_cols + extra = src_data_cols - spec_data_cols + if missing: + data_issues.append(f"missing data columns: {sorted(missing)}") + if extra: + data_issues.append(f"unexpected data columns: {sorted(extra)}") + + if tag_issues or data_issues: + raise SourceSpecMismatchError( + f"SourceNode '{self._name}' is not compatible with the provided source. " + + "; ".join(tag_issues + data_issues) + ) + + # ------------------------------------------------------------------ + # Abstract + # ------------------------------------------------------------------ + + @abstractmethod + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Yield ``(tag, data)`` pairs, or raise if data is unavailable.""" + ... + + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(name={self._name!r}, " + f"tag_schema={dict(self._tag_schema)!r}, " + f"data_schema={dict(self._data_schema)!r})" + ) + + +class SourceNode(SourceNodeBase): + """Schema-only input-slot declaration for ``Pipeline`` recording. + + Replaces ``SourceSpec`` as the user-facing way to declare typed pipeline + inputs. Pass a ``SourceNode`` inside a ``with pipeline:`` block as the + upstream for any pod invocation. + + Example:: + + slot = SourceNode(name="data", tag_schema={"id": int}, data_schema={"v": float}) + with pipeline: + result = my_pod(slot) + + job = PipelineJob.from_pipeline(pipeline, store=db, sources={"data": my_source}) + job.run() + + Hash-stability note: + ``identity_structure()`` returns ``("SourceSpec", name, tag_schema, data_schema)`` + — identical to the old ``SourceSpec`` — so existing DB paths remain valid. + """ + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Raise ``UnboundSourceError`` — ``SourceNode`` carries no data. + + Raises: + UnboundSourceError: Always. + """ + raise UnboundSourceError( + f"SourceNode '{self._name}' is not bound to a concrete source. " + "Use PipelineJob.from_pipeline(..., sources={'': source}) " + "or job.bind(sources={'': source}) to attach data." + ) + + +class SourceJobNode(SourceNodeBase): + """Execution-ready source node wrapping an optional concrete stream. + + Used inside ``PipelineJob._persistent_node_map``. The ``_concrete`` + field is **mutable** — ``PipelineJob.bind(sources={...})`` updates it + in-place so that downstream ``FunctionJobNode`` objects (which hold a + reference to this same object) automatically see the new concrete source + without cascading reference updates. + + Hash behaviour: + + * ``content_hash()`` — delegates to ``_concrete.content_hash()`` when + bound; falls back to schema-based ``SourceNodeBase.content_hash()`` (== + ``SourceNode.content_hash()``) when unbound. + * ``pipeline_hash()`` — always schema-based (inherited); never + data-inclusive. This invariant keeps DB paths stable across different + data sources bound to the same slot. + + Args: + name: Slot name. + tag_schema: Tag schema. + data_schema: Data schema. + concrete: Optional concrete stream. Can be set or replaced later via + ``job_node._concrete = source``. + data_context: Optional data context override. + """ + + def __init__( + self, + name: str, + tag_schema: Schema, + data_schema: Schema, + concrete: StreamProtocol | None = None, + data_context: str | contexts.DataContext | None = None, + ) -> None: + super().__init__( + name=name, + tag_schema=tag_schema, + data_schema=data_schema, + data_context=data_context, + ) + self._concrete: StreamProtocol | None = concrete + + def content_hash(self, hasher=None) -> ContentHash: + """Return data-inclusive hash when bound; schema-based hash when unbound. + + Args: + hasher: Optional semantic hasher. + + Returns: + ``_concrete.content_hash(hasher)`` when bound, otherwise + ``SourceNodeBase.content_hash(hasher)``. + """ + if self._concrete is not None: + if hasher is None: + hasher = self.data_context.semantic_hasher + return self._concrete.content_hash(hasher) + return super().content_hash(hasher) + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Delegate to concrete source, or raise if unbound. + + Raises: + UnboundSourceError: When no concrete source is attached. + """ + if self._concrete is None: + raise UnboundSourceError( + f"SourceJobNode '{self._name}' has no concrete source bound. " + "Call job.bind(sources={'': source}) before running." + ) + return self._concrete.iter_data() + + def as_node(self) -> SourceNode: + """Return the lightweight ``SourceNode`` equivalent of this job node. + + Returns: + A new ``SourceNode`` with the same name and schemas. + """ + return SourceNode( + name=self._name, + tag_schema=self._tag_schema, + data_schema=self._data_schema, + ) +``` + +- [ ] **Step 1.5: Run new source_node tests** + +```bash +uv run pytest tests/test_core/nodes/test_source_node.py -v +``` + +Expected: All 14 tests pass. + +- [ ] **Step 1.6: Update nodes/__init__.py to export new types** + +Replace `src/orcapod/core/nodes/__init__.py` content: + +```python +from typing import TypeAlias + +from .function_node import FunctionNode +from .operator_node import OperatorNode +from .source_node import SourceJobNode, SourceNode, SourceNodeBase + +GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode + +__all__ = [ + "FunctionNode", + "GraphNode", + "OperatorNode", + "SourceJobNode", + "SourceNode", + "SourceNodeBase", +] +``` + +- [ ] **Step 1.7: Update Pipeline.compile() to accept SourceNode directly** + +In `src/orcapod/pipeline/graph.py`, find the `compile()` method leaf-handling block (around line 218) and replace: + +```python +# OLD — lines ~218-229 +from orcapod.core.sources.source_spec import SourceSpec +stream = self._upstreams[node_hash] +if not isinstance(stream, SourceSpec): + raise ValueError( + f"Pipeline: all leaf inputs must be SourceSpec instances, " + f"but found {type(stream).__name__!r}. " + "Use 'with PipelineJob:' to record a pipeline with concrete sources, " + "or replace concrete sources with SourceSpec declarations." + ) +node = SourceNode(stream=stream) +``` + +With: + +```python +# NEW +from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass +stream = self._upstreams[node_hash] +if not isinstance(stream, SourceNodeClass): + raise ValueError( + f"Pipeline: all leaf inputs must be SourceNode instances, " + f"but found {type(stream).__name__!r}. " + "Use 'with PipelineJob:' to record a pipeline with concrete sources, " + "or replace concrete sources with SourceNode declarations." + ) +node = stream # SourceNode IS the leaf — no wrapping needed +``` + +Also update the `isinstance` checks later in `compile()` that reference `SourceNode` for node_type annotation — they now just check `isinstance(node, SourceNodeClass)`. + +- [ ] **Step 1.8: Update PipelineJob._ensure_spec → _ensure_source_node** + +In `src/orcapod/pipeline/job.py`: + +1. Rename `_ensure_spec` to `_ensure_source_node` and update it: + +```python +def _ensure_source_node(self, source: cp.StreamProtocol) -> "SourceNode": + """Promote *source* to a SourceNode, storing the concrete binding. + + If the slot already exists (same label/hash key), returns the cached node. + + Args: + source: A concrete ``RootSource`` to promote. + + Returns: + The ``SourceNode`` slot declaration for this source. + """ + from orcapod.core.nodes.source_node import SourceNode + + has_label = source.has_assigned_label + if has_label: + name = source.label # type: ignore[attr-defined] + else: + name = source.content_hash().to_string() + + if name not in self._spec_by_name: + tag_schema, data_schema = source.output_schema() + node = SourceNode(name=name, tag_schema=tag_schema, data_schema=data_schema) + self._spec_by_name[name] = node # type: ignore[assignment] + self._sources[name] = source + return self._spec_by_name[name] # type: ignore[return-value] +``` + +2. Update `_is_concrete_source` to not exclude `SourceNode`: + +```python +@staticmethod +def _is_concrete_source(stream: cp.StreamProtocol) -> bool: + """True if *stream* is a concrete RootSource (not a SourceNode).""" + from orcapod.core.sources.base import RootSource + from orcapod.core.nodes.source_node import SourceNode + + return isinstance(stream, RootSource) and not isinstance(stream, SourceNode) +``` + +3. Rename `_to_spec_stream` → `_to_node_stream` and update: + +```python +def _to_node_stream(self, stream: cp.StreamProtocol) -> cp.StreamProtocol: + """Convert *stream* to a node-based equivalent for consistent hash recording. + + Concrete ``RootSource`` instances are promoted to ``SourceNode`` via + ``_ensure_source_node``. ``DynamicPodStream`` instances have their + upstreams recursively converted. + + Args: + stream: The upstream stream to convert. + + Returns: + A node-based stream with a stable hash for recording. + """ + from orcapod.core.operators.static_output_pod import DynamicPodStream + + if self._is_concrete_source(stream): + return self._ensure_source_node(stream) + if isinstance(stream, DynamicPodStream): + node_upstreams = tuple(self._to_node_stream(s) for s in stream.upstreams) + return DynamicPodStream( + pod=stream._pod, + upstreams=node_upstreams, + label=stream._label, + ) + return stream +``` + +4. Update `record_function_pod_invocation` to call `_to_node_stream`: + +```python +def record_function_pod_invocation(self, pod, input_stream, label=None): + from orcapod.core.nodes import FunctionNode + input_stream = self._to_node_stream(input_stream) + input_hash = input_stream.content_hash().to_string() + function_node = FunctionNode(function_pod=pod, input_stream=input_stream, label=label) + fn_hash = function_node.content_hash().to_string() + self._rec_node_lut[fn_hash] = function_node + self._rec_upstreams[input_hash] = input_stream + self._rec_graph_edges.append((input_hash, fn_hash)) +``` + +5. Update `record_operator_pod_invocation` to call `_to_node_stream`: + +```python +def record_operator_pod_invocation(self, pod, upstreams=(), label=None): + from orcapod.core.nodes import OperatorNode + processed = tuple(self._to_node_stream(s) for s in upstreams) + operator_node = OperatorNode(operator=pod, input_streams=processed, label=label) + op_hash = operator_node.content_hash().to_string() + self._rec_node_lut[op_hash] = operator_node + for upstream in processed: + up_hash = upstream.content_hash().to_string() + self._rec_upstreams[up_hash] = upstream + self._rec_graph_edges.append((up_hash, op_hash)) +``` + +6. Update `unbound_specs` → `unbound_source_nodes`: + +```python +def unbound_source_nodes(self) -> list["SourceNode"]: + """Return all SourceNode slots not yet bound in this job. + + Returns: + List of unbound ``SourceNode`` instances, in graph order. + """ + from orcapod.core.nodes.source_node import SourceNode + + if self._compiled_pipeline is None: + return [] + + unbound: list[SourceNode] = [] + seen: set[str] = set() + for node in self._compiled_pipeline._persistent_node_map.values(): + if ( + isinstance(node, SourceNode) + and node.name not in self._sources + and node.name not in seen + ): + unbound.append(node) + seen.add(node.name) + return unbound +``` + +Keep `unbound_specs` as a deprecated alias that calls `unbound_source_nodes` (will be removed in Task 8): + +```python +def unbound_specs(self): + """Deprecated — use unbound_source_nodes() instead.""" + return self.unbound_source_nodes() +``` + +7. Update `is_complete` to use `unbound_source_nodes`: + +```python +def is_complete(self) -> bool: + return self._store is not None and len(self.unbound_source_nodes()) == 0 +``` + +8. Update `is_runnable` to use `SourceNode` instead of `SourceSpec`: + +```python +def is_runnable(self, node_label: str) -> bool: + from orcapod.core.nodes.source_node import SourceNode + + pipeline = self._compiled_pipeline + if pipeline is None: + return False + target = pipeline._nodes.get(node_label) + if target is None: + return False + if pipeline._node_graph is None: + return False + + import networkx as nx + + for node in nx.ancestors(pipeline._node_graph, target) | {target}: + if isinstance(node, SourceNode) and node.name not in self._sources: + return False + return True +``` + +9. Update `bind()` validation: + +```python +def bind(self, sources=None, store=None, execution_context=None) -> "PipelineJob": + from orcapod.core.nodes.source_node import SourceNode + + merged_sources = dict(self._sources) + if sources is not None: + pipeline = self._compiled_pipeline + if pipeline is not None: + spec_names = { + node.name + for node in pipeline._persistent_node_map.values() + if isinstance(node, SourceNode) + } + for name, source in sources.items(): + for node in pipeline._persistent_node_map.values(): + if isinstance(node, SourceNode) and node.name == name: + node.validate(source) + break + unknown = set(sources.keys()) - spec_names + if unknown: + raise ValueError( + f"bind() received source keys with no matching SourceNode in the pipeline: " + f"{sorted(unknown)}. Known names: {sorted(spec_names)}" + ) + merged_sources.update(sources) + + return PipelineJob( + name=self._pipeline_name, + store=store if store is not None else self._store, + execution_context=( + execution_context if execution_context is not None else self._execution_context + ), + _pipeline=self._compiled_pipeline, + sources=merged_sources, + ) +``` + +- [ ] **Step 1.9: Run pipeline test suite** + +```bash +uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 +``` + +Expected: All existing pipeline tests pass. Failures here indicate missed `SourceSpec` references — fix each one. + +- [ ] **Step 1.10: Commit** + +```bash +git add src/orcapod/errors.py \ + src/orcapod/core/nodes/source_node.py \ + src/orcapod/core/nodes/__init__.py \ + src/orcapod/pipeline/graph.py \ + src/orcapod/pipeline/job.py \ + tests/test_core/nodes/__init__.py \ + tests/test_core/nodes/test_source_node.py +git commit -m "refactor(nodes): replace SourceSpec with schema-only SourceNode + add SourceJobNode" +``` + +--- + +## Task 2: Split function_node.py into FunctionNodeBase + FunctionNode + FunctionJobNode + +**Files:** +- Rewrite: `src/orcapod/core/nodes/function_node.py` +- Modify: `src/orcapod/core/nodes/__init__.py` +- Create: `tests/test_core/nodes/test_function_node_split.py` + +**Strategy:** `FunctionJobNode` is essentially the current `FunctionNode` renamed. The new thin `FunctionNode` just raises `PipelineJobRequiredError` on `iter_data()`. `FunctionNodeBase` holds the shared constructor, properties, and `from_descriptor` logic. + +- [ ] **Step 2.1: Write failing tests** + +Create `tests/test_core/nodes/test_function_node_split.py`: + +```python +"""Tests for the FunctionNode / FunctionJobNode split.""" +from __future__ import annotations + +import pytest + +from orcapod.errors import PipelineJobRequiredError +from orcapod.types import Schema + + +@pytest.fixture +def simple_pipeline(tmp_path): + """A minimal in-memory store + function pod + source node fixture.""" + from orcapod.core.function_pod import FunctionPod + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.dict_source import DictSource + from orcapod.databases.in_memory_database import InMemoryDatabase + + tag_schema = Schema({"id": int}) + data_schema = Schema({"value": float}) + + source_node = SourceNode(name="src", tag_schema=tag_schema, data_schema=data_schema) + src = DictSource(records=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + db = InMemoryDatabase() + + @FunctionPod.from_function + def double(value: float) -> dict: + return {"result": value * 2} + + return { + "source_node": source_node, + "source": src, + "db": db, + "pod": double, + "tag_schema": tag_schema, + "data_schema": data_schema, + } + + +class TestThinFunctionNode: + def test_iter_data_raises_pipeline_job_required(self, simple_pipeline): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + with pytest.raises(PipelineJobRequiredError): + list(fn.iter_data()) + + def test_content_hash_is_stable(self, simple_pipeline): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + h1 = fn.content_hash() + h2 = fn.content_hash() + assert h1 == h2 + + def test_node_type(self, simple_pipeline): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + assert fn.node_type == "function" + + def test_output_schema(self, simple_pipeline): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + tag_s, data_s = fn.output_schema() + assert "result" in data_s + + +class TestFunctionJobNodeHashParity: + """FunctionJobNode must have identical content_hash / pipeline_hash to FunctionNode.""" + + def test_content_hash_matches_function_node(self, simple_pipeline): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + fn = FunctionNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + fjn = FunctionJobNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + assert fn.content_hash() == fjn.content_hash() + + def test_pipeline_hash_matches_function_node(self, simple_pipeline): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + fn = FunctionNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + fjn = FunctionJobNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + assert fn.pipeline_hash() == fjn.pipeline_hash() + + def test_as_node_returns_function_node(self, simple_pipeline): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + fjn = FunctionJobNode( + function_pod=simple_pipeline["pod"], + input_stream=simple_pipeline["source_node"], + ) + fn = fjn.as_node() + assert isinstance(fn, FunctionNode) + assert fn.content_hash() == fjn.content_hash() +``` + +- [ ] **Step 2.2: Run tests to confirm failure** + +```bash +uv run pytest tests/test_core/nodes/test_function_node_split.py -v 2>&1 | tail -20 +``` + +Expected: `ImportError: cannot import name 'FunctionJobNode'` + +- [ ] **Step 2.3: Refactor function_node.py** + +The current `FunctionNode` class (1466 lines) becomes the new `FunctionJobNode`. The new thin `FunctionNode` only raises on `iter_data()`. Both share `FunctionNodeBase` which holds the constructor, identity, properties, and `from_descriptor` logic. + +At the top of `src/orcapod/core/nodes/function_node.py`, after the existing imports, add: + +```python +from orcapod.errors import PipelineJobRequiredError +``` + +Then restructure the class hierarchy as follows (keeping all existing logic intact): + +**a) Rename the current `FunctionNode` class to `FunctionJobNode`** by adding a `FunctionJobNode` alias at the bottom of the file **first**, then creating the new thin `FunctionNode`: + +At the **bottom** of `function_node.py`, after the existing `FunctionNode` class, add: + +```python +# FunctionJobNode is the DB-backed execution variant of FunctionNode. +# It is the existing FunctionNode class renamed. +FunctionJobNode = FunctionNode + + +class FunctionNode(FunctionJobNode): # type: ignore[no-redef] + """Lightweight blueprint node for ``Pipeline`` recording. + + Carries no database references. Calling ``iter_data()`` raises + ``PipelineJobRequiredError`` — wrap the containing ``Pipeline`` in a + ``PipelineJob`` to obtain an executable ``FunctionJobNode``. + + All identity methods (``content_hash``, ``pipeline_hash``, + ``output_schema``) are inherited from ``FunctionJobNode`` and produce + values identical to those of the corresponding ``FunctionJobNode`` + constructed with the same arguments. + + Args: + function_pod: The wrapped function pod. + input_stream: The upstream stream (must be a ``SourceNode`` or + another blueprint node). + label: Optional display label. + """ + + def __init__( + self, + function_pod: "FunctionPodProtocol", + input_stream: "StreamProtocol", + tracker_manager: "TrackerManagerProtocol | None" = None, + label: str | None = None, + config: "Config | None" = None, + ) -> None: + # Construct without any DB parameters + super().__init__( + function_pod=function_pod, + input_stream=input_stream, + tracker_manager=tracker_manager, + label=label, + config=config, + ) + + def iter_data(self): + """Raise PipelineJobRequiredError — blueprint node cannot produce data. + + Raises: + PipelineJobRequiredError: Always. + """ + raise PipelineJobRequiredError( + f"FunctionNode '{self.label}' is a blueprint node and cannot produce data. " + "Wrap the containing Pipeline in a PipelineJob to execute:\n" + " job = PipelineJob.from_pipeline(pipeline, store=db, sources={...})\n" + " job.run()" + ) + + def as_node(self) -> "FunctionNode": + """Return self — already a lightweight node. + + Returns: + ``self`` + """ + return self + + +# Patch FunctionJobNode.as_node() to return the lightweight FunctionNode variant +def _function_job_node_as_node(self) -> "FunctionNode": + """Return a lightweight ``FunctionNode`` with the same identity. + + Returns: + A new ``FunctionNode`` with the same function_pod, input_stream, and label. + """ + return FunctionNode( + function_pod=self._function_pod, + input_stream=self._input_stream, + label=self._label, + ) + + +FunctionJobNode.as_node = _function_job_node_as_node # type: ignore[method-assign] +``` + +> **Note on architecture:** This "alias + subclass" approach avoids duplicating 1400 lines of code. `FunctionJobNode` = the existing class unchanged. `FunctionNode` = thin subclass that overrides only `__init__` and `iter_data`. Both have identical `content_hash()` / `pipeline_hash()` because `__init__` sets identical state. + +- [ ] **Step 2.4: Run new tests** + +```bash +uv run pytest tests/test_core/nodes/test_function_node_split.py -v +``` + +Expected: All 7 tests pass. + +- [ ] **Step 2.5: Update nodes/__init__.py** + +```python +from typing import TypeAlias + +from .function_node import FunctionJobNode, FunctionNode +from .operator_node import OperatorNode +from .source_node import SourceJobNode, SourceNode, SourceNodeBase + +GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode +JobNode: TypeAlias = SourceJobNode | FunctionJobNode + +__all__ = [ + "FunctionJobNode", + "FunctionNode", + "GraphNode", + "JobNode", + "OperatorNode", + "SourceJobNode", + "SourceNode", + "SourceNodeBase", +] +``` + +- [ ] **Step 2.6: Run full pipeline test suite** + +```bash +uv run pytest tests/test_pipeline/ tests/test_core/ -v --tb=short 2>&1 | tail -40 +``` + +Expected: All tests pass. If any tests create `FunctionNode(..., pipeline_database=...)` directly, those still work because `FunctionNode` subclasses `FunctionJobNode` and passes kwargs up. + +- [ ] **Step 2.7: Commit** + +```bash +git add src/orcapod/core/nodes/function_node.py \ + src/orcapod/core/nodes/__init__.py \ + tests/test_core/nodes/test_function_node_split.py +git commit -m "refactor(nodes): split FunctionNode into thin FunctionNode + FunctionJobNode" +``` + +--- + +## Task 3: Split operator_node.py into OperatorNodeBase + OperatorNode + OperatorJobNode + +**Files:** +- Rewrite: `src/orcapod/core/nodes/operator_node.py` +- Modify: `src/orcapod/core/nodes/__init__.py` +- Create: `tests/test_core/nodes/test_operator_node_split.py` + +Same pattern as Task 2: `OperatorJobNode` = existing `OperatorNode` renamed. Thin `OperatorNode` overrides `iter_data()` to raise `PipelineJobRequiredError`. + +- [ ] **Step 3.1: Write failing tests** + +Create `tests/test_core/nodes/test_operator_node_split.py`: + +```python +"""Tests for the OperatorNode / OperatorJobNode split.""" +from __future__ import annotations + +import pytest + +from orcapod.errors import PipelineJobRequiredError +from orcapod.types import Schema + + +@pytest.fixture +def source_pair(): + from orcapod.core.nodes.source_node import SourceNode + + tag_schema = Schema({"id": int}) + data_schema_a = Schema({"a": float}) + data_schema_b = Schema({"b": float}) + node_a = SourceNode(name="src_a", tag_schema=tag_schema, data_schema=data_schema_a) + node_b = SourceNode(name="src_b", tag_schema=tag_schema, data_schema=data_schema_b) + return node_a, node_b + + +class TestThinOperatorNode: + def test_iter_data_raises_pipeline_job_required(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) + with pytest.raises(PipelineJobRequiredError): + list(op_node.iter_data()) + + def test_node_type(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) + assert op_node.node_type == "operator" + + +class TestOperatorJobNodeHashParity: + def test_content_hash_matches_operator_node(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + thin = OperatorNode(operator=op, input_streams=(node_a, node_b)) + job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) + assert thin.content_hash() == job.content_hash() + + def test_pipeline_hash_matches_operator_node(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + thin = OperatorNode(operator=op, input_streams=(node_a, node_b)) + job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) + assert thin.pipeline_hash() == job.pipeline_hash() + + def test_as_node_returns_operator_node(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) + thin = job.as_node() + assert isinstance(thin, OperatorNode) + assert thin.content_hash() == job.content_hash() +``` + +- [ ] **Step 3.2: Run tests to confirm failure** + +```bash +uv run pytest tests/test_core/nodes/test_operator_node_split.py -v 2>&1 | tail -10 +``` + +Expected: `ImportError: cannot import name 'OperatorJobNode'` + +- [ ] **Step 3.3: Refactor operator_node.py** + +At the bottom of `src/orcapod/core/nodes/operator_node.py`, add: + +```python +# OperatorJobNode is the DB-backed execution variant of OperatorNode. +# It is the existing OperatorNode class renamed. +OperatorJobNode = OperatorNode + + +class OperatorNode(OperatorJobNode): # type: ignore[no-redef] + """Lightweight blueprint node for ``Pipeline`` recording. + + Carries no database references. Calling ``iter_data()`` raises + ``PipelineJobRequiredError``. All identity methods are inherited from + ``OperatorJobNode`` and produce identical hashes. + + Args: + operator: The wrapped operator pod. + input_streams: Upstream streams (``SourceNode`` instances or other + blueprint nodes). + label: Optional display label. + """ + + def __init__( + self, + operator: "OperatorPodProtocol", + input_streams: "tuple[StreamProtocol, ...] | list[StreamProtocol]", + tracker_manager: "TrackerManagerProtocol | None" = None, + label: str | None = None, + config: "Config | None" = None, + ) -> None: + # Construct without any DB parameters + super().__init__( + operator=operator, + input_streams=input_streams, + tracker_manager=tracker_manager, + label=label, + config=config, + ) + + def iter_data(self): + """Raise PipelineJobRequiredError — blueprint node cannot produce data. + + Raises: + PipelineJobRequiredError: Always. + """ + from orcapod.errors import PipelineJobRequiredError + + raise PipelineJobRequiredError( + f"OperatorNode '{self.label}' is a blueprint node and cannot produce data. " + "Wrap the containing Pipeline in a PipelineJob to execute:\n" + " job = PipelineJob.from_pipeline(pipeline, store=db, sources={...})\n" + " job.run()" + ) + + def as_node(self) -> "OperatorNode": + """Return self — already a lightweight node.""" + return self + + +# Patch OperatorJobNode.as_node() to return the lightweight OperatorNode variant +def _operator_job_node_as_node(self) -> "OperatorNode": + """Return a lightweight ``OperatorNode`` with the same identity.""" + return OperatorNode( + operator=self._operator, + input_streams=self._input_streams, + label=self._label, + ) + + +OperatorJobNode.as_node = _operator_job_node_as_node # type: ignore[method-assign] +``` + +Also add this import at the top of the file (with the other imports): + +```python +# (PipelineJobRequiredError is imported lazily inside iter_data to avoid circular import) +``` + +- [ ] **Step 3.4: Update nodes/__init__.py to export OperatorJobNode** + +```python +from typing import TypeAlias + +from .function_node import FunctionJobNode, FunctionNode +from .operator_node import OperatorJobNode, OperatorNode +from .source_node import SourceJobNode, SourceNode, SourceNodeBase + +GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode +JobNode: TypeAlias = SourceJobNode | FunctionJobNode | OperatorJobNode + +__all__ = [ + "FunctionJobNode", + "FunctionNode", + "GraphNode", + "JobNode", + "OperatorJobNode", + "OperatorNode", + "SourceJobNode", + "SourceNode", + "SourceNodeBase", +] +``` + +- [ ] **Step 3.5: Run new and existing tests** + +```bash +uv run pytest tests/test_core/nodes/test_operator_node_split.py tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 +``` + +Expected: All pass. + +- [ ] **Step 3.6: Commit** + +```bash +git add src/orcapod/core/nodes/operator_node.py \ + src/orcapod/core/nodes/__init__.py \ + tests/test_core/nodes/test_operator_node_split.py +git commit -m "refactor(nodes): split OperatorNode into thin OperatorNode + OperatorJobNode" +``` + +--- + +## Task 4: Remove Pipeline.bind() and add PipelineJob.from_pipeline() + as_pipeline() + +**Files:** +- Modify: `src/orcapod/pipeline/graph.py` — remove `bind()` +- Modify: `src/orcapod/pipeline/job.py` — add `from_pipeline()`, `as_pipeline()`, make `bind()` mutating + +This is the heart of the public API change. `Pipeline.bind()` is deleted; `PipelineJob.from_pipeline()` is the new way to create a job from a compiled blueprint. + +- [ ] **Step 4.1: Write failing tests** + +Add these tests to `tests/test_pipeline/test_pipeline_job.py` (at the bottom, after existing tests): + +```python +class TestFromPipeline: + """PipelineJob.from_pipeline() creates a runnable job from a compiled Pipeline.""" + + def test_from_pipeline_creates_pipeline_job(self, compiled_pipeline, db): + """from_pipeline returns a PipelineJob with the same topology.""" + from orcapod.pipeline.job import PipelineJob + + job = PipelineJob.from_pipeline(compiled_pipeline, store=db) + assert isinstance(job, PipelineJob) + + def test_from_pipeline_with_sources_binds_them(self, compiled_pipeline, db, source_a): + """Sources passed to from_pipeline are immediately bound.""" + from orcapod.pipeline.job import PipelineJob + + job = PipelineJob.from_pipeline( + compiled_pipeline, store=db, sources={"slot_a": source_a} + ) + assert "slot_a" in job._sources + + def test_pipeline_bind_removed(self, compiled_pipeline): + """Pipeline.bind() no longer exists.""" + assert not hasattr(compiled_pipeline, "bind"), ( + "Pipeline.bind() must be removed — use PipelineJob.from_pipeline() instead" + ) + + +class TestMutatingBind: + """PipelineJob.bind() mutates in place and returns None.""" + + def test_bind_returns_none(self, pipeline_job_with_pipeline, source_a): + result = pipeline_job_with_pipeline.bind(sources={"slot_a": source_a}) + assert result is None + + def test_bind_mutates_sources(self, pipeline_job_with_pipeline, source_a): + pipeline_job_with_pipeline.bind(sources={"slot_a": source_a}) + assert "slot_a" in pipeline_job_with_pipeline._sources + + def test_bind_mutates_store(self, pipeline_job_with_pipeline, db): + pipeline_job_with_pipeline.bind(store=db) + assert pipeline_job_with_pipeline._store is db + + +class TestAsPipeline: + """PipelineJob.as_pipeline() returns a lightweight Pipeline.""" + + def test_as_pipeline_returns_pipeline(self, pipeline_job_with_sources_and_store): + from orcapod.pipeline.graph import Pipeline + + pipeline = pipeline_job_with_sources_and_store.as_pipeline() + assert isinstance(pipeline, Pipeline) + + def test_as_pipeline_node_hashes_match(self, pipeline_job_with_sources_and_store): + """as_pipeline() nodes have matching pipeline_hash to job nodes.""" + job = pipeline_job_with_sources_and_store + pipeline = job.as_pipeline() + + for node_hash in job._persistent_node_map: + assert node_hash in pipeline._persistent_node_map +``` + +> **Note:** The fixture names above (`compiled_pipeline`, `db`, `source_a`, `pipeline_job_with_pipeline`, `pipeline_job_with_sources_and_store`) must be defined in `tests/test_pipeline/conftest.py` or at the top of the test file. Add them as needed based on what already exists. + +- [ ] **Step 4.2: Check existing fixtures in test files** + +```bash +grep -n "^def compiled_pipeline\|^def pipeline_job_with_pipeline\|^def source_a\|^@pytest.fixture" \ + tests/test_pipeline/test_pipeline_job.py | head -20 +``` + +Identify what fixtures already exist and add only the missing ones. + +- [ ] **Step 4.3: Remove Pipeline.bind()** + +In `src/orcapod/pipeline/graph.py`, delete the entire `bind()` method (lines ~313–340 currently): + +```python +# DELETE this entire block: +def bind( + self, + sources: "dict[str, cp.StreamProtocol] | None" = None, + store: "dbp.ArrowDatabaseProtocol | None" = None, + execution_context: "ExecutionContext | None" = None, +) -> "PipelineJob": + ... +``` + +Also remove the `TYPE_CHECKING` import of `PipelineJob` from graph.py if it was only used by `bind()`. + +- [ ] **Step 4.4: Add PipelineJob.from_pipeline() classmethod** + +In `src/orcapod/pipeline/job.py`, add the following classmethod to `PipelineJob`: + +```python +@classmethod +def from_pipeline( + cls, + pipeline: "Pipeline", + store: "ArrowDatabaseProtocol | None" = None, + sources: "dict[str, cp.StreamProtocol] | None" = None, + execution_context: "ExecutionContext | None" = None, +) -> "PipelineJob": + """Create a runnable ``PipelineJob`` from a compiled ``Pipeline``. + + Walks the pipeline's ``_persistent_node_map`` topologically and + creates corresponding ``JobNode`` variants: + + * ``SourceNode`` → ``SourceJobNode(name, schemas, concrete=sources.get(name))`` + * ``FunctionNode`` → ``FunctionJobNode(function_pod, upstream_job_node, label)`` + * ``OperatorNode`` → ``OperatorJobNode(operator, upstream_job_nodes, label)`` + + If *store* is set, ``_distribute_databases()`` is called immediately + so that all ``FunctionJobNode`` / ``OperatorJobNode`` objects have live + DB references before the first ``run()``. + + Args: + pipeline: A compiled ``Pipeline`` (``pipeline._compiled`` must be + ``True``). + store: Database for result caching and operator records. + sources: Mapping of ``SourceNode.name`` → concrete source. + execution_context: Optional execution configuration. + + Returns: + A new ``PipelineJob`` ready to run (or ``bind()`` further). + + Raises: + ValueError: If *pipeline* has not been compiled. + """ + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.utils.lazy_module import LazyModule + + nx = LazyModule("networkx") + + if not pipeline._compiled: + raise ValueError( + "Pipeline must be compiled before creating a PipelineJob from it. " + "Call pipeline.compile() or use auto_compile=True." + ) + + bound_sources: dict[str, cp.StreamProtocol] = dict(sources or {}) + + # Build a topological ordering of the persistent node map + G = pipeline._hash_graph + job_node_map: dict[str, object] = {} # content_hash_str → JobNode + + import networkx as _nx + + for node_hash in _nx.topological_sort(G): + if node_hash not in pipeline._persistent_node_map: + continue + + node = pipeline._persistent_node_map[node_hash] + + if isinstance(node, SourceNode): + concrete = bound_sources.get(node.name) + job_node = SourceJobNode( + name=node.name, + tag_schema=node.tag_schema, + data_schema=node.data_schema, + concrete=concrete, + ) + + elif isinstance(node, FunctionNode): + # Rewire input to the already-built SourceJobNode / FunctionJobNode + original_input_hash = node._input_stream.content_hash().to_string() + upstream_job_node = job_node_map[original_input_hash] + job_node = FunctionJobNode( + function_pod=node._function_pod, + input_stream=upstream_job_node, # type: ignore[arg-type] + label=node._label, + table_scope=node._table_scope, + ) + + elif isinstance(node, OperatorNode): + # Rewire all inputs to already-built job nodes + upstream_job_nodes = tuple( + job_node_map[s.content_hash().to_string()] + for s in node._input_streams + ) + job_node = OperatorJobNode( + operator=node._operator, + input_streams=upstream_job_nodes, # type: ignore[arg-type] + label=node._label, + cache_mode=node._cache_mode, + table_scope=node._table_scope, + ) + + else: + raise TypeError( + f"Unknown node type in pipeline._persistent_node_map: {type(node)}" + ) + + job_node_map[node_hash] = job_node + + # Construct the PipelineJob and inject the built node map + job = cls.__new__(cls) + super(PipelineJob, job).__init__() + job._store = store + job._execution_context = execution_context + job._sources = bound_sources + job._pipeline_name = pipeline._name + job._unresolved_specs = [] + job._has_run = False + job._run_id = None + job._rec_graph_edges = [] + job._rec_upstreams = {} + job._rec_node_lut = {} + job._spec_by_name = {} + + # Copy pipeline graph structure + job._compiled_pipeline = pipeline + job._persistent_node_map = job_node_map # type: ignore[assignment] + job._nodes = {} # populated below + + # Build label → node map by copying from pipeline._nodes + for label, node in pipeline._nodes.items(): + node_hash = node.content_hash().to_string() + if node_hash in job_node_map: + job._nodes[label] = job_node_map[node_hash] # type: ignore[assignment] + + # Wire databases if store is provided + if store is not None: + job._distribute_databases() + + return job +``` + +- [ ] **Step 4.5: Make PipelineJob.bind() mutating (returns None)** + +Replace the existing `bind()` method in `src/orcapod/pipeline/job.py`: + +```python +def bind( + self, + sources: "dict[str, cp.StreamProtocol] | None" = None, + store: "ArrowDatabaseProtocol | None" = None, + execution_context: "ExecutionContext | None" = None, +) -> None: + """Update bindings in place. + + Mutating — modifies ``self`` directly. Existing bindings not mentioned + in this call are preserved. + + When *sources* is provided, each concrete source is validated against + its matching ``SourceNode`` slot schema, then the corresponding + ``SourceJobNode._concrete`` is updated in-place (downstream + ``FunctionJobNode`` objects that hold a reference to the same + ``SourceJobNode`` object automatically see the new concrete without + any cascading reference updates). + + When *store* is provided and differs from the current store, + ``_distribute_databases()`` is called so that all job nodes receive + live DB references immediately. + + Args: + sources: Mapping of ``SourceNode.name`` → concrete source. + store: Replaces the current store and triggers DB redistribution. + execution_context: Replaces the current execution context. + + Raises: + SourceSpecMismatchError: If any source's schema is incompatible. + ValueError: If a source key has no matching ``SourceNode`` slot. + """ + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + store_changed = store is not None and store is not self._store + + if store is not None: + self._store = store + + if sources is not None: + pipeline = self._compiled_pipeline + if pipeline is not None: + spec_names = { + node.name + for node in pipeline._persistent_node_map.values() + if isinstance(node, SourceNode) + } + unknown = set(sources.keys()) - spec_names + if unknown: + raise ValueError( + f"bind() received source keys with no matching SourceNode: " + f"{sorted(unknown)}. Known names: {sorted(spec_names)}" + ) + # Validate schema for each supplied source + for node in pipeline._persistent_node_map.values(): + if isinstance(node, SourceNode) and node.name in sources: + node.validate(sources[node.name]) + + # Update SourceJobNode._concrete in-place + for job_node in (self._persistent_node_map or {}).values(): + if isinstance(job_node, SourceJobNode) and job_node.name in sources: + job_node._concrete = sources[job_node.name] + # Clear cached hashes so content_hash() reflects new concrete + job_node._content_hash_cache.clear() + + self._sources.update(sources) + + if execution_context is not None: + self._execution_context = execution_context + + if store_changed: + self._distribute_databases() +``` + +- [ ] **Step 4.6: Add _distribute_databases() to PipelineJob** + +Check whether `_distribute_databases()` already exists in `job.py`. If not (it may be in `_build_execution_graph`), add it: + +```python +def _distribute_databases(self) -> None: + """Wire live DB references to all FunctionJobNode and OperatorJobNode objects. + + Called by ``bind()`` when *store* is changed and by ``from_pipeline()`` + when *store* is provided at construction time. + + Raises: + RuntimeError: If ``_store`` is not set. + """ + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + + if self._store is None: + raise RuntimeError( + "Cannot distribute databases: no store is set. " + "Call bind(store=...) or from_pipeline(..., store=...) first." + ) + + pipeline_db = self._store.at(*self._pipeline_name) + result_db = pipeline_db.at("_result") + + for node in (self._persistent_node_map or {}).values(): + if isinstance(node, FunctionJobNode): + node.attach_databases( + pipeline_database=pipeline_db, + result_database=result_db, + ) + elif isinstance(node, OperatorJobNode): + node.attach_databases(pipeline_database=pipeline_db) +``` + +- [ ] **Step 4.7: Add as_pipeline() to PipelineJob** + +```python +def as_pipeline(self) -> "Pipeline": + """Return the lightweight ``Pipeline`` blueprint for this job. + + Walks ``_persistent_node_map`` and calls ``.as_node()`` on each + ``JobNode`` to obtain the corresponding lightweight ``Node``. + Upstream references in the returned ``Pipeline`` point at the + lightweight nodes, not the ``JobNode`` objects. + + Returns: + A compiled ``Pipeline`` whose ``_persistent_node_map`` contains + only lightweight ``SourceNode`` / ``FunctionNode`` / ``OperatorNode`` + objects with identical ``content_hash()`` and ``pipeline_hash()`` + values to their ``JobNode`` counterparts. + """ + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.pipeline.graph import Pipeline + + import networkx as _nx + + if self._compiled_pipeline is None: + raise RuntimeError( + "PipelineJob has no compiled pipeline. " + "Either use 'with job:' to record a DAG, " + "or create the job via PipelineJob.from_pipeline()." + ) + + G = self._compiled_pipeline._hash_graph + node_map: dict[str, object] = {} + + for node_hash in _nx.topological_sort(G): + if node_hash not in (self._persistent_node_map or {}): + continue + job_node = self._persistent_node_map[node_hash] # type: ignore[index] + node_map[node_hash] = job_node.as_node() + + pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) + pipeline._graph_edges = list(self._compiled_pipeline._graph_edges) + pipeline._upstreams = dict(self._compiled_pipeline._upstreams) + pipeline._node_lut = dict(self._compiled_pipeline._node_lut) + pipeline._hash_graph = self._compiled_pipeline._hash_graph + pipeline._persistent_node_map = node_map # type: ignore[assignment] + pipeline._nodes = { + label: node_map[node.content_hash().to_string()] # type: ignore[index] + for label, node in self._compiled_pipeline._nodes.items() + if node.content_hash().to_string() in node_map + } + pipeline._compiled = True + + return pipeline +``` + +- [ ] **Step 4.8: Run new and full test suite** + +```bash +uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 +``` + +Fix any test that calls `pipeline.bind(...)` — replace with: + +```python +# OLD: +job = pipeline.bind(sources={"slot_a": src}, store=db) + +# NEW: +from orcapod.pipeline.job import PipelineJob +job = PipelineJob.from_pipeline(pipeline, store=db, sources={"slot_a": src}) +``` + +- [ ] **Step 4.9: Commit** + +```bash +git add src/orcapod/pipeline/graph.py \ + src/orcapod/pipeline/job.py \ + tests/test_pipeline/test_pipeline_job.py +git commit -m "refactor(pipeline): remove Pipeline.bind(); add PipelineJob.from_pipeline(), as_pipeline(), mutating bind()" +``` + +--- + +## Task 5: AbstractPipelineBase in pipeline/base.py + +**Files:** +- Create: `src/orcapod/pipeline/base.py` +- Modify: `src/orcapod/pipeline/graph.py` — inherit from `AbstractPipelineBase` +- Modify: `src/orcapod/pipeline/job.py` — inherit from `AbstractPipelineBase` + +This task extracts the shared recording machinery (`_node_lut`, `_upstreams`, `_graph_edges`, `_hash_graph`, `reset()`, `__exit__`, `__getattr__`, `graph` property, etc.) into a common base. + +- [ ] **Step 5.1: Create src/orcapod/pipeline/base.py** + +```python +"""AbstractPipelineBase — shared recording mechanism for Pipeline and PipelineJob.""" +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +from orcapod.core.tracker import AutoRegisteringContextBasedTracker +from orcapod.protocols import core_protocols as cp +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import networkx as nx + from orcapod.core.nodes import GraphNode + +else: + nx = LazyModule("networkx") + +logger = logging.getLogger(__name__) + + +class AbstractPipelineBase(AutoRegisteringContextBasedTracker, ABC): + """Shared recording mechanism and graph state for Pipeline and PipelineJob. + + Manages the ``with``-block recording phase: accumulating graph edges, + node LUT entries, and upstream stream references. Subclasses specialise + which node types are created (blueprint vs. job nodes). + + Args: + name: Pipeline name (string or tuple). Used to scope database paths. + tracker_manager: Optional tracker manager override. + """ + + def __init__( + self, + name: str | tuple[str, ...] = "pipeline", + tracker_manager: cp.TrackerManagerProtocol | None = None, + ) -> None: + super().__init__(tracker_manager=tracker_manager) + self._name: tuple[str, ...] = (name,) if isinstance(name, str) else tuple(name) + self._node_lut: dict[str, Any] = {} + self._upstreams: dict[str, cp.StreamProtocol] = {} + self._graph_edges: list[tuple[str, str]] = [] + self._hash_graph: nx.DiGraph = nx.DiGraph() + self._persistent_node_map: dict[str, Any] = {} + self._nodes: dict[str, Any] = {} + self._node_graph: nx.DiGraph | None = None + self._compiled: bool = False + + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ + + @property + def name(self) -> tuple[str, ...]: + """Pipeline name tuple.""" + return self._name + + @property + def graph(self) -> nx.DiGraph: + """Directed hash graph of accumulated pipeline structure.""" + return self._hash_graph + + @property + def compiled_nodes(self) -> dict[str, Any]: + """Copy of the compiled nodes dict (label → node).""" + return self._nodes.copy() + + # ------------------------------------------------------------------ + # Recording helpers + # ------------------------------------------------------------------ + + def reset(self) -> None: + """Clear session-scoped recorded state (node LUT, upstreams, edge list). + + Note: + ``_hash_graph`` and ``_persistent_node_map`` are intentionally + *not* cleared — they accumulate across ``with`` blocks. + """ + self._node_lut.clear() + self._upstreams.clear() + self._graph_edges.clear() + + def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: + super().__exit__(exc_type, exc_value, traceback) + if exc_type is None: + self.compile() + + def __getattr__(self, item: str) -> Any: + """Look up compiled nodes by label as attribute access.""" + if item.startswith("_"): + raise AttributeError(item) + nodes = object.__getattribute__(self, "_nodes") + if item in nodes: + return nodes[item] + raise AttributeError( + f"{type(self).__name__!r} has no attribute {item!r}. " + f"Available node labels: {sorted(nodes.keys())}" + ) + + # ------------------------------------------------------------------ + # Abstract — specialised per subclass + # ------------------------------------------------------------------ + + @abstractmethod + def record_function_pod_invocation( + self, + pod: cp.FunctionPodProtocol, + input_stream: cp.StreamProtocol, + label: str | None = None, + ) -> None: + """Record a function pod invocation into the graph.""" + ... + + @abstractmethod + def record_operator_pod_invocation( + self, + pod: cp.OperatorPodProtocol, + upstreams: tuple[cp.StreamProtocol, ...] = (), + label: str | None = None, + ) -> None: + """Record an operator pod invocation into the graph.""" + ... + + @abstractmethod + def compile(self) -> None: + """Compile recorded invocations into a frozen DAG.""" + ... +``` + +- [ ] **Step 5.2: Update Pipeline to inherit from AbstractPipelineBase** + +In `src/orcapod/pipeline/graph.py`: + +1. Add import: `from orcapod.pipeline.base import AbstractPipelineBase` +2. Change class definition: `class Pipeline(AbstractPipelineBase):` (remove `AutoRegisteringContextBasedTracker` from the inheritance since it's now in the base) +3. Remove from `Pipeline.__init__` the attributes already in `AbstractPipelineBase` (`_node_lut`, `_upstreams`, `_graph_edges`, `_hash_graph`, `_name`, `_nodes`, `_persistent_node_map`, `_node_graph`, `_compiled`) +4. Update `super().__init__` call to pass `name=name` and `tracker_manager=tracker_manager` +5. Remove the `reset()` method (now in base), `graph` property (now in base), `compiled_nodes` property (now in base), and `__exit__` if it just calls `compile()` (now in base handles this) +6. Keep `name` property if it does something different, otherwise remove it (base has it) +7. Keep `_auto_compile` flag in Pipeline (base doesn't have it) + +Verify `__exit__` in Pipeline: the base calls `compile()` unconditionally; Pipeline uses `_auto_compile`. Update base's `__exit__` to be abstract (not calling `compile()`), or handle `_auto_compile` differently: + +Actually, keep Pipeline's `__exit__` override: + +```python +def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: + super(AutoRegisteringContextBasedTracker, self).__exit__(exc_type, exc_value, traceback) + if exc_type is None and self._auto_compile: + self.compile() +``` + +Or just keep `_auto_compile` handling by overriding `__exit__` in `Pipeline`: + +```python +def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: + # Skip AbstractPipelineBase.__exit__ (which calls compile() unconditionally) + # and call compile() only if auto_compile is True. + AutoRegisteringContextBasedTracker.__exit__(self, exc_type, exc_value, traceback) + if exc_type is None and self._auto_compile: + self.compile() +``` + +- [ ] **Step 5.3: Update PipelineJob to inherit from AbstractPipelineBase** + +In `src/orcapod/pipeline/job.py`: + +1. Add import: `from orcapod.pipeline.base import AbstractPipelineBase` +2. Change class definition: `class PipelineJob(AbstractPipelineBase):` +3. Update `__init__` to call `super().__init__(name=name, tracker_manager=tracker_manager)` and remove duplicate state initialization + +- [ ] **Step 5.4: Run full test suite** + +```bash +uv run pytest tests/ -v --tb=short 2>&1 | tail -40 +``` + +Expected: All tests pass. + +- [ ] **Step 5.5: Commit** + +```bash +git add src/orcapod/pipeline/base.py \ + src/orcapod/pipeline/graph.py \ + src/orcapod/pipeline/job.py +git commit -m "refactor(pipeline): extract AbstractPipelineBase with shared recording mechanism" +``` + +--- + +## Task 6: Update PipelineJob to use FunctionJobNode / OperatorJobNode during recording + +**Files:** +- Modify: `src/orcapod/pipeline/job.py` + +Currently `PipelineJob.record_function_pod_invocation` creates a `FunctionNode`. After this task it creates a `FunctionJobNode` (DB-ready). Same for operators. PipelineJob's `compile()` then builds `SourceJobNode` leaves and wires job nodes together. + +- [ ] **Step 6.1: Write failing test** + +Add to `tests/test_pipeline/test_pipeline_job.py`: + +```python +class TestPipelineJobUsesJobNodes: + """PipelineJob._persistent_node_map must contain only JobNode variants.""" + + def test_persistent_map_contains_source_job_nodes(self, pipeline_job_with_sources): + from orcapod.core.nodes.source_node import SourceJobNode + + for node in pipeline_job_with_sources._persistent_node_map.values(): + from orcapod.core.nodes.source_node import SourceNodeBase + if isinstance(node, SourceNodeBase): + assert isinstance(node, SourceJobNode), ( + f"Expected SourceJobNode, got {type(node).__name__}" + ) + + def test_persistent_map_contains_function_job_nodes(self, pipeline_job_with_sources): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + for node in pipeline_job_with_sources._persistent_node_map.values(): + if node.node_type == "function": + assert isinstance(node, FunctionJobNode), ( + f"Expected FunctionJobNode, got {type(node).__name__}" + ) + # Thin FunctionNode should NOT appear in PipelineJob + assert type(node) is not FunctionNode, ( + "PipelineJob should use FunctionJobNode, not thin FunctionNode" + ) +``` + +- [ ] **Step 6.2: Run test to confirm failure** + +```bash +uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobUsesJobNodes -v 2>&1 | tail -20 +``` + +Expected: FAIL (currently creates `FunctionNode` in `_persistent_node_map`) + +- [ ] **Step 6.3: Update PipelineJob recording to use FunctionJobNode/OperatorJobNode** + +In `src/orcapod/pipeline/job.py`, update `record_function_pod_invocation`: + +```python +def record_function_pod_invocation( + self, + pod: cp.FunctionPodProtocol, + input_stream: cp.StreamProtocol, + label: str | None = None, +) -> None: + from orcapod.core.nodes.function_node import FunctionJobNode + + input_stream = self._to_node_stream(input_stream) + input_hash = input_stream.content_hash().to_string() + node = FunctionJobNode(function_pod=pod, input_stream=input_stream, label=label) + fn_hash = node.content_hash().to_string() + self._rec_node_lut[fn_hash] = node + self._rec_upstreams[input_hash] = input_stream + self._rec_graph_edges.append((input_hash, fn_hash)) +``` + +Update `record_operator_pod_invocation`: + +```python +def record_operator_pod_invocation( + self, + pod: cp.OperatorPodProtocol, + upstreams: tuple[cp.StreamProtocol, ...] = (), + label: str | None = None, +) -> None: + from orcapod.core.nodes.operator_node import OperatorJobNode + + processed = tuple(self._to_node_stream(s) for s in upstreams) + node = OperatorJobNode(operator=pod, input_streams=processed, label=label) + op_hash = node.content_hash().to_string() + self._rec_node_lut[op_hash] = node + for upstream in processed: + up_hash = upstream.content_hash().to_string() + self._rec_upstreams[up_hash] = upstream + self._rec_graph_edges.append((up_hash, op_hash)) +``` + +- [ ] **Step 6.4: Update PipelineJob._compile_from_recording() to create SourceJobNode leaves** + +In `src/orcapod/pipeline/job.py`, update `_compile_from_recording`: + +```python +def _compile_from_recording(self) -> None: + """Compile recorded edges + node LUT into a pure Pipeline + job node map.""" + from orcapod.pipeline.graph import Pipeline + + # Build the pure Pipeline (SourceNode leaves, thin FunctionNode/OperatorNode) + pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) + pipeline._graph_edges = list(self._rec_graph_edges) + pipeline._upstreams = dict(self._rec_upstreams) + # Convert FunctionJobNode → FunctionNode and OperatorJobNode → OperatorNode + # for the blueprint pipeline + pipeline._node_lut = { + h: node.as_node() for h, node in self._rec_node_lut.items() + } + for edge in self._rec_graph_edges: + pipeline._hash_graph.add_edge(*edge) + for node_hash, node in self._rec_node_lut.items(): + if node_hash in pipeline._hash_graph.nodes: + pipeline._hash_graph.nodes[node_hash]["node_type"] = node.node_type + if node._label: + pipeline._hash_graph.nodes[node_hash]["label"] = node._label + for node_hash, stream in self._rec_upstreams.items(): + if node_hash in pipeline._hash_graph.nodes: + if not pipeline._hash_graph.nodes[node_hash].get("node_type"): + pipeline._hash_graph.nodes[node_hash]["node_type"] = "source" + pipeline.compile() + self._compiled_pipeline = pipeline + + # Build PipelineJob's own job node map using SourceJobNode for leaves + import networkx as _nx + + job_node_map: dict[str, object] = {} + G = pipeline._hash_graph + + for node_hash in _nx.topological_sort(G): + if node_hash not in pipeline._persistent_node_map: + continue + bp_node = pipeline._persistent_node_map[node_hash] + + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + + if isinstance(bp_node, SourceNode): + concrete = self._sources.get(bp_node.name) + job_node = SourceJobNode( + name=bp_node.name, + tag_schema=bp_node.tag_schema, + data_schema=bp_node.data_schema, + concrete=concrete, + ) + + elif isinstance(bp_node, FunctionNode): + original_input_hash = bp_node._input_stream.content_hash().to_string() + upstream_job_node = job_node_map[original_input_hash] + # Get the original FunctionJobNode from rec_node_lut + rec_node = self._rec_node_lut.get(node_hash) + table_scope = rec_node._table_scope if rec_node is not None else "pipeline_hash" + job_node = FunctionJobNode( + function_pod=bp_node._function_pod, + input_stream=upstream_job_node, # type: ignore[arg-type] + label=bp_node._label, + table_scope=table_scope, + ) + + elif isinstance(bp_node, OperatorNode): + upstream_job_nodes = tuple( + job_node_map[s.content_hash().to_string()] + for s in bp_node._input_streams + ) + rec_node = self._rec_node_lut.get(node_hash) + table_scope = rec_node._table_scope if rec_node is not None else "pipeline_hash" + cache_mode = rec_node._cache_mode if rec_node is not None else None + from orcapod.types import CacheMode + job_node = OperatorJobNode( + operator=bp_node._operator, + input_streams=upstream_job_nodes, # type: ignore[arg-type] + label=bp_node._label, + cache_mode=cache_mode or CacheMode.OFF, + table_scope=table_scope, + ) + + else: + raise TypeError( + f"Unknown blueprint node type in pipeline._persistent_node_map: {type(bp_node)}" + ) + + job_node_map[node_hash] = job_node + + self._persistent_node_map = job_node_map # type: ignore[assignment] + + # Build label → job node map + self._nodes = { + label: job_node_map[node.content_hash().to_string()] # type: ignore[index] + for label, node in pipeline._nodes.items() + if node.content_hash().to_string() in job_node_map + } + + # Wire databases if store is set + if self._store is not None: + self._distribute_databases() +``` + +- [ ] **Step 6.5: Run test suite** + +```bash +uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 +``` + +Fix any failures. Common issues: +- Tests that access `job._compiled_pipeline._persistent_node_map` expecting `FunctionNode` — now it has `FunctionNode` (blueprint), but `job._persistent_node_map` has `FunctionJobNode`. Update assertions. +- Tests calling `_build_execution_graph()` — check if that method still works or needs updating. + +- [ ] **Step 6.6: Commit** + +```bash +git add src/orcapod/pipeline/job.py +git commit -m "refactor(pipeline): PipelineJob recording now creates FunctionJobNode/OperatorJobNode/SourceJobNode" +``` + +--- + +## Task 7: Update serialization for the new node format + +**Files:** +- Modify: `src/orcapod/pipeline/serialization.py` +- Modify: `src/orcapod/pipeline/graph.py` (save/load methods) +- Modify: `tests/test_pipeline/test_serialization.py` + +The key change: serialized source nodes no longer wrap a `SourceSpec` — they are plain `SourceNode` objects with `source_type: "node"`. Format versions bump. + +- [ ] **Step 7.1: Write failing serialization tests** + +Add to `tests/test_pipeline/test_serialization.py`: + +```python +class TestNewSerializationFormat: + def test_save_load_roundtrip_with_source_node(self, tmp_path, compiled_pipeline): + """Pipeline.save/load round-trip preserves SourceNode slots.""" + from orcapod.core.nodes.source_node import SourceNode + from orcapod.pipeline.graph import Pipeline + + save_path = tmp_path / "test_pipeline.json" + compiled_pipeline.save(save_path) + + loaded = Pipeline.load(save_path) + assert loaded._compiled + + for node in loaded._persistent_node_map.values(): + if node.node_type == "source": + assert isinstance(node, SourceNode) + + def test_saved_format_has_source_node_type(self, tmp_path, compiled_pipeline): + import json + + save_path = tmp_path / "test_pipeline.json" + compiled_pipeline.save(save_path) + + with open(save_path) as f: + data = json.load(f) + + for node_data in data["nodes"].values(): + if node_data["node_type"] == "source": + assert node_data.get("source_config", {}).get("source_type") == "node", ( + "Source nodes should be serialized with source_type='node'" + ) + + def test_format_version_is_0_3(self, tmp_path, compiled_pipeline): + import json + + save_path = tmp_path / "test_pipeline.json" + compiled_pipeline.save(save_path) + + with open(save_path) as f: + data = json.load(f) + + assert data["orcapod_pipeline_version"] == "0.3" +``` + +- [ ] **Step 7.2: Run tests to confirm failure** + +```bash +uv run pytest tests/test_pipeline/test_serialization.py::TestNewSerializationFormat -v 2>&1 | tail -20 +``` + +- [ ] **Step 7.3: Update Pipeline.save() for new SourceNode format** + +In `src/orcapod/pipeline/graph.py`, find `save()` and update the source-node serialization block. Find the block that serializes source nodes (currently writes `SourceSpec` fields): + +```python +# OLD (inside save() — look for the SourceNode handling block) +from orcapod.core.sources.source_spec import SourceSpec +if isinstance(node.stream, SourceSpec): + spec = node.stream + nodes[node_hash] = { + "node_type": "source", + "label": node.label, + "source_config": { + "source_type": "spec", + "name": spec.name, + "tag_schema": serialize_schema(spec.tag_schema), + "data_schema": serialize_schema(spec.data_schema), + }, + } +``` + +Replace with: + +```python +# NEW — SourceNode is the leaf directly +from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass +if isinstance(node, SourceNodeClass): + nodes[node_hash] = { + "node_type": "source", + "label": node.label, + "source_config": { + "source_type": "node", + "name": node.name, + "tag_schema": serialize_schema(node.tag_schema), + "data_schema": serialize_schema(node.data_schema), + }, + } +``` + +Update the format version constant in `src/orcapod/pipeline/serialization.py`: + +```python +# OLD: +PIPELINE_FORMAT_VERSION = "0.2" +# NEW: +PIPELINE_FORMAT_VERSION = "0.3" +``` + +- [ ] **Step 7.4: Update Pipeline.load() to reconstruct SourceNode from new format** + +In `src/orcapod/pipeline/graph.py`, find `load()` and update source node reconstruction: + +```python +# NEW — inside load(), source node handling +if node_data["node_type"] == "source": + source_config = node_data.get("source_config", {}) + source_type = source_config.get("source_type") + if source_type == "node": + from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass + from orcapod.pipeline.serialization import deserialize_schema + node = SourceNodeClass( + name=source_config["name"], + tag_schema=deserialize_schema(source_config["tag_schema"]), + data_schema=deserialize_schema(source_config["data_schema"]), + ) + elif source_type == "spec": + # Backward-compat: load old format (v0.2) that used SourceSpec + from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass + from orcapod.pipeline.serialization import deserialize_schema + node = SourceNodeClass( + name=source_config["name"], + tag_schema=deserialize_schema(source_config["tag_schema"]), + data_schema=deserialize_schema(source_config["data_schema"]), + ) + else: + raise ValueError( + f"Unknown source_type {source_type!r} in pipeline descriptor." + ) +``` + +> **Note:** For backward compatibility with v0.2 format, `source_type == "spec"` reconstructs a `SourceNode` with the same name/schemas — identical hashes are preserved since `SourceNode.identity_structure()` matches old `SourceSpec.identity_structure()`. + +- [ ] **Step 7.5: Run serialization tests** + +```bash +uv run pytest tests/test_pipeline/test_serialization.py -v --tb=short 2>&1 | tail -40 +``` + +Expected: All tests pass (including the new ones and old ones). + +- [ ] **Step 7.6: Commit** + +```bash +git add src/orcapod/pipeline/graph.py \ + src/orcapod/pipeline/serialization.py \ + tests/test_pipeline/test_serialization.py +git commit -m "refactor(serialization): update Pipeline save/load for SourceNode format; bump to v0.3" +``` + +--- + +## Task 8: Delete SourceSpec and clean up all references + +**Files:** +- Delete: `src/orcapod/core/sources/source_spec.py` +- Delete: `tests/test_core/sources/test_source_spec.py` +- Modify: `src/orcapod/core/sources/__init__.py` +- Modify: `src/orcapod/__init__.py` +- Modify: `src/orcapod/errors.py` — rename `SourceSpecMismatchError` → keep as alias +- Modify: all remaining test files that reference `SourceSpec` + +- [ ] **Step 8.1: Find all remaining SourceSpec references** + +```bash +grep -rn "SourceSpec\|source_spec" src/ tests/ --include="*.py" | grep -v "\.pyc" +``` + +Record every file that still imports or references `SourceSpec`. + +- [ ] **Step 8.2: Update src/orcapod/__init__.py** + +Replace `SourceSpec` export with `SourceNode`: + +```python +# OLD: +from .core.sources.source_spec import SourceSpec + +# NEW: +from .core.nodes.source_node import SourceNode +``` + +- [ ] **Step 8.3: Update src/orcapod/core/sources/__init__.py** + +Remove `SourceSpec` from the exports. If the file only exported `SourceSpec`, the file can be left with just the remaining exports (or emptied). + +- [ ] **Step 8.4: Update errors.py — rename SourceSpecMismatchError** + +`SourceSpecMismatchError` is the right name since it describes a schema mismatch on a source node slot. Keep the name but update the docstring: + +```python +class SourceSpecMismatchError(ValueError): + """Raised when a concrete source's schema is incompatible with a SourceNode slot. + + Previously named in terms of ``SourceSpec``; the error class name is preserved + for compatibility with any code that catches it by name. + """ +``` + +- [ ] **Step 8.5: Update remaining SourceSpec references in test files** + +For each file from Step 8.1 that still imports `SourceSpec`: + +**`tests/test_pipeline/test_pipeline.py`** — replace: +```python +# OLD: +from orcapod.core.sources.source_spec import SourceSpec +... +spec_a = SourceSpec(name="a", tag_schema=..., data_schema=...) +with pipeline: + result = my_pod(spec_a) + +# NEW: +from orcapod.core.nodes.source_node import SourceNode +... +spec_a = SourceNode(name="a", tag_schema=..., data_schema=...) +with pipeline: + result = my_pod(spec_a) +``` + +**`tests/test_pipeline/test_pipeline_job.py`** — same pattern. + +**`tests/test_core/test_tracker.py`** — if it uses `SourceSpec`, replace with `SourceNode`. + +Run after each file update: + +```bash +uv run pytest -v --tb=short 2>&1 | tail -20 +``` + +- [ ] **Step 8.6: Delete source_spec.py** + +```bash +git rm src/orcapod/core/sources/source_spec.py +git rm tests/test_core/sources/test_source_spec.py +``` + +- [ ] **Step 8.7: Run full test suite** + +```bash +uv run pytest tests/ -v --tb=short 2>&1 | tail -40 +``` + +Expected: All tests pass, no `SourceSpec` references remaining. + +- [ ] **Step 8.8: Final grep to confirm SourceSpec is gone** + +```bash +grep -rn "SourceSpec\|source_spec" src/ tests/ --include="*.py" +``` + +Expected: No output (zero matches). + +- [ ] **Step 8.9: Commit** + +```bash +git add -A +git commit -m "refactor(sources): delete SourceSpec; update all references to SourceNode (ENG-493)" +``` + +--- + +## Task 9: Integration test sweep and PR + +**Files:** +- Run all tests, fix stragglers +- Create PR + +- [ ] **Step 9.1: Run full test suite including integration tests** + +```bash +uv run pytest tests/ -v --tb=short 2>&1 | tee /tmp/test_results.txt +grep -E "FAILED|ERROR" /tmp/test_results.txt +``` + +- [ ] **Step 9.2: Fix any remaining failures** + +Common failure patterns at this stage: +- `_build_execution_graph()` in `job.py` may still reference `SourceSpec` — update to `SourceNode` +- Serialization load tests for old v0.2 format — verify backward-compat path works +- Orchestrator tests that build execution graphs — update to use `from_pipeline()` pattern + +For each failure, diagnose root cause, implement fix, re-run the specific test. + +- [ ] **Step 9.3: Verify hash stability end-to-end** + +```bash +uv run python -c " +from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.sources.source_spec import SourceSpec +from orcapod.types import Schema + +tag = Schema({'id': int}) +data = Schema({'value': float}) + +# These MUST be equal for DB path stability +old_hash = SourceSpec(name='x', tag_schema=tag, data_schema=data).content_hash() +new_hash = SourceNode(name='x', tag_schema=tag, data_schema=data).content_hash() + +assert old_hash == new_hash, f'Hash mismatch! old={old_hash}, new={new_hash}' +print('Hash stability: PASS') + +old_pipe = SourceSpec(name='x', tag_schema=tag, data_schema=data).pipeline_hash() +new_pipe = SourceNode(name='x', tag_schema=tag, data_schema=data).pipeline_hash() +assert old_pipe == new_pipe, f'Pipeline hash mismatch!' +print('Pipeline hash stability: PASS') +" +``` + +> **Note:** This check must pass before the PR is created. If SourceSpec has already been deleted at this point, adjust to compare against a recorded reference hash instead. + +- [ ] **Step 9.4: Update src/orcapod/__init__.py exports** + +Ensure `SourceNode` is properly exported (was done in Task 8, verify): + +```python +from .core.nodes.source_node import SourceNode +from .pipeline.job import PipelineJob +from .pipeline.graph import Pipeline +``` + +- [ ] **Step 9.5: Final test run** + +```bash +uv run pytest tests/ --tb=short 2>&1 | tail -10 +``` + +Expected: `X passed, 0 failed, 0 errors` + +- [ ] **Step 9.6: Commit any remaining fixes** + +```bash +git add -A +git commit -m "fix(pipeline): address integration test failures after pure-descriptor refactor" +``` + +- [ ] **Step 9.7: Push and create PR** + +```bash +git push -u origin eywalker/eng-493-refactor-pipeline-into-a-pure-computational-descriptor +``` + +Create PR targeting `dev`: + +```bash +gh pr create \ + --title "refactor(pipeline): Pipeline pure-descriptor refactor — ENG-493" \ + --base dev \ + --body "$(cat <<'EOF' +## Summary + +- `Pipeline` now stores only lightweight blueprint nodes (`FunctionNode`, `OperatorNode`, `SourceNode`) with no DB references +- `PipelineJob` stores DB-backed job nodes (`FunctionJobNode`, `OperatorJobNode`, `SourceJobNode`) +- `SourceSpec` deleted; replaced by `SourceNode` with bit-identical `content_hash()` / `pipeline_hash()` (DB paths preserved) +- `Pipeline.bind()` removed; replaced by `PipelineJob.from_pipeline(pipeline, store=..., sources=...)` +- `PipelineJob.bind()` is now mutating (returns `None`); updates `SourceJobNode._concrete` in-place +- `PipelineJob.as_pipeline()` produces a lightweight `Pipeline` from a job +- `AbstractPipelineBase` extracted to `pipeline/base.py` — shared recording machinery +- Serialization format bumped to v0.3 (backward-compatible load of v0.2) + +Closes ENG-493 + +## Test plan +- [ ] `uv run pytest tests/test_core/nodes/` — new node hierarchy unit tests +- [ ] `uv run pytest tests/test_pipeline/` — full pipeline test suite +- [ ] `uv run pytest tests/` — complete test suite + +🤖 Generated with [Claude Code](https://claude.com/claude-code) +EOF +)" +``` + +--- + +## Self-Review Against Spec + +### Spec coverage check + +| Spec requirement | Covered by task | +|---|---| +| `Pipeline._persistent_node_map` contains only lightweight nodes | Tasks 5, 6 | +| `PipelineJob._persistent_node_map` contains only JobNodes | Task 6 | +| `SourceSpec` eliminated | Task 8 | +| `SourceNode` as user-facing input slot | Task 1 | +| `Pipeline.bind()` removed | Task 4 | +| `PipelineJob.from_pipeline()` classmethod | Task 4 | +| `PipelineJob.bind()` mutating | Task 4 | +| `PipelineJob.as_pipeline()` | Task 4 | +| `SourceJobNode._concrete` mutable for in-place update | Task 1 | +| Hash stability: `SourceNode.content_hash() == SourceSpec.content_hash()` | Task 1 (tests) | +| `pipeline_hash()` always schema-based for `SourceJobNode` | Task 1 (tests) | +| `FunctionNode.iter_data()` raises `PipelineJobRequiredError` | Task 2 | +| `FunctionJobNode.as_node()` returns `FunctionNode` | Task 2 | +| `OperatorNode.iter_data()` raises `PipelineJobRequiredError` | Task 3 | +| `OperatorJobNode.as_node()` returns `OperatorNode` | Task 3 | +| `AbstractPipelineBase` in `pipeline/base.py` | Task 5 | +| Serialization format v0.3, backward-compat v0.2 load | Task 7 | +| All tests updated | Tasks 1–8 | + +### Notes on `_build_execution_graph()` + +The current `PipelineJob._build_execution_graph()` creates `FunctionNode`/`OperatorNode` with DB attached from scratch. After Task 6, the `_persistent_node_map` already has `FunctionJobNode`/`OperatorJobNode`. The `_build_execution_graph()` method may need updating or removal — it may be replaced by `_distribute_databases()` + the new job node map. Check what calls it (orchestrators) and update those call sites. + +### `tracker_manager` parameter in AbstractPipelineBase + +`PipelineJob.__init__` currently does not accept `name` or `tracker_manager` via parent `__init__` in the same form. Ensure the `super().__init__` chains are correct after Task 5. From a82b19fb7c054fad3a2d7b0875e64fe76d856e6d Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 00:17:16 +0000 Subject: [PATCH 03/24] =?UTF-8?q?docs(pipeline):=20correct=20FunctionNode/?= =?UTF-8?q?OperatorNode=20split=20=E2=80=94=20proper=20sibling=20base=20cl?= =?UTF-8?q?ass=20hierarchy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...-05-21-eng-493-pipeline-pure-descriptor.md | 546 ++++++++++++++---- 1 file changed, 429 insertions(+), 117 deletions(-) diff --git a/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md b/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md index 453c95eb1..635074e6f 100644 --- a/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md +++ b/superpowers/plans/2026-05-21-eng-493-pipeline-pure-descriptor.md @@ -898,7 +898,37 @@ git commit -m "refactor(nodes): replace SourceSpec with schema-only SourceNode + - Modify: `src/orcapod/core/nodes/__init__.py` - Create: `tests/test_core/nodes/test_function_node_split.py` -**Strategy:** `FunctionJobNode` is essentially the current `FunctionNode` renamed. The new thin `FunctionNode` just raises `PipelineJobRequiredError` on `iter_data()`. `FunctionNodeBase` holds the shared constructor, properties, and `from_descriptor` logic. +**Class hierarchy (both siblings inherit from the same base — neither inherits from the other):** + +``` +FunctionNodeBase(StreamBase) +│ __init__(function_pod, input_stream, label, tracker_manager, config, table_scope) +│ Shared: load_status, producer, data_context, data_context_key, executor, +│ upstreams property/setter, content_hash() override (read-only mode), +│ pipeline_hash() override (read-only mode), output_schema(), keys(), +│ node_identity_path, node_uri, clear_cache() (identity cache only) +│ +├── FunctionNode(FunctionNodeBase) +│ __init__: same params as base, no DB params +│ iter_data(): raises PipelineJobRequiredError +│ as_node(): returns self +│ +└── FunctionJobNode(FunctionNodeBase) + __init__: adds pipeline_database, result_database params + Additional state: _pipeline_database, _cached_function_pod, _output_schema_hash, + _cached_output_datas, _cached_output_table, _cached_content_hash_column + attach_databases(), _require_pipeline_database(), _filter_by_content_hash() + from_descriptor() classmethod + clear_cache() override: clears DB cache state too + execute(), execute_data(), iter_data() (two-phase), async methods, + get_cached_results(), as_source() + as_node(): returns FunctionNode(function_pod, input_stream, label, table_scope) +``` + +**Split strategy for the 1466-line current `FunctionNode`:** +- `FunctionNodeBase` gets everything from the current `__init__` that is *not* DB state, plus all shared properties (lines ~76–534 of current file minus DB fields). +- `FunctionNode` is a new thin class with no extra state. +- `FunctionJobNode` gets the remaining DB state + all execution methods (lines ~530–1466 plus DB fields from `__init__`). - [ ] **Step 2.1: Write failing tests** @@ -1034,70 +1064,121 @@ uv run pytest tests/test_core/nodes/test_function_node_split.py -v 2>&1 | tail - Expected: `ImportError: cannot import name 'FunctionJobNode'` -- [ ] **Step 2.3: Refactor function_node.py** +- [ ] **Step 2.3: Rewrite function_node.py with proper base class** -The current `FunctionNode` class (1466 lines) becomes the new `FunctionJobNode`. The new thin `FunctionNode` only raises on `iter_data()`. Both share `FunctionNodeBase` which holds the constructor, identity, properties, and `from_descriptor` logic. +Restructure `src/orcapod/core/nodes/function_node.py` into three classes. Keep all existing +logic intact — just redistribute it. The file will still be long; the important thing is +clean ownership of state and methods. -At the top of `src/orcapod/core/nodes/function_node.py`, after the existing imports, add: +**a) `FunctionNodeBase(StreamBase)` — new class, replaces the top of the current `FunctionNode`** + +Carries everything that does not touch a database. Its `__init__` is the current +`FunctionNode.__init__` with the DB parameters and DB-state initialisation removed: ```python -from orcapod.errors import PipelineJobRequiredError -``` +class FunctionNodeBase(StreamBase): + """Shared identity, properties, and schema logic for FunctionNode and FunctionJobNode. -Then restructure the class hierarchy as follows (keeping all existing logic intact): + Neither subtype is a subtype of the other — both inherit directly from this base. + This class carries no database references. + """ -**a) Rename the current `FunctionNode` class to `FunctionJobNode`** by adding a `FunctionJobNode` alias at the bottom of the file **first**, then creating the new thin `FunctionNode`: + node_type = "function" -At the **bottom** of `function_node.py`, after the existing `FunctionNode` class, add: + def __init__( + self, + function_pod: FunctionPodProtocol, + input_stream: StreamProtocol, + tracker_manager: TrackerManagerProtocol | None = None, + label: str | None = None, + config: Config | None = None, + table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", + ) -> None: + if tracker_manager is None: + tracker_manager = DEFAULT_TRACKER_MANAGER + self.tracker_manager = tracker_manager + self._data_function = function_pod.data_function + self._function_pod = function_pod + super().__init__(label=label, config=config) + + # Schema validation (skip for UNAVAILABLE streams) + from orcapod.pipeline.serialization import LoadStatus + _stream_unavailable = ( + hasattr(input_stream, "load_status") + and input_stream.load_status == LoadStatus.UNAVAILABLE + ) + if not _stream_unavailable: + _, incoming_data_types = input_stream.output_schema() + expected_data_schema = self._data_function.input_data_schema + if not schema_utils.check_schema_compatibility( + incoming_data_types, expected_data_schema + ): + raise ValueError( + f"Incoming data type {incoming_data_types} from {input_stream} " + f"is not compatible with expected input schema {expected_data_schema}" + ) -```python -# FunctionJobNode is the DB-backed execution variant of FunctionNode. -# It is the existing FunctionNode class renamed. -FunctionJobNode = FunctionNode + self._input_stream = input_stream + if table_scope not in ("pipeline_hash", "content_hash"): + raise ValueError( + f"Unknown table_scope {table_scope!r}. " + "Expected one of: 'pipeline_hash', 'content_hash'." + ) + self._table_scope = table_scope + self._node_identity_path_cache: tuple[str, ...] | None = None + + # Descriptor fields for read-only / UNAVAILABLE deserialized nodes + from orcapod.pipeline.serialization import LoadStatus + self._load_status: LoadStatus = LoadStatus.FULL + self._stored_content_hash: str | None = None + self._stored_pipeline_hash: str | None = None + self._stored_schema: dict = {} + self._stored_node_uri: tuple[str, ...] = () + self._stored_pipeline_path: tuple[str, ...] = () + self._stored_result_record_path: tuple[str, ...] = () + self._descriptor: dict = {} + + # Copy these shared properties verbatim from the current FunctionNode: + # load_status, producer, data_context, data_context_key, executor (+ setter), + # upstreams (+ setter), content_hash() override, pipeline_hash() override, + # output_schema(), keys(), node_identity_path, node_uri + # + # clear_cache() in base clears only the identity path cache: + + def clear_cache(self) -> None: + self._node_identity_path_cache = None + self._update_modified_time() +``` + +**b) `FunctionNode(FunctionNodeBase)` — thin blueprint node** -class FunctionNode(FunctionJobNode): # type: ignore[no-redef] +```python +class FunctionNode(FunctionNodeBase): """Lightweight blueprint node for ``Pipeline`` recording. - Carries no database references. Calling ``iter_data()`` raises - ``PipelineJobRequiredError`` — wrap the containing ``Pipeline`` in a - ``PipelineJob`` to obtain an executable ``FunctionJobNode``. - - All identity methods (``content_hash``, ``pipeline_hash``, - ``output_schema``) are inherited from ``FunctionJobNode`` and produce - values identical to those of the corresponding ``FunctionJobNode`` - constructed with the same arguments. + Carries no database references. ``iter_data()`` raises + ``PipelineJobRequiredError``. Use ``PipelineJob.from_pipeline()`` to + obtain an executable ``FunctionJobNode``. Args: function_pod: The wrapped function pod. - input_stream: The upstream stream (must be a ``SourceNode`` or - another blueprint node). + input_stream: Upstream stream (``SourceNode`` or another blueprint node). + tracker_manager: Optional tracker manager override. label: Optional display label. + config: Optional node config. + table_scope: DB table scoping strategy (preserved for hash stability when + later converted to a ``FunctionJobNode``). """ - def __init__( - self, - function_pod: "FunctionPodProtocol", - input_stream: "StreamProtocol", - tracker_manager: "TrackerManagerProtocol | None" = None, - label: str | None = None, - config: "Config | None" = None, - ) -> None: - # Construct without any DB parameters - super().__init__( - function_pod=function_pod, - input_stream=input_stream, - tracker_manager=tracker_manager, - label=label, - config=config, - ) - def iter_data(self): - """Raise PipelineJobRequiredError — blueprint node cannot produce data. + """Raise PipelineJobRequiredError — blueprint nodes cannot produce data. Raises: PipelineJobRequiredError: Always. """ + from orcapod.errors import PipelineJobRequiredError + raise PipelineJobRequiredError( f"FunctionNode '{self.label}' is a blueprint node and cannot produce data. " "Wrap the containing Pipeline in a PipelineJob to execute:\n" @@ -1106,32 +1187,99 @@ class FunctionNode(FunctionJobNode): # type: ignore[no-redef] ) def as_node(self) -> "FunctionNode": - """Return self — already a lightweight node. - - Returns: - ``self`` - """ + """Return self — already a lightweight blueprint node.""" return self +``` +**c) `FunctionJobNode(FunctionNodeBase)` — DB-backed execution node** -# Patch FunctionJobNode.as_node() to return the lightweight FunctionNode variant -def _function_job_node_as_node(self) -> "FunctionNode": - """Return a lightweight ``FunctionNode`` with the same identity. +```python +class FunctionJobNode(FunctionNodeBase): + """DB-backed execution node for ``PipelineJob`` graphs. - Returns: - A new ``FunctionNode`` with the same function_pod, input_stream, and label. + Adds database references and all execution logic on top of + ``FunctionNodeBase``. ``FunctionNode`` and ``FunctionJobNode`` + are siblings — neither inherits from the other. + + Args: + function_pod: The wrapped function pod. + input_stream: Upstream stream (``SourceJobNode`` or another job node). + tracker_manager: Optional tracker manager override. + label: Optional display label. + config: Optional node config. + table_scope: DB table scoping strategy. + pipeline_database: Optional database for pipeline records. + result_database: Optional database for cached results. """ - return FunctionNode( - function_pod=self._function_pod, - input_stream=self._input_stream, - label=self._label, - ) + def __init__( + self, + function_pod: FunctionPodProtocol, + input_stream: StreamProtocol, + tracker_manager: TrackerManagerProtocol | None = None, + label: str | None = None, + config: Config | None = None, + table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", + pipeline_database: ArrowDatabaseProtocol | None = None, + result_database: ArrowDatabaseProtocol | None = None, + ) -> None: + super().__init__( + function_pod=function_pod, + input_stream=input_stream, + tracker_manager=tracker_manager, + label=label, + config=config, + table_scope=table_scope, + ) + # DB-specific state + self._cached_output_datas: dict[str, tuple[TagProtocol, DataProtocol | None]] = {} + self._cached_output_table: pa.Table | None = None + self._cached_content_hash_column: pa.Array | None = None + self._pipeline_database: ArrowDatabaseProtocol | None = None + self._cached_function_pod: CachedFunctionPod | None = None + self._output_schema_hash: str | None = None + + if pipeline_database is not None: + self.attach_databases( + pipeline_database=pipeline_database, + result_database=result_database, + ) + + def clear_cache(self) -> None: + """Clear all caches including DB-backed output cache.""" + self._cached_output_datas.clear() + self._cached_output_table = None + self._cached_content_hash_column = None + self._node_identity_path_cache = None + self._update_modified_time() + + def as_node(self) -> FunctionNode: + """Return a lightweight ``FunctionNode`` with the same identity. -FunctionJobNode.as_node = _function_job_node_as_node # type: ignore[method-assign] + Returns: + A new ``FunctionNode`` carrying the same function_pod, input_stream, + label, and table_scope — and therefore identical content_hash() and + pipeline_hash() values. + """ + return FunctionNode( + function_pod=self._function_pod, + input_stream=self._input_stream, + label=self._label, + table_scope=self._table_scope, + ) + + # Move these verbatim from the current FunctionNode (they already exist there): + # attach_databases(), _require_pipeline_database(), _filter_by_content_hash(), + # from_descriptor() classmethod, execute_data(), execute(), iter_data() (two-phase), + # _process_data_internal(), get_cached_results(), as_source(), async methods. ``` -> **Note on architecture:** This "alias + subclass" approach avoids duplicating 1400 lines of code. `FunctionJobNode` = the existing class unchanged. `FunctionNode` = thin subclass that overrides only `__init__` and `iter_data`. Both have identical `content_hash()` / `pipeline_hash()` because `__init__` sets identical state. +> **Implementation note:** The bulk of `FunctionJobNode` is a verbatim move of the current +> `FunctionNode`'s DB methods. No logic changes are needed — only re-homing the code into +> the new class and adjusting the `__init__` to call `super().__init__()` without DB params. +> Any place that currently says `isinstance(node, FunctionNode)` in execution-path code +> should be updated to `isinstance(node, FunctionJobNode)` where the intent is "DB-capable +> node"; use `isinstance(node, FunctionNodeBase)` where the intent is "any function node". - [ ] **Step 2.4: Run new tests** @@ -1141,12 +1289,33 @@ uv run pytest tests/test_core/nodes/test_function_node_split.py -v Expected: All 7 tests pass. -- [ ] **Step 2.5: Update nodes/__init__.py** +- [ ] **Step 2.5: Verify isinstance hierarchy is correct** + +```bash +uv run python -c " +from orcapod.core.nodes.function_node import FunctionNode, FunctionJobNode, FunctionNodeBase +from orcapod.core.streams.base import StreamBase + +# Neither should be a subtype of the other +assert not issubclass(FunctionNode, FunctionJobNode), 'FunctionNode must NOT inherit from FunctionJobNode' +assert not issubclass(FunctionJobNode, FunctionNode), 'FunctionJobNode must NOT inherit from FunctionNode' + +# Both should share the base +assert issubclass(FunctionNode, FunctionNodeBase) +assert issubclass(FunctionJobNode, FunctionNodeBase) +assert issubclass(FunctionNodeBase, StreamBase) +print('isinstance hierarchy: CORRECT') +" +``` + +Expected: `isinstance hierarchy: CORRECT` + +- [ ] **Step 2.6: Update nodes/__init__.py** ```python from typing import TypeAlias -from .function_node import FunctionJobNode, FunctionNode +from .function_node import FunctionJobNode, FunctionNode, FunctionNodeBase from .operator_node import OperatorNode from .source_node import SourceJobNode, SourceNode, SourceNodeBase @@ -1156,6 +1325,7 @@ JobNode: TypeAlias = SourceJobNode | FunctionJobNode __all__ = [ "FunctionJobNode", "FunctionNode", + "FunctionNodeBase", "GraphNode", "JobNode", "OperatorNode", @@ -1165,21 +1335,23 @@ __all__ = [ ] ``` -- [ ] **Step 2.6: Run full pipeline test suite** +- [ ] **Step 2.7: Run full pipeline test suite** ```bash uv run pytest tests/test_pipeline/ tests/test_core/ -v --tb=short 2>&1 | tail -40 ``` -Expected: All tests pass. If any tests create `FunctionNode(..., pipeline_database=...)` directly, those still work because `FunctionNode` subclasses `FunctionJobNode` and passes kwargs up. +Any test that instantiated `FunctionNode(..., pipeline_database=...)` directly will now +fail because `FunctionNode` no longer accepts DB params. Update those call sites to use +`FunctionJobNode` instead. -- [ ] **Step 2.7: Commit** +- [ ] **Step 2.8: Commit** ```bash git add src/orcapod/core/nodes/function_node.py \ src/orcapod/core/nodes/__init__.py \ tests/test_core/nodes/test_function_node_split.py -git commit -m "refactor(nodes): split FunctionNode into thin FunctionNode + FunctionJobNode" +git commit -m "refactor(nodes): split FunctionNode → FunctionNodeBase + FunctionNode + FunctionJobNode" ``` --- @@ -1191,7 +1363,34 @@ git commit -m "refactor(nodes): split FunctionNode into thin FunctionNode + Func - Modify: `src/orcapod/core/nodes/__init__.py` - Create: `tests/test_core/nodes/test_operator_node_split.py` -Same pattern as Task 2: `OperatorJobNode` = existing `OperatorNode` renamed. Thin `OperatorNode` overrides `iter_data()` to raise `PipelineJobRequiredError`. +**Class hierarchy (same sibling pattern as Task 2 — neither inherits from the other):** + +``` +OperatorNodeBase(StreamBase) +│ __init__(operator, input_streams, label, tracker_manager, config, table_scope) +│ Shared: load_status, identity_structure(), pipeline_identity_structure(), +│ content_hash() override, pipeline_hash() override, producer, +│ data_context, data_context_key, upstreams property/setter, +│ keys(), output_schema(), node_identity_path, node_uri +│ +├── OperatorNode(OperatorNodeBase) +│ iter_data(): raises PipelineJobRequiredError +│ as_node(): returns self +│ +└── OperatorJobNode(OperatorNodeBase) + __init__: adds pipeline_database, cache_mode params + Additional state: _pipeline_database, _cache_mode, + _cached_output_stream, _cached_output_table + attach_databases(), from_descriptor() classmethod, + execute(), run(), iter_data() (cache-mode aware), + as_table(), get_all_records(), as_source(), async methods + as_node(): returns OperatorNode(operator, input_streams, label, table_scope) +``` + +**Split strategy for the current 903-line `OperatorNode`:** +- `OperatorNodeBase` gets everything from `__init__` that is not DB state, plus all shared properties/identity methods. +- `OperatorNode` is a thin class with no extra state. +- `OperatorJobNode` gets the DB state + all execution methods. - [ ] **Step 3.1: Write failing tests** @@ -1281,49 +1480,82 @@ uv run pytest tests/test_core/nodes/test_operator_node_split.py -v 2>&1 | tail - Expected: `ImportError: cannot import name 'OperatorJobNode'` -- [ ] **Step 3.3: Refactor operator_node.py** +- [ ] **Step 3.3: Rewrite operator_node.py with proper base class** -At the bottom of `src/orcapod/core/nodes/operator_node.py`, add: +Restructure `src/orcapod/core/nodes/operator_node.py` into three classes following the +same pattern as Task 2. -```python -# OperatorJobNode is the DB-backed execution variant of OperatorNode. -# It is the existing OperatorNode class renamed. -OperatorJobNode = OperatorNode - - -class OperatorNode(OperatorJobNode): # type: ignore[no-redef] - """Lightweight blueprint node for ``Pipeline`` recording. +**a) `OperatorNodeBase(StreamBase)` — new class** - Carries no database references. Calling ``iter_data()`` raises - ``PipelineJobRequiredError``. All identity methods are inherited from - ``OperatorJobNode`` and produce identical hashes. +```python +class OperatorNodeBase(StreamBase): + """Shared identity, properties, and schema logic for OperatorNode and OperatorJobNode. - Args: - operator: The wrapped operator pod. - input_streams: Upstream streams (``SourceNode`` instances or other - blueprint nodes). - label: Optional display label. + Neither subtype is a subtype of the other — both inherit directly from this base. + This class carries no database references. """ + node_type = "operator" + HASH_COLUMN_NAME = "_record_hash" + def __init__( self, - operator: "OperatorPodProtocol", - input_streams: "tuple[StreamProtocol, ...] | list[StreamProtocol]", - tracker_manager: "TrackerManagerProtocol | None" = None, + operator: OperatorPodProtocol, + input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], + tracker_manager: TrackerManagerProtocol | None = None, label: str | None = None, - config: "Config | None" = None, + config: Config | None = None, + table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", ) -> None: - # Construct without any DB parameters - super().__init__( - operator=operator, - input_streams=input_streams, - tracker_manager=tracker_manager, - label=label, - config=config, - ) + if tracker_manager is None: + tracker_manager = DEFAULT_TRACKER_MANAGER + self.tracker_manager = tracker_manager + self._operator = operator + self._input_streams = tuple(input_streams) + super().__init__(label=label, config=config) + + # Eager input validation + self._operator.validate_inputs(*self._input_streams) + + if table_scope not in ("pipeline_hash", "content_hash"): + raise ValueError( + f"Unknown table_scope {table_scope!r}. " + "Expected one of: 'pipeline_hash', 'content_hash'." + ) + self._table_scope = table_scope + self._node_identity_path_cache: tuple[str, ...] | None = None + self._set_modified_time(None) + + # Descriptor fields for read-only / UNAVAILABLE deserialized nodes + from orcapod.pipeline.serialization import LoadStatus + self._load_status: LoadStatus = LoadStatus.FULL + self._stored_content_hash: str | None = None + self._stored_pipeline_hash: str | None = None + self._stored_schema: dict = {} + self._stored_node_uri: tuple[str, ...] = () + self._stored_pipeline_path: tuple[str, ...] = () + self._descriptor: dict = {} + + # Copy these verbatim from the current OperatorNode (they already exist there): + # load_status property, identity_structure(), pipeline_identity_structure(), + # content_hash() override, pipeline_hash() override, + # producer, data_context, data_context_key, upstreams property/setter, + # keys(), output_schema(), node_identity_path, node_uri +``` + +**b) `OperatorNode(OperatorNodeBase)` — thin blueprint node** + +```python +class OperatorNode(OperatorNodeBase): + """Lightweight blueprint node for ``Pipeline`` recording. + + Carries no database references. ``iter_data()`` raises + ``PipelineJobRequiredError``. Use ``PipelineJob.from_pipeline()`` to + obtain an executable ``OperatorJobNode``. + """ def iter_data(self): - """Raise PipelineJobRequiredError — blueprint node cannot produce data. + """Raise PipelineJobRequiredError — blueprint nodes cannot produce data. Raises: PipelineJobRequiredError: Always. @@ -1338,36 +1570,113 @@ class OperatorNode(OperatorJobNode): # type: ignore[no-redef] ) def as_node(self) -> "OperatorNode": - """Return self — already a lightweight node.""" + """Return self — already a lightweight blueprint node.""" return self +``` +**c) `OperatorJobNode(OperatorNodeBase)` — DB-backed execution node** -# Patch OperatorJobNode.as_node() to return the lightweight OperatorNode variant -def _operator_job_node_as_node(self) -> "OperatorNode": - """Return a lightweight ``OperatorNode`` with the same identity.""" - return OperatorNode( - operator=self._operator, - input_streams=self._input_streams, - label=self._label, - ) +```python +class OperatorJobNode(OperatorNodeBase): + """DB-backed execution node for ``PipelineJob`` graphs. + + ``OperatorNode`` and ``OperatorJobNode`` are siblings — neither inherits + from the other. + Args: + operator: The wrapped operator pod. + input_streams: Upstream job nodes. + tracker_manager: Optional tracker manager override. + label: Optional display label. + config: Optional node config. + table_scope: DB table scoping strategy. + pipeline_database: Optional database for pipeline records. + cache_mode: Caching behaviour (OFF / LOG / REPLAY). + """ -OperatorJobNode.as_node = _operator_job_node_as_node # type: ignore[method-assign] + def __init__( + self, + operator: OperatorPodProtocol, + input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], + tracker_manager: TrackerManagerProtocol | None = None, + label: str | None = None, + config: Config | None = None, + table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", + pipeline_database: ArrowDatabaseProtocol | None = None, + cache_mode: CacheMode = CacheMode.OFF, + ) -> None: + super().__init__( + operator=operator, + input_streams=input_streams, + tracker_manager=tracker_manager, + label=label, + config=config, + table_scope=table_scope, + ) + # DB-specific state + self._cached_output_stream: StreamProtocol | None = None + self._cached_output_table: pa.Table | None = None + self._pipeline_database: ArrowDatabaseProtocol | None = None + self._cache_mode = CacheMode.OFF + + if pipeline_database is not None: + self.attach_databases( + pipeline_database=pipeline_database, + cache_mode=cache_mode, + ) + + def as_node(self) -> OperatorNode: + """Return a lightweight ``OperatorNode`` with the same identity. + + Returns: + A new ``OperatorNode`` carrying the same operator, input_streams, + label, and table_scope — with identical content_hash() and + pipeline_hash() values. + """ + return OperatorNode( + operator=self._operator, + input_streams=self._input_streams, + label=self._label, + table_scope=self._table_scope, + ) + + # Move these verbatim from the current OperatorNode (they already exist there): + # attach_databases(), from_descriptor() classmethod, + # execute(), run(), iter_data() (cache-mode aware), + # as_table(), get_all_records(), as_source(), async methods. ``` -Also add this import at the top of the file (with the other imports): +> **Implementation note:** Same approach as Task 2. The bulk of `OperatorJobNode` is a +> verbatim move of the current `OperatorNode`'s DB methods. Update any `isinstance(node, +> OperatorNode)` in execution-path code to `isinstance(node, OperatorJobNode)` where the +> intent is "DB-capable node"; use `isinstance(node, OperatorNodeBase)` where the intent +> is "any operator node". -```python -# (PipelineJobRequiredError is imported lazily inside iter_data to avoid circular import) +- [ ] **Step 3.4: Verify isinstance hierarchy is correct** + +```bash +uv run python -c " +from orcapod.core.nodes.operator_node import OperatorNode, OperatorJobNode, OperatorNodeBase +from orcapod.core.streams.base import StreamBase + +assert not issubclass(OperatorNode, OperatorJobNode), 'OperatorNode must NOT inherit from OperatorJobNode' +assert not issubclass(OperatorJobNode, OperatorNode), 'OperatorJobNode must NOT inherit from OperatorNode' +assert issubclass(OperatorNode, OperatorNodeBase) +assert issubclass(OperatorJobNode, OperatorNodeBase) +assert issubclass(OperatorNodeBase, StreamBase) +print('isinstance hierarchy: CORRECT') +" ``` -- [ ] **Step 3.4: Update nodes/__init__.py to export OperatorJobNode** +Expected: `isinstance hierarchy: CORRECT` + +- [ ] **Step 3.5: Update nodes/__init__.py to export all new types** ```python from typing import TypeAlias -from .function_node import FunctionJobNode, FunctionNode -from .operator_node import OperatorJobNode, OperatorNode +from .function_node import FunctionJobNode, FunctionNode, FunctionNodeBase +from .operator_node import OperatorJobNode, OperatorNode, OperatorNodeBase from .source_node import SourceJobNode, SourceNode, SourceNodeBase GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode @@ -1376,31 +1685,34 @@ JobNode: TypeAlias = SourceJobNode | FunctionJobNode | OperatorJobNode __all__ = [ "FunctionJobNode", "FunctionNode", + "FunctionNodeBase", "GraphNode", "JobNode", "OperatorJobNode", "OperatorNode", + "OperatorNodeBase", "SourceJobNode", "SourceNode", "SourceNodeBase", ] ``` -- [ ] **Step 3.5: Run new and existing tests** +- [ ] **Step 3.6: Run new and existing tests** ```bash uv run pytest tests/test_core/nodes/test_operator_node_split.py tests/test_pipeline/ -v --tb=short 2>&1 | tail -40 ``` -Expected: All pass. +Any test that instantiated `OperatorNode(..., pipeline_database=...)` directly will now +fail — update those to `OperatorJobNode` instead. -- [ ] **Step 3.6: Commit** +- [ ] **Step 3.7: Commit** ```bash git add src/orcapod/core/nodes/operator_node.py \ src/orcapod/core/nodes/__init__.py \ tests/test_core/nodes/test_operator_node_split.py -git commit -m "refactor(nodes): split OperatorNode into thin OperatorNode + OperatorJobNode" +git commit -m "refactor(nodes): split OperatorNode → OperatorNodeBase + OperatorNode + OperatorJobNode" ``` --- From 65fc68f4c48fa3ef0e6ea2492fb3f248265dde04 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 01:19:14 +0000 Subject: [PATCH 04/24] refactor(nodes): replace SourceSpec with schema-only SourceNode + add SourceJobNode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements Task 1 of ENG-493. Introduces a new two-class hierarchy: - SourceNode: schema-only input-slot declaration (replaces SourceSpec as the user-facing leaf for Pipeline recording). Raises UnboundSourceError on data access. - SourceJobNode: execution-ready variant wrapping a concrete StreamProtocol. Used internally by PipelineJob._build_execution_graph(). Both classes share SourceNodeBase which provides hash-stable identity via identity_structure() = ("SourceSpec", name, tag_schema, data_schema) — identical to the old SourceSpec — preserving all existing DB paths. Pipeline.compile() now accepts SourceNode instances directly as leaf inputs (no wrapping). PipelineJob creates SourceJobNode with bound concrete sources at run time. Serialization uses source_type="node" with node_name key. All 3048 tests pass (160 skipped as expected). Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/__init__.py | 4 +- src/orcapod/core/nodes/source_node.py | 580 +++++++++++------- src/orcapod/errors.py | 9 + src/orcapod/pipeline/graph.py | 71 ++- src/orcapod/pipeline/job.py | 144 ++--- tests/test_core/nodes/test_source_node.py | 170 +++++ tests/test_core/test_tracker.py | 181 +++--- tests/test_pipeline/test_node_descriptors.py | 154 ++--- tests/test_pipeline/test_node_protocols.py | 30 +- tests/test_pipeline/test_orchestrator.py | 22 +- tests/test_pipeline/test_pipeline.py | 20 +- tests/test_pipeline/test_pipeline_job.py | 81 +-- tests/test_pipeline/test_serialization.py | 28 +- tests/test_pipeline/test_sync_orchestrator.py | 12 +- tests/test_protocols/test_node_protocols.py | 10 +- 15 files changed, 949 insertions(+), 567 deletions(-) create mode 100644 tests/test_core/nodes/test_source_node.py diff --git a/src/orcapod/core/nodes/__init__.py b/src/orcapod/core/nodes/__init__.py index 5d2ef1ee3..65a7cf069 100644 --- a/src/orcapod/core/nodes/__init__.py +++ b/src/orcapod/core/nodes/__init__.py @@ -2,7 +2,7 @@ from .function_node import FunctionNode from .operator_node import OperatorNode -from .source_node import SourceNode +from .source_node import SourceJobNode, SourceNode, SourceNodeBase GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode @@ -10,5 +10,7 @@ "FunctionNode", "GraphNode", "OperatorNode", + "SourceJobNode", "SourceNode", + "SourceNodeBase", ] diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index c189083c7..3cec7cf7f 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -1,208 +1,168 @@ -"""SourceNode — wraps a root source stream in the computation graph.""" +"""Source node hierarchy for Pipeline and PipelineJob. +SourceNode — schema-only input-slot declaration (replaces SourceSpec). +SourceJobNode — execution variant that wraps a concrete StreamProtocol. +Both share SourceNodeBase which provides hash-stable identity. + +Hash-stability guarantee: + SourceNode(name=n, tag_schema=t, data_schema=d).content_hash() + == SourceSpec(name=n, tag_schema=t, data_schema=d).content_hash() + +This is achieved by using identical identity_structure(): + ("SourceSpec", name, tag_schema, data_schema) +""" from __future__ import annotations import logging +from abc import ABC, abstractmethod from collections.abc import Iterator from typing import TYPE_CHECKING, Any -logger = logging.getLogger(__name__) - from orcapod import contexts -from orcapod.channels import WritableChannel -from orcapod.config import Config, DEFAULT_CONFIG -from orcapod.core.streams.base import StreamBase -from orcapod.protocols import core_protocols as cp +from orcapod.core.base import TraceableBase +from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.types import ColumnConfig, ContentHash, Schema if TYPE_CHECKING: import pyarrow as pa + from orcapod.channels import WritableChannel + from orcapod.protocols.core_protocols import StreamProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol +logger = logging.getLogger(__name__) + + +class SourceNodeBase(TraceableBase, ABC): + """Abstract base for SourceNode and SourceJobNode. + + Provides schema-based identity (content_hash, pipeline_hash) and + shared properties. Both sub-types carry identical schemas so their + pipeline_hash() values always match; content_hash() diverges only + when SourceJobNode has a concrete source bound. -class SourceNode(StreamBase): - """Represents a root source stream in the computation graph.""" + Args: + name: The input-slot name used as the key in + ``PipelineJob.bind(sources={name: source})``. + tag_schema: Mapping of tag column names to Python types. + data_schema: Mapping of data column names to Python types. + data_context: Optional data context override. + """ node_type = "source" def __init__( self, - stream: cp.StreamProtocol, - label: str | None = None, - config: Config | None = None, - ): - super().__init__(label=label, config=config) - self.stream = stream - self._cached_results: list[tuple[cp.TagProtocol, cp.DataProtocol]] | None = ( - None - ) + name: str, + tag_schema: Schema, + data_schema: Schema, + data_context: str | contexts.DataContext | None = None, + ) -> None: + super().__init__(data_context=data_context) + self._name = name + self._tag_schema = tag_schema + self._data_schema = data_schema # ------------------------------------------------------------------ - # from_descriptor — reconstruct from a serialized pipeline descriptor + # Identity — hash-stable against old SourceSpec # ------------------------------------------------------------------ - @classmethod - def from_descriptor( - cls, - descriptor: dict[str, Any], - stream: cp.StreamProtocol | None, - databases: dict[str, Any], - ) -> SourceNode: - """Construct a SourceNode from a serialized descriptor. - - When *stream* is provided the node operates in full mode — all - delegation goes through the live stream. When *stream* is ``None`` - the node is created in read-only mode with metadata from the - descriptor; data-access methods (``iter_data``, ``as_table``) - will raise ``RuntimeError``. - - Args: - descriptor: The serialized node descriptor dict. - stream: An optional live stream to wrap. ``None`` for - read-only mode. - databases: Mapping of database role names to database - instances (currently unused for source nodes but kept - for interface consistency with other node types). + def identity_structure(self) -> Any: + """Return the content identity: ``("SourceSpec", name, tag_schema, data_schema)``. - Returns: - A new ``SourceNode`` instance. + Deliberately matches ``SourceSpec.identity_structure()`` so that a + ``SourceNode`` constructed with the same arguments as a ``SourceSpec`` + produces an identical ``content_hash()``. This preserves all DB paths + computed from pre-refactor pipelines. """ - from orcapod.pipeline.serialization import LoadStatus - - if stream is not None: - node = cls(stream=stream, label=descriptor.get("label")) - node._descriptor = descriptor - node._load_status = LoadStatus.FULL - return node + return ("SourceSpec", self._name, self._tag_schema, self._data_schema) - # Read-only mode: bypass __init__, set minimum required state - node = cls.__new__(cls) + def pipeline_identity_structure(self) -> Any: + """Return the pipeline identity: ``(tag_schema, data_schema)`` (name-independent). - # From LabelableMixin - node._label = descriptor.get("label") + Matches ``RootSource.pipeline_identity_structure()`` so that sources + with identical schemas share the same DB table paths regardless of name. + """ + return (self._tag_schema, self._data_schema) - # From DataContextMixin - node._data_context = contexts.resolve_context( - descriptor.get("data_context_key") - ) - node._orcapod_config = DEFAULT_CONFIG + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ - # From ContentIdentifiableBase - node._content_hash_cache = {} - node._cached_int_hash = None + @property + def name(self) -> str: + """The input-slot name used as the key in ``PipelineJob.bind(sources={...})``.""" + return self._name - # From PipelineElementBase - node._pipeline_hash_cache = {} + def computed_label(self) -> str | None: + """Resolve the node label to the slot name. - # From TemporalMixin - node._modified_time = None + Implements ``LabelableMixin.computed_label()`` so that ``self.label`` + resolves to the slot name without an explicit label assignment. - # SourceNode's own state - node.stream = None - node._cached_results = None - node._descriptor = descriptor - node._load_status = LoadStatus.UNAVAILABLE - node._stored_schema = descriptor.get("output_schema", {}) - node._stored_content_hash = descriptor.get("content_hash") - node._stored_pipeline_hash = descriptor.get("pipeline_hash") - node._stored_node_uri = tuple(descriptor.get("node_uri") or []) + Returns: + The slot name. + """ + return self._name - return node + @property + def tag_schema(self) -> Schema: + """Tag schema for this input slot.""" + return self._tag_schema - # ------------------------------------------------------------------ - # node_uri - # ------------------------------------------------------------------ + @property + def data_schema(self) -> Schema: + """Data schema for this input slot.""" + return self._data_schema @property def node_uri(self) -> tuple[str, ...]: - """Canonical URI tuple identifying this source. + """Canonical URI tuple for this source node. - At runtime: derives from stream config (source_type, source_id). - In read-only (deserialized) mode: returns stored value from descriptor. + Returns a tuple identifying this node as a named schema-only source slot. """ - if self.stream is None: - uri = tuple(getattr(self, "_stored_node_uri", ())) - logger.debug("SourceNode.node_uri: read-only mode, returning stored URI %r", uri) - return uri - stream = self.stream - if hasattr(stream, "to_config"): - cfg = stream.to_config() - stream_type = cfg.get("source_type", "unknown") - source_id = cfg.get("source_id") or getattr(stream, "source_id", "") - uri = (stream_type, str(source_id or "")) - logger.debug("SourceNode.node_uri: live stream, derived URI %r from to_config()", uri) - return uri - uri = (type(stream).__name__,) - logger.debug("SourceNode.node_uri: live stream without to_config, using type name %r", uri) - return uri - - # ------------------------------------------------------------------ - # load_status - # ------------------------------------------------------------------ + return ("source_node", self._name) @property - def load_status(self) -> Any: - """Return the load status of this node. + def producer(self) -> None: + """Source nodes have no producer pod — they are root nodes. Returns: - The ``LoadStatus`` enum value indicating how this node was - loaded. Defaults to ``FULL`` for nodes created via - ``__init__``. + Always ``None``. """ - from orcapod.pipeline.serialization import LoadStatus - - return getattr(self, "_load_status", LoadStatus.FULL) - - # ------------------------------------------------------------------ - # Delegation — with read-only guards - # ------------------------------------------------------------------ - - @property - def data_context(self) -> contexts.DataContext: - if self.stream is None: - return self._data_context - return contexts.resolve_context(self.stream.data_context_key) + return None @property - def data_context_key(self) -> str: - if self.stream is None: - return self._data_context.context_key - return self.stream.data_context_key + def upstreams(self) -> "tuple[StreamProtocol, ...]": + """Source nodes have no upstream streams — they are root nodes. - def computed_label(self) -> str | None: - if self.stream is None: - return None - return self.stream.label + Returns: + Always an empty tuple. + """ + return () - def identity_structure(self) -> Any: - if self.stream is None: - raise RuntimeError( - "SourceNode in read-only mode has no stream data available" - ) - # TODO: revisit this logic for case where stream is not a root source - return self.stream.identity_structure() + @upstreams.setter + def upstreams(self, value: "tuple[StreamProtocol, ...]") -> None: + if len(value) != 0: + raise ValueError("SourceNode upstreams must be empty") - def pipeline_identity_structure(self) -> Any: - if self.stream is None: - raise RuntimeError( - "SourceNode in read-only mode has no stream data available" - ) - return self.stream.pipeline_identity_structure() + def output_schema( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """Return ``(tag_schema, data_schema)``. - def content_hash(self, hasher=None) -> ContentHash: - """Return the content hash, using stored value in read-only mode.""" - stored = getattr(self, "_stored_content_hash", None) - if self.stream is None and stored is not None: - return ContentHash.from_string(stored) - return super().content_hash(hasher) + Args: + columns: Ignored. + all_info: Ignored. - def pipeline_hash(self, hasher=None) -> ContentHash: - """Return the pipeline hash, using stored value in read-only mode.""" - stored = getattr(self, "_stored_pipeline_hash", None) - if self.stream is None and stored is not None: - return ContentHash.from_string(stored) - return super().pipeline_hash(hasher) + Returns: + Tuple of ``(tag_schema, data_schema)``. + """ + return (self._tag_schema, self._data_schema) def keys( self, @@ -210,117 +170,301 @@ def keys( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> tuple[tuple[str, ...], tuple[str, ...]]: - if self.stream is None: - stored = getattr(self, "_stored_schema", {}) - tag_keys = tuple(stored.get("tag", {}).keys()) - data_keys = tuple(stored.get("data", {}).keys()) - return tag_keys, data_keys - return self.stream.keys(columns=columns, all_info=all_info) + """Return ``(tag_keys, data_keys)``. - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - if self.stream is None: - stored = getattr(self, "_stored_schema", {}) - tag = Schema(stored.get("tag", {})) - data = Schema(stored.get("data", {})) - return tag, data - return self.stream.output_schema(columns=columns, all_info=all_info) + Args: + columns: Ignored. + all_info: Ignored. - @property - def producer(self) -> None: - return None + Returns: + Tuple of ``(tag_column_names, data_column_names)``. + """ + return (tuple(self._tag_schema.keys()), tuple(self._data_schema.keys())) - @property - def upstreams(self) -> tuple[cp.StreamProtocol, ...]: - return () + # ------------------------------------------------------------------ + # Validation + # ------------------------------------------------------------------ - @upstreams.setter - def upstreams(self, value: tuple[cp.StreamProtocol, ...]) -> None: - if len(value) != 0: - raise ValueError("SourceNode upstreams must be empty") + def validate(self, source: "StreamProtocol") -> None: + """Check that *source* is schema-compatible with this node's declared schema. - def __repr__(self) -> str: - return f"SourceNode(stream={self.stream!r}, label={self.label!r})" + Args: + source: A concrete stream to validate. + + Raises: + SourceSpecMismatchError: If schema columns don't match. + """ + source_tag, source_data = source.output_schema() + + tag_issues: list[str] = [] + data_issues: list[str] = [] + + spec_tag_cols = set(self._tag_schema.keys()) + src_tag_cols = set(source_tag.keys()) + if spec_tag_cols != src_tag_cols: + missing = spec_tag_cols - src_tag_cols + extra = src_tag_cols - spec_tag_cols + if missing: + tag_issues.append(f"missing tag columns: {sorted(missing)}") + if extra: + tag_issues.append(f"unexpected tag columns: {sorted(extra)}") + + spec_data_cols = set(self._data_schema.keys()) + src_data_cols = set(source_data.keys()) + if spec_data_cols != src_data_cols: + missing = spec_data_cols - src_data_cols + extra = src_data_cols - spec_data_cols + if missing: + data_issues.append(f"missing data columns: {sorted(missing)}") + if extra: + data_issues.append(f"unexpected data columns: {sorted(extra)}") + + if tag_issues or data_issues: + raise SourceSpecMismatchError( + f"SourceNode '{self._name}' is not compatible with the provided source. " + + "; ".join(tag_issues + data_issues) + ) def as_table( self, *, columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, - ) -> pa.Table: - if self.stream is None: - raise RuntimeError( - "SourceNode in read-only mode has no stream data available" - ) - return self.stream.as_table(columns=columns, all_info=all_info) + ) -> "pa.Table": + """Materialize stream as a PyArrow Table. - def iter_data(self) -> Iterator[tuple[cp.TagProtocol, cp.DataProtocol]]: - if self.stream is None: - raise RuntimeError( - "SourceNode in read-only mode has no stream data available" - ) - if self._cached_results is not None: - return iter(self._cached_results) - return self.stream.iter_data() + Delegates to the concrete source (SourceJobNode), or raises for + schema-only SourceNode. + + Args: + columns: Column selection config. + all_info: If True, include all metadata columns. + + Raises: + UnboundSourceError: When no concrete data is available. + """ + # Calling iter_data() will raise UnboundSourceError for SourceNode, + # or delegate to concrete for SourceJobNode. + # For SourceJobNode with a concrete source, delegate directly. + raise UnboundSourceError( + f"SourceNode '{self._name}' is not bound to a concrete source. " + "Use PipelineJob.bind() to attach data before calling as_table()." + ) + + async def async_iter_data(self): + """Asynchronous iterator over (tag, data) pairs. + + Raises: + UnboundSourceError: When no concrete data is available. + """ + for pair in self.iter_data(): + yield pair + + # ------------------------------------------------------------------ + # Abstract + # ------------------------------------------------------------------ + + @abstractmethod + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Yield ``(tag, data)`` pairs, or raise if data is unavailable.""" + ... def execute( self, *, - observer: ExecutionObserverProtocol | None = None, - ) -> list[tuple[cp.TagProtocol, cp.DataProtocol]]: - """Execute this source: materialize data and return. + observer: "ExecutionObserverProtocol | None" = None, + ) -> list[tuple[TagProtocol, DataProtocol]]: + """Execute this source node: materialize and return data. Args: - observer: Optional execution observer for hooks. + observer: Optional execution observer. Returns: List of (tag, data) tuples. + + Raises: + UnboundSourceError: When no concrete data is available. """ - if self.stream is None: - raise RuntimeError( - "SourceNode in read-only mode has no stream data available" - ) node_label = self.label node_hash = "" if observer is not None: observer.on_node_start(node_label, node_hash) - result = list(self.stream.iter_data()) - self._cached_results = result + result = list(self.iter_data()) if observer is not None: observer.on_node_end(node_label, node_hash) return result - def run(self) -> None: - """No-op for source nodes — data is already available.""" - async def async_execute( self, - output: WritableChannel[tuple[cp.TagProtocol, cp.DataProtocol]], + output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", *, - observer: ExecutionObserverProtocol | None = None, + observer: "ExecutionObserverProtocol | None" = None, ) -> None: - """Push all (tag, data) pairs from the wrapped stream to the output channel. + """Push all (tag, data) pairs to the output channel. Args: output: Channel to write results to. - observer: Optional execution observer for hooks. + observer: Optional execution observer. + + Raises: + UnboundSourceError: When no concrete data is available. """ - if self.stream is None: - raise RuntimeError( - "SourceNode in read-only mode has no stream data available" - ) node_label = self.label node_hash = "" try: if observer is not None: observer.on_node_start(node_label, node_hash) - for tag, data in self.stream.iter_data(): + for tag, data in self.iter_data(): await output.send((tag, data)) if observer is not None: observer.on_node_end(node_label, node_hash) finally: await output.close() + + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(name={self._name!r}, " + f"tag_schema={dict(self._tag_schema)!r}, " + f"data_schema={dict(self._data_schema)!r})" + ) + + +class SourceNode(SourceNodeBase): + """Schema-only input-slot declaration for ``Pipeline`` recording. + + Replaces ``SourceSpec`` as the user-facing way to declare typed pipeline + inputs. Pass a ``SourceNode`` inside a ``with pipeline:`` block as the + upstream for any pod invocation. + + Example:: + + slot = SourceNode(name="data", tag_schema={"id": int}, data_schema={"v": float}) + with pipeline: + result = my_pod(slot) + + job = PipelineJob.from_pipeline(pipeline, store=db, sources={"data": my_source}) + job.run() + + Hash-stability note: + ``identity_structure()`` returns ``("SourceSpec", name, tag_schema, data_schema)`` + — identical to the old ``SourceSpec`` — so existing DB paths remain valid. + """ + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Raise ``UnboundSourceError`` — ``SourceNode`` carries no data. + + Raises: + UnboundSourceError: Always. + """ + raise UnboundSourceError( + f"SourceNode '{self._name}' is not bound to a concrete source. " + "Use PipelineJob.from_pipeline(..., sources={'': source}) " + "or job.bind(sources={'': source}) to attach data." + ) + + +class SourceJobNode(SourceNodeBase): + """Execution-ready source node wrapping an optional concrete stream. + + Used inside ``PipelineJob._persistent_node_map``. The ``_concrete`` + field is **mutable** — ``PipelineJob.bind(sources={...})`` updates it + in-place so that downstream ``FunctionJobNode`` objects (which hold a + reference to this same object) automatically see the new concrete source + without cascading reference updates. + + Hash behaviour: + + * ``content_hash()`` — delegates to ``_concrete.content_hash()`` when + bound; falls back to schema-based ``SourceNodeBase.content_hash()`` (== + ``SourceNode.content_hash()``) when unbound. + * ``pipeline_hash()`` — always schema-based (inherited); never + data-inclusive. This invariant keeps DB paths stable across different + data sources bound to the same slot. + + Args: + name: Slot name. + tag_schema: Tag schema. + data_schema: Data schema. + concrete: Optional concrete stream. Can be set or replaced later via + ``job_node._concrete = source``. + data_context: Optional data context override. + """ + + def __init__( + self, + name: str, + tag_schema: Schema, + data_schema: Schema, + concrete: "StreamProtocol | None" = None, + data_context: str | contexts.DataContext | None = None, + ) -> None: + super().__init__( + name=name, + tag_schema=tag_schema, + data_schema=data_schema, + data_context=data_context, + ) + self._concrete: "StreamProtocol | None" = concrete + + def content_hash(self, hasher=None) -> ContentHash: + """Return data-inclusive hash when bound; schema-based hash when unbound. + + Args: + hasher: Optional semantic hasher. + + Returns: + ``_concrete.content_hash(hasher)`` when bound, otherwise + ``SourceNodeBase.content_hash(hasher)``. + """ + if self._concrete is not None: + if hasher is None: + hasher = self.data_context.semantic_hasher + return self._concrete.content_hash(hasher) + return super().content_hash(hasher) + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Delegate to concrete source, or raise if unbound. + + Raises: + UnboundSourceError: When no concrete source is attached. + """ + if self._concrete is None: + raise UnboundSourceError( + f"SourceJobNode '{self._name}' has no concrete source bound. " + "Call job.bind(sources={'': source}) before running." + ) + return self._concrete.iter_data() + + def as_table( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> "pa.Table": + """Materialize the concrete source as a PyArrow Table. + + Args: + columns: Column selection config. + all_info: If True, include all metadata columns. + + Raises: + UnboundSourceError: When no concrete source is attached. + """ + if self._concrete is None: + raise UnboundSourceError( + f"SourceJobNode '{self._name}' has no concrete source bound. " + "Call job.bind(sources={'': source}) before calling as_table()." + ) + return self._concrete.as_table(columns=columns, all_info=all_info) + + def as_node(self) -> SourceNode: + """Return the lightweight ``SourceNode`` equivalent of this job node. + + Returns: + A new ``SourceNode`` with the same name and schemas. + """ + return SourceNode( + name=self._name, + tag_schema=self._tag_schema, + data_schema=self._data_schema, + ) diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index 03458ae9f..a7a7c01ba 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -50,3 +50,12 @@ class SourceSpecMismatchError(ValueError): Contains the spec name and a description of the incompatible field(s). Raised at ``bind()`` time — schema mismatches are rejected before execution. """ + + +class PipelineJobRequiredError(RuntimeError): + """Raised when a lightweight blueprint node is asked to produce data. + + Blueprint nodes (``FunctionNode``, ``OperatorNode``) carry no database + references. Wrap the containing ``Pipeline`` in a ``PipelineJob`` to + obtain executable ``FunctionJobNode`` / ``OperatorJobNode`` variants. + """ diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index db1f19fff..54fbe2ab3 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -216,17 +216,17 @@ def compile(self) -> None: continue if node_hash not in self._node_lut: - # -- Leaf stream: must be a SourceSpec in the new design -- - from orcapod.core.sources.source_spec import SourceSpec + # -- Leaf stream: must be a SourceNode in the new design -- + from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass stream = self._upstreams[node_hash] - if not isinstance(stream, SourceSpec): + if not isinstance(stream, SourceNodeClass): raise ValueError( - f"Pipeline: all leaf inputs must be SourceSpec instances, " + f"Pipeline: all leaf inputs must be SourceNode instances, " f"but found {type(stream).__name__!r}. " "Use 'with PipelineJob:' to record a pipeline with concrete sources, " - "or replace concrete sources with SourceSpec declarations." + "or replace concrete sources with SourceNode declarations." ) - node = SourceNode(stream=stream) + node = stream # SourceNode IS the leaf — no wrapping needed persistent_node_map[node_hash] = node else: node = self._node_lut[node_hash] @@ -383,8 +383,8 @@ def save(self, path: str | Path) -> None: PIPELINE_FORMAT_VERSION, serialize_schema, ) - from orcapod.core.sources.source_spec import SourceSpec from orcapod.core.nodes import OperatorNode, FunctionNode + from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass nodes: dict[str, Any] = {} for content_hash_str, node in self._persistent_node_map.items(): @@ -416,16 +416,12 @@ def save(self, path: str | Path) -> None: "data_context_key": data_context_key, } - if isinstance(node, SourceNode): - if isinstance(node.stream, SourceSpec): - descriptor["source_config"] = { - "source_type": "spec", - "spec_name": node.stream.name, - } - descriptor["reconstructable"] = True - else: - descriptor["source_config"] = None - descriptor["reconstructable"] = False + if isinstance(node, SourceNodeClass): + descriptor["source_config"] = { + "source_type": "node", + "node_name": node.name, + } + descriptor["reconstructable"] = True elif isinstance(node, FunctionNode): if node._function_pod is not None: @@ -473,8 +469,8 @@ def load(cls, path: str | Path) -> "Pipeline": SUPPORTED_FORMAT_VERSIONS, deserialize_schema, ) - from orcapod.core.sources.source_spec import SourceSpec from orcapod.core.nodes import FunctionNode, OperatorNode + from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass from orcapod.types import Schema path = Path(path) @@ -517,19 +513,26 @@ def load(cls, path: str | Path) -> "Pipeline": source_config = descriptor.get("source_config") or {} if node_type == "source": - if source_config.get("source_type") == "spec": - spec_name = source_config["spec_name"] - tag_schema = Schema(deserialize_schema(descriptor["output_schema"]["tag"])) - data_schema = Schema(deserialize_schema(descriptor["output_schema"]["data"])) - stream = SourceSpec( - name=spec_name, - tag_schema=tag_schema, - data_schema=data_schema, - ) + tag_schema = Schema(deserialize_schema(descriptor["output_schema"]["tag"])) + data_schema = Schema(deserialize_schema(descriptor["output_schema"]["data"])) + # Support both old "spec" format and new "node" format + if source_config.get("source_type") == "node": + node_name = source_config["node_name"] + elif source_config.get("source_type") == "spec": + # Legacy format compatibility: spec_name becomes node name + node_name = source_config["spec_name"] else: - stream = None # non-spec source — schema known but not rebuildable - - node = SourceNode.from_descriptor(descriptor, stream=stream, databases={}) + # Fall back to stored label + node_name = descriptor.get("label") or "unknown" + node = SourceNodeClass( + name=node_name, + tag_schema=tag_schema, + data_schema=data_schema, + ) + # Restore label from descriptor if set explicitly + stored_label = descriptor.get("label") + if stored_label and stored_label != node_name: + node._label = stored_label reconstructed[node_hash] = node elif node_type == "function": @@ -610,12 +613,14 @@ def load(cls, path: str | Path) -> "Pipeline": pipeline._node_lut = { h: n for h, n in reconstructed.items() - if not isinstance(n, SourceNode) + if not isinstance(n, SourceNodeClass) } + # SourceNode IS the upstream — store it directly so _build_execution_graph() + # can find it by hash and substitute a concrete source at run time. pipeline._upstreams = { - h: n.stream + h: n for h, n in reconstructed.items() - if isinstance(n, SourceNode) and n.stream is not None + if isinstance(n, SourceNodeClass) } pipeline._compiled = True diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index c85d47048..efff2a92a 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -10,8 +10,8 @@ from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: - from orcapod.core.nodes import FunctionNode, GraphNode, OperatorNode, SourceNode - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.core.nodes import FunctionNode, GraphNode, OperatorNode + from orcapod.core.nodes.source_node import SourceNode from orcapod.pipeline.execution_context import ExecutionContext from orcapod.pipeline.graph import Pipeline from orcapod.protocols.database_protocols import ArrowDatabaseProtocol @@ -71,7 +71,7 @@ def __init__( self._rec_graph_edges: list[tuple[str, str]] = [] self._rec_upstreams: dict[str, cp.StreamProtocol] = {} self._rec_node_lut: dict[str, "GraphNode"] = {} - self._spec_by_name: dict[str, "SourceSpec"] = {} + self._spec_by_name: dict[str, "SourceNode"] = {} self._pipeline_name: tuple[str, ...] = (name,) if isinstance(name, str) else tuple(name) self._unresolved_specs: list[str] = [] self._has_run: bool = False @@ -123,19 +123,19 @@ def _compile_from_recording(self) -> None: pipeline.compile() self._compiled_pipeline = pipeline - def _ensure_spec(self, source: cp.StreamProtocol) -> "SourceSpec": - """Promote *source* to a SourceSpec, storing the concrete binding. + def _ensure_source_node(self, source: cp.StreamProtocol) -> "SourceNode": + """Promote *source* to a SourceNode, storing the concrete binding. - If the spec already exists (same label/hash key), returns the cached spec. + If the node already exists (same label/hash key), returns the cached node. When a source has an explicitly assigned label, that label is used as the - SourceSpec name. When no label is assigned (the source falls back to its + SourceNode name. When no label is assigned (the source falls back to its class name), the source's content hash is used to ensure uniqueness. """ - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.core.nodes.source_node import SourceNode # Use explicit label when set; otherwise fall back to content hash - # to avoid two unlabeled sources getting the same spec name. + # to avoid two unlabeled sources getting the same node name. has_label = source.has_assigned_label if has_label: name = source.label # type: ignore[attr-defined] @@ -144,28 +144,28 @@ class name), the source's content hash is used to ensure uniqueness. if name not in self._spec_by_name: tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name=name, tag_schema=tag_schema, data_schema=data_schema) - self._spec_by_name[name] = spec + node = SourceNode(name=name, tag_schema=tag_schema, data_schema=data_schema) + self._spec_by_name[name] = node self._sources[name] = source return self._spec_by_name[name] @staticmethod def _is_concrete_source(stream: cp.StreamProtocol) -> bool: - """True if *stream* is a concrete RootSource (not a SourceSpec).""" + """True if *stream* is a concrete RootSource (not a SourceNode).""" from orcapod.core.sources.base import RootSource - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.core.nodes.source_node import SourceNode - return isinstance(stream, RootSource) and not isinstance(stream, SourceSpec) + return isinstance(stream, RootSource) and not isinstance(stream, SourceNode) # ------------------------------------------------------------------ # TrackerProtocol — recording with source interception # ------------------------------------------------------------------ - def _to_spec_stream(self, stream: cp.StreamProtocol) -> cp.StreamProtocol: - """Convert *stream* to a spec-based equivalent for consistent hash recording. + def _to_node_stream(self, stream: cp.StreamProtocol) -> cp.StreamProtocol: + """Convert *stream* to a node-based equivalent for consistent hash recording. - Concrete ``RootSource`` instances are promoted to ``SourceSpec`` via - ``_ensure_spec``. ``DynamicPodStream`` instances have their upstreams + Concrete ``RootSource`` instances are promoted to ``SourceNode`` via + ``_ensure_source_node``. ``DynamicPodStream`` instances have their upstreams recursively converted so that their content hash matches the ``OperatorNode`` recorded in ``_rec_node_lut``. @@ -173,17 +173,17 @@ def _to_spec_stream(self, stream: cp.StreamProtocol) -> cp.StreamProtocol: stream: The upstream stream to convert. Returns: - A spec-based stream with a stable hash for recording. + A node-based stream with a stable hash for recording. """ from orcapod.core.operators.static_output_pod import DynamicPodStream if self._is_concrete_source(stream): - return self._ensure_spec(stream) + return self._ensure_source_node(stream) if isinstance(stream, DynamicPodStream): - spec_upstreams = tuple(self._to_spec_stream(s) for s in stream.upstreams) + node_upstreams = tuple(self._to_node_stream(s) for s in stream.upstreams) return DynamicPodStream( pod=stream._pod, - upstreams=spec_upstreams, + upstreams=node_upstreams, label=stream._label, ) return stream @@ -203,7 +203,7 @@ def record_function_pod_invocation( """ from orcapod.core.nodes import FunctionNode - input_stream = self._to_spec_stream(input_stream) + input_stream = self._to_node_stream(input_stream) input_hash = input_stream.content_hash().to_string() function_node = FunctionNode(function_pod=pod, input_stream=input_stream, label=label) @@ -228,7 +228,7 @@ def record_operator_pod_invocation( """ from orcapod.core.nodes import OperatorNode - processed = tuple(self._to_spec_stream(s) for s in upstreams) + processed = tuple(self._to_node_stream(s) for s in upstreams) operator_node = OperatorNode(operator=pod, input_streams=processed, label=label) op_hash = operator_node.content_hash().to_string() @@ -302,32 +302,30 @@ def bind( Raises: SourceSpecMismatchError: If any source's schema is incompatible. """ - from orcapod.core.nodes import SourceNode - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.core.nodes.source_node import SourceNode merged_sources = dict(self._sources) if sources is not None: - # Validate each supplied source against its SourceSpec + # Validate each supplied source against its SourceNode pipeline = self._compiled_pipeline if pipeline is not None: for node in pipeline._persistent_node_map.values(): if ( isinstance(node, SourceNode) - and isinstance(node.stream, SourceSpec) - and node.stream.name in sources + and node.name in sources ): - node.stream.validate(sources[node.stream.name]) - # Check that every provided key corresponds to a SourceSpec leaf - spec_names = { - node.stream.name + node.validate(sources[node.name]) + # Check that every provided key corresponds to a SourceNode leaf + node_names = { + node.name for node in pipeline._persistent_node_map.values() - if isinstance(node, SourceNode) and isinstance(node.stream, SourceSpec) + if isinstance(node, SourceNode) } - unknown = set(sources.keys()) - spec_names + unknown = set(sources.keys()) - node_names if unknown: raise ValueError( - f"bind() received source keys with no matching SourceSpec in the pipeline: " - f"{sorted(unknown)}. Known spec names: {sorted(spec_names)}" + f"bind() received source keys with no matching SourceNode in the pipeline: " + f"{sorted(unknown)}. Known node names: {sorted(node_names)}" ) merged_sources.update(sources) @@ -343,39 +341,41 @@ def bind( # Completeness introspection # ------------------------------------------------------------------ - def unbound_specs(self) -> "list[SourceSpec]": - """Return all SourceSpec slots not yet bound in this job. + def unbound_source_nodes(self) -> "list[SourceNode]": + """Return all SourceNode slots not yet bound in this job. Returns: - List of unbound ``SourceSpec`` instances, in order of appearance + List of unbound ``SourceNode`` instances, in order of appearance in the pipeline graph. """ - from orcapod.core.nodes import SourceNode - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.core.nodes.source_node import SourceNode if self._compiled_pipeline is None: return [] - unbound = [] + unbound: list[SourceNode] = [] seen: set[str] = set() for node in self._compiled_pipeline._persistent_node_map.values(): if ( isinstance(node, SourceNode) - and isinstance(node.stream, SourceSpec) - and node.stream.name not in self._sources - and node.stream.name not in seen + and node.name not in self._sources + and node.name not in seen ): - unbound.append(node.stream) - seen.add(node.stream.name) + unbound.append(node) + seen.add(node.name) return unbound + def unbound_specs(self) -> "list[SourceNode]": + """Deprecated — use unbound_source_nodes() instead.""" + return self.unbound_source_nodes() + def is_complete(self) -> bool: - """Return ``True`` when all specs are bound and a store is set. + """Return ``True`` when all source nodes are bound and a store is set. Returns: - ``True`` if all SourceSpec slots are bound and a store is set. + ``True`` if all SourceNode slots are bound and a store is set. """ - return self._store is not None and len(self.unbound_specs()) == 0 + return self._store is not None and len(self.unbound_source_nodes()) == 0 def is_runnable(self, node_label: str) -> bool: """Return ``True`` if all upstream inputs of *node_label* are resolved. @@ -386,8 +386,7 @@ def is_runnable(self, node_label: str) -> bool: Returns: ``True`` if the node can be executed with current bindings. """ - from orcapod.core.nodes import SourceNode - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.core.nodes.source_node import SourceNode pipeline = self._compiled_pipeline if pipeline is None: @@ -405,8 +404,7 @@ def is_runnable(self, node_label: str) -> bool: for node in nx.ancestors(pipeline._node_graph, target) | {target}: if ( isinstance(node, SourceNode) - and isinstance(node.stream, SourceSpec) - and node.stream.name not in self._sources + and node.name not in self._sources ): return False return True @@ -465,8 +463,8 @@ def _build_execution_graph(self) -> "tuple[Any, list[str], Pipeline]": # Any = RuntimeError: If no compiled pipeline is available. """ import networkx as nx - from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.core.nodes import FunctionNode, OperatorNode + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.executors.local import LocalPythonFunctionExecutor pipeline = self._compiled_pipeline @@ -513,25 +511,29 @@ def _build_execution_graph(self) -> "tuple[Any, list[str], Pipeline]": # Any = if node_hash not in pipeline._node_lut: # Leaf stream — must be in _upstreams - stream = pipeline._upstreams.get(node_hash) - if stream is None: + upstream = pipeline._upstreams.get(node_hash) + if upstream is None: continue - if isinstance(stream, SourceSpec): - if stream.name in self._sources: - # Bound — wrap concrete source in a SourceNode. - # Label is not injected explicitly; SourceNode.computed_label() - # delegates to stream.label (the concrete source's label). - concrete = self._sources[stream.name] - exec_node_map[node_hash] = SourceNode(stream=concrete) + if isinstance(upstream, SourceNode): + if upstream.name in self._sources: + # Bound — create a SourceJobNode with the concrete source. + concrete = self._sources[upstream.name] + exec_job_node = SourceJobNode( + name=upstream.name, + tag_schema=upstream.tag_schema, + data_schema=upstream.data_schema, + concrete=concrete, + ) + exec_node_map[node_hash] = exec_job_node else: # Unbound — exclude this branch excluded_hashes.add(node_hash) - if stream.name not in unresolved_specs: - unresolved_specs.append(stream.name) + if upstream.name not in unresolved_specs: + unresolved_specs.append(upstream.name) else: - # Raw (non-spec) stream: SourceNode.computed_label() delegates - # to stream.label automatically — no explicit label needed. - exec_node_map[node_hash] = SourceNode(stream=stream) + # Raw non-SourceNode stream (shouldn't happen in new design, + # but handle gracefully for robustness). + exec_node_map[node_hash] = upstream else: template = pipeline._node_lut[node_hash] preds = list(G.predecessors(node_hash)) diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py new file mode 100644 index 000000000..443025d6e --- /dev/null +++ b/tests/test_core/nodes/test_source_node.py @@ -0,0 +1,170 @@ +"""Tests for SourceNode (schema-only slot) and SourceJobNode (execution variant).""" +from __future__ import annotations + +import pytest + +from orcapod.errors import SourceSpecMismatchError, UnboundSourceError +from orcapod.types import Schema + + +@pytest.fixture +def tag_schema(): + return Schema({"id": int}) + + +@pytest.fixture +def data_schema(): + return Schema({"value": float}) + + +class TestSourceNodeHashStability: + """SourceNode must produce bit-identical hashes to SourceSpec with the same args.""" + + def test_content_hash_matches_source_spec(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.source_spec import SourceSpec + + spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + assert node.content_hash() == spec.content_hash() + + def test_pipeline_hash_matches_source_spec(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.source_spec import SourceSpec + + spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + assert node.pipeline_hash() == spec.pipeline_hash() + + def test_different_names_different_content_hash(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + b = SourceNode(name="slot_b", tag_schema=tag_schema, data_schema=data_schema) + assert a.content_hash() != b.content_hash() + + def test_different_names_same_pipeline_hash(self, tag_schema, data_schema): + """pipeline_hash is schema-only, name-independent.""" + from orcapod.core.nodes.source_node import SourceNode + + a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + b = SourceNode(name="slot_b", tag_schema=tag_schema, data_schema=data_schema) + assert a.pipeline_hash() == b.pipeline_hash() + + +class TestSourceNodeInterface: + def test_iter_data_raises_unbound_error(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + with pytest.raises(UnboundSourceError): + list(node.iter_data()) + + def test_output_schema(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + t, d = node.output_schema() + assert t == tag_schema + assert d == data_schema + + def test_label_resolves_to_name(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="my_slot", tag_schema=tag_schema, data_schema=data_schema) + assert node.label == "my_slot" + + def test_node_type(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert node.node_type == "source" + + def test_name_property(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="my_slot", tag_schema=tag_schema, data_schema=data_schema) + assert node.name == "my_slot" + + def test_validate_compatible_source(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.dict_source import DictSource + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + node.validate(src) # must not raise + + def test_validate_incompatible_raises(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.sources.dict_source import DictSource + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + src = DictSource(data=[{"id": 1, "wrong": 1.0}], tag_columns=["id"]) + with pytest.raises(SourceSpecMismatchError): + node.validate(src) + + +class TestSourceJobNode: + def test_unbound_iter_data_raises(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceJobNode + + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + with pytest.raises(UnboundSourceError): + list(job_node.iter_data()) + + def test_unbound_content_hash_matches_source_node(self, tag_schema, data_schema): + """Unbound SourceJobNode has same content_hash as SourceNode.""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert job_node.content_hash() == node.content_hash() + + def test_pipeline_hash_matches_source_node(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert job_node.pipeline_hash() == node.pipeline_hash() + + def test_bound_content_hash_is_concrete_hash(self, tag_schema, data_schema): + """Bound SourceJobNode content_hash() == concrete.content_hash().""" + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.sources.dict_source import DictSource + + src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + ) + assert job_node.content_hash() == src.content_hash() + + def test_bound_pipeline_hash_still_schema_based(self, tag_schema, data_schema): + """pipeline_hash stays schema-based even when concrete is bound.""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.core.sources.dict_source import DictSource + + src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + ) + assert job_node.pipeline_hash() == node.pipeline_hash() + + def test_as_node_returns_source_node(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + node = job_node.as_node() + assert isinstance(node, SourceNode) + assert node.content_hash() == job_node.content_hash() + + def test_mutable_concrete_updates_in_place(self, tag_schema, data_schema): + """Binding concrete mutates _concrete in-place.""" + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.sources.dict_source import DictSource + + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + assert job_node._concrete is None + src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + job_node._concrete = src + assert job_node._concrete is src diff --git a/tests/test_core/test_tracker.py b/tests/test_core/test_tracker.py index 1356b6cdc..fd4f9fc85 100644 --- a/tests/test_core/test_tracker.py +++ b/tests/test_core/test_tracker.py @@ -21,10 +21,10 @@ from orcapod.core.function_pod import FunctionPod, function_pod from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode +from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.operators import Join, SelectTagColumns from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources.arrow_table_source import ArrowTableSource -from orcapod.core.sources.source_spec import SourceSpec from orcapod.core.streams import ArrowTableStream from orcapod.core.tracker import BasicTrackerManager from orcapod.pipeline import Pipeline @@ -54,27 +54,33 @@ def _make_pipeline( ) -def _make_spec(name: str = "test_spec") -> SourceSpec: - """SourceSpec matching the schema of _make_stream().""" - return SourceSpec( +def _make_source_node(name: str = "test_spec") -> SourceNode: + """SourceNode matching the schema of _make_stream().""" + return SourceNode( name=name, tag_schema=Schema({"id": int}), data_schema=Schema({"x": int}), ) -def _make_two_col_spec(name: str = "test_two_col_spec") -> SourceSpec: - """SourceSpec matching the schema of _make_two_col_stream().""" - return SourceSpec( +# Keep backward-compat alias used throughout tests +def _make_spec(name: str = "test_spec") -> SourceNode: + """SourceNode matching the schema of _make_stream().""" + return _make_source_node(name) + + +def _make_two_col_spec(name: str = "test_two_col_spec") -> SourceNode: + """SourceNode matching the schema of _make_two_col_stream().""" + return SourceNode( name=name, tag_schema=Schema({"id": int}), data_schema=Schema({"a": int, "b": int}), ) -def _make_y_spec(name: str = "test_y_spec") -> SourceSpec: - """SourceSpec matching the schema of _make_y_stream().""" - return SourceSpec( +def _make_y_spec(name: str = "test_y_spec") -> SourceNode: + """SourceNode matching the schema of _make_y_stream().""" + return SourceNode( name=name, tag_schema=Schema({"id": int}), data_schema=Schema({"y": int}), @@ -131,97 +137,111 @@ def _make_y_stream(n: int = 3) -> ArrowTableStream: class TestSourceNode: + """Tests for the new SourceNode (schema-only) and SourceJobNode (concrete-bound) APIs.""" + + def _make_job_node(self, stream=None, name="test_source"): + """Create a SourceJobNode wrapping an optional concrete stream.""" + if stream is None: + stream = _make_stream() + tag_schema = Schema({"id": int}) + data_schema = Schema({"x": int}) + return SourceJobNode( + name=name, + tag_schema=tag_schema, + data_schema=data_schema, + concrete=stream, + ) + def test_construction(self): stream = _make_stream() - node = SourceNode(stream=stream) - assert node.stream is stream + node = self._make_job_node(stream) + assert node._concrete is stream assert node.node_type == "source" assert node.producer is None assert node.upstreams == () def test_label_from_argument(self): stream = _make_stream() - node = SourceNode(stream=stream, label="my_source") + node = self._make_job_node(stream, name="my_source") + # label resolves to the name via computed_label() assert node.label == "my_source" - def test_label_from_stream(self): + def test_label_set_explicitly(self): stream = _make_stream() - stream._label = "stream_label" - node = SourceNode(stream=stream) - # computed_label defers to wrapped stream's label - assert node.label == "stream_label" - - def test_label_defaults_to_stream_label(self): - stream = _make_stream() - node = SourceNode(stream=stream) - # No explicit label → computed_label defers to stream.label - assert node.label == stream.label + node = self._make_job_node(stream, name="my_source") + node._label = "explicit" + assert node.label == "explicit" - def test_label_argument_overrides_stream(self): + def test_label_defaults_to_name(self): stream = _make_stream() - stream._label = "stream_label" - node = SourceNode(stream=stream, label="explicit") - assert node.label == "explicit" + node = self._make_job_node(stream, name="my_source") + # No explicit label → computed_label() returns the slot name + assert node.label == "my_source" def test_repr(self): stream = _make_stream() - node = SourceNode(stream=stream, label="test") + node = self._make_job_node(stream, name="test") r = repr(node) - assert "SourceNode" in r + assert "SourceJobNode" in r assert "test" in r - def test_content_hash_matches_stream(self): + def test_content_hash_delegates_to_concrete(self): stream = _make_stream() - node = SourceNode(stream=stream) + node = self._make_job_node(stream) + # SourceJobNode with concrete delegates content_hash to the concrete source assert node.content_hash() == stream.content_hash() def test_upstreams_setter_rejects_nonempty(self): - stream = _make_stream() - node = SourceNode(stream=stream) + node = self._make_job_node() with pytest.raises(ValueError, match="empty"): node.upstreams = (_make_stream(),) - def test_delegates_output_schema(self): + def test_output_schema(self): stream = _make_stream() - node = SourceNode(stream=stream) - assert node.output_schema() == stream.output_schema() + node = self._make_job_node(stream) + tag_s, data_s = node.output_schema() + assert set(tag_s.keys()) == {"id"} + assert set(data_s.keys()) == {"x"} - def test_delegates_keys(self): + def test_keys(self): stream = _make_stream() - node = SourceNode(stream=stream) - assert node.keys() == stream.keys() + node = self._make_job_node(stream) + tag_keys, data_keys = node.keys() + assert set(tag_keys) == {"id"} + assert set(data_keys) == {"x"} def test_delegates_as_table(self): stream = _make_stream() - node = SourceNode(stream=stream) + node = self._make_job_node(stream) node_table = node.as_table() stream_table = stream.as_table() assert node_table.equals(stream_table) def test_delegates_iter_data(self): stream = _make_stream() - node = SourceNode(stream=stream) + node = self._make_job_node(stream) node_data = list(node.iter_data()) stream_data = list(stream.iter_data()) assert len(node_data) == len(stream_data) - def test_run_is_noop(self): + def test_run_equivalent(self): + """execute() on SourceJobNode returns all data rows.""" stream = _make_stream() - node = SourceNode(stream=stream) - # run() should succeed without side effects - node.run() - # Data is still accessible after run() - assert node.as_table().num_rows == stream.as_table().num_rows + node = self._make_job_node(stream) + result = node.execute() + assert len(result) == stream.as_table().num_rows - def test_delegates_data_context_key(self): + def test_data_context_key(self): stream = _make_stream() - node = SourceNode(stream=stream) - assert node.data_context_key == stream.data_context_key + node = self._make_job_node(stream) + # SourceJobNode uses its own data context (not the concrete stream's) + assert node.data_context_key is not None - def test_delegates_data_context(self): + def test_data_context(self): stream = _make_stream() - node = SourceNode(stream=stream) - assert node.data_context.context_key == stream.data_context_key + node = self._make_job_node(stream) + assert node.data_context is not None + assert node.data_context.context_key is not None # --------------------------------------------------------------------------- @@ -234,9 +254,15 @@ class TestNodeContextDelegation: def test_source_node_context_matches_stream(self): stream = _make_stream() - node = SourceNode(stream=stream) - assert node.data_context_key == stream.data_context_key - assert node.data_context.context_key == stream.data_context_key + node = SourceJobNode( + name="test", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int}), + concrete=stream, + ) + # SourceJobNode has its own data context (not delegated to concrete) + assert node.data_context_key is not None + assert node.data_context is not None def test_function_node_context_matches_pod(self): stream = _make_stream() @@ -255,10 +281,16 @@ def test_operator_node_context_matches_operator(self): def test_source_node_hash_consistent_with_stream(self): stream = _make_stream() - node = SourceNode(stream=stream) - # Both should use the same hasher (from the same data context) + node = SourceJobNode( + name="test", + tag_schema=Schema({"id": int}), + data_schema=Schema({"x": int}), + concrete=stream, + ) + # SourceJobNode delegates content_hash() to concrete when bound assert node.content_hash() == stream.content_hash() - assert node.pipeline_hash() == stream.pipeline_hash() + # pipeline_hash() is schema-based (stable across different data) + assert node.pipeline_hash() is not None def test_function_node_hash_uses_pod_context(self): stream = _make_stream() @@ -471,14 +503,14 @@ def _persistent_nodes(pipeline: Pipeline) -> list: return list(pipeline._persistent_node_map.values()) def test_compile_single_function_pod(self): - """Source stream -> FunctionNode: compile creates SourceNode and wires upstream.""" + """Source node -> FunctionNode: compile wires SourceNode as upstream.""" pf = PythonDataFunction(_double, output_keys="result") pod = FunctionPod(data_function=pf) - spec = _make_spec() + source_node = _make_spec() # returns SourceNode mgr = BasicTrackerManager() with _make_pipeline(tracker_manager=mgr) as tracker: - tracker.record_function_pod_invocation(pod, spec) + tracker.record_function_pod_invocation(pod, source_node) tracker.compile() # After compile: 1 SourceNode + 1 FunctionNode in persistent map @@ -489,21 +521,20 @@ def test_compile_single_function_pod(self): assert len(source_nodes) == 1 assert len(fn_nodes) == 1 - # SourceNode wraps the original spec - assert source_nodes[0].stream is spec + # SourceNode IS the leaf — no stream wrapping assert source_nodes[0].upstreams == () # FunctionNode's upstream is now the SourceNode assert fn_nodes[0].upstreams == (source_nodes[0],) def test_compile_single_operator(self): - """Source stream -> Operator: compile creates SourceNode and wires upstream.""" - spec = _make_spec() + """Source node -> Operator: compile wires SourceNode as upstream.""" + source_node = _make_spec() op = SelectTagColumns(columns=["id"]) mgr = BasicTrackerManager() with _make_pipeline(tracker_manager=mgr) as tracker: - tracker.record_operator_pod_invocation(op, upstreams=(spec,)) + tracker.record_operator_pod_invocation(op, upstreams=(source_node,)) tracker.compile() all_nodes = self._persistent_nodes(tracker) @@ -515,7 +546,7 @@ def test_compile_single_operator(self): assert op_nodes[0].upstreams == (source_nodes[0],) def test_compile_operator_with_two_inputs(self): - """Two source streams -> Join: compile creates 2 SourceNodes.""" + """Two source nodes -> Join: compile creates 2 SourceNodes.""" spec_a = _make_spec("spec_a") spec_b = _make_y_spec("spec_b") op = Join() @@ -928,12 +959,12 @@ def test_pipeline_output_values(self, sources, expected_bmi): def test_compiled_graph_structure(self): """After compile(), the graph has the expected node types and count.""" - heights_spec = SourceSpec( + heights_spec = SourceNode( name="heights", tag_schema=Schema({"person_id": int}), data_schema=Schema({"height_cm": int}), ) - weights_spec = SourceSpec( + weights_spec = SourceNode( name="weights", tag_schema=Schema({"person_id": int}), data_schema=Schema({"weight_kg": int}), @@ -958,12 +989,12 @@ def test_compiled_graph_structure(self): def test_compiled_graph_all_upstreams_are_nodes(self): """Every upstream reference is a graph node after compile().""" - heights_spec = SourceSpec( + heights_spec = SourceNode( name="heights", tag_schema=Schema({"person_id": int}), data_schema=Schema({"height_cm": int}), ) - weights_spec = SourceSpec( + weights_spec = SourceNode( name="weights", tag_schema=Schema({"person_id": int}), data_schema=Schema({"weight_kg": int}), @@ -986,12 +1017,12 @@ def test_compiled_graph_all_upstreams_are_nodes(self): def test_compiled_graph_wiring(self): """Verify specific upstream wiring: cm_to_m<-source, join<-(cm_to_m, source), bmi<-join.""" - heights_spec = SourceSpec( + heights_spec = SourceNode( name="heights", tag_schema=Schema({"person_id": int}), data_schema=Schema({"height_cm": int}), ) - weights_spec = SourceSpec( + weights_spec = SourceNode( name="weights", tag_schema=Schema({"person_id": int}), data_schema=Schema({"weight_kg": int}), diff --git a/tests/test_pipeline/test_node_descriptors.py b/tests/test_pipeline/test_node_descriptors.py index 51cc2779b..95205d932 100644 --- a/tests/test_pipeline/test_node_descriptors.py +++ b/tests/test_pipeline/test_node_descriptors.py @@ -5,114 +5,92 @@ from orcapod.core.nodes.source_node import SourceNode from orcapod.core.sources.dict_source import DictSource from orcapod.databases.in_memory_databases import InMemoryArrowDatabase +from orcapod.errors import UnboundSourceError from orcapod.pipeline.serialization import LoadStatus +from orcapod.types import Schema class TestSourceNodeFromDescriptor: - def _make_source_and_descriptor(self): - source = DictSource( - data=[{"a": 1, "b": 2}, {"a": 3, "b": 4}], - tag_columns=["a"], - source_id="test", - ) - node = SourceNode(stream=source, label="my_source") - tag_schema, data_schema = node.output_schema() - descriptor = { - "node_type": "source", - "label": "my_source", - "content_hash": node.content_hash().to_string(), - "pipeline_hash": node.pipeline_hash().to_string(), - "data_context_key": node.data_context_key, - "output_schema": { - "tag": {k: str(v) for k, v in tag_schema.items()}, - "data": {k: str(v) for k, v in data_schema.items()}, - }, - "stream_type": "dict", - "source_id": "test", - "reconstructable": False, - } - return source, node, descriptor + """Tests for SourceNode construction — the new schema-only design. + + SourceNode no longer wraps a stream; instead it stores name + schemas + and raises UnboundSourceError on data access. + """ + + def _make_source_node(self): + tag_schema = Schema({"a": int}) + data_schema = Schema({"b": int}) + node = SourceNode(name="my_source", tag_schema=tag_schema, data_schema=data_schema) + return node def test_from_descriptor_with_stream(self): - source, original, descriptor = self._make_source_and_descriptor() - loaded = SourceNode.from_descriptor( - descriptor=descriptor, - stream=source, - databases={}, - ) - assert loaded.load_status == LoadStatus.FULL - assert loaded.label == "my_source" + """SourceNode can be constructed with name and schemas (new API).""" + node = self._make_source_node() + assert node.label == "my_source" + assert node.name == "my_source" def test_from_descriptor_without_stream_read_only(self): - _, original, descriptor = self._make_source_and_descriptor() - db = InMemoryArrowDatabase() - loaded = SourceNode.from_descriptor( - descriptor=descriptor, - stream=None, - databases={"pipeline": db}, - ) - assert loaded.load_status in (LoadStatus.READ_ONLY, LoadStatus.UNAVAILABLE) + """Unbound SourceNode has no concrete data.""" + node = self._make_source_node() + with pytest.raises(UnboundSourceError): + list(node.iter_data()) def test_from_descriptor_output_schema_from_metadata(self): - _, original, descriptor = self._make_source_and_descriptor() - loaded = SourceNode.from_descriptor( - descriptor=descriptor, - stream=None, - databases={}, - ) - tag_schema, data_schema = loaded.output_schema() - assert set(tag_schema.keys()) == set(descriptor["output_schema"]["tag"].keys()) - assert set(data_schema.keys()) == set( - descriptor["output_schema"]["data"].keys() - ) + """SourceNode output_schema returns the declared tag and data schemas.""" + tag_schema = Schema({"a": int}) + data_schema = Schema({"b": int}) + node = SourceNode(name="test_node", tag_schema=tag_schema, data_schema=data_schema) + t, d = node.output_schema() + assert set(t.keys()) == {"a"} + assert set(d.keys()) == {"b"} def test_from_descriptor_full_mode_delegates_to_stream(self): - """Full-mode node should delegate output_schema, iter_data, as_table to stream.""" - source, _, descriptor = self._make_source_and_descriptor() - loaded = SourceNode.from_descriptor( - descriptor=descriptor, - stream=source, - databases={}, + """SourceNode with bound concrete (via SourceJobNode) delegates iter_data.""" + from orcapod.core.nodes.source_node import SourceJobNode + tag_schema = Schema({"a": int}) + data_schema = Schema({"b": int}) + source = DictSource( + data=[{"a": 1, "b": 2}, {"a": 3, "b": 4}], + tag_columns=["a"], + source_id="test", ) - tag_schema, data_schema = loaded.output_schema() - assert "a" in tag_schema - assert "b" in data_schema - # iter_data should work - data = list(loaded.iter_data()) + job_node = SourceJobNode( + name="my_source", + tag_schema=tag_schema, + data_schema=data_schema, + concrete=source, + ) + t, d = job_node.output_schema() + assert "a" in t + assert "b" in d + data = list(job_node.iter_data()) assert len(data) == 2 def test_from_descriptor_read_only_iter_data_raises(self): - """Read-only node should raise when iter_data is called.""" - _, _, descriptor = self._make_source_and_descriptor() - loaded = SourceNode.from_descriptor( - descriptor=descriptor, - stream=None, - databases={}, - ) - with pytest.raises(RuntimeError, match="read-only mode"): - list(loaded.iter_data()) + """Unbound SourceNode should raise UnboundSourceError on iter_data.""" + node = self._make_source_node() + with pytest.raises(UnboundSourceError): + list(node.iter_data()) def test_from_descriptor_read_only_as_table_raises(self): - """Read-only node should raise when as_table is called.""" - _, _, descriptor = self._make_source_and_descriptor() - loaded = SourceNode.from_descriptor( - descriptor=descriptor, - stream=None, - databases={}, - ) - with pytest.raises(RuntimeError, match="read-only mode"): - loaded.as_table() + """Unbound SourceNode should raise UnboundSourceError on as_table.""" + node = self._make_source_node() + with pytest.raises(UnboundSourceError): + node.as_table() def test_from_descriptor_stored_hashes(self): - """Read-only node should return stored content_hash and pipeline_hash.""" - _, original, descriptor = self._make_source_and_descriptor() - loaded = SourceNode.from_descriptor( - descriptor=descriptor, - stream=None, - databases={}, - ) - assert loaded.content_hash().to_string() == descriptor["content_hash"] - assert loaded.pipeline_hash().to_string() == descriptor["pipeline_hash"] + """SourceNode produces stable hashes based on name and schemas.""" + node = self._make_source_node() + ch = node.content_hash() + ph = node.pipeline_hash() + # Same args → same hashes + node2 = SourceNode( + name="my_source", + tag_schema=Schema({"a": int}), + data_schema=Schema({"b": int}), + ) + assert node2.content_hash() == ch + assert node2.pipeline_hash() == ph from orcapod.core.nodes.function_node import FunctionNode diff --git a/tests/test_pipeline/test_node_protocols.py b/tests/test_pipeline/test_node_protocols.py index ce2086294..2c7b4a59b 100644 --- a/tests/test_pipeline/test_node_protocols.py +++ b/tests/test_pipeline/test_node_protocols.py @@ -144,7 +144,20 @@ def test_dispatch_operator(self): import pyarrow as pa from orcapod.core.sources import ArrowTableSource -from orcapod.core.nodes import SourceNode +from orcapod.core.nodes.source_node import SourceJobNode +from orcapod.types import Schema + + +def _make_source_job_node(table, tag_col="key"): + """Helper: create a SourceJobNode wrapping a concrete source.""" + src = ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) + tag_schema, data_schema = src.output_schema() + return SourceJobNode( + name="test_source", + tag_schema=tag_schema, + data_schema=data_schema, + concrete=src, + ) class TestSourceNodeExecute: @@ -153,8 +166,7 @@ def _make_source_node(self): "key": pa.array(["a", "b", "c"], type=pa.large_string()), "value": pa.array([1, 2, 3], type=pa.int64()), }) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - return SourceNode(src) + return _make_source_job_node(table) def test_execute_returns_list(self): node = self._make_source_node() @@ -163,10 +175,10 @@ def test_execute_returns_list(self): assert len(result) == 3 def test_execute_populates_cached_results(self): + """execute() on SourceJobNode returns correct data (no caching field).""" node = self._make_source_node() - node.execute() - assert node._cached_results is not None - assert len(node._cached_results) == 3 + result = node.execute() + assert len(result) == 3 def test_execute_with_observer(self): node = self._make_source_node() @@ -204,8 +216,7 @@ async def test_tightened_signature(self): "key": pa.array(["a", "b"], type=pa.large_string()), "value": pa.array([1, 2], type=pa.int64()), }) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - node = SourceNode(src) + node = _make_source_job_node(table) output_ch = Channel(buffer_size=16) await node.async_execute(output_ch.writer, observer=None) @@ -218,8 +229,7 @@ async def test_async_execute_with_observer(self): "key": pa.array(["a"], type=pa.large_string()), "value": pa.array([1], type=pa.int64()), }) - src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) - node = SourceNode(src) + node = _make_source_job_node(table) events = [] class Obs: diff --git a/tests/test_pipeline/test_orchestrator.py b/tests/test_pipeline/test_orchestrator.py index c5cd493ba..b656057fa 100644 --- a/tests/test_pipeline/test_orchestrator.py +++ b/tests/test_pipeline/test_orchestrator.py @@ -25,6 +25,7 @@ from orcapod.channels import Channel from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode +from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.operators import SelectDataColumns from orcapod.core.operators.join import Join from orcapod.core.operators.mappers import MapData @@ -72,10 +73,19 @@ def add_values(value: int, score: int) -> int: class TestSourceNodeAsyncExecute: + def _make_job_node(self, src): + tag_schema, data_schema = src.output_schema() + return SourceJobNode( + name="test_src", + tag_schema=tag_schema, + data_schema=data_schema, + concrete=src, + ) + @pytest.mark.asyncio async def test_pushes_all_rows_to_output(self): src = _make_source("key", "value", {"key": ["a", "b", "c"], "value": [1, 2, 3]}) - node = SourceNode(src) + node = self._make_job_node(src) output_ch = Channel(buffer_size=16) await node.async_execute(output_ch.writer) @@ -86,7 +96,7 @@ async def test_pushes_all_rows_to_output(self): @pytest.mark.asyncio async def test_closes_channel_on_completion(self): src = _make_source("key", "value", {"key": ["a"], "value": [1]}) - node = SourceNode(src) + node = self._make_job_node(src) output_ch = Channel(buffer_size=4) await node.async_execute(output_ch.writer) @@ -419,7 +429,13 @@ def test_single_terminal_source(self): import networkx as nx src = _make_source("key", "value", {"key": ["a"], "value": [1]}) - node = SourceNode(src) + tag_schema, data_schema = src.output_schema() + node = SourceJobNode( + name="test_src", + tag_schema=tag_schema, + data_schema=data_schema, + concrete=src, + ) G = nx.DiGraph() G.add_node(node) diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index 8b4d31aec..a74ecaaf4 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -23,10 +23,10 @@ OperatorNode, SourceNode, ) +from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, CachedSource -from orcapod.core.sources.source_spec import SourceSpec from orcapod.databases import InMemoryArrowDatabase from orcapod.pipeline import Pipeline from orcapod.pipeline.job import PipelineJob @@ -85,16 +85,16 @@ def test_pipeline_no_database_params(self): assert pipeline._compiled def test_pipeline_with_spec_leaves_compiles(self): - """Pipeline with SourceSpec leaves compiles without error.""" + """Pipeline with SourceNode leaves compiles without error.""" src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("input_a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("input_b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="input_a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="input_b", tag_schema=tag_b, data_schema=data_b) pipeline = Pipeline(name="spec_pipe") with pipeline: - Join()(spec_a, spec_b) + Join()(node_a, node_b) assert pipeline._compiled source_nodes = [ @@ -103,11 +103,11 @@ def test_pipeline_with_spec_leaves_compiles(self): assert len(source_nodes) == 2 def test_pipeline_with_concrete_leaf_raises(self): - """Pipeline.compile() raises ValueError if any leaf is not a SourceSpec.""" + """Pipeline.compile() raises ValueError if any leaf is not a SourceNode.""" src_a, src_b = _make_two_sources() pipeline = Pipeline(name="bad_pipe") - with pytest.raises(ValueError, match="SourceSpec"): + with pytest.raises(ValueError, match="SourceNode"): with pipeline: Join()(src_a, src_b) @@ -116,12 +116,12 @@ def test_pipeline_bind_returns_pipeline_job(self): src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="b", tag_schema=tag_b, data_schema=data_b) pipeline = Pipeline(name="p") with pipeline: - Join()(spec_a, spec_b) + Join()(node_a, node_b) db = InMemoryArrowDatabase() job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=db) diff --git a/tests/test_pipeline/test_pipeline_job.py b/tests/test_pipeline/test_pipeline_job.py index bf0b86418..0bbe64659 100644 --- a/tests/test_pipeline/test_pipeline_job.py +++ b/tests/test_pipeline/test_pipeline_job.py @@ -9,13 +9,14 @@ from orcapod.core.data_function import PythonDataFunction from orcapod.core.function_pod import FunctionPod -from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode +from orcapod.core.nodes import FunctionNode, OperatorNode +from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join from orcapod.core.sources import ArrowTableSource -from orcapod.core.sources.source_spec import SourceSpec from orcapod.databases import InMemoryArrowDatabase from orcapod.errors import SourceSpecMismatchError from orcapod.pipeline.job import PipelineJob +from orcapod.types import Schema # --------------------------------------------------------------------------- @@ -60,18 +61,19 @@ def store(): class TestPipelineJobRecording: def test_with_concrete_sources_auto_creates_specs(self, store): - """Concrete sources in with-block become SourceSpecs in job.pipeline.""" + """Concrete sources in with-block become SourceNodes in job.pipeline.""" src_a, src_b = _make_two_sources() job = PipelineJob(store=store) with job: Join()(src_a, src_b) - # Pipeline should have SourceSpec leaf nodes + # Pipeline should have SourceNode leaf nodes source_nodes = [ n for n in job.pipeline._node_graph.nodes() if isinstance(n, SourceNode) ] - assert all(isinstance(n.stream, SourceSpec) for n in source_nodes) + assert len(source_nodes) == 2 + assert all(isinstance(n, SourceNode) for n in source_nodes) def test_concrete_source_stored_in_sources(self, store): """Concrete sources from with-block are stored by label in job.sources.""" @@ -89,14 +91,14 @@ def test_concrete_source_stored_in_sources(self, store): assert job.sources["source_b"] is src_b def test_spec_leaf_not_added_to_sources(self, store): - """SourceSpec leaves are NOT added to job.sources (they're unbound).""" + """SourceNode leaves are NOT added to job.sources (they're unbound).""" src_a, _ = _make_two_sources() tag_b, data_b = _make_source("key", "score", {"key": ["a"], "score": [1]}).output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) + node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) job = PipelineJob(store=store) with job: - Join()(src_a, spec_b) + Join()(src_a, node_b) assert "spec_b" not in job.sources @@ -124,12 +126,12 @@ def test_bind_sources_returns_new_job(self, store): src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="b", tag_schema=tag_b, data_schema=data_b) job = PipelineJob(store=store) with job: - Join()(spec_a, spec_b) + Join()(node_a, node_b) job2 = job.bind(sources={"a": src_a, "b": src_b}) assert job2 is not job @@ -152,12 +154,12 @@ def test_bind_preserves_existing_sources(self, store): src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="b", tag_schema=tag_b, data_schema=data_b) job = PipelineJob(store=store) with job: - Join()(spec_a, spec_b) + Join()(node_a, node_b) job2 = job.bind(sources={"a": src_a}) job3 = job2.bind(sources={"b": src_b}) @@ -169,13 +171,12 @@ def test_bind_validates_schema_at_bind_time(self, store): """bind() raises SourceSpecMismatchError for incompatible sources.""" src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() - # Create a spec that requires an extra column the source doesn't have - from orcapod.types import Schema - wrong_spec = SourceSpec("a", tag_schema=tag_a, data_schema=Schema({"value": int, "extra": str})) + # Create a node that requires an extra column the source doesn't have + wrong_node = SourceNode(name="a", tag_schema=tag_a, data_schema=Schema({"value": int, "extra": str})) job = PipelineJob(store=store) with job: - Join()(wrong_spec, src_b) + Join()(wrong_node, src_b) with pytest.raises(SourceSpecMismatchError): job.bind(sources={"a": src_a}) @@ -187,12 +188,12 @@ def test_pipeline_bind_wraps_in_job(self, store): src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="b", tag_schema=tag_b, data_schema=data_b) pipeline = Pipeline(name="p") with pipeline: - Join()(spec_a, spec_b) + Join()(node_a, node_b) job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=store) assert isinstance(job, PipelineJob) @@ -205,12 +206,12 @@ def test_pipeline_bind_propagates_name(self, store): src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="b", tag_schema=tag_b, data_schema=data_b) pipeline = Pipeline(name="my_pipeline") with pipeline: - Join()(spec_a, spec_b) + Join()(node_a, node_b) job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=store) assert job._pipeline_name == ("my_pipeline",), ( @@ -225,14 +226,14 @@ def test_pipeline_bind_propagates_name(self, store): class TestPipelineJobCompleteness: def test_unbound_specs_lists_unbound(self, store): - """unbound_specs() lists SourceSpec names not in job.sources.""" + """unbound_specs() lists SourceNode names not in job.sources.""" src_a, src_b = _make_two_sources() tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) + node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) job = PipelineJob(store=store) with job: - Join()(src_a, spec_b) + Join()(src_a, node_b) unbound = job.unbound_specs() assert len(unbound) == 1 @@ -265,11 +266,11 @@ def test_is_complete_false_when_store_missing(self): def test_is_complete_false_when_specs_unbound(self, store): src_a, src_b = _make_two_sources() tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) + node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) job = PipelineJob(store=store) with job: - Join()(src_a, spec_b) + Join()(src_a, node_b) assert not job.is_complete() @@ -291,14 +292,14 @@ def test_is_runnable_true_when_all_upstreams_bound(self, store): assert job.is_runnable("joiner") def test_is_runnable_false_when_spec_unbound(self, store): - """is_runnable returns False when an upstream SourceSpec is unbound.""" + """is_runnable returns False when an upstream SourceNode is unbound.""" src_a, src_b = _make_two_sources() tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) + node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) job = PipelineJob(store=store) with job: - Join()(src_a, spec_b, label="joiner") + Join()(src_a, node_b, label="joiner") assert not job.is_runnable("joiner") @@ -363,16 +364,16 @@ def test_run_produces_correct_values(self, store): assert totals == [110, 220] # a: 10+100, b: 20+200 def test_run_partial_execution_skips_unbound_subgraph(self, store): - """Nodes with unbound upstream SourceSpecs are excluded from execution.""" + """Nodes with unbound upstream SourceNodes are excluded from execution.""" src_a, src_b = _make_two_sources() tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("spec_b", tag_schema=tag_b, data_schema=data_b) + node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) pf = PythonDataFunction(add_values, output_keys="total") pod = FunctionPod(data_function=pf) job = PipelineJob(store=store) with job: - joined = Join()(src_a, spec_b) + joined = Join()(src_a, node_b) pod(joined, label="adder") result = job.run() @@ -488,14 +489,14 @@ def test_bind_then_run(self, store): src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("src_a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("src_b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="src_a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="src_b", tag_schema=tag_b, data_schema=data_b) pf = PythonDataFunction(add_values, output_keys="total") pod = FunctionPod(data_function=pf) pipeline = Pipeline(name="bp") with pipeline: - joined = Join()(spec_a, spec_b) + joined = Join()(node_a, node_b) pod(joined, label="adder") job = pipeline.bind( @@ -643,12 +644,12 @@ def test_load_after_partial_run_restores_unresolved_specs(self, store, tmp_path) """Loaded job preserves unresolved_specs from a partial run.""" src_a, src_b = _make_two_sources() tag_b, data_b = src_b.output_schema() - spec_b = SourceSpec("unbound_b", tag_schema=tag_b, data_schema=data_b) + node_b = SourceNode(name="unbound_b", tag_schema=tag_b, data_schema=data_b) pf = PythonDataFunction(add_values, output_keys="total") pod = FunctionPod(data_function=pf) job = PipelineJob(store=store) with job: - joined = Join()(src_a, spec_b) + joined = Join()(src_a, node_b) pod(joined, label="adder") result = job.run() assert "unbound_b" in result.unresolved_specs diff --git a/tests/test_pipeline/test_serialization.py b/tests/test_pipeline/test_serialization.py index 24478ad85..fbefa9058 100644 --- a/tests/test_pipeline/test_serialization.py +++ b/tests/test_pipeline/test_serialization.py @@ -8,9 +8,9 @@ import pytest from orcapod.core.nodes import SourceNode +from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join from orcapod.core.sources import ArrowTableSource -from orcapod.core.sources.source_spec import SourceSpec from orcapod.databases.in_memory_databases import InMemoryArrowDatabase from orcapod.pipeline import Pipeline from orcapod.pipeline.serialization import PIPELINE_FORMAT_VERSION @@ -18,7 +18,7 @@ @pytest.fixture def spec_pipeline(tmp_path): - """A compiled Pipeline using SourceSpec leaves.""" + """A compiled Pipeline using SourceNode leaves.""" def _src(tag, data): tbl = pa.table({tag: pa.array(["a"], type=pa.large_string()), data: pa.array([1], type=pa.int64())}) return ArrowTableSource(tbl, tag_columns=[tag], infer_nullable=True) @@ -28,12 +28,12 @@ def _src(tag, data): tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() - spec_a = SourceSpec("source_a", tag_schema=tag_a, data_schema=data_a) - spec_b = SourceSpec("source_b", tag_schema=tag_b, data_schema=data_b) + node_a = SourceNode(name="source_a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="source_b", tag_schema=tag_b, data_schema=data_b) pipeline = Pipeline(name="spec_pipe") with pipeline: - Join()(spec_a, spec_b, label="joiner") + Join()(node_a, node_b, label="joiner") return pipeline, tmp_path @@ -61,17 +61,17 @@ def test_save_no_databases_block(self, spec_pipeline): assert "databases" not in data def test_save_source_spec_nodes(self, spec_pipeline): - """SourceSpec nodes must serialize with source_type='spec'.""" + """SourceNode nodes must serialize with source_type='node'.""" pipeline, tmp_path = spec_pipeline path = tmp_path / "pipeline.json" pipeline.save(str(path)) data = json.loads(path.read_text()) - spec_nodes = [ + source_nodes = [ n for n in data["nodes"].values() if n.get("node_type") == "source" - and n.get("source_config", {}).get("source_type") == "spec" + and n.get("source_config", {}).get("source_type") == "node" ] - assert len(spec_nodes) == 2 + assert len(source_nodes) == 2 def test_save_load_roundtrip_preserves_topology(self, spec_pipeline): """load() reconstructs the same number of nodes and edges.""" @@ -83,17 +83,17 @@ def test_save_load_roundtrip_preserves_topology(self, spec_pipeline): assert len(list(loaded._node_graph.edges())) == len(list(pipeline._node_graph.edges())) def test_save_load_restores_spec_names(self, spec_pipeline): - """SourceSpec names must survive save/load.""" + """SourceNode names must survive save/load.""" pipeline, tmp_path = spec_pipeline path = tmp_path / "pipeline.json" pipeline.save(str(path)) loaded = Pipeline.load(str(path)) - spec_names = { - node.stream.name + node_names = { + node.name for node in loaded._persistent_node_map.values() - if isinstance(node, SourceNode) and isinstance(node.stream, SourceSpec) + if isinstance(node, SourceNode) } - assert spec_names == {"source_a", "source_b"} + assert node_names == {"source_a", "source_b"} class TestPipelineBlueprintLoad: diff --git a/tests/test_pipeline/test_sync_orchestrator.py b/tests/test_pipeline/test_sync_orchestrator.py index 381855114..19b64ced8 100644 --- a/tests/test_pipeline/test_sync_orchestrator.py +++ b/tests/test_pipeline/test_sync_orchestrator.py @@ -330,9 +330,15 @@ class TestMaterializedStreamIdentity: def test_materialized_stream_has_same_pipeline_hash(self): """Stream reconstructed from buffer should have same pipeline_hash as original.""" src = _make_source("key", "value", {"key": ["a", "b"], "value": [1, 2]}) - from orcapod.core.nodes import SourceNode - - node = SourceNode(src) + from orcapod.core.nodes.source_node import SourceJobNode + + tag_schema, data_schema = src.output_schema() + node = SourceJobNode( + name="test_src", + tag_schema=tag_schema, + data_schema=data_schema, + concrete=src, + ) buf = list(node.iter_data()) stream = SyncPipelineOrchestrator._materialize_as_stream(buf, node) diff --git a/tests/test_protocols/test_node_protocols.py b/tests/test_protocols/test_node_protocols.py index 072766203..f14f33b11 100644 --- a/tests/test_protocols/test_node_protocols.py +++ b/tests/test_protocols/test_node_protocols.py @@ -6,11 +6,13 @@ import pytest from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode +from orcapod.core.nodes.source_node import SourceJobNode from orcapod.protocols.node_protocols import ( is_function_node, is_operator_node, is_source_node, ) +from orcapod.types import Schema @pytest.fixture @@ -28,7 +30,13 @@ def _sample_source(): @pytest.fixture def source_node(_sample_source): - return SourceNode(_sample_source) + tag_schema, data_schema = _sample_source.output_schema() + return SourceJobNode( + name="test_source", + tag_schema=tag_schema, + data_schema=data_schema, + concrete=_sample_source, + ) @pytest.fixture From 2c1784432025a9d27784d15fe58bab5a3c2365a8 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 01:25:03 +0000 Subject: [PATCH 05/24] fix(nodes): clear content_hash cache on SourceJobNode._concrete mutation; remove deprecated unbound_specs alias - Add __setattr__ override on SourceJobNode to clear _content_hash_cache whenever _concrete is mutated, preventing stale schema-based hashes from being returned after a concrete source is bound in-place - Remove unbound_specs() alias from PipelineJob; update two test callers to use unbound_source_nodes() per project no-alias policy - Update PipelineJob class/method docstrings to replace SourceSpec references with SourceNode throughout - Add Yields section to SourceNodeBase.async_iter_data() docstring - Add tests: SourceNode.as_table() raises UnboundSourceError, bound SourceJobNode.as_table() delegates to concrete; also test that _concrete mutation clears the content hash cache Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/source_node.py | 18 +++++++++- src/orcapod/pipeline/job.py | 20 +++++------- tests/test_core/nodes/test_source_node.py | 40 +++++++++++++++++++++++ tests/test_pipeline/test_pipeline_job.py | 6 ++-- 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 3cec7cf7f..86ff543d3 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -254,6 +254,10 @@ def as_table( async def async_iter_data(self): """Asynchronous iterator over (tag, data) pairs. + Yields: + tuple[TagProtocol, DataProtocol]: A ``(tag, data)`` pair from the + concrete source. Raises before yielding anything when unbound. + Raises: UnboundSourceError: When no concrete data is available. """ @@ -404,7 +408,19 @@ def __init__( data_schema=data_schema, data_context=data_context, ) - self._concrete: "StreamProtocol | None" = concrete + # Use object.__setattr__ to bypass the property setter during __init__ + # (the cache doesn't exist yet at this point). + object.__setattr__(self, "_concrete", concrete) + + def __setattr__(self, name: str, value: object) -> None: + """Clear ``_content_hash_cache`` whenever ``_concrete`` is mutated. + + This prevents a stale schema-based hash from being returned by the + parent ``content_hash()`` cache after the concrete source is updated. + """ + object.__setattr__(self, name, value) + if name == "_concrete" and hasattr(self, "_content_hash_cache"): + self._content_hash_cache.clear() def content_hash(self, hasher=None) -> ContentHash: """Return data-inclusive hash when bound; schema-based hash when unbound. diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index efff2a92a..3908db80d 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -28,12 +28,12 @@ class PipelineJob(AutoRegisteringContextBasedTracker): ``PipelineJob`` is the everyday working object. It is built incrementally: its ``with``-block records both the DAG structure and any concrete source bindings simultaneously. Concrete sources are automatically promoted to - ``SourceSpec`` declarations in the underlying ``Pipeline``, with their + ``SourceNode`` declarations in the underlying ``Pipeline``, with their concrete instances stored in ``job.sources``. After the ``with`` block, ``job.pipeline`` is a fully compiled, pure - ``Pipeline`` (SourceSpec-only leaves). ``job.run()`` executes the - resolvable subgraph — nodes whose upstream SourceSpecs are all bound. + ``Pipeline`` (SourceNode-only leaves). ``job.run()`` executes the + resolvable subgraph — nodes whose upstream SourceNodes are all bound. ``PipelineJob`` can also be created from a ``Pipeline`` via ``pipeline.bind(sources=..., store=...)`` for the "explicit blueprint" @@ -245,7 +245,7 @@ def record_operator_pod_invocation( @property def pipeline(self) -> "Pipeline": - """The compiled pure Pipeline (SourceSpec-only leaves). + """The compiled pure Pipeline (SourceNode-only leaves). Raises: RuntimeError: If the with-block has not been completed yet. @@ -259,7 +259,7 @@ def pipeline(self) -> "Pipeline": @property def sources(self) -> dict[str, cp.StreamProtocol]: - """Mapping of SourceSpec name to bound concrete source.""" + """Mapping of SourceNode name to bound concrete source.""" return dict(self._sources) @property @@ -287,12 +287,12 @@ def bind( Non-mutating — the original ``PipelineJob`` is unchanged. Existing bindings not mentioned in this call are carried forward. - ``SourceSpec.validate()`` is called for each source in *sources*; + ``SourceNode.validate()`` is called for each source in *sources*; ``SourceSpecMismatchError`` is raised on schema mismatch. Args: - sources: Mapping of SourceSpec name to concrete source. Each - source is validated against the matching SourceSpec. + sources: Mapping of SourceNode name to concrete source. Each + source is validated against the matching SourceNode. store: Replaces the current store. execution_context: Replaces the current execution context. @@ -365,10 +365,6 @@ def unbound_source_nodes(self) -> "list[SourceNode]": seen.add(node.name) return unbound - def unbound_specs(self) -> "list[SourceNode]": - """Deprecated — use unbound_source_nodes() instead.""" - return self.unbound_source_nodes() - def is_complete(self) -> bool: """Return ``True`` when all source nodes are bound and a store is set. diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 443025d6e..0d9ea2d4d 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -168,3 +168,43 @@ def test_mutable_concrete_updates_in_place(self, tag_schema, data_schema): src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) job_node._concrete = src assert job_node._concrete is src + + def test_concrete_mutation_clears_content_hash_cache(self, tag_schema, data_schema): + """Setting _concrete clears the content_hash cache so stale values are not returned.""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + from orcapod.core.sources.dict_source import DictSource + + job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + schema_hash = job_node.content_hash() # populates cache with schema-based hash + + src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + job_node._concrete = src # should clear the cache + + assert job_node._content_hash_cache == {} # cache cleared + bound_hash = job_node.content_hash() + assert bound_hash != schema_hash # now returns concrete-based hash + assert bound_hash == src.content_hash() + + +class TestSourceNodeAsTable: + def test_source_node_as_table_raises_unbound_error(self, tag_schema, data_schema): + """SourceNode.as_table() raises UnboundSourceError (no concrete data).""" + from orcapod.core.nodes.source_node import SourceNode + + node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) + with pytest.raises(UnboundSourceError): + node.as_table() + + def test_source_job_node_as_table_delegates_to_concrete(self, tag_schema, data_schema): + """Bound SourceJobNode.as_table() delegates to concrete and does not raise.""" + import pyarrow as pa + + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.sources.dict_source import DictSource + + src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) + job_node = SourceJobNode( + name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + ) + table = job_node.as_table() + assert isinstance(table, pa.Table) diff --git a/tests/test_pipeline/test_pipeline_job.py b/tests/test_pipeline/test_pipeline_job.py index 0bbe64659..572f84bf4 100644 --- a/tests/test_pipeline/test_pipeline_job.py +++ b/tests/test_pipeline/test_pipeline_job.py @@ -226,7 +226,7 @@ def test_pipeline_bind_propagates_name(self, store): class TestPipelineJobCompleteness: def test_unbound_specs_lists_unbound(self, store): - """unbound_specs() lists SourceNode names not in job.sources.""" + """unbound_source_nodes() lists SourceNode names not in job.sources.""" src_a, src_b = _make_two_sources() tag_b, data_b = src_b.output_schema() node_b = SourceNode(name="spec_b", tag_schema=tag_b, data_schema=data_b) @@ -235,7 +235,7 @@ def test_unbound_specs_lists_unbound(self, store): with job: Join()(src_a, node_b) - unbound = job.unbound_specs() + unbound = job.unbound_source_nodes() assert len(unbound) == 1 assert unbound[0].name == "spec_b" @@ -245,7 +245,7 @@ def test_unbound_specs_empty_when_all_bound(self, store): with job: Join()(src_a, src_b) # both auto-bound via content-hash-based spec names - assert job.unbound_specs() == [] + assert job.unbound_source_nodes() == [] def test_is_complete_true_when_all_bound_with_store(self, store): src_a, src_b = _make_two_sources() From 4f4010b1de85f6fda6abbd5936ba9cefb775cb83 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 01:59:41 +0000 Subject: [PATCH 06/24] =?UTF-8?q?refactor(nodes):=20split=20FunctionNode?= =?UTF-8?q?=20=E2=86=92=20FunctionNodeBase=20+=20FunctionNode=20+=20Functi?= =?UTF-8?q?onJobNode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a sibling class hierarchy where FunctionNode (blueprint descriptor) and FunctionJobNode (DB-backed execution node) both inherit directly from FunctionNodeBase, with neither inheriting from the other. FunctionNode.iter_data() raises PipelineJobRequiredError; all DB logic, two-phase caching, and execution methods live on FunctionJobNode. Updates all callers, tests, and pipeline/job.py to use FunctionJobNode where DB-backed execution is required. Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/__init__.py | 6 +- src/orcapod/core/nodes/function_node.py | 860 ++++++++++-------- src/orcapod/pipeline/job.py | 3 +- .../test_copilot_review_issues.py | 5 +- .../test_channels/test_node_async_execute.py | 54 +- .../test_core/data_function/test_executor.py | 18 +- .../test_function_node_attach_db.py | 27 +- .../test_function_node_caching.py | 3 +- .../function_pod/test_function_pod_node.py | 35 +- .../test_function_pod_node_stream.py | 15 +- .../test_pipeline_hash_integration.py | 55 +- .../nodes/test_function_node_get_cached.py | 5 +- .../nodes/test_function_node_iteration.py | 7 +- .../nodes/test_function_node_split.py | 148 +++ tests/test_core/nodes/test_node_execute.py | 5 +- .../test_core/sources/test_derived_source.py | 13 +- tests/test_core/test_caching_integration.py | 15 +- tests/test_core/test_table_scope.py | 37 +- .../test_function_node_nullability.py | 8 +- tests/test_pipeline/test_node_descriptors.py | 3 +- tests/test_pipeline/test_node_protocols.py | 7 +- tests/test_pipeline/test_orchestrator.py | 3 +- tests/test_pipeline/test_pipeline.py | 11 +- 23 files changed, 827 insertions(+), 516 deletions(-) create mode 100644 tests/test_core/nodes/test_function_node_split.py diff --git a/src/orcapod/core/nodes/__init__.py b/src/orcapod/core/nodes/__init__.py index 65a7cf069..e4b7f8ee6 100644 --- a/src/orcapod/core/nodes/__init__.py +++ b/src/orcapod/core/nodes/__init__.py @@ -1,14 +1,18 @@ from typing import TypeAlias -from .function_node import FunctionNode +from .function_node import FunctionJobNode, FunctionNode, FunctionNodeBase from .operator_node import OperatorNode from .source_node import SourceJobNode, SourceNode, SourceNodeBase GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode +JobNode: TypeAlias = SourceJobNode | FunctionJobNode __all__ = [ + "FunctionJobNode", "FunctionNode", + "FunctionNodeBase", "GraphNode", + "JobNode", "OperatorNode", "SourceJobNode", "SourceNode", diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 3d2bc8300..affbb9ab5 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1,4 +1,19 @@ -"""FunctionNode — stream node for data function invocations with optional DB persistence.""" +"""FunctionNode hierarchy — pure blueprint + DB-backed execution node. + +Three classes: + +* ``FunctionNodeBase`` — shared base; no DB state. Holds identity, + schema, and all non-DB properties. +* ``FunctionNode`` — thin blueprint descriptor. Raises + ``PipelineJobRequiredError`` on ``iter_data()``. This is the node + recorded in a ``Pipeline`` and serialized to disk. +* ``FunctionJobNode`` — DB-backed execution node; carries all DB logic + from the original ``FunctionNode``. Created by ``PipelineJob`` at + run time. + +``FunctionNode`` and ``FunctionJobNode`` are *siblings*: both inherit +directly from ``FunctionNodeBase``, neither from the other. +""" from __future__ import annotations @@ -14,6 +29,7 @@ from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import StreamBase from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.errors import PipelineJobRequiredError from orcapod.protocols.core_protocols import ( FunctionPodProtocol, DataFunctionExecutorProtocol, @@ -61,14 +77,16 @@ def _executor_supports_concurrent( return executor is not None and executor.supports_concurrent_execution -class FunctionNode(StreamBase): - """Stream node representing a data function invocation with optional DB persistence. +# --------------------------------------------------------------------------- +# FunctionNodeBase — shared base (no DB) +# --------------------------------------------------------------------------- + - When constructed without database parameters, provides the core stream - interface (identity, schema, iteration) without any persistence. When - databases are provided (either at construction or via ``attach_databases``), - adds result caching via ``CachedFunctionPod``, pipeline record storage, - and two-phase iteration (cached first, then compute missing). +class FunctionNodeBase(StreamBase): + """Shared base for ``FunctionNode`` and ``FunctionJobNode``. + + Carries all non-DB state: identity, schema, upstreams, and properties + shared by both the blueprint and the execution variant. """ node_type = "function" @@ -80,9 +98,6 @@ def __init__( tracker_manager: TrackerManagerProtocol | None = None, label: str | None = None, config: Config | None = None, - # Optional DB params for persistent mode: - pipeline_database: ArrowDatabaseProtocol | None = None, - result_database: ArrowDatabaseProtocol | None = None, table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", ): if tracker_manager is None: @@ -90,7 +105,7 @@ def __init__( self.tracker_manager = tracker_manager self._data_function = function_pod.data_function - # FunctionPod used for the `producer` property and pipeline identity + # FunctionPod used for the ``producer`` property and pipeline identity self._function_pod = function_pod super().__init__(label=label, config=config) @@ -117,20 +132,8 @@ def __init__( self._input_stream = input_stream - # stream-level caching state - self._cached_output_datas: dict[ - str, tuple[TagProtocol, DataProtocol | None] - ] = {} - self._cached_output_table: pa.Table | None = None - self._cached_content_hash_column: pa.Array | None = None - - # DB persistence state (initially None; set via __init__ params or attach_databases) - self._pipeline_database: ArrowDatabaseProtocol | None = None - self._cached_function_pod: CachedFunctionPod | None = None - self._output_schema_hash: str | None = None - # Descriptor fields — populated by from_descriptor() for read-only/UNAVAILABLE - # nodes. Initialized here so they are always present on the concrete class + # nodes. Initialized here so they are always present on the concrete class # (avoids getattr access for possibly-absent attributes). from orcapod.pipeline.serialization import LoadStatus self._load_status: LoadStatus = LoadStatus.FULL @@ -141,6 +144,7 @@ def __init__( self._stored_pipeline_path: tuple[str, ...] = () self._stored_result_record_path: tuple[str, ...] = () self._descriptor: dict = {} + if table_scope not in ("pipeline_hash", "content_hash"): raise ValueError( f"Unknown table_scope {table_scope!r}. " @@ -149,100 +153,318 @@ def __init__( self._table_scope = table_scope self._node_identity_path_cache: tuple[str, ...] | None = None - if pipeline_database is not None: - self.attach_databases( - pipeline_database=pipeline_database, - result_database=result_database, - ) + # ------------------------------------------------------------------ + # load_status + # ------------------------------------------------------------------ + + @property + def load_status(self) -> Any: + """Return the load status of this node. + + Returns: + The ``LoadStatus`` enum value indicating how this node was + loaded. Defaults to ``FULL`` for nodes created via + ``__init__``. + """ + return self._load_status # ------------------------------------------------------------------ - # attach_databases + # Core properties # ------------------------------------------------------------------ - def attach_databases( + @property + def producer(self) -> FunctionPodProtocol: + return self._function_pod + + @property + def data_context(self) -> contexts.DataContext: + return contexts.resolve_context(self._function_pod.data_context_key) + + @property + def data_context_key(self) -> str: + return self._function_pod.data_context_key + + @property + def executor(self) -> DataFunctionExecutorProtocol | None: + """The executor set on the underlying data function.""" + return self._data_function.executor + + @executor.setter + def executor(self, executor: DataFunctionExecutorProtocol | None) -> None: + """Set or clear the executor on the underlying data function.""" + self._data_function.executor = executor + + @property + def upstreams(self) -> tuple[StreamProtocol, ...]: + return (self._input_stream,) + + @upstreams.setter + def upstreams(self, value: tuple[StreamProtocol, ...]) -> None: + if len(value) != 1: + raise ValueError("FunctionPod can only have one upstream") + self._input_stream = value[0] + + # ------------------------------------------------------------------ + # Read-only overrides (for deserialized nodes without live function_pod) + # ------------------------------------------------------------------ + + def content_hash(self, hasher=None) -> ContentHash: + """Return the content hash, using stored value in read-only mode.""" + if self._function_pod is None and self._stored_content_hash is not None: + from orcapod.types import ContentHash as CH + + return CH.from_string(self._stored_content_hash) + return super().content_hash(hasher) + + def pipeline_hash(self, hasher=None) -> ContentHash: + """Return the pipeline hash, using stored value in read-only mode.""" + if self._function_pod is None and self._stored_pipeline_hash is not None: + from orcapod.types import ContentHash as CH + + return CH.from_string(self._stored_pipeline_hash) + return super().pipeline_hash(hasher) + + def output_schema( self, - pipeline_database: ArrowDatabaseProtocol, - result_database: ArrowDatabaseProtocol | None = None, - ) -> None: - """Attach databases for persistent caching and pipeline records. + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """Return output schema, using stored value in read-only mode.""" + if self._function_pod is None: + tag = Schema(self._stored_schema.get("tag", {})) + data = Schema(self._stored_schema.get("data", {})) + return tag, data + tag_schema = self._input_stream.output_schema( + columns=columns, all_info=all_info + )[0] + return tag_schema, self._data_function.output_data_schema - Creates a ``CachedFunctionPod`` wrapping the original function pod - for result caching. The pipeline database is used separately for - pipeline-level provenance records (tag + data hash). + def keys( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + if self._function_pod is None: + tag_keys = tuple(self._stored_schema.get("tag", {}).keys()) + data_keys = tuple(self._stored_schema.get("data", {}).keys()) + return tag_keys, data_keys + tag_schema, data_schema = self.output_schema( + columns=columns, all_info=all_info + ) + return tuple(tag_schema.keys()), tuple(data_schema.keys()) - The databases are expected to be pre-scoped by the pipeline (via - ``db.at(*pipeline_name).at("_result")`` etc.) so no additional path - prefix is needed here. + # ------------------------------------------------------------------ + # Pipeline path + # ------------------------------------------------------------------ - Args: - pipeline_database: Database for pipeline records. - result_database: Database for cached results. Defaults to - pipeline_database. + @property + def node_identity_path(self) -> tuple[str, ...]: + """Return the node identity path for observer contextualization. + + When ``table_scope="pipeline_hash"`` (default) the path is + ``pod.uri + (schema:{pipeline_hash},)`` — all runs that share the same + pipeline structure are routed to one shared table, with per-run + disambiguation via the ``_node_content_hash`` row-level column. + + When ``table_scope="content_hash"`` the legacy path is returned: + ``pod.uri + (schema:{pipeline_hash}, instance:{content_hash})``. + + In read-only/UNAVAILABLE mode (no pod) the path stored from the + deserialized descriptor is returned (empty tuple when absent). """ - if result_database is None: - # Default result database is pipeline_database scoped to "_result" - # so that results are stored separately from pipeline-level records. - result_database = pipeline_database.at("_result") + if self._data_function is None: + return self._stored_pipeline_path + if self._node_identity_path_cache is not None: + return self._node_identity_path_cache + pf = self._function_pod + path = pf.uri + (f"schema:{self.pipeline_hash().to_string()}",) + if self._table_scope != "pipeline_hash": + path += (f"instance:{self.content_hash().to_string()}",) + self._node_identity_path_cache = path + return path - # Always wrap the original function_pod (not a previous cached wrapper) - self._cached_function_pod = CachedFunctionPod( - self._function_pod, - result_database=result_database, - ) + @property + def node_uri(self) -> tuple[str, ...]: + """Canonical URI tuple identifying this computation. - self._pipeline_database = pipeline_database + Identical to ``data_function.uri`` at runtime. + Returns stored value in read-only (deserialized) mode. + """ + if self._data_function is None: + return self._stored_node_uri + return self._data_function.uri - # Clear all caches - self._node_identity_path_cache = None - self.clear_cache() - self._content_hash_cache.clear() - self._pipeline_hash_cache.clear() + # ------------------------------------------------------------------ + # Caching + # ------------------------------------------------------------------ - # Compute output schema hash - self._output_schema_hash = self.data_context.semantic_hasher.hash_object( - self._data_function.output_data_schema - ).to_string() + def clear_cache(self) -> None: + """Clear the node identity path cache.""" + self._node_identity_path_cache = None + self._update_modified_time() # ------------------------------------------------------------------ - # Internal helpers + # as_table # ------------------------------------------------------------------ - def _require_pipeline_database(self) -> None: - """Raise a clear RuntimeError if no pipeline database is attached. + def as_table( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> "pa.Table": + if self._cached_output_table is None: + all_tags = [] + all_data = [] + tag_schema, data_schema = None, None + for tag, data in self.iter_data(): + if tag_schema is None: + tag_schema = tag.arrow_schema(all_info=True) + if data_schema is None: + data_schema = data.arrow_schema(all_info=True) + all_tags.append(tag.as_dict(all_info=True)) + all_data.append(data.as_dict(all_info=True)) - Called at the top of methods that unconditionally access - ``self._pipeline_database``. Provides an actionable error message - instead of an opaque ``AttributeError: 'NoneType' object has no - attribute ...`` when a definition-level pipeline is executed without - supplying a database. - """ - if self._pipeline_database is None: - raise RuntimeError( - f"FunctionNode '{self.label}' has no pipeline database attached. " - "Either construct the pipeline with a pipeline_database argument, " - "or supply one via Pipeline.load(..., pipeline_database=)." + if not all_tags: + self._cached_output_table = pa.table({}) + + converter = self.data_context.type_converter + + # Derive the Python schema from the Arrow schema when available, + # rather than re-inferring from dict values. This preserves precise + # types for empty containers (e.g. {} infers as dict[Any, Any] but + # the Arrow schema knows it's dict[str, str]). + data_python_schema = ( + converter.arrow_schema_to_python_schema(data_schema) + if data_schema is not None + else None + ) + struct_data = converter.python_dicts_to_struct_dicts( + all_data, python_schema=data_python_schema + ) + all_tags_as_tables: pa.Table = pa.Table.from_pylist( + all_tags, schema=tag_schema + ) + if constants.CONTEXT_KEY in all_tags_as_tables.column_names: + all_tags_as_tables = all_tags_as_tables.drop([constants.CONTEXT_KEY]) + all_data_as_tables: pa.Table = pa.Table.from_pylist( + struct_data, schema=data_schema ) - def _filter_by_content_hash(self, table: pa.Table) -> pa.Table: - """Filter *table* to rows whose ``NODE_CONTENT_HASH_COL`` matches this node. + self._cached_output_table = arrow_utils.hstack_tables( + all_tags_as_tables, all_data_as_tables + ) + if self._cached_output_table is None: + self._cached_output_table = pa.table({}) - Only applied when ``table_scope="pipeline_hash"`` because in that mode - multiple runs share the same DB table and must be disambiguated at read - time. In ``"content_hash"`` mode every run has its own table so no - filtering is needed. - """ - if self._table_scope != "pipeline_hash": - return table - col_name = constants.NODE_CONTENT_HASH_COL - if col_name not in table.column_names: - raise ValueError( - f"Cannot isolate records for table_scope='pipeline_hash': " - f"required column {col_name!r} is missing from the stored table. " - "This may indicate records written by an older version of the code." + column_config = ColumnConfig.handle_config(columns, all_info=all_info) + + drop_columns = [] + if not column_config.system_tags: + drop_columns.extend( + [ + c + for c in self._cached_output_table.column_names + if c.startswith(constants.SYSTEM_TAG_PREFIX) + ] ) - own_hash = self.content_hash().to_string() - mask = pc.equal(table.column(col_name), own_hash) - return table.filter(mask) + if not column_config.source: + drop_columns.extend(f"{constants.SOURCE_PREFIX}{c}" for c in self.keys()[1]) + if not column_config.context: + drop_columns.append(constants.CONTEXT_KEY) + if not column_config.meta: + drop_columns.extend( + c + for c in self._cached_output_table.column_names + if c.startswith(constants.META_PREFIX) + ) + elif not isinstance(column_config.meta, bool): + # Collection[str]: keep only meta columns matching the specified prefixes + drop_columns.extend( + c + for c in self._cached_output_table.column_names + if c.startswith(constants.META_PREFIX) + and not any(c.startswith(p) for p in column_config.meta) + ) + output_table = self._cached_output_table.drop( + [c for c in drop_columns if c in self._cached_output_table.column_names] + ) + + if column_config.content_hash: + if self._cached_content_hash_column is None: + content_hashes = [] + for tag, data in self.iter_data(): + content_hashes.append(data.content_hash().to_string()) + self._cached_content_hash_column = pa.array( + content_hashes, type=pa.large_string() + ) + assert self._cached_content_hash_column is not None, ( + "_cached_content_hash_column should not be None here." + ) + hash_column_name = ( + "_content_hash" + if column_config.content_hash is True + else column_config.content_hash + ) + output_table = output_table.append_column( + hash_column_name, self._cached_content_hash_column + ) + + if column_config.sort_by_tags: + output_table_schema = output_table.schema + output_table = ( + pl.DataFrame(output_table) + .sort(by=self.keys()[0], descending=False) + .to_arrow() + ) + output_table = arrow_utils.restore_schema_nullability(output_table, output_table_schema) + return output_table + + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(data_function={self._data_function!r}, " + f"input_stream={self._input_stream!r})" + ) + + +# --------------------------------------------------------------------------- +# FunctionNode — thin blueprint (no DB) +# --------------------------------------------------------------------------- + + +class FunctionNode(FunctionNodeBase): + """Thin blueprint descriptor for a function pod invocation. + + Carries no database references. Calling ``iter_data()`` raises + ``PipelineJobRequiredError`` — wrap the containing ``Pipeline`` in a + ``PipelineJob`` to obtain an executable ``FunctionJobNode``. + + This is the node type recorded inside a ``Pipeline`` context manager + and serialized to disk via ``Pipeline.save()``. + """ + + def __init__( + self, + function_pod: FunctionPodProtocol, + input_stream: StreamProtocol, + tracker_manager: TrackerManagerProtocol | None = None, + label: str | None = None, + config: Config | None = None, + table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", + ): + super().__init__( + function_pod=function_pod, + input_stream=input_stream, + tracker_manager=tracker_manager, + label=label, + config=config, + table_scope=table_scope, + ) + # Blueprint nodes have no in-memory output table cache + self._cached_output_table: "pa.Table | None" = None + self._cached_content_hash_column: "pa.Array | None" = None # ------------------------------------------------------------------ # from_descriptor — reconstruct from a serialized pipeline descriptor @@ -262,7 +484,7 @@ def from_descriptor( operates in full mode -- constructed normally via ``__init__``. When *function_pod* is ``None`` the node is created in read-only mode with metadata from the descriptor; computation methods will - raise ``RuntimeError``. + raise ``PipelineJobRequiredError``. Args: descriptor: The serialized node descriptor dict. @@ -278,9 +500,6 @@ def from_descriptor( """ from orcapod.pipeline.serialization import LoadStatus - pipeline_db = databases.get("pipeline") - result_db = databases.get("result") # pre-scoped; None if not provided - if "table_scope" not in descriptor: raise ValueError( f"FunctionNode descriptor is missing required 'table_scope' field: " @@ -300,8 +519,6 @@ def from_descriptor( node = cls( function_pod=function_pod, input_stream=input_stream, - pipeline_database=pipeline_db, - result_database=result_db, label=descriptor.get("label"), table_scope=table_scope, ) @@ -346,20 +563,16 @@ def from_descriptor( # From TemporalMixin node._modified_time = None - # From FunctionNode + # From FunctionNodeBase node._function_pod = None node._data_function = None node._input_stream = None node.tracker_manager = DEFAULT_TRACKER_MANAGER - node._cached_output_datas = {} + + # Blueprint-level table caches node._cached_output_table = None node._cached_content_hash_column = None - # DB persistence state - node._pipeline_database = pipeline_db - node._cached_function_pod = None - node._output_schema_hash = None - # Descriptor metadata for read-only access node._descriptor = descriptor node._stored_schema = descriptor.get("output_schema", {}) @@ -373,167 +586,213 @@ def from_descriptor( node._table_scope = table_scope node._node_identity_path_cache = None - # Determine load status based on DB availability + # FunctionNode loaded read-only is always UNAVAILABLE (no DB) node._load_status = LoadStatus.UNAVAILABLE - if pipeline_db is not None: - node._load_status = LoadStatus.READ_ONLY return node # ------------------------------------------------------------------ - # load_status + # iter_data — raises PipelineJobRequiredError # ------------------------------------------------------------------ - @property - def load_status(self) -> Any: - """Return the load status of this node. + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Raise ``PipelineJobRequiredError`` — blueprint nodes cannot produce data. - Returns: - The ``LoadStatus`` enum value indicating how this node was - loaded. Defaults to ``FULL`` for nodes created via - ``__init__``. + Raises: + PipelineJobRequiredError: Always. """ - return self._load_status - - # ------------------------------------------------------------------ - # Core properties - # ------------------------------------------------------------------ - - @property - def producer(self) -> FunctionPodProtocol: - return self._function_pod - - @property - def data_context(self) -> contexts.DataContext: - return contexts.resolve_context(self._function_pod.data_context_key) - - @property - def data_context_key(self) -> str: - return self._function_pod.data_context_key - - @property - def executor(self) -> DataFunctionExecutorProtocol | None: - """The executor set on the underlying data function.""" - return self._data_function.executor - - @executor.setter - def executor(self, executor: DataFunctionExecutorProtocol | None) -> None: - """Set or clear the executor on the underlying data function.""" - self._data_function.executor = executor - - @property - def upstreams(self) -> tuple[StreamProtocol, ...]: - return (self._input_stream,) + raise PipelineJobRequiredError( + f"FunctionNode '{self.label}' is a blueprint — it carries no database " + "references and cannot produce data directly. " + "Wrap the containing Pipeline in a PipelineJob to obtain an executable " + "FunctionJobNode." + ) + # yield is needed to satisfy the Iterator return type annotation + return # pragma: no cover + yield # pragma: no cover - @upstreams.setter - def upstreams(self, value: tuple[StreamProtocol, ...]) -> None: - if len(value) != 1: - raise ValueError("FunctionPod can only have one upstream") - self._input_stream = value[0] + def as_node(self) -> "FunctionNode": + """Return ``self`` — already the lightweight blueprint form. - # ------------------------------------------------------------------ - # Read-only overrides (for deserialized nodes without live function_pod) - # ------------------------------------------------------------------ + Returns: + This instance. + """ + return self - def content_hash(self, hasher=None) -> ContentHash: - """Return the content hash, using stored value in read-only mode.""" - if self._function_pod is None and self._stored_content_hash is not None: - from orcapod.types import ContentHash as CH - return CH.from_string(self._stored_content_hash) - return super().content_hash(hasher) +# --------------------------------------------------------------------------- +# FunctionJobNode — DB-backed execution node +# --------------------------------------------------------------------------- - def pipeline_hash(self, hasher=None) -> ContentHash: - """Return the pipeline hash, using stored value in read-only mode.""" - if self._function_pod is None and self._stored_pipeline_hash is not None: - from orcapod.types import ContentHash as CH - return CH.from_string(self._stored_pipeline_hash) - return super().pipeline_hash(hasher) +class FunctionJobNode(FunctionNodeBase): + """DB-backed execution node for function pod invocations. - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return output schema, using stored value in read-only mode.""" - if self._function_pod is None: - tag = Schema(self._stored_schema.get("tag", {})) - data = Schema(self._stored_schema.get("data", {})) - return tag, data - tag_schema = self._input_stream.output_schema( - columns=columns, all_info=all_info - )[0] - return tag_schema, self._data_function.output_data_schema + Created by ``PipelineJob`` at run time; never recorded inside a plain + ``Pipeline``. Carries all persistence logic: ``CachedFunctionPod`` + wrapping, pipeline records, and two-phase ``iter_data()`` / async + execution. + """ - def keys( + def __init__( self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - if self._function_pod is None: - tag_keys = tuple(self._stored_schema.get("tag", {}).keys()) - data_keys = tuple(self._stored_schema.get("data", {}).keys()) - return tag_keys, data_keys - tag_schema, data_schema = self.output_schema( - columns=columns, all_info=all_info + function_pod: FunctionPodProtocol, + input_stream: StreamProtocol, + tracker_manager: TrackerManagerProtocol | None = None, + label: str | None = None, + config: Config | None = None, + # Optional DB params for persistent mode: + pipeline_database: ArrowDatabaseProtocol | None = None, + result_database: ArrowDatabaseProtocol | None = None, + table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", + ): + super().__init__( + function_pod=function_pod, + input_stream=input_stream, + tracker_manager=tracker_manager, + label=label, + config=config, + table_scope=table_scope, ) - return tuple(tag_schema.keys()), tuple(data_schema.keys()) + + # stream-level caching state + self._cached_output_datas: dict[ + str, tuple[TagProtocol, DataProtocol | None] + ] = {} + self._cached_output_table: "pa.Table | None" = None + self._cached_content_hash_column: "pa.Array | None" = None + + # DB persistence state (initially None; set via __init__ params or attach_databases) + self._pipeline_database: ArrowDatabaseProtocol | None = None + self._cached_function_pod: CachedFunctionPod | None = None + self._output_schema_hash: str | None = None + + if pipeline_database is not None: + self.attach_databases( + pipeline_database=pipeline_database, + result_database=result_database, + ) # ------------------------------------------------------------------ - # Pipeline path + # attach_databases # ------------------------------------------------------------------ - @property - def node_identity_path(self) -> tuple[str, ...]: - """Return the node identity path for observer contextualization. + def attach_databases( + self, + pipeline_database: ArrowDatabaseProtocol, + result_database: ArrowDatabaseProtocol | None = None, + ) -> None: + """Attach databases for persistent caching and pipeline records. - When ``table_scope="pipeline_hash"`` (default) the path is - ``pod.uri + (schema:{pipeline_hash},)`` — all runs that share the same - pipeline structure are routed to one shared table, with per-run - disambiguation via the ``_node_content_hash`` row-level column. + Creates a ``CachedFunctionPod`` wrapping the original function pod + for result caching. The pipeline database is used separately for + pipeline-level provenance records (tag + data hash). - When ``table_scope="content_hash"`` the legacy path is returned: - ``pod.uri + (schema:{pipeline_hash}, instance:{content_hash})``. + The databases are expected to be pre-scoped by the pipeline (via + ``db.at(*pipeline_name).at("_result")`` etc.) so no additional path + prefix is needed here. - In read-only/UNAVAILABLE mode (no pod) the path stored from the - deserialized descriptor is returned (empty tuple when absent). + Args: + pipeline_database: Database for pipeline records. + result_database: Database for cached results. Defaults to + pipeline_database. """ - if self._data_function is None: - return self._stored_pipeline_path - if self._node_identity_path_cache is not None: - return self._node_identity_path_cache - pf = self._function_pod - path = pf.uri + (f"schema:{self.pipeline_hash().to_string()}",) - if self._table_scope != "pipeline_hash": - path += (f"instance:{self.content_hash().to_string()}",) - self._node_identity_path_cache = path - return path + if result_database is None: + # Default result database is pipeline_database scoped to "_result" + # so that results are stored separately from pipeline-level records. + result_database = pipeline_database.at("_result") - @property - def node_uri(self) -> tuple[str, ...]: - """Canonical URI tuple identifying this computation. + # Always wrap the original function_pod (not a previous cached wrapper) + self._cached_function_pod = CachedFunctionPod( + self._function_pod, + result_database=result_database, + ) - Identical to ``data_function.uri`` at runtime. - Returns stored value in read-only (deserialized) mode. - """ - if self._data_function is None: - return self._stored_node_uri - return self._data_function.uri + self._pipeline_database = pipeline_database + + # Clear all caches + self._node_identity_path_cache = None + self.clear_cache() + self._content_hash_cache.clear() + self._pipeline_hash_cache.clear() + + # Compute output schema hash + self._output_schema_hash = self.data_context.semantic_hasher.hash_object( + self._data_function.output_data_schema + ).to_string() # ------------------------------------------------------------------ - # Caching + # Override clear_cache to also clear DB caches # ------------------------------------------------------------------ def clear_cache(self) -> None: + """Clear in-memory output caches and the node identity path cache.""" self._cached_output_datas.clear() self._cached_output_table = None self._cached_content_hash_column = None self._node_identity_path_cache = None self._update_modified_time() + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _require_pipeline_database(self) -> None: + """Raise a clear RuntimeError if no pipeline database is attached. + + Called at the top of methods that unconditionally access + ``self._pipeline_database``. Provides an actionable error message + instead of an opaque ``AttributeError: 'NoneType' object has no + attribute ...`` when a definition-level pipeline is executed without + supplying a database. + """ + if self._pipeline_database is None: + raise RuntimeError( + f"FunctionJobNode '{self.label}' has no pipeline database attached. " + "Either construct the pipeline with a pipeline_database argument, " + "or supply one via Pipeline.load(..., pipeline_database=)." + ) + + def _filter_by_content_hash(self, table: "pa.Table") -> "pa.Table": + """Filter *table* to rows whose ``NODE_CONTENT_HASH_COL`` matches this node. + + Only applied when ``table_scope="pipeline_hash"`` because in that mode + multiple runs share the same DB table and must be disambiguated at read + time. In ``"content_hash"`` mode every run has its own table so no + filtering is needed. + """ + if self._table_scope != "pipeline_hash": + return table + col_name = constants.NODE_CONTENT_HASH_COL + if col_name not in table.column_names: + raise ValueError( + f"Cannot isolate records for table_scope='pipeline_hash': " + f"required column {col_name!r} is missing from the stored table. " + "This may indicate records written by an older version of the code." + ) + own_hash = self.content_hash().to_string() + mask = pc.equal(table.column(col_name), own_hash) + return table.filter(mask) + + # ------------------------------------------------------------------ + # as_node — return the lightweight FunctionNode equivalent + # ------------------------------------------------------------------ + + def as_node(self) -> FunctionNode: + """Return the lightweight ``FunctionNode`` equivalent of this job node. + + Returns: + A new ``FunctionNode`` with the same function pod, input stream, + label, and table scope. Its ``content_hash()`` / ``pipeline_hash()`` + are identical to those of this ``FunctionJobNode``. + """ + return FunctionNode( + function_pod=self._function_pod, + input_stream=self._input_stream, + label=self._label, + table_scope=self._table_scope, + ) + # ------------------------------------------------------------------ # Data processing # ------------------------------------------------------------------ @@ -891,7 +1150,7 @@ def get_all_records( self, columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, - ) -> pa.Table | None: + ) -> "pa.Table | None": """Return all computed results joined with their pipeline tag records. Args: @@ -1117,7 +1376,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: status = self.load_status if status == LoadStatus.UNAVAILABLE: raise RuntimeError( - f"FunctionNode {self.label!r} is unavailable: " + f"FunctionJobNode {self.label!r} is unavailable: " "no function pod and no database attached." ) @@ -1165,7 +1424,7 @@ def run(self) -> None: if self._load_status == LoadStatus.UNAVAILABLE: raise RuntimeError( - f"FunctionNode {self.label!r} is unavailable: " + f"FunctionJobNode {self.label!r} is unavailable: " "no function pod and no database attached." ) if self._load_status in (LoadStatus.CACHE_ONLY, LoadStatus.READ_ONLY): @@ -1179,135 +1438,18 @@ def run(self) -> None: self.clear_cache() self.execute(self._input_stream) - # ------------------------------------------------------------------ - # as_table - # ------------------------------------------------------------------ - - def as_table( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> pa.Table: - if self._cached_output_table is None: - all_tags = [] - all_data = [] - tag_schema, data_schema = None, None - for tag, data in self.iter_data(): - if tag_schema is None: - tag_schema = tag.arrow_schema(all_info=True) - if data_schema is None: - data_schema = data.arrow_schema(all_info=True) - all_tags.append(tag.as_dict(all_info=True)) - all_data.append(data.as_dict(all_info=True)) - - if not all_tags: - self._cached_output_table = pa.table({}) - - converter = self.data_context.type_converter - - # Derive the Python schema from the Arrow schema when available, - # rather than re-inferring from dict values. This preserves precise - # types for empty containers (e.g. {} infers as dict[Any, Any] but - # the Arrow schema knows it's dict[str, str]). - data_python_schema = ( - converter.arrow_schema_to_python_schema(data_schema) - if data_schema is not None - else None - ) - struct_data = converter.python_dicts_to_struct_dicts( - all_data, python_schema=data_python_schema - ) - all_tags_as_tables: pa.Table = pa.Table.from_pylist( - all_tags, schema=tag_schema - ) - if constants.CONTEXT_KEY in all_tags_as_tables.column_names: - all_tags_as_tables = all_tags_as_tables.drop([constants.CONTEXT_KEY]) - all_data_as_tables: pa.Table = pa.Table.from_pylist( - struct_data, schema=data_schema - ) - - self._cached_output_table = arrow_utils.hstack_tables( - all_tags_as_tables, all_data_as_tables - ) - if self._cached_output_table is None: - self._cached_output_table = pa.table({}) - - column_config = ColumnConfig.handle_config(columns, all_info=all_info) - - drop_columns = [] - if not column_config.system_tags: - drop_columns.extend( - [ - c - for c in self._cached_output_table.column_names - if c.startswith(constants.SYSTEM_TAG_PREFIX) - ] - ) - if not column_config.source: - drop_columns.extend(f"{constants.SOURCE_PREFIX}{c}" for c in self.keys()[1]) - if not column_config.context: - drop_columns.append(constants.CONTEXT_KEY) - if not column_config.meta: - drop_columns.extend( - c - for c in self._cached_output_table.column_names - if c.startswith(constants.META_PREFIX) - ) - elif not isinstance(column_config.meta, bool): - # Collection[str]: keep only meta columns matching the specified prefixes - drop_columns.extend( - c - for c in self._cached_output_table.column_names - if c.startswith(constants.META_PREFIX) - and not any(c.startswith(p) for p in column_config.meta) - ) - output_table = self._cached_output_table.drop( - [c for c in drop_columns if c in self._cached_output_table.column_names] - ) - - if column_config.content_hash: - if self._cached_content_hash_column is None: - content_hashes = [] - for tag, data in self.iter_data(): - content_hashes.append(data.content_hash().to_string()) - self._cached_content_hash_column = pa.array( - content_hashes, type=pa.large_string() - ) - assert self._cached_content_hash_column is not None, ( - "_cached_content_hash_column should not be None here." - ) - hash_column_name = ( - "_content_hash" - if column_config.content_hash is True - else column_config.content_hash - ) - output_table = output_table.append_column( - hash_column_name, self._cached_content_hash_column - ) - - if column_config.sort_by_tags: - output_table_schema = output_table.schema - output_table = ( - pl.DataFrame(output_table) - .sort(by=self.keys()[0], descending=False) - .to_arrow() - ) - output_table = arrow_utils.restore_schema_nullability(output_table, output_table_schema) - return output_table - # ------------------------------------------------------------------ # Async channel execution # ------------------------------------------------------------------ async def async_execute( self, - input_channel: ReadableChannel[tuple[TagProtocol, DataProtocol]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], + input_channel: "ReadableChannel[tuple[TagProtocol, DataProtocol]]", + output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", *, observer: ExecutionObserverProtocol | None = None, ) -> None: - """Streaming async execution for FunctionNode. + """Streaming async execution for FunctionJobNode. When a database is attached, uses two-phase execution: replay cached results first, then compute missing data concurrently. Otherwise, @@ -1332,7 +1474,7 @@ async def async_execute( if status == LoadStatus.UNAVAILABLE: await output.close() raise RuntimeError( - f"FunctionNode {self.label!r} is unavailable: " + f"FunctionJobNode {self.label!r} is unavailable: " "no function pod and no database attached." ) @@ -1432,7 +1574,7 @@ async def _async_execute_one_data( self, tag: TagProtocol, data: DataProtocol, - output: WritableChannel[tuple[TagProtocol, DataProtocol]], + output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", *, observer: ExecutionObserverProtocol, node_label: str, @@ -1458,9 +1600,3 @@ async def _async_execute_one_data( ) if result_data is not None: await output.send((tag_out, result_data)) - - def __repr__(self) -> str: - return ( - f"{type(self).__name__}(data_function={self._data_function!r}, " - f"input_stream={self._input_stream!r})" - ) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 3908db80d..27173b82e 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -460,6 +460,7 @@ def _build_execution_graph(self) -> "tuple[Any, list[str], Pipeline]": # Any = """ import networkx as nx from orcapod.core.nodes import FunctionNode, OperatorNode + from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.executors.local import LocalPythonFunctionExecutor @@ -543,7 +544,7 @@ def _build_execution_graph(self) -> "tuple[Any, list[str], Pipeline]": # Any = excluded_hashes.add(node_hash) continue input_node = exec_node_map[preds[0]] - new_fn = FunctionNode( + new_fn = FunctionJobNode( function_pod=template._function_pod, input_stream=input_node, label=template._label, diff --git a/tests/test_channels/test_copilot_review_issues.py b/tests/test_channels/test_copilot_review_issues.py index 81cc5aacb..73e9d8857 100644 --- a/tests/test_channels/test_copilot_review_issues.py +++ b/tests/test_channels/test_copilot_review_issues.py @@ -24,6 +24,7 @@ from orcapod.core.datagrams import Data from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.streams import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase @@ -85,7 +86,7 @@ async def tracked_double(x: int) -> int: pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=5)) db = InMemoryArrowDatabase() stream = make_stream(5) - node = FunctionNode(pod, stream, pipeline_database=db) + node = FunctionJobNode(pod, stream, pipeline_database=db) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -276,7 +277,7 @@ async def double(x: int) -> int: pf = PythonDataFunction(double, output_keys="result") pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=0)) stream = make_stream(1) - node = FunctionNode(pod, stream) + node = FunctionJobNode(pod, stream) input_ch = Channel(buffer_size=4) output_ch = Channel(buffer_size=4) diff --git a/tests/test_channels/test_node_async_execute.py b/tests/test_channels/test_node_async_execute.py index c8483e357..61c954691 100644 --- a/tests/test_channels/test_node_async_execute.py +++ b/tests/test_channels/test_node_async_execute.py @@ -24,6 +24,7 @@ FunctionNode, OperatorNode, ) +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.operators import SelectDataColumns from orcapod.core.operators.join import Join from orcapod.core.operators.semijoin import SemiJoin @@ -178,19 +179,22 @@ def double(x: int) -> int: # --------------------------------------------------------------------------- -class TestFunctionNodeAsyncExecute: +class TestFunctionJobNodeAsyncExecuteSimple: + """Test FunctionJobNode.async_execute without a DB (simple non-cached path).""" + @pytest.mark.asyncio async def test_basic_streaming_matches_sync(self): _, pod = make_double_pod() stream = make_stream(5) - # Sync results - node_sync = FunctionNode(pod, stream) + # Sync results via sync run + node_sync = FunctionJobNode(pod, stream) + node_sync.execute(stream) sync_results = list(node_sync.iter_data()) sync_values = sorted(pkt.as_dict()["result"] for _, pkt in sync_results) # Async results - node_async = FunctionNode(pod, make_stream(5)) + node_async = FunctionJobNode(pod, make_stream(5)) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -204,7 +208,7 @@ async def test_basic_streaming_matches_sync(self): @pytest.mark.asyncio async def test_empty_input_closes_cleanly(self): _, pod = make_double_pod() - node = FunctionNode(pod, make_stream(1)) + node = FunctionJobNode(pod, make_stream(1)) input_ch = Channel(buffer_size=4) output_ch = Channel(buffer_size=4) @@ -219,7 +223,7 @@ async def test_empty_input_closes_cleanly(self): async def test_tags_preserved(self): """Tags should pass through unchanged.""" _, pod = make_double_pod() - node = FunctionNode(pod, make_stream(3)) + node = FunctionJobNode(pod, make_stream(3)) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -233,7 +237,7 @@ async def test_tags_preserved(self): # --------------------------------------------------------------------------- -# 4. FunctionNode.async_execute +# 4. FunctionJobNode.async_execute (DB path) # --------------------------------------------------------------------------- @@ -244,7 +248,7 @@ async def test_no_cache_processes_all_inputs(self): pf, pod = make_double_pod() db = InMemoryArrowDatabase() stream = make_stream(3) - node = FunctionNode(pod, stream, pipeline_database=db) + node = FunctionJobNode(pod, stream, pipeline_database=db) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -265,12 +269,12 @@ async def test_sync_run_then_async_emits_from_cache(self): stream = make_stream(3) # Sync run to populate DB - node1 = FunctionNode(pod, stream, pipeline_database=db) + node1 = FunctionJobNode(pod, stream, pipeline_database=db) node1.run() # New node with same DB — send same data, expect cached hits input_stream = make_stream(3) - node2 = FunctionNode(pod, input_stream, pipeline_database=db) + node2 = FunctionJobNode(pod, input_stream, pipeline_database=db) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -295,11 +299,11 @@ async def test_two_phase_cached_and_new(self): # Sync run with 3 items to populate DB stream = make_stream(3) - node1 = FunctionNode(pod, stream, pipeline_database=db) + node1 = FunctionJobNode(pod, stream, pipeline_database=db) node1.run() # Now run async with 5 items (3 cached + 2 new) - node2 = FunctionNode(pod, make_stream(5), pipeline_database=db) + node2 = FunctionJobNode(pod, make_stream(5), pipeline_database=db) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -332,7 +336,7 @@ async def tracked_double(x: int) -> int: pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=5)) db = InMemoryArrowDatabase() stream = make_stream(5) - node = FunctionNode(pod, stream, pipeline_database=db) + node = FunctionJobNode(pod, stream, pipeline_database=db) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -369,7 +373,7 @@ async def tracked_double(x: int) -> int: pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=1)) db = InMemoryArrowDatabase() stream = make_stream(5) - node = FunctionNode(pod, stream, pipeline_database=db) + node = FunctionJobNode(pod, stream, pipeline_database=db) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -405,7 +409,7 @@ async def tracked_double(x: int) -> int: pod = FunctionPod(pf, node_config=NodeConfig(max_concurrency=5)) stream = make_stream(5) # No pipeline_database — exercises the simple (non-DB) path - node = FunctionNode(pod, stream) + node = FunctionJobNode(pod, stream) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -427,7 +431,7 @@ async def test_db_records_created(self): pf, pod = make_double_pod() db = InMemoryArrowDatabase() stream = make_stream(3) - node = FunctionNode(pod, stream, pipeline_database=db) + node = FunctionJobNode(pod, stream, pipeline_database=db) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -668,12 +672,12 @@ async def test_replay_empty_db_returns_empty(self): class TestExecuteDataRouting: def test_function_node_sequential_uses_execute_data(self): - """Verify FunctionNode routes through execute_data (not raw pf.call).""" + """Verify FunctionJobNode routes through execute_data (not raw pf.call).""" call_log = [] _, pod = make_double_pod() stream = make_stream(3) - node = FunctionNode(pod, stream) + node = FunctionJobNode(pod, stream) # Monkey-patch to verify routing through internal path original = node._process_data_internal @@ -691,12 +695,12 @@ def patched(tag, data, *, logger=None): @pytest.mark.asyncio async def test_function_node_async_uses_async_process_data_internal(self): - """Verify FunctionNode.async_execute routes through _async_process_data_internal.""" + """Verify FunctionJobNode.async_execute routes through _async_process_data_internal.""" call_log = [] _, pod = make_double_pod() stream = make_stream(3) - node = FunctionNode(pod, stream) + node = FunctionJobNode(pod, stream) original = node._async_process_data_internal @@ -724,7 +728,7 @@ async def patched(tag, data, **kwargs): class TestEndToEnd: @pytest.mark.asyncio async def test_source_to_function_node_pipeline(self): - """Source → FunctionNode async pipeline.""" + """Source → FunctionJobNode async pipeline.""" def triple(x: int) -> int: return x * 3 @@ -732,7 +736,7 @@ def triple(x: int) -> int: pf = PythonDataFunction(triple, output_keys="result") pod = FunctionPod(pf) stream = make_stream(4) - node = FunctionNode(pod, stream) + node = FunctionJobNode(pod, stream) ch1 = Channel(buffer_size=16) ch2 = Channel(buffer_size=16) @@ -805,7 +809,7 @@ def double(x: int) -> int: db = InMemoryArrowDatabase() stream = make_stream(5) # ids 0..4, x values 0..4 - node = FunctionNode(pod, stream, pipeline_database=db) + node = FunctionJobNode(pod, stream, pipeline_database=db) # --- Async pipeline execution --- input_ch = Channel(buffer_size=16) @@ -834,7 +838,7 @@ async def source_producer(): assert sorted(result_col) == [0, 2, 4, 6, 8] # A *new* node sharing the same DB can also read these records - node2 = FunctionNode(pod, make_stream(5), pipeline_database=db) + node2 = FunctionJobNode(pod, make_stream(5), pipeline_database=db) records2 = node2.get_all_records() assert records2 is not None assert records2.num_rows == 5 @@ -907,7 +911,7 @@ def double(x: int) -> int: fn_db = InMemoryArrowDatabase() stream = make_stream(3) # ids 0..2, x 0..2 - fn_node = FunctionNode(pod, stream, pipeline_database=fn_db) + fn_node = FunctionJobNode(pod, stream, pipeline_database=fn_db) # --- Setup stage 2: select only "result" column --- # Build a placeholder stream for schema purposes (OperatorNode needs diff --git a/tests/test_core/data_function/test_executor.py b/tests/test_core/data_function/test_executor.py index 430cef8dc..86c44b715 100644 --- a/tests/test_core/data_function/test_executor.py +++ b/tests/test_core/data_function/test_executor.py @@ -479,14 +479,14 @@ def test_node_executor_set_targets_data_function(self): def test_node_iter_uses_executor(self): from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes import FunctionNode + from orcapod.core.nodes.function_node import FunctionJobNode spy = SpyExecutor() pf = PythonDataFunction(add, output_keys="result") pf.executor = spy pod = FunctionPod(pf) - node = FunctionNode(pod, _make_add_stream()) + node = FunctionJobNode(pod, _make_add_stream()) node.run() results = list(node.iter_data()) @@ -634,18 +634,18 @@ def test_function_pod_stream_uses_async_path(self): assert len(spy.sync_calls) == 0 def test_function_node_uses_sync_path_via_run(self): - """FunctionNode.run() delegates to execute(), which is always sequential + """FunctionJobNode.run() delegates to execute(), which is always sequential (synchronous). The async path is only used through async_execute() in the async pipeline orchestrator. Even with a ConcurrentSpyExecutor attached, run() → execute() → sync executor path.""" from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes import FunctionNode + from orcapod.core.nodes.function_node import FunctionJobNode spy = ConcurrentSpyExecutor() pf = PythonDataFunction(add, output_keys="result", executor=spy) pod = FunctionPod(pf) - node = FunctionNode(pod, _make_add_stream()) + node = FunctionJobNode(pod, _make_add_stream()) node.run() results = list(node.iter_data()) @@ -659,13 +659,13 @@ def test_function_node_uses_sync_path_via_run(self): def test_non_concurrent_executor_uses_sync_path(self): """SpyExecutor has supports_concurrent_execution=False (default).""" from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes import FunctionNode + from orcapod.core.nodes.function_node import FunctionJobNode spy = SpyExecutor() pf = PythonDataFunction(add, output_keys="result", executor=spy) pod = FunctionPod(pf) - node = FunctionNode(pod, _make_add_stream()) + node = FunctionJobNode(pod, _make_add_stream()) node.run() results = list(node.iter_data()) @@ -675,12 +675,12 @@ def test_non_concurrent_executor_uses_sync_path(self): def test_no_executor_uses_sync_path(self): from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes import FunctionNode + from orcapod.core.nodes.function_node import FunctionJobNode pf = PythonDataFunction(add, output_keys="result") pod = FunctionPod(pf) - node = FunctionNode(pod, _make_add_stream()) + node = FunctionJobNode(pod, _make_add_stream()) node.run() results = list(node.iter_data()) diff --git a/tests/test_core/function_pod/test_function_node_attach_db.py b/tests/test_core/function_pod/test_function_node_attach_db.py index 9118c96b5..2bcc7c7e1 100644 --- a/tests/test_core/function_pod/test_function_node_attach_db.py +++ b/tests/test_core/function_pod/test_function_node_attach_db.py @@ -7,6 +7,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase @@ -40,29 +41,29 @@ def _make_stream(n=3): class TestFunctionNodeWithoutDatabase: def test_construction_without_database(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) assert node._pipeline_database is None def test_iter_data_without_database(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream(n=3)) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream(n=3)) node.run() results = list(node.iter_data()) assert len(results) == 3 assert results[0][1]["result"] == 0 def test_get_all_records_without_database_returns_none(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) assert node.get_all_records() is None def test_as_source_without_database_raises(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) with pytest.raises(RuntimeError): node.as_source() class TestFunctionNodeAttachDatabases: def test_attach_databases_sets_pipeline_db(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) db = InMemoryArrowDatabase() node.attach_databases(pipeline_database=db, result_database=db) assert node._pipeline_database is db @@ -70,13 +71,13 @@ def test_attach_databases_sets_pipeline_db(self): def test_attach_databases_creates_cached_function_pod(self): from orcapod.core.cached_function_pod import CachedFunctionPod - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) db = InMemoryArrowDatabase() node.attach_databases(pipeline_database=db, result_database=db) assert isinstance(node._cached_function_pod, CachedFunctionPod) def test_attach_databases_clears_caches(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) node.run() # populate cache assert len(node._cached_output_datas) > 0 db = InMemoryArrowDatabase() @@ -84,7 +85,7 @@ def test_attach_databases_clears_caches(self): assert len(node._cached_output_datas) == 0 def test_attach_databases_computes_node_identity_path(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) db = InMemoryArrowDatabase() node.attach_databases(pipeline_database=db, result_database=db) assert node.node_identity_path is not None @@ -93,7 +94,7 @@ def test_attach_databases_computes_node_identity_path(self): def test_double_attach_does_not_double_wrap(self): from orcapod.core.cached_function_pod import CachedFunctionPod - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream()) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream()) db = InMemoryArrowDatabase() node.attach_databases(pipeline_database=db, result_database=db) assert isinstance(node._cached_function_pod, CachedFunctionPod) @@ -105,7 +106,7 @@ def test_double_attach_does_not_double_wrap(self): ) def test_iter_data_after_attach_works(self): - node = FunctionNode(function_pod=_make_pod(), input_stream=_make_stream(n=2)) + node = FunctionJobNode(function_pod=_make_pod(), input_stream=_make_stream(n=2)) db = InMemoryArrowDatabase() node.attach_databases(pipeline_database=db, result_database=db) node.run() @@ -116,7 +117,7 @@ def test_iter_data_after_attach_works(self): class TestFunctionNodeWithDatabase: def test_construction_with_database(self): db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=_make_pod(), input_stream=_make_stream(), pipeline_database=db, @@ -126,7 +127,7 @@ def test_construction_with_database(self): def test_node_identity_path_with_database(self): db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=_make_pod(), input_stream=_make_stream(), pipeline_database=db, @@ -136,7 +137,7 @@ def test_node_identity_path_with_database(self): def test_iter_data_with_database(self): db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=_make_pod(), input_stream=_make_stream(n=3), pipeline_database=db, diff --git a/tests/test_core/function_pod/test_function_node_caching.py b/tests/test_core/function_pod/test_function_node_caching.py index 8c5bb0d54..0dd6f1ccf 100644 --- a/tests/test_core/function_pod/test_function_node_caching.py +++ b/tests/test_core/function_pod/test_function_node_caching.py @@ -18,6 +18,7 @@ from orcapod.core.datagrams import Data, Tag from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource from orcapod.core.streams.arrow_table_stream import ArrowTableStream @@ -70,7 +71,7 @@ def _make_node(stream, db=None): pod = _make_pod() if db is None: db = InMemoryArrowDatabase() - return FunctionNode( + return FunctionJobNode( function_pod=pod, input_stream=stream, pipeline_database=db, diff --git a/tests/test_core/function_pod/test_function_pod_node.py b/tests/test_core/function_pod/test_function_pod_node.py index ea2906646..f9007bc71 100644 --- a/tests/test_core/function_pod/test_function_pod_node.py +++ b/tests/test_core/function_pod/test_function_pod_node.py @@ -20,6 +20,7 @@ from orcapod.core.datagrams import Data, Tag from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.streams import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase @@ -41,7 +42,7 @@ def _make_node( ) -> FunctionNode: if db is None: db = InMemoryArrowDatabase() - return FunctionNode( + return FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=n), pipeline_database=db, @@ -72,7 +73,7 @@ def _make_node_with_system_tags( schema=schema, ) stream = ArrowTableStream(table, tag_columns=["id"], system_tag_columns=["run"]) - return FunctionNode( + return FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=stream, pipeline_database=db, @@ -94,7 +95,7 @@ class TestFunctionNodeConstruction: def node(self, double_pf) -> FunctionNode: db = InMemoryArrowDatabase() stream = make_int_stream(n=3) - return FunctionNode( + return FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=stream, pipeline_database=db, @@ -158,7 +159,7 @@ def test_incompatible_stream_raises_on_construction(self, double_pf): tag_columns=["id"], ) with pytest.raises(ValueError): - FunctionNode( + FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=bad_stream, pipeline_database=db, @@ -166,7 +167,7 @@ def test_incompatible_stream_raises_on_construction(self, double_pf): def test_result_database_defaults_to_pipeline_database(self, double_pf): db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=2), pipeline_database=db, @@ -176,7 +177,7 @@ def test_result_database_defaults_to_pipeline_database(self, double_pf): def test_separate_result_database_accepted(self, double_pf): pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=2), pipeline_database=pipeline_db, @@ -194,7 +195,7 @@ class TestFunctionNodeOutputSchema: @pytest.fixture def node(self, double_pf) -> FunctionNode: db = InMemoryArrowDatabase() - return FunctionNode( + return FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, @@ -230,7 +231,7 @@ class TestFunctionNodeExecuteData: @pytest.fixture def node(self, double_pf) -> FunctionNode: db = InMemoryArrowDatabase() - return FunctionNode( + return FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, @@ -302,7 +303,7 @@ class TestFunctionNodeStreamInterface: @pytest.fixture def node(self, double_pf) -> FunctionNode: db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, @@ -334,12 +335,12 @@ def test_run_fills_database(self, node): class TestFunctionNodePipelineIdentity: def test_pipeline_hash_same_schema_same_hash(self, double_pf): db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=5), # different data, same schema pipeline_database=db, @@ -362,12 +363,12 @@ def test_pipeline_hash_different_data_same_hash(self, double_pf): ), tag_columns=["id"], ) - node_a = FunctionNode( + node_a = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=stream_a, pipeline_database=db, ) - node_b = FunctionNode( + node_b = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=stream_b, pipeline_database=db, @@ -386,12 +387,12 @@ def test_pipeline_node_hash_in_uri_is_schema_based(self, double_pf): Two nodes with same schema share the same full path; per-run isolation is achieved via the _node_content_hash row column.""" db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=99), # different data pipeline_database=db, @@ -653,7 +654,7 @@ def test_all_info_data_columns_match_default(self, filled_node): class TestFunctionNodeIdentityPath: def test_node_identity_path_starts_with_pf_uri(self, double_pf): db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=2), pipeline_database=db, @@ -676,7 +677,7 @@ def test_result_records_stored_under_pod_uri_path(self, double_pf): since the database is pre-scoped at compile time (ENG-340/ENG-349).""" db = InMemoryArrowDatabase() pod = FunctionPod(data_function=double_pf) - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=make_int_stream(n=2), pipeline_database=db, diff --git a/tests/test_core/function_pod/test_function_pod_node_stream.py b/tests/test_core/function_pod/test_function_pod_node_stream.py index 6442d3ccc..207a63bb8 100644 --- a/tests/test_core/function_pod/test_function_pod_node_stream.py +++ b/tests/test_core/function_pod/test_function_pod_node_stream.py @@ -21,6 +21,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.streams import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase @@ -40,7 +41,7 @@ def _make_node( ) -> FunctionNode: if db is None: db = InMemoryArrowDatabase() - return FunctionNode( + return FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=n), pipeline_database=db, @@ -61,7 +62,7 @@ class TestFunctionNodeStreamBasic: @pytest.fixture def node(self, double_pf) -> FunctionNode: db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, @@ -136,7 +137,7 @@ def test_as_table_sort_by_tags(self, double_pf): ), ) input_stream = ArrowTableStream(reversed_table, tag_columns=["id"]) - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=input_stream, pipeline_database=db, @@ -379,7 +380,7 @@ def test_is_stale_true_after_upstream_modified(self, double_pf): db = InMemoryArrowDatabase() input_stream = make_int_stream(n=3) - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=input_stream, pipeline_database=db, @@ -396,7 +397,7 @@ def test_is_stale_false_after_clear_cache(self, double_pf): db = InMemoryArrowDatabase() input_stream = make_int_stream(n=3) - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=input_stream, pipeline_database=db, @@ -440,7 +441,7 @@ def test_iter_data_auto_detects_stale_and_repopulates(self, double_pf): db = InMemoryArrowDatabase() input_stream = make_int_stream(n=3) - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=input_stream, pipeline_database=db, @@ -460,7 +461,7 @@ def test_as_table_auto_detects_stale_and_repopulates(self, double_pf): db = InMemoryArrowDatabase() input_stream = make_int_stream(n=3) - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=input_stream, pipeline_database=db, diff --git a/tests/test_core/function_pod/test_pipeline_hash_integration.py b/tests/test_core/function_pod/test_pipeline_hash_integration.py index 45f78d022..1ba003f16 100644 --- a/tests/test_core/function_pod/test_pipeline_hash_integration.py +++ b/tests/test_core/function_pod/test_pipeline_hash_integration.py @@ -38,6 +38,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, DictSource from orcapod.core.streams import ArrowTableStream @@ -56,7 +57,7 @@ class TestPipelineElementBase: """Verify PipelineElementBase invariants on concrete instances.""" def test_function_node_pipeline_hash_returns_content_hash(self, double_pf): - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=InMemoryArrowDatabase(), @@ -65,7 +66,7 @@ def test_function_node_pipeline_hash_returns_content_hash(self, double_pf): assert isinstance(h, ContentHash) def test_pipeline_hash_is_cached(self, double_pf): - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=InMemoryArrowDatabase(), @@ -75,7 +76,7 @@ def test_pipeline_hash_is_cached(self, double_pf): def test_pipeline_hash_not_equal_to_content_hash(self, double_pf): """pipeline_hash (schema+topology) must differ from content_hash (data-inclusive) when the input stream contains real data.""" - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=InMemoryArrowDatabase(), @@ -83,7 +84,7 @@ def test_pipeline_hash_not_equal_to_content_hash(self, double_pf): assert node.pipeline_hash() != node.content_hash() def test_source_satisfies_pipeline_element_protocol(self, double_pf): - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=InMemoryArrowDatabase(), @@ -137,12 +138,12 @@ def test_function_pod_pipeline_hash_determines_function_node_pipeline_hash( have different pipeline_hashes because the FunctionPod hashes differ.""" db = InMemoryArrowDatabase() stream = make_two_col_stream(n=3) - node_double = FunctionNode( + node_double = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) - node_add = FunctionNode( + node_add = FunctionJobNode( function_pod=FunctionPod(data_function=add_pf), input_stream=stream, pipeline_database=db, @@ -289,12 +290,12 @@ class TestFunctionNodePipelineHashFix: def test_different_data_same_schema_share_pipeline_path(self, double_pf): db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=5), pipeline_database=db, @@ -308,12 +309,12 @@ def test_different_data_same_schema_share_pipeline_path(self, double_pf): def test_different_data_same_schema_share_uri(self, double_pf): """With pipeline_hash scope, two nodes with same schema share the full path.""" db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=ArrowTableStream( pa.table( @@ -338,12 +339,12 @@ def test_different_data_same_schema_share_uri(self, double_pf): def test_different_data_yields_different_content_hash(self, double_pf): """Same schema, different actual data → content_hash must differ.""" db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=ArrowTableStream( pa.table( @@ -367,12 +368,12 @@ def test_different_data_yields_different_content_hash(self, double_pf): def test_different_function_different_pipeline_path(self, double_pf, add_pf): """Different functions → different pipeline_hash → different pipeline_path.""" db = InMemoryArrowDatabase() - node_double = FunctionNode( + node_double = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) - node_add = FunctionNode( + node_add = FunctionJobNode( function_pod=FunctionPod(data_function=add_pf), input_stream=make_two_col_stream(n=3), pipeline_database=db, @@ -380,7 +381,7 @@ def test_different_function_different_pipeline_path(self, double_pf, add_pf): assert node_double.node_identity_path != node_add.node_identity_path def test_node_identity_path_starts_with_pf_uri(self, double_pf): - node = FunctionNode( + node = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=2), pipeline_database=InMemoryArrowDatabase(), @@ -427,12 +428,12 @@ def counting_double(x: int) -> int: pf = PythonDataFunction(counting_double, output_keys="result") db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=3), # x in {0,1,2} pipeline_database=db, ) - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=5), # x in {0,1,2,3,4} pipeline_database=db, @@ -465,12 +466,12 @@ def counting_double(x: int) -> int: pf = PythonDataFunction(counting_double, output_keys="result") db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=5), pipeline_database=db, ) - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=3), # strict subset of node1's data pipeline_database=db, @@ -487,14 +488,14 @@ def test_shared_db_results_are_correct_values(self, double_pf): """Correctness: DB-served results from a shared pipeline have correct values.""" db = InMemoryArrowDatabase() - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=3), pipeline_database=db, ) node1.run() - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=make_int_stream(n=5), pipeline_database=db, @@ -517,13 +518,13 @@ def counting_double(x: int) -> int: pf = PythonDataFunction(counting_double, output_keys="result") n = 3 - FunctionNode( + FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=n), pipeline_database=InMemoryArrowDatabase(), ).run() - FunctionNode( + FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=n), pipeline_database=InMemoryArrowDatabase(), @@ -548,12 +549,12 @@ def test_pipeline_hash_chain_root_to_function_node(self, double_pf): # Level 0 (root): same schema → same pipeline_hash assert stream_a.pipeline_hash() == stream_b.pipeline_hash() - node_a = FunctionNode( + node_a = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=stream_a, pipeline_database=db, ) - node_b = FunctionNode( + node_b = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=stream_b, pipeline_database=db, @@ -575,7 +576,7 @@ def test_chained_nodes_share_pipeline_path(self, double_pf): # Pipeline A: stream(n=3) → node1_a → source_a → node2_a stream_a = make_int_stream(n=3) - node1_a = FunctionNode( + node1_a = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=stream_a, pipeline_database=db, @@ -585,7 +586,7 @@ def test_chained_nodes_share_pipeline_path(self, double_pf): # Pipeline B: stream(n=5) → node1_b → source_b → node2_b stream_b = make_int_stream(n=5) - node1_b = FunctionNode( + node1_b = FunctionJobNode( function_pod=FunctionPod(data_function=double_pf), input_stream=stream_b, pipeline_database=db, diff --git a/tests/test_core/nodes/test_function_node_get_cached.py b/tests/test_core/nodes/test_function_node_get_cached.py index a11aac4ce..150655922 100644 --- a/tests/test_core/nodes/test_function_node_get_cached.py +++ b/tests/test_core/nodes/test_function_node_get_cached.py @@ -7,6 +7,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource from orcapod.databases import InMemoryArrowDatabase @@ -29,7 +30,7 @@ def function_node_with_db(): pod = FunctionPod(pf) pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( pod, src, pipeline_database=pipeline_db, @@ -49,7 +50,7 @@ def test_returns_empty_dict_when_no_db(self): src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) - node = FunctionNode(pod, src) + node = FunctionJobNode(pod, src) assert node.get_cached_results([]) == {} def test_returns_empty_dict_when_db_empty(self, function_node_with_db): diff --git a/tests/test_core/nodes/test_function_node_iteration.py b/tests/test_core/nodes/test_function_node_iteration.py index 3dc212659..ae67fcdbc 100644 --- a/tests/test_core/nodes/test_function_node_iteration.py +++ b/tests/test_core/nodes/test_function_node_iteration.py @@ -13,6 +13,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource from orcapod.databases import InMemoryArrowDatabase @@ -35,14 +36,14 @@ def _make_source(n: int = 3) -> ArrowTableSource: return ArrowTableSource(table, tag_columns=["id"]) -def _make_node(n: int = 3, db: InMemoryArrowDatabase | None = None) -> FunctionNode: +def _make_node(n: int = 3, db: InMemoryArrowDatabase | None = None) -> FunctionJobNode: def double(x: int) -> int: return x * 2 pf = PythonDataFunction(double, output_keys="result") pod = FunctionPod(pf) pipeline_db = db if db is not None else InMemoryArrowDatabase() - return FunctionNode(pod, _make_source(n=n), pipeline_database=pipeline_db) + return FunctionJobNode(pod, _make_source(n=n), pipeline_database=pipeline_db) class TestIterDatasReadOnly: @@ -149,7 +150,7 @@ def sometimes_fail(x: int) -> int: pf.executor = LocalPythonFunctionExecutor() # supports_concurrent_execution is False pod = FunctionPod(pf) db = InMemoryArrowDatabase() - node = FunctionNode(pod, _make_source(n=3), pipeline_database=db) + node = FunctionJobNode(pod, _make_source(n=3), pipeline_database=db) from orcapod.pipeline.observer import NoOpObserver diff --git a/tests/test_core/nodes/test_function_node_split.py b/tests/test_core/nodes/test_function_node_split.py new file mode 100644 index 000000000..e1cfadd1e --- /dev/null +++ b/tests/test_core/nodes/test_function_node_split.py @@ -0,0 +1,148 @@ +"""Tests for the FunctionNode / FunctionJobNode split.""" +from __future__ import annotations + +import pytest + +from orcapod.errors import PipelineJobRequiredError +from orcapod.types import Schema + + +@pytest.fixture +def simple_setup(): + """A minimal function pod + source node fixture.""" + from orcapod.core.function_pod import FunctionPod + from orcapod.core.data_function import PythonDataFunction + from orcapod.core.nodes.source_node import SourceNode + + tag_schema = Schema({"id": int}) + data_schema = Schema({"value": float}) + + source_node = SourceNode(name="src", tag_schema=tag_schema, data_schema=data_schema) + + def double(value: float) -> float: + return value * 2 + + pf = PythonDataFunction(double, output_keys="result") + pod = FunctionPod(pf) + + return { + "source_node": source_node, + "pod": pod, + } + + +class TestThinFunctionNode: + def test_iter_data_raises_pipeline_job_required(self, simple_setup): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + with pytest.raises(PipelineJobRequiredError): + list(fn.iter_data()) + + def test_content_hash_is_stable(self, simple_setup): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + h1 = fn.content_hash() + h2 = fn.content_hash() + assert h1 == h2 + + def test_node_type(self, simple_setup): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + assert fn.node_type == "function" + + def test_output_schema(self, simple_setup): + from orcapod.core.nodes.function_node import FunctionNode + + fn = FunctionNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + tag_s, data_s = fn.output_schema() + assert "result" in data_s + + def test_function_node_does_not_accept_db_params(self, simple_setup): + """FunctionNode must NOT accept pipeline_database param.""" + from orcapod.core.nodes.function_node import FunctionNode + import inspect + + sig = inspect.signature(FunctionNode.__init__) + assert "pipeline_database" not in sig.parameters, ( + "FunctionNode must not accept pipeline_database — use FunctionJobNode instead" + ) + + +class TestFunctionJobNodeHashParity: + """FunctionJobNode must have identical content_hash / pipeline_hash to FunctionNode.""" + + def test_content_hash_matches_function_node(self, simple_setup): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + fn = FunctionNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + fjn = FunctionJobNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + assert fn.content_hash() == fjn.content_hash() + + def test_pipeline_hash_matches_function_node(self, simple_setup): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + fn = FunctionNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + fjn = FunctionJobNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + assert fn.pipeline_hash() == fjn.pipeline_hash() + + def test_as_node_returns_function_node(self, simple_setup): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + fjn = FunctionJobNode( + function_pod=simple_setup["pod"], + input_stream=simple_setup["source_node"], + ) + fn = fjn.as_node() + assert isinstance(fn, FunctionNode) + assert fn.content_hash() == fjn.content_hash() + + +class TestSiblingHierarchy: + """FunctionNode and FunctionJobNode must be siblings, not parent/child.""" + + def test_function_node_not_subclass_of_function_job_node(self): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + assert not issubclass(FunctionNode, FunctionJobNode), ( + "FunctionNode must NOT inherit from FunctionJobNode" + ) + + def test_function_job_node_not_subclass_of_function_node(self): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + + assert not issubclass(FunctionJobNode, FunctionNode), ( + "FunctionJobNode must NOT inherit from FunctionNode" + ) + + def test_both_inherit_from_base(self): + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode, FunctionNodeBase + + assert issubclass(FunctionNode, FunctionNodeBase) + assert issubclass(FunctionJobNode, FunctionNodeBase) diff --git a/tests/test_core/nodes/test_node_execute.py b/tests/test_core/nodes/test_node_execute.py index 9875d737c..6b74d71e6 100644 --- a/tests/test_core/nodes/test_node_execute.py +++ b/tests/test_core/nodes/test_node_execute.py @@ -11,6 +11,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource from orcapod.databases import InMemoryArrowDatabase @@ -33,7 +34,7 @@ def function_node_with_db(): pod = FunctionPod(pf) pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( pod, src, pipeline_database=pipeline_db, @@ -53,7 +54,7 @@ def function_node_no_db(): src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) - return FunctionNode(pod, src) + return FunctionJobNode(pod, src) class TestFunctionNodeExecuteData: diff --git a/tests/test_core/sources/test_derived_source.py b/tests/test_core/sources/test_derived_source.py index 8865c6f41..8674fca08 100644 --- a/tests/test_core/sources/test_derived_source.py +++ b/tests/test_core/sources/test_derived_source.py @@ -28,6 +28,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.sources import DerivedSource, RootSource from orcapod.core.streams import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase @@ -41,13 +42,13 @@ # --------------------------------------------------------------------------- -def _make_node(n: int = 3, db: InMemoryArrowDatabase | None = None) -> FunctionNode: +def _make_node(n: int = 3, db: InMemoryArrowDatabase | None = None) -> FunctionJobNode: from orcapod.core.data_function import PythonDataFunction if db is None: db = InMemoryArrowDatabase() pf = PythonDataFunction(double, output_keys="result") - return FunctionNode( + return FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=make_int_stream(n=n), pipeline_database=db, @@ -226,7 +227,7 @@ def test_derived_source_can_feed_downstream_node(self): result_stream = ArrowTableStream(result_table, tag_columns=["id"]) double_result = PythonDataFunction(double, output_keys="result") - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=FunctionPod(data_function=double_result), input_stream=result_stream, pipeline_database=InMemoryArrowDatabase(), # fresh DB @@ -358,7 +359,7 @@ def triple(x: int) -> tuple[int, int]: node_double.run() src_double = node_double.as_source() - node_triple = FunctionNode( + node_triple = FunctionJobNode( function_pod=FunctionPod(data_function=triple_pf), input_stream=make_int_stream(n=3), pipeline_database=db, @@ -379,12 +380,12 @@ def test_same_data_different_origin_content_hash_differs(self): pf = PythonDataFunction(double, output_keys="result") stream = make_int_stream(n=3) - node_a = FunctionNode( + node_a = FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=stream, pipeline_database=InMemoryArrowDatabase(), ) - node_b = FunctionNode( + node_b = FunctionJobNode( function_pod=FunctionPod(data_function=pf), input_stream=stream, pipeline_database=InMemoryArrowDatabase(), diff --git a/tests/test_core/test_caching_integration.py b/tests/test_core/test_caching_integration.py index 61c463ca8..2c1c569b5 100644 --- a/tests/test_core/test_caching_integration.py +++ b/tests/test_core/test_caching_integration.py @@ -22,6 +22,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode, OperatorNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, DeltaTableSource, CachedSource @@ -294,7 +295,7 @@ def test_function_node_stores_records( ) joined = Join()(patients, labs) - fn_node = FunctionNode( + fn_node = FunctionJobNode( function_pod=pod, input_stream=joined, pipeline_database=pipeline_db, @@ -324,7 +325,7 @@ def test_cross_source_sharing_same_pipeline_path( DeltaTableSource(labs_a, tag_columns=["patient_id"]), cache_database=source_db, ) - fn_a = FunctionNode( + fn_a = FunctionJobNode( function_pod=pod, input_stream=Join()(pa_src, la_src), pipeline_database=pipeline_db, @@ -340,7 +341,7 @@ def test_cross_source_sharing_same_pipeline_path( DeltaTableSource(labs_b, tag_columns=["patient_id"]), cache_database=source_db, ) - fn_b = FunctionNode( + fn_b = FunctionJobNode( function_pod=pod, input_stream=Join()(pb_src, lb_src), pipeline_database=pipeline_db, @@ -362,7 +363,7 @@ def test_cross_source_records_accumulate_in_shared_table( patients_b, labs_b = clinic_b # Pipeline A: 3 patients - fn_a = FunctionNode( + fn_a = FunctionJobNode( function_pod=pod, input_stream=Join()( CachedSource( @@ -381,7 +382,7 @@ def test_cross_source_records_accumulate_in_shared_table( assert fn_a.get_all_records().num_rows == 3 # Pipeline B: 2 patients, different source identity, same schema - fn_b = FunctionNode( + fn_b = FunctionJobNode( function_pod=pod, input_stream=Join()( CachedSource( @@ -553,7 +554,7 @@ def test_full_pipeline_source_to_function_to_operator( # Step 2: Join + FunctionNode joined = Join()(patients, labs) - fn_node = FunctionNode( + fn_node = FunctionJobNode( function_pod=pod, input_stream=joined, pipeline_database=pipeline_db, @@ -585,7 +586,7 @@ def test_full_pipeline_source_to_function_to_operator( # Step 5: Second clinic uses same function but different data → own pipeline_path patients_b, labs_b = clinic_b - fn_node_b = FunctionNode( + fn_node_b = FunctionJobNode( function_pod=pod, input_stream=Join()( CachedSource( diff --git a/tests/test_core/test_table_scope.py b/tests/test_core/test_table_scope.py index 2010c4262..d71603e43 100644 --- a/tests/test_core/test_table_scope.py +++ b/tests/test_core/test_table_scope.py @@ -14,6 +14,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode, OperatorNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, DictSource @@ -83,14 +84,14 @@ def test_default_scope_is_pipeline_hash(self): db = InMemoryArrowDatabase() pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) - node = FunctionNode(function_pod=pod, input_stream=src, pipeline_database=db) + node = FunctionJobNode(function_pod=pod, input_stream=src, pipeline_database=db) assert node._table_scope == "pipeline_hash" def test_node_identity_path_ends_with_schema_only(self): db = InMemoryArrowDatabase() pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) - node = FunctionNode(function_pod=pod, input_stream=src, pipeline_database=db) + node = FunctionJobNode(function_pod=pod, input_stream=src, pipeline_database=db) path = node.node_identity_path assert path[-1].startswith("schema:"), f"Expected schema:... got {path[-1]!r}" assert not any(seg.startswith("instance:") for seg in path) @@ -102,8 +103,8 @@ def test_two_nodes_same_function_same_schema_share_path(self): # Both use same source_id → same schema structure → same pipeline_hash src_a = _make_source([{"x": 1, "y": 10}], source_id="src") src_b = _make_source([{"x": 2, "y": 20}], source_id="src") - node_a = FunctionNode(function_pod=pod, input_stream=src_a, pipeline_database=db) - node_b = FunctionNode(function_pod=pod, input_stream=src_b, pipeline_database=db) + node_a = FunctionJobNode(function_pod=pod, input_stream=src_a, pipeline_database=db) + node_b = FunctionJobNode(function_pod=pod, input_stream=src_b, pipeline_database=db) assert node_a.node_identity_path == node_b.node_identity_path assert node_a.pipeline_hash() == node_b.pipeline_hash() @@ -113,7 +114,7 @@ def test_node_content_hash_col_in_pipeline_records(self): db = InMemoryArrowDatabase() pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) - node = FunctionNode(function_pod=pod, input_stream=src, pipeline_database=db) + node = FunctionJobNode(function_pod=pod, input_stream=src, pipeline_database=db) node.run() col_name = SystemConstant().NODE_CONTENT_HASH_COL @@ -126,7 +127,7 @@ def test_get_all_records_drops_node_content_hash_col(self): db = InMemoryArrowDatabase() pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) - node = FunctionNode(function_pod=pod, input_stream=src, pipeline_database=db) + node = FunctionJobNode(function_pod=pod, input_stream=src, pipeline_database=db) node.run() records = node.get_all_records() @@ -140,8 +141,8 @@ def test_isolation_two_nodes_share_table_see_only_own_records(self): # Different source_ids → different content_hash → different _node_content_hash rows src_a = _make_source([{"x": 1, "y": 10}], source_id="source_a") src_b = _make_source([{"x": 2, "y": 20}], source_id="source_b") - node_a = FunctionNode(function_pod=pod, input_stream=src_a, pipeline_database=db) - node_b = FunctionNode(function_pod=pod, input_stream=src_b, pipeline_database=db) + node_a = FunctionJobNode(function_pod=pod, input_stream=src_a, pipeline_database=db) + node_b = FunctionJobNode(function_pod=pod, input_stream=src_b, pipeline_database=db) node_a.run() node_b.run() @@ -170,7 +171,7 @@ def test_scope_set_to_content_hash(self): db = InMemoryArrowDatabase() pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=src, pipeline_database=db, table_scope="content_hash" ) assert node._table_scope == "content_hash" @@ -179,7 +180,7 @@ def test_node_identity_path_ends_with_schema_and_instance(self): db = InMemoryArrowDatabase() pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=src, pipeline_database=db, table_scope="content_hash" ) path = node.node_identity_path @@ -194,10 +195,10 @@ def test_two_nodes_different_source_id_have_different_paths(self): pod = _make_pod() src_a = _make_source([{"x": 1, "y": 10}], source_id="source_a") src_b = _make_source([{"x": 2, "y": 20}], source_id="source_b") - node_a = FunctionNode( + node_a = FunctionJobNode( function_pod=pod, input_stream=src_a, pipeline_database=db, table_scope="content_hash" ) - node_b = FunctionNode( + node_b = FunctionJobNode( function_pod=pod, input_stream=src_b, pipeline_database=db, table_scope="content_hash" ) assert node_a.content_hash() != node_b.content_hash() @@ -210,10 +211,10 @@ def test_pipeline_hash_still_equal_across_content_hash_nodes(self): # Same source_id → same pipeline_hash src_a = _make_source([{"x": 1, "y": 10}], source_id="src") src_b = _make_source([{"x": 2, "y": 20}], source_id="src") - node_a = FunctionNode( + node_a = FunctionJobNode( function_pod=pod, input_stream=src_a, pipeline_database=db, table_scope="content_hash" ) - node_b = FunctionNode( + node_b = FunctionJobNode( function_pod=pod, input_stream=src_b, pipeline_database=db, table_scope="content_hash" ) # pipeline_hash same → same schema: segment @@ -232,7 +233,7 @@ def test_from_descriptor_missing_table_scope_raises(self): pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) db = InMemoryArrowDatabase() - node = FunctionNode(function_pod=pod, input_stream=src, pipeline_database=db) + node = FunctionJobNode(function_pod=pod, input_stream=src, pipeline_database=db) tag_schema, data_schema = node.output_schema() descriptor = { "node_type": "function", @@ -258,7 +259,7 @@ def test_from_descriptor_preserves_pipeline_hash_scope(self): pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) db = InMemoryArrowDatabase() - node = FunctionNode(function_pod=pod, input_stream=src, pipeline_database=db) + node = FunctionJobNode(function_pod=pod, input_stream=src, pipeline_database=db) tag_schema, data_schema = node.output_schema() descriptor = { "node_type": "function", @@ -284,7 +285,7 @@ def test_from_descriptor_preserves_content_hash_scope(self): pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=src, pipeline_database=db, table_scope="content_hash" ) tag_schema, data_schema = node.output_schema() @@ -573,7 +574,7 @@ def test_function_node_cache_cleared_on_clear_cache(self): db = InMemoryArrowDatabase() pod = _make_pod() src = _make_source([{"x": 1, "y": 2}]) - node = FunctionNode(function_pod=pod, input_stream=src, pipeline_database=db) + node = FunctionJobNode(function_pod=pod, input_stream=src, pipeline_database=db) _ = node.node_identity_path # populate cache assert node._node_identity_path_cache is not None node.clear_cache() diff --git a/tests/test_data/test_polars_nullability/test_function_node_nullability.py b/tests/test_data/test_polars_nullability/test_function_node_nullability.py index 84eec1295..cbe84d673 100644 --- a/tests/test_data/test_polars_nullability/test_function_node_nullability.py +++ b/tests/test_data/test_polars_nullability/test_function_node_nullability.py @@ -8,7 +8,7 @@ import pyarrow as pa import orcapod as op -from orcapod.core.nodes.function_node import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.databases import InMemoryArrowDatabase from orcapod.pipeline import PipelineJob from orcapod.pipeline.graph import Pipeline @@ -19,9 +19,9 @@ # --------------------------------------------------------------------------- -def _get_function_nodes(pipeline: Pipeline) -> list[FunctionNode]: - """Return all FunctionNode instances from compiled pipeline nodes.""" - return [n for n in pipeline.compiled_nodes.values() if isinstance(n, FunctionNode)] +def _get_function_nodes(pipeline: Pipeline) -> list[FunctionJobNode]: + """Return all FunctionJobNode instances from compiled pipeline nodes.""" + return [n for n in pipeline.compiled_nodes.values() if isinstance(n, FunctionJobNode)] # --------------------------------------------------------------------------- diff --git a/tests/test_pipeline/test_node_descriptors.py b/tests/test_pipeline/test_node_descriptors.py index 95205d932..7b61acafb 100644 --- a/tests/test_pipeline/test_node_descriptors.py +++ b/tests/test_pipeline/test_node_descriptors.py @@ -3,6 +3,7 @@ import pytest from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.sources.dict_source import DictSource from orcapod.databases.in_memory_databases import InMemoryArrowDatabase from orcapod.errors import UnboundSourceError @@ -113,7 +114,7 @@ def _make_function_node_descriptor(self): pod = FunctionPod(data_function=pf) db = InMemoryArrowDatabase() scoped_db = db.at("test_pipeline") - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=source, pipeline_database=scoped_db, diff --git a/tests/test_pipeline/test_node_protocols.py b/tests/test_pipeline/test_node_protocols.py index 2c7b4a59b..452b75cc8 100644 --- a/tests/test_pipeline/test_node_protocols.py +++ b/tests/test_pipeline/test_node_protocols.py @@ -254,6 +254,7 @@ def on_data_end(self, node_label, t, ip, op, cached): from orcapod.core.function_pod import FunctionPod from orcapod.core.data_function import PythonDataFunction from orcapod.core.nodes import FunctionNode +from orcapod.core.nodes.function_node import FunctionJobNode def double_value(value: int) -> int: @@ -269,7 +270,7 @@ def _make_function_node(self): src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) - return FunctionNode(pod, src) + return FunctionJobNode(pod, src) def test_execute_with_observer(self): node = self._make_function_node() @@ -326,7 +327,7 @@ async def test_tightened_signature(self): src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) - node = FunctionNode(pod, src) + node = FunctionJobNode(pod, src) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -350,7 +351,7 @@ async def test_async_execute_with_observer(self): src = ArrowTableSource(table, tag_columns=["key"], infer_nullable=True) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) - node = FunctionNode(pod, src) + node = FunctionJobNode(pod, src) events = [] diff --git a/tests/test_pipeline/test_orchestrator.py b/tests/test_pipeline/test_orchestrator.py index b656057fa..134d43311 100644 --- a/tests/test_pipeline/test_orchestrator.py +++ b/tests/test_pipeline/test_orchestrator.py @@ -25,6 +25,7 @@ from orcapod.channels import Channel from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.operators import SelectDataColumns from orcapod.core.operators.join import Join @@ -141,7 +142,7 @@ async def test_processes_data(self): src = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) pf = PythonDataFunction(double_value, output_keys="result") pod = FunctionPod(pf) - node = FunctionNode(pod, src) + node = FunctionJobNode(pod, src) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index a74ecaaf4..555090902 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -23,6 +23,7 @@ OperatorNode, SourceNode, ) +from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction @@ -227,7 +228,7 @@ def test_exec_nodes_have_pipeline_database_after_run(self, pipeline_db): # After run, compiled_nodes["adder"] is the exec node in the returned result exec_node = result.pipeline.compiled_nodes["adder"] - assert isinstance(exec_node, FunctionNode) + assert isinstance(exec_node, FunctionJobNode) assert exec_node._pipeline_database is not None def test_exec_operator_nodes_have_pipeline_database_after_run(self, pipeline_db): @@ -261,7 +262,7 @@ def test_result_database_scoped_to_pipeline_name(self, pipeline_db): result = job.run() exec_node = result.pipeline.compiled_nodes["adder"] - assert isinstance(exec_node, FunctionNode) + assert isinstance(exec_node, FunctionJobNode) # Verify the exec node has databases attached assert exec_node._pipeline_database is not None # The result DB is scoped as pipeline_name/_result internally. @@ -592,8 +593,10 @@ def test_compile_does_not_trigger_source_materialization(self, pipeline_db): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - # After compile but before run, adder node has no records - assert job.pipeline.compiled_nodes["adder"].get_all_records() is None + # After compile but before run, adder node is a blueprint FunctionNode with no DB + pre_run_node = job.pipeline.compiled_nodes["adder"] + assert isinstance(pre_run_node, FunctionNode) + assert not isinstance(pre_run_node, FunctionJobNode) # Running should work correctly result = job.run() table = result.pipeline.compiled_nodes["adder"].as_table() From 3e3e2a85245159c4721d85a8adb43f9cf5bdd366 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:03:51 +0000 Subject: [PATCH 07/24] fix(nodes): FunctionJobNode.clear_cache uses super(); as_node forwards tracker_manager --- src/orcapod/core/nodes/function_node.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index affbb9ab5..61df1ceaf 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -726,12 +726,11 @@ def attach_databases( # ------------------------------------------------------------------ def clear_cache(self) -> None: - """Clear in-memory output caches and the node identity path cache.""" + """Clear in-memory output caches, content hash cache, and node identity path cache.""" + super().clear_cache() self._cached_output_datas.clear() self._cached_output_table = None self._cached_content_hash_column = None - self._node_identity_path_cache = None - self._update_modified_time() # ------------------------------------------------------------------ # Internal helpers @@ -783,14 +782,15 @@ def as_node(self) -> FunctionNode: Returns: A new ``FunctionNode`` with the same function pod, input stream, - label, and table scope. Its ``content_hash()`` / ``pipeline_hash()`` - are identical to those of this ``FunctionJobNode``. + label, table scope, and tracker manager. Its ``content_hash()`` / + ``pipeline_hash()`` are identical to those of this ``FunctionJobNode``. """ return FunctionNode( function_pod=self._function_pod, input_stream=self._input_stream, label=self._label, table_scope=self._table_scope, + tracker_manager=self.tracker_manager, ) # ------------------------------------------------------------------ From 7a23d1dd46961f2200c57b58443a3c17333f86a6 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:22:06 +0000 Subject: [PATCH 08/24] =?UTF-8?q?refactor(nodes):=20split=20OperatorNode?= =?UTF-8?q?=20=E2=86=92=20OperatorNodeBase=20+=20OperatorNode=20+=20Operat?= =?UTF-8?q?orJobNode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the Task 2 FunctionNode split. OperatorNode and OperatorJobNode are siblings inheriting from OperatorNodeBase(StreamBase): OperatorNode is a thin blueprint that raises PipelineJobRequiredError on iter_data/as_table; OperatorJobNode is the DB-backed execution node. Update Pipeline graph/job to create OperatorJobNode at execution time and deserialize via OperatorJobNode.from_descriptor. All 3075 tests pass. Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/__init__.py | 6 +- src/orcapod/core/nodes/operator_node.py | 529 +++++++++++------- src/orcapod/pipeline/graph.py | 3 +- src/orcapod/pipeline/job.py | 9 +- .../test_channels/test_node_async_execute.py | 29 +- tests/test_core/nodes/test_node_execute.py | 2 +- .../nodes/test_operator_node_split.py | 116 ++++ .../test_core/operators/test_operator_node.py | 8 +- .../operators/test_operator_node_attach_db.py | 32 +- .../test_operator_node_non_active.py | 8 +- tests/test_core/test_caching_integration.py | 21 +- tests/test_core/test_table_scope.py | 45 +- tests/test_pipeline/test_node_descriptors.py | 8 +- tests/test_pipeline/test_node_protocols.py | 2 +- tests/test_pipeline/test_orchestrator.py | 3 +- tests/test_pipeline/test_pipeline.py | 3 +- tests/test_pipeline/test_sync_orchestrator.py | 8 +- 17 files changed, 557 insertions(+), 275 deletions(-) create mode 100644 tests/test_core/nodes/test_operator_node_split.py diff --git a/src/orcapod/core/nodes/__init__.py b/src/orcapod/core/nodes/__init__.py index e4b7f8ee6..d5f2e99b5 100644 --- a/src/orcapod/core/nodes/__init__.py +++ b/src/orcapod/core/nodes/__init__.py @@ -1,11 +1,11 @@ from typing import TypeAlias from .function_node import FunctionJobNode, FunctionNode, FunctionNodeBase -from .operator_node import OperatorNode +from .operator_node import OperatorJobNode, OperatorNode, OperatorNodeBase from .source_node import SourceJobNode, SourceNode, SourceNodeBase GraphNode: TypeAlias = SourceNode | FunctionNode | OperatorNode -JobNode: TypeAlias = SourceJobNode | FunctionJobNode +JobNode: TypeAlias = SourceJobNode | FunctionJobNode | OperatorJobNode __all__ = [ "FunctionJobNode", @@ -13,7 +13,9 @@ "FunctionNodeBase", "GraphNode", "JobNode", + "OperatorJobNode", "OperatorNode", + "OperatorNodeBase", "SourceJobNode", "SourceNode", "SourceNodeBase", diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 3cf44c7fa..ede63a2de 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1,4 +1,19 @@ -"""OperatorNode — stream node for operator invocations with optional DB persistence.""" +"""OperatorNode hierarchy — pure blueprint + DB-backed execution node. + +Three classes: + +* ``OperatorNodeBase`` — shared base; no DB state. Holds identity, + schema, and all non-DB properties. +* ``OperatorNode`` — thin blueprint descriptor. Raises + ``PipelineJobRequiredError`` on ``iter_data()``. This is the node + recorded in a ``Pipeline`` and serialized to disk. +* ``OperatorJobNode`` — DB-backed execution node; carries all DB logic + from the original ``OperatorNode``. Created by ``PipelineJob`` at + run time. + +``OperatorNode`` and ``OperatorJobNode`` are *siblings*: both inherit +directly from ``OperatorNodeBase``, neither from the other. +""" from __future__ import annotations @@ -14,6 +29,7 @@ from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.core.streams.base import StreamBase from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.errors import PipelineJobRequiredError from orcapod.protocols.core_protocols import ( DataProtocol, StreamProtocol, @@ -38,30 +54,16 @@ pc = LazyModule("pyarrow.compute") -class OperatorNode(StreamBase): - """Stream node representing an operator invocation with optional DB persistence. - - When constructed without database parameters, provides the core stream - interface (identity, schema, iteration) without any persistence. When - databases are provided (either at construction or via ``attach_databases``), - adds pipeline record storage with per-row deduplication, ``get_all_records()`` - for retrieving stored results, ``as_source()`` for creating a - ``DerivedSource`` from DB records, and three-tier cache mode - (OFF / LOG / REPLAY). +# --------------------------------------------------------------------------- +# OperatorNodeBase — shared base (no DB) +# --------------------------------------------------------------------------- - Node identity path structure:: - operator.uri / schema:{pipeline_hash} / instance:{content_hash} +class OperatorNodeBase(StreamBase): + """Shared base for ``OperatorNode`` and ``OperatorJobNode``. - Where ``pipeline_hash`` encodes the pipeline structure (operator + - upstream topology) and ``instance:{content_hash}`` is the - data-inclusive hash that encodes upstream source identities, ensuring - each unique source combination gets its own cache table. - - Cache modes: - - **OFF** (default): compute, don't write to DB. - - **LOG**: compute AND write to DB (append-only historical record). - - **REPLAY**: skip computation, flow cached results downstream. + Carries all non-DB state: identity, schema, upstreams, and properties + shared by both the blueprint and the execution variant. """ node_type = "operator" @@ -74,9 +76,6 @@ def __init__( tracker_manager: TrackerManagerProtocol | None = None, label: str | None = None, config: Config | None = None, - # Optional DB params for persistent mode: - pipeline_database: ArrowDatabaseProtocol | None = None, - cache_mode: CacheMode = CacheMode.OFF, table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", ): if tracker_manager is None: @@ -91,15 +90,8 @@ def __init__( # Validate inputs eagerly self._operator.validate_inputs(*self._input_streams) - # Stream-level caching state - self._cached_output_stream: StreamProtocol | None = None - self._cached_output_table: pa.Table | None = None self._set_modified_time(None) - # DB persistence state (initially None; set via __init__ params or attach_databases) - self._pipeline_database: ArrowDatabaseProtocol | None = None - self._cache_mode = CacheMode.OFF - # Descriptor fields — populated by from_descriptor() for read-only/UNAVAILABLE # nodes. Initialized here so they are always present on the concrete class # (avoids getattr access for possibly-absent attributes). @@ -111,6 +103,7 @@ def __init__( self._stored_node_uri: tuple[str, ...] = () self._stored_pipeline_path: tuple[str, ...] = () self._descriptor: dict = {} + if table_scope not in ("pipeline_hash", "content_hash"): raise ValueError( f"Unknown table_scope {table_scope!r}. " @@ -119,12 +112,316 @@ def __init__( self._table_scope = table_scope self._node_identity_path_cache: tuple[str, ...] | None = None + # ------------------------------------------------------------------ + # load_status + # ------------------------------------------------------------------ + + @property + def load_status(self) -> Any: + """Return the load status of this node. + + Returns: + The ``LoadStatus`` enum value indicating how this node was + loaded. Defaults to ``FULL`` for nodes created via + ``__init__``. + """ + return self._load_status + + # ------------------------------------------------------------------ + # Identity + # ------------------------------------------------------------------ + + def identity_structure(self) -> Any: + return (self._operator, self._operator.argument_symmetry(self._input_streams)) + + def pipeline_identity_structure(self) -> Any: + return (self._operator, self._operator.argument_symmetry(self._input_streams)) + + # ------------------------------------------------------------------ + # Read-only overrides (for deserialized nodes without live operator) + # ------------------------------------------------------------------ + + def content_hash(self, hasher=None) -> ContentHash: + """Return the content hash, using stored value in read-only mode.""" + if self._operator is None and self._stored_content_hash is not None: + return ContentHash.from_string(self._stored_content_hash) + return super().content_hash(hasher) + + def pipeline_hash(self, hasher=None) -> ContentHash: + """Return the pipeline hash, using stored value in read-only mode.""" + if self._operator is None and self._stored_pipeline_hash is not None: + return ContentHash.from_string(self._stored_pipeline_hash) + return super().pipeline_hash(hasher) + + # ------------------------------------------------------------------ + # Stream interface + # ------------------------------------------------------------------ + + @property + def producer(self) -> OperatorPodProtocol: + return self._operator + + @property + def data_context(self) -> contexts.DataContext: + return contexts.resolve_context(self._operator.data_context_key) + + @property + def data_context_key(self) -> str: + return self._operator.data_context_key + + @property + def upstreams(self) -> tuple[StreamProtocol, ...]: + return self._input_streams + + @upstreams.setter + def upstreams(self, value: tuple[StreamProtocol, ...]) -> None: + self._input_streams = value + + def keys( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + if self._operator is None: + tag_keys = tuple(self._stored_schema.get("tag", {}).keys()) + data_keys = tuple(self._stored_schema.get("data", {}).keys()) + return tag_keys, data_keys + tag_schema, data_schema = self.output_schema( + columns=columns, all_info=all_info + ) + return tuple(tag_schema.keys()), tuple(data_schema.keys()) + + def output_schema( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """Return output schema, using stored value in read-only mode.""" + if self._operator is None: + tag = Schema(self._stored_schema.get("tag", {})) + data = Schema(self._stored_schema.get("data", {})) + return tag, data + return self._operator.output_schema( + *self._input_streams, + columns=columns, + all_info=all_info, + ) + + # ------------------------------------------------------------------ + # Node identity path + # ------------------------------------------------------------------ + + @property + def node_identity_path(self) -> tuple[str, ...]: + """Return the node identity path for observer contextualization. + + When ``table_scope="pipeline_hash"`` (default) the path is + ``operator.uri + (schema:{pipeline_hash},)`` — all runs that share + the same pipeline structure use one shared table, with per-run + disambiguation via the ``_node_content_hash`` row-level column. + + When ``table_scope="content_hash"`` the legacy path is returned: + ``operator.uri + (schema:{pipeline_hash}, instance:{content_hash})``. + + In read-only/UNAVAILABLE mode (no operator) the path stored from the + deserialized descriptor is returned (empty tuple when absent). + """ + if self._operator is None: + return self._stored_pipeline_path + if self._node_identity_path_cache is not None: + return self._node_identity_path_cache + path = self._operator.uri + (f"schema:{self.pipeline_hash().to_string()}",) + if self._table_scope != "pipeline_hash": + path += (f"instance:{self.content_hash().to_string()}",) + self._node_identity_path_cache = path + return path + + @property + def node_uri(self) -> tuple[str, ...]: + """Canonical URI tuple identifying this computation. + + Identical to ``operator.uri`` at runtime. + Returns stored value in read-only (deserialized) mode. + """ + if self._operator is None: + return self._stored_node_uri + return self._operator.uri + + # ------------------------------------------------------------------ + # Caching + # ------------------------------------------------------------------ + + def clear_cache(self) -> None: + """Clear the node identity path cache.""" + self._node_identity_path_cache = None + self._update_modified_time() + + def __repr__(self) -> str: + return ( + f"{type(self).__name__}(operator={self._operator!r}, " + f"upstreams={self._input_streams!r})" + ) + + +# --------------------------------------------------------------------------- +# OperatorNode — thin blueprint (no DB) +# --------------------------------------------------------------------------- + + +class OperatorNode(OperatorNodeBase): + """Thin blueprint descriptor for an operator pod invocation. + + Carries no database references. Calling ``iter_data()`` raises + ``PipelineJobRequiredError`` — wrap the containing ``Pipeline`` in a + ``PipelineJob`` to obtain an executable ``OperatorJobNode``. + + This is the node type recorded inside a ``Pipeline`` context manager + and serialized to disk via ``Pipeline.save()``. + """ + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Raise ``PipelineJobRequiredError`` — blueprint nodes cannot produce data. + + Raises: + PipelineJobRequiredError: Always. + """ + raise PipelineJobRequiredError( + f"OperatorNode '{self.label}' is a blueprint — it carries no database " + "references and cannot produce data directly. " + "Wrap the containing Pipeline in a PipelineJob to obtain an executable " + "OperatorJobNode." + ) + # yield is needed to satisfy the Iterator return type annotation + return # pragma: no cover + yield # pragma: no cover + + def as_table( + self, + *, + columns: ColumnConfig | dict[Any, Any] | None = None, + all_info: bool = False, + ) -> "pa.Table": + """Raise ``PipelineJobRequiredError`` — blueprint nodes cannot produce data. + + Raises: + PipelineJobRequiredError: Always. + """ + raise PipelineJobRequiredError( + f"OperatorNode '{self.label}' is a blueprint — it carries no database " + "references and cannot produce data directly. " + "Wrap the containing Pipeline in a PipelineJob to obtain an executable " + "OperatorJobNode." + ) + + def as_node(self) -> "OperatorNode": + """Return ``self`` — already the lightweight blueprint form. + + Returns: + This instance. + """ + return self + + +# --------------------------------------------------------------------------- +# OperatorJobNode — DB-backed execution node +# --------------------------------------------------------------------------- + + +class OperatorJobNode(OperatorNodeBase): + """DB-backed execution node for operator pod invocations. + + Stream node representing an operator invocation with optional DB persistence. + + When constructed without database parameters, provides the core stream + interface (identity, schema, iteration) without any persistence. When + databases are provided (either at construction or via ``attach_databases``), + adds pipeline record storage with per-row deduplication, ``get_all_records()`` + for retrieving stored results, ``as_source()`` for creating a + ``DerivedSource`` from DB records, and three-tier cache mode + (OFF / LOG / REPLAY). + + Node identity path structure:: + + operator.uri / schema:{pipeline_hash} / instance:{content_hash} + + Where ``pipeline_hash`` encodes the pipeline structure (operator + + upstream topology) and ``instance:{content_hash}`` is the + data-inclusive hash that encodes upstream source identities, ensuring + each unique source combination gets its own cache table. + + Cache modes: + - **OFF** (default): compute, don't write to DB. + - **LOG**: compute AND write to DB (append-only historical record). + - **REPLAY**: skip computation, flow cached results downstream. + """ + + def __init__( + self, + operator: OperatorPodProtocol, + input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], + tracker_manager: TrackerManagerProtocol | None = None, + label: str | None = None, + config: Config | None = None, + # Optional DB params for persistent mode: + pipeline_database: ArrowDatabaseProtocol | None = None, + cache_mode: CacheMode = CacheMode.OFF, + table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", + ): + super().__init__( + operator=operator, + input_streams=input_streams, + tracker_manager=tracker_manager, + label=label, + config=config, + table_scope=table_scope, + ) + + # Stream-level caching state + self._cached_output_stream: StreamProtocol | None = None + self._cached_output_table: pa.Table | None = None + + # DB persistence state (initially None; set via __init__ params or attach_databases) + self._pipeline_database: ArrowDatabaseProtocol | None = None + self._cache_mode = CacheMode.OFF + if pipeline_database is not None: self.attach_databases( pipeline_database=pipeline_database, cache_mode=cache_mode, ) + # ------------------------------------------------------------------ + # as_node — return the lightweight OperatorNode equivalent + # ------------------------------------------------------------------ + + def as_node(self) -> OperatorNode: + """Return the lightweight ``OperatorNode`` equivalent of this job node. + + Returns: + A new ``OperatorNode`` with the same operator, input streams, + label, table scope, and tracker manager. Its ``content_hash()`` / + ``pipeline_hash()`` are identical to those of this ``OperatorJobNode``. + """ + return OperatorNode( + operator=self._operator, + input_streams=self._input_streams, + label=self._label, + table_scope=self._table_scope, + tracker_manager=self.tracker_manager, + ) + + # ------------------------------------------------------------------ + # Override clear_cache to also clear output stream caches + # ------------------------------------------------------------------ + + def clear_cache(self) -> None: + """Clear output caches and node identity path cache.""" + super().clear_cache() # clears _node_identity_path_cache + _update_modified_time + self._cached_output_stream = None + self._cached_output_table = None + # ------------------------------------------------------------------ # attach_databases # ------------------------------------------------------------------ @@ -160,8 +457,8 @@ def from_descriptor( operator: OperatorPodProtocol | None, input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], databases: dict[str, Any], - ) -> "OperatorNode": - """Construct an OperatorNode from a serialized descriptor. + ) -> "OperatorJobNode": + """Construct an OperatorJobNode from a serialized descriptor. When *operator* and *input_streams* are provided the node operates in full mode — constructed normally via ``__init__``. When @@ -179,19 +476,19 @@ def from_descriptor( to database instances. Returns: - A new ``OperatorNode`` instance. + A new ``OperatorJobNode`` instance. """ from orcapod.pipeline.serialization import LoadStatus if "table_scope" not in descriptor: raise ValueError( - f"OperatorNode descriptor is missing required 'table_scope' field: " + f"OperatorJobNode descriptor is missing required 'table_scope' field: " f"{descriptor.get('label', '')}" ) raw_table_scope = descriptor["table_scope"] if raw_table_scope not in ("pipeline_hash", "content_hash"): raise ValueError( - f"OperatorNode descriptor has invalid 'table_scope' value " + f"OperatorJobNode descriptor has invalid 'table_scope' value " f"{raw_table_scope!r} for {descriptor.get('label', '')}; " "expected one of ('pipeline_hash', 'content_hash')" ) @@ -245,10 +542,12 @@ def from_descriptor( # From TemporalMixin node._modified_time = None - # From OperatorNode + # From OperatorNodeBase node._operator = None node._input_streams = () node.tracker_manager = DEFAULT_TRACKER_MANAGER + + # From OperatorJobNode node._cached_output_stream = None node._cached_output_table = None @@ -278,131 +577,9 @@ def from_descriptor( return node # ------------------------------------------------------------------ - # load_status + # Internal helpers # ------------------------------------------------------------------ - @property - def load_status(self) -> Any: - """Return the load status of this node. - - Returns: - The ``LoadStatus`` enum value indicating how this node was - loaded. Defaults to ``FULL`` for nodes created via - ``__init__``. - """ - return self._load_status - - # ------------------------------------------------------------------ - # Identity - # ------------------------------------------------------------------ - - def identity_structure(self) -> Any: - return (self._operator, self._operator.argument_symmetry(self._input_streams)) - - def pipeline_identity_structure(self) -> Any: - return (self._operator, self._operator.argument_symmetry(self._input_streams)) - - # ------------------------------------------------------------------ - # Read-only overrides (for deserialized nodes without live operator) - # ------------------------------------------------------------------ - - def content_hash(self, hasher=None) -> ContentHash: - """Return the content hash, using stored value in read-only mode.""" - if self._operator is None and self._stored_content_hash is not None: - return ContentHash.from_string(self._stored_content_hash) - return super().content_hash(hasher) - - def pipeline_hash(self, hasher=None) -> ContentHash: - """Return the pipeline hash, using stored value in read-only mode.""" - if self._operator is None and self._stored_pipeline_hash is not None: - return ContentHash.from_string(self._stored_pipeline_hash) - return super().pipeline_hash(hasher) - - # ------------------------------------------------------------------ - # Stream interface - # ------------------------------------------------------------------ - - @property - def producer(self) -> OperatorPodProtocol: - return self._operator - - @property - def data_context(self) -> contexts.DataContext: - return contexts.resolve_context(self._operator.data_context_key) - - @property - def data_context_key(self) -> str: - return self._operator.data_context_key - - @property - def upstreams(self) -> tuple[StreamProtocol, ...]: - return self._input_streams - - @upstreams.setter - def upstreams(self, value: tuple[StreamProtocol, ...]) -> None: - self._input_streams = value - - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - if self._operator is None: - tag_keys = tuple(self._stored_schema.get("tag", {}).keys()) - data_keys = tuple(self._stored_schema.get("data", {}).keys()) - return tag_keys, data_keys - tag_schema, data_schema = self.output_schema( - columns=columns, all_info=all_info - ) - return tuple(tag_schema.keys()), tuple(data_schema.keys()) - - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return output schema, using stored value in read-only mode.""" - if self._operator is None: - tag = Schema(self._stored_schema.get("tag", {})) - data = Schema(self._stored_schema.get("data", {})) - return tag, data - return self._operator.output_schema( - *self._input_streams, - columns=columns, - all_info=all_info, - ) - - # ------------------------------------------------------------------ - # Node identity path - # ------------------------------------------------------------------ - - @property - def node_identity_path(self) -> tuple[str, ...]: - """Return the node identity path for observer contextualization. - - When ``table_scope="pipeline_hash"`` (default) the path is - ``operator.uri + (schema:{pipeline_hash},)`` — all runs that share - the same pipeline structure use one shared table, with per-run - disambiguation via the ``_node_content_hash`` row-level column. - - When ``table_scope="content_hash"`` the legacy path is returned: - ``operator.uri + (schema:{pipeline_hash}, instance:{content_hash})``. - - In read-only/UNAVAILABLE mode (no operator) the path stored from the - deserialized descriptor is returned (empty tuple when absent). - """ - if self._operator is None: - return self._stored_pipeline_path - if self._node_identity_path_cache is not None: - return self._node_identity_path_cache - path = self._operator.uri + (f"schema:{self.pipeline_hash().to_string()}",) - if self._table_scope != "pipeline_hash": - path += (f"instance:{self.content_hash().to_string()}",) - self._node_identity_path_cache = path - return path - def _filter_by_content_hash(self, table: pa.Table) -> pa.Table: """Filter *table* to rows whose ``NODE_CONTENT_HASH_COL`` matches this node. @@ -423,28 +600,10 @@ def _filter_by_content_hash(self, table: pa.Table) -> pa.Table: mask = pc.equal(table.column(col_name), own_hash) return table.filter(mask) - @property - def node_uri(self) -> tuple[str, ...]: - """Canonical URI tuple identifying this computation. - - Identical to ``operator.uri`` at runtime. - Returns stored value in read-only (deserialized) mode. - """ - if self._operator is None: - return self._stored_node_uri - return self._operator.uri - # ------------------------------------------------------------------ # Computation and caching # ------------------------------------------------------------------ - def clear_cache(self) -> None: - """Discard all in-memory cached state.""" - self._cached_output_stream = None - self._cached_output_table = None - self._node_identity_path_cache = None - self._update_modified_time() - def _store_output_stream(self, stream: StreamProtocol) -> None: """Materialize stream and store in the pipeline database with per-row dedup.""" output_table = stream.as_table( @@ -569,7 +728,7 @@ def get_cached_output(self) -> StreamProtocol | None: def execute( self, *input_streams: StreamProtocol, - observer: ExecutionObserverProtocol | None = None, + observer: "ExecutionObserverProtocol | None" = None, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute input streams: compute, persist, and cache. @@ -738,7 +897,7 @@ def get_all_records( self, columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, - ) -> pa.Table | None: + ) -> "pa.Table | None": """Retrieve all stored records from the pipeline database. Returns the stored output table with column filtering applied @@ -817,7 +976,7 @@ async def async_execute( inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, - observer: ExecutionObserverProtocol | None = None, + observer: "ExecutionObserverProtocol | None" = None, ) -> None: """Async execution with cache mode handling when DB is attached. @@ -895,9 +1054,3 @@ async def forward() -> None: ctx_obs.on_node_end(node_label, node_hash) finally: await output.close() - - def __repr__(self) -> str: - return ( - f"{type(self).__name__}(operator={self._operator!r}, " - f"upstreams={self._input_streams!r})" - ) diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 54fbe2ab3..480da0de8 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -12,6 +12,7 @@ OperatorNode, SourceNode, ) +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.tracker import AutoRegisteringContextBasedTracker from orcapod.protocols import core_protocols as cp from orcapod.protocols import database_protocols as dbp @@ -564,7 +565,7 @@ def load(cls, path: str | Path) -> "Pipeline": op_config.get("class_name"), exc, ) - node = OperatorNode.from_descriptor( + node = OperatorJobNode.from_descriptor( descriptor, operator=operator, input_streams=upstream_nodes, databases={} ) reconstructed[node_hash] = node diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 27173b82e..814c7168d 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -7,6 +7,7 @@ from orcapod.core.tracker import AutoRegisteringContextBasedTracker from orcapod.protocols import core_protocols as cp +from orcapod.types import CacheMode from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: @@ -461,6 +462,7 @@ def _build_execution_graph(self) -> "tuple[Any, list[str], Pipeline]": # Any = import networkx as nx from orcapod.core.nodes import FunctionNode, OperatorNode from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.nodes.source_node import SourceJobNode, SourceNode from orcapod.core.executors.local import LocalPythonFunctionExecutor @@ -569,14 +571,17 @@ def _build_execution_graph(self) -> "tuple[Any, list[str], Pipeline]": # Any = f"OperatorNode predecessor missing from exec_node_map: {missing}" ) upstream_nodes = tuple(exec_node_map[p] for p in preds) - new_op = OperatorNode( + # blueprint OperatorNode has no _cache_mode; default is OFF + op_cache_mode = getattr(template, "_cache_mode", None) or CacheMode.OFF + new_op = OperatorJobNode( operator=template._operator, input_streams=upstream_nodes, label=template._label, + table_scope=template._table_scope, ) new_op.attach_databases( pipeline_database=pipeline_db, - cache_mode=template._cache_mode, + cache_mode=op_cache_mode, ) exec_node_map[node_hash] = new_op diff --git a/tests/test_channels/test_node_async_execute.py b/tests/test_channels/test_node_async_execute.py index 61c954691..78b82bd30 100644 --- a/tests/test_channels/test_node_async_execute.py +++ b/tests/test_channels/test_node_async_execute.py @@ -25,6 +25,7 @@ OperatorNode, ) from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import SelectDataColumns from orcapod.core.operators.join import Join from orcapod.core.operators.semijoin import SemiJoin @@ -456,7 +457,7 @@ class TestOperatorNodeAsyncExecute: async def test_unary_op_delegation(self): stream = make_two_col_stream(3) op = SelectDataColumns(["x"]) - node = OperatorNode(op, [stream]) + node = OperatorJobNode(op, [stream]) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -483,7 +484,7 @@ async def test_binary_op_delegation(self): right = ArrowTableStream(right_table, tag_columns=["id"]) op = SemiJoin() - node = OperatorNode(op, [left, right]) + node = OperatorJobNode(op, [left, right]) left_ch = Channel(buffer_size=16) right_ch = Channel(buffer_size=16) @@ -516,7 +517,7 @@ async def test_nary_op_delegation(self): left = ArrowTableStream(left_table, tag_columns=["id"]) right = ArrowTableStream(right_table, tag_columns=["id"]) op = Join() - node = OperatorNode(op, [left, right]) + node = OperatorJobNode(op, [left, right]) left_ch = Channel(buffer_size=16) right_ch = Channel(buffer_size=16) @@ -541,13 +542,13 @@ async def test_results_match_sync(self): op = SelectDataColumns(["x"]) # Sync - node_sync = OperatorNode(op, [stream]) + node_sync = OperatorJobNode(op, [stream]) node_sync.run() sync_table = node_sync.as_table() sync_x = sorted(sync_table.column("x").to_pylist()) # Async - node_async = OperatorNode(op, [make_two_col_stream(4)]) + node_async = OperatorJobNode(op, [make_two_col_stream(4)]) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) @@ -570,7 +571,7 @@ async def test_off_mode_no_db_write(self): stream = make_two_col_stream(3) op = SelectDataColumns(["x"]) db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( op, [stream], pipeline_database=db, cache_mode=CacheMode.OFF ) @@ -592,7 +593,7 @@ async def test_log_mode_stores_results(self): stream = make_two_col_stream(3) op = SelectDataColumns(["x"]) db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( op, [stream], pipeline_database=db, cache_mode=CacheMode.LOG ) @@ -617,13 +618,13 @@ async def test_replay_mode_emits_from_db(self): db = InMemoryArrowDatabase() # First: sync LOG to populate DB - node1 = OperatorNode( + node1 = OperatorJobNode( op, [stream], pipeline_database=db, cache_mode=CacheMode.LOG ) node1.run() # Second: async REPLAY from DB - node2 = OperatorNode( + node2 = OperatorJobNode( op, [make_two_col_stream(3)], pipeline_database=db, @@ -648,7 +649,7 @@ async def test_replay_empty_db_returns_empty(self): op = SelectDataColumns(["x"]) db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( op, [stream], pipeline_database=db, @@ -760,7 +761,7 @@ async def test_source_to_operator_node_pipeline(self): """Source → OperatorNode (SelectDataColumns) async pipeline.""" stream = make_two_col_stream(3) op = SelectDataColumns(["x"]) - node = OperatorNode(op, [stream]) + node = OperatorJobNode(op, [stream]) ch1 = Channel(buffer_size=16) ch2 = Channel(buffer_size=16) @@ -852,7 +853,7 @@ async def test_persistent_operator_node_log_then_sync_db_retrieval(self): op = SelectDataColumns(["x"]) db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( op, [stream], pipeline_database=db, cache_mode=CacheMode.LOG ) @@ -883,7 +884,7 @@ async def source_producer(): assert "y" not in records.column_names # --- REPLAY from DB via a new node (no computation) --- - replay_node = OperatorNode( + replay_node = OperatorJobNode( op, [make_two_col_stream(4)], pipeline_database=db, @@ -925,7 +926,7 @@ def double(x: int) -> int: stage1_stream = ArrowTableStream(stage1_table, tag_columns=["id"]) op = SelectDataColumns(["result"]) op_db = InMemoryArrowDatabase() - op_node = OperatorNode( + op_node = OperatorJobNode( op, [stage1_stream], pipeline_database=op_db, cache_mode=CacheMode.LOG ) diff --git a/tests/test_core/nodes/test_node_execute.py b/tests/test_core/nodes/test_node_execute.py index 6b74d71e6..6949c3431 100644 --- a/tests/test_core/nodes/test_node_execute.py +++ b/tests/test_core/nodes/test_node_execute.py @@ -118,7 +118,7 @@ def test_caches_internally(self, function_node_with_db): # OperatorNode.execute() tests # ------------------------------------------------------------------ -from orcapod.core.nodes import OperatorNode +from orcapod.core.nodes.operator_node import OperatorJobNode as OperatorNode from orcapod.core.operators import SelectDataColumns from orcapod.types import CacheMode diff --git a/tests/test_core/nodes/test_operator_node_split.py b/tests/test_core/nodes/test_operator_node_split.py new file mode 100644 index 000000000..7bca166d9 --- /dev/null +++ b/tests/test_core/nodes/test_operator_node_split.py @@ -0,0 +1,116 @@ +"""Tests for the OperatorNode / OperatorJobNode split.""" +from __future__ import annotations + +import pytest + +from orcapod.errors import PipelineJobRequiredError +from orcapod.types import Schema + + +@pytest.fixture +def source_pair(): + from orcapod.core.nodes.source_node import SourceNode + + tag_schema = Schema({"id": int}) + data_schema_a = Schema({"a": float}) + data_schema_b = Schema({"b": float}) + node_a = SourceNode(name="src_a", tag_schema=tag_schema, data_schema=data_schema_a) + node_b = SourceNode(name="src_b", tag_schema=tag_schema, data_schema=data_schema_b) + return node_a, node_b + + +class TestThinOperatorNode: + def test_iter_data_raises_pipeline_job_required(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) + with pytest.raises(PipelineJobRequiredError): + list(op_node.iter_data()) + + def test_node_type(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) + assert op_node.node_type == "operator" + + def test_operator_node_does_not_accept_db_params(self, source_pair): + """OperatorNode must NOT accept pipeline_database param.""" + from orcapod.core.nodes.operator_node import OperatorNode + import inspect + + sig = inspect.signature(OperatorNode.__init__) + assert "pipeline_database" not in sig.parameters, ( + "OperatorNode must not accept pipeline_database — use OperatorJobNode instead" + ) + + def test_as_node_returns_self(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) + assert op_node.as_node() is op_node + + +class TestOperatorJobNodeHashParity: + def test_content_hash_matches_operator_node(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + thin = OperatorNode(operator=op, input_streams=(node_a, node_b)) + job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) + assert thin.content_hash() == job.content_hash() + + def test_pipeline_hash_matches_operator_node(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + thin = OperatorNode(operator=op, input_streams=(node_a, node_b)) + job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) + assert thin.pipeline_hash() == job.pipeline_hash() + + def test_as_node_returns_operator_node(self, source_pair): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.operators.join import Join + + op = Join() + node_a, node_b = source_pair + job = OperatorJobNode(operator=op, input_streams=(node_a, node_b)) + thin = job.as_node() + assert isinstance(thin, OperatorNode) + assert thin.content_hash() == job.content_hash() + + +class TestSiblingHierarchy: + """OperatorNode and OperatorJobNode must be siblings, not parent/child.""" + + def test_operator_node_not_subclass_of_operator_job_node(self): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + + assert not issubclass(OperatorNode, OperatorJobNode), ( + "OperatorNode must NOT inherit from OperatorJobNode" + ) + + def test_operator_job_node_not_subclass_of_operator_node(self): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + + assert not issubclass(OperatorJobNode, OperatorNode), ( + "OperatorJobNode must NOT inherit from OperatorNode" + ) + + def test_both_inherit_from_base(self): + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode, OperatorNodeBase + + assert issubclass(OperatorNode, OperatorNodeBase) + assert issubclass(OperatorJobNode, OperatorNodeBase) diff --git a/tests/test_core/operators/test_operator_node.py b/tests/test_core/operators/test_operator_node.py index df3cdeca4..e37830753 100644 --- a/tests/test_core/operators/test_operator_node.py +++ b/tests/test_core/operators/test_operator_node.py @@ -17,7 +17,7 @@ import pyarrow as pa import pytest -from orcapod.core.nodes import OperatorNode +from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode from orcapod.core.operators import ( DropDataColumns, Join, @@ -99,10 +99,10 @@ def _make_node( streams: tuple[ArrowTableStream, ...], db: InMemoryArrowDatabase | None = None, cache_mode: CacheMode = CacheMode.OFF, -) -> OperatorNode: +) -> OperatorJobNode: if db is None: db = InMemoryArrowDatabase() - return OperatorNode( + return OperatorJobNode( operator=operator, input_streams=streams, pipeline_database=db, @@ -442,4 +442,4 @@ def test_repr(self, simple_stream): op = MapData({"x": "renamed_x"}) node = _make_node(op, (simple_stream,)) r = repr(node) - assert "OperatorNode" in r + assert "OperatorJobNode" in r diff --git a/tests/test_core/operators/test_operator_node_attach_db.py b/tests/test_core/operators/test_operator_node_attach_db.py index d93c6d2e9..9c49079f0 100644 --- a/tests/test_core/operators/test_operator_node_attach_db.py +++ b/tests/test_core/operators/test_operator_node_attach_db.py @@ -1,11 +1,11 @@ -"""Tests for OperatorNode with optional database backing.""" +"""Tests for OperatorJobNode with optional database backing.""" from __future__ import annotations import pyarrow as pa import pytest -from orcapod.core.nodes import OperatorNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators.join import Join from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase @@ -23,32 +23,32 @@ def _make_stream(name="x", n=3): ) -class TestOperatorNodeWithoutDatabase: +class TestOperatorJobNodeWithoutDatabase: def test_construction_without_database(self): - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), ) assert node._pipeline_database is None def test_iter_data_without_database(self): - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), ) - node.run() # <-- add this line + node.run() results = list(node.iter_data()) assert len(results) == 3 def test_get_all_records_without_database_returns_none(self): - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), ) assert node.get_all_records() is None def test_as_source_without_database_raises(self): - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), ) @@ -56,9 +56,9 @@ def test_as_source_without_database_raises(self): node.as_source() -class TestOperatorNodeAttachDatabases: +class TestOperatorJobNodeAttachDatabases: def test_attach_databases_sets_pipeline_db(self): - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), ) @@ -67,7 +67,7 @@ def test_attach_databases_sets_pipeline_db(self): assert node._pipeline_database is db def test_attach_databases_computes_node_identity_path(self): - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), ) @@ -77,7 +77,7 @@ def test_attach_databases_computes_node_identity_path(self): assert len(node.node_identity_path) > 0 def test_attach_databases_clears_caches(self): - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), ) @@ -88,10 +88,10 @@ def test_attach_databases_clears_caches(self): assert node._cached_output_stream is None -class TestOperatorNodeWithDatabase: +class TestOperatorJobNodeWithDatabase: def test_construction_with_database(self): db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), pipeline_database=db, @@ -100,11 +100,11 @@ def test_construction_with_database(self): def test_iter_data_with_database(self): db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(_make_stream("a"), _make_stream("b")), pipeline_database=db, ) - node.run() # <-- add this line + node.run() results = list(node.iter_data()) assert len(results) == 3 diff --git a/tests/test_core/operators/test_operator_node_non_active.py b/tests/test_core/operators/test_operator_node_non_active.py index 593355a6d..c245b61b8 100644 --- a/tests/test_core/operators/test_operator_node_non_active.py +++ b/tests/test_core/operators/test_operator_node_non_active.py @@ -8,7 +8,7 @@ import pyarrow as pa import pytest -from orcapod.core.nodes import OperatorNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import MapData from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.databases import InMemoryArrowDatabase @@ -40,11 +40,11 @@ def map_op() -> MapData: def _node(operator, streams, *, db=None, cache_mode=CacheMode.OFF): - """Build an OperatorNode, attaching DB only when provided.""" + """Build an OperatorJobNode, attaching DB only when provided.""" kwargs: dict = dict(operator=operator, input_streams=streams, cache_mode=cache_mode) if db is not None: kwargs["pipeline_database"] = db - return OperatorNode(**kwargs) + return OperatorJobNode(**kwargs) # --------------------------------------------------------------------------- @@ -147,7 +147,7 @@ def test_pipeline_run_then_iterate(self, simple_source): class TestCacheModeNonActive: def test_iter_data_no_db_no_run_returns_empty(self, simple_source, map_op): """No DB, no run() → empty (step 3 fallback).""" - node = OperatorNode(operator=map_op, input_streams=(simple_source,)) + node = OperatorJobNode(operator=map_op, input_streams=(simple_source,)) assert list(node.iter_data()) == [] def test_iter_data_replay_mode_no_records_returns_empty( diff --git a/tests/test_core/test_caching_integration.py b/tests/test_core/test_caching_integration.py index 2c1c569b5..c88efd669 100644 --- a/tests/test_core/test_caching_integration.py +++ b/tests/test_core/test_caching_integration.py @@ -23,6 +23,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode, OperatorNode from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, DeltaTableSource, CachedSource @@ -422,7 +423,7 @@ def _make_joined_streams(self, clinic_a, source_db): def test_off_computes_without_db_writes(self, clinic_a, source_db, operator_db): patients, labs = self._make_joined_streams(clinic_a, source_db) - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=[patients, labs], pipeline_database=operator_db, @@ -434,7 +435,7 @@ def test_off_computes_without_db_writes(self, clinic_a, source_db, operator_db): def test_log_computes_and_writes(self, clinic_a, source_db, operator_db): patients, labs = self._make_joined_streams(clinic_a, source_db) - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=[patients, labs], pipeline_database=operator_db, @@ -450,7 +451,7 @@ def test_replay_loads_from_cache(self, clinic_a, source_db, operator_db): patients, labs = self._make_joined_streams(clinic_a, source_db) # First LOG to populate - log_node = OperatorNode( + log_node = OperatorJobNode( operator=Join(), input_streams=[patients, labs], pipeline_database=operator_db, @@ -459,7 +460,7 @@ def test_replay_loads_from_cache(self, clinic_a, source_db, operator_db): log_node.run() # Then REPLAY from cache - replay_node = OperatorNode( + replay_node = OperatorJobNode( operator=Join(), input_streams=[patients, labs], pipeline_database=operator_db, @@ -470,7 +471,7 @@ def test_replay_loads_from_cache(self, clinic_a, source_db, operator_db): def test_replay_empty_cache_returns_empty_stream(self, clinic_a, source_db): patients, labs = self._make_joined_streams(clinic_a, source_db) - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=[patients, labs], pipeline_database=InMemoryArrowDatabase(), @@ -509,13 +510,13 @@ def test_content_hash_scoping_isolates_source_combinations( cache_database=source_db, ) - node_a = OperatorNode( + node_a = OperatorJobNode( operator=Join(), input_streams=[pa_src, la_src], pipeline_database=operator_db, cache_mode=CacheMode.LOG, ) - node_b = OperatorNode( + node_b = OperatorJobNode( operator=Join(), input_streams=[pb_src, lb_src], pipeline_database=operator_db, @@ -563,9 +564,9 @@ def test_full_pipeline_source_to_function_to_operator( fn_node.run() assert fn_node.get_all_records().num_rows == 3 - # Step 3: OperatorNode (LOG) + # Step 3: OperatorJobNode (LOG) # Use fn_node output as input to an operator - op_node = OperatorNode( + op_node = OperatorJobNode( operator=Join(), input_streams=[patients, labs], pipeline_database=operator_db, @@ -575,7 +576,7 @@ def test_full_pipeline_source_to_function_to_operator( assert operator_db.get_all_records(op_node.node_identity_path).num_rows == 3 # Step 4: REPLAY from operator cache - op_replay = OperatorNode( + op_replay = OperatorJobNode( operator=Join(), input_streams=[patients, labs], pipeline_database=operator_db, diff --git a/tests/test_core/test_table_scope.py b/tests/test_core/test_table_scope.py index d71603e43..b52f5e606 100644 --- a/tests/test_core/test_table_scope.py +++ b/tests/test_core/test_table_scope.py @@ -15,6 +15,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode, OperatorNode from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, DictSource @@ -319,7 +320,7 @@ class TestOperatorNodePipelineHashScope: def test_default_scope_is_pipeline_hash(self): db = InMemoryArrowDatabase() src_a, src_b = _make_join_streams([1, 2], "x") - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(src_a, src_b), pipeline_database=db, @@ -329,7 +330,7 @@ def test_default_scope_is_pipeline_hash(self): def test_node_identity_path_ends_with_schema_only(self): db = InMemoryArrowDatabase() src_a, src_b = _make_join_streams([1, 2], "x") - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(src_a, src_b), pipeline_database=db, @@ -344,12 +345,12 @@ def test_two_nodes_same_operator_same_schema_share_path(self): # Same source_id_suffix → same source_id → same pipeline_hash src_a1, src_b1 = _make_join_streams([1, 2], "x") src_a2, src_b2 = _make_join_streams([3, 4], "x") - node1 = OperatorNode( + node1 = OperatorJobNode( operator=Join(), input_streams=(src_a1, src_b1), pipeline_database=db, ) - node2 = OperatorNode( + node2 = OperatorJobNode( operator=Join(), input_streams=(src_a2, src_b2), pipeline_database=db, @@ -358,16 +359,16 @@ def test_two_nodes_same_operator_same_schema_share_path(self): assert node1.pipeline_hash() == node2.pipeline_hash() def test_two_nodes_different_source_ids_have_different_content_hash(self): - """Different source_ids → different content_hash for OperatorNode.""" + """Different source_ids → different content_hash for OperatorJobNode.""" db = InMemoryArrowDatabase() src_a1, src_b1 = _make_join_streams([1, 2], "run1") src_a2, src_b2 = _make_join_streams([1, 2], "run2") - node1 = OperatorNode( + node1 = OperatorJobNode( operator=Join(), input_streams=(src_a1, src_b1), pipeline_database=db, ) - node2 = OperatorNode( + node2 = OperatorJobNode( operator=Join(), input_streams=(src_a2, src_b2), pipeline_database=db, @@ -375,17 +376,17 @@ def test_two_nodes_different_source_ids_have_different_content_hash(self): assert node1.content_hash() != node2.content_hash() def test_isolation_two_nodes_share_table_see_only_own_records(self): - """Two OperatorNodes sharing a DB path each see only their own records.""" + """Two OperatorJobNodes sharing a DB path each see only their own records.""" db = InMemoryArrowDatabase() src_a1, src_b1 = _make_join_streams([1, 2], "run1") src_a2, src_b2 = _make_join_streams([3, 4], "run2") - node1 = OperatorNode( + node1 = OperatorJobNode( operator=Join(), input_streams=(src_a1, src_b1), pipeline_database=db, cache_mode=CacheMode.LOG, ) - node2 = OperatorNode( + node2 = OperatorJobNode( operator=Join(), input_streams=(src_a2, src_b2), pipeline_database=db, @@ -421,7 +422,7 @@ class TestOperatorNodeContentHashScope: def test_node_identity_path_ends_with_schema_and_instance(self): db = InMemoryArrowDatabase() src_a, src_b = _make_join_streams([1, 2], "x") - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(src_a, src_b), pipeline_database=db, @@ -437,13 +438,13 @@ def test_two_nodes_different_source_ids_have_different_paths(self): db = InMemoryArrowDatabase() src_a1, src_b1 = _make_join_streams([1, 2], "run1") src_a2, src_b2 = _make_join_streams([1, 2], "run2") - node1 = OperatorNode( + node1 = OperatorJobNode( operator=Join(), input_streams=(src_a1, src_b1), pipeline_database=db, table_scope="content_hash", ) - node2 = OperatorNode( + node2 = OperatorJobNode( operator=Join(), input_streams=(src_a2, src_b2), pipeline_database=db, @@ -459,13 +460,13 @@ def test_pipeline_hash_same_across_content_hash_nodes_with_same_schema(self): # (ArrowTableSource hashes by schema/source_id, not raw data values) src_a1, src_b1 = _make_join_streams([1, 2], "x") src_a2, src_b2 = _make_join_streams([3, 4], "x") - node1 = OperatorNode( + node1 = OperatorJobNode( operator=Join(), input_streams=(src_a1, src_b1), pipeline_database=db, table_scope="content_hash", ) - node2 = OperatorNode( + node2 = OperatorJobNode( operator=Join(), input_streams=(src_a2, src_b2), pipeline_database=db, @@ -483,7 +484,7 @@ def test_pipeline_hash_same_across_content_hash_nodes_with_same_schema(self): class TestOperatorNodeDescriptorTableScope: def test_from_descriptor_missing_table_scope_raises(self): - from orcapod.core.nodes.operator_node import OperatorNode as ON + from orcapod.core.nodes.operator_node import OperatorJobNode as OJN db = InMemoryArrowDatabase() descriptor = { @@ -502,7 +503,7 @@ def test_from_descriptor_missing_table_scope_raises(self): "cache_mode": "OFF", } with pytest.raises(ValueError, match="table_scope"): - ON.from_descriptor( + OJN.from_descriptor( descriptor=descriptor, operator=None, input_streams=(), @@ -510,7 +511,7 @@ def test_from_descriptor_missing_table_scope_raises(self): ) def test_from_descriptor_preserves_pipeline_hash_scope(self): - from orcapod.core.nodes.operator_node import OperatorNode as ON + from orcapod.core.nodes.operator_node import OperatorJobNode as OJN db = InMemoryArrowDatabase() descriptor = { @@ -528,7 +529,7 @@ def test_from_descriptor_preserves_pipeline_hash_scope(self): }, "cache_mode": "OFF", } - loaded = ON.from_descriptor( + loaded = OJN.from_descriptor( descriptor=descriptor, operator=None, input_streams=(), @@ -537,7 +538,7 @@ def test_from_descriptor_preserves_pipeline_hash_scope(self): assert loaded._table_scope == "pipeline_hash" def test_from_descriptor_preserves_content_hash_scope(self): - from orcapod.core.nodes.operator_node import OperatorNode as ON + from orcapod.core.nodes.operator_node import OperatorJobNode as OJN db = InMemoryArrowDatabase() descriptor = { @@ -555,7 +556,7 @@ def test_from_descriptor_preserves_content_hash_scope(self): }, "cache_mode": "OFF", } - loaded = ON.from_descriptor( + loaded = OJN.from_descriptor( descriptor=descriptor, operator=None, input_streams=(), @@ -583,7 +584,7 @@ def test_function_node_cache_cleared_on_clear_cache(self): def test_operator_node_cache_cleared_on_clear_cache(self): db = InMemoryArrowDatabase() src_a, src_b = _make_join_streams([1], "x") - node = OperatorNode( + node = OperatorJobNode( operator=Join(), input_streams=(src_a, src_b), pipeline_database=db, diff --git a/tests/test_pipeline/test_node_descriptors.py b/tests/test_pipeline/test_node_descriptors.py index 7b61acafb..d52ec2e2c 100644 --- a/tests/test_pipeline/test_node_descriptors.py +++ b/tests/test_pipeline/test_node_descriptors.py @@ -165,7 +165,7 @@ def test_from_descriptor_read_only(self): assert loaded.load_status in (LoadStatus.READ_ONLY, LoadStatus.UNAVAILABLE) -from orcapod.core.nodes.operator_node import OperatorNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import Join @@ -191,7 +191,7 @@ def test_from_descriptor_read_only(self): "cache_mode": "OFF", "pipeline_path": ["test", "Join", "hash", "schema:fake_pipeline_hash", "instance:fake_content_hash"], } - loaded = OperatorNode.from_descriptor( + loaded = OperatorJobNode.from_descriptor( descriptor=descriptor, operator=None, input_streams=(), @@ -206,7 +206,7 @@ def test_from_descriptor_full_mode(self): source1 = DictSource(data=[{"a": 1, "b": 2}], tag_columns=["a"], source_id="s1") source2 = DictSource(data=[{"a": 1, "c": 3}], tag_columns=["a"], source_id="s2") op = Join() - node = OperatorNode( + node = OperatorJobNode( operator=op, input_streams=(source1, source2), pipeline_database=scoped_db, @@ -226,7 +226,7 @@ def test_from_descriptor_full_mode(self): "cache_mode": "OFF", "pipeline_path": list(node.node_identity_path), } - loaded = OperatorNode.from_descriptor( + loaded = OperatorJobNode.from_descriptor( descriptor=descriptor, operator=op, input_streams=(source1, source2), diff --git a/tests/test_pipeline/test_node_protocols.py b/tests/test_pipeline/test_node_protocols.py index 452b75cc8..543aec681 100644 --- a/tests/test_pipeline/test_node_protocols.py +++ b/tests/test_pipeline/test_node_protocols.py @@ -387,7 +387,7 @@ def create_data_logger(self, t, p, **kwargs): # OperatorNode.execute() with observer + cache check # =========================================================================== -from orcapod.core.nodes import OperatorNode +from orcapod.core.nodes.operator_node import OperatorJobNode as OperatorNode from orcapod.core.operators.join import Join diff --git a/tests/test_pipeline/test_orchestrator.py b/tests/test_pipeline/test_orchestrator.py index 134d43311..42443a84b 100644 --- a/tests/test_pipeline/test_orchestrator.py +++ b/tests/test_pipeline/test_orchestrator.py @@ -26,6 +26,7 @@ from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.operators import SelectDataColumns from orcapod.core.operators.join import Join @@ -116,7 +117,7 @@ class TestOperatorNodeAsyncExecute: async def test_delegates_to_operator(self): src = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) op = SelectDataColumns(columns=["value"]) - op_node = OperatorNode(op, input_streams=[src]) + op_node = OperatorJobNode(op, input_streams=[src]) input_ch = Channel(buffer_size=16) output_ch = Channel(buffer_size=16) diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index 555090902..ba2623c9d 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -24,6 +24,7 @@ SourceNode, ) from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction @@ -240,7 +241,7 @@ def test_exec_operator_nodes_have_pipeline_database_after_run(self, pipeline_db) result = job.run() exec_node = result.pipeline.compiled_nodes["joiner"] - assert isinstance(exec_node, OperatorNode) + assert isinstance(exec_node, OperatorJobNode) assert exec_node._pipeline_database is not None diff --git a/tests/test_pipeline/test_sync_orchestrator.py b/tests/test_pipeline/test_sync_orchestrator.py index 19b64ced8..bc0f7dea2 100644 --- a/tests/test_pipeline/test_sync_orchestrator.py +++ b/tests/test_pipeline/test_sync_orchestrator.py @@ -354,9 +354,9 @@ def test_materialized_stream_has_same_content_hash(self): """ src_a = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) src_b = _make_source("key", "score", {"key": ["a", "b"], "score": [100, 200]}) - from orcapod.core.nodes import OperatorNode + from orcapod.core.nodes.operator_node import OperatorJobNode - op_node = OperatorNode(Join(), input_streams=[src_a, src_b]) + op_node = OperatorJobNode(Join(), input_streams=[src_a, src_b]) op_node.run() buf = list(op_node.iter_data()) @@ -368,10 +368,10 @@ def test_materialized_stream_preserves_system_tags(self): src_a = _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) src_b = _make_source("key", "score", {"key": ["a", "b"], "score": [100, 200]}) from orcapod.core.operators.join import Join - from orcapod.core.nodes import OperatorNode + from orcapod.core.nodes.operator_node import OperatorJobNode op = Join() - op_node = OperatorNode(op, input_streams=[src_a, src_b]) + op_node = OperatorJobNode(op, input_streams=[src_a, src_b]) op_node.run() buf = list(op_node.iter_data()) From 2704d854ae491e5954d1cd78bee0e367ae095a49 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:33:24 +0000 Subject: [PATCH 09/24] refactor(pipeline): remove Pipeline.bind(); add PipelineJob.from_pipeline(), as_pipeline(), mutating bind() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove Pipeline.bind() — callers should use PipelineJob.from_pipeline() instead - Add PipelineJob.from_pipeline() classmethod that creates a runnable job from a compiled Pipeline - Make PipelineJob.bind() mutating (returns None instead of new object) - Add PipelineJob.as_pipeline() method that returns lightweight Pipeline blueprint from job - Add PipelineJob._distribute_databases() helper that wires DB references to job nodes - Handle OperatorJobNode/FunctionJobNode/SourceNodeBase in from_pipeline() for loaded pipelines - Update all call sites (tests) to use new API Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/pipeline/graph.py | 35 --- src/orcapod/pipeline/job.py | 300 +++++++++++++++++++--- tests/test_pipeline/test_pipeline.py | 8 +- tests/test_pipeline/test_pipeline_job.py | 166 ++++++++++-- tests/test_pipeline/test_serialization.py | 5 +- 5 files changed, 412 insertions(+), 102 deletions(-) diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 480da0de8..b963b60f8 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -15,13 +15,11 @@ from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.tracker import AutoRegisteringContextBasedTracker from orcapod.protocols import core_protocols as cp -from orcapod.protocols import database_protocols as dbp from orcapod.utils.lazy_module import LazyModule if TYPE_CHECKING: import networkx as nx from orcapod.pipeline.execution_context import ExecutionContext - from orcapod.pipeline.job import PipelineJob else: nx = LazyModule("networkx") @@ -307,39 +305,6 @@ def compile(self) -> None: self._compiled = True - # ------------------------------------------------------------------ - # Bind - # ------------------------------------------------------------------ - - def bind( - self, - sources: "dict[str, cp.StreamProtocol] | None" = None, - store: "dbp.ArrowDatabaseProtocol | None" = None, - execution_context: "ExecutionContext | None" = None, - ) -> "PipelineJob": - """Wrap this pipeline in a ``PipelineJob`` with the given bindings. - - Non-mutating — returns a fresh ``PipelineJob``; this ``Pipeline`` - is unchanged. - - Args: - sources: Mapping of SourceSpec name to concrete source. - store: Database for result caching and operator records. - execution_context: Optional execution configuration. - - Returns: - A new ``PipelineJob`` with this pipeline and the given bindings. - """ - from orcapod.pipeline.job import PipelineJob - - return PipelineJob( - name=self._name, - _pipeline=self, - sources=sources or {}, - store=store, - execution_context=execution_context, - ) - # ------------------------------------------------------------------ # Graph display # ------------------------------------------------------------------ diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 814c7168d..43426eb81 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -78,6 +78,10 @@ def __init__( self._has_run: bool = False self._run_id: str | None = None + # Job-node map (populated by from_pipeline(); None for with-block-created jobs) + self._persistent_node_map: "dict[str, Any] | None" = None + self._nodes: "dict[str, Any]" = {} + # ------------------------------------------------------------------ # Context manager — recording # ------------------------------------------------------------------ @@ -274,7 +278,134 @@ def execution_context(self) -> "ExecutionContext | None": return self._execution_context # ------------------------------------------------------------------ - # bind() — non-mutating + # from_pipeline() — classmethod constructor + # ------------------------------------------------------------------ + + @classmethod + def from_pipeline( + cls, + pipeline: "Pipeline", + store: "ArrowDatabaseProtocol | None" = None, + sources: "dict[str, cp.StreamProtocol] | None" = None, + execution_context: "ExecutionContext | None" = None, + ) -> "PipelineJob": + """Create a runnable ``PipelineJob`` from a compiled ``Pipeline``. + + Walks the pipeline's ``_persistent_node_map`` topologically and + creates corresponding ``JobNode`` variants: + + * ``SourceNode`` → ``SourceJobNode(name, schemas, concrete=sources.get(name))`` + * ``FunctionNode`` → ``FunctionJobNode(function_pod, upstream_job_node, label)`` + * ``OperatorNode`` → ``OperatorJobNode(operator, upstream_job_nodes, label)`` + + Args: + pipeline: A compiled ``Pipeline`` (``pipeline._compiled`` must be ``True``). + store: Database for result caching and operator records. + sources: Mapping of ``SourceNode.name`` → concrete source. + execution_context: Optional execution configuration. + + Returns: + A new ``PipelineJob`` ready to run (or ``bind()`` further). + + Raises: + ValueError: If *pipeline* has not been compiled. + """ + import networkx as _nx + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode, FunctionNodeBase + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode, OperatorNodeBase + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode, SourceNodeBase + + if not pipeline._compiled: + raise ValueError( + "Pipeline must be compiled before creating a PipelineJob from it. " + "Call pipeline.compile() or use auto_compile=True." + ) + + bound_sources: dict[str, cp.StreamProtocol] = dict(sources or {}) + + G = pipeline._hash_graph + job_node_map: dict[str, object] = {} + + for node_hash in _nx.topological_sort(G): + if node_hash not in pipeline._persistent_node_map: + continue + + node = pipeline._persistent_node_map[node_hash] + + if isinstance(node, SourceNodeBase): + # Handles both SourceNode (blueprint) and SourceJobNode (loaded pipeline) + concrete = bound_sources.get(node.name) + job_node = SourceJobNode( + name=node.name, + tag_schema=node.tag_schema, + data_schema=node.data_schema, + concrete=concrete, + ) + + elif isinstance(node, FunctionNodeBase): + # Handles both FunctionNode (blueprint) and FunctionJobNode (loaded pipeline) + original_input_hash = node._input_stream.content_hash().to_string() + upstream_job_node = job_node_map[original_input_hash] + job_node = FunctionJobNode( + function_pod=node._function_pod, + input_stream=upstream_job_node, + label=node._label, + table_scope=node._table_scope, + tracker_manager=node.tracker_manager, + ) + + elif isinstance(node, OperatorNodeBase): + # Handles both OperatorNode (blueprint) and OperatorJobNode (loaded pipeline) + upstream_job_nodes = tuple( + job_node_map[s.content_hash().to_string()] + for s in node._input_streams + ) + job_node = OperatorJobNode( + operator=node._operator, + input_streams=upstream_job_nodes, + label=node._label, + table_scope=node._table_scope, + tracker_manager=node.tracker_manager, + ) + + else: + raise TypeError( + f"Unknown node type in pipeline._persistent_node_map: {type(node)}" + ) + + job_node_map[node_hash] = job_node + + # Construct the PipelineJob using __new__ to bypass __init__ + job = cls.__new__(cls) + super(PipelineJob, job).__init__() + job._store = store + job._execution_context = execution_context + job._sources = bound_sources + job._pipeline_name = pipeline._name + job._has_run = False + job._run_id = None + job._unresolved_specs = [] + job._rec_graph_edges = [] + job._rec_upstreams = {} + job._rec_node_lut = {} + job._spec_by_name = {} + + job._compiled_pipeline = pipeline + job._persistent_node_map = job_node_map + job._nodes = {} + + for label, node in pipeline._nodes.items(): + node_hash = node.content_hash().to_string() + if node_hash in job_node_map: + job._nodes[label] = job_node_map[node_hash] + + if store is not None: + job._distribute_databases() + + return job + + # ------------------------------------------------------------------ + # bind() — mutating # ------------------------------------------------------------------ def bind( @@ -282,61 +413,160 @@ def bind( sources: "dict[str, cp.StreamProtocol] | None" = None, store: "ArrowDatabaseProtocol | None" = None, execution_context: "ExecutionContext | None" = None, - ) -> "PipelineJob": - """Return a new ``PipelineJob`` with updated bindings. + ) -> None: + """Update bindings in place. Returns ``None``. + + Mutating — modifies ``self`` directly. Existing bindings not mentioned + in this call are preserved. - Non-mutating — the original ``PipelineJob`` is unchanged. Existing - bindings not mentioned in this call are carried forward. + When *sources* is provided, each concrete source is validated against + its matching ``SourceNode`` slot schema, then the corresponding + ``SourceJobNode._concrete`` is updated in-place. - ``SourceNode.validate()`` is called for each source in *sources*; - ``SourceSpecMismatchError`` is raised on schema mismatch. + When *store* is provided and differs from the current store, + ``_distribute_databases()`` is called so that all job nodes receive + live DB references immediately. Args: - sources: Mapping of SourceNode name to concrete source. Each - source is validated against the matching SourceNode. - store: Replaces the current store. + sources: Mapping of ``SourceNode.name`` → concrete source. + store: Replaces the current store and triggers DB redistribution. execution_context: Replaces the current execution context. - Returns: - A new ``PipelineJob`` with merged bindings. - Raises: SourceSpecMismatchError: If any source's schema is incompatible. + ValueError: If a source key has no matching ``SourceNode`` slot. """ - from orcapod.core.nodes.source_node import SourceNode + from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + + store_changed = store is not None and store is not self._store + + if store is not None: + self._store = store - merged_sources = dict(self._sources) if sources is not None: - # Validate each supplied source against its SourceNode pipeline = self._compiled_pipeline if pipeline is not None: - for node in pipeline._persistent_node_map.values(): - if ( - isinstance(node, SourceNode) - and node.name in sources - ): - node.validate(sources[node.name]) - # Check that every provided key corresponds to a SourceNode leaf - node_names = { + spec_names = { node.name for node in pipeline._persistent_node_map.values() if isinstance(node, SourceNode) } - unknown = set(sources.keys()) - node_names + unknown = set(sources.keys()) - spec_names if unknown: raise ValueError( - f"bind() received source keys with no matching SourceNode in the pipeline: " - f"{sorted(unknown)}. Known node names: {sorted(node_names)}" + f"bind() received source keys with no matching SourceNode: " + f"{sorted(unknown)}. Known names: {sorted(spec_names)}" ) - merged_sources.update(sources) + for node in pipeline._persistent_node_map.values(): + if isinstance(node, SourceNode) and node.name in sources: + node.validate(sources[node.name]) - return PipelineJob( - name=self._pipeline_name, - store=store if store is not None else self._store, - execution_context=execution_context if execution_context is not None else self._execution_context, - _pipeline=self._compiled_pipeline, - sources=merged_sources, - ) + for job_node in (self._persistent_node_map or {}).values(): + if isinstance(job_node, SourceJobNode) and job_node.name in sources: + job_node._concrete = sources[job_node.name] + + self._sources.update(sources) + + if execution_context is not None: + self._execution_context = execution_context + + if store_changed: + self._distribute_databases() + + # ------------------------------------------------------------------ + # _distribute_databases() + # ------------------------------------------------------------------ + + def _distribute_databases(self) -> None: + """Wire live DB references to all FunctionJobNode and OperatorJobNode objects. + + Called by ``bind()`` when *store* is changed and by ``from_pipeline()`` + when *store* is provided at construction time. + + Raises: + RuntimeError: If ``_store`` is not set. + """ + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.types import CacheMode + + if self._store is None: + raise RuntimeError( + "Cannot distribute databases: no store is set. " + "Call bind(store=...) or from_pipeline(..., store=...) first." + ) + + pipeline = self._compiled_pipeline + pipeline_name = pipeline.name if pipeline is not None else self._pipeline_name + pipeline_db = self._store.at(*pipeline_name) + result_db = pipeline_db.at("_result") + + for node in (self._persistent_node_map or {}).values(): + if isinstance(node, FunctionJobNode): + node.attach_databases( + pipeline_database=pipeline_db, + result_database=result_db, + ) + elif isinstance(node, OperatorJobNode): + op_cache_mode = getattr(node, "_cache_mode", None) or CacheMode.OFF + node.attach_databases( + pipeline_database=pipeline_db, + cache_mode=op_cache_mode, + ) + + # ------------------------------------------------------------------ + # as_pipeline() + # ------------------------------------------------------------------ + + def as_pipeline(self) -> "Pipeline": + """Return the lightweight ``Pipeline`` blueprint for this job. + + Walks ``_persistent_node_map`` and calls ``.as_node()`` on each + ``JobNode`` to obtain the corresponding lightweight node. The returned + ``Pipeline`` has identical ``_persistent_node_map`` keys (content hashes) + as this job, but with lightweight blueprint nodes instead of job nodes. + + Returns: + A compiled ``Pipeline`` whose ``_persistent_node_map`` contains + only lightweight ``SourceNode`` / ``FunctionNode`` / ``OperatorNode`` + objects. + + Raises: + RuntimeError: If this job has no compiled pipeline. + """ + import networkx as _nx + from orcapod.pipeline.graph import Pipeline + + if self._compiled_pipeline is None: + raise RuntimeError( + "PipelineJob has no compiled pipeline. " + "Either use 'with job:' to record a DAG, " + "or create the job via PipelineJob.from_pipeline()." + ) + + G = self._compiled_pipeline._hash_graph + node_map: dict[str, object] = {} + + for node_hash in _nx.topological_sort(G): + if node_hash not in (self._persistent_node_map or {}): + continue + job_node = self._persistent_node_map[node_hash] + node_map[node_hash] = job_node.as_node() + + pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) + pipeline._graph_edges = list(self._compiled_pipeline._graph_edges) + pipeline._upstreams = dict(self._compiled_pipeline._upstreams) + pipeline._node_lut = dict(self._compiled_pipeline._node_lut) + pipeline._hash_graph = self._compiled_pipeline._hash_graph + pipeline._persistent_node_map = node_map + pipeline._nodes = { + label: node_map[node.content_hash().to_string()] + for label, node in self._compiled_pipeline._nodes.items() + if node.content_hash().to_string() in node_map + } + pipeline._compiled = True + + return pipeline # ------------------------------------------------------------------ # Completeness introspection diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index ba2623c9d..6d4ba9695 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -113,8 +113,8 @@ def test_pipeline_with_concrete_leaf_raises(self): with pipeline: Join()(src_a, src_b) - def test_pipeline_bind_returns_pipeline_job(self): - """Pipeline.bind() returns a PipelineJob without modifying the pipeline.""" + def test_pipeline_from_pipeline_returns_pipeline_job(self): + """PipelineJob.from_pipeline() returns a PipelineJob without modifying the pipeline.""" src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() @@ -126,10 +126,10 @@ def test_pipeline_bind_returns_pipeline_job(self): Join()(node_a, node_b) db = InMemoryArrowDatabase() - job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=db) + job = PipelineJob.from_pipeline(pipeline, sources={"a": src_a, "b": src_b}, store=db) assert isinstance(job, PipelineJob) - assert job.pipeline is pipeline + assert job._compiled_pipeline is pipeline assert job.store is db diff --git a/tests/test_pipeline/test_pipeline_job.py b/tests/test_pipeline/test_pipeline_job.py index 572f84bf4..040794dd5 100644 --- a/tests/test_pipeline/test_pipeline_job.py +++ b/tests/test_pipeline/test_pipeline_job.py @@ -121,8 +121,8 @@ def test_pipeline_extracted_after_with_block(self, store): class TestPipelineJobBind: - def test_bind_sources_returns_new_job(self, store): - """bind(sources=...) returns a new PipelineJob; original is unchanged.""" + def test_bind_sources_mutates_job(self, store): + """bind(sources=...) mutates the job in place and returns None.""" src_a, src_b = _make_two_sources() tag_a, data_a = src_a.output_schema() tag_b, data_b = src_b.output_schema() @@ -133,21 +133,21 @@ def test_bind_sources_returns_new_job(self, store): with job: Join()(node_a, node_b) - job2 = job.bind(sources={"a": src_a, "b": src_b}) - assert job2 is not job - assert job.sources == {} # original unchanged - assert "a" in job2.sources and "b" in job2.sources + result = job.bind(sources={"a": src_a, "b": src_b}) + assert result is None # bind() is now mutating and returns None + assert "a" in job.sources and "b" in job.sources - def test_bind_store_returns_new_job(self, store): + def test_bind_store_mutates_job(self, store): + """bind(store=...) updates the store in place and returns None.""" src_a, src_b = _make_two_sources() job = PipelineJob() with job: Join()(src_a, src_b) new_store = InMemoryArrowDatabase() - job2 = job.bind(store=new_store) - assert job2.store is new_store - assert job.store is None # original unchanged + result = job.bind(store=new_store) + assert result is None # bind() returns None + assert job.store is new_store def test_bind_preserves_existing_sources(self, store): """bind(sources=...) merges new sources with existing ones.""" @@ -161,11 +161,11 @@ def test_bind_preserves_existing_sources(self, store): with job: Join()(node_a, node_b) - job2 = job.bind(sources={"a": src_a}) - job3 = job2.bind(sources={"b": src_b}) + job.bind(sources={"a": src_a}) + job.bind(sources={"b": src_b}) - assert "a" in job3.sources - assert "b" in job3.sources + assert "a" in job.sources + assert "b" in job.sources def test_bind_validates_schema_at_bind_time(self, store): """bind() raises SourceSpecMismatchError for incompatible sources.""" @@ -181,8 +181,8 @@ def test_bind_validates_schema_at_bind_time(self, store): with pytest.raises(SourceSpecMismatchError): job.bind(sources={"a": src_a}) - def test_pipeline_bind_wraps_in_job(self, store): - """Pipeline.bind() returns a PipelineJob holding that pipeline.""" + def test_from_pipeline_wraps_in_job(self, store): + """PipelineJob.from_pipeline() returns a PipelineJob holding that pipeline.""" from orcapod.pipeline.graph import Pipeline src_a, src_b = _make_two_sources() @@ -195,12 +195,12 @@ def test_pipeline_bind_wraps_in_job(self, store): with pipeline: Join()(node_a, node_b) - job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=store) + job = PipelineJob.from_pipeline(pipeline, sources={"a": src_a, "b": src_b}, store=store) assert isinstance(job, PipelineJob) - assert job.pipeline is pipeline + assert job._compiled_pipeline is pipeline - def test_pipeline_bind_propagates_name(self, store): - """Pipeline.bind() must propagate the pipeline's name to the job.""" + def test_from_pipeline_propagates_name(self, store): + """PipelineJob.from_pipeline() must propagate the pipeline's name to the job.""" from orcapod.pipeline.graph import Pipeline src_a, src_b = _make_two_sources() @@ -213,9 +213,9 @@ def test_pipeline_bind_propagates_name(self, store): with pipeline: Join()(node_a, node_b) - job = pipeline.bind(sources={"a": src_a, "b": src_b}, store=store) + job = PipelineJob.from_pipeline(pipeline, sources={"a": src_a, "b": src_b}, store=store) assert job._pipeline_name == ("my_pipeline",), ( - "PipelineJob._pipeline_name should match Pipeline.name after bind()" + "PipelineJob._pipeline_name should match Pipeline.name after from_pipeline()" ) @@ -400,7 +400,8 @@ def test_run_does_not_mutate_blueprint_nodes(self, store): The pipeline blueprint is a shared, reusable object. Running one job must not replace blueprint template nodes with live exec nodes — that would break - subsequent jobs (and ``Pipeline.bind()`` callers) that share the same pipeline. + subsequent jobs (and ``PipelineJob.from_pipeline()`` callers) that share the + same pipeline. """ src_a, src_b = _make_two_sources() pf = PythonDataFunction(add_values, output_keys="total") @@ -482,8 +483,8 @@ def test_end_to_end_source_join_function(self, store): totals = sorted(cast(list[int], table.column("total").to_pylist())) assert totals == [110, 220] - def test_bind_then_run(self, store): - """Pipeline.bind() + job.run() produces correct results.""" + def test_from_pipeline_then_run(self, store): + """PipelineJob.from_pipeline() + job.run() produces correct results.""" from orcapod.pipeline.graph import Pipeline src_a, src_b = _make_two_sources() @@ -499,7 +500,8 @@ def test_bind_then_run(self, store): joined = Join()(node_a, node_b) pod(joined, label="adder") - job = pipeline.bind( + job = PipelineJob.from_pipeline( + pipeline, sources={"src_a": src_a, "src_b": src_b}, store=store, ) @@ -657,3 +659,115 @@ def test_load_after_partial_run_restores_unresolved_specs(self, store, tmp_path) result.save(str(path)) loaded = PipelineJob.load(str(path), store=store) assert "unbound_b" in loaded.unresolved_specs + + +# --------------------------------------------------------------------------- +# Tests: PipelineJob.from_pipeline() +# --------------------------------------------------------------------------- + + +@pytest.fixture +def compiled_pipeline(): + """A compiled Pipeline with SourceNode leaves (no concrete sources).""" + from orcapod.pipeline.graph import Pipeline + + src_a, src_b = _make_two_sources() + tag_a, data_a = src_a.output_schema() + tag_b, data_b = src_b.output_schema() + node_a = SourceNode(name="slot_a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="slot_b", tag_schema=tag_b, data_schema=data_b) + + pipeline = Pipeline(name="test_pipeline") + with pipeline: + Join()(node_a, node_b, label="joiner") + return pipeline + + +@pytest.fixture +def db(): + """An in-memory database.""" + return InMemoryArrowDatabase() + + +@pytest.fixture +def source_a(): + """Concrete source matching slot_a schema (key tag, value data).""" + return _make_source("key", "value", {"key": ["a", "b"], "value": [10, 20]}) + + +@pytest.fixture +def source_b(): + """Concrete source matching slot_b schema (key tag, score data).""" + return _make_source("key", "score", {"key": ["a", "b"], "score": [100, 200]}) + + +@pytest.fixture +def pipeline_job(compiled_pipeline): + """A PipelineJob with a compiled pipeline but not fully bound.""" + return PipelineJob.from_pipeline(compiled_pipeline) + + +@pytest.fixture +def pipeline_job_complete(compiled_pipeline, db, source_a, source_b): + """A PipelineJob with store and all sources bound.""" + return PipelineJob.from_pipeline( + compiled_pipeline, + store=db, + sources={"slot_a": source_a, "slot_b": source_b}, + ) + + +class TestFromPipeline: + """PipelineJob.from_pipeline() creates a runnable job from a compiled Pipeline.""" + + def test_from_pipeline_creates_pipeline_job(self, compiled_pipeline, db): + """from_pipeline returns a PipelineJob with the same topology.""" + job = PipelineJob.from_pipeline(compiled_pipeline, store=db) + assert isinstance(job, PipelineJob) + + def test_from_pipeline_with_sources_binds_them(self, compiled_pipeline, db, source_a): + """Sources passed to from_pipeline are immediately bound.""" + job = PipelineJob.from_pipeline( + compiled_pipeline, store=db, sources={"slot_a": source_a} + ) + assert "slot_a" in job._sources + + def test_pipeline_bind_removed(self, compiled_pipeline): + """Pipeline.bind() no longer exists.""" + assert not hasattr(compiled_pipeline, "bind"), ( + "Pipeline.bind() must be removed — use PipelineJob.from_pipeline() instead" + ) + + +class TestMutatingBind: + """PipelineJob.bind() mutates in place and returns None.""" + + def test_bind_returns_none(self, pipeline_job, source_a): + result = pipeline_job.bind(sources={"slot_a": source_a}) + assert result is None + + def test_bind_mutates_sources(self, pipeline_job, source_a): + pipeline_job.bind(sources={"slot_a": source_a}) + assert "slot_a" in pipeline_job._sources + + def test_bind_mutates_store(self, pipeline_job, db): + pipeline_job.bind(store=db) + assert pipeline_job._store is db + + +class TestAsPipeline: + """PipelineJob.as_pipeline() returns a lightweight Pipeline.""" + + def test_as_pipeline_returns_pipeline(self, pipeline_job_complete): + from orcapod.pipeline.graph import Pipeline + + pipeline = pipeline_job_complete.as_pipeline() + assert isinstance(pipeline, Pipeline) + + def test_as_pipeline_node_hashes_match(self, pipeline_job_complete): + """as_pipeline() blueprint nodes have matching hashes to job nodes.""" + job = pipeline_job_complete + pipeline = job.as_pipeline() + + for node_hash in job._persistent_node_map: + assert node_hash in pipeline._persistent_node_map diff --git a/tests/test_pipeline/test_serialization.py b/tests/test_pipeline/test_serialization.py index fbefa9058..637e82c94 100644 --- a/tests/test_pipeline/test_serialization.py +++ b/tests/test_pipeline/test_serialization.py @@ -13,6 +13,7 @@ from orcapod.core.sources import ArrowTableSource from orcapod.databases.in_memory_databases import InMemoryArrowDatabase from orcapod.pipeline import Pipeline +from orcapod.pipeline.job import PipelineJob from orcapod.pipeline.serialization import PIPELINE_FORMAT_VERSION @@ -133,7 +134,7 @@ def test_load_version_mismatch_raises(self, spec_pipeline, tmp_path): Pipeline.load(str(path)) def test_load_bindable_and_runnable(self, spec_pipeline): - """Loaded pipeline can be bound to concrete sources and run.""" + """Loaded pipeline can be used to create a runnable PipelineJob and run.""" pipeline, tmp_path = spec_pipeline path = tmp_path / "pipeline.json" pipeline.save(str(path)) @@ -157,7 +158,7 @@ def test_load_bindable_and_runnable(self, spec_pipeline): infer_nullable=True, ) store = InMemoryArrowDatabase() - job = loaded.bind(sources={"source_a": src_a, "source_b": src_b}, store=store) + job = PipelineJob.from_pipeline(loaded, sources={"source_a": src_a, "source_b": src_b}, store=store) completed = job.run() assert completed._has_run is True # Verify all source specs were resolved (no unresolved specs) From 3153b481a92af2a50a5b4c014070802d4231db73 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:36:39 +0000 Subject: [PATCH 10/24] docs(pipeline): update docstrings to reference PipelineJob.from_pipeline() instead of removed Pipeline.bind() --- src/orcapod/pipeline/graph.py | 8 ++++---- src/orcapod/pipeline/job.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index b963b60f8..089ee08a0 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -42,8 +42,8 @@ class Pipeline(AutoRegisteringContextBasedTracker): - Function pod invocations → ``FunctionNode`` - Operator invocations → ``OperatorNode`` - All leaf inputs must be ``SourceSpec`` instances. To run a ``Pipeline``, - use ``Pipeline.bind(sources=..., store=...)`` to create a ``PipelineJob``. + All leaf inputs must be ``SourceNode`` instances. To run a ``Pipeline``, + use ``PipelineJob.from_pipeline(pipeline, sources=..., store=...)`` to create a ``PipelineJob``. Parameters: name: Pipeline name (string or tuple). Used as the path prefix for @@ -418,8 +418,8 @@ def load(cls, path: str | Path) -> "Pipeline": """Deserialize a pure pipeline blueprint from a JSON file. Reconstructs topology and SourceSpec declarations. The loaded - pipeline is topology-only — to run it, call - ``pipeline.bind(sources=..., store=...)`` first. + pipeline is topology-only — to run it, use + ``PipelineJob.from_pipeline(pipeline, sources=..., store=...)``. Args: path: Path to the JSON file produced by :meth:`save`. diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 43426eb81..05bfc1c2b 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -37,7 +37,7 @@ class PipelineJob(AutoRegisteringContextBasedTracker): resolvable subgraph — nodes whose upstream SourceNodes are all bound. ``PipelineJob`` can also be created from a ``Pipeline`` via - ``pipeline.bind(sources=..., store=...)`` for the "explicit blueprint" + ``PipelineJob.from_pipeline(pipeline, sources=..., store=...)`` for the "explicit blueprint" workflow. Args: @@ -47,8 +47,8 @@ class PipelineJob(AutoRegisteringContextBasedTracker): store: Database for result caching and operator records. execution_context: Optional execution configuration. tracker_manager: Optional tracker manager override. - _pipeline: Internal — pre-built pipeline (used by Pipeline.bind()). - sources: Internal — pre-bound sources (used by Pipeline.bind() / + _pipeline: Internal — pre-built pipeline (used by PipelineJob.from_pipeline()). + sources: Internal — pre-bound sources (used by PipelineJob.from_pipeline() / bind()). """ From 694d72d0e4ef29037d9096510ac2b1ae2bff1f62 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:44:26 +0000 Subject: [PATCH 11/24] refactor(pipeline): extract AbstractPipelineBase with shared recording mechanism Introduces AbstractPipelineBase in src/orcapod/pipeline/base.py as a shared ABC inheriting from AutoRegisteringContextBasedTracker. Pipeline and PipelineJob now inherit from it, de-duplicating _name, _node_lut, _upstreams, _graph_edges, _hash_graph, _persistent_node_map, _nodes, _node_graph, _compiled initialization plus reset(), graph property, compiled_nodes property, __getattr__(), and __exit__() with compile() dispatch. PipelineJob._pipeline_name unified into inherited _name. Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/pipeline/base.py | 129 +++++++++++++++++++++++ src/orcapod/pipeline/graph.py | 66 ++---------- src/orcapod/pipeline/job.py | 25 ++--- tests/test_pipeline/test_pipeline.py | 2 +- tests/test_pipeline/test_pipeline_job.py | 10 +- 5 files changed, 155 insertions(+), 77 deletions(-) create mode 100644 src/orcapod/pipeline/base.py diff --git a/src/orcapod/pipeline/base.py b/src/orcapod/pipeline/base.py new file mode 100644 index 000000000..ea27206df --- /dev/null +++ b/src/orcapod/pipeline/base.py @@ -0,0 +1,129 @@ +"""AbstractPipelineBase — shared recording mechanism for Pipeline and PipelineJob.""" +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +from orcapod.core.tracker import AutoRegisteringContextBasedTracker +from orcapod.protocols import core_protocols as cp +from orcapod.utils.lazy_module import LazyModule + +if TYPE_CHECKING: + import networkx as nx +else: + nx = LazyModule("networkx") + +logger = logging.getLogger(__name__) + + +class AbstractPipelineBase(AutoRegisteringContextBasedTracker, ABC): + """Shared recording mechanism and graph state for Pipeline and PipelineJob. + + Manages the ``with``-block recording phase: accumulating graph edges, + node LUT entries, and upstream stream references. Subclasses specialise + which node types are created (blueprint vs. job nodes) and compile them + into executable graphs. + + Args: + name: Pipeline name (string or tuple). Used to scope database paths. + tracker_manager: Optional tracker manager override. + """ + + def __init__( + self, + name: str | tuple[str, ...] = "pipeline", + tracker_manager: "cp.TrackerManagerProtocol | None" = None, + ) -> None: + super().__init__(tracker_manager=tracker_manager) + self._name: tuple[str, ...] = (name,) if isinstance(name, str) else tuple(name) + self._node_lut: dict[str, Any] = {} + self._upstreams: dict[str, Any] = {} + self._graph_edges: list[tuple[str, str]] = [] + self._hash_graph: "nx.DiGraph" = nx.DiGraph() + self._persistent_node_map: dict[str, Any] = {} + self._nodes: dict[str, Any] = {} + self._node_graph: "nx.DiGraph | None" = None + self._compiled: bool = False + + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ + + @property + def name(self) -> tuple[str, ...]: + """Pipeline name tuple.""" + return self._name + + @property + def graph(self) -> "nx.DiGraph": + """Directed hash graph of accumulated pipeline structure.""" + return self._hash_graph + + @property + def compiled_nodes(self) -> dict[str, Any]: + """Copy of the compiled nodes dict (label to node).""" + return self._nodes.copy() + + # ------------------------------------------------------------------ + # Recording helpers + # ------------------------------------------------------------------ + + def reset(self) -> None: + """Clear session-scoped recorded state (node LUT, upstreams, edge list). + + Note: + ``_hash_graph`` and ``_persistent_node_map`` are intentionally + *not* cleared -- they accumulate across ``with`` blocks. + """ + self._node_lut.clear() + self._upstreams.clear() + self._graph_edges.clear() + + def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: + """Exit the recording context, compiling if no exception occurred.""" + super().__exit__(exc_type, exc_value, traceback) + if exc_type is None: + self.compile() + + def __getattr__(self, item: str) -> Any: + """Look up compiled nodes by label as attribute access.""" + if item.startswith("_"): + raise AttributeError(item) + # Use object.__getattribute__ to avoid recursion + nodes = object.__getattribute__(self, "_nodes") + if item in nodes: + return nodes[item] + raise AttributeError( + f"{type(self).__name__!r} has no attribute {item!r}. " + f"Available node labels: {sorted(nodes.keys())}" + ) + + # ------------------------------------------------------------------ + # Abstract -- specialised per subclass + # ------------------------------------------------------------------ + + @abstractmethod + def record_function_pod_invocation( + self, + pod: "cp.FunctionPodProtocol", + input_stream: "cp.StreamProtocol", + label: str | None = None, + ) -> None: + """Record a function pod invocation into the graph.""" + ... + + @abstractmethod + def record_operator_pod_invocation( + self, + pod: "cp.OperatorPodProtocol", + upstreams: "tuple[cp.StreamProtocol, ...]" = (), + label: str | None = None, + ) -> None: + """Record an operator pod invocation into the graph.""" + ... + + @abstractmethod + def compile(self) -> None: + """Compile recorded invocations into a frozen DAG.""" + ... diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 089ee08a0..9419b652c 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -14,6 +14,7 @@ ) from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.tracker import AutoRegisteringContextBasedTracker +from orcapod.pipeline.base import AbstractPipelineBase from orcapod.protocols import core_protocols as cp from orcapod.utils.lazy_module import LazyModule @@ -31,7 +32,7 @@ # --------------------------------------------------------------------------- -class Pipeline(AutoRegisteringContextBasedTracker): +class Pipeline(AbstractPipelineBase): """A pure computational blueprint recording operator and function pod invocations. During the ``with`` block, operator and function pod invocations are @@ -69,17 +70,8 @@ def __init__( auto_compile: If ``True`` (default), ``compile()`` is called automatically when the context manager exits. """ - super().__init__(tracker_manager=tracker_manager) - self._node_lut: dict[str, GraphNode] = {} - self._upstreams: dict[str, cp.StreamProtocol] = {} - self._graph_edges: list[tuple[str, str]] = [] - self._hash_graph: "nx.DiGraph" = nx.DiGraph() - self._name = (name,) if isinstance(name, str) else tuple(name) - self._nodes: dict[str, GraphNode] = {} - self._persistent_node_map: dict[str, GraphNode] = {} - self._node_graph: "nx.DiGraph | None" = None + super().__init__(name=name, tracker_manager=tracker_manager) self._auto_compile = auto_compile - self._compiled = False # ------------------------------------------------------------------ # Recording (TrackerProtocol) @@ -131,48 +123,15 @@ def nodes(self) -> list[GraphNode]: """Return the list of recorded (non-persistent) nodes.""" return list(self._node_lut.values()) - @property - def graph(self) -> "nx.DiGraph": - """Directed graph of content-hash strings representing the accumulated - pipeline structure. Vertices are ``content_hash`` strings; node - attributes include ``node_type`` ("source" / "function" / "operator") - and, after ``compile()``, ``label`` and ``pipeline_hash``. - - The graph accumulates across multiple ``with`` blocks and is never - cleared by ``reset()``. - """ - return self._hash_graph - - def reset(self) -> None: - """Clear session-scoped recorded state (node LUT, upstreams, edge list). - - Note: ``_hash_graph`` is intentionally *not* cleared -- it accumulates - the pipeline structure across ``with`` blocks. - """ - self._node_lut.clear() - self._upstreams.clear() - self._graph_edges.clear() - - # ------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------ - - @property - def name(self) -> tuple[str, ...]: - return self._name - - @property - def compiled_nodes(self) -> dict[str, GraphNode]: - """Return a copy of the compiled nodes dict.""" - return self._nodes.copy() - # ------------------------------------------------------------------ # Context manager # ------------------------------------------------------------------ def __exit__(self, exc_type=None, exc_value=None, traceback=None): - super().__exit__(exc_type, exc_value, traceback) - if self._auto_compile: + # Call AutoRegisteringContextBasedTracker.__exit__ directly (deactivates the tracker) + # but NOT AbstractPipelineBase.__exit__ (which calls compile() unconditionally). + AutoRegisteringContextBasedTracker.__exit__(self, exc_type, exc_value, traceback) + if exc_type is None and self._auto_compile: self.compile() # ------------------------------------------------------------------ @@ -625,17 +584,6 @@ def _clone_for_execution(self) -> "Pipeline": clone._nodes = dict(self._nodes) return clone - # ------------------------------------------------------------------ - # Node access by label - # ------------------------------------------------------------------ - - def __getattr__(self, item: str) -> Any: - # Use __dict__ to avoid recursion during __init__ - nodes = self.__dict__.get("_nodes", {}) - if item in nodes: - return nodes[item] - raise AttributeError(f"Pipeline has no attribute '{item}'") - def __dir__(self) -> list[str]: return list(super().__dir__()) + list(self._nodes.keys()) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 05bfc1c2b..7ca92e00e 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any from orcapod.core.tracker import AutoRegisteringContextBasedTracker +from orcapod.pipeline.base import AbstractPipelineBase from orcapod.protocols import core_protocols as cp from orcapod.types import CacheMode from orcapod.utils.lazy_module import LazyModule @@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) -class PipelineJob(AutoRegisteringContextBasedTracker): +class PipelineJob(AbstractPipelineBase): """Pipeline + source bindings + execution context. ``PipelineJob`` is the everyday working object. It is built incrementally: @@ -62,7 +63,7 @@ def __init__( _pipeline: "Pipeline | None" = None, sources: "dict[str, cp.StreamProtocol] | None" = None, ) -> None: - super().__init__(tracker_manager=tracker_manager) + super().__init__(name=name, tracker_manager=tracker_manager) self._store = store self._execution_context = execution_context self._compiled_pipeline: "Pipeline | None" = _pipeline @@ -73,12 +74,13 @@ def __init__( self._rec_upstreams: dict[str, cp.StreamProtocol] = {} self._rec_node_lut: dict[str, "GraphNode"] = {} self._spec_by_name: dict[str, "SourceNode"] = {} - self._pipeline_name: tuple[str, ...] = (name,) if isinstance(name, str) else tuple(name) self._unresolved_specs: list[str] = [] self._has_run: bool = False self._run_id: str | None = None # Job-node map (populated by from_pipeline(); None for with-block-created jobs) + # Note: _persistent_node_map and _nodes are initialized by AbstractPipelineBase.__init__ + # but overridden here for PipelineJob-specific types. self._persistent_node_map: "dict[str, Any] | None" = None self._nodes: "dict[str, Any]" = {} @@ -94,16 +96,15 @@ def __enter__(self) -> "PipelineJob": self._spec_by_name = {} return super().__enter__() # type: ignore[return-value] - def __exit__(self, exc_type=None, exc_value=None, traceback=None) -> None: - super().__exit__(exc_type, exc_value, traceback) - if exc_type is None: - self._compile_from_recording() + def compile(self) -> None: + """Compile recorded invocations into a Pipeline (implements AbstractPipelineBase.compile).""" + self._compile_from_recording() def _compile_from_recording(self) -> None: """Compile the recorded edges into a pure Pipeline.""" from orcapod.pipeline.graph import Pipeline - pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) + pipeline = Pipeline(name=self._name, auto_compile=False) # Inject the recording state into the pipeline pipeline._graph_edges = list(self._rec_graph_edges) pipeline._upstreams = dict(self._rec_upstreams) @@ -381,7 +382,7 @@ def from_pipeline( job._store = store job._execution_context = execution_context job._sources = bound_sources - job._pipeline_name = pipeline._name + job._name = pipeline._name job._has_run = False job._run_id = None job._unresolved_specs = [] @@ -497,7 +498,7 @@ def _distribute_databases(self) -> None: ) pipeline = self._compiled_pipeline - pipeline_name = pipeline.name if pipeline is not None else self._pipeline_name + pipeline_name = pipeline.name if pipeline is not None else self._name pipeline_db = self._store.at(*pipeline_name) result_db = pipeline_db.at("_result") @@ -553,7 +554,7 @@ def as_pipeline(self) -> "Pipeline": job_node = self._persistent_node_map[node_hash] node_map[node_hash] = job_node.as_node() - pipeline = Pipeline(name=self._pipeline_name, auto_compile=False) + pipeline = Pipeline(name=self._name, auto_compile=False) pipeline._graph_edges = list(self._compiled_pipeline._graph_edges) pipeline._upstreams = dict(self._compiled_pipeline._upstreams) pipeline._node_lut = dict(self._compiled_pipeline._node_lut) @@ -913,7 +914,7 @@ def run( # Return new job (different object); uses exec_pipeline (clone with exec nodes) result = PipelineJob( - name=self._pipeline_name, + name=self._name, store=self._store, execution_context=self._execution_context, _pipeline=exec_pipeline, # exec_pipeline (clone), not self._compiled_pipeline diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index 6d4ba9695..8c7e0ea1d 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -321,7 +321,7 @@ def test_getattr_raises_for_unknown(self, pipeline_db): with job: pass # empty pipeline - with pytest.raises(AttributeError, match="Pipeline has no attribute"): + with pytest.raises(AttributeError, match="has no attribute"): _ = job.pipeline.nonexistent def test_dir_includes_node_labels(self, pipeline_db): diff --git a/tests/test_pipeline/test_pipeline_job.py b/tests/test_pipeline/test_pipeline_job.py index 040794dd5..c8e266ea4 100644 --- a/tests/test_pipeline/test_pipeline_job.py +++ b/tests/test_pipeline/test_pipeline_job.py @@ -214,8 +214,8 @@ def test_from_pipeline_propagates_name(self, store): Join()(node_a, node_b) job = PipelineJob.from_pipeline(pipeline, sources={"a": src_a, "b": src_b}, store=store) - assert job._pipeline_name == ("my_pipeline",), ( - "PipelineJob._pipeline_name should match Pipeline.name after from_pipeline()" + assert job._name == ("my_pipeline",), ( + "PipelineJob._name should match Pipeline.name after from_pipeline()" ) @@ -624,7 +624,7 @@ def test_load_after_run_restores_has_run_true(self, store, tmp_path): assert loaded._has_run is True def test_load_restores_pipeline_name(self, store, tmp_path): - """PipelineJob.load() must restore _pipeline_name from the blueprint.""" + """PipelineJob.load() must restore _name from the blueprint.""" src_a, src_b = _make_two_sources() pf = PythonDataFunction(add_values, output_keys="total") pod = FunctionPod(data_function=pf) @@ -638,8 +638,8 @@ def test_load_restores_pipeline_name(self, store, tmp_path): job.save(str(path)) loaded = PipelineJob.load(str(path), store=store) - assert loaded._pipeline_name == ("named_pipeline",), ( - "PipelineJob.load() should restore _pipeline_name from the saved pipeline name" + assert loaded._name == ("named_pipeline",), ( + "PipelineJob.load() should restore _name from the saved pipeline name" ) def test_load_after_partial_run_restores_unresolved_specs(self, store, tmp_path): From df3c785edea71bdf933eb596219c7ce135a961c8 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:48:08 +0000 Subject: [PATCH 12/24] docs(pipeline): add Google-style docstrings to AbstractPipelineBase and Pipeline recording methods --- src/orcapod/pipeline/base.py | 31 ++++++++++++++++++++++++++++--- src/orcapod/pipeline/graph.py | 26 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/orcapod/pipeline/base.py b/src/orcapod/pipeline/base.py index ea27206df..d0fdb2bbb 100644 --- a/src/orcapod/pipeline/base.py +++ b/src/orcapod/pipeline/base.py @@ -35,6 +35,14 @@ def __init__( name: str | tuple[str, ...] = "pipeline", tracker_manager: "cp.TrackerManagerProtocol | None" = None, ) -> None: + """Initialize shared pipeline state. + + Args: + name: Pipeline name (string or tuple). Used to scope database paths. + Stored internally as a tuple. + tracker_manager: Optional tracker manager override. Uses the default + tracker manager if ``None``. + """ super().__init__(tracker_manager=tracker_manager) self._name: tuple[str, ...] = (name,) if isinstance(name, str) else tuple(name) self._node_lut: dict[str, Any] = {} @@ -110,7 +118,13 @@ def record_function_pod_invocation( input_stream: "cp.StreamProtocol", label: str | None = None, ) -> None: - """Record a function pod invocation into the graph.""" + """Record a function pod invocation into the graph. + + Args: + pod: The function pod being invoked. + input_stream: The upstream stream. + label: Optional display label for the resulting node. + """ ... @abstractmethod @@ -120,10 +134,21 @@ def record_operator_pod_invocation( upstreams: "tuple[cp.StreamProtocol, ...]" = (), label: str | None = None, ) -> None: - """Record an operator pod invocation into the graph.""" + """Record an operator pod invocation into the graph. + + Args: + pod: The operator pod being invoked. + upstreams: Upstream streams for this operator. + label: Optional display label for the resulting node. + """ ... @abstractmethod def compile(self) -> None: - """Compile recorded invocations into a frozen DAG.""" + """Compile recorded invocations into a frozen DAG. + + Transforms accumulated ``_node_lut``, ``_upstreams``, and + ``_graph_edges`` into ``_persistent_node_map`` and ``_nodes``. + Sets ``_compiled = True`` on completion. + """ ... diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 9419b652c..b4d6c1c56 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -83,6 +83,19 @@ def record_function_pod_invocation( input_stream: cp.StreamProtocol, label: str | None = None, ) -> None: + """Record a function pod invocation and return its stream. + + Called by ``FunctionPod.__call__`` when used inside a ``with pipeline:`` + block. Creates a lightweight ``FunctionNode`` blueprint. + + Args: + pod: The function pod being invoked. + input_stream: The upstream stream. + label: Optional display label for the resulting node. + + Returns: + The ``FunctionNode`` representing this invocation. + """ input_stream_hash = input_stream.content_hash().to_string() function_node = FunctionNode( function_pod=pod, @@ -103,6 +116,19 @@ def record_operator_pod_invocation( upstreams: tuple[cp.StreamProtocol, ...] = (), label: str | None = None, ) -> None: + """Record an operator pod invocation and return its stream. + + Called by operator pods when used inside a ``with pipeline:`` + block. Creates a lightweight ``OperatorNode`` blueprint. + + Args: + pod: The operator pod being invoked. + upstreams: Upstream streams for this operator. + label: Optional display label for the resulting node. + + Returns: + The ``OperatorNode`` representing this invocation. + """ operator_node = OperatorNode( operator=pod, input_streams=upstreams, From b3ece05bf817da805f2d996b7af850ccf2b94a42 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 02:57:07 +0000 Subject: [PATCH 13/24] refactor(pipeline): PipelineJob recording creates FunctionJobNode/OperatorJobNode; compile builds SourceJobNode leaves --- src/orcapod/pipeline/job.py | 99 +++- ...-22-eng-493-task6-pipelinejob-job-nodes.md | 487 ++++++++++++++++++ tests/test_pipeline/test_pipeline_job.py | 74 +++ 3 files changed, 653 insertions(+), 7 deletions(-) create mode 100644 superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 7ca92e00e..6960100a0 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -101,14 +101,34 @@ def compile(self) -> None: self._compile_from_recording() def _compile_from_recording(self) -> None: - """Compile the recorded edges into a pure Pipeline.""" + """Compile the recorded edges into a pure Pipeline and build the job node map. + + ``_rec_node_lut`` now contains ``FunctionJobNode`` / ``OperatorJobNode`` objects + (set by ``record_function_pod_invocation`` / ``record_operator_pod_invocation``). + This method: + + 1. Converts each recorded job node to its lightweight blueprint counterpart via + ``.as_node()`` and injects the result into ``pipeline._node_lut`` so that + ``Pipeline.compile()`` sees only ``FunctionNode`` / ``OperatorNode`` objects. + 2. After compiling the blueprint pipeline, walks it topologically to build + ``self._persistent_node_map`` using ``SourceJobNode`` for leaf nodes + (concrete sources are taken from ``self._sources``) and fresh + ``FunctionJobNode`` / ``OperatorJobNode`` objects rewired to upstream job nodes + for non-leaf nodes. + """ + import networkx as _nx from orcapod.pipeline.graph import Pipeline + from orcapod.core.nodes.source_node import SourceJobNode, SourceNodeBase + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNodeBase + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase pipeline = Pipeline(name=self._name, auto_compile=False) - # Inject the recording state into the pipeline + # Inject the recording state into the pipeline, converting job nodes → blueprint nodes pipeline._graph_edges = list(self._rec_graph_edges) pipeline._upstreams = dict(self._rec_upstreams) - pipeline._node_lut = dict(self._rec_node_lut) + pipeline._node_lut = { + h: node.as_node() for h, node in self._rec_node_lut.items() + } # Rebuild hash graph from edges for edge in self._rec_graph_edges: pipeline._hash_graph.add_edge(*edge) @@ -129,6 +149,71 @@ def _compile_from_recording(self) -> None: pipeline.compile() self._compiled_pipeline = pipeline + # Build PipelineJob's own job node map walking compiled pipeline topologically. + # Leaf nodes (SourceNode) become SourceJobNode with concrete binding from _sources. + # Non-leaf nodes are fresh FunctionJobNode/OperatorJobNode rewired to upstream job nodes. + G = pipeline._hash_graph + job_node_map: dict[str, object] = {} + + for node_hash in _nx.topological_sort(G): + if node_hash not in pipeline._persistent_node_map: + continue + + bp_node = pipeline._persistent_node_map[node_hash] + + if isinstance(bp_node, SourceNodeBase): + concrete = self._sources.get(bp_node.name) + job_node: object = SourceJobNode( + name=bp_node.name, + tag_schema=bp_node.tag_schema, + data_schema=bp_node.data_schema, + concrete=concrete, + ) + elif isinstance(bp_node, FunctionNodeBase): + # Create fresh FunctionJobNode rewired to the upstream job node. + rec_node = self._rec_node_lut[node_hash] + original_input_hash = bp_node._input_stream.content_hash().to_string() + upstream_job_node = job_node_map[original_input_hash] + job_node = FunctionJobNode( + function_pod=rec_node._function_pod, + input_stream=upstream_job_node, + label=rec_node._label, + table_scope=rec_node._table_scope, + tracker_manager=rec_node.tracker_manager, + ) + elif isinstance(bp_node, OperatorNodeBase): + rec_node = self._rec_node_lut[node_hash] + upstream_job_nodes = tuple( + job_node_map[s.content_hash().to_string()] + for s in bp_node._input_streams + ) + job_node = OperatorJobNode( + operator=rec_node._operator, + input_streams=upstream_job_nodes, + label=rec_node._label, + table_scope=rec_node._table_scope, + tracker_manager=rec_node.tracker_manager, + ) + else: + raise TypeError( + f"Unknown blueprint node type in compiled pipeline: {type(bp_node)}" + ) + + job_node_map[node_hash] = job_node + + self._persistent_node_map = job_node_map + + # Build label → job node map from pipeline._nodes + self._nodes = { + label: job_node_map[node.content_hash().to_string()] + for label, node in pipeline._nodes.items() + if node.content_hash().to_string() in job_node_map + } + + # Wire databases if store is already set + if self._store is not None: + self._distribute_databases() + def _ensure_source_node(self, source: cp.StreamProtocol) -> "SourceNode": """Promote *source* to a SourceNode, storing the concrete binding. @@ -207,12 +292,12 @@ def record_function_pod_invocation( input_stream: The upstream stream (concrete source or spec). label: Optional label for the resulting node. """ - from orcapod.core.nodes import FunctionNode + from orcapod.core.nodes.function_node import FunctionJobNode input_stream = self._to_node_stream(input_stream) input_hash = input_stream.content_hash().to_string() - function_node = FunctionNode(function_pod=pod, input_stream=input_stream, label=label) + function_node = FunctionJobNode(function_pod=pod, input_stream=input_stream, label=label) fn_hash = function_node.content_hash().to_string() self._rec_node_lut[fn_hash] = function_node @@ -232,11 +317,11 @@ def record_operator_pod_invocation( upstreams: Upstream streams (concrete sources or specs). label: Optional label for the resulting node. """ - from orcapod.core.nodes import OperatorNode + from orcapod.core.nodes.operator_node import OperatorJobNode processed = tuple(self._to_node_stream(s) for s in upstreams) - operator_node = OperatorNode(operator=pod, input_streams=processed, label=label) + operator_node = OperatorJobNode(operator=pod, input_streams=processed, label=label) op_hash = operator_node.content_hash().to_string() self._rec_node_lut[op_hash] = operator_node diff --git a/superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md b/superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md new file mode 100644 index 000000000..6fd4e8927 --- /dev/null +++ b/superpowers/plans/2026-05-22-eng-493-task6-pipelinejob-job-nodes.md @@ -0,0 +1,487 @@ +# ENG-493 Task 6: PipelineJob Recording Uses JobNode Types + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After a `with job:` block, `job._persistent_node_map` contains only `JobNode` variants (`SourceJobNode`, `FunctionJobNode`, `OperatorJobNode`), not blueprint types. + +**Architecture:** `record_function_pod_invocation` and `record_operator_pod_invocation` now create `FunctionJobNode`/`OperatorJobNode` in `_rec_node_lut`. `_compile_from_recording` converts these to lightweight `FunctionNode`/`OperatorNode` (via `.as_node()`) for the blueprint `Pipeline._node_lut`, then builds `job._persistent_node_map` with `SourceJobNode` leaves by walking the compiled pipeline topologically. `_build_execution_graph` already uses `FunctionNode`/`OperatorNode` from the blueprint pipeline and is unchanged. + +**Tech Stack:** Python, uv, pytest, networkx, orcapod nodes hierarchy. + +--- + +## File Map + +| File | Change | +|---|---| +| `src/orcapod/pipeline/job.py` | Modify `record_function_pod_invocation`, `record_operator_pod_invocation`, and `_compile_from_recording` | +| `tests/test_pipeline/test_pipeline_job.py` | Add `TestPipelineJobUsesJobNodes` class with fixture | + +--- + +### Task 1: Write failing tests for job node types in `_persistent_node_map` + +**Files:** +- Modify: `tests/test_pipeline/test_pipeline_job.py` + +- [ ] **Step 1: Add the `pipeline_job_with_sources` fixture and `TestPipelineJobUsesJobNodes` class** + +Open `tests/test_pipeline/test_pipeline_job.py` and add the following immediately before the final `class TestFromPipeline:` block (after the `compiled_pipeline`/`db`/`source_a`/`source_b`/`pipeline_job`/`pipeline_job_complete` fixtures, around line 720): + +```python +@pytest.fixture +def pipeline_job_with_sources(store): + """A PipelineJob created via with-block using concrete sources + a FunctionPod.""" + src_a, src_b = _make_two_sources() + pf = PythonDataFunction(add_values, output_keys="total") + pod = FunctionPod(data_function=pf) + + job = PipelineJob(store=store) + with job: + joined = Join()(src_a, src_b, label="joiner") + pod(joined, label="adder") + return job + + +class TestPipelineJobUsesJobNodes: + """PipelineJob._persistent_node_map must contain only JobNode variants after recording.""" + + def test_persistent_map_has_source_job_nodes(self, pipeline_job_with_sources): + """Source entries in PipelineJob._persistent_node_map must be SourceJobNode.""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNodeBase + + assert pipeline_job_with_sources._persistent_node_map is not None, ( + "_persistent_node_map must be set after with-block" + ) + for node in pipeline_job_with_sources._persistent_node_map.values(): + if isinstance(node, SourceNodeBase): + assert isinstance(node, SourceJobNode), ( + f"Expected SourceJobNode but got {type(node).__name__}" + ) + + def test_persistent_map_has_function_job_nodes(self, pipeline_job_with_sources): + """Function entries in PipelineJob._persistent_node_map must be FunctionJobNode.""" + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNodeBase + + assert pipeline_job_with_sources._persistent_node_map is not None + fn_nodes = [ + n for n in pipeline_job_with_sources._persistent_node_map.values() + if isinstance(n, FunctionNodeBase) + ] + assert len(fn_nodes) >= 1, "Expected at least one FunctionJobNode" + for node in fn_nodes: + assert isinstance(node, FunctionJobNode), ( + f"Expected FunctionJobNode but got {type(node).__name__}" + ) + + def test_persistent_map_has_operator_job_nodes(self, pipeline_job_with_sources): + """Operator entries in PipelineJob._persistent_node_map must be OperatorJobNode.""" + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase + + assert pipeline_job_with_sources._persistent_node_map is not None + op_nodes = [ + n for n in pipeline_job_with_sources._persistent_node_map.values() + if isinstance(n, OperatorNodeBase) + ] + assert len(op_nodes) >= 1, "Expected at least one OperatorJobNode" + for node in op_nodes: + assert isinstance(node, OperatorJobNode), ( + f"Expected OperatorJobNode but got {type(node).__name__}" + ) + + def test_blueprint_pipeline_still_has_lightweight_nodes(self, pipeline_job_with_sources): + """The compiled pipeline's _persistent_node_map still has lightweight nodes.""" + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + + for node in pipeline_job_with_sources.pipeline._persistent_node_map.values(): + assert not isinstance(node, FunctionJobNode), ( + "Blueprint pipeline must not contain FunctionJobNode" + ) + assert not isinstance(node, OperatorJobNode), ( + "Blueprint pipeline must not contain OperatorJobNode" + ) +``` + +- [ ] **Step 2: Run the new tests to verify they fail** + +```bash +cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ +uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobUsesJobNodes -v 2>&1 | tail -20 +``` + +Expected: FAIL — `_persistent_node_map` is `None` after with-block (returns `AssertionError: _persistent_node_map must be set after with-block`), and/or `FunctionNode`/`OperatorNode` found instead of job variants. + +--- + +### Task 2: Update `record_function_pod_invocation` to create `FunctionJobNode` + +**Files:** +- Modify: `src/orcapod/pipeline/job.py` (lines ~197–221) + +- [ ] **Step 1: Replace the body of `record_function_pod_invocation`** + +Find this block in `src/orcapod/pipeline/job.py`: + +```python + def record_function_pod_invocation( + self, + pod: cp.FunctionPodProtocol, + input_stream: cp.StreamProtocol, + label: str | None = None, + ) -> None: + """Record a function pod invocation, promoting concrete sources to specs. + + Args: + pod: The function pod being invoked. + input_stream: The upstream stream (concrete source or spec). + label: Optional label for the resulting node. + """ + from orcapod.core.nodes import FunctionNode + + input_stream = self._to_node_stream(input_stream) + + input_hash = input_stream.content_hash().to_string() + function_node = FunctionNode(function_pod=pod, input_stream=input_stream, label=label) + fn_hash = function_node.content_hash().to_string() + + self._rec_node_lut[fn_hash] = function_node + self._rec_upstreams[input_hash] = input_stream + self._rec_graph_edges.append((input_hash, fn_hash)) +``` + +Replace with: + +```python + def record_function_pod_invocation( + self, + pod: cp.FunctionPodProtocol, + input_stream: cp.StreamProtocol, + label: str | None = None, + ) -> None: + """Record a function pod invocation, promoting concrete sources to specs. + + Args: + pod: The function pod being invoked. + input_stream: The upstream stream (concrete source or spec). + label: Optional label for the resulting node. + """ + from orcapod.core.nodes.function_node import FunctionJobNode + + input_stream = self._to_node_stream(input_stream) + + input_hash = input_stream.content_hash().to_string() + function_node = FunctionJobNode(function_pod=pod, input_stream=input_stream, label=label) + fn_hash = function_node.content_hash().to_string() + + self._rec_node_lut[fn_hash] = function_node + self._rec_upstreams[input_hash] = input_stream + self._rec_graph_edges.append((input_hash, fn_hash)) +``` + +The only change is `FunctionNode` → `FunctionJobNode` in the import and construction. The hash is identical because `FunctionJobNode` inherits `content_hash()` from `FunctionNodeBase` with the same identity structure. + +--- + +### Task 3: Update `record_operator_pod_invocation` to create `OperatorJobNode` + +**Files:** +- Modify: `src/orcapod/pipeline/job.py` (lines ~222–246) + +- [ ] **Step 1: Replace the body of `record_operator_pod_invocation`** + +Find this block in `src/orcapod/pipeline/job.py`: + +```python + def record_operator_pod_invocation( + self, + pod: cp.OperatorPodProtocol, + upstreams: tuple[cp.StreamProtocol, ...] = (), + label: str | None = None, + ) -> None: + """Record an operator pod invocation, promoting concrete sources to specs. + + Args: + pod: The operator pod being invoked. + upstreams: Upstream streams (concrete sources or specs). + label: Optional label for the resulting node. + """ + from orcapod.core.nodes import OperatorNode + + processed = tuple(self._to_node_stream(s) for s in upstreams) + + operator_node = OperatorNode(operator=pod, input_streams=processed, label=label) + op_hash = operator_node.content_hash().to_string() + + self._rec_node_lut[op_hash] = operator_node + for upstream in processed: + up_hash = upstream.content_hash().to_string() + self._rec_upstreams[up_hash] = upstream + self._rec_graph_edges.append((up_hash, op_hash)) +``` + +Replace with: + +```python + def record_operator_pod_invocation( + self, + pod: cp.OperatorPodProtocol, + upstreams: tuple[cp.StreamProtocol, ...] = (), + label: str | None = None, + ) -> None: + """Record an operator pod invocation, promoting concrete sources to specs. + + Args: + pod: The operator pod being invoked. + upstreams: Upstream streams (concrete sources or specs). + label: Optional label for the resulting node. + """ + from orcapod.core.nodes.operator_node import OperatorJobNode + + processed = tuple(self._to_node_stream(s) for s in upstreams) + + operator_node = OperatorJobNode(operator=pod, input_streams=processed, label=label) + op_hash = operator_node.content_hash().to_string() + + self._rec_node_lut[op_hash] = operator_node + for upstream in processed: + up_hash = upstream.content_hash().to_string() + self._rec_upstreams[up_hash] = upstream + self._rec_graph_edges.append((up_hash, op_hash)) +``` + +The only change is `OperatorNode` → `OperatorJobNode` in the import and construction. + +--- + +### Task 4: Update `_compile_from_recording` to convert JobNodes for blueprint and build job `_persistent_node_map` + +**Files:** +- Modify: `src/orcapod/pipeline/job.py` (lines ~103–130) + +**Key insight:** `Pipeline.compile()` expects `FunctionNode`/`OperatorNode` in `pipeline._node_lut` — it raises `TypeError` for any other type. So `_compile_from_recording` must convert `FunctionJobNode`→`FunctionNode` and `OperatorJobNode`→`OperatorNode` via `.as_node()` before calling `pipeline.compile()`. Then it builds `job._persistent_node_map` by walking the compiled pipeline topologically, creating `SourceJobNode` leaves from the concrete sources captured in `_sources`. + +- [ ] **Step 1: Replace `_compile_from_recording`** + +Find this entire method in `src/orcapod/pipeline/job.py`: + +```python + def _compile_from_recording(self) -> None: + """Compile the recorded edges into a pure Pipeline.""" + from orcapod.pipeline.graph import Pipeline + + pipeline = Pipeline(name=self._name, auto_compile=False) + # Inject the recording state into the pipeline + pipeline._graph_edges = list(self._rec_graph_edges) + pipeline._upstreams = dict(self._rec_upstreams) + pipeline._node_lut = dict(self._rec_node_lut) + # Rebuild hash graph from edges + for edge in self._rec_graph_edges: + pipeline._hash_graph.add_edge(*edge) + + # Annotate node_type on each recorded node (function/operator). + for node_hash, node in self._rec_node_lut.items(): + if node_hash in pipeline._hash_graph.nodes: + pipeline._hash_graph.nodes[node_hash]["node_type"] = node.node_type + if node.label: + pipeline._hash_graph.nodes[node_hash]["label"] = node.label + + # Annotate upstream (source) nodes that are not in _rec_node_lut. + for node_hash, stream in self._rec_upstreams.items(): + if node_hash in pipeline._hash_graph.nodes: + if not pipeline._hash_graph.nodes[node_hash].get("node_type"): + pipeline._hash_graph.nodes[node_hash]["node_type"] = "source" + + pipeline.compile() + self._compiled_pipeline = pipeline +``` + +Replace with: + +```python + def _compile_from_recording(self) -> None: + """Compile the recorded edges into a pure Pipeline and build the job node map. + + ``_rec_node_lut`` now contains ``FunctionJobNode`` / ``OperatorJobNode`` objects + (set by ``record_function_pod_invocation`` / ``record_operator_pod_invocation``). + This method: + + 1. Converts each recorded job node to its lightweight blueprint counterpart via + ``.as_node()`` and injects the result into ``pipeline._node_lut`` so that + ``Pipeline.compile()`` sees only ``FunctionNode`` / ``OperatorNode`` objects. + 2. After compiling the blueprint pipeline, walks it topologically to build + ``self._persistent_node_map`` using ``SourceJobNode`` for leaf nodes + (concrete sources are taken from ``self._sources``) and the original recorded + ``FunctionJobNode`` / ``OperatorJobNode`` objects for non-leaf nodes. + """ + import networkx as _nx + from orcapod.pipeline.graph import Pipeline + from orcapod.core.nodes.source_node import SourceJobNode, SourceNodeBase + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNodeBase + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase + + pipeline = Pipeline(name=self._name, auto_compile=False) + # Inject the recording state into the pipeline, converting job nodes → blueprint nodes + pipeline._graph_edges = list(self._rec_graph_edges) + pipeline._upstreams = dict(self._rec_upstreams) + pipeline._node_lut = { + h: node.as_node() for h, node in self._rec_node_lut.items() + } + # Rebuild hash graph from edges + for edge in self._rec_graph_edges: + pipeline._hash_graph.add_edge(*edge) + + # Annotate node_type on each recorded node (function/operator). + for node_hash, node in self._rec_node_lut.items(): + if node_hash in pipeline._hash_graph.nodes: + pipeline._hash_graph.nodes[node_hash]["node_type"] = node.node_type + if node.label: + pipeline._hash_graph.nodes[node_hash]["label"] = node.label + + # Annotate upstream (source) nodes that are not in _rec_node_lut. + for node_hash, stream in self._rec_upstreams.items(): + if node_hash in pipeline._hash_graph.nodes: + if not pipeline._hash_graph.nodes[node_hash].get("node_type"): + pipeline._hash_graph.nodes[node_hash]["node_type"] = "source" + + pipeline.compile() + self._compiled_pipeline = pipeline + + # Build PipelineJob's own job node map walking compiled pipeline topologically. + # Leaf nodes (SourceNode) become SourceJobNode with concrete binding from _sources. + # Non-leaf nodes reuse the FunctionJobNode/OperatorJobNode from _rec_node_lut. + G = pipeline._hash_graph + job_node_map: dict[str, object] = {} + + for node_hash in _nx.topological_sort(G): + if node_hash not in pipeline._persistent_node_map: + continue + + bp_node = pipeline._persistent_node_map[node_hash] + + if isinstance(bp_node, SourceNodeBase): + concrete = self._sources.get(bp_node.name) + job_node: object = SourceJobNode( + name=bp_node.name, + tag_schema=bp_node.tag_schema, + data_schema=bp_node.data_schema, + concrete=concrete, + ) + elif isinstance(bp_node, FunctionNodeBase): + # Reuse the FunctionJobNode from _rec_node_lut, rewired to upstream job node. + # The original rec node has the correct table_scope and tracker_manager. + rec_node = self._rec_node_lut[node_hash] + original_input_hash = bp_node._input_stream.content_hash().to_string() + upstream_job_node = job_node_map[original_input_hash] + job_node = FunctionJobNode( + function_pod=rec_node._function_pod, + input_stream=upstream_job_node, + label=rec_node._label, + table_scope=rec_node._table_scope, + tracker_manager=rec_node.tracker_manager, + ) + elif isinstance(bp_node, OperatorNodeBase): + rec_node = self._rec_node_lut[node_hash] + upstream_job_nodes = tuple( + job_node_map[s.content_hash().to_string()] + for s in bp_node._input_streams + ) + job_node = OperatorJobNode( + operator=rec_node._operator, + input_streams=upstream_job_nodes, + label=rec_node._label, + table_scope=rec_node._table_scope, + tracker_manager=rec_node.tracker_manager, + ) + else: + raise TypeError( + f"Unknown blueprint node type in compiled pipeline: {type(bp_node)}" + ) + + job_node_map[node_hash] = job_node + + self._persistent_node_map = job_node_map + + # Build label → job node map from pipeline._nodes + self._nodes = { + label: job_node_map[node.content_hash().to_string()] + for label, node in pipeline._nodes.items() + if node.content_hash().to_string() in job_node_map + } + + # Wire databases if store is already set + if self._store is not None: + self._distribute_databases() +``` + +--- + +### Task 5: Run the new tests — verify they pass + +- [ ] **Step 1: Run `TestPipelineJobUsesJobNodes`** + +```bash +cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ +uv run pytest tests/test_pipeline/test_pipeline_job.py::TestPipelineJobUsesJobNodes -v 2>&1 | tail -20 +``` + +Expected: All 4 tests PASS. + +- [ ] **Step 2: Run the full pipeline test suite** + +```bash +cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ +uv run pytest tests/test_pipeline/ -v --tb=short 2>&1 | tail -50 +``` + +Expected: All tests pass. Watch for: +- `TestPipelineJobEndToEnd::test_end_to_end_source_join_function` — checks `compiled_nodes["joiner"]` is `OperatorNode` and `compiled_nodes["adder"]` is `FunctionNode`. These look at `job.pipeline.compiled_nodes` (the blueprint pipeline), not `job._persistent_node_map`, so they should still pass. +- `TestPipelineJobRun` — exercises `job.run()` via `_build_execution_graph` which reads from `pipeline._node_lut` (blueprint); should be unaffected. + +If any test fails due to type checks on `_persistent_node_map` expecting `FunctionNode`/`OperatorNode`, update those assertions to check for `FunctionJobNode`/`OperatorJobNode` respectively. + +- [ ] **Step 3: Run the full test suite** + +```bash +cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ +uv run pytest tests/ -v --tb=short 2>&1 | tail -60 +``` + +Expected: All tests pass. + +--- + +### Task 6: Commit + +- [ ] **Step 1: Commit the changes** + +```bash +cd /home/kurouto/kurouto-jobs/5bda6bb8-f5e1-4b33-b256-7eef168aa769/orcapod-python && \ +git add src/orcapod/pipeline/job.py tests/test_pipeline/test_pipeline_job.py && \ +git commit -m "refactor(pipeline): PipelineJob recording creates FunctionJobNode/OperatorJobNode; compile builds SourceJobNode leaves" +``` + +--- + +## Self-Review + +**Spec coverage:** +- `record_function_pod_invocation` → `FunctionJobNode`: Task 2. ✓ +- `record_operator_pod_invocation` → `OperatorJobNode`: Task 3. ✓ +- `_compile_from_recording` converts via `.as_node()` for blueprint: Task 4. ✓ +- `_compile_from_recording` builds `_persistent_node_map` with `SourceJobNode`: Task 4. ✓ +- `job._nodes` label map updated: Task 4. ✓ +- `_distribute_databases()` called when store set: Task 4. ✓ +- Blueprint pipeline still has lightweight nodes: tested in Task 1, step 1 (4th test). ✓ +- Existing tests not broken: Task 5 full suite run. ✓ + +**Placeholder scan:** No TBDs, TODOs, or vague steps found. + +**Type consistency:** +- `FunctionJobNode` referenced in Tasks 2, 4 — both use `from orcapod.core.nodes.function_node import FunctionJobNode`. ✓ +- `OperatorJobNode` referenced in Tasks 3, 4 — both use `from orcapod.core.nodes.operator_node import OperatorJobNode`. ✓ +- `SourceJobNode` referenced in Task 4 — uses `from orcapod.core.nodes.source_node import SourceJobNode`. ✓ +- `bp_node._input_stream` (Task 4, FunctionNodeBase branch) — `FunctionNodeBase` has `_input_stream` attribute. ✓ +- `bp_node._input_streams` (Task 4, OperatorNodeBase branch) — `OperatorNodeBase` has `_input_streams` attribute. ✓ +- `rec_node._function_pod`, `rec_node._table_scope`, `rec_node.tracker_manager` (Task 4) — all present on `FunctionJobNode` via `FunctionNodeBase`. ✓ +- `rec_node._operator`, `rec_node._input_streams`, `rec_node._table_scope`, `rec_node.tracker_manager` (Task 4) — all present on `OperatorJobNode` via `OperatorNodeBase`. ✓ diff --git a/tests/test_pipeline/test_pipeline_job.py b/tests/test_pipeline/test_pipeline_job.py index c8e266ea4..27a547c45 100644 --- a/tests/test_pipeline/test_pipeline_job.py +++ b/tests/test_pipeline/test_pipeline_job.py @@ -717,6 +717,80 @@ def pipeline_job_complete(compiled_pipeline, db, source_a, source_b): ) +@pytest.fixture +def pipeline_job_with_sources(store): + """A PipelineJob created via with-block using concrete sources + a FunctionPod.""" + src_a, src_b = _make_two_sources() + pf = PythonDataFunction(add_values, output_keys="total") + pod = FunctionPod(data_function=pf) + + job = PipelineJob(store=store) + with job: + joined = Join()(src_a, src_b, label="joiner") + pod(joined, label="adder") + return job + + +class TestPipelineJobUsesJobNodes: + """PipelineJob._persistent_node_map must contain only JobNode variants after recording.""" + + def test_persistent_map_has_source_job_nodes(self, pipeline_job_with_sources): + """Source entries in PipelineJob._persistent_node_map must be SourceJobNode.""" + from orcapod.core.nodes.source_node import SourceJobNode, SourceNodeBase + + assert pipeline_job_with_sources._persistent_node_map is not None, ( + "_persistent_node_map must be set after with-block" + ) + for node in pipeline_job_with_sources._persistent_node_map.values(): + if isinstance(node, SourceNodeBase): + assert isinstance(node, SourceJobNode), ( + f"Expected SourceJobNode but got {type(node).__name__}" + ) + + def test_persistent_map_has_function_job_nodes(self, pipeline_job_with_sources): + """Function entries in PipelineJob._persistent_node_map must be FunctionJobNode.""" + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNodeBase + + assert pipeline_job_with_sources._persistent_node_map is not None + fn_nodes = [ + n for n in pipeline_job_with_sources._persistent_node_map.values() + if isinstance(n, FunctionNodeBase) + ] + assert len(fn_nodes) >= 1, "Expected at least one FunctionJobNode" + for node in fn_nodes: + assert isinstance(node, FunctionJobNode), ( + f"Expected FunctionJobNode but got {type(node).__name__}" + ) + + def test_persistent_map_has_operator_job_nodes(self, pipeline_job_with_sources): + """Operator entries in PipelineJob._persistent_node_map must be OperatorJobNode.""" + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase + + assert pipeline_job_with_sources._persistent_node_map is not None + op_nodes = [ + n for n in pipeline_job_with_sources._persistent_node_map.values() + if isinstance(n, OperatorNodeBase) + ] + assert len(op_nodes) >= 1, "Expected at least one OperatorJobNode" + for node in op_nodes: + assert isinstance(node, OperatorJobNode), ( + f"Expected OperatorJobNode but got {type(node).__name__}" + ) + + def test_blueprint_pipeline_still_has_lightweight_nodes(self, pipeline_job_with_sources): + """The compiled pipeline's _persistent_node_map still has lightweight nodes.""" + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + + for node in pipeline_job_with_sources.pipeline._persistent_node_map.values(): + assert not isinstance(node, FunctionJobNode), ( + "Blueprint pipeline must not contain FunctionJobNode" + ) + assert not isinstance(node, OperatorJobNode), ( + "Blueprint pipeline must not contain OperatorJobNode" + ) + + class TestFromPipeline: """PipelineJob.from_pipeline() creates a runnable job from a compiled Pipeline.""" From 07930dd0e7aebd0261a25b934892e34d014e78b4 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 03:03:35 +0000 Subject: [PATCH 14/24] refactor(serialization): update Pipeline save/load for SourceNode format; bump version to v0.3 - Bump PIPELINE_FORMAT_VERSION from "0.1.0" to "0.3" - Add "0.1.0", "0.2", "0.3" to SUPPORTED_FORMAT_VERSIONS for backward compat - save(): use source_config key "name" (was "node_name") and include tag_schema/data_schema in source_config - load(): handle new "name" key (v0.3) and legacy "node_name" (v0.1.0), plus "spec" source_type (v0.2 backward compat) - Add TestNewSerializationFormat with 4 tests covering round-trip, format fields, version, and v0.2 backward compat Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/pipeline/graph.py | 56 +++++++---- src/orcapod/pipeline/serialization.py | 4 +- tests/test_pipeline/test_serialization.py | 97 +++++++++++++++++++ .../test_serialization_helpers.py | 2 +- 4 files changed, 136 insertions(+), 23 deletions(-) diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index b4d6c1c56..e1cd4ab80 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -370,7 +370,9 @@ def save(self, path: str | Path) -> None: if isinstance(node, SourceNodeClass): descriptor["source_config"] = { "source_type": "node", - "node_name": node.name, + "name": node.name, + "tag_schema": serialize_schema(node.tag_schema, type_converter), + "data_schema": serialize_schema(node.data_schema, type_converter), } descriptor["reconstructable"] = True @@ -464,26 +466,40 @@ def load(cls, path: str | Path) -> "Pipeline": source_config = descriptor.get("source_config") or {} if node_type == "source": - tag_schema = Schema(deserialize_schema(descriptor["output_schema"]["tag"])) - data_schema = Schema(deserialize_schema(descriptor["output_schema"]["data"])) - # Support both old "spec" format and new "node" format - if source_config.get("source_type") == "node": - node_name = source_config["node_name"] - elif source_config.get("source_type") == "spec": - # Legacy format compatibility: spec_name becomes node name - node_name = source_config["spec_name"] + source_type = source_config.get("source_type") + if source_type in ("node", "spec"): + # "node" is the v0.3 format; "spec" is the v0.2 backward-compat format. + # Both reconstruct as SourceNode — hashes are preserved because + # SourceNode.identity_structure() matches old SourceSpec.identity_structure(). + if source_type == "node": + # v0.3 format uses "name"; v0.1.0 used "node_name" — support both + node_name = source_config.get("name") or source_config.get("node_name") + else: + # v0.2 backward-compat: spec_name becomes node name + node_name = source_config.get("spec_name") + if not node_name: + node_name = descriptor.get("label") or "unknown" + # Prefer tag/data schemas from source_config when present (v0.3+); + # fall back to output_schema for older formats. + if "tag_schema" in source_config and "data_schema" in source_config: + tag_schema = Schema(deserialize_schema(source_config["tag_schema"])) + data_schema = Schema(deserialize_schema(source_config["data_schema"])) + else: + tag_schema = Schema(deserialize_schema(descriptor["output_schema"]["tag"])) + data_schema = Schema(deserialize_schema(descriptor["output_schema"]["data"])) + node = SourceNodeClass( + name=node_name, + tag_schema=tag_schema, + data_schema=data_schema, + ) + # Restore label from descriptor if set explicitly + stored_label = descriptor.get("label") + if stored_label and stored_label != node_name: + node._label = stored_label else: - # Fall back to stored label - node_name = descriptor.get("label") or "unknown" - node = SourceNodeClass( - name=node_name, - tag_schema=tag_schema, - data_schema=data_schema, - ) - # Restore label from descriptor if set explicitly - stored_label = descriptor.get("label") - if stored_label and stored_label != node_name: - node._label = stored_label + raise ValueError( + f"Unknown source_type {source_type!r} in pipeline descriptor." + ) reconstructed[node_hash] = node elif node_type == "function": diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index f255634ac..a9867f3e3 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -17,8 +17,8 @@ # Format version # --------------------------------------------------------------------------- -PIPELINE_FORMAT_VERSION = "0.1.0" -SUPPORTED_FORMAT_VERSIONS = frozenset({"0.1.0"}) +PIPELINE_FORMAT_VERSION = "0.3" +SUPPORTED_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2", "0.3"}) # --------------------------------------------------------------------------- # LoadStatus diff --git a/tests/test_pipeline/test_serialization.py b/tests/test_pipeline/test_serialization.py index 637e82c94..7e2354bfa 100644 --- a/tests/test_pipeline/test_serialization.py +++ b/tests/test_pipeline/test_serialization.py @@ -178,3 +178,100 @@ def test_load_hash_graph_has_node_types(self, spec_pipeline): assert "node_type" in attrs, ( f"Node {node_hash} missing node_type in _hash_graph" ) + + +@pytest.fixture +def compiled_pipeline(): + """A compiled Pipeline using SourceNode leaves (no tmp_path coupling).""" + def _src(tag, data): + tbl = pa.table({tag: pa.array(["a"], type=pa.large_string()), data: pa.array([1], type=pa.int64())}) + return ArrowTableSource(tbl, tag_columns=[tag], infer_nullable=True) + + from orcapod.core.operators import Join + + src_a = _src("key", "value") + src_b = _src("key", "score") + tag_a, data_a = src_a.output_schema() + tag_b, data_b = src_b.output_schema() + + node_a = SourceNode(name="source_a", tag_schema=tag_a, data_schema=data_a) + node_b = SourceNode(name="source_b", tag_schema=tag_b, data_schema=data_b) + + pipeline = Pipeline(name="test_pipe") + with pipeline: + Join()(node_a, node_b, label="joiner") + + return pipeline + + +class TestNewSerializationFormat: + """v0.3 serialization format tests.""" + + def test_save_load_roundtrip_with_source_node(self, tmp_path, compiled_pipeline): + """Pipeline.save/load round-trip preserves SourceNode slots.""" + save_path = tmp_path / "test_pipeline.json" + compiled_pipeline.save(save_path) + + loaded = Pipeline.load(save_path) + assert loaded._compiled + + for node in loaded._persistent_node_map.values(): + if node.node_type == "source": + assert isinstance(node, SourceNode), ( + f"Expected SourceNode after load, got {type(node).__name__}" + ) + + def test_saved_format_has_source_node_type(self, tmp_path, compiled_pipeline): + """Saved format uses source_type='node'.""" + save_path = tmp_path / "test_pipeline.json" + compiled_pipeline.save(save_path) + + with open(save_path) as f: + data = json.load(f) + + for node_data in data.get("nodes", {}).values(): + if node_data.get("node_type") == "source": + assert node_data.get("source_config", {}).get("source_type") == "node", ( + f"Expected source_type='node', got {node_data.get('source_config')}" + ) + + def test_format_version_is_0_3(self, tmp_path, compiled_pipeline): + """Saved format version is 0.3.""" + save_path = tmp_path / "test_pipeline.json" + compiled_pipeline.save(save_path) + + with open(save_path) as f: + data = json.load(f) + + assert data.get("orcapod_pipeline_version") == "0.3", ( + f"Expected version '0.3', got {data.get('orcapod_pipeline_version')!r}" + ) + + def test_backward_compat_load_v0_2_spec_format(self, tmp_path, compiled_pipeline): + """Loading a v0.2 pipeline with source_type='spec' produces SourceNode.""" + # Save, then hack the JSON to simulate v0.2 format + save_path = tmp_path / "old_pipeline.json" + compiled_pipeline.save(save_path) + + with open(save_path) as f: + data = json.load(f) + + # Downgrade to v0.2 format + data["orcapod_pipeline_version"] = "0.2" + for node_data in data.get("nodes", {}).values(): + if node_data.get("node_type") == "source": + if "source_config" in node_data: + node_data["source_config"]["source_type"] = "spec" + + old_path = tmp_path / "old_format.json" + with open(old_path, "w") as f: + json.dump(data, f) + + # Load should work and produce SourceNode + loaded = Pipeline.load(old_path) + + for node in loaded._persistent_node_map.values(): + if node.node_type == "source": + assert isinstance(node, SourceNode), ( + f"Expected SourceNode from v0.2 load, got {type(node).__name__}" + ) diff --git a/tests/test_pipeline/test_serialization_helpers.py b/tests/test_pipeline/test_serialization_helpers.py index 5e85f74c4..9a0885036 100644 --- a/tests/test_pipeline/test_serialization_helpers.py +++ b/tests/test_pipeline/test_serialization_helpers.py @@ -196,7 +196,7 @@ def test_resolve_unknown_raises(self): class TestPipelineFormatVersion: def test_version_is_string(self): assert isinstance(PIPELINE_FORMAT_VERSION, str) - assert PIPELINE_FORMAT_VERSION == "0.1.0" + assert PIPELINE_FORMAT_VERSION == "0.3" # --------------------------------------------------------------------------- From 0f1c7ab3af90050c240d0080fd2d6e2ac9a2df1b Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 03:10:02 +0000 Subject: [PATCH 15/24] refactor(sources): delete SourceSpec; update all references to SourceNode (ENG-493) - Delete src/orcapod/core/sources/source_spec.py - Delete tests/test_core/sources/test_source_spec.py - Replace SourceSpec export in src/orcapod/__init__.py with SourceNode - Remove SourceSpec from src/orcapod/core/sources/__init__.py - Update src/orcapod/errors.py docstrings for UnboundSourceError and SourceSpecMismatchError (class name preserved for catch-by-name compat) - Replace SourceSpec imports/usage in test-objective/unit/test_tracker.py with SourceNode - Replace SourceSpec hash-comparison tests in test_source_node.py with deterministic + stable-value tests; anchor hash digests documented in docstring Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/__init__.py | 4 +- src/orcapod/core/sources/__init__.py | 3 - src/orcapod/core/sources/source_spec.py | 243 -------------------- src/orcapod/errors.py | 11 +- test-objective/unit/test_tracker.py | 6 +- tests/test_core/nodes/test_source_node.py | 60 ++++- tests/test_core/sources/test_source_spec.py | 129 ----------- 7 files changed, 63 insertions(+), 393 deletions(-) delete mode 100644 src/orcapod/core/sources/source_spec.py delete mode 100644 tests/test_core/sources/test_source_spec.py diff --git a/src/orcapod/__init__.py b/src/orcapod/__init__.py index 50ca05161..7e07bd1d2 100644 --- a/src/orcapod/__init__.py +++ b/src/orcapod/__init__.py @@ -2,7 +2,7 @@ FunctionPod, function_pod, ) -from .core.sources.source_spec import SourceSpec +from .core.nodes.source_node import SourceNode from .pipeline import Pipeline, PipelineJob # Subpackage re-exports for clean public API @@ -18,7 +18,7 @@ "function_pod", "Pipeline", "PipelineJob", - "SourceSpec", + "SourceNode", "databases", "nodes", "operators", diff --git a/src/orcapod/core/sources/__init__.py b/src/orcapod/core/sources/__init__.py index b2e046d57..f7ad5b63c 100644 --- a/src/orcapod/core/sources/__init__.py +++ b/src/orcapod/core/sources/__init__.py @@ -13,8 +13,6 @@ from .spiraldb_table_source import SpiralDBTableSource from .sqlite_table_source import SQLiteTableSource from .postgresql_table_source import PostgreSQLTableSource -from .source_spec import SourceSpec - __all__ = [ "RootSource", "ArrowTableSource", @@ -28,7 +26,6 @@ "ListSource", "SourceProxy", "SourceRegistry", - "SourceSpec", "SpiralDBTableSource", "SQLiteTableSource", "PostgreSQLTableSource", diff --git a/src/orcapod/core/sources/source_spec.py b/src/orcapod/core/sources/source_spec.py deleted file mode 100644 index 176ce2927..000000000 --- a/src/orcapod/core/sources/source_spec.py +++ /dev/null @@ -1,243 +0,0 @@ -"""SourceSpec — a named schema declaration for pipeline input slots.""" - -from __future__ import annotations - -from collections.abc import Iterator -from typing import TYPE_CHECKING, Any - -import orcapod.contexts as contexts -from orcapod.core.base import TraceableBase -from orcapod.errors import SourceSpecMismatchError, UnboundSourceError -from orcapod.protocols.core_protocols import DataProtocol, TagProtocol -from orcapod.types import ColumnConfig, Schema - -if TYPE_CHECKING: - import pyarrow as pa - - from orcapod.protocols.core_protocols import StreamProtocol - - -class SourceSpec(TraceableBase): - """A named schema declaration for a pipeline input slot. - - ``SourceSpec`` describes what a pipeline input looks like — its key schema - and data schema — without referencing any concrete data source. It is used - as the typed input slot concept for both ``Pipeline`` and ``PipelineJob``. - - Note: - ``SourceSpec`` is designed to be treated as immutable — all attributes - are stored as private members and exposed only through read-only - properties. Immutability is a convention rather than a type-system - guarantee; external mutation of private attributes is unsupported. - - A ``SourceSpec`` can appear as an upstream in operator chains during a - ``with Pipeline:`` or ``with PipelineJob:`` recording block. Calling data- - producing methods (``iter_data``, ``as_table``) raises ``UnboundSourceError`` - until the spec is bound to a concrete source via ``PipelineJob.bind()``. - - Identity and hashing: - - ``pipeline_hash()`` — schema-only, ignoring ``name``. Matches a - schema-compatible ``RootSource.pipeline_hash()``, enabling DB path - reuse across different sources bound to the same spec. - - ``content_hash()`` — includes ``name``. Two specs with identical - schemas but different names are distinct elements. - - Args: - name: Human-readable identifier for this input slot. Used as the - source label when auto-promoting concrete sources in - ``PipelineJob``. Must be unique within a pipeline. - tag_schema: Mapping of tag column names to Python types. - data_schema: Mapping of data column names to Python types. - data_context: Optional data context override. Defaults to the default data context. - """ - - def __init__( - self, - name: str, - tag_schema: Schema, - data_schema: Schema, - data_context: str | contexts.DataContext | None = None, - ) -> None: - super().__init__(data_context=data_context) - self._name = name - self._tag_schema = tag_schema - self._data_schema = data_schema - - # ------------------------------------------------------------------ - # Identity - # ------------------------------------------------------------------ - - @property - def name(self) -> str: - """Human-readable name for this input slot.""" - return self._name - - def computed_label(self) -> str | None: - """Return the spec name as the computed label. - - Implements the ``LabelableMixin.computed_label()`` hook so that - ``self.label`` resolves to the spec name without needing an explicit - label assignment. An explicit ``label`` assignment (via the setter) - would still take priority, but is not expected for immutable specs. - - Returns: - The spec name. - """ - return self._name - - @property - def tag_schema(self) -> Schema: - """Key schema for this input slot.""" - return self._tag_schema - - @property - def data_schema(self) -> Schema: - """Data schema for this input slot.""" - return self._data_schema - - # ------------------------------------------------------------------ - # ContentIdentifiableBase - # ------------------------------------------------------------------ - - def identity_structure(self) -> Any: - """Content identity includes name + both schemas.""" - return ("SourceSpec", self._name, self._tag_schema, self._data_schema) - - # ------------------------------------------------------------------ - # PipelineElementBase - # ------------------------------------------------------------------ - - def pipeline_identity_structure(self) -> Any: - """Pipeline identity is schema-only (no name). - - Matches ``RootSource.pipeline_identity_structure()`` so that a - schema-compatible concrete source and this spec share the same - pipeline hash — and therefore the same DB table paths. - """ - return (self._tag_schema, self._data_schema) - - # ------------------------------------------------------------------ - # StreamProtocol surface (minimal — no data access) - # ------------------------------------------------------------------ - - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """Return ``(tag_schema, data_schema)``. - - Args: - columns: Ignored — SourceSpec always returns the full declared schemas. - all_info: Ignored — SourceSpec always returns the full declared schemas. - - Returns: - Tuple of ``(tag_schema, data_schema)``. - """ - return (self._tag_schema, self._data_schema) - - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Return ``(tag_keys, data_keys)``. - - Args: - columns: Ignored. - all_info: Ignored. - - Returns: - Tuple of ``(tag_column_names, data_column_names)``. - """ - return (tuple(self._tag_schema.keys()), tuple(self._data_schema.keys())) - - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Raise ``UnboundSourceError`` — spec is not bound to a concrete source. - - Raises: - UnboundSourceError: Always. - """ - raise UnboundSourceError( - f"SourceSpec '{self._name}' is not bound to a concrete source. " - "Call PipelineJob.bind(sources={...}) to attach a source before running." - ) - - def as_table( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> "pa.Table": - """Raise ``UnboundSourceError`` — spec is not bound to a concrete source. - - Raises: - UnboundSourceError: Always. - """ - raise UnboundSourceError( - f"SourceSpec '{self._name}' is not bound to a concrete source. " - "Call PipelineJob.bind(sources={...}) to attach a source before running." - ) - - # ------------------------------------------------------------------ - # Validation - # ------------------------------------------------------------------ - - def validate(self, source: "StreamProtocol") -> None: - """Check that *source* is schema-compatible with this spec. - - Validates that the source's tag and data schemas have exactly the - same columns as this spec (no extra, no missing columns). Type - compatibility is not checked here — Arrow conversion handles - coercions at runtime. - - Args: - source: The concrete source to validate. - - Raises: - SourceSpecMismatchError: If the source schema does not match. - """ - source_tag, source_data = source.output_schema() - - tag_issues: list[str] = [] - data_issues: list[str] = [] - - spec_tag_cols = set(self._tag_schema.keys()) - src_tag_cols = set(source_tag.keys()) - if spec_tag_cols != src_tag_cols: - missing = spec_tag_cols - src_tag_cols - extra = src_tag_cols - spec_tag_cols - if missing: - tag_issues.append(f"missing tag columns: {sorted(missing)}") - if extra: - tag_issues.append(f"unexpected tag columns: {sorted(extra)}") - - spec_data_cols = set(self._data_schema.keys()) - src_data_cols = set(source_data.keys()) - if spec_data_cols != src_data_cols: - missing = spec_data_cols - src_data_cols - extra = src_data_cols - spec_data_cols - if missing: - data_issues.append(f"missing data columns: {sorted(missing)}") - if extra: - data_issues.append(f"unexpected data columns: {sorted(extra)}") - - if tag_issues or data_issues: - all_issues = tag_issues + data_issues - raise SourceSpecMismatchError( - f"SourceSpec '{self._name}' is not compatible with the provided source. " - + "; ".join(all_issues) - ) - - # ------------------------------------------------------------------ - # Repr - # ------------------------------------------------------------------ - - def __repr__(self) -> str: - return ( - f"SourceSpec(name={self._name!r}, " - f"tag_schema={dict(self._tag_schema)!r}, " - f"data_schema={dict(self._data_schema)!r})" - ) diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index a7a7c01ba..91e601710 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -37,17 +37,20 @@ class FieldNotResolvableError(LookupError): class UnboundSourceError(RuntimeError): - """Raised when a data-producing method is called on an unbound SourceSpec. + """Raised when a data-producing method is called on an unbound SourceNode. - Occurs when ``iter_data()`` or ``as_table()`` is called on a ``SourceSpec`` + Occurs when ``iter_data()`` or ``as_table()`` is called on a ``SourceNode`` that has not been bound to a concrete source in a ``PipelineJob``. """ class SourceSpecMismatchError(ValueError): - """Raised when a concrete source's schema is incompatible with a SourceSpec. + """Raised when a concrete source's schema is incompatible with a SourceNode slot. - Contains the spec name and a description of the incompatible field(s). + The class name ``SourceSpecMismatchError`` is preserved for compatibility + with any code that catches it by name. + + Contains the slot name and a description of the incompatible field(s). Raised at ``bind()`` time — schema mismatches are rejected before execution. """ diff --git a/test-objective/unit/test_tracker.py b/test-objective/unit/test_tracker.py index a5eda45d7..f3a577499 100644 --- a/test-objective/unit/test_tracker.py +++ b/test-objective/unit/test_tracker.py @@ -13,7 +13,7 @@ from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource -from orcapod.core.sources.source_spec import SourceSpec +from orcapod.core.nodes.source_node import SourceNode from orcapod.core.tracker import BasicTrackerManager from orcapod.pipeline import Pipeline from orcapod.types import Schema @@ -111,8 +111,8 @@ def test_compile_builds_graph(self): pf = PythonDataFunction(_double, output_keys="result") pod = FunctionPod(data_function=pf) - # compile() enforces SourceSpec-only leaves — use SourceSpec instead of raw stream - spec = SourceSpec( + # compile() enforces SourceNode-only leaves — use SourceNode instead of raw stream + spec = SourceNode( name="test_spec", tag_schema=Schema({"id": int}), data_schema=Schema({"x": int}), diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 0d9ea2d4d..12ce88b00 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -18,23 +18,65 @@ def data_schema(): class TestSourceNodeHashStability: - """SourceNode must produce bit-identical hashes to SourceSpec with the same args.""" + """SourceNode must produce stable, deterministic hashes. - def test_content_hash_matches_source_spec(self, tag_schema, data_schema): + Hash values below were verified to be bit-identical to SourceSpec + with the same arguments during the SourceSpec→SourceNode migration (ENG-493). + SourceSpec has since been deleted; these values serve as the stability anchor. + + For tag_schema=Schema({"id": int}), data_schema=Schema({"value": float}), + name="slot_a": + content_hash = semantic_v0.1:df0cba56fd880f86584ef89b35ef850bd813c95c114ac3bc84818e195b2175cb + pipeline_hash = semantic_v0.1:3e32b07447e313318744ce498086c21ad136a40596f833c05162088e840ad16e + """ + + def test_content_hash_is_deterministic(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode + + node_a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + node_b = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + assert node_a.content_hash() == node_b.content_hash() + + def test_content_hash_stable_value(self, tag_schema, data_schema): + """content_hash must match the value anchored at migration time. + + Digest (hex): df0cba56fd880f86584ef89b35ef850bd813c95c114ac3bc84818e195b2175cb + """ from orcapod.core.nodes.source_node import SourceNode - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.types import ContentHash - spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - assert node.content_hash() == spec.content_hash() + expected = ContentHash( + method="semantic_v0.1", + digest=bytes.fromhex( + "df0cba56fd880f86584ef89b35ef850bd813c95c114ac3bc84818e195b2175cb" + ), + ) + assert node.content_hash() == expected + + def test_pipeline_hash_is_deterministic(self, tag_schema, data_schema): + from orcapod.core.nodes.source_node import SourceNode - def test_pipeline_hash_matches_source_spec(self, tag_schema, data_schema): + node_a = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + node_b = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) + assert node_a.pipeline_hash() == node_b.pipeline_hash() + + def test_pipeline_hash_stable_value(self, tag_schema, data_schema): + """pipeline_hash must match the value anchored at migration time. + + Digest (hex): 3e32b07447e313318744ce498086c21ad136a40596f833c05162088e840ad16e + """ from orcapod.core.nodes.source_node import SourceNode - from orcapod.core.sources.source_spec import SourceSpec + from orcapod.types import ContentHash - spec = SourceSpec(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) node = SourceNode(name="slot_a", tag_schema=tag_schema, data_schema=data_schema) - assert node.pipeline_hash() == spec.pipeline_hash() + expected = ContentHash( + method="semantic_v0.1", + digest=bytes.fromhex( + "3e32b07447e313318744ce498086c21ad136a40596f833c05162088e840ad16e" + ), + ) + assert node.pipeline_hash() == expected def test_different_names_different_content_hash(self, tag_schema, data_schema): from orcapod.core.nodes.source_node import SourceNode diff --git a/tests/test_core/sources/test_source_spec.py b/tests/test_core/sources/test_source_spec.py deleted file mode 100644 index 07a6e3743..000000000 --- a/tests/test_core/sources/test_source_spec.py +++ /dev/null @@ -1,129 +0,0 @@ -from __future__ import annotations - -import pyarrow as pa -import pytest - -from orcapod.core.sources import ArrowTableSource -from orcapod.core.sources.source_spec import SourceSpec -from orcapod.errors import UnboundSourceError, SourceSpecMismatchError -from orcapod.types import Schema - - -def _make_source(tag_col: str, data_col: str) -> ArrowTableSource: - table = pa.table( - { - tag_col: pa.array(["a", "b"], type=pa.large_string()), - data_col: pa.array([1, 2], type=pa.int64()), - } - ) - return ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) - - -class TestSourceSpecConstruction: - def test_construct_with_name_and_schemas(self): - spec = SourceSpec( - name="my_source", - tag_schema=Schema({"key": str}), - data_schema=Schema({"value": int}), - ) - assert spec.name == "my_source" - assert "key" in spec.tag_schema - assert "value" in spec.data_schema - - def test_output_schema_returns_tag_and_data(self): - tag = Schema({"key": str}) - data = Schema({"value": int}) - spec = SourceSpec(name="s", tag_schema=tag, data_schema=data) - out_tag, out_data = spec.output_schema() - assert out_tag == tag - assert out_data == data - - def test_keys_returns_tag_column_names(self): - spec = SourceSpec( - name="s", - tag_schema=Schema({"key": str, "group": str}), - data_schema=Schema({"value": int}), - ) - tag_keys, data_keys = spec.keys() - assert set(tag_keys) == {"key", "group"} - assert set(data_keys) == {"value"} - - def test_label_returns_name(self): - spec = SourceSpec(name="my_spec", tag_schema=Schema({"k": str}), data_schema=Schema({"v": int})) - assert spec.label == "my_spec" - - -class TestSourceSpecHashing: - def test_pipeline_hash_matches_compatible_source(self): - """SourceSpec.pipeline_hash() must equal a schema-compatible source's pipeline_hash().""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name="my_source", tag_schema=tag_schema, data_schema=data_schema) - - assert spec.pipeline_hash() == source.pipeline_hash() - - def test_content_hash_differs_by_name(self): - """Two specs with the same schema but different names must have different content hashes.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec_a = SourceSpec(name="source_a", tag_schema=tag_schema, data_schema=data_schema) - spec_b = SourceSpec(name="source_b", tag_schema=tag_schema, data_schema=data_schema) - - assert spec_a.content_hash() != spec_b.content_hash() - - def test_pipeline_hash_same_for_different_names(self): - """SourceSpec.pipeline_hash() must be schema-only (ignoring name).""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec_a = SourceSpec(name="a", tag_schema=tag_schema, data_schema=data_schema) - spec_b = SourceSpec(name="b", tag_schema=tag_schema, data_schema=data_schema) - - assert spec_a.pipeline_hash() == spec_b.pipeline_hash() - - def test_content_hash_stable(self): - """Same name + schemas → same content hash across calls.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name="s", tag_schema=tag_schema, data_schema=data_schema) - assert spec.content_hash() == spec.content_hash() - - -class TestSourceSpecUnboundBehavior: - def test_iter_data_raises_unbound_error(self): - spec = SourceSpec(name="s", tag_schema=Schema({"k": str}), data_schema=Schema({"v": int})) - with pytest.raises(UnboundSourceError, match="s"): - list(spec.iter_data()) - - def test_as_table_raises_unbound_error(self): - spec = SourceSpec(name="s", tag_schema=Schema({"k": str}), data_schema=Schema({"v": int})) - with pytest.raises(UnboundSourceError, match="s"): - spec.as_table() - - -class TestSourceSpecValidate: - def test_validate_passes_for_compatible_source(self): - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - spec = SourceSpec(name="s", tag_schema=tag_schema, data_schema=data_schema) - # Should not raise - spec.validate(source) - - def test_validate_raises_for_extra_tag_column(self): - """Spec requires an extra tag column not present in source → SourceSpecMismatchError.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - # Spec declares an extra tag column ("unexpected") that the source does not have - extra_tag = Schema({"key": str, "unexpected": str}) - spec = SourceSpec(name="s", tag_schema=extra_tag, data_schema=data_schema) - with pytest.raises(SourceSpecMismatchError): - spec.validate(source) - - def test_validate_raises_for_missing_data_column(self): - """Source missing a required data column → SourceSpecMismatchError.""" - source = _make_source("key", "value") - tag_schema, data_schema = source.output_schema() - # Spec requires an extra data column the source doesn't have - wider_data = Schema({"value": int, "extra": str}) - spec = SourceSpec(name="s", tag_schema=tag_schema, data_schema=wider_data) - with pytest.raises(SourceSpecMismatchError): - spec.validate(source) From ed7f7bf211fe4fe3bfbe385351962d2422640080 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 07:22:28 +0000 Subject: [PATCH 16/24] =?UTF-8?q?fix(pipeline):=20address=20Copilot=20PR?= =?UTF-8?q?=20review=20=E2=80=94=20validate=20sources=20in=20from=5Fpipeli?= =?UTF-8?q?ne(),=20rewire=20as=5Fpipeline()=20upstreams,=20unskip=20node?= =?UTF-8?q?=5Furi=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues addressed from PR #141 review: 1. PipelineJob.from_pipeline() now validates the `sources` dict against SourceNode schemas before creating job nodes — mirrors the identical validation already present in bind(). Raises ValueError on unknown source keys; raises SourceSpecMismatchError on schema mismatch. 2. PipelineJob.as_pipeline() rewires FunctionNode/OperatorNode upstreams topologically so returned blueprint nodes reference other blueprint nodes (not job nodes). Builds a reverse id→blueprint-hash lookup to preserve original _input_streams order for non-commutative operators. This ensures every node's content_hash() matches its key in _persistent_node_map. 3. Unskip test_function_node_has_node_uri and test_operator_node_has_node_uri in test_serialization_helpers.py. Updated to use SourceNode + Schema declarations (new Pipeline API) instead of ArrowTableSource + old pipeline_database constructor argument. Closes ENG-493 (review items 3, 6, 9). Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/pipeline/job.py | 79 +++++++++++++++++-- .../test_serialization_helpers.py | 36 +++------ 2 files changed, 82 insertions(+), 33 deletions(-) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 6960100a0..df4d6cf43 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -409,6 +409,23 @@ def from_pipeline( bound_sources: dict[str, cp.StreamProtocol] = dict(sources or {}) + # Validate sources against SourceNode schemas (mirrors bind() validation). + if bound_sources: + spec_names = { + node.name + for node in pipeline._persistent_node_map.values() + if isinstance(node, SourceNodeBase) + } + unknown = set(bound_sources.keys()) - spec_names + if unknown: + raise ValueError( + f"from_pipeline() received source keys with no matching SourceNode: " + f"{sorted(unknown)}. Known names: {sorted(spec_names)}" + ) + for node in pipeline._persistent_node_map.values(): + if isinstance(node, SourceNodeBase) and node.name in bound_sources: + node.validate(bound_sources[node.name]) + G = pipeline._hash_graph job_node_map: dict[str, object] = {} @@ -607,20 +624,24 @@ def _distribute_databases(self) -> None: def as_pipeline(self) -> "Pipeline": """Return the lightweight ``Pipeline`` blueprint for this job. - Walks ``_persistent_node_map`` and calls ``.as_node()`` on each - ``JobNode`` to obtain the corresponding lightweight node. The returned - ``Pipeline`` has identical ``_persistent_node_map`` keys (content hashes) - as this job, but with lightweight blueprint nodes instead of job nodes. + Walks ``_persistent_node_map`` topologically and rewires each job node + to a fresh blueprint node whose upstreams point at the already-built + blueprint nodes in ``node_map`` (not at job nodes). This ensures that + the content_hash of every node in the returned ``Pipeline`` matches its + key in ``_persistent_node_map``. Returns: A compiled ``Pipeline`` whose ``_persistent_node_map`` contains only lightweight ``SourceNode`` / ``FunctionNode`` / ``OperatorNode`` - objects. + objects with blueprint (non-job) upstream references. Raises: RuntimeError: If this job has no compiled pipeline. """ import networkx as _nx + from orcapod.core.nodes.function_node import FunctionJobNode, FunctionNode + from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNode + from orcapod.core.nodes.source_node import SourceJobNode from orcapod.pipeline.graph import Pipeline if self._compiled_pipeline is None: @@ -631,13 +652,55 @@ def as_pipeline(self) -> "Pipeline": ) G = self._compiled_pipeline._hash_graph + persistent = self._persistent_node_map or {} node_map: dict[str, object] = {} + # Build a reverse lookup from Python object identity to blueprint hash so + # that FunctionJobNode._input_stream and OperatorJobNode._input_streams + # (which are themselves job nodes) can be mapped to their blueprint-hash keys. + job_id_to_bp_hash: dict[int, str] = { + id(job_node): bp_hash for bp_hash, job_node in persistent.items() + } + for node_hash in _nx.topological_sort(G): - if node_hash not in (self._persistent_node_map or {}): + if node_hash not in persistent: continue - job_node = self._persistent_node_map[node_hash] - node_map[node_hash] = job_node.as_node() + job_node = persistent[node_hash] + + if isinstance(job_node, SourceJobNode): + # SourceNode has no upstream — as_node() is safe as-is. + node_map[node_hash] = job_node.as_node() + + elif isinstance(job_node, FunctionJobNode): + # Wire _input_stream to the already-built blueprint upstream so + # the resulting FunctionNode.content_hash() == node_hash. + upstream_bp_hash = job_id_to_bp_hash[id(job_node._input_stream)] + node_map[node_hash] = FunctionNode( + function_pod=job_node._function_pod, + input_stream=node_map[upstream_bp_hash], + label=job_node._label, + table_scope=job_node._table_scope, + tracker_manager=job_node.tracker_manager, + ) + + elif isinstance(job_node, OperatorJobNode): + # Preserve original _input_streams order (important for non-commutative + # operators such as SemiJoin) via the object-identity reverse lookup. + blueprint_upstreams = tuple( + node_map[job_id_to_bp_hash[id(s)]] + for s in job_node._input_streams + ) + node_map[node_hash] = OperatorNode( + operator=job_node._operator, + input_streams=blueprint_upstreams, + label=job_node._label, + table_scope=job_node._table_scope, + tracker_manager=job_node.tracker_manager, + ) + + else: + # Fallback for any future node types — may not rewire upstreams. + node_map[node_hash] = job_node.as_node() pipeline = Pipeline(name=self._name, auto_compile=False) pipeline._graph_edges = list(self._compiled_pipeline._graph_edges) diff --git a/tests/test_pipeline/test_serialization_helpers.py b/tests/test_pipeline/test_serialization_helpers.py index 9a0885036..616e612c2 100644 --- a/tests/test_pipeline/test_serialization_helpers.py +++ b/tests/test_pipeline/test_serialization_helpers.py @@ -723,31 +723,25 @@ def test_source_proxy_from_config_backward_compat(): # --------------------------------------------------------------------------- -@pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") def test_function_node_has_node_uri(): - import pyarrow as pa - from orcapod.core.function_pod import FunctionPod - from orcapod.core.nodes import FunctionNode + from orcapod.core.nodes import FunctionNode, SourceNode from orcapod.core.data_function import PythonDataFunction - from orcapod.core.sources import ArrowTableSource from orcapod.pipeline import Pipeline - - db = InMemoryArrowDatabase() + from orcapod.types import Schema def add_one(x: int) -> int: return x + 1 - table = pa.table({"id": pa.array(["a", "b"], type=pa.large_string()), "x": pa.array([1, 2], type=pa.int64())}) - source = ArrowTableSource(table, tag_columns=["id"], infer_nullable=True) + source = SourceNode("source", tag_schema=Schema({"id": str}), data_schema=Schema({"x": int})) pf = PythonDataFunction(add_one, output_keys="result") pod = FunctionPod(data_function=pf) - pipeline = Pipeline(name="test", pipeline_database=db) + pipeline = Pipeline(name="test") with pipeline: pod(source, label="fn") - fn_node = pipeline.compiled_nodes["fn"] + fn_node = pipeline._nodes["fn"] assert isinstance(fn_node, FunctionNode) assert hasattr(fn_node, "node_uri") uri = fn_node.node_uri @@ -773,28 +767,20 @@ def test_function_node_stored_node_uri_from_descriptor(): assert node.node_uri == ("add_one", "v0", "python.function.v0", "schema_repr") -@pytest.mark.skip(reason="Migrating to PipelineJob-based API — pending migration task") def test_operator_node_has_node_uri(): - import pyarrow as pa - - from orcapod.core.nodes import OperatorNode + from orcapod.core.nodes import OperatorNode, SourceNode from orcapod.core.operators import Join - from orcapod.core.sources import ArrowTableSource from orcapod.pipeline import Pipeline + from orcapod.types import Schema - db = InMemoryArrowDatabase() - table_a = pa.table({"key": pa.array(["a"], type=pa.large_string()), "val_a": pa.array([10], type=pa.int64())}) - table_b = pa.table( - {"key": pa.array(["a"], type=pa.large_string()), "val_b": pa.array([1], type=pa.int64())} - ) - src_a = ArrowTableSource(table_a, tag_columns=["key"], infer_nullable=True) - src_b = ArrowTableSource(table_b, tag_columns=["key"], infer_nullable=True) + src_a = SourceNode("src_a", tag_schema=Schema({"key": str}), data_schema=Schema({"val_a": int})) + src_b = SourceNode("src_b", tag_schema=Schema({"key": str}), data_schema=Schema({"val_b": int})) - pipeline = Pipeline(name="test", pipeline_database=db) + pipeline = Pipeline(name="test") with pipeline: Join()(src_a, src_b, label="joined") - op_node = pipeline.compiled_nodes["joined"] + op_node = pipeline._nodes["joined"] assert isinstance(op_node, OperatorNode) assert hasattr(op_node, "node_uri") uri = op_node.node_uri From 72e99e1fe04ebccdcb6348b19984741ca3c3f85e Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 15:22:47 +0000 Subject: [PATCH 17/24] test(test-objective): migrate test-objective node tests to job-node API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FunctionNode and OperatorNode are now pure blueprints (PipelineJobRequiredError on iter_data/as_table). Update spec-derived tests to use FunctionJobNode and OperatorJobNode (the executable variants) so CI passes. - TestFunctionNode: iter_data/process_data/clear_cache → FunctionJobNode with run() and execute_data() - TestOperatorNode: delegates/clear_cache → OperatorJobNode (no database) - test_caching_flows.py: already fixed in prior commit (FunctionJobNode/ OperatorJobNode imports + usage) Co-Authored-By: Claude Sonnet 4.6 --- .../integration/test_caching_flows.py | 16 ++++--- test-objective/unit/test_nodes.py | 47 +++++++++++-------- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/test-objective/integration/test_caching_flows.py b/test-objective/integration/test_caching_flows.py index 9afb838db..e6c0dc952 100644 --- a/test-objective/integration/test_caching_flows.py +++ b/test-objective/integration/test_caching_flows.py @@ -14,6 +14,8 @@ FunctionNode, OperatorNode, ) +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import Join from orcapod.core.data_function import CachedDataFunction, PythonDataFunction from orcapod.core.sources import ArrowTableSource, DerivedSource @@ -55,7 +57,7 @@ def test_first_run_computes_all(self): source = _make_source(3) pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=source, pipeline_database=pipeline_db, @@ -74,7 +76,7 @@ def test_second_run_uses_cache(self): result_db = InMemoryArrowDatabase() # First run - node1 = FunctionNode( + node1 = FunctionJobNode( function_pod=pod, input_stream=source, pipeline_database=pipeline_db, @@ -83,7 +85,7 @@ def test_second_run_uses_cache(self): node1.run() # Second run with same inputs — should use cached results - node2 = FunctionNode( + node2 = FunctionJobNode( function_pod=pod, input_stream=source, pipeline_database=pipeline_db, @@ -103,7 +105,7 @@ def test_derived_source_as_pipeline_input(self): pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=source, pipeline_database=pipeline_db, @@ -151,7 +153,7 @@ def test_log_mode_stores_results(self): ) join = Join() db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( operator=join, input_streams=[source_a, source_b], pipeline_database=db, @@ -187,7 +189,7 @@ def test_replay_mode_loads_from_db(self): db = InMemoryArrowDatabase() # First: LOG - node1 = OperatorNode( + node1 = OperatorJobNode( operator=join, input_streams=[source_a, source_b], pipeline_database=db, @@ -196,7 +198,7 @@ def test_replay_mode_loads_from_db(self): node1.run() # Second: REPLAY - node2 = OperatorNode( + node2 = OperatorJobNode( operator=join, input_streams=[source_a, source_b], pipeline_database=db, diff --git a/test-objective/unit/test_nodes.py b/test-objective/unit/test_nodes.py index 39acdcb27..4a57e8387 100644 --- a/test-objective/unit/test_nodes.py +++ b/test-objective/unit/test_nodes.py @@ -18,6 +18,8 @@ FunctionNode, OperatorNode, ) +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, DerivedSource @@ -75,7 +77,9 @@ def test_iter_data(self): pf = PythonDataFunction(_double, output_keys="result") pod = FunctionPod(data_function=pf) stream = _make_stream(3) - node = FunctionNode(function_pod=pod, input_stream=stream) + # FunctionNode is a blueprint; FunctionJobNode is the executable variant + node = FunctionJobNode(function_pod=pod, input_stream=stream) + node.run() data = list(node.iter_data()) assert len(data) == 3 for tag, data in data: @@ -85,10 +89,11 @@ def test_process_data(self): pf = PythonDataFunction(_double, output_keys="result") pod = FunctionPod(data_function=pf) stream = _make_stream() - node = FunctionNode(function_pod=pod, input_stream=stream) + # FunctionJobNode.execute_data() is the per-item processing method + node = FunctionJobNode(function_pod=pod, input_stream=stream) # Get first tag/data from input tag, data = next(iter(stream.iter_data())) - out_tag, out_data = node.process_data(tag, data) + out_tag, out_data = node.execute_data(tag, data) assert out_data is not None assert "result" in out_data.keys() @@ -110,10 +115,12 @@ def test_clear_cache(self): pf = PythonDataFunction(_double, output_keys="result") pod = FunctionPod(data_function=pf) stream = _make_stream() - node = FunctionNode(function_pod=pod, input_stream=stream) - list(node.iter_data()) + # FunctionJobNode is the executable variant that supports clear_cache + re-run + node = FunctionJobNode(function_pod=pod, input_stream=stream) + node.run() node.clear_cache() - # Should be able to iterate again after clearing + # Should be able to run and iterate again after clearing + node.run() data = list(node.iter_data()) assert len(data) == 3 @@ -123,7 +130,7 @@ def test_clear_cache(self): # =================================================================== -class TestFunctionNode: +class TestFunctionJobNode: """Per design: two-phase iteration — Phase 1 returns cached records, Phase 2 computes missing. Uses pipeline_hash for DB path scoping.""" @@ -133,7 +140,7 @@ def test_caches_computed_results(self): stream = _make_stream(3) pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=stream, pipeline_database=pipeline_db, @@ -150,7 +157,7 @@ def test_run_eagerly_processes_all(self): stream = _make_stream(3) pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=stream, pipeline_database=pipeline_db, @@ -168,7 +175,7 @@ def test_as_source_returns_derived_source(self): stream = _make_stream(3) pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=stream, pipeline_database=pipeline_db, @@ -184,7 +191,7 @@ def test_node_identity_path_uses_pipeline_hash(self): stream = _make_stream() pipeline_db = InMemoryArrowDatabase() result_db = InMemoryArrowDatabase() - node = FunctionNode( + node = FunctionJobNode( function_pod=pod, input_stream=stream, pipeline_database=pipeline_db, @@ -206,7 +213,8 @@ class TestOperatorNode: def test_delegates_to_operator(self): join = Join() s1, s2 = _make_joinable_streams() - node = OperatorNode(operator=join, input_streams=[s1, s2]) + # OperatorNode is a blueprint; OperatorJobNode is the executable variant + node = OperatorJobNode(operator=join, input_streams=[s1, s2]) node.run() table = node.as_table() assert table.num_rows == 2 # Inner join on id=2, id=3 @@ -214,7 +222,8 @@ def test_delegates_to_operator(self): def test_clear_cache(self): join = Join() s1, s2 = _make_joinable_streams() - node = OperatorNode(operator=join, input_streams=[s1, s2]) + # OperatorJobNode is the executable variant that supports clear_cache + re-run + node = OperatorJobNode(operator=join, input_streams=[s1, s2]) node.run() node.clear_cache() # Should be able to run again @@ -228,7 +237,7 @@ def test_clear_cache(self): # =================================================================== -class TestOperatorNode: +class TestOperatorJobNode: """Per design, supports CacheMode: OFF (always compute), LOG (compute+store), REPLAY (load from DB).""" @@ -236,7 +245,7 @@ def test_cache_mode_off(self): join = Join() s1, s2 = _make_joinable_streams() db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( operator=join, input_streams=[s1, s2], pipeline_database=db, @@ -250,7 +259,7 @@ def test_cache_mode_log(self): join = Join() s1, s2 = _make_joinable_streams() db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( operator=join, input_streams=[s1, s2], pipeline_database=db, @@ -268,7 +277,7 @@ def test_cache_mode_replay(self): db = InMemoryArrowDatabase() # First: LOG to populate DB - node1 = OperatorNode( + node1 = OperatorJobNode( operator=join, input_streams=[s1, s2], pipeline_database=db, @@ -277,7 +286,7 @@ def test_cache_mode_replay(self): node1.run() # Second: REPLAY to load from DB - node2 = OperatorNode( + node2 = OperatorJobNode( operator=join, input_streams=[s1, s2], pipeline_database=db, @@ -291,7 +300,7 @@ def test_as_source_returns_derived_source(self): join = Join() s1, s2 = _make_joinable_streams() db = InMemoryArrowDatabase() - node = OperatorNode( + node = OperatorJobNode( operator=join, input_streams=[s1, s2], pipeline_database=db, From e8d414c9eb93ce143798cbdf0e3673eab316904f Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 15:48:23 +0000 Subject: [PATCH 18/24] fix(pipeline): address Copilot review comments round 2 - operator_node: add OperatorNode.from_descriptor() so Pipeline.load() can reconstruct loaded operator nodes as OperatorNode blueprints (not OperatorJobNode); fixes isinstance(template, OperatorNode) checks in Pipeline.save() and PipelineJob._build_execution_graph() - graph: Pipeline.load() now calls OperatorNode.from_descriptor() and removes the OperatorJobNode import that is no longer needed here - graph: Pipeline.load() now passes data_context_key from the descriptor to SourceNode so non-default contexts survive a save/load round-trip - graph: remove stale "Returns:" entries from record_function_pod_invocation() and record_operator_pod_invocation() docstrings (both return None) - test_serialization: remove duplicate SourceNode import - test_node_protocols: remove unused Schema import Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/operator_node.py | 104 ++++++++++++++++++++ src/orcapod/pipeline/graph.py | 14 +-- tests/test_pipeline/test_serialization.py | 1 - tests/test_protocols/test_node_protocols.py | 2 - 4 files changed, 108 insertions(+), 13 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index ede63a2de..c0a03f125 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -315,6 +315,110 @@ def as_table( "OperatorJobNode." ) + @classmethod + def from_descriptor( + cls, + descriptor: dict[str, Any], + operator: OperatorPodProtocol | None, + input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], + databases: dict[str, Any], + ) -> "OperatorNode": + """Construct an OperatorNode from a serialized descriptor. + + When *operator* and *input_streams* are provided the node operates + in full mode — constructed normally via ``__init__``. When + *operator* is ``None`` the node is created in read-only mode with + metadata from the descriptor; computation methods will raise + ``PipelineJobRequiredError``. + + Args: + descriptor: The serialized node descriptor dict. + operator: An optional live operator instance. ``None`` for + read-only mode. + input_streams: Input streams for the operator. Empty tuple + for read-only mode. + databases: Unused — kept for API parity with + ``OperatorJobNode.from_descriptor``. + + Returns: + A new ``OperatorNode`` instance. + + Raises: + ValueError: If ``table_scope`` is missing or invalid. + """ + from orcapod.pipeline.serialization import LoadStatus + + if "table_scope" not in descriptor: + raise ValueError( + f"OperatorNode descriptor is missing required 'table_scope' field: " + f"{descriptor.get('label', '')}" + ) + raw_table_scope = descriptor["table_scope"] + if raw_table_scope not in ("pipeline_hash", "content_hash"): + raise ValueError( + f"OperatorNode descriptor has invalid 'table_scope' value " + f"{raw_table_scope!r} for {descriptor.get('label', '')}; " + "expected one of ('pipeline_hash', 'content_hash')" + ) + table_scope: Literal["pipeline_hash", "content_hash"] = raw_table_scope + + if operator is not None and input_streams: + # Full mode: construct normally. + node = cls( + operator=operator, + input_streams=input_streams, + label=descriptor.get("label"), + table_scope=table_scope, + ) + node._descriptor = descriptor + node._load_status = LoadStatus.FULL + return node + + # Read-only mode: bypass __init__ (which calls validate_inputs) and + # manually set the minimum required state. + node = cls.__new__(cls) + + # From LabelableMixin + node._label = descriptor.get("label") + + # From DataContextMixin + from orcapod.config import DEFAULT_CONFIG + + node._data_context = contexts.resolve_context( + descriptor.get("data_context_key") + ) + node._orcapod_config = DEFAULT_CONFIG + + # From ContentIdentifiableBase + node._content_hash_cache = {} + node._cached_int_hash = None + + # From PipelineElementBase + node._pipeline_hash_cache = {} + + # From TemporalMixin + node._modified_time = None + + # From OperatorNodeBase + node._operator = None + node._input_streams = () + node.tracker_manager = DEFAULT_TRACKER_MANAGER + + # Descriptor metadata for read-only access + node._descriptor = descriptor + node._stored_schema = descriptor.get("output_schema", {}) + node._stored_content_hash = descriptor.get("content_hash") + node._stored_pipeline_hash = descriptor.get("pipeline_hash") + node._stored_pipeline_path = tuple(descriptor.get("pipeline_path", ())) + node._stored_node_uri = tuple(descriptor.get("node_uri") or []) + node._table_scope = table_scope + node._node_identity_path_cache = None + + # Blueprint nodes loaded read-only are always UNAVAILABLE (no DB) + node._load_status = LoadStatus.UNAVAILABLE + + return node + def as_node(self) -> "OperatorNode": """Return ``self`` — already the lightweight blueprint form. diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index e1cd4ab80..243bc8ec9 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -12,7 +12,6 @@ OperatorNode, SourceNode, ) -from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.tracker import AutoRegisteringContextBasedTracker from orcapod.pipeline.base import AbstractPipelineBase from orcapod.protocols import core_protocols as cp @@ -83,7 +82,7 @@ def record_function_pod_invocation( input_stream: cp.StreamProtocol, label: str | None = None, ) -> None: - """Record a function pod invocation and return its stream. + """Record a function pod invocation. Called by ``FunctionPod.__call__`` when used inside a ``with pipeline:`` block. Creates a lightweight ``FunctionNode`` blueprint. @@ -92,9 +91,6 @@ def record_function_pod_invocation( pod: The function pod being invoked. input_stream: The upstream stream. label: Optional display label for the resulting node. - - Returns: - The ``FunctionNode`` representing this invocation. """ input_stream_hash = input_stream.content_hash().to_string() function_node = FunctionNode( @@ -116,7 +112,7 @@ def record_operator_pod_invocation( upstreams: tuple[cp.StreamProtocol, ...] = (), label: str | None = None, ) -> None: - """Record an operator pod invocation and return its stream. + """Record an operator pod invocation. Called by operator pods when used inside a ``with pipeline:`` block. Creates a lightweight ``OperatorNode`` blueprint. @@ -125,9 +121,6 @@ def record_operator_pod_invocation( pod: The operator pod being invoked. upstreams: Upstream streams for this operator. label: Optional display label for the resulting node. - - Returns: - The ``OperatorNode`` representing this invocation. """ operator_node = OperatorNode( operator=pod, @@ -491,6 +484,7 @@ def load(cls, path: str | Path) -> "Pipeline": name=node_name, tag_schema=tag_schema, data_schema=data_schema, + data_context=descriptor.get("data_context_key"), ) # Restore label from descriptor if set explicitly stored_label = descriptor.get("label") @@ -531,7 +525,7 @@ def load(cls, path: str | Path) -> "Pipeline": op_config.get("class_name"), exc, ) - node = OperatorJobNode.from_descriptor( + node = OperatorNode.from_descriptor( descriptor, operator=operator, input_streams=upstream_nodes, databases={} ) reconstructed[node_hash] = node diff --git a/tests/test_pipeline/test_serialization.py b/tests/test_pipeline/test_serialization.py index 7e2354bfa..9fa588bfe 100644 --- a/tests/test_pipeline/test_serialization.py +++ b/tests/test_pipeline/test_serialization.py @@ -8,7 +8,6 @@ import pytest from orcapod.core.nodes import SourceNode -from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join from orcapod.core.sources import ArrowTableSource from orcapod.databases.in_memory_databases import InMemoryArrowDatabase diff --git a/tests/test_protocols/test_node_protocols.py b/tests/test_protocols/test_node_protocols.py index f14f33b11..2e59aa82f 100644 --- a/tests/test_protocols/test_node_protocols.py +++ b/tests/test_protocols/test_node_protocols.py @@ -12,8 +12,6 @@ is_operator_node, is_source_node, ) -from orcapod.types import Schema - @pytest.fixture def _sample_source(): From 3bdcfc9a28c5932db5137e14110f1f6d6fb23c37 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 16:52:04 +0000 Subject: [PATCH 19/24] fix(nodes): address Copilot review comments round 3 - source_node: SourceNodeBase.execute() and async_execute() now pass self.content_hash().to_string() as node_hash to observer hooks instead of an empty string, so observers that key storage/logging by node hash work correctly - source_node: SourceJobNode.content_hash() no longer forces self.data_context.semantic_hasher when hasher=None; it now delegates as self._concrete.content_hash(hasher) so the concrete source's own hasher is used, preserving the "delegates to concrete" contract - graph: update save()/load()/compile()/class docstrings to replace all stale "SourceSpec" references with "SourceNode" - test_pipeline: remove duplicate SourceNode import (orcapod.core.nodes.source_node shadowed the orcapod.core.nodes export) - test_pipeline/test_node_protocols: remove unused Schema import - test_protocols/test_node_protocols: remove unused SourceNode import (only SourceJobNode is used in the fixtures) Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/source_node.py | 6 ++---- src/orcapod/pipeline/graph.py | 12 ++++++------ tests/test_pipeline/test_node_protocols.py | 1 - tests/test_pipeline/test_pipeline.py | 1 - tests/test_protocols/test_node_protocols.py | 2 +- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 86ff543d3..12621c23b 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -290,7 +290,7 @@ def execute( UnboundSourceError: When no concrete data is available. """ node_label = self.label - node_hash = "" + node_hash = self.content_hash().to_string() if observer is not None: observer.on_node_start(node_label, node_hash) result = list(self.iter_data()) @@ -314,7 +314,7 @@ async def async_execute( UnboundSourceError: When no concrete data is available. """ node_label = self.label - node_hash = "" + node_hash = self.content_hash().to_string() try: if observer is not None: observer.on_node_start(node_label, node_hash) @@ -433,8 +433,6 @@ def content_hash(self, hasher=None) -> ContentHash: ``SourceNodeBase.content_hash(hasher)``. """ if self._concrete is not None: - if hasher is None: - hasher = self.data_context.semantic_hasher return self._concrete.content_hash(hasher) return super().content_hash(hasher) diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 243bc8ec9..ff9b9966a 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -38,7 +38,7 @@ class Pipeline(AbstractPipelineBase): recorded into an internal graph. On context exit, ``compile()`` rewires the graph into a frozen DAG: - - Leaf ``SourceSpec`` declarations → ``SourceNode`` (schema-only placeholders) + - Leaf ``SourceNode`` declarations (schema-only placeholders) - Function pod invocations → ``FunctionNode`` - Operator invocations → ``OperatorNode`` @@ -162,8 +162,8 @@ def compile(self) -> None: Walks the graph in topological order and: - - Verifies leaf streams are ``SourceSpec`` instances (raises ``ValueError`` otherwise) - - Wraps ``SourceSpec`` leaves in ``SourceNode`` + - Verifies leaf streams are ``SourceNode`` instances (raises ``ValueError`` otherwise) + - Leaves ``SourceNode`` leaves as-is (no wrapping needed) - Rewires upstream references on recorded ``FunctionNode`` / ``OperatorNode`` to point at persistent (compiled) nodes @@ -307,7 +307,7 @@ def show_graph(self, **kwargs) -> str | None: def save(self, path: str | Path) -> None: """Serialize the pure pipeline blueprint to a JSON file. - Saves topology and SourceSpec declarations only — no databases, + Saves topology and SourceNode declarations only — no databases, no execution context, no run metadata. Args: @@ -397,7 +397,7 @@ def save(self, path: str | Path) -> None: def load(cls, path: str | Path) -> "Pipeline": """Deserialize a pure pipeline blueprint from a JSON file. - Reconstructs topology and SourceSpec declarations. The loaded + Reconstructs topology and SourceNode declarations. The loaded pipeline is topology-only — to run it, use ``PipelineJob.from_pipeline(pipeline, sources=..., store=...)``. @@ -405,7 +405,7 @@ def load(cls, path: str | Path) -> "Pipeline": path: Path to the JSON file produced by :meth:`save`. Returns: - A compiled ``Pipeline`` instance with SourceSpec leaf nodes. + A compiled ``Pipeline`` instance with SourceNode leaf nodes. Raises: ValueError: If the file's format version is unsupported. diff --git a/tests/test_pipeline/test_node_protocols.py b/tests/test_pipeline/test_node_protocols.py index 543aec681..e6af2485f 100644 --- a/tests/test_pipeline/test_node_protocols.py +++ b/tests/test_pipeline/test_node_protocols.py @@ -145,7 +145,6 @@ def test_dispatch_operator(self): import pyarrow as pa from orcapod.core.sources import ArrowTableSource from orcapod.core.nodes.source_node import SourceJobNode -from orcapod.types import Schema def _make_source_job_node(table, tag_col="key"): diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index 8c7e0ea1d..627bd28dd 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -25,7 +25,6 @@ ) from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.nodes.operator_node import OperatorJobNode, OperatorNodeBase -from orcapod.core.nodes.source_node import SourceNode from orcapod.core.operators import Join from orcapod.core.data_function import PythonDataFunction from orcapod.core.sources import ArrowTableSource, CachedSource diff --git a/tests/test_protocols/test_node_protocols.py b/tests/test_protocols/test_node_protocols.py index 2e59aa82f..f959a9790 100644 --- a/tests/test_protocols/test_node_protocols.py +++ b/tests/test_protocols/test_node_protocols.py @@ -5,7 +5,7 @@ import pyarrow as pa import pytest -from orcapod.core.nodes import FunctionNode, OperatorNode, SourceNode +from orcapod.core.nodes import FunctionNode, OperatorNode from orcapod.core.nodes.source_node import SourceJobNode from orcapod.protocols.node_protocols import ( is_function_node, From 566a935b50896af6d01e27f4d4e56a0462d626cd Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 17:01:47 +0000 Subject: [PATCH 20/24] revert(nodes): restore SourceJobNode.content_hash() hasher override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit removed the `if hasher is None: hasher = self.data_context.semantic_hasher` guard on the advice of a Copilot reviewer. That advice was incorrect. SourceJobNode is the canonical entry point for hash computation in the pipeline context. When no hasher is supplied, the job node should resolve a consistent default from its own data_context and propagate it into the concrete source — not let each layer pick its own default independently. This ensures a single, consistent hasher is used across all nodes within one hashing session, regardless of what data_context the concrete source carries. The original behavior is correct. Callers who need a specific hasher can still pass one in explicitly; hasher=None means "let the job node decide". Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/source_node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 12621c23b..3c0e915c9 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -433,6 +433,8 @@ def content_hash(self, hasher=None) -> ContentHash: ``SourceNodeBase.content_hash(hasher)``. """ if self._concrete is not None: + if hasher is None: + hasher = self.data_context.semantic_hasher return self._concrete.content_hash(hasher) return super().content_hash(hasher) From 39247a414e83ebf8526d5a0aedb224d4018fbc9c Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 23:51:35 +0000 Subject: [PATCH 21/24] refactor(pipeline): address eywalker PR review comments (ENG-493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `_invalidate_content_hash_cache()` / `_invalidate_pipeline_hash_cache()` to `ContentIdentifiableBase` and `PipelineElementBase` as first-class methods; replace all direct `_content_hash_cache.clear()` call sites in FunctionNode, OperatorJobNode, and FunctionJobNode. - `SourceJobNode`: replace fragile `__setattr__` override and `content_hash()` bypass with an explicit `bound_source` property (getter + setter); setter calls `_invalidate_content_hash_cache()` on write. Rename constructor arg from `concrete` to `bound_source` throughout (source, tests, job.py). - `SourceJobNode.identity_structure()`: delegate to `bound_source.identity_structure()` when bound — correct extension point so `content_hash()` caching/resolver logic is not bypassed. - `SourceNodeBase.identity_structure()`: use prefix `"source_node"` instead of `"SourceSpec"` (intentional hash break; stability test updated). - Add `SourceNode.from_stream()` classmethod; use it in `Pipeline.compile()` to auto-wrap concrete leaf streams as `SourceNode` instead of raising `ValueError`. - `Pipeline.compile()`: assert node_type consistency on incremental recompile. - `Pipeline.save()`: replace `isinstance` chain with `match`/`case` class patterns. - `Pipeline.load()`: remove `"spec"` backward-compat branch; only `source_type=="node"`. - `OperatorNodeBase.identity_structure()`: delegate to `pipeline_identity_structure()`. - `OperatorNodeBase.__init__`: broaden `input_streams` type to `Collection[StreamProtocol]`. - `OperatorNode.iter_data()` / `FunctionNode.iter_data()`: add detailed generator semantics comment above `return; yield`. - `as_node()` on OperatorNode / FunctionNode: return fresh clone (not `self`); UNAVAILABLE stubs still return `self`. - Revert `PIPELINE_FORMAT_VERSION` to `"0.1.0"`; remove support for `"0.2"` / `"0.3"`. - Remove all forward-reference quotes from type hints (modules use `from __future__ import annotations`). - Remove unused `compile()` wrapper in PipelineJob. Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/base.py | 20 +++ src/orcapod/core/nodes/function_node.py | 47 +++--- src/orcapod/core/nodes/operator_node.py | 63 +++++--- src/orcapod/core/nodes/source_node.py | 148 +++++++++++------- src/orcapod/pipeline/base.py | 2 +- src/orcapod/pipeline/graph.py | 81 +++++----- src/orcapod/pipeline/job.py | 18 +-- src/orcapod/pipeline/serialization.py | 4 +- .../nodes/test_operator_node_split.py | 8 +- tests/test_core/nodes/test_source_node.py | 32 ++-- tests/test_core/test_tracker.py | 8 +- tests/test_pipeline/test_node_descriptors.py | 2 +- tests/test_pipeline/test_node_protocols.py | 2 +- tests/test_pipeline/test_orchestrator.py | 4 +- tests/test_pipeline/test_pipeline.py | 17 +- tests/test_pipeline/test_serialization.py | 33 ++-- .../test_serialization_helpers.py | 2 +- tests/test_pipeline/test_sync_orchestrator.py | 2 +- tests/test_protocols/test_node_protocols.py | 2 +- 19 files changed, 283 insertions(+), 212 deletions(-) diff --git a/src/orcapod/core/base.py b/src/orcapod/core/base.py index ddd178a12..00c5fc096 100644 --- a/src/orcapod/core/base.py +++ b/src/orcapod/core/base.py @@ -201,6 +201,17 @@ def __eq__(self, other: object) -> bool: return self.identity_structure() == other.identity_structure() + def _invalidate_content_hash_cache(self) -> None: + """Invalidate the cached content hash. + + Call this after any mutation that changes the object's semantic + content so that the next call to ``content_hash()`` recomputes + from scratch. Subclasses must use this method rather than + accessing ``_content_hash_cache`` directly. + """ + self._content_hash_cache.clear() + self._cached_int_hash = None + class PipelineElementBase(DataContextMixin, ABC): """ @@ -270,6 +281,15 @@ def pipeline_resolver(obj: Any) -> ContentHash: ) return self._pipeline_hash_cache[cache_key] + def _invalidate_pipeline_hash_cache(self) -> None: + """Invalidate the cached pipeline hash. + + Call this after any structural mutation (e.g. attaching a database) + that changes this element's pipeline identity. Subclasses must use + this method rather than accessing ``_pipeline_hash_cache`` directly. + """ + self._pipeline_hash_cache.clear() + class TemporalMixin: """ diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 61df1ceaf..c57652503 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -7,9 +7,8 @@ * ``FunctionNode`` — thin blueprint descriptor. Raises ``PipelineJobRequiredError`` on ``iter_data()``. This is the node recorded in a ``Pipeline`` and serialized to disk. -* ``FunctionJobNode`` — DB-backed execution node; carries all DB logic - from the original ``FunctionNode``. Created by ``PipelineJob`` at - run time. +* ``FunctionJobNode`` — DB-backed execution node with full persistence + logic. Created by ``PipelineJob`` at run time. ``FunctionNode`` and ``FunctionJobNode`` are *siblings*: both inherit directly from ``FunctionNodeBase``, neither from the other. @@ -178,7 +177,7 @@ def producer(self) -> FunctionPodProtocol: @property def data_context(self) -> contexts.DataContext: - return contexts.resolve_context(self._function_pod.data_context_key) + return contexts.resolve_context(self.data_context_key) @property def data_context_key(self) -> str: @@ -611,13 +610,29 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: return # pragma: no cover yield # pragma: no cover - def as_node(self) -> "FunctionNode": - """Return ``self`` — already the lightweight blueprint form. + def as_node(self) -> FunctionNode: + """Return the lightweight blueprint equivalent of this node. + + For UNAVAILABLE read-only stubs (loaded with no live function pod), + returns ``self`` — there is nothing to clone. For normal instances, + returns a fresh ``FunctionNode`` with the same function pod, input + stream, label, table scope, and tracker manager. Its + ``content_hash()`` / ``pipeline_hash()`` are identical to those of + this node. Returns: - This instance. + This instance (if unavailable) or a new equivalent ``FunctionNode``. """ - return self + if self._function_pod is None: + # UNAVAILABLE stub — no live function pod, cannot meaningfully clone + return self + return FunctionNode( + function_pod=self._function_pod, + input_stream=self._input_stream, + label=self._label, + table_scope=self._table_scope, + tracker_manager=self.tracker_manager, + ) # --------------------------------------------------------------------------- @@ -665,7 +680,6 @@ def __init__( # DB persistence state (initially None; set via __init__ params or attach_databases) self._pipeline_database: ArrowDatabaseProtocol | None = None self._cached_function_pod: CachedFunctionPod | None = None - self._output_schema_hash: str | None = None if pipeline_database is not None: self.attach_databases( @@ -713,13 +727,8 @@ def attach_databases( # Clear all caches self._node_identity_path_cache = None self.clear_cache() - self._content_hash_cache.clear() - self._pipeline_hash_cache.clear() - - # Compute output schema hash - self._output_schema_hash = self.data_context.semantic_hasher.hash_object( - self._data_function.output_data_schema - ).to_string() + self._invalidate_content_hash_cache() + self._invalidate_pipeline_hash_cache() # ------------------------------------------------------------------ # Override clear_cache to also clear DB caches @@ -1150,7 +1159,7 @@ def get_all_records( self, columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, - ) -> "pa.Table | None": + ) -> pa.Table | None: """Return all computed results joined with their pipeline tag records. Args: @@ -1444,8 +1453,8 @@ def run(self) -> None: async def async_execute( self, - input_channel: "ReadableChannel[tuple[TagProtocol, DataProtocol]]", - output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", + input_channel: ReadableChannel[tuple[TagProtocol, DataProtocol]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, observer: ExecutionObserverProtocol | None = None, ) -> None: diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index c0a03f125..8ebbea0da 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -7,9 +7,8 @@ * ``OperatorNode`` — thin blueprint descriptor. Raises ``PipelineJobRequiredError`` on ``iter_data()``. This is the node recorded in a ``Pipeline`` and serialized to disk. -* ``OperatorJobNode`` — DB-backed execution node; carries all DB logic - from the original ``OperatorNode``. Created by ``PipelineJob`` at - run time. +* ``OperatorJobNode`` — DB-backed execution node with full persistence + logic. Created by ``PipelineJob`` at run time. ``OperatorNode`` and ``OperatorJobNode`` are *siblings*: both inherit directly from ``OperatorNodeBase``, neither from the other. @@ -19,7 +18,7 @@ import asyncio import logging -from collections.abc import Iterator, Sequence +from collections.abc import Collection, Iterator, Sequence from typing import TYPE_CHECKING, Any, Literal from orcapod import contexts @@ -72,7 +71,7 @@ class OperatorNodeBase(StreamBase): def __init__( self, operator: OperatorPodProtocol, - input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], + input_streams: Collection[StreamProtocol], tracker_manager: TrackerManagerProtocol | None = None, label: str | None = None, config: Config | None = None, @@ -132,7 +131,7 @@ def load_status(self) -> Any: # ------------------------------------------------------------------ def identity_structure(self) -> Any: - return (self._operator, self._operator.argument_symmetry(self._input_streams)) + return self.pipeline_identity_structure() def pipeline_identity_structure(self) -> Any: return (self._operator, self._operator.argument_symmetry(self._input_streams)) @@ -163,7 +162,7 @@ def producer(self) -> OperatorPodProtocol: @property def data_context(self) -> contexts.DataContext: - return contexts.resolve_context(self._operator.data_context_key) + return contexts.resolve_context(self.data_context_key) @property def data_context_key(self) -> str: @@ -232,7 +231,7 @@ def node_identity_path(self) -> tuple[str, ...]: return self._stored_pipeline_path if self._node_identity_path_cache is not None: return self._node_identity_path_cache - path = self._operator.uri + (f"schema:{self.pipeline_hash().to_string()}",) + path = self.node_uri + (f"schema:{self.pipeline_hash().to_string()}",) if self._table_scope != "pipeline_hash": path += (f"instance:{self.content_hash().to_string()}",) self._node_identity_path_cache = path @@ -293,7 +292,19 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: "Wrap the containing Pipeline in a PipelineJob to obtain an executable " "OperatorJobNode." ) - # yield is needed to satisfy the Iterator return type annotation + # Python classifies a function as a generator (returning an Iterator + # lazily) based purely on the *syntactic* presence of a ``yield`` + # statement anywhere in the function body — this is determined at + # compile time, not at runtime. Without ``yield``, this would be a + # plain function: calling it would execute the body immediately and + # raise ``PipelineJobRequiredError`` before the caller ever gets an + # iterator object. With ``yield`` present (even though it is + # unreachable due to the ``return`` above), Python compiles the + # function as a generator function. Calling it therefore returns a + # generator object instantly without executing any body code; the + # ``raise`` is deferred until the caller first calls ``next()`` on + # the iterator (i.e., when iteration actually begins). This matches + # the expected ``Iterator`` return-type contract. return # pragma: no cover yield # pragma: no cover @@ -302,7 +313,7 @@ def as_table( *, columns: ColumnConfig | dict[Any, Any] | None = None, all_info: bool = False, - ) -> "pa.Table": + ) -> pa.Table: """Raise ``PipelineJobRequiredError`` — blueprint nodes cannot produce data. Raises: @@ -419,13 +430,29 @@ def from_descriptor( return node - def as_node(self) -> "OperatorNode": - """Return ``self`` — already the lightweight blueprint form. + def as_node(self) -> OperatorNode: + """Return the lightweight blueprint equivalent of this node. + + For UNAVAILABLE read-only stubs (loaded with no live operator), + returns ``self`` — there is nothing to clone. For normal instances, + returns a fresh ``OperatorNode`` with the same operator, input + streams, label, table scope, and tracker manager. Its + ``content_hash()`` / ``pipeline_hash()`` are identical to those of + this node. Returns: - This instance. + This instance (if unavailable) or a new equivalent ``OperatorNode``. """ - return self + if self._operator is None: + # UNAVAILABLE stub — no live operator, cannot meaningfully clone + return self + return OperatorNode( + operator=self._operator, + input_streams=self._input_streams, + label=self._label, + table_scope=self._table_scope, + tracker_manager=self.tracker_manager, + ) # --------------------------------------------------------------------------- @@ -547,8 +574,8 @@ def attach_databases( # Clear caches self._node_identity_path_cache = None self.clear_cache() - self._content_hash_cache.clear() - self._pipeline_hash_cache.clear() + self._invalidate_content_hash_cache() + self._invalidate_pipeline_hash_cache() # ------------------------------------------------------------------ # from_descriptor — reconstruct from a serialized pipeline descriptor @@ -561,7 +588,7 @@ def from_descriptor( operator: OperatorPodProtocol | None, input_streams: tuple[StreamProtocol, ...] | list[StreamProtocol], databases: dict[str, Any], - ) -> "OperatorJobNode": + ) -> OperatorJobNode: """Construct an OperatorJobNode from a serialized descriptor. When *operator* and *input_streams* are provided the node operates @@ -832,7 +859,7 @@ def get_cached_output(self) -> StreamProtocol | None: def execute( self, *input_streams: StreamProtocol, - observer: "ExecutionObserverProtocol | None" = None, + observer: ExecutionObserverProtocol | None = None, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute input streams: compute, persist, and cache. diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 3c0e915c9..1afc4ef43 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -1,15 +1,8 @@ """Source node hierarchy for Pipeline and PipelineJob. -SourceNode — schema-only input-slot declaration (replaces SourceSpec). +SourceNode — schema-only input-slot declaration. SourceJobNode — execution variant that wraps a concrete StreamProtocol. -Both share SourceNodeBase which provides hash-stable identity. - -Hash-stability guarantee: - SourceNode(name=n, tag_schema=t, data_schema=d).content_hash() - == SourceSpec(name=n, tag_schema=t, data_schema=d).content_hash() - -This is achieved by using identical identity_structure(): - ("SourceSpec", name, tag_schema, data_schema) +Both share SourceNodeBase which provides schema-based identity. """ from __future__ import annotations @@ -19,10 +12,11 @@ from typing import TYPE_CHECKING, Any from orcapod import contexts +from orcapod.config import Config from orcapod.core.base import TraceableBase from orcapod.errors import SourceSpecMismatchError, UnboundSourceError from orcapod.protocols.core_protocols import DataProtocol, TagProtocol -from orcapod.types import ColumnConfig, ContentHash, Schema +from orcapod.types import ColumnConfig, Schema if TYPE_CHECKING: import pyarrow as pa @@ -48,6 +42,8 @@ class SourceNodeBase(TraceableBase, ABC): tag_schema: Mapping of tag column names to Python types. data_schema: Mapping of data column names to Python types. data_context: Optional data context override. + label: Optional display label override. + config: Optional config override. """ node_type = "source" @@ -58,31 +54,27 @@ def __init__( tag_schema: Schema, data_schema: Schema, data_context: str | contexts.DataContext | None = None, + label: str | None = None, + config: Config | None = None, ) -> None: - super().__init__(data_context=data_context) + super().__init__(label=label, data_context=data_context, config=config) self._name = name self._tag_schema = tag_schema self._data_schema = data_schema # ------------------------------------------------------------------ - # Identity — hash-stable against old SourceSpec + # Identity # ------------------------------------------------------------------ def identity_structure(self) -> Any: - """Return the content identity: ``("SourceSpec", name, tag_schema, data_schema)``. - - Deliberately matches ``SourceSpec.identity_structure()`` so that a - ``SourceNode`` constructed with the same arguments as a ``SourceSpec`` - produces an identical ``content_hash()``. This preserves all DB paths - computed from pre-refactor pipelines. - """ - return ("SourceSpec", self._name, self._tag_schema, self._data_schema) + """Return the content identity: ``("source_node", name, tag_schema, data_schema)``.""" + return ("source_node", self._name, self._tag_schema, self._data_schema) def pipeline_identity_structure(self) -> Any: """Return the pipeline identity: ``(tag_schema, data_schema)`` (name-independent). - Matches ``RootSource.pipeline_identity_structure()`` so that sources - with identical schemas share the same DB table paths regardless of name. + Sources with identical schemas share the same DB table paths regardless + of name. """ return (self._tag_schema, self._data_schema) @@ -104,7 +96,7 @@ def computed_label(self) -> str | None: Returns: The slot name. """ - return self._name + return self.name @property def tag_schema(self) -> Schema: @@ -230,7 +222,7 @@ def as_table( *, columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, - ) -> "pa.Table": + ) -> pa.Table: """Materialize stream as a PyArrow Table. Delegates to the concrete source (SourceJobNode), or raises for @@ -276,7 +268,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: def execute( self, *, - observer: "ExecutionObserverProtocol | None" = None, + observer: ExecutionObserverProtocol | None = None, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute this source node: materialize and return data. @@ -302,7 +294,7 @@ async def async_execute( self, output: "WritableChannel[tuple[TagProtocol, DataProtocol]]", *, - observer: "ExecutionObserverProtocol | None" = None, + observer: ExecutionObserverProtocol | None = None, ) -> None: """Push all (tag, data) pairs to the output channel. @@ -366,6 +358,35 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: "or job.bind(sources={'': source}) to attach data." ) + @classmethod + def from_stream( + cls, + stream: StreamProtocol, + name: str | None = None, + ) -> SourceNode: + """Wrap *stream* in a ``SourceNode``, or return it unchanged if already one. + + Derives tag and data schemas from ``stream.output_schema()``. The + slot name defaults to ``stream.label`` — no fallback is applied. + + Args: + stream: The upstream stream to wrap. + name: Optional explicit slot name. Defaults to ``stream.label``. + + Returns: + The original *stream* if it is already a ``SourceNode``; otherwise + a new ``SourceNode`` with schemas derived from *stream*. + """ + if isinstance(stream, SourceNode): + return stream + tag_schema, data_schema = stream.output_schema() + slot_name = name if name is not None else stream.label + return cls( + name=slot_name, + tag_schema=tag_schema, + data_schema=data_schema, + ) + class SourceJobNode(SourceNodeBase): """Execution-ready source node wrapping an optional concrete stream. @@ -378,9 +399,9 @@ class SourceJobNode(SourceNodeBase): Hash behaviour: - * ``content_hash()`` — delegates to ``_concrete.content_hash()`` when - bound; falls back to schema-based ``SourceNodeBase.content_hash()`` (== - ``SourceNode.content_hash()``) when unbound. + * ``content_hash()`` — delegates to the bound source's + ``identity_structure()`` when bound (via ``identity_structure()`` + override); falls back to schema-based identity when unbound. * ``pipeline_hash()`` — always schema-based (inherited); never data-inclusive. This invariant keeps DB paths stable across different data sources bound to the same slot. @@ -389,8 +410,8 @@ class SourceJobNode(SourceNodeBase): name: Slot name. tag_schema: Tag schema. data_schema: Data schema. - concrete: Optional concrete stream. Can be set or replaced later via - ``job_node._concrete = source``. + bound_source: Optional concrete stream. Can be set or replaced later + via ``job_node.bound_source = source``. data_context: Optional data context override. """ @@ -399,7 +420,7 @@ def __init__( name: str, tag_schema: Schema, data_schema: Schema, - concrete: "StreamProtocol | None" = None, + bound_source: StreamProtocol | None = None, data_context: str | contexts.DataContext | None = None, ) -> None: super().__init__( @@ -408,35 +429,44 @@ def __init__( data_schema=data_schema, data_context=data_context, ) - # Use object.__setattr__ to bypass the property setter during __init__ - # (the cache doesn't exist yet at this point). - object.__setattr__(self, "_concrete", concrete) + # Direct assignment to the backing attribute — super().__init__() has + # already initialised _content_hash_cache, so the property setter is + # safe to use here; we bypass it only to make the init path explicit. + self._bound_source: StreamProtocol | None = bound_source - def __setattr__(self, name: str, value: object) -> None: - """Clear ``_content_hash_cache`` whenever ``_concrete`` is mutated. + # ------------------------------------------------------------------ + # bound_source property — explicit binding with cache invalidation + # ------------------------------------------------------------------ - This prevents a stale schema-based hash from being returned by the - parent ``content_hash()`` cache after the concrete source is updated. - """ - object.__setattr__(self, name, value) - if name == "_concrete" and hasattr(self, "_content_hash_cache"): - self._content_hash_cache.clear() + @property + def bound_source(self) -> StreamProtocol | None: + """The concrete stream currently bound to this slot, or ``None``.""" + return self._bound_source - def content_hash(self, hasher=None) -> ContentHash: - """Return data-inclusive hash when bound; schema-based hash when unbound. + @bound_source.setter + def bound_source(self, value: StreamProtocol | None) -> None: + """Bind *value* as the concrete source and invalidate the content hash cache.""" + self._bound_source = value + self._invalidate_content_hash_cache() - Args: - hasher: Optional semantic hasher. + # ------------------------------------------------------------------ + # Identity — delegate to bound source when set + # ------------------------------------------------------------------ + + def identity_structure(self) -> Any: + """Delegate to ``bound_source.identity_structure()`` when bound. + + This is the correct extension point: content_hash() flows from + identity_structure(), so overriding here avoids bypassing the + caching and resolver logic in ContentIdentifiableBase.content_hash(). Returns: - ``_concrete.content_hash(hasher)`` when bound, otherwise - ``SourceNodeBase.content_hash(hasher)``. + Bound source's identity structure when bound; schema-based + identity (inherited from SourceNodeBase) when unbound. """ - if self._concrete is not None: - if hasher is None: - hasher = self.data_context.semantic_hasher - return self._concrete.content_hash(hasher) - return super().content_hash(hasher) + if self._bound_source is not None: + return self._bound_source.identity_structure() + return super().identity_structure() def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: """Delegate to concrete source, or raise if unbound. @@ -444,19 +474,19 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: Raises: UnboundSourceError: When no concrete source is attached. """ - if self._concrete is None: + if self._bound_source is None: raise UnboundSourceError( f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before running." ) - return self._concrete.iter_data() + return self._bound_source.iter_data() def as_table( self, *, columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, - ) -> "pa.Table": + ) -> pa.Table: """Materialize the concrete source as a PyArrow Table. Args: @@ -466,12 +496,12 @@ def as_table( Raises: UnboundSourceError: When no concrete source is attached. """ - if self._concrete is None: + if self._bound_source is None: raise UnboundSourceError( f"SourceJobNode '{self._name}' has no concrete source bound. " "Call job.bind(sources={'': source}) before calling as_table()." ) - return self._concrete.as_table(columns=columns, all_info=all_info) + return self._bound_source.as_table(columns=columns, all_info=all_info) def as_node(self) -> SourceNode: """Return the lightweight ``SourceNode`` equivalent of this job node. diff --git a/src/orcapod/pipeline/base.py b/src/orcapod/pipeline/base.py index d0fdb2bbb..732b95c1f 100644 --- a/src/orcapod/pipeline/base.py +++ b/src/orcapod/pipeline/base.py @@ -33,7 +33,7 @@ class AbstractPipelineBase(AutoRegisteringContextBasedTracker, ABC): def __init__( self, name: str | tuple[str, ...] = "pipeline", - tracker_manager: "cp.TrackerManagerProtocol | None" = None, + tracker_manager: cp.TrackerManagerProtocol | None = None, ) -> None: """Initialize shared pipeline state. diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index ff9b9966a..9fa8e7285 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -185,25 +185,25 @@ def compile(self) -> None: for node_hash in nx.topological_sort(G): if node_hash in persistent_node_map: - # Already compiled — reuse, but track for label assignment + # Already compiled — reuse, but verify type consistency across + # incremental compiles (a hash must never change node type). existing_node = persistent_node_map[node_hash] + new_node = self._node_lut.get(node_hash) + if new_node is not None: + assert existing_node.node_type == new_node.node_type, ( + f"Node type changed for hash {node_hash!r}: " + f"was {existing_node.node_type!r}, now {new_node.node_type!r}" + ) name_candidates.setdefault(existing_node.label, []).append( existing_node ) continue if node_hash not in self._node_lut: - # -- Leaf stream: must be a SourceNode in the new design -- + # Leaf stream — wrap in SourceNode if not already one. from orcapod.core.nodes.source_node import SourceNode as SourceNodeClass stream = self._upstreams[node_hash] - if not isinstance(stream, SourceNodeClass): - raise ValueError( - f"Pipeline: all leaf inputs must be SourceNode instances, " - f"but found {type(stream).__name__!r}. " - "Use 'with PipelineJob:' to record a pipeline with concrete sources, " - "or replace concrete sources with SourceNode declarations." - ) - node = stream # SourceNode IS the leaf — no wrapping needed + node = SourceNodeClass.from_stream(stream) persistent_node_map[node_hash] = node else: node = self._node_lut[node_hash] @@ -307,8 +307,10 @@ def show_graph(self, **kwargs) -> str | None: def save(self, path: str | Path) -> None: """Serialize the pure pipeline blueprint to a JSON file. - Saves topology and SourceNode declarations only — no databases, - no execution context, no run metadata. + Saves the full pipeline topology: SourceNode declarations, function + and operator pod configurations, and all edge connections. Runtime + state — databases, execution context, and run metadata — is not + persisted. Args: path: File path to write JSON output to. @@ -360,24 +362,25 @@ def save(self, path: str | Path) -> None: "data_context_key": data_context_key, } - if isinstance(node, SourceNodeClass): - descriptor["source_config"] = { - "source_type": "node", - "name": node.name, - "tag_schema": serialize_schema(node.tag_schema, type_converter), - "data_schema": serialize_schema(node.data_schema, type_converter), - } - descriptor["reconstructable"] = True - - elif isinstance(node, FunctionNode): - if node._function_pod is not None: - descriptor["function_config"] = node._function_pod.to_config() - descriptor["table_scope"] = node._table_scope - - elif isinstance(node, OperatorNode): - if node._operator is not None: - descriptor["operator_config"] = node._operator.to_config() - descriptor["table_scope"] = node._table_scope + match node: + case SourceNodeClass(): + descriptor["source_config"] = { + "source_type": "node", + "name": node.name, + "tag_schema": serialize_schema(node.tag_schema, type_converter), + "data_schema": serialize_schema(node.data_schema, type_converter), + } + descriptor["reconstructable"] = True + + case FunctionNode(): + if node._function_pod is not None: + descriptor["function_config"] = node._function_pod.to_config() + descriptor["table_scope"] = node._table_scope + + case OperatorNode(): + if node._operator is not None: + descriptor["operator_config"] = node._operator.to_config() + descriptor["table_scope"] = node._table_scope nodes[content_hash_str] = descriptor @@ -460,20 +463,10 @@ def load(cls, path: str | Path) -> "Pipeline": if node_type == "source": source_type = source_config.get("source_type") - if source_type in ("node", "spec"): - # "node" is the v0.3 format; "spec" is the v0.2 backward-compat format. - # Both reconstruct as SourceNode — hashes are preserved because - # SourceNode.identity_structure() matches old SourceSpec.identity_structure(). - if source_type == "node": - # v0.3 format uses "name"; v0.1.0 used "node_name" — support both - node_name = source_config.get("name") or source_config.get("node_name") - else: - # v0.2 backward-compat: spec_name becomes node name - node_name = source_config.get("spec_name") + if source_type == "node": + node_name = source_config.get("name") or source_config.get("node_name") if not node_name: node_name = descriptor.get("label") or "unknown" - # Prefer tag/data schemas from source_config when present (v0.3+); - # fall back to output_schema for older formats. if "tag_schema" in source_config and "data_schema" in source_config: tag_schema = Schema(deserialize_schema(source_config["tag_schema"])) data_schema = Schema(deserialize_schema(source_config["data_schema"])) @@ -574,14 +567,14 @@ def load(cls, path: str | Path) -> "Pipeline": pipeline._node_lut = { h: n for h, n in reconstructed.items() - if not isinstance(n, SourceNodeClass) + if n.node_type != "source" } # SourceNode IS the upstream — store it directly so _build_execution_graph() # can find it by hash and substitute a concrete source at run time. pipeline._upstreams = { h: n for h, n in reconstructed.items() - if isinstance(n, SourceNodeClass) + if n.node_type == "source" } pipeline._compiled = True diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index df4d6cf43..b2dd9f7c7 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -97,10 +97,6 @@ def __enter__(self) -> "PipelineJob": return super().__enter__() # type: ignore[return-value] def compile(self) -> None: - """Compile recorded invocations into a Pipeline (implements AbstractPipelineBase.compile).""" - self._compile_from_recording() - - def _compile_from_recording(self) -> None: """Compile the recorded edges into a pure Pipeline and build the job node map. ``_rec_node_lut`` now contains ``FunctionJobNode`` / ``OperatorJobNode`` objects @@ -167,7 +163,7 @@ def _compile_from_recording(self) -> None: name=bp_node.name, tag_schema=bp_node.tag_schema, data_schema=bp_node.data_schema, - concrete=concrete, + bound_source=concrete, ) elif isinstance(bp_node, FunctionNodeBase): # Create fresh FunctionJobNode rewired to the upstream job node. @@ -252,7 +248,7 @@ def _is_concrete_source(stream: cp.StreamProtocol) -> bool: # TrackerProtocol — recording with source interception # ------------------------------------------------------------------ - def _to_node_stream(self, stream: cp.StreamProtocol) -> cp.StreamProtocol: + def _to_node_stream(self, stream: cp.StreamProtocol) -> SourceNode | cp.StreamProtocol: """Convert *stream* to a node-based equivalent for consistent hash recording. Concrete ``RootSource`` instances are promoted to ``SourceNode`` via @@ -380,7 +376,7 @@ def from_pipeline( Walks the pipeline's ``_persistent_node_map`` topologically and creates corresponding ``JobNode`` variants: - * ``SourceNode`` → ``SourceJobNode(name, schemas, concrete=sources.get(name))`` + * ``SourceNode`` → ``SourceJobNode(name, schemas, bound_source=sources.get(name))`` * ``FunctionNode`` → ``FunctionJobNode(function_pod, upstream_job_node, label)`` * ``OperatorNode`` → ``OperatorJobNode(operator, upstream_job_nodes, label)`` @@ -442,7 +438,7 @@ def from_pipeline( name=node.name, tag_schema=node.tag_schema, data_schema=node.data_schema, - concrete=concrete, + bound_source=concrete, ) elif isinstance(node, FunctionNodeBase): @@ -524,7 +520,7 @@ def bind( When *sources* is provided, each concrete source is validated against its matching ``SourceNode`` slot schema, then the corresponding - ``SourceJobNode._concrete`` is updated in-place. + ``SourceJobNode.bound_source`` is updated in-place. When *store* is provided and differs from the current store, ``_distribute_databases()`` is called so that all job nodes receive @@ -566,7 +562,7 @@ def bind( for job_node in (self._persistent_node_map or {}).values(): if isinstance(job_node, SourceJobNode) and job_node.name in sources: - job_node._concrete = sources[job_node.name] + job_node.bound_source = sources[job_node.name] self._sources.update(sources) @@ -900,7 +896,7 @@ def _build_execution_graph(self) -> "tuple[Any, list[str], Pipeline]": # Any = name=upstream.name, tag_schema=upstream.tag_schema, data_schema=upstream.data_schema, - concrete=concrete, + bound_source=concrete, ) exec_node_map[node_hash] = exec_job_node else: diff --git a/src/orcapod/pipeline/serialization.py b/src/orcapod/pipeline/serialization.py index a9867f3e3..f255634ac 100644 --- a/src/orcapod/pipeline/serialization.py +++ b/src/orcapod/pipeline/serialization.py @@ -17,8 +17,8 @@ # Format version # --------------------------------------------------------------------------- -PIPELINE_FORMAT_VERSION = "0.3" -SUPPORTED_FORMAT_VERSIONS = frozenset({"0.1.0", "0.2", "0.3"}) +PIPELINE_FORMAT_VERSION = "0.1.0" +SUPPORTED_FORMAT_VERSIONS = frozenset({"0.1.0"}) # --------------------------------------------------------------------------- # LoadStatus diff --git a/tests/test_core/nodes/test_operator_node_split.py b/tests/test_core/nodes/test_operator_node_split.py index 7bca166d9..a45084d38 100644 --- a/tests/test_core/nodes/test_operator_node_split.py +++ b/tests/test_core/nodes/test_operator_node_split.py @@ -49,14 +49,18 @@ def test_operator_node_does_not_accept_db_params(self, source_pair): "OperatorNode must not accept pipeline_database — use OperatorJobNode instead" ) - def test_as_node_returns_self(self, source_pair): + def test_as_node_returns_clone(self, source_pair): from orcapod.core.nodes.operator_node import OperatorNode from orcapod.core.operators.join import Join op = Join() node_a, node_b = source_pair op_node = OperatorNode(operator=op, input_streams=(node_a, node_b)) - assert op_node.as_node() is op_node + cloned = op_node.as_node() + assert isinstance(cloned, OperatorNode) + assert cloned is not op_node + assert cloned.content_hash() == op_node.content_hash() + assert cloned.pipeline_hash() == op_node.pipeline_hash() class TestOperatorJobNodeHashParity: diff --git a/tests/test_core/nodes/test_source_node.py b/tests/test_core/nodes/test_source_node.py index 12ce88b00..c15df180c 100644 --- a/tests/test_core/nodes/test_source_node.py +++ b/tests/test_core/nodes/test_source_node.py @@ -40,7 +40,10 @@ def test_content_hash_is_deterministic(self, tag_schema, data_schema): def test_content_hash_stable_value(self, tag_schema, data_schema): """content_hash must match the value anchored at migration time. - Digest (hex): df0cba56fd880f86584ef89b35ef850bd813c95c114ac3bc84818e195b2175cb + Digest (hex): b2779d890c22b601f0ed71eb2817138205cc509581b9d7e23186c2f0ec815695 + + Note: hash changed in ENG-493 when identity_structure() prefix was updated + from ``"SourceSpec"`` to ``"source_node"`` to better reflect the node type. """ from orcapod.core.nodes.source_node import SourceNode from orcapod.types import ContentHash @@ -49,7 +52,7 @@ def test_content_hash_stable_value(self, tag_schema, data_schema): expected = ContentHash( method="semantic_v0.1", digest=bytes.fromhex( - "df0cba56fd880f86584ef89b35ef850bd813c95c114ac3bc84818e195b2175cb" + "b2779d890c22b601f0ed71eb2817138205cc509581b9d7e23186c2f0ec815695" ), ) assert node.content_hash() == expected @@ -176,7 +179,7 @@ def test_bound_content_hash_is_concrete_hash(self, tag_schema, data_schema): src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) job_node = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) assert job_node.content_hash() == src.content_hash() @@ -188,7 +191,7 @@ def test_bound_pipeline_hash_still_schema_based(self, tag_schema, data_schema): src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) node = SourceNode(name="x", tag_schema=tag_schema, data_schema=data_schema) job_node = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) assert job_node.pipeline_hash() == node.pipeline_hash() @@ -200,29 +203,28 @@ def test_as_node_returns_source_node(self, tag_schema, data_schema): assert isinstance(node, SourceNode) assert node.content_hash() == job_node.content_hash() - def test_mutable_concrete_updates_in_place(self, tag_schema, data_schema): - """Binding concrete mutates _concrete in-place.""" + def test_bound_source_property_updates_in_place(self, tag_schema, data_schema): + """Setting bound_source mutates the node in-place.""" from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) - assert job_node._concrete is None + assert job_node.bound_source is None src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - job_node._concrete = src - assert job_node._concrete is src + job_node.bound_source = src + assert job_node.bound_source is src - def test_concrete_mutation_clears_content_hash_cache(self, tag_schema, data_schema): - """Setting _concrete clears the content_hash cache so stale values are not returned.""" - from orcapod.core.nodes.source_node import SourceJobNode, SourceNode + def test_bound_source_setter_clears_content_hash_cache(self, tag_schema, data_schema): + """Setting bound_source clears the content_hash cache so stale values are not returned.""" + from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.dict_source import DictSource job_node = SourceJobNode(name="x", tag_schema=tag_schema, data_schema=data_schema) schema_hash = job_node.content_hash() # populates cache with schema-based hash src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) - job_node._concrete = src # should clear the cache + job_node.bound_source = src # should clear the cache - assert job_node._content_hash_cache == {} # cache cleared bound_hash = job_node.content_hash() assert bound_hash != schema_hash # now returns concrete-based hash assert bound_hash == src.content_hash() @@ -246,7 +248,7 @@ def test_source_job_node_as_table_delegates_to_concrete(self, tag_schema, data_s src = DictSource(data=[{"id": 1, "value": 1.0}], tag_columns=["id"]) job_node = SourceJobNode( - name="x", tag_schema=tag_schema, data_schema=data_schema, concrete=src + name="x", tag_schema=tag_schema, data_schema=data_schema, bound_source=src ) table = job_node.as_table() assert isinstance(table, pa.Table) diff --git a/tests/test_core/test_tracker.py b/tests/test_core/test_tracker.py index fd4f9fc85..efc22d75b 100644 --- a/tests/test_core/test_tracker.py +++ b/tests/test_core/test_tracker.py @@ -149,13 +149,13 @@ def _make_job_node(self, stream=None, name="test_source"): name=name, tag_schema=tag_schema, data_schema=data_schema, - concrete=stream, + bound_source=stream, ) def test_construction(self): stream = _make_stream() node = self._make_job_node(stream) - assert node._concrete is stream + assert node.bound_source is stream assert node.node_type == "source" assert node.producer is None assert node.upstreams == () @@ -258,7 +258,7 @@ def test_source_node_context_matches_stream(self): name="test", tag_schema=Schema({"id": int}), data_schema=Schema({"x": int}), - concrete=stream, + bound_source=stream, ) # SourceJobNode has its own data context (not delegated to concrete) assert node.data_context_key is not None @@ -285,7 +285,7 @@ def test_source_node_hash_consistent_with_stream(self): name="test", tag_schema=Schema({"id": int}), data_schema=Schema({"x": int}), - concrete=stream, + bound_source=stream, ) # SourceJobNode delegates content_hash() to concrete when bound assert node.content_hash() == stream.content_hash() diff --git a/tests/test_pipeline/test_node_descriptors.py b/tests/test_pipeline/test_node_descriptors.py index d52ec2e2c..42f9cad9c 100644 --- a/tests/test_pipeline/test_node_descriptors.py +++ b/tests/test_pipeline/test_node_descriptors.py @@ -59,7 +59,7 @@ def test_from_descriptor_full_mode_delegates_to_stream(self): name="my_source", tag_schema=tag_schema, data_schema=data_schema, - concrete=source, + bound_source=source, ) t, d = job_node.output_schema() assert "a" in t diff --git a/tests/test_pipeline/test_node_protocols.py b/tests/test_pipeline/test_node_protocols.py index e6af2485f..103165c08 100644 --- a/tests/test_pipeline/test_node_protocols.py +++ b/tests/test_pipeline/test_node_protocols.py @@ -155,7 +155,7 @@ def _make_source_job_node(table, tag_col="key"): name="test_source", tag_schema=tag_schema, data_schema=data_schema, - concrete=src, + bound_source=src, ) diff --git a/tests/test_pipeline/test_orchestrator.py b/tests/test_pipeline/test_orchestrator.py index 42443a84b..8d627bf2a 100644 --- a/tests/test_pipeline/test_orchestrator.py +++ b/tests/test_pipeline/test_orchestrator.py @@ -81,7 +81,7 @@ def _make_job_node(self, src): name="test_src", tag_schema=tag_schema, data_schema=data_schema, - concrete=src, + bound_source=src, ) @pytest.mark.asyncio @@ -436,7 +436,7 @@ def test_single_terminal_source(self): name="test_src", tag_schema=tag_schema, data_schema=data_schema, - concrete=src, + bound_source=src, ) G = nx.DiGraph() G.add_node(node) diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index 627bd28dd..aab35aaf0 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -1,7 +1,7 @@ """Tests for the Pipeline and PipelineJob classes. Verifies that Pipeline correctly wraps all nodes during compile(): -- Leaf streams → SourceNode (SourceSpec-only; concrete sources raise ValueError) +- Leaf streams → SourceNode (auto-wrapped via SourceNode.from_stream()) - Function pod invocations → FunctionNode - Operator invocations → OperatorNode @@ -103,14 +103,17 @@ def test_pipeline_with_spec_leaves_compiles(self): ] assert len(source_nodes) == 2 - def test_pipeline_with_concrete_leaf_raises(self): - """Pipeline.compile() raises ValueError if any leaf is not a SourceNode.""" + def test_pipeline_with_concrete_leaf_auto_wraps_as_source_node(self): + """Pipeline.compile() auto-wraps concrete leaf streams as SourceNode.""" src_a, src_b = _make_two_sources() - pipeline = Pipeline(name="bad_pipe") - with pytest.raises(ValueError, match="SourceNode"): - with pipeline: - Join()(src_a, src_b) + pipeline = Pipeline(name="auto_wrap_pipe") + with pipeline: + Join()(src_a, src_b) + + # Concrete streams at the leaves are automatically wrapped as SourceNode + source_nodes = [n for n in pipeline._node_graph.nodes() if isinstance(n, SourceNode)] + assert len(source_nodes) == 2 def test_pipeline_from_pipeline_returns_pipeline_job(self): """PipelineJob.from_pipeline() returns a PipelineJob without modifying the pipeline.""" diff --git a/tests/test_pipeline/test_serialization.py b/tests/test_pipeline/test_serialization.py index 9fa588bfe..a9d9b12c4 100644 --- a/tests/test_pipeline/test_serialization.py +++ b/tests/test_pipeline/test_serialization.py @@ -204,7 +204,7 @@ def _src(tag, data): class TestNewSerializationFormat: - """v0.3 serialization format tests.""" + """v0.1.0 serialization format tests.""" def test_save_load_roundtrip_with_source_node(self, tmp_path, compiled_pipeline): """Pipeline.save/load round-trip preserves SourceNode slots.""" @@ -234,43 +234,30 @@ def test_saved_format_has_source_node_type(self, tmp_path, compiled_pipeline): f"Expected source_type='node', got {node_data.get('source_config')}" ) - def test_format_version_is_0_3(self, tmp_path, compiled_pipeline): - """Saved format version is 0.3.""" + def test_format_version_is_0_1_0(self, tmp_path, compiled_pipeline): + """Saved format version is 0.1.0.""" save_path = tmp_path / "test_pipeline.json" compiled_pipeline.save(save_path) with open(save_path) as f: data = json.load(f) - assert data.get("orcapod_pipeline_version") == "0.3", ( - f"Expected version '0.3', got {data.get('orcapod_pipeline_version')!r}" + assert data.get("orcapod_pipeline_version") == "0.1.0", ( + f"Expected version '0.1.0', got {data.get('orcapod_pipeline_version')!r}" ) - def test_backward_compat_load_v0_2_spec_format(self, tmp_path, compiled_pipeline): - """Loading a v0.2 pipeline with source_type='spec' produces SourceNode.""" - # Save, then hack the JSON to simulate v0.2 format - save_path = tmp_path / "old_pipeline.json" + def test_unsupported_version_raises(self, tmp_path, compiled_pipeline): + """Loading a pipeline with an unsupported version string raises ValueError.""" + save_path = tmp_path / "pipeline.json" compiled_pipeline.save(save_path) with open(save_path) as f: data = json.load(f) - # Downgrade to v0.2 format data["orcapod_pipeline_version"] = "0.2" - for node_data in data.get("nodes", {}).values(): - if node_data.get("node_type") == "source": - if "source_config" in node_data: - node_data["source_config"]["source_type"] = "spec" - old_path = tmp_path / "old_format.json" with open(old_path, "w") as f: json.dump(data, f) - # Load should work and produce SourceNode - loaded = Pipeline.load(old_path) - - for node in loaded._persistent_node_map.values(): - if node.node_type == "source": - assert isinstance(node, SourceNode), ( - f"Expected SourceNode from v0.2 load, got {type(node).__name__}" - ) + with pytest.raises(ValueError, match="version"): + Pipeline.load(old_path) diff --git a/tests/test_pipeline/test_serialization_helpers.py b/tests/test_pipeline/test_serialization_helpers.py index 616e612c2..c6f47d953 100644 --- a/tests/test_pipeline/test_serialization_helpers.py +++ b/tests/test_pipeline/test_serialization_helpers.py @@ -196,7 +196,7 @@ def test_resolve_unknown_raises(self): class TestPipelineFormatVersion: def test_version_is_string(self): assert isinstance(PIPELINE_FORMAT_VERSION, str) - assert PIPELINE_FORMAT_VERSION == "0.3" + assert PIPELINE_FORMAT_VERSION == "0.1.0" # --------------------------------------------------------------------------- diff --git a/tests/test_pipeline/test_sync_orchestrator.py b/tests/test_pipeline/test_sync_orchestrator.py index bc0f7dea2..5f8cffe70 100644 --- a/tests/test_pipeline/test_sync_orchestrator.py +++ b/tests/test_pipeline/test_sync_orchestrator.py @@ -337,7 +337,7 @@ def test_materialized_stream_has_same_pipeline_hash(self): name="test_src", tag_schema=tag_schema, data_schema=data_schema, - concrete=src, + bound_source=src, ) buf = list(node.iter_data()) diff --git a/tests/test_protocols/test_node_protocols.py b/tests/test_protocols/test_node_protocols.py index f959a9790..5295b08f0 100644 --- a/tests/test_protocols/test_node_protocols.py +++ b/tests/test_protocols/test_node_protocols.py @@ -33,7 +33,7 @@ def source_node(_sample_source): name="test_source", tag_schema=tag_schema, data_schema=data_schema, - concrete=_sample_source, + bound_source=_sample_source, ) From 01119dab64aee71722ff4bb0effb5069e3378cdd Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 23:54:48 +0000 Subject: [PATCH 22/24] docs(operator_node): add Google-style docstring to OperatorNodeBase.__init__() Documents all constructor arguments (operator, input_streams, tracker_manager, label, config, table_scope). Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/operator_node.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 8ebbea0da..339174439 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -76,7 +76,24 @@ def __init__( label: str | None = None, config: Config | None = None, table_scope: Literal["pipeline_hash", "content_hash"] = "pipeline_hash", - ): + ) -> None: + """Initialize the shared operator-node state. + + Args: + operator: The operator pod that defines the transformation logic + and output schema. + input_streams: One or more upstream streams consumed by this + operator. Passed through as a tuple. + tracker_manager: Optional tracker manager override. Defaults to + ``DEFAULT_TRACKER_MANAGER``. + label: Optional display label for this node. + config: Optional config override; defaults to the global config. + table_scope: Determines how the database table path is scoped. + ``"pipeline_hash"`` (default) shares a table across all runs + with the same topology and schemas regardless of which concrete + data is bound. ``"content_hash"`` creates a separate table per + unique data combination. + """ if tracker_manager is None: tracker_manager = DEFAULT_TRACKER_MANAGER self.tracker_manager = tracker_manager From 3f155777a723adffa7f406c57a01998ad1f35afe Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Sat, 23 May 2026 00:38:44 +0000 Subject: [PATCH 23/24] fix(merge): resolve conflicts from origin/main merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - errors.py: keep both CursorInvalidatedError (from main) and PipelineJobRequiredError (from branch); preserve ordering. - source_node.py: resolve async_execute conflict by using `async for tag, data in self.async_iter_data()` (matches the async_iter_data method defined on SourceNodeBase). - test_polling_source.py: update SourceNode(stream) usage (old API) to SourceJobNode(name=..., tag_schema=..., data_schema=..., bound_source=stream) — the new split API. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_channels/test_polling_source.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/test_channels/test_polling_source.py b/tests/test_channels/test_polling_source.py index 0832fa5cf..1ead0e0f1 100644 --- a/tests/test_channels/test_polling_source.py +++ b/tests/test_channels/test_polling_source.py @@ -10,7 +10,7 @@ import pytest from orcapod.channels import Channel -from orcapod.core.nodes.source_node import SourceNode +from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.sources.polling_source import PollingSource from orcapod.core.streams.arrow_table_stream import ArrowTableStream from orcapod.errors import CursorInvalidatedError @@ -197,9 +197,16 @@ async def test_default_yields_same_items_as_iter_data(self): @pytest.mark.asyncio async def test_source_node_async_execute_uses_async_iter(self): - """SourceNode.async_execute routes through async_iter_data — no regression.""" + """SourceJobNode.async_execute routes through async_iter_data — no regression.""" + from orcapod.types import Schema + stream = _make_arrow_stream(2) - node = SourceNode(stream) + node = SourceJobNode( + name="test", + tag_schema=Schema({"id": int}), + data_schema=Schema({"val": int}), + bound_source=stream, + ) ch = Channel(buffer_size=8) await node.async_execute(ch.writer) @@ -209,8 +216,15 @@ async def test_source_node_async_execute_uses_async_iter(self): @pytest.mark.asyncio async def test_source_node_closes_channel_after_exhaustion(self): + from orcapod.types import Schema + stream = _make_arrow_stream(1) - node = SourceNode(stream) + node = SourceJobNode( + name="test", + tag_schema=Schema({"id": int}), + data_schema=Schema({"val": int}), + bound_source=stream, + ) ch = Channel(buffer_size=4) await node.async_execute(ch.writer) From 33150c4eeb3b96fa55af7c90a23e35baba775613 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Sat, 23 May 2026 00:58:49 +0000 Subject: [PATCH 24/24] fix(nodes): raise RuntimeError from UNAVAILABLE as_node(); invalidate pipeline hash on bind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - FunctionJobNode.as_node() and OperatorJobNode.as_node() now raise RuntimeError instead of returning self when the node is UNAVAILABLE (no live pod was provided at load time). Returning self from a mutable type is an unsafe referential copy; callers must handle the UNAVAILABLE status before calling as_node(). - SourceJobNode.bound_source setter now calls both _invalidate_content_hash_cache() and _invalidate_pipeline_hash_cache() for defensive consistency. - Add TODO(ENG-512) comment in job.py flagging the potential O(depth²) cost of repeated _to_node_stream recursive traversal. - Add ENG-513 note in test_node_protocols.py helper noting that explicit schema passing is currently required but will be made optional. Co-Authored-By: Claude Sonnet 4.6 --- src/orcapod/core/nodes/function_node.py | 21 +++++++++++++------ src/orcapod/core/nodes/operator_node.py | 24 ++++++++++++++-------- src/orcapod/core/nodes/source_node.py | 3 ++- src/orcapod/pipeline/job.py | 3 +++ tests/test_pipeline/test_node_protocols.py | 8 +++++++- 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index c57652503..9254967cf 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -613,19 +613,28 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: def as_node(self) -> FunctionNode: """Return the lightweight blueprint equivalent of this node. - For UNAVAILABLE read-only stubs (loaded with no live function pod), - returns ``self`` — there is nothing to clone. For normal instances, - returns a fresh ``FunctionNode`` with the same function pod, input + Returns a fresh ``FunctionNode`` with the same function pod, input stream, label, table scope, and tracker manager. Its ``content_hash()`` / ``pipeline_hash()`` are identical to those of this node. Returns: - This instance (if unavailable) or a new equivalent ``FunctionNode``. + A new equivalent ``FunctionNode``. + + Raises: + RuntimeError: If this node is in UNAVAILABLE state (no live + function pod). UNAVAILABLE nodes cannot be cloned into a + usable blueprint — callers must handle this status before + invoking ``as_node()``. """ if self._function_pod is None: - # UNAVAILABLE stub — no live function pod, cannot meaningfully clone - return self + from orcapod.pipeline.serialization import LoadStatus + + raise RuntimeError( + f"Cannot clone FunctionNode {self._label!r} into a blueprint: " + "the node is UNAVAILABLE (no live function pod was provided when " + "it was loaded). Only FULL or READ_ONLY nodes support as_node()." + ) return FunctionNode( function_pod=self._function_pod, input_stream=self._input_stream, diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 339174439..a38fde042 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -450,19 +450,25 @@ def from_descriptor( def as_node(self) -> OperatorNode: """Return the lightweight blueprint equivalent of this node. - For UNAVAILABLE read-only stubs (loaded with no live operator), - returns ``self`` — there is nothing to clone. For normal instances, - returns a fresh ``OperatorNode`` with the same operator, input - streams, label, table scope, and tracker manager. Its - ``content_hash()`` / ``pipeline_hash()`` are identical to those of - this node. + Returns a fresh ``OperatorNode`` with the same operator, input streams, + label, table scope, and tracker manager. Its ``content_hash()`` / + ``pipeline_hash()`` are identical to those of this node. Returns: - This instance (if unavailable) or a new equivalent ``OperatorNode``. + A new equivalent ``OperatorNode``. + + Raises: + RuntimeError: If this node is in UNAVAILABLE state (no live + operator). UNAVAILABLE nodes cannot be cloned into a usable + blueprint — callers must handle this status before invoking + ``as_node()``. """ if self._operator is None: - # UNAVAILABLE stub — no live operator, cannot meaningfully clone - return self + raise RuntimeError( + f"Cannot clone OperatorNode {self._label!r} into a blueprint: " + "the node is UNAVAILABLE (no live operator was provided when " + "it was loaded). Only FULL or READ_ONLY nodes support as_node()." + ) return OperatorNode( operator=self._operator, input_streams=self._input_streams, diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index c7c4ce453..92ec12599 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -449,9 +449,10 @@ def bound_source(self) -> StreamProtocol | None: @bound_source.setter def bound_source(self, value: StreamProtocol | None) -> None: - """Bind *value* as the concrete source and invalidate the content hash cache.""" + """Bind *value* as the concrete source and invalidate both hash caches.""" self._bound_source = value self._invalidate_content_hash_cache() + self._invalidate_pipeline_hash_cache() # ------------------------------------------------------------------ # Identity — delegate to bound source when set diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index b2dd9f7c7..e2012263a 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -290,6 +290,9 @@ def record_function_pod_invocation( """ from orcapod.core.nodes.function_node import FunctionJobNode + # TODO(ENG-512): _to_node_stream recurses into upstreams on every recording call; + # for deep graphs this can be O(depth²). Consider memoizing or doing a single + # topological pass at job-creation time. input_stream = self._to_node_stream(input_stream) input_hash = input_stream.content_hash().to_string() diff --git a/tests/test_pipeline/test_node_protocols.py b/tests/test_pipeline/test_node_protocols.py index 103165c08..99ab04fdd 100644 --- a/tests/test_pipeline/test_node_protocols.py +++ b/tests/test_pipeline/test_node_protocols.py @@ -148,7 +148,13 @@ def test_dispatch_operator(self): def _make_source_job_node(table, tag_col="key"): - """Helper: create a SourceJobNode wrapping a concrete source.""" + """Helper: create a SourceJobNode wrapping a concrete source. + + Note: schemas are passed explicitly here because SourceJobNode currently + requires them even when bound_source is provided. ENG-513 tracks adding + schema inference so that `SourceJobNode(name=..., bound_source=src)` works + without explicit tag_schema/data_schema. + """ src = ArrowTableSource(table, tag_columns=[tag_col], infer_nullable=True) tag_schema, data_schema = src.output_schema() return SourceJobNode(