From 7aadae1736f4a6ba7c13129d5e3979e19a8f18fd Mon Sep 17 00:00:00 2001 From: Vishesh 'ironeagle' Bangotra Date: Wed, 16 Sep 2026 15:43:14 +0530 Subject: [PATCH] feat: collect dagpipe wiki, refresh lib and mcp artifacts Enables dagpipe wiki on the docs hub alongside lib and mcp, removes stale nested lib pages, and picks up regenerated GSDFC docstrings in the MCP bundle. --- _index/index.html | 1 + config.yml | 1 + dagpipe/lib/dagpipe/engine/index.html | 1799 ----- dagpipe/lib/dagpipe/graph/index.html | 1428 ---- dagpipe/lib/dagpipe/index.html | 5881 --------------- dagpipe/lib/dagpipe/node/index.html | 2391 ------ dagpipe/lib/dagpipe/state/index.html | 2324 ------ dagpipe/lib/dagpipe/yaml_loader/index.html | 1158 --- dagpipe/lib/engine/index.html | 414 + dagpipe/lib/graph/index.html | 66 +- dagpipe/lib/index.html | 1290 +++- dagpipe/lib/node/index.html | 658 +- dagpipe/lib/objects.inv | Bin 921 -> 962 bytes dagpipe/lib/search/search_index.json | 2 +- dagpipe/lib/sitemap.xml.gz | Bin 127 -> 127 bytes dagpipe/lib/state/index.html | 152 +- dagpipe/wiki/01_overview/index.html | 1210 +++ dagpipe/wiki/02_components/index.html | 1388 ++++ .../01_basic_linear_pipeline/index.html | 1206 +++ .../02_graph_execution/index.html | 1296 ++++ .../03_use_cases/03_yaml_pipeline/index.html | 1270 ++++ .../04_schema_and_payload/index.html | 1264 ++++ .../05_state_fork_and_lineage/index.html | 1249 +++ .../06_async_execution/index.html | 1204 +++ .../03_use_cases/07_step_execution/index.html | 1258 ++++ dagpipe/wiki/04_best_practices/index.html | 1197 +++ dagpipe/wiki/06_error_handling/index.html | 1246 +++ dagpipe/wiki/07_testing_example/index.html | 1364 ++++ dagpipe/wiki/404.html | 951 +++ dagpipe/wiki/assets/images/favicon.png | Bin 0 -> 1870 bytes .../assets/javascripts/bundle.f55a23d4.min.js | 16 + .../javascripts/bundle.f55a23d4.min.js.map | 7 + .../javascripts/lunr/min/lunr.ar.min.js | 1 + .../javascripts/lunr/min/lunr.da.min.js | 18 + .../javascripts/lunr/min/lunr.de.min.js | 18 + .../javascripts/lunr/min/lunr.du.min.js | 18 + .../javascripts/lunr/min/lunr.el.min.js | 1 + .../javascripts/lunr/min/lunr.es.min.js | 18 + .../javascripts/lunr/min/lunr.fi.min.js | 18 + .../javascripts/lunr/min/lunr.fr.min.js | 18 + .../javascripts/lunr/min/lunr.he.min.js | 1 + .../javascripts/lunr/min/lunr.hi.min.js | 1 + .../javascripts/lunr/min/lunr.hu.min.js | 18 + .../javascripts/lunr/min/lunr.hy.min.js | 1 + .../javascripts/lunr/min/lunr.it.min.js | 18 + .../javascripts/lunr/min/lunr.ja.min.js | 1 + .../javascripts/lunr/min/lunr.jp.min.js | 1 + .../javascripts/lunr/min/lunr.kn.min.js | 1 + .../javascripts/lunr/min/lunr.ko.min.js | 1 + .../javascripts/lunr/min/lunr.multi.min.js | 1 + .../javascripts/lunr/min/lunr.nl.min.js | 18 + .../javascripts/lunr/min/lunr.no.min.js | 18 + .../javascripts/lunr/min/lunr.pt.min.js | 18 + .../javascripts/lunr/min/lunr.ro.min.js | 18 + .../javascripts/lunr/min/lunr.ru.min.js | 18 + .../javascripts/lunr/min/lunr.sa.min.js | 1 + .../lunr/min/lunr.stemmer.support.min.js | 1 + .../javascripts/lunr/min/lunr.sv.min.js | 18 + .../javascripts/lunr/min/lunr.ta.min.js | 1 + .../javascripts/lunr/min/lunr.te.min.js | 1 + .../javascripts/lunr/min/lunr.th.min.js | 1 + .../javascripts/lunr/min/lunr.tr.min.js | 18 + .../javascripts/lunr/min/lunr.vi.min.js | 1 + .../javascripts/lunr/min/lunr.zh.min.js | 1 + .../wiki/assets/javascripts/lunr/tinyseg.js | 206 + .../wiki/assets/javascripts/lunr/wordcut.js | 6708 +++++++++++++++++ .../workers/search.973d3a69.min.js | 42 + .../workers/search.973d3a69.min.js.map | 7 + .../assets/stylesheets/main.84d31ad4.min.css | 1 + .../stylesheets/main.84d31ad4.min.css.map | 1 + .../stylesheets/palette.06af60db.min.css | 1 + .../stylesheets/palette.06af60db.min.css.map | 1 + dagpipe/wiki/index.html | 1215 +++ dagpipe/wiki/search/search_index.json | 1 + dagpipe/wiki/sitemap.xml | 3 + dagpipe/wiki/sitemap.xml.gz | Bin 0 -> 127 bytes mcp/dagpipe/modules/dagpipe.engine.json | 155 +- mcp/dagpipe/modules/dagpipe.graph.json | 41 +- mcp/dagpipe/modules/dagpipe.json | 735 +- mcp/dagpipe/modules/dagpipe.node.json | 117 +- mcp/dagpipe/modules/dagpipe.state.json | 111 +- mcp/dagpipe/modules/dagpipe.yaml_loader.json | 151 +- 82 files changed, 27200 insertions(+), 16274 deletions(-) delete mode 100644 dagpipe/lib/dagpipe/engine/index.html delete mode 100644 dagpipe/lib/dagpipe/graph/index.html delete mode 100644 dagpipe/lib/dagpipe/index.html delete mode 100644 dagpipe/lib/dagpipe/node/index.html delete mode 100644 dagpipe/lib/dagpipe/state/index.html delete mode 100644 dagpipe/lib/dagpipe/yaml_loader/index.html create mode 100644 dagpipe/wiki/01_overview/index.html create mode 100644 dagpipe/wiki/02_components/index.html create mode 100644 dagpipe/wiki/03_use_cases/01_basic_linear_pipeline/index.html create mode 100644 dagpipe/wiki/03_use_cases/02_graph_execution/index.html create mode 100644 dagpipe/wiki/03_use_cases/03_yaml_pipeline/index.html create mode 100644 dagpipe/wiki/03_use_cases/04_schema_and_payload/index.html create mode 100644 dagpipe/wiki/03_use_cases/05_state_fork_and_lineage/index.html create mode 100644 dagpipe/wiki/03_use_cases/06_async_execution/index.html create mode 100644 dagpipe/wiki/03_use_cases/07_step_execution/index.html create mode 100644 dagpipe/wiki/04_best_practices/index.html create mode 100644 dagpipe/wiki/06_error_handling/index.html create mode 100644 dagpipe/wiki/07_testing_example/index.html create mode 100644 dagpipe/wiki/404.html create mode 100644 dagpipe/wiki/assets/images/favicon.png create mode 100644 dagpipe/wiki/assets/javascripts/bundle.f55a23d4.min.js create mode 100644 dagpipe/wiki/assets/javascripts/bundle.f55a23d4.min.js.map create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.ar.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.da.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.de.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.du.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.el.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.es.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.fi.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.fr.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.he.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.hi.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.hu.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.hy.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.it.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.ja.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.jp.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.kn.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.ko.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.multi.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.nl.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.no.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.pt.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.ro.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.ru.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.sa.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.stemmer.support.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.sv.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.ta.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.te.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.th.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.tr.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.vi.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/min/lunr.zh.min.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/tinyseg.js create mode 100644 dagpipe/wiki/assets/javascripts/lunr/wordcut.js create mode 100644 dagpipe/wiki/assets/javascripts/workers/search.973d3a69.min.js create mode 100644 dagpipe/wiki/assets/javascripts/workers/search.973d3a69.min.js.map create mode 100644 dagpipe/wiki/assets/stylesheets/main.84d31ad4.min.css create mode 100644 dagpipe/wiki/assets/stylesheets/main.84d31ad4.min.css.map create mode 100644 dagpipe/wiki/assets/stylesheets/palette.06af60db.min.css create mode 100644 dagpipe/wiki/assets/stylesheets/palette.06af60db.min.css.map create mode 100644 dagpipe/wiki/index.html create mode 100644 dagpipe/wiki/search/search_index.json create mode 100644 dagpipe/wiki/sitemap.xml create mode 100644 dagpipe/wiki/sitemap.xml.gz diff --git a/_index/index.html b/_index/index.html index 6b28e36..b54550d 100644 --- a/_index/index.html +++ b/_index/index.html @@ -143,6 +143,7 @@

DAG Pipe

Deterministic pipeline framework for executing state transformations through a directed acyclic graph (DAG).

+ view wiki view lib
diff --git a/config.yml b/config.yml index d927854..32dec7c 100644 --- a/config.yml +++ b/config.yml @@ -47,6 +47,7 @@ repos: description: Deterministic pipeline framework for executing state transformations through a directed acyclic graph (DAG). section: libraries docs: + wiki: site lib: site mcp: { bundle: docs/mcp, server: dagpipe, port: 8006 } diff --git a/dagpipe/lib/dagpipe/engine/index.html b/dagpipe/lib/dagpipe/engine/index.html deleted file mode 100644 index 533e953..0000000 --- a/dagpipe/lib/dagpipe/engine/index.html +++ /dev/null @@ -1,1799 +0,0 @@ - - - - - - - - - - - - - - - - - - - Engine - dagpipe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - Skip to content - - -
-
- -
- - - - - - -
- - -
- -
- - - - - - -
-
- - - -
-
-
- - - - - -
-
-
- - - -
-
-
- - - -
-
-
- - - -
-
- - - - - -

Engine

- - -
- - - -

- dagpipe.engine - - -

- -
- -

Summary

-

Execution engine responsible for running pipelines and graphs.

-

The Engine executes Node objects and propagates immutable State instances -through either a linear sequence or a directed acyclic graph (Graph). -It orchestrates execution order, branching, and state propagation.

-
-

Guarantees

-
    -
  • Deterministic execution and consistent state lineage.
    • -
  • Orchestrates execution without modifying Graph, Node, or State objects.
    • -
- - - -
- - - - - - -

Classes

- -
- - - -

- Engine - - -

-
1
-2
-3
-4
-5
Engine(
-    nodes_or_graph: Sequence[Node] | Graph,
-    *,
-    on_step: StepHook | None = None
-)
-
- -
- - -

Execution engine responsible for running pipeline logic.

- - -
- Notes -

Responsibilities:

-
1
-2
-3
-4
-5
-6
-7
- Accepts either a linear sequence of `Node` objects or a `Graph`
-  defining execution topology.
-- Propagates immutable `State` objects through `Node` objects and
-  collects terminal states.
-- Supports synchronous (`run`) and asynchronous (`run_async`)
-  execution, dispatching per-node.
-- Supports step-wise / resumable execution and progress hooks.
-
-

Guarantees:

-
1
-2
-3
-4
-5
- Never mutates `State`, `Node`, or `Graph` instances.
-- `State` objects are never modified in place; each branch produces
-  independent instances.
-- Execution order is deterministic and follows graph or pipeline topology.
-- Thread-safe for concurrent execution.
-
-
- - - -
- - - - - -
Attributes
- -
- - - -
- nodes - - - - property - - -
-
1
nodes: tuple[Node, ...]
-
- -
- -

Return nodes managed by this engine.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Ordered sequence in linear mode or all nodes in graph mode.

-
-
-
- -
- -
Functions
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return the canonical string representation of the object.

- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
str - str - -
-

Representation that uniquely identifies the object and its configuration.

-
-
- -
- -
- -
- - -
- run - - -
-
1
run(root: State) -> list[State]
-
- -
- -

Execute the pipeline starting from a root State.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- list[State] - -
-

list[State]: -Terminal execution states produced by the pipeline.

-
-
- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If root is not a State instance.

-
-
- RuntimeError - -
-

If the engine execution mode is invalid.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Selects execution mode, propagates state through nodes, creates
-  new instances for branches, and collects terminal states.
-
-
-
- -
- -
- - -
- run_async - - - - async - - -
-
1
run_async(root: State) -> list[State]
-
- -
- -

Execute the pipeline starting from root, dispatching sync vs async nodes.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- list[State] - -
-

list[State]: -Terminal execution states produced by the pipeline.

-
-
- - -
- Notes -

Each node is executed with Node.run when synchronous and -AsyncNode.run_async when asynchronous. Linear and graph topologies -are both supported.

-
-
- -
- -
- - -
- run_steps - - -
-
1
-2
-3
-4
-5
-6
run_steps(
-    root: State,
-    *,
-    resume_from: int | None = None,
-    on_step: StepHook | None = None
-) -> Iterator[StepResult]
-
- -
- -

Execute the pipeline step-by-step, yielding one StepResult per step.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
resume_from - int | None - -
-

Skip steps at index < resume_from (for resume-after-partial). -Steps are 0-indexed.

-
- 		- None -
on_step - StepHook | None - -
-

Callback (step, status, message) invoked per step; falls back -to the engine-level hook when unset.

-
- 		- None -
- - -

Yields:

- - - - - - - - - - - - - -
Name TypeDescription
StepResult - StepResult - -
-

One per executed node/step, carrying the produced states.

-
-
- - -
- Notes -

This is a synchronous, generator-based checkpoint interface compatible -with the imperative resume-by-step behaviour of the legacy -orchestrator. Use run_steps_async for async nodes.

-
-
- -
- -
- - -
- run_steps_async - - - - async - - -
-
1
-2
-3
-4
-5
-6
run_steps_async(
-    root: State,
-    *,
-    resume_from: int | None = None,
-    on_step: AsyncStepHook | None = None
-) -> AsyncIterator[StepResult]
-
- -
- -

Async variant of run_steps supporting AsyncNode execution.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
resume_from - int | None - -
-

Skip steps at index < resume_from.

-
- 		- None -
on_step - AsyncStepHook | None - -
-

Async callback (step, status, message) invoked per step.

-
- 		- None -
- - -

Yields:

- - - - - - - - - - - - - -
Name TypeDescription
StepResult - AsyncIterator[StepResult] - -
-

One per executed node/step.

-
-
- -
- -
- - - -
- -
- -
- -
- - - -

- ProgressMessage - - -

-
 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
-10
-11
ProgressMessage(
-    *,
-    lines: int | None = None,
-    blocks: int | None = None,
-    count: int | None = None,
-    unit: str | None = None,
-    raw_ocr_line: str | None = None,
-    error: str | None = None,
-    step: str = "",
-    status: str = ""
-)
-
- -
- - -

Lightweight progress payload emitted by engine step hooks.

-

Mirrors the imperative ProgressMessage used by the legacy orchestrator so -callers can surface counts/lines/errors without coupling the engine to pydantic.

- - - - -
- - - - - - - - - - - -
- -
- -
- -
- - - -

- StepResult - - -

-
1
-2
-3
-4
-5
-6
StepResult(
-    index: int,
-    node_id: str,
-    states: tuple[State, ...],
-    completed: bool,
-)
-
- -
- - -

A single checkpointed step within an async/resumable engine run.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
index - int - -
-

Ordinal index of the step.

-
-
node_id - str - -
-

Identifier of the node associated with this step.

-
-
states - tuple[State, ...] - -
-

States produced by running this step.

-
-
completed - bool - -
-

Whether this step succeeded (vs. paused/interrupted).

-
-
- -

Initialise StepResult.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
index - int - -
-

Ordinal index of the step.

-
- 		- required -
node_id - str - -
-

Identifier of the node associated with this step.

-
- 		- required -
states - tuple[State, ...] - -
-

States produced by running this step.

-
- 		- required -
completed - bool - -
-

Whether this step succeeded (vs. paused/interrupted).

-
- 		- required -
- - - - -
- - - - - - - -
Functions
- - - -
- -
- -
- - - - -
- -
- -
- - - - - - - - - - - - - -
-
- - - - - -
- - - -
- - - -
-
-
-
- - - - - - - - - - - - \ No newline at end of file diff --git a/dagpipe/lib/dagpipe/graph/index.html b/dagpipe/lib/dagpipe/graph/index.html deleted file mode 100644 index a5f0c80..0000000 --- a/dagpipe/lib/dagpipe/graph/index.html +++ /dev/null @@ -1,1428 +0,0 @@ - - - - - - - - - - - - - - - - - - - Graph - dagpipe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - Skip to content - - -
-
- -
- - - - - - -
- - -
- -
- - - - - - -
-
- - - -
-
-
- - - - - -
-
-
- - - -
-
-
- - - -
-
-
- - - -
-
- - - - - -

Graph

- - -
- - - -

- dagpipe.graph - - -

- -
- -

Summary

-

Defines DAG structure connecting nodes.

-

A Graph describes execution topology only. It does not execute nodes or manage -State. Execution is handled by an Engine.

-
-

Responsibilities

-
    -
  • Multiple roots, branching, and merging support.
    • -
  • Deterministic traversal based on topology.
    • -
  • Graph is mutable during construction but treated as immutable at runtime.
    • -
- - - -
- - - - - - -

Classes

- -
- - - -

- Graph - - -

-
1
Graph()
-
- -
- - -

Directed Acyclic Graph defining execution topology of Node objects.

- - -
- Notes -

Responsibilities:

-
1
-2
- Stores node connectivity and validates that the topology remains acyclic.
-- Structure determines how `State` flows between nodes during execution.
-
-

Guarantees:

-
1
-2
- Topology is acyclic. Node relationships remain consistent.
-- Thread-safe for concurrent reads after construction.
-
-
-

Create an empty Graph.

-
Initializes node registry and edge mappings.
- - - - -
- - - - - - - -
Functions
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return debug representation.

-
Returns
-

str

- -
- -
- -
- - -
- add_edge - - -
-
1
add_edge(src: Node, dst: Node) -> None
-
- -
- -

Add a directed edge from src to dst.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
src - Node - -
-

Source node.

-
- 		- required -
dst - Node - -
-

Destination node.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If src or dst is not a Node.

-
-
- ValueError - -
-

If the edge would create a cycle or if src and dst are common.

-
-
- - -
- Notes -
    -
  • Validates node types.
    • -
  • Prevents cycles.
    • -
  • Registers nodes if not present.
    • -
  • Updates parent and child mappings.
    • -
-
-
- -
- -
- - -
- add_root - - -
-
1
add_root(node: Node) -> None
-
- -
- -

Add a root node with no parents.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node - Node - -
-

Node to add as a root.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If node is not a Node instance.

-
-
- -
- -
- -
- - -
- children - - -
-
1
children(node: Node) -> tuple[Node, ...]
-
- -
- -

Return child nodes of a node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node - Node - -
-

Node to query.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Outgoing neighbors.

-
-
- -
- -
- -
- - -
- nodes - - -
-
1
nodes() -> tuple[Node, ...]
-
- -
- -

Return all nodes in the graph.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -All registered nodes.

-
-
- -
- -
- -
- - -
- parents - - -
-
1
parents(node: Node) -> tuple[Node, ...]
-
- -
- -

Return parent nodes of a node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node - Node - -
-

Node to query.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Incoming neighbors.

-
-
- -
- -
- -
- - -
- roots - - -
-
1
roots() -> tuple[Node, ...]
-
- -
- -

Return root nodes (nodes with no incoming edges).

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Entry point nodes.

-
-
- -
- -
- - - -
- -
- -
- - - - -
- -
- -
- - - - - - - - - - - - - -
-
- - - - - -
- - - -
- - - -
-
-
-
- - - - - - - - - - - - \ No newline at end of file diff --git a/dagpipe/lib/dagpipe/index.html b/dagpipe/lib/dagpipe/index.html deleted file mode 100644 index 852f554..0000000 --- a/dagpipe/lib/dagpipe/index.html +++ /dev/null @@ -1,5881 +0,0 @@ - - - - - - - - - - - - - - - - - - - Dagpipe - dagpipe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - Skip to content - - -
-
- -
- - - - - - -
- - -
- -
- - - - - - -
-
- - - -
-
-
- - - - - -
-
-
- - - -
-
-
- - - -
-
-
- - - -
-
- - - - - -

Dagpipe

- - -
- - - -

- dagpipe - - -

- -
- -

Summary

-

Directed acyclic graph execution framework for deterministic state propagation.

-

dagpipe executes pipelines composed of nodes connected in a directed acyclic -graph (DAG). Each node receives an immutable State and optionally produces -derived states for downstream nodes.

-

Installation

-

Install using pip:

-
1
pip install dagpipe
-
-
-

Quick Start

-
 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
-10
-11
-12
-13
-14
-15
-16
-17
from dagpipe import State, Payload, Schema, Graph, Engine
-from dagpipe.node import Node
-
-class HelloNode(Node):
-    id = "hello"
-    def resolve(self, state):
-        yield self.fork(state, payload_update={"msg": "hello"})
-
-# Build and run
-graph = Graph()
-graph.add_root(HelloNode())
-engine = Engine(graph)
-
-class MyState(State):
-    schema = Schema({})
-
-results = engine.run(MyState(payload=Payload({})))
-
-
-

Public API

-

This package re-exports the core pipeline components. -Consumers should import from this namespace for standard usage.

-

Execution Core

-
    -
  • Engine: Responsible for orchestrating node execution and state propagation.
    • -
  • Graph: Defines the execution topology and node relationships.
    • -
  • Node: Base class for defining execution logic and transformations.
    • -
-

State & Data

-
    -
  • State: Represents an immutable execution snapshot at a point in time.
    • -
  • Payload: Immutable hierarchical container for execution data.
    • -
  • Schema: Defines and validates the allowed structure of payloads.
    • -
  • SchemaError: Raised when data violates the declared schema.
    • -
-

Declarative Pipelines

-
    -
  • Pipeline: High-level wrapper for an engine, state type, and initial payload.
    • -
  • load_pipeline: Factory function to create a pipeline from YAML.
    • -
-
- - - -
- - - - - - -

Classes

- -
- - - -

- AsyncNode - - -

- - -
-

- Bases: Node

- - -

Base class for nodes whose execution is asynchronous.

-

Subclasses implement resolve_async (an async generator yielding derived -State objects). The engine dispatches to resolve_async when running an -async traversal (see Engine.run_async).

-

Sync-only engines (and the base Node.run) treat an AsyncNode as a no-op -consumer: calling run on an AsyncNode returns no states, signalling that -an async engine is required.

- - - - -
- - - - - - - -
Functions
- -
- - -
- __hash__ - - -
-
1
__hash__() -> int
-
- -
- -

Return stable hash based on node ID.

-
Returns
-

int

- -
- -
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return debug representation.

-
Returns
-

str

- -
- -
- -
- - -
- __str__ - - -
-
1
__str__() -> str
-
- -
- -

Return display representation.

-
Returns
-

str

- -
- -
- -
- - -
- clean_id_and_name - - - - classmethod - - -
-
1
clean_id_and_name() -> None
-
- -
- -

Normalize and validate node ID and display name.

- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If ID is not a string.

-
-
- ValueError - -
-

If ID format is invalid.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
-3
- Generates ID from module and class name if missing.
-- Validates ID format.
-- Generates human-readable name if missing.
-
-
-
- -
- -
- - -
- fork - - -
-
1
-2
-3
-4
-5
-6
-7
fork(
-    state: State,
-    *,
-    payload_update: Any = None,
-    confidence_delta: float = 0.0,
-    metadata_update: Any = None
-) -> State
-
- -
- -

Create a child State attributed to this node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Parent execution state.

-
- 		- required -
payload_update - Any - -
-

Dot-path payload updates.

-
- 		- None -
confidence_delta - float - -
-

Confidence adjustment.

-
- 		- 0.0 -
metadata_update - Any - -
-

Metadata updates.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
State - State - -
-

New child execution state.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Convenience wrapper around `State.fork()` that automatically
-  records this node's ID in state history.
-
-
-
- -
- -
- - -
- is_async - - -
-
1
is_async() -> bool
-
- -
- -

Return whether this node executes asynchronously.

- -
- -
- -
- - -
- node_id_to_name - - - - staticmethod - - -
-
1
node_id_to_name(node_id: str) -> str
-
- -
- -

Convert a dotted snake_case node ID into a human-readable name.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node_id - str - -
-

Unique node identifier (e.g., 'entity.resolve.numeric_merchant').

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
str - str - -
-

Human-readable display name (e.g., 'Entity › Resolve › Numeric Merchant').

-
-
- -
- -
- -
- - -
- resolve_async - - - - async - - -
-
1
resolve_async(state: State) -> Iterable[State]
-
- -
- -

Execute node logic asynchronously.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Iterable[State] - -
-

Iterable[State]: -Derived execution state(s).

-
-
- - -
- Notes -

Subclasses implement this. Must not mutate the input state. -Should use fork() to create child states.

-
-
- -
- -
- - -
- run - - -
-
1
run(state: State) -> tuple[State, ...]
-
- -
- -

Execute this node on a State.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[State, ...] - -
-

tuple[State, ...]: -Derived execution states.

-
-
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If resolve() yields a non-State object.

-
-
- -
- -
- -
- - -
- run_async - - - - async - - -
-
1
run_async(state: State) -> tuple[State, ...]
-
- -
- -

Execute this node asynchronously on a state, validating outputs.

- -
- -
- - - -
- -
- -
- -
- - - -

- Engine - - -

-
1
-2
-3
-4
-5
Engine(
-    nodes_or_graph: Sequence[Node] | Graph,
-    *,
-    on_step: StepHook | None = None
-)
-
- -
- - -

Execution engine responsible for running pipeline logic.

- - -
- Notes -

Responsibilities:

-
1
-2
-3
-4
-5
-6
-7
- Accepts either a linear sequence of `Node` objects or a `Graph`
-  defining execution topology.
-- Propagates immutable `State` objects through `Node` objects and
-  collects terminal states.
-- Supports synchronous (`run`) and asynchronous (`run_async`)
-  execution, dispatching per-node.
-- Supports step-wise / resumable execution and progress hooks.
-
-

Guarantees:

-
1
-2
-3
-4
-5
- Never mutates `State`, `Node`, or `Graph` instances.
-- `State` objects are never modified in place; each branch produces
-  independent instances.
-- Execution order is deterministic and follows graph or pipeline topology.
-- Thread-safe for concurrent execution.
-
-
- - - -
- - - - - -
Attributes
- -
- - - -
- nodes - - - - property - - -
-
1
nodes: tuple[Node, ...]
-
- -
- -

Return nodes managed by this engine.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Ordered sequence in linear mode or all nodes in graph mode.

-
-
-
- -
- -
Functions
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return the canonical string representation of the object.

- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
str - str - -
-

Representation that uniquely identifies the object and its configuration.

-
-
- -
- -
- -
- - -
- run - - -
-
1
run(root: State) -> list[State]
-
- -
- -

Execute the pipeline starting from a root State.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- list[State] - -
-

list[State]: -Terminal execution states produced by the pipeline.

-
-
- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If root is not a State instance.

-
-
- RuntimeError - -
-

If the engine execution mode is invalid.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Selects execution mode, propagates state through nodes, creates
-  new instances for branches, and collects terminal states.
-
-
-
- -
- -
- - -
- run_async - - - - async - - -
-
1
run_async(root: State) -> list[State]
-
- -
- -

Execute the pipeline starting from root, dispatching sync vs async nodes.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- list[State] - -
-

list[State]: -Terminal execution states produced by the pipeline.

-
-
- - -
- Notes -

Each node is executed with Node.run when synchronous and -AsyncNode.run_async when asynchronous. Linear and graph topologies -are both supported.

-
-
- -
- -
- - -
- run_steps - - -
-
1
-2
-3
-4
-5
-6
run_steps(
-    root: State,
-    *,
-    resume_from: int | None = None,
-    on_step: StepHook | None = None
-) -> Iterator[StepResult]
-
- -
- -

Execute the pipeline step-by-step, yielding one StepResult per step.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
resume_from - int | None - -
-

Skip steps at index < resume_from (for resume-after-partial). -Steps are 0-indexed.

-
- 		- None -
on_step - StepHook | None - -
-

Callback (step, status, message) invoked per step; falls back -to the engine-level hook when unset.

-
- 		- None -
- - -

Yields:

- - - - - - - - - - - - - -
Name TypeDescription
StepResult - StepResult - -
-

One per executed node/step, carrying the produced states.

-
-
- - -
- Notes -

This is a synchronous, generator-based checkpoint interface compatible -with the imperative resume-by-step behaviour of the legacy -orchestrator. Use run_steps_async for async nodes.

-
-
- -
- -
- - -
- run_steps_async - - - - async - - -
-
1
-2
-3
-4
-5
-6
run_steps_async(
-    root: State,
-    *,
-    resume_from: int | None = None,
-    on_step: AsyncStepHook | None = None
-) -> AsyncIterator[StepResult]
-
- -
- -

Async variant of run_steps supporting AsyncNode execution.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
root - State - -
-

Initial execution state.

-
- 		- required -
resume_from - int | None - -
-

Skip steps at index < resume_from.

-
- 		- None -
on_step - AsyncStepHook | None - -
-

Async callback (step, status, message) invoked per step.

-
- 		- None -
- - -

Yields:

- - - - - - - - - - - - - -
Name TypeDescription
StepResult - AsyncIterator[StepResult] - -
-

One per executed node/step.

-
-
- -
- -
- - - -
- -
- -
- -
- - - -

- Graph - - -

-
1
Graph()
-
- -
- - -

Directed Acyclic Graph defining execution topology of Node objects.

- - -
- Notes -

Responsibilities:

-
1
-2
- Stores node connectivity and validates that the topology remains acyclic.
-- Structure determines how `State` flows between nodes during execution.
-
-

Guarantees:

-
1
-2
- Topology is acyclic. Node relationships remain consistent.
-- Thread-safe for concurrent reads after construction.
-
-
-

Create an empty Graph.

-
Initializes node registry and edge mappings.
- - - - -
- - - - - - - -
Functions
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return debug representation.

-
Returns
-

str

- -
- -
- -
- - -
- add_edge - - -
-
1
add_edge(src: Node, dst: Node) -> None
-
- -
- -

Add a directed edge from src to dst.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
src - Node - -
-

Source node.

-
- 		- required -
dst - Node - -
-

Destination node.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If src or dst is not a Node.

-
-
- ValueError - -
-

If the edge would create a cycle or if src and dst are common.

-
-
- - -
- Notes -
    -
  • Validates node types.
    • -
  • Prevents cycles.
    • -
  • Registers nodes if not present.
    • -
  • Updates parent and child mappings.
    • -
-
-
- -
- -
- - -
- add_root - - -
-
1
add_root(node: Node) -> None
-
- -
- -

Add a root node with no parents.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node - Node - -
-

Node to add as a root.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If node is not a Node instance.

-
-
- -
- -
- -
- - -
- children - - -
-
1
children(node: Node) -> tuple[Node, ...]
-
- -
- -

Return child nodes of a node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node - Node - -
-

Node to query.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Outgoing neighbors.

-
-
- -
- -
- -
- - -
- nodes - - -
-
1
nodes() -> tuple[Node, ...]
-
- -
- -

Return all nodes in the graph.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -All registered nodes.

-
-
- -
- -
- -
- - -
- parents - - -
-
1
parents(node: Node) -> tuple[Node, ...]
-
- -
- -

Return parent nodes of a node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node - Node - -
-

Node to query.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Incoming neighbors.

-
-
- -
- -
- -
- - -
- roots - - -
-
1
roots() -> tuple[Node, ...]
-
- -
- -

Return root nodes (nodes with no incoming edges).

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[Node, ...] - -
-

tuple[Node, ...]: -Entry point nodes.

-
-
- -
- -
- - - -
- -
- -
- -
- - - -

- Node - - -

- - -
-

- Bases: ABC

- - -

Base class for all dagpipe execution nodes.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
id - str - -
-

Unique identifier of the node (snake_case dotted format).

-
-
name - str - -
-

Human-readable display name.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Represents a deterministic unit of execution in the pipeline graph.
-- Consumes one `State` and produces zero, one, or many derived states.
-- Defines execution logic and enables branching, filtering, and transformation.
-
-

Guarantees:

-
1
-2
- Nodes must never mutate the input `State`.
-- Instances are singletons per subclass and reused across executions.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- __hash__ - - -
-
1
__hash__() -> int
-
- -
- -

Return stable hash based on node ID.

-
Returns
-

int

- -
- -
- -
- - -
- __new__ - - -
-
1
__new__(*args: Any, **kwargs: Any) -> Node
-
- -
- -

Create or reuse a Node instance.

-

Stateless subclasses (no parameterized __init__) share one singleton -instance per class — matching the original dagpipe behaviour underpinning -set_registry-style configuration. Subclasses that declare an __init__ -requiring instance-state arguments get a fresh instance per construction -so pipeline builders can inject per-run dependencies.

- -
- -
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return debug representation.

-
Returns
-

str

- -
- -
- -
- - -
- __str__ - - -
-
1
__str__() -> str
-
- -
- -

Return display representation.

-
Returns
-

str

- -
- -
- -
- - -
- clean_id_and_name - - - - classmethod - - -
-
1
clean_id_and_name() -> None
-
- -
- -

Normalize and validate node ID and display name.

- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If ID is not a string.

-
-
- ValueError - -
-

If ID format is invalid.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
-3
- Generates ID from module and class name if missing.
-- Validates ID format.
-- Generates human-readable name if missing.
-
-
-
- -
- -
- - -
- fork - - -
-
1
-2
-3
-4
-5
-6
-7
fork(
-    state: State,
-    *,
-    payload_update: Any = None,
-    confidence_delta: float = 0.0,
-    metadata_update: Any = None
-) -> State
-
- -
- -

Create a child State attributed to this node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Parent execution state.

-
- 		- required -
payload_update - Any - -
-

Dot-path payload updates.

-
- 		- None -
confidence_delta - float - -
-

Confidence adjustment.

-
- 		- 0.0 -
metadata_update - Any - -
-

Metadata updates.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
State - State - -
-

New child execution state.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Convenience wrapper around `State.fork()` that automatically
-  records this node's ID in state history.
-
-
-
- -
- -
- - -
- is_async - - -
-
1
is_async() -> bool
-
- -
- -

Return whether this node executes asynchronously.

- -
- -
- -
- - -
- node_id_to_name - - - - staticmethod - - -
-
1
node_id_to_name(node_id: str) -> str
-
- -
- -

Convert a dotted snake_case node ID into a human-readable name.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node_id - str - -
-

Unique node identifier (e.g., 'entity.resolve.numeric_merchant').

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
str - str - -
-

Human-readable display name (e.g., 'Entity › Resolve › Numeric Merchant').

-
-
- -
- -
- -
- - -
- resolve - - - - abstractmethod - - -
-
1
resolve(state: State) -> Iterable[State]
-
- -
- -

Execute node logic.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Iterable[State] - -
-

Iterable[State]: -Derived execution state(s).

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
-4
- Subclasses implement specific resolution behavior.
-- Must not mutate input state.
-- Should use `fork()` to create child states.
-- May yield zero states to terminate a branch.
-
-
-
- -
- -
- - -
- run - - -
-
1
run(state: State) -> tuple[State, ...]
-
- -
- -

Execute this node on a State.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[State, ...] - -
-

tuple[State, ...]: -Derived execution states.

-
-
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If resolve() yields a non-State object.

-
-
- -
- -
- - - -
- -
- -
- -
- - - -

- Payload - - - - dataclass - - -

-
1
Payload(_data: Mapping[str, Any])
-
- -
- - -

Immutable hierarchical container with dot-path access.

- - -

Attributes:

- - - - - - - - - - - - - - - -
NameTypeDescription
_data - Mapping[str, Any] - -
-

Immutable hierarchical data structure.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Stores execution data used by `State`.
-- Supports efficient atomic updates without modifying existing instances.
-- `Payload` instances are fully thread-safe due to immutability.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- as_dict - - -
-
1
as_dict() -> Mapping[str, Any]
-
- -
- -

Return underlying mapping.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Mapping[str, Any] - -
-

Mapping[str, Any]: -Read-only view of the underlying data.

-
-
- -
- -
- -
- - -
- get - - -
-
1
get(path: str, default: Any = None) -> Any
-
- -
- -

Retrieve value using dot-path.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
path - str - -
-

Dot-separated path to the value.

-
- 		- required -
default - Any - -
-

Default value if path doesn't exist.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Any - Any - -
-

The retrieved value or default.

-
-
- -
- -
- -
- - -
- has - - -
-
1
has(path: str) -> bool
-
- -
- -

Return True if path exists.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
path - str - -
-

Dot-separated path to check.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
bool - bool - -
-

Existence of the path.

-
-
- -
- -
- -
- - -
- iter_paths - - - - classmethod - - -
-
1
-2
-3
iter_paths(
-    data: Mapping[str, Any], prefix: str = ""
-) -> Iterable[str]
-
- -
- -

Recursively yield dot-paths for all leaf nodes.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
data - Mapping[str, Any] - -
-

The mapping to iterate over.

-
- 		- required -
prefix - str - -
-

Current path prefix.

-
- 		- '' -
- - -

Yields:

- - - - - - - - - - - - - -
Name TypeDescription
str - Iterable[str] - -
-

Dot-path for each leaf node.

-
-
- -
- -
- -
- - -
- keys - - -
-
1
keys() -> Iterable[str]
-
- -
- -

Return top-level keys.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Iterable[str] - -
-

Iterable[str]: -Iterator over top-level keys.

-
-
- -
- -
- -
- - -
- update - - -
-
1
update(updates: Mapping[str, Any]) -> Payload
-
- -
- -

Create a new Payload with dot-path updates applied.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
updates - Mapping[str, Any] - -
-

Dot-path to value mapping.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Payload - Payload - -
-

New immutable payload instance with updates.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
- Preserves existing data by copying only modified branches.
-- Returns a new immutable `Payload`.
-
-
-
- -
- - - -
- -
- -
- -
- - - -

- Pipeline - - - - dataclass - - -

-
1
-2
-3
-4
-5
Pipeline(
-    engine: Engine,
-    state_cls: type[State],
-    initial_payload: Payload,
-)
-
- -
- - -

Executable pipeline created from YAML configuration.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
engine - Engine - -
-

Execution engine responsible for running the pipeline.

-
-
state_cls - Type[State] - -
-

Dynamically created State subclass with configured schema.

-
-
initial_payload - Payload - -
-

Default payload used when execution begins.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Encapsulates engine, state type, and initial payload.
-- Provides a simplified interface for executing configured pipelines.
-- Safe for concurrent execution if underlying nodes are thread-safe.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- run - - -
-
1
-2
-3
run(
-    payload_override: Mapping[str, Any] | None = None,
-) -> list[State]
-
- -
- -

Execute the pipeline.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
payload_override - Mapping[str, Any] | None - -
-

Payload values overriding initial payload.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- list[State] - -
-

list[State]: -Terminal execution states.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Merges override payload with initial payload.
-- Creates root `State` and executes engine.
-
-
-
- -
- - - -
- -
- -
- -
- - - -

- ProgressMessage - - -

-
 1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
-10
-11
ProgressMessage(
-    *,
-    lines: int | None = None,
-    blocks: int | None = None,
-    count: int | None = None,
-    unit: str | None = None,
-    raw_ocr_line: str | None = None,
-    error: str | None = None,
-    step: str = "",
-    status: str = ""
-)
-
- -
- - -

Lightweight progress payload emitted by engine step hooks.

-

Mirrors the imperative ProgressMessage used by the legacy orchestrator so -callers can surface counts/lines/errors without coupling the engine to pydantic.

- - - - -
- - - - - - - - - - - -
- -
- -
- -
- - - -

- Schema - - - - dataclass - - -

-
1
Schema(tree: Mapping[str, SchemaNode])
-
- -
- - -

Immutable hierarchical schema defining allowed payload structure.

- - -

Attributes:

- - - - - - - - - - - - - - - -
NameTypeDescription
tree - Mapping[str, SchemaNode] - -
-

Hierarchical schema definition.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Validates `State` payloads and updates.
-- Reusable across all `State` instances.
-- Fully thread-safe due to immutability.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- validate_payload - - -
-
1
validate_payload(payload: Payload) -> None
-
- -
- -

Validate complete payload structure.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
payload - Payload - -
-

Payload to validate.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- SchemaError - -
-

If payload violates schema.

-
-
- -
- -
- -
- - -
- validate_update - - -
-
1
validate_update(updates: Mapping[str, Any]) -> None
-
- -
- -

Validate payload update paths.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
updates - Mapping[str, Any] - -
-

Dot-path updates to validate.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- SchemaError - -
-

If any path is invalid according to the schema.

-
-
- -
- -
- - - -
- -
- -
- -
- - - -

- SchemaError - - -

- - -
-

- Bases: Exception

- - -

Raised when payload data violates the declared schema.

-
Indicates invalid structure, invalid path, or invalid type.
- - -
- -
- -
- - - -

- State - - - - dataclass - - -

-
1
-2
-3
-4
-5
-6
-7
-8
State(
-    payload: Payload,
-    confidence: float = 1.0,
-    parent: State | None = None,
-    depth: int = 0,
-    history: tuple[str, ...] = tuple(),
-    metadata: dict[str, Any] = dict(),
-)
-
- -
- - -

Immutable execution state propagated through dagpipe pipeline.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
payload - Payload - -
-

Execution data container.

-
-
schema - ClassVar[Schema] - -
-

Payload validation schema.

-
-
confidence - float - -
-

Execution confidence score.

-
-
parent - Optional[State] - -
-

Parent state reference.

-
-
depth - int - -
-

Execution depth.

-
-
history - Tuple[str, ...] - -
-

Ordered node execution lineage.

-
-
metadata - Dict[str, Any] - -
-

Execution metadata.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
-4
- Represents a complete execution snapshot at a specific point in
-  pipeline traversal.
-- Fundamental unit of execution in `dagpipe`.
-- Fully thread-safe due to immutability.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Concise debug representation.

-

Avoids printing full data for large states.

- -
- -
- -
- - -
- fork - - -
-
1
-2
-3
-4
-5
-6
-7
fork(
-    *,
-    payload_update: Mapping[str, Any] | None = None,
-    confidence_delta: float = 0.0,
-    node_id: str | None = None,
-    metadata_update: Mapping[str, Any] | None = None
-) -> State
-
- -
- -

Create a new child State derived from this state.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
payload_update - Mapping[str, Any] | None - -
-

Dot-path updates applied to the payload.

-
- 		- None -
confidence_delta - float - -
-

Adjustment applied to current confidence.

-
- 		- 0.0 -
node_id - str | None - -
-

Identifier of the node creating this state.

-
- 		- None -
metadata_update - Mapping[str, Any] | None - -
-

Updates merged into state metadata.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
State - State - -
-

A new immutable State instance.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
-3
- This is the only supported mechanism for modifying execution data.
-- Validates payload updates, preserves lineage, increments depth,
-  and appends to history.
-
-
-
- -
- -
- - -
- get - - -
-
1
get(key: str, default: Any = None) -> Any
-
- -
- -

Retrieve payload value.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
key - str - -
-

Dot-path key.

-
- 		- required -
default - Any - -
-

Fallback value.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Any - Any - -
-

Stored value or default.

-
-
- -
- -
- -
- - -
- has - - -
-
1
has(key: str) -> bool
-
- -
- -

Check whether payload contains key.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
key - str - -
-

Dot-path key.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
bool - bool - -
-

Existence of the key.

-
-
- -
- -
- -
- - -
- lineage - - -
-
1
lineage() -> tuple[State, ...]
-
- -
- -

Return lineage from root to this State.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[State, ...] - -
-

tuple[State, ...]: -Ordered execution lineage (root first).

-
-
- -
- -
- - - -
- -
- -
- -
- - - -

- StepResult - - -

-
1
-2
-3
-4
-5
-6
StepResult(
-    index: int,
-    node_id: str,
-    states: tuple[State, ...],
-    completed: bool,
-)
-
- -
- - -

A single checkpointed step within an async/resumable engine run.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
index - int - -
-

Ordinal index of the step.

-
-
node_id - str - -
-

Identifier of the node associated with this step.

-
-
states - tuple[State, ...] - -
-

States produced by running this step.

-
-
completed - bool - -
-

Whether this step succeeded (vs. paused/interrupted).

-
-
- -

Initialise StepResult.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
index - int - -
-

Ordinal index of the step.

-
- 		- required -
node_id - str - -
-

Identifier of the node associated with this step.

-
- 		- required -
states - tuple[State, ...] - -
-

States produced by running this step.

-
- 		- required -
completed - bool - -
-

Whether this step succeeded (vs. paused/interrupted).

-
- 		- required -
- - - - -
- - - - - - - -
Functions
- - - -
- -
- -
-

Functions

- -
- - -

- load_pipeline - - -

-
1
load_pipeline(path: str) -> Pipeline
-
- -
- -

Load pipeline from YAML file.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
path - str - -
-

Path to YAML configuration file.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Pipeline - Pipeline - -
-

Executable pipeline instance.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Loads YAML configuration and builds schema.
-- Creates `State` subclass and loads `Node` instances.
-- Builds `Graph` topology and initializes `Engine`.
-
-
-
- -
- - - -
- -
- -
- - - - - - - - - - - - - -
-
- - - - - -
- - - -
- - - -
-
-
-
- - - - - - - - - - - - \ No newline at end of file diff --git a/dagpipe/lib/dagpipe/node/index.html b/dagpipe/lib/dagpipe/node/index.html deleted file mode 100644 index aecbabb..0000000 --- a/dagpipe/lib/dagpipe/node/index.html +++ /dev/null @@ -1,2391 +0,0 @@ - - - - - - - - - - - - - - - - - - - Node - dagpipe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - Skip to content - - -
-
- -
- - - - - - -
- - -
- -
- - - - - - -
-
- - - -
-
-
- - - - - -
-
-
- - - -
-
-
- - - -
-
-
- - - -
-
- - - - - -

Node

- - -
- - - -

- dagpipe.node - - -

- -
- -

Summary

-

Defines the Node abstraction used by dagpipe.

-

A node represents a single unit of pipeline execution logic. It consumes one -State and produces zero, one, or many new State objects.

-

Nodes are connected using a Graph and executed by an Engine.

-
-

Design principles

-
    -
  • Pure: Must not mutate input state.
    • -
  • Deterministic: Same input produces same output.
    • -
  • Stateless: Recommended to be stateless for reuse.
    • -
  • Composable: Nodes enable branching execution graphs.
    • -
- - - -
- - - - - - -

Classes

- -
- - - -

- AsyncNode - - -

- - -
-

- Bases: Node

- - -

Base class for nodes whose execution is asynchronous.

-

Subclasses implement resolve_async (an async generator yielding derived -State objects). The engine dispatches to resolve_async when running an -async traversal (see Engine.run_async).

-

Sync-only engines (and the base Node.run) treat an AsyncNode as a no-op -consumer: calling run on an AsyncNode returns no states, signalling that -an async engine is required.

- - - - -
- - - - - - - -
Functions
- -
- - -
- __hash__ - - -
-
1
__hash__() -> int
-
- -
- -

Return stable hash based on node ID.

-
Returns
-

int

- -
- -
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return debug representation.

-
Returns
-

str

- -
- -
- -
- - -
- __str__ - - -
-
1
__str__() -> str
-
- -
- -

Return display representation.

-
Returns
-

str

- -
- -
- -
- - -
- clean_id_and_name - - - - classmethod - - -
-
1
clean_id_and_name() -> None
-
- -
- -

Normalize and validate node ID and display name.

- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If ID is not a string.

-
-
- ValueError - -
-

If ID format is invalid.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
-3
- Generates ID from module and class name if missing.
-- Validates ID format.
-- Generates human-readable name if missing.
-
-
-
- -
- -
- - -
- fork - - -
-
1
-2
-3
-4
-5
-6
-7
fork(
-    state: State,
-    *,
-    payload_update: Any = None,
-    confidence_delta: float = 0.0,
-    metadata_update: Any = None
-) -> State
-
- -
- -

Create a child State attributed to this node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Parent execution state.

-
- 		- required -
payload_update - Any - -
-

Dot-path payload updates.

-
- 		- None -
confidence_delta - float - -
-

Confidence adjustment.

-
- 		- 0.0 -
metadata_update - Any - -
-

Metadata updates.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
State - State - -
-

New child execution state.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Convenience wrapper around `State.fork()` that automatically
-  records this node's ID in state history.
-
-
-
- -
- -
- - -
- is_async - - -
-
1
is_async() -> bool
-
- -
- -

Return whether this node executes asynchronously.

- -
- -
- -
- - -
- node_id_to_name - - - - staticmethod - - -
-
1
node_id_to_name(node_id: str) -> str
-
- -
- -

Convert a dotted snake_case node ID into a human-readable name.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node_id - str - -
-

Unique node identifier (e.g., 'entity.resolve.numeric_merchant').

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
str - str - -
-

Human-readable display name (e.g., 'Entity › Resolve › Numeric Merchant').

-
-
- -
- -
- -
- - -
- resolve_async - - - - async - - -
-
1
resolve_async(state: State) -> Iterable[State]
-
- -
- -

Execute node logic asynchronously.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Iterable[State] - -
-

Iterable[State]: -Derived execution state(s).

-
-
- - -
- Notes -

Subclasses implement this. Must not mutate the input state. -Should use fork() to create child states.

-
-
- -
- -
- - -
- run - - -
-
1
run(state: State) -> tuple[State, ...]
-
- -
- -

Execute this node on a State.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[State, ...] - -
-

tuple[State, ...]: -Derived execution states.

-
-
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If resolve() yields a non-State object.

-
-
- -
- -
- -
- - -
- run_async - - - - async - - -
-
1
run_async(state: State) -> tuple[State, ...]
-
- -
- -

Execute this node asynchronously on a state, validating outputs.

- -
- -
- - - -
- -
- -
- -
- - - -

- Node - - -

- - -
-

- Bases: ABC

- - -

Base class for all dagpipe execution nodes.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
id - str - -
-

Unique identifier of the node (snake_case dotted format).

-
-
name - str - -
-

Human-readable display name.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Represents a deterministic unit of execution in the pipeline graph.
-- Consumes one `State` and produces zero, one, or many derived states.
-- Defines execution logic and enables branching, filtering, and transformation.
-
-

Guarantees:

-
1
-2
- Nodes must never mutate the input `State`.
-- Instances are singletons per subclass and reused across executions.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- __hash__ - - -
-
1
__hash__() -> int
-
- -
- -

Return stable hash based on node ID.

-
Returns
-

int

- -
- -
- -
- - -
- __new__ - - -
-
1
__new__(*args: Any, **kwargs: Any) -> Node
-
- -
- -

Create or reuse a Node instance.

-

Stateless subclasses (no parameterized __init__) share one singleton -instance per class — matching the original dagpipe behaviour underpinning -set_registry-style configuration. Subclasses that declare an __init__ -requiring instance-state arguments get a fresh instance per construction -so pipeline builders can inject per-run dependencies.

- -
- -
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Return debug representation.

-
Returns
-

str

- -
- -
- -
- - -
- __str__ - - -
-
1
__str__() -> str
-
- -
- -

Return display representation.

-
Returns
-

str

- -
- -
- -
- - -
- clean_id_and_name - - - - classmethod - - -
-
1
clean_id_and_name() -> None
-
- -
- -

Normalize and validate node ID and display name.

- - -

Raises:

- - - - - - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If ID is not a string.

-
-
- ValueError - -
-

If ID format is invalid.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
-3
- Generates ID from module and class name if missing.
-- Validates ID format.
-- Generates human-readable name if missing.
-
-
-
- -
- -
- - -
- fork - - -
-
1
-2
-3
-4
-5
-6
-7
fork(
-    state: State,
-    *,
-    payload_update: Any = None,
-    confidence_delta: float = 0.0,
-    metadata_update: Any = None
-) -> State
-
- -
- -

Create a child State attributed to this node.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Parent execution state.

-
- 		- required -
payload_update - Any - -
-

Dot-path payload updates.

-
- 		- None -
confidence_delta - float - -
-

Confidence adjustment.

-
- 		- 0.0 -
metadata_update - Any - -
-

Metadata updates.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
State - State - -
-

New child execution state.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Convenience wrapper around `State.fork()` that automatically
-  records this node's ID in state history.
-
-
-
- -
- -
- - -
- is_async - - -
-
1
is_async() -> bool
-
- -
- -

Return whether this node executes asynchronously.

- -
- -
- -
- - -
- node_id_to_name - - - - staticmethod - - -
-
1
node_id_to_name(node_id: str) -> str
-
- -
- -

Convert a dotted snake_case node ID into a human-readable name.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
node_id - str - -
-

Unique node identifier (e.g., 'entity.resolve.numeric_merchant').

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
str - str - -
-

Human-readable display name (e.g., 'Entity › Resolve › Numeric Merchant').

-
-
- -
- -
- -
- - -
- resolve - - - - abstractmethod - - -
-
1
resolve(state: State) -> Iterable[State]
-
- -
- -

Execute node logic.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Iterable[State] - -
-

Iterable[State]: -Derived execution state(s).

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
-4
- Subclasses implement specific resolution behavior.
-- Must not mutate input state.
-- Should use `fork()` to create child states.
-- May yield zero states to terminate a branch.
-
-
-
- -
- -
- - -
- run - - -
-
1
run(state: State) -> tuple[State, ...]
-
- -
- -

Execute this node on a State.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
state - State - -
-

Input execution state.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[State, ...] - -
-

tuple[State, ...]: -Derived execution states.

-
-
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- TypeError - -
-

If resolve() yields a non-State object.

-
-
- -
- -
- - - -
- -
- -
- - - - -
- -
- -
- - - - - - - - - - - - - -
-
- - - - - -
- - - -
- - - -
-
-
-
- - - - - - - - - - - - \ No newline at end of file diff --git a/dagpipe/lib/dagpipe/state/index.html b/dagpipe/lib/dagpipe/state/index.html deleted file mode 100644 index e318fc0..0000000 --- a/dagpipe/lib/dagpipe/state/index.html +++ /dev/null @@ -1,2324 +0,0 @@ - - - - - - - - - - - - - - - - - - - State - dagpipe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - Skip to content - - -
-
- -
- - - - - - -
- - -
- -
- - - - - - -
-
- - - -
-
-
- - - - - -
-
-
- - - -
-
-
- - - -
-
-
- - - -
-
- - - - - -

State

- - -
- - - -

- dagpipe.state - - -

- -
- -

Summary

-

Defines the core State object used by dagpipe.

-

The State represents a single point in pipeline execution. It contains -arbitrary data and metadata and is designed to be immutable. Instead of -modifying an existing state, nodes create new child states via fork().

-
-

Design principles

-
    -
  • Immutability: States must never be modified after creation. - All transformations must create a new state via fork().
    • -
  • Cheap cloning: Forking must be efficient since branching may create many states.
    • -
  • Lineage tracking: Each state maintains a reference to its parent and - execution metadata for debugging and observability.
    • -
  • Domain agnostic: State contains generic key-value data and does not - assume any schema.
    • -
  • Engine-friendly: State contains execution metadata such as depth and history.
    • -
- - - -
- - - - - - -

Classes

- -
- - - -

- Payload - - - - dataclass - - -

-
1
Payload(_data: Mapping[str, Any])
-
- -
- - -

Immutable hierarchical container with dot-path access.

- - -

Attributes:

- - - - - - - - - - - - - - - -
NameTypeDescription
_data - Mapping[str, Any] - -
-

Immutable hierarchical data structure.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Stores execution data used by `State`.
-- Supports efficient atomic updates without modifying existing instances.
-- `Payload` instances are fully thread-safe due to immutability.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- as_dict - - -
-
1
as_dict() -> Mapping[str, Any]
-
- -
- -

Return underlying mapping.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Mapping[str, Any] - -
-

Mapping[str, Any]: -Read-only view of the underlying data.

-
-
- -
- -
- -
- - -
- get - - -
-
1
get(path: str, default: Any = None) -> Any
-
- -
- -

Retrieve value using dot-path.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
path - str - -
-

Dot-separated path to the value.

-
- 		- required -
default - Any - -
-

Default value if path doesn't exist.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Any - Any - -
-

The retrieved value or default.

-
-
- -
- -
- -
- - -
- has - - -
-
1
has(path: str) -> bool
-
- -
- -

Return True if path exists.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
path - str - -
-

Dot-separated path to check.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
bool - bool - -
-

Existence of the path.

-
-
- -
- -
- -
- - -
- iter_paths - - - - classmethod - - -
-
1
-2
-3
iter_paths(
-    data: Mapping[str, Any], prefix: str = ""
-) -> Iterable[str]
-
- -
- -

Recursively yield dot-paths for all leaf nodes.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
data - Mapping[str, Any] - -
-

The mapping to iterate over.

-
- 		- required -
prefix - str - -
-

Current path prefix.

-
- 		- '' -
- - -

Yields:

- - - - - - - - - - - - - -
Name TypeDescription
str - Iterable[str] - -
-

Dot-path for each leaf node.

-
-
- -
- -
- -
- - -
- keys - - -
-
1
keys() -> Iterable[str]
-
- -
- -

Return top-level keys.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- Iterable[str] - -
-

Iterable[str]: -Iterator over top-level keys.

-
-
- -
- -
- -
- - -
- update - - -
-
1
update(updates: Mapping[str, Any]) -> Payload
-
- -
- -

Create a new Payload with dot-path updates applied.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
updates - Mapping[str, Any] - -
-

Dot-path to value mapping.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Payload - Payload - -
-

New immutable payload instance with updates.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
- Preserves existing data by copying only modified branches.
-- Returns a new immutable `Payload`.
-
-
-
- -
- - - -
- -
- -
- -
- - - -

- Schema - - - - dataclass - - -

-
1
Schema(tree: Mapping[str, SchemaNode])
-
- -
- - -

Immutable hierarchical schema defining allowed payload structure.

- - -

Attributes:

- - - - - - - - - - - - - - - -
NameTypeDescription
tree - Mapping[str, SchemaNode] - -
-

Hierarchical schema definition.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Validates `State` payloads and updates.
-- Reusable across all `State` instances.
-- Fully thread-safe due to immutability.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- validate_payload - - -
-
1
validate_payload(payload: Payload) -> None
-
- -
- -

Validate complete payload structure.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
payload - Payload - -
-

Payload to validate.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- SchemaError - -
-

If payload violates schema.

-
-
- -
- -
- -
- - -
- validate_update - - -
-
1
validate_update(updates: Mapping[str, Any]) -> None
-
- -
- -

Validate payload update paths.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
updates - Mapping[str, Any] - -
-

Dot-path updates to validate.

-
- 		- required -
- - -

Raises:

- - - - - - - - - - - - - -
TypeDescription
- SchemaError - -
-

If any path is invalid according to the schema.

-
-
- -
- -
- - - -
- -
- -
- -
- - - -

- SchemaError - - -

- - -
-

- Bases: Exception

- - -

Raised when payload data violates the declared schema.

-
Indicates invalid structure, invalid path, or invalid type.
- - -
- -
- -
- - - -

- State - - - - dataclass - - -

-
1
-2
-3
-4
-5
-6
-7
-8
State(
-    payload: Payload,
-    confidence: float = 1.0,
-    parent: State | None = None,
-    depth: int = 0,
-    history: tuple[str, ...] = tuple(),
-    metadata: dict[str, Any] = dict(),
-)
-
- -
- - -

Immutable execution state propagated through dagpipe pipeline.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
payload - Payload - -
-

Execution data container.

-
-
schema - ClassVar[Schema] - -
-

Payload validation schema.

-
-
confidence - float - -
-

Execution confidence score.

-
-
parent - Optional[State] - -
-

Parent state reference.

-
-
depth - int - -
-

Execution depth.

-
-
history - Tuple[str, ...] - -
-

Ordered node execution lineage.

-
-
metadata - Dict[str, Any] - -
-

Execution metadata.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
-4
- Represents a complete execution snapshot at a specific point in
-  pipeline traversal.
-- Fundamental unit of execution in `dagpipe`.
-- Fully thread-safe due to immutability.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- __repr__ - - -
-
1
__repr__() -> str
-
- -
- -

Concise debug representation.

-

Avoids printing full data for large states.

- -
- -
- -
- - -
- fork - - -
-
1
-2
-3
-4
-5
-6
-7
fork(
-    *,
-    payload_update: Mapping[str, Any] | None = None,
-    confidence_delta: float = 0.0,
-    node_id: str | None = None,
-    metadata_update: Mapping[str, Any] | None = None
-) -> State
-
- -
- -

Create a new child State derived from this state.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
payload_update - Mapping[str, Any] | None - -
-

Dot-path updates applied to the payload.

-
- 		- None -
confidence_delta - float - -
-

Adjustment applied to current confidence.

-
- 		- 0.0 -
node_id - str | None - -
-

Identifier of the node creating this state.

-
- 		- None -
metadata_update - Mapping[str, Any] | None - -
-

Updates merged into state metadata.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
State - State - -
-

A new immutable State instance.

-
-
- - -
- Notes -

Guarantees:

-
1
-2
-3
- This is the only supported mechanism for modifying execution data.
-- Validates payload updates, preserves lineage, increments depth,
-  and appends to history.
-
-
-
- -
- -
- - -
- get - - -
-
1
get(key: str, default: Any = None) -> Any
-
- -
- -

Retrieve payload value.

- - -

Parameters:

- - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
key - str - -
-

Dot-path key.

-
- 		- required -
default - Any - -
-

Fallback value.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Any - Any - -
-

Stored value or default.

-
-
- -
- -
- -
- - -
- has - - -
-
1
has(key: str) -> bool
-
- -
- -

Check whether payload contains key.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
key - str - -
-

Dot-path key.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
bool - bool - -
-

Existence of the key.

-
-
- -
- -
- -
- - -
- lineage - - -
-
1
lineage() -> tuple[State, ...]
-
- -
- -

Return lineage from root to this State.

- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- tuple[State, ...] - -
-

tuple[State, ...]: -Ordered execution lineage (root first).

-
-
- -
- -
- - - -
- -
- -
- - - - -
- -
- -
- - - - - - - - - - - - - -
-
- - - - - -
- - - -
- - - -
-
-
-
- - - - - - - - - - - - \ No newline at end of file diff --git a/dagpipe/lib/dagpipe/yaml_loader/index.html b/dagpipe/lib/dagpipe/yaml_loader/index.html deleted file mode 100644 index a5f1eb8..0000000 --- a/dagpipe/lib/dagpipe/yaml_loader/index.html +++ /dev/null @@ -1,1158 +0,0 @@ - - - - - - - - - - - - - - - - - - - Yaml Loader - dagpipe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - Skip to content - - -
-
- -
- - - - - - -
- - -
- -
- - - - - - -
-
- - - -
-
-
- - - - - -
-
-
- - - -
-
-
- - - -
-
-
- - - -
-
- - - - - -

Yaml Loader

- - -
- - - -

- dagpipe.yaml_loader - - -

- -
- -

Summary

-

Loads dagpipe pipelines from YAML configuration.

-

Creates fully configured pipeline objects from declarative YAML definitions, -including Schema, State subclasses, Node instances, Graph topology, -and initial payloads.

- - - -
- - - - - - -

Classes

- -
- - - -

- Pipeline - - - - dataclass - - -

-
1
-2
-3
-4
-5
Pipeline(
-    engine: Engine,
-    state_cls: type[State],
-    initial_payload: Payload,
-)
-
- -
- - -

Executable pipeline created from YAML configuration.

- - -

Attributes:

- - - - - - - - - - - - - - - - - - - - - - - - - -
NameTypeDescription
engine - Engine - -
-

Execution engine responsible for running the pipeline.

-
-
state_cls - Type[State] - -
-

Dynamically created State subclass with configured schema.

-
-
initial_payload - Payload - -
-

Default payload used when execution begins.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Encapsulates engine, state type, and initial payload.
-- Provides a simplified interface for executing configured pipelines.
-- Safe for concurrent execution if underlying nodes are thread-safe.
-
-
- - - -
- - - - - - - -
Functions
- -
- - -
- run - - -
-
1
-2
-3
run(
-    payload_override: Mapping[str, Any] | None = None,
-) -> list[State]
-
- -
- -

Execute the pipeline.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
payload_override - Mapping[str, Any] | None - -
-

Payload values overriding initial payload.

-
- 		- None -
- - -

Returns:

- - - - - - - - - - - - - -
TypeDescription
- list[State] - -
-

list[State]: -Terminal execution states.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
- Merges override payload with initial payload.
-- Creates root `State` and executes engine.
-
-
-
- -
- - - -
- -
- -
-

Functions

- -
- - -

- load_pipeline - - -

-
1
load_pipeline(path: str) -> Pipeline
-
- -
- -

Load pipeline from YAML file.

- - -

Parameters:

- - - - - - - - - - - - - - - - - -
NameTypeDescriptionDefault
path - str - -
-

Path to YAML configuration file.

-
- 		- required -
- - -

Returns:

- - - - - - - - - - - - - -
Name TypeDescription
Pipeline - Pipeline - -
-

Executable pipeline instance.

-
-
- - -
- Notes -

Responsibilities:

-
1
-2
-3
- Loads YAML configuration and builds schema.
-- Creates `State` subclass and loads `Node` instances.
-- Builds `Graph` topology and initializes `Engine`.
-
-
-
- -
- - - -
- -
- -
- - - - - - - - - - - - - -
-
- - - - - -
- - - -
- - - -
-
-
-
- - - - - - - - - - - - \ No newline at end of file diff --git a/dagpipe/lib/engine/index.html b/dagpipe/lib/engine/index.html index ad01711..4ba8069 100644 --- a/dagpipe/lib/engine/index.html +++ b/dagpipe/lib/engine/index.html @@ -728,6 +728,36 @@ + +
  • @@ -1029,6 +1059,36 @@ + +
  • @@ -1174,6 +1234,77 @@ It orchestrates execution order, branching, and state propagation.

    - Thread-safe for concurrent execution. +

    Create an engine from a node sequence or a graph.

    + + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    nodes_or_graph + Sequence[Node] | Graph + +
    +

    Either an ordered sequence of Node instances (linear mode) or +a Graph defining the execution topology (graph mode).

    +
    +     		+ required +
    on_step + StepHook | None + +
    +

    Default per-step callback (step, status, message) used when a +step run does not supply its own hook.

    +
    +     		+ None +
    + + +

    Raises:

    + + + + + + + + + + + + + +
    TypeDescription
    + TypeError + +
    +

    If a sequence element is not a Node, or if +nodes_or_graph is neither a Sequence[Node] nor a Graph.

    +
    +
    + @@ -1752,6 +1883,246 @@ orchestrator. Use run_steps_async for async nodes.

    callers can surface counts/lines/errors without coupling the engine to pydantic.

    +

    Attributes:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescription
    lines + int | None + +
    +

    Optional count of processed lines.

    +
    +
    blocks + int | None + +
    +

    Optional count of processed blocks.

    +
    +
    count + int | None + +
    +

    Optional generic item count.

    +
    +
    unit + str | None + +
    +

    Optional unit for the count (e.g., 'pages').

    +
    +
    raw_ocr_line + str | None + +
    +

    Optional raw OCR line payload.

    +
    +
    error + str | None + +
    +

    Optional error description.

    +
    +
    step + str + +
    +

    Identifier of the step that emitted the message.

    +
    +
    status + str + +
    +

    Status label associated with the step.

    +
    +
    + + +
    + Notes +

    Guarantees:

    +
    1
+2
    - Immutable after construction (attributes are never reassigned).
+- Independent of pydantic; safe to construct in the engine core.
+
    +
    +

    Create a progress message.

    + + +

    Parameters:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameTypeDescriptionDefault
    lines + int | None + +
    +

    Optional count of processed lines.

    +
    +     		+ None +
    blocks + int | None + +
    +

    Optional count of processed blocks.

    +
    +     		+ None +
    count + int | None + +
    +

    Optional generic item count.

    +
    +     		+ None +
    unit + str | None + +
    +

    Optional unit for the count (e.g., 'pages').

    +
    +     		+ None +
    raw_ocr_line + str | None + +
    +

    Optional raw OCR line payload.

    +
    +     		+ None +
    error + str | None + +
    +

    Optional error description.

    +
    +     		+ None +
    step + str + +
    +

    Identifier of the step that emitted the message.

    +
    +     		+ '' +
    status + str + +
    +

    Status label associated with the step.

    +
    +     		+ '' +
    + +
    @@ -1762,7 +2133,50 @@ callers can surface counts/lines/errors without coupling the engine to pydantic. +
    Functions
    +
    + + +
    + as_dict + + +
    +
    1
    as_dict() -> dict[str, Any]
+
    + +
    + +

    Return the message as a plain dictionary.

    + + +

    Returns:

    + + + + + + + + + + + + + +
    TypeDescription
    + dict[str, Any] + +
    +

    dict[str, Any]: +All attribute values keyed by their attribute name.

    +
    +
    + +
    + +
    diff --git a/dagpipe/lib/graph/index.html b/dagpipe/lib/graph/index.html index b4391d7..0e0eb5e 100644 --- a/dagpipe/lib/graph/index.html +++ b/dagpipe/lib/graph/index.html @@ -553,15 +553,6 @@

    Create an empty Graph.

    -
    Initializes node registry and edge mappings.
    +

    Initializes node registry and edge mappings.

    @@ -1137,9 +1101,31 @@
    -

    Return debug representation.

    -
    Returns
    -

    str

    +

    Return a compact graph description.

    + + +

    Returns:

    + + + + + + + + + + + + + +
    Name TypeDescription
    str + str + +
    +

    A string describing the graph as Graph(nodes=N, edges=M) where +N and M describe the current registry size.

    +
    +
    diff --git a/dagpipe/lib/index.html b/dagpipe/lib/index.html index f0c557c..c19232f 100644 --- a/dagpipe/lib/index.html +++ b/dagpipe/lib/index.html @@ -393,9 +393,9 @@
  • - + - Returns + __new__ @@ -408,15 +408,6 @@ -
    • - -
  • - - - Returns - - -
  • @@ -426,15 +417,6 @@ -
    • - -
  • - - - Returns - - -
  • @@ -471,6 +453,15 @@ +
    • + +
  • + + + resolve + + +
  • @@ -619,15 +610,6 @@