aetos/docs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b3b1c3404bb2f6ff013f7df2dc2f5a93b38471a7
docs/dagpipe/wiki/02_components/index.html
Vishesh 'ironeagle' Bangotra 7aadae1736 feat: collect dagpipe wiki, refresh lib and mcp artifacts 
Enables dagpipe wiki on the docs hub alongside lib and mcp, removes stale nested lib pages, and picks up regenerated GSDFC docstrings in the MCP bundle.
2026-09-16 15:43:14 +05:30

1388 lines
49 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="prev" href="../01_overview/">
<link rel="next" href="../03_use_cases/01_basic_linear_pipeline/">
<link rel="icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.23">
<title>Core Components - DAG Pipe Documentation</title>
<link rel="stylesheet" href="../assets/stylesheets/main.84d31ad4.min.css">
<link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CJetBrains+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"JetBrains Mono"}</style>
<link rel="stylesheet" href="https://unpkg.com/dracula-prism/dist/css/dracula-prism.css">
<script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="blue-grey" data-md-color-accent="teal">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#core-components" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href=".." title="DAG Pipe Documentation" class="md-header__button md-logo" aria-label="DAG Pipe Documentation" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 3C7.58 3 4 4.79 4 7s3.58 4 8 4 8-1.79 8-4-3.58-4-8-4M4 9v3c0 2.21 3.58 4 8 4s8-1.79 8-4V9c0 2.21-3.58 4-8 4s-8-1.79-8-4m0 5v3c0 2.21 3.58 4 8 4s8-1.79 8-4v-3c0 2.21-3.58 4-8 4s-8-1.79-8-4"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
DAG Pipe Documentation
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Core Components
</span>
</div>
</div>
</div>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7s-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91s2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href=".." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../01_overview/" class="md-tabs__link">
Overview
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="./" class="md-tabs__link">
Core Components
</a>
</li>
<li class="md-tabs__item">
<a href="../03_use_cases/01_basic_linear_pipeline/" class="md-tabs__link">
Use Cases
</a>
</li>
<li class="md-tabs__item">
<a href="../04_best_practices/" class="md-tabs__link">
Best Practices
</a>
</li>
<li class="md-tabs__item">
<a href="../06_error_handling/" class="md-tabs__link">
Error Handling
</a>
</li>
<li class="md-tabs__item">
<a href="../07_testing_example/" class="md-tabs__link">
Testing
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted md-nav--integrated" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href=".." title="DAG Pipe Documentation" class="md-nav__button md-logo" aria-label="DAG Pipe Documentation" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 3C7.58 3 4 4.79 4 7s3.58 4 8 4 8-1.79 8-4-3.58-4-8-4M4 9v3c0 2.21 3.58 4 8 4s8-1.79 8-4V9c0 2.21-3.58 4-8 4s-8-1.79-8-4m0 5v3c0 2.21 3.58 4 8 4s8-1.79 8-4v-3c0 2.21-3.58 4-8 4s-8-1.79-8-4"/></svg>
</a>
DAG Pipe Documentation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href=".." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../01_overview/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Core Components
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Core Components
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#execution-core" class="md-nav__link">
<span class="md-ellipsis">
⚡ Execution Core
</span>
</a>
<nav class="md-nav" aria-label="⚡ Execution Core">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#node" class="md-nav__link">
<span class="md-ellipsis">
Node
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#asyncnode" class="md-nav__link">
<span class="md-ellipsis">
AsyncNode
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#graph" class="md-nav__link">
<span class="md-ellipsis">
Graph
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#engine" class="md-nav__link">
<span class="md-ellipsis">
Engine
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#state-data" class="md-nav__link">
<span class="md-ellipsis">
🧊 State &amp; Data
</span>
</a>
<nav class="md-nav" aria-label="🧊 State &amp; Data">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#state" class="md-nav__link">
<span class="md-ellipsis">
State
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#payload" class="md-nav__link">
<span class="md-ellipsis">
Payload
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#schema" class="md-nav__link">
<span class="md-ellipsis">
Schema
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#schemaerror" class="md-nav__link">
<span class="md-ellipsis">
SchemaError
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#declarative-pipelines" class="md-nav__link">
<span class="md-ellipsis">
📜 Declarative Pipelines
</span>
</a>
<nav class="md-nav" aria-label="📜 Declarative Pipelines">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#pipeline" class="md-nav__link">
<span class="md-ellipsis">
Pipeline
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#load_pipelinepath" class="md-nav__link">
<span class="md-ellipsis">
load_pipeline(path)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#progress-types" class="md-nav__link">
<span class="md-ellipsis">
🔁 Progress Types
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#read-next" class="md-nav__link">
<span class="md-ellipsis">
📚 Read Next
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4" >
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
<span class="md-ellipsis">
Use Cases
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4">
<span class="md-nav__icon md-icon"></span>
Use Cases
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4_1" >
<label class="md-nav__link" for="__nav_4_1" id="__nav_4_1_label" tabindex="0">
<span class="md-ellipsis">
Getting Started
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_1_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_1">
<span class="md-nav__icon md-icon"></span>
Getting Started
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../03_use_cases/01_basic_linear_pipeline/" class="md-nav__link">
<span class="md-ellipsis">
Basic Linear Pipeline
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../03_use_cases/02_graph_execution/" class="md-nav__link">
<span class="md-ellipsis">
Graph Execution
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../03_use_cases/03_yaml_pipeline/" class="md-nav__link">
<span class="md-ellipsis">
YAML Pipelines
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4_2" >
<label class="md-nav__link" for="__nav_4_2" id="__nav_4_2_label" tabindex="0">
<span class="md-ellipsis">
State & Data
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_2">
<span class="md-nav__icon md-icon"></span>
State & Data
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../03_use_cases/04_schema_and_payload/" class="md-nav__link">
<span class="md-ellipsis">
Schema and Payload
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../03_use_cases/05_state_fork_and_lineage/" class="md-nav__link">
<span class="md-ellipsis">
State Fork and Lineage
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4_3" >
<label class="md-nav__link" for="__nav_4_3" id="__nav_4_3_label" tabindex="0">
<span class="md-ellipsis">
Async & Steps
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4_3">
<span class="md-nav__icon md-icon"></span>
Async & Steps
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../03_use_cases/06_async_execution/" class="md-nav__link">
<span class="md-ellipsis">
Async Execution
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../03_use_cases/07_step_execution/" class="md-nav__link">
<span class="md-ellipsis">
Step-wise Execution
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../04_best_practices/" class="md-nav__link">
<span class="md-ellipsis">
Best Practices
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../06_error_handling/" class="md-nav__link">
<span class="md-ellipsis">
Error Handling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../07_testing_example/" class="md-nav__link">
<span class="md-ellipsis">
Testing
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="core-components">🧩 Core Components</h1>
<p>This page is the validated reference for the public API surface. For
step-by-step recipes see the <a href="../">use cases index</a>. For exact signatures
and docstrings, see the library reference (<code>docs/lib</code>) or the MCP bundle
(<code>docs/mcp</code>).</p>
<hr />
<h2 id="execution-core">⚡ Execution Core</h2>
<h3 id="node"><code>Node</code></h3>
<p>Base class for all execution nodes. It is an abstract base class: subclasses
implement <code>resolve()</code> and declare an <code>id</code> (dotted snake_case).</p>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1">1</a></span>
<span class="normal"><a href="#__codelineno-0-2">2</a></span>
<span class="normal"><a href="#__codelineno-0-3">3</a></span>
<span class="normal"><a href="#__codelineno-0-4">4</a></span>
<span class="normal"><a href="#__codelineno-0-5">5</a></span>
<span class="normal"><a href="#__codelineno-0-6">6</a></span>
<span class="normal"><a href="#__codelineno-0-7">7</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">dagpipe</span><span class="w"> </span><span class="kn">import</span> <span class="n">Node</span><span class="p">,</span> <span class="n">State</span>
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2"></a>
</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3"></a><span class="k">class</span><span class="w"> </span><span class="nc">Increment</span><span class="p">(</span><span class="n">Node</span><span class="p">):</span>
</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4"></a> <span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;math.increment&quot;</span>
</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5"></a>
</span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6"></a> <span class="k">def</span><span class="w"> </span><span class="nf">resolve</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">:</span> <span class="n">State</span><span class="p">):</span>
</span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7"></a> <span class="k">yield</span> <span class="bp">self</span><span class="o">.</span><span class="n">fork</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">payload_update</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="n">state</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;value&quot;</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">})</span>
</span></code></pre></div></td></tr></table></div>
<p>Key facts:</p>
<ul>
<li><strong>Singleton per subclass</strong> — stateless subclasses share one instance. <code>A() is A()</code>
is <code>True</code>. Subclasses that declare a custom <code>__init__</code> get one instance per
construction (use this to inject per-run dependencies).</li>
<li><strong><code>fork()</code></strong> — convenience wrapper around <code>State.fork()</code> that records the node
ID into state history.</li>
<li><strong><code>resolve()</code> must yield <code>State</code></strong> — anything else raises <code>TypeError</code>.</li>
<li><strong><code>node_id_to_name(node_id)</code></strong> — converts <code>entity.resolve.numeric_merchant</code>
into <code>Entity › Resolve › Numeric Merchant</code>.</li>
<li><strong><code>name</code></strong> — auto-derived from the ID unless set explicitly.</li>
</ul>
<h3 id="asyncnode"><code>AsyncNode</code></h3>
<p>Base class for asynchronous nodes. Subclasses implement <code>resolve_async()</code> (an
async generator yielding <code>State</code> objects). See
<a href="../03_use_cases/06_async_execution/">use case 06</a>.</p>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-1-1">1</a></span>
<span class="normal"><a href="#__codelineno-1-2">2</a></span>
<span class="normal"><a href="#__codelineno-1-3">3</a></span>
<span class="normal"><a href="#__codelineno-1-4">4</a></span>
<span class="normal"><a href="#__codelineno-1-5">5</a></span>
<span class="normal"><a href="#__codelineno-1-6">6</a></span>
<span class="normal"><a href="#__codelineno-1-7">7</a></span>
<span class="normal"><a href="#__codelineno-1-8">8</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">dagpipe</span><span class="w"> </span><span class="kn">import</span> <span class="n">AsyncNode</span>
</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2"></a>
</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3"></a><span class="k">class</span><span class="w"> </span><span class="nc">FetchRemote</span><span class="p">(</span><span class="n">AsyncNode</span><span class="p">):</span>
</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4"></a> <span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;web.fetch&quot;</span>
</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5"></a>
</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6"></a> <span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">resolve_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">:</span> <span class="n">State</span><span class="p">):</span>
</span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7"></a> <span class="n">data</span> <span class="o">=</span> <span class="k">await</span> <span class="n">fetch</span><span class="p">(</span><span class="n">state</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">))</span>
</span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8"></a> <span class="k">yield</span> <span class="bp">self</span><span class="o">.</span><span class="n">fork</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">payload_update</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;body&quot;</span><span class="p">:</span> <span class="n">data</span><span class="p">})</span>
</span></code></pre></div></td></tr></table></div>
<blockquote>
<p>A sync engine that hits an <code>AsyncNode</code> treats it as a no-op (yields no
states). Use <code>Engine.run_async</code> when the graph contains <code>AsyncNode</code>s.</p>
</blockquote>
<h3 id="graph"><code>Graph</code></h3>
<p>DAG topology container. Stores connectivity only — it never executes nodes.</p>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-2-1"> 1</a></span>
<span class="normal"><a href="#__codelineno-2-2"> 2</a></span>
<span class="normal"><a href="#__codelineno-2-3"> 3</a></span>
<span class="normal"><a href="#__codelineno-2-4"> 4</a></span>
<span class="normal"><a href="#__codelineno-2-5"> 5</a></span>
<span class="normal"><a href="#__codelineno-2-6"> 6</a></span>
<span class="normal"><a href="#__codelineno-2-7"> 7</a></span>
<span class="normal"><a href="#__codelineno-2-8"> 8</a></span>
<span class="normal"><a href="#__codelineno-2-9"> 9</a></span>
<span class="normal"><a href="#__codelineno-2-10">10</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">dagpipe</span><span class="w"> </span><span class="kn">import</span> <span class="n">Graph</span>
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2"></a>
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3"></a><span class="n">graph</span> <span class="o">=</span> <span class="n">Graph</span><span class="p">()</span>
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4"></a><span class="n">graph</span><span class="o">.</span><span class="n">add_root</span><span class="p">(</span><span class="n">a</span><span class="p">)</span> <span class="c1"># node with no parents</span>
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5"></a><span class="n">graph</span><span class="o">.</span><span class="n">add_edge</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">)</span> <span class="c1"># b is a child of a</span>
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6"></a><span class="n">graph</span><span class="o">.</span><span class="n">add_edge</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">c</span><span class="p">)</span>
</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7"></a><span class="n">graph</span><span class="o">.</span><span class="n">children</span><span class="p">(</span><span class="n">a</span><span class="p">)</span> <span class="c1"># (b, c)</span>
</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8"></a><span class="n">graph</span><span class="o">.</span><span class="n">parents</span><span class="p">(</span><span class="n">b</span><span class="p">)</span> <span class="c1"># (a,)</span>
</span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9"></a><span class="n">graph</span><span class="o">.</span><span class="n">roots</span><span class="p">()</span> <span class="c1"># nodes with no incoming edges</span>
</span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10"></a><span class="n">graph</span><span class="o">.</span><span class="n">nodes</span><span class="p">()</span> <span class="c1"># all registered nodes</span>
</span></code></pre></div></td></tr></table></div>
<ul>
<li>Cycle detection is automatic: adding an edge that would create a cycle raises
<code>ValueError</code> (self-edges too).</li>
<li>Nodes are registered implicitly by <code>add_edge</code> / <code>add_root</code>.</li>
<li>The graph is mutable during construction but treated as immutable at runtime.</li>
</ul>
<h3 id="engine"><code>Engine</code></h3>
<p>Orchestrator that runs a linear <code>Sequence[Node]</code> or a <code>Graph</code>.</p>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-3-1">1</a></span>
<span class="normal"><a href="#__codelineno-3-2">2</a></span>
<span class="normal"><a href="#__codelineno-3-3">3</a></span>
<span class="normal"><a href="#__codelineno-3-4">4</a></span>
<span class="normal"><a href="#__codelineno-3-5">5</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">dagpipe</span><span class="w"> </span><span class="kn">import</span> <span class="n">Engine</span>
</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2"></a>
</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3"></a><span class="n">engine</span> <span class="o">=</span> <span class="n">Engine</span><span class="p">(</span><span class="n">graph</span><span class="p">)</span> <span class="c1"># or Engine([n1, n2, n3])</span>
</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4"></a><span class="n">results</span> <span class="o">=</span> <span class="n">engine</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">root_state</span><span class="p">)</span> <span class="c1"># list[State] of terminal states</span>
</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5"></a><span class="n">results_async</span> <span class="o">=</span> <span class="k">await</span> <span class="n">engine</span><span class="o">.</span><span class="n">run_async</span><span class="p">(</span><span class="n">root_state</span><span class="p">)</span>
</span></code></pre></div></td></tr></table></div>
<ul>
<li><strong>Linear mode</strong> — each node feeds every downstream state it receives, in order.</li>
<li><strong>Graph mode</strong> — BFS traversal from all roots; states fan out along edges and
are collected as terminal states at nodes with no children.</li>
<li>Also exposes <code>run_steps</code> / <code>run_steps_async</code> and a <code>nodes</code> property.</li>
<li>Never mutates <code>State</code>, <code>Node</code>, or <code>Graph</code> instances.</li>
</ul>
<hr />
<h2 id="state-data">🧊 State &amp; Data</h2>
<h3 id="state"><code>State</code></h3>
<p>Immutable execution snapshot at one point in traversal. Subclass and bind a
<code>schema</code>:</p>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-4-1">1</a></span>
<span class="normal"><a href="#__codelineno-4-2">2</a></span>
<span class="normal"><a href="#__codelineno-4-3">3</a></span>
<span class="normal"><a href="#__codelineno-4-4">4</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">dagpipe</span><span class="w"> </span><span class="kn">import</span> <span class="n">State</span><span class="p">,</span> <span class="n">Schema</span>
</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2"></a>
</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3"></a><span class="k">class</span><span class="w"> </span><span class="nc">MyState</span><span class="p">(</span><span class="n">State</span><span class="p">):</span>
</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4"></a> <span class="n">schema</span> <span class="o">=</span> <span class="n">Schema</span><span class="p">({</span><span class="s2">&quot;value&quot;</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="s2">&quot;label&quot;</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">})</span>
</span></code></pre></div></td></tr></table></div>
<ul>
<li>Validates its <code>payload</code> against <code>schema</code> at construction (<code>SchemaError</code> on
violation).</li>
<li><strong><code>fork()</code></strong> is the only supported mechanism for producing a new state. Use
<code>State.fork</code> directly, or the <code>Node.fork</code> convenience wrapper.</li>
<li>Tracks <code>confidence</code>, <code>parent</code>, <code>depth</code>, and <code>history</code> (ordered node-ID
lineage) for observability.</li>
<li><strong><code>lineage()</code></strong> — root-to-this ordered tuple.</li>
<li><code>get(key)</code> / <code>has(key)</code> read dot-paths from the underlying <code>Payload</code>.</li>
</ul>
<h3 id="payload"><code>Payload</code></h3>
<p>Immutable hierarchical container with dot-path access.</p>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-5-1">1</a></span>
<span class="normal"><a href="#__codelineno-5-2">2</a></span>
<span class="normal"><a href="#__codelineno-5-3">3</a></span>
<span class="normal"><a href="#__codelineno-5-4">4</a></span>
<span class="normal"><a href="#__codelineno-5-5">5</a></span>
<span class="normal"><a href="#__codelineno-5-6">6</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1"></a><span class="n">payload</span> <span class="o">=</span> <span class="n">Payload</span><span class="p">({</span><span class="s2">&quot;user&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;address&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;city&quot;</span><span class="p">:</span> <span class="s2">&quot;Mumbai&quot;</span><span class="p">}}})</span>
</span><span id="__span-5-2"><a id="__codelineno-5-2" name="__codelineno-5-2"></a><span class="n">payload</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;user.address.city&quot;</span><span class="p">)</span> <span class="c1"># &#39;Mumbai&#39;</span>
</span><span id="__span-5-3"><a id="__codelineno-5-3" name="__codelineno-5-3"></a><span class="n">payload</span><span class="o">.</span><span class="n">has</span><span class="p">(</span><span class="s2">&quot;user.address.zip&quot;</span><span class="p">)</span> <span class="c1"># False</span>
</span><span id="__span-5-4"><a id="__codelineno-5-4" name="__codelineno-5-4"></a><span class="n">payload</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;user.address.zip&quot;</span><span class="p">:</span> <span class="mi">400001</span><span class="p">})</span> <span class="c1"># new Payload, original untouched</span>
</span><span id="__span-5-5"><a id="__codelineno-5-5" name="__codelineno-5-5"></a><span class="n">payload</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span> <span class="c1"># (&#39;user&#39;,)</span>
</span><span id="__span-5-6"><a id="__codelineno-5-6" name="__codelineno-5-6"></a><span class="n">payload</span><span class="o">.</span><span class="n">as_dict</span><span class="p">()</span> <span class="c1"># read-only view of underlying mapping</span>
</span></code></pre></div></td></tr></table></div>
<ul>
<li>Updates are atomic and cheap — only modified branches are copied.</li>
</ul>
<h3 id="schema"><code>Schema</code></h3>
<p>Immutable hierarchical validation tree. Leaf nodes are types or PEP-604 unions;
nested <code>Schema</code> instances describe nested structure.</p>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-6-1">1</a></span>
<span class="normal"><a href="#__codelineno-6-2">2</a></span>
<span class="normal"><a href="#__codelineno-6-3">3</a></span>
<span class="normal"><a href="#__codelineno-6-4">4</a></span>
<span class="normal"><a href="#__codelineno-6-5">5</a></span>
<span class="normal"><a href="#__codelineno-6-6">6</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1"></a><span class="n">AddressSchema</span> <span class="o">=</span> <span class="n">Schema</span><span class="p">({</span><span class="s2">&quot;city&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="s2">&quot;zip&quot;</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span><span class="p">})</span>
</span><span id="__span-6-2"><a id="__codelineno-6-2" name="__codelineno-6-2"></a><span class="n">UserSchema</span> <span class="o">=</span> <span class="n">Schema</span><span class="p">({</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="s2">&quot;address&quot;</span><span class="p">:</span> <span class="n">AddressSchema</span><span class="p">})</span>
</span><span id="__span-6-3"><a id="__codelineno-6-3" name="__codelineno-6-3"></a>
</span><span id="__span-6-4"><a id="__codelineno-6-4" name="__codelineno-6-4"></a><span class="n">user_payload</span> <span class="o">=</span> <span class="n">Payload</span><span class="p">({</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="s2">&quot;Ada&quot;</span><span class="p">,</span> <span class="s2">&quot;address&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;city&quot;</span><span class="p">:</span> <span class="s2">&quot;London&quot;</span><span class="p">}})</span>
</span><span id="__span-6-5"><a id="__codelineno-6-5" name="__codelineno-6-5"></a><span class="n">UserSchema</span><span class="o">.</span><span class="n">validate_payload</span><span class="p">(</span><span class="n">user_payload</span><span class="p">)</span> <span class="c1"># no-op on success</span>
</span><span id="__span-6-6"><a id="__codelineno-6-6" name="__codelineno-6-6"></a><span class="n">UserSchema</span><span class="o">.</span><span class="n">validate_update</span><span class="p">({</span><span class="s2">&quot;address.city&quot;</span><span class="p">:</span> <span class="s2">&quot;Paris&quot;</span><span class="p">})</span>
</span></code></pre></div></td></tr></table></div>
<ul>
<li><code>validate_payload</code> — full structure check.</li>
<li><code>validate_update</code> — path existence check for <code>fork()</code> updates.</li>
</ul>
<h3 id="schemaerror"><code>SchemaError</code></h3>
<p>Raised when payload data violates the declared schema: invalid structure,
undefined path, or invalid type. See <a href="../06_error_handling/">Error Handling</a>.</p>
<hr />
<h2 id="declarative-pipelines">📜 Declarative Pipelines</h2>
<h3 id="pipeline"><code>Pipeline</code></h3>
<p>Dataclass wrapping <code>engine</code>, <code>state_cls</code>, and <code>initial_payload</code>. Executes with
<code>run(payload_override=None)</code> and returns terminal states.</p>
<h3 id="load_pipelinepath"><code>load_pipeline(path)</code></h3>
<p>Factory that builds a <code>Pipeline</code> from one YAML file:</p>
<div class="language-yaml highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-7-1"> 1</a></span>
<span class="normal"><a href="#__codelineno-7-2"> 2</a></span>
<span class="normal"><a href="#__codelineno-7-3"> 3</a></span>
<span class="normal"><a href="#__codelineno-7-4"> 4</a></span>
<span class="normal"><a href="#__codelineno-7-5"> 5</a></span>
<span class="normal"><a href="#__codelineno-7-6"> 6</a></span>
<span class="normal"><a href="#__codelineno-7-7"> 7</a></span>
<span class="normal"><a href="#__codelineno-7-8"> 8</a></span>
<span class="normal"><a href="#__codelineno-7-9"> 9</a></span>
<span class="normal"><a href="#__codelineno-7-10">10</a></span>
<span class="normal"><a href="#__codelineno-7-11">11</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1"></a><span class="nt">version</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span>
</span><span id="__span-7-2"><a id="__codelineno-7-2" name="__codelineno-7-2"></a><span class="nt">schema</span><span class="p">:</span>
</span><span id="__span-7-3"><a id="__codelineno-7-3" name="__codelineno-7-3"></a><span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">int</span>
</span><span id="__span-7-4"><a id="__codelineno-7-4" name="__codelineno-7-4"></a><span class="nt">initial</span><span class="p">:</span>
</span><span id="__span-7-5"><a id="__codelineno-7-5" name="__codelineno-7-5"></a><span class="w"> </span><span class="nt">value</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1</span>
</span><span id="__span-7-6"><a id="__codelineno-7-6" name="__codelineno-7-6"></a><span class="nt">nodes</span><span class="p">:</span>
</span><span id="__span-7-7"><a id="__codelineno-7-7" name="__codelineno-7-7"></a><span class="w"> </span><span class="nt">step1</span><span class="p">:</span>
</span><span id="__span-7-8"><a id="__codelineno-7-8" name="__codelineno-7-8"></a><span class="w"> </span><span class="nt">class</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">mymod.Increment</span>
</span><span id="__span-7-9"><a id="__codelineno-7-9" name="__codelineno-7-9"></a><span class="nt">graph</span><span class="p">:</span>
</span><span id="__span-7-10"><a id="__codelineno-7-10" name="__codelineno-7-10"></a><span class="w"> </span><span class="nt">roots</span><span class="p">:</span>
</span><span id="__span-7-11"><a id="__codelineno-7-11" name="__codelineno-7-11"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">step1</span>
</span></code></pre></div></td></tr></table></div>
<p>See <a href="../03_use_cases/03_yaml_pipeline/">use case 03</a> for the full walkthrough.</p>
<hr />
<h2 id="progress-types">🔁 Progress Types</h2>
<ul>
<li><strong><code>StepResult</code></strong> — <code>(index, node_id, states, completed)</code> for one executed step.</li>
<li><strong><code>ProgressMessage</code></strong> — keyword-only, optional fields (<code>lines</code>, <code>blocks</code>,
<code>count</code>, <code>unit</code>, <code>raw_ocr_line</code>, <code>error</code>, <code>step</code>, <code>status</code>) plus <code>as_dict()</code>.
Passed to step hooks for progress reporting.</li>
</ul>
<p>Both are produced by <code>Engine.run_steps</code> / <code>run_steps_async</code> — see
<a href="../03_use_cases/07_step_execution/">use case 07</a>.</p>
<hr />
<h2 id="read-next">📚 Read Next</h2>
<ul>
<li><a href="../03_use_cases/04_schema_and_payload/">Use case 04 – Schema and payload</a></li>
<li><a href="../03_use_cases/05_state_fork_and_lineage/">Use case 05 – State fork and lineage</a></li>
<li><a href="../04_best_practices/">Best Practices</a></li>
</ul>
</article>
</div>
<script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var labels=set.querySelector(".tabbed-labels");for(var tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": "..", "features": ["navigation.sections", "navigation.expand", "navigation.top", "navigation.instant", "navigation.tracking", "navigation.indexes", "content.code.copy", "content.code.annotate", "content.tabs.link", "content.action.edit", "search.highlight", "search.share", "search.suggest", "navigation.tabs", "toc.integrate", "header.autohide", "announce.dismiss", "footer.social", "content.code.select", "content.code.line_numbers", "content.tooltips"], "search": "../assets/javascripts/workers/search.973d3a69.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../assets/javascripts/bundle.f55a23d4.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink