diff --git a/Dockerfile b/Dockerfile index ad175c7..6fc508a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,13 +40,20 @@ COPY py-jwt /usr/share/nginx/html/py-jwt COPY hexa /usr/share/nginx/html/hexa COPY mongo-ops /usr/share/nginx/html/mongo-ops COPY auth-server /usr/share/nginx/html/auth-server -COPY blog /usr/share/nginx/html/blog COPY media-manager /usr/share/nginx/html/media-manager COPY khata-app /usr/share/nginx/html/khata-app COPY standards /usr/share/nginx/html/standards -# --- MCP bundles ----------------------------------------------------------- -COPY mcp /app/mcp +# --- MCP bundles (one per repo under /app//mcp) ---------------------- +COPY openapi-first/mcp /app/openapi-first/mcp +COPY doc-forge/mcp /app/doc-forge/mcp +COPY dagpipe/mcp /app/dagpipe/mcp +COPY mail-intake/mcp /app/mail-intake/mcp +COPY omniread/mcp /app/omniread/mcp +COPY py-jwt/mcp /app/py-jwt/mcp +COPY hexa/mcp /app/hexa/mcp +COPY mongo-ops/mcp /app/mongo-ops/mcp +COPY standards/mcp /app/standards/mcp EXPOSE 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 9000 diff --git a/build.py b/build.py index 6df817f..ae9a097 100644 --- a/build.py +++ b/build.py @@ -7,7 +7,7 @@ hub. Any top-level folder with ``src/mkdocs.*.yml`` gets, for each config: 1. a MkDocs HTML build into the target dir the config declares (e.g. ``standards/src/mkdocs.wiki.yml`` -> ``standards/wiki/``), and -2. a doc-forge-shaped MCP bundle under ``mcp//`` +2. a doc-forge-shaped MCP bundle under ``/mcp/`` (``index.json``, ``nav.json``, ``modules/*.json``), regenerated from the lib pages (``src/lib/*.md``) so the bundle mirrors the lib site. @@ -100,7 +100,7 @@ def _read_title(path: Path) -> str: def build_bundle(folder: Path, dry_run: bool = False) -> None: - """Regenerate ``mcp/`` from ``folder/src/lib`` markdown. + """Regenerate ``/mcp`` from ``folder/src/lib`` markdown. The bundle mirrors the lib site (in the same way doc-forge's MCP output mirrors its lib documentation). Writes ``index.json``, ``nav.json`` and @@ -115,7 +115,7 @@ def build_bundle(folder: Path, dry_run: bool = False) -> None: lib = folder / "src" / "lib" if not lib.is_dir(): return - out_dir = ROOT / "mcp" / name + out_dir = folder / "mcp" modules_dir = out_dir / "modules" if dry_run: print(f"[MCP] {name} {lib.relative_to(ROOT)}/*.md " diff --git a/config.py b/config.py index b8551b5..2a97fa6 100644 --- a/config.py +++ b/config.py @@ -69,7 +69,7 @@ def load_config(path: Path | str = "config.yml") -> ServiceConfig: mcp_static: dict[str, MCPBundle] = {} for entry in data.get("mcp_static", []): mcp_static[entry["name"]] = MCPBundle( - root=Path("mcp") / entry["bundle"], + root=Path(entry["bundle"]), name=entry.get("server") or entry["name"], port=entry["port"], kind=entry.get("kind", "docs"), diff --git a/config.yml b/config.yml index 9ccffdd..1cbdd67 100644 --- a/config.yml +++ b/config.yml @@ -143,7 +143,7 @@ mcp_static: # Hand-authored bundles that live in this repo (not collected from a source repo). - name: standards title: Standards - bundle: standards + bundle: standards/mcp server: standards port: 8009 kind: registry diff --git a/registry.py b/registry.py index 2104b7c..140b320 100644 --- a/registry.py +++ b/registry.py @@ -44,7 +44,7 @@ class StandardsMCPServer(MCPServer): "path": str(path), "libraries": [], } - return json.loads(path.read_text(encoding="utf-8")) + return json.loads(path.read_bytes()) def _register_standards(self) -> None: @self.app.resource("registry://libraries") diff --git a/standards/lib/13_library_registry/index.html b/standards/lib/13_library_registry/index.html index 261311b..7a3ab34 100644 --- a/standards/lib/13_library_registry/index.html +++ b/standards/lib/13_library_registry/index.html @@ -639,7 +639,7 @@ every application. Use them. Writing a private copy of any capa is a defect.

Each library also exposes its own documentation as an MCP server. An AI agent can read a library's full API by connecting to its MCP and reading -docs://index, docs://nav, and docs://modules/{module}.

+doc://index, doc://nav, and doc://modules/{module}.

Catalog

diff --git a/standards/lib/search/search_index.json b/standards/lib/search/search_index.json index e603bce..3e37c33 100644 --- a/standards/lib/search/search_index.json +++ b/standards/lib/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Standards Reference","text":"

The reference half of the Standards description: the enforceable rules, patterns, and the internal library catalog that agents consult while coding.

  • Layering Rules \u2014 the strict request \u2192 domain \u2192 repository layering and the rules that enforce it.
  • Composition Root \u2014 a single composition point wiring the whole app.
  • Spec-First Routing \u2014 the OpenAPI contract as the routing source of truth.
  • CRUD Template \u2014 the standardized generic resource endpoint scaffold.
  • Pipeline Scaffold \u2014 the deterministic pipeline skeleton for multi-stage jobs.
  • Worker & Progress \u2014 queued work, progress reporting, and visibility.
  • Human-in-the-Loop \u2014 guardrails and approvals inside automations.
  • Infra Adapters \u2014 how infrastructure connects without leaking into domain.
  • Genericity Matrix \u2014 what is generic vs app-specific.
  • Library Registry \u2014 the catalog of internal libraries behind all of it.

Read the Wiki for the philosophy and rationale; these pages are the reference that mirrors the standards MCP bundle.

"},{"location":"02_layering_rules/","title":"Layering Rules","text":""},{"location":"02_layering_rules/#summary","title":"Summary","text":"

The dependency direction is fixed. Violating it is a design defect.

"},{"location":"02_layering_rules/#allowed-imports","title":"Allowed imports","text":"
interfaces/*      ->  domain/*            (services, ports, models)\ninterfaces/*      ->  infra/adapters/*    (only read-only value objects)\ndomain/*          ->  domain/*            (same layer)\ndomain/*          ->  pipelines/*         (call a pipeline via a port)\ninfra/adapters/*  ->  domain/ports/*      (implement the interface)\nmain.py/init.py   ->  everything          (composition root only)\n
"},{"location":"02_layering_rules/#forbidden-imports","title":"Forbidden imports","text":"
domain/*          ->  interfaces/*        (never)\ndomain/*          ->  fastapi/starlette   (never)\ninfra/adapters/*  ->  interfaces/*        (never)\ninterfaces/*      ->  infra/db/*          (state lives in resources, not imports)\n
"},{"location":"02_layering_rules/#why","title":"Why","text":"
  • A pure domain is testable without a server, a database, or an event loop.
  • Adapters can be swapped (in-memory fake, real database) without touching business logic.
  • The HTTP edge stays thin and replaceable.
"},{"location":"02_layering_rules/#the-portadapter-contract","title":"The port/adapter contract","text":"

A port is an abstract class or protocol declared in domain/<x>/ports/. An adapter implements it in infra/adapters/. The composition root binds the two.

domain/<resource>/ports/db.py       -> repository port + document model\ndomain/<resource>/ports/service.py  -> service + factory\ninfra/db/repos/<vendor>.py          -> repository adapter factory\ninfra/adapters/service.py           -> generic CRUD service base\n
"},{"location":"02_layering_rules/#enforcement","title":"Enforcement","text":"
  • Keep every domain module importable with only the standard library and Pydantic. If a domain import pulls in fastapi, the layering is broken.
  • Reviewers (and CI) reject import fastapi inside domain/ or infra/.
"},{"location":"03_composition_root/","title":"Composition Root","text":""},{"location":"03_composition_root/#summary","title":"Summary","text":"

All wiring happens in exactly one module. It builds a flat resource registry and exposes it to the HTTP layer and to scripts.

"},{"location":"03_composition_root/#the-resource-registry","title":"The resource registry","text":"

init.py returns a plain dict[str, Any]. Keys follow two conventions:

  • <resource>_repo \u2014 a repository adapter for a resource.
  • <resource>_svc \u2014 a service for a resource.

Plus named infrastructure adaptors:

  • http_cache_store \u2014 response cache backend.
  • progress_bus \u2014 progress/event bus.
  • <vendor>_<kind> \u2014 fetchers, clients, stores.

The API stores the dict on app.state.resources during lifespan startup. Handlers read from it; nothing imports concrete adapters.

"},{"location":"03_composition_root/#shape","title":"Shape","text":"
_resources: dict = {}\n_initialized = False\n\nasync def init(settings=None) -> dict:\n    global _initialized\n    if _initialized:\n        return _resources\n\n    await setup_database(uri=settings.mongo_uri, db_name=settings.mongo_db)\n\n    # 1. leaf repositories\n    user_repo = create_user_repo()\n\n    # 2. dependent repositories (dependencies passed by keyword)\n    wallet_repo = create_wallet_repo(user_repo=user_repo)\n\n    # 3. services (build via the DI library, injecting the repo)\n    _resources[\"user_repo\"]   = user_repo\n    _resources[\"wallet_repo\"] = wallet_repo\n    _resources[\"user_svc\"]    = get_user_service(user_repo)\n    _resources[\"wallet_svc\"]  = get_wallet_service(wallet_repo)\n\n    # 4. infrastructure adaptors (optional)\n    _resources[\"http_cache_store\"] = HttpCacheStore(...)\n    _resources[\"progress_bus\"]     = ProgressEventBus()\n\n    _initialized = True\n    return _resources\n\nasync def shutdown() -> None:\n    global _initialized\n    await close_stores(_resources)\n    await shutdown_database()\n    _resources.clear()\n    _initialized = False\n
"},{"location":"03_composition_root/#typed-accessors","title":"Typed accessors","text":"

Expose a get_svc(name) helper with one @overload per key so scripts get typing without importing concrete classes:

@overload\ndef get_svc(name: Literal[\"user_svc\"]) -> UserService: ...\n@overload\ndef get_svc(name: Literal[\"wallet_svc\"]) -> WalletService: ...\n\ndef get_svc(name: str):\n    return _resources[name]\n

Also expose get_db() (raw database handle) and pure helpers such as compute_fingerprint(payload) (canonical hash used for deduplication).

"},{"location":"03_composition_root/#rules","title":"Rules","text":"
  • init() is idempotent.
  • The composition root is the only place that imports concrete adapters.
  • Handlers never call init(); they read request.app.state.resources.
  • Scripts call init() / shutdown() around their work.
"},{"location":"04_spec_first_routing/","title":"Spec-First Routing","text":""},{"location":"04_spec_first_routing/#summary","title":"Summary","text":"

The OpenAPI document is the single source of truth for the HTTP contract. Handlers are bound to it by name; there is no decorator router.

"},{"location":"04_spec_first_routing/#the-contract","title":"The contract","text":"
  • openapi.yaml (or .json) declares paths, schemas, parameters, responses, and security.
  • A custom operationId names the handler function.
  • The framework (openapi_first) validates requests/responses against the spec and dispatches to the handler whose name equals operationId.
"},{"location":"04_spec_first_routing/#app-bootstrap","title":"App bootstrap","text":"
# main.py\nfrom openapi_first.app import OpenAPIFirstApp\n\napp = OpenAPIFirstApp(\n    openapi_path=\"openapi.yaml\",\n    routes_module=interfaces.openapi,   # module whose namespace holds handlers\n    title=\"<App> API\",\n    version=\"0.1.0\",\n    lifespan=lifespan,\n)\n

lifespan calls the composition root:

@asynccontextmanager\nasync def lifespan(app):\n    app.state.resources = await init()\n    yield\n    await shutdown()\n
"},{"location":"04_spec_first_routing/#handler-registry","title":"Handler registry","text":"

interfaces/openapi.py is a flat re-export of every handler, so the framework can resolve operationId across the whole app:

from interfaces.item.routes import *\nfrom interfaces.wallet.routes import *\nfrom interfaces.progress.sse import *\n
"},{"location":"04_spec_first_routing/#handler-shape","title":"Handler shape","text":"

Handlers are thin and synchronous with respect to business logic:

async def create_item(request: Request, body: dict):\n    svc = request.app.state.resources[\"item_svc\"]\n    return await svc.create(Item.model_validate(body))\n
"},{"location":"04_spec_first_routing/#conventions","title":"Conventions","text":"
  • operationId naming: list_<resource>, create_<resource>, get_<resource>, update_<resource>, delete_<resource>.
  • Read-only endpoints may use a cache decorator keyed on path + query + user.
  • Validation errors -> 400; not found -> 404; delete -> 204.
  • Security is declared once in the spec and enforced by the framework; handlers contain no auth code.
"},{"location":"04_spec_first_routing/#codegen","title":"Codegen","text":"

The openapi_first library can scaffold models and route stubs from the spec, and ships app templates (CRUD app, model app, service). Prefer codegen over hand-writing boilerplate.

"},{"location":"05_crud_template/","title":"CRUD Template","text":""},{"location":"05_crud_template/#summary","title":"Summary","text":"

Every resource follows the same four-file slice. Repetition is intentional: it makes the codebase predictable and generatable.

"},{"location":"05_crud_template/#the-slice","title":"The slice","text":"
domain/<resource>/\n    models.py               # pure Pydantic model (no DB fields)\n    ports/\n        db.py               # document model + repository factory\n        service.py          # service class + factory\n        converters.py       # (optional) transforms\n
"},{"location":"05_crud_template/#modelspy","title":"models.py","text":"

A framework-free Pydantic model describing the business shape.

class Item(BaseModel):\n    name: str\n    quantity: int = 0\n
"},{"location":"05_crud_template/#portsdbpy","title":"ports/db.py","text":"

The persistence model extends the business model and adds database concerns. The repository factory selects the right repository flavour.

class ItemDocument(Item, BaseDocument):   # BaseDocument adds id/created_at/...\n    pass\n\ndef create_item_repo(*, owner_repo=None) -> ItemRepository:\n    return repo_factory(\n        collection=\"items\",\n        document=ItemDocument,\n        populate_rules=[PopulateRule(field=\"owner\", collection=\"users\")],\n        cache=None,               # or a CachedBaseRepository config\n    )\n
"},{"location":"05_crud_template/#portsservicepy","title":"ports/service.py","text":"

The service is built through the DI library, injecting the repository. The generic CRUD base provides create/read/update/patch/delete/search/....

class ItemService(CrudsServiceAdaptor[Item, ItemDocument]):\n    model = Item\n    schema = ItemDocument\n\ndef get_item_service(repo) -> ItemService:\n    return build(ItemService, instances={\"repo\": repo})\n
"},{"location":"05_crud_template/#naming-conventions","title":"Naming conventions","text":"Concept Convention collection plural snake case (items) document <Resource>Document repository factory create_<resource>_repo service <Resource>Service service factory get_<resource>_service resource key <resource>_svc operationId list_items, create_item, ..."},{"location":"05_crud_template/#optional-files","title":"Optional files","text":"
  • converters.py \u2014 pure functions mapping between domain types.
  • ports/enrichment.py \u2014 adapts a pipeline into an async callable the service can invoke (used when a resource is populated by a pipeline).
"},{"location":"06_pipeline_scaffold/","title":"Pipeline Scaffold","text":""},{"location":"06_pipeline_scaffold/#summary","title":"Summary","text":"

Data-driven processing is expressed as a composition graph. The graph is built by a DI library; execution is driven by a DAG engine. Stages are pure and individually testable.

"},{"location":"06_pipeline_scaffold/#when-to-use-a-pipeline","title":"When to use a pipeline","text":"

Use a pipeline when a resource is produced by a multi-step transformation of an input (a document, a file, a message) rather than by a simple CRUD write.

"},{"location":"06_pipeline_scaffold/#structure-of-a-family","title":"Structure of a family","text":"

A family is one input shape (e.g. a file format or a message kind). Each family has the same skeleton:

pipelines/<family>/\n    abc.py            # port contracts (stage interfaces + root interface)\n    impl.py           # concrete stages + root implementation\n    pipeline.yaml     # graph spec: nodes, edges, roots, schema\n    resources/\n        pipeline.yaml # runtime spec consumed by the loader\n        patterns/*.yaml  # rule fragments (pattern registry)\n    banks/<variant>.py   # optional variants that re-pin a few slots\n
"},{"location":"06_pipeline_scaffold/#shared-scaffolding","title":"Shared scaffolding","text":"
pipelines/shared/\n    ports.py     # RegistryPort, FinalizePort, etc.\n    loader.py    # build(root_cls) -> wires the graph + engine\n    registry.py  # resolve_root(...) -> root class for a (kind, variant)\n
"},{"location":"06_pipeline_scaffold/#the-loader","title":"The loader","text":"
def build(root_cls, *, instances=None, config=None):\n    # merge resources/patterns/*.yaml into a pattern registry\n    # bind registry/config into each graph node by id\n    # build the schema (state) class and the engine\n    root = root_cls()\n    root._engine = Engine(graph, state_cls, payload_cls)\n    return root\n
"},{"location":"06_pipeline_scaffold/#the-registry","title":"The registry","text":"

A single lookup replaces hand-rolled if/else selection:

_FAMILIES = {\n    \"csv\":  {\"acme\": {\"heuristic\": AcmeCsvPipeline}},\n    \"json\": {\"globex\": {\"heuristic\": GlobexJsonPipeline}},\n}\n\ndef resolve_root(kind, variant, mode=\"heuristic\"):\n    return _FAMILIES[kind][variant][mode]\n

Adding a new variant means adding one class and one registry entry.

"},{"location":"06_pipeline_scaffold/#driving-a-pipeline","title":"Driving a pipeline","text":"

Stages are driven explicitly so intermediate state can be persisted:

pipeline = build(resolve_root(kind, variant))\ncontent = pipeline.ingest.materialize(raw_bytes)\nrecords = pipeline.reader.read(content)\nitems   = pipeline.normalize.build(records)\n
"},{"location":"06_pipeline_scaffold/#testing","title":"Testing","text":"

Because stages are pure, test each stage in isolation and test the assembled root against fixture inputs whose expected outputs are stored as golden files. A parity test replays fixtures headlessly.

"},{"location":"07_worker_and_progress/","title":"Worker & Progress","text":""},{"location":"07_worker_and_progress/#summary","title":"Summary","text":"

Long-running work runs in a background worker. Jobs are durable documents that move through an explicit status machine. Progress is streamed to clients.

"},{"location":"07_worker_and_progress/#the-status-machine","title":"The status machine","text":"
pending -> processing -> completed\n                     \\-> failed    (retried up to N times)\n                     \\-> paused    (waiting for human input)\n
  • pending \u2014 queued, eligible for pickup.
  • processing \u2014 claimed by a worker.
  • paused \u2014 blocked on a human-in-the-loop question.
  • failed \u2014 error; retry_count increments until a max, then skipped.
  • completed \u2014 terminal success.
"},{"location":"07_worker_and_progress/#the-poll-loop","title":"The poll loop","text":"
async def poll(resources):\n    backoff = MIN_POLL\n    while True:\n        jobs = await job_svc.search(filter={\"status\": \"pending\"}, limit=100)\n        backoff = MIN_POLL if jobs else min(backoff * 2, MAX_POLL)\n        for job in jobs:\n            if (job.retry_count or 0) >= MAX_RETRIES:\n                continue\n            await job_svc.update_status(job.id, \"processing\")\n            try:\n                await process(job, resources)\n            except PauseForResolution:\n                await job_svc.update_status(job.id, \"paused\")\n            except Exception as e:\n                await job_svc.update_status(\n                    job.id, \"failed\", error=str(e),\n                    retry_count=(job.retry_count or 0) + 1,\n                )\n        await asyncio.sleep(backoff)\n

Tunables live in environment variables: WORKER_POLL_MIN, WORKER_POLL_MAX, WORKER_MAX_RETRIES.

"},{"location":"07_worker_and_progress/#progress-events","title":"Progress events","text":"

A worker publishes structured progress events to a bus (a port). The HTTP edge exposes a streaming endpoint that replays events for a given job id.

class ProgressEvent(BaseModel):\n    job_id: str\n    step: str\n    status: str            # started | progress | completed | skipped | paused\n    message: dict | None = None\n

The bus is an in-memory adapter by default; because it is a port, it can be swapped for a durable broker. Emit from the orchestrator at each stage:

await self.bus.publish(ProgressEvent(job_id=jid, step=\"normalize\",\n                                     status=\"completed\", message={\"count\": n}))\n
"},{"location":"07_worker_and_progress/#streaming-endpoint","title":"Streaming endpoint","text":"
async def stream_job_events(request, id):\n    bus = request.app.state.resources.get(\"progress_bus\")\n    if bus is None:\n        return {\"events\": []}\n\n    async def gen():\n        sent = 0\n        while True:\n            events = bus.get_events(id)\n            for e in events[sent:]:\n                yield f\"data: {e.model_dump_json(exclude_none=True)}\\n\\n\"\n                sent += 1\n            await asyncio.sleep(0.5)\n\n    return StreamingResponse(gen(), media_type=\"text/event-stream\")\n

Known limitation. The default in-memory bus is per-process. If the worker and the API run as separate processes, events published by the worker are not visible to the API. For production, back the bus with a shared broker. The port boundary makes this a one-adapter change.

"},{"location":"08_human_in_loop/","title":"Human-in-the-Loop","text":""},{"location":"08_human_in_loop/#summary","title":"Summary","text":"

Some inputs are ambiguous and must be resolved by a human before processing can continue. The blueprint models this as a durable pause: an exception is raised, a question is recorded, the job pauses, and later resumes.

"},{"location":"08_human_in_loop/#the-mechanism","title":"The mechanism","text":"
  1. During processing, a stage raises a domain exception carrying the context needed to ask the question:
raise PauseForResolution(\n    job_id=jid,\n    reference=\"<stable key for this decision>\",\n    options=[...],\n    context={...},\n)\n
  1. The orchestrator catches it, records a question document, and re-raises so the worker marks the job paused.
  2. A client lists pending questions and answers one:
GET  /jobs/{id}/questions            -> list pending questions\nPOST /questions/{qid}/resolve  {choice}  -> record the choice\n
  1. When the last pending question is resolved, the resolve endpoint flips the parent job back to pending.
  2. The worker picks it up again, rebuilds a resolution map from previously answered questions, and seeds it into the pipeline so the same decision is not asked twice.
"},{"location":"08_human_in_loop/#the-question-document","title":"The question document","text":"
class Question(BaseDocument):\n    job_id: str\n    reference: str            # stable key: stage|field|value\n    status: str               # pending | resolved\n    options: list[dict]\n    context: dict\n    chosen: dict | None = None\n    resolved_at: str | None = None\n
"},{"location":"08_human_in_loop/#resume-map","title":"Resume map","text":"

The worker turns answered questions into a lookup keyed by reference, then injects it into the pipeline so that stage can skip re-raising:

resolved = {q.reference: q.chosen for q in await svc.search(\n    filter={\"job_id\": jid, \"status\": \"resolved\"})}\n\npipeline = build(root, instances={\"pause\": handler}, config={\"resolved\": resolved})\n
"},{"location":"08_human_in_loop/#resolve-endpoint","title":"Resolve endpoint","text":"
async def resolve_question(request, id, body):\n    svc = request.app.state.resources[\"question_svc\"]\n    q = await svc.read(id)\n    await svc.update(id, {\"chosen\": body[\"chosen\"], \"status\": \"resolved\",\n                          \"resolved_at\": now()})\n\n    pending = await svc.search(filter={\"job_id\": q.job_id, \"status\": \"pending\"},\n                               limit=1)\n    if not pending:\n        jobs = request.app.state.resources[\"job_svc\"]\n        job = await jobs.read(q.job_id)\n        if job and job.status == \"paused\":\n            await jobs.update_status(q.job_id, \"pending\")\n    return q\n
"},{"location":"08_human_in_loop/#cli-parity","title":"CLI parity","text":"

The same flow is available through scripts/CLI: list, get, create, questions, resolve, retry.

"},{"location":"09_infra_adaptors/","title":"Infra Adapters","text":""},{"location":"09_infra_adaptors/#summary","title":"Summary","text":"

All external systems are reached through adapters in infra/. Prefer an existing internal library over writing a new adapter.

"},{"location":"09_infra_adaptors/#database-mongodb","title":"Database (MongoDB)","text":"
  • A single connection manager owns the client lifecycle. setup_database(uri, db_name) / shutdown_database().
  • A repository factory selects the repository flavour:
repo_factory(collection, document, engine=None,\n             populate_rules=None, cache=None)\n
  • base repository \u2014 plain CRUD;
  • populating repository \u2014 resolves referenced documents (foreign keys);

  • cached repository \u2014 an LRU/TTL cache in front of reads.

  • Referenced documents are described declaratively:

PopulateRule(field=\"owner\", collection=\"users\",\n             nested_rules=[], max_depth=2, projection=None)\n
  • Transactions use a session context manager provided by the library.

Library: the Mongo operations library (see registry). Do not write raw Motor calls in the domain.

"},{"location":"09_infra_adaptors/#http-response-cache","title":"HTTP response cache","text":"

A generic GET cache sits behind a decorator.

@cache_response(ttl=60)\nasync def list_items(request): ...\n
  • Keyed on path + query + user.
  • No-op when no store is wired.
  • invalidate(*resources) bumps a generation counter for write-triggered invalidation.
"},{"location":"09_infra_adaptors/#event-progress-bus","title":"Event / progress bus","text":"

A port with an in-memory default adapter (publish, get_events). Swap for a durable broker in production (see Worker & Progress).

"},{"location":"09_infra_adaptors/#external-clients-ingestion","title":"External clients (ingestion)","text":"

External fetchers (files, mail, HTTP) are adapters that wrap ingestion libraries. Construct them in the composition root and inject into orchestrators.

Libraries: the ingestion library (formats) and the mail library (email).

"},{"location":"09_infra_adaptors/#llm-clients","title":"LLM clients","text":"

When a pipeline calls a model provider, route through a pool adaptor that handles provider selection, retries, and client lifecycle:

pool = setup_async_pool(provider=settings.llm_provider)\nresult = await pool.complete(model=\"...\", messages=[...])\n

Providers are selected by LLM_PROVIDER and configured by environment.

"},{"location":"09_infra_adaptors/#key-value-store","title":"Key-value store","text":"

For report/snapshot storage, provide a pluggable store port with a Redis adapter and an in-memory fallback.

"},{"location":"09_infra_adaptors/#rule","title":"Rule","text":"

If a capability already exists as a library (DB, DI, ingestion, mail, auth, docs), use it. Writing a private copy of a library is a defect.

"},{"location":"12_genericity_matrix/","title":"Genericity Matrix","text":""},{"location":"12_genericity_matrix/#summary","title":"Summary","text":"

This page states precisely what is reusable as-is, what is a template to adapt, and what is app-specific and must be authored. Use it to scope a new build.

"},{"location":"12_genericity_matrix/#reusable-as-is-do-not-rewrite","title":"Reusable as-is (do not rewrite)","text":"Area Artifact Layering & dependency rule the package layout and import rules Composition root the init.py shape, idempotency, get_svc overloading Web edge OpenAPIFirstApp bootstrap, handler-name = operationId, thin handlers CRUD slice the four-file pattern (model / ports-db / ports-service / converters) Generic CRUD service the base service adaptor Repository factory base / populating / cached selection Worker the poll loop, backoff, retry, status machine Progress bus the bus port + in-memory adapter + streaming endpoint Human-in-the-loop the durable pause + question document + resolve endpoint + resume map Infra adaptors HTTP cache decorator, KV store port, LLM pool scaffold Testing integration harness with fakes, fixture-parity pattern Docs GSDFC + generated stubs + MCP bundle pipeline"},{"location":"12_genericity_matrix/#template-to-adapt-copy-then-change-names","title":"Template to adapt (copy, then change names)","text":"Artifact Change pyproject.toml project name, dependencies Dockerfile / docker-compose.yaml service name, ports, services .env.example connection strings, provider keys openapi.yaml every path/schema/operationId scripts/ seed and operator tasks"},{"location":"12_genericity_matrix/#app-specific-must-be-authored","title":"App-specific (must be authored)","text":"Artifact Why Resource models the business shape Repository populate/cache rules the relationships Service business logic domain behaviour Pipeline families (if any) the transformation graph Pattern/rule resources data-driven policy Fixtures & golden outputs test data"},{"location":"12_genericity_matrix/#the-two-axes","title":"The two axes","text":"

Everything app-specific reduces to two axes:

  1. Resource model \u2014 the set of resources, their fields, relationships, and the OpenAPI contract.
  2. Pipeline family \u2014 only if inputs require multi-step transformation.

If you can specify those two axes, the rest of the app follows mechanically.

"},{"location":"13_library_registry/","title":"Library Registry","text":""},{"location":"13_library_registry/#summary","title":"Summary","text":"

The Aetos ecosystem provides internal libraries for the shared concerns of every application. Use them. Writing a private copy of any capability below is a defect.

Each library also exposes its own documentation as an MCP server. An AI agent can read a library's full API by connecting to its MCP and reading docs://index, docs://nav, and docs://modules/{module}.

"},{"location":"13_library_registry/#catalog","title":"Catalog","text":"Library Import Purpose Use instead of MCP DAG Pipe dagpipe Deterministic state-transformation DAG engine hand-rolled orchestration/topological ordering :8006 Hexa hexa Declarative DI / composition-root tooling manual wiring, hand-written factories (see note) Mongo Ops mongo_ops Async MongoDB repositories, cache, populate, transactions raw Motor CRUD and connection lifecycle :8007 Omniread omniread Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) custom scrapers/parsers :8003 OpenAPI First openapi_first Spec-first FastAPI bootstrap, client, codegen decorator routers and hand-written clients :8004 Mail Intake mail_intake Provider-agnostic read-only email ingestion Gmail OAuth + MIME parsing :8002 Py JWT jwtlib JWT auth primitives (register/login/introspect) password hashing + token code :8005 Doc Forge docforge Docstring compiler \u2192 MkDocs + MCP docs custom doc generators :8001 Auth Server (service) Central identity service (register/login/me/introspect) building auth endpoints yourself \u2014"},{"location":"13_library_registry/#how-to-choose","title":"How to choose","text":"You need to\u2026 Use persist and query documents Mongo Ops compose dependencies Hexa run a multi-step transformation DAG Pipe read/parse an arbitrary input format Omniread expose a contract-first HTTP API OpenAPI First ingest email Mail Intake authenticate users / verify tokens Py JWT (+ Auth Server) generate documentation Doc Forge"},{"location":"13_library_registry/#discovery","title":"Discovery","text":"

A single standards MCP exposes find_library(capability), which returns the matching library, its import name, its \"use instead of\" guidance, and its MCP endpoint. Query it before writing any infrastructure.

"},{"location":"13_library_registry/#notes","title":"Notes","text":"
  • Hexa ships an MCP bundle but is not yet assigned a dedicated port in the hub config; it is reachable via the standards registry entry and will be wired to a port.
  • Library MCP servers expose documentation resources (not executable tools): they tell an agent the available API surface so it does not invent code.
"}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Standards Reference","text":"

The reference half of the Standards description: the enforceable rules, patterns, and the internal library catalog that agents consult while coding.

  • Layering Rules \u2014 the strict request \u2192 domain \u2192 repository layering and the rules that enforce it.
  • Composition Root \u2014 a single composition point wiring the whole app.
  • Spec-First Routing \u2014 the OpenAPI contract as the routing source of truth.
  • CRUD Template \u2014 the standardized generic resource endpoint scaffold.
  • Pipeline Scaffold \u2014 the deterministic pipeline skeleton for multi-stage jobs.
  • Worker & Progress \u2014 queued work, progress reporting, and visibility.
  • Human-in-the-Loop \u2014 guardrails and approvals inside automations.
  • Infra Adapters \u2014 how infrastructure connects without leaking into domain.
  • Genericity Matrix \u2014 what is generic vs app-specific.
  • Library Registry \u2014 the catalog of internal libraries behind all of it.

Read the Wiki for the philosophy and rationale; these pages are the reference that mirrors the standards MCP bundle.

"},{"location":"02_layering_rules/","title":"Layering Rules","text":""},{"location":"02_layering_rules/#summary","title":"Summary","text":"

The dependency direction is fixed. Violating it is a design defect.

"},{"location":"02_layering_rules/#allowed-imports","title":"Allowed imports","text":"
interfaces/*      ->  domain/*            (services, ports, models)\ninterfaces/*      ->  infra/adapters/*    (only read-only value objects)\ndomain/*          ->  domain/*            (same layer)\ndomain/*          ->  pipelines/*         (call a pipeline via a port)\ninfra/adapters/*  ->  domain/ports/*      (implement the interface)\nmain.py/init.py   ->  everything          (composition root only)\n
"},{"location":"02_layering_rules/#forbidden-imports","title":"Forbidden imports","text":"
domain/*          ->  interfaces/*        (never)\ndomain/*          ->  fastapi/starlette   (never)\ninfra/adapters/*  ->  interfaces/*        (never)\ninterfaces/*      ->  infra/db/*          (state lives in resources, not imports)\n
"},{"location":"02_layering_rules/#why","title":"Why","text":"
  • A pure domain is testable without a server, a database, or an event loop.
  • Adapters can be swapped (in-memory fake, real database) without touching business logic.
  • The HTTP edge stays thin and replaceable.
"},{"location":"02_layering_rules/#the-portadapter-contract","title":"The port/adapter contract","text":"

A port is an abstract class or protocol declared in domain/<x>/ports/. An adapter implements it in infra/adapters/. The composition root binds the two.

domain/<resource>/ports/db.py       -> repository port + document model\ndomain/<resource>/ports/service.py  -> service + factory\ninfra/db/repos/<vendor>.py          -> repository adapter factory\ninfra/adapters/service.py           -> generic CRUD service base\n
"},{"location":"02_layering_rules/#enforcement","title":"Enforcement","text":"
  • Keep every domain module importable with only the standard library and Pydantic. If a domain import pulls in fastapi, the layering is broken.
  • Reviewers (and CI) reject import fastapi inside domain/ or infra/.
"},{"location":"03_composition_root/","title":"Composition Root","text":""},{"location":"03_composition_root/#summary","title":"Summary","text":"

All wiring happens in exactly one module. It builds a flat resource registry and exposes it to the HTTP layer and to scripts.

"},{"location":"03_composition_root/#the-resource-registry","title":"The resource registry","text":"

init.py returns a plain dict[str, Any]. Keys follow two conventions:

  • <resource>_repo \u2014 a repository adapter for a resource.
  • <resource>_svc \u2014 a service for a resource.

Plus named infrastructure adaptors:

  • http_cache_store \u2014 response cache backend.
  • progress_bus \u2014 progress/event bus.
  • <vendor>_<kind> \u2014 fetchers, clients, stores.

The API stores the dict on app.state.resources during lifespan startup. Handlers read from it; nothing imports concrete adapters.

"},{"location":"03_composition_root/#shape","title":"Shape","text":"
_resources: dict = {}\n_initialized = False\n\nasync def init(settings=None) -> dict:\n    global _initialized\n    if _initialized:\n        return _resources\n\n    await setup_database(uri=settings.mongo_uri, db_name=settings.mongo_db)\n\n    # 1. leaf repositories\n    user_repo = create_user_repo()\n\n    # 2. dependent repositories (dependencies passed by keyword)\n    wallet_repo = create_wallet_repo(user_repo=user_repo)\n\n    # 3. services (build via the DI library, injecting the repo)\n    _resources[\"user_repo\"]   = user_repo\n    _resources[\"wallet_repo\"] = wallet_repo\n    _resources[\"user_svc\"]    = get_user_service(user_repo)\n    _resources[\"wallet_svc\"]  = get_wallet_service(wallet_repo)\n\n    # 4. infrastructure adaptors (optional)\n    _resources[\"http_cache_store\"] = HttpCacheStore(...)\n    _resources[\"progress_bus\"]     = ProgressEventBus()\n\n    _initialized = True\n    return _resources\n\nasync def shutdown() -> None:\n    global _initialized\n    await close_stores(_resources)\n    await shutdown_database()\n    _resources.clear()\n    _initialized = False\n
"},{"location":"03_composition_root/#typed-accessors","title":"Typed accessors","text":"

Expose a get_svc(name) helper with one @overload per key so scripts get typing without importing concrete classes:

@overload\ndef get_svc(name: Literal[\"user_svc\"]) -> UserService: ...\n@overload\ndef get_svc(name: Literal[\"wallet_svc\"]) -> WalletService: ...\n\ndef get_svc(name: str):\n    return _resources[name]\n

Also expose get_db() (raw database handle) and pure helpers such as compute_fingerprint(payload) (canonical hash used for deduplication).

"},{"location":"03_composition_root/#rules","title":"Rules","text":"
  • init() is idempotent.
  • The composition root is the only place that imports concrete adapters.
  • Handlers never call init(); they read request.app.state.resources.
  • Scripts call init() / shutdown() around their work.
"},{"location":"04_spec_first_routing/","title":"Spec-First Routing","text":""},{"location":"04_spec_first_routing/#summary","title":"Summary","text":"

The OpenAPI document is the single source of truth for the HTTP contract. Handlers are bound to it by name; there is no decorator router.

"},{"location":"04_spec_first_routing/#the-contract","title":"The contract","text":"
  • openapi.yaml (or .json) declares paths, schemas, parameters, responses, and security.
  • A custom operationId names the handler function.
  • The framework (openapi_first) validates requests/responses against the spec and dispatches to the handler whose name equals operationId.
"},{"location":"04_spec_first_routing/#app-bootstrap","title":"App bootstrap","text":"
# main.py\nfrom openapi_first.app import OpenAPIFirstApp\n\napp = OpenAPIFirstApp(\n    openapi_path=\"openapi.yaml\",\n    routes_module=interfaces.openapi,   # module whose namespace holds handlers\n    title=\"<App> API\",\n    version=\"0.1.0\",\n    lifespan=lifespan,\n)\n

lifespan calls the composition root:

@asynccontextmanager\nasync def lifespan(app):\n    app.state.resources = await init()\n    yield\n    await shutdown()\n
"},{"location":"04_spec_first_routing/#handler-registry","title":"Handler registry","text":"

interfaces/openapi.py is a flat re-export of every handler, so the framework can resolve operationId across the whole app:

from interfaces.item.routes import *\nfrom interfaces.wallet.routes import *\nfrom interfaces.progress.sse import *\n
"},{"location":"04_spec_first_routing/#handler-shape","title":"Handler shape","text":"

Handlers are thin and synchronous with respect to business logic:

async def create_item(request: Request, body: dict):\n    svc = request.app.state.resources[\"item_svc\"]\n    return await svc.create(Item.model_validate(body))\n
"},{"location":"04_spec_first_routing/#conventions","title":"Conventions","text":"
  • operationId naming: list_<resource>, create_<resource>, get_<resource>, update_<resource>, delete_<resource>.
  • Read-only endpoints may use a cache decorator keyed on path + query + user.
  • Validation errors -> 400; not found -> 404; delete -> 204.
  • Security is declared once in the spec and enforced by the framework; handlers contain no auth code.
"},{"location":"04_spec_first_routing/#codegen","title":"Codegen","text":"

The openapi_first library can scaffold models and route stubs from the spec, and ships app templates (CRUD app, model app, service). Prefer codegen over hand-writing boilerplate.

"},{"location":"05_crud_template/","title":"CRUD Template","text":""},{"location":"05_crud_template/#summary","title":"Summary","text":"

Every resource follows the same four-file slice. Repetition is intentional: it makes the codebase predictable and generatable.

"},{"location":"05_crud_template/#the-slice","title":"The slice","text":"
domain/<resource>/\n    models.py               # pure Pydantic model (no DB fields)\n    ports/\n        db.py               # document model + repository factory\n        service.py          # service class + factory\n        converters.py       # (optional) transforms\n
"},{"location":"05_crud_template/#modelspy","title":"models.py","text":"

A framework-free Pydantic model describing the business shape.

class Item(BaseModel):\n    name: str\n    quantity: int = 0\n
"},{"location":"05_crud_template/#portsdbpy","title":"ports/db.py","text":"

The persistence model extends the business model and adds database concerns. The repository factory selects the right repository flavour.

class ItemDocument(Item, BaseDocument):   # BaseDocument adds id/created_at/...\n    pass\n\ndef create_item_repo(*, owner_repo=None) -> ItemRepository:\n    return repo_factory(\n        collection=\"items\",\n        document=ItemDocument,\n        populate_rules=[PopulateRule(field=\"owner\", collection=\"users\")],\n        cache=None,               # or a CachedBaseRepository config\n    )\n
"},{"location":"05_crud_template/#portsservicepy","title":"ports/service.py","text":"

The service is built through the DI library, injecting the repository. The generic CRUD base provides create/read/update/patch/delete/search/....

class ItemService(CrudsServiceAdaptor[Item, ItemDocument]):\n    model = Item\n    schema = ItemDocument\n\ndef get_item_service(repo) -> ItemService:\n    return build(ItemService, instances={\"repo\": repo})\n
"},{"location":"05_crud_template/#naming-conventions","title":"Naming conventions","text":"Concept Convention collection plural snake case (items) document <Resource>Document repository factory create_<resource>_repo service <Resource>Service service factory get_<resource>_service resource key <resource>_svc operationId list_items, create_item, ..."},{"location":"05_crud_template/#optional-files","title":"Optional files","text":"
  • converters.py \u2014 pure functions mapping between domain types.
  • ports/enrichment.py \u2014 adapts a pipeline into an async callable the service can invoke (used when a resource is populated by a pipeline).
"},{"location":"06_pipeline_scaffold/","title":"Pipeline Scaffold","text":""},{"location":"06_pipeline_scaffold/#summary","title":"Summary","text":"

Data-driven processing is expressed as a composition graph. The graph is built by a DI library; execution is driven by a DAG engine. Stages are pure and individually testable.

"},{"location":"06_pipeline_scaffold/#when-to-use-a-pipeline","title":"When to use a pipeline","text":"

Use a pipeline when a resource is produced by a multi-step transformation of an input (a document, a file, a message) rather than by a simple CRUD write.

"},{"location":"06_pipeline_scaffold/#structure-of-a-family","title":"Structure of a family","text":"

A family is one input shape (e.g. a file format or a message kind). Each family has the same skeleton:

pipelines/<family>/\n    abc.py            # port contracts (stage interfaces + root interface)\n    impl.py           # concrete stages + root implementation\n    pipeline.yaml     # graph spec: nodes, edges, roots, schema\n    resources/\n        pipeline.yaml # runtime spec consumed by the loader\n        patterns/*.yaml  # rule fragments (pattern registry)\n    banks/<variant>.py   # optional variants that re-pin a few slots\n
"},{"location":"06_pipeline_scaffold/#shared-scaffolding","title":"Shared scaffolding","text":"
pipelines/shared/\n    ports.py     # RegistryPort, FinalizePort, etc.\n    loader.py    # build(root_cls) -> wires the graph + engine\n    registry.py  # resolve_root(...) -> root class for a (kind, variant)\n
"},{"location":"06_pipeline_scaffold/#the-loader","title":"The loader","text":"
def build(root_cls, *, instances=None, config=None):\n    # merge resources/patterns/*.yaml into a pattern registry\n    # bind registry/config into each graph node by id\n    # build the schema (state) class and the engine\n    root = root_cls()\n    root._engine = Engine(graph, state_cls, payload_cls)\n    return root\n
"},{"location":"06_pipeline_scaffold/#the-registry","title":"The registry","text":"

A single lookup replaces hand-rolled if/else selection:

_FAMILIES = {\n    \"csv\":  {\"acme\": {\"heuristic\": AcmeCsvPipeline}},\n    \"json\": {\"globex\": {\"heuristic\": GlobexJsonPipeline}},\n}\n\ndef resolve_root(kind, variant, mode=\"heuristic\"):\n    return _FAMILIES[kind][variant][mode]\n

Adding a new variant means adding one class and one registry entry.

"},{"location":"06_pipeline_scaffold/#driving-a-pipeline","title":"Driving a pipeline","text":"

Stages are driven explicitly so intermediate state can be persisted:

pipeline = build(resolve_root(kind, variant))\ncontent = pipeline.ingest.materialize(raw_bytes)\nrecords = pipeline.reader.read(content)\nitems   = pipeline.normalize.build(records)\n
"},{"location":"06_pipeline_scaffold/#testing","title":"Testing","text":"

Because stages are pure, test each stage in isolation and test the assembled root against fixture inputs whose expected outputs are stored as golden files. A parity test replays fixtures headlessly.

"},{"location":"07_worker_and_progress/","title":"Worker & Progress","text":""},{"location":"07_worker_and_progress/#summary","title":"Summary","text":"

Long-running work runs in a background worker. Jobs are durable documents that move through an explicit status machine. Progress is streamed to clients.

"},{"location":"07_worker_and_progress/#the-status-machine","title":"The status machine","text":"
pending -> processing -> completed\n                     \\-> failed    (retried up to N times)\n                     \\-> paused    (waiting for human input)\n
  • pending \u2014 queued, eligible for pickup.
  • processing \u2014 claimed by a worker.
  • paused \u2014 blocked on a human-in-the-loop question.
  • failed \u2014 error; retry_count increments until a max, then skipped.
  • completed \u2014 terminal success.
"},{"location":"07_worker_and_progress/#the-poll-loop","title":"The poll loop","text":"
async def poll(resources):\n    backoff = MIN_POLL\n    while True:\n        jobs = await job_svc.search(filter={\"status\": \"pending\"}, limit=100)\n        backoff = MIN_POLL if jobs else min(backoff * 2, MAX_POLL)\n        for job in jobs:\n            if (job.retry_count or 0) >= MAX_RETRIES:\n                continue\n            await job_svc.update_status(job.id, \"processing\")\n            try:\n                await process(job, resources)\n            except PauseForResolution:\n                await job_svc.update_status(job.id, \"paused\")\n            except Exception as e:\n                await job_svc.update_status(\n                    job.id, \"failed\", error=str(e),\n                    retry_count=(job.retry_count or 0) + 1,\n                )\n        await asyncio.sleep(backoff)\n

Tunables live in environment variables: WORKER_POLL_MIN, WORKER_POLL_MAX, WORKER_MAX_RETRIES.

"},{"location":"07_worker_and_progress/#progress-events","title":"Progress events","text":"

A worker publishes structured progress events to a bus (a port). The HTTP edge exposes a streaming endpoint that replays events for a given job id.

class ProgressEvent(BaseModel):\n    job_id: str\n    step: str\n    status: str            # started | progress | completed | skipped | paused\n    message: dict | None = None\n

The bus is an in-memory adapter by default; because it is a port, it can be swapped for a durable broker. Emit from the orchestrator at each stage:

await self.bus.publish(ProgressEvent(job_id=jid, step=\"normalize\",\n                                     status=\"completed\", message={\"count\": n}))\n
"},{"location":"07_worker_and_progress/#streaming-endpoint","title":"Streaming endpoint","text":"
async def stream_job_events(request, id):\n    bus = request.app.state.resources.get(\"progress_bus\")\n    if bus is None:\n        return {\"events\": []}\n\n    async def gen():\n        sent = 0\n        while True:\n            events = bus.get_events(id)\n            for e in events[sent:]:\n                yield f\"data: {e.model_dump_json(exclude_none=True)}\\n\\n\"\n                sent += 1\n            await asyncio.sleep(0.5)\n\n    return StreamingResponse(gen(), media_type=\"text/event-stream\")\n

Known limitation. The default in-memory bus is per-process. If the worker and the API run as separate processes, events published by the worker are not visible to the API. For production, back the bus with a shared broker. The port boundary makes this a one-adapter change.

"},{"location":"08_human_in_loop/","title":"Human-in-the-Loop","text":""},{"location":"08_human_in_loop/#summary","title":"Summary","text":"

Some inputs are ambiguous and must be resolved by a human before processing can continue. The blueprint models this as a durable pause: an exception is raised, a question is recorded, the job pauses, and later resumes.

"},{"location":"08_human_in_loop/#the-mechanism","title":"The mechanism","text":"
  1. During processing, a stage raises a domain exception carrying the context needed to ask the question:
raise PauseForResolution(\n    job_id=jid,\n    reference=\"<stable key for this decision>\",\n    options=[...],\n    context={...},\n)\n
  1. The orchestrator catches it, records a question document, and re-raises so the worker marks the job paused.
  2. A client lists pending questions and answers one:
GET  /jobs/{id}/questions            -> list pending questions\nPOST /questions/{qid}/resolve  {choice}  -> record the choice\n
  1. When the last pending question is resolved, the resolve endpoint flips the parent job back to pending.
  2. The worker picks it up again, rebuilds a resolution map from previously answered questions, and seeds it into the pipeline so the same decision is not asked twice.
"},{"location":"08_human_in_loop/#the-question-document","title":"The question document","text":"
class Question(BaseDocument):\n    job_id: str\n    reference: str            # stable key: stage|field|value\n    status: str               # pending | resolved\n    options: list[dict]\n    context: dict\n    chosen: dict | None = None\n    resolved_at: str | None = None\n
"},{"location":"08_human_in_loop/#resume-map","title":"Resume map","text":"

The worker turns answered questions into a lookup keyed by reference, then injects it into the pipeline so that stage can skip re-raising:

resolved = {q.reference: q.chosen for q in await svc.search(\n    filter={\"job_id\": jid, \"status\": \"resolved\"})}\n\npipeline = build(root, instances={\"pause\": handler}, config={\"resolved\": resolved})\n
"},{"location":"08_human_in_loop/#resolve-endpoint","title":"Resolve endpoint","text":"
async def resolve_question(request, id, body):\n    svc = request.app.state.resources[\"question_svc\"]\n    q = await svc.read(id)\n    await svc.update(id, {\"chosen\": body[\"chosen\"], \"status\": \"resolved\",\n                          \"resolved_at\": now()})\n\n    pending = await svc.search(filter={\"job_id\": q.job_id, \"status\": \"pending\"},\n                               limit=1)\n    if not pending:\n        jobs = request.app.state.resources[\"job_svc\"]\n        job = await jobs.read(q.job_id)\n        if job and job.status == \"paused\":\n            await jobs.update_status(q.job_id, \"pending\")\n    return q\n
"},{"location":"08_human_in_loop/#cli-parity","title":"CLI parity","text":"

The same flow is available through scripts/CLI: list, get, create, questions, resolve, retry.

"},{"location":"09_infra_adaptors/","title":"Infra Adapters","text":""},{"location":"09_infra_adaptors/#summary","title":"Summary","text":"

All external systems are reached through adapters in infra/. Prefer an existing internal library over writing a new adapter.

"},{"location":"09_infra_adaptors/#database-mongodb","title":"Database (MongoDB)","text":"
  • A single connection manager owns the client lifecycle. setup_database(uri, db_name) / shutdown_database().
  • A repository factory selects the repository flavour:
repo_factory(collection, document, engine=None,\n             populate_rules=None, cache=None)\n
  • base repository \u2014 plain CRUD;
  • populating repository \u2014 resolves referenced documents (foreign keys);

  • cached repository \u2014 an LRU/TTL cache in front of reads.

  • Referenced documents are described declaratively:

PopulateRule(field=\"owner\", collection=\"users\",\n             nested_rules=[], max_depth=2, projection=None)\n
  • Transactions use a session context manager provided by the library.

Library: the Mongo operations library (see registry). Do not write raw Motor calls in the domain.

"},{"location":"09_infra_adaptors/#http-response-cache","title":"HTTP response cache","text":"

A generic GET cache sits behind a decorator.

@cache_response(ttl=60)\nasync def list_items(request): ...\n
  • Keyed on path + query + user.
  • No-op when no store is wired.
  • invalidate(*resources) bumps a generation counter for write-triggered invalidation.
"},{"location":"09_infra_adaptors/#event-progress-bus","title":"Event / progress bus","text":"

A port with an in-memory default adapter (publish, get_events). Swap for a durable broker in production (see Worker & Progress).

"},{"location":"09_infra_adaptors/#external-clients-ingestion","title":"External clients (ingestion)","text":"

External fetchers (files, mail, HTTP) are adapters that wrap ingestion libraries. Construct them in the composition root and inject into orchestrators.

Libraries: the ingestion library (formats) and the mail library (email).

"},{"location":"09_infra_adaptors/#llm-clients","title":"LLM clients","text":"

When a pipeline calls a model provider, route through a pool adaptor that handles provider selection, retries, and client lifecycle:

pool = setup_async_pool(provider=settings.llm_provider)\nresult = await pool.complete(model=\"...\", messages=[...])\n

Providers are selected by LLM_PROVIDER and configured by environment.

"},{"location":"09_infra_adaptors/#key-value-store","title":"Key-value store","text":"

For report/snapshot storage, provide a pluggable store port with a Redis adapter and an in-memory fallback.

"},{"location":"09_infra_adaptors/#rule","title":"Rule","text":"

If a capability already exists as a library (DB, DI, ingestion, mail, auth, docs), use it. Writing a private copy of a library is a defect.

"},{"location":"12_genericity_matrix/","title":"Genericity Matrix","text":""},{"location":"12_genericity_matrix/#summary","title":"Summary","text":"

This page states precisely what is reusable as-is, what is a template to adapt, and what is app-specific and must be authored. Use it to scope a new build.

"},{"location":"12_genericity_matrix/#reusable-as-is-do-not-rewrite","title":"Reusable as-is (do not rewrite)","text":"Area Artifact Layering & dependency rule the package layout and import rules Composition root the init.py shape, idempotency, get_svc overloading Web edge OpenAPIFirstApp bootstrap, handler-name = operationId, thin handlers CRUD slice the four-file pattern (model / ports-db / ports-service / converters) Generic CRUD service the base service adaptor Repository factory base / populating / cached selection Worker the poll loop, backoff, retry, status machine Progress bus the bus port + in-memory adapter + streaming endpoint Human-in-the-loop the durable pause + question document + resolve endpoint + resume map Infra adaptors HTTP cache decorator, KV store port, LLM pool scaffold Testing integration harness with fakes, fixture-parity pattern Docs GSDFC + generated stubs + MCP bundle pipeline"},{"location":"12_genericity_matrix/#template-to-adapt-copy-then-change-names","title":"Template to adapt (copy, then change names)","text":"Artifact Change pyproject.toml project name, dependencies Dockerfile / docker-compose.yaml service name, ports, services .env.example connection strings, provider keys openapi.yaml every path/schema/operationId scripts/ seed and operator tasks"},{"location":"12_genericity_matrix/#app-specific-must-be-authored","title":"App-specific (must be authored)","text":"Artifact Why Resource models the business shape Repository populate/cache rules the relationships Service business logic domain behaviour Pipeline families (if any) the transformation graph Pattern/rule resources data-driven policy Fixtures & golden outputs test data"},{"location":"12_genericity_matrix/#the-two-axes","title":"The two axes","text":"

Everything app-specific reduces to two axes:

  1. Resource model \u2014 the set of resources, their fields, relationships, and the OpenAPI contract.
  2. Pipeline family \u2014 only if inputs require multi-step transformation.

If you can specify those two axes, the rest of the app follows mechanically.

"},{"location":"13_library_registry/","title":"Library Registry","text":""},{"location":"13_library_registry/#summary","title":"Summary","text":"

The Aetos ecosystem provides internal libraries for the shared concerns of every application. Use them. Writing a private copy of any capability below is a defect.

Each library also exposes its own documentation as an MCP server. An AI agent can read a library's full API by connecting to its MCP and reading doc://index, doc://nav, and doc://modules/{module}.

"},{"location":"13_library_registry/#catalog","title":"Catalog","text":"Library Import Purpose Use instead of MCP DAG Pipe dagpipe Deterministic state-transformation DAG engine hand-rolled orchestration/topological ordering :8006 Hexa hexa Declarative DI / composition-root tooling manual wiring, hand-written factories (see note) Mongo Ops mongo_ops Async MongoDB repositories, cache, populate, transactions raw Motor CRUD and connection lifecycle :8007 Omniread omniread Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) custom scrapers/parsers :8003 OpenAPI First openapi_first Spec-first FastAPI bootstrap, client, codegen decorator routers and hand-written clients :8004 Mail Intake mail_intake Provider-agnostic read-only email ingestion Gmail OAuth + MIME parsing :8002 Py JWT jwtlib JWT auth primitives (register/login/introspect) password hashing + token code :8005 Doc Forge docforge Docstring compiler \u2192 MkDocs + MCP docs custom doc generators :8001 Auth Server (service) Central identity service (register/login/me/introspect) building auth endpoints yourself \u2014"},{"location":"13_library_registry/#how-to-choose","title":"How to choose","text":"You need to\u2026 Use persist and query documents Mongo Ops compose dependencies Hexa run a multi-step transformation DAG Pipe read/parse an arbitrary input format Omniread expose a contract-first HTTP API OpenAPI First ingest email Mail Intake authenticate users / verify tokens Py JWT (+ Auth Server) generate documentation Doc Forge"},{"location":"13_library_registry/#discovery","title":"Discovery","text":"

A single standards MCP exposes find_library(capability), which returns the matching library, its import name, its \"use instead of\" guidance, and its MCP endpoint. Query it before writing any infrastructure.

"},{"location":"13_library_registry/#notes","title":"Notes","text":"
  • Hexa ships an MCP bundle but is not yet assigned a dedicated port in the hub config; it is reachable via the standards registry entry and will be wired to a port.
  • Library MCP servers expose documentation resources (not executable tools): they tell an agent the available API surface so it does not invent code.
"}]} \ No newline at end of file diff --git a/standards/mcp/libraries.json b/standards/mcp/libraries.json index facdfb0..ba58017 100644 Binary files a/standards/mcp/libraries.json and b/standards/mcp/libraries.json differ diff --git a/standards/mcp/modules/library_registry.json b/standards/mcp/modules/library_registry.json index b8096de..bee7171 100644 --- a/standards/mcp/modules/library_registry.json +++ b/standards/mcp/modules/library_registry.json @@ -2,7 +2,7 @@ "module": "library_registry", "content": { "path": "library_registry", - "docstring": "# Library Registry\n\n## Summary\n\nThe Aetos ecosystem provides internal libraries for the shared concerns of\nevery application. **Use them.** Writing a private copy of any capability below\nis a defect.\n\nEach library also exposes its own documentation as an MCP server. An AI agent\ncan read a library's full API by connecting to its MCP and reading\n`docs://index`, `docs://nav`, and `docs://modules/{module}`.\n\n---\n\n## Catalog\n\n| Library | Import | Purpose | Use instead of | MCP |\n|---|---|---|---|---|\n| DAG Pipe | `dagpipe` | Deterministic state-transformation DAG engine | hand-rolled orchestration/topological ordering | `:8006` |\n| Hexa | `hexa` | Declarative DI / composition-root tooling | manual wiring, hand-written factories | *(see note)* |\n| Mongo Ops | `mongo_ops` | Async MongoDB repositories, cache, populate, transactions | raw Motor CRUD and connection lifecycle | `:8007` |\n| Omniread | `omniread` | Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) | custom scrapers/parsers | `:8003` |\n| OpenAPI First | `openapi_first` | Spec-first FastAPI bootstrap, client, codegen | decorator routers and hand-written clients | `:8004` |\n| Mail Intake | `mail_intake` | Provider-agnostic read-only email ingestion | Gmail OAuth + MIME parsing | `:8002` |\n| Py JWT | `jwtlib` | JWT auth primitives (register/login/introspect) | password hashing + token code | `:8005` |\n| Doc Forge | `docforge` | Docstring compiler โ†’ MkDocs + MCP docs | custom doc generators | `:8001` |\n| Auth Server | *(service)* | Central identity service (register/login/me/introspect) | building auth endpoints yourself | โ€” |\n\n## How to choose\n\n| You need toโ€ฆ | Use |\n|---|---|\n| persist and query documents | Mongo Ops |\n| compose dependencies | Hexa |\n| run a multi-step transformation | DAG Pipe |\n| read/parse an arbitrary input format | Omniread |\n| expose a contract-first HTTP API | OpenAPI First |\n| ingest email | Mail Intake |\n| authenticate users / verify tokens | Py JWT (+ Auth Server) |\n| generate documentation | Doc Forge |\n\n## Discovery\n\nA single **standards MCP** exposes `find_library(capability)`, which returns the\nmatching library, its import name, its \"use instead of\" guidance, and its MCP\nendpoint. Query it before writing any infrastructure.\n\n## Notes\n\n- **Hexa** ships an MCP bundle but is not yet assigned a dedicated port in the\n hub config; it is reachable via the standards registry entry and will be\n wired to a port.\n- Library MCP servers expose **documentation resources** (not executable tools):\n they tell an agent the available API surface so it does not invent code.\n", + "docstring": "# Library Registry\n\n## Summary\n\nThe Aetos ecosystem provides internal libraries for the shared concerns of\nevery application. **Use them.** Writing a private copy of any capability below\nis a defect.\n\nEach library also exposes its own documentation as an MCP server. An AI agent\ncan read a library's full API by connecting to its MCP and reading\n`doc://index`, `doc://nav`, and `doc://modules/{module}`.\n\n---\n\n## Catalog\n\n| Library | Import | Purpose | Use instead of | MCP |\n|---|---|---|---|---|\n| DAG Pipe | `dagpipe` | Deterministic state-transformation DAG engine | hand-rolled orchestration/topological ordering | `:8006` |\n| Hexa | `hexa` | Declarative DI / composition-root tooling | manual wiring, hand-written factories | *(see note)* |\n| Mongo Ops | `mongo_ops` | Async MongoDB repositories, cache, populate, transactions | raw Motor CRUD and connection lifecycle | `:8007` |\n| Omniread | `omniread` | Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) | custom scrapers/parsers | `:8003` |\n| OpenAPI First | `openapi_first` | Spec-first FastAPI bootstrap, client, codegen | decorator routers and hand-written clients | `:8004` |\n| Mail Intake | `mail_intake` | Provider-agnostic read-only email ingestion | Gmail OAuth + MIME parsing | `:8002` |\n| Py JWT | `jwtlib` | JWT auth primitives (register/login/introspect) | password hashing + token code | `:8005` |\n| Doc Forge | `docforge` | Docstring compiler โ†’ MkDocs + MCP docs | custom doc generators | `:8001` |\n| Auth Server | *(service)* | Central identity service (register/login/me/introspect) | building auth endpoints yourself | โ€” |\n\n## How to choose\n\n| You need toโ€ฆ | Use |\n|---|---|\n| persist and query documents | Mongo Ops |\n| compose dependencies | Hexa |\n| run a multi-step transformation | DAG Pipe |\n| read/parse an arbitrary input format | Omniread |\n| expose a contract-first HTTP API | OpenAPI First |\n| ingest email | Mail Intake |\n| authenticate users / verify tokens | Py JWT (+ Auth Server) |\n| generate documentation | Doc Forge |\n\n## Discovery\n\nA single **standards MCP** exposes `find_library(capability)`, which returns the\nmatching library, its import name, its \"use instead of\" guidance, and its MCP\nendpoint. Query it before writing any infrastructure.\n\n## Notes\n\n- **Hexa** ships an MCP bundle but is not yet assigned a dedicated port in the\n hub config; it is reachable via the standards registry entry and will be\n wired to a port.\n- Library MCP servers expose **documentation resources** (not executable tools):\n they tell an agent the available API surface so it does not invent code.\n", "objects": {} } } \ No newline at end of file diff --git a/standards/mcp/standards/index.json b/standards/mcp/standards/index.json deleted file mode 100644 index b06d01e..0000000 --- a/standards/mcp/standards/index.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "project": "standards", - "type": "standards", - "modules_count": 11, - "source": "hand-authored" -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/composition_root.json b/standards/mcp/standards/modules/composition_root.json deleted file mode 100644 index 0011498..0000000 --- a/standards/mcp/standards/modules/composition_root.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "composition_root", - "content": { - "path": "composition_root", - "docstring": "# Composition Root\n\n## Summary\n\nAll wiring happens in exactly one module. It builds a **flat resource\nregistry** and exposes it to the HTTP layer and to scripts.\n\n---\n\n## The resource registry\n\n`init.py` returns a plain `dict[str, Any]`. Keys follow two conventions:\n\n- `_repo` โ€” a repository adapter for a resource.\n- `_svc` โ€” a service for a resource.\n\nPlus named infrastructure adaptors:\n\n- `http_cache_store` โ€” response cache backend.\n- `progress_bus` โ€” progress/event bus.\n- `_` โ€” fetchers, clients, stores.\n\nThe API stores the dict on `app.state.resources` during lifespan startup.\nHandlers read from it; nothing imports concrete adapters.\n\n## Shape\n\n```python\n_resources: dict = {}\n_initialized = False\n\nasync def init(settings=None) -> dict:\n global _initialized\n if _initialized:\n return _resources\n\n await setup_database(uri=settings.mongo_uri, db_name=settings.mongo_db)\n\n # 1. leaf repositories\n user_repo = create_user_repo()\n\n # 2. dependent repositories (dependencies passed by keyword)\n wallet_repo = create_wallet_repo(user_repo=user_repo)\n\n # 3. services (build via the DI library, injecting the repo)\n _resources[\"user_repo\"] = user_repo\n _resources[\"wallet_repo\"] = wallet_repo\n _resources[\"user_svc\"] = get_user_service(user_repo)\n _resources[\"wallet_svc\"] = get_wallet_service(wallet_repo)\n\n # 4. infrastructure adaptors (optional)\n _resources[\"http_cache_store\"] = HttpCacheStore(...)\n _resources[\"progress_bus\"] = ProgressEventBus()\n\n _initialized = True\n return _resources\n\nasync def shutdown() -> None:\n global _initialized\n await close_stores(_resources)\n await shutdown_database()\n _resources.clear()\n _initialized = False\n```\n\n## Typed accessors\n\nExpose a `get_svc(name)` helper with one `@overload` per key so scripts get\ntyping without importing concrete classes:\n\n```python\n@overload\ndef get_svc(name: Literal[\"user_svc\"]) -> UserService: ...\n@overload\ndef get_svc(name: Literal[\"wallet_svc\"]) -> WalletService: ...\n\ndef get_svc(name: str):\n return _resources[name]\n```\n\nAlso expose `get_db()` (raw database handle) and pure helpers such as\n`compute_fingerprint(payload)` (canonical hash used for deduplication).\n\n## Rules\n\n- `init()` is **idempotent**.\n- The composition root is the only place that imports concrete adapters.\n- Handlers never call `init()`; they read `request.app.state.resources`.\n- Scripts call `init()` / `shutdown()` around their work.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/crud_template.json b/standards/mcp/standards/modules/crud_template.json deleted file mode 100644 index f2d855a..0000000 --- a/standards/mcp/standards/modules/crud_template.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "crud_template", - "content": { - "path": "crud_template", - "docstring": "# CRUD Template\n\n## Summary\n\nEvery resource follows the same four-file slice. Repetition is intentional: it\nmakes the codebase predictable and generatable.\n\n---\n\n## The slice\n\n```\ndomain//\n models.py # pure Pydantic model (no DB fields)\n ports/\n db.py # document model + repository factory\n service.py # service class + factory\n converters.py # (optional) transforms\n```\n\n## models.py\n\nA framework-free Pydantic model describing the business shape.\n\n```python\nclass Item(BaseModel):\n name: str\n quantity: int = 0\n```\n\n## ports/db.py\n\nThe persistence model extends the business model and adds database concerns.\nThe repository factory selects the right repository flavour.\n\n```python\nclass ItemDocument(Item, BaseDocument): # BaseDocument adds id/created_at/...\n pass\n\ndef create_item_repo(*, owner_repo=None) -> ItemRepository:\n return repo_factory(\n collection=\"items\",\n document=ItemDocument,\n populate_rules=[PopulateRule(field=\"owner\", collection=\"users\")],\n cache=None, # or a CachedBaseRepository config\n )\n```\n\n## ports/service.py\n\nThe service is built through the DI library, injecting the repository. The\ngeneric CRUD base provides `create/read/update/patch/delete/search/...`.\n\n```python\nclass ItemService(CrudsServiceAdaptor[Item, ItemDocument]):\n model = Item\n schema = ItemDocument\n\ndef get_item_service(repo) -> ItemService:\n return build(ItemService, instances={\"repo\": repo})\n```\n\n## Naming conventions\n\n| Concept | Convention |\n|---|---|\n| collection | plural snake case (`items`) |\n| document | `Document` |\n| repository factory | `create__repo` |\n| service | `Service` |\n| service factory | `get__service` |\n| resource key | `_svc` |\n| operationId | `list_items`, `create_item`, ... |\n\n## Optional files\n\n- `converters.py` โ€” pure functions mapping between domain types.\n- `ports/enrichment.py` โ€” adapts a pipeline into an async callable the service\n can invoke (used when a resource is populated by a pipeline).\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/genericity_matrix.json b/standards/mcp/standards/modules/genericity_matrix.json deleted file mode 100644 index 18253f3..0000000 --- a/standards/mcp/standards/modules/genericity_matrix.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "genericity_matrix", - "content": { - "path": "genericity_matrix", - "docstring": "# Genericity Matrix\n\n## Summary\n\nThis page states precisely what is **reusable as-is**, what is a **template to\nadapt**, and what is **app-specific and must be authored**. Use it to scope a\nnew build.\n\n---\n\n## Reusable as-is (do not rewrite)\n\n| Area | Artifact |\n|---|---|\n| Layering & dependency rule | the package layout and import rules |\n| Composition root | the `init.py` shape, idempotency, `get_svc` overloading |\n| Web edge | `OpenAPIFirstApp` bootstrap, handler-name = `operationId`, thin handlers |\n| CRUD slice | the four-file pattern (model / ports-db / ports-service / converters) |\n| Generic CRUD service | the base service adaptor |\n| Repository factory | base / populating / cached selection |\n| Worker | the poll loop, backoff, retry, status machine |\n| Progress bus | the bus port + in-memory adapter + streaming endpoint |\n| Human-in-the-loop | the durable pause + question document + resolve endpoint + resume map |\n| Infra adaptors | HTTP cache decorator, KV store port, LLM pool scaffold |\n| Testing | integration harness with fakes, fixture-parity pattern |\n| Docs | GSDFC + generated stubs + MCP bundle pipeline |\n\n## Template to adapt (copy, then change names)\n\n| Artifact | Change |\n|---|---|\n| `pyproject.toml` | project name, dependencies |\n| `Dockerfile` / `docker-compose.yaml` | service name, ports, services |\n| `.env.example` | connection strings, provider keys |\n| `openapi.yaml` | every path/schema/operationId |\n| `scripts/` | seed and operator tasks |\n\n## App-specific (must be authored)\n\n| Artifact | Why |\n|---|---|\n| Resource models | the business shape |\n| Repository populate/cache rules | the relationships |\n| Service business logic | domain behaviour |\n| Pipeline families (if any) | the transformation graph |\n| Pattern/rule resources | data-driven policy |\n| Fixtures & golden outputs | test data |\n\n## The two axes\n\nEverything app-specific reduces to **two axes**:\n\n1. **Resource model** โ€” the set of resources, their fields, relationships, and\n the OpenAPI contract.\n2. **Pipeline family** โ€” only if inputs require multi-step transformation.\n\nIf you can specify those two axes, the rest of the app follows mechanically.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/human_in_loop.json b/standards/mcp/standards/modules/human_in_loop.json deleted file mode 100644 index ea11e76..0000000 --- a/standards/mcp/standards/modules/human_in_loop.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "human_in_loop", - "content": { - "path": "human_in_loop", - "docstring": "# Human-in-the-Loop\n\n## Summary\n\nSome inputs are ambiguous and must be resolved by a human before processing can\ncontinue. The blueprint models this as a **durable pause**: an exception is\nraised, a question is recorded, the job pauses, and later resumes.\n\n---\n\n## The mechanism\n\n1. During processing, a stage raises a domain exception carrying the context\n needed to ask the question:\n\n ```python\n raise PauseForResolution(\n job_id=jid,\n reference=\"\",\n options=[...],\n context={...},\n )\n ```\n\n2. The orchestrator catches it, records a **question document**, and re-raises\n so the worker marks the job `paused`.\n3. A client lists pending questions and answers one:\n\n ```\n GET /jobs/{id}/questions -> list pending questions\n POST /questions/{qid}/resolve {choice} -> record the choice\n ```\n\n4. When the **last** pending question is resolved, the resolve endpoint flips\n the parent job back to `pending`.\n5. The worker picks it up again, rebuilds a **resolution map** from previously\n answered questions, and seeds it into the pipeline so the same decision is\n not asked twice.\n\n## The question document\n\n```python\nclass Question(BaseDocument):\n job_id: str\n reference: str # stable key: stage|field|value\n status: str # pending | resolved\n options: list[dict]\n context: dict\n chosen: dict | None = None\n resolved_at: str | None = None\n```\n\n## Resume map\n\nThe worker turns answered questions into a lookup keyed by `reference`, then\ninjects it into the pipeline so that stage can skip re-raising:\n\n```python\nresolved = {q.reference: q.chosen for q in await svc.search(\n filter={\"job_id\": jid, \"status\": \"resolved\"})}\n\npipeline = build(root, instances={\"pause\": handler}, config={\"resolved\": resolved})\n```\n\n## Resolve endpoint\n\n```python\nasync def resolve_question(request, id, body):\n svc = request.app.state.resources[\"question_svc\"]\n q = await svc.read(id)\n await svc.update(id, {\"chosen\": body[\"chosen\"], \"status\": \"resolved\",\n \"resolved_at\": now()})\n\n pending = await svc.search(filter={\"job_id\": q.job_id, \"status\": \"pending\"},\n limit=1)\n if not pending:\n jobs = request.app.state.resources[\"job_svc\"]\n job = await jobs.read(q.job_id)\n if job and job.status == \"paused\":\n await jobs.update_status(q.job_id, \"pending\")\n return q\n```\n\n## CLI parity\n\nThe same flow is available through scripts/CLI: `list`, `get`, `create`,\n`questions`, `resolve`, `retry`.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/index.json b/standards/mcp/standards/modules/index.json deleted file mode 100644 index 6ea50b1..0000000 --- a/standards/mcp/standards/modules/index.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "index", - "content": { - "path": "index", - "docstring": "# Standards Reference\n\nThe reference half of the Standards description: the enforceable rules,\npatterns, and the internal library catalog that agents consult while coding.\n\n- **Layering Rules** โ€” the strict request โ†’ domain โ†’ repository layering and\n the rules that enforce it.\n- **Composition Root** โ€” a single composition point wiring the whole app.\n- **Spec-First Routing** โ€” the OpenAPI contract as the routing source of truth.\n- **CRUD Template** โ€” the standardized generic resource endpoint scaffold.\n- **Pipeline Scaffold** โ€” the deterministic pipeline skeleton for multi-stage\n jobs.\n- **Worker & Progress** โ€” queued work, progress reporting, and visibility.\n- **Human-in-the-Loop** โ€” guardrails and approvals inside automations.\n- **Infra Adapters** โ€” how infrastructure connects without leaking into domain.\n- **Genericity Matrix** โ€” what is generic vs app-specific.\n- **Library Registry** โ€” the catalog of internal libraries behind all of it.\n\nRead the **Wiki** for the philosophy and rationale; these pages are the\nreference that mirrors the `standards` MCP bundle.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/infra_adaptors.json b/standards/mcp/standards/modules/infra_adaptors.json deleted file mode 100644 index bb00edb..0000000 --- a/standards/mcp/standards/modules/infra_adaptors.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "infra_adaptors", - "content": { - "path": "infra_adaptors", - "docstring": "# Infra Adapters\n\n## Summary\n\nAll external systems are reached through adapters in `infra/`. Prefer an\nexisting internal library over writing a new adapter.\n\n---\n\n## Database (MongoDB)\n\n- A single connection manager owns the client lifecycle.\n `setup_database(uri, db_name)` / `shutdown_database()`.\n- A repository **factory** selects the repository flavour:\n\n ```python\n repo_factory(collection, document, engine=None,\n populate_rules=None, cache=None)\n ```\n\n - base repository โ€” plain CRUD;\n - populating repository โ€” resolves referenced documents (foreign keys);\n - cached repository โ€” an LRU/TTL cache in front of reads.\n\n- Referenced documents are described declaratively:\n\n ```python\n PopulateRule(field=\"owner\", collection=\"users\",\n nested_rules=[], max_depth=2, projection=None)\n ```\n\n- Transactions use a session context manager provided by the library.\n\n**Library:** the Mongo operations library (see registry). Do not write raw\nMotor calls in the domain.\n\n## HTTP response cache\n\nA generic GET cache sits behind a decorator.\n\n```python\n@cache_response(ttl=60)\nasync def list_items(request): ...\n```\n\n- Keyed on path + query + user.\n- No-op when no store is wired.\n- `invalidate(*resources)` bumps a generation counter for write-triggered\n invalidation.\n\n## Event / progress bus\n\nA port with an in-memory default adapter (`publish`, `get_events`). Swap for a\ndurable broker in production (see Worker & Progress).\n\n## External clients (ingestion)\n\nExternal fetchers (files, mail, HTTP) are adapters that wrap ingestion\nlibraries. Construct them in the composition root and inject into\norchestrators.\n\n**Libraries:** the ingestion library (formats) and the mail library (email).\n\n## LLM clients\n\nWhen a pipeline calls a model provider, route through a pool adaptor that\nhandles provider selection, retries, and client lifecycle:\n\n```python\npool = setup_async_pool(provider=settings.llm_provider)\nresult = await pool.complete(model=\"...\", messages=[...])\n```\n\nProviders are selected by `LLM_PROVIDER` and configured by environment.\n\n## Key-value store\n\nFor report/snapshot storage, provide a pluggable store port with a Redis\nadapter and an in-memory fallback.\n\n## Rule\n\nIf a capability already exists as a library (DB, DI, ingestion, mail, auth,\ndocs), **use it**. Writing a private copy of a library is a defect.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/layering_rules.json b/standards/mcp/standards/modules/layering_rules.json deleted file mode 100644 index 594b7e1..0000000 --- a/standards/mcp/standards/modules/layering_rules.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "layering_rules", - "content": { - "path": "layering_rules", - "docstring": "# Layering Rules\n\n## Summary\n\nThe dependency direction is fixed. Violating it is a design defect.\n\n---\n\n## Allowed imports\n\n```\ninterfaces/* -> domain/* (services, ports, models)\ninterfaces/* -> infra/adapters/* (only read-only value objects)\ndomain/* -> domain/* (same layer)\ndomain/* -> pipelines/* (call a pipeline via a port)\ninfra/adapters/* -> domain/ports/* (implement the interface)\nmain.py/init.py -> everything (composition root only)\n```\n\n## Forbidden imports\n\n```\ndomain/* -> interfaces/* (never)\ndomain/* -> fastapi/starlette (never)\ninfra/adapters/* -> interfaces/* (never)\ninterfaces/* -> infra/db/* (state lives in resources, not imports)\n```\n\n## Why\n\n- A pure domain is testable without a server, a database, or an event loop.\n- Adapters can be swapped (in-memory fake, real database) without touching\n business logic.\n- The HTTP edge stays thin and replaceable.\n\n## The port/adapter contract\n\nA **port** is an abstract class or protocol declared in `domain//ports/`.\nAn **adapter** implements it in `infra/adapters/`. The composition root binds\nthe two.\n\n```\ndomain//ports/db.py -> repository port + document model\ndomain//ports/service.py -> service + factory\ninfra/db/repos/.py -> repository adapter factory\ninfra/adapters/service.py -> generic CRUD service base\n```\n\n## Enforcement\n\n- Keep every domain module importable with only the standard library and\n Pydantic. If a domain import pulls in `fastapi`, the layering is broken.\n- Reviewers (and CI) reject `import fastapi` inside `domain/` or `infra/`.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/library_registry.json b/standards/mcp/standards/modules/library_registry.json deleted file mode 100644 index b8096de..0000000 --- a/standards/mcp/standards/modules/library_registry.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "library_registry", - "content": { - "path": "library_registry", - "docstring": "# Library Registry\n\n## Summary\n\nThe Aetos ecosystem provides internal libraries for the shared concerns of\nevery application. **Use them.** Writing a private copy of any capability below\nis a defect.\n\nEach library also exposes its own documentation as an MCP server. An AI agent\ncan read a library's full API by connecting to its MCP and reading\n`docs://index`, `docs://nav`, and `docs://modules/{module}`.\n\n---\n\n## Catalog\n\n| Library | Import | Purpose | Use instead of | MCP |\n|---|---|---|---|---|\n| DAG Pipe | `dagpipe` | Deterministic state-transformation DAG engine | hand-rolled orchestration/topological ordering | `:8006` |\n| Hexa | `hexa` | Declarative DI / composition-root tooling | manual wiring, hand-written factories | *(see note)* |\n| Mongo Ops | `mongo_ops` | Async MongoDB repositories, cache, populate, transactions | raw Motor CRUD and connection lifecycle | `:8007` |\n| Omniread | `omniread` | Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) | custom scrapers/parsers | `:8003` |\n| OpenAPI First | `openapi_first` | Spec-first FastAPI bootstrap, client, codegen | decorator routers and hand-written clients | `:8004` |\n| Mail Intake | `mail_intake` | Provider-agnostic read-only email ingestion | Gmail OAuth + MIME parsing | `:8002` |\n| Py JWT | `jwtlib` | JWT auth primitives (register/login/introspect) | password hashing + token code | `:8005` |\n| Doc Forge | `docforge` | Docstring compiler โ†’ MkDocs + MCP docs | custom doc generators | `:8001` |\n| Auth Server | *(service)* | Central identity service (register/login/me/introspect) | building auth endpoints yourself | โ€” |\n\n## How to choose\n\n| You need toโ€ฆ | Use |\n|---|---|\n| persist and query documents | Mongo Ops |\n| compose dependencies | Hexa |\n| run a multi-step transformation | DAG Pipe |\n| read/parse an arbitrary input format | Omniread |\n| expose a contract-first HTTP API | OpenAPI First |\n| ingest email | Mail Intake |\n| authenticate users / verify tokens | Py JWT (+ Auth Server) |\n| generate documentation | Doc Forge |\n\n## Discovery\n\nA single **standards MCP** exposes `find_library(capability)`, which returns the\nmatching library, its import name, its \"use instead of\" guidance, and its MCP\nendpoint. Query it before writing any infrastructure.\n\n## Notes\n\n- **Hexa** ships an MCP bundle but is not yet assigned a dedicated port in the\n hub config; it is reachable via the standards registry entry and will be\n wired to a port.\n- Library MCP servers expose **documentation resources** (not executable tools):\n they tell an agent the available API surface so it does not invent code.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/pipeline_scaffold.json b/standards/mcp/standards/modules/pipeline_scaffold.json deleted file mode 100644 index c15078e..0000000 --- a/standards/mcp/standards/modules/pipeline_scaffold.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "pipeline_scaffold", - "content": { - "path": "pipeline_scaffold", - "docstring": "# Pipeline Scaffold\n\n## Summary\n\nData-driven processing is expressed as a **composition graph**. The graph is\nbuilt by a DI library; execution is driven by a DAG engine. Stages are pure and\nindividually testable.\n\n---\n\n## When to use a pipeline\n\nUse a pipeline when a resource is produced by a **multi-step transformation**\nof an input (a document, a file, a message) rather than by a simple CRUD write.\n\n## Structure of a family\n\nA **family** is one input shape (e.g. a file format or a message kind). Each\nfamily has the same skeleton:\n\n```\npipelines//\n abc.py # port contracts (stage interfaces + root interface)\n impl.py # concrete stages + root implementation\n pipeline.yaml # graph spec: nodes, edges, roots, schema\n resources/\n pipeline.yaml # runtime spec consumed by the loader\n patterns/*.yaml # rule fragments (pattern registry)\n banks/.py # optional variants that re-pin a few slots\n```\n\n## Shared scaffolding\n\n```\npipelines/shared/\n ports.py # RegistryPort, FinalizePort, etc.\n loader.py # build(root_cls) -> wires the graph + engine\n registry.py # resolve_root(...) -> root class for a (kind, variant)\n```\n\n## The loader\n\n```python\ndef build(root_cls, *, instances=None, config=None):\n # merge resources/patterns/*.yaml into a pattern registry\n # bind registry/config into each graph node by id\n # build the schema (state) class and the engine\n root = root_cls()\n root._engine = Engine(graph, state_cls, payload_cls)\n return root\n```\n\n## The registry\n\nA single lookup replaces hand-rolled `if/else` selection:\n\n```python\n_FAMILIES = {\n \"csv\": {\"acme\": {\"heuristic\": AcmeCsvPipeline}},\n \"json\": {\"globex\": {\"heuristic\": GlobexJsonPipeline}},\n}\n\ndef resolve_root(kind, variant, mode=\"heuristic\"):\n return _FAMILIES[kind][variant][mode]\n```\n\nAdding a new variant means adding one class and one registry entry.\n\n## Driving a pipeline\n\nStages are driven explicitly so intermediate state can be persisted:\n\n```python\npipeline = build(resolve_root(kind, variant))\ncontent = pipeline.ingest.materialize(raw_bytes)\nrecords = pipeline.reader.read(content)\nitems = pipeline.normalize.build(records)\n```\n\n## Testing\n\nBecause stages are pure, test each stage in isolation and test the assembled\nroot against **fixture inputs** whose expected outputs are stored as golden\nfiles. A parity test replays fixtures headlessly.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/spec_first_routing.json b/standards/mcp/standards/modules/spec_first_routing.json deleted file mode 100644 index 9db3023..0000000 --- a/standards/mcp/standards/modules/spec_first_routing.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "spec_first_routing", - "content": { - "path": "spec_first_routing", - "docstring": "# Spec-First Routing\n\n## Summary\n\nThe OpenAPI document is the single source of truth for the HTTP contract.\nHandlers are bound to it by name; there is no decorator router.\n\n---\n\n## The contract\n\n- `openapi.yaml` (or `.json`) declares paths, schemas, parameters, responses,\n and security.\n- A custom `operationId` names the handler function.\n- The framework (`openapi_first`) validates requests/responses against the\n spec and dispatches to the handler whose name equals `operationId`.\n\n## App bootstrap\n\n```python\n# main.py\nfrom openapi_first.app import OpenAPIFirstApp\n\napp = OpenAPIFirstApp(\n openapi_path=\"openapi.yaml\",\n routes_module=interfaces.openapi, # module whose namespace holds handlers\n title=\" API\",\n version=\"0.1.0\",\n lifespan=lifespan,\n)\n```\n\n`lifespan` calls the composition root:\n\n```python\n@asynccontextmanager\nasync def lifespan(app):\n app.state.resources = await init()\n yield\n await shutdown()\n```\n\n## Handler registry\n\n`interfaces/openapi.py` is a flat re-export of every handler, so the framework\ncan resolve `operationId` across the whole app:\n\n```python\nfrom interfaces.item.routes import *\nfrom interfaces.wallet.routes import *\nfrom interfaces.progress.sse import *\n```\n\n## Handler shape\n\nHandlers are thin and synchronous with respect to business logic:\n\n```python\nasync def create_item(request: Request, body: dict):\n svc = request.app.state.resources[\"item_svc\"]\n return await svc.create(Item.model_validate(body))\n```\n\n## Conventions\n\n- `operationId` naming: `list_`, `create_`,\n `get_`, `update_`, `delete_`.\n- Read-only endpoints may use a cache decorator keyed on path + query + user.\n- Validation errors -> `400`; not found -> `404`; delete -> `204`.\n- Security is declared once in the spec and enforced by the framework; handlers\n contain no auth code.\n\n## Codegen\n\nThe `openapi_first` library can scaffold models and route stubs from the spec,\nand ships app templates (CRUD app, model app, service). Prefer codegen over\nhand-writing boilerplate.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/modules/worker_and_progress.json b/standards/mcp/standards/modules/worker_and_progress.json deleted file mode 100644 index 09d79ef..0000000 --- a/standards/mcp/standards/modules/worker_and_progress.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "module": "worker_and_progress", - "content": { - "path": "worker_and_progress", - "docstring": "# Worker & Progress\n\n## Summary\n\nLong-running work runs in a background worker. Jobs are durable documents that\nmove through an explicit status machine. Progress is streamed to clients.\n\n---\n\n## The status machine\n\n```\npending -> processing -> completed\n \\-> failed (retried up to N times)\n \\-> paused (waiting for human input)\n```\n\n- `pending` โ€” queued, eligible for pickup.\n- `processing` โ€” claimed by a worker.\n- `paused` โ€” blocked on a human-in-the-loop question.\n- `failed` โ€” error; `retry_count` increments until a max, then skipped.\n- `completed` โ€” terminal success.\n\n## The poll loop\n\n```python\nasync def poll(resources):\n backoff = MIN_POLL\n while True:\n jobs = await job_svc.search(filter={\"status\": \"pending\"}, limit=100)\n backoff = MIN_POLL if jobs else min(backoff * 2, MAX_POLL)\n for job in jobs:\n if (job.retry_count or 0) >= MAX_RETRIES:\n continue\n await job_svc.update_status(job.id, \"processing\")\n try:\n await process(job, resources)\n except PauseForResolution:\n await job_svc.update_status(job.id, \"paused\")\n except Exception as e:\n await job_svc.update_status(\n job.id, \"failed\", error=str(e),\n retry_count=(job.retry_count or 0) + 1,\n )\n await asyncio.sleep(backoff)\n```\n\nTunables live in environment variables: `WORKER_POLL_MIN`, `WORKER_POLL_MAX`,\n`WORKER_MAX_RETRIES`.\n\n## Progress events\n\nA worker publishes structured progress events to a **bus** (a port). The HTTP\nedge exposes a streaming endpoint that replays events for a given job id.\n\n```python\nclass ProgressEvent(BaseModel):\n job_id: str\n step: str\n status: str # started | progress | completed | skipped | paused\n message: dict | None = None\n```\n\nThe bus is an in-memory adapter by default; because it is a port, it can be\nswapped for a durable broker. Emit from the orchestrator at each stage:\n\n```python\nawait self.bus.publish(ProgressEvent(job_id=jid, step=\"normalize\",\n status=\"completed\", message={\"count\": n}))\n```\n\n## Streaming endpoint\n\n```python\nasync def stream_job_events(request, id):\n bus = request.app.state.resources.get(\"progress_bus\")\n if bus is None:\n return {\"events\": []}\n\n async def gen():\n sent = 0\n while True:\n events = bus.get_events(id)\n for e in events[sent:]:\n yield f\"data: {e.model_dump_json(exclude_none=True)}\\n\\n\"\n sent += 1\n await asyncio.sleep(0.5)\n\n return StreamingResponse(gen(), media_type=\"text/event-stream\")\n```\n\n> **Known limitation.** The default in-memory bus is per-process. If the worker\n> and the API run as separate processes, events published by the worker are not\n> visible to the API. For production, back the bus with a shared broker. The\n> port boundary makes this a one-adapter change.\n", - "objects": {} - } -} \ No newline at end of file diff --git a/standards/mcp/standards/nav.json b/standards/mcp/standards/nav.json deleted file mode 100644 index 6a08db3..0000000 --- a/standards/mcp/standards/nav.json +++ /dev/null @@ -1,57 +0,0 @@ -[ - { - "module": "layering_rules", - "resource": "doc://modules/layering_rules", - "title": "Layering Rules" - }, - { - "module": "composition_root", - "resource": "doc://modules/composition_root", - "title": "Composition Root" - }, - { - "module": "spec_first_routing", - "resource": "doc://modules/spec_first_routing", - "title": "Spec-First Routing" - }, - { - "module": "crud_template", - "resource": "doc://modules/crud_template", - "title": "CRUD Template" - }, - { - "module": "pipeline_scaffold", - "resource": "doc://modules/pipeline_scaffold", - "title": "Pipeline Scaffold" - }, - { - "module": "worker_and_progress", - "resource": "doc://modules/worker_and_progress", - "title": "Worker & Progress" - }, - { - "module": "human_in_loop", - "resource": "doc://modules/human_in_loop", - "title": "Human-in-the-Loop" - }, - { - "module": "infra_adaptors", - "resource": "doc://modules/infra_adaptors", - "title": "Infra Adapters" - }, - { - "module": "genericity_matrix", - "resource": "doc://modules/genericity_matrix", - "title": "Genericity Matrix" - }, - { - "module": "library_registry", - "resource": "doc://modules/library_registry", - "title": "Library Registry" - }, - { - "module": "index", - "resource": "doc://modules/index", - "title": "Standards Reference" - } -] \ No newline at end of file diff --git a/standards/src/lib/13_library_registry.md b/standards/src/lib/13_library_registry.md index 0af34c0..5e8ef84 100644 --- a/standards/src/lib/13_library_registry.md +++ b/standards/src/lib/13_library_registry.md @@ -8,7 +8,7 @@ is a defect. Each library also exposes its own documentation as an MCP server. An AI agent can read a library's full API by connecting to its MCP and reading -`docs://index`, `docs://nav`, and `docs://modules/{module}`. +`doc://index`, `doc://nav`, and `doc://modules/{module}`. --- diff --git a/standards/src/mkdocs.wiki.yml b/standards/src/mkdocs.wiki.yml index 6d1962f..14b8210 100644 --- a/standards/src/mkdocs.wiki.yml +++ b/standards/src/mkdocs.wiki.yml @@ -29,7 +29,11 @@ theme: - search.suggest markdown_extensions: - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.inlinehilite - pymdownx.snippets - admonition @@ -52,8 +56,12 @@ markdown_extensions: plugins: - search +extra_javascript: + - https://unpkg.com/mermaid@10.9.1/dist/mermaid.min.js + nav: - Home: index.md - Overview: 01_overview.md - - Testing: 10_testing.md - - Build Recipe: 11_build_recipe.md \ No newline at end of file + - App Components: 02_components.md + - Build Recipe: 03_build_recipe.md + - Testing & Quality: 04_development/01_testing.md \ No newline at end of file diff --git a/standards/templates/README.md b/standards/src/templates/README.md similarity index 98% rename from standards/templates/README.md rename to standards/src/templates/README.md index 65b18bb..1a7ea62 100644 --- a/standards/templates/README.md +++ b/standards/src/templates/README.md @@ -13,7 +13,7 @@ than delete unused code. The domain is a neutral "item", not an app noun. ## Instantiate a new app - cp -r templates/app apps/ + cp -r src/templates/app apps/ cd apps/ 1. `pyproject.toml` โ€” rename the package; keep the internal library pins. diff --git a/standards/templates/app/.dockerignore b/standards/src/templates/app/.dockerignore similarity index 100% rename from standards/templates/app/.dockerignore rename to standards/src/templates/app/.dockerignore diff --git a/standards/templates/app/.env.example b/standards/src/templates/app/.env.example similarity index 100% rename from standards/templates/app/.env.example rename to standards/src/templates/app/.env.example diff --git a/standards/templates/app/Dockerfile b/standards/src/templates/app/Dockerfile similarity index 100% rename from standards/templates/app/Dockerfile rename to standards/src/templates/app/Dockerfile diff --git a/standards/templates/app/config.py b/standards/src/templates/app/config.py similarity index 100% rename from standards/templates/app/config.py rename to standards/src/templates/app/config.py diff --git a/standards/templates/app/docker-compose.yaml b/standards/src/templates/app/docker-compose.yaml similarity index 100% rename from standards/templates/app/docker-compose.yaml rename to standards/src/templates/app/docker-compose.yaml diff --git a/standards/templates/app/domain/__init__.py b/standards/src/templates/app/domain/__init__.py similarity index 100% rename from standards/templates/app/domain/__init__.py rename to standards/src/templates/app/domain/__init__.py diff --git a/standards/templates/app/domain/item/__init__.py b/standards/src/templates/app/domain/item/__init__.py similarity index 100% rename from standards/templates/app/domain/item/__init__.py rename to standards/src/templates/app/domain/item/__init__.py diff --git a/standards/templates/app/domain/item/models.py b/standards/src/templates/app/domain/item/models.py similarity index 100% rename from standards/templates/app/domain/item/models.py rename to standards/src/templates/app/domain/item/models.py diff --git a/standards/templates/app/domain/item/ports/__init__.py b/standards/src/templates/app/domain/item/ports/__init__.py similarity index 100% rename from standards/templates/app/domain/item/ports/__init__.py rename to standards/src/templates/app/domain/item/ports/__init__.py diff --git a/standards/templates/app/domain/item/ports/db.py b/standards/src/templates/app/domain/item/ports/db.py similarity index 100% rename from standards/templates/app/domain/item/ports/db.py rename to standards/src/templates/app/domain/item/ports/db.py diff --git a/standards/templates/app/domain/item/ports/service.py b/standards/src/templates/app/domain/item/ports/service.py similarity index 100% rename from standards/templates/app/domain/item/ports/service.py rename to standards/src/templates/app/domain/item/ports/service.py diff --git a/standards/templates/app/domain/job/__init__.py b/standards/src/templates/app/domain/job/__init__.py similarity index 100% rename from standards/templates/app/domain/job/__init__.py rename to standards/src/templates/app/domain/job/__init__.py diff --git a/standards/templates/app/domain/job/models.py b/standards/src/templates/app/domain/job/models.py similarity index 100% rename from standards/templates/app/domain/job/models.py rename to standards/src/templates/app/domain/job/models.py diff --git a/standards/templates/app/domain/job/ports/__init__.py b/standards/src/templates/app/domain/job/ports/__init__.py similarity index 100% rename from standards/templates/app/domain/job/ports/__init__.py rename to standards/src/templates/app/domain/job/ports/__init__.py diff --git a/standards/templates/app/domain/job/ports/db.py b/standards/src/templates/app/domain/job/ports/db.py similarity index 100% rename from standards/templates/app/domain/job/ports/db.py rename to standards/src/templates/app/domain/job/ports/db.py diff --git a/standards/templates/app/domain/job/ports/service.py b/standards/src/templates/app/domain/job/ports/service.py similarity index 100% rename from standards/templates/app/domain/job/ports/service.py rename to standards/src/templates/app/domain/job/ports/service.py diff --git a/standards/templates/app/infra/__init__.py b/standards/src/templates/app/infra/__init__.py similarity index 100% rename from standards/templates/app/infra/__init__.py rename to standards/src/templates/app/infra/__init__.py diff --git a/standards/templates/app/infra/adapters/__init__.py b/standards/src/templates/app/infra/adapters/__init__.py similarity index 100% rename from standards/templates/app/infra/adapters/__init__.py rename to standards/src/templates/app/infra/adapters/__init__.py diff --git a/standards/templates/app/infra/adapters/service.py b/standards/src/templates/app/infra/adapters/service.py similarity index 100% rename from standards/templates/app/infra/adapters/service.py rename to standards/src/templates/app/infra/adapters/service.py diff --git a/standards/templates/app/infra/db/__init__.py b/standards/src/templates/app/infra/db/__init__.py similarity index 100% rename from standards/templates/app/infra/db/__init__.py rename to standards/src/templates/app/infra/db/__init__.py diff --git a/standards/templates/app/infra/db/repos/__init__.py b/standards/src/templates/app/infra/db/repos/__init__.py similarity index 100% rename from standards/templates/app/infra/db/repos/__init__.py rename to standards/src/templates/app/infra/db/repos/__init__.py diff --git a/standards/templates/app/infra/db/repos/mongo.py b/standards/src/templates/app/infra/db/repos/mongo.py similarity index 100% rename from standards/templates/app/infra/db/repos/mongo.py rename to standards/src/templates/app/infra/db/repos/mongo.py diff --git a/standards/templates/app/infra/db/setup.py b/standards/src/templates/app/infra/db/setup.py similarity index 100% rename from standards/templates/app/infra/db/setup.py rename to standards/src/templates/app/infra/db/setup.py diff --git a/standards/templates/app/init.py b/standards/src/templates/app/init.py similarity index 100% rename from standards/templates/app/init.py rename to standards/src/templates/app/init.py diff --git a/standards/templates/app/interfaces/__init__.py b/standards/src/templates/app/interfaces/__init__.py similarity index 100% rename from standards/templates/app/interfaces/__init__.py rename to standards/src/templates/app/interfaces/__init__.py diff --git a/standards/templates/app/interfaces/item/__init__.py b/standards/src/templates/app/interfaces/item/__init__.py similarity index 100% rename from standards/templates/app/interfaces/item/__init__.py rename to standards/src/templates/app/interfaces/item/__init__.py diff --git a/standards/templates/app/interfaces/item/routes.py b/standards/src/templates/app/interfaces/item/routes.py similarity index 100% rename from standards/templates/app/interfaces/item/routes.py rename to standards/src/templates/app/interfaces/item/routes.py diff --git a/standards/templates/app/interfaces/job/__init__.py b/standards/src/templates/app/interfaces/job/__init__.py similarity index 100% rename from standards/templates/app/interfaces/job/__init__.py rename to standards/src/templates/app/interfaces/job/__init__.py diff --git a/standards/templates/app/interfaces/job/routes.py b/standards/src/templates/app/interfaces/job/routes.py similarity index 100% rename from standards/templates/app/interfaces/job/routes.py rename to standards/src/templates/app/interfaces/job/routes.py diff --git a/standards/templates/app/interfaces/openapi.py b/standards/src/templates/app/interfaces/openapi.py similarity index 100% rename from standards/templates/app/interfaces/openapi.py rename to standards/src/templates/app/interfaces/openapi.py diff --git a/standards/templates/app/main.py b/standards/src/templates/app/main.py similarity index 100% rename from standards/templates/app/main.py rename to standards/src/templates/app/main.py diff --git a/standards/templates/app/openapi.yaml b/standards/src/templates/app/openapi.yaml similarity index 100% rename from standards/templates/app/openapi.yaml rename to standards/src/templates/app/openapi.yaml diff --git a/standards/templates/app/pipelines/__init__.py b/standards/src/templates/app/pipelines/__init__.py similarity index 100% rename from standards/templates/app/pipelines/__init__.py rename to standards/src/templates/app/pipelines/__init__.py diff --git a/standards/templates/app/pipelines/item_sync.py b/standards/src/templates/app/pipelines/item_sync.py similarity index 100% rename from standards/templates/app/pipelines/item_sync.py rename to standards/src/templates/app/pipelines/item_sync.py diff --git a/standards/templates/app/pyproject.toml b/standards/src/templates/app/pyproject.toml similarity index 100% rename from standards/templates/app/pyproject.toml rename to standards/src/templates/app/pyproject.toml diff --git a/standards/templates/app/tests/__init__.py b/standards/src/templates/app/tests/__init__.py similarity index 100% rename from standards/templates/app/tests/__init__.py rename to standards/src/templates/app/tests/__init__.py diff --git a/standards/templates/app/tests/integration/__init__.py b/standards/src/templates/app/tests/integration/__init__.py similarity index 100% rename from standards/templates/app/tests/integration/__init__.py rename to standards/src/templates/app/tests/integration/__init__.py diff --git a/standards/templates/app/tests/integration/test_item_routes.py b/standards/src/templates/app/tests/integration/test_item_routes.py similarity index 100% rename from standards/templates/app/tests/integration/test_item_routes.py rename to standards/src/templates/app/tests/integration/test_item_routes.py diff --git a/standards/templates/app/tests/unit/__init__.py b/standards/src/templates/app/tests/unit/__init__.py similarity index 100% rename from standards/templates/app/tests/unit/__init__.py rename to standards/src/templates/app/tests/unit/__init__.py diff --git a/standards/templates/app/tests/unit/test_item_models.py b/standards/src/templates/app/tests/unit/test_item_models.py similarity index 100% rename from standards/templates/app/tests/unit/test_item_models.py rename to standards/src/templates/app/tests/unit/test_item_models.py diff --git a/standards/templates/app/tests/unit/test_item_sync.py b/standards/src/templates/app/tests/unit/test_item_sync.py similarity index 100% rename from standards/templates/app/tests/unit/test_item_sync.py rename to standards/src/templates/app/tests/unit/test_item_sync.py diff --git a/standards/templates/app/worker.py b/standards/src/templates/app/worker.py similarity index 100% rename from standards/templates/app/worker.py rename to standards/src/templates/app/worker.py diff --git a/standards/src/wiki/01_overview.md b/standards/src/wiki/01_overview.md index dd0d46a..0bb90d2 100644 --- a/standards/src/wiki/01_overview.md +++ b/standards/src/wiki/01_overview.md @@ -1,54 +1,99 @@ # Overview -## Summary - -The blueprint is a **hexagonal (ports and adapters) application** with a +The blueprint is a **hexagonal (ports & adapters) application** with a **spec-first** web edge and a **single composition root**. It favors explicit -wiring over implicit magic, and it prefers existing internal libraries over -hand-written infrastructure. +wiring over implicit magic and existing internal libraries over hand-written +infrastructure. Start here to build the mental model; the enforceable details +live in the reference pages. --- -## The three layers +## ๐Ÿ—๏ธ Architecture -``` -interfaces/ โ† HTTP edge: thin handlers bound to an OpenAPI operationId -domain/ โ† business core: models, ports (interfaces), services, orchestrators -infra/ โ† adapters: databases, caches, external clients, hardware +```mermaid +flowchart LR + subgraph EDGE["interfaces/ โ€” HTTP edge"] + OAS["openapi.yaml
operationId contract"] + RT["/routes.py
thin handlers"] + end + + subgraph CORE["domain/ โ€” pure business core"] + MD["models"] + PT["ports (abstract)"] + SV["services"] + end + + subgraph INFRA["infra/ โ€” adapters"] + RP["mongo repos (mongo-ops)"] + AD["cache ยท stores ยท llm ยท mail"] + DB[("MongoDB")] + end + + CR["init.py
composition root"] + + OAS --> RT + RT --> SV + SV --> PT + SV --> RP + PT -. depends on .- RP + CR --- RT + CR --- SV + CR --- RP + RP --> DB + AD --- CR ``` -Plus two optional layers: +The rule holds at every edge: code in an outer ring may depend on a ring closer +to the core, never the reverse. `domain/` never imports a web framework or a +database driver. -``` -pipelines/ โ† data-driven transformation graphs (extraction, enrichment) -scripts/ โ† operator scripts that reuse the composition root -``` +> **Legend** โ€” solid arrows are compile-time calls; dotted arrows are the +> port/adapter contract, resolved by the composition root at startup. -## Core principles +--- -1. **One-way dependencies.** `interfaces -> domain -> infra` is allowed; - the reverse is forbidden. The domain never imports a framework it does not - own. -2. **Ports in the domain, adapters in infra.** A service depends on an - abstract port; the composition root injects a concrete adapter. +## ๐Ÿ”„ The one-way dependency rule + +| Ring | May import | Never imports | +|-------------|------------------------------------|----------------------------------| +| `interfaces/` | `domain`, framework glue (glue only) | business logic, `infra` directly | +| `domain/` | nothing but its own ports | web frameworks, drivers, `infra` | +| `infra/` | `domain` ports, drivers, libraries | โ€” | +| `init.py` | everything (the wiring point) | โ€” | + +--- + +## ๐Ÿงญ Core principles + +1. **One-way dependencies.** `interfaces -> domain -> infra` is allowed; the + reverse is forbidden. +2. **Ports in the domain, adapters in infra.** A service depends on an abstract + port; the composition root injects a concrete adapter. 3. **One composition root.** Nothing constructs its own dependencies. All - wiring lives in one module (`init.py`) and is exposed as a flat resource - registry. -4. **Spec-first.** The HTTP contract is declared in an OpenAPI document. Route + wiring lives in one module (`init.py`) exposed as a flat resource registry. +4. **Spec-first.** The HTTP contract is declared in `openapi.yaml`; route handlers are looked up by `operationId`. There are no decorator routers. -5. **Reuse the libraries.** Pipeline orchestration, DI, Mongo repositories, - ingestion, auth, and docs are provided by internal libraries. Do not - reimplement them โ€” see the library registry. +5. **Reuse the libraries.** Orchestration, DI, Mongo repositories, ingestion, + auth, and docs come from internal libraries โ€” see the [Library + Registry](../lib/13_library_registry/). 6. **Explicit state machines.** Long-running work moves through explicit, durable statuses and can pause for human input. -## Runtime processes +--- -A standard app may run as up to three processes that share the same -composition root: +## โš™๏ธ Runtime processes -| Process | Entry point | Responsibility | -|---|---|---| -| API | `main.py` | serve HTTP; start/stop resources via lifespan | -| Worker | `worker.py` | poll durable jobs, run pipelines, publish progress | -| Scripts | `python -m scripts.` | one-shot operator tasks | +A standard app may run as up to three processes that share the same composition +root: + +| Process | Entry point | Responsibility | +|---------|------------------------|-------------------------------------------| +| API | `main.py` | serve HTTP; start/stop resources via lifespan | +| Worker | `worker.py` | poll durable jobs, run pipelines, publish progress | +| Scripts | `python -m scripts.` | one-shot operator tasks | + +--- + +## โžก๏ธ Read Next + +- [02 โ€“ App Components](02_components.md) ยท [03 โ€“ Build Recipe](03_build_recipe.md) \ No newline at end of file diff --git a/standards/src/wiki/02_components.md b/standards/src/wiki/02_components.md new file mode 100644 index 0000000..9dfcfe4 --- /dev/null +++ b/standards/src/wiki/02_components.md @@ -0,0 +1,137 @@ +# App Components + +What ships in every standard app, and how the pieces relate. This is the +blueprint's component model โ€” `templates/app/` is a runnable instance of it. + +--- + +## ๐Ÿ—‚๏ธ Component model + +```mermaid +flowchart LR + subgraph EDGE["interfaces/ โ€” HTTP edge"] + OAS["openapi.yaml"] + RU["resource routes"] + JU["job routes"] + end + + subgraph CORE["domain/ โ€” business core"] + RI[" ยท models ยท ports ยท services"] + JI[" ยท models ยท ports ยท services"] + PI["pipelines/"] + end + + subgraph INFRA["infra/ โ€” adapters"] + DB[("MongoDB repos")] + CA["redis cache"] + ST["stores ยท progress bus"] + LL["llm clients"] + end + + W["worker.py"] + CR["init.py โ€” composition root"] + + OAS --> RU + OAS --> JU + RU --> RI + JU --> JI + JI --> PI + PI --> DB + PI --> LL + RI --> CA + RI --> DB + JI --> ST + W --> JI + W --> PI + + CR === RU + CR === JU + CR === RI + CR === JI + CR === PI + CR === W + CR === DB + CR === CA + CR === ST +``` + +> **Legend** โ€” solid arrows are direct calls; thick links are wires created by +> the composition root. Every component is injected, never self-constructed. + +--- + +## ๐Ÿ“ฆ Component roles + +### 1. `interfaces/` โ€” the edge + +Thin handlers bound to `openapi.yaml` `operationId`s. They parse the request, +call a service, and shape the response. No business rules, no I/O. + +- `interfaces/openapi.py` re-exports every route module so the OpenAPI-first + app can resolve handlers. +- Routes may mount the `cache` decorator on read endpoints. + +### 2. `domain/` โ€” the core + +Pure models, **ports** (abstract contracts), and services (use cases). No +framework imports, no drivers. + +- `/models.py` โ€” dataclasses, enums, document shapes. +- `/ports/db.py` โ€” repo contract; `/ports/service.py` โ€” + service contract. +- Services compose ports and enforce the business rules; the composition root + injects the implementations. + +### 3. `infra/` โ€” the adapters + +Concrete implementations of the domain ports plus cross-cutting infrastructure: + +- `db/` โ€” mongo-ops repositories and the connection lifecycle. +- `adapters/` โ€” cache store, progress bus, storages, fetchers (library-backed). +- Anything else the domain needs, implemented against its ports. + +### 4. `pipelines/` โ€” data-driven families *(optional)* + +Extraction and enrichment families, each a deterministic step chain +(`abc.py` / `impl.py` / `pipeline.yaml` / `resources/`). Roots are registered +in the pattern registry and driven either by a worker job or a CLI. + +### 5. `worker.py` โ€” the job worker *(optional)* + +A separate process that polls durable job collections, runs pipeline roots, and +publishes progress to the progress bus. It shares the same composition root as +the API. + +### 6. `init.py` โ€” the composition root + +The only place anything is constructed: leaf repositories โ†’ dependent +repositories โ†’ services โ†’ adapters. Exposes `get_svc` overloads and starts / +stops resources on the app lifespan. + +--- + +## ๐Ÿ”— How the request flows + +```mermaid +sequenceDiagram + participant C as Client + participant R as route (interfaces) + participant S as service (domain) + participant P as repo (infra) + participant DB as MongoDB + + C->>R: POST /items (operationId) + R->>S: svc.create(payload) + S->>P: repo.insert(doc) + P->>DB: insert + DB-->>P: ack + P-->>S: doc + S-->>R: result + R-->>C: 200 JSON +``` + +--- + +## Related + +- [01 โ€“ Overview](01_overview.md) ยท [03 โ€“ Build Recipe](03_build_recipe.md) ยท [04 โ€“ Testing & Quality](04_development/01_testing.md) \ No newline at end of file diff --git a/standards/src/wiki/03_build_recipe.md b/standards/src/wiki/03_build_recipe.md new file mode 100644 index 0000000..9021a59 --- /dev/null +++ b/standards/src/wiki/03_build_recipe.md @@ -0,0 +1,124 @@ +# Build Recipe + +The ordered, deterministic checklist for generating a new application of the +standard shape. Every step is fixed; only steps 2 and 3 need app-specific +authoring. Give an AI agent the copyable prompt below (or read it from the +`standards` MCP bundle as `doc://modules/build_recipe`) and it can produce a +complete app in one pass. + +--- + +## ๐Ÿš€ Quick start + +```bash +# Copy the runnable skeleton (from the standards docs checkout) +cp -r /standards/src/templates/app apps/ +cd apps/ +``` + +Then follow the numbered steps in the prompt. + +--- + +## ๐Ÿ“‹ Agent prompt (copyable) + +````markdown +# Build a new application from the Aetos Standards blueprint + +You are coding a complete new backend application conforming to the Aetos +standard app blueprint. Execute every step in order. Only steps 2 and 3 use the +app-specific details provided by the user; everything else is deterministic. + +Before writing any code, connect to the `standards` MCP server and read: +- `doc://nav` (the module list) +- `doc://modules/layering_rules` (the dependency rules) +- `doc://modules/genericity_matrix` (what is generic vs app-specific) +Then call `find_library` with capabilities like `db`, `di`, `routing`, +`ingestion`, `auth`, `docs`, `pipeline`, and note the libraries that already +exist. Never reimplement one. + +1. **Scaffold** โ€” copy `src/templates/app/` into `apps//`. Keep the + package tree: `domain/`, `infra/`, `interfaces/`, `pipelines/`, `tests/`, + `scripts/`, plus `pyproject.toml`, `openapi.yaml`, `config.py`, `init.py`, + `main.py`, `worker.py`, `Dockerfile`, `docker-compose.yaml`, `.env.example`. + +2. **Author the spec (app-specific)** โ€” write `openapi.yaml`: resources, + schemas, `operationId`s, security, and extension metadata. This is the + contract and drives all routing. + +3. **Write each CRUD slice (app-specific)** โ€” per resource create the + four-file slice: `models.py`, `ports/db.py`, `ports/service.py`, and + optional `converters.py`. Follow the CRUD template. + +4. **Wire the composition root** โ€” in `init.py`, build leaf repositories, then + dependent repositories, then services; add infra adapters (db, cache, + stores, progress bus, fetchers). Expose `get_svc` overloads. + +5. **Add routes** โ€” create `interfaces//routes.py` thin handlers + bound to `operationId`s and re-export them in `interfaces/openapi.py`. Add + the cache decorator to read endpoints. + +6. **Bootstrap app and worker** โ€” confirm `main.py` runs the OpenAPI-first app + with its lifespan. If the app has long-running jobs, adapt `worker.py` to + poll the job collection, run the pipeline, and publish progress. + +7. **Add pipeline families (only if data-driven)** โ€” create + `pipelines//` with `abc.py`, `impl.py`, `pipeline.yaml`, + `resources/`, and register the root. Add fixtures and a parity test. + +8. **Add infrastructure only where needed** โ€” prefer a library-backed adapter. + Never hand-write what an internal library already provides. + +9. **Tests** โ€” integration tests with fakes at the HTTP edge; unit tests for + domain and pipeline logic; a config-consistency test for rule-driven + pipelines. + +10. **Scripts and seeds** โ€” operator scripts under `scripts/` using + `init()` / `get_svc()` / `shutdown()`. + +11. **Docs** โ€” author GSDFC docstrings, generate `.pyi` stubs, mark the package + `py.typed`, and build the docs. + +**Definition of done:** +- `python -c "import main"` succeeds. +- `pytest` passes. +- The app serves every operation in the spec; the worker drains its queue. +- No domain module imports a web framework. +- No infrastructure was reimplemented that a library already provides. +```` + +--- + +## ๐Ÿงญ Where the steps land + +The steps map onto the layers as follows: + +| Step | Package / file | Depends on | +|---------------------|-----------------------------------|---------------------------------------------| +| 1 ยท Scaffold | whole `templates/app/` | โ€” | +| 2 ยท Spec | `openapi.yaml` | โ€” | +| 3 ยท CRUD slices | `domain//` | spec, CRUD template | +| 4 ยท Wiring | `init.py` | every slice | +| 5 ยท Routes | `interfaces//` | services, openapi route registry | +| 6 ยท Bootstrap | `main.py`, `worker.py` | composition root | +| 7 ยท Pipelines | `pipelines//` | library pipeline engine | +| 8 ยท Infra | `infra/` | domain ports | +| 9 ยท Tests | `tests/` | app + fixtures | +| 10 ยท Scripts | `scripts/` | composition root | +| 11 ยท Docs | docstrings + docs build | code, Doc Forge | + +--- + +## โš ๏ธ Definition of done + +- `python -c "import main"` succeeds. +- `pytest` passes. +- The app serves the spec's operations; the worker drains its queue. +- No domain module imports a web framework. +- No infrastructure was reimplemented that a library already provides. + +--- + +## Related + +- [01 โ€“ Overview](01_overview.md) ยท [02 โ€“ App Components](02_components.md) ยท [04 โ€“ Testing & Quality](04_development/01_testing.md) \ No newline at end of file diff --git a/standards/src/wiki/10_testing.md b/standards/src/wiki/04_development/01_testing.md similarity index 63% rename from standards/src/wiki/10_testing.md rename to standards/src/wiki/04_development/01_testing.md index e6d8e7e..b6af74a 100644 --- a/standards/src/wiki/10_testing.md +++ b/standards/src/wiki/04_development/01_testing.md @@ -1,13 +1,12 @@ -# Testing +# Testing & Quality Gates -## Summary - -Two test suites: **integration** (the HTTP edge with fakes) and **unit** (pure -domain and pipeline logic). Both run under pytest with asyncio auto mode. +Two suites run under pytest with asyncio auto mode: **integration** (the HTTP +edge against fakes) and **unit** (pure domain and pipeline logic). Finish a +blueprint app only when every gate below passes. --- -## Layout +## ๐Ÿ—‚๏ธ Layout ``` tests/ @@ -16,7 +15,9 @@ tests/ conftest.py # fixtures ``` -## Integration tests +--- + +## ๐Ÿš€ Integration tests - Build the app with `TestClient`. - Replace `app.state.resources` with fake repositories/services so no database @@ -35,22 +36,46 @@ def test_create_item(): assert r.status_code == 200 ``` -## Unit tests +--- + +## ๐Ÿงช Unit tests - Domain models and converters: pure, no I/O. - Pipeline stages: feed a fixture state, assert the transformed state. - Assembled pipeline roots: **fixture parity** โ€” replay a stored fixture input and compare against a golden output file. -## Config-consistency tests +--- + +## ๐Ÿ” Config-consistency tests For rule-driven pipelines, add a test that cross-references rules against the declared pattern registry: every flag emitted must be defined; every defined flag must be reachable. This catches drift without running the pipeline. -## Conventions +--- + +## โœ… Quality gates + +A **definition of done** for any standard app: + +- `python -c "import main"` succeeds. +- `pytest` passes. +- The app serves the spec's operations; the worker drains its queue. +- No domain module imports a web framework. +- No infrastructure was reimplemented that a library already provides. + +--- + +## ๐Ÿ’ก Conventions - `asyncio_mode = "auto"`; no manual event loops. - Fixtures are checked into the repo (inputs + golden outputs). - Tests skip (not fail) when a required fixture is absent. - Do not mock the domain; mock only the infrastructure edge. + +--- + +## Related + +- [02 โ€“ App Components](02_components.md) ยท [03 โ€“ Build Recipe](03_build_recipe.md) ยท [Overview](../01_overview.md) \ No newline at end of file diff --git a/standards/src/wiki/11_build_recipe.md b/standards/src/wiki/11_build_recipe.md deleted file mode 100644 index 069e933..0000000 --- a/standards/src/wiki/11_build_recipe.md +++ /dev/null @@ -1,82 +0,0 @@ -# Build Recipe - -## Summary - -Follow this ordered checklist to generate a new application of the standard -shape. Every step is deterministic; only steps 2 and 7 require app-specific -authoring. - ---- - -## 0. Query the registry - -Before writing anything, read the **library registry** and note which internal -libraries provide DB, DI, routing, ingestion, auth, and docs. Do not -reimplement them. - -## 1. Scaffold the repository - -Copy the `templates/` skeleton. It provides: `pyproject.toml`, `openapi.yaml`, -`main.py`, `init.py`, `worker.py`, `config.py`, `Dockerfile`, -`docker-compose.yaml`, `.env.example`, and the package tree -(`domain/ infra/ interfaces/ pipelines/ tests/ scripts/`). - -## 2. Author the spec *(app-specific)* - -Write `openapi.yaml`: resources, schemas, `operationId`s, security, and any -extension metadata. This is the contract. - -## 3. Write each CRUD slice *(app-specific)* - -For each resource, create the four-file slice (see **CRUD Template**): -`models.py`, `ports/db.py`, `ports/service.py`, optional `converters.py`. - -## 4. Wire the composition root - -In `init.py`: build leaf repositories, then dependent repositories, then -services; add infrastructure adaptors (cache store, progress bus, stores, -fetchers). Add `get_svc` overloads. - -## 5. Add routes - -Create `interfaces//routes.py` (thin handlers) and re-export them in -`interfaces/openapi.py`. Add a cache decorator to read endpoints. - -## 6. Bootstrap the app and worker - -Confirm `main.py` wires the OpenAPI-first app with the lifespan. If the app has -long-running jobs, adapt `worker.py` to poll the relevant job collection. - -## 7. Add a pipeline family *(only if data-driven)* - -Create `pipelines//` with `abc.py`, `impl.py`, `pipeline.yaml`, -`resources/`, and register the root in `registry.py`. Add fixtures and a parity -test. - -## 8. Add infrastructure as needed - -Only where the standard adaptors do not cover the need. Prefer library-backed -adapters. - -## 9. Tests - -Add integration tests with fakes and unit tests for domain/pipeline logic; add -a config-consistency test if using rule-driven pipelines. - -## 10. Scripts and seeds - -Add operator scripts under `scripts/` that use `init()` / `get_svc()` / -`shutdown()`. - -## 11. Docs - -Author GSDFC docstrings, generate the type stubs (`.pyi`), mark the package -`py.typed`, and build docs. - -## Definition of done - -- `python -c "import main"` succeeds. -- `pytest` passes. -- The app serves the spec's operations; the worker drains its queue. -- No domain module imports a web framework. -- No infrastructure was reimplemented that a library already provides. diff --git a/standards/src/wiki/index.md b/standards/src/wiki/index.md index 98589c8..b01d3f9 100644 --- a/standards/src/wiki/index.md +++ b/standards/src/wiki/index.md @@ -1,45 +1,68 @@ -# The Aetos Standard App Blueprint +# ๐Ÿงฉ Standards โ€” the Aetos App Blueprint -A single, reusable architecture for backend applications in the Aetos ecosystem. +One reusable architecture for every backend application in the Aetos ecosystem: +a spec-first HTTP edge, a pure domain core, a pluggable infrastructure layer, +and a single composition root. -Every application built with this blueprint has the same shape: +Fill in **two axes only** and a new app is done: your **resource model**, and +(if needed) your **pipeline family**. The blueprint itself never names a +business domain โ€” expenses, health metrics, inventory, and analytics are all +the same shape underneath. -- a **spec-first HTTP layer** driven by an OpenAPI document; -- a **domain layer** of pure models, ports, and services; -- an **infrastructure layer** of adapters (MongoDB, cache, workers, LLM, mail); -- a **composition root** that wires everything in exactly one place; -- an optional **pipeline layer** for data-driven extraction/enrichment; -- an optional **background worker** that runs long-running jobs and reports progress. +> **Doc model:** this wiki is written for humans โ€” how-to guides, the build +> recipe, and quality gates. The enforceable rules (layering, composition root, +> CRUD template, pipeline scaffold) and the library registry live in the +> **reference** pages, mirrored by the machine-readable `standards` MCP bundle. +> Query `find_library` before writing any infrastructure. -> This blueprints is deliberately **app-agnostic**. It never names a concrete -> business domain. Any application โ€” expenses, health metrics, inventory, -> analytics โ€” is produced by filling in **two axes only**: the resource model -> and (if needed) the pipeline family. +--- -## How to read this +## ๐Ÿš€ Key Features -| Page | What it covers | -|---|---| -| Overview | The philosophy and the one-way dependency rule | -| Layering Rules | Which package may import which | -| Composition Root | The single wiring point | -| Spec-First Routing | OpenAPI as the source of truth | -| CRUD Template | The canonical resource slice | -| Pipeline Scaffold | Data-driven processing families | -| Worker & Progress | Async jobs, status machine, streaming progress | -| Human-in-the-Loop | Pausing a job to ask a question | -| Infra Adapters | MongoDB, cache, stores, LLM, mail | -| Testing | Integration + unit conventions | -| Build Recipe | The ordered checklist to generate a new app | -| Genericity Matrix | What is reusable vs what you must write | +- ๐Ÿงญ **Spec-first routing** โ€” the OpenAPI document, not decorator routers, is + the source of truth (operationId lookup) +- ๐Ÿ—๏ธ **One-way layering** โ€” `interfaces โ†’ domain โ†’ infra`, enforced +- ๐Ÿ”Œ **Single composition root** โ€” every dependency is wired in exactly one place +- ๐Ÿงฉ **Generic, not app-specific** โ€” reusable rules versus two authoring axes +- ๐Ÿ“š **Library-backed** โ€” DB, DI, pipelines, ingestion, auth, and docs come from + internal libraries, never hand-written +- ๐Ÿšฆ **Explicit state machines** โ€” durable job statuses, progress, and + human-in-the-loop pauses -## For AI agents +--- -This content is served as MCP resources under `docs://index`, `docs://nav`, -and `docs://modules/{module}`. To build a new app: +## ๐Ÿ“ Documentation Structure -1. Read `build_recipe` first. -2. Read `genericity_matrix` to know what you must author. -3. Query the **library registry** (`find_library`) before writing any - infrastructure yourself. -4. Copy the `templates/` skeleton and fill the two app-specific axes. +| Section | Description | +|-----------------------------------------------------|--------------------------------------------------------| +| [Overview](01_overview.md) | Philosophy, the one-way dependency rule, architecture | +| [App Components](02_components.md) | The component model and how the pieces wire together | +| [Build Recipe](03_build_recipe.md) | The ordered, agentic checklist for a new app | +| [Testing & Quality](04_development/01_testing.md) | Test layout and acceptance gates | + +**Reference** (separate site, mirrors the MCP bundle): the enforceable +rules โ€” [Layering](../lib/02_layering_rules/), [Composition +Root](../lib/03_composition_root/), [Spec-First +Routing](../lib/04_spec_first_routing/), [CRUD +Template](../lib/05_crud_template/), [Pipeline +Scaffold](../lib/06_pipeline_scaffold/), [Worker & +Progress](../lib/07_worker_and_progress/), [Human-in-the-Loop](../lib/08_human_in_loop/), +[Infra Adapters](../lib/09_infra_adaptors/), [Genericity +Matrix](../lib/12_genericity_matrix/), [Library +Registry](../lib/13_library_registry/). + +--- + +## ๐Ÿ”— Related Resources + +- **MCP bundle** โ€” `standards` server on `:8009`: read `doc://index`, + `doc://nav`, `doc://modules/{module}`; call `find_library(capability)` +- **Library registry** โ€” the catalog of internal libraries behind every app + (`libraries.json`, served as `registry://libraries`) +- **Template** โ€” the runnable skeleton at `standards/src/templates/app/` +- **Copy the repo's `AGENTS.md`-style build prompt** from + [Build Recipe](03_build_recipe.md) + +--- + +ยฉ Aetoskia Internal โ€” `standards` \ No newline at end of file diff --git a/standards/wiki/01_overview/index.html b/standards/wiki/01_overview/index.html index 56ed155..29d83c9 100644 --- a/standards/wiki/01_overview/index.html +++ b/standards/wiki/01_overview/index.html @@ -14,7 +14,7 @@ - + @@ -297,18 +297,18 @@ @@ -411,18 +442,18 @@ 
1
-2
-3
interfaces/   โ† HTTP edge: thin handlers bound to an OpenAPI operationId
-domain/       โ† business core: models, ports (interfaces), services, orchestrators
-infra/        โ† adapters: databases, caches, external clients, hardware
-
-

Plus two optional layers:

-
1
-2
pipelines/    โ† data-driven transformation graphs (extraction, enrichment)
-scripts/      โ† operator scripts that reuse the composition root
-
-

Core principles

+

๐Ÿ—๏ธ Architecture

+
flowchart LR
+    subgraph EDGE["interfaces/ โ€” HTTP edge"]
+        OAS["openapi.yaml<br/>operationId contract"]
+        RT["<resource>/routes.py<br/>thin handlers"]
+    end
+
+    subgraph CORE["domain/ โ€” pure business core"]
+        MD["models"]
+        PT["ports (abstract)"]
+        SV["services"]
+    end
+
+    subgraph INFRA["infra/ โ€” adapters"]
+        RP["mongo repos (mongo-ops)"]
+        AD["cache ยท stores ยท llm ยท mail"]
+        DB[("MongoDB")]
+    end
+
+    CR["init.py<br/>composition root"]
+
+    OAS --> RT
+    RT --> SV
+    SV --> PT
+    SV --> RP
+    PT -. depends on .- RP
+    CR --- RT
+    CR --- SV
+    CR --- RP
+    RP --> DB
+    AD --- CR
+

The rule holds at every edge: code in an outer ring may depend on a ring closer +to the core, never the reverse. domain/ never imports a web framework or a +database driver.

+
+

Legend โ€” solid arrows are compile-time calls; dotted arrows are the +port/adapter contract, resolved by the composition root at startup.

+
+
+

๐Ÿ”„ The one-way dependency rule

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
RingMay importNever imports
interfaces/domain, framework glue (glue only)business logic, infra directly
domain/nothing but its own portsweb frameworks, drivers, infra
infra/domain ports, drivers, librariesโ€”
init.pyeverything (the wiring point)โ€”
+
+

๐Ÿงญ Core principles

    -
  1. One-way dependencies. interfaces -> domain -> infra is allowed; - the reverse is forbidden. The domain never imports a framework it does not - own.
    2. -
  2. Ports in the domain, adapters in infra. A service depends on an - abstract port; the composition root injects a concrete adapter.
    3. +
  3. One-way dependencies. interfaces -> domain -> infra is allowed; the + reverse is forbidden.
    4. +
  4. Ports in the domain, adapters in infra. A service depends on an abstract + port; the composition root injects a concrete adapter.
  5. One composition root. Nothing constructs its own dependencies. All - wiring lives in one module (init.py) and is exposed as a flat resource - registry.
    6. -
  6. Spec-first. The HTTP contract is declared in an OpenAPI document. Route + wiring lives in one module (init.py) exposed as a flat resource registry.
    7. +
  7. Spec-first. The HTTP contract is declared in openapi.yaml; route handlers are looked up by operationId. There are no decorator routers.
    8. -
  8. Reuse the libraries. Pipeline orchestration, DI, Mongo repositories, - ingestion, auth, and docs are provided by internal libraries. Do not - reimplement them โ€” see the library registry.
    9. +
  9. Reuse the libraries. Orchestration, DI, Mongo repositories, ingestion, + auth, and docs come from internal libraries โ€” see the Library + Registry.
  10. Explicit state machines. Long-running work moves through explicit, durable statuses and can pause for human input.
-

Runtime processes

-

A standard app may run as up to three processes that share the same -composition root:

+
+

โš™๏ธ Runtime processes

+

A standard app may run as up to three processes that share the same composition +root:

@@ -528,6 +627,11 @@ composition root:

+
+ + @@ -589,6 +693,8 @@ composition root:

+ + \ No newline at end of file diff --git a/standards/wiki/11_build_recipe/index.html b/standards/wiki/02_components/index.html similarity index 65% rename from standards/wiki/11_build_recipe/index.html rename to standards/wiki/02_components/index.html index a82897e..5fca5e7 100644 --- a/standards/wiki/11_build_recipe/index.html +++ b/standards/wiki/02_components/index.html @@ -11,16 +11,18 @@ - + + + - Build Recipe - Aetos Standards โ€” Wiki + App Components - Aetos Standards โ€” Wiki @@ -72,7 +74,7 @@
- + Skip to content @@ -108,7 +110,7 @@
- Build Recipe + App Components
@@ -264,28 +266,6 @@ - - -
  • - - - - - - Testing - - - - - -
    • - - - - - - - @@ -303,7 +283,7 @@ - Build Recipe + App Components @@ -316,7 +296,7 @@ - Build Recipe + App Components @@ -339,126 +319,96 @@
    @@ -499,126 +493,96 @@
    • - + - Summary + ๐Ÿ—‚๏ธ Component model
    • - + - 0. Query the registry + ๐Ÿ“ฆ Component roles + + + + + +
      • + +
    • + + + ๐Ÿ”— How the request flows
    • - + - 1. Scaffold the repository - - - -
      • - -
    • - - - 2. Author the spec (app-specific) - - - -
      • - -
    • - - - 3. Write each CRUD slice (app-specific) - - - -
      • - -
    • - - - 4. Wire the composition root - - - -
      • - -
    • - - - 5. Add routes - - - -
      • - -
    • - - - 6. Bootstrap the app and worker - - - -
      • - -
    • - - - 7. Add a pipeline family (only if data-driven) - - - -
      • - -
    • - - - 8. Add infrastructure as needed - - - -
      • - -
    • - - - 9. Tests - - - -
      • - -
    • - - - 10. Scripts and seeds - - - -
      • - -
    • - - - 11. Docs - - - -
      • - -
    • - - - Definition of done + Related @@ -640,60 +604,120 @@ -

      Build Recipe

      -

      Summary

      -

      Follow this ordered checklist to generate a new application of the standard -shape. Every step is deterministic; only steps 2 and 7 require app-specific -authoring.

      +

      App Components

      +

      What ships in every standard app, and how the pieces relate. This is the +blueprint's component model โ€” templates/app/ is a runnable instance of it.

      -

      0. Query the registry

      -

      Before writing anything, read the library registry and note which internal -libraries provide DB, DI, routing, ingestion, auth, and docs. Do not -reimplement them.

      -

      1. Scaffold the repository

      -

      Copy the templates/ skeleton. It provides: pyproject.toml, openapi.yaml, -main.py, init.py, worker.py, config.py, Dockerfile, -docker-compose.yaml, .env.example, and the package tree -(domain/ infra/ interfaces/ pipelines/ tests/ scripts/).

      -

      2. Author the spec (app-specific)

      -

      Write openapi.yaml: resources, schemas, operationIds, security, and any -extension metadata. This is the contract.

      -

      3. Write each CRUD slice (app-specific)

      -

      For each resource, create the four-file slice (see CRUD Template): -models.py, ports/db.py, ports/service.py, optional converters.py.

      -

      4. Wire the composition root

      -

      In init.py: build leaf repositories, then dependent repositories, then -services; add infrastructure adaptors (cache store, progress bus, stores, -fetchers). Add get_svc overloads.

      -

      5. Add routes

      -

      Create interfaces/<resource>/routes.py (thin handlers) and re-export them in -interfaces/openapi.py. Add a cache decorator to read endpoints.

      -

      6. Bootstrap the app and worker

      -

      Confirm main.py wires the OpenAPI-first app with the lifespan. If the app has -long-running jobs, adapt worker.py to poll the relevant job collection.

      -

      7. Add a pipeline family (only if data-driven)

      -

      Create pipelines/<family>/ with abc.py, impl.py, pipeline.yaml, -resources/, and register the root in registry.py. Add fixtures and a parity -test.

      -

      8. Add infrastructure as needed

      -

      Only where the standard adaptors do not cover the need. Prefer library-backed -adapters.

      -

      9. Tests

      -

      Add integration tests with fakes and unit tests for domain/pipeline logic; add -a config-consistency test if using rule-driven pipelines.

      -

      10. Scripts and seeds

      -

      Add operator scripts under scripts/ that use init() / get_svc() / -shutdown().

      -

      11. Docs

      -

      Author GSDFC docstrings, generate the type stubs (.pyi), mark the package -py.typed, and build docs.

      -

      Definition of done

      +

      ๐Ÿ—‚๏ธ Component model

      +
      flowchart LR
+    subgraph EDGE["interfaces/ โ€” HTTP edge"]
+        OAS["openapi.yaml"]
+        RU["resource routes"]
+        JU["job routes"]
+    end
+
+    subgraph CORE["domain/ โ€” business core"]
+        RI["<resource> ยท models ยท ports ยท services"]
+        JI["<job/worker> ยท models ยท ports ยท services"]
+        PI["pipelines/<family>"]
+    end
+
+    subgraph INFRA["infra/ โ€” adapters"]
+        DB[("MongoDB repos")]
+        CA["redis cache"]
+        ST["stores ยท progress bus"]
+        LL["llm clients"]
+    end
+
+    W["worker.py"]
+    CR["init.py โ€” composition root"]
+
+    OAS --> RU
+    OAS --> JU
+    RU --> RI
+    JU --> JI
+    JI --> PI
+    PI --> DB
+    PI --> LL
+    RI --> CA
+    RI --> DB
+    JI --> ST
+    W --> JI
+    W --> PI
+
+    CR === RU
+    CR === JU
+    CR === RI
+    CR === JI
+    CR === PI
+    CR === W
+    CR === DB
+    CR === CA
+    CR === ST
      +
      +

      Legend โ€” solid arrows are direct calls; thick links are wires created by +the composition root. Every component is injected, never self-constructed.

      +
      +
      +

      ๐Ÿ“ฆ Component roles

      +

      1. interfaces/ โ€” the edge

      +

      Thin handlers bound to openapi.yaml operationIds. They parse the request, +call a service, and shape the response. No business rules, no I/O.

        -
      • python -c "import main" succeeds.
        • -
      • pytest passes.
        • -
      • The app serves the spec's operations; the worker drains its queue.
        • -
      • No domain module imports a web framework.
        • -
      • No infrastructure was reimplemented that a library already provides.
        • +
      • interfaces/openapi.py re-exports every route module so the OpenAPI-first + app can resolve handlers.
        • +
      • Routes may mount the cache decorator on read endpoints.
        • +
      +

      2. domain/ โ€” the core

      +

      Pure models, ports (abstract contracts), and services (use cases). No +framework imports, no drivers.

      +
        +
      • <aggregate>/models.py โ€” dataclasses, enums, document shapes.
        • +
      • <aggregate>/ports/db.py โ€” repo contract; <aggregate>/ports/service.py โ€” + service contract.
        • +
      • Services compose ports and enforce the business rules; the composition root + injects the implementations.
        • +
      +

      3. infra/ โ€” the adapters

      +

      Concrete implementations of the domain ports plus cross-cutting infrastructure:

      +
        +
      • db/ โ€” mongo-ops repositories and the connection lifecycle.
        • +
      • adapters/ โ€” cache store, progress bus, storages, fetchers (library-backed).
        • +
      • Anything else the domain needs, implemented against its ports.
        • +
      +

      4. pipelines/ โ€” data-driven families (optional)

      +

      Extraction and enrichment families, each a deterministic step chain +(abc.py / impl.py / pipeline.yaml / resources/). Roots are registered +in the pattern registry and driven either by a worker job or a CLI.

      +

      5. worker.py โ€” the job worker (optional)

      +

      A separate process that polls durable job collections, runs pipeline roots, and +publishes progress to the progress bus. It shares the same composition root as +the API.

      +

      6. init.py โ€” the composition root

      +

      The only place anything is constructed: leaf repositories โ†’ dependent +repositories โ†’ services โ†’ adapters. Exposes get_svc overloads and starts / +stops resources on the app lifespan.

      +
      +

      ๐Ÿ”— How the request flows

      +
      sequenceDiagram
+    participant C as Client
+    participant R as route (interfaces)
+    participant S as service (domain)
+    participant P as repo (infra)
+    participant DB as MongoDB
+
+    C->>R: POST /items (operationId)
+    R->>S: svc.create(payload)
+    S->>P: repo.insert(doc)
+    P->>DB: insert
+    DB-->>P: ack
+    P-->>S: doc
+    S-->>R: result
+    R-->>C: 200 JSON
      +
      + + @@ -756,6 +780,8 @@ a config-consistency test if using rule-driven pipelines.

      + + \ No newline at end of file diff --git a/standards/wiki/03_build_recipe/index.html b/standards/wiki/03_build_recipe/index.html new file mode 100644 index 0000000..1a83289 --- /dev/null +++ b/standards/wiki/03_build_recipe/index.html @@ -0,0 +1,794 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Build Recipe - Aetos Standards โ€” Wiki + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      + +
      + + + + + + +
      + + +
      + +
      + + + + + + +
      +
      + + + +
      +
      +
      + + + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      +
      + + + +
      +
      + + + + + +

      Build Recipe

      +

      The ordered, deterministic checklist for generating a new application of the +standard shape. Every step is fixed; only steps 2 and 3 need app-specific +authoring. Give an AI agent the copyable prompt below (or read it from the +standards MCP bundle as doc://modules/build_recipe) and it can produce a +complete app in one pass.

      +
      +

      ๐Ÿš€ Quick start

      +
      1
+2
+3
      # Copy the runnable skeleton (from the standards docs checkout)
+cp -r <docs>/standards/src/templates/app apps/<your-app>
+cd apps/<your-app>
+
      +

      Then follow the numbered steps in the prompt.

      +
      +

      ๐Ÿ“‹ Agent prompt (copyable)

      +
       1
+ 2
+ 3
+ 4
+ 5
+ 6
+ 7
+ 8
+ 9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
      # Build a new application from the Aetos Standards blueprint
+
+You are coding a complete new backend application conforming to the Aetos
+standard app blueprint. Execute every step in order. Only steps 2 and 3 use the
+app-specific details provided by the user; everything else is deterministic.
+
+Before writing any code, connect to the `standards` MCP server and read:
+- `doc://nav` (the module list)
+- `doc://modules/layering_rules` (the dependency rules)
+- `doc://modules/genericity_matrix` (what is generic vs app-specific)
+Then call `find_library` with capabilities like `db`, `di`, `routing`,
+`ingestion`, `auth`, `docs`, `pipeline`, and note the libraries that already
+exist. Never reimplement one.
+
+1. **Scaffold** โ€” copy `src/templates/app/` into `apps/<your-app>/`. Keep the
+   package tree: `domain/`, `infra/`, `interfaces/`, `pipelines/`, `tests/`,
+   `scripts/`, plus `pyproject.toml`, `openapi.yaml`, `config.py`, `init.py`,
+   `main.py`, `worker.py`, `Dockerfile`, `docker-compose.yaml`, `.env.example`.
+
+2. **Author the spec (app-specific)** โ€” write `openapi.yaml`: resources,
+   schemas, `operationId`s, security, and extension metadata. This is the
+   contract and drives all routing.
+
+3. **Write each CRUD slice (app-specific)** โ€” per resource create the
+   four-file slice: `models.py`, `ports/db.py`, `ports/service.py`, and
+   optional `converters.py`. Follow the CRUD template.
+
+4. **Wire the composition root** โ€” in `init.py`, build leaf repositories, then
+   dependent repositories, then services; add infra adapters (db, cache,
+   stores, progress bus, fetchers). Expose `get_svc` overloads.
+
+5. **Add routes** โ€” create `interfaces/<resource>/routes.py` thin handlers
+   bound to `operationId`s and re-export them in `interfaces/openapi.py`. Add
+   the cache decorator to read endpoints.
+
+6. **Bootstrap app and worker** โ€” confirm `main.py` runs the OpenAPI-first app
+   with its lifespan. If the app has long-running jobs, adapt `worker.py` to
+   poll the job collection, run the pipeline, and publish progress.
+
+7. **Add pipeline families (only if data-driven)** โ€” create
+   `pipelines/<family>/` with `abc.py`, `impl.py`, `pipeline.yaml`,
+   `resources/`, and register the root. Add fixtures and a parity test.
+
+8. **Add infrastructure only where needed** โ€” prefer a library-backed adapter.
+   Never hand-write what an internal library already provides.
+
+9. **Tests** โ€” integration tests with fakes at the HTTP edge; unit tests for
+   domain and pipeline logic; a config-consistency test for rule-driven
+   pipelines.
+
+10. **Scripts and seeds** โ€” operator scripts under `scripts/` using
+    `init()` / `get_svc()` / `shutdown()`.
+
+11. **Docs** โ€” author GSDFC docstrings, generate `.pyi` stubs, mark the package
+    `py.typed`, and build the docs.
+
+**Definition of done:**
+- `python -c "import main"` succeeds.
+- `pytest` passes.
+- The app serves every operation in the spec; the worker drains its queue.
+- No domain module imports a web framework.
+- No infrastructure was reimplemented that a library already provides.
+
      +
      +

      ๐Ÿงญ Where the steps land

      +

      The steps map onto the layers as follows:

      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      StepPackage / fileDepends on
      1 ยท Scaffoldwhole templates/app/โ€”
      2 ยท Specopenapi.yamlโ€”
      3 ยท CRUD slicesdomain/<resource>/spec, CRUD template
      4 ยท Wiringinit.pyevery slice
      5 ยท Routesinterfaces/<resource>/services, openapi route registry
      6 ยท Bootstrapmain.py, worker.pycomposition root
      7 ยท Pipelinespipelines/<family>/library pipeline engine
      8 ยท Infrainfra/domain ports
      9 ยท Teststests/app + fixtures
      10 ยท Scriptsscripts/composition root
      11 ยท Docsdocstrings + docs buildcode, Doc Forge
      +
      +

      โš ๏ธ Definition of done

      +
        +
      • python -c "import main" succeeds.
        • +
      • pytest passes.
        • +
      • The app serves the spec's operations; the worker drains its queue.
        • +
      • No domain module imports a web framework.
        • +
      • No infrastructure was reimplemented that a library already provides.
        • +
      +
      + + + + + + + + + + + + + + + +
      +
      + + + + + +
      + + + +
      + + + +
      +
      +
      +
      + + + + + + + + + + + + + + \ No newline at end of file diff --git a/standards/wiki/10_testing/index.html b/standards/wiki/04_development/01_testing/index.html similarity index 80% rename from standards/wiki/10_testing/index.html rename to standards/wiki/04_development/01_testing/index.html index 0332a38..6893020 100644 --- a/standards/wiki/10_testing/index.html +++ b/standards/wiki/04_development/01_testing/index.html @@ -11,25 +11,23 @@ - + - - - + - Testing - Aetos Standards โ€” Wiki + Testing & Quality - Aetos Standards โ€” Wiki - + - + @@ -48,7 +46,7 @@ - + @@ -74,7 +72,7 @@
      - + Skip to content @@ -90,7 +88,7 @@
      @@ -428,19 +457,10 @@
        -
      • - - - Summary - - - -
        • -
      • - Layout + ๐Ÿ—‚๏ธ Layout @@ -449,7 +469,7 @@
      • - Integration tests + ๐Ÿš€ Integration tests @@ -458,7 +478,7 @@
      • - Unit tests + ๐Ÿงช Unit tests @@ -467,7 +487,16 @@
      • - Config-consistency tests + ๐Ÿ” Config-consistency tests + + + +
        • + +
      • + + + โœ… Quality gates @@ -476,7 +505,16 @@
      • - Conventions + ๐Ÿ’ก Conventions + + + +
        • + +
      • + + + Related @@ -498,12 +536,12 @@ -

        Testing

        -

        Summary

        -

        Two test suites: integration (the HTTP edge with fakes) and unit (pure -domain and pipeline logic). Both run under pytest with asyncio auto mode.

        +

        Testing & Quality Gates

        +

        Two suites run under pytest with asyncio auto mode: integration (the HTTP +edge against fakes) and unit (pure domain and pipeline logic). Finish a +blueprint app only when every gate below passes.

        -

        Layout

        +

        ๐Ÿ—‚๏ธ Layout

        1
 2
 3
@@ -512,7 +550,8 @@ domain and pipeline logic). Both run under pytest with asyncio auto mode.
        
     unit/            # models, converters, pipeline stages/roots
     conftest.py      # fixtures
        -

        Integration tests

        +
        +

        ๐Ÿš€ Integration tests

        • Build the app with TestClient.
        • Replace app.state.resources with fake repositories/services so no database @@ -534,24 +573,42 @@ domain and pipeline logic). Both run under pytest with asyncio auto mode.

          r = client.post("/items", json={"name": "x"}) assert r.status_code == 200 -

          Unit tests

          +
          +

          ๐Ÿงช Unit tests

          • Domain models and converters: pure, no I/O.
          • Pipeline stages: feed a fixture state, assert the transformed state.
          • Assembled pipeline roots: fixture parity โ€” replay a stored fixture input and compare against a golden output file.
          -

          Config-consistency tests

          +
          +

          ๐Ÿ” Config-consistency tests

          For rule-driven pipelines, add a test that cross-references rules against the declared pattern registry: every flag emitted must be defined; every defined flag must be reachable. This catches drift without running the pipeline.

          -

          Conventions

          +
          +

          โœ… Quality gates

          +

          A definition of done for any standard app:

          +
            +
          • python -c "import main" succeeds.
            • +
          • pytest passes.
            • +
          • The app serves the spec's operations; the worker drains its queue.
            • +
          • No domain module imports a web framework.
            • +
          • No infrastructure was reimplemented that a library already provides.
            • +
          +
          +

          ๐Ÿ’ก Conventions

          • asyncio_mode = "auto"; no manual event loops.
          • Fixtures are checked into the repo (inputs + golden outputs).
          • Tests skip (not fail) when a required fixture is absent.
          • Do not mock the domain; mock only the infrastructure edge.
          +
          + + @@ -608,10 +665,12 @@ flag must be reachable. This catches drift without running the pipeline.

          - + - + + + diff --git a/standards/wiki/404.html b/standards/wiki/404.html index d9d0abf..974a310 100644 --- a/standards/wiki/404.html +++ b/standards/wiki/404.html @@ -260,12 +260,12 @@
        • - + - Testing + App Components @@ -282,7 +282,7 @@
        • - + @@ -297,6 +297,28 @@ + + + + + + +
        • + + + + + + Testing & Quality + + + + + +
          • + + +
        @@ -375,6 +397,8 @@ + + \ No newline at end of file diff --git a/standards/wiki/index.html b/standards/wiki/index.html index 07cc74d..914594f 100644 --- a/standards/wiki/index.html +++ b/standards/wiki/index.html @@ -72,7 +72,7 @@ @@ -391,18 +422,27 @@
        • - + - How to read this + ๐Ÿš€ Key Features
        • - + - For AI agents + ๐Ÿ“ Documentation Structure + + + +
          • + +
        • + + + ๐Ÿ”— Related Resources @@ -424,92 +464,85 @@ -

          The Aetos Standard App Blueprint

          -

          A single, reusable architecture for backend applications in the Aetos ecosystem.

          -

          Every application built with this blueprint has the same shape:

          -
            -
          • a spec-first HTTP layer driven by an OpenAPI document;
            • -
          • a domain layer of pure models, ports, and services;
            • -
          • an infrastructure layer of adapters (MongoDB, cache, workers, LLM, mail);
            • -
          • a composition root that wires everything in exactly one place;
            • -
          • an optional pipeline layer for data-driven extraction/enrichment;
            • -
          • an optional background worker that runs long-running jobs and reports progress.
            • -
          +

          ๐Ÿงฉ Standards โ€” the Aetos App Blueprint

          +

          One reusable architecture for every backend application in the Aetos ecosystem: +a spec-first HTTP edge, a pure domain core, a pluggable infrastructure layer, +and a single composition root.

          +

          Fill in two axes only and a new app is done: your resource model, and +(if needed) your pipeline family. The blueprint itself never names a +business domain โ€” expenses, health metrics, inventory, and analytics are all +the same shape underneath.

          -

          This blueprints is deliberately app-agnostic. It never names a concrete -business domain. Any application โ€” expenses, health metrics, inventory, -analytics โ€” is produced by filling in two axes only: the resource model -and (if needed) the pipeline family.

          +

          Doc model: this wiki is written for humans โ€” how-to guides, the build +recipe, and quality gates. The enforceable rules (layering, composition root, +CRUD template, pipeline scaffold) and the library registry live in the +reference pages, mirrored by the machine-readable standards MCP bundle. +Query find_library before writing any infrastructure.

          -

          How to read this

          +
          +

          ๐Ÿš€ Key Features

          +
            +
          • ๐Ÿงญ Spec-first routing โ€” the OpenAPI document, not decorator routers, is + the source of truth (operationId lookup)
            • +
          • ๐Ÿ—๏ธ One-way layering โ€” interfaces โ†’ domain โ†’ infra, enforced
            • +
          • ๐Ÿ”Œ Single composition root โ€” every dependency is wired in exactly one place
            • +
          • ๐Ÿงฉ Generic, not app-specific โ€” reusable rules versus two authoring axes
            • +
          • ๐Ÿ“š Library-backed โ€” DB, DI, pipelines, ingestion, auth, and docs come from + internal libraries, never hand-written
            • +
          • ๐Ÿšฆ Explicit state machines โ€” durable job statuses, progress, and + human-in-the-loop pauses
            • +
          +
          +

          ๐Ÿ“ Documentation Structure

          - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +
          PageWhat it coversSectionDescription
          OverviewThe philosophy and the one-way dependency ruleOverviewPhilosophy, the one-way dependency rule, architecture
          Layering RulesWhich package may import whichApp ComponentsThe component model and how the pieces wire together
          Composition RootThe single wiring pointBuild RecipeThe ordered, agentic checklist for a new app
          Spec-First RoutingOpenAPI as the source of truth
          CRUD TemplateThe canonical resource slice
          Pipeline ScaffoldData-driven processing families
          Worker & ProgressAsync jobs, status machine, streaming progress
          Human-in-the-LoopPausing a job to ask a question
          Infra AdaptersMongoDB, cache, stores, LLM, mail
          TestingIntegration + unit conventions
          Build RecipeThe ordered checklist to generate a new app
          Genericity MatrixWhat is reusable vs what you must writeTesting & QualityTest layout and acceptance gates
          -

          For AI agents

          -

          This content is served as MCP resources under docs://index, docs://nav, -and docs://modules/{module}. To build a new app:

          -
            -
          1. Read build_recipe first.
            2. -
          2. Read genericity_matrix to know what you must author.
            3. -
          3. Query the library registry (find_library) before writing any - infrastructure yourself.
            4. -
          4. Copy the templates/ skeleton and fill the two app-specific axes.
            5. -
          +

          Reference (separate site, mirrors the MCP bundle): the enforceable +rules โ€” Layering, Composition +Root, Spec-First +Routing, CRUD +Template, Pipeline +Scaffold, Worker & +Progress, Human-in-the-Loop, +Infra Adapters, Genericity +Matrix, Library +Registry.

          +
          + +
            +
          • MCP bundle โ€” standards server on :8009: read doc://index, + doc://nav, doc://modules/{module}; call find_library(capability)
            • +
          • Library registry โ€” the catalog of internal libraries behind every app + (libraries.json, served as registry://libraries)
            • +
          • Template โ€” the runnable skeleton at standards/src/templates/app/
            • +
          • Copy the repo's AGENTS.md-style build prompt from + Build Recipe
            • +
          +
          +

          ยฉ Aetoskia Internal โ€” standards

          @@ -571,6 +604,8 @@ and docs://modules/{module}. To build a new app:

          + + \ No newline at end of file diff --git a/standards/wiki/search/search_index.json b/standards/wiki/search/search_index.json index 83bb912..b971223 100644 --- a/standards/wiki/search/search_index.json +++ b/standards/wiki/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"The Aetos Standard App Blueprint","text":"

          A single, reusable architecture for backend applications in the Aetos ecosystem.

          Every application built with this blueprint has the same shape:

          • a spec-first HTTP layer driven by an OpenAPI document;
          • a domain layer of pure models, ports, and services;
          • an infrastructure layer of adapters (MongoDB, cache, workers, LLM, mail);
          • a composition root that wires everything in exactly one place;
          • an optional pipeline layer for data-driven extraction/enrichment;
          • an optional background worker that runs long-running jobs and reports progress.

          This blueprints is deliberately app-agnostic. It never names a concrete business domain. Any application \u2014 expenses, health metrics, inventory, analytics \u2014 is produced by filling in two axes only: the resource model and (if needed) the pipeline family.

          "},{"location":"#how-to-read-this","title":"How to read this","text":"Page What it covers Overview The philosophy and the one-way dependency rule Layering Rules Which package may import which Composition Root The single wiring point Spec-First Routing OpenAPI as the source of truth CRUD Template The canonical resource slice Pipeline Scaffold Data-driven processing families Worker & Progress Async jobs, status machine, streaming progress Human-in-the-Loop Pausing a job to ask a question Infra Adapters MongoDB, cache, stores, LLM, mail Testing Integration + unit conventions Build Recipe The ordered checklist to generate a new app Genericity Matrix What is reusable vs what you must write"},{"location":"#for-ai-agents","title":"For AI agents","text":"

          This content is served as MCP resources under docs://index, docs://nav, and docs://modules/{module}. To build a new app:

          1. Read build_recipe first.
          2. Read genericity_matrix to know what you must author.
          3. Query the library registry (find_library) before writing any infrastructure yourself.
          4. Copy the templates/ skeleton and fill the two app-specific axes.
          "},{"location":"01_overview/","title":"Overview","text":""},{"location":"01_overview/#summary","title":"Summary","text":"

          The blueprint is a hexagonal (ports and adapters) application with a spec-first web edge and a single composition root. It favors explicit wiring over implicit magic, and it prefers existing internal libraries over hand-written infrastructure.

          "},{"location":"01_overview/#the-three-layers","title":"The three layers","text":"
          interfaces/   \u2190 HTTP edge: thin handlers bound to an OpenAPI operationId\ndomain/       \u2190 business core: models, ports (interfaces), services, orchestrators\ninfra/        \u2190 adapters: databases, caches, external clients, hardware\n

          Plus two optional layers:

          pipelines/    \u2190 data-driven transformation graphs (extraction, enrichment)\nscripts/      \u2190 operator scripts that reuse the composition root\n
          "},{"location":"01_overview/#core-principles","title":"Core principles","text":"
          1. One-way dependencies. interfaces -> domain -> infra is allowed; the reverse is forbidden. The domain never imports a framework it does not own.
          2. Ports in the domain, adapters in infra. A service depends on an abstract port; the composition root injects a concrete adapter.
          3. One composition root. Nothing constructs its own dependencies. All wiring lives in one module (init.py) and is exposed as a flat resource registry.
          4. Spec-first. The HTTP contract is declared in an OpenAPI document. Route handlers are looked up by operationId. There are no decorator routers.
          5. Reuse the libraries. Pipeline orchestration, DI, Mongo repositories, ingestion, auth, and docs are provided by internal libraries. Do not reimplement them \u2014 see the library registry.
          6. Explicit state machines. Long-running work moves through explicit, durable statuses and can pause for human input.
          "},{"location":"01_overview/#runtime-processes","title":"Runtime processes","text":"

          A standard app may run as up to three processes that share the same composition root:

          Process Entry point Responsibility API main.py serve HTTP; start/stop resources via lifespan Worker worker.py poll durable jobs, run pipelines, publish progress Scripts python -m scripts.<name> one-shot operator tasks"},{"location":"10_testing/","title":"Testing","text":""},{"location":"10_testing/#summary","title":"Summary","text":"

          Two test suites: integration (the HTTP edge with fakes) and unit (pure domain and pipeline logic). Both run under pytest with asyncio auto mode.

          "},{"location":"10_testing/#layout","title":"Layout","text":"
          tests/\n    integration/     # TestClient(app) + mocked resources\n    unit/            # models, converters, pipeline stages/roots\n    conftest.py      # fixtures\n
          "},{"location":"10_testing/#integration-tests","title":"Integration tests","text":"
          • Build the app with TestClient.
          • Replace app.state.resources with fake repositories/services so no database is required.
          • Patch the auth introspection call (httpx.AsyncClient.post) to return a canned identity.
          • Assert status codes and bodies for list/get/create/update/delete, plus 400, 404, 204, and the streaming endpoint's fallback.
          client = TestClient(app)\napp.state.resources = {\"item_svc\": FakeItemService(), ...}\n\ndef test_create_item():\n    r = client.post(\"/items\", json={\"name\": \"x\"})\n    assert r.status_code == 200\n
          "},{"location":"10_testing/#unit-tests","title":"Unit tests","text":"
          • Domain models and converters: pure, no I/O.
          • Pipeline stages: feed a fixture state, assert the transformed state.
          • Assembled pipeline roots: fixture parity \u2014 replay a stored fixture input and compare against a golden output file.
          "},{"location":"10_testing/#config-consistency-tests","title":"Config-consistency tests","text":"

          For rule-driven pipelines, add a test that cross-references rules against the declared pattern registry: every flag emitted must be defined; every defined flag must be reachable. This catches drift without running the pipeline.

          "},{"location":"10_testing/#conventions","title":"Conventions","text":"
          • asyncio_mode = \"auto\"; no manual event loops.
          • Fixtures are checked into the repo (inputs + golden outputs).
          • Tests skip (not fail) when a required fixture is absent.
          • Do not mock the domain; mock only the infrastructure edge.
          "},{"location":"11_build_recipe/","title":"Build Recipe","text":""},{"location":"11_build_recipe/#summary","title":"Summary","text":"

          Follow this ordered checklist to generate a new application of the standard shape. Every step is deterministic; only steps 2 and 7 require app-specific authoring.

          "},{"location":"11_build_recipe/#0-query-the-registry","title":"0. Query the registry","text":"

          Before writing anything, read the library registry and note which internal libraries provide DB, DI, routing, ingestion, auth, and docs. Do not reimplement them.

          "},{"location":"11_build_recipe/#1-scaffold-the-repository","title":"1. Scaffold the repository","text":"

          Copy the templates/ skeleton. It provides: pyproject.toml, openapi.yaml, main.py, init.py, worker.py, config.py, Dockerfile, docker-compose.yaml, .env.example, and the package tree (domain/ infra/ interfaces/ pipelines/ tests/ scripts/).

          "},{"location":"11_build_recipe/#2-author-the-spec-app-specific","title":"2. Author the spec (app-specific)","text":"

          Write openapi.yaml: resources, schemas, operationIds, security, and any extension metadata. This is the contract.

          "},{"location":"11_build_recipe/#3-write-each-crud-slice-app-specific","title":"3. Write each CRUD slice (app-specific)","text":"

          For each resource, create the four-file slice (see CRUD Template): models.py, ports/db.py, ports/service.py, optional converters.py.

          "},{"location":"11_build_recipe/#4-wire-the-composition-root","title":"4. Wire the composition root","text":"

          In init.py: build leaf repositories, then dependent repositories, then services; add infrastructure adaptors (cache store, progress bus, stores, fetchers). Add get_svc overloads.

          "},{"location":"11_build_recipe/#5-add-routes","title":"5. Add routes","text":"

          Create interfaces/<resource>/routes.py (thin handlers) and re-export them in interfaces/openapi.py. Add a cache decorator to read endpoints.

          "},{"location":"11_build_recipe/#6-bootstrap-the-app-and-worker","title":"6. Bootstrap the app and worker","text":"

          Confirm main.py wires the OpenAPI-first app with the lifespan. If the app has long-running jobs, adapt worker.py to poll the relevant job collection.

          "},{"location":"11_build_recipe/#7-add-a-pipeline-family-only-if-data-driven","title":"7. Add a pipeline family (only if data-driven)","text":"

          Create pipelines/<family>/ with abc.py, impl.py, pipeline.yaml, resources/, and register the root in registry.py. Add fixtures and a parity test.

          "},{"location":"11_build_recipe/#8-add-infrastructure-as-needed","title":"8. Add infrastructure as needed","text":"

          Only where the standard adaptors do not cover the need. Prefer library-backed adapters.

          "},{"location":"11_build_recipe/#9-tests","title":"9. Tests","text":"

          Add integration tests with fakes and unit tests for domain/pipeline logic; add a config-consistency test if using rule-driven pipelines.

          "},{"location":"11_build_recipe/#10-scripts-and-seeds","title":"10. Scripts and seeds","text":"

          Add operator scripts under scripts/ that use init() / get_svc() / shutdown().

          "},{"location":"11_build_recipe/#11-docs","title":"11. Docs","text":"

          Author GSDFC docstrings, generate the type stubs (.pyi), mark the package py.typed, and build docs.

          "},{"location":"11_build_recipe/#definition-of-done","title":"Definition of done","text":"
          • python -c \"import main\" succeeds.
          • pytest passes.
          • The app serves the spec's operations; the worker drains its queue.
          • No domain module imports a web framework.
          • No infrastructure was reimplemented that a library already provides.
          "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"\ud83e\udde9 Standards \u2014 the Aetos App Blueprint","text":"

          One reusable architecture for every backend application in the Aetos ecosystem: a spec-first HTTP edge, a pure domain core, a pluggable infrastructure layer, and a single composition root.

          Fill in two axes only and a new app is done: your resource model, and (if needed) your pipeline family. The blueprint itself never names a business domain \u2014 expenses, health metrics, inventory, and analytics are all the same shape underneath.

          Doc model: this wiki is written for humans \u2014 how-to guides, the build recipe, and quality gates. The enforceable rules (layering, composition root, CRUD template, pipeline scaffold) and the library registry live in the reference pages, mirrored by the machine-readable standards MCP bundle. Query find_library before writing any infrastructure.

          "},{"location":"#key-features","title":"\ud83d\ude80 Key Features","text":"
          • \ud83e\udded Spec-first routing \u2014 the OpenAPI document, not decorator routers, is the source of truth (operationId lookup)
          • \ud83c\udfd7\ufe0f One-way layering \u2014 interfaces \u2192 domain \u2192 infra, enforced
          • \ud83d\udd0c Single composition root \u2014 every dependency is wired in exactly one place
          • \ud83e\udde9 Generic, not app-specific \u2014 reusable rules versus two authoring axes
          • \ud83d\udcda Library-backed \u2014 DB, DI, pipelines, ingestion, auth, and docs come from internal libraries, never hand-written
          • \ud83d\udea6 Explicit state machines \u2014 durable job statuses, progress, and human-in-the-loop pauses
          "},{"location":"#documentation-structure","title":"\ud83d\udcc1 Documentation Structure","text":"Section Description Overview Philosophy, the one-way dependency rule, architecture App Components The component model and how the pieces wire together Build Recipe The ordered, agentic checklist for a new app Testing & Quality Test layout and acceptance gates

          Reference (separate site, mirrors the MCP bundle): the enforceable rules \u2014 Layering, Composition Root, Spec-First Routing, CRUD Template, Pipeline Scaffold, Worker & Progress, Human-in-the-Loop, Infra Adapters, Genericity Matrix, Library Registry.

          "},{"location":"#related-resources","title":"\ud83d\udd17 Related Resources","text":"
          • MCP bundle \u2014 standards server on :8009: read doc://index, doc://nav, doc://modules/{module}; call find_library(capability)
          • Library registry \u2014 the catalog of internal libraries behind every app (libraries.json, served as registry://libraries)
          • Template \u2014 the runnable skeleton at standards/src/templates/app/
          • Copy the repo's AGENTS.md-style build prompt from Build Recipe

          \u00a9 Aetoskia Internal \u2014 standards

          "},{"location":"01_overview/","title":"Overview","text":"

          The blueprint is a hexagonal (ports & adapters) application with a spec-first web edge and a single composition root. It favors explicit wiring over implicit magic and existing internal libraries over hand-written infrastructure. Start here to build the mental model; the enforceable details live in the reference pages.

          "},{"location":"01_overview/#architecture","title":"\ud83c\udfd7\ufe0f Architecture","text":"
          flowchart LR\n    subgraph EDGE[\"interfaces/ \u2014 HTTP edge\"]\n        OAS[\"openapi.yaml<br/>operationId contract\"]\n        RT[\"<resource>/routes.py<br/>thin handlers\"]\n    end\n\n    subgraph CORE[\"domain/ \u2014 pure business core\"]\n        MD[\"models\"]\n        PT[\"ports (abstract)\"]\n        SV[\"services\"]\n    end\n\n    subgraph INFRA[\"infra/ \u2014 adapters\"]\n        RP[\"mongo repos (mongo-ops)\"]\n        AD[\"cache \u00b7 stores \u00b7 llm \u00b7 mail\"]\n        DB[(\"MongoDB\")]\n    end\n\n    CR[\"init.py<br/>composition root\"]\n\n    OAS --> RT\n    RT --> SV\n    SV --> PT\n    SV --> RP\n    PT -. depends on .- RP\n    CR --- RT\n    CR --- SV\n    CR --- RP\n    RP --> DB\n    AD --- CR

          The rule holds at every edge: code in an outer ring may depend on a ring closer to the core, never the reverse. domain/ never imports a web framework or a database driver.

          Legend \u2014 solid arrows are compile-time calls; dotted arrows are the port/adapter contract, resolved by the composition root at startup.

          "},{"location":"01_overview/#the-one-way-dependency-rule","title":"\ud83d\udd04 The one-way dependency rule","text":"Ring May import Never imports interfaces/ domain, framework glue (glue only) business logic, infra directly domain/ nothing but its own ports web frameworks, drivers, infra infra/ domain ports, drivers, libraries \u2014 init.py everything (the wiring point) \u2014"},{"location":"01_overview/#core-principles","title":"\ud83e\udded Core principles","text":"
          1. One-way dependencies. interfaces -> domain -> infra is allowed; the reverse is forbidden.
          2. Ports in the domain, adapters in infra. A service depends on an abstract port; the composition root injects a concrete adapter.
          3. One composition root. Nothing constructs its own dependencies. All wiring lives in one module (init.py) exposed as a flat resource registry.
          4. Spec-first. The HTTP contract is declared in openapi.yaml; route handlers are looked up by operationId. There are no decorator routers.
          5. Reuse the libraries. Orchestration, DI, Mongo repositories, ingestion, auth, and docs come from internal libraries \u2014 see the Library Registry.
          6. Explicit state machines. Long-running work moves through explicit, durable statuses and can pause for human input.
          "},{"location":"01_overview/#runtime-processes","title":"\u2699\ufe0f Runtime processes","text":"

          A standard app may run as up to three processes that share the same composition root:

          Process Entry point Responsibility API main.py serve HTTP; start/stop resources via lifespan Worker worker.py poll durable jobs, run pipelines, publish progress Scripts python -m scripts.<name> one-shot operator tasks"},{"location":"01_overview/#read-next","title":"\u27a1\ufe0f Read Next","text":"
          • 02 \u2013 App Components \u00b7 03 \u2013 Build Recipe
          "},{"location":"02_components/","title":"App Components","text":"

          What ships in every standard app, and how the pieces relate. This is the blueprint's component model \u2014 templates/app/ is a runnable instance of it.

          "},{"location":"02_components/#component-model","title":"\ud83d\uddc2\ufe0f Component model","text":"
          flowchart LR\n    subgraph EDGE[\"interfaces/ \u2014 HTTP edge\"]\n        OAS[\"openapi.yaml\"]\n        RU[\"resource routes\"]\n        JU[\"job routes\"]\n    end\n\n    subgraph CORE[\"domain/ \u2014 business core\"]\n        RI[\"<resource> \u00b7 models \u00b7 ports \u00b7 services\"]\n        JI[\"<job/worker> \u00b7 models \u00b7 ports \u00b7 services\"]\n        PI[\"pipelines/<family>\"]\n    end\n\n    subgraph INFRA[\"infra/ \u2014 adapters\"]\n        DB[(\"MongoDB repos\")]\n        CA[\"redis cache\"]\n        ST[\"stores \u00b7 progress bus\"]\n        LL[\"llm clients\"]\n    end\n\n    W[\"worker.py\"]\n    CR[\"init.py \u2014 composition root\"]\n\n    OAS --> RU\n    OAS --> JU\n    RU --> RI\n    JU --> JI\n    JI --> PI\n    PI --> DB\n    PI --> LL\n    RI --> CA\n    RI --> DB\n    JI --> ST\n    W --> JI\n    W --> PI\n\n    CR === RU\n    CR === JU\n    CR === RI\n    CR === JI\n    CR === PI\n    CR === W\n    CR === DB\n    CR === CA\n    CR === ST

          Legend \u2014 solid arrows are direct calls; thick links are wires created by the composition root. Every component is injected, never self-constructed.

          "},{"location":"02_components/#component-roles","title":"\ud83d\udce6 Component roles","text":""},{"location":"02_components/#1-interfaces-the-edge","title":"1. interfaces/ \u2014 the edge","text":"

          Thin handlers bound to openapi.yaml operationIds. They parse the request, call a service, and shape the response. No business rules, no I/O.

          • interfaces/openapi.py re-exports every route module so the OpenAPI-first app can resolve handlers.
          • Routes may mount the cache decorator on read endpoints.
          "},{"location":"02_components/#2-domain-the-core","title":"2. domain/ \u2014 the core","text":"

          Pure models, ports (abstract contracts), and services (use cases). No framework imports, no drivers.

          • <aggregate>/models.py \u2014 dataclasses, enums, document shapes.
          • <aggregate>/ports/db.py \u2014 repo contract; <aggregate>/ports/service.py \u2014 service contract.
          • Services compose ports and enforce the business rules; the composition root injects the implementations.
          "},{"location":"02_components/#3-infra-the-adapters","title":"3. infra/ \u2014 the adapters","text":"

          Concrete implementations of the domain ports plus cross-cutting infrastructure:

          • db/ \u2014 mongo-ops repositories and the connection lifecycle.
          • adapters/ \u2014 cache store, progress bus, storages, fetchers (library-backed).
          • Anything else the domain needs, implemented against its ports.
          "},{"location":"02_components/#4-pipelines-data-driven-families-optional","title":"4. pipelines/ \u2014 data-driven families (optional)","text":"

          Extraction and enrichment families, each a deterministic step chain (abc.py / impl.py / pipeline.yaml / resources/). Roots are registered in the pattern registry and driven either by a worker job or a CLI.

          "},{"location":"02_components/#5-workerpy-the-job-worker-optional","title":"5. worker.py \u2014 the job worker (optional)","text":"

          A separate process that polls durable job collections, runs pipeline roots, and publishes progress to the progress bus. It shares the same composition root as the API.

          "},{"location":"02_components/#6-initpy-the-composition-root","title":"6. init.py \u2014 the composition root","text":"

          The only place anything is constructed: leaf repositories \u2192 dependent repositories \u2192 services \u2192 adapters. Exposes get_svc overloads and starts / stops resources on the app lifespan.

          "},{"location":"02_components/#how-the-request-flows","title":"\ud83d\udd17 How the request flows","text":"
          sequenceDiagram\n    participant C as Client\n    participant R as route (interfaces)\n    participant S as service (domain)\n    participant P as repo (infra)\n    participant DB as MongoDB\n\n    C->>R: POST /items (operationId)\n    R->>S: svc.create(payload)\n    S->>P: repo.insert(doc)\n    P->>DB: insert\n    DB-->>P: ack\n    P-->>S: doc\n    S-->>R: result\n    R-->>C: 200 JSON
          "},{"location":"02_components/#related","title":"Related","text":"
          • 01 \u2013 Overview \u00b7 03 \u2013 Build Recipe \u00b7 04 \u2013 Testing & Quality
          "},{"location":"03_build_recipe/","title":"Build Recipe","text":"

          The ordered, deterministic checklist for generating a new application of the standard shape. Every step is fixed; only steps 2 and 3 need app-specific authoring. Give an AI agent the copyable prompt below (or read it from the standards MCP bundle as doc://modules/build_recipe) and it can produce a complete app in one pass.

          "},{"location":"03_build_recipe/#quick-start","title":"\ud83d\ude80 Quick start","text":"
          # Copy the runnable skeleton (from the standards docs checkout)\ncp -r <docs>/standards/src/templates/app apps/<your-app>\ncd apps/<your-app>\n

          Then follow the numbered steps in the prompt.

          "},{"location":"03_build_recipe/#agent-prompt-copyable","title":"\ud83d\udccb Agent prompt (copyable)","text":"
          # Build a new application from the Aetos Standards blueprint\n\nYou are coding a complete new backend application conforming to the Aetos\nstandard app blueprint. Execute every step in order. Only steps 2 and 3 use the\napp-specific details provided by the user; everything else is deterministic.\n\nBefore writing any code, connect to the `standards` MCP server and read:\n- `doc://nav` (the module list)\n- `doc://modules/layering_rules` (the dependency rules)\n- `doc://modules/genericity_matrix` (what is generic vs app-specific)\nThen call `find_library` with capabilities like `db`, `di`, `routing`,\n`ingestion`, `auth`, `docs`, `pipeline`, and note the libraries that already\nexist. Never reimplement one.\n\n1. **Scaffold** \u2014 copy `src/templates/app/` into `apps/<your-app>/`. Keep the\n   package tree: `domain/`, `infra/`, `interfaces/`, `pipelines/`, `tests/`,\n   `scripts/`, plus `pyproject.toml`, `openapi.yaml`, `config.py`, `init.py`,\n   `main.py`, `worker.py`, `Dockerfile`, `docker-compose.yaml`, `.env.example`.\n\n2. **Author the spec (app-specific)** \u2014 write `openapi.yaml`: resources,\n   schemas, `operationId`s, security, and extension metadata. This is the\n   contract and drives all routing.\n\n3. **Write each CRUD slice (app-specific)** \u2014 per resource create the\n   four-file slice: `models.py`, `ports/db.py`, `ports/service.py`, and\n   optional `converters.py`. Follow the CRUD template.\n\n4. **Wire the composition root** \u2014 in `init.py`, build leaf repositories, then\n   dependent repositories, then services; add infra adapters (db, cache,\n   stores, progress bus, fetchers). Expose `get_svc` overloads.\n\n5. **Add routes** \u2014 create `interfaces/<resource>/routes.py` thin handlers\n   bound to `operationId`s and re-export them in `interfaces/openapi.py`. Add\n   the cache decorator to read endpoints.\n\n6. **Bootstrap app and worker** \u2014 confirm `main.py` runs the OpenAPI-first app\n   with its lifespan. If the app has long-running jobs, adapt `worker.py` to\n   poll the job collection, run the pipeline, and publish progress.\n\n7. **Add pipeline families (only if data-driven)** \u2014 create\n   `pipelines/<family>/` with `abc.py`, `impl.py`, `pipeline.yaml`,\n   `resources/`, and register the root. Add fixtures and a parity test.\n\n8. **Add infrastructure only where needed** \u2014 prefer a library-backed adapter.\n   Never hand-write what an internal library already provides.\n\n9. **Tests** \u2014 integration tests with fakes at the HTTP edge; unit tests for\n   domain and pipeline logic; a config-consistency test for rule-driven\n   pipelines.\n\n10. **Scripts and seeds** \u2014 operator scripts under `scripts/` using\n    `init()` / `get_svc()` / `shutdown()`.\n\n11. **Docs** \u2014 author GSDFC docstrings, generate `.pyi` stubs, mark the package\n    `py.typed`, and build the docs.\n\n**Definition of done:**\n- `python -c \"import main\"` succeeds.\n- `pytest` passes.\n- The app serves every operation in the spec; the worker drains its queue.\n- No domain module imports a web framework.\n- No infrastructure was reimplemented that a library already provides.\n
          "},{"location":"03_build_recipe/#where-the-steps-land","title":"\ud83e\udded Where the steps land","text":"

          The steps map onto the layers as follows:

          Step Package / file Depends on 1 \u00b7 Scaffold whole templates/app/ \u2014 2 \u00b7 Spec openapi.yaml \u2014 3 \u00b7 CRUD slices domain/<resource>/ spec, CRUD template 4 \u00b7 Wiring init.py every slice 5 \u00b7 Routes interfaces/<resource>/ services, openapi route registry 6 \u00b7 Bootstrap main.py, worker.py composition root 7 \u00b7 Pipelines pipelines/<family>/ library pipeline engine 8 \u00b7 Infra infra/ domain ports 9 \u00b7 Tests tests/ app + fixtures 10 \u00b7 Scripts scripts/ composition root 11 \u00b7 Docs docstrings + docs build code, Doc Forge"},{"location":"03_build_recipe/#definition-of-done","title":"\u26a0\ufe0f Definition of done","text":"
          • python -c \"import main\" succeeds.
          • pytest passes.
          • The app serves the spec's operations; the worker drains its queue.
          • No domain module imports a web framework.
          • No infrastructure was reimplemented that a library already provides.
          "},{"location":"03_build_recipe/#related","title":"Related","text":"
          • 01 \u2013 Overview \u00b7 02 \u2013 App Components \u00b7 04 \u2013 Testing & Quality
          "},{"location":"04_development/01_testing/","title":"Testing & Quality Gates","text":"

          Two suites run under pytest with asyncio auto mode: integration (the HTTP edge against fakes) and unit (pure domain and pipeline logic). Finish a blueprint app only when every gate below passes.

          "},{"location":"04_development/01_testing/#layout","title":"\ud83d\uddc2\ufe0f Layout","text":"
          tests/\n    integration/     # TestClient(app) + mocked resources\n    unit/            # models, converters, pipeline stages/roots\n    conftest.py      # fixtures\n
          "},{"location":"04_development/01_testing/#integration-tests","title":"\ud83d\ude80 Integration tests","text":"
          • Build the app with TestClient.
          • Replace app.state.resources with fake repositories/services so no database is required.
          • Patch the auth introspection call (httpx.AsyncClient.post) to return a canned identity.
          • Assert status codes and bodies for list/get/create/update/delete, plus 400, 404, 204, and the streaming endpoint's fallback.
          client = TestClient(app)\napp.state.resources = {\"item_svc\": FakeItemService(), ...}\n\ndef test_create_item():\n    r = client.post(\"/items\", json={\"name\": \"x\"})\n    assert r.status_code == 200\n
          "},{"location":"04_development/01_testing/#unit-tests","title":"\ud83e\uddea Unit tests","text":"
          • Domain models and converters: pure, no I/O.
          • Pipeline stages: feed a fixture state, assert the transformed state.
          • Assembled pipeline roots: fixture parity \u2014 replay a stored fixture input and compare against a golden output file.
          "},{"location":"04_development/01_testing/#config-consistency-tests","title":"\ud83d\udd10 Config-consistency tests","text":"

          For rule-driven pipelines, add a test that cross-references rules against the declared pattern registry: every flag emitted must be defined; every defined flag must be reachable. This catches drift without running the pipeline.

          "},{"location":"04_development/01_testing/#quality-gates","title":"\u2705 Quality gates","text":"

          A definition of done for any standard app:

          • python -c \"import main\" succeeds.
          • pytest passes.
          • The app serves the spec's operations; the worker drains its queue.
          • No domain module imports a web framework.
          • No infrastructure was reimplemented that a library already provides.
          "},{"location":"04_development/01_testing/#conventions","title":"\ud83d\udca1 Conventions","text":"
          • asyncio_mode = \"auto\"; no manual event loops.
          • Fixtures are checked into the repo (inputs + golden outputs).
          • Tests skip (not fail) when a required fixture is absent.
          • Do not mock the domain; mock only the infrastructure edge.
          "},{"location":"04_development/01_testing/#related","title":"Related","text":"
          • 02 \u2013 App Components \u00b7 03 \u2013 Build Recipe \u00b7 Overview
          "}]} \ No newline at end of file