From 6c0d859a1e02259153211c26ebee985ef21d9c38 Mon Sep 17 00:00:00 2001 From: Vishesh 'ironeagle' Bangotra Date: Tue, 13 Oct 2026 19:35:12 +0530 Subject: [PATCH] feat(standards): fix F11-F16 and rebuild bundle (43 components) + trial findings - F11 PopulateRule field_name/collection_name corrections - F12 service adaptor update handles PopulatingRepository - F13 nested-references section; F14 authless /users/me guidance - F15 template ruff-clean; F16 stop vendoring .ruff_cache, ship .gitignore - document --template-style in build_recipe/spec_first_routing; refresh collected openapi-first + standards bundle (12 modules, 43 components) - add trial.md and trial_findings.md (F1-F16, re-trial A/B evidence) --- build.py | 7 +- .../mcp/modules/openapi_first.codegen.json | 4 +- .../modules/openapi_first.codegen_routes.json | 6 +- openapi-first/mcp/modules/openapi_first.json | 10 +- standards/lib/01_build_recipe/index.html | 104 +++++++------- .../lib/04_spec_first_routing/index.html | 13 +- standards/lib/05_crud_template/index.html | 33 ++++- standards/lib/09_infra_adaptors/index.html | 2 +- standards/lib/search/search_index.json | 2 +- standards/lib/sitemap.xml.gz | Bin 127 -> 127 bytes standards/mcp/components.json | 6 + standards/mcp/modules/build_recipe.json | 2 +- standards/mcp/modules/crud_template.json | 2 +- standards/mcp/modules/infra_adaptors.json | 2 +- standards/mcp/modules/spec_first_routing.json | 2 +- standards/mcp/templates/app/.gitignore | 23 ++++ .../mcp/templates/app/domain/item/ports/db.py | 3 +- .../mcp/templates/app/domain/job/models.py | 1 + .../mcp/templates/app/domain/job/ports/db.py | 3 +- .../mcp/templates/app/infra/adapters/kv.py | 23 ++-- .../templates/app/infra/adapters/service.py | 24 +++- .../mcp/templates/app/infra/adapters/sse.py | 6 +- .../mcp/templates/app/infra/db/repos/mongo.py | 13 +- .../mcp/templates/app/infra/http_cache.py | 9 +- standards/mcp/templates/app/init.py | 6 +- .../mcp/templates/app/interfaces/openapi.py | 1 + standards/mcp/templates/app/main.py | 1 + .../templates/app/pipelines/shared/loader.py | 1 - .../app/pipelines/shared/registry.py | 7 +- standards/mcp/templates/app/pyproject.toml | 1 + .../app/tests/integration/test_item_routes.py | 4 +- .../integration/test_openapi_app_builds.py | 3 +- standards/src/lib/01_build_recipe.md | 11 +- standards/src/lib/04_spec_first_routing.md | 15 +- standards/src/lib/05_crud_template.md | 17 ++- standards/src/lib/09_infra_adaptors.md | 2 +- standards/src/templates/app/.gitignore | 23 ++++ .../src/templates/app/domain/item/ports/db.py | 3 +- .../src/templates/app/domain/job/models.py | 1 + .../src/templates/app/domain/job/ports/db.py | 3 +- .../src/templates/app/infra/adapters/kv.py | 23 ++-- .../templates/app/infra/adapters/service.py | 24 +++- .../src/templates/app/infra/adapters/sse.py | 6 +- .../src/templates/app/infra/db/repos/mongo.py | 13 +- .../src/templates/app/infra/http_cache.py | 9 +- standards/src/templates/app/init.py | 6 +- .../src/templates/app/interfaces/openapi.py | 1 + standards/src/templates/app/main.py | 1 + .../templates/app/pipelines/shared/loader.py | 1 - .../app/pipelines/shared/registry.py | 7 +- standards/src/templates/app/pyproject.toml | 1 + .../app/tests/integration/test_item_routes.py | 4 +- .../integration/test_openapi_app_builds.py | 3 +- standards/wiki/sitemap.xml.gz | Bin 127 -> 127 bytes trial.md | 58 ++++++++ trial_findings.md | 130 ++++++++++++++++++ 56 files changed, 521 insertions(+), 165 deletions(-) create mode 100644 standards/mcp/templates/app/.gitignore create mode 100644 standards/src/templates/app/.gitignore create mode 100644 trial.md create mode 100644 trial_findings.md diff --git a/build.py b/build.py index ff2b40f..798bf20 100644 --- a/build.py +++ b/build.py @@ -241,6 +241,8 @@ _TEMPLATE_COMPONENTS: list[dict[str, str]] = [ "description": "API + worker image with private-index wiring."}, {"kind": "dockerignore", "path": ".dockerignore", "placeholder": "", "description": "Docker build-context exclusions."}, + {"kind": "gitignore", "path": ".gitignore", "placeholder": "", + "description": "VCS exclusions (caches, venv, generated _gen/)."}, {"kind": "compose", "path": "docker-compose.yaml", "placeholder": "", "description": "api + worker + mongo compose file."}, {"kind": "env-example", "path": ".env.example", "placeholder": "", @@ -319,7 +321,10 @@ def build_templates(folder: Path, dry_run: bool = False) -> None: out_dir.parent.mkdir(parents=True, exist_ok=True) shutil.copytree( src, out_dir, - ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), + ignore=shutil.ignore_patterns( + "__pycache__", "*.pyc", ".ruff_cache", ".pytest_cache", + ".mypy_cache", ".venv", "venv", ".git", "*.egg-info", "htmlcov", + ), ) manifest = build_components_manifest(out_dir) (folder / "mcp" / "components.json").write_text( diff --git a/openapi-first/mcp/modules/openapi_first.codegen.json b/openapi-first/mcp/modules/openapi_first.codegen.json index 9e239df..2974ab1 100644 --- a/openapi-first/mcp/modules/openapi_first.codegen.json +++ b/openapi-first/mcp/modules/openapi_first.codegen.json @@ -8,8 +8,8 @@ "name": "generate_routes", "kind": "function", "path": "openapi_first.codegen.generate_routes", - "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False)", - "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." + "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False, template_style: bool = False)", + "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n template_style (bool, optional):\n If ``True``, emit ``async def`` handlers with a JSON body parameter\n named ``body`` (the standards-template convention) plus a\n service-resolution hint comment, instead of sync ``def`` handlers\n taking ``payload``.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." }, "generate_models": { "name": "generate_models", diff --git a/openapi-first/mcp/modules/openapi_first.codegen_routes.json b/openapi-first/mcp/modules/openapi_first.codegen_routes.json index 6cd5010..379e7f9 100644 --- a/openapi-first/mcp/modules/openapi_first.codegen_routes.json +++ b/openapi-first/mcp/modules/openapi_first.codegen_routes.json @@ -2,7 +2,7 @@ "module": "openapi_first.codegen_routes", "content": { "path": "openapi_first.codegen_routes", - "docstring": "# Summary\n\nRoute handler code generation from OpenAPI specifications.\n\nThis module generates Python route handler stubs from an OpenAPI 3.x\nspecification. Each resource (derived from the first path segment)\ngets its own file under the output directory. Every OpenAPI operation\nmust define an ``operationId``, which becomes the handler function name.\n\nNotes:\n **Design constraints:**\n\n - ``operationId`` is required on every operation (matching\n ``binder.bind_routes``).\n - Handlers are stubs raising ``NotImplementedError``.\n - Sub-resources (e.g. ``/pets/{id}/photo``) are grouped with their\n parent resource (``pets``).\n - Parameter types and defaults are inferred from the spec.\n - ``response: Response`` is injected for non-200 success codes.\n - Resource names are sanitized to valid Python module identifiers,\n so a path such as ``/fetch-requests`` yields ``fetch_requests.py``.\n - Passing ``inject_request=True`` prepends ``request: Request`` to\n every handler (the composition-root / DI convention).\n - ``multipart/form-data`` and\n ``application/x-www-form-urlencoded`` request bodies expand to one\n parameter per schema property (``UploadFile`` / ``File`` / ``Form``)\n instead of an opaque ``payload: dict``.\n - Passing ``package=True`` writes ``/routes.py`` (with\n ``__init__.py`` sidecars) instead of a flat ``.py``.", + "docstring": "# Summary\n\nRoute handler code generation from OpenAPI specifications.\n\nThis module generates Python route handler stubs from an OpenAPI 3.x\nspecification. Each resource (derived from the first path segment)\ngets its own file under the output directory. Every OpenAPI operation\nmust define an ``operationId``, which becomes the handler function name.\n\nNotes:\n **Design constraints:**\n\n - ``operationId`` is required on every operation (matching\n ``binder.bind_routes``).\n - Handlers are stubs raising ``NotImplementedError``.\n - Sub-resources (e.g. ``/pets/{id}/photo``) are grouped with their\n parent resource (``pets``).\n - Parameter types and defaults are inferred from the spec.\n - ``response: Response`` is injected for non-200 success codes.\n - Resource names are sanitized to valid Python module identifiers,\n so a path such as ``/fetch-requests`` yields ``fetch_requests.py``.\n - Passing ``inject_request=True`` prepends ``request: Request`` to\n every handler (the composition-root / DI convention).\n - ``multipart/form-data`` and\n ``application/x-www-form-urlencoded`` request bodies expand to one\n parameter per schema property (``UploadFile`` / ``File`` / ``Form``)\n instead of an opaque ``payload: dict``.\n - Passing ``package=True`` writes ``/routes.py`` (with\n ``__init__.py`` sidecars) instead of a flat ``.py``.\n - Passing ``template_style=True`` emits ``async def`` handlers whose\n JSON body parameter is named ``body`` (matching the standards\n template) and adds a service-resolution hint comment.", "objects": { "load_openapi": { "name": "load_openapi", @@ -15,8 +15,8 @@ "name": "generate_routes", "kind": "function", "path": "openapi_first.codegen_routes.generate_routes", - "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False) -> list[Path]", - "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." + "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False, template_style: bool = False) -> list[Path]", + "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n template_style (bool, optional):\n If ``True``, emit ``async def`` handlers with a JSON body parameter\n named ``body`` (the standards-template convention) plus a\n service-resolution hint comment, instead of sync ``def`` handlers\n taking ``payload``.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." } } } diff --git a/openapi-first/mcp/modules/openapi_first.json b/openapi-first/mcp/modules/openapi_first.json index 8d7eed1..be874c6 100644 --- a/openapi-first/mcp/modules/openapi_first.json +++ b/openapi-first/mcp/modules/openapi_first.json @@ -188,8 +188,8 @@ "name": "generate_routes", "kind": "function", "path": "openapi_first.codegen.generate_routes", - "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False)", - "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." + "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False, template_style: bool = False)", + "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n template_style (bool, optional):\n If ``True``, emit ``async def`` handlers with a JSON body parameter\n named ``body`` (the standards-template convention) plus a\n service-resolution hint comment, instead of sync ``def`` handlers\n taking ``payload``.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." }, "generate_models": { "name": "generate_models", @@ -205,7 +205,7 @@ "kind": "module", "path": "openapi_first.codegen_routes", "signature": null, - "docstring": "# Summary\n\nRoute handler code generation from OpenAPI specifications.\n\nThis module generates Python route handler stubs from an OpenAPI 3.x\nspecification. Each resource (derived from the first path segment)\ngets its own file under the output directory. Every OpenAPI operation\nmust define an ``operationId``, which becomes the handler function name.\n\nNotes:\n **Design constraints:**\n\n - ``operationId`` is required on every operation (matching\n ``binder.bind_routes``).\n - Handlers are stubs raising ``NotImplementedError``.\n - Sub-resources (e.g. ``/pets/{id}/photo``) are grouped with their\n parent resource (``pets``).\n - Parameter types and defaults are inferred from the spec.\n - ``response: Response`` is injected for non-200 success codes.\n - Resource names are sanitized to valid Python module identifiers,\n so a path such as ``/fetch-requests`` yields ``fetch_requests.py``.\n - Passing ``inject_request=True`` prepends ``request: Request`` to\n every handler (the composition-root / DI convention).\n - ``multipart/form-data`` and\n ``application/x-www-form-urlencoded`` request bodies expand to one\n parameter per schema property (``UploadFile`` / ``File`` / ``Form``)\n instead of an opaque ``payload: dict``.\n - Passing ``package=True`` writes ``/routes.py`` (with\n ``__init__.py`` sidecars) instead of a flat ``.py``.", + "docstring": "# Summary\n\nRoute handler code generation from OpenAPI specifications.\n\nThis module generates Python route handler stubs from an OpenAPI 3.x\nspecification. Each resource (derived from the first path segment)\ngets its own file under the output directory. Every OpenAPI operation\nmust define an ``operationId``, which becomes the handler function name.\n\nNotes:\n **Design constraints:**\n\n - ``operationId`` is required on every operation (matching\n ``binder.bind_routes``).\n - Handlers are stubs raising ``NotImplementedError``.\n - Sub-resources (e.g. ``/pets/{id}/photo``) are grouped with their\n parent resource (``pets``).\n - Parameter types and defaults are inferred from the spec.\n - ``response: Response`` is injected for non-200 success codes.\n - Resource names are sanitized to valid Python module identifiers,\n so a path such as ``/fetch-requests`` yields ``fetch_requests.py``.\n - Passing ``inject_request=True`` prepends ``request: Request`` to\n every handler (the composition-root / DI convention).\n - ``multipart/form-data`` and\n ``application/x-www-form-urlencoded`` request bodies expand to one\n parameter per schema property (``UploadFile`` / ``File`` / ``Form``)\n instead of an opaque ``payload: dict``.\n - Passing ``package=True`` writes ``/routes.py`` (with\n ``__init__.py`` sidecars) instead of a flat ``.py``.\n - Passing ``template_style=True`` emits ``async def`` handlers whose\n JSON body parameter is named ``body`` (matching the standards\n template) and adds a service-resolution hint comment.", "members": { "load_openapi": { "name": "load_openapi", @@ -218,8 +218,8 @@ "name": "generate_routes", "kind": "function", "path": "openapi_first.codegen_routes.generate_routes", - "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False) -> list[Path]", - "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." + "signature": "generate_routes(spec_path: Path, output_dir: Path, *, use_models: bool = False, models_module: str = 'models', inject_request: bool = False, package: bool = False, template_style: bool = False) -> list[Path]", + "docstring": "Generate route handler stubs from an OpenAPI specification.\n\nCreates one route module per resource in *output_dir*. Resources are\nderived from the first path segment (e.g. ``/pets`` and ``/pets/{id}``\nboth group under ``pets``) and sanitized to valid module identifiers.\n\nArgs:\n spec_path (Path):\n Path to the OpenAPI specification file (YAML or JSON).\n output_dir (Path):\n Directory where the generated route files are written.\n Created automatically if it does not exist.\n use_models (bool, optional):\n If ``True``, import Pydantic models from *models_module*\n for request-body schemas referenced via ``$ref``.\n models_module (str, optional):\n Dotted Python module path from which to import models\n (e.g. ``\"models\"``, ``\"app.models\"``).\n inject_request (bool, optional):\n If ``True``, prepend ``request: Request`` to every handler so the\n stub can later resolve services from ``request.app.state``.\n package (bool, optional):\n If ``True``, emit ``/routes.py`` packages (each with an\n ``__init__.py``) instead of flat ``.py`` modules.\n template_style (bool, optional):\n If ``True``, emit ``async def`` handlers with a JSON body parameter\n named ``body`` (the standards-template convention) plus a\n service-resolution hint comment, instead of sync ``def`` handlers\n taking ``payload``.\n\nReturns:\n list[Path]:\n Absolute paths of every generated route file.\n\nRaises:\n OpenAPISpecLoadError:\n If the spec cannot be loaded or validated.\n ValueError:\n If any operation is missing ``operationId``." } } }, diff --git a/standards/lib/01_build_recipe/index.html b/standards/lib/01_build_recipe/index.html index 8bfff72..2cbd208 100644 --- a/standards/lib/01_build_recipe/index.html +++ b/standards/lib/01_build_recipe/index.html @@ -811,7 +811,10 @@ environment-only; never commit them.

85 86 87 -88
# Build a new application from the Aetos Standards blueprint
+88
+89
+90
+91
# Build a new application from the Aetos Standards blueprint
 
 You are coding a complete new backend application conforming to the Aetos
 standard app blueprint. Execute every step in order. Only steps 2–4 use the
@@ -848,57 +851,60 @@ environment-only; never commit them.

    ```bash
    openapi-first models openapi.yaml -o _gen/models.py
    openapi-first routes openapi.yaml -o _gen/routes \
-     --use-models --models-module _gen.models --inject-request --package
+     --use-models --models-module _gen.models --inject-request --package --template-style
    ```
    `_gen/models.py` is the **wire contract** (Pydantic, one class per
    `components.schemas` entry). `_gen/routes/<resource>/routes.py` lists every
-   `operationId` with its exact parameter signature. Treat `_gen/` as generated
-   (never hand-edit it) — the canonical handlers live in `interfaces/`.
-
-4. **Write each CRUD slice (app-specific)** — per resource create the four-file
-   slice: `models.py` (domain model, **no `id`**), `ports/db.py`
-   (`class XDocument(BaseDocument, X)` — id comes from `BaseDocument`),
-   `ports/service.py`, and optional `converters.py`. Follow the CRUD template.
-
-5. **Wire the composition root** — in `init.py`, build leaf repositories, then
-   dependent repositories, then services; add infra adapters (db, cache, stores,
-   progress bus, fetchers). Expose `get_svc` overloads.
-
-6. **Add routes** — create `interfaces/<resource>/routes.py` thin handlers whose
-   names match the `operationId`s (use the signatures from step 3 as your
-   checklist) and re-export them in `interfaces/openapi.py`. Add the cache
-   decorator to read endpoints.
-
-7. **Bootstrap app and worker** — confirm `main.py` runs the OpenAPI-first app
-   with its lifespan. If the app has long-running jobs, adapt `worker.py` to poll
-   the job collection, run the pipeline, and publish progress.
-
-8. **Add pipeline families (only if data-driven)** — copy `pipelines/example/`
-   to `pipelines/<family>/` with `abc.py`, `impl.py`, `pipeline.yaml`,
-   `resources/`, and reuse `pipelines/shared/` (loader + registry). Register the
-   root. Add fixtures and a parity test.
-
-9. **Add infrastructure only where needed** — Tier A capability: use the
-   library. Tier B capability: copy the template. Never hand-write a Tier A
-   library.
-
-10. **Tests** — integration tests with fakes at the HTTP edge; unit tests for
-    domain and pipeline logic; a config-consistency test for rule-driven
-    pipelines. Include `tests/integration/test_openapi_app_builds.py` (imports
-    the app, asserts `import main` binds every operation).
-
-11. **Scripts and seeds** — operator scripts under `scripts/` using
-    `init()` / `get_svc()` / `shutdown()`; run as `python -m scripts.<name>`.
+   `operationId` with its exact parameter signature. `--template-style` makes
+   those stubs `async def` with a `body` parameter, matching the handlers you
+   keep in `interfaces/` (without it they are sync `def` stubs taking
+   `payload`; both bind the same way). Treat `_gen/` as generated
+   (never hand-edit it) — the canonical handlers live in `interfaces/`.
+
+4. **Write each CRUD slice (app-specific)** — per resource create the four-file
+   slice: `models.py` (domain model, **no `id`**), `ports/db.py`
+   (`class XDocument(BaseDocument, X)` — id comes from `BaseDocument`),
+   `ports/service.py`, and optional `converters.py`. Follow the CRUD template.
+
+5. **Wire the composition root** — in `init.py`, build leaf repositories, then
+   dependent repositories, then services; add infra adapters (db, cache, stores,
+   progress bus, fetchers). Expose `get_svc` overloads.
+
+6. **Add routes** — create `interfaces/<resource>/routes.py` thin handlers whose
+   names match the `operationId`s (use the signatures from step 3 as your
+   checklist) and re-export them in `interfaces/openapi.py`. Add the cache
+   decorator to read endpoints.
+
+7. **Bootstrap app and worker** — confirm `main.py` runs the OpenAPI-first app
+   with its lifespan. If the app has long-running jobs, adapt `worker.py` to poll
+   the job collection, run the pipeline, and publish progress.
+
+8. **Add pipeline families (only if data-driven)** — copy `pipelines/example/`
+   to `pipelines/<family>/` with `abc.py`, `impl.py`, `pipeline.yaml`,
+   `resources/`, and reuse `pipelines/shared/` (loader + registry). Register the
+   root. Add fixtures and a parity test.
+
+9. **Add infrastructure only where needed** — Tier A capability: use the
+   library. Tier B capability: copy the template. Never hand-write a Tier A
+   library.
+
+10. **Tests** — integration tests with fakes at the HTTP edge; unit tests for
+    domain and pipeline logic; a config-consistency test for rule-driven
+    pipelines. Include `tests/integration/test_openapi_app_builds.py` (imports
+    the app, asserts `import main` binds every operation).
 
-12. **Docs** — author GSDFC docstrings, generate `.pyi` stubs, mark the package
-    `py.typed`, and build the docs.
+11. **Scripts and seeds** — operator scripts under `scripts/` using
+    `init()` / `get_svc()` / `shutdown()`; run as `python -m scripts.<name>`.
 
-**Definition of done:**
-- `python -c "import main"` succeeds.
-- `pytest` passes.
-- The app serves every operation in the spec; the worker drains its queue.
-- No domain module imports a web framework.
-- No Tier A library was reimplemented.
+12. **Docs** — author GSDFC docstrings, generate `.pyi` stubs, mark the package
+    `py.typed`, and build the docs.
+
+**Definition of done:**
+- `python -c "import main"` succeeds.
+- `pytest` passes.
+- The app serves every operation in the spec; the worker drains its queue.
+- No domain module imports a web framework.
+- No Tier A library was reimplemented.

🧭 Where the steps land

@@ -1005,6 +1011,10 @@ the two mechanical parts of the build:

domain/<resource>/models.py is your business shape. If the spec's schema has a required id: str, keep it off the domain model and let the document get its id from BaseDocument. +
  • Use --template-style. Stubs are then async def ...(request, body: ...) + matching interfaces/; without it they are sync def ...(payload: ...). + Either binds (the router introspects the signature), but --template-style + keeps the scaffold consistent with the code you keep.
  • Resource names are sanitized. /fetch-requests → fetch_requests.py.
  • Multipart bodies expand to UploadFile = File(...) / Form(...) params.
  • _gen/ is disposable. Regenerate on every spec change; git diff shows diff --git a/standards/lib/04_spec_first_routing/index.html b/standards/lib/04_spec_first_routing/index.html index 5c63cf1..0e63bc7 100644 --- a/standards/lib/04_spec_first_routing/index.html +++ b/standards/lib/04_spec_first_routing/index.html @@ -772,6 +772,13 @@ can resolve operationId across the whole app:

    svc = request.app.state.db_resources["item_svc"] return await svc.create(Item.model_validate(body))     +

    Generated stubs. openapi-first routes emits this exact shape (async def +handlers taking body) when given --template-style; without that flag the +stub is def create_item(payload: ItemInput) — sync, with a payload +parameter. Both shapes bind identically: the router resolves handlers by +operationId and introspects the FastAPI signature, and it never inspects the +request body. Pass --template-style so the generated signatures match the +handlers you keep in interfaces/.

    Conventions

    • operationId naming: list_<resource>, create_<resource>, @@ -794,6 +801,10 @@ just omit the security keys:

      global security block exists. Leaving every operation without a security key is the cleanest authless spec.
    +

    Profile endpoints (/users/me) and profile headers (x-profile) still exist in +an authless app. Define a stand-in identity in the composition root — for +example get_current_user returns or lazily creates a fixed "Demo User" — so +handlers stay auth-agnostic and only the wiring changes when real auth arrives.

    The handlers, services, and composition root are unchanged. When auth is needed later, add the global security block and an introspecting dependency; only handlers that must opt out get security: [].

    @@ -804,7 +815,7 @@ handlers that must opt out get security: [].

    2 3
    openapi-first models openapi.yaml -o _gen/models.py
 openapi-first routes openapi.yaml -o _gen/routes \
-  --use-models --models-module _gen.models --inject-request --package
+  --use-models --models-module _gen.models --inject-request --package --template-style

    Prefer codegen over hand-writing boilerplate. The result is the wire contract and a set of operationId-named signatures — not the canonical handlers.

    diff --git a/standards/lib/05_crud_template/index.html b/standards/lib/05_crud_template/index.html index e9473db..b5d8f5e 100644 --- a/standards/lib/05_crud_template/index.html +++ b/standards/lib/05_crud_template/index.html @@ -427,6 +427,15 @@ +
    • + +
  • + + + Nested references + + +
  • @@ -665,6 +674,15 @@ +
    • + +
  • + + + Nested references + + +
  • @@ -802,10 +820,23 @@ The repository factory selects the right repository flavour.

    return repo_factory( collection="items", document=ItemDocument, - populate_rules=[PopulateRule(field="owner", collection="users")], + populate_rules=[PopulateRule(field_name="owner", collection_name="users")], cache=None, # or a CachedBaseRepository config ) +

    Nested references

    +

    The wire contract often declares a reference as an expanded object +(expense.entity: Entity), while the domain model stays id-free. Pick one +convention and apply it consistently:

    +
      +
    • Embed (default for small, stable refs) — store the object as a dict / + list[dict] on the document; the API returns it verbatim. No populate wiring.
      • +
    • Populate — store an id and declare a PopulateRule; mongo-ops expands it + on read. The referenced repo must be registered with the PopulationEngine.
      • +
    • Id only — store the id and let the client resolve it.
      • +
    +

    For populate-backed repositories change a reference with update, not patch: +patch rejects changes to populated FK fields by design.

    ports/service.py

    The service is built through the DI library, injecting the repository. The generic CRUD base provides create/read/update/patch/delete/search/....

    diff --git a/standards/lib/09_infra_adaptors/index.html b/standards/lib/09_infra_adaptors/index.html index d2a91d7..15ce400 100644 --- a/standards/lib/09_infra_adaptors/index.html +++ b/standards/lib/09_infra_adaptors/index.html @@ -738,7 +738,7 @@ Tier A / B / C split.

    • 1
-2
    PopulateRule(field="owner", collection="users",
+2
    PopulateRule(field_name="owner", collection_name="users",
              nested_rules=[], max_depth=2, projection=None)
      diff --git a/standards/lib/search/search_index.json b/standards/lib/search/search_index.json index de0fab5..f2dfe1a 100644 --- a/standards/lib/search/search_index.json +++ b/standards/lib/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Standards Reference","text":"

      The reference half of the Standards description: the enforceable rules, patterns, and the internal library catalog that agents consult while coding.

      • Build Recipe \u2014 the ordered, agentic checklist for a new app.
      • Layering Rules \u2014 the strict request \u2192 domain \u2192 repository layering and the rules that enforce it.
      • Composition Root \u2014 a single composition point wiring the whole app.
      • Spec-First Routing \u2014 the OpenAPI contract as the routing source of truth.
      • CRUD Template \u2014 the standardized generic resource endpoint scaffold.
      • Pipeline Scaffold \u2014 the deterministic pipeline skeleton for multi-stage jobs.
      • Worker & Progress \u2014 queued work, progress reporting, and visibility.
      • Human-in-the-Loop \u2014 guardrails and approvals inside automations.
      • Infra Adapters \u2014 how infrastructure connects without leaking into domain.
      • Genericity Matrix \u2014 what is library-backed, template-provided, or authored.
      • Library Registry \u2014 the catalog of internal libraries behind all of it.

      Read the Wiki for the philosophy and rationale; these pages are the reference that mirrors the standards MCP bundle.

      "},{"location":"01_build_recipe/","title":"Build Recipe","text":""},{"location":"01_build_recipe/#summary","title":"Summary","text":"

      The ordered, deterministic checklist for generating a new application of the standard shape. Every step is fixed; only the resource model (step 2), the codegen pass (step 3), and the pipeline family (step 8) need app-specific work.

      This page is served by the standards MCP as docs://modules/build_recipe. The start_app prompt on that server collects the two axes and hands off here; pull exact files with get_app_component.

      "},{"location":"01_build_recipe/#bootstrap","title":"\ud83d\ude80 Bootstrap","text":"

      From the standards docs checkout:

      cp -r <docs>/standards/src/templates/app apps/<your-app>\ncd apps/<your-app>\n

      Install the internal libraries \u2014 editable local checkouts (preferred in development) or the private index (CI / fresh machines). Credentials are environment-only; never commit them.

      # local checkouts\npip install -e ../aetos/openapi-first -e ../aetos/mongo-ops -e ../aetos/hexa \\\n            -e ../aetos/dagpipe -e ../aetos/doc-forge\n# private index\npip install --extra-index-url \\\n  \"https://$PIP_REGISTRY_USERNAME:$PIP_REGISTRY_PASSWORD@pip.aetoskia.com/simple\" .\n
      "},{"location":"01_build_recipe/#agent-prompt-copyable","title":"\ud83d\udccb Agent prompt (copyable)","text":"
      # Build a new application from the Aetos Standards blueprint\n\nYou are coding a complete new backend application conforming to the Aetos\nstandard app blueprint. Execute every step in order. Only steps 2\u20134 use the\napp-specific details provided by the user; everything else is deterministic.\n\n## Before writing any code\n\nConnect to the `standards` MCP server and read:\n- `docs://nav` (the module list)\n- `docs://modules/genericity_matrix` (Tier A library / Tier B template / Tier C authored)\n- `docs://modules/layering_rules` (the dependency rules)\n\nThen:\n- call `find_library` with `db`, `di`, `routing`, `ingestion`, `pipeline`, `docs`\n  and note the Tier A libraries that already exist \u2014 never reimplement one;\n- call `list_app_components()` and pull each file you need with\n  `get_app_component(kind, resource=..., family=...)` instead of writing it from\n  scratch.\n\n## Steps\n\n1. **Scaffold** \u2014 copy `standards/src/templates/app/` into `apps/<your-app>/`\n   (or pull the same tree via `get_app_template`). Keep the tree: `domain/`,\n   `infra/`, `interfaces/`, `pipelines/`, `tests/`, `scripts/`, plus\n   `pyproject.toml`, `openapi.yaml`, `config.py`, `init.py`, `main.py`,\n   `worker.py`, `Dockerfile`, `docker-compose.yaml`, `.env.example`.\n\n2. **Author the spec (app-specific)** \u2014 write `openapi.yaml`: resources,\n   schemas, `operationId`s, security, and extension metadata. This is the\n   contract and drives all routing. Every operation MUST have an `operationId`.\n\n3. **Generate the scaffold (codegen)** \u2014 from the spec, generate the wire\n   models and the exact handler signatures:\n   ```bash\n   openapi-first models openapi.yaml -o _gen/models.py\n   openapi-first routes openapi.yaml -o _gen/routes \\\n     --use-models --models-module _gen.models --inject-request --package\n   ```\n   `_gen/models.py` is the **wire contract** (Pydantic, one class per\n   `components.schemas` entry). `_gen/routes/<resource>/routes.py` lists every\n   `operationId` with its exact parameter signature. Treat `_gen/` as generated\n   (never hand-edit it) \u2014 the canonical handlers live in `interfaces/`.\n\n4. **Write each CRUD slice (app-specific)** \u2014 per resource create the four-file\n   slice: `models.py` (domain model, **no `id`**), `ports/db.py`\n   (`class XDocument(BaseDocument, X)` \u2014 id comes from `BaseDocument`),\n   `ports/service.py`, and optional `converters.py`. Follow the CRUD template.\n\n5. **Wire the composition root** \u2014 in `init.py`, build leaf repositories, then\n   dependent repositories, then services; add infra adapters (db, cache, stores,\n   progress bus, fetchers). Expose `get_svc` overloads.\n\n6. **Add routes** \u2014 create `interfaces/<resource>/routes.py` thin handlers whose\n   names match the `operationId`s (use the signatures from step 3 as your\n   checklist) and re-export them in `interfaces/openapi.py`. Add the cache\n   decorator to read endpoints.\n\n7. **Bootstrap app and worker** \u2014 confirm `main.py` runs the OpenAPI-first app\n   with its lifespan. If the app has long-running jobs, adapt `worker.py` to poll\n   the job collection, run the pipeline, and publish progress.\n\n8. **Add pipeline families (only if data-driven)** \u2014 copy `pipelines/example/`\n   to `pipelines/<family>/` with `abc.py`, `impl.py`, `pipeline.yaml`,\n   `resources/`, and reuse `pipelines/shared/` (loader + registry). Register the\n   root. Add fixtures and a parity test.\n\n9. **Add infrastructure only where needed** \u2014 Tier A capability: use the\n   library. Tier B capability: copy the template. Never hand-write a Tier A\n   library.\n\n10. **Tests** \u2014 integration tests with fakes at the HTTP edge; unit tests for\n    domain and pipeline logic; a config-consistency test for rule-driven\n    pipelines. Include `tests/integration/test_openapi_app_builds.py` (imports\n    the app, asserts `import main` binds every operation).\n\n11. **Scripts and seeds** \u2014 operator scripts under `scripts/` using\n    `init()` / `get_svc()` / `shutdown()`; run as `python -m scripts.<name>`.\n\n12. **Docs** \u2014 author GSDFC docstrings, generate `.pyi` stubs, mark the package\n    `py.typed`, and build the docs.\n\n**Definition of done:**\n- `python -c \"import main\"` succeeds.\n- `pytest` passes.\n- The app serves every operation in the spec; the worker drains its queue.\n- No domain module imports a web framework.\n- No Tier A library was reimplemented.\n
      "},{"location":"01_build_recipe/#where-the-steps-land","title":"\ud83e\udded Where the steps land","text":"

      The steps map onto the layers as follows:

      Step Package / file Depends on 1 \u00b7 Scaffold whole templates/app/ \u2014 2 \u00b7 Spec openapi.yaml \u2014 3 \u00b7 Codegen _gen/ (generated) spec, openapi-first 4 \u00b7 CRUD slices domain/<resource>/ spec, CRUD template 5 \u00b7 Wiring init.py every slice 6 \u00b7 Routes interfaces/<resource>/ services, openapi route registry 7 \u00b7 Bootstrap main.py, worker.py composition root 8 \u00b7 Pipelines pipelines/<family>/, shared/ library pipeline engine 9 \u00b7 Infra infra/ domain ports 10 \u00b7 Tests tests/ app + fixtures 11 \u00b7 Scripts scripts/ composition root 12 \u00b7 Docs docstrings + docs build code, Doc Forge"},{"location":"01_build_recipe/#the-codegen-fast-path-in-detail","title":"\ud83d\udd27 The codegen fast path, in detail","text":"

      openapi-first is spec \u2192 code only (it never generates a spec). Use it to erase the two mechanical parts of the build:

      Generated artifact Replaces Notes _gen/models.py hand-writing Pydantic wire models one class per components.schemas entry _gen/routes/<resource>/routes.py hand-typing handler signatures names == operationIds; request: Request injected

      Rules of the road:

      • Wire models are not domain models. _gen/models.py is the HTTP contract; domain/<resource>/models.py is your business shape. If the spec's schema has a required id: str, keep it off the domain model and let the document get its id from BaseDocument.
      • Resource names are sanitized. /fetch-requests \u2192 fetch_requests.py.
      • Multipart bodies expand to UploadFile = File(...) / Form(...) params.
      • _gen/ is disposable. Regenerate on every spec change; git diff shows exactly what moved.
      "},{"location":"01_build_recipe/#definition-of-done","title":"\u26a0\ufe0f Definition of done","text":"
      • python -c \"import main\" succeeds.
      • pytest passes.
      • The app serves the spec's operations; the worker drains its queue.
      • No domain module imports a web framework.
      • No Tier A library was reimplemented.
      "},{"location":"01_build_recipe/#related","title":"Related","text":"
      • 02 \u2013 Layering Rules \u00b7 05 \u2013 CRUD Template \u00b7 06 \u2013 Pipeline Scaffold \u00b7 12 \u2013 Genericity Matrix
      "},{"location":"02_layering_rules/","title":"Layering Rules","text":""},{"location":"02_layering_rules/#summary","title":"Summary","text":"

      The dependency direction is fixed. Violating it is a design defect.

      "},{"location":"02_layering_rules/#allowed-imports","title":"Allowed imports","text":"
      interfaces/*      ->  domain/*            (services, ports, models)\ninterfaces/*      ->  infra/adapters/*    (only read-only value objects)\ndomain/*          ->  domain/*            (same layer)\ndomain/*          ->  pipelines/*         (call a pipeline via a port)\ndomain/*/ports/*  ->  infra/*             (bind adapters + libraries at the seam)\ninfra/adapters/*  ->  domain/ports/*      (implement the interface)\nmain.py/init.py   ->  everything          (composition root only)\n
      "},{"location":"02_layering_rules/#forbidden-imports","title":"Forbidden imports","text":"
      domain/*          ->  interfaces/*        (never)\ndomain/*          ->  fastapi/starlette   (never)\ninfra/adapters/*  ->  interfaces/*        (never)\ninterfaces/*      ->  infra/db/*          (state lives in resources, not imports)\n
      "},{"location":"02_layering_rules/#why","title":"Why","text":"
      • The business logic is testable without a server or an event loop.
      • Adapters can be swapped (in-memory fake, real database) without touching business logic.
      • The HTTP edge stays thin and replaceable.
      "},{"location":"02_layering_rules/#the-portadapter-contract","title":"The port/adapter contract","text":"

      The ports/ package is the seam where the domain meets infrastructure. Each domain/<resource>/ports/ module declares the resource's models/document shape and exposes factory functions that bind a concrete adapter \u2014 so a port module is allowed to import from infra/ and from the persistence libraries. Everything outside ports/ in domain/ stays framework- and driver-free.

      domain/<resource>/ports/db.py       -> document model + create_<resource>_repo()\ndomain/<resource>/ports/service.py  -> service + get_<resource>_service()\ninfra/db/repos/<vendor>.py          -> repository adapter factory\ninfra/adapters/service.py           -> generic CRUD service base\n

      The composition root calls the port factories; nothing else constructs an adapter.

      "},{"location":"02_layering_rules/#enforcement","title":"Enforcement","text":"
      • domain/ must never import the web framework (fastapi / starlette) or interfaces/. Only the ports/ seam, infra/, and the composition root touch drivers.
      • Reviewers (and CI) reject import fastapi inside domain/ or infra/.
      "},{"location":"03_composition_root/","title":"Composition Root","text":""},{"location":"03_composition_root/#summary","title":"Summary","text":"

      All wiring happens in exactly one module. It builds a flat resource registry and exposes it to the HTTP layer and to scripts.

      "},{"location":"03_composition_root/#the-resource-registry","title":"The resource registry","text":"

      init.py returns a plain dict[str, Any]. Keys follow two conventions:

      • <resource>_repo \u2014 a repository adapter for a resource.
      • <resource>_svc \u2014 a service for a resource.

      Plus named infrastructure adaptors:

      • http_cache_store \u2014 response cache backend.
      • progress_bus \u2014 progress/event bus.
      • <vendor>_<kind> \u2014 fetchers, clients, stores.

      The API stores the dict on app.state.db_resources during lifespan startup. Handlers read from it; nothing imports concrete adapters.

      "},{"location":"03_composition_root/#shape","title":"Shape","text":"
      _resources: dict = {}\n_initialized = False\n\nasync def init(settings=None) -> dict:\n    global _initialized\n    if _initialized:\n        return _resources\n\n    await setup_database(uri=settings.mongo_uri, db_name=settings.mongo_db)\n\n    # 1. leaf repositories\n    user_repo = create_user_repo()\n\n    # 2. dependent repositories (dependencies passed by keyword)\n    wallet_repo = create_wallet_repo(user_repo=user_repo)\n\n    # 3. services (build via the DI library, injecting the repo)\n    _resources[\"user_repo\"]   = user_repo\n    _resources[\"wallet_repo\"] = wallet_repo\n    _resources[\"user_svc\"]    = get_user_service(user_repo)\n    _resources[\"wallet_svc\"]  = get_wallet_service(wallet_repo)\n\n    # 4. infrastructure adaptors (optional)\n    _resources[\"http_cache_store\"] = HttpCacheStore(...)\n    _resources[\"progress_bus\"]     = ProgressEventBus()\n\n    _initialized = True\n    return _resources\n\nasync def shutdown() -> None:\n    global _initialized\n    await close_stores(_resources)\n    await shutdown_database()\n    _resources.clear()\n    _initialized = False\n
      "},{"location":"03_composition_root/#typed-accessors","title":"Typed accessors","text":"

      Expose a get_svc(name) helper with one @overload per key so scripts get typing without importing concrete classes:

      @overload\ndef get_svc(name: Literal[\"user_svc\"]) -> UserService: ...\n@overload\ndef get_svc(name: Literal[\"wallet_svc\"]) -> WalletService: ...\n\ndef get_svc(name: str):\n    return _resources[name]\n

      Also expose get_db() (raw database handle) and pure helpers such as compute_fingerprint(payload) (canonical hash used for deduplication).

      "},{"location":"03_composition_root/#rules","title":"Rules","text":"
      • init() is idempotent.
      • The composition root is the only place that imports concrete adapters.
      • Handlers never call init(); they read request.app.state.db_resources.
      • Scripts call init() / shutdown() around their work.
      "},{"location":"04_spec_first_routing/","title":"Spec-First Routing","text":""},{"location":"04_spec_first_routing/#summary","title":"Summary","text":"

      The OpenAPI document is the single source of truth for the HTTP contract. Handlers are bound to it by name; there is no decorator router.

      "},{"location":"04_spec_first_routing/#the-contract","title":"The contract","text":"
      • openapi.yaml (or .json) declares paths, schemas, parameters, responses, and security.
      • A custom operationId names the handler function.
      • The framework (openapi_first) validates requests/responses against the spec and dispatches to the handler whose name equals operationId.
      "},{"location":"04_spec_first_routing/#app-bootstrap","title":"App bootstrap","text":"
      # main.py\nfrom openapi_first.app import OpenAPIFirstApp\n\napp = OpenAPIFirstApp(\n    openapi_path=\"openapi.yaml\",\n    routes_module=interfaces.openapi,   # module whose namespace holds handlers\n    title=\"<App> API\",\n    version=\"0.1.0\",\n    lifespan=lifespan,\n)\n

      lifespan calls the composition root:

      @asynccontextmanager\nasync def lifespan(app):\n    app.state.db_resources = await init()\n    yield\n    await shutdown()\n
      "},{"location":"04_spec_first_routing/#handler-registry","title":"Handler registry","text":"

      interfaces/openapi.py is a flat re-export of every handler, so the framework can resolve operationId across the whole app:

      from interfaces.item.routes import *\nfrom interfaces.wallet.routes import *\nfrom interfaces.progress.sse import *\n
      "},{"location":"04_spec_first_routing/#handler-shape","title":"Handler shape","text":"

      Handlers are thin and synchronous with respect to business logic:

      async def create_item(request: Request, body: dict):\n    svc = request.app.state.db_resources[\"item_svc\"]\n    return await svc.create(Item.model_validate(body))\n
      "},{"location":"04_spec_first_routing/#conventions","title":"Conventions","text":"
      • operationId naming: list_<resource>, create_<resource>, get_<resource>, update_<resource>, delete_<resource>.
      • Read-only endpoints may use a cache decorator keyed on path + query + user.
      • Validation errors -> 400; not found -> 404; delete -> 204.
      • Security is declared once in the spec and enforced by the framework; handlers contain no auth code.
      "},{"location":"04_spec_first_routing/#authless-apps","title":"Authless apps","text":"

      Auth is deferred in the standard: the same spec-first app runs with or without it, because security lives entirely in the spec, never in handlers. For an app that needs no auth yet, do not invent per-handler auth checks \u2014 just omit the security keys:

      • Remove the top-level security: block.
      • Remove components.securitySchemes.
      • Do not add security: [] to operations. An explicit empty array is a per-operation override that means \"public\", and it only matters when a global security block exists. Leaving every operation without a security key is the cleanest authless spec.

      The handlers, services, and composition root are unchanged. When auth is needed later, add the global security block and an introspecting dependency; only handlers that must opt out get security: [].

      "},{"location":"04_spec_first_routing/#codegen","title":"Codegen","text":"

      openapi_first scaffolds models and route stubs from the spec (see the Build Recipe, step 3):

      openapi-first models openapi.yaml -o _gen/models.py\nopenapi-first routes openapi.yaml -o _gen/routes \\\n  --use-models --models-module _gen.models --inject-request --package\n

      Prefer codegen over hand-writing boilerplate. The result is the wire contract and a set of operationId-named signatures \u2014 not the canonical handlers.

      "},{"location":"05_crud_template/","title":"CRUD Template","text":""},{"location":"05_crud_template/#summary","title":"Summary","text":"

      Every resource follows the same four-file slice. Repetition is intentional: it makes the codebase predictable and generatable.

      "},{"location":"05_crud_template/#the-slice","title":"The slice","text":"
      domain/<resource>/\n    models.py               # pure Pydantic model (no DB fields)\n    ports/\n        db.py               # document model + repository factory\n        service.py          # service class + factory\n        converters.py       # (optional) transforms\n
      "},{"location":"05_crud_template/#the-id-rule-model-vs-document","title":"The id rule (model vs. document)","text":"

      Two different objects, two different jobs:

      Object Lives in Has id? Purpose Domain model domain/<resource>/models.py No business shape, framework-free Document domain/<resource>/ports/db.py Yes (from BaseDocument) Mongo persistence shape

      BaseDocument (mongo-ops) supplies id: Optional[PyObjectId] (aliased to _id) plus created_at / updated_at. The domain model must not declare id; Mongo owns it.

      Trap \u2014 generated wire models. openapi-first models produces the HTTP contract (see the Build Recipe, step 3). Response schemas frequently declare a required id: str, so a generated class is not a domain model. If you write class ItemDocument(Item, BaseDocument) and Item re-exports a required id: str, that field shadows BaseDocument.id and no Mongo _id can ever be set. Fix it one of three ways:

      1. Preferred: keep the domain model id-free (hand-author it from the CRUD template), and keep the required id only on the wire model.
      2. Order the bases so the document wins: class ItemDocument(BaseDocument, Item).
      3. Keep a separate document type and translate with ports/converters.py.
      "},{"location":"05_crud_template/#modelspy","title":"models.py","text":"

      A framework-free Pydantic model describing the business shape (no id).

      class Item(BaseModel):\n    name: str\n    quantity: int = 0\n
      "},{"location":"05_crud_template/#portsdbpy","title":"ports/db.py","text":"

      The persistence model extends the business model and adds database concerns. The repository factory selects the right repository flavour.

      # Item is id-free; BaseDocument adds id/created_at/updated_at.\nclass ItemDocument(Item, BaseDocument):\n    pass\n\ndef create_item_repo(*, owner_repo=None) -> ItemRepository:\n    return repo_factory(\n        collection=\"items\",\n        document=ItemDocument,\n        populate_rules=[PopulateRule(field=\"owner\", collection=\"users\")],\n        cache=None,               # or a CachedBaseRepository config\n    )\n
      "},{"location":"05_crud_template/#portsservicepy","title":"ports/service.py","text":"

      The service is built through the DI library, injecting the repository. The generic CRUD base provides create/read/update/patch/delete/search/....

      class ItemService(CrudsServiceAdaptor[Item, ItemDocument]):\n    model = Item\n    schema = ItemDocument\n\ndef get_item_service(repo) -> ItemService:\n    return build(ItemService, instances={\"repo\": repo})\n
      "},{"location":"05_crud_template/#naming-conventions","title":"Naming conventions","text":"Concept Convention collection plural snake case (items) document <Resource>Document repository factory create_<resource>_repo service <Resource>Service service factory get_<resource>_service resource key <resource>_svc operationId list_items, create_item, ..."},{"location":"05_crud_template/#optional-files","title":"Optional files","text":"
      • converters.py \u2014 pure functions mapping between domain types.
      • ports/enrichment.py \u2014 adapts a pipeline into an async callable the service can invoke (used when a resource is populated by a pipeline).
      "},{"location":"06_pipeline_scaffold/","title":"Pipeline Scaffold","text":""},{"location":"06_pipeline_scaffold/#summary","title":"Summary","text":"

      Data-driven processing is expressed as a composition graph. The graph is built by a DI library; execution is driven by a DAG engine. Stages are pure and individually testable.

      "},{"location":"06_pipeline_scaffold/#when-to-use-a-pipeline","title":"When to use a pipeline","text":"

      Use a pipeline when a resource is produced by a multi-step transformation of an input (a document, a file, a message) rather than by a simple CRUD write.

      "},{"location":"06_pipeline_scaffold/#structure-of-a-family","title":"Structure of a family","text":"

      A family is one input shape (e.g. a file format or a message kind). Each family has the same skeleton:

      pipelines/<family>/\n    abc.py            # port contracts (stage interfaces + root interface)\n    impl.py           # concrete stages + root implementation\n    pipeline.yaml     # graph spec: nodes, edges, roots, schema\n    resources/\n        pipeline.yaml # runtime spec consumed by the loader\n        patterns/*.yaml  # rule fragments (pattern registry)\n    banks/<variant>.py   # optional variants that re-pin a few slots\n
      "},{"location":"06_pipeline_scaffold/#shared-scaffolding","title":"Shared scaffolding","text":"
      pipelines/shared/\n    ports.py     # RegistryPort, FinalizePort, etc.\n    loader.py    # build(root_cls) -> wires the graph + engine\n    registry.py  # resolve_root(...) -> root class for a (kind, variant)\n
      "},{"location":"06_pipeline_scaffold/#the-loader","title":"The loader","text":"
      def build(root_cls, *, instances=None, config=None):\n    # merge resources/patterns/*.yaml into a pattern registry\n    # bind registry/config into each graph node by id\n    # build the schema (state) class and the engine\n    root = root_cls()\n    root._engine = Engine(graph, state_cls, payload_cls)\n    return root\n
      "},{"location":"06_pipeline_scaffold/#the-registry","title":"The registry","text":"

      A single lookup replaces hand-rolled if/else selection:

      _FAMILIES = {\n    \"csv\":  {\"acme\": {\"heuristic\": AcmeCsvPipeline}},\n    \"json\": {\"globex\": {\"heuristic\": GlobexJsonPipeline}},\n}\n\ndef resolve_root(kind, variant, mode=\"heuristic\"):\n    return _FAMILIES[kind][variant][mode]\n

      Adding a new variant means adding one class and one registry entry.

      "},{"location":"06_pipeline_scaffold/#driving-a-pipeline","title":"Driving a pipeline","text":"

      Stages are driven explicitly so intermediate state can be persisted:

      pipeline = build(resolve_root(kind, variant))\ncontent = pipeline.ingest.materialize(raw_bytes)\nrecords = pipeline.reader.read(content)\nitems   = pipeline.normalize.build(records)\n
      "},{"location":"06_pipeline_scaffold/#testing","title":"Testing","text":"

      Because stages are pure, test each stage in isolation and test the assembled root against fixture inputs whose expected outputs are stored as golden files. A parity test replays fixtures headlessly.

      "},{"location":"07_worker_and_progress/","title":"Worker & Progress","text":""},{"location":"07_worker_and_progress/#summary","title":"Summary","text":"

      Long-running work runs in a background worker. Jobs are durable documents that move through an explicit status machine. Progress is streamed to clients.

      "},{"location":"07_worker_and_progress/#the-status-machine","title":"The status machine","text":"
      pending -> processing -> completed\n                     \\-> failed    (retried up to N times)\n                     \\-> paused    (waiting for human input)\n
      • pending \u2014 queued, eligible for pickup.
      • processing \u2014 claimed by a worker.
      • paused \u2014 blocked on a human-in-the-loop question.
      • failed \u2014 error; retry_count increments until a max, then skipped.
      • completed \u2014 terminal success.
      "},{"location":"07_worker_and_progress/#the-poll-loop","title":"The poll loop","text":"
      async def poll(resources):\n    backoff = MIN_POLL\n    while True:\n        jobs = await job_svc.search(filter={\"status\": \"pending\"}, limit=100)\n        backoff = MIN_POLL if jobs else min(backoff * 2, MAX_POLL)\n        for job in jobs:\n            if (job.retry_count or 0) >= MAX_RETRIES:\n                continue\n            await job_svc.update_status(job.id, \"processing\")\n            try:\n                await process(job, resources)\n            except PauseForResolution:\n                await job_svc.update_status(job.id, \"paused\")\n            except Exception as e:\n                await job_svc.update_status(\n                    job.id, \"failed\", error=str(e),\n                    retry_count=(job.retry_count or 0) + 1,\n                )\n        await asyncio.sleep(backoff)\n

      Tunables live in environment variables: WORKER_POLL_MIN, WORKER_POLL_MAX, WORKER_MAX_RETRIES.

      "},{"location":"07_worker_and_progress/#progress-events","title":"Progress events","text":"

      A worker publishes structured progress events to a bus (a port). The HTTP edge exposes a streaming endpoint that replays events for a given job id.

      class ProgressEvent(BaseModel):\n    job_id: str\n    step: str\n    status: str            # started | progress | completed | skipped | paused\n    message: dict | None = None\n

      The bus is an in-memory adapter by default; because it is a port, it can be swapped for a durable broker. Emit from the orchestrator at each stage:

      await self.bus.publish(ProgressEvent(job_id=jid, step=\"normalize\",\n                                     status=\"completed\", message={\"count\": n}))\n
      "},{"location":"07_worker_and_progress/#streaming-endpoint","title":"Streaming endpoint","text":"
      async def stream_job_events(request, id):\n    bus = request.app.state.db_resources.get(\"progress_bus\")\n    if bus is None:\n        return {\"events\": []}\n\n    async def gen():\n        sent = 0\n        while True:\n            events = bus.get_events(id)\n            for e in events[sent:]:\n                yield f\"data: {e.model_dump_json(exclude_none=True)}\\n\\n\"\n                sent += 1\n            await asyncio.sleep(0.5)\n\n    return StreamingResponse(gen(), media_type=\"text/event-stream\")\n

      Known limitation. The default in-memory bus is per-process. If the worker and the API run as separate processes, events published by the worker are not visible to the API. For production, back the bus with a shared broker. The port boundary makes this a one-adapter change.

      "},{"location":"08_human_in_loop/","title":"Human-in-the-Loop","text":""},{"location":"08_human_in_loop/#summary","title":"Summary","text":"

      Some inputs are ambiguous and must be resolved by a human before processing can continue. The blueprint models this as a durable pause: an exception is raised, a question is recorded, the job pauses, and later resumes.

      "},{"location":"08_human_in_loop/#the-mechanism","title":"The mechanism","text":"
      1. During processing, a stage raises a domain exception carrying the context needed to ask the question:
      raise PauseForResolution(\n    job_id=jid,\n    reference=\"<stable key for this decision>\",\n    options=[...],\n    context={...},\n)\n
      1. The orchestrator catches it, records a question document, and re-raises so the worker marks the job paused.
      2. A client lists pending questions and answers one:
      GET  /jobs/{id}/questions            -> list pending questions\nPOST /questions/{qid}/resolve  {choice}  -> record the choice\n
      1. When the last pending question is resolved, the resolve endpoint flips the parent job back to pending.
      2. The worker picks it up again, rebuilds a resolution map from previously answered questions, and seeds it into the pipeline so the same decision is not asked twice.
      "},{"location":"08_human_in_loop/#the-question-document","title":"The question document","text":"
      class Question(BaseDocument):\n    job_id: str\n    reference: str            # stable key: stage|field|value\n    status: str               # pending | resolved\n    options: list[dict]\n    context: dict\n    chosen: dict | None = None\n    resolved_at: str | None = None\n
      "},{"location":"08_human_in_loop/#resume-map","title":"Resume map","text":"

      The worker turns answered questions into a lookup keyed by reference, then injects it into the pipeline so that stage can skip re-raising:

      resolved = {q.reference: q.chosen for q in await svc.search(\n    filter={\"job_id\": jid, \"status\": \"resolved\"})}\n\npipeline = build(root, instances={\"pause\": handler}, config={\"resolved\": resolved})\n
      "},{"location":"08_human_in_loop/#resolve-endpoint","title":"Resolve endpoint","text":"
      async def resolve_question(request, id, body):\n    svc = request.app.state.db_resources[\"question_svc\"]\n    q = await svc.read(id)\n    await svc.update(id, {\"chosen\": body[\"chosen\"], \"status\": \"resolved\",\n                          \"resolved_at\": now()})\n\n    pending = await svc.search(filter={\"job_id\": q.job_id, \"status\": \"pending\"},\n                               limit=1)\n    if not pending:\n        jobs = request.app.state.db_resources[\"job_svc\"]\n        job = await jobs.read(q.job_id)\n        if job and job.status == \"paused\":\n            await jobs.update_status(q.job_id, \"pending\")\n    return q\n
      "},{"location":"08_human_in_loop/#cli-parity","title":"CLI parity","text":"

      The same flow is available through scripts/CLI: list, get, create, questions, resolve, retry.

      "},{"location":"09_infra_adaptors/","title":"Infra Adapters","text":""},{"location":"09_infra_adaptors/#summary","title":"Summary","text":"

      All external systems are reached through adapters in infra/. Prefer an existing internal library over writing a new adapter; where no library exists, copy the Tier B scaffold from templates/app. See the Genericity Matrix for the Tier A / B / C split.

      "},{"location":"09_infra_adaptors/#database-mongodb","title":"Database (MongoDB)","text":"
      • A single connection manager owns the client lifecycle. setup_database(uri, db_name) / shutdown_database().
      • A repository factory selects the repository flavour:
      repo_factory(collection, document, engine=None,\n             populate_rules=None, cache=None)\n
      • base repository \u2014 plain CRUD;
      • populating repository \u2014 resolves referenced documents (foreign keys);

      • cached repository \u2014 an LRU/TTL cache in front of reads.

      • Referenced documents are described declaratively:

      PopulateRule(field=\"owner\", collection=\"users\",\n             nested_rules=[], max_depth=2, projection=None)\n
      • Transactions use a session context manager provided by the library.

      Tier A \u2014 Library (Mongo Ops). Do not write raw Motor calls in the domain; consult the registry and the library's MCP before touching the driver.

      "},{"location":"09_infra_adaptors/#http-response-cache","title":"HTTP response cache","text":"

      Tier B \u2014 template. Copy from templates/app; not a standalone library.

      A generic GET cache sits behind a decorator.

      @cache_response(ttl=60)\nasync def list_items(request): ...\n
      • Keyed on path + query + user.
      • No-op when no store is wired.
      • invalidate(*resources) bumps a generation counter for write-triggered invalidation.
      "},{"location":"09_infra_adaptors/#event-progress-bus","title":"Event / progress bus","text":"

      Tier B \u2014 template. Copy from templates/app.

      A port with an in-memory default adapter (publish, get_events). Swap for a durable broker in production (see Worker & Progress).

      "},{"location":"09_infra_adaptors/#external-clients-ingestion","title":"External clients (ingestion)","text":"

      External fetchers (files, mail, HTTP) are adapters that wrap ingestion libraries. Construct them in the composition root and inject into orchestrators.

      Tier A \u2014 Libraries: Omniread (formats) and Mail Intake (email).

      "},{"location":"09_infra_adaptors/#llm-clients","title":"LLM clients","text":"

      Tier B \u2014 template. Copy the pool scaffold from templates/app.

      When a pipeline calls a model provider, route through a pool adaptor that handles provider selection, retries, and client lifecycle:

      pool = setup_async_pool(provider=settings.llm_provider)\nresult = await pool.complete(model=\"...\", messages=[...])\n

      Providers are selected by LLM_PROVIDER and configured by environment.

      "},{"location":"09_infra_adaptors/#key-value-store","title":"Key-value store","text":"

      Tier B \u2014 template. Copy from templates/app.

      For report/snapshot storage, provide a pluggable store port with a Redis adapter and an in-memory fallback.

      "},{"location":"09_infra_adaptors/#rule","title":"Rule","text":"
      • Tier A (libraries: DB, DI, ingestion, mail, auth, docs, pipelines) \u2014 use the library; writing a private copy is a defect. Query find_library first.
      • Tier B (cache, progress bus, LLM pool, KV store, composition shape, worker, tests) \u2014 these are template code, not libraries: copy them from templates/app rather than searching the registry.
      • Tier C (models, business logic, pipeline families, rules) \u2014 author them yourself.
      "},{"location":"12_genericity_matrix/","title":"Genericity Matrix","text":""},{"location":"12_genericity_matrix/#summary","title":"Summary","text":"

      This page states precisely what is library-backed, what is template-provided, and what is app-specific and must be authored. Use it to scope a new build and to know, for any given capability, whether to call a library, copy template code, or write it yourself.

      "},{"location":"12_genericity_matrix/#tier-a-library-backed-use-never-reimplement","title":"Tier A \u2014 Library-backed (use; never reimplement)","text":"

      These capabilities already exist as internal libraries. Reach them through the library registry MCP (find_library(capability)) and read the library's own MCP for its API. Writing a private copy of any of these is a defect.

      Capability Library MCP Persistence: repositories, cache, populate, transactions Mongo Ops (mongo_ops) :8007 Dependency injection / composition Hexa (hexa) :8008 Deterministic multi-step transformation engine DAG Pipe (dagpipe) :8006 Spec-first HTTP edge, client, codegen OpenAPI First (openapi_first) :8004 Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) Omniread (omniread) :8003 Read-only email ingestion Mail Intake (mail_intake) :8002 JWT auth primitives (register, login, introspect) Py JWT (jwtlib) :8005 Documentation build (MkDocs + MCP) Doc Forge (docforge) :8001"},{"location":"12_genericity_matrix/#tier-b-template-provided-copy-from-templatesapp-then-adapt","title":"Tier B \u2014 Template-provided (copy from templates/app, then adapt)","text":"

      These are reusable internal components shipped inside the template \u2014 they are not independent libraries. Copy them as-is, then rename/populate only what is app-specific. They give every app the same wiring shape as the reference app. Fetch them by kind with get_app_component, or the whole tree with get_app_template.

      Component Template location Action Shipped Layering & dependency rule the package layout + import rules keep \u2705 Composition root shape init.py (get_svc/get_db, idempotency) keep \u2705 Spec-first web edge interfaces/openapi.py, thin handlers bound to operationId keep \u2705 CRUD slice pattern domain/<resource>/{models.py, ports/db.py, ports/service.py, converters.py} rename + fill \u2705 Generic CRUD service infra/adapters/service.py (CrudsServiceAdaptor) keep \u2705 Repository factory infra/db/repos/mongo.py (_repo_factory) keep \u2705 Worker poll + status loop worker.py keep \u2705 Progress bus infra/adapters/sse.py (keyed in-memory bus) + an app SSE endpoint keep \u2705 HTTP response cache infra/http_cache.py (cache_response + generation invalidation) keep \u2705 Key-value store infra/adapters/kv.py (port + in-memory + Redis adaptor) keep \u2705 Pipeline scaffold pipelines/shared/{ports,loader,registry}.py + pipelines/<family>/ rename + fill \u2705 Operator scripts scripts/ (init() / get_svc() / shutdown()) rename \u2705 Testing harness integration fakes + fixture-parity pattern keep \u2705 Project scaffolding pyproject.toml, Dockerfile, docker-compose.yaml, .env.example, openapi.yaml rename + fill \u2705"},{"location":"12_genericity_matrix/#tier-b-patterns-port-from-the-reference-app","title":"Tier B (patterns) \u2014 port from the reference app","text":"

      A few Tier B capabilities are real and reusable, but are shaped by the app or the provider, so the template does not ship them verbatim \u2014 copying them blindly would carry app-specific names. Treat them as patterns: lift the shape from the reference app and rename per app.

      Component Reference location Action Shipped Human-in-the-loop durable pause + question document + resolve endpoint + resume map port + reshape \u2717 LLM client pool provider-select + retry/rotation pool scaffold port + reshape \u2717"},{"location":"12_genericity_matrix/#tier-c-app-specific-must-be-authored","title":"Tier C \u2014 App-specific (must be authored)","text":"Artifact Why Resource models the business shape Repository populate/cache rules the relationships Service business logic domain behaviour Pipeline families the transformation graph Pattern/rule resources data-driven policy Fixtures & golden outputs test data"},{"location":"12_genericity_matrix/#the-two-axes","title":"The two axes","text":"

      Everything app-specific reduces to two axes:

      1. Resource model \u2014 the set of resources, their fields, relationships, and the OpenAPI contract.
      2. Pipeline family \u2014 only if inputs require multi-step transformation.

      If you can specify those two axes, the rest of the app follows mechanically: Tier A is imported, Tier B is copied, and Tier C is the only thing you author.

      "},{"location":"13_library_registry/","title":"Library Registry","text":""},{"location":"13_library_registry/#summary","title":"Summary","text":"

      The Aetos ecosystem provides internal libraries for the shared concerns of every application. Use them. Writing a private copy of any capability below is a defect.

      Each library also exposes its own documentation as an MCP server. An AI agent can read a library's full API by connecting to its MCP and reading docs://index, docs://nav, and docs://modules/{module}.

      "},{"location":"13_library_registry/#catalog","title":"Catalog","text":"Library Import Purpose Use instead of MCP DAG Pipe dagpipe Deterministic state-transformation DAG engine hand-rolled orchestration/topological ordering :8006 Hexa hexa Declarative DI / composition-root tooling manual wiring, hand-written factories :8008 Mongo Ops mongo_ops Async MongoDB repositories, cache, populate, transactions raw Motor CRUD and connection lifecycle :8007 Omniread omniread Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) custom scrapers/parsers :8003 OpenAPI First openapi_first Spec-first FastAPI bootstrap, client, codegen decorator routers and hand-written clients :8004 Mail Intake mail_intake Provider-agnostic read-only email ingestion Gmail OAuth + MIME parsing :8002 Py JWT jwtlib JWT auth primitives (register/login/introspect) password hashing + token code :8005 Doc Forge docforge Docstring compiler \u2192 MkDocs + MCP docs custom doc generators :8001 Auth Server (service) Central identity service (register/login/me/introspect) building auth endpoints yourself (deferred)"},{"location":"13_library_registry/#how-to-choose","title":"How to choose","text":"You need to\u2026 Use persist and query documents Mongo Ops compose dependencies Hexa run a multi-step transformation DAG Pipe read/parse an arbitrary input format Omniread expose a contract-first HTTP API OpenAPI First ingest email Mail Intake authenticate users / verify tokens Py JWT (+ Auth Server) generate documentation Doc Forge"},{"location":"13_library_registry/#discovery","title":"Discovery","text":"

      A single standards MCP exposes find_library(capability), which returns the matching library, its import name, its \"use instead of\" guidance, and its MCP endpoint. Query it before writing any infrastructure.

      "},{"location":"13_library_registry/#notes","title":"Notes","text":"
      • Hexa is a full library with its own MCP server on :8008: read docs://index, docs://nav, and docs://modules/{module} (e.g. docs://modules/hexa.container) to discover the DI surface \u2014 build(cls, *, instances, config) and the PortNode model.
      • Auth Server is deferred: it has no pyproject.toml/importable package and no MCP bundle. Do not reach for it yet; py-jwt covers token primitives.
      • Library MCP servers expose documentation resources (not executable tools): they tell an agent the available API surface so it does not invent code.
      "}]} \ No newline at end of file +{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Standards Reference","text":"

      The reference half of the Standards description: the enforceable rules, patterns, and the internal library catalog that agents consult while coding.

      • Build Recipe \u2014 the ordered, agentic checklist for a new app.
      • Layering Rules \u2014 the strict request \u2192 domain \u2192 repository layering and the rules that enforce it.
      • Composition Root \u2014 a single composition point wiring the whole app.
      • Spec-First Routing \u2014 the OpenAPI contract as the routing source of truth.
      • CRUD Template \u2014 the standardized generic resource endpoint scaffold.
      • Pipeline Scaffold \u2014 the deterministic pipeline skeleton for multi-stage jobs.
      • Worker & Progress \u2014 queued work, progress reporting, and visibility.
      • Human-in-the-Loop \u2014 guardrails and approvals inside automations.
      • Infra Adapters \u2014 how infrastructure connects without leaking into domain.
      • Genericity Matrix \u2014 what is library-backed, template-provided, or authored.
      • Library Registry \u2014 the catalog of internal libraries behind all of it.

      Read the Wiki for the philosophy and rationale; these pages are the reference that mirrors the standards MCP bundle.

      "},{"location":"01_build_recipe/","title":"Build Recipe","text":""},{"location":"01_build_recipe/#summary","title":"Summary","text":"

      The ordered, deterministic checklist for generating a new application of the standard shape. Every step is fixed; only the resource model (step 2), the codegen pass (step 3), and the pipeline family (step 8) need app-specific work.

      This page is served by the standards MCP as docs://modules/build_recipe. The start_app prompt on that server collects the two axes and hands off here; pull exact files with get_app_component.

      "},{"location":"01_build_recipe/#bootstrap","title":"\ud83d\ude80 Bootstrap","text":"

      From the standards docs checkout:

      cp -r <docs>/standards/src/templates/app apps/<your-app>\ncd apps/<your-app>\n

      Install the internal libraries \u2014 editable local checkouts (preferred in development) or the private index (CI / fresh machines). Credentials are environment-only; never commit them.

      # local checkouts\npip install -e ../aetos/openapi-first -e ../aetos/mongo-ops -e ../aetos/hexa \\\n            -e ../aetos/dagpipe -e ../aetos/doc-forge\n# private index\npip install --extra-index-url \\\n  \"https://$PIP_REGISTRY_USERNAME:$PIP_REGISTRY_PASSWORD@pip.aetoskia.com/simple\" .\n
      "},{"location":"01_build_recipe/#agent-prompt-copyable","title":"\ud83d\udccb Agent prompt (copyable)","text":"
      # Build a new application from the Aetos Standards blueprint\n\nYou are coding a complete new backend application conforming to the Aetos\nstandard app blueprint. Execute every step in order. Only steps 2\u20134 use the\napp-specific details provided by the user; everything else is deterministic.\n\n## Before writing any code\n\nConnect to the `standards` MCP server and read:\n- `docs://nav` (the module list)\n- `docs://modules/genericity_matrix` (Tier A library / Tier B template / Tier C authored)\n- `docs://modules/layering_rules` (the dependency rules)\n\nThen:\n- call `find_library` with `db`, `di`, `routing`, `ingestion`, `pipeline`, `docs`\n  and note the Tier A libraries that already exist \u2014 never reimplement one;\n- call `list_app_components()` and pull each file you need with\n  `get_app_component(kind, resource=..., family=...)` instead of writing it from\n  scratch.\n\n## Steps\n\n1. **Scaffold** \u2014 copy `standards/src/templates/app/` into `apps/<your-app>/`\n   (or pull the same tree via `get_app_template`). Keep the tree: `domain/`,\n   `infra/`, `interfaces/`, `pipelines/`, `tests/`, `scripts/`, plus\n   `pyproject.toml`, `openapi.yaml`, `config.py`, `init.py`, `main.py`,\n   `worker.py`, `Dockerfile`, `docker-compose.yaml`, `.env.example`.\n\n2. **Author the spec (app-specific)** \u2014 write `openapi.yaml`: resources,\n   schemas, `operationId`s, security, and extension metadata. This is the\n   contract and drives all routing. Every operation MUST have an `operationId`.\n\n3. **Generate the scaffold (codegen)** \u2014 from the spec, generate the wire\n   models and the exact handler signatures:\n   ```bash\n   openapi-first models openapi.yaml -o _gen/models.py\n   openapi-first routes openapi.yaml -o _gen/routes \\\n     --use-models --models-module _gen.models --inject-request --package --template-style\n   ```\n   `_gen/models.py` is the **wire contract** (Pydantic, one class per\n   `components.schemas` entry). `_gen/routes/<resource>/routes.py` lists every\n   `operationId` with its exact parameter signature. `--template-style` makes\n   those stubs `async def` with a `body` parameter, matching the handlers you\n   keep in `interfaces/` (without it they are sync `def` stubs taking\n   `payload`; both bind the same way). Treat `_gen/` as generated\n   (never hand-edit it) \u2014 the canonical handlers live in `interfaces/`.\n\n4. **Write each CRUD slice (app-specific)** \u2014 per resource create the four-file\n   slice: `models.py` (domain model, **no `id`**), `ports/db.py`\n   (`class XDocument(BaseDocument, X)` \u2014 id comes from `BaseDocument`),\n   `ports/service.py`, and optional `converters.py`. Follow the CRUD template.\n\n5. **Wire the composition root** \u2014 in `init.py`, build leaf repositories, then\n   dependent repositories, then services; add infra adapters (db, cache, stores,\n   progress bus, fetchers). Expose `get_svc` overloads.\n\n6. **Add routes** \u2014 create `interfaces/<resource>/routes.py` thin handlers whose\n   names match the `operationId`s (use the signatures from step 3 as your\n   checklist) and re-export them in `interfaces/openapi.py`. Add the cache\n   decorator to read endpoints.\n\n7. **Bootstrap app and worker** \u2014 confirm `main.py` runs the OpenAPI-first app\n   with its lifespan. If the app has long-running jobs, adapt `worker.py` to poll\n   the job collection, run the pipeline, and publish progress.\n\n8. **Add pipeline families (only if data-driven)** \u2014 copy `pipelines/example/`\n   to `pipelines/<family>/` with `abc.py`, `impl.py`, `pipeline.yaml`,\n   `resources/`, and reuse `pipelines/shared/` (loader + registry). Register the\n   root. Add fixtures and a parity test.\n\n9. **Add infrastructure only where needed** \u2014 Tier A capability: use the\n   library. Tier B capability: copy the template. Never hand-write a Tier A\n   library.\n\n10. **Tests** \u2014 integration tests with fakes at the HTTP edge; unit tests for\n    domain and pipeline logic; a config-consistency test for rule-driven\n    pipelines. Include `tests/integration/test_openapi_app_builds.py` (imports\n    the app, asserts `import main` binds every operation).\n\n11. **Scripts and seeds** \u2014 operator scripts under `scripts/` using\n    `init()` / `get_svc()` / `shutdown()`; run as `python -m scripts.<name>`.\n\n12. **Docs** \u2014 author GSDFC docstrings, generate `.pyi` stubs, mark the package\n    `py.typed`, and build the docs.\n\n**Definition of done:**\n- `python -c \"import main\"` succeeds.\n- `pytest` passes.\n- The app serves every operation in the spec; the worker drains its queue.\n- No domain module imports a web framework.\n- No Tier A library was reimplemented.\n
      "},{"location":"01_build_recipe/#where-the-steps-land","title":"\ud83e\udded Where the steps land","text":"

      The steps map onto the layers as follows:

      Step Package / file Depends on 1 \u00b7 Scaffold whole templates/app/ \u2014 2 \u00b7 Spec openapi.yaml \u2014 3 \u00b7 Codegen _gen/ (generated) spec, openapi-first 4 \u00b7 CRUD slices domain/<resource>/ spec, CRUD template 5 \u00b7 Wiring init.py every slice 6 \u00b7 Routes interfaces/<resource>/ services, openapi route registry 7 \u00b7 Bootstrap main.py, worker.py composition root 8 \u00b7 Pipelines pipelines/<family>/, shared/ library pipeline engine 9 \u00b7 Infra infra/ domain ports 10 \u00b7 Tests tests/ app + fixtures 11 \u00b7 Scripts scripts/ composition root 12 \u00b7 Docs docstrings + docs build code, Doc Forge"},{"location":"01_build_recipe/#the-codegen-fast-path-in-detail","title":"\ud83d\udd27 The codegen fast path, in detail","text":"

      openapi-first is spec \u2192 code only (it never generates a spec). Use it to erase the two mechanical parts of the build:

      Generated artifact Replaces Notes _gen/models.py hand-writing Pydantic wire models one class per components.schemas entry _gen/routes/<resource>/routes.py hand-typing handler signatures names == operationIds; request: Request injected

      Rules of the road:

      • Wire models are not domain models. _gen/models.py is the HTTP contract; domain/<resource>/models.py is your business shape. If the spec's schema has a required id: str, keep it off the domain model and let the document get its id from BaseDocument.
      • Use --template-style. Stubs are then async def ...(request, body: ...) matching interfaces/; without it they are sync def ...(payload: ...). Either binds (the router introspects the signature), but --template-style keeps the scaffold consistent with the code you keep.
      • Resource names are sanitized. /fetch-requests \u2192 fetch_requests.py.
      • Multipart bodies expand to UploadFile = File(...) / Form(...) params.
      • _gen/ is disposable. Regenerate on every spec change; git diff shows exactly what moved.
      "},{"location":"01_build_recipe/#definition-of-done","title":"\u26a0\ufe0f Definition of done","text":"
      • python -c \"import main\" succeeds.
      • pytest passes.
      • The app serves the spec's operations; the worker drains its queue.
      • No domain module imports a web framework.
      • No Tier A library was reimplemented.
      "},{"location":"01_build_recipe/#related","title":"Related","text":"
      • 02 \u2013 Layering Rules \u00b7 05 \u2013 CRUD Template \u00b7 06 \u2013 Pipeline Scaffold \u00b7 12 \u2013 Genericity Matrix
      "},{"location":"02_layering_rules/","title":"Layering Rules","text":""},{"location":"02_layering_rules/#summary","title":"Summary","text":"

      The dependency direction is fixed. Violating it is a design defect.

      "},{"location":"02_layering_rules/#allowed-imports","title":"Allowed imports","text":"
      interfaces/*      ->  domain/*            (services, ports, models)\ninterfaces/*      ->  infra/adapters/*    (only read-only value objects)\ndomain/*          ->  domain/*            (same layer)\ndomain/*          ->  pipelines/*         (call a pipeline via a port)\ndomain/*/ports/*  ->  infra/*             (bind adapters + libraries at the seam)\ninfra/adapters/*  ->  domain/ports/*      (implement the interface)\nmain.py/init.py   ->  everything          (composition root only)\n
      "},{"location":"02_layering_rules/#forbidden-imports","title":"Forbidden imports","text":"
      domain/*          ->  interfaces/*        (never)\ndomain/*          ->  fastapi/starlette   (never)\ninfra/adapters/*  ->  interfaces/*        (never)\ninterfaces/*      ->  infra/db/*          (state lives in resources, not imports)\n
      "},{"location":"02_layering_rules/#why","title":"Why","text":"
      • The business logic is testable without a server or an event loop.
      • Adapters can be swapped (in-memory fake, real database) without touching business logic.
      • The HTTP edge stays thin and replaceable.
      "},{"location":"02_layering_rules/#the-portadapter-contract","title":"The port/adapter contract","text":"

      The ports/ package is the seam where the domain meets infrastructure. Each domain/<resource>/ports/ module declares the resource's models/document shape and exposes factory functions that bind a concrete adapter \u2014 so a port module is allowed to import from infra/ and from the persistence libraries. Everything outside ports/ in domain/ stays framework- and driver-free.

      domain/<resource>/ports/db.py       -> document model + create_<resource>_repo()\ndomain/<resource>/ports/service.py  -> service + get_<resource>_service()\ninfra/db/repos/<vendor>.py          -> repository adapter factory\ninfra/adapters/service.py           -> generic CRUD service base\n

      The composition root calls the port factories; nothing else constructs an adapter.

      "},{"location":"02_layering_rules/#enforcement","title":"Enforcement","text":"
      • domain/ must never import the web framework (fastapi / starlette) or interfaces/. Only the ports/ seam, infra/, and the composition root touch drivers.
      • Reviewers (and CI) reject import fastapi inside domain/ or infra/.
      "},{"location":"03_composition_root/","title":"Composition Root","text":""},{"location":"03_composition_root/#summary","title":"Summary","text":"

      All wiring happens in exactly one module. It builds a flat resource registry and exposes it to the HTTP layer and to scripts.

      "},{"location":"03_composition_root/#the-resource-registry","title":"The resource registry","text":"

      init.py returns a plain dict[str, Any]. Keys follow two conventions:

      • <resource>_repo \u2014 a repository adapter for a resource.
      • <resource>_svc \u2014 a service for a resource.

      Plus named infrastructure adaptors:

      • http_cache_store \u2014 response cache backend.
      • progress_bus \u2014 progress/event bus.
      • <vendor>_<kind> \u2014 fetchers, clients, stores.

      The API stores the dict on app.state.db_resources during lifespan startup. Handlers read from it; nothing imports concrete adapters.

      "},{"location":"03_composition_root/#shape","title":"Shape","text":"
      _resources: dict = {}\n_initialized = False\n\nasync def init(settings=None) -> dict:\n    global _initialized\n    if _initialized:\n        return _resources\n\n    await setup_database(uri=settings.mongo_uri, db_name=settings.mongo_db)\n\n    # 1. leaf repositories\n    user_repo = create_user_repo()\n\n    # 2. dependent repositories (dependencies passed by keyword)\n    wallet_repo = create_wallet_repo(user_repo=user_repo)\n\n    # 3. services (build via the DI library, injecting the repo)\n    _resources[\"user_repo\"]   = user_repo\n    _resources[\"wallet_repo\"] = wallet_repo\n    _resources[\"user_svc\"]    = get_user_service(user_repo)\n    _resources[\"wallet_svc\"]  = get_wallet_service(wallet_repo)\n\n    # 4. infrastructure adaptors (optional)\n    _resources[\"http_cache_store\"] = HttpCacheStore(...)\n    _resources[\"progress_bus\"]     = ProgressEventBus()\n\n    _initialized = True\n    return _resources\n\nasync def shutdown() -> None:\n    global _initialized\n    await close_stores(_resources)\n    await shutdown_database()\n    _resources.clear()\n    _initialized = False\n
      "},{"location":"03_composition_root/#typed-accessors","title":"Typed accessors","text":"

      Expose a get_svc(name) helper with one @overload per key so scripts get typing without importing concrete classes:

      @overload\ndef get_svc(name: Literal[\"user_svc\"]) -> UserService: ...\n@overload\ndef get_svc(name: Literal[\"wallet_svc\"]) -> WalletService: ...\n\ndef get_svc(name: str):\n    return _resources[name]\n

      Also expose get_db() (raw database handle) and pure helpers such as compute_fingerprint(payload) (canonical hash used for deduplication).

      "},{"location":"03_composition_root/#rules","title":"Rules","text":"
      • init() is idempotent.
      • The composition root is the only place that imports concrete adapters.
      • Handlers never call init(); they read request.app.state.db_resources.
      • Scripts call init() / shutdown() around their work.
      "},{"location":"04_spec_first_routing/","title":"Spec-First Routing","text":""},{"location":"04_spec_first_routing/#summary","title":"Summary","text":"

      The OpenAPI document is the single source of truth for the HTTP contract. Handlers are bound to it by name; there is no decorator router.

      "},{"location":"04_spec_first_routing/#the-contract","title":"The contract","text":"
      • openapi.yaml (or .json) declares paths, schemas, parameters, responses, and security.
      • A custom operationId names the handler function.
      • The framework (openapi_first) validates requests/responses against the spec and dispatches to the handler whose name equals operationId.
      "},{"location":"04_spec_first_routing/#app-bootstrap","title":"App bootstrap","text":"
      # main.py\nfrom openapi_first.app import OpenAPIFirstApp\n\napp = OpenAPIFirstApp(\n    openapi_path=\"openapi.yaml\",\n    routes_module=interfaces.openapi,   # module whose namespace holds handlers\n    title=\"<App> API\",\n    version=\"0.1.0\",\n    lifespan=lifespan,\n)\n

      lifespan calls the composition root:

      @asynccontextmanager\nasync def lifespan(app):\n    app.state.db_resources = await init()\n    yield\n    await shutdown()\n
      "},{"location":"04_spec_first_routing/#handler-registry","title":"Handler registry","text":"

      interfaces/openapi.py is a flat re-export of every handler, so the framework can resolve operationId across the whole app:

      from interfaces.item.routes import *\nfrom interfaces.wallet.routes import *\nfrom interfaces.progress.sse import *\n
      "},{"location":"04_spec_first_routing/#handler-shape","title":"Handler shape","text":"

      Handlers are thin and synchronous with respect to business logic:

      async def create_item(request: Request, body: dict):\n    svc = request.app.state.db_resources[\"item_svc\"]\n    return await svc.create(Item.model_validate(body))\n

      Generated stubs. openapi-first routes emits this exact shape (async def handlers taking body) when given --template-style; without that flag the stub is def create_item(payload: ItemInput) \u2014 sync, with a payload parameter. Both shapes bind identically: the router resolves handlers by operationId and introspects the FastAPI signature, and it never inspects the request body. Pass --template-style so the generated signatures match the handlers you keep in interfaces/.

      "},{"location":"04_spec_first_routing/#conventions","title":"Conventions","text":"
      • operationId naming: list_<resource>, create_<resource>, get_<resource>, update_<resource>, delete_<resource>.
      • Read-only endpoints may use a cache decorator keyed on path + query + user.
      • Validation errors -> 400; not found -> 404; delete -> 204.
      • Security is declared once in the spec and enforced by the framework; handlers contain no auth code.
      "},{"location":"04_spec_first_routing/#authless-apps","title":"Authless apps","text":"

      Auth is deferred in the standard: the same spec-first app runs with or without it, because security lives entirely in the spec, never in handlers. For an app that needs no auth yet, do not invent per-handler auth checks \u2014 just omit the security keys:

      • Remove the top-level security: block.
      • Remove components.securitySchemes.
      • Do not add security: [] to operations. An explicit empty array is a per-operation override that means \"public\", and it only matters when a global security block exists. Leaving every operation without a security key is the cleanest authless spec.

      Profile endpoints (/users/me) and profile headers (x-profile) still exist in an authless app. Define a stand-in identity in the composition root \u2014 for example get_current_user returns or lazily creates a fixed \"Demo User\" \u2014 so handlers stay auth-agnostic and only the wiring changes when real auth arrives.

      The handlers, services, and composition root are unchanged. When auth is needed later, add the global security block and an introspecting dependency; only handlers that must opt out get security: [].

      "},{"location":"04_spec_first_routing/#codegen","title":"Codegen","text":"

      openapi_first scaffolds models and route stubs from the spec (see the Build Recipe, step 3):

      openapi-first models openapi.yaml -o _gen/models.py\nopenapi-first routes openapi.yaml -o _gen/routes \\\n  --use-models --models-module _gen.models --inject-request --package --template-style\n

      Prefer codegen over hand-writing boilerplate. The result is the wire contract and a set of operationId-named signatures \u2014 not the canonical handlers.

      "},{"location":"05_crud_template/","title":"CRUD Template","text":""},{"location":"05_crud_template/#summary","title":"Summary","text":"

      Every resource follows the same four-file slice. Repetition is intentional: it makes the codebase predictable and generatable.

      "},{"location":"05_crud_template/#the-slice","title":"The slice","text":"
      domain/<resource>/\n    models.py               # pure Pydantic model (no DB fields)\n    ports/\n        db.py               # document model + repository factory\n        service.py          # service class + factory\n        converters.py       # (optional) transforms\n
      "},{"location":"05_crud_template/#the-id-rule-model-vs-document","title":"The id rule (model vs. document)","text":"

      Two different objects, two different jobs:

      Object Lives in Has id? Purpose Domain model domain/<resource>/models.py No business shape, framework-free Document domain/<resource>/ports/db.py Yes (from BaseDocument) Mongo persistence shape

      BaseDocument (mongo-ops) supplies id: Optional[PyObjectId] (aliased to _id) plus created_at / updated_at. The domain model must not declare id; Mongo owns it.

      Trap \u2014 generated wire models. openapi-first models produces the HTTP contract (see the Build Recipe, step 3). Response schemas frequently declare a required id: str, so a generated class is not a domain model. If you write class ItemDocument(Item, BaseDocument) and Item re-exports a required id: str, that field shadows BaseDocument.id and no Mongo _id can ever be set. Fix it one of three ways:

      1. Preferred: keep the domain model id-free (hand-author it from the CRUD template), and keep the required id only on the wire model.
      2. Order the bases so the document wins: class ItemDocument(BaseDocument, Item).
      3. Keep a separate document type and translate with ports/converters.py.
      "},{"location":"05_crud_template/#modelspy","title":"models.py","text":"

      A framework-free Pydantic model describing the business shape (no id).

      class Item(BaseModel):\n    name: str\n    quantity: int = 0\n
      "},{"location":"05_crud_template/#portsdbpy","title":"ports/db.py","text":"

      The persistence model extends the business model and adds database concerns. The repository factory selects the right repository flavour.

      # Item is id-free; BaseDocument adds id/created_at/updated_at.\nclass ItemDocument(Item, BaseDocument):\n    pass\n\ndef create_item_repo(*, owner_repo=None) -> ItemRepository:\n    return repo_factory(\n        collection=\"items\",\n        document=ItemDocument,\n        populate_rules=[PopulateRule(field_name=\"owner\", collection_name=\"users\")],\n        cache=None,               # or a CachedBaseRepository config\n    )\n
      "},{"location":"05_crud_template/#nested-references","title":"Nested references","text":"

      The wire contract often declares a reference as an expanded object (expense.entity: Entity), while the domain model stays id-free. Pick one convention and apply it consistently:

      • Embed (default for small, stable refs) \u2014 store the object as a dict / list[dict] on the document; the API returns it verbatim. No populate wiring.
      • Populate \u2014 store an id and declare a PopulateRule; mongo-ops expands it on read. The referenced repo must be registered with the PopulationEngine.
      • Id only \u2014 store the id and let the client resolve it.

      For populate-backed repositories change a reference with update, not patch: patch rejects changes to populated FK fields by design.

      "},{"location":"05_crud_template/#portsservicepy","title":"ports/service.py","text":"

      The service is built through the DI library, injecting the repository. The generic CRUD base provides create/read/update/patch/delete/search/....

      class ItemService(CrudsServiceAdaptor[Item, ItemDocument]):\n    model = Item\n    schema = ItemDocument\n\ndef get_item_service(repo) -> ItemService:\n    return build(ItemService, instances={\"repo\": repo})\n
      "},{"location":"05_crud_template/#naming-conventions","title":"Naming conventions","text":"Concept Convention collection plural snake case (items) document <Resource>Document repository factory create_<resource>_repo service <Resource>Service service factory get_<resource>_service resource key <resource>_svc operationId list_items, create_item, ..."},{"location":"05_crud_template/#optional-files","title":"Optional files","text":"
      • converters.py \u2014 pure functions mapping between domain types.
      • ports/enrichment.py \u2014 adapts a pipeline into an async callable the service can invoke (used when a resource is populated by a pipeline).
      "},{"location":"06_pipeline_scaffold/","title":"Pipeline Scaffold","text":""},{"location":"06_pipeline_scaffold/#summary","title":"Summary","text":"

      Data-driven processing is expressed as a composition graph. The graph is built by a DI library; execution is driven by a DAG engine. Stages are pure and individually testable.

      "},{"location":"06_pipeline_scaffold/#when-to-use-a-pipeline","title":"When to use a pipeline","text":"

      Use a pipeline when a resource is produced by a multi-step transformation of an input (a document, a file, a message) rather than by a simple CRUD write.

      "},{"location":"06_pipeline_scaffold/#structure-of-a-family","title":"Structure of a family","text":"

      A family is one input shape (e.g. a file format or a message kind). Each family has the same skeleton:

      pipelines/<family>/\n    abc.py            # port contracts (stage interfaces + root interface)\n    impl.py           # concrete stages + root implementation\n    pipeline.yaml     # graph spec: nodes, edges, roots, schema\n    resources/\n        pipeline.yaml # runtime spec consumed by the loader\n        patterns/*.yaml  # rule fragments (pattern registry)\n    banks/<variant>.py   # optional variants that re-pin a few slots\n
      "},{"location":"06_pipeline_scaffold/#shared-scaffolding","title":"Shared scaffolding","text":"
      pipelines/shared/\n    ports.py     # RegistryPort, FinalizePort, etc.\n    loader.py    # build(root_cls) -> wires the graph + engine\n    registry.py  # resolve_root(...) -> root class for a (kind, variant)\n
      "},{"location":"06_pipeline_scaffold/#the-loader","title":"The loader","text":"
      def build(root_cls, *, instances=None, config=None):\n    # merge resources/patterns/*.yaml into a pattern registry\n    # bind registry/config into each graph node by id\n    # build the schema (state) class and the engine\n    root = root_cls()\n    root._engine = Engine(graph, state_cls, payload_cls)\n    return root\n
      "},{"location":"06_pipeline_scaffold/#the-registry","title":"The registry","text":"

      A single lookup replaces hand-rolled if/else selection:

      _FAMILIES = {\n    \"csv\":  {\"acme\": {\"heuristic\": AcmeCsvPipeline}},\n    \"json\": {\"globex\": {\"heuristic\": GlobexJsonPipeline}},\n}\n\ndef resolve_root(kind, variant, mode=\"heuristic\"):\n    return _FAMILIES[kind][variant][mode]\n

      Adding a new variant means adding one class and one registry entry.

      "},{"location":"06_pipeline_scaffold/#driving-a-pipeline","title":"Driving a pipeline","text":"

      Stages are driven explicitly so intermediate state can be persisted:

      pipeline = build(resolve_root(kind, variant))\ncontent = pipeline.ingest.materialize(raw_bytes)\nrecords = pipeline.reader.read(content)\nitems   = pipeline.normalize.build(records)\n
      "},{"location":"06_pipeline_scaffold/#testing","title":"Testing","text":"

      Because stages are pure, test each stage in isolation and test the assembled root against fixture inputs whose expected outputs are stored as golden files. A parity test replays fixtures headlessly.

      "},{"location":"07_worker_and_progress/","title":"Worker & Progress","text":""},{"location":"07_worker_and_progress/#summary","title":"Summary","text":"

      Long-running work runs in a background worker. Jobs are durable documents that move through an explicit status machine. Progress is streamed to clients.

      "},{"location":"07_worker_and_progress/#the-status-machine","title":"The status machine","text":"
      pending -> processing -> completed\n                     \\-> failed    (retried up to N times)\n                     \\-> paused    (waiting for human input)\n
      • pending \u2014 queued, eligible for pickup.
      • processing \u2014 claimed by a worker.
      • paused \u2014 blocked on a human-in-the-loop question.
      • failed \u2014 error; retry_count increments until a max, then skipped.
      • completed \u2014 terminal success.
      "},{"location":"07_worker_and_progress/#the-poll-loop","title":"The poll loop","text":"
      async def poll(resources):\n    backoff = MIN_POLL\n    while True:\n        jobs = await job_svc.search(filter={\"status\": \"pending\"}, limit=100)\n        backoff = MIN_POLL if jobs else min(backoff * 2, MAX_POLL)\n        for job in jobs:\n            if (job.retry_count or 0) >= MAX_RETRIES:\n                continue\n            await job_svc.update_status(job.id, \"processing\")\n            try:\n                await process(job, resources)\n            except PauseForResolution:\n                await job_svc.update_status(job.id, \"paused\")\n            except Exception as e:\n                await job_svc.update_status(\n                    job.id, \"failed\", error=str(e),\n                    retry_count=(job.retry_count or 0) + 1,\n                )\n        await asyncio.sleep(backoff)\n

      Tunables live in environment variables: WORKER_POLL_MIN, WORKER_POLL_MAX, WORKER_MAX_RETRIES.

      "},{"location":"07_worker_and_progress/#progress-events","title":"Progress events","text":"

      A worker publishes structured progress events to a bus (a port). The HTTP edge exposes a streaming endpoint that replays events for a given job id.

      class ProgressEvent(BaseModel):\n    job_id: str\n    step: str\n    status: str            # started | progress | completed | skipped | paused\n    message: dict | None = None\n

      The bus is an in-memory adapter by default; because it is a port, it can be swapped for a durable broker. Emit from the orchestrator at each stage:

      await self.bus.publish(ProgressEvent(job_id=jid, step=\"normalize\",\n                                     status=\"completed\", message={\"count\": n}))\n
      "},{"location":"07_worker_and_progress/#streaming-endpoint","title":"Streaming endpoint","text":"
      async def stream_job_events(request, id):\n    bus = request.app.state.db_resources.get(\"progress_bus\")\n    if bus is None:\n        return {\"events\": []}\n\n    async def gen():\n        sent = 0\n        while True:\n            events = bus.get_events(id)\n            for e in events[sent:]:\n                yield f\"data: {e.model_dump_json(exclude_none=True)}\\n\\n\"\n                sent += 1\n            await asyncio.sleep(0.5)\n\n    return StreamingResponse(gen(), media_type=\"text/event-stream\")\n

      Known limitation. The default in-memory bus is per-process. If the worker and the API run as separate processes, events published by the worker are not visible to the API. For production, back the bus with a shared broker. The port boundary makes this a one-adapter change.

      "},{"location":"08_human_in_loop/","title":"Human-in-the-Loop","text":""},{"location":"08_human_in_loop/#summary","title":"Summary","text":"

      Some inputs are ambiguous and must be resolved by a human before processing can continue. The blueprint models this as a durable pause: an exception is raised, a question is recorded, the job pauses, and later resumes.

      "},{"location":"08_human_in_loop/#the-mechanism","title":"The mechanism","text":"
      1. During processing, a stage raises a domain exception carrying the context needed to ask the question:
      raise PauseForResolution(\n    job_id=jid,\n    reference=\"<stable key for this decision>\",\n    options=[...],\n    context={...},\n)\n
      1. The orchestrator catches it, records a question document, and re-raises so the worker marks the job paused.
      2. A client lists pending questions and answers one:
      GET  /jobs/{id}/questions            -> list pending questions\nPOST /questions/{qid}/resolve  {choice}  -> record the choice\n
      1. When the last pending question is resolved, the resolve endpoint flips the parent job back to pending.
      2. The worker picks it up again, rebuilds a resolution map from previously answered questions, and seeds it into the pipeline so the same decision is not asked twice.
      "},{"location":"08_human_in_loop/#the-question-document","title":"The question document","text":"
      class Question(BaseDocument):\n    job_id: str\n    reference: str            # stable key: stage|field|value\n    status: str               # pending | resolved\n    options: list[dict]\n    context: dict\n    chosen: dict | None = None\n    resolved_at: str | None = None\n
      "},{"location":"08_human_in_loop/#resume-map","title":"Resume map","text":"

      The worker turns answered questions into a lookup keyed by reference, then injects it into the pipeline so that stage can skip re-raising:

      resolved = {q.reference: q.chosen for q in await svc.search(\n    filter={\"job_id\": jid, \"status\": \"resolved\"})}\n\npipeline = build(root, instances={\"pause\": handler}, config={\"resolved\": resolved})\n
      "},{"location":"08_human_in_loop/#resolve-endpoint","title":"Resolve endpoint","text":"
      async def resolve_question(request, id, body):\n    svc = request.app.state.db_resources[\"question_svc\"]\n    q = await svc.read(id)\n    await svc.update(id, {\"chosen\": body[\"chosen\"], \"status\": \"resolved\",\n                          \"resolved_at\": now()})\n\n    pending = await svc.search(filter={\"job_id\": q.job_id, \"status\": \"pending\"},\n                               limit=1)\n    if not pending:\n        jobs = request.app.state.db_resources[\"job_svc\"]\n        job = await jobs.read(q.job_id)\n        if job and job.status == \"paused\":\n            await jobs.update_status(q.job_id, \"pending\")\n    return q\n
      "},{"location":"08_human_in_loop/#cli-parity","title":"CLI parity","text":"

      The same flow is available through scripts/CLI: list, get, create, questions, resolve, retry.

      "},{"location":"09_infra_adaptors/","title":"Infra Adapters","text":""},{"location":"09_infra_adaptors/#summary","title":"Summary","text":"

      All external systems are reached through adapters in infra/. Prefer an existing internal library over writing a new adapter; where no library exists, copy the Tier B scaffold from templates/app. See the Genericity Matrix for the Tier A / B / C split.

      "},{"location":"09_infra_adaptors/#database-mongodb","title":"Database (MongoDB)","text":"
      • A single connection manager owns the client lifecycle. setup_database(uri, db_name) / shutdown_database().
      • A repository factory selects the repository flavour:
      repo_factory(collection, document, engine=None,\n             populate_rules=None, cache=None)\n
      • base repository \u2014 plain CRUD;
      • populating repository \u2014 resolves referenced documents (foreign keys);

      • cached repository \u2014 an LRU/TTL cache in front of reads.

      • Referenced documents are described declaratively:

      PopulateRule(field_name=\"owner\", collection_name=\"users\",\n             nested_rules=[], max_depth=2, projection=None)\n
      • Transactions use a session context manager provided by the library.

      Tier A \u2014 Library (Mongo Ops). Do not write raw Motor calls in the domain; consult the registry and the library's MCP before touching the driver.

      "},{"location":"09_infra_adaptors/#http-response-cache","title":"HTTP response cache","text":"

      Tier B \u2014 template. Copy from templates/app; not a standalone library.

      A generic GET cache sits behind a decorator.

      @cache_response(ttl=60)\nasync def list_items(request): ...\n
      • Keyed on path + query + user.
      • No-op when no store is wired.
      • invalidate(*resources) bumps a generation counter for write-triggered invalidation.
      "},{"location":"09_infra_adaptors/#event-progress-bus","title":"Event / progress bus","text":"

      Tier B \u2014 template. Copy from templates/app.

      A port with an in-memory default adapter (publish, get_events). Swap for a durable broker in production (see Worker & Progress).

      "},{"location":"09_infra_adaptors/#external-clients-ingestion","title":"External clients (ingestion)","text":"

      External fetchers (files, mail, HTTP) are adapters that wrap ingestion libraries. Construct them in the composition root and inject into orchestrators.

      Tier A \u2014 Libraries: Omniread (formats) and Mail Intake (email).

      "},{"location":"09_infra_adaptors/#llm-clients","title":"LLM clients","text":"

      Tier B \u2014 template. Copy the pool scaffold from templates/app.

      When a pipeline calls a model provider, route through a pool adaptor that handles provider selection, retries, and client lifecycle:

      pool = setup_async_pool(provider=settings.llm_provider)\nresult = await pool.complete(model=\"...\", messages=[...])\n

      Providers are selected by LLM_PROVIDER and configured by environment.

      "},{"location":"09_infra_adaptors/#key-value-store","title":"Key-value store","text":"

      Tier B \u2014 template. Copy from templates/app.

      For report/snapshot storage, provide a pluggable store port with a Redis adapter and an in-memory fallback.

      "},{"location":"09_infra_adaptors/#rule","title":"Rule","text":"
      • Tier A (libraries: DB, DI, ingestion, mail, auth, docs, pipelines) \u2014 use the library; writing a private copy is a defect. Query find_library first.
      • Tier B (cache, progress bus, LLM pool, KV store, composition shape, worker, tests) \u2014 these are template code, not libraries: copy them from templates/app rather than searching the registry.
      • Tier C (models, business logic, pipeline families, rules) \u2014 author them yourself.
      "},{"location":"12_genericity_matrix/","title":"Genericity Matrix","text":""},{"location":"12_genericity_matrix/#summary","title":"Summary","text":"

      This page states precisely what is library-backed, what is template-provided, and what is app-specific and must be authored. Use it to scope a new build and to know, for any given capability, whether to call a library, copy template code, or write it yourself.

      "},{"location":"12_genericity_matrix/#tier-a-library-backed-use-never-reimplement","title":"Tier A \u2014 Library-backed (use; never reimplement)","text":"

      These capabilities already exist as internal libraries. Reach them through the library registry MCP (find_library(capability)) and read the library's own MCP for its API. Writing a private copy of any of these is a defect.

      Capability Library MCP Persistence: repositories, cache, populate, transactions Mongo Ops (mongo_ops) :8007 Dependency injection / composition Hexa (hexa) :8008 Deterministic multi-step transformation engine DAG Pipe (dagpipe) :8006 Spec-first HTTP edge, client, codegen OpenAPI First (openapi_first) :8004 Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) Omniread (omniread) :8003 Read-only email ingestion Mail Intake (mail_intake) :8002 JWT auth primitives (register, login, introspect) Py JWT (jwtlib) :8005 Documentation build (MkDocs + MCP) Doc Forge (docforge) :8001"},{"location":"12_genericity_matrix/#tier-b-template-provided-copy-from-templatesapp-then-adapt","title":"Tier B \u2014 Template-provided (copy from templates/app, then adapt)","text":"

      These are reusable internal components shipped inside the template \u2014 they are not independent libraries. Copy them as-is, then rename/populate only what is app-specific. They give every app the same wiring shape as the reference app. Fetch them by kind with get_app_component, or the whole tree with get_app_template.

      Component Template location Action Shipped Layering & dependency rule the package layout + import rules keep \u2705 Composition root shape init.py (get_svc/get_db, idempotency) keep \u2705 Spec-first web edge interfaces/openapi.py, thin handlers bound to operationId keep \u2705 CRUD slice pattern domain/<resource>/{models.py, ports/db.py, ports/service.py, converters.py} rename + fill \u2705 Generic CRUD service infra/adapters/service.py (CrudsServiceAdaptor) keep \u2705 Repository factory infra/db/repos/mongo.py (_repo_factory) keep \u2705 Worker poll + status loop worker.py keep \u2705 Progress bus infra/adapters/sse.py (keyed in-memory bus) + an app SSE endpoint keep \u2705 HTTP response cache infra/http_cache.py (cache_response + generation invalidation) keep \u2705 Key-value store infra/adapters/kv.py (port + in-memory + Redis adaptor) keep \u2705 Pipeline scaffold pipelines/shared/{ports,loader,registry}.py + pipelines/<family>/ rename + fill \u2705 Operator scripts scripts/ (init() / get_svc() / shutdown()) rename \u2705 Testing harness integration fakes + fixture-parity pattern keep \u2705 Project scaffolding pyproject.toml, Dockerfile, docker-compose.yaml, .env.example, openapi.yaml rename + fill \u2705"},{"location":"12_genericity_matrix/#tier-b-patterns-port-from-the-reference-app","title":"Tier B (patterns) \u2014 port from the reference app","text":"

      A few Tier B capabilities are real and reusable, but are shaped by the app or the provider, so the template does not ship them verbatim \u2014 copying them blindly would carry app-specific names. Treat them as patterns: lift the shape from the reference app and rename per app.

      Component Reference location Action Shipped Human-in-the-loop durable pause + question document + resolve endpoint + resume map port + reshape \u2717 LLM client pool provider-select + retry/rotation pool scaffold port + reshape \u2717"},{"location":"12_genericity_matrix/#tier-c-app-specific-must-be-authored","title":"Tier C \u2014 App-specific (must be authored)","text":"Artifact Why Resource models the business shape Repository populate/cache rules the relationships Service business logic domain behaviour Pipeline families the transformation graph Pattern/rule resources data-driven policy Fixtures & golden outputs test data"},{"location":"12_genericity_matrix/#the-two-axes","title":"The two axes","text":"

      Everything app-specific reduces to two axes:

      1. Resource model \u2014 the set of resources, their fields, relationships, and the OpenAPI contract.
      2. Pipeline family \u2014 only if inputs require multi-step transformation.

      If you can specify those two axes, the rest of the app follows mechanically: Tier A is imported, Tier B is copied, and Tier C is the only thing you author.

      "},{"location":"13_library_registry/","title":"Library Registry","text":""},{"location":"13_library_registry/#summary","title":"Summary","text":"

      The Aetos ecosystem provides internal libraries for the shared concerns of every application. Use them. Writing a private copy of any capability below is a defect.

      Each library also exposes its own documentation as an MCP server. An AI agent can read a library's full API by connecting to its MCP and reading docs://index, docs://nav, and docs://modules/{module}.

      "},{"location":"13_library_registry/#catalog","title":"Catalog","text":"Library Import Purpose Use instead of MCP DAG Pipe dagpipe Deterministic state-transformation DAG engine hand-rolled orchestration/topological ordering :8006 Hexa hexa Declarative DI / composition-root tooling manual wiring, hand-written factories :8008 Mongo Ops mongo_ops Async MongoDB repositories, cache, populate, transactions raw Motor CRUD and connection lifecycle :8007 Omniread omniread Format-agnostic fetch + parse (HTML, PDF, CSV, XLSX) custom scrapers/parsers :8003 OpenAPI First openapi_first Spec-first FastAPI bootstrap, client, codegen decorator routers and hand-written clients :8004 Mail Intake mail_intake Provider-agnostic read-only email ingestion Gmail OAuth + MIME parsing :8002 Py JWT jwtlib JWT auth primitives (register/login/introspect) password hashing + token code :8005 Doc Forge docforge Docstring compiler \u2192 MkDocs + MCP docs custom doc generators :8001 Auth Server (service) Central identity service (register/login/me/introspect) building auth endpoints yourself (deferred)"},{"location":"13_library_registry/#how-to-choose","title":"How to choose","text":"You need to\u2026 Use persist and query documents Mongo Ops compose dependencies Hexa run a multi-step transformation DAG Pipe read/parse an arbitrary input format Omniread expose a contract-first HTTP API OpenAPI First ingest email Mail Intake authenticate users / verify tokens Py JWT (+ Auth Server) generate documentation Doc Forge"},{"location":"13_library_registry/#discovery","title":"Discovery","text":"

      A single standards MCP exposes find_library(capability), which returns the matching library, its import name, its \"use instead of\" guidance, and its MCP endpoint. Query it before writing any infrastructure.

      "},{"location":"13_library_registry/#notes","title":"Notes","text":"
      • Hexa is a full library with its own MCP server on :8008: read docs://index, docs://nav, and docs://modules/{module} (e.g. docs://modules/hexa.container) to discover the DI surface \u2014 build(cls, *, instances, config) and the PortNode model.
      • Auth Server is deferred: it has no pyproject.toml/importable package and no MCP bundle. Do not reach for it yet; py-jwt covers token primitives.
      • Library MCP servers expose documentation resources (not executable tools): they tell an agent the available API surface so it does not invent code.
      "}]} \ No newline at end of file diff --git a/standards/lib/sitemap.xml.gz b/standards/lib/sitemap.xml.gz index 36fff951d658b8fa1c2624e6fcea07a1ebb3772b..f0957ead3fa71accf94fc2db32d8580ce204bea1 100644 GIT binary patch delta 13 Ucmb=gXP58h;Akj0JCVHt03HJbkpKVy delta 13 Ucmb=gXP58h;9yWbGm*Ul02yinzW@LL diff --git a/standards/mcp/components.json b/standards/mcp/components.json index f76d541..9adb280 100644 --- a/standards/mcp/components.json +++ b/standards/mcp/components.json @@ -192,6 +192,12 @@ "placeholder": "", "description": "Docker build-context exclusions." }, + { + "kind": "gitignore", + "path": ".gitignore", + "placeholder": "", + "description": "VCS exclusions (caches, venv, generated _gen/)." + }, { "kind": "compose", "path": "docker-compose.yaml", diff --git a/standards/mcp/modules/build_recipe.json b/standards/mcp/modules/build_recipe.json index 3604400..e7ebbb6 100644 --- a/standards/mcp/modules/build_recipe.json +++ b/standards/mcp/modules/build_recipe.json @@ -2,7 +2,7 @@ "module": "build_recipe", "content": { "path": "build_recipe", - "docstring": "# Build Recipe\n\n## Summary\n\nThe ordered, deterministic checklist for generating a new application of the\nstandard shape. Every step is fixed; only the resource model (step 2), the\ncodegen pass (step 3), and the pipeline family (step 8) need app-specific work.\n\nThis page is served by the `standards` MCP as `docs://modules/build_recipe`. The\n`start_app` prompt on that server collects the two axes and hands off here; pull\nexact files with `get_app_component`.\n\n---\n\n## 🚀 Bootstrap\n\nFrom the standards docs checkout:\n\n```bash\ncp -r /standards/src/templates/app apps/\ncd apps/\n```\n\nInstall the internal libraries — editable local checkouts (preferred in\ndevelopment) or the private index (CI / fresh machines). Credentials are\nenvironment-only; never commit them.\n\n```bash\n# local checkouts\npip install -e ../aetos/openapi-first -e ../aetos/mongo-ops -e ../aetos/hexa \\\n -e ../aetos/dagpipe -e ../aetos/doc-forge\n# private index\npip install --extra-index-url \\\n \"https://$PIP_REGISTRY_USERNAME:$PIP_REGISTRY_PASSWORD@pip.aetoskia.com/simple\" .\n```\n\n---\n\n## 📋 Agent prompt (copyable)\n\n````markdown\n# Build a new application from the Aetos Standards blueprint\n\nYou are coding a complete new backend application conforming to the Aetos\nstandard app blueprint. Execute every step in order. Only steps 2–4 use the\napp-specific details provided by the user; everything else is deterministic.\n\n## Before writing any code\n\nConnect to the `standards` MCP server and read:\n- `docs://nav` (the module list)\n- `docs://modules/genericity_matrix` (Tier A library / Tier B template / Tier C authored)\n- `docs://modules/layering_rules` (the dependency rules)\n\nThen:\n- call `find_library` with `db`, `di`, `routing`, `ingestion`, `pipeline`, `docs`\n and note the Tier A libraries that already exist — never reimplement one;\n- call `list_app_components()` and pull each file you need with\n `get_app_component(kind, resource=..., family=...)` instead of writing it from\n scratch.\n\n## Steps\n\n1. **Scaffold** — copy `standards/src/templates/app/` into `apps//`\n (or pull the same tree via `get_app_template`). Keep the tree: `domain/`,\n `infra/`, `interfaces/`, `pipelines/`, `tests/`, `scripts/`, plus\n `pyproject.toml`, `openapi.yaml`, `config.py`, `init.py`, `main.py`,\n `worker.py`, `Dockerfile`, `docker-compose.yaml`, `.env.example`.\n\n2. **Author the spec (app-specific)** — write `openapi.yaml`: resources,\n schemas, `operationId`s, security, and extension metadata. This is the\n contract and drives all routing. Every operation MUST have an `operationId`.\n\n3. **Generate the scaffold (codegen)** — from the spec, generate the wire\n models and the exact handler signatures:\n ```bash\n openapi-first models openapi.yaml -o _gen/models.py\n openapi-first routes openapi.yaml -o _gen/routes \\\n --use-models --models-module _gen.models --inject-request --package\n ```\n `_gen/models.py` is the **wire contract** (Pydantic, one class per\n `components.schemas` entry). `_gen/routes//routes.py` lists every\n `operationId` with its exact parameter signature. Treat `_gen/` as generated\n (never hand-edit it) — the canonical handlers live in `interfaces/`.\n\n4. **Write each CRUD slice (app-specific)** — per resource create the four-file\n slice: `models.py` (domain model, **no `id`**), `ports/db.py`\n (`class XDocument(BaseDocument, X)` — id comes from `BaseDocument`),\n `ports/service.py`, and optional `converters.py`. Follow the CRUD template.\n\n5. **Wire the composition root** — in `init.py`, build leaf repositories, then\n dependent repositories, then services; add infra adapters (db, cache, stores,\n progress bus, fetchers). Expose `get_svc` overloads.\n\n6. **Add routes** — create `interfaces//routes.py` thin handlers whose\n names match the `operationId`s (use the signatures from step 3 as your\n checklist) and re-export them in `interfaces/openapi.py`. Add the cache\n decorator to read endpoints.\n\n7. **Bootstrap app and worker** — confirm `main.py` runs the OpenAPI-first app\n with its lifespan. If the app has long-running jobs, adapt `worker.py` to poll\n the job collection, run the pipeline, and publish progress.\n\n8. **Add pipeline families (only if data-driven)** — copy `pipelines/example/`\n to `pipelines//` with `abc.py`, `impl.py`, `pipeline.yaml`,\n `resources/`, and reuse `pipelines/shared/` (loader + registry). Register the\n root. Add fixtures and a parity test.\n\n9. **Add infrastructure only where needed** — Tier A capability: use the\n library. Tier B capability: copy the template. Never hand-write a Tier A\n library.\n\n10. **Tests** — integration tests with fakes at the HTTP edge; unit tests for\n domain and pipeline logic; a config-consistency test for rule-driven\n pipelines. Include `tests/integration/test_openapi_app_builds.py` (imports\n the app, asserts `import main` binds every operation).\n\n11. **Scripts and seeds** — operator scripts under `scripts/` using\n `init()` / `get_svc()` / `shutdown()`; run as `python -m scripts.`.\n\n12. **Docs** — author GSDFC docstrings, generate `.pyi` stubs, mark the package\n `py.typed`, and build the docs.\n\n**Definition of done:**\n- `python -c \"import main\"` succeeds.\n- `pytest` passes.\n- The app serves every operation in the spec; the worker drains its queue.\n- No domain module imports a web framework.\n- No Tier A library was reimplemented.\n````\n\n---\n\n## 🧭 Where the steps land\n\nThe steps map onto the layers as follows:\n\n| Step | Package / file | Depends on |\n|---------------------|-----------------------------------|---------------------------------------------|\n| 1 · Scaffold | whole `templates/app/` | — |\n| 2 · Spec | `openapi.yaml` | — |\n| 3 · Codegen | `_gen/` (generated) | spec, openapi-first |\n| 4 · CRUD slices | `domain//` | spec, CRUD template |\n| 5 · Wiring | `init.py` | every slice |\n| 6 · Routes | `interfaces//` | services, openapi route registry |\n| 7 · Bootstrap | `main.py`, `worker.py` | composition root |\n| 8 · Pipelines | `pipelines//`, `shared/` | library pipeline engine |\n| 9 · Infra | `infra/` | domain ports |\n| 10 · Tests | `tests/` | app + fixtures |\n| 11 · Scripts | `scripts/` | composition root |\n| 12 · Docs | docstrings + docs build | code, Doc Forge |\n\n---\n\n## 🔧 The codegen fast path, in detail\n\n`openapi-first` is spec → code only (it never generates a spec). Use it to erase\nthe two mechanical parts of the build:\n\n| Generated artifact | Replaces | Notes |\n|--------------------|----------|-------|\n| `_gen/models.py` | hand-writing Pydantic wire models | one class per `components.schemas` entry |\n| `_gen/routes//routes.py` | hand-typing handler signatures | names == `operationId`s; `request: Request` injected |\n\nRules of the road:\n\n- **Wire models are not domain models.** `_gen/models.py` is the HTTP contract;\n `domain//models.py` is your business shape. If the spec's schema has\n a required `id: str`, keep it off the domain model and let the document get its\n id from `BaseDocument`.\n- **Resource names are sanitized.** `/fetch-requests` → `fetch_requests.py`.\n- **Multipart bodies expand** to `UploadFile = File(...)` / `Form(...)` params.\n- **`_gen/` is disposable.** Regenerate on every spec change; `git diff` shows\n exactly what moved.\n\n---\n\n## ⚠️ Definition of done\n\n- `python -c \"import main\"` succeeds.\n- `pytest` passes.\n- The app serves the spec's operations; the worker drains its queue.\n- No domain module imports a web framework.\n- No Tier A library was reimplemented.\n\n---\n\n## Related\n\n- [02 – Layering Rules](02_layering_rules.md) · [05 – CRUD Template](05_crud_template.md) · [06 – Pipeline Scaffold](06_pipeline_scaffold.md) · [12 – Genericity Matrix](12_genericity_matrix.md)\n", + "docstring": "# Build Recipe\n\n## Summary\n\nThe ordered, deterministic checklist for generating a new application of the\nstandard shape. Every step is fixed; only the resource model (step 2), the\ncodegen pass (step 3), and the pipeline family (step 8) need app-specific work.\n\nThis page is served by the `standards` MCP as `docs://modules/build_recipe`. The\n`start_app` prompt on that server collects the two axes and hands off here; pull\nexact files with `get_app_component`.\n\n---\n\n## 🚀 Bootstrap\n\nFrom the standards docs checkout:\n\n```bash\ncp -r /standards/src/templates/app apps/\ncd apps/\n```\n\nInstall the internal libraries — editable local checkouts (preferred in\ndevelopment) or the private index (CI / fresh machines). Credentials are\nenvironment-only; never commit them.\n\n```bash\n# local checkouts\npip install -e ../aetos/openapi-first -e ../aetos/mongo-ops -e ../aetos/hexa \\\n -e ../aetos/dagpipe -e ../aetos/doc-forge\n# private index\npip install --extra-index-url \\\n \"https://$PIP_REGISTRY_USERNAME:$PIP_REGISTRY_PASSWORD@pip.aetoskia.com/simple\" .\n```\n\n---\n\n## 📋 Agent prompt (copyable)\n\n````markdown\n# Build a new application from the Aetos Standards blueprint\n\nYou are coding a complete new backend application conforming to the Aetos\nstandard app blueprint. Execute every step in order. Only steps 2–4 use the\napp-specific details provided by the user; everything else is deterministic.\n\n## Before writing any code\n\nConnect to the `standards` MCP server and read:\n- `docs://nav` (the module list)\n- `docs://modules/genericity_matrix` (Tier A library / Tier B template / Tier C authored)\n- `docs://modules/layering_rules` (the dependency rules)\n\nThen:\n- call `find_library` with `db`, `di`, `routing`, `ingestion`, `pipeline`, `docs`\n and note the Tier A libraries that already exist — never reimplement one;\n- call `list_app_components()` and pull each file you need with\n `get_app_component(kind, resource=..., family=...)` instead of writing it from\n scratch.\n\n## Steps\n\n1. **Scaffold** — copy `standards/src/templates/app/` into `apps//`\n (or pull the same tree via `get_app_template`). Keep the tree: `domain/`,\n `infra/`, `interfaces/`, `pipelines/`, `tests/`, `scripts/`, plus\n `pyproject.toml`, `openapi.yaml`, `config.py`, `init.py`, `main.py`,\n `worker.py`, `Dockerfile`, `docker-compose.yaml`, `.env.example`.\n\n2. **Author the spec (app-specific)** — write `openapi.yaml`: resources,\n schemas, `operationId`s, security, and extension metadata. This is the\n contract and drives all routing. Every operation MUST have an `operationId`.\n\n3. **Generate the scaffold (codegen)** — from the spec, generate the wire\n models and the exact handler signatures:\n ```bash\n openapi-first models openapi.yaml -o _gen/models.py\n openapi-first routes openapi.yaml -o _gen/routes \\\n --use-models --models-module _gen.models --inject-request --package --template-style\n ```\n `_gen/models.py` is the **wire contract** (Pydantic, one class per\n `components.schemas` entry). `_gen/routes//routes.py` lists every\n `operationId` with its exact parameter signature. `--template-style` makes\n those stubs `async def` with a `body` parameter, matching the handlers you\n keep in `interfaces/` (without it they are sync `def` stubs taking\n `payload`; both bind the same way). Treat `_gen/` as generated\n (never hand-edit it) — the canonical handlers live in `interfaces/`.\n\n4. **Write each CRUD slice (app-specific)** — per resource create the four-file\n slice: `models.py` (domain model, **no `id`**), `ports/db.py`\n (`class XDocument(BaseDocument, X)` — id comes from `BaseDocument`),\n `ports/service.py`, and optional `converters.py`. Follow the CRUD template.\n\n5. **Wire the composition root** — in `init.py`, build leaf repositories, then\n dependent repositories, then services; add infra adapters (db, cache, stores,\n progress bus, fetchers). Expose `get_svc` overloads.\n\n6. **Add routes** — create `interfaces//routes.py` thin handlers whose\n names match the `operationId`s (use the signatures from step 3 as your\n checklist) and re-export them in `interfaces/openapi.py`. Add the cache\n decorator to read endpoints.\n\n7. **Bootstrap app and worker** — confirm `main.py` runs the OpenAPI-first app\n with its lifespan. If the app has long-running jobs, adapt `worker.py` to poll\n the job collection, run the pipeline, and publish progress.\n\n8. **Add pipeline families (only if data-driven)** — copy `pipelines/example/`\n to `pipelines//` with `abc.py`, `impl.py`, `pipeline.yaml`,\n `resources/`, and reuse `pipelines/shared/` (loader + registry). Register the\n root. Add fixtures and a parity test.\n\n9. **Add infrastructure only where needed** — Tier A capability: use the\n library. Tier B capability: copy the template. Never hand-write a Tier A\n library.\n\n10. **Tests** — integration tests with fakes at the HTTP edge; unit tests for\n domain and pipeline logic; a config-consistency test for rule-driven\n pipelines. Include `tests/integration/test_openapi_app_builds.py` (imports\n the app, asserts `import main` binds every operation).\n\n11. **Scripts and seeds** — operator scripts under `scripts/` using\n `init()` / `get_svc()` / `shutdown()`; run as `python -m scripts.`.\n\n12. **Docs** — author GSDFC docstrings, generate `.pyi` stubs, mark the package\n `py.typed`, and build the docs.\n\n**Definition of done:**\n- `python -c \"import main\"` succeeds.\n- `pytest` passes.\n- The app serves every operation in the spec; the worker drains its queue.\n- No domain module imports a web framework.\n- No Tier A library was reimplemented.\n````\n\n---\n\n## 🧭 Where the steps land\n\nThe steps map onto the layers as follows:\n\n| Step | Package / file | Depends on |\n|---------------------|-----------------------------------|---------------------------------------------|\n| 1 · Scaffold | whole `templates/app/` | — |\n| 2 · Spec | `openapi.yaml` | — |\n| 3 · Codegen | `_gen/` (generated) | spec, openapi-first |\n| 4 · CRUD slices | `domain//` | spec, CRUD template |\n| 5 · Wiring | `init.py` | every slice |\n| 6 · Routes | `interfaces//` | services, openapi route registry |\n| 7 · Bootstrap | `main.py`, `worker.py` | composition root |\n| 8 · Pipelines | `pipelines//`, `shared/` | library pipeline engine |\n| 9 · Infra | `infra/` | domain ports |\n| 10 · Tests | `tests/` | app + fixtures |\n| 11 · Scripts | `scripts/` | composition root |\n| 12 · Docs | docstrings + docs build | code, Doc Forge |\n\n---\n\n## 🔧 The codegen fast path, in detail\n\n`openapi-first` is spec → code only (it never generates a spec). Use it to erase\nthe two mechanical parts of the build:\n\n| Generated artifact | Replaces | Notes |\n|--------------------|----------|-------|\n| `_gen/models.py` | hand-writing Pydantic wire models | one class per `components.schemas` entry |\n| `_gen/routes//routes.py` | hand-typing handler signatures | names == `operationId`s; `request: Request` injected |\n\nRules of the road:\n\n- **Wire models are not domain models.** `_gen/models.py` is the HTTP contract;\n `domain//models.py` is your business shape. If the spec's schema has\n a required `id: str`, keep it off the domain model and let the document get its\n id from `BaseDocument`.\n- **Use `--template-style`.** Stubs are then `async def ...(request, body: ...)`\n matching `interfaces/`; without it they are sync `def ...(payload: ...)`.\n Either binds (the router introspects the signature), but `--template-style`\n keeps the scaffold consistent with the code you keep.\n- **Resource names are sanitized.** `/fetch-requests` → `fetch_requests.py`.\n- **Multipart bodies expand** to `UploadFile = File(...)` / `Form(...)` params.\n- **`_gen/` is disposable.** Regenerate on every spec change; `git diff` shows\n exactly what moved.\n\n---\n\n## ⚠️ Definition of done\n\n- `python -c \"import main\"` succeeds.\n- `pytest` passes.\n- The app serves the spec's operations; the worker drains its queue.\n- No domain module imports a web framework.\n- No Tier A library was reimplemented.\n\n---\n\n## Related\n\n- [02 – Layering Rules](02_layering_rules.md) · [05 – CRUD Template](05_crud_template.md) · [06 – Pipeline Scaffold](06_pipeline_scaffold.md) · [12 – Genericity Matrix](12_genericity_matrix.md)\n", "objects": {} } } \ No newline at end of file diff --git a/standards/mcp/modules/crud_template.json b/standards/mcp/modules/crud_template.json index c9b8f98..2489dae 100644 --- a/standards/mcp/modules/crud_template.json +++ b/standards/mcp/modules/crud_template.json @@ -2,7 +2,7 @@ "module": "crud_template", "content": { "path": "crud_template", - "docstring": "# CRUD Template\n\n## Summary\n\nEvery resource follows the same four-file slice. Repetition is intentional: it\nmakes the codebase predictable and generatable.\n\n---\n\n## The slice\n\n```\ndomain//\n models.py # pure Pydantic model (no DB fields)\n ports/\n db.py # document model + repository factory\n service.py # service class + factory\n converters.py # (optional) transforms\n```\n\n## The id rule (model vs. document)\n\nTwo different objects, two different jobs:\n\n| Object | Lives in | Has `id`? | Purpose |\n|---|---|---|---|\n| **Domain model** | `domain//models.py` | **No** | business shape, framework-free |\n| **Document** | `domain//ports/db.py` | **Yes** (from `BaseDocument`) | Mongo persistence shape |\n\n`BaseDocument` (mongo-ops) supplies `id: Optional[PyObjectId]` (aliased to\n`_id`) plus `created_at` / `updated_at`. The domain model must **not** declare\n`id`; Mongo owns it.\n\n> **Trap — generated wire models.** `openapi-first models` produces the HTTP\n> *contract* (see the [Build Recipe](01_build_recipe.md), step 3). Response\n> schemas frequently declare a required `id: str`, so a generated class is\n> **not** a domain model. If you write\n> `class ItemDocument(Item, BaseDocument)` and `Item` re-exports a required\n> `id: str`, that field shadows `BaseDocument.id` and no Mongo `_id` can ever be\n> set. Fix it one of three ways:\n>\n> 1. **Preferred:** keep the domain model id-free (hand-author it from the CRUD\n> template), and keep the required `id` only on the wire model.\n> 2. Order the bases so the document wins:\n> `class ItemDocument(BaseDocument, Item)`.\n> 3. Keep a separate document type and translate with `ports/converters.py`.\n\n## models.py\n\nA framework-free Pydantic model describing the business shape (no `id`).\n\n```python\nclass Item(BaseModel):\n name: str\n quantity: int = 0\n```\n\n## ports/db.py\n\nThe persistence model extends the business model and adds database concerns.\nThe repository factory selects the right repository flavour.\n\n```python\n# Item is id-free; BaseDocument adds id/created_at/updated_at.\nclass ItemDocument(Item, BaseDocument):\n pass\n\ndef create_item_repo(*, owner_repo=None) -> ItemRepository:\n return repo_factory(\n collection=\"items\",\n document=ItemDocument,\n populate_rules=[PopulateRule(field=\"owner\", collection=\"users\")],\n cache=None, # or a CachedBaseRepository config\n )\n```\n\n## ports/service.py\n\nThe service is built through the DI library, injecting the repository. The\ngeneric CRUD base provides `create/read/update/patch/delete/search/...`.\n\n```python\nclass ItemService(CrudsServiceAdaptor[Item, ItemDocument]):\n model = Item\n schema = ItemDocument\n\ndef get_item_service(repo) -> ItemService:\n return build(ItemService, instances={\"repo\": repo})\n```\n\n## Naming conventions\n\n| Concept | Convention |\n|---|---|\n| collection | plural snake case (`items`) |\n| document | `Document` |\n| repository factory | `create__repo` |\n| service | `Service` |\n| service factory | `get__service` |\n| resource key | `_svc` |\n| operationId | `list_items`, `create_item`, ... |\n\n## Optional files\n\n- `converters.py` — pure functions mapping between domain types.\n- `ports/enrichment.py` — adapts a pipeline into an async callable the service\n can invoke (used when a resource is populated by a pipeline).\n", + "docstring": "# CRUD Template\n\n## Summary\n\nEvery resource follows the same four-file slice. Repetition is intentional: it\nmakes the codebase predictable and generatable.\n\n---\n\n## The slice\n\n```\ndomain//\n models.py # pure Pydantic model (no DB fields)\n ports/\n db.py # document model + repository factory\n service.py # service class + factory\n converters.py # (optional) transforms\n```\n\n## The id rule (model vs. document)\n\nTwo different objects, two different jobs:\n\n| Object | Lives in | Has `id`? | Purpose |\n|---|---|---|---|\n| **Domain model** | `domain//models.py` | **No** | business shape, framework-free |\n| **Document** | `domain//ports/db.py` | **Yes** (from `BaseDocument`) | Mongo persistence shape |\n\n`BaseDocument` (mongo-ops) supplies `id: Optional[PyObjectId]` (aliased to\n`_id`) plus `created_at` / `updated_at`. The domain model must **not** declare\n`id`; Mongo owns it.\n\n> **Trap — generated wire models.** `openapi-first models` produces the HTTP\n> *contract* (see the [Build Recipe](01_build_recipe.md), step 3). Response\n> schemas frequently declare a required `id: str`, so a generated class is\n> **not** a domain model. If you write\n> `class ItemDocument(Item, BaseDocument)` and `Item` re-exports a required\n> `id: str`, that field shadows `BaseDocument.id` and no Mongo `_id` can ever be\n> set. Fix it one of three ways:\n>\n> 1. **Preferred:** keep the domain model id-free (hand-author it from the CRUD\n> template), and keep the required `id` only on the wire model.\n> 2. Order the bases so the document wins:\n> `class ItemDocument(BaseDocument, Item)`.\n> 3. Keep a separate document type and translate with `ports/converters.py`.\n\n## models.py\n\nA framework-free Pydantic model describing the business shape (no `id`).\n\n```python\nclass Item(BaseModel):\n name: str\n quantity: int = 0\n```\n\n## ports/db.py\n\nThe persistence model extends the business model and adds database concerns.\nThe repository factory selects the right repository flavour.\n\n```python\n# Item is id-free; BaseDocument adds id/created_at/updated_at.\nclass ItemDocument(Item, BaseDocument):\n pass\n\ndef create_item_repo(*, owner_repo=None) -> ItemRepository:\n return repo_factory(\n collection=\"items\",\n document=ItemDocument,\n populate_rules=[PopulateRule(field_name=\"owner\", collection_name=\"users\")],\n cache=None, # or a CachedBaseRepository config\n )\n```\n\n## Nested references\n\nThe wire contract often declares a reference as an expanded object\n(`expense.entity: Entity`), while the domain model stays id-free. Pick one\nconvention and apply it consistently:\n\n- **Embed (default for small, stable refs)** — store the object as a `dict` /\n `list[dict]` on the document; the API returns it verbatim. No populate wiring.\n- **Populate** — store an id and declare a `PopulateRule`; mongo-ops expands it\n on read. The referenced repo must be registered with the `PopulationEngine`.\n- **Id only** — store the id and let the client resolve it.\n\nFor populate-backed repositories change a reference with `update`, not `patch`:\n`patch` rejects changes to populated FK fields by design.\n\n## ports/service.py\n\nThe service is built through the DI library, injecting the repository. The\ngeneric CRUD base provides `create/read/update/patch/delete/search/...`.\n\n```python\nclass ItemService(CrudsServiceAdaptor[Item, ItemDocument]):\n model = Item\n schema = ItemDocument\n\ndef get_item_service(repo) -> ItemService:\n return build(ItemService, instances={\"repo\": repo})\n```\n\n## Naming conventions\n\n| Concept | Convention |\n|---|---|\n| collection | plural snake case (`items`) |\n| document | `Document` |\n| repository factory | `create__repo` |\n| service | `Service` |\n| service factory | `get__service` |\n| resource key | `_svc` |\n| operationId | `list_items`, `create_item`, ... |\n\n## Optional files\n\n- `converters.py` — pure functions mapping between domain types.\n- `ports/enrichment.py` — adapts a pipeline into an async callable the service\n can invoke (used when a resource is populated by a pipeline).\n", "objects": {} } } \ No newline at end of file diff --git a/standards/mcp/modules/infra_adaptors.json b/standards/mcp/modules/infra_adaptors.json index fb3833a..3802809 100644 --- a/standards/mcp/modules/infra_adaptors.json +++ b/standards/mcp/modules/infra_adaptors.json @@ -2,7 +2,7 @@ "module": "infra_adaptors", "content": { "path": "infra_adaptors", - "docstring": "# Infra Adapters\n\n## Summary\n\nAll external systems are reached through adapters in `infra/`. Prefer an\nexisting internal library over writing a new adapter; where no library exists,\ncopy the Tier B scaffold from `templates/app`. See the Genericity Matrix for the\nTier A / B / C split.\n\n---\n\n## Database (MongoDB)\n\n- A single connection manager owns the client lifecycle.\n `setup_database(uri, db_name)` / `shutdown_database()`.\n- A repository **factory** selects the repository flavour:\n\n ```python\n repo_factory(collection, document, engine=None,\n populate_rules=None, cache=None)\n ```\n\n - base repository — plain CRUD;\n - populating repository — resolves referenced documents (foreign keys);\n - cached repository — an LRU/TTL cache in front of reads.\n\n- Referenced documents are described declaratively:\n\n ```python\n PopulateRule(field=\"owner\", collection=\"users\",\n nested_rules=[], max_depth=2, projection=None)\n ```\n\n- Transactions use a session context manager provided by the library.\n\n**Tier A — Library (Mongo Ops).** Do not write raw Motor calls in the domain;\nconsult the registry and the library's MCP before touching the driver.\n\n## HTTP response cache\n\n**Tier B — template.** Copy from `templates/app`; not a standalone library.\n\nA generic GET cache sits behind a decorator.\n\n```python\n@cache_response(ttl=60)\nasync def list_items(request): ...\n```\n\n- Keyed on path + query + user.\n- No-op when no store is wired.\n- `invalidate(*resources)` bumps a generation counter for write-triggered\n invalidation.\n\n## Event / progress bus\n\n**Tier B — template.** Copy from `templates/app`.\n\nA port with an in-memory default adapter (`publish`, `get_events`). Swap for a\ndurable broker in production (see Worker & Progress).\n\n## External clients (ingestion)\n\nExternal fetchers (files, mail, HTTP) are adapters that wrap ingestion\nlibraries. Construct them in the composition root and inject into\norchestrators.\n\n**Tier A — Libraries:** Omniread (formats) and Mail Intake (email).\n\n## LLM clients\n\n**Tier B — template.** Copy the pool scaffold from `templates/app`.\n\nWhen a pipeline calls a model provider, route through a pool adaptor that\nhandles provider selection, retries, and client lifecycle:\n\n```python\npool = setup_async_pool(provider=settings.llm_provider)\nresult = await pool.complete(model=\"...\", messages=[...])\n```\n\nProviders are selected by `LLM_PROVIDER` and configured by environment.\n\n## Key-value store\n\n**Tier B — template.** Copy from `templates/app`.\n\nFor report/snapshot storage, provide a pluggable store port with a Redis\nadapter and an in-memory fallback.\n\n## Rule\n\n- **Tier A (libraries: DB, DI, ingestion, mail, auth, docs, pipelines)** — use\n the library; writing a private copy is a defect. Query `find_library` first.\n- **Tier B (cache, progress bus, LLM pool, KV store, composition shape, worker,\n tests)** — these are template code, not libraries: copy them from\n `templates/app` rather than searching the registry.\n- **Tier C (models, business logic, pipeline families, rules)** — author them\n yourself.\n", + "docstring": "# Infra Adapters\n\n## Summary\n\nAll external systems are reached through adapters in `infra/`. Prefer an\nexisting internal library over writing a new adapter; where no library exists,\ncopy the Tier B scaffold from `templates/app`. See the Genericity Matrix for the\nTier A / B / C split.\n\n---\n\n## Database (MongoDB)\n\n- A single connection manager owns the client lifecycle.\n `setup_database(uri, db_name)` / `shutdown_database()`.\n- A repository **factory** selects the repository flavour:\n\n ```python\n repo_factory(collection, document, engine=None,\n populate_rules=None, cache=None)\n ```\n\n - base repository — plain CRUD;\n - populating repository — resolves referenced documents (foreign keys);\n - cached repository — an LRU/TTL cache in front of reads.\n\n- Referenced documents are described declaratively:\n\n ```python\n PopulateRule(field_name=\"owner\", collection_name=\"users\",\n nested_rules=[], max_depth=2, projection=None)\n ```\n\n- Transactions use a session context manager provided by the library.\n\n**Tier A — Library (Mongo Ops).** Do not write raw Motor calls in the domain;\nconsult the registry and the library's MCP before touching the driver.\n\n## HTTP response cache\n\n**Tier B — template.** Copy from `templates/app`; not a standalone library.\n\nA generic GET cache sits behind a decorator.\n\n```python\n@cache_response(ttl=60)\nasync def list_items(request): ...\n```\n\n- Keyed on path + query + user.\n- No-op when no store is wired.\n- `invalidate(*resources)` bumps a generation counter for write-triggered\n invalidation.\n\n## Event / progress bus\n\n**Tier B — template.** Copy from `templates/app`.\n\nA port with an in-memory default adapter (`publish`, `get_events`). Swap for a\ndurable broker in production (see Worker & Progress).\n\n## External clients (ingestion)\n\nExternal fetchers (files, mail, HTTP) are adapters that wrap ingestion\nlibraries. Construct them in the composition root and inject into\norchestrators.\n\n**Tier A — Libraries:** Omniread (formats) and Mail Intake (email).\n\n## LLM clients\n\n**Tier B — template.** Copy the pool scaffold from `templates/app`.\n\nWhen a pipeline calls a model provider, route through a pool adaptor that\nhandles provider selection, retries, and client lifecycle:\n\n```python\npool = setup_async_pool(provider=settings.llm_provider)\nresult = await pool.complete(model=\"...\", messages=[...])\n```\n\nProviders are selected by `LLM_PROVIDER` and configured by environment.\n\n## Key-value store\n\n**Tier B — template.** Copy from `templates/app`.\n\nFor report/snapshot storage, provide a pluggable store port with a Redis\nadapter and an in-memory fallback.\n\n## Rule\n\n- **Tier A (libraries: DB, DI, ingestion, mail, auth, docs, pipelines)** — use\n the library; writing a private copy is a defect. Query `find_library` first.\n- **Tier B (cache, progress bus, LLM pool, KV store, composition shape, worker,\n tests)** — these are template code, not libraries: copy them from\n `templates/app` rather than searching the registry.\n- **Tier C (models, business logic, pipeline families, rules)** — author them\n yourself.\n", "objects": {} } } \ No newline at end of file diff --git a/standards/mcp/modules/spec_first_routing.json b/standards/mcp/modules/spec_first_routing.json index 6b155e4..56cae85 100644 --- a/standards/mcp/modules/spec_first_routing.json +++ b/standards/mcp/modules/spec_first_routing.json @@ -2,7 +2,7 @@ "module": "spec_first_routing", "content": { "path": "spec_first_routing", - "docstring": "# Spec-First Routing\n\n## Summary\n\nThe OpenAPI document is the single source of truth for the HTTP contract.\nHandlers are bound to it by name; there is no decorator router.\n\n---\n\n## The contract\n\n- `openapi.yaml` (or `.json`) declares paths, schemas, parameters, responses,\n and security.\n- A custom `operationId` names the handler function.\n- The framework (`openapi_first`) validates requests/responses against the\n spec and dispatches to the handler whose name equals `operationId`.\n\n## App bootstrap\n\n```python\n# main.py\nfrom openapi_first.app import OpenAPIFirstApp\n\napp = OpenAPIFirstApp(\n openapi_path=\"openapi.yaml\",\n routes_module=interfaces.openapi, # module whose namespace holds handlers\n title=\" API\",\n version=\"0.1.0\",\n lifespan=lifespan,\n)\n```\n\n`lifespan` calls the composition root:\n\n```python\n@asynccontextmanager\nasync def lifespan(app):\n app.state.db_resources = await init()\n yield\n await shutdown()\n```\n\n## Handler registry\n\n`interfaces/openapi.py` is a flat re-export of every handler, so the framework\ncan resolve `operationId` across the whole app:\n\n```python\nfrom interfaces.item.routes import *\nfrom interfaces.wallet.routes import *\nfrom interfaces.progress.sse import *\n```\n\n## Handler shape\n\nHandlers are thin and synchronous with respect to business logic:\n\n```python\nasync def create_item(request: Request, body: dict):\n svc = request.app.state.db_resources[\"item_svc\"]\n return await svc.create(Item.model_validate(body))\n```\n\n## Conventions\n\n- `operationId` naming: `list_`, `create_`,\n `get_`, `update_`, `delete_`.\n- Read-only endpoints may use a cache decorator keyed on path + query + user.\n- Validation errors -> `400`; not found -> `404`; delete -> `204`.\n- Security is declared once in the spec and enforced by the framework; handlers\n contain no auth code.\n\n## Authless apps\n\nAuth is **deferred** in the standard: the same spec-first app runs with or\nwithout it, because security lives entirely in the spec, never in handlers.\nFor an app that needs no auth yet, do **not** invent per-handler auth checks —\njust omit the security keys:\n\n- Remove the top-level `security:` block.\n- Remove `components.securitySchemes`.\n- Do **not** add `security: []` to operations. An explicit empty array is a\n per-operation *override* that means \"public\", and it only matters when a\n global `security` block exists. Leaving every operation without a `security`\n key is the cleanest authless spec.\n\nThe handlers, services, and composition root are unchanged. When auth is needed\nlater, add the global `security` block and an introspecting dependency; only\nhandlers that must opt out get `security: []`.\n\n## Codegen\n\n`openapi_first` scaffolds models and route stubs from the spec (see the\n[Build Recipe](01_build_recipe.md), step 3):\n\n```bash\nopenapi-first models openapi.yaml -o _gen/models.py\nopenapi-first routes openapi.yaml -o _gen/routes \\\n --use-models --models-module _gen.models --inject-request --package\n```\n\nPrefer codegen over hand-writing boilerplate. The result is the wire contract\nand a set of `operationId`-named signatures — not the canonical handlers.\n", + "docstring": "# Spec-First Routing\n\n## Summary\n\nThe OpenAPI document is the single source of truth for the HTTP contract.\nHandlers are bound to it by name; there is no decorator router.\n\n---\n\n## The contract\n\n- `openapi.yaml` (or `.json`) declares paths, schemas, parameters, responses,\n and security.\n- A custom `operationId` names the handler function.\n- The framework (`openapi_first`) validates requests/responses against the\n spec and dispatches to the handler whose name equals `operationId`.\n\n## App bootstrap\n\n```python\n# main.py\nfrom openapi_first.app import OpenAPIFirstApp\n\napp = OpenAPIFirstApp(\n openapi_path=\"openapi.yaml\",\n routes_module=interfaces.openapi, # module whose namespace holds handlers\n title=\" API\",\n version=\"0.1.0\",\n lifespan=lifespan,\n)\n```\n\n`lifespan` calls the composition root:\n\n```python\n@asynccontextmanager\nasync def lifespan(app):\n app.state.db_resources = await init()\n yield\n await shutdown()\n```\n\n## Handler registry\n\n`interfaces/openapi.py` is a flat re-export of every handler, so the framework\ncan resolve `operationId` across the whole app:\n\n```python\nfrom interfaces.item.routes import *\nfrom interfaces.wallet.routes import *\nfrom interfaces.progress.sse import *\n```\n\n## Handler shape\n\nHandlers are thin and synchronous with respect to business logic:\n\n```python\nasync def create_item(request: Request, body: dict):\n svc = request.app.state.db_resources[\"item_svc\"]\n return await svc.create(Item.model_validate(body))\n```\n\n**Generated stubs.** `openapi-first routes` emits this exact shape (`async def`\nhandlers taking `body`) when given `--template-style`; without that flag the\nstub is `def create_item(payload: ItemInput)` — sync, with a `payload`\nparameter. Both shapes bind identically: the router resolves handlers by\n`operationId` and introspects the FastAPI signature, and it never inspects the\nrequest body. Pass `--template-style` so the generated signatures match the\nhandlers you keep in `interfaces/`.\n\n## Conventions\n\n- `operationId` naming: `list_`, `create_`,\n `get_`, `update_`, `delete_`.\n- Read-only endpoints may use a cache decorator keyed on path + query + user.\n- Validation errors -> `400`; not found -> `404`; delete -> `204`.\n- Security is declared once in the spec and enforced by the framework; handlers\n contain no auth code.\n\n## Authless apps\n\nAuth is **deferred** in the standard: the same spec-first app runs with or\nwithout it, because security lives entirely in the spec, never in handlers.\nFor an app that needs no auth yet, do **not** invent per-handler auth checks —\njust omit the security keys:\n\n- Remove the top-level `security:` block.\n- Remove `components.securitySchemes`.\n- Do **not** add `security: []` to operations. An explicit empty array is a\n per-operation *override* that means \"public\", and it only matters when a\n global `security` block exists. Leaving every operation without a `security`\n key is the cleanest authless spec.\n\nProfile endpoints (`/users/me`) and profile headers (`x-profile`) still exist in\nan authless app. Define a stand-in identity in the composition root — for\nexample `get_current_user` returns or lazily creates a fixed \"Demo User\" — so\nhandlers stay auth-agnostic and only the wiring changes when real auth arrives.\n\nThe handlers, services, and composition root are unchanged. When auth is needed\nlater, add the global `security` block and an introspecting dependency; only\nhandlers that must opt out get `security: []`.\n\n## Codegen\n\n`openapi_first` scaffolds models and route stubs from the spec (see the\n[Build Recipe](01_build_recipe.md), step 3):\n\n```bash\nopenapi-first models openapi.yaml -o _gen/models.py\nopenapi-first routes openapi.yaml -o _gen/routes \\\n --use-models --models-module _gen.models --inject-request --package --template-style\n```\n\nPrefer codegen over hand-writing boilerplate. The result is the wire contract\nand a set of `operationId`-named signatures — not the canonical handlers.\n", "objects": {} } } \ No newline at end of file diff --git a/standards/mcp/templates/app/.gitignore b/standards/mcp/templates/app/.gitignore new file mode 100644 index 0000000..cebb4e9 --- /dev/null +++ b/standards/mcp/templates/app/.gitignore @@ -0,0 +1,23 @@ +# Byte-compiled / caches +__pycache__/ +*.py[cod] +.ruff_cache/ +.pytest_cache/ +.mypy_cache/ + +# Virtual environments +.venv/ +venv/ + +# Env +.env + +# Generated wire contract (regenerate from openapi.yaml) +_gen/ + +# Packaging / coverage +dist/ +build/ +*.egg-info/ +htmlcov/ +.coverage diff --git a/standards/mcp/templates/app/domain/item/ports/db.py b/standards/mcp/templates/app/domain/item/ports/db.py index d1dbbb5..d8905d2 100644 --- a/standards/mcp/templates/app/domain/item/ports/db.py +++ b/standards/mcp/templates/app/domain/item/ports/db.py @@ -2,9 +2,10 @@ Mongo persistence for items: document schema and repository factory. """ +from mongo_ops import BaseDocument, BaseRepository + from domain.item.models import Item from infra.db.repos.mongo import _repo_factory as mongo_repo_factory -from mongo_ops import BaseDocument, BaseRepository class ItemDocument(Item, BaseDocument): diff --git a/standards/mcp/templates/app/domain/job/models.py b/standards/mcp/templates/app/domain/job/models.py index 17ec51e..968d8c2 100644 --- a/standards/mcp/templates/app/domain/job/models.py +++ b/standards/mcp/templates/app/domain/job/models.py @@ -3,6 +3,7 @@ Job domain model — a pipeline work item consumed by the background worker. """ from typing import Literal + from pydantic import BaseModel, Field diff --git a/standards/mcp/templates/app/domain/job/ports/db.py b/standards/mcp/templates/app/domain/job/ports/db.py index 547e2b5..c085eb3 100644 --- a/standards/mcp/templates/app/domain/job/ports/db.py +++ b/standards/mcp/templates/app/domain/job/ports/db.py @@ -2,9 +2,10 @@ Mongo persistence for jobs: document schema and repository factory. """ +from mongo_ops import BaseDocument, BaseRepository + from domain.job.models import Job from infra.db.repos.mongo import _repo_factory as mongo_repo_factory -from mongo_ops import BaseDocument, BaseRepository class JobDocument(Job, BaseDocument): diff --git a/standards/mcp/templates/app/infra/adapters/kv.py b/standards/mcp/templates/app/infra/adapters/kv.py index 00e3ff9..04b75a3 100644 --- a/standards/mcp/templates/app/infra/adapters/kv.py +++ b/standards/mcp/templates/app/infra/adapters/kv.py @@ -21,18 +21,17 @@ Example: from __future__ import annotations from abc import ABC, abstractmethod -from typing import Dict, List, Optional class KVStore(ABC): """Abstract key-value store port.""" @abstractmethod - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Return the value at ``key``, or ``None``.""" @abstractmethod - async def set(self, key: str, value: str, ttl: Optional[int] = None) -> None: + async def set(self, key: str, value: str, ttl: int | None = None) -> None: """Store ``value`` at ``key`` with an optional TTL in seconds.""" @abstractmethod @@ -40,7 +39,7 @@ class KVStore(ABC): """Delete ``key`` if present.""" @abstractmethod - async def scan(self, pattern: str) -> List[str]: + async def scan(self, pattern: str) -> list[str]: """Return keys matching a glob ``pattern``.""" async def clear(self, pattern: str = "*") -> None: @@ -67,8 +66,8 @@ class InMemoryKV(KVStore): def __init__(self) -> None: """Initialize an empty store with no imports or connections.""" - self._data: Dict[str, str] = {} - self._expiry: Dict[str, float] = {} + self._data: dict[str, str] = {} + self._expiry: dict[str, float] = {} def _expired(self, key: str) -> bool: import time @@ -80,13 +79,13 @@ class InMemoryKV(KVStore): return True return False - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Return the value at ``key``, or ``None`` when absent/expired.""" if self._expired(key): return None return self._data.get(key) - async def set(self, key: str, value: str, ttl: Optional[int] = None) -> None: + async def set(self, key: str, value: str, ttl: int | None = None) -> None: """Store ``value`` at ``key`` with an optional TTL in seconds.""" import time @@ -98,7 +97,7 @@ class InMemoryKV(KVStore): self._data.pop(key, None) self._expiry.pop(key, None) - async def scan(self, pattern: str) -> List[str]: + async def scan(self, pattern: str) -> list[str]: """Return live keys matching a glob ``pattern``.""" import fnmatch @@ -126,11 +125,11 @@ class RedisKV(KVStore): self._redis = Redis.from_url(url, decode_responses=decode_responses) - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Return the value at ``key``, or ``None``.""" return await self._redis.get(key) - async def set(self, key: str, value: str, ttl: Optional[int] = None) -> None: + async def set(self, key: str, value: str, ttl: int | None = None) -> None: """Store ``value`` at ``key`` with an optional TTL in seconds.""" await self._redis.set(key, value, ex=ttl) @@ -138,7 +137,7 @@ class RedisKV(KVStore): """Delete ``key`` if present.""" await self._redis.delete(key) - async def scan(self, pattern: str) -> List[str]: + async def scan(self, pattern: str) -> list[str]: """Return keys matching a glob ``pattern``.""" return [key async for key in self._redis.scan_iter(match=pattern)] diff --git a/standards/mcp/templates/app/infra/adapters/service.py b/standards/mcp/templates/app/infra/adapters/service.py index dfa46d8..9454d83 100644 --- a/standards/mcp/templates/app/infra/adapters/service.py +++ b/standards/mcp/templates/app/infra/adapters/service.py @@ -5,14 +5,14 @@ Generic CRUD service adaptor over a Mongo repository. Bridge pydantic models and mongo-ops documents, exposing create, read, update, patch, delete, and search operations for every domain that needs them. """ -from typing import Generic, Type, TypeVar +from typing import Generic, TypeVar + +from mongo_ops import BaseDocument, PopulatingRepository +from mongo_ops.models import PyObjectId +from pydantic import BaseModel from infra.db.repos import mongo -from pydantic import BaseModel -from mongo_ops import BaseDocument -from mongo_ops.models import PyObjectId - Model = TypeVar("Model", bound=BaseModel) Schema = TypeVar("Schema", bound=BaseDocument) @@ -27,8 +27,8 @@ class CrudsServiceAdaptor(Generic[Model, Schema]): """ repo: mongo.BaseRepository - model: Type[Model] - schema: Type[Schema] + model: type[Model] + schema: type[Schema] def __init__(self, *, repo=None): """Initialize the adaptor with a backing repository. @@ -83,7 +83,17 @@ class CrudsServiceAdaptor(Generic[Model, Schema]): Returns: Schema | None: Updated document, or ``None`` when absent. + + Notes: + A populating repository expects a document *model* (so FK fields can + be depopulated), while a base repository expects a raw mapping. This + method normalizes both inputs to whatever the repo requires. """ + if isinstance(self.repo, PopulatingRepository): + if not isinstance(data, self.schema): + fields = data if isinstance(data, dict) else self.model_to_dict(data) + data = self.schema(**fields) + return await self.repo.update(id, data) return await self.repo.update( id, data if isinstance(data, dict) else self.model_to_dict(data), diff --git a/standards/mcp/templates/app/infra/adapters/sse.py b/standards/mcp/templates/app/infra/adapters/sse.py index 70c6f31..bd1a799 100644 --- a/standards/mcp/templates/app/infra/adapters/sse.py +++ b/standards/mcp/templates/app/infra/adapters/sse.py @@ -7,7 +7,7 @@ SSEInMemoryBus — in-memory event bus for SSE progress events. Buffer published progress events so they can be replayed (and streamed) per correlation key, such as a job, upload, or fetch-request id. """ -from typing import Generic, List, TypeVar +from typing import Generic, TypeVar T = TypeVar("T") @@ -27,7 +27,7 @@ class SSEInMemoryBus(Generic[T]): key_attr (str): Attribute name on each event holding the stream correlation key (for example ``"job_id"``). """ - self._events: List[T] = [] + self._events: list[T] = [] self._key_attr = key_attr async def publish(self, event: T) -> None: @@ -38,7 +38,7 @@ class SSEInMemoryBus(Generic[T]): """ self._events.append(event) - def get_events(self, key: str) -> List[T]: + def get_events(self, key: str) -> list[T]: """Return buffered events for a correlation key. Args: diff --git a/standards/mcp/templates/app/infra/db/repos/mongo.py b/standards/mcp/templates/app/infra/db/repos/mongo.py index a68f8f3..2fe8bb8 100644 --- a/standards/mcp/templates/app/infra/db/repos/mongo.py +++ b/standards/mcp/templates/app/infra/db/repos/mongo.py @@ -5,21 +5,20 @@ Generic MongoDB CRUD repo — single implementation for all domains. Select a concrete mongo-ops repository (base, cached, or populating) from the options supplied to the factory. """ -from typing import List, Optional, Type from mongo_ops import BaseRepository, CachedBaseRepository, PopulatingRepository from mongo_ops.cache import CacheBackend, CacheConfig from mongo_ops.models import BaseDocument -from mongo_ops.populate import PopulationEngine, PopulateRule +from mongo_ops.populate import PopulateRule, PopulationEngine def _repo_factory( collection: str, - db_model: Type[BaseDocument], - cache_backend: Optional[CacheBackend] = None, - cache_config: Optional[CacheConfig] = None, - engine: Optional[PopulationEngine] = None, - populate_rules: Optional[List[PopulateRule]] = None, + db_model: type[BaseDocument], + cache_backend: CacheBackend | None = None, + cache_config: CacheConfig | None = None, + engine: PopulationEngine | None = None, + populate_rules: list[PopulateRule] | None = None, ) -> BaseRepository: """Build the appropriate mongo-ops repository. diff --git a/standards/mcp/templates/app/infra/http_cache.py b/standards/mcp/templates/app/infra/http_cache.py index 8debf88..dfc36e2 100644 --- a/standards/mcp/templates/app/infra/http_cache.py +++ b/standards/mcp/templates/app/infra/http_cache.py @@ -23,8 +23,9 @@ Notes: """ import hashlib import json +from collections.abc import Callable from functools import wraps -from typing import Any, Callable, Optional +from typing import Any from fastapi import Request, Response from fastapi.encoders import jsonable_encoder @@ -55,7 +56,7 @@ class HttpCacheStore: self._redis = Redis.from_url(url, decode_responses=decode_responses) - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Read the value stored at ``key``. Args: @@ -173,7 +174,7 @@ def _cache_key(request: Request, generation: str = "0") -> str: return hashlib.sha256("|".join(parts).encode()).hexdigest() -def cache_response(ttl: Optional[int] = None) -> Callable: +def cache_response(ttl: int | None = None) -> Callable: """Decorate a GET-style route handler to cache its JSON response. Args: @@ -203,7 +204,7 @@ def cache_response(ttl: Optional[int] = None) -> Callable: return await func(*args, **kwargs) db_resources = getattr(request.app.state, "db_resources", None) or {} - store: Optional[HttpCacheStore] = db_resources.get("http_cache_store") + store: HttpCacheStore | None = db_resources.get("http_cache_store") if store is None: return await func(*args, **kwargs) diff --git a/standards/mcp/templates/app/init.py b/standards/mcp/templates/app/init.py index 68a6fac..46b1a7f 100644 --- a/standards/mcp/templates/app/init.py +++ b/standards/mcp/templates/app/init.py @@ -4,7 +4,7 @@ Infrastructure initialization — connects MongoDB and wires all repos and servi Shared bootstrap for both the FastAPI app and the worker, guarded against repeated initialization via a module-level flag. """ -from typing import Any, Dict +from typing import Any from config import settings from domain.item.ports.db import create_item_repo @@ -13,11 +13,11 @@ from domain.job.ports.db import create_job_repo from domain.job.ports.service import get_job_service from infra.db.setup import setup_database, shutdown_database -_resources: Dict[str, Any] = {} +_resources: dict[str, Any] = {} _initialized: bool = False -async def init() -> Dict[str, Any]: +async def init() -> dict[str, Any]: """Connect MongoDB, wire all repos and services, and return the resources dict. Idempotent: repeated calls short-circuit and return the cached resources. diff --git a/standards/mcp/templates/app/interfaces/openapi.py b/standards/mcp/templates/app/interfaces/openapi.py index 8c43aa7..489f65a 100644 --- a/standards/mcp/templates/app/interfaces/openapi.py +++ b/standards/mcp/templates/app/interfaces/openapi.py @@ -4,5 +4,6 @@ All handler functions — names must match openapi.yaml operationId values. The OpenAPI-first router resolves each operation to a handler by name, so this module re-exports every route handler from the subpackages into one flat namespace. """ +# ruff: noqa: F403 from interfaces.item.routes import * from interfaces.job.routes import * \ No newline at end of file diff --git a/standards/mcp/templates/app/main.py b/standards/mcp/templates/app/main.py index 0a451e3..a569f7b 100644 --- a/standards/mcp/templates/app/main.py +++ b/standards/mcp/templates/app/main.py @@ -4,6 +4,7 @@ FastAPI application bootstrap — uses OpenAPIFirstApp for spec-driven routing. Handlers are looked up by operationId in interfaces.openapi, so no router tables are declared here; the lifespan initializes and shuts down app resources. """ +# ruff: noqa: E402 import logging from contextlib import asynccontextmanager diff --git a/standards/mcp/templates/app/pipelines/shared/loader.py b/standards/mcp/templates/app/pipelines/shared/loader.py index 10a3c2b..d6e0d0d 100644 --- a/standards/mcp/templates/app/pipelines/shared/loader.py +++ b/standards/mcp/templates/app/pipelines/shared/loader.py @@ -9,7 +9,6 @@ from pathlib import Path from typing import Any import yaml - from hexa import build diff --git a/standards/mcp/templates/app/pipelines/shared/registry.py b/standards/mcp/templates/app/pipelines/shared/registry.py index d5de929..d449aba 100644 --- a/standards/mcp/templates/app/pipelines/shared/registry.py +++ b/standards/mcp/templates/app/pipelines/shared/registry.py @@ -4,13 +4,12 @@ Family registry — resolve a ``(kind, variant, mode)`` triple to a root class. A single lookup replaces hand-rolled ``if/else`` family selection. """ -from typing import Dict, Type # Populated by ``register`` as families import. -_FAMILIES: Dict[str, Dict[str, Dict[str, Type]]] = {} +_FAMILIES: dict[str, dict[str, dict[str, type]]] = {} -def register(kind: str, variant: str, root_cls: Type, mode: str = "default") -> Type: +def register(kind: str, variant: str, root_cls: type, mode: str = "default") -> type: """Register ``root_cls`` for a ``(kind, variant, mode)`` triple. Returns: @@ -20,7 +19,7 @@ def register(kind: str, variant: str, root_cls: Type, mode: str = "default") -> return root_cls -def resolve_root(kind: str, variant: str, mode: str = "default") -> Type: +def resolve_root(kind: str, variant: str, mode: str = "default") -> type: """Return the root class for a ``(kind, variant, mode)`` triple. Raises: diff --git a/standards/mcp/templates/app/pyproject.toml b/standards/mcp/templates/app/pyproject.toml index 408c0e1..8da9f77 100644 --- a/standards/mcp/templates/app/pyproject.toml +++ b/standards/mcp/templates/app/pyproject.toml @@ -40,6 +40,7 @@ testpaths = ["tests"] [tool.ruff] line-length = 100 target-version = "py311" +extend-exclude = ["_gen"] [tool.ruff.lint] select = ["E", "F", "I", "UP"] \ No newline at end of file diff --git a/standards/mcp/templates/app/tests/integration/test_item_routes.py b/standards/mcp/templates/app/tests/integration/test_item_routes.py index a7ecc0e..3525d88 100644 --- a/standards/mcp/templates/app/tests/integration/test_item_routes.py +++ b/standards/mcp/templates/app/tests/integration/test_item_routes.py @@ -1,9 +1,9 @@ import importlib -import interfaces.openapi as routes_module - from openapi_first.app import OpenAPIFirstApp +import interfaces.openapi as routes_module + def test_openapi_app_builds(): """The spec-first app must build and plant each route handler.""" diff --git a/standards/mcp/templates/app/tests/integration/test_openapi_app_builds.py b/standards/mcp/templates/app/tests/integration/test_openapi_app_builds.py index fe811c0..2d499c3 100644 --- a/standards/mcp/templates/app/tests/integration/test_openapi_app_builds.py +++ b/standards/mcp/templates/app/tests/integration/test_openapi_app_builds.py @@ -1,11 +1,10 @@ import importlib import yaml +from openapi_first.app import OpenAPIFirstApp import interfaces.openapi as routes_module -from openapi_first.app import OpenAPIFirstApp - _HTTP_METHODS = {"get", "post", "put", "patch", "delete", "head", "options"} diff --git a/standards/src/lib/01_build_recipe.md b/standards/src/lib/01_build_recipe.md index c3734e1..1cf3157 100644 --- a/standards/src/lib/01_build_recipe.md +++ b/standards/src/lib/01_build_recipe.md @@ -76,11 +76,14 @@ Then: ```bash openapi-first models openapi.yaml -o _gen/models.py openapi-first routes openapi.yaml -o _gen/routes \ - --use-models --models-module _gen.models --inject-request --package + --use-models --models-module _gen.models --inject-request --package --template-style ``` `_gen/models.py` is the **wire contract** (Pydantic, one class per `components.schemas` entry). `_gen/routes//routes.py` lists every - `operationId` with its exact parameter signature. Treat `_gen/` as generated + `operationId` with its exact parameter signature. `--template-style` makes + those stubs `async def` with a `body` parameter, matching the handlers you + keep in `interfaces/` (without it they are sync `def` stubs taking + `payload`; both bind the same way). Treat `_gen/` as generated (never hand-edit it) — the canonical handlers live in `interfaces/`. 4. **Write each CRUD slice (app-specific)** — per resource create the four-file @@ -168,6 +171,10 @@ Rules of the road: `domain//models.py` is your business shape. If the spec's schema has a required `id: str`, keep it off the domain model and let the document get its id from `BaseDocument`. +- **Use `--template-style`.** Stubs are then `async def ...(request, body: ...)` + matching `interfaces/`; without it they are sync `def ...(payload: ...)`. + Either binds (the router introspects the signature), but `--template-style` + keeps the scaffold consistent with the code you keep. - **Resource names are sanitized.** `/fetch-requests` → `fetch_requests.py`. - **Multipart bodies expand** to `UploadFile = File(...)` / `Form(...)` params. - **`_gen/` is disposable.** Regenerate on every spec change; `git diff` shows diff --git a/standards/src/lib/04_spec_first_routing.md b/standards/src/lib/04_spec_first_routing.md index b7fca3c..3cc98a5 100644 --- a/standards/src/lib/04_spec_first_routing.md +++ b/standards/src/lib/04_spec_first_routing.md @@ -61,6 +61,14 @@ async def create_item(request: Request, body: dict): return await svc.create(Item.model_validate(body)) ``` +**Generated stubs.** `openapi-first routes` emits this exact shape (`async def` +handlers taking `body`) when given `--template-style`; without that flag the +stub is `def create_item(payload: ItemInput)` — sync, with a `payload` +parameter. Both shapes bind identically: the router resolves handlers by +`operationId` and introspects the FastAPI signature, and it never inspects the +request body. Pass `--template-style` so the generated signatures match the +handlers you keep in `interfaces/`. + ## Conventions - `operationId` naming: `list_`, `create_`, @@ -84,6 +92,11 @@ just omit the security keys: global `security` block exists. Leaving every operation without a `security` key is the cleanest authless spec. +Profile endpoints (`/users/me`) and profile headers (`x-profile`) still exist in +an authless app. Define a stand-in identity in the composition root — for +example `get_current_user` returns or lazily creates a fixed "Demo User" — so +handlers stay auth-agnostic and only the wiring changes when real auth arrives. + The handlers, services, and composition root are unchanged. When auth is needed later, add the global `security` block and an introspecting dependency; only handlers that must opt out get `security: []`. @@ -96,7 +109,7 @@ handlers that must opt out get `security: []`. ```bash openapi-first models openapi.yaml -o _gen/models.py openapi-first routes openapi.yaml -o _gen/routes \ - --use-models --models-module _gen.models --inject-request --package + --use-models --models-module _gen.models --inject-request --package --template-style ``` Prefer codegen over hand-writing boilerplate. The result is the wire contract diff --git a/standards/src/lib/05_crud_template.md b/standards/src/lib/05_crud_template.md index fec4cd6..628f21c 100644 --- a/standards/src/lib/05_crud_template.md +++ b/standards/src/lib/05_crud_template.md @@ -69,11 +69,26 @@ def create_item_repo(*, owner_repo=None) -> ItemRepository: return repo_factory( collection="items", document=ItemDocument, - populate_rules=[PopulateRule(field="owner", collection="users")], + populate_rules=[PopulateRule(field_name="owner", collection_name="users")], cache=None, # or a CachedBaseRepository config ) ``` +## Nested references + +The wire contract often declares a reference as an expanded object +(`expense.entity: Entity`), while the domain model stays id-free. Pick one +convention and apply it consistently: + +- **Embed (default for small, stable refs)** — store the object as a `dict` / + `list[dict]` on the document; the API returns it verbatim. No populate wiring. +- **Populate** — store an id and declare a `PopulateRule`; mongo-ops expands it + on read. The referenced repo must be registered with the `PopulationEngine`. +- **Id only** — store the id and let the client resolve it. + +For populate-backed repositories change a reference with `update`, not `patch`: +`patch` rejects changes to populated FK fields by design. + ## ports/service.py The service is built through the DI library, injecting the repository. The diff --git a/standards/src/lib/09_infra_adaptors.md b/standards/src/lib/09_infra_adaptors.md index ea1d2e9..6da0559 100644 --- a/standards/src/lib/09_infra_adaptors.md +++ b/standards/src/lib/09_infra_adaptors.md @@ -27,7 +27,7 @@ Tier A / B / C split. - Referenced documents are described declaratively: ```python - PopulateRule(field="owner", collection="users", + PopulateRule(field_name="owner", collection_name="users", nested_rules=[], max_depth=2, projection=None) ``` diff --git a/standards/src/templates/app/.gitignore b/standards/src/templates/app/.gitignore new file mode 100644 index 0000000..cebb4e9 --- /dev/null +++ b/standards/src/templates/app/.gitignore @@ -0,0 +1,23 @@ +# Byte-compiled / caches +__pycache__/ +*.py[cod] +.ruff_cache/ +.pytest_cache/ +.mypy_cache/ + +# Virtual environments +.venv/ +venv/ + +# Env +.env + +# Generated wire contract (regenerate from openapi.yaml) +_gen/ + +# Packaging / coverage +dist/ +build/ +*.egg-info/ +htmlcov/ +.coverage diff --git a/standards/src/templates/app/domain/item/ports/db.py b/standards/src/templates/app/domain/item/ports/db.py index d1dbbb5..d8905d2 100644 --- a/standards/src/templates/app/domain/item/ports/db.py +++ b/standards/src/templates/app/domain/item/ports/db.py @@ -2,9 +2,10 @@ Mongo persistence for items: document schema and repository factory. """ +from mongo_ops import BaseDocument, BaseRepository + from domain.item.models import Item from infra.db.repos.mongo import _repo_factory as mongo_repo_factory -from mongo_ops import BaseDocument, BaseRepository class ItemDocument(Item, BaseDocument): diff --git a/standards/src/templates/app/domain/job/models.py b/standards/src/templates/app/domain/job/models.py index 17ec51e..968d8c2 100644 --- a/standards/src/templates/app/domain/job/models.py +++ b/standards/src/templates/app/domain/job/models.py @@ -3,6 +3,7 @@ Job domain model — a pipeline work item consumed by the background worker. """ from typing import Literal + from pydantic import BaseModel, Field diff --git a/standards/src/templates/app/domain/job/ports/db.py b/standards/src/templates/app/domain/job/ports/db.py index 547e2b5..c085eb3 100644 --- a/standards/src/templates/app/domain/job/ports/db.py +++ b/standards/src/templates/app/domain/job/ports/db.py @@ -2,9 +2,10 @@ Mongo persistence for jobs: document schema and repository factory. """ +from mongo_ops import BaseDocument, BaseRepository + from domain.job.models import Job from infra.db.repos.mongo import _repo_factory as mongo_repo_factory -from mongo_ops import BaseDocument, BaseRepository class JobDocument(Job, BaseDocument): diff --git a/standards/src/templates/app/infra/adapters/kv.py b/standards/src/templates/app/infra/adapters/kv.py index 00e3ff9..04b75a3 100644 --- a/standards/src/templates/app/infra/adapters/kv.py +++ b/standards/src/templates/app/infra/adapters/kv.py @@ -21,18 +21,17 @@ Example: from __future__ import annotations from abc import ABC, abstractmethod -from typing import Dict, List, Optional class KVStore(ABC): """Abstract key-value store port.""" @abstractmethod - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Return the value at ``key``, or ``None``.""" @abstractmethod - async def set(self, key: str, value: str, ttl: Optional[int] = None) -> None: + async def set(self, key: str, value: str, ttl: int | None = None) -> None: """Store ``value`` at ``key`` with an optional TTL in seconds.""" @abstractmethod @@ -40,7 +39,7 @@ class KVStore(ABC): """Delete ``key`` if present.""" @abstractmethod - async def scan(self, pattern: str) -> List[str]: + async def scan(self, pattern: str) -> list[str]: """Return keys matching a glob ``pattern``.""" async def clear(self, pattern: str = "*") -> None: @@ -67,8 +66,8 @@ class InMemoryKV(KVStore): def __init__(self) -> None: """Initialize an empty store with no imports or connections.""" - self._data: Dict[str, str] = {} - self._expiry: Dict[str, float] = {} + self._data: dict[str, str] = {} + self._expiry: dict[str, float] = {} def _expired(self, key: str) -> bool: import time @@ -80,13 +79,13 @@ class InMemoryKV(KVStore): return True return False - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Return the value at ``key``, or ``None`` when absent/expired.""" if self._expired(key): return None return self._data.get(key) - async def set(self, key: str, value: str, ttl: Optional[int] = None) -> None: + async def set(self, key: str, value: str, ttl: int | None = None) -> None: """Store ``value`` at ``key`` with an optional TTL in seconds.""" import time @@ -98,7 +97,7 @@ class InMemoryKV(KVStore): self._data.pop(key, None) self._expiry.pop(key, None) - async def scan(self, pattern: str) -> List[str]: + async def scan(self, pattern: str) -> list[str]: """Return live keys matching a glob ``pattern``.""" import fnmatch @@ -126,11 +125,11 @@ class RedisKV(KVStore): self._redis = Redis.from_url(url, decode_responses=decode_responses) - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Return the value at ``key``, or ``None``.""" return await self._redis.get(key) - async def set(self, key: str, value: str, ttl: Optional[int] = None) -> None: + async def set(self, key: str, value: str, ttl: int | None = None) -> None: """Store ``value`` at ``key`` with an optional TTL in seconds.""" await self._redis.set(key, value, ex=ttl) @@ -138,7 +137,7 @@ class RedisKV(KVStore): """Delete ``key`` if present.""" await self._redis.delete(key) - async def scan(self, pattern: str) -> List[str]: + async def scan(self, pattern: str) -> list[str]: """Return keys matching a glob ``pattern``.""" return [key async for key in self._redis.scan_iter(match=pattern)] diff --git a/standards/src/templates/app/infra/adapters/service.py b/standards/src/templates/app/infra/adapters/service.py index dfa46d8..9454d83 100644 --- a/standards/src/templates/app/infra/adapters/service.py +++ b/standards/src/templates/app/infra/adapters/service.py @@ -5,14 +5,14 @@ Generic CRUD service adaptor over a Mongo repository. Bridge pydantic models and mongo-ops documents, exposing create, read, update, patch, delete, and search operations for every domain that needs them. """ -from typing import Generic, Type, TypeVar +from typing import Generic, TypeVar + +from mongo_ops import BaseDocument, PopulatingRepository +from mongo_ops.models import PyObjectId +from pydantic import BaseModel from infra.db.repos import mongo -from pydantic import BaseModel -from mongo_ops import BaseDocument -from mongo_ops.models import PyObjectId - Model = TypeVar("Model", bound=BaseModel) Schema = TypeVar("Schema", bound=BaseDocument) @@ -27,8 +27,8 @@ class CrudsServiceAdaptor(Generic[Model, Schema]): """ repo: mongo.BaseRepository - model: Type[Model] - schema: Type[Schema] + model: type[Model] + schema: type[Schema] def __init__(self, *, repo=None): """Initialize the adaptor with a backing repository. @@ -83,7 +83,17 @@ class CrudsServiceAdaptor(Generic[Model, Schema]): Returns: Schema | None: Updated document, or ``None`` when absent. + + Notes: + A populating repository expects a document *model* (so FK fields can + be depopulated), while a base repository expects a raw mapping. This + method normalizes both inputs to whatever the repo requires. """ + if isinstance(self.repo, PopulatingRepository): + if not isinstance(data, self.schema): + fields = data if isinstance(data, dict) else self.model_to_dict(data) + data = self.schema(**fields) + return await self.repo.update(id, data) return await self.repo.update( id, data if isinstance(data, dict) else self.model_to_dict(data), diff --git a/standards/src/templates/app/infra/adapters/sse.py b/standards/src/templates/app/infra/adapters/sse.py index 70c6f31..bd1a799 100644 --- a/standards/src/templates/app/infra/adapters/sse.py +++ b/standards/src/templates/app/infra/adapters/sse.py @@ -7,7 +7,7 @@ SSEInMemoryBus — in-memory event bus for SSE progress events. Buffer published progress events so they can be replayed (and streamed) per correlation key, such as a job, upload, or fetch-request id. """ -from typing import Generic, List, TypeVar +from typing import Generic, TypeVar T = TypeVar("T") @@ -27,7 +27,7 @@ class SSEInMemoryBus(Generic[T]): key_attr (str): Attribute name on each event holding the stream correlation key (for example ``"job_id"``). """ - self._events: List[T] = [] + self._events: list[T] = [] self._key_attr = key_attr async def publish(self, event: T) -> None: @@ -38,7 +38,7 @@ class SSEInMemoryBus(Generic[T]): """ self._events.append(event) - def get_events(self, key: str) -> List[T]: + def get_events(self, key: str) -> list[T]: """Return buffered events for a correlation key. Args: diff --git a/standards/src/templates/app/infra/db/repos/mongo.py b/standards/src/templates/app/infra/db/repos/mongo.py index a68f8f3..2fe8bb8 100644 --- a/standards/src/templates/app/infra/db/repos/mongo.py +++ b/standards/src/templates/app/infra/db/repos/mongo.py @@ -5,21 +5,20 @@ Generic MongoDB CRUD repo — single implementation for all domains. Select a concrete mongo-ops repository (base, cached, or populating) from the options supplied to the factory. """ -from typing import List, Optional, Type from mongo_ops import BaseRepository, CachedBaseRepository, PopulatingRepository from mongo_ops.cache import CacheBackend, CacheConfig from mongo_ops.models import BaseDocument -from mongo_ops.populate import PopulationEngine, PopulateRule +from mongo_ops.populate import PopulateRule, PopulationEngine def _repo_factory( collection: str, - db_model: Type[BaseDocument], - cache_backend: Optional[CacheBackend] = None, - cache_config: Optional[CacheConfig] = None, - engine: Optional[PopulationEngine] = None, - populate_rules: Optional[List[PopulateRule]] = None, + db_model: type[BaseDocument], + cache_backend: CacheBackend | None = None, + cache_config: CacheConfig | None = None, + engine: PopulationEngine | None = None, + populate_rules: list[PopulateRule] | None = None, ) -> BaseRepository: """Build the appropriate mongo-ops repository. diff --git a/standards/src/templates/app/infra/http_cache.py b/standards/src/templates/app/infra/http_cache.py index 8debf88..dfc36e2 100644 --- a/standards/src/templates/app/infra/http_cache.py +++ b/standards/src/templates/app/infra/http_cache.py @@ -23,8 +23,9 @@ Notes: """ import hashlib import json +from collections.abc import Callable from functools import wraps -from typing import Any, Callable, Optional +from typing import Any from fastapi import Request, Response from fastapi.encoders import jsonable_encoder @@ -55,7 +56,7 @@ class HttpCacheStore: self._redis = Redis.from_url(url, decode_responses=decode_responses) - async def get(self, key: str) -> Optional[str]: + async def get(self, key: str) -> str | None: """Read the value stored at ``key``. Args: @@ -173,7 +174,7 @@ def _cache_key(request: Request, generation: str = "0") -> str: return hashlib.sha256("|".join(parts).encode()).hexdigest() -def cache_response(ttl: Optional[int] = None) -> Callable: +def cache_response(ttl: int | None = None) -> Callable: """Decorate a GET-style route handler to cache its JSON response. Args: @@ -203,7 +204,7 @@ def cache_response(ttl: Optional[int] = None) -> Callable: return await func(*args, **kwargs) db_resources = getattr(request.app.state, "db_resources", None) or {} - store: Optional[HttpCacheStore] = db_resources.get("http_cache_store") + store: HttpCacheStore | None = db_resources.get("http_cache_store") if store is None: return await func(*args, **kwargs) diff --git a/standards/src/templates/app/init.py b/standards/src/templates/app/init.py index 68a6fac..46b1a7f 100644 --- a/standards/src/templates/app/init.py +++ b/standards/src/templates/app/init.py @@ -4,7 +4,7 @@ Infrastructure initialization — connects MongoDB and wires all repos and servi Shared bootstrap for both the FastAPI app and the worker, guarded against repeated initialization via a module-level flag. """ -from typing import Any, Dict +from typing import Any from config import settings from domain.item.ports.db import create_item_repo @@ -13,11 +13,11 @@ from domain.job.ports.db import create_job_repo from domain.job.ports.service import get_job_service from infra.db.setup import setup_database, shutdown_database -_resources: Dict[str, Any] = {} +_resources: dict[str, Any] = {} _initialized: bool = False -async def init() -> Dict[str, Any]: +async def init() -> dict[str, Any]: """Connect MongoDB, wire all repos and services, and return the resources dict. Idempotent: repeated calls short-circuit and return the cached resources. diff --git a/standards/src/templates/app/interfaces/openapi.py b/standards/src/templates/app/interfaces/openapi.py index 8c43aa7..489f65a 100644 --- a/standards/src/templates/app/interfaces/openapi.py +++ b/standards/src/templates/app/interfaces/openapi.py @@ -4,5 +4,6 @@ All handler functions — names must match openapi.yaml operationId values. The OpenAPI-first router resolves each operation to a handler by name, so this module re-exports every route handler from the subpackages into one flat namespace. """ +# ruff: noqa: F403 from interfaces.item.routes import * from interfaces.job.routes import * \ No newline at end of file diff --git a/standards/src/templates/app/main.py b/standards/src/templates/app/main.py index 0a451e3..a569f7b 100644 --- a/standards/src/templates/app/main.py +++ b/standards/src/templates/app/main.py @@ -4,6 +4,7 @@ FastAPI application bootstrap — uses OpenAPIFirstApp for spec-driven routing. Handlers are looked up by operationId in interfaces.openapi, so no router tables are declared here; the lifespan initializes and shuts down app resources. """ +# ruff: noqa: E402 import logging from contextlib import asynccontextmanager diff --git a/standards/src/templates/app/pipelines/shared/loader.py b/standards/src/templates/app/pipelines/shared/loader.py index 10a3c2b..d6e0d0d 100644 --- a/standards/src/templates/app/pipelines/shared/loader.py +++ b/standards/src/templates/app/pipelines/shared/loader.py @@ -9,7 +9,6 @@ from pathlib import Path from typing import Any import yaml - from hexa import build diff --git a/standards/src/templates/app/pipelines/shared/registry.py b/standards/src/templates/app/pipelines/shared/registry.py index d5de929..d449aba 100644 --- a/standards/src/templates/app/pipelines/shared/registry.py +++ b/standards/src/templates/app/pipelines/shared/registry.py @@ -4,13 +4,12 @@ Family registry — resolve a ``(kind, variant, mode)`` triple to a root class. A single lookup replaces hand-rolled ``if/else`` family selection. """ -from typing import Dict, Type # Populated by ``register`` as families import. -_FAMILIES: Dict[str, Dict[str, Dict[str, Type]]] = {} +_FAMILIES: dict[str, dict[str, dict[str, type]]] = {} -def register(kind: str, variant: str, root_cls: Type, mode: str = "default") -> Type: +def register(kind: str, variant: str, root_cls: type, mode: str = "default") -> type: """Register ``root_cls`` for a ``(kind, variant, mode)`` triple. Returns: @@ -20,7 +19,7 @@ def register(kind: str, variant: str, root_cls: Type, mode: str = "default") -> return root_cls -def resolve_root(kind: str, variant: str, mode: str = "default") -> Type: +def resolve_root(kind: str, variant: str, mode: str = "default") -> type: """Return the root class for a ``(kind, variant, mode)`` triple. Raises: diff --git a/standards/src/templates/app/pyproject.toml b/standards/src/templates/app/pyproject.toml index 408c0e1..8da9f77 100644 --- a/standards/src/templates/app/pyproject.toml +++ b/standards/src/templates/app/pyproject.toml @@ -40,6 +40,7 @@ testpaths = ["tests"] [tool.ruff] line-length = 100 target-version = "py311" +extend-exclude = ["_gen"] [tool.ruff.lint] select = ["E", "F", "I", "UP"] \ No newline at end of file diff --git a/standards/src/templates/app/tests/integration/test_item_routes.py b/standards/src/templates/app/tests/integration/test_item_routes.py index a7ecc0e..3525d88 100644 --- a/standards/src/templates/app/tests/integration/test_item_routes.py +++ b/standards/src/templates/app/tests/integration/test_item_routes.py @@ -1,9 +1,9 @@ import importlib -import interfaces.openapi as routes_module - from openapi_first.app import OpenAPIFirstApp +import interfaces.openapi as routes_module + def test_openapi_app_builds(): """The spec-first app must build and plant each route handler.""" diff --git a/standards/src/templates/app/tests/integration/test_openapi_app_builds.py b/standards/src/templates/app/tests/integration/test_openapi_app_builds.py index fe811c0..2d499c3 100644 --- a/standards/src/templates/app/tests/integration/test_openapi_app_builds.py +++ b/standards/src/templates/app/tests/integration/test_openapi_app_builds.py @@ -1,11 +1,10 @@ import importlib import yaml +from openapi_first.app import OpenAPIFirstApp import interfaces.openapi as routes_module -from openapi_first.app import OpenAPIFirstApp - _HTTP_METHODS = {"get", "post", "put", "patch", "delete", "head", "options"} diff --git a/standards/wiki/sitemap.xml.gz b/standards/wiki/sitemap.xml.gz index 36fff951d658b8fa1c2624e6fcea07a1ebb3772b..f0957ead3fa71accf94fc2db32d8580ce204bea1 100644 GIT binary patch delta 13 Ucmb=gXP58h;Akj0JCVHt03HJbkpKVy delta 13 Ucmb=gXP58h;9yWbGm*Ul02yinzW@LL diff --git a/trial.md b/trial.md new file mode 100644 index 0000000..9dfb505 --- /dev/null +++ b/trial.md @@ -0,0 +1,58 @@ +# Trial: build an Aetos app from the Standards blueprint (self-run, no auth) + +You are a zero-context builder. Test the Aetos *standards* by reproducing the +reference app "khata-app" WITHOUT auth into `apps/khata-trial`, using ONLY the +standard. You may consult the existing khata-app repo ONLY as the answer key to +the intake questions below. Never copy its implementation code. + +## Sources of truth (read these first) +- Standards MCP: http://localhost:8009/mcp + tools: find_library, list_app_components, get_app_component + resources: docs://nav, docs://modules/{module}, registry://libraries, registry://components + prompt: start_app +- Tier A library MCPs (read a library's own docs before using it): + openapi-first :8004, mongo-ops :8007, hexa :8008, dagpipe :8006, + omniread :8003, mail-intake :8002, py-jwt :8005, doc-forge :8001 +- Standard docs: docs://modules/build_recipe, docs://modules/genericity_matrix, + docs://modules/layering_rules +- If a server is not running, start all of them from the `aetos/docs` checkout: + `.venv\Scripts\python.exe main.py` + +## Step 0 — Intake (ask yourself the standard's questions) +1. App name + one-line purpose +2. Axis 1 — resources: each resource, fields, relationships (or paste openapi.yaml) +3. Axis 2 — pipelines: multi-step inputs? families + formats (or none) +4. Auth needed now? (yes/no) +5. Integrations: Mongo / ingestion formats / email / LLM provider + +## Step 1 — Answer the intake from khata-app ONLY (answers only) +Resolve each answer from these artifacts, and nothing else: +- resources + fields + relationships + operationIds <- openapi.yaml (schemas + operationId names) +- pipeline families + input formats <- directory layout under pipelines/ (+ each pipeline.yaml name) +- integrations <- pyproject.toml deps + .env.example variable names +- auth <- the GLOBAL `security` block in openapi.yaml (this trial: NONE) +Do NOT read or copy khata-app implementation modules. Write the answers down first. + +## Step 2 — Build (follow the standard) +Follow docs://modules/build_recipe step by step: +- Scaffold from the standard template. Prefer pulling files via get_app_component; + fallback: copy /standards/src/templates/app into apps/khata-trial. +- Author openapi.yaml for the resolved resources, OMITTING auth (no global + `security`, no bearer scheme). +- Author every CRUD slice; wire the composition root; add thin routes; bootstrap + app + worker; add pipeline families; add tests and operator scripts. +- Tier A = call the library (never reimplement); Tier B = copy the template; + Tier C = author. + +## Step 3 — Verify and report +- python -c "import main" succeeds +- pytest passes +- every no-auth khata-app operationId is served -> report matched / missing +- no domain/ module imports fastapi/starlette; no Tier A library reimplemented + +## Step 4 — Report standard gaps (primary output) +List every point where the standard was insufficient, wrong, or missing: a file +the docs said to copy but that didn't exist, an MCP tool that couldn't retrieve +a needed file, an intake question that wasn't answerable, etc. + +Output: apps/khata-trial \ No newline at end of file diff --git a/trial_findings.md b/trial_findings.md new file mode 100644 index 0000000..06e215c --- /dev/null +++ b/trial_findings.md @@ -0,0 +1,130 @@ +# Trial findings — build a khata app (no auth) from the Standards blueprint + +Scope: reproduce `apps/khata-app` WITHOUT auth into `apps/khata-trial`, using ONLY +the standards (MCP bundle + library MCPs + `docs://`). khata-app consulted as the +answer key for intake only; implementation never copied. + +Date: 2026-10-13. Status: trial build GREEN (imports, tests, live HTTP, pipeline). + +## Verification evidence (what the standard successfully produced) + +- `python -c "import main"` — OK; app builds with 46 routes (42 operationIds + 4 + FastAPI defaults). Every no-auth operationId resolved to a handler. +- `pytest tests -q` — 6 passed, incl. the generic + `tests/integration/test_openapi_app_builds.py` that asserts each spec + operationId resolves against `interfaces.openapi`. +- Live end-to-end (real MongoDB, `TestClient`): 19 endpoints across all 8 + resources returned 2xx — users(create/list/me/get/put), tags, accounts, + entities(+logo), expenses(create/list), fetch-requests(create/list/get/ + ambiguities), reports(create/list). `FAILURES: none`. +- Deletes: `DELETE` returns 204 for entities/tags/accounts/expenses/ + fetch-requests/reports; the follow-up `GET` correctly returns 404. +- Pipeline axis: `run_fetch_request` advanced status pending→completed and + published 7 progress events on the bus (`raw_lines`, `txn_blocks`, + `raw_expenses`, `enrich_entity`, `enrich_tags`, `save_expenses`). +- Layering respected: no `domain/` module imports fastapi/starlette; no Tier A + library reimplemented. + +## Findings + +Severity: **H**igh (blocks a 1-shot build), **M**edium (agent must work around), +**L**ow (rough edge). + +| # | Area | Sev | Finding | Proposed standard fix | +|---|------|-----|---------|-----------------------| +| F1 | build_recipe | H | Recipe never mentioned the spec→code step; agent had to discover `python -m openapi_first.cli {models,routes}` unaided. | FIXED (P1): codegen codified in recipe + prompt + wiki. | +| F2 | codegen | H | `openapi-first` CLI output is not consumable as-is: models land in one file with globally renumbered enums (`Type`, `Type1`, `Type2`…), routes land in per-tag packages with `payload:`/sync signatures. | FIXED (P3): `--inject-request`, `--package`, `--use-models`, `--models-module`; documented shape. | +| F3 | wire vs doc | M | Generated wire models require top-level `id` + `created_at`/`updated_at`, but Mongo docs own those via `BaseDocument`; `class XDoc(X, BaseDocument)` keeps `id` required and fights the repo. | FIXED (P4): API-model→document convention documented; keep domain models id-free. | +| F4 | spec→slices | M | No documented bridge from spec tags/operationIds to domain slices; agent inferred the 9-slice split. | Build recipe now shows tag→slice mapping (P1). | +| F5 | authless | M | Authless path (strip global security + per-op `security: []` + `securitySchemes`) and the `/users/me` behaviour were manual/undefined. | FIXED (P5): authless guidance added. | +| F6 | components | H | `get_app_component` omitted `__init__.py` files and `interfaces/openapi.py`; no tool returned the whole tree. | FIXED (P6a): `get_app_template` returns the full tree; package-inits registered. | +| F7 | docs:// | H | Library MCP resource URIs regressed to `doc://`. | FIXED (P0): restored `docs://`. | +| F8 | Tier B | M | Tier-B files referenced by the docs (`http_cache`, progress bus, KV store) were not shipped. | FIXED (P6b): shipped `infra/http_cache.py`, `infra/adapters/{sse,kv}.py`. | +| F9 | prompts | M | Only `start_app` existed; no build/intake/trial prompts. | FIXED (P7): `build`, `intake`, `trial` added. | +| **F10** | codegen shape | L | Generated route stubs use `payload: `, sync `def`, body `raise NotImplementedError`; the template convention is `body: dict` + async. Both bind (binder introspects FastAPI signatures) but the mismatch is undocumented. | Document both accepted handler shapes, or emit template-style stubs. | +| **F11** | docs defect | H | `05_crud_template.md` and `09_infra_adaptors.md` show `PopulateRule(field="owner", collection="users")`, but the real dataclass fields are **`field_name`** / **`collection_name`** (`mongo_ops/populate/rules.py:37`). Copying the doc raises `TypeError`. | Correct the two doc examples to `field_name=`/`collection_name=`. **FIXED.** | +| **F12** | lib interop | M | `CrudsServiceAdaptor.update` (`infra/adapters/service.py`) always forwards a **dict**, but `PopulatingRepository.update(id, data: T)` calls `_depopulate(data)` → `document.model_dump(...)`, requiring a **model** (`mongo_ops/repository.py:404,384`). Populate + update through the adaptor raises `AttributeError`. | Adaptor now detects `PopulatingRepository` and builds the document model before calling `update`. **FIXED.** | +| **F13** | modeling | M | Wire contract declares refs expanded (`expense.entity: Entity`, `account.owner: User`), while id-free domain models store FKs. Trial chose embedded dicts / string FKs. No single documented convention. | Added a "Nested references" section to `05_crud_template.md` (embed / populate / id). **FIXED.** | +| **F14** | authless | L | `/users/me` (`get_current_user`, `update_current_user`, `x-profile: true`) has no defined authless semantics; agent invented "auto-create Demo User". | Added authless profile-endpoint guidance to `04_spec_first_routing.md`. **FIXED.** | +| **F15** | template lint | M | The template's own `pyproject.toml` sets `ruff select = ["E","F","I","UP"]` but template-shipped files violate it: `main.py` E402, `interfaces/openapi.py` F403, `infra/adapters/{kv,sse,service}.py` + `infra/http_cache.py` UP/I, `pipelines/shared/{registry,loader}.py` UP/I. | Fixed typing/imports, added file-level `noqa`s, and `extend-exclude = ["_gen"]`; template now `ruff check` clean. **FIXED.** | +| **F16** | template ship | M | The template vendors a `.ruff_cache/` directory into every generated app (`build.py` `copytree` ignore list only skipped `__pycache__`/`*.pyc`), and ships no `.gitignore`. | Hardened the `copytree` ignore list (`.ruff_cache`, `.pytest_cache`, `.mypy_cache`, `.venv`, `venv`, `.git`, `*.egg-info`, `htmlcov`), added a first-class `gitignore` component + template `.gitignore`. Component count 42→**43**. **FIXED.** | + +## F10 resolution (emitter) + re-trial evidence + +`openapi-first` now emits template-convention stubs when asked: + +- `generate_routes(..., template_style=True)` / CLI `--template-style` emits + `async def`, a JSON body parameter named `body` (both `body: dict` and + `body: `), and (with `--inject-request`) a service-lookup hint comment + `# Resolve the service: request.app.state.db_resources["_svc"]` + before `raise NotImplementedError`. Default behaviour is unchanged + (`def` / `payload`). +- Tests: `tests/test_codegen_routes.py` gained + `test_template_style_async_body_model` and + `test_template_style_dict_body_without_models`; suite now **32 passed** (was + 30); `ruff check` clean. Wiki `03_use_cases/04_codegen.md` documents the flag. + MCP bundle regenerated (`docs/mcp/modules/openapi_first.json` contains + `template_style`). Standards `spec_first_routing`/`build_recipe` modules + updated to reference `--template-style`. + +**Re-trial A (template-level):** scaffolded the template into `apps/_tpl-check`, +`pytest` → 7 passed; `import main` → 11 routes; live `TestClient` Mongo smoke +→ `FAILURES: none`; targeted F12 populate+update check → `FAILURES: none`. +Scratch dir + DB removed. + +**Re-trial B (full khata rebuild):** scaffolded `apps/khata-trial`, stripped auth +(2286-line spec), ran codegen with `--template-style` → 9 route files + `_gen` +models, authored all 9 domain slices, 42 handlers, composition root, worker, and +fetch pipeline. Then: + +- `import main` → 46 routes (42 operationIds + 4 FastAPI defaults); every + operationId resolved. +- `pytest` → 2 passed. +- Live end-to-end (real MongoDB, `httpx.ASGITransport`): **43 checks, 0 + failures** — all 9 resources CRUD, entity-logo multipart, `/users/me` authless + auto-create, deletes 204 + follow-up 404, pipeline pending→completed, SSE + events replay, raw octet-stream upload + download, report create/get/delete. +- One emitter-adjacent trap re-confirmed: a handler parameter named `response` + must be annotated (`response: Response`) or FastAPI binds it as a query param; + returning `Response(status_code=204)` from deletes is the safe template idiom. +- `apps/khata-trial` + `khata_trial` DB removed after verification. + +## Recommended standard changes (from F10–F16) + +1. Fix the `PopulateRule` examples (F11) — highest value, one-line corrections. +2. Resolve the populate/update interop gap (F12) — either fix the adaptor or + document the incompatibility. +3. Add a "nested refs" modeling subsection (F13) and authless `/users/me` + behavior (F14). +4. Make the shipped template pass its own ruff config and exclude generated code + (F15). +5. Document the two accepted codegen handler shapes (F10). +6. Stop vendoring `.ruff_cache` into apps and ship a `.gitignore` (F16). + +## Resolution + +F11–F16 applied to the standard sources and the MCP bundle rebuilt +(`build.py --name standards`: 12 modules, **43 components**, lib + wiki HTML): + +- F11 — `field=`/`collection=` → `field_name=`/`collection_name=` in + `05_crud_template.md`, `09_infra_adaptors.md`. +- F12 — `infra/adapters/service.py` `update` handles `PopulatingRepository`. +- F13 — new "Nested references" section in `05_crud_template.md`. +- F14 — authless profile-endpoint guidance in `04_spec_first_routing.md`. +- F15 — template `ruff check` clean (`extend-exclude = ["_gen"]`, file-level + `noqa`s, typing/import fixes); template recompiles. +- F16 — `build.py` ignore list hardened; `gitignore` component + template + `.gitignore` added. + +The running `docs` image bakes the bundle, so it was rebuilt +(`aetos-docs:standards5`) and the `docs` container swapped onto it; MCP +`:8009` live-verified (tools/prompts/resources unchanged, 43 components, +`gitignore` present). + +## Outcome + +Trials (original + re-trial A + re-trial B) proved the standard buildable +end-to-end; F1–F16 recorded and F11–F16 fixed. F10 resolved in the +`openapi-first` emitter. `apps/khata-trial` removed after verification. Remaining +items in "Recommended standard changes" are cosmetic/documentation-only and will +not meaningfully change build outcome quality.