Vishesh 'ironeagle' Bangotra
6c0d859a1e
- F11 PopulateRule field_name/collection_name corrections - F12 service adaptor update handles PopulatingRepository - F13 nested-references section; F14 authless /users/me guidance - F15 template ruff-clean; F16 stop vendoring .ruff_cache, ship .gitignore - document --template-style in build_recipe/spec_first_routing; refresh collected openapi-first + standards bundle (12 modules, 43 components) - add trial.md and trial_findings.md (F1-F16, re-trial A/B evidence)
360 lines
16 KiB
Python
360 lines
16 KiB
Python
#!/usr/bin/env python
|
|
"""Build hand-authored documentation folders from their markdown sources.
|
|
|
|
Companion to collect.py. While collect.py pulls *finished* HTML and MCP bundles
|
|
out of source repos, build.py compiles folders that keep their sources in this
|
|
hub. Any top-level folder with ``src/mkdocs.*.yml`` gets, for each config:
|
|
|
|
1. a MkDocs HTML build into the target dir the config declares
|
|
(e.g. ``standards/src/mkdocs.wiki.yml`` -> ``standards/wiki/``), and
|
|
2. a doc-forge-shaped MCP bundle under ``<name>/mcp/``
|
|
(``index.json``, ``nav.json``, ``modules/*.json``), regenerated from the
|
|
lib pages (``src/lib/*.md``) so the bundle mirrors the lib site.
|
|
|
|
Each MkDocs config owns a dedicated site dir (declared via ``site_dir``), so
|
|
builds never touch ``src/`` or any other kind's output.
|
|
|
|
Usage:
|
|
python build.py # build every folder with src/mkdocs.*.yml
|
|
python build.py --name standards
|
|
python build.py --dry-run
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
|
|
NUM_PREFIX = re.compile(r"^\d+[_-]?")
|
|
|
|
|
|
def find_src_dirs(root: Path = ROOT) -> list[Path]:
|
|
"""Return top-level folders containing ``src/mkdocs.*.yml``.
|
|
|
|
Skips hidden directories, the mcp bundle container, and the generated
|
|
portal index.
|
|
"""
|
|
found: list[Path] = []
|
|
for child in sorted(root.iterdir()):
|
|
if not child.is_dir() or child.name.startswith("."):
|
|
continue
|
|
if child.name in ("mcp", "_index"):
|
|
continue
|
|
src = child / "src"
|
|
if src.is_dir() and list(src.glob("mkdocs.*.yml")):
|
|
found.append(child)
|
|
return found
|
|
|
|
|
|
def build_html(folder: Path, dry_run: bool = False) -> None:
|
|
"""Compile each ``src/mkdocs.*.yml`` config of ``folder`` into HTML.
|
|
|
|
Each config declares its own ``docs_dir`` and ``site_dir``, mirroring
|
|
doc-forge (``mkdocs.wiki.yml`` -> ``<name>/wiki/``, ``mkdocs.lib.yml`` ->
|
|
``<name>/lib/``).
|
|
|
|
Args:
|
|
folder (Path): The hub folder owning ``src/mkdocs.*.yml``.
|
|
dry_run (bool): Print the plan without executing.
|
|
"""
|
|
name = folder.name
|
|
for cfg in sorted((folder / "src").glob("mkdocs.*.yml")):
|
|
site = cfg.name.removeprefix("mkdocs.").removesuffix(".yml")
|
|
if dry_run:
|
|
print(f"[BLD] {name} mkdocs build -f {cfg.relative_to(ROOT)} "
|
|
f"(-> {name}/{site}/)")
|
|
continue
|
|
|
|
result = subprocess.run(
|
|
[sys.executable, "-m", "mkdocs", "build", "-f", str(cfg)],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode != 0:
|
|
print(f"[ERR] {name} mkdocs failed:\n{result.stdout}\n{result.stderr}",
|
|
file=sys.stderr)
|
|
raise SystemExit(1)
|
|
print(f"[BLD] {name} {site} site built into {folder.relative_to(ROOT)}/{site}/")
|
|
|
|
|
|
# ── MCP bundle ───────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _module_name(stem: str) -> str:
|
|
"""Map a markdown filename to a module name (strip the NN_ prefix)."""
|
|
return NUM_PREFIX.sub("", stem) or stem
|
|
|
|
|
|
def _read_title(path: Path) -> str:
|
|
"""Return the first markdown H1 title, or a prettified fallback."""
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
if line.startswith("# "):
|
|
return line[2:].strip()
|
|
return _module_name(path.stem).replace("_", " ").title()
|
|
|
|
|
|
def build_bundle(folder: Path, dry_run: bool = False) -> None:
|
|
"""Regenerate ``<name>/mcp`` from ``folder/src/lib`` markdown.
|
|
|
|
The bundle mirrors the lib site (in the same way doc-forge's MCP output
|
|
mirrors its lib documentation). Writes ``index.json``, ``nav.json`` and
|
|
``modules/*.json`` in the doc-forge MCP bundle shape. Hand-authored extras
|
|
(e.g. ``libraries.json``) are left untouched.
|
|
|
|
Args:
|
|
folder (Path): The hub folder owning a ``src/lib``.
|
|
dry_run (bool): Print the plan without executing.
|
|
"""
|
|
name = folder.name
|
|
lib = folder / "src" / "lib"
|
|
if not lib.is_dir():
|
|
return
|
|
out_dir = folder / "mcp"
|
|
modules_dir = out_dir / "modules"
|
|
if dry_run:
|
|
print(f"[MCP] {name} {lib.relative_to(ROOT)}/*.md "
|
|
f"-> {out_dir.relative_to(ROOT)}/")
|
|
return
|
|
|
|
if modules_dir.exists():
|
|
shutil.rmtree(modules_dir)
|
|
modules_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def write_module(module: str, body: str) -> None:
|
|
payload = {
|
|
"module": module,
|
|
"content": {
|
|
"path": module,
|
|
"docstring": body.strip() + "\n",
|
|
"objects": {},
|
|
},
|
|
}
|
|
(modules_dir / f"{module}.json").write_text(
|
|
json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
|
|
nav = []
|
|
for path in sorted(lib.glob("*.md")):
|
|
module = _module_name(path.stem)
|
|
write_module(module, path.read_text(encoding="utf-8"))
|
|
nav.append({
|
|
"module": module,
|
|
"resource": f"docs://modules/{module}",
|
|
"title": _read_title(path),
|
|
})
|
|
|
|
(out_dir / "nav.json").write_text(
|
|
json.dumps(nav, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
(out_dir / "index.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"project": name,
|
|
"type": name,
|
|
"modules_count": len(nav),
|
|
"source": "hand-authored",
|
|
},
|
|
indent=2,
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
print(f"[MCP] {name} bundle written to {out_dir.relative_to(ROOT)}/ ({len(nav)} modules)")
|
|
|
|
|
|
# ── Template bundle ──────────────────────────────────────────────────────────
|
|
|
|
# Component kinds shipped in the vendored template. ``placeholder`` names the
|
|
# seed token an agent renames: "resource" -> item/Item, "family" -> example/
|
|
# Example; "" means ship as-is.
|
|
_TEMPLATE_COMPONENTS: list[dict[str, str]] = [
|
|
# Per-resource slice (seed: item/Item).
|
|
{"kind": "model", "path": "domain/item/models.py", "placeholder": "resource",
|
|
"description": "Business Pydantic model for a resource."},
|
|
{"kind": "ports-db", "path": "domain/item/ports/db.py", "placeholder": "resource",
|
|
"description": "Document model + repository factory for a resource."},
|
|
{"kind": "ports-service", "path": "domain/item/ports/service.py", "placeholder": "resource",
|
|
"description": "CRUD service wired through the DI library."},
|
|
{"kind": "converters", "path": "domain/item/ports/converters.py", "placeholder": "resource",
|
|
"description": "Optional pure converters between domain types."},
|
|
{"kind": "routes", "path": "interfaces/item/routes.py", "placeholder": "resource",
|
|
"description": "Thin HTTP handlers bound to operationIds."},
|
|
{"kind": "openapi-routes", "path": "interfaces/openapi.py", "placeholder": "",
|
|
"description": "Route registry: flat re-export of every handler namespace."},
|
|
{"kind": "test-model", "path": "tests/unit/test_item_models.py", "placeholder": "resource",
|
|
"description": "Unit test for the resource model."},
|
|
{"kind": "test-routes", "path": "tests/integration/test_item_routes.py", "placeholder": "resource",
|
|
"description": "Integration test at the HTTP edge."},
|
|
{"kind": "test-openapi-build", "path": "tests/integration/test_openapi_app_builds.py", "placeholder": "",
|
|
"description": "Generic test: every spec operationId resolves and the app builds."},
|
|
{"kind": "test-pipeline", "path": "tests/unit/test_item_sync.py", "placeholder": "",
|
|
"description": "Unit test for the smallest pipeline form."},
|
|
# Per-family pipeline scaffold (seed: example/Example).
|
|
{"kind": "pipeline-abc", "path": "pipelines/example/abc.py", "placeholder": "family",
|
|
"description": "Port contracts for a pipeline family."},
|
|
{"kind": "pipeline-impl", "path": "pipelines/example/impl.py", "placeholder": "family",
|
|
"description": "Concrete stages, wired root, and family registration."},
|
|
{"kind": "pipeline-yaml", "path": "pipelines/example/pipeline.yaml", "placeholder": "family",
|
|
"description": "Graph spec (nodes, edges, roots, schema)."},
|
|
{"kind": "pipeline-resources", "path": "pipelines/example/resources/pipeline.yaml", "placeholder": "family",
|
|
"description": "Runtime spec consumed by the shared loader."},
|
|
{"kind": "pipeline-pattern", "path": "pipelines/example/resources/patterns/example.yaml", "placeholder": "family",
|
|
"description": "Pattern/rule fragment merged by the loader."},
|
|
# Shared pipeline scaffolding (ship as-is).
|
|
{"kind": "pipeline-ports", "path": "pipelines/shared/ports.py", "placeholder": "",
|
|
"description": "Shared pipeline ports (FinalizePort, RegistryPort)."},
|
|
{"kind": "pipeline-loader", "path": "pipelines/shared/loader.py", "placeholder": "",
|
|
"description": "Shared loader: DI wiring + pattern merge."},
|
|
{"kind": "pipeline-registry", "path": "pipelines/shared/registry.py", "placeholder": "",
|
|
"description": "Shared registry: (kind, variant, mode) -> root class."},
|
|
{"kind": "pipeline-simple", "path": "pipelines/item_sync.py", "placeholder": "",
|
|
"description": "Smallest pipeline form (plain async steps)."},
|
|
# Second resource slice (seed: job).
|
|
{"kind": "job-model", "path": "domain/job/models.py", "placeholder": "",
|
|
"description": "Seeded second resource model (job)."},
|
|
{"kind": "job-ports-db", "path": "domain/job/ports/db.py", "placeholder": "",
|
|
"description": "Seeded second resource repository (job)."},
|
|
{"kind": "job-ports-service", "path": "domain/job/ports/service.py", "placeholder": "",
|
|
"description": "Seeded second resource service (job)."},
|
|
{"kind": "job-routes", "path": "interfaces/job/routes.py", "placeholder": "",
|
|
"description": "Seeded second resource routes (job)."},
|
|
# Root / shared files (ship as-is).
|
|
{"kind": "config", "path": "config.py", "placeholder": "",
|
|
"description": "Settings dataclass + singleton."},
|
|
{"kind": "init", "path": "init.py", "placeholder": "",
|
|
"description": "Composition root (init/shutdown, get_svc)."},
|
|
{"kind": "main", "path": "main.py", "placeholder": "",
|
|
"description": "Spec-first app bootstrap + lifespan."},
|
|
{"kind": "worker", "path": "worker.py", "placeholder": "",
|
|
"description": "Poll loop + job status machine."},
|
|
{"kind": "openapi", "path": "openapi.yaml", "placeholder": "",
|
|
"description": "OpenAPI contract skeleton."},
|
|
{"kind": "pyproject", "path": "pyproject.toml", "placeholder": "",
|
|
"description": "Packaging + dependency pins."},
|
|
{"kind": "dockerfile", "path": "Dockerfile", "placeholder": "",
|
|
"description": "API + worker image with private-index wiring."},
|
|
{"kind": "dockerignore", "path": ".dockerignore", "placeholder": "",
|
|
"description": "Docker build-context exclusions."},
|
|
{"kind": "gitignore", "path": ".gitignore", "placeholder": "",
|
|
"description": "VCS exclusions (caches, venv, generated _gen/)."},
|
|
{"kind": "compose", "path": "docker-compose.yaml", "placeholder": "",
|
|
"description": "api + worker + mongo compose file."},
|
|
{"kind": "env-example", "path": ".env.example", "placeholder": "",
|
|
"description": "Environment variable template."},
|
|
{"kind": "db-setup", "path": "infra/db/setup.py", "placeholder": "",
|
|
"description": "Mongo connection lifecycle."},
|
|
{"kind": "repo", "path": "infra/db/repos/mongo.py", "placeholder": "",
|
|
"description": "Repository factory (base/populating/cached)."},
|
|
{"kind": "service-adapter", "path": "infra/adapters/service.py", "placeholder": "",
|
|
"description": "Generic CRUD service adaptor."},
|
|
{"kind": "http-cache", "path": "infra/http_cache.py", "placeholder": "",
|
|
"description": "Response cache decorator + Redis store (generation invalidation)."},
|
|
{"kind": "progress-bus", "path": "infra/adapters/sse.py", "placeholder": "",
|
|
"description": "In-memory SSE progress bus (keyed by an event attribute)."},
|
|
{"kind": "kv-store", "path": "infra/adapters/kv.py", "placeholder": "",
|
|
"description": "KV port + in-memory fallback + optional Redis adaptor."},
|
|
{"kind": "package-init", "path": "domain/__init__.py", "placeholder": "",
|
|
"description": "Empty package marker; add an __init__.py to every package dir."},
|
|
{"kind": "script-seed", "path": "scripts/seed.py", "placeholder": "",
|
|
"description": "Operator seed script."},
|
|
{"kind": "script-drop", "path": "scripts/drop.py", "placeholder": "",
|
|
"description": "Operator drop script."},
|
|
]
|
|
|
|
|
|
def build_components_manifest(template_root: Path) -> dict[str, object]:
|
|
"""Describe every component in the vendored template.
|
|
|
|
Args:
|
|
template_root (Path): The vendored ``templates/app`` directory.
|
|
|
|
Returns:
|
|
dict[str, object]: Manifest with the seed tokens and component entries.
|
|
"""
|
|
components: list[dict[str, str]] = []
|
|
for entry in _TEMPLATE_COMPONENTS:
|
|
rel = entry["path"]
|
|
if not (template_root / rel).exists():
|
|
raise SystemExit(f"[ERR] component path missing from template: {rel}")
|
|
components.append({
|
|
"kind": entry["kind"],
|
|
"path": rel,
|
|
"placeholder": entry["placeholder"],
|
|
"description": entry["description"],
|
|
})
|
|
return {
|
|
"project": "standards",
|
|
"template": "templates/app",
|
|
"seed": {"resource": "item", "family": "example"},
|
|
"components": components,
|
|
}
|
|
|
|
|
|
def build_templates(folder: Path, dry_run: bool = False) -> None:
|
|
"""Vendor ``folder/src/templates/app`` into ``<name>/mcp/templates/app``.
|
|
|
|
Ships the runnable skeleton in the MCP bundle so an agent can pull exact,
|
|
current component code, and emits ``components.json`` describing the kinds.
|
|
Excludes ``__pycache__`` / ``*.pyc``.
|
|
|
|
Args:
|
|
folder (Path): The hub folder owning ``src/templates/app``.
|
|
dry_run (bool): Print the plan without executing.
|
|
"""
|
|
name = folder.name
|
|
src = folder / "src" / "templates" / "app"
|
|
if not src.is_dir():
|
|
return
|
|
out_dir = folder / "mcp" / "templates" / "app"
|
|
if dry_run:
|
|
print(f"[TPL] {name} {src.relative_to(ROOT)}/** -> {out_dir.relative_to(ROOT)}/**")
|
|
return
|
|
|
|
if out_dir.exists():
|
|
shutil.rmtree(out_dir)
|
|
out_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copytree(
|
|
src, out_dir,
|
|
ignore=shutil.ignore_patterns(
|
|
"__pycache__", "*.pyc", ".ruff_cache", ".pytest_cache",
|
|
".mypy_cache", ".venv", "venv", ".git", "*.egg-info", "htmlcov",
|
|
),
|
|
)
|
|
manifest = build_components_manifest(out_dir)
|
|
(folder / "mcp" / "components.json").write_text(
|
|
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
print(f"[TPL] {name} template vendored to {out_dir.relative_to(ROOT)}/ "
|
|
f"({len(manifest['components'])} components)")
|
|
|
|
|
|
# ── CLI ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Build hub-authored doc folders (src/ -> HTML + MCP bundle)"
|
|
)
|
|
parser.add_argument("--name", help="Only build a single folder")
|
|
parser.add_argument("--dry-run", action="store_true", help="Show the plan without writing")
|
|
args = parser.parse_args()
|
|
|
|
folders = [f for f in find_src_dirs() if not args.name or f.name == args.name]
|
|
if not folders:
|
|
print("No src/ doc folders found. Run build.py from the hub root.")
|
|
return
|
|
|
|
for folder in folders:
|
|
build_bundle(folder, dry_run=args.dry_run)
|
|
build_templates(folder, dry_run=args.dry_run)
|
|
build_html(folder, dry_run=args.dry_run)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |