Files
docs/build.py
Vishesh 'ironeagle' Bangotra 104bf3eb1e feat(standards): drop the site CI variant + require changelog reconciliation before release
- Remove the push-triggered build-and-deploy-site pipeline and every "site"
  reference; Type B is now a single tag-triggered service pipeline
  (service-docker.yaml).
- Inventory now covers packages + the auth-server service only (repos.json
  11 -> 9; docs/mcp are no longer part of the standard).
- Versioning standard: reconcile [Unreleased] against the last tag before
  promoting, and always show proposed entries to the user for approval.
- Update registry.py compliance/release text and list_repos types, build.py
  component descriptions, and rebuild the MCP bundle + lib/wiki HTML.
2026-10-17 00:41:52 +05:30

576 lines
25 KiB
Python

#!/usr/bin/env python
"""Build hand-authored documentation folders from their markdown sources.
Companion to collect.py. While collect.py pulls *finished* HTML and MCP bundles
out of source repos, build.py compiles folders that keep their sources in this
hub. Any top-level folder with ``src/mkdocs.*.yml`` gets, for each config:
1. a MkDocs HTML build into the target dir the config declares
(e.g. ``standards/src/mkdocs.wiki.yml`` -> ``standards/wiki/``), and
2. a doc-forge-shaped MCP bundle under ``<name>/mcp/``
(``index.json``, ``nav.json``, ``modules/*.json``), regenerated from the
lib pages (``src/lib/*.md``) so the bundle mirrors the lib site.
Each MkDocs config owns a dedicated site dir (declared via ``site_dir``), so
builds never touch ``src/`` or any other kind's output.
Usage:
python build.py # build every folder with src/mkdocs.*.yml
python build.py --name standards
python build.py --dry-run
"""
from __future__ import annotations
import argparse
import json
import re
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent
NUM_PREFIX = re.compile(r"^\d+[_-]?")
def find_src_dirs(root: Path = ROOT) -> list[Path]:
"""Return top-level folders containing ``src/mkdocs.*.yml``.
Skips hidden directories, the mcp bundle container, and the generated
portal index.
"""
found: list[Path] = []
for child in sorted(root.iterdir()):
if not child.is_dir() or child.name.startswith("."):
continue
if child.name in ("mcp", "_index"):
continue
src = child / "src"
if src.is_dir() and list(src.glob("mkdocs.*.yml")):
found.append(child)
return found
def build_html(folder: Path, dry_run: bool = False) -> None:
"""Compile each ``src/mkdocs.*.yml`` config of ``folder`` into HTML.
Each config declares its own ``docs_dir`` and ``site_dir``, mirroring
doc-forge (``mkdocs.wiki.yml`` -> ``<name>/wiki/``, ``mkdocs.lib.yml`` ->
``<name>/lib/``).
Args:
folder (Path): The hub folder owning ``src/mkdocs.*.yml``.
dry_run (bool): Print the plan without executing.
"""
name = folder.name
for cfg in sorted((folder / "src").glob("mkdocs.*.yml")):
site = cfg.name.removeprefix("mkdocs.").removesuffix(".yml")
if dry_run:
print(f"[BLD] {name} mkdocs build -f {cfg.relative_to(ROOT)} "
f"(-> {name}/{site}/)")
continue
result = subprocess.run(
[sys.executable, "-m", "mkdocs", "build", "-f", str(cfg)],
capture_output=True,
text=True,
)
if result.returncode != 0:
print(f"[ERR] {name} mkdocs failed:\n{result.stdout}\n{result.stderr}",
file=sys.stderr)
raise SystemExit(1)
print(f"[BLD] {name} {site} site built into {folder.relative_to(ROOT)}/{site}/")
# ── MCP bundle ───────────────────────────────────────────────────────────────
def _module_name(stem: str) -> str:
"""Map a markdown filename to a module name (strip the NN_ prefix)."""
return NUM_PREFIX.sub("", stem) or stem
def _read_title(path: Path) -> str:
"""Return the first markdown H1 title, or a prettified fallback."""
for line in path.read_text(encoding="utf-8").splitlines():
if line.startswith("# "):
return line[2:].strip()
return _module_name(path.stem).replace("_", " ").title()
def _wiki_module_name(rel: Path) -> str:
"""Map a wiki page path to a namespaced module name.
``04_development/02_releasing.md`` -> ``wiki_development_releasing``; the
``wiki_`` prefix keeps wiki pages from colliding with lib modules (both have
an ``index``).
"""
parts = [NUM_PREFIX.sub("", part) or part for part in rel.with_suffix("").parts]
return "wiki_" + "_".join(parts)
def build_bundle(folder: Path, dry_run: bool = False) -> None:
"""Regenerate ``<name>/mcp`` from ``folder/src/lib`` and ``folder/src/wiki``.
The bundle mirrors the lib site (in the same way doc-forge's MCP output
mirrors its lib documentation). Writes ``index.json``, ``nav.json`` and
``modules/*.json`` in the doc-forge MCP bundle shape. Hand-authored extras
(e.g. ``libraries.json``) are left untouched. Wiki pages (``src/wiki/**``)
are also bundled, under a ``wiki_`` namespaced module name, so the runbooks
are reachable from the MCP.
Args:
folder (Path): The hub folder owning a ``src/lib``.
dry_run (bool): Print the plan without executing.
"""
name = folder.name
lib = folder / "src" / "lib"
if not lib.is_dir():
return
out_dir = folder / "mcp"
modules_dir = out_dir / "modules"
if dry_run:
print(f"[MCP] {name} {lib.relative_to(ROOT)}/*.md "
f"-> {out_dir.relative_to(ROOT)}/")
return
if modules_dir.exists():
shutil.rmtree(modules_dir)
modules_dir.mkdir(parents=True, exist_ok=True)
def write_module(module: str, body: str) -> None:
payload = {
"module": module,
"content": {
"path": module,
"docstring": body.strip() + "\n",
"objects": {},
},
}
(modules_dir / f"{module}.json").write_text(
json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8"
)
nav = []
for path in sorted(lib.glob("*.md")):
module = _module_name(path.stem)
write_module(module, path.read_text(encoding="utf-8"))
nav.append({
"module": module,
"resource": f"docs://modules/{module}",
"title": _read_title(path),
})
wiki = folder / "src" / "wiki"
if wiki.is_dir():
for path in sorted(wiki.rglob("*.md")):
module = _wiki_module_name(path.relative_to(wiki))
write_module(module, path.read_text(encoding="utf-8"))
nav.append({
"module": module,
"resource": f"docs://modules/{module}",
"title": _read_title(path),
})
(out_dir / "nav.json").write_text(
json.dumps(nav, indent=2, ensure_ascii=False), encoding="utf-8"
)
(out_dir / "index.json").write_text(
json.dumps(
{
"project": name,
"type": name,
"modules_count": len(nav),
"source": "hand-authored",
},
indent=2,
ensure_ascii=False,
),
encoding="utf-8",
)
print(f"[MCP] {name} bundle written to {out_dir.relative_to(ROOT)}/ ({len(nav)} modules)")
# ── Template bundle ──────────────────────────────────────────────────────────
# Component kinds shipped in the vendored template. ``placeholder`` names the
# seed token an agent renames: "resource" -> item/Item, "family" -> example/
# Example; "" means ship as-is.
_TEMPLATE_COMPONENTS: list[dict[str, str]] = [
# Per-resource slice (seed: item/Item).
{"kind": "model", "path": "domain/item/models.py", "placeholder": "resource",
"description": "Business Pydantic model for a resource."},
{"kind": "ports-db", "path": "domain/item/ports/db.py", "placeholder": "resource",
"description": "Document model + repository factory for a resource."},
{"kind": "ports-service", "path": "domain/item/ports/service.py", "placeholder": "resource",
"description": "CRUD service wired through the DI library."},
{"kind": "converters", "path": "domain/item/ports/converters.py", "placeholder": "resource",
"description": "Optional pure converters between domain types."},
{"kind": "routes", "path": "interfaces/item/routes.py", "placeholder": "resource",
"description": "Thin HTTP handlers bound to operationIds."},
{"kind": "openapi-routes", "path": "interfaces/openapi.py", "placeholder": "",
"description": "Route registry: flat re-export of every handler namespace."},
{"kind": "auth", "path": "interfaces/auth/security.yaml", "placeholder": "",
"description": "Mergeable bearerAuth security scheme + global security for openapi.yaml."},
{"kind": "test-model", "path": "tests/unit/test_item_models.py", "placeholder": "resource",
"description": "Unit test for the resource model."},
{"kind": "test-routes", "path": "tests/integration/test_item_routes.py", "placeholder": "resource",
"description": "Integration test at the HTTP edge."},
{"kind": "test-openapi-build", "path": "tests/integration/test_openapi_app_builds.py", "placeholder": "",
"description": "Generic test: every spec operationId resolves and the app builds."},
{"kind": "test-pipeline", "path": "tests/unit/test_item_sync.py", "placeholder": "",
"description": "Unit test for the smallest pipeline form."},
# Per-family pipeline scaffold (seed: example/Example).
{"kind": "pipeline-abc", "path": "pipelines/example/abc.py", "placeholder": "family",
"description": "Port contracts for a pipeline family."},
{"kind": "pipeline-impl", "path": "pipelines/example/impl.py", "placeholder": "family",
"description": "Concrete stages, wired root, and family registration."},
{"kind": "pipeline-yaml", "path": "pipelines/example/pipeline.yaml", "placeholder": "family",
"description": "Graph spec (nodes, edges, roots, schema)."},
{"kind": "pipeline-resources", "path": "pipelines/example/resources/pipeline.yaml", "placeholder": "family",
"description": "Runtime spec consumed by the shared loader."},
{"kind": "pipeline-pattern", "path": "pipelines/example/resources/patterns/example.yaml", "placeholder": "family",
"description": "Pattern/rule fragment merged by the loader."},
# Shared pipeline scaffolding (ship as-is).
{"kind": "pipeline-ports", "path": "pipelines/shared/ports.py", "placeholder": "",
"description": "Shared pipeline ports (FinalizePort, RegistryPort)."},
{"kind": "pipeline-loader", "path": "pipelines/shared/loader.py", "placeholder": "",
"description": "Shared loader: DI wiring + pattern merge."},
{"kind": "pipeline-registry", "path": "pipelines/shared/registry.py", "placeholder": "",
"description": "Shared registry: (kind, variant, mode) -> root class."},
{"kind": "pipeline-simple", "path": "pipelines/item_sync.py", "placeholder": "",
"description": "Smallest pipeline form (plain async steps)."},
# Second resource slice (seed: job).
{"kind": "job-model", "path": "domain/job/models.py", "placeholder": "",
"description": "Seeded second resource model (job)."},
{"kind": "job-ports-db", "path": "domain/job/ports/db.py", "placeholder": "",
"description": "Seeded second resource repository (job)."},
{"kind": "job-ports-service", "path": "domain/job/ports/service.py", "placeholder": "",
"description": "Seeded second resource service (job)."},
{"kind": "job-routes", "path": "interfaces/job/routes.py", "placeholder": "",
"description": "Seeded second resource routes (job)."},
# Root / shared files (ship as-is).
{"kind": "config", "path": "config.py", "placeholder": "",
"description": "Settings dataclass + singleton."},
{"kind": "init", "path": "init.py", "placeholder": "",
"description": "Composition root (init/shutdown, get_svc)."},
{"kind": "main", "path": "main.py", "placeholder": "",
"description": "Spec-first app bootstrap + lifespan."},
{"kind": "worker", "path": "worker.py", "placeholder": "",
"description": "Poll loop + job status machine."},
{"kind": "openapi", "path": "openapi.yaml", "placeholder": "",
"description": "OpenAPI contract skeleton."},
{"kind": "pyproject", "path": "pyproject.toml", "placeholder": "",
"description": "Packaging + dependency pins."},
{"kind": "changelog", "path": "CHANGELOG.md", "placeholder": "",
"description": "Keep a Changelog file with an [Unreleased] section (versioning standard)."},
{"kind": "dockerfile", "path": "Dockerfile", "placeholder": "",
"description": "API + worker image with private-index wiring."},
{"kind": "dockerignore", "path": ".dockerignore", "placeholder": "",
"description": "Docker build-context exclusions."},
{"kind": "gitignore", "path": ".gitignore", "placeholder": "",
"description": "VCS exclusions (caches, venv, generated _gen/)."},
{"kind": "compose", "path": "docker-compose.yaml", "placeholder": "",
"description": "api + worker + mongo compose file."},
{"kind": "env-example", "path": ".env.example", "placeholder": "",
"description": "Environment variable template."},
{"kind": "db-setup", "path": "infra/db/setup.py", "placeholder": "",
"description": "Mongo connection lifecycle."},
{"kind": "repo", "path": "infra/db/repos/mongo.py", "placeholder": "",
"description": "Repository factory (base/populating/cached)."},
{"kind": "service-adapter", "path": "infra/adapters/service.py", "placeholder": "",
"description": "Generic CRUD service adaptor."},
{"kind": "http-cache", "path": "infra/http_cache.py", "placeholder": "",
"description": "Response cache decorator + Redis store (generation invalidation)."},
{"kind": "progress-bus", "path": "infra/adapters/sse.py", "placeholder": "",
"description": "In-memory SSE progress bus (keyed by an event attribute)."},
{"kind": "kv-store", "path": "infra/adapters/kv.py", "placeholder": "",
"description": "KV port + in-memory fallback + optional Redis adaptor."},
{"kind": "package-init", "path": "domain/__init__.py", "placeholder": "",
"description": "Empty package marker; add an __init__.py to every package dir."},
{"kind": "script-seed", "path": "scripts/seed.py", "placeholder": "",
"description": "Operator seed script."},
{"kind": "script-drop", "path": "scripts/drop.py", "placeholder": "",
"description": "Operator drop script."},
]
def build_components_manifest(template_root: Path) -> dict[str, object]:
"""Describe every component in the vendored template.
Args:
template_root (Path): The vendored ``templates/app`` directory.
Returns:
dict[str, object]: Manifest with the seed tokens and component entries.
"""
components: list[dict[str, str]] = []
for entry in _TEMPLATE_COMPONENTS:
rel = entry["path"]
if not (template_root / rel).exists():
raise SystemExit(f"[ERR] component path missing from template: {rel}")
components.append({
"kind": entry["kind"],
"path": rel,
"placeholder": entry["placeholder"],
"description": entry["description"],
})
return {
"project": "standards",
"template": "templates/app",
"seed": {"resource": "item", "family": "example"},
"components": components,
}
def build_templates(folder: Path, dry_run: bool = False) -> None:
"""Vendor ``folder/src/templates/app`` into ``<name>/mcp/templates/app``.
Ships the runnable skeleton in the MCP bundle so an agent can pull exact,
current component code, and emits ``components.json`` describing the kinds.
Excludes ``__pycache__`` / ``*.pyc``.
Args:
folder (Path): The hub folder owning ``src/templates/app``.
dry_run (bool): Print the plan without executing.
"""
name = folder.name
src = folder / "src" / "templates" / "app"
if not src.is_dir():
return
out_dir = folder / "mcp" / "templates" / "app"
if dry_run:
print(f"[TPL] {name} {src.relative_to(ROOT)}/** -> {out_dir.relative_to(ROOT)}/**")
return
if out_dir.exists():
shutil.rmtree(out_dir)
out_dir.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(
src, out_dir,
ignore=shutil.ignore_patterns(
"__pycache__", "*.pyc", ".ruff_cache", ".pytest_cache",
".mypy_cache", ".venv", "venv", ".git", "*.egg-info", "htmlcov",
),
)
manifest = build_components_manifest(out_dir)
(folder / "mcp" / "components.json").write_text(
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
)
print(f"[TPL] {name} template vendored to {out_dir.relative_to(ROOT)}/ "
f"({len(manifest['components'])} components)")
# ── Drone standard bundle ─────────────────────────────────────────────────────
# The two CI shapes shipped in ``src/templates/drone``. There are exactly two
# types of ``.drone.yml`` in the org: a Python package (private PyPI) and a
# Docker deployment (build / push / run). See standards "10_drone_pipelines".
_DRONE_COMPONENTS: list[dict[str, str]] = [
{"kind": "drone-pypi", "path": "python-lib.yaml", "placeholder": "",
"description": "Drone pipeline: build + publish a Python package to the private PyPI index."},
{"kind": "drone-service", "path": "service-docker.yaml", "placeholder": "",
"description": "Drone pipeline: build, push, and deploy a Docker service."},
]
def build_drone_manifest(template_root: Path) -> dict[str, object]:
"""Describe every CI template in the vendored drone standard.
Args:
template_root (Path): The vendored ``templates/drone`` directory.
Returns:
dict[str, object]: Manifest with the two component entries.
"""
components: list[dict[str, str]] = []
for entry in _DRONE_COMPONENTS:
rel = entry["path"]
if not (template_root / rel).exists():
raise SystemExit(f"[ERR] drone template missing: {rel}")
components.append({
"kind": entry["kind"],
"path": rel,
"placeholder": entry["placeholder"],
"description": entry["description"],
})
return {
"project": "standards",
"template": "templates/drone",
"types": ["pypi", "docker"],
"components": components,
}
def build_drone_templates(folder: Path, dry_run: bool = False) -> None:
"""Vendor ``folder/src/templates/drone`` into ``<name>/mcp/templates/drone``.
Ships the two ``.drone.yml`` shapes (PyPI package, Docker deployment) in the
MCP bundle so an agent can pull the exact CI file, and emits ``drone.json``
describing them.
Args:
folder (Path): The hub folder owning ``src/templates/drone``.
dry_run (bool): Print the plan without executing.
"""
name = folder.name
src = folder / "src" / "templates" / "drone"
if not src.is_dir():
return
out_dir = folder / "mcp" / "templates" / "drone"
if dry_run:
print(f"[DRN] {name} {src.relative_to(ROOT)}/** -> {out_dir.relative_to(ROOT)}/**")
return
if out_dir.exists():
shutil.rmtree(out_dir)
out_dir.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(
src, out_dir,
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".git"),
)
manifest = build_drone_manifest(out_dir)
(folder / "mcp" / "drone.json").write_text(
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
)
print(f"[DRN] {name} drone templates vendored to {out_dir.relative_to(ROOT)}/ "
f"({len(manifest['components'])} components)")
# ── Release standard bundle ───────────────────────────────────────────────────
# The release artefacts shipped in ``src/templates/release``. One canonical
# changelog (Keep a Changelog + SemVer) and the service version file. See
# standards "11_versioning_and_changelog".
_RELEASE_COMPONENTS: list[dict[str, str]] = [
{"kind": "changelog", "path": "CHANGELOG.md", "placeholder": "",
"description": "Keep a Changelog skeleton: one [Unreleased] section + compare links."},
{"kind": "version-file", "path": "VERSION", "placeholder": "",
"description": "Single-line SemVer source of truth for services."},
]
def build_release_manifest(template_root: Path) -> dict[str, object]:
"""Describe every artefact in the vendored release standard.
Args:
template_root (Path): The vendored ``templates/release`` directory.
Returns:
dict[str, object]: Manifest with the component entries and the
tag == version invariant.
"""
components: list[dict[str, str]] = []
for entry in _RELEASE_COMPONENTS:
rel = entry["path"]
if not (template_root / rel).exists():
raise SystemExit(f"[ERR] release template missing: {rel}")
components.append({
"kind": entry["kind"],
"path": rel,
"placeholder": entry["placeholder"],
"description": entry["description"],
})
return {
"project": "standards",
"template": "templates/release",
"invariant": "git tag == version file (enforced by the Drone guard)",
"components": components,
}
def build_release_templates(folder: Path, dry_run: bool = False) -> None:
"""Vendor ``folder/src/templates/release`` into ``<name>/mcp/templates/release``.
Ships the canonical ``CHANGELOG.md`` and ``VERSION`` in the MCP bundle so an
agent can drop them into a repo, and emits ``release.json`` describing them.
Args:
folder (Path): The hub folder owning ``src/templates/release``.
dry_run (bool): Print the plan without executing.
"""
name = folder.name
src = folder / "src" / "templates" / "release"
if not src.is_dir():
return
out_dir = folder / "mcp" / "templates" / "release"
if dry_run:
print(f"[REL] {name} {src.relative_to(ROOT)}/** -> {out_dir.relative_to(ROOT)}/**")
return
if out_dir.exists():
shutil.rmtree(out_dir)
out_dir.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(
src, out_dir,
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".git"),
)
manifest = build_release_manifest(out_dir)
(folder / "mcp" / "release.json").write_text(
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
)
print(f"[REL] {name} release templates vendored to {out_dir.relative_to(ROOT)}/ "
f"({len(manifest['components'])} components)")
# ── Repository inventory bundle ───────────────────────────────────────────────
def build_repos(folder: Path, dry_run: bool = False) -> None:
"""Publish ``folder/src/repos.json`` as ``<name>/mcp/repos.json``.
The repository inventory (name, type, version source, Drone shape) is a
single hand-authored JSON file, surfaced to agents through the
``registry://repos`` resource and the ``list_repos`` tool. See standards
"15_repository_inventory".
Args:
folder (Path): The hub folder owning ``src/repos.json``.
dry_run (bool): Print the plan without executing.
"""
name = folder.name
src = folder / "src" / "repos.json"
if not src.is_file():
return
out = folder / "mcp" / "repos.json"
if dry_run:
print(f"[REP] {name} {src.relative_to(ROOT)} -> {out.relative_to(ROOT)}")
return
data = json.loads(src.read_text(encoding="utf-8"))
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(
json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8"
)
print(f"[REP] {name} repo inventory written to {out.relative_to(ROOT)} "
f"({len(data.get('repos', []))} repos)")
# ── CLI ──────────────────────────────────────────────────────────────────────
def main() -> None:
parser = argparse.ArgumentParser(
description="Build hub-authored doc folders (src/ -> HTML + MCP bundle)"
)
parser.add_argument("--name", help="Only build a single folder")
parser.add_argument("--dry-run", action="store_true", help="Show the plan without writing")
args = parser.parse_args()
folders = [f for f in find_src_dirs() if not args.name or f.name == args.name]
if not folders:
print("No src/ doc folders found. Run build.py from the hub root.")
return
for folder in folders:
build_bundle(folder, dry_run=args.dry_run)
build_templates(folder, dry_run=args.dry_run)
build_drone_templates(folder, dry_run=args.dry_run)
build_release_templates(folder, dry_run=args.dry_run)
build_repos(folder, dry_run=args.dry_run)
build_html(folder, dry_run=args.dry_run)
if __name__ == "__main__":
main()