feat: add standards wiki, MCP registry, and app template
This commit is contained in:
67
templates/app/worker.py
Normal file
67
templates/app/worker.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""# Summary
|
||||
|
||||
Background worker — polls for pending jobs and runs the item-sync pipeline.
|
||||
The poll loop is the pipeline axis of the blueprint: resources are created by
|
||||
the API axis, workers consume them on a fixed cadence.
|
||||
|
||||
Notes:
|
||||
- This process is separate from the API process. Any in-memory event bus is per
|
||||
process — if the pipeline must surface live progress to the API, swap the bus
|
||||
adaptor for a broker-backed one (see standards 07_worker_and_progress).
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from config import settings
|
||||
from pipelines.item_sync import run_item_sync
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def poll_jobs(resources: dict):
|
||||
"""Poll for pending jobs and run the item-sync pipeline on each.
|
||||
|
||||
Args:
|
||||
resources (dict): Wired services from :func:`init.init`.
|
||||
"""
|
||||
job_svc = resources["job_svc"]
|
||||
logger.info("WORKER STARTED — poll=%ss max_retries=%d",
|
||||
settings.worker_poll_seconds, settings.worker_max_retries)
|
||||
|
||||
while True:
|
||||
pending = await job_svc.search(filter={"status": "pending"}, limit=10)
|
||||
logger.info("Found %d pending job(s)", len(pending) if pending else 0)
|
||||
|
||||
for job in pending:
|
||||
job_id = str(job.id)
|
||||
if (job.retry_count or 0) >= settings.worker_max_retries:
|
||||
logger.warning("SKIPPING job %s — retries exhausted", job_id)
|
||||
continue
|
||||
|
||||
await job_svc.patch(job_id, {"status": "processing"})
|
||||
try:
|
||||
await run_item_sync(job, resources)
|
||||
await job_svc.patch(job_id, {"status": "done", "error_message": ""})
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("job %s FAILED: %s", job_id, exc)
|
||||
await job_svc.patch(job_id, {"status": "failed", "error_message": str(exc)})
|
||||
|
||||
await asyncio.sleep(settings.worker_poll_seconds)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from init import init, shutdown
|
||||
|
||||
async def _run():
|
||||
resources = await init()
|
||||
try:
|
||||
await poll_jobs(resources)
|
||||
finally:
|
||||
await shutdown()
|
||||
|
||||
asyncio.run(_run())
|
||||
Reference in New Issue
Block a user