67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
"""# Summary
|
|
|
|
Background worker — polls for pending jobs and runs the item-sync pipeline.
|
|
The poll loop is the pipeline axis of the blueprint: resources are created by
|
|
the API axis, workers consume them on a fixed cadence.
|
|
|
|
Notes:
|
|
- This process is separate from the API process. Any in-memory event bus is per
|
|
process — if the pipeline must surface live progress to the API, swap the bus
|
|
adaptor for a broker-backed one (see standards 07_worker_and_progress).
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
|
|
from config import settings
|
|
from pipelines.item_sync import run_item_sync
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def poll_jobs(resources: dict):
|
|
"""Poll for pending jobs and run the item-sync pipeline on each.
|
|
|
|
Args:
|
|
resources (dict): Wired services from :func:`init.init`.
|
|
"""
|
|
job_svc = resources["job_svc"]
|
|
logger.info("WORKER STARTED — poll=%ss max_retries=%d",
|
|
settings.worker_poll_seconds, settings.worker_max_retries)
|
|
|
|
while True:
|
|
pending = await job_svc.search(filter={"status": "pending"}, limit=10)
|
|
logger.info("Found %d pending job(s)", len(pending) if pending else 0)
|
|
|
|
for job in pending:
|
|
job_id = str(job.id)
|
|
if (job.retry_count or 0) >= settings.worker_max_retries:
|
|
logger.warning("SKIPPING job %s — retries exhausted", job_id)
|
|
continue
|
|
|
|
await job_svc.patch(job_id, {"status": "processing"})
|
|
try:
|
|
await run_item_sync(job, resources)
|
|
await job_svc.patch(job_id, {"status": "done", "error_message": ""})
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.exception("job %s FAILED: %s", job_id, exc)
|
|
await job_svc.patch(job_id, {"status": "failed", "error_message": str(exc)})
|
|
|
|
await asyncio.sleep(settings.worker_poll_seconds)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from init import init, shutdown
|
|
|
|
async def _run():
|
|
resources = await init()
|
|
try:
|
|
await poll_jobs(resources)
|
|
finally:
|
|
await shutdown()
|
|
|
|
asyncio.run(_run()) |