"""Worker-Überwachung: Herzschlag in der Datenbank und ein kleiner HTTP-Server für /live, /ready und /metrics (der Worker hat sonst keinen Port).""" import json import logging import socket import threading import time from datetime import datetime, timedelta from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from sqlalchemy import DateTime, Integer, String, delete from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Mapped, mapped_column from app.core import health, metrics from app.core.config import Settings from app.core.database import Base, Database, now log = logging.getLogger(__name__) class WorkerHeartbeat(Base): """Ein Eintrag je laufendem Worker; last_seen wird regelmäßig erneuert.""" __tablename__ = "worker_heartbeats" worker_id: Mapped[str] = mapped_column(String(200), primary_key=True) hostname: Mapped[str] = mapped_column(String(200)) started: Mapped[datetime] = mapped_column(DateTime(timezone=True)) last_seen: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True) runs: Mapped[int] = mapped_column(Integer, default=0) def beat(db: Database, worker_id: str, started: datetime, runs: int) -> None: with db.session() as session: session.execute(insert(WorkerHeartbeat).values( worker_id=worker_id, hostname=socket.gethostname()[:200], started=started, last_seen=now(), runs=runs) .on_conflict_do_update(index_elements=["worker_id"], set_={"last_seen": now(), "runs": runs})) def forget(db: Database, worker_id: str) -> None: with db.session() as session: session.execute(delete(WorkerHeartbeat).where(WorkerHeartbeat.worker_id == worker_id)) def prune(db: Database, older_than: timedelta = timedelta(days=1)) -> None: with db.session() as session: session.execute(delete(WorkerHeartbeat).where(WorkerHeartbeat.last_seen < now() - older_than)) class Probe: """Zustand der Worker-Schleife für /live: wann sie zuletzt durchlief.""" def __init__(self) -> None: self.last = time.monotonic() def tick(self) -> None: self.last = time.monotonic() metrics.worker_loop.set_to_current_time() def alive(self, settings: Settings) -> bool: return time.monotonic() - self.last < settings.worker_stale_seconds def serve(db: Database, settings: Settings, probe: Probe) -> ThreadingHTTPServer | None: """HTTP-Server des Workers in einem Daemon-Thread; Port 0 in den Einstellungen = aus.""" if not settings.worker_http_port: return None class Handler(BaseHTTPRequestHandler): def do_GET(self): path = self.path.split("?", 1)[0] if path in ("/live", "/healthz"): ok = probe.alive(settings) self.reply(200 if ok else 503, {"status": "ok" if ok else "Worker-Schleife hängt"}) elif path in ("/ready", "/readyz"): report = health.ready(db, settings) self.reply(200 if report.ok else 503, report.body()) elif path == "/metrics": if not metrics.authorized(self.headers.get("Authorization"), settings): self.reply(404 if not settings.metrics_token else 401, {"status": "nicht erlaubt"}) return body = metrics.render() self.send_response(200) self.send_header("Content-Type", metrics.CONTENT_TYPE) self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) else: self.reply(404, {"status": "unbekannt"}) def reply(self, code: int, payload: dict) -> None: body = json.dumps(payload, ensure_ascii=False).encode() self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *args) -> None: # Probes alle paar Sekunden: nicht ins Log pass server = ThreadingHTTPServer((settings.worker_http_host, settings.worker_http_port), Handler) threading.Thread(target=server.serve_forever, name="worker-http", daemon=True).start() log.info("Worker-HTTP auf %s:%s", settings.worker_http_host, server.server_port) return server