107 lines
4.3 KiB
Python
107 lines
4.3 KiB
Python
"""Worker-Überwachung: Herzschlag in der Datenbank und ein kleiner HTTP-Server
|
|
für /live, /ready und /metrics (der Worker hat sonst keinen Port)."""
|
|
|
|
import json
|
|
import logging
|
|
import socket
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
from sqlalchemy import DateTime, Integer, String, delete
|
|
from sqlalchemy.dialects.postgresql import insert
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from app.core import health, metrics
|
|
from app.core.config import Settings
|
|
from app.core.database import Base, Database, now
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class WorkerHeartbeat(Base):
|
|
"""Ein Eintrag je laufendem Worker; last_seen wird regelmäßig erneuert."""
|
|
|
|
__tablename__ = "worker_heartbeats"
|
|
|
|
worker_id: Mapped[str] = mapped_column(String(200), primary_key=True)
|
|
hostname: Mapped[str] = mapped_column(String(200))
|
|
started: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
last_seen: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
|
runs: Mapped[int] = mapped_column(Integer, default=0)
|
|
|
|
|
|
def beat(db: Database, worker_id: str, started: datetime, runs: int) -> None:
|
|
with db.session() as session:
|
|
session.execute(insert(WorkerHeartbeat).values(
|
|
worker_id=worker_id, hostname=socket.gethostname()[:200], started=started, last_seen=now(), runs=runs)
|
|
.on_conflict_do_update(index_elements=["worker_id"], set_={"last_seen": now(), "runs": runs}))
|
|
|
|
|
|
def forget(db: Database, worker_id: str) -> None:
|
|
with db.session() as session:
|
|
session.execute(delete(WorkerHeartbeat).where(WorkerHeartbeat.worker_id == worker_id))
|
|
|
|
|
|
def prune(db: Database, older_than: timedelta = timedelta(days=1)) -> None:
|
|
with db.session() as session:
|
|
session.execute(delete(WorkerHeartbeat).where(WorkerHeartbeat.last_seen < now() - older_than))
|
|
|
|
|
|
class Probe:
|
|
"""Zustand der Worker-Schleife für /live: wann sie zuletzt durchlief."""
|
|
|
|
def __init__(self) -> None:
|
|
self.last = time.monotonic()
|
|
|
|
def tick(self) -> None:
|
|
self.last = time.monotonic()
|
|
metrics.worker_loop.set_to_current_time()
|
|
|
|
def alive(self, settings: Settings) -> bool:
|
|
return time.monotonic() - self.last < settings.worker_stale_seconds
|
|
|
|
|
|
def serve(db: Database, settings: Settings, probe: Probe) -> ThreadingHTTPServer | None:
|
|
"""HTTP-Server des Workers in einem Daemon-Thread; Port 0 in den Einstellungen = aus."""
|
|
if not settings.worker_http_port:
|
|
return None
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
path = self.path.split("?", 1)[0]
|
|
if path in ("/live", "/healthz"):
|
|
ok = probe.alive(settings)
|
|
self.reply(200 if ok else 503, {"status": "ok" if ok else "Worker-Schleife hängt"})
|
|
elif path in ("/ready", "/readyz"):
|
|
report = health.ready(db, settings)
|
|
self.reply(200 if report.ok else 503, report.body())
|
|
elif path == "/metrics":
|
|
if not metrics.authorized(self.headers.get("Authorization"), settings):
|
|
self.reply(404 if not settings.metrics_token else 401, {"status": "nicht erlaubt"})
|
|
return
|
|
body = metrics.render()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", metrics.CONTENT_TYPE)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
else:
|
|
self.reply(404, {"status": "unbekannt"})
|
|
|
|
def reply(self, code: int, payload: dict) -> None:
|
|
body = json.dumps(payload, ensure_ascii=False).encode()
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def log_message(self, *args) -> None: # Probes alle paar Sekunden: nicht ins Log
|
|
pass
|
|
|
|
server = ThreadingHTTPServer((settings.worker_http_host, settings.worker_http_port), Handler)
|
|
threading.Thread(target=server.serve_forever, name="worker-http", daemon=True).start()
|
|
log.info("Worker-HTTP auf %s:%s", settings.worker_http_host, server.server_port)
|
|
return server
|