160 lines
9 KiB
Python
160 lines
9 KiB
Python
"""Metriken im Prometheus-Format (prometheus_client).
|
|
|
|
Zähler und Histogramme gelten je Prozess: Web zählt Requests, der Worker Runs,
|
|
Modell- und Tool-Aufrufe und Zustellungen. Der Zustand aus der Datenbank (wartende
|
|
Runs, offene Freigaben, Worker-Herzschläge …) wird nur vom Web-Prozess beim Abruf
|
|
gelesen (StateCollector), damit er nicht doppelt erscheint.
|
|
|
|
Labels bleiben klein: Route-Muster statt Pfade, keine IDs, keine Nutzer.
|
|
"""
|
|
|
|
import hmac
|
|
import logging
|
|
from datetime import timedelta
|
|
|
|
from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram, generate_latest
|
|
from prometheus_client.core import GaugeMetricFamily
|
|
from prometheus_client.registry import Collector
|
|
from sqlalchemy import func, select
|
|
|
|
from app.core.config import Settings
|
|
from app.core.database import Database, now
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
|
|
REGISTRY = CollectorRegistry()
|
|
|
|
http_requests = Counter("aiw_http_requests_total", "HTTP-Requests", ["method", "route", "status"],
|
|
registry=REGISTRY)
|
|
http_duration = Histogram("aiw_http_request_duration_seconds", "Dauer der HTTP-Requests", ["method", "route"],
|
|
buckets=(0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10), registry=REGISTRY)
|
|
runs_finished = Counter("aiw_runs_finished_total", "Beendete Run-Versuche", ["kind", "status"], registry=REGISTRY)
|
|
run_duration = Histogram("aiw_run_duration_seconds", "Dauer eines Run-Versuchs im Worker", ["kind"],
|
|
buckets=(1, 2.5, 5, 10, 20, 30, 60, 120, 300, 600), registry=REGISTRY)
|
|
model_calls = Counter("aiw_model_calls_total", "Modellaufrufe", ["model", "outcome"], registry=REGISTRY)
|
|
model_duration = Histogram("aiw_model_call_duration_seconds", "Dauer der Modellaufrufe", ["model"],
|
|
buckets=(0.5, 1, 2.5, 5, 10, 20, 30, 60, 120), registry=REGISTRY)
|
|
model_tokens = Counter("aiw_model_tokens_total", "Verbrauchte Tokens", ["model", "kind"], registry=REGISTRY)
|
|
tool_calls = Counter("aiw_tool_calls_total", "Tool-Aufrufe", ["tool", "status"], registry=REGISTRY)
|
|
tool_duration = Histogram("aiw_tool_call_duration_seconds", "Dauer der Tool-Aufrufe", ["tool"],
|
|
buckets=(0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30), registry=REGISTRY)
|
|
notifications_sent = Counter("aiw_notifications_total", "Zustellversuche von Benachrichtigungen",
|
|
["channel", "result"], registry=REGISTRY)
|
|
untrusted_signals = Counter("aiw_untrusted_signals_total",
|
|
"Fremdinhalte, die wie Anweisungen an eine KI aussehen (→ app.security)", ["source"],
|
|
registry=REGISTRY)
|
|
worker_loop = Gauge("aiw_worker_last_loop_timestamp_seconds", "Letzter Durchlauf der Worker-Schleife",
|
|
registry=REGISTRY)
|
|
|
|
|
|
def tool_label(name: str) -> str:
|
|
"""MCP-Tools nach Server zusammenfassen, damit die Zahl der Labels begrenzt bleibt."""
|
|
return name.split("__", 1)[0] + "__*" if "__" in name else name
|
|
|
|
|
|
def untrusted_signal(source: str) -> None:
|
|
untrusted_signals.labels(tool_label(source)[:60]).inc()
|
|
|
|
|
|
class StateCollector(Collector):
|
|
"""Zustand aus der Datenbank, beim Abruf gelesen (nur im Web-Prozess registriert)."""
|
|
|
|
def __init__(self, db: Database, settings: Settings):
|
|
self.db, self.settings = db, settings
|
|
|
|
def collect(self):
|
|
from app.approvals.models import ApprovalRequest
|
|
from app.notify.models import Notification
|
|
from app.tasks.models import TaskRun
|
|
from app.tasks.monitor import WorkerHeartbeat
|
|
from app.uploads.models import Upload
|
|
from app.usage.models import UsageRecord
|
|
from app.workspaces.models import Workspace
|
|
up = GaugeMetricFamily("aiw_state_up", "1, wenn der Zustand aus der Datenbank gelesen werden konnte")
|
|
try:
|
|
with self.db.session() as session:
|
|
runs = GaugeMetricFamily("aiw_runs", "Runs nach Status", labels=["status"])
|
|
for status, count in session.execute(select(TaskRun.status, func.count()).group_by(TaskRun.status)
|
|
.where(TaskRun.status.in_(ACTIVE_STATES))):
|
|
runs.add_metric([status], count)
|
|
yield runs
|
|
oldest = session.scalar(select(func.min(TaskRun.available_at)).where(TaskRun.status == "queued",
|
|
TaskRun.available_at <= func.now()))
|
|
yield GaugeMetricFamily("aiw_runs_oldest_queued_seconds",
|
|
"Wartezeit des ältesten fälligen Runs (0 = keiner)",
|
|
value=(now() - oldest).total_seconds() if oldest else 0)
|
|
yield GaugeMetricFamily("aiw_approvals_pending", "Offene Freigabeanträge", value=session.scalar(
|
|
select(func.count()).select_from(ApprovalRequest).where(ApprovalRequest.status == "pending")))
|
|
notes = GaugeMetricFamily("aiw_notifications", "Benachrichtigungen nach Status", labels=["status"])
|
|
for status, count in session.execute(select(Notification.status, func.count())
|
|
.where(Notification.status.in_(("pending", "failed")))
|
|
.group_by(Notification.status)):
|
|
notes.add_metric([status], count)
|
|
yield notes
|
|
beats = list(session.scalars(select(WorkerHeartbeat.last_seen)))
|
|
alive = now() - timedelta(seconds=self.settings.worker_stale_seconds)
|
|
yield GaugeMetricFamily("aiw_workers_alive", "Worker mit frischem Herzschlag",
|
|
value=sum(1 for b in beats if b >= alive))
|
|
yield GaugeMetricFamily("aiw_worker_last_heartbeat_seconds",
|
|
"Sekunden seit dem jüngsten Worker-Herzschlag (-1 = nie)",
|
|
value=(now() - max(beats)).total_seconds() if beats else -1)
|
|
yield GaugeMetricFamily("aiw_workspaces_trashed", "Workspaces im Papierkorb", value=session.scalar(
|
|
select(func.count()).select_from(Workspace).where(Workspace.deleted.is_not(None))))
|
|
yield GaugeMetricFamily("aiw_uploads", "Hochgeladene Dateien", value=session.scalar(
|
|
select(func.count()).select_from(Upload)))
|
|
used = (select(Upload.workspace_id, func.sum(Upload.stored_size).label("bytes"))
|
|
.group_by(Upload.workspace_id).subquery())
|
|
rows = session.execute(select(Workspace.storage_limit_mb, used.c.bytes)
|
|
.join(used, used.c.workspace_id == Workspace.id)
|
|
.where(Workspace.deleted.is_(None))).all()
|
|
default = self.settings.workspace_storage_mb
|
|
yield GaugeMetricFamily("aiw_upload_bytes", "Belegter Speicher aller Uploads (DuckDB-Dateien)",
|
|
value=sum(int(b or 0) for _, b in rows))
|
|
yield GaugeMetricFamily("aiw_workspaces_storage_high",
|
|
"Workspaces mit mindestens 80 % belegtem Speicherkontingent",
|
|
value=sum(1 for limit, b in rows
|
|
if int(b or 0) >= 0.8 * (limit if limit is not None else default)
|
|
* 1024 * 1024))
|
|
today = now().replace(hour=0, minute=0, second=0, microsecond=0)
|
|
cost = session.scalar(select(func.coalesce(func.sum(UsageRecord.cost), 0))
|
|
.where(UsageRecord.time >= today))
|
|
yield GaugeMetricFamily("aiw_cost_today", f"Kosten seit Mitternacht (UTC) in "
|
|
f"{self.settings.cost_currency}", value=float(cost or 0))
|
|
up.add_metric([], 1)
|
|
except Exception:
|
|
log.exception("Zustand für /metrics nicht lesbar")
|
|
up.add_metric([], 0)
|
|
yield up
|
|
|
|
|
|
ACTIVE_STATES = ("queued", "running", "waiting_approval", "waiting_input", "needs_reconciliation")
|
|
|
|
|
|
def authorized(header: str | None, settings: Settings) -> bool:
|
|
"""Ohne AIW_METRICS_TOKEN ist /metrics aus; sonst muss der Bearer-Token passen."""
|
|
if not settings.metrics_token:
|
|
return False
|
|
sent = (header or "").removeprefix("Bearer ").strip()
|
|
return hmac.compare_digest(sent.encode(), settings.metrics_token.encode())
|
|
|
|
|
|
def render(registry: CollectorRegistry = REGISTRY) -> bytes:
|
|
return generate_latest(registry)
|
|
|
|
|
|
def web_registry(db: Database, settings: Settings) -> CollectorRegistry:
|
|
"""Prozess-Zähler plus Zustand aus der Datenbank für den Web-Prozess."""
|
|
registry = CollectorRegistry()
|
|
registry.register(_Forward(REGISTRY))
|
|
registry.register(StateCollector(db, settings))
|
|
return registry
|
|
|
|
|
|
class _Forward(Collector):
|
|
def __init__(self, source: CollectorRegistry):
|
|
self.source = source
|
|
|
|
def collect(self):
|
|
yield from self.source.collect()
|
|
|