"""Sicherung der Daten außerhalb der Datenbank und Prüfung des Bestands (→ Betrieb). Die Datenbank wird zentral gesichert. Außerhalb liegen nur die Uploads (DuckDB-Dateien, nach dem Hochladen unveränderlich) und der Git-Cache (jederzeit neu klonbar, nicht gesichert). Der Master-Key AIW_SECRET_KEY gehört in den Secret-Store; ohne ihn sind die verschlüsselten Zugangsdaten in der Datenbank wertlos. `check` prüft deshalb auch, ob er sie entschlüsseln kann. aiw backup Archiv uploads-.tar.gz mit manifest.json (Prüfsummen) aiw restore Dateien zurückspielen, Prüfsummen prüfen aiw check Datenbank, Migrationen, Uploads und Master-Key prüfen """ import hashlib import io import json import tarfile from dataclasses import dataclass, field from datetime import UTC, datetime from pathlib import Path, PurePosixPath from sqlalchemy import select, text from app.core import health from app.core.config import Settings from app.core.crypto import Vault from app.core.database import Database MANIFEST = "manifest.json" FORMAT = 1 def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() def referenced(db: Database, settings: Settings) -> list[tuple[int, int, Path]]: """(Upload-ID, Workspace-ID, Datei) aller verarbeiteten Uploads laut Datenbank.""" from app.uploads.models import Upload from app.uploads.service import path_for with db.session() as session: return [(u.id, u.workspace_id, path_for(settings, u)) for u in session.scalars(select(Upload).where(Upload.status == "ok").order_by(Upload.id))] @dataclass class BackupResult: archive: Path files: int bytes: int missing: list[int] = field(default_factory=list) # Upload-IDs ohne Datei def backup(db: Database, settings: Settings, target: Path) -> BackupResult: target.mkdir(parents=True, exist_ok=True) stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") archive = target / f"uploads-{stamp}.tar.gz" partial = archive.with_suffix(".partial") root = Path(settings.upload_dir) entries, missing, total = [], [], 0 with tarfile.open(partial, "w:gz") as tar: for upload_id, workspace_id, path in referenced(db, settings): if not path.is_file(): missing.append(upload_id) continue name = f"uploads/{path.relative_to(root).as_posix()}" tar.add(path, arcname=name, recursive=False) size = path.stat().st_size total += size entries.append({"path": name, "upload_id": upload_id, "workspace_id": workspace_id, "size": size, "sha256": sha256(path)}) manifest = {"format": FORMAT, "created": stamp, "migration": migration(db), "files": entries, "missing": missing} data = json.dumps(manifest, indent=2).encode() info = tarfile.TarInfo(MANIFEST) info.size, info.mtime = len(data), int(datetime.now(UTC).timestamp()) tar.addfile(info, io.BytesIO(data)) partial.rename(archive) # erst vollständige Archive tragen den endgültigen Namen return BackupResult(archive, len(entries), total, missing) @dataclass class RestoreResult: restored: int = 0 unchanged: int = 0 conflicts: list[str] = field(default_factory=list) # vorhanden, aber anderer Inhalt (ohne --force) unknown: list[int] = field(default_factory=list) # Upload gibt es in der Datenbank nicht mehr def restore(db: Database, settings: Settings, archive: Path, force: bool = False) -> RestoreResult: root = Path(settings.upload_dir).resolve() known = {upload_id for upload_id, _, _ in referenced(db, settings)} result = RestoreResult() with tarfile.open(archive, "r:gz") as tar: manifest = json.load(tar.extractfile(MANIFEST)) if manifest.get("format") != FORMAT: raise ValueError(f"Unbekanntes Archivformat {manifest.get('format')!r}") for entry in manifest["files"]: if entry["upload_id"] not in known: result.unknown.append(entry["upload_id"]) continue relative = PurePosixPath(entry["path"]) # Nur uploads//.duckdb, nie Pfade außerhalb des Upload-Verzeichnisses if relative.parts[0] != "uploads" or ".." in relative.parts or len(relative.parts) != 3: raise ValueError(f"Ungültiger Pfad im Archiv: {entry['path']}") destination = (root / Path(*relative.parts[1:])).resolve() if root not in destination.parents: raise ValueError(f"Ungültiger Pfad im Archiv: {entry['path']}") if destination.is_file(): if sha256(destination) == entry["sha256"]: result.unchanged += 1 continue if not force: result.conflicts.append(entry["path"]) continue source = tar.extractfile(entry["path"]) if source is None: raise ValueError(f"Datei fehlt im Archiv: {entry['path']}") destination.parent.mkdir(parents=True, exist_ok=True) temporary = destination.with_suffix(".restore") digest = hashlib.sha256() with temporary.open("wb") as out: for chunk in iter(lambda source=source: source.read(1 << 20), b""): digest.update(chunk) out.write(chunk) if digest.hexdigest() != entry["sha256"]: temporary.unlink() raise ValueError(f"Prüfsumme stimmt nicht: {entry['path']}") temporary.replace(destination) result.restored += 1 return result @dataclass class CheckResult: checks: dict[str, str] = field(default_factory=dict) # Name → "ok" oder Problem details: dict[str, list] = field(default_factory=dict) @property def ok(self) -> bool: return all(v == "ok" for v in self.checks.values()) def migration(db: Database) -> str | None: with db.engine.connect() as connection: return connection.execute(text("SELECT version_num FROM alembic_version")).scalar() def check(db: Database, settings: Settings) -> CheckResult: result = CheckResult() ready = health.ready(db, settings) result.checks.update(ready.checks) if ready.checks.get("database") != "ok": return result # Uploads: Datenbank und Dateien abgleichen root = Path(settings.upload_dir) rows = referenced(db, settings) missing = [upload_id for upload_id, _, path in rows if not path.is_file()] expected = {path.resolve() for _, _, path in rows} orphans = sorted(str(p.relative_to(root)) for p in root.glob("*/*.duckdb") if p.resolve() not in expected) \ if root.is_dir() else [] result.checks["upload_files"] = "ok" if not missing else f"{len(missing)} Uploads ohne Datei" result.details["missing_uploads"] = missing # Verwaiste Dateien sind kein Fehler (z. B. abgebrochene Verarbeitung), nur ein Hinweis result.details["orphan_files"] = orphans result.checks["secrets"] = secrets(db, settings, result) return result def secrets(db: Database, settings: Settings, result: CheckResult) -> str: """Kann AIW_SECRET_KEY alle gespeicherten Geheimnisse entschlüsseln?""" from app.ai.models import ProviderKey from app.data_sources.models import DataSource from app.mcp.models import McpServer, WorkspaceMcpServer from app.workspaces.models import Workspace if not settings.secret_key: return "AIW_SECRET_KEY fehlt" vault = Vault(settings.secret_key) failed: list[str] = [] total = 0 with db.session() as session: for label, column, key in (("Datenquelle", DataSource.secret, DataSource.id), ("API-Key", ProviderKey.secret, ProviderKey.id), ("MCP-Server", McpServer.secret, McpServer.id), ("MCP-Header", McpServer.headers, McpServer.id), ("MCP-Freischaltung", WorkspaceMcpServer.secret, WorkspaceMcpServer.workspace_id), ("Webhook", Workspace.webhook, Workspace.id)): for row_id, value in session.execute(select(key, column).where(column.is_not(None))): total += 1 try: vault.decrypt(value) except Exception: failed.append(f"{label} {row_id}") result.details["secrets_failed"] = failed result.details["secrets_total"] = [total] return "ok" if not failed else f"{len(failed)} von {total} Geheimnissen nicht entschlüsselbar"