bifroest/app/backup.py

"""Sicherung der Daten außerhalb der Datenbank und Prüfung des Bestands (→ Betrieb).

Die Datenbank wird zentral gesichert. Außerhalb liegen nur die Uploads (DuckDB-Dateien,
nach dem Hochladen unveränderlich) und der Git-Cache (jederzeit neu klonbar, nicht gesichert).
Der Master-Key AIW_SECRET_KEY gehört in den Secret-Store; ohne ihn sind die verschlüsselten
Zugangsdaten in der Datenbank wertlos. `check` prüft deshalb auch, ob er sie entschlüsseln kann.

  aiw backup <verzeichnis>   Archiv uploads-<zeit>.tar.gz mit manifest.json (Prüfsummen)
  aiw restore <archiv>       Dateien zurückspielen, Prüfsummen prüfen
  aiw check                  Datenbank, Migrationen, Uploads und Master-Key prüfen
"""

import hashlib
import io
import json
import tarfile
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath

from sqlalchemy import select, text

from app.core import health
from app.core.config import Settings
from app.core.crypto import Vault
from app.core.database import Database

MANIFEST = "manifest.json"
FORMAT = 1


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            digest.update(chunk)
    return digest.hexdigest()


def referenced(db: Database, settings: Settings) -> list[tuple[int, int, Path]]:
    """(Upload-ID, Workspace-ID, Datei) aller verarbeiteten Uploads laut Datenbank."""
    from app.uploads.models import Upload
    from app.uploads.service import path_for
    with db.session() as session:
        return [(u.id, u.workspace_id, path_for(settings, u))
                for u in session.scalars(select(Upload).where(Upload.status == "ok").order_by(Upload.id))]


@dataclass
class BackupResult:
    archive: Path
    files: int
    bytes: int
    missing: list[int] = field(default_factory=list)  # Upload-IDs ohne Datei


def backup(db: Database, settings: Settings, target: Path) -> BackupResult:
    target.mkdir(parents=True, exist_ok=True)
    stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
    archive = target / f"uploads-{stamp}.tar.gz"
    partial = archive.with_suffix(".partial")
    root = Path(settings.upload_dir)
    entries, missing, total = [], [], 0
    with tarfile.open(partial, "w:gz") as tar:
        for upload_id, workspace_id, path in referenced(db, settings):
            if not path.is_file():
                missing.append(upload_id)
                continue
            name = f"uploads/{path.relative_to(root).as_posix()}"
            tar.add(path, arcname=name, recursive=False)
            size = path.stat().st_size
            total += size
            entries.append({"path": name, "upload_id": upload_id, "workspace_id": workspace_id, "size": size,
                            "sha256": sha256(path)})
        manifest = {"format": FORMAT, "created": stamp, "migration": migration(db), "files": entries,
                    "missing": missing}
        data = json.dumps(manifest, indent=2).encode()
        info = tarfile.TarInfo(MANIFEST)
        info.size, info.mtime = len(data), int(datetime.now(UTC).timestamp())
        tar.addfile(info, io.BytesIO(data))
    partial.rename(archive)  # erst vollständige Archive tragen den endgültigen Namen
    return BackupResult(archive, len(entries), total, missing)


@dataclass
class RestoreResult:
    restored: int = 0
    unchanged: int = 0
    conflicts: list[str] = field(default_factory=list)  # vorhanden, aber anderer Inhalt (ohne --force)
    unknown: list[int] = field(default_factory=list)  # Upload gibt es in der Datenbank nicht mehr


def restore(db: Database, settings: Settings, archive: Path, force: bool = False) -> RestoreResult:
    root = Path(settings.upload_dir).resolve()
    known = {upload_id for upload_id, _, _ in referenced(db, settings)}
    result = RestoreResult()
    with tarfile.open(archive, "r:gz") as tar:
        manifest = json.load(tar.extractfile(MANIFEST))
        if manifest.get("format") != FORMAT:
            raise ValueError(f"Unbekanntes Archivformat {manifest.get('format')!r}")
        for entry in manifest["files"]:
            if entry["upload_id"] not in known:
                result.unknown.append(entry["upload_id"])
                continue
            relative = PurePosixPath(entry["path"])
            # Nur uploads/<workspace>/<id>.duckdb, nie Pfade außerhalb des Upload-Verzeichnisses
            if relative.parts[0] != "uploads" or ".." in relative.parts or len(relative.parts) != 3:
                raise ValueError(f"Ungültiger Pfad im Archiv: {entry['path']}")
            destination = (root / Path(*relative.parts[1:])).resolve()
            if root not in destination.parents:
                raise ValueError(f"Ungültiger Pfad im Archiv: {entry['path']}")
            if destination.is_file():
                if sha256(destination) == entry["sha256"]:
                    result.unchanged += 1
                    continue
                if not force:
                    result.conflicts.append(entry["path"])
                    continue
            source = tar.extractfile(entry["path"])
            if source is None:
                raise ValueError(f"Datei fehlt im Archiv: {entry['path']}")
            destination.parent.mkdir(parents=True, exist_ok=True)
            temporary = destination.with_suffix(".restore")
            digest = hashlib.sha256()
            with temporary.open("wb") as out:
                for chunk in iter(lambda source=source: source.read(1 << 20), b""):
                    digest.update(chunk)
                    out.write(chunk)
            if digest.hexdigest() != entry["sha256"]:
                temporary.unlink()
                raise ValueError(f"Prüfsumme stimmt nicht: {entry['path']}")
            temporary.replace(destination)
            result.restored += 1
    return result


@dataclass
class CheckResult:
    checks: dict[str, str] = field(default_factory=dict)  # Name → "ok" oder Problem
    details: dict[str, list] = field(default_factory=dict)

    @property
    def ok(self) -> bool:
        return all(v == "ok" for v in self.checks.values())


def migration(db: Database) -> str | None:
    with db.engine.connect() as connection:
        return connection.execute(text("SELECT version_num FROM alembic_version")).scalar()


def check(db: Database, settings: Settings) -> CheckResult:
    result = CheckResult()
    ready = health.ready(db, settings)
    result.checks.update(ready.checks)
    if ready.checks.get("database") != "ok":
        return result
    # Uploads: Datenbank und Dateien abgleichen
    root = Path(settings.upload_dir)
    rows = referenced(db, settings)
    missing = [upload_id for upload_id, _, path in rows if not path.is_file()]
    expected = {path.resolve() for _, _, path in rows}
    orphans = sorted(str(p.relative_to(root)) for p in root.glob("*/*.duckdb") if p.resolve() not in expected) \
        if root.is_dir() else []
    result.checks["upload_files"] = "ok" if not missing else f"{len(missing)} Uploads ohne Datei"
    result.details["missing_uploads"] = missing
    # Verwaiste Dateien sind kein Fehler (z. B. abgebrochene Verarbeitung), nur ein Hinweis
    result.details["orphan_files"] = orphans
    result.checks["secrets"] = secrets(db, settings, result)
    return result


def secrets(db: Database, settings: Settings, result: CheckResult) -> str:
    """Kann AIW_SECRET_KEY alle gespeicherten Geheimnisse entschlüsseln?"""
    from app.ai.models import ProviderKey
    from app.data_sources.models import DataSource
    from app.mcp.models import McpServer, WorkspaceMcpServer
    from app.workspaces.models import Workspace
    if not settings.secret_key:
        return "AIW_SECRET_KEY fehlt"
    vault = Vault(settings.secret_key)
    failed: list[str] = []
    total = 0
    with db.session() as session:
        for label, column, key in (("Datenquelle", DataSource.secret, DataSource.id),
                                   ("API-Key", ProviderKey.secret, ProviderKey.id),
                                   ("MCP-Server", McpServer.secret, McpServer.id),
                                   ("MCP-Header", McpServer.headers, McpServer.id),
                                   ("MCP-Freischaltung", WorkspaceMcpServer.secret, WorkspaceMcpServer.workspace_id),
                                   ("Webhook", Workspace.webhook, Workspace.id)):
            for row_id, value in session.execute(select(key, column).where(column.is_not(None))):
                total += 1
                try:
                    vault.decrypt(value)
                except Exception:
                    failed.append(f"{label} {row_id}")
    result.details["secrets_failed"] = failed
    result.details["secrets_total"] = [total]
    return "ok" if not failed else f"{len(failed)} von {total} Geheimnissen nicht entschlüsselbar"