"""Uploads je Workspace (→ Konzept 7.7). Mitglieder eines Workspace dürfen Dateien hochladen und ihre eigenen löschen, Workspace-Admins jede. Die Originaldatei wird nach der Umwandlung verworfen; erhalten bleibt nur die DuckDB-Datei unter upload_dir//.duckdb. """ import re import tempfile from dataclasses import dataclass from pathlib import Path, PurePath from typing import BinaryIO from sqlalchemy import func, select from sqlalchemy.orm import Session from app import audit from app.core.config import Settings from app.core.errors import Forbidden, Invalid, NotFound from app.identity import Principal from app.tools.registry import ToolError from app.uploads import duck from app.uploads.models import FORMATS, Upload from app.workspaces import service as workspaces from app.workspaces.models import Workspace CHUNK = 1024 * 1024 MB = 1024 * 1024 QUOTA_FULL = ("Das Speicherkontingent des Workspace ist ausgeschöpft. Nicht mehr benötigte Dateien löschen " "oder den Plattform-Admin um mehr Speicher bitten.") def path_for(settings: Settings, upload: Upload) -> Path: return Path(settings.upload_dir) / str(upload.workspace_id) / f"{upload.id}.duckdb" def list_for(session: Session, principal: Principal, workspace_id: int) -> list[Upload]: workspaces.get(session, principal, workspace_id) return list(session.scalars(select(Upload).where(Upload.workspace_id == workspace_id).order_by(Upload.table_name))) def tables(session: Session, settings: Settings, workspace_id: int) -> list[duck.Table]: rows = session.scalars(select(Upload).where(Upload.workspace_id == workspace_id, Upload.status == "ok") .order_by(Upload.table_name)) return [duck.Table(u.table_name, path_for(settings, u)) for u in rows] def has_tables(session: Session, workspace_id: int) -> bool: return session.scalar(select(Upload.id).where(Upload.workspace_id == workspace_id, Upload.status == "ok") .limit(1)) is not None def table_name(session: Session, workspace_id: int, filename: str) -> str: """Aus dem Dateinamen ein einfacher SQL-Name, eindeutig im Workspace (umsatz_2025, umsatz_2025_2 …).""" stem = re.sub(r"[^a-z0-9]+", "_", PurePath(filename).stem.lower()).strip("_")[:40] or "daten" if stem[0].isdigit(): stem = f"t_{stem}" taken = set(session.scalars(select(Upload.table_name).where(Upload.workspace_id == workspace_id))) name, n = stem, 2 while name in taken: name, n = f"{stem}_{n}", n + 1 return name @dataclass(frozen=True) class Storage: used: int # Bytes limit: int # Bytes @property def free(self) -> int: return max(self.limit - self.used, 0) @property def percent(self) -> int: return min(round(self.used * 100 / self.limit), 100) if self.limit else 100 def storage(session: Session, settings: Settings, workspace: Workspace) -> Storage: from app.documents.service import used_bytes used = (session.scalar(select(func.coalesce(func.sum(Upload.stored_size), 0)) .where(Upload.workspace_id == workspace.id)) or 0) + used_bytes(session, workspace.id) limit_mb = workspace.storage_limit_mb if workspace.storage_limit_mb is not None else settings.workspace_storage_mb return Storage(int(used), limit_mb * MB) def save(session: Session, principal: Principal, settings: Settings, workspace_id: int, filename: str, stream: BinaryIO) -> Upload: workspace = workspaces.get(session, principal, workspace_id).workspace # Sperre je Workspace: parallele Uploads dürfen das Kontingent nicht gemeinsam überschreiten session.execute(select(Workspace.id).where(Workspace.id == workspace_id).with_for_update()) if storage(session, settings, workspace).free <= 0: raise Invalid(QUOTA_FULL) filename = PurePath(filename or "").name[:255] fmt = FORMATS.get(PurePath(filename).suffix.lower()) if fmt is None: raise Invalid("Erlaubt sind CSV, TSV, JSON, JSON Lines und Parquet.") limit = settings.upload_max_mb * 1024 * 1024 with tempfile.TemporaryDirectory() as tmp: raw = Path(tmp) / f"upload{PurePath(filename).suffix.lower()}" size = 0 with raw.open("wb") as out: while chunk := stream.read(CHUNK): size += len(chunk) if size > limit: raise Invalid(f"Die Datei ist größer als {settings.upload_max_mb} MB.") out.write(chunk) if size == 0: raise Invalid("Die Datei ist leer.") upload = Upload(workspace_id=workspace_id, filename=filename, format=fmt, size=size, table_name=table_name(session, workspace_id, filename), uploaded_by=principal.user_id) session.add(upload) session.flush() target = path_for(settings, upload) target.parent.mkdir(parents=True, exist_ok=True) try: upload.columns, upload.row_count = duck.convert(raw, fmt, target, settings) upload.status = "ok" except ToolError as e: upload.status, upload.error = "error", str(e) upload.stored_size = target.stat().st_size if target.is_file() else 0 quota = storage(session, settings, workspace) if quota.used > quota.limit: target.unlink(missing_ok=True) needed = (quota.used - quota.limit) / MB raise Invalid(f"Das Speicherkontingent des Workspace reicht nicht: Die Datei braucht als Tabelle " f"{upload.stored_size / MB:.1f} MB, es fehlen {needed:.1f} MB. Nicht mehr benötigte " "Dateien löschen oder den Plattform-Admin um mehr Speicher bitten.") audit.record(session, "upload.created", user_id=principal.user_id, workspace_id=workspace_id, target_kind="upload", target_id=upload.id, filename=filename, size=size, status=upload.status) return upload def delete(session: Session, principal: Principal, settings: Settings, upload_id: int) -> int: upload = session.get(Upload, upload_id) if upload is None: raise NotFound membership = workspaces.get(session, principal, upload.workspace_id) if upload.uploaded_by != principal.user_id and not membership.is_admin: raise Forbidden path_for(settings, upload).unlink(missing_ok=True) audit.record(session, "upload.deleted", user_id=principal.user_id, workspace_id=upload.workspace_id, target_kind="upload", target_id=upload.id, filename=upload.filename) session.delete(upload) return upload.workspace_id