147 lines
6.5 KiB
Python
147 lines
6.5 KiB
Python
"""Uploads je Workspace (→ Konzept 7.7).
|
|
|
|
Mitglieder eines Workspace dürfen Dateien hochladen und ihre eigenen löschen,
|
|
Workspace-Admins jede. Die Originaldatei wird nach der Umwandlung verworfen;
|
|
erhalten bleibt nur die DuckDB-Datei unter upload_dir/<workspace>/<upload>.duckdb.
|
|
"""
|
|
|
|
import re
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path, PurePath
|
|
from typing import BinaryIO
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import audit
|
|
from app.core.config import Settings
|
|
from app.core.errors import Forbidden, Invalid, NotFound
|
|
from app.identity import Principal
|
|
from app.tools.registry import ToolError
|
|
from app.uploads import duck
|
|
from app.uploads.models import FORMATS, Upload
|
|
from app.workspaces import service as workspaces
|
|
from app.workspaces.models import Workspace
|
|
|
|
CHUNK = 1024 * 1024
|
|
|
|
|
|
MB = 1024 * 1024
|
|
QUOTA_FULL = ("Das Speicherkontingent des Workspace ist ausgeschöpft. Nicht mehr benötigte Dateien löschen "
|
|
"oder den Plattform-Admin um mehr Speicher bitten.")
|
|
|
|
|
|
def path_for(settings: Settings, upload: Upload) -> Path:
|
|
return Path(settings.upload_dir) / str(upload.workspace_id) / f"{upload.id}.duckdb"
|
|
|
|
|
|
def list_for(session: Session, principal: Principal, workspace_id: int) -> list[Upload]:
|
|
workspaces.get(session, principal, workspace_id)
|
|
return list(session.scalars(select(Upload).where(Upload.workspace_id == workspace_id).order_by(Upload.table_name)))
|
|
|
|
|
|
def tables(session: Session, settings: Settings, workspace_id: int) -> list[duck.Table]:
|
|
rows = session.scalars(select(Upload).where(Upload.workspace_id == workspace_id, Upload.status == "ok")
|
|
.order_by(Upload.table_name))
|
|
return [duck.Table(u.table_name, path_for(settings, u)) for u in rows]
|
|
|
|
|
|
def has_tables(session: Session, workspace_id: int) -> bool:
|
|
return session.scalar(select(Upload.id).where(Upload.workspace_id == workspace_id, Upload.status == "ok")
|
|
.limit(1)) is not None
|
|
|
|
|
|
def table_name(session: Session, workspace_id: int, filename: str) -> str:
|
|
"""Aus dem Dateinamen ein einfacher SQL-Name, eindeutig im Workspace (umsatz_2025, umsatz_2025_2 …)."""
|
|
stem = re.sub(r"[^a-z0-9]+", "_", PurePath(filename).stem.lower()).strip("_")[:40] or "daten"
|
|
if stem[0].isdigit():
|
|
stem = f"t_{stem}"
|
|
taken = set(session.scalars(select(Upload.table_name).where(Upload.workspace_id == workspace_id)))
|
|
name, n = stem, 2
|
|
while name in taken:
|
|
name, n = f"{stem}_{n}", n + 1
|
|
return name
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Storage:
|
|
used: int # Bytes
|
|
limit: int # Bytes
|
|
|
|
@property
|
|
def free(self) -> int:
|
|
return max(self.limit - self.used, 0)
|
|
|
|
@property
|
|
def percent(self) -> int:
|
|
return min(round(self.used * 100 / self.limit), 100) if self.limit else 100
|
|
|
|
|
|
def storage(session: Session, settings: Settings, workspace: Workspace) -> Storage:
|
|
from app.documents.service import used_bytes
|
|
used = (session.scalar(select(func.coalesce(func.sum(Upload.stored_size), 0))
|
|
.where(Upload.workspace_id == workspace.id)) or 0) + used_bytes(session, workspace.id)
|
|
limit_mb = workspace.storage_limit_mb if workspace.storage_limit_mb is not None else settings.workspace_storage_mb
|
|
return Storage(int(used), limit_mb * MB)
|
|
|
|
|
|
def save(session: Session, principal: Principal, settings: Settings, workspace_id: int, filename: str,
|
|
stream: BinaryIO) -> Upload:
|
|
workspace = workspaces.get(session, principal, workspace_id).workspace
|
|
# Sperre je Workspace: parallele Uploads dürfen das Kontingent nicht gemeinsam überschreiten
|
|
session.execute(select(Workspace.id).where(Workspace.id == workspace_id).with_for_update())
|
|
if storage(session, settings, workspace).free <= 0:
|
|
raise Invalid(QUOTA_FULL)
|
|
filename = PurePath(filename or "").name[:255]
|
|
fmt = FORMATS.get(PurePath(filename).suffix.lower())
|
|
if fmt is None:
|
|
raise Invalid("Erlaubt sind CSV, TSV, JSON, JSON Lines und Parquet.")
|
|
limit = settings.upload_max_mb * 1024 * 1024
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
raw = Path(tmp) / f"upload{PurePath(filename).suffix.lower()}"
|
|
size = 0
|
|
with raw.open("wb") as out:
|
|
while chunk := stream.read(CHUNK):
|
|
size += len(chunk)
|
|
if size > limit:
|
|
raise Invalid(f"Die Datei ist größer als {settings.upload_max_mb} MB.")
|
|
out.write(chunk)
|
|
if size == 0:
|
|
raise Invalid("Die Datei ist leer.")
|
|
upload = Upload(workspace_id=workspace_id, filename=filename, format=fmt, size=size,
|
|
table_name=table_name(session, workspace_id, filename), uploaded_by=principal.user_id)
|
|
session.add(upload)
|
|
session.flush()
|
|
target = path_for(settings, upload)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
upload.columns, upload.row_count = duck.convert(raw, fmt, target, settings)
|
|
upload.status = "ok"
|
|
except ToolError as e:
|
|
upload.status, upload.error = "error", str(e)
|
|
upload.stored_size = target.stat().st_size if target.is_file() else 0
|
|
quota = storage(session, settings, workspace)
|
|
if quota.used > quota.limit:
|
|
target.unlink(missing_ok=True)
|
|
needed = (quota.used - quota.limit) / MB
|
|
raise Invalid(f"Das Speicherkontingent des Workspace reicht nicht: Die Datei braucht als Tabelle "
|
|
f"{upload.stored_size / MB:.1f} MB, es fehlen {needed:.1f} MB. Nicht mehr benötigte "
|
|
"Dateien löschen oder den Plattform-Admin um mehr Speicher bitten.")
|
|
audit.record(session, "upload.created", user_id=principal.user_id, workspace_id=workspace_id,
|
|
target_kind="upload", target_id=upload.id, filename=filename, size=size, status=upload.status)
|
|
return upload
|
|
|
|
|
|
def delete(session: Session, principal: Principal, settings: Settings, upload_id: int) -> int:
|
|
upload = session.get(Upload, upload_id)
|
|
if upload is None:
|
|
raise NotFound
|
|
membership = workspaces.get(session, principal, upload.workspace_id)
|
|
if upload.uploaded_by != principal.user_id and not membership.is_admin:
|
|
raise Forbidden
|
|
path_for(settings, upload).unlink(missing_ok=True)
|
|
audit.record(session, "upload.deleted", user_id=principal.user_id, workspace_id=upload.workspace_id,
|
|
target_kind="upload", target_id=upload.id, filename=upload.filename)
|
|
session.delete(upload)
|
|
return upload.workspace_id
|