330 lines
16 KiB
Python
330 lines
16 KiB
Python
"""Abgeglichene Ordner: anlegen, abgleichen (Liste, Datei, Bereinigen), auslesen, Sichtbarkeit.
|
|
|
|
Abgleich: Der Browser holt die Liste (Pfad, Größe, Änderungszeit), lädt nur Neues und Geändertes einzeln hoch
|
|
und meldet zum Schluss alle vorhandenen Pfade; was fehlt, wird entfernt. Textdateien liest Bifröst sofort,
|
|
Office, PDF und Bilder der Worker über den MCP-Server mit der Rolle „documents“. Bis dahin liegt das Original
|
|
im Upload-Verzeichnis und wird nach der Auslese gelöscht.
|
|
"""
|
|
|
|
import base64
|
|
import dataclasses
|
|
import hashlib
|
|
import logging
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import BinaryIO
|
|
|
|
from sqlalchemy import func, or_, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import audit
|
|
from app.core.config import Settings
|
|
from app.core.crypto import Vault
|
|
from app.core.database import Database, now
|
|
from app.core.errors import Forbidden, Invalid, NotFound
|
|
from app.documents.models import ERROR, OK, PENDING, PRIVATE, UNSUPPORTED, WORKSPACE, DocFile, DocFolder
|
|
from app.identity import Principal
|
|
from app.tools.registry import ToolError
|
|
from app.workspaces import service as workspaces
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
TEXT = {".md", ".markdown", ".txt", ".text", ".csv", ".tsv", ".json", ".yaml", ".yml", ".xml", ".html", ".htm",
|
|
".rst", ".log", ".ini", ".toml", ".adoc"}
|
|
# Liest der MCP-Server mit der Rolle „documents“
|
|
EXTRACTED = {".pdf", ".docx", ".doc", ".docm", ".xlsx", ".xls", ".xlsm", ".pptx", ".ppt", ".odt", ".ods", ".odp",
|
|
".rtf", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".tif", ".tiff", ".bmp"}
|
|
SCOPES = {PRIVATE: "Nur ich", WORKSPACE: "Ganzer Workspace"}
|
|
CHUNK = 1024 * 1024
|
|
BATCH = 5 # Dateien je Auslese-Durchlauf des Workers
|
|
|
|
|
|
def suffix(path: str) -> str:
|
|
return PurePosixPath(path).suffix.lower()
|
|
|
|
|
|
def allowed(path: str) -> bool:
|
|
name = PurePosixPath(path).name
|
|
return suffix(path) in TEXT | EXTRACTED and not name.startswith((".", "~$"))
|
|
|
|
|
|
def check_path(path: str) -> str:
|
|
"""Relativer Pfad im Ordner, ohne .., ohne führenden Schrägstrich."""
|
|
path = path.replace("\\", "/").strip().strip("/")
|
|
parts = [p for p in path.split("/") if p not in ("", ".")]
|
|
if not parts or ".." in parts or len(path) > 500 or any(c in path for c in "\0\n\r"):
|
|
raise Invalid("Ungültiger Dateipfad.")
|
|
return "/".join(parts)
|
|
|
|
|
|
def pending_files(session: Session, settings: Settings, workspace_id: int) -> list[Path]:
|
|
"""Zwischendateien noch nicht ausgelesener Dateien eines Workspace (zum Aufräumen beim Löschen)."""
|
|
rows = session.scalars(select(DocFile).join(DocFolder, DocFolder.id == DocFile.folder_id)
|
|
.where(DocFolder.workspace_id == workspace_id, DocFile.status == PENDING))
|
|
return [pending_path(settings, f) for f in rows]
|
|
|
|
|
|
def pending_path(settings: Settings, file: DocFile) -> Path:
|
|
return Path(settings.upload_dir) / "documents" / f"{file.id}.bin"
|
|
|
|
|
|
# --- Ordner und Sichtbarkeit ---
|
|
|
|
def visible(query, user_id: int):
|
|
"""Ordner, die jemand sieht: alle des Workspace und die eigenen privaten."""
|
|
return query.where(or_(DocFolder.scope == WORKSPACE, DocFolder.owner_id == user_id))
|
|
|
|
|
|
def folders(session: Session, principal: Principal, workspace_id: int) -> list[DocFolder]:
|
|
workspaces.get(session, principal, workspace_id)
|
|
return list(session.scalars(visible(select(DocFolder).where(DocFolder.workspace_id == workspace_id),
|
|
principal.user_id).order_by(DocFolder.name)))
|
|
|
|
|
|
def for_run(session: Session, workspace_id: int, user_id: int | None, chat_id: int | None) -> list[DocFolder]:
|
|
"""Ordner, die ein Run lesen darf: die des Workspace und private nur, wenn die Person ihn in ihrem
|
|
eigenen Chat ausgelöst hat (in fremden Chats sähen andere die Antworten)."""
|
|
from app.chat.models import ChatSession
|
|
owner = None
|
|
if user_id is not None and chat_id is not None:
|
|
chat = session.get(ChatSession, chat_id)
|
|
owner = user_id if chat is not None and chat.created_by == user_id else None
|
|
condition = DocFolder.scope == WORKSPACE
|
|
if owner is not None:
|
|
condition = or_(condition, DocFolder.owner_id == owner)
|
|
return list(session.scalars(select(DocFolder).where(DocFolder.workspace_id == workspace_id, condition)
|
|
.order_by(DocFolder.name)))
|
|
|
|
|
|
def prompt_note(found: list[DocFolder]) -> str:
|
|
if not found:
|
|
return ""
|
|
lines = []
|
|
for f in found:
|
|
age = f"zuletzt abgeglichen {f.synced.astimezone().strftime('%d.%m.%Y %H:%M')}" if f.synced else "noch nie abgeglichen"
|
|
lines.append(f"- {f.name} ({'nur für die Person, die fragt' if f.scope == PRIVATE else 'Workspace'}; {age})")
|
|
return ("\nAbgeglichene Ordner mit Dokumenten (lesen mit docs_list, docs_search, docs_read; der Stand ist der "
|
|
"des letzten Abgleichs, nenne ihn bei Bedarf):\n" + "\n".join(lines) + "\n")
|
|
|
|
|
|
def folder(session: Session, principal: Principal, folder_id: int, write: bool = False) -> DocFolder:
|
|
"""Ordner, den die Person sieht; abgleichen darf nur, wer ihn angelegt hat."""
|
|
found = session.get(DocFolder, folder_id)
|
|
if found is None:
|
|
raise NotFound
|
|
workspaces.get(session, principal, found.workspace_id)
|
|
if found.scope == PRIVATE and found.owner_id != principal.user_id:
|
|
raise NotFound
|
|
if write and found.owner_id != principal.user_id:
|
|
raise Forbidden("Abgleichen kann nur, wer den Ordner verbunden hat.")
|
|
return found
|
|
|
|
|
|
def create(session: Session, principal: Principal, workspace_id: int, name: str, scope: str,
|
|
confirmed: bool) -> DocFolder:
|
|
workspaces.get(session, principal, workspace_id)
|
|
name = name.strip()[:100]
|
|
if not name:
|
|
raise Invalid("Name fehlt.")
|
|
if scope not in SCOPES:
|
|
raise Invalid("Unbekannte Sichtbarkeit.")
|
|
if scope == WORKSPACE and not confirmed:
|
|
raise Invalid("Bitte bestätigen, dass alle Mitglieder des Workspace diese Dokumente sehen dürfen.")
|
|
created = DocFolder(workspace_id=workspace_id, owner_id=principal.user_id, name=name, scope=scope,
|
|
exposure_confirmed=scope == WORKSPACE)
|
|
session.add(created)
|
|
session.flush()
|
|
audit.record(session, "documents.folder_created", user_id=principal.user_id, workspace_id=workspace_id,
|
|
target_kind="doc_folder", target_id=created.id, name=name, scope=scope)
|
|
return created
|
|
|
|
|
|
def delete(session: Session, principal: Principal, settings: Settings, folder_id: int) -> int:
|
|
"""Löschen darf, wer den Ordner verbunden hat, bei Ordnern des Workspace auch ein Workspace-Admin."""
|
|
found = folder(session, principal, folder_id)
|
|
admin = found.scope == WORKSPACE and workspaces.get(session, principal, found.workspace_id).is_admin
|
|
if found.owner_id != principal.user_id and not admin:
|
|
raise Forbidden("Löschen kann nur, wer den Ordner verbunden hat, oder ein Workspace-Admin.")
|
|
for file in session.scalars(select(DocFile).where(DocFile.folder_id == found.id, DocFile.status == PENDING)):
|
|
pending_path(settings, file).unlink(missing_ok=True)
|
|
audit.record(session, "documents.folder_deleted", user_id=principal.user_id, workspace_id=found.workspace_id,
|
|
target_kind="doc_folder", target_id=found.id, name=found.name)
|
|
workspace_id = found.workspace_id
|
|
session.delete(found)
|
|
return workspace_id
|
|
|
|
|
|
@dataclasses.dataclass
|
|
class Summary:
|
|
files: int = 0
|
|
ok: int = 0
|
|
pending: int = 0
|
|
failed: int = 0 # Fehler oder ohne Auslese
|
|
size: int = 0 # Text in Bytes
|
|
|
|
|
|
def summaries(session: Session, folder_ids: list[int]) -> dict[int, Summary]:
|
|
result = {i: Summary() for i in folder_ids}
|
|
if not folder_ids:
|
|
return result
|
|
rows = session.execute(select(DocFile.folder_id, DocFile.status, func.count(),
|
|
func.coalesce(func.sum(func.octet_length(DocFile.text)), 0))
|
|
.where(DocFile.folder_id.in_(folder_ids)).group_by(DocFile.folder_id, DocFile.status))
|
|
for folder_id, status, count, size in rows:
|
|
s = result[folder_id]
|
|
s.files += count
|
|
s.size += int(size)
|
|
if status == OK:
|
|
s.ok += count
|
|
elif status == PENDING:
|
|
s.pending += count
|
|
else:
|
|
s.failed += count
|
|
return result
|
|
|
|
|
|
def files(session: Session, folder_id: int) -> list[DocFile]:
|
|
return list(session.scalars(select(DocFile).where(DocFile.folder_id == folder_id).order_by(DocFile.path)))
|
|
|
|
|
|
def used_bytes(session: Session, workspace_id: int) -> int:
|
|
"""Speicher der Ordner eines Workspace (ausgelesener Text); zählt fürs Kontingent."""
|
|
return int(session.scalar(select(func.coalesce(func.sum(func.octet_length(DocFile.text)), 0))
|
|
.join(DocFolder, DocFolder.id == DocFile.folder_id)
|
|
.where(DocFolder.workspace_id == workspace_id)) or 0)
|
|
|
|
|
|
# --- Abgleich ---
|
|
|
|
def manifest(session: Session, principal: Principal, folder_id: int) -> dict:
|
|
found = folder(session, principal, folder_id, write=True)
|
|
# Prüfsumme für Browser, deren Dateiangaben beim erneuten Wählen veraltet sein können (Safari)
|
|
rows = session.execute(select(DocFile.path, DocFile.size, DocFile.modified, DocFile.sha256)
|
|
.where(DocFile.folder_id == found.id))
|
|
return {"files": {path: [size, modified, sha] for path, size, modified, sha in rows},
|
|
"suffixes": sorted(TEXT | EXTRACTED)}
|
|
|
|
|
|
def put(session: Session, principal: Principal, settings: Settings, folder_id: int, path: str, modified: int,
|
|
stream: BinaryIO) -> DocFile:
|
|
"""Eine Datei hinzufügen oder ersetzen. Gleicher Inhalt (Prüfsumme) wird nicht erneut ausgelesen."""
|
|
from app.uploads import service as uploads
|
|
found = folder(session, principal, folder_id, write=True)
|
|
path = check_path(path)
|
|
if not allowed(path):
|
|
raise Invalid(f"{path}: Format wird nicht abgeglichen.")
|
|
limit = settings.doc_max_mb * 1024 * 1024
|
|
digest, size, data = hashlib.sha256(), 0, bytearray()
|
|
while chunk := stream.read(CHUNK):
|
|
size += len(chunk)
|
|
if size > limit:
|
|
raise Invalid(f"{path}: größer als {settings.doc_max_mb} MB.")
|
|
digest.update(chunk)
|
|
data += chunk
|
|
sha = digest.hexdigest()
|
|
file = session.scalar(select(DocFile).where(DocFile.folder_id == found.id, DocFile.path == path)
|
|
.with_for_update())
|
|
if file is not None and file.sha256 == sha and file.status != ERROR:
|
|
file.modified, file.size = int(modified), size
|
|
return file
|
|
if file is None:
|
|
count = session.scalar(select(func.count()).select_from(DocFile).where(DocFile.folder_id == found.id)) or 0
|
|
if count >= settings.doc_max_files:
|
|
raise Invalid(f"Höchstens {settings.doc_max_files} Dateien je Ordner.")
|
|
workspace = workspaces.get(session, principal, found.workspace_id).workspace
|
|
if uploads.storage(session, settings, workspace).free <= 0:
|
|
raise Invalid(uploads.QUOTA_FULL)
|
|
file = DocFile(folder_id=found.id, path=path, size=size, modified=int(modified), sha256=sha)
|
|
session.add(file)
|
|
file.size, file.modified, file.sha256, file.error, file.updated = size, int(modified), sha, "", now()
|
|
if suffix(path) in TEXT:
|
|
file.text, file.status = cut(bytes(data).decode("utf-8", "replace"), settings), OK
|
|
else:
|
|
file.text, file.status = "", PENDING
|
|
session.flush()
|
|
target = pending_path(settings, file)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(bytes(data))
|
|
return file
|
|
|
|
|
|
def prune(session: Session, principal: Principal, settings: Settings, folder_id: int, paths: list[str]) -> int:
|
|
"""Abgleich abschließen: Dateien, die es im Ordner nicht mehr gibt, entfernen."""
|
|
found = folder(session, principal, folder_id, write=True)
|
|
keep = {check_path(p) for p in paths if p}
|
|
removed = 0
|
|
for file in session.scalars(select(DocFile).where(DocFile.folder_id == found.id)):
|
|
if file.path not in keep:
|
|
pending_path(settings, file).unlink(missing_ok=True)
|
|
session.delete(file)
|
|
removed += 1
|
|
found.synced = now()
|
|
audit.record(session, "documents.synced", user_id=principal.user_id, workspace_id=found.workspace_id,
|
|
target_kind="doc_folder", target_id=found.id, files=len(keep), removed=removed)
|
|
return removed
|
|
|
|
|
|
def cut(text: str, settings: Settings) -> str:
|
|
limit = settings.doc_max_text_kb * 1024
|
|
text = text.replace("\0", "")
|
|
return text if len(text.encode()) <= limit else text.encode()[:limit].decode("utf-8", "ignore")
|
|
|
|
|
|
# --- Auslese (Worker) ---
|
|
|
|
def extractor(session: Session):
|
|
"""Aktiver zentraler MCP-Server mit der Rolle „documents“ und festgelegtem Tool, sonst None."""
|
|
from app.mcp.models import McpServer
|
|
from app.mcp.service import DOCUMENTS
|
|
server = session.scalar(select(McpServer).where(McpServer.role == DOCUMENTS, McpServer.active.is_(True),
|
|
McpServer.workspace_id.is_(None)).order_by(McpServer.id))
|
|
return server if server is not None and (server.config or {}).get("tool") else None
|
|
|
|
|
|
def extract_pending(db: Database, settings: Settings, limit: int = BATCH) -> int:
|
|
"""Wartende Dateien über den MCP-Server auslesen. Ohne Server bleiben sie mit Hinweis liegen."""
|
|
from app.mcp import client
|
|
from app.mcp import service as mcp
|
|
done = 0
|
|
vault = Vault(settings.secret_key)
|
|
for _ in range(limit):
|
|
with db.session() as session:
|
|
file = session.scalar(select(DocFile).where(DocFile.status == PENDING).order_by(DocFile.id)
|
|
.limit(1).with_for_update(skip_locked=True))
|
|
if file is None:
|
|
return done
|
|
raw = pending_path(settings, file)
|
|
server = extractor(session)
|
|
if server is None:
|
|
file.status, file.error = UNSUPPORTED, ("Keine Dokumenten-Auslese eingerichtet (Plattform-Admin: "
|
|
"MCP-Server mit der Art „Dokumenten-Auslese“).")
|
|
raw.unlink(missing_ok=True)
|
|
continue
|
|
if not raw.is_file():
|
|
file.status, file.error = ERROR, "Original fehlt; bitte erneut abgleichen."
|
|
continue
|
|
config = dict(server.config)
|
|
target = dataclasses.replace(mcp.target(server, vault, settings), timeout=settings.doc_extract_timeout_seconds)
|
|
arguments = {config.get("content") or "content": base64.b64encode(raw.read_bytes()).decode()}
|
|
if config.get("filename"):
|
|
arguments[config["filename"]] = PurePosixPath(file.path).name
|
|
file_id, tool = file.id, config["tool"]
|
|
# Aufruf ohne offene Transaktion: die Auslese kann dauern
|
|
try:
|
|
text, error = client.call(target, tool, arguments), ""
|
|
except ToolError as e:
|
|
text, error = "", str(e)[:500]
|
|
except Exception as e:
|
|
text, error = "", f"Dokumenten-Auslese nicht erreichbar ({client.describe(e)})."
|
|
with db.session() as session:
|
|
file = session.get(DocFile, file_id)
|
|
if file is None or file.status != PENDING:
|
|
continue # inzwischen ersetzt oder gelöscht
|
|
if error:
|
|
file.status, file.error = ERROR, error
|
|
log.warning("Auslese fehlgeschlagen", extra={"event": "doc_extract_failed", "file_id": file_id})
|
|
else:
|
|
file.status, file.text, file.error = OK, cut(text, settings), ""
|
|
file.updated = now()
|
|
pending_path(settings, file).unlink(missing_ok=True)
|
|
done += 1
|
|
return done
|