159 lines
8.2 KiB
Python
159 lines
8.2 KiB
Python
"""Datenquellen anlegen, anzeigen und testen (→ Konzept 4.2, 5.5)."""
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app import audit
|
|
from app.core.config import Settings
|
|
from app.core.crypto import Vault
|
|
from app.core.database import now
|
|
from app.core.errors import Forbidden, Invalid, NotFound
|
|
from app.data_sources.connectors import TestResult
|
|
from app.data_sources.connectors.git import GitConnector, check_config, check_username
|
|
from app.data_sources.connectors.pull_requests import BITBUCKET_URL, PROVIDERS
|
|
from app.data_sources.models import DataSource
|
|
from app.identity import Principal
|
|
from app.uploads.service import has_tables
|
|
from app.workspaces import service as workspaces
|
|
|
|
|
|
def list_for(session: Session, principal: Principal, workspace_id: int) -> list[DataSource]:
|
|
workspaces.get(session, principal, workspace_id)
|
|
return list(session.scalars(select(DataSource).where(DataSource.workspace_id == workspace_id)
|
|
.order_by(DataSource.name)))
|
|
|
|
|
|
def get(session: Session, principal: Principal, source_id: int) -> DataSource:
|
|
source = session.get(DataSource, source_id)
|
|
if source is None:
|
|
raise NotFound
|
|
workspaces.get(session, principal, source.workspace_id) # NotFound für Nichtmitglieder
|
|
return source
|
|
|
|
|
|
def check_pr_provider(kind: str, url: str) -> str:
|
|
if kind not in PROVIDERS:
|
|
raise Invalid("Unbekannter Pull-Request-Anbieter.")
|
|
if kind == "bitbucket_dc" and not BITBUCKET_URL.match(url):
|
|
raise Invalid("Für Bitbucket Data Center muss die URL die Form https://host/scm/PROJEKT/repo.git haben.")
|
|
return kind
|
|
|
|
|
|
def check_token(token: str) -> str:
|
|
token = token.strip()
|
|
if token and (len(token) < 8 or any(c.isspace() for c in token)):
|
|
raise Invalid("Das sieht nicht nach einem Token aus.")
|
|
return token
|
|
|
|
|
|
def create_git(session: Session, principal: Principal, settings: Settings, vault: Vault, workspace_id: int, *,
|
|
name: str, url: str, branch: str, token: str, exposure_confirmed: bool,
|
|
pr_provider: str = "none", delete_branches: bool = True, username: str = "") -> DataSource:
|
|
workspaces.require_admin(session, principal, workspace_id)
|
|
if not name.strip():
|
|
raise Invalid("Name fehlt.")
|
|
if not exposure_confirmed:
|
|
raise Invalid("Bitte bestätigen, dass alle Mitglieder des Workspace diese Daten sehen dürfen.")
|
|
config = check_config(url, branch, settings)
|
|
token, username = check_token(token), check_username(username)
|
|
if username and not token:
|
|
raise Invalid("Der Nutzername gilt nur zusammen mit einem Token.")
|
|
source = DataSource(workspace_id=workspace_id, kind="git", name=name.strip()[:200],
|
|
config={"url": config.url, "branch": config.branch,
|
|
"pr_provider": check_pr_provider(pr_provider, config.url),
|
|
"delete_branches": delete_branches, "username": username},
|
|
secret=vault.encrypt(token) if token else None, secret_hint=token[-4:] if token else "",
|
|
exposure_confirmed_by=principal.user_id, created_by=principal.user_id)
|
|
session.add(source)
|
|
session.flush()
|
|
audit.record(session, "source.created", user_id=principal.user_id, workspace_id=workspace_id,
|
|
target_kind="source", target_id=source.id, kind="git", url=config.url, branch=config.branch,
|
|
token=bool(token), auth="basic" if username else "bearer")
|
|
audit.record(session, "source.exposure_confirmed", user_id=principal.user_id, workspace_id=workspace_id,
|
|
target_kind="source", target_id=source.id)
|
|
return source
|
|
|
|
|
|
def update_git(session: Session, principal: Principal, settings: Settings, vault: Vault, source_id: int, *,
|
|
name: str, branch: str, token: str, remove_token: bool, pr_provider: str = "none",
|
|
delete_branches: bool = True, username: str = "") -> DataSource:
|
|
"""Name, Branch und Token ändern. Die URL bleibt: eine andere URL ist eine neue Quelle
|
|
mit neuer Bestätigung. Nach einer Änderung muss die Quelle neu getestet werden."""
|
|
source = require_testable(session, principal, source_id)
|
|
if source.kind != "git":
|
|
raise Invalid("Das ist keine Git-Quelle.")
|
|
if not name.strip():
|
|
raise Invalid("Name fehlt.")
|
|
config = check_config(source.config["url"], branch, settings)
|
|
token, username = check_token(token), check_username(username)
|
|
if username and not (token or (source.secret and not remove_token)):
|
|
raise Invalid("Der Nutzername gilt nur zusammen mit einem Token.")
|
|
changed = (config.branch != source.config["branch"] or bool(token) or (remove_token and source.secret)
|
|
or username != source.config.get("username", ""))
|
|
source.name = name.strip()[:200]
|
|
source.config = {**source.config, "branch": config.branch,
|
|
"pr_provider": check_pr_provider(pr_provider, source.config["url"]),
|
|
"delete_branches": delete_branches, "username": username}
|
|
if token:
|
|
source.secret, source.secret_hint = vault.encrypt(token), token[-4:]
|
|
elif remove_token:
|
|
source.secret, source.secret_hint = None, ""
|
|
if changed:
|
|
source.status, source.status_message = "new", "Geändert, bitte neu testen."
|
|
audit.record(session, "source.updated", user_id=principal.user_id, workspace_id=source.workspace_id,
|
|
target_kind="source", target_id=source.id, branch=config.branch,
|
|
token="ersetzt" if token else "entfernt" if remove_token else "unverändert",
|
|
auth="basic" if username else "bearer")
|
|
return source
|
|
|
|
|
|
def delete(session: Session, principal: Principal, source_id: int) -> int:
|
|
source = require_testable(session, principal, source_id)
|
|
workspace_id = source.workspace_id
|
|
audit.record(session, "source.deleted", user_id=principal.user_id, workspace_id=workspace_id,
|
|
target_kind="source", target_id=source.id, kind=source.kind,
|
|
target=source.config.get("url") or source.config.get("host", ""))
|
|
session.delete(source)
|
|
return workspace_id
|
|
|
|
|
|
def connector(source: DataSource, settings: Settings, vault: Vault) -> GitConnector:
|
|
token = vault.decrypt(source.secret) if source.secret else None
|
|
return GitConnector(check_config(source.config["url"], source.config["branch"], settings), settings, token,
|
|
source.config.get("username", ""))
|
|
|
|
|
|
def run_test(source: DataSource, settings: Settings, vault: Vault) -> TestResult:
|
|
"""Ohne offene Transaktion aufrufen: der Test kann bis zum Zeitlimit dauern."""
|
|
try:
|
|
return connector(source, settings, vault).test_connection()
|
|
except Invalid as e: # z. B. Host inzwischen nicht mehr freigegeben
|
|
return TestResult(False, str(e))
|
|
except RuntimeError as e: # Token lässt sich nicht entschlüsseln (anderer AIW_SECRET_KEY)
|
|
return TestResult(False, str(e))
|
|
|
|
|
|
def require_testable(session: Session, principal: Principal, source_id: int) -> DataSource:
|
|
source = get(session, principal, source_id)
|
|
if not workspaces.get(session, principal, source.workspace_id).is_admin:
|
|
raise Forbidden
|
|
return source
|
|
|
|
|
|
def save_test(session: Session, principal: Principal, source_id: int, result: TestResult) -> None:
|
|
source = require_testable(session, principal, source_id)
|
|
source.status = "ok" if result.ok else "error"
|
|
source.status_message = result.message
|
|
source.last_tested = now()
|
|
audit.record(session, "source.tested", user_id=principal.user_id, workspace_id=source.workspace_id,
|
|
target_kind="source", target_id=source.id, ok=result.ok, message=result.message)
|
|
|
|
|
|
def chat_ready(session: Session, workspace_id: int) -> bool:
|
|
"""Context first (→ Konzept 1): Chat erst mit mindestens einer erfolgreich getesteten Quelle,
|
|
einer erfolgreich eingelesenen Datei oder einem abgeglichenen Ordner."""
|
|
from app.documents.models import DocFolder
|
|
return session.scalar(select(DataSource.id).where(
|
|
DataSource.workspace_id == workspace_id, DataSource.status == "ok").limit(1)) is not None \
|
|
or has_tables(session, workspace_id) \
|
|
or session.scalar(select(DocFolder.id).where(DocFolder.workspace_id == workspace_id).limit(1)) is not None
|