"""Rückmeldungen zu Antworten, Beispielfragen und der Demo-Workspace (aiw demo).""" import subprocess import pytest from sqlalchemy import create_engine, func, select, text from sqlalchemy.engine import make_url from sqlalchemy.exc import ProgrammingError from app import demo from app.audit.models import AuditEvent from app.chat.models import ChatSession, Message from app.data_sources.models import DataSource from app.data_sources.service import chat_ready from app.demo import shop_db from app.documents.models import DocFile from app.feedback.models import AnswerFeedback from app.identity.models import User from app.tasks.worker import Worker from app.uploads.models import Upload from app.workspaces.models import Workspace, WorkspaceMembership from helpers import connect_repo, create_workspace @pytest.fixture def answered(browser, people, db, settings, git_repo, fake_provider): """Workspace mit Repo, Chat von anna mit einer Antwort; ben Mitglied, dora fremd.""" with db.session() as session: session.get(User, people["anna"]).is_admin = True anna = browser("anna") workspace_id = create_workspace(anna, people["team_demo"]) anna.post(f"/workspaces/{workspace_id}/members", {"username": "ben", "role": "member"}) connect_repo(anna, workspace_id, git_repo) chat_id = int(anna.post(f"/workspaces/{workspace_id}/chats").headers["location"].rsplit("/", 1)[1]) fake_provider.script = ["Die Antwort."] anna.post(f"/chats/{chat_id}/messages", {"content": "Wie ist das Projekt aufgebaut?"}, headers={"HX-Request": "true"}) Worker(db, settings).run_once() with db.session() as session: answer = session.scalar(select(Message.id).where(Message.role == "assistant")) question = session.scalar(select(Message.id).where(Message.role == "user")) return {"anna": anna, "ben": browser("ben"), "dora": browser("dora"), "workspace_id": workspace_id, "chat_id": chat_id, "answer": answer, "question": question} def feedback_of(db, message_id): with db.session() as session: return {(f.user_id, f.rating, f.comment) for f in session.scalars( select(AnswerFeedback).where(AnswerFeedback.message_id == message_id))} def test_rueckmeldung_geben_aendern_zuruecknehmen(answered, db, people): anna, ben, answer = answered["anna"], answered["ben"], answered["answer"] page = anna.get(f"/chats/{answered['chat_id']}").text assert f'id="feedback-{answer}"' in page and "Hilfreich?" in page part = ben.post(f"/messages/{answer}/feedback", {"rating": "-1"}).text assert 'aria-pressed="true"' in part and "Was hat gefehlt" in part part = ben.post(f"/messages/{answer}/feedback", {"rating": "-1", "comment": " Quelle fehlte "}).text assert "Danke für die Rückmeldung" in part and "„Quelle fehlte“" in part anna.post(f"/messages/{answer}/feedback", {"rating": "1"}) assert feedback_of(db, answer) == {(people["ben"], -1, "Quelle fehlte"), (people["anna"], 1, "")} # Umentscheiden behält den Kommentar; nochmal klicken nimmt zurück ben.post(f"/messages/{answer}/feedback", {"rating": "1"}) assert (people["ben"], 1, "Quelle fehlte") in feedback_of(db, answer) assert "aiw-feedback-on" in ben.get(f"/chats/{answered['chat_id']}").text ben.post(f"/messages/{answer}/feedback", {"rating": "0"}) assert feedback_of(db, answer) == {(people["anna"], 1, "")} # Nur Antworten, nur Mitglieder, nur gültige Werte assert ben.post(f"/messages/{answered['question']}/feedback", {"rating": "1"}).status_code == 404 assert answered["dora"].post(f"/messages/{answer}/feedback", {"rating": "1"}).status_code == 404 assert ben.post(f"/messages/{answer}/feedback", {"rating": "5"}).status_code == 400 assert ben.post(f"/messages/{answer}/feedback", {"rating": "1", "_csrf": ""}).status_code == 403 def test_auswertung_und_csv(answered, db): anna, ben, answer = answered["anna"], answered["ben"], answered["answer"] ben.post(f"/messages/{answer}/feedback", {"rating": "-1", "comment": "Zu allgemein"}) anna.post(f"/messages/{answer}/feedback", {"rating": "1"}) page = anna.get(f"/workspaces/{answered['workspace_id']}/usage").text assert "Rückmeldungen zu Antworten" in page and "50 %" in page and "Zu allgemein" in page assert f"/chats/{answered['chat_id']}#message-{answer}" in page csv = anna.get(f"/workspaces/{answered['workspace_id']}/usage/feedback.csv").text assert "Bewertung;Kommentar" in csv and "nicht hilfreich;Zu allgemein;Ben" in csv and "hilfreich;;Anna" in csv assert "Rückmeldungen zu Antworten" in anna.get("/admin/usage").text # Mitglieder sehen die Auswertung nicht assert ben.get(f"/workspaces/{answered['workspace_id']}/usage/feedback.csv").status_code == 403 def test_beispielfragen(answered, db, settings, fake_provider): anna, ben, workspace_id = answered["anna"], answered["ben"], answered["workspace_id"] too_many = "\n".join(f"Frage {n}" for n in range(9)) assert anna.post(f"/workspaces/{workspace_id}/examples", {"examples": too_many}).status_code == 400 assert ben.post(f"/workspaces/{workspace_id}/examples", {"examples": "x"}).status_code == 403 anna.post(f"/workspaces/{workspace_id}/examples", {"examples": "Was ist neu?\n\n Wer committet am meisten? \n"}) with db.session() as session: assert session.get(Workspace, workspace_id).settings["examples"] == ["Was ist neu?", "Wer committet am meisten?"] assert "Was ist neu?\nWer committet am meisten?" in anna.get(f"/workspaces/{workspace_id}/settings").text page = ben.get(f"/workspaces/{workspace_id}").text assert "Ausprobieren" in page and 'name="question" value="Was ist neu?"' in page # Klick in der Übersicht: neuer Chat, Frage gestellt response = ben.post(f"/workspaces/{workspace_id}/chats", {"question": "Was ist neu?"}) chat_id = int(response.headers["location"].rsplit("/", 1)[1]) with db.session() as session: assert session.get(ChatSession, chat_id).title == "Was ist neu?" assert session.scalar(select(Message.content).where(Message.chat_session_id == chat_id)) == "Was ist neu?" # Leerer Chat zeigt die Fragen; Klick stellt sie empty = int(ben.post(f"/workspaces/{workspace_id}/chats").headers["location"].rsplit("/", 1)[1]) page = ben.get(f"/chats/{empty}").text assert "Zum Beispiel:" in page and f'action="/chats/{empty}/ask"' in page assert ben.post(f"/chats/{empty}/ask", {"content": "Wer committet am meisten?"}).status_code == 303 assert "Zum Beispiel:" not in ben.get(f"/chats/{empty}").text # Leeren entfernt sie anna.post(f"/workspaces/{workspace_id}/examples", {"examples": ""}) assert "Ausprobieren" not in ben.get(f"/workspaces/{workspace_id}").text def test_demo_workspaces(db, people, settings, capsys, monkeypatch): # Ohne DB-Dienst: die Beispieldatenbank wird mit Hinweis übersprungen (eigener Test unten) monkeypatch.setattr(shop_db, "reachable", lambda: False) demos = demo.all_demos() assert demo.run(settings) == 0 first = {} with db.session() as session: # Nutzer heißen nach ihrer Rolle; freigabe ist Freigeber in jeder Demo admin = session.scalar(select(User).where(User.username == "admin")) assert admin.is_admin and admin.display_name == "Admin" for d in demos.values(): workspace = session.scalar(select(Workspace).where(Workspace.name == d.name)) assert workspace.settings["examples"] == d.examples roles = dict(session.execute(select(User.username, WorkspaceMembership.role).join( WorkspaceMembership, WorkspaceMembership.user_id == User.id).where( WorkspaceMembership.workspace_id == workspace.id)).all()) assert roles == {"admin": "admin", "entwicklung": "member", "fachbereich": "member", "freigabe": "approver"} first[d.key] = workspace.id if d.key == "chat": # Nur Chat: keine Quelle, keine Datei, kein Ordner, trotzdem bereit assert workspace.settings["chat_only"] is True assert session.scalar(select(DataSource).where(DataSource.workspace_id == workspace.id)) is None assert session.scalar(select(Upload).where(Upload.workspace_id == workspace.id)) is None assert chat_ready(session, workspace.id) continue assert "chat_only" not in workspace.settings source = session.scalar(select(DataSource).where(DataSource.workspace_id == workspace.id)) assert source.status == "ok" and source.name == d.repo upload = session.scalar(select(Upload).where(Upload.workspace_id == workspace.id)) assert upload.status == "ok" and upload.filename == d.upload[0] assert len(session.scalars(select(DocFile)).all()) == sum(len(d.documents or {}) for d in demos.values()) out = capsys.readouterr().out assert "Datenbank übersprungen: kein DB-Dienst" in out assert out.count("angelegt (ID") == 3 and out.count("verbunden") == 2 and "1200 Zeilen" in out assert "Keine Datenquellen: Chat ohne Kontext eingeschaltet" in out # Zweiter Lauf lässt alles, wie es ist; --reset legt nur die gewählte Demo neu an assert demo.run(settings) == 0 and capsys.readouterr().out.count("existiert schon") == 3 assert demo.run(settings, ["mcp"], reset=True) == 0 with db.session() as session: ids = {d.key: session.scalar(select(Workspace.id).where(Workspace.name == d.name)) for d in demos.values()} assert ids["bestellservice"] == first["bestellservice"] and ids["mcp"] != first["mcp"] # Repositories mit Verlauf und Feature-Branch, Autoren nach Rolle; Daten reproduzierbar for d in (d for d in demos.values() if d.repo): repo = demo.repo_path(settings, d) log = subprocess.run(["git", "log", "--format=%s", "main"], capture_output=True, text=True, cwd=repo) assert log.stdout.splitlines()[0] == d.commits[-1][2] and len(log.stdout.splitlines()) == len(d.commits) branches = subprocess.run(["git", "branch", "--format=%(refname:short)"], capture_output=True, text=True, cwd=repo).stdout.split() assert d.branch[0] in branches authors = subprocess.run(["git", "log", "--format=%an", "--all"], capture_output=True, text=True, cwd=repo).stdout.split() assert set(authors) <= {"Admin", "Entwicklung", "Fachbereich"} assert d.upload[1]() == d.upload[1]() # Der eingebaute Fehler der MCP-Demo steht im Code, die Korrektur nur im Protokoll server = demos["mcp"].commits[-2][3]["ticket_mcp/server.py"] assert 'title="Ticket schließen", readOnlyHint=True' in server and "limit: int = 20" in server def test_chat_ohne_datenquellen(answered, db, browser, people, fake_provider, settings): anna, ben = answered["anna"], answered["ben"] workspace_id = create_workspace(anna, people["team_demo"], name="Nur Chat") anna.post(f"/workspaces/{workspace_id}/members", {"username": "ben", "role": "member"}) # Ohne Quelle kein Chat (Context first) … assert "Erste Schritte" in ben.get(f"/workspaces/{workspace_id}").text assert anna.post(f"/workspaces/{workspace_id}/chats").status_code == 400 # … außer der Admin schaltet es ausdrücklich ein assert ben.post(f"/workspaces/{workspace_id}/chat-only", {"enabled": "1"}).status_code == 403 anna.post(f"/workspaces/{workspace_id}/chat-only", {"enabled": "1"}) assert 'name="enabled" value="1" checked' in anna.get(f"/workspaces/{workspace_id}/settings").text page = ben.get(f"/workspaces/{workspace_id}").text assert "Nur Chat, keine Datenquellen" in page and "Erste Schritte" not in page chat_id = int(ben.post(f"/workspaces/{workspace_id}/chats").headers["location"].rsplit("/", 1)[1]) fake_provider.script = ["Allgemeine Antwort."] ben.post(f"/chats/{chat_id}/messages", {"content": "Was ist ein MCP-Server?"}, headers={"HX-Request": "true"}) Worker(db, settings).run_once() prompt = fake_provider.calls[-1]["instructions"] assert "keine Daten angebunden" in prompt # Wieder aus: kein neuer Chat; im Audit-Log steht beides anna.post(f"/workspaces/{workspace_id}/chat-only", {}) assert anna.post(f"/workspaces/{workspace_id}/chats").status_code == 400 with db.session() as session: assert session.scalar(select(func.count()).select_from(AuditEvent).where( AuditEvent.action == "workspace.chat_only_set")) == 2 # Workspaces mit Daten bekommen den Hinweis nicht chat = answered["chat_id"] fake_provider.script = ["Antwort."] anna.post(f"/chats/{chat}/messages", {"content": "Und jetzt?"}, headers={"HX-Request": "true"}) Worker(db, settings).run_once() assert "keine Daten angebunden" not in fake_provider.calls[-1]["instructions"] def test_beispieldatenbank(settings): name = shop_db.build(settings) url = make_url(settings.database_url) try: assert shop_db.build(settings) == name # erneut: neu befüllt, nicht doppelt reader = create_engine(url.set(database=name, username=shop_db.READER, password=shop_db.READER_PASSWORD)) with reader.connect() as c: counts = {t: c.scalar(text(f"SELECT count(*) FROM {t}")) for t in shop_db.rows()} assert counts == {t: len(v) for t, v in shop_db.rows().items()} assert counts["kunden"] == 300 and counts["retouren"] > 100 with pytest.raises(ProgrammingError): c.execute(text("DELETE FROM retouren")) reader.dispose() assert shop_db.rows() == shop_db.rows() finally: admin = create_engine(url.set(database="postgres"), isolation_level="AUTOCOMMIT") with admin.connect() as c: c.execute(text(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)')) admin.dispose()