227 lines
14 KiB
Python
227 lines
14 KiB
Python
"""Rückmeldungen zu Antworten, Beispielfragen und der Demo-Workspace (aiw demo)."""
|
|
|
|
import subprocess
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, func, select, text
|
|
from sqlalchemy.engine import make_url
|
|
from sqlalchemy.exc import ProgrammingError
|
|
|
|
from app import demo
|
|
from app.audit.models import AuditEvent
|
|
from app.chat.models import ChatSession, Message
|
|
from app.data_sources.models import DataSource
|
|
from app.data_sources.service import chat_ready
|
|
from app.demo import shop_db
|
|
from app.documents.models import DocFile
|
|
from app.feedback.models import AnswerFeedback
|
|
from app.identity.models import User
|
|
from app.tasks.worker import Worker
|
|
from app.uploads.models import Upload
|
|
from app.workspaces.models import Workspace, WorkspaceMembership
|
|
from helpers import connect_repo, create_workspace
|
|
|
|
|
|
@pytest.fixture
|
|
def answered(browser, people, db, settings, git_repo, fake_provider):
|
|
"""Workspace mit Repo, Chat von anna mit einer Antwort; ben Mitglied, dora fremd."""
|
|
with db.session() as session:
|
|
session.get(User, people["anna"]).is_admin = True
|
|
anna = browser("anna")
|
|
workspace_id = create_workspace(anna, people["team_demo"])
|
|
anna.post(f"/workspaces/{workspace_id}/members", {"username": "ben", "role": "member"})
|
|
connect_repo(anna, workspace_id, git_repo)
|
|
chat_id = int(anna.post(f"/workspaces/{workspace_id}/chats").headers["location"].rsplit("/", 1)[1])
|
|
fake_provider.script = ["Die Antwort."]
|
|
anna.post(f"/chats/{chat_id}/messages", {"content": "Wie ist das Projekt aufgebaut?"},
|
|
headers={"HX-Request": "true"})
|
|
Worker(db, settings).run_once()
|
|
with db.session() as session:
|
|
answer = session.scalar(select(Message.id).where(Message.role == "assistant"))
|
|
question = session.scalar(select(Message.id).where(Message.role == "user"))
|
|
return {"anna": anna, "ben": browser("ben"), "dora": browser("dora"), "workspace_id": workspace_id,
|
|
"chat_id": chat_id, "answer": answer, "question": question}
|
|
|
|
|
|
def feedback_of(db, message_id):
|
|
with db.session() as session:
|
|
return {(f.user_id, f.rating, f.comment) for f in session.scalars(
|
|
select(AnswerFeedback).where(AnswerFeedback.message_id == message_id))}
|
|
|
|
|
|
def test_rueckmeldung_geben_aendern_zuruecknehmen(answered, db, people):
|
|
anna, ben, answer = answered["anna"], answered["ben"], answered["answer"]
|
|
page = anna.get(f"/chats/{answered['chat_id']}").text
|
|
assert f'id="feedback-{answer}"' in page and "Hilfreich?" in page
|
|
part = ben.post(f"/messages/{answer}/feedback", {"rating": "-1"}).text
|
|
assert 'aria-pressed="true"' in part and "Was hat gefehlt" in part
|
|
part = ben.post(f"/messages/{answer}/feedback", {"rating": "-1", "comment": " Quelle fehlte "}).text
|
|
assert "Danke für die Rückmeldung" in part and "„Quelle fehlte“" in part
|
|
anna.post(f"/messages/{answer}/feedback", {"rating": "1"})
|
|
assert feedback_of(db, answer) == {(people["ben"], -1, "Quelle fehlte"), (people["anna"], 1, "")}
|
|
# Umentscheiden behält den Kommentar; nochmal klicken nimmt zurück
|
|
ben.post(f"/messages/{answer}/feedback", {"rating": "1"})
|
|
assert (people["ben"], 1, "Quelle fehlte") in feedback_of(db, answer)
|
|
assert "aiw-feedback-on" in ben.get(f"/chats/{answered['chat_id']}").text
|
|
ben.post(f"/messages/{answer}/feedback", {"rating": "0"})
|
|
assert feedback_of(db, answer) == {(people["anna"], 1, "")}
|
|
# Nur Antworten, nur Mitglieder, nur gültige Werte
|
|
assert ben.post(f"/messages/{answered['question']}/feedback", {"rating": "1"}).status_code == 404
|
|
assert answered["dora"].post(f"/messages/{answer}/feedback", {"rating": "1"}).status_code == 404
|
|
assert ben.post(f"/messages/{answer}/feedback", {"rating": "5"}).status_code == 400
|
|
assert ben.post(f"/messages/{answer}/feedback", {"rating": "1", "_csrf": ""}).status_code == 403
|
|
|
|
|
|
def test_auswertung_und_csv(answered, db):
|
|
anna, ben, answer = answered["anna"], answered["ben"], answered["answer"]
|
|
ben.post(f"/messages/{answer}/feedback", {"rating": "-1", "comment": "Zu allgemein"})
|
|
anna.post(f"/messages/{answer}/feedback", {"rating": "1"})
|
|
page = anna.get(f"/workspaces/{answered['workspace_id']}/usage").text
|
|
assert "Rückmeldungen zu Antworten" in page and "50 %" in page and "Zu allgemein" in page
|
|
assert f"/chats/{answered['chat_id']}#message-{answer}" in page
|
|
csv = anna.get(f"/workspaces/{answered['workspace_id']}/usage/feedback.csv").text
|
|
assert "Bewertung;Kommentar" in csv and "nicht hilfreich;Zu allgemein;Ben" in csv and "hilfreich;;Anna" in csv
|
|
assert "Rückmeldungen zu Antworten" in anna.get("/admin/usage").text
|
|
# Mitglieder sehen die Auswertung nicht
|
|
assert ben.get(f"/workspaces/{answered['workspace_id']}/usage/feedback.csv").status_code == 403
|
|
|
|
|
|
def test_beispielfragen(answered, db, settings, fake_provider):
|
|
anna, ben, workspace_id = answered["anna"], answered["ben"], answered["workspace_id"]
|
|
too_many = "\n".join(f"Frage {n}" for n in range(9))
|
|
assert anna.post(f"/workspaces/{workspace_id}/examples", {"examples": too_many}).status_code == 400
|
|
assert ben.post(f"/workspaces/{workspace_id}/examples", {"examples": "x"}).status_code == 403
|
|
anna.post(f"/workspaces/{workspace_id}/examples", {"examples": "Was ist neu?\n\n Wer committet am meisten? \n"})
|
|
with db.session() as session:
|
|
assert session.get(Workspace, workspace_id).settings["examples"] == ["Was ist neu?", "Wer committet am meisten?"]
|
|
assert "Was ist neu?\nWer committet am meisten?" in anna.get(f"/workspaces/{workspace_id}/settings").text
|
|
page = ben.get(f"/workspaces/{workspace_id}").text
|
|
assert "Ausprobieren" in page and 'name="question" value="Was ist neu?"' in page
|
|
# Klick in der Übersicht: neuer Chat, Frage gestellt
|
|
response = ben.post(f"/workspaces/{workspace_id}/chats", {"question": "Was ist neu?"})
|
|
chat_id = int(response.headers["location"].rsplit("/", 1)[1])
|
|
with db.session() as session:
|
|
assert session.get(ChatSession, chat_id).title == "Was ist neu?"
|
|
assert session.scalar(select(Message.content).where(Message.chat_session_id == chat_id)) == "Was ist neu?"
|
|
# Leerer Chat zeigt die Fragen; Klick stellt sie
|
|
empty = int(ben.post(f"/workspaces/{workspace_id}/chats").headers["location"].rsplit("/", 1)[1])
|
|
page = ben.get(f"/chats/{empty}").text
|
|
assert "Zum Beispiel:" in page and f'action="/chats/{empty}/ask"' in page
|
|
assert ben.post(f"/chats/{empty}/ask", {"content": "Wer committet am meisten?"}).status_code == 303
|
|
assert "Zum Beispiel:" not in ben.get(f"/chats/{empty}").text
|
|
# Leeren entfernt sie
|
|
anna.post(f"/workspaces/{workspace_id}/examples", {"examples": ""})
|
|
assert "Ausprobieren" not in ben.get(f"/workspaces/{workspace_id}").text
|
|
|
|
|
|
def test_demo_workspaces(db, people, settings, capsys, monkeypatch):
|
|
# Ohne DB-Dienst: die Beispieldatenbank wird mit Hinweis übersprungen (eigener Test unten)
|
|
monkeypatch.setattr(shop_db, "reachable", lambda: False)
|
|
demos = demo.all_demos()
|
|
assert demo.run(settings) == 0
|
|
first = {}
|
|
with db.session() as session:
|
|
# Nutzer heißen nach ihrer Rolle; freigabe ist Freigeber in jeder Demo
|
|
admin = session.scalar(select(User).where(User.username == "admin"))
|
|
assert admin.is_admin and admin.display_name == "Admin"
|
|
for d in demos.values():
|
|
workspace = session.scalar(select(Workspace).where(Workspace.name == d.name))
|
|
assert workspace.settings["examples"] == d.examples
|
|
roles = dict(session.execute(select(User.username, WorkspaceMembership.role).join(
|
|
WorkspaceMembership, WorkspaceMembership.user_id == User.id).where(
|
|
WorkspaceMembership.workspace_id == workspace.id)).all())
|
|
assert roles == {"admin": "admin", "entwicklung": "member", "fachbereich": "member",
|
|
"freigabe": "approver"}
|
|
first[d.key] = workspace.id
|
|
if d.key == "chat":
|
|
# Nur Chat: keine Quelle, keine Datei, kein Ordner, trotzdem bereit
|
|
assert workspace.settings["chat_only"] is True
|
|
assert session.scalar(select(DataSource).where(DataSource.workspace_id == workspace.id)) is None
|
|
assert session.scalar(select(Upload).where(Upload.workspace_id == workspace.id)) is None
|
|
assert chat_ready(session, workspace.id)
|
|
continue
|
|
assert "chat_only" not in workspace.settings
|
|
source = session.scalar(select(DataSource).where(DataSource.workspace_id == workspace.id))
|
|
assert source.status == "ok" and source.name == d.repo
|
|
upload = session.scalar(select(Upload).where(Upload.workspace_id == workspace.id))
|
|
assert upload.status == "ok" and upload.filename == d.upload[0]
|
|
assert len(session.scalars(select(DocFile)).all()) == sum(len(d.documents or {}) for d in demos.values())
|
|
out = capsys.readouterr().out
|
|
assert "Datenbank übersprungen: kein DB-Dienst" in out
|
|
assert out.count("angelegt (ID") == 3 and out.count("verbunden") == 2 and "1200 Zeilen" in out
|
|
assert "Keine Datenquellen: Chat ohne Kontext eingeschaltet" in out
|
|
# Zweiter Lauf lässt alles, wie es ist; --reset legt nur die gewählte Demo neu an
|
|
assert demo.run(settings) == 0 and capsys.readouterr().out.count("existiert schon") == 3
|
|
assert demo.run(settings, ["mcp"], reset=True) == 0
|
|
with db.session() as session:
|
|
ids = {d.key: session.scalar(select(Workspace.id).where(Workspace.name == d.name)) for d in demos.values()}
|
|
assert ids["bestellservice"] == first["bestellservice"] and ids["mcp"] != first["mcp"]
|
|
# Repositories mit Verlauf und Feature-Branch, Autoren nach Rolle; Daten reproduzierbar
|
|
for d in (d for d in demos.values() if d.repo):
|
|
repo = demo.repo_path(settings, d)
|
|
log = subprocess.run(["git", "log", "--format=%s", "main"], capture_output=True, text=True, cwd=repo)
|
|
assert log.stdout.splitlines()[0] == d.commits[-1][2] and len(log.stdout.splitlines()) == len(d.commits)
|
|
branches = subprocess.run(["git", "branch", "--format=%(refname:short)"], capture_output=True, text=True,
|
|
cwd=repo).stdout.split()
|
|
assert d.branch[0] in branches
|
|
authors = subprocess.run(["git", "log", "--format=%an", "--all"], capture_output=True, text=True,
|
|
cwd=repo).stdout.split()
|
|
assert set(authors) <= {"Admin", "Entwicklung", "Fachbereich"}
|
|
assert d.upload[1]() == d.upload[1]()
|
|
# Der eingebaute Fehler der MCP-Demo steht im Code, die Korrektur nur im Protokoll
|
|
server = demos["mcp"].commits[-2][3]["ticket_mcp/server.py"]
|
|
assert 'title="Ticket schließen", readOnlyHint=True' in server and "limit: int = 20" in server
|
|
|
|
|
|
def test_chat_ohne_datenquellen(answered, db, browser, people, fake_provider, settings):
|
|
anna, ben = answered["anna"], answered["ben"]
|
|
workspace_id = create_workspace(anna, people["team_demo"], name="Nur Chat")
|
|
anna.post(f"/workspaces/{workspace_id}/members", {"username": "ben", "role": "member"})
|
|
# Ohne Quelle kein Chat (Context first) …
|
|
assert "Erste Schritte" in ben.get(f"/workspaces/{workspace_id}").text
|
|
assert anna.post(f"/workspaces/{workspace_id}/chats").status_code == 400
|
|
# … außer der Admin schaltet es ausdrücklich ein
|
|
assert ben.post(f"/workspaces/{workspace_id}/chat-only", {"enabled": "1"}).status_code == 403
|
|
anna.post(f"/workspaces/{workspace_id}/chat-only", {"enabled": "1"})
|
|
assert 'name="enabled" value="1" checked' in anna.get(f"/workspaces/{workspace_id}/settings").text
|
|
page = ben.get(f"/workspaces/{workspace_id}").text
|
|
assert "Nur Chat, keine Datenquellen" in page and "Erste Schritte" not in page
|
|
chat_id = int(ben.post(f"/workspaces/{workspace_id}/chats").headers["location"].rsplit("/", 1)[1])
|
|
fake_provider.script = ["Allgemeine Antwort."]
|
|
ben.post(f"/chats/{chat_id}/messages", {"content": "Was ist ein MCP-Server?"}, headers={"HX-Request": "true"})
|
|
Worker(db, settings).run_once()
|
|
prompt = fake_provider.calls[-1]["instructions"]
|
|
assert "keine Daten angebunden" in prompt
|
|
# Wieder aus: kein neuer Chat; im Audit-Log steht beides
|
|
anna.post(f"/workspaces/{workspace_id}/chat-only", {})
|
|
assert anna.post(f"/workspaces/{workspace_id}/chats").status_code == 400
|
|
with db.session() as session:
|
|
assert session.scalar(select(func.count()).select_from(AuditEvent).where(
|
|
AuditEvent.action == "workspace.chat_only_set")) == 2
|
|
# Workspaces mit Daten bekommen den Hinweis nicht
|
|
chat = answered["chat_id"]
|
|
fake_provider.script = ["Antwort."]
|
|
anna.post(f"/chats/{chat}/messages", {"content": "Und jetzt?"}, headers={"HX-Request": "true"})
|
|
Worker(db, settings).run_once()
|
|
assert "keine Daten angebunden" not in fake_provider.calls[-1]["instructions"]
|
|
|
|
|
|
def test_beispieldatenbank(settings):
|
|
name = shop_db.build(settings)
|
|
url = make_url(settings.database_url)
|
|
try:
|
|
assert shop_db.build(settings) == name # erneut: neu befüllt, nicht doppelt
|
|
reader = create_engine(url.set(database=name, username=shop_db.READER, password=shop_db.READER_PASSWORD))
|
|
with reader.connect() as c:
|
|
counts = {t: c.scalar(text(f"SELECT count(*) FROM {t}")) for t in shop_db.rows()}
|
|
assert counts == {t: len(v) for t, v in shop_db.rows().items()}
|
|
assert counts["kunden"] == 300 and counts["retouren"] > 100
|
|
with pytest.raises(ProgrammingError):
|
|
c.execute(text("DELETE FROM retouren"))
|
|
reader.dispose()
|
|
assert shop_db.rows() == shop_db.rows()
|
|
finally:
|
|
admin = create_engine(url.set(database="postgres"), isolation_level="AUTOCOMMIT")
|
|
with admin.connect() as c:
|
|
c.execute(text(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)'))
|
|
admin.dispose()
|