148 lines
7.7 KiB
Python
148 lines
7.7 KiB
Python
"""Lange Chats: Verlauf nach Budget, Arbeitsnotiz, einstellbare Tool-Schritte und „Weiterarbeiten“."""
|
|
|
|
import re
|
|
import uuid
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from app.chat import context as chat_context
|
|
from app.chat.models import ChatSession, Message
|
|
from app.chat.runner import LAST_ROUND
|
|
from app.identity import Principal
|
|
from app.tasks.worker import Worker
|
|
from app.tools import service as tools
|
|
from app.workspaces.models import Workspace
|
|
from helpers import connect_repo, create_workspace
|
|
|
|
|
|
@pytest.fixture
|
|
def setup(browser, people, db, git_repo):
|
|
anna = browser("anna")
|
|
workspace_id = create_workspace(anna, people["team_demo"])
|
|
connect_repo(anna, workspace_id, git_repo)
|
|
anna.post(f"/workspaces/{workspace_id}/members", {"username": "ben", "role": "member"})
|
|
admin = Principal(user_id=people["anna"], username="anna", display_name="Anna", is_admin=True,
|
|
team_ids=frozenset({people["team_demo"]}))
|
|
with db.session() as session:
|
|
tools.classify_suggested(session, admin)
|
|
chat_id = int(anna.post(f"/workspaces/{workspace_id}/chats").headers["location"].rsplit("/", 1)[1])
|
|
return {"anna": anna, "ben": browser("ben"), "chat_id": chat_id, "workspace_id": workspace_id}
|
|
|
|
|
|
def ask(b, chat_id: int, text: str = "Frage") -> uuid.UUID:
|
|
response = b.post(f"/chats/{chat_id}/messages", {"content": text}, headers={"HX-Request": "true"})
|
|
return uuid.UUID(re.search(r"/runs/([0-9a-f-]+)/events", response.text).group(1))
|
|
|
|
|
|
def message(role: str, content: str, items=None, id_: int = 0) -> Message:
|
|
return Message(id=id_, role=role, content=content, items=items)
|
|
|
|
|
|
def test_verlauf_nach_budget_und_gekuerzte_tool_ausgaben():
|
|
big = "x" * 5000
|
|
|
|
def answer(i):
|
|
return message("assistant", f"Antwort {i}", [
|
|
{"type": "function_call", "call_id": f"c{i}", "name": "git_read_file", "arguments": "{}"},
|
|
{"type": "function_call_output", "call_id": f"c{i}", "output": big},
|
|
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": f"Antwort {i}"}]}],
|
|
id_=2 * i + 1)
|
|
messages = [m for i in range(4) for m in (message("user", f"Frage {i}", id_=2 * i), answer(i))]
|
|
messages.append(message("user", "Neue Frage", id_=99))
|
|
plan = chat_context.plan(messages, budget=100_000)
|
|
assert not plan.dropped
|
|
outputs = [i["output"] for i in plan.input if i.get("type") == "function_call_output"]
|
|
# Die beiden letzten Antworten vollständig, ältere gekürzt; Aufruf und Ausgabe bleiben paarweise
|
|
assert [len(o) == 5000 for o in outputs] == [False, False, True, True]
|
|
assert "in älterer Antwort gekürzt, 5000 Zeichen" in outputs[0]
|
|
assert sum(i.get("type") == "function_call" for i in plan.input) == 4
|
|
# Knappes Budget: Älteste fallen heraus, die neue Frage ist immer dabei
|
|
small = chat_context.plan(messages, budget=4000)
|
|
assert small.dropped and small.dropped[0].id == 0 and small.input[-1] == {"role": "user", "content": "Neue Frage"}
|
|
assert small.tokens <= 4000
|
|
assert chat_context.plan([message("user", big * 10)], budget=10).input # zu groß, trotzdem gesendet
|
|
|
|
|
|
def test_arbeitsnotiz_fasst_herausfallende_nachrichten_zusammen(setup, db, settings, fake_provider):
|
|
tight = settings.model_copy(update={"history_max_tokens": 60})
|
|
for i in range(3):
|
|
fake_provider.script = [f"Antwort {i} " + "lang " * 30]
|
|
ask(setup["anna"], setup["chat_id"], f"Frage {i} " + "Details " * 20)
|
|
Worker(db, tight).run_once()
|
|
summaries = [c for c in fake_provider.calls if c["instructions"] == chat_context.SUMMARY_PROMPT]
|
|
assert summaries and "Frage 0" in summaries[0]["input"][0]["content"]
|
|
with db.session() as session:
|
|
chat = session.get(ChatSession, setup["chat_id"])
|
|
assert chat.summary and chat.summary_upto
|
|
upto = chat.summary_upto
|
|
last = fake_provider.calls[-1]
|
|
assert "Arbeitsnotiz zum früheren Verlauf" in last["instructions"]
|
|
assert not any("Frage 0" in str(i.get("content", "")) for i in last["input"])
|
|
# Bereits zusammengefasste Nachrichten werden nicht erneut zusammengefasst
|
|
assert all("Frage 0 " not in c["input"][0]["content"] for c in summaries[1:])
|
|
page = setup["anna"].get(f"/chats/{setup['chat_id']}").text
|
|
assert "mit Arbeitsnotiz" in page and "Arbeitsnotiz der KI" in page
|
|
assert upto > 0
|
|
|
|
|
|
def test_arbeitsnotiz_scheitert_ohne_den_run_abzubrechen(setup, db, settings, fake_provider, monkeypatch):
|
|
from app.ai.provider import ProviderError
|
|
tight = settings.model_copy(update={"history_max_tokens": 60})
|
|
fake_provider.script = ["Erste " + "lang " * 30]
|
|
ask(setup["anna"], setup["chat_id"], "Frage 0 " + "Details " * 20)
|
|
Worker(db, tight).run_once()
|
|
original = fake_provider.respond
|
|
|
|
def respond(self, target, *, instructions, **kw):
|
|
if instructions == chat_context.SUMMARY_PROMPT:
|
|
raise ProviderError("Gateway weg", retryable=True, user_message="weg")
|
|
return original(self, target, instructions=instructions, **kw)
|
|
monkeypatch.setattr(fake_provider, "respond", respond)
|
|
fake_provider.script = ["Trotzdem beantwortet."]
|
|
ask(setup["anna"], setup["chat_id"], "Frage 1")
|
|
Worker(db, tight).run_once()
|
|
with db.session() as session:
|
|
assert session.scalar(select(Message.content).order_by(Message.id.desc())) == "Trotzdem beantwortet."
|
|
assert session.get(ChatSession, setup["chat_id"]).summary == ""
|
|
|
|
|
|
def test_tool_schritte_einstellbar_und_weiterarbeiten(setup, db, settings, fake_provider):
|
|
url = f"/workspaces/{setup['workspace_id']}/tool-rounds"
|
|
assert setup["ben"].post(url, {"value": "20"}).status_code == 403
|
|
for bad in ("3", "41", "zehn"):
|
|
assert setup["anna"].post(url, {"value": bad}).status_code == 400, bad
|
|
assert setup["anna"].post(url, {"value": "4"}).status_code == 303
|
|
assert "Aktuell 4." in setup["anna"].get(f"/workspaces/{setup['workspace_id']}/settings?tab=general").text
|
|
read = [("git_read_file", {"path": "README.md"})]
|
|
fake_provider.script = [read] * 10
|
|
fake_provider.usage = {"input_tokens": 1000, "output_tokens": 200, "total_tokens": 1200}
|
|
ask(setup["anna"], setup["chat_id"], "Großer Auftrag")
|
|
Worker(db, settings).run_once()
|
|
calls = fake_provider.calls
|
|
assert len(calls) == 5 and calls[-1]["tools"] is None and LAST_ROUND in calls[-1]["instructions"]
|
|
with db.session() as session:
|
|
reply = session.scalar(select(Message).where(Message.role == "assistant"))
|
|
assert reply.usage["round_limit"] is True and reply.usage["context_window"] == 128000
|
|
page = setup["anna"].get(f"/chats/{setup['chat_id']}").text
|
|
assert "Limit an Tool-Schritten" in page and "Weiterarbeiten" in page and "Kontext " in page
|
|
assert "Bisher 6.000 Tokens" in page
|
|
fake_provider.script = ["Fertig."]
|
|
response = setup["anna"].post(f"/chats/{setup['chat_id']}/messages",
|
|
{"content": "Bitte mach dort weiter, wo du aufgehört hast."},
|
|
headers={"HX-Request": "true"})
|
|
assert response.status_code == 200
|
|
Worker(db, settings).run_once()
|
|
page = setup["anna"].get(f"/chats/{setup['chat_id']}").text
|
|
assert page.count("Weiterarbeiten</button>") == 0 # nur an der letzten Antwort
|
|
# Zurück auf den Standard
|
|
setup["anna"].post(url, {"value": ""})
|
|
with db.session() as session:
|
|
assert "max_tool_rounds" not in (session.get(Workspace, setup["workspace_id"]).settings or {})
|
|
|
|
|
|
def test_limit_der_plattform_gilt_auch_fuer_gespeicherte_werte(settings):
|
|
workspace = Workspace(settings={"max_tool_rounds": 99})
|
|
from app.workspaces import service as workspaces
|
|
assert workspaces.tool_rounds(workspace, settings) == settings.max_tool_rounds_limit
|
|
assert workspaces.tool_rounds(Workspace(settings={}), settings) == settings.max_tool_rounds
|