174 lines
7.7 KiB
Python
174 lines
7.7 KiB
Python
"""Robustheit bei Ausfällen: Worker stirbt mitten im Run, Datenbank kurz weg, Git-Server antwortet
|
|
nicht, Uhren von Web, Worker und Datenbank weichen ab (→ Konzept 9.2, 9.3)."""
|
|
|
|
import sys
|
|
import threading
|
|
from datetime import timedelta
|
|
|
|
import pytest
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.exc import OperationalError
|
|
from test_branches import README, ask, branch, commit, fake_prs, git, hx, outputs, remote_branches, setup # noqa: F401
|
|
|
|
from app.changes.models import BranchCommit
|
|
from app.core import database
|
|
from app.data_sources.connectors.git_repo import GitError, GitRepo
|
|
from app.tasks import engine, states
|
|
from app.tasks.models import TaskRun
|
|
from app.tasks.worker import Worker
|
|
|
|
|
|
class Crash(BaseException):
|
|
"""Wie ein beendeter Prozess: nichts im Worker fängt das ab, der Run bleibt mit Lease liegen."""
|
|
|
|
|
|
def fail_on_call(fake_provider, monkeypatch, number: int, error: BaseException) -> None:
|
|
"""Den number-ten Modellaufruf dieses Tests scheitern lassen."""
|
|
original, seen = fake_provider.respond, []
|
|
|
|
def respond(self, *args, **kwargs):
|
|
seen.append(1)
|
|
if len(seen) == number:
|
|
raise error
|
|
return original(self, *args, **kwargs)
|
|
|
|
monkeypatch.setattr(fake_provider, "respond", respond)
|
|
|
|
|
|
def commits(db) -> list[BranchCommit]:
|
|
with db.session() as session:
|
|
return list(session.scalars(select(BranchCommit)))
|
|
|
|
|
|
def status(db, run_id) -> str:
|
|
with db.session() as session:
|
|
return session.get(TaskRun, run_id).status
|
|
|
|
|
|
def test_worker_stirbt_nach_commit_neuer_worker_setzt_fort(setup, db, settings, fake_provider, monkeypatch):
|
|
fake_provider.script = [[("commit_change", {"message": "README ergänzen", "changes": [README]})], "Erledigt."]
|
|
fail_on_call(fake_provider, monkeypatch, 2, Crash())
|
|
run_id = ask(setup["ben"], setup["chat_id"])
|
|
with pytest.raises(Crash):
|
|
Worker(db, settings, "worker-a").run_once()
|
|
assert status(db, run_id) == states.RUNNING and len(commits(db)) == 1
|
|
# Lease läuft ab, ein anderer Worker übernimmt
|
|
with db.session() as session:
|
|
session.execute(update(TaskRun).values(locked_until=database.now() - timedelta(seconds=1)))
|
|
assert Worker(db, settings, "worker-b").run_once()
|
|
assert status(db, run_id) == states.COMPLETED
|
|
# Fortgesetzt statt wiederholt: kein zweiter Commit, das Modell kennt das Ergebnis des ersten
|
|
assert len(commits(db)) == 1
|
|
assert any("auf Branch" in o for o in outputs(fake_provider, -1))
|
|
|
|
|
|
def test_gateway_kurz_weg_nach_commit(setup, db, settings, fake_provider, monkeypatch):
|
|
from app.ai.provider import ProviderError
|
|
fake_provider.script = [[("commit_change", {"message": "README ergänzen", "changes": [README]})], "Erledigt."]
|
|
fail_on_call(fake_provider, monkeypatch, 2, ProviderError("Timeout", user_message="Zeitüberschreitung",
|
|
retryable=True))
|
|
run_id = ask(setup["ben"], setup["chat_id"])
|
|
Worker(db, settings).run_once()
|
|
assert status(db, run_id) == states.QUEUED # neuer Versuch mit Wartezeit
|
|
with db.session() as session:
|
|
session.execute(update(TaskRun).values(available_at=database.now() - timedelta(seconds=1)))
|
|
Worker(db, settings).run_once()
|
|
assert status(db, run_id) == states.COMPLETED and len(commits(db)) == 1
|
|
assert any("auf Branch" in o for o in outputs(fake_provider, -1))
|
|
|
|
|
|
@pytest.mark.parametrize("offset", [3600, -3600])
|
|
def test_uhren_weichen_ab(setup, db, settings, fake_provider, monkeypatch, offset):
|
|
"""Web und Worker mit einer Stunde Abweichung zur Datenbank: Runs werden sofort bearbeitet,
|
|
laufende Leases gelten nicht als abgelaufen."""
|
|
real = database.now
|
|
skewed = lambda: real() + timedelta(seconds=offset)
|
|
for module in list(sys.modules.values()):
|
|
if getattr(module, "now", None) is real:
|
|
monkeypatch.setattr(module, "now", skewed)
|
|
run_id = ask(setup["ben"], setup["chat_id"], "Hallo")
|
|
with db.session() as session:
|
|
assert engine.claim(session, "worker-a", settings) is not None
|
|
assert engine.recover_expired(db, settings) == 0
|
|
with db.session() as session:
|
|
session.execute(update(TaskRun).values(status=states.QUEUED, locked_by=None, locked_until=None))
|
|
assert Worker(db, settings).run_once() and status(db, run_id) == states.COMPLETED
|
|
|
|
|
|
def test_worker_uebersteht_datenbankausfall(setup, db, settings, fake_provider, monkeypatch, caplog):
|
|
run_id = ask(setup["ben"], setup["chat_id"], "Hallo")
|
|
original, failures = engine.recover_expired, []
|
|
|
|
def flaky(*args, **kwargs):
|
|
if len(failures) < 2:
|
|
failures.append(1)
|
|
raise OperationalError("SELECT 1", {}, Exception("connection refused"))
|
|
return original(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(engine, "recover_expired", flaky)
|
|
monkeypatch.setattr(settings, "worker_poll_seconds", 0.01)
|
|
stop, worker = threading.Event(), Worker(db, settings)
|
|
monkeypatch.setattr("app.tasks.monitor.serve", lambda *a: None)
|
|
thread = threading.Thread(target=worker.run_forever, args=(stop,))
|
|
thread.start()
|
|
for _ in range(500):
|
|
if status(db, run_id) == states.COMPLETED:
|
|
break
|
|
threading.Event().wait(0.01)
|
|
stop.set()
|
|
thread.join(5)
|
|
assert status(db, run_id) == states.COMPLETED and len(failures) == 2
|
|
assert "Fehler in der Worker-Schleife" in caplog.text
|
|
|
|
|
|
def test_git_server_antwortet_nicht_beim_push(setup, db, settings, fake_provider, monkeypatch):
|
|
commit(setup, db, settings, fake_provider)
|
|
b = branch(db)
|
|
|
|
def silent(self, *args, **kwargs):
|
|
raise GitError("Keine Antwort vom Git-Server nach 30 Sekunden.")
|
|
|
|
monkeypatch.setattr(GitRepo, "remote", silent)
|
|
monkeypatch.setattr(GitRepo, "remote_head", silent)
|
|
monkeypatch.setattr(GitRepo, "sync", lambda self: None)
|
|
response = hx(setup["ben"], f"/branches/{b.id}/push")
|
|
assert response.status_code == 400 and "Keine Antwort vom Git-Server" in response.text
|
|
assert branch(db).pushed is None and remote_branches(setup["repo"]) == []
|
|
monkeypatch.undo()
|
|
assert hx(setup["ben"], f"/branches/{b.id}/push").status_code == 200
|
|
assert branch(db).pushed == b.head == git(setup["repo"], "rev-parse", b.name)
|
|
|
|
|
|
def test_push_angekommen_aber_antwort_verloren(setup, db, settings, fake_provider, monkeypatch):
|
|
commit(setup, db, settings, fake_provider)
|
|
b = branch(db)
|
|
original = GitRepo.push
|
|
|
|
def lost(self, *args, **kwargs):
|
|
original(self, *args, **kwargs)
|
|
raise GitError("Verbindung abgebrochen.")
|
|
|
|
monkeypatch.setattr(GitRepo, "push", lost)
|
|
response = hx(setup["ben"], f"/branches/{b.id}/push")
|
|
assert response.status_code == 200 and "Übertragen." in response.text
|
|
assert branch(db).pushed == b.head == git(setup["repo"], "rev-parse", b.name)
|
|
|
|
|
|
def test_pull_request_anlegen_nach_abbruch_ohne_doppelten(setup, db, settings, fake_provider, fake_prs,
|
|
monkeypatch):
|
|
"""Bitbucket legt den Pull Request an, die Antwort geht verloren: der nächste Versuch findet ihn."""
|
|
commit(setup, db, settings, fake_provider)
|
|
b = branch(db)
|
|
original = fake_prs.create
|
|
|
|
def lost(*args, **kwargs):
|
|
original(*args, **kwargs)
|
|
raise GitError("Bitbucket nicht erreichbar (ReadTimeout).")
|
|
|
|
monkeypatch.setattr(fake_prs, "create", lost)
|
|
response = hx(setup["ben"], f"/branches/{b.id}/pull-request", {"title": "README ergänzen"})
|
|
assert response.status_code == 400 and "nicht erreichbar" in response.text
|
|
assert remote_branches(setup["repo"]) == [b.name] # Push ist durch
|
|
monkeypatch.setattr(fake_prs, "create", original)
|
|
assert hx(setup["ben"], f"/branches/{b.id}/pull-request", {"title": "README ergänzen"}).status_code == 200
|
|
assert len(fake_prs.created) == 1 and branch(db).pr_id == "42"
|