"""Robustheit bei Ausfällen: Worker stirbt mitten im Run, Datenbank kurz weg, Git-Server antwortet nicht, Uhren von Web, Worker und Datenbank weichen ab (→ Konzept 9.2, 9.3).""" import sys import threading from datetime import timedelta import pytest from sqlalchemy import select, update from sqlalchemy.exc import OperationalError from test_branches import README, ask, branch, commit, fake_prs, git, hx, outputs, remote_branches, setup # noqa: F401 from app.changes.models import BranchCommit from app.core import database from app.data_sources.connectors.git_repo import GitError, GitRepo from app.tasks import engine, states from app.tasks.models import TaskRun from app.tasks.worker import Worker class Crash(BaseException): """Wie ein beendeter Prozess: nichts im Worker fängt das ab, der Run bleibt mit Lease liegen.""" def fail_on_call(fake_provider, monkeypatch, number: int, error: BaseException) -> None: """Den number-ten Modellaufruf dieses Tests scheitern lassen.""" original, seen = fake_provider.respond, [] def respond(self, *args, **kwargs): seen.append(1) if len(seen) == number: raise error return original(self, *args, **kwargs) monkeypatch.setattr(fake_provider, "respond", respond) def commits(db) -> list[BranchCommit]: with db.session() as session: return list(session.scalars(select(BranchCommit))) def status(db, run_id) -> str: with db.session() as session: return session.get(TaskRun, run_id).status def test_worker_stirbt_nach_commit_neuer_worker_setzt_fort(setup, db, settings, fake_provider, monkeypatch): fake_provider.script = [[("commit_change", {"message": "README ergänzen", "changes": [README]})], "Erledigt."] fail_on_call(fake_provider, monkeypatch, 2, Crash()) run_id = ask(setup["ben"], setup["chat_id"]) with pytest.raises(Crash): Worker(db, settings, "worker-a").run_once() assert status(db, run_id) == states.RUNNING and len(commits(db)) == 1 # Lease läuft ab, ein anderer Worker übernimmt with db.session() as session: session.execute(update(TaskRun).values(locked_until=database.now() - timedelta(seconds=1))) assert Worker(db, settings, "worker-b").run_once() assert status(db, run_id) == states.COMPLETED # Fortgesetzt statt wiederholt: kein zweiter Commit, das Modell kennt das Ergebnis des ersten assert len(commits(db)) == 1 assert any("auf Branch" in o for o in outputs(fake_provider, -1)) def test_gateway_kurz_weg_nach_commit(setup, db, settings, fake_provider, monkeypatch): from app.ai.provider import ProviderError fake_provider.script = [[("commit_change", {"message": "README ergänzen", "changes": [README]})], "Erledigt."] fail_on_call(fake_provider, monkeypatch, 2, ProviderError("Timeout", user_message="Zeitüberschreitung", retryable=True)) run_id = ask(setup["ben"], setup["chat_id"]) Worker(db, settings).run_once() assert status(db, run_id) == states.QUEUED # neuer Versuch mit Wartezeit with db.session() as session: session.execute(update(TaskRun).values(available_at=database.now() - timedelta(seconds=1))) Worker(db, settings).run_once() assert status(db, run_id) == states.COMPLETED and len(commits(db)) == 1 assert any("auf Branch" in o for o in outputs(fake_provider, -1)) @pytest.mark.parametrize("offset", [3600, -3600]) def test_uhren_weichen_ab(setup, db, settings, fake_provider, monkeypatch, offset): """Web und Worker mit einer Stunde Abweichung zur Datenbank: Runs werden sofort bearbeitet, laufende Leases gelten nicht als abgelaufen.""" real = database.now skewed = lambda: real() + timedelta(seconds=offset) for module in list(sys.modules.values()): if getattr(module, "now", None) is real: monkeypatch.setattr(module, "now", skewed) run_id = ask(setup["ben"], setup["chat_id"], "Hallo") with db.session() as session: assert engine.claim(session, "worker-a", settings) is not None assert engine.recover_expired(db, settings) == 0 with db.session() as session: session.execute(update(TaskRun).values(status=states.QUEUED, locked_by=None, locked_until=None)) assert Worker(db, settings).run_once() and status(db, run_id) == states.COMPLETED def test_worker_uebersteht_datenbankausfall(setup, db, settings, fake_provider, monkeypatch, caplog): run_id = ask(setup["ben"], setup["chat_id"], "Hallo") original, failures = engine.recover_expired, [] def flaky(*args, **kwargs): if len(failures) < 2: failures.append(1) raise OperationalError("SELECT 1", {}, Exception("connection refused")) return original(*args, **kwargs) monkeypatch.setattr(engine, "recover_expired", flaky) monkeypatch.setattr(settings, "worker_poll_seconds", 0.01) stop, worker = threading.Event(), Worker(db, settings) monkeypatch.setattr("app.tasks.monitor.serve", lambda *a: None) thread = threading.Thread(target=worker.run_forever, args=(stop,)) thread.start() for _ in range(500): if status(db, run_id) == states.COMPLETED: break threading.Event().wait(0.01) stop.set() thread.join(5) assert status(db, run_id) == states.COMPLETED and len(failures) == 2 assert "Fehler in der Worker-Schleife" in caplog.text def test_git_server_antwortet_nicht_beim_push(setup, db, settings, fake_provider, monkeypatch): commit(setup, db, settings, fake_provider) b = branch(db) def silent(self, *args, **kwargs): raise GitError("Keine Antwort vom Git-Server nach 30 Sekunden.") monkeypatch.setattr(GitRepo, "remote", silent) monkeypatch.setattr(GitRepo, "remote_head", silent) monkeypatch.setattr(GitRepo, "sync", lambda self: None) response = hx(setup["ben"], f"/branches/{b.id}/push") assert response.status_code == 400 and "Keine Antwort vom Git-Server" in response.text assert branch(db).pushed is None and remote_branches(setup["repo"]) == [] monkeypatch.undo() assert hx(setup["ben"], f"/branches/{b.id}/push").status_code == 200 assert branch(db).pushed == b.head == git(setup["repo"], "rev-parse", b.name) def test_push_angekommen_aber_antwort_verloren(setup, db, settings, fake_provider, monkeypatch): commit(setup, db, settings, fake_provider) b = branch(db) original = GitRepo.push def lost(self, *args, **kwargs): original(self, *args, **kwargs) raise GitError("Verbindung abgebrochen.") monkeypatch.setattr(GitRepo, "push", lost) response = hx(setup["ben"], f"/branches/{b.id}/push") assert response.status_code == 200 and "Übertragen." in response.text assert branch(db).pushed == b.head == git(setup["repo"], "rev-parse", b.name) def test_pull_request_anlegen_nach_abbruch_ohne_doppelten(setup, db, settings, fake_provider, fake_prs, monkeypatch): """Bitbucket legt den Pull Request an, die Antwort geht verloren: der nächste Versuch findet ihn.""" commit(setup, db, settings, fake_provider) b = branch(db) original = fake_prs.create def lost(*args, **kwargs): original(*args, **kwargs) raise GitError("Bitbucket nicht erreichbar (ReadTimeout).") monkeypatch.setattr(fake_prs, "create", lost) response = hx(setup["ben"], f"/branches/{b.id}/pull-request", {"title": "README ergänzen"}) assert response.status_code == 400 and "nicht erreichbar" in response.text assert remote_branches(setup["repo"]) == [b.name] # Push ist durch monkeypatch.setattr(fake_prs, "create", original) assert hx(setup["ben"], f"/branches/{b.id}/pull-request", {"title": "README ergänzen"}).status_code == 200 assert len(fake_prs.created) == 1 and branch(db).pr_id == "42"