323 lines
10 KiB
Python
323 lines
10 KiB
Python
from collections.abc import Iterable
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from agentci.engine.model import Workflow, WorkflowKind
|
|
from agentci.workflows.model import (
|
|
AgentResult,
|
|
PlanArtifact,
|
|
ReviewFinding,
|
|
ReviewReport,
|
|
ReviewSeverity,
|
|
)
|
|
from agentci.workflows.review import review_implementation_loop, review_plan_loop
|
|
from tests.workflow_support import WorkflowHarness, make_workflow_harness
|
|
|
|
|
|
def serious_report(
|
|
summary: str = "Needs work",
|
|
*,
|
|
severity: ReviewSeverity = ReviewSeverity.MAJOR,
|
|
) -> ReviewReport:
|
|
return ReviewReport(
|
|
summary=summary,
|
|
findings=[
|
|
ReviewFinding(
|
|
severity=severity,
|
|
title="Missing check",
|
|
detail="A check is absent.",
|
|
recommendation="Add it.",
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
def clean_report() -> ReviewReport:
|
|
return ReviewReport(summary="Ready", findings=[])
|
|
|
|
|
|
def minor_report() -> ReviewReport:
|
|
return ReviewReport(
|
|
summary="Optional improvement",
|
|
findings=[
|
|
ReviewFinding(
|
|
severity=ReviewSeverity.MINOR,
|
|
title="Clarify wording",
|
|
detail="The wording could be clearer.",
|
|
recommendation="Tighten it when convenient.",
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
class FakeRepository:
|
|
def __init__(self, trace: list[tuple[object, ...]] | None = None) -> None:
|
|
self.saved_workflows: list[Workflow] = []
|
|
self.trace = trace
|
|
|
|
async def save_workflow(self, workflow: Workflow) -> None:
|
|
self.saved_workflows.append(workflow)
|
|
if self.trace is not None:
|
|
self.trace.append(("save_workflow", workflow.reviewer_session_id))
|
|
|
|
|
|
def workflow(*, reviewer_session_id: str | None = None) -> Workflow:
|
|
return Workflow(
|
|
id="flow",
|
|
kind=WorkflowKind.IMPLEMENT,
|
|
repo_owner="org",
|
|
repo_name="repo",
|
|
issue_number=1,
|
|
workspace_path=Path("/workspace/repo"),
|
|
base_sha="abc",
|
|
primary_session_id="primary-session",
|
|
reviewer_session_id=reviewer_session_id,
|
|
)
|
|
|
|
|
|
def review_harness(
|
|
responses: Iterable[BaseModel],
|
|
repository: FakeRepository,
|
|
*,
|
|
trace: list[tuple[object, ...]] | None = None,
|
|
plan_rounds: int = 4,
|
|
implementation_rounds: int = 3,
|
|
) -> WorkflowHarness:
|
|
return make_workflow_harness(
|
|
settings=SimpleNamespace(
|
|
plan_review_rounds=plan_rounds,
|
|
plan_model="provider/plan",
|
|
plan_variant="high",
|
|
implement_review_rounds=implementation_rounds,
|
|
implement_model="provider/implement",
|
|
implement_variant="high",
|
|
),
|
|
repository=repository,
|
|
gitea=object(),
|
|
responses=responses,
|
|
trace=trace,
|
|
)
|
|
|
|
|
|
async def test_implementation_loop_persists_reviewed_revision_and_stops_clean() -> None:
|
|
revised = AgentResult(summary_markdown="revision 1", tests=["pytest: passed"])
|
|
repository = FakeRepository()
|
|
harness = review_harness(
|
|
[serious_report(), revised, clean_report()],
|
|
repository,
|
|
implementation_rounds=4,
|
|
)
|
|
original = workflow()
|
|
|
|
updated, result, report = await review_implementation_loop(
|
|
original,
|
|
"issue context",
|
|
"canonical plan",
|
|
AgentResult(summary_markdown="initial", tests=[]),
|
|
harness.run,
|
|
harness.services,
|
|
)
|
|
|
|
assert result == revised
|
|
assert report == clean_report()
|
|
assert original.reviewer_session_id is None
|
|
assert updated.reviewer_session_id == "implementation-review-session"
|
|
assert updated.artifact == revised.model_dump_json()
|
|
assert updated.review_json == clean_report().model_dump_json()
|
|
assert repository.saved_workflows[-1] == updated
|
|
assert repository.saved_workflows[0].reviewer_session_id == ("implementation-review-session")
|
|
assert harness.run.stages == [
|
|
"reviewing implementation 1/4",
|
|
"reviewing implementation 2/4",
|
|
]
|
|
assert harness.opencode.created_sessions == [(original.workspace_path, "implementation-review")]
|
|
assert [call["session_id"] for call in harness.opencode.resume_calls] == [
|
|
"implementation-review-session",
|
|
"primary-session",
|
|
"implementation-review-session",
|
|
]
|
|
assert [call["result_type"] for call in harness.opencode.resume_calls] == [
|
|
ReviewReport,
|
|
AgentResult,
|
|
ReviewReport,
|
|
]
|
|
assert [name for name, _ in harness.prompts.calls] == [
|
|
"implementation_review",
|
|
"implementation_revision",
|
|
"implementation_review",
|
|
]
|
|
assert harness.opencode.responses == []
|
|
|
|
|
|
async def test_implementation_loop_never_makes_unreviewed_final_revision() -> None:
|
|
final_report = serious_report(
|
|
"Still failing after the last review",
|
|
severity=ReviewSeverity.BLOCKING,
|
|
)
|
|
repository = FakeRepository()
|
|
harness = review_harness(
|
|
[
|
|
serious_report("round 1"),
|
|
AgentResult(summary_markdown="revision 1", tests=[]),
|
|
serious_report("round 2"),
|
|
AgentResult(summary_markdown="revision 2", tests=[]),
|
|
final_report,
|
|
],
|
|
repository,
|
|
implementation_rounds=3,
|
|
)
|
|
|
|
updated, result, report = await review_implementation_loop(
|
|
workflow(),
|
|
"issue context",
|
|
"canonical plan",
|
|
AgentResult(summary_markdown="initial", tests=[]),
|
|
harness.run,
|
|
harness.services,
|
|
)
|
|
|
|
assert result.summary_markdown == "revision 2"
|
|
assert report is final_report
|
|
assert updated.artifact == result.model_dump_json()
|
|
assert updated.review_json == final_report.model_dump_json()
|
|
assert repository.saved_workflows[-1] == updated
|
|
assert [call["result_type"] for call in harness.opencode.resume_calls] == [
|
|
ReviewReport,
|
|
AgentResult,
|
|
ReviewReport,
|
|
AgentResult,
|
|
ReviewReport,
|
|
]
|
|
assert harness.run.stages == [
|
|
"reviewing implementation 1/3",
|
|
"reviewing implementation 2/3",
|
|
"reviewing implementation 3/3",
|
|
]
|
|
assert harness.opencode.responses == []
|
|
|
|
|
|
async def test_plan_loop_revises_serious_finding_then_persists_clean_result() -> None:
|
|
revised = PlanArtifact(plan_markdown="Revised plan")
|
|
trace: list[tuple[object, ...]] = []
|
|
repository = FakeRepository(trace)
|
|
harness = review_harness(
|
|
[serious_report(), revised, clean_report()],
|
|
repository,
|
|
trace=trace,
|
|
plan_rounds=4,
|
|
)
|
|
original = workflow()
|
|
initial = PlanArtifact(plan_markdown="Initial plan")
|
|
|
|
updated, artifact, report = await review_plan_loop(
|
|
original,
|
|
"issue context",
|
|
initial,
|
|
harness.run,
|
|
harness.services,
|
|
)
|
|
|
|
assert artifact is revised
|
|
assert report == clean_report()
|
|
assert original.reviewer_session_id is None
|
|
assert updated.reviewer_session_id == "plan-review-session"
|
|
assert updated.artifact == "Revised plan"
|
|
assert updated.review_json == clean_report().model_dump_json()
|
|
assert repository.saved_workflows[-1] == updated
|
|
assert harness.run.stages == ["reviewing plan 1/4", "reviewing plan 2/4"]
|
|
assert harness.opencode.created_sessions == [(original.workspace_path, "plan-review")]
|
|
assert harness.opencode.resume_calls[0] == {
|
|
"session_id": "plan-review-session",
|
|
"workspace": original.workspace_path,
|
|
"prompt": "rendered plan_review",
|
|
"model": "provider/plan",
|
|
"variant": "high",
|
|
"schema_name": "review.json",
|
|
"result_type": ReviewReport,
|
|
}
|
|
assert harness.prompts.calls[0] == (
|
|
"plan_review",
|
|
{"context": "issue context", "artifact": "Initial plan"},
|
|
)
|
|
assert trace[:3] == [
|
|
("create_session", "plan-review", "plan-review-session"),
|
|
("save_workflow", "plan-review-session"),
|
|
("resume", "plan-review-session", ReviewReport),
|
|
]
|
|
assert [call["session_id"] for call in harness.opencode.resume_calls] == [
|
|
"plan-review-session",
|
|
"primary-session",
|
|
"plan-review-session",
|
|
]
|
|
assert [name for name, _ in harness.prompts.calls] == [
|
|
"plan_review",
|
|
"plan_revision",
|
|
"plan_review",
|
|
]
|
|
assert harness.opencode.responses == []
|
|
|
|
|
|
async def test_plan_loop_stops_at_round_boundary_without_unreviewed_revision() -> None:
|
|
final_report = serious_report("round 2")
|
|
repository = FakeRepository()
|
|
harness = review_harness(
|
|
[
|
|
serious_report("round 1"),
|
|
PlanArtifact(plan_markdown="Only revision"),
|
|
final_report,
|
|
],
|
|
repository,
|
|
plan_rounds=2,
|
|
)
|
|
|
|
updated, artifact, report = await review_plan_loop(
|
|
workflow(),
|
|
"issue context",
|
|
PlanArtifact(plan_markdown="Initial plan"),
|
|
harness.run,
|
|
harness.services,
|
|
)
|
|
|
|
assert artifact.plan_markdown == "Only revision"
|
|
assert report is final_report
|
|
assert updated.artifact == "Only revision"
|
|
assert updated.review_json == final_report.model_dump_json()
|
|
assert repository.saved_workflows[-1] == updated
|
|
assert [call["result_type"] for call in harness.opencode.resume_calls] == [
|
|
ReviewReport,
|
|
PlanArtifact,
|
|
ReviewReport,
|
|
]
|
|
assert harness.run.stages == ["reviewing plan 1/2", "reviewing plan 2/2"]
|
|
assert harness.opencode.responses == []
|
|
|
|
|
|
async def test_minor_findings_end_review_loop_without_revision() -> None:
|
|
repository = FakeRepository()
|
|
harness = review_harness(
|
|
[minor_report()],
|
|
repository,
|
|
implementation_rounds=5,
|
|
)
|
|
initial = AgentResult(summary_markdown="initial", tests=[])
|
|
|
|
updated, result, report = await review_implementation_loop(
|
|
workflow(),
|
|
"issue context",
|
|
"canonical plan",
|
|
initial,
|
|
harness.run,
|
|
harness.services,
|
|
)
|
|
|
|
assert result is initial
|
|
assert report == minor_report()
|
|
assert not report.has_serious_findings
|
|
assert updated.artifact == initial.model_dump_json()
|
|
assert updated.review_json == minor_report().model_dump_json()
|
|
assert repository.saved_workflows[-1] == updated
|
|
assert [call["result_type"] for call in harness.opencode.resume_calls] == [ReviewReport]
|
|
assert harness.run.stages == ["reviewing implementation 1/5"]
|