518 lines
16 KiB
Python
518 lines
16 KiB
Python
import asyncio
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from agentci.integrations.opencode.client import OpenCode, OpenCodeError
|
|
from agentci.workflows.model import AgentResult
|
|
|
|
API_DOCUMENT = {
|
|
"paths": {
|
|
"/global/health": {"get": {}},
|
|
"/provider": {"get": {}},
|
|
"/session": {"post": {}},
|
|
"/session/{sessionID}/message": {"post": {}},
|
|
"/session/{sessionID}/abort": {"post": {}},
|
|
}
|
|
}
|
|
PROVIDERS = {
|
|
"connected": ["openai"],
|
|
"all": [
|
|
{
|
|
"id": "openai",
|
|
"models": {
|
|
"model": {
|
|
"status": "active",
|
|
"capabilities": {"toolcall": True},
|
|
"variants": {"high": {}},
|
|
}
|
|
},
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
class FakeCodeGraph:
|
|
def __init__(self) -> None:
|
|
self.prepared: list[Path] = []
|
|
|
|
async def prepare(self, workspace: Path) -> None:
|
|
self.prepared.append(workspace)
|
|
|
|
|
|
def client(tmp_path: Path, handler, codegraph: FakeCodeGraph | None = None) -> OpenCode:
|
|
selected_codegraph = codegraph or FakeCodeGraph()
|
|
return OpenCode(
|
|
base_url="http://opencode:4096",
|
|
username="opencode",
|
|
password="server-secret",
|
|
schemas_dir=Path(__file__).parents[1] / "src" / "agentci" / "prompts" / "schemas",
|
|
health_directory=tmp_path,
|
|
required_models=(("openai/model", None),),
|
|
timeout_seconds=60,
|
|
codegraph=selected_codegraph, # type: ignore[arg-type]
|
|
transport=httpx.MockTransport(handler),
|
|
)
|
|
|
|
|
|
async def test_ready_requires_healthy_server_and_connected_provider(tmp_path: Path) -> None:
|
|
expected_directory = str(tmp_path)
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.headers["authorization"].startswith("Basic ")
|
|
if request.url.path == "/global/health":
|
|
return httpx.Response(200, json={"healthy": True, "version": "1.18.4"})
|
|
if request.url.path == "/doc":
|
|
return httpx.Response(200, json=API_DOCUMENT)
|
|
assert request.headers["x-opencode-directory"] == expected_directory
|
|
return httpx.Response(200, json=PROVIDERS)
|
|
|
|
value = client(tmp_path, handler)
|
|
assert await value.ready()
|
|
await value.close()
|
|
|
|
|
|
async def test_concurrent_readiness_checks_share_one_probe(tmp_path: Path) -> None:
|
|
health_started = asyncio.Event()
|
|
release_health = asyncio.Event()
|
|
paths: list[str] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
paths.append(request.url.path)
|
|
if request.url.path == "/global/health":
|
|
health_started.set()
|
|
await release_health.wait()
|
|
return httpx.Response(200, json={"healthy": True, "version": "1.18.4"})
|
|
if request.url.path == "/doc":
|
|
return httpx.Response(200, json=API_DOCUMENT)
|
|
return httpx.Response(200, json=PROVIDERS)
|
|
|
|
value = client(tmp_path, handler)
|
|
first = asyncio.create_task(value.ready())
|
|
await health_started.wait()
|
|
second = asyncio.create_task(value.ready())
|
|
release_health.set()
|
|
|
|
assert await asyncio.gather(first, second) == [True, True]
|
|
assert paths == ["/global/health", "/doc", "/provider"]
|
|
await value.close()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("health", "document", "providers", "expected_paths"),
|
|
[
|
|
(
|
|
{"healthy": False, "version": "1.18.4"},
|
|
API_DOCUMENT,
|
|
PROVIDERS,
|
|
["/global/health"],
|
|
),
|
|
(
|
|
{"healthy": True, "version": "2.0.0"},
|
|
API_DOCUMENT,
|
|
PROVIDERS,
|
|
["/global/health"],
|
|
),
|
|
(
|
|
{"healthy": True, "version": "1.18.4"},
|
|
{"paths": {}},
|
|
PROVIDERS,
|
|
["/global/health", "/doc"],
|
|
),
|
|
(
|
|
{"healthy": True, "version": "1.18.4"},
|
|
API_DOCUMENT,
|
|
{**PROVIDERS, "connected": ["anthropic"]},
|
|
["/global/health", "/doc", "/provider"],
|
|
),
|
|
(
|
|
{"healthy": True, "version": "1.18.4"},
|
|
API_DOCUMENT,
|
|
{
|
|
"connected": ["openai"],
|
|
"all": [
|
|
{
|
|
"id": "openai",
|
|
"models": {
|
|
"model": {
|
|
"status": "active",
|
|
"capabilities": {"toolcall": False},
|
|
"variants": {"high": {}},
|
|
}
|
|
},
|
|
}
|
|
],
|
|
},
|
|
["/global/health", "/doc", "/provider"],
|
|
),
|
|
],
|
|
)
|
|
async def test_ready_rejects_incomplete_runtime_matrix(
|
|
tmp_path: Path,
|
|
health: object,
|
|
document: object,
|
|
providers: object,
|
|
expected_paths: list[str],
|
|
) -> None:
|
|
paths: list[str] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
paths.append(request.url.path)
|
|
if request.url.path == "/global/health":
|
|
return httpx.Response(200, json=health)
|
|
if request.url.path == "/doc":
|
|
return httpx.Response(200, json=document)
|
|
return httpx.Response(200, json=providers)
|
|
|
|
value = client(tmp_path, handler)
|
|
assert not await value.ready()
|
|
assert paths == expected_paths
|
|
await value.close()
|
|
|
|
|
|
@pytest.mark.parametrize("response_kind", ["http_error", "invalid_json"])
|
|
async def test_ready_converts_probe_errors_to_not_ready(tmp_path: Path, response_kind: str) -> None:
|
|
async def handler(_request: httpx.Request) -> httpx.Response:
|
|
if response_kind == "http_error":
|
|
return httpx.Response(503)
|
|
return httpx.Response(200, content=b"not-json")
|
|
|
|
value = client(tmp_path, handler)
|
|
assert not await value.ready()
|
|
await value.close()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("response_kind", "message"),
|
|
[
|
|
("invalid_json", "OpenCode request failed: POST /session"),
|
|
("array", "OpenCode returned an invalid response for POST /session"),
|
|
("http_error", "OpenCode request failed: POST /session: gateway detail"),
|
|
],
|
|
)
|
|
async def test_create_session_reports_malformed_and_error_responses(
|
|
tmp_path: Path, response_kind: str, message: str
|
|
) -> None:
|
|
async def handler(_request: httpx.Request) -> httpx.Response:
|
|
if response_kind == "invalid_json":
|
|
return httpx.Response(200, content=b"not-json")
|
|
if response_kind == "array":
|
|
return httpx.Response(200, json=[])
|
|
return httpx.Response(502, text="gateway detail")
|
|
|
|
value = client(tmp_path, handler)
|
|
with pytest.raises(OpenCodeError, match=message):
|
|
await value.create_session(tmp_path, "agent_result.json")
|
|
await value.close()
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [{}, {"id": ""}, {"id": 42}])
|
|
async def test_create_session_requires_nonempty_string_id(
|
|
tmp_path: Path, payload: dict[str, object]
|
|
) -> None:
|
|
async def handler(_request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json=payload)
|
|
|
|
value = client(tmp_path, handler)
|
|
with pytest.raises(OpenCodeError, match="did not return a session ID"):
|
|
await value.create_session(tmp_path, "agent_result.json")
|
|
await value.close()
|
|
|
|
|
|
async def test_creates_and_resumes_structured_session_in_workspace(tmp_path: Path) -> None:
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
codegraph = FakeCodeGraph()
|
|
requests: list[httpx.Request] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
requests.append(request)
|
|
assert request.headers["x-opencode-directory"] == str(workspace.resolve())
|
|
if request.url.path == "/session":
|
|
return httpx.Response(200, json={"id": "ses_new"})
|
|
body = json.loads(request.content)
|
|
assert body["model"] == {"providerID": "openai", "modelID": "model"}
|
|
assert body["agent"] == "build"
|
|
assert body["variant"] == "high"
|
|
assert body["format"]["type"] == "json_schema"
|
|
assert body["format"]["schema"]["required"] == ["summary_markdown", "tests"]
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"info": {
|
|
"role": "assistant",
|
|
"sessionID": "ses_new",
|
|
"structured": {"summary_markdown": "summary", "tests": []},
|
|
},
|
|
"parts": [],
|
|
},
|
|
)
|
|
|
|
value = client(tmp_path, handler, codegraph)
|
|
session_id = await value.create_session(workspace, "agent_result.json")
|
|
result = await value.resume(
|
|
session_id=session_id,
|
|
workspace=workspace,
|
|
prompt="implement",
|
|
model="openai/model",
|
|
variant="high",
|
|
schema_name="agent_result.json",
|
|
result_type=AgentResult,
|
|
)
|
|
|
|
assert session_id == "ses_new"
|
|
assert result.summary_markdown == "summary"
|
|
assert codegraph.prepared == [workspace]
|
|
assert [request.url.path for request in requests] == [
|
|
"/session",
|
|
"/session/ses_new/message",
|
|
]
|
|
await value.close()
|
|
|
|
|
|
async def test_retries_invalid_structured_result_on_same_session(tmp_path: Path) -> None:
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
prompts: list[str] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
body = json.loads(request.content)
|
|
prompts.append(body["parts"][0]["text"])
|
|
if len(prompts) == 1:
|
|
return httpx.Response(200, json={"info": {"error": {"name": "StructuredOutputError"}}})
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"info": {"structured": {"summary_markdown": "fixed", "tests": []}},
|
|
"parts": [],
|
|
},
|
|
)
|
|
|
|
value = client(tmp_path, handler)
|
|
result = await value.resume(
|
|
session_id="ses_existing",
|
|
workspace=workspace,
|
|
prompt="continue",
|
|
model="openai/model",
|
|
variant=None,
|
|
schema_name="agent_result.json",
|
|
result_type=AgentResult,
|
|
)
|
|
|
|
assert result.summary_markdown == "fixed"
|
|
assert prompts[0] == "continue"
|
|
assert "without repeating repository work" in prompts[1]
|
|
await value.close()
|
|
|
|
|
|
async def test_structured_validation_retry_exhaustion_reports_final_error(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
prompts: list[str] = []
|
|
paths: list[str] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
paths.append(request.url.path)
|
|
if request.url.path.endswith("/abort"):
|
|
return httpx.Response(200, json=True)
|
|
body = json.loads(request.content)
|
|
prompts.append(body["parts"][0]["text"])
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"info": {"structured": {"summary_markdown": "", "tests": "invalid"}},
|
|
"parts": [],
|
|
},
|
|
)
|
|
|
|
value = client(tmp_path, handler)
|
|
with pytest.raises(
|
|
OpenCodeError,
|
|
match="structured result failed validation",
|
|
):
|
|
await value.resume(
|
|
session_id="ses_existing",
|
|
workspace=workspace,
|
|
prompt="continue",
|
|
model="openai/model",
|
|
variant=None,
|
|
schema_name="agent_result.json",
|
|
result_type=AgentResult,
|
|
)
|
|
|
|
assert len(prompts) == 2
|
|
assert prompts[0] == "continue"
|
|
assert "without repeating repository work" in prompts[1]
|
|
assert paths[-1] == "/session/ses_existing/abort"
|
|
await value.close()
|
|
|
|
|
|
async def test_cleanup_failure_does_not_mask_turn_error(tmp_path: Path) -> None:
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
paths: list[str] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
paths.append(request.url.path)
|
|
if request.url.path.endswith("/abort"):
|
|
return httpx.Response(500, text="abort failed")
|
|
return httpx.Response(200, json={"unexpected": True})
|
|
|
|
value = client(tmp_path, handler)
|
|
with pytest.raises(
|
|
OpenCodeError,
|
|
match="OpenCode response did not include assistant metadata",
|
|
):
|
|
await value.resume(
|
|
session_id="ses_existing",
|
|
workspace=workspace,
|
|
prompt="continue",
|
|
model="openai/model",
|
|
variant=None,
|
|
schema_name="agent_result.json",
|
|
result_type=AgentResult,
|
|
)
|
|
|
|
assert paths == [
|
|
"/session/ses_existing/message",
|
|
"/session/ses_existing/abort",
|
|
]
|
|
await value.close()
|
|
|
|
|
|
async def test_close_aborts_an_active_session(tmp_path: Path) -> None:
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
message_started = asyncio.Event()
|
|
release_message = asyncio.Event()
|
|
paths: list[str] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
paths.append(request.url.path)
|
|
if request.url.path.endswith("/abort"):
|
|
assert request.headers["x-opencode-directory"] == str(workspace.resolve())
|
|
return httpx.Response(200, json=True)
|
|
message_started.set()
|
|
await release_message.wait()
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"info": {"structured": {"summary_markdown": "finished", "tests": []}},
|
|
"parts": [],
|
|
},
|
|
)
|
|
|
|
value = client(tmp_path, handler)
|
|
turn = asyncio.create_task(
|
|
value.resume(
|
|
session_id="ses_active",
|
|
workspace=workspace,
|
|
prompt="continue",
|
|
model="openai/model",
|
|
variant=None,
|
|
schema_name="agent_result.json",
|
|
result_type=AgentResult,
|
|
)
|
|
)
|
|
await message_started.wait()
|
|
|
|
await value.close()
|
|
release_message.set()
|
|
result = await turn
|
|
|
|
assert result.summary_markdown == "finished"
|
|
assert paths == ["/session/ses_active/message", "/session/ses_active/abort"]
|
|
assert value.client.is_closed
|
|
|
|
|
|
async def test_close_continues_after_active_session_abort_failure(tmp_path: Path) -> None:
|
|
paths: list[str] = []
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
paths.append(request.url.path)
|
|
if "/failed/" in request.url.path:
|
|
return httpx.Response(503)
|
|
return httpx.Response(200, json=True)
|
|
|
|
value = client(tmp_path, handler)
|
|
value._active_sessions.update(
|
|
{
|
|
"failed": tmp_path / "first",
|
|
"succeeds": tmp_path / "second",
|
|
}
|
|
)
|
|
|
|
await value.close()
|
|
|
|
assert paths == ["/session/failed/abort", "/session/succeeds/abort"]
|
|
assert value.client.is_closed
|
|
|
|
|
|
async def test_aborts_timed_out_session(tmp_path: Path) -> None:
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
abort_count = 0
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal abort_count
|
|
if request.url.path.endswith("/abort"):
|
|
abort_count += 1
|
|
return httpx.Response(200, json=True)
|
|
raise httpx.ReadTimeout("slow", request=request)
|
|
|
|
value = client(tmp_path, handler)
|
|
with pytest.raises(OpenCodeError, match="exceeded 60 seconds"):
|
|
await value.resume(
|
|
session_id="ses_existing",
|
|
workspace=workspace,
|
|
prompt="continue",
|
|
model="openai/model",
|
|
variant=None,
|
|
schema_name="agent_result.json",
|
|
result_type=AgentResult,
|
|
)
|
|
|
|
assert abort_count == 1
|
|
await value.close()
|
|
|
|
|
|
async def test_aborts_cancelled_session(tmp_path: Path) -> None:
|
|
workspace = tmp_path / "workspace"
|
|
workspace.mkdir()
|
|
started = asyncio.Event()
|
|
aborted = False
|
|
|
|
async def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal aborted
|
|
if request.url.path.endswith("/abort"):
|
|
aborted = True
|
|
return httpx.Response(200, json=True)
|
|
started.set()
|
|
await asyncio.Event().wait()
|
|
raise AssertionError("unreachable")
|
|
|
|
value = client(tmp_path, handler)
|
|
turn = asyncio.create_task(
|
|
value.resume(
|
|
session_id="ses_existing",
|
|
workspace=workspace,
|
|
prompt="continue",
|
|
model="openai/model",
|
|
variant=None,
|
|
schema_name="agent_result.json",
|
|
result_type=AgentResult,
|
|
)
|
|
)
|
|
await started.wait()
|
|
turn.cancel()
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await turn
|
|
|
|
assert aborted
|
|
await value.close()
|