494 lines
18 KiB
Python
494 lines
18 KiB
Python
"""Tests for preflight analysis engine."""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from cmdforge.preflight import (
|
|
PreflightReport,
|
|
analyze_tool,
|
|
_check_config_integrity,
|
|
_check_similar_tools,
|
|
_is_semver,
|
|
)
|
|
from cmdforge.registry_client import PaginatedResponse, RegistryError
|
|
|
|
|
|
class TestPreflightReport:
|
|
def test_empty_report_is_ok(self):
|
|
report = PreflightReport()
|
|
assert report.ok
|
|
|
|
def test_errors_make_not_ok(self):
|
|
report = PreflightReport(errors=["bad"])
|
|
assert not report.ok
|
|
|
|
def test_merge(self):
|
|
a = PreflightReport(errors=["e1"], warnings=["w1"])
|
|
b = PreflightReport(errors=["e2"], suggestions=["s1"])
|
|
c = a.merge(b)
|
|
assert c.errors == ["e1", "e2"]
|
|
assert c.warnings == ["w1"]
|
|
assert c.suggestions == ["s1"]
|
|
|
|
def test_to_dict_returns_independent_lists(self):
|
|
report = PreflightReport(errors=["bad"])
|
|
serialized = report.to_dict()
|
|
serialized["errors"].append("another")
|
|
assert report.errors == ["bad"]
|
|
|
|
def test_to_dict_copies_contract_proposal(self):
|
|
report = PreflightReport(
|
|
contract_proposal={"input_schema": {"type": "string"}}
|
|
)
|
|
serialized = report.to_dict()
|
|
serialized["contract_proposal"]["input_schema"]["type"] = "integer"
|
|
assert report.contract_proposal["input_schema"]["type"] == "string"
|
|
|
|
|
|
class TestAnalyzeTool:
|
|
def test_no_steps_warns(self, tmp_path):
|
|
from cmdforge.tool import Tool, save_tool
|
|
|
|
with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"), patch(
|
|
"cmdforge.tool.BIN_DIR", tmp_path / "bin"
|
|
):
|
|
tool = Tool(name="empty-tool")
|
|
save_tool(tool)
|
|
|
|
report = analyze_tool(tool)
|
|
assert report.ok
|
|
assert any("no steps" in w.lower() for w in report.warnings)
|
|
|
|
def test_missing_name(self, tmp_path):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = PreflightReport()
|
|
tool = Tool(name="")
|
|
_check_config_integrity(tool, report)
|
|
assert not report.ok
|
|
|
|
def test_invalid_version(self, tmp_path):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = PreflightReport()
|
|
tool = Tool(name="test", version="not-semver")
|
|
_check_config_integrity(tool, report)
|
|
assert not report.ok
|
|
|
|
def test_ok_tool(self, tmp_path):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = PreflightReport()
|
|
tool = Tool(name="ok-tool", version="1.0.0")
|
|
_check_config_integrity(tool, report)
|
|
assert report.ok
|
|
|
|
def test_constant_tool_without_steps_is_not_passthrough_warning(self):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = analyze_tool(Tool(name="constant", output="hello"))
|
|
assert not any("passthrough" in warning for warning in report.warnings)
|
|
|
|
def test_secret_pattern_in_prompt(self, tmp_path):
|
|
from cmdforge.tool import Tool, PromptStep
|
|
|
|
tool = Tool(
|
|
name="leaky",
|
|
steps=[PromptStep(
|
|
prompt="Use api_key: abc123 to auth",
|
|
provider="mock",
|
|
output_var="out",
|
|
)],
|
|
)
|
|
report = PreflightReport()
|
|
from cmdforge.preflight import _check_secrets
|
|
_check_secrets(tool, report)
|
|
assert any("secret" in w.lower() for w in report.warnings)
|
|
|
|
@pytest.mark.parametrize("version", ["1.2.3garbage", "1.2.3.4", "1.2"])
|
|
def test_semver_rejects_trailing_or_incomplete_values(self, version):
|
|
assert _is_semver(version) is False
|
|
|
|
def test_registry_similarity_uses_paginated_data(self):
|
|
class Client:
|
|
def search_tools(self, query, per_page):
|
|
assert (query, per_page) == ("summarize", 5)
|
|
return PaginatedResponse(data=[{
|
|
"owner": "official",
|
|
"name": "summary",
|
|
"description": "Summarize text",
|
|
"downloads": 10,
|
|
}])
|
|
|
|
report = PreflightReport()
|
|
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
|
|
assert report.similar_tools[0]["name"] == "official/summary"
|
|
|
|
def test_server_audit_does_not_claim_skipped_local_checks(self):
|
|
from cmdforge.tool import Tool, ToolStep
|
|
|
|
report = analyze_tool(
|
|
Tool(name="server", steps=[ToolStep(tool="child", output_var="out")]),
|
|
check_local_dependencies=False,
|
|
)
|
|
checks = report.audit_evidence["checks_run"]
|
|
assert "dependencies" not in checks
|
|
assert "toolstep_compatibility" not in checks
|
|
assert "reuse_opportunities" not in checks
|
|
assert report.compatibility == []
|
|
|
|
def test_unrelated_code_steps_are_not_reuse_evidence(self):
|
|
from cmdforge.tool import CodeStep, Tool
|
|
|
|
current = Tool(
|
|
name="current", input_schema={}, output_schema={},
|
|
steps=[
|
|
CodeStep(code="x = 1", output_var="x"),
|
|
CodeStep(code="y = 2", output_var="y"),
|
|
],
|
|
)
|
|
other = Tool(
|
|
name="other", input_schema={}, output_schema={},
|
|
steps=[
|
|
CodeStep(code="a = 99", output_var="a"),
|
|
CodeStep(code="b = 100", output_var="b"),
|
|
],
|
|
)
|
|
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
|
|
"cmdforge.tool.load_tool", return_value=other
|
|
):
|
|
report = analyze_tool(current)
|
|
assert report.reuse_opportunities == []
|
|
|
|
def test_exact_contracted_duplicate_has_evidence(self):
|
|
from cmdforge.tool import PromptStep, Tool
|
|
|
|
def make(name):
|
|
return Tool(
|
|
name=name, input_schema={}, output_schema={"type": "string"},
|
|
steps=[
|
|
PromptStep(
|
|
prompt="first", provider="mock", output_var="one",
|
|
output_schema={"type": "string"},
|
|
),
|
|
PromptStep(
|
|
prompt="second", provider="mock", output_var="two",
|
|
output_schema={"type": "string"},
|
|
),
|
|
],
|
|
)
|
|
|
|
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
|
|
"cmdforge.tool.load_tool", return_value=make("other")
|
|
):
|
|
report = analyze_tool(make("current"))
|
|
duplicate = [
|
|
item for item in report.reuse_opportunities
|
|
if item["type"] == "duplicate_sequence"
|
|
]
|
|
assert duplicate
|
|
assert "explicit tool contracts" in duplicate[0]["evidence"]
|
|
|
|
def test_registry_error_becomes_warning(self):
|
|
class Client:
|
|
def search_tools(self, query, per_page):
|
|
raise RegistryError("CONNECTION_ERROR", "offline")
|
|
|
|
report = PreflightReport()
|
|
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
|
|
assert report.warnings == ["Could not search registry for similar tools"]
|
|
|
|
def test_programming_error_is_not_swallowed(self):
|
|
class Client:
|
|
def search_tools(self, query, per_page):
|
|
raise TypeError("broken adapter")
|
|
|
|
with pytest.raises(TypeError, match="broken adapter"):
|
|
_check_similar_tools(
|
|
SimpleNamespace(name="summarize"), PreflightReport(), Client()
|
|
)
|
|
|
|
|
|
class TestContractInference:
|
|
def test_no_arguments_no_input(self):
|
|
from cmdforge.tool import Tool
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
tool = Tool(name="empty", output="{response}")
|
|
result = infer_contracts(tool)
|
|
assert result["input_schema"] is None
|
|
assert result["output_schema"] is None
|
|
assert result["confidence"] == "low"
|
|
|
|
def test_argument_becomes_property(self):
|
|
from cmdforge.tool import Tool, ToolArgument
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
tool = Tool(
|
|
name="greet",
|
|
arguments=[ToolArgument(flag="--name", variable="name", description="Who to greet")],
|
|
output="Hello",
|
|
)
|
|
result = infer_contracts(tool)
|
|
assert result["input_schema"] is not None
|
|
assert "name" in result["input_schema"]["properties"]
|
|
assert result["input_schema"]["properties"]["name"]["description"] == "Who to greet"
|
|
|
|
def test_stdin_detected_when_input_used(self):
|
|
from cmdforge.tool import Tool, PromptStep
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
tool = Tool(
|
|
name="echo",
|
|
steps=[PromptStep(prompt="Repeat: {input}", provider="mock", output_var="response")],
|
|
output="{response}",
|
|
)
|
|
result = infer_contracts(tool)
|
|
assert result["input_schema"] is not None
|
|
assert "input" in result["input_schema"]["properties"]
|
|
assert any("stdin" in e.lower() for e in result["evidence"])
|
|
|
|
def test_enum_propagates(self):
|
|
from cmdforge.tool import Tool, ToolArgument
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
arg = ToolArgument(flag="--mode", variable="mode")
|
|
arg.enum = ["fast", "accurate"]
|
|
tool = Tool(name="tool", arguments=[arg], output="done")
|
|
result = infer_contracts(tool)
|
|
assert result["input_schema"]["properties"]["mode"]["enum"] == ["fast", "accurate"]
|
|
|
|
def test_explicit_empty_schema_not_overwritten(self):
|
|
from cmdforge.tool import Tool
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
tool = Tool(name="tool", input_schema={}, output_schema={})
|
|
result = infer_contracts(tool)
|
|
assert result["input_schema"] is None
|
|
assert result["output_schema"] is None
|
|
|
|
def test_output_propagated_from_step_schema(self):
|
|
from cmdforge.tool import Tool, PromptStep
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
step = PromptStep(
|
|
prompt="Analyze: {input}",
|
|
provider="mock",
|
|
output_var="analysis",
|
|
)
|
|
step.output_schema = {
|
|
"type": "object",
|
|
"properties": {"summary": {"type": "string"}},
|
|
}
|
|
tool = Tool(name="analyze", steps=[step], output="{analysis}")
|
|
result = infer_contracts(tool)
|
|
assert result["output_schema"] is not None
|
|
assert result["output_schema"]["type"] == "object"
|
|
assert "summary" in result["output_schema"]["properties"]
|
|
assert result["confidence"] == "high"
|
|
|
|
@pytest.mark.parametrize("schema_type", ["string", "number", "array", "boolean"])
|
|
def test_non_object_output_schema_is_propagated(self, schema_type):
|
|
from cmdforge.tool import PromptStep, Tool
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
schema = {"type": schema_type}
|
|
step = PromptStep(
|
|
prompt="Generate", provider="mock", output_var="result",
|
|
output_schema=schema,
|
|
)
|
|
proposal = infer_contracts(
|
|
Tool(name="typed", steps=[step], output="{result}")
|
|
)
|
|
assert proposal["output_schema"] == schema
|
|
|
|
def test_proposed_schema_does_not_alias_step_schema(self):
|
|
from cmdforge.tool import PromptStep, Tool
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
step = PromptStep(
|
|
prompt="Generate", provider="mock", output_var="result",
|
|
output_schema={
|
|
"type": "object",
|
|
"properties": {"value": {"type": "string"}},
|
|
},
|
|
)
|
|
proposal = infer_contracts(
|
|
Tool(name="copy", steps=[step], output="{result}")
|
|
)
|
|
proposal["output_schema"]["properties"]["value"]["type"] = "integer"
|
|
assert step.output_schema["properties"]["value"]["type"] == "string"
|
|
|
|
def test_stdin_detected_in_nested_tool_input(self):
|
|
from cmdforge.tool import Tool, ToolStep
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
proposal = infer_contracts(Tool(
|
|
name="nested",
|
|
steps=[ToolStep(tool="child", output_var="result")],
|
|
output="{result}",
|
|
))
|
|
assert "input" in proposal["input_schema"]["properties"]
|
|
|
|
def test_evidence_uses_real_argument_flag(self):
|
|
from cmdforge.tool import Tool, ToolArgument
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
proposal = infer_contracts(Tool(
|
|
name="flags",
|
|
arguments=[ToolArgument(flag="-n", variable="name")],
|
|
output="constant",
|
|
))
|
|
assert any("argument -n" in item for item in proposal["evidence"])
|
|
|
|
def test_explicit_contracts_produce_high_confidence(self):
|
|
from cmdforge.tool import Tool
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
proposal = infer_contracts(
|
|
Tool(name="explicit", input_schema={}, output_schema={})
|
|
)
|
|
assert proposal["confidence"] == "high"
|
|
|
|
def test_bare_variable_output_stays_bare(self):
|
|
from cmdforge.tool import Tool
|
|
from cmdforge.preflight import infer_contracts
|
|
|
|
tool = Tool(name="passthrough", steps=[], output="{response}")
|
|
result = infer_contracts(tool)
|
|
assert result["output_schema"] is None
|
|
assert any("source not found" in e.lower() for e in result["evidence"])
|
|
|
|
|
|
class TestConformanceIntegration:
|
|
def test_contract_tests_are_opt_in_for_server_side_preflight(self):
|
|
from cmdforge.tool import Tool
|
|
|
|
tool = Tool(name="server-safe", input_schema={}, output_schema={})
|
|
with patch("cmdforge.contract_testing.run_contract_tests") as runner:
|
|
analyze_tool(tool)
|
|
runner.assert_not_called()
|
|
|
|
def test_analyze_tool_records_passing_contract_case(self):
|
|
from cmdforge.tool import PromptStep, Tool
|
|
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {"answer": {"type": "string"}},
|
|
"required": ["answer"],
|
|
}
|
|
tool = Tool(
|
|
name="contracted",
|
|
input_schema={},
|
|
output_schema=schema,
|
|
steps=[PromptStep(
|
|
prompt="Answer", provider="paid-provider", output_var="answer",
|
|
output_schema=schema,
|
|
)],
|
|
output="{answer}",
|
|
)
|
|
report = analyze_tool(tool, include_contract_tests=True)
|
|
assert report.ok
|
|
assert report.generated_tests[0]["state"] == "passed"
|
|
|
|
def test_unsupported_execution_is_advisory_not_failure(self):
|
|
from cmdforge.tool import CodeStep, Tool
|
|
|
|
tool = Tool(
|
|
name="code-tool",
|
|
input_schema={},
|
|
output_schema={},
|
|
steps=[CodeStep(code="result = 1", output_var="result")],
|
|
output="{result}",
|
|
)
|
|
report = analyze_tool(tool, include_contract_tests=True)
|
|
assert report.ok
|
|
assert report.generated_tests[0]["state"] == "unsupported"
|
|
assert any("unsupported" in warning.lower() for warning in report.warnings)
|
|
|
|
def test_saved_baseline_is_compared_during_local_preflight(self, tmp_path):
|
|
from cmdforge.contract_testing import (
|
|
run_contract_tests, save_conformance_baseline,
|
|
)
|
|
from cmdforge.tool import Tool
|
|
|
|
baseline_tool = Tool(
|
|
name="regression", version="1.0.0", input_schema={},
|
|
output_schema={"type": "string"}, output="before",
|
|
path=tmp_path / "config.yaml",
|
|
)
|
|
save_conformance_baseline(
|
|
baseline_tool, run_contract_tests(baseline_tool), tmp_path
|
|
)
|
|
current = Tool(
|
|
name="regression", version="2.0.0", input_schema={},
|
|
output_schema={"type": "string"}, output="after",
|
|
path=tmp_path / "config.yaml",
|
|
)
|
|
report = analyze_tool(current, include_contract_tests=True)
|
|
assert report.regression["has_regressions"] is True
|
|
assert report.regression["current_version"] == "2.0.0"
|
|
assert any("Regression comparison" in warning for warning in report.warnings)
|
|
|
|
def test_baseline_inputs_are_rerun_after_input_schema_change(self, tmp_path):
|
|
from cmdforge.contract_testing import (
|
|
run_contract_tests, save_conformance_baseline,
|
|
)
|
|
from cmdforge.tool import Tool
|
|
|
|
baseline_tool = Tool(
|
|
name="input-change", version="1.0.0", input_schema={},
|
|
output_schema={"type": "string"}, output="stable",
|
|
path=tmp_path / "config.yaml",
|
|
)
|
|
save_conformance_baseline(
|
|
baseline_tool, run_contract_tests(baseline_tool), tmp_path
|
|
)
|
|
current = Tool(
|
|
name="input-change", version="2.0.0",
|
|
input_schema={
|
|
"type": "object",
|
|
"properties": {"name": {"type": "string"}},
|
|
"required": ["name"],
|
|
},
|
|
output_schema={"type": "string"}, output="stable",
|
|
path=tmp_path / "config.yaml",
|
|
)
|
|
report = analyze_tool(current, include_contract_tests=True)
|
|
assert report.regression["has_regressions"] is True
|
|
assert len(report.regression["regressed"]) == 1
|
|
assert any(
|
|
result["input_value"] == {} and result["state"] == "failed"
|
|
for result in report.generated_tests
|
|
)
|
|
|
|
def test_invalid_baseline_is_an_advisory(self, tmp_path):
|
|
from cmdforge.contract_testing import CONFORMANCE_FILE
|
|
from cmdforge.tool import Tool
|
|
|
|
(tmp_path / CONFORMANCE_FILE).write_text("not json")
|
|
tool = Tool(
|
|
name="invalid-baseline", input_schema={}, output_schema={},
|
|
output="ok", path=tmp_path / "config.yaml",
|
|
)
|
|
report = analyze_tool(tool, include_contract_tests=True)
|
|
assert report.ok
|
|
assert any("baseline is invalid" in warning.lower() for warning in report.warnings)
|
|
|
|
def test_partial_unsupported_conformance_is_not_hidden_by_a_pass(self, monkeypatch):
|
|
from cmdforge.contract_testing import ConformanceReport, ConformanceResult
|
|
from cmdforge.tool import Tool
|
|
|
|
monkeypatch.setattr(
|
|
"cmdforge.contract_testing.run_contract_tests",
|
|
lambda tool, test_inputs=None: ConformanceReport([
|
|
ConformanceResult("case-a", "passed", "valid"),
|
|
ConformanceResult("input-generation", "unsupported", "unsupported keyword"),
|
|
]),
|
|
)
|
|
tool = Tool(name="mixed", input_schema={}, output_schema={})
|
|
report = analyze_tool(tool, include_contract_tests=True)
|
|
assert any("unsupported keyword" in warning for warning in report.warnings)
|