369 lines
13 KiB
Python
369 lines
13 KiB
Python
"""Tests for deterministic contract conformance (M8.V1)."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from jsonschema import validate
|
|
|
|
from cmdforge.contract_testing import (
|
|
ConformanceResult,
|
|
ConformanceReport,
|
|
UnsupportedContract,
|
|
_generate_inputs,
|
|
_fallback_value,
|
|
run_contract_tests,
|
|
)
|
|
from cmdforge.tool import Tool
|
|
|
|
|
|
class TestInputGeneration:
|
|
def test_enum_generates_variants(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"mode": {"type": "string", "enum": ["fast", "accurate", "balanced"]},
|
|
"limit": {"type": "integer", "default": 10},
|
|
},
|
|
"required": ["mode"],
|
|
}
|
|
cases = _generate_inputs(schema)
|
|
assert len(cases) >= 1
|
|
assert cases[0]["mode"] in ("fast", "accurate", "balanced")
|
|
|
|
def test_fallback_string(self):
|
|
cases = _generate_inputs({
|
|
"type": "object",
|
|
"properties": {"name": {"type": "string"}},
|
|
"required": ["name"],
|
|
})
|
|
assert "test-name" in cases[0]["name"]
|
|
|
|
def test_fallback_integer(self):
|
|
cases = _generate_inputs({
|
|
"type": "object",
|
|
"properties": {"count": {"type": "integer"}},
|
|
"required": ["count"],
|
|
})
|
|
assert cases[0]["count"] == 0
|
|
|
|
def test_default_used(self):
|
|
cases = _generate_inputs({
|
|
"type": "object",
|
|
"properties": {"name": {"type": "string", "default": "World"}},
|
|
"required": ["name"],
|
|
})
|
|
assert cases[0]["name"] == "World"
|
|
|
|
def test_scalar_schema_generates_scalar(self):
|
|
cases = _generate_inputs({"type": "string"})
|
|
assert cases == ["test-input"]
|
|
|
|
def test_generated_value_respects_numeric_bounds(self):
|
|
schema = {
|
|
"type": "object",
|
|
"properties": {
|
|
"count": {"type": "integer", "minimum": 1, "maximum": 3}
|
|
},
|
|
"required": ["count"],
|
|
}
|
|
case = _generate_inputs(schema)[0]
|
|
validate(case, schema)
|
|
assert 1 <= case["count"] <= 3
|
|
|
|
def test_array_and_nested_object_are_supported(self):
|
|
schema = {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {"name": {"type": "string"}},
|
|
"required": ["name"],
|
|
},
|
|
}
|
|
value = _generate_inputs(schema)[0]
|
|
validate(value, schema)
|
|
|
|
def test_unsynthesizable_pattern_is_unsupported(self):
|
|
with pytest.raises(UnsupportedContract):
|
|
_generate_inputs({"type": "string", "pattern": "^z{20}$"})
|
|
|
|
def test_invalid_default_is_unsupported_not_returned(self):
|
|
with pytest.raises(UnsupportedContract, match="valid input"):
|
|
_generate_inputs({"type": "integer", "default": "wrong"})
|
|
|
|
|
|
class TestFallbackValue:
|
|
def test_explicit_default(self):
|
|
assert _fallback_value({"default": "hi"}, "x", []) == "hi"
|
|
|
|
def test_examples(self):
|
|
assert _fallback_value({"examples": ["a", "b"]}, "x", []) == "a"
|
|
|
|
def test_enum(self):
|
|
assert _fallback_value({"enum": ["yes", "no"]}, "x", []) == "yes"
|
|
|
|
def test_type_fallback(self):
|
|
assert _fallback_value({"type": "boolean"}, "flag", []) is True
|
|
|
|
|
|
class TestConformanceReport:
|
|
def test_empty_is_not_run_not_pass(self):
|
|
report = ConformanceReport()
|
|
assert report.outcome == "not_run"
|
|
assert not report.passed
|
|
|
|
def test_invalid_state_is_rejected(self):
|
|
with pytest.raises(ValueError, match="Unknown conformance state"):
|
|
ConformanceResult("case", "unknown")
|
|
|
|
def test_summary_counts(self):
|
|
report = ConformanceReport(results=[
|
|
ConformanceResult("a", "passed"),
|
|
ConformanceResult("b", "passed"),
|
|
ConformanceResult("c", "failed", "error"),
|
|
])
|
|
assert report.summary == {"passed": 2, "failed": 1, "unsupported": 0, "not_run": 0}
|
|
assert not report.passed
|
|
|
|
|
|
class TestRunContractTests:
|
|
def test_missing_schemas_returns_not_run(self, tmp_path):
|
|
from unittest.mock import patch
|
|
from cmdforge.tool import Tool
|
|
|
|
with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
|
|
tool = Tool(name="bare")
|
|
report = run_contract_tests(tool)
|
|
assert report.results[0].state == "not_run"
|
|
|
|
def test_simple_tool_passes(self, tmp_path):
|
|
from unittest.mock import patch
|
|
from cmdforge.tool import Tool, PromptStep
|
|
|
|
with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
|
|
tool = Tool(
|
|
name="greet",
|
|
input_schema={
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {"type": "string", "default": "World"},
|
|
},
|
|
},
|
|
output_schema={
|
|
"type": "object",
|
|
"properties": {"result": {"type": "string"}},
|
|
"required": ["result"],
|
|
},
|
|
steps=[
|
|
PromptStep(
|
|
prompt="Greet {name}",
|
|
provider="real-provider-that-must-not-run",
|
|
output_var="response",
|
|
output_schema={
|
|
"type": "object",
|
|
"properties": {"result": {"type": "string"}},
|
|
"required": ["result"],
|
|
},
|
|
),
|
|
],
|
|
output="{response}",
|
|
)
|
|
report = run_contract_tests(tool)
|
|
assert report.outcome == "passed"
|
|
assert report.results[0].state == "passed"
|
|
|
|
def test_empty_schemas_are_explicit_and_runnable(self):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = run_contract_tests(
|
|
Tool(name="anything", input_schema={}, output_schema={}, output="ok")
|
|
)
|
|
assert report.outcome == "passed"
|
|
|
|
@pytest.mark.parametrize("step_type", ["code", "tool", "mcp"])
|
|
def test_side_effecting_steps_are_unsupported(self, step_type):
|
|
from cmdforge.tool import CodeStep, McpStep, Tool, ToolStep
|
|
|
|
steps = {
|
|
"code": CodeStep(code="raise AssertionError('must not run')", output_var="x"),
|
|
"tool": ToolStep(tool="external", output_var="x"),
|
|
"mcp": McpStep(server="remote", tool="external", output_var="x"),
|
|
}
|
|
tool = Tool(
|
|
name="unsafe",
|
|
input_schema={},
|
|
output_schema={},
|
|
steps=[steps[step_type]],
|
|
output="{x}",
|
|
)
|
|
report = run_contract_tests(tool)
|
|
assert report.outcome == "unsupported"
|
|
|
|
def test_schema_mismatch_is_a_failure(self):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = run_contract_tests(
|
|
Tool(
|
|
name="mismatch",
|
|
input_schema={},
|
|
output_schema={"type": "integer"},
|
|
output="not-an-integer",
|
|
)
|
|
)
|
|
assert report.outcome == "failed"
|
|
|
|
|
|
class TestRegressionComparison:
|
|
@staticmethod
|
|
def contracted_tool(version="1.0.0", output_schema=None):
|
|
from cmdforge.tool import Tool
|
|
|
|
return Tool(
|
|
name="test",
|
|
version=version,
|
|
input_schema={},
|
|
output_schema=output_schema or {"type": "string"},
|
|
output="stable",
|
|
)
|
|
|
|
def passing_report(self, tool=None):
|
|
return run_contract_tests(tool or self.contracted_tool())
|
|
|
|
def test_save_and_load_baseline(self, tmp_path):
|
|
from cmdforge.contract_testing import (
|
|
save_conformance_baseline, load_conformance_baseline,
|
|
)
|
|
import stat
|
|
|
|
tool = self.contracted_tool()
|
|
report = self.passing_report(tool)
|
|
path = save_conformance_baseline(tool, report, tmp_path)
|
|
assert path.exists()
|
|
assert stat.S_IMODE(path.stat().st_mode) == 0o600
|
|
|
|
loaded = load_conformance_baseline(tmp_path)
|
|
assert loaded["format_version"] == 1
|
|
assert loaded["tool"]["version"] == "1.0.0"
|
|
assert loaded["contract_hash"]
|
|
assert len(loaded["results"]) == 1
|
|
assert loaded["results"][0]["case_id"].startswith("case-")
|
|
assert loaded["results"][0]["output_shape"] == {"type": "string"}
|
|
|
|
def test_missing_baseline(self, tmp_path):
|
|
from cmdforge.contract_testing import load_conformance_baseline
|
|
assert load_conformance_baseline(tmp_path) is None
|
|
|
|
def test_corrupt_baseline_is_rejected(self, tmp_path):
|
|
from cmdforge.contract_testing import (
|
|
BaselineFormatError, CONFORMANCE_FILE, load_conformance_baseline,
|
|
)
|
|
|
|
(tmp_path / CONFORMANCE_FILE).write_text("[]")
|
|
with pytest.raises(BaselineFormatError, match="root"):
|
|
load_conformance_baseline(tmp_path)
|
|
|
|
def test_tampered_baseline_evidence_is_rejected(self, tmp_path):
|
|
from cmdforge.contract_testing import (
|
|
BaselineFormatError, CONFORMANCE_FILE, load_conformance_baseline,
|
|
save_conformance_baseline,
|
|
)
|
|
import json
|
|
|
|
tool = self.contracted_tool()
|
|
save_conformance_baseline(tool, self.passing_report(tool), tmp_path)
|
|
path = tmp_path / CONFORMANCE_FILE
|
|
data = json.loads(path.read_text())
|
|
data["results"][0]["input_value"] = {"tampered": True}
|
|
path.write_text(json.dumps(data))
|
|
|
|
with pytest.raises(BaselineFormatError, match="fingerprint mismatch"):
|
|
load_conformance_baseline(tmp_path)
|
|
|
|
def test_failed_report_cannot_replace_baseline(self, tmp_path):
|
|
from cmdforge.contract_testing import save_conformance_baseline
|
|
|
|
report = run_contract_tests(Tool(
|
|
name="failed", input_schema={}, output_schema={"type": "integer"},
|
|
output="text",
|
|
))
|
|
with pytest.raises(ValueError, match="passing"):
|
|
save_conformance_baseline(self.contracted_tool(), report, tmp_path)
|
|
|
|
def baseline_and_report(self, tmp_path):
|
|
from cmdforge.contract_testing import (
|
|
load_conformance_baseline, save_conformance_baseline,
|
|
)
|
|
|
|
tool = self.contracted_tool()
|
|
report = self.passing_report(tool)
|
|
save_conformance_baseline(tool, report, tmp_path)
|
|
return load_conformance_baseline(tmp_path), report
|
|
|
|
def test_detects_regression(self):
|
|
from dataclasses import replace
|
|
from cmdforge.contract_testing import (
|
|
ConformanceReport, compare_conformance,
|
|
)
|
|
import tempfile
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
baseline, report = self.baseline_and_report(Path(directory))
|
|
current = ConformanceReport([
|
|
replace(report.results[0], state="failed", detail="now broken")
|
|
])
|
|
delta = compare_conformance(
|
|
baseline, current, self.contracted_tool(version="2.0.0")
|
|
)
|
|
assert len(delta.regressed) == 1
|
|
assert delta.has_regressions
|
|
assert delta.current_version == "2.0.0"
|
|
|
|
def test_lost_coverage_is_regression(self, tmp_path):
|
|
from dataclasses import replace
|
|
from cmdforge.contract_testing import (
|
|
ConformanceReport, compare_conformance,
|
|
)
|
|
baseline, report = self.baseline_and_report(tmp_path)
|
|
current = ConformanceReport([
|
|
replace(report.results[0], state="unsupported", detail="lost")
|
|
])
|
|
delta = compare_conformance(baseline, current, self.contracted_tool())
|
|
assert len(delta.coverage_losses) == 1
|
|
assert delta.has_regressions
|
|
|
|
def test_removed_case_is_regression(self, tmp_path):
|
|
from cmdforge.contract_testing import (
|
|
ConformanceReport, compare_conformance,
|
|
)
|
|
baseline, _ = self.baseline_and_report(tmp_path)
|
|
delta = compare_conformance(
|
|
baseline, ConformanceReport(), self.contracted_tool()
|
|
)
|
|
assert len(delta.removed_cases) == 1
|
|
assert delta.has_regressions
|
|
|
|
def test_output_shape_or_value_change_is_flagged(self, tmp_path):
|
|
from dataclasses import replace
|
|
from cmdforge.contract_testing import (
|
|
ConformanceReport, compare_conformance,
|
|
)
|
|
baseline, report = self.baseline_and_report(tmp_path)
|
|
current = ConformanceReport([
|
|
replace(
|
|
report.results[0],
|
|
output_shape={"type": "object", "properties": {}},
|
|
output_fingerprint="changed",
|
|
)
|
|
])
|
|
delta = compare_conformance(baseline, current, self.contracted_tool())
|
|
assert len(delta.output_changes) == 1
|
|
assert delta.has_regressions
|
|
|
|
def test_contract_change_is_recorded(self, tmp_path):
|
|
from cmdforge.contract_testing import compare_conformance
|
|
|
|
baseline, report = self.baseline_and_report(tmp_path)
|
|
changed = self.contracted_tool(output_schema={"type": ["string", "null"]})
|
|
delta = compare_conformance(baseline, report, changed)
|
|
assert delta.contract_changed
|