Add evidence-based regression and schema compatibility checks
This commit is contained in:
@@ -589,6 +589,25 @@ def test_inspect_shows_contract_proposal_and_passing_conformance(
|
||||
assert "PASSED" in output
|
||||
|
||||
|
||||
def test_inspect_can_save_passing_baseline(tmp_path, monkeypatch, capsys):
|
||||
from cmdforge.cli import cmd_inspect
|
||||
from cmdforge.contract_testing import CONFORMANCE_FILE
|
||||
|
||||
tool = Tool(
|
||||
name="baseline", version="1.0.0", input_schema={},
|
||||
output_schema={"type": "string"}, output="stable",
|
||||
path=tmp_path / "config.yaml",
|
||||
)
|
||||
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
|
||||
|
||||
args = SimpleNamespace(
|
||||
name="baseline", registry=False, save_baseline=True
|
||||
)
|
||||
assert cmd_inspect(args) == 0
|
||||
assert (tmp_path / CONFORMANCE_FILE).exists()
|
||||
assert "Saved conformance baseline" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_switch_to_existing_tool_closes_creation_page_first():
|
||||
pytest.importorskip("PySide6")
|
||||
from cmdforge.gui.pages.tool_builder_page import _switch_to_existing_tool
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Tests for deterministic contract conformance (M8.V1)."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from jsonschema import validate
|
||||
|
||||
@@ -11,6 +13,7 @@ from cmdforge.contract_testing import (
|
||||
_fallback_value,
|
||||
run_contract_tests,
|
||||
)
|
||||
from cmdforge.tool import Tool
|
||||
|
||||
|
||||
class TestInputGeneration:
|
||||
@@ -208,3 +211,158 @@ class TestRunContractTests:
|
||||
)
|
||||
)
|
||||
assert report.outcome == "failed"
|
||||
|
||||
|
||||
class TestRegressionComparison:
|
||||
@staticmethod
|
||||
def contracted_tool(version="1.0.0", output_schema=None):
|
||||
from cmdforge.tool import Tool
|
||||
|
||||
return Tool(
|
||||
name="test",
|
||||
version=version,
|
||||
input_schema={},
|
||||
output_schema=output_schema or {"type": "string"},
|
||||
output="stable",
|
||||
)
|
||||
|
||||
def passing_report(self, tool=None):
|
||||
return run_contract_tests(tool or self.contracted_tool())
|
||||
|
||||
def test_save_and_load_baseline(self, tmp_path):
|
||||
from cmdforge.contract_testing import (
|
||||
save_conformance_baseline, load_conformance_baseline,
|
||||
)
|
||||
import stat
|
||||
|
||||
tool = self.contracted_tool()
|
||||
report = self.passing_report(tool)
|
||||
path = save_conformance_baseline(tool, report, tmp_path)
|
||||
assert path.exists()
|
||||
assert stat.S_IMODE(path.stat().st_mode) == 0o600
|
||||
|
||||
loaded = load_conformance_baseline(tmp_path)
|
||||
assert loaded["format_version"] == 1
|
||||
assert loaded["tool"]["version"] == "1.0.0"
|
||||
assert loaded["contract_hash"]
|
||||
assert len(loaded["results"]) == 1
|
||||
assert loaded["results"][0]["case_id"].startswith("case-")
|
||||
assert loaded["results"][0]["output_shape"] == {"type": "string"}
|
||||
|
||||
def test_missing_baseline(self, tmp_path):
|
||||
from cmdforge.contract_testing import load_conformance_baseline
|
||||
assert load_conformance_baseline(tmp_path) is None
|
||||
|
||||
def test_corrupt_baseline_is_rejected(self, tmp_path):
|
||||
from cmdforge.contract_testing import (
|
||||
BaselineFormatError, CONFORMANCE_FILE, load_conformance_baseline,
|
||||
)
|
||||
|
||||
(tmp_path / CONFORMANCE_FILE).write_text("[]")
|
||||
with pytest.raises(BaselineFormatError, match="root"):
|
||||
load_conformance_baseline(tmp_path)
|
||||
|
||||
def test_tampered_baseline_evidence_is_rejected(self, tmp_path):
|
||||
from cmdforge.contract_testing import (
|
||||
BaselineFormatError, CONFORMANCE_FILE, load_conformance_baseline,
|
||||
save_conformance_baseline,
|
||||
)
|
||||
import json
|
||||
|
||||
tool = self.contracted_tool()
|
||||
save_conformance_baseline(tool, self.passing_report(tool), tmp_path)
|
||||
path = tmp_path / CONFORMANCE_FILE
|
||||
data = json.loads(path.read_text())
|
||||
data["results"][0]["input_value"] = {"tampered": True}
|
||||
path.write_text(json.dumps(data))
|
||||
|
||||
with pytest.raises(BaselineFormatError, match="fingerprint mismatch"):
|
||||
load_conformance_baseline(tmp_path)
|
||||
|
||||
def test_failed_report_cannot_replace_baseline(self, tmp_path):
|
||||
from cmdforge.contract_testing import save_conformance_baseline
|
||||
|
||||
report = run_contract_tests(Tool(
|
||||
name="failed", input_schema={}, output_schema={"type": "integer"},
|
||||
output="text",
|
||||
))
|
||||
with pytest.raises(ValueError, match="passing"):
|
||||
save_conformance_baseline(self.contracted_tool(), report, tmp_path)
|
||||
|
||||
def baseline_and_report(self, tmp_path):
|
||||
from cmdforge.contract_testing import (
|
||||
load_conformance_baseline, save_conformance_baseline,
|
||||
)
|
||||
|
||||
tool = self.contracted_tool()
|
||||
report = self.passing_report(tool)
|
||||
save_conformance_baseline(tool, report, tmp_path)
|
||||
return load_conformance_baseline(tmp_path), report
|
||||
|
||||
def test_detects_regression(self):
|
||||
from dataclasses import replace
|
||||
from cmdforge.contract_testing import (
|
||||
ConformanceReport, compare_conformance,
|
||||
)
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
baseline, report = self.baseline_and_report(Path(directory))
|
||||
current = ConformanceReport([
|
||||
replace(report.results[0], state="failed", detail="now broken")
|
||||
])
|
||||
delta = compare_conformance(
|
||||
baseline, current, self.contracted_tool(version="2.0.0")
|
||||
)
|
||||
assert len(delta.regressed) == 1
|
||||
assert delta.has_regressions
|
||||
assert delta.current_version == "2.0.0"
|
||||
|
||||
def test_lost_coverage_is_regression(self, tmp_path):
|
||||
from dataclasses import replace
|
||||
from cmdforge.contract_testing import (
|
||||
ConformanceReport, compare_conformance,
|
||||
)
|
||||
baseline, report = self.baseline_and_report(tmp_path)
|
||||
current = ConformanceReport([
|
||||
replace(report.results[0], state="unsupported", detail="lost")
|
||||
])
|
||||
delta = compare_conformance(baseline, current, self.contracted_tool())
|
||||
assert len(delta.coverage_losses) == 1
|
||||
assert delta.has_regressions
|
||||
|
||||
def test_removed_case_is_regression(self, tmp_path):
|
||||
from cmdforge.contract_testing import (
|
||||
ConformanceReport, compare_conformance,
|
||||
)
|
||||
baseline, _ = self.baseline_and_report(tmp_path)
|
||||
delta = compare_conformance(
|
||||
baseline, ConformanceReport(), self.contracted_tool()
|
||||
)
|
||||
assert len(delta.removed_cases) == 1
|
||||
assert delta.has_regressions
|
||||
|
||||
def test_output_shape_or_value_change_is_flagged(self, tmp_path):
|
||||
from dataclasses import replace
|
||||
from cmdforge.contract_testing import (
|
||||
ConformanceReport, compare_conformance,
|
||||
)
|
||||
baseline, report = self.baseline_and_report(tmp_path)
|
||||
current = ConformanceReport([
|
||||
replace(
|
||||
report.results[0],
|
||||
output_shape={"type": "object", "properties": {}},
|
||||
output_fingerprint="changed",
|
||||
)
|
||||
])
|
||||
delta = compare_conformance(baseline, current, self.contracted_tool())
|
||||
assert len(delta.output_changes) == 1
|
||||
assert delta.has_regressions
|
||||
|
||||
def test_contract_change_is_recorded(self, tmp_path):
|
||||
from cmdforge.contract_testing import compare_conformance
|
||||
|
||||
baseline, report = self.baseline_and_report(tmp_path)
|
||||
changed = self.contracted_tool(output_schema={"type": ["string", "null"]})
|
||||
delta = compare_conformance(baseline, report, changed)
|
||||
assert delta.contract_changed
|
||||
|
||||
@@ -342,3 +342,87 @@ class TestConformanceIntegration:
|
||||
assert report.ok
|
||||
assert report.generated_tests[0]["state"] == "unsupported"
|
||||
assert any("unsupported" in warning.lower() for warning in report.warnings)
|
||||
|
||||
def test_saved_baseline_is_compared_during_local_preflight(self, tmp_path):
|
||||
from cmdforge.contract_testing import (
|
||||
run_contract_tests, save_conformance_baseline,
|
||||
)
|
||||
from cmdforge.tool import Tool
|
||||
|
||||
baseline_tool = Tool(
|
||||
name="regression", version="1.0.0", input_schema={},
|
||||
output_schema={"type": "string"}, output="before",
|
||||
path=tmp_path / "config.yaml",
|
||||
)
|
||||
save_conformance_baseline(
|
||||
baseline_tool, run_contract_tests(baseline_tool), tmp_path
|
||||
)
|
||||
current = Tool(
|
||||
name="regression", version="2.0.0", input_schema={},
|
||||
output_schema={"type": "string"}, output="after",
|
||||
path=tmp_path / "config.yaml",
|
||||
)
|
||||
report = analyze_tool(current, include_contract_tests=True)
|
||||
assert report.regression["has_regressions"] is True
|
||||
assert report.regression["current_version"] == "2.0.0"
|
||||
assert any("Regression comparison" in warning for warning in report.warnings)
|
||||
|
||||
def test_baseline_inputs_are_rerun_after_input_schema_change(self, tmp_path):
|
||||
from cmdforge.contract_testing import (
|
||||
run_contract_tests, save_conformance_baseline,
|
||||
)
|
||||
from cmdforge.tool import Tool
|
||||
|
||||
baseline_tool = Tool(
|
||||
name="input-change", version="1.0.0", input_schema={},
|
||||
output_schema={"type": "string"}, output="stable",
|
||||
path=tmp_path / "config.yaml",
|
||||
)
|
||||
save_conformance_baseline(
|
||||
baseline_tool, run_contract_tests(baseline_tool), tmp_path
|
||||
)
|
||||
current = Tool(
|
||||
name="input-change", version="2.0.0",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}},
|
||||
"required": ["name"],
|
||||
},
|
||||
output_schema={"type": "string"}, output="stable",
|
||||
path=tmp_path / "config.yaml",
|
||||
)
|
||||
report = analyze_tool(current, include_contract_tests=True)
|
||||
assert report.regression["has_regressions"] is True
|
||||
assert len(report.regression["regressed"]) == 1
|
||||
assert any(
|
||||
result["input_value"] == {} and result["state"] == "failed"
|
||||
for result in report.generated_tests
|
||||
)
|
||||
|
||||
def test_invalid_baseline_is_an_advisory(self, tmp_path):
|
||||
from cmdforge.contract_testing import CONFORMANCE_FILE
|
||||
from cmdforge.tool import Tool
|
||||
|
||||
(tmp_path / CONFORMANCE_FILE).write_text("not json")
|
||||
tool = Tool(
|
||||
name="invalid-baseline", input_schema={}, output_schema={},
|
||||
output="ok", path=tmp_path / "config.yaml",
|
||||
)
|
||||
report = analyze_tool(tool, include_contract_tests=True)
|
||||
assert report.ok
|
||||
assert any("baseline is invalid" in warning.lower() for warning in report.warnings)
|
||||
|
||||
def test_partial_unsupported_conformance_is_not_hidden_by_a_pass(self, monkeypatch):
|
||||
from cmdforge.contract_testing import ConformanceReport, ConformanceResult
|
||||
from cmdforge.tool import Tool
|
||||
|
||||
monkeypatch.setattr(
|
||||
"cmdforge.contract_testing.run_contract_tests",
|
||||
lambda tool, test_inputs=None: ConformanceReport([
|
||||
ConformanceResult("case-a", "passed", "valid"),
|
||||
ConformanceResult("input-generation", "unsupported", "unsupported keyword"),
|
||||
]),
|
||||
)
|
||||
tool = Tool(name="mixed", input_schema={}, output_schema={})
|
||||
report = analyze_tool(tool, include_contract_tests=True)
|
||||
assert any("unsupported keyword" in warning for warning in report.warnings)
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Tests for conservative ToolStep JSON Schema compatibility."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from cmdforge.preflight import (
|
||||
PreflightReport, _check_toolstep_compatibility, analyze_tool,
|
||||
)
|
||||
from cmdforge.schema_compat import compare_json_schemas
|
||||
from cmdforge.tool import McpStep, PromptStep, Tool, ToolStep
|
||||
|
||||
|
||||
class TestSchemaCompatibility:
|
||||
def test_type_mismatch_is_incompatible(self):
|
||||
result = compare_json_schemas(
|
||||
{"type": "string"}, {"type": "integer"}
|
||||
)
|
||||
assert result.state == "incompatible"
|
||||
|
||||
def test_integer_is_accepted_by_number(self):
|
||||
result = compare_json_schemas(
|
||||
{"type": "integer"}, {"type": "number"}
|
||||
)
|
||||
assert result.state == "compatible"
|
||||
|
||||
def test_missing_required_property_is_incompatible(self):
|
||||
result = compare_json_schemas(
|
||||
{"type": "object", "properties": {}},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}},
|
||||
"required": ["name"],
|
||||
},
|
||||
)
|
||||
assert result.state == "incompatible"
|
||||
assert "required" in result.detail
|
||||
|
||||
def test_matching_object_property_is_compatible(self):
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string"}},
|
||||
"required": ["name"],
|
||||
}
|
||||
assert compare_json_schemas(schema, schema).state == "compatible"
|
||||
|
||||
def test_literal_enum_value_is_checked(self):
|
||||
assert compare_json_schemas(
|
||||
{"const": "fast"}, {"type": "string", "enum": ["fast", "slow"]}
|
||||
).state == "compatible"
|
||||
assert compare_json_schemas(
|
||||
{"const": "invalid"}, {"type": "string", "enum": ["fast"]}
|
||||
).state == "incompatible"
|
||||
|
||||
def test_unconstrained_producer_does_not_satisfy_consumer_enum(self):
|
||||
result = compare_json_schemas(
|
||||
{"type": "string"}, {"type": "string", "enum": ["fast"]}
|
||||
)
|
||||
assert result.state == "incompatible"
|
||||
|
||||
def test_closed_consumer_rejects_open_producer_object(self):
|
||||
result = compare_json_schemas(
|
||||
{"type": "object", "properties": {}},
|
||||
{"type": "object", "properties": {}, "additionalProperties": False},
|
||||
)
|
||||
assert result.state == "incompatible"
|
||||
|
||||
def test_open_producer_cannot_guarantee_typed_optional_consumer_field(self):
|
||||
result = compare_json_schemas(
|
||||
{"type": "object", "properties": {}},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"limit": {"type": "integer"}},
|
||||
},
|
||||
)
|
||||
assert result.state == "incompatible"
|
||||
assert "limit" in result.detail
|
||||
|
||||
def test_unproven_pattern_is_unknown(self):
|
||||
result = compare_json_schemas(
|
||||
{"type": "string"}, {"type": "string", "pattern": "^[a-z]+$"}
|
||||
)
|
||||
assert result.state == "unknown"
|
||||
|
||||
|
||||
class TestToolStepCompatibility:
|
||||
@staticmethod
|
||||
def child(input_schema):
|
||||
return Tool(name="child", input_schema=input_schema)
|
||||
|
||||
def analyze(self, parent, child):
|
||||
report = PreflightReport()
|
||||
with patch("cmdforge.tool.load_tool", return_value=child):
|
||||
_check_toolstep_compatibility(parent, report)
|
||||
return report
|
||||
|
||||
def test_compatible_stdin_contract(self):
|
||||
parent = Tool(
|
||||
name="parent",
|
||||
steps=[ToolStep(tool="child", output_var="result")],
|
||||
)
|
||||
child = self.child({
|
||||
"type": "object",
|
||||
"properties": {"input": {"type": "string"}},
|
||||
"required": ["input"],
|
||||
})
|
||||
report = self.analyze(parent, child)
|
||||
assert report.compatibility[0]["state"] == "compatible"
|
||||
|
||||
def test_upstream_serialization_mismatch_is_reported(self):
|
||||
producer = PromptStep(
|
||||
prompt="produce", provider="mock", output_var="value",
|
||||
output_schema={"type": "string"},
|
||||
)
|
||||
parent = Tool(
|
||||
name="parent",
|
||||
steps=[
|
||||
producer,
|
||||
ToolStep(
|
||||
tool="child", output_var="result", input_template="{value}"
|
||||
),
|
||||
],
|
||||
)
|
||||
child = self.child({
|
||||
"type": "object",
|
||||
"properties": {"input": {"type": "integer"}},
|
||||
"required": ["input"],
|
||||
})
|
||||
report = self.analyze(parent, child)
|
||||
finding = report.compatibility[0]
|
||||
assert finding["state"] == "incompatible"
|
||||
assert finding["upstream_schema"] == {"type": "string"}
|
||||
assert any("incompatible" in warning for warning in report.warnings)
|
||||
|
||||
def test_missing_required_argument_is_incompatible(self):
|
||||
parent = Tool(
|
||||
name="parent",
|
||||
steps=[ToolStep(tool="child", output_var="result")],
|
||||
)
|
||||
child = self.child({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"input": {"type": "string"},
|
||||
"limit": {"type": "integer"},
|
||||
},
|
||||
"required": ["input", "limit"],
|
||||
})
|
||||
report = self.analyze(parent, child)
|
||||
assert report.compatibility[0]["state"] == "incompatible"
|
||||
assert "limit" in report.compatibility[0]["detail"]
|
||||
|
||||
def test_explicit_argument_reflects_string_transport(self):
|
||||
parent = Tool(
|
||||
name="parent",
|
||||
steps=[ToolStep(
|
||||
tool="child", output_var="result", args={"limit": "{input}"}
|
||||
)],
|
||||
)
|
||||
child = self.child({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"input": {"type": "string"},
|
||||
"limit": {"type": "integer"},
|
||||
},
|
||||
"required": ["input", "limit"],
|
||||
})
|
||||
assert self.analyze(parent, child).compatibility[0]["state"] == "incompatible"
|
||||
|
||||
def test_mcp_steps_are_ignored(self):
|
||||
parent = Tool(
|
||||
name="parent",
|
||||
steps=[McpStep(server="remote", tool="same-name", output_var="result")],
|
||||
)
|
||||
report = PreflightReport()
|
||||
with patch("cmdforge.tool.load_tool") as loader:
|
||||
_check_toolstep_compatibility(parent, report)
|
||||
loader.assert_not_called()
|
||||
assert report.compatibility == []
|
||||
|
||||
def test_server_mode_does_not_resolve_local_tools(self):
|
||||
parent = Tool(
|
||||
name="published",
|
||||
steps=[ToolStep(tool="child", output_var="result")],
|
||||
)
|
||||
with patch("cmdforge.tool.load_tool") as loader:
|
||||
report = analyze_tool(parent, check_local_dependencies=False)
|
||||
loader.assert_not_called()
|
||||
assert report.compatibility == []
|
||||
Reference in New Issue
Block a user