Add evidence-based regression and schema compatibility checks

tests/test_contract_testing.py

@@ -1,5 +1,7 @@
 """Tests for deterministic contract conformance (M8.V1)."""
 
+from pathlib import Path
+
 import pytest
 from jsonschema import validate
 
@@ -11,6 +13,7 @@ from cmdforge.contract_testing import (
     _fallback_value,
     run_contract_tests,
 )
+from cmdforge.tool import Tool
 
 
 class TestInputGeneration:
@@ -208,3 +211,158 @@ class TestRunContractTests:
             )
         )
         assert report.outcome == "failed"
+
+
+class TestRegressionComparison:
+    @staticmethod
+    def contracted_tool(version="1.0.0", output_schema=None):
+        from cmdforge.tool import Tool
+
+        return Tool(
+            name="test",
+            version=version,
+            input_schema={},
+            output_schema=output_schema or {"type": "string"},
+            output="stable",
+        )
+
+    def passing_report(self, tool=None):
+        return run_contract_tests(tool or self.contracted_tool())
+
+    def test_save_and_load_baseline(self, tmp_path):
+        from cmdforge.contract_testing import (
+            save_conformance_baseline, load_conformance_baseline,
+        )
+        import stat
+
+        tool = self.contracted_tool()
+        report = self.passing_report(tool)
+        path = save_conformance_baseline(tool, report, tmp_path)
+        assert path.exists()
+        assert stat.S_IMODE(path.stat().st_mode) == 0o600
+
+        loaded = load_conformance_baseline(tmp_path)
+        assert loaded["format_version"] == 1
+        assert loaded["tool"]["version"] == "1.0.0"
+        assert loaded["contract_hash"]
+        assert len(loaded["results"]) == 1
+        assert loaded["results"][0]["case_id"].startswith("case-")
+        assert loaded["results"][0]["output_shape"] == {"type": "string"}
+
+    def test_missing_baseline(self, tmp_path):
+        from cmdforge.contract_testing import load_conformance_baseline
+        assert load_conformance_baseline(tmp_path) is None
+
+    def test_corrupt_baseline_is_rejected(self, tmp_path):
+        from cmdforge.contract_testing import (
+            BaselineFormatError, CONFORMANCE_FILE, load_conformance_baseline,
+        )
+
+        (tmp_path / CONFORMANCE_FILE).write_text("[]")
+        with pytest.raises(BaselineFormatError, match="root"):
+            load_conformance_baseline(tmp_path)
+
+    def test_tampered_baseline_evidence_is_rejected(self, tmp_path):
+        from cmdforge.contract_testing import (
+            BaselineFormatError, CONFORMANCE_FILE, load_conformance_baseline,
+            save_conformance_baseline,
+        )
+        import json
+
+        tool = self.contracted_tool()
+        save_conformance_baseline(tool, self.passing_report(tool), tmp_path)
+        path = tmp_path / CONFORMANCE_FILE
+        data = json.loads(path.read_text())
+        data["results"][0]["input_value"] = {"tampered": True}
+        path.write_text(json.dumps(data))
+
+        with pytest.raises(BaselineFormatError, match="fingerprint mismatch"):
+            load_conformance_baseline(tmp_path)
+
+    def test_failed_report_cannot_replace_baseline(self, tmp_path):
+        from cmdforge.contract_testing import save_conformance_baseline
+
+        report = run_contract_tests(Tool(
+            name="failed", input_schema={}, output_schema={"type": "integer"},
+            output="text",
+        ))
+        with pytest.raises(ValueError, match="passing"):
+            save_conformance_baseline(self.contracted_tool(), report, tmp_path)
+
+    def baseline_and_report(self, tmp_path):
+        from cmdforge.contract_testing import (
+            load_conformance_baseline, save_conformance_baseline,
+        )
+
+        tool = self.contracted_tool()
+        report = self.passing_report(tool)
+        save_conformance_baseline(tool, report, tmp_path)
+        return load_conformance_baseline(tmp_path), report
+
+    def test_detects_regression(self):
+        from dataclasses import replace
+        from cmdforge.contract_testing import (
+            ConformanceReport, compare_conformance,
+        )
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as directory:
+            baseline, report = self.baseline_and_report(Path(directory))
+        current = ConformanceReport([
+            replace(report.results[0], state="failed", detail="now broken")
+        ])
+        delta = compare_conformance(
+            baseline, current, self.contracted_tool(version="2.0.0")
+        )
+        assert len(delta.regressed) == 1
+        assert delta.has_regressions
+        assert delta.current_version == "2.0.0"
+
+    def test_lost_coverage_is_regression(self, tmp_path):
+        from dataclasses import replace
+        from cmdforge.contract_testing import (
+            ConformanceReport, compare_conformance,
+        )
+        baseline, report = self.baseline_and_report(tmp_path)
+        current = ConformanceReport([
+            replace(report.results[0], state="unsupported", detail="lost")
+        ])
+        delta = compare_conformance(baseline, current, self.contracted_tool())
+        assert len(delta.coverage_losses) == 1
+        assert delta.has_regressions
+
+    def test_removed_case_is_regression(self, tmp_path):
+        from cmdforge.contract_testing import (
+            ConformanceReport, compare_conformance,
+        )
+        baseline, _ = self.baseline_and_report(tmp_path)
+        delta = compare_conformance(
+            baseline, ConformanceReport(), self.contracted_tool()
+        )
+        assert len(delta.removed_cases) == 1
+        assert delta.has_regressions
+
+    def test_output_shape_or_value_change_is_flagged(self, tmp_path):
+        from dataclasses import replace
+        from cmdforge.contract_testing import (
+            ConformanceReport, compare_conformance,
+        )
+        baseline, report = self.baseline_and_report(tmp_path)
+        current = ConformanceReport([
+            replace(
+                report.results[0],
+                output_shape={"type": "object", "properties": {}},
+                output_fingerprint="changed",
+            )
+        ])
+        delta = compare_conformance(baseline, current, self.contracted_tool())
+        assert len(delta.output_changes) == 1
+        assert delta.has_regressions
+
+    def test_contract_change_is_recorded(self, tmp_path):
+        from cmdforge.contract_testing import compare_conformance
+
+        baseline, report = self.baseline_and_report(tmp_path)
+        changed = self.contracted_tool(output_schema={"type": ["string", "null"]})
+        delta = compare_conformance(baseline, report, changed)
+        assert delta.contract_changed