Add evidence-based regression and schema compatibility checks

2026-07-20 14:21:42 -03:00
parent d81286e322
commit 501e6cd589
13 changed files with 1496 additions and 17 deletions
@@ -342,3 +342,87 @@ class TestConformanceIntegration:
        assert report.ok
        assert report.generated_tests[0]["state"] == "unsupported"
        assert any("unsupported" in warning.lower() for warning in report.warnings)
+
+    def test_saved_baseline_is_compared_during_local_preflight(self, tmp_path):
+        from cmdforge.contract_testing import (
+            run_contract_tests, save_conformance_baseline,
+        )
+        from cmdforge.tool import Tool
+
+        baseline_tool = Tool(
+            name="regression", version="1.0.0", input_schema={},
+            output_schema={"type": "string"}, output="before",
+            path=tmp_path / "config.yaml",
+        )
+        save_conformance_baseline(
+            baseline_tool, run_contract_tests(baseline_tool), tmp_path
+        )
+        current = Tool(
+            name="regression", version="2.0.0", input_schema={},
+            output_schema={"type": "string"}, output="after",
+            path=tmp_path / "config.yaml",
+        )
+        report = analyze_tool(current, include_contract_tests=True)
+        assert report.regression["has_regressions"] is True
+        assert report.regression["current_version"] == "2.0.0"
+        assert any("Regression comparison" in warning for warning in report.warnings)
+
+    def test_baseline_inputs_are_rerun_after_input_schema_change(self, tmp_path):
+        from cmdforge.contract_testing import (
+            run_contract_tests, save_conformance_baseline,
+        )
+        from cmdforge.tool import Tool
+
+        baseline_tool = Tool(
+            name="input-change", version="1.0.0", input_schema={},
+            output_schema={"type": "string"}, output="stable",
+            path=tmp_path / "config.yaml",
+        )
+        save_conformance_baseline(
+            baseline_tool, run_contract_tests(baseline_tool), tmp_path
+        )
+        current = Tool(
+            name="input-change", version="2.0.0",
+            input_schema={
+                "type": "object",
+                "properties": {"name": {"type": "string"}},
+                "required": ["name"],
+            },
+            output_schema={"type": "string"}, output="stable",
+            path=tmp_path / "config.yaml",
+        )
+        report = analyze_tool(current, include_contract_tests=True)
+        assert report.regression["has_regressions"] is True
+        assert len(report.regression["regressed"]) == 1
+        assert any(
+            result["input_value"] == {} and result["state"] == "failed"
+            for result in report.generated_tests
+        )
+
+    def test_invalid_baseline_is_an_advisory(self, tmp_path):
+        from cmdforge.contract_testing import CONFORMANCE_FILE
+        from cmdforge.tool import Tool
+
+        (tmp_path / CONFORMANCE_FILE).write_text("not json")
+        tool = Tool(
+            name="invalid-baseline", input_schema={}, output_schema={},
+            output="ok", path=tmp_path / "config.yaml",
+        )
+        report = analyze_tool(tool, include_contract_tests=True)
+        assert report.ok
+        assert any("baseline is invalid" in warning.lower() for warning in report.warnings)
+
+    def test_partial_unsupported_conformance_is_not_hidden_by_a_pass(self, monkeypatch):
+        from cmdforge.contract_testing import ConformanceReport, ConformanceResult
+        from cmdforge.tool import Tool
+
+        monkeypatch.setattr(
+            "cmdforge.contract_testing.run_contract_tests",
+            lambda tool, test_inputs=None: ConformanceReport([
+                ConformanceResult("case-a", "passed", "valid"),
+                ConformanceResult("input-generation", "unsupported", "unsupported keyword"),
+            ]),
+        )
+        tool = Tool(name="mixed", input_schema={}, output_schema={})
+        report = analyze_tool(tool, include_contract_tests=True)
+        assert any("unsupported keyword" in warning for warning in report.warnings)