M8.0/M8.V1: Add contract inference engine and deterministic conformance testing

2026-07-20 13:34:13 -03:00
parent ba29e4dc4e
commit d81286e322
12 changed files with 1191 additions and 5 deletions
@@ -0,0 +1,210 @@
+"""Tests for deterministic contract conformance (M8.V1)."""
+
+import pytest
+from jsonschema import validate
+
+from cmdforge.contract_testing import (
+    ConformanceResult,
+    ConformanceReport,
+    UnsupportedContract,
+    _generate_inputs,
+    _fallback_value,
+    run_contract_tests,
+)
+
+
+class TestInputGeneration:
+    def test_enum_generates_variants(self):
+        schema = {
+            "type": "object",
+            "properties": {
+                "mode": {"type": "string", "enum": ["fast", "accurate", "balanced"]},
+                "limit": {"type": "integer", "default": 10},
+            },
+            "required": ["mode"],
+        }
+        cases = _generate_inputs(schema)
+        assert len(cases) >= 1
+        assert cases[0]["mode"] in ("fast", "accurate", "balanced")
+
+    def test_fallback_string(self):
+        cases = _generate_inputs({
+            "type": "object",
+            "properties": {"name": {"type": "string"}},
+            "required": ["name"],
+        })
+        assert "test-name" in cases[0]["name"]
+
+    def test_fallback_integer(self):
+        cases = _generate_inputs({
+            "type": "object",
+            "properties": {"count": {"type": "integer"}},
+            "required": ["count"],
+        })
+        assert cases[0]["count"] == 0
+
+    def test_default_used(self):
+        cases = _generate_inputs({
+            "type": "object",
+            "properties": {"name": {"type": "string", "default": "World"}},
+            "required": ["name"],
+        })
+        assert cases[0]["name"] == "World"
+
+    def test_scalar_schema_generates_scalar(self):
+        cases = _generate_inputs({"type": "string"})
+        assert cases == ["test-input"]
+
+    def test_generated_value_respects_numeric_bounds(self):
+        schema = {
+            "type": "object",
+            "properties": {
+                "count": {"type": "integer", "minimum": 1, "maximum": 3}
+            },
+            "required": ["count"],
+        }
+        case = _generate_inputs(schema)[0]
+        validate(case, schema)
+        assert 1 <= case["count"] <= 3
+
+    def test_array_and_nested_object_are_supported(self):
+        schema = {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "type": "object",
+                "properties": {"name": {"type": "string"}},
+                "required": ["name"],
+            },
+        }
+        value = _generate_inputs(schema)[0]
+        validate(value, schema)
+
+    def test_unsynthesizable_pattern_is_unsupported(self):
+        with pytest.raises(UnsupportedContract):
+            _generate_inputs({"type": "string", "pattern": "^z{20}$"})
+
+    def test_invalid_default_is_unsupported_not_returned(self):
+        with pytest.raises(UnsupportedContract, match="valid input"):
+            _generate_inputs({"type": "integer", "default": "wrong"})
+
+
+class TestFallbackValue:
+    def test_explicit_default(self):
+        assert _fallback_value({"default": "hi"}, "x", []) == "hi"
+
+    def test_examples(self):
+        assert _fallback_value({"examples": ["a", "b"]}, "x", []) == "a"
+
+    def test_enum(self):
+        assert _fallback_value({"enum": ["yes", "no"]}, "x", []) == "yes"
+
+    def test_type_fallback(self):
+        assert _fallback_value({"type": "boolean"}, "flag", []) is True
+
+
+class TestConformanceReport:
+    def test_empty_is_not_run_not_pass(self):
+        report = ConformanceReport()
+        assert report.outcome == "not_run"
+        assert not report.passed
+
+    def test_invalid_state_is_rejected(self):
+        with pytest.raises(ValueError, match="Unknown conformance state"):
+            ConformanceResult("case", "unknown")
+
+    def test_summary_counts(self):
+        report = ConformanceReport(results=[
+            ConformanceResult("a", "passed"),
+            ConformanceResult("b", "passed"),
+            ConformanceResult("c", "failed", "error"),
+        ])
+        assert report.summary == {"passed": 2, "failed": 1, "unsupported": 0, "not_run": 0}
+        assert not report.passed
+
+
+class TestRunContractTests:
+    def test_missing_schemas_returns_not_run(self, tmp_path):
+        from unittest.mock import patch
+        from cmdforge.tool import Tool
+
+        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
+            tool = Tool(name="bare")
+            report = run_contract_tests(tool)
+            assert report.results[0].state == "not_run"
+
+    def test_simple_tool_passes(self, tmp_path):
+        from unittest.mock import patch
+        from cmdforge.tool import Tool, PromptStep
+
+        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
+            tool = Tool(
+                name="greet",
+                input_schema={
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "default": "World"},
+                    },
+                },
+                output_schema={
+                    "type": "object",
+                    "properties": {"result": {"type": "string"}},
+                    "required": ["result"],
+                },
+                steps=[
+                    PromptStep(
+                        prompt="Greet {name}",
+                        provider="real-provider-that-must-not-run",
+                        output_var="response",
+                        output_schema={
+                            "type": "object",
+                            "properties": {"result": {"type": "string"}},
+                            "required": ["result"],
+                        },
+                    ),
+                ],
+                output="{response}",
+            )
+            report = run_contract_tests(tool)
+            assert report.outcome == "passed"
+            assert report.results[0].state == "passed"
+
+    def test_empty_schemas_are_explicit_and_runnable(self):
+        from cmdforge.tool import Tool
+
+        report = run_contract_tests(
+            Tool(name="anything", input_schema={}, output_schema={}, output="ok")
+        )
+        assert report.outcome == "passed"
+
+    @pytest.mark.parametrize("step_type", ["code", "tool", "mcp"])
+    def test_side_effecting_steps_are_unsupported(self, step_type):
+        from cmdforge.tool import CodeStep, McpStep, Tool, ToolStep
+
+        steps = {
+            "code": CodeStep(code="raise AssertionError('must not run')", output_var="x"),
+            "tool": ToolStep(tool="external", output_var="x"),
+            "mcp": McpStep(server="remote", tool="external", output_var="x"),
+        }
+        tool = Tool(
+            name="unsafe",
+            input_schema={},
+            output_schema={},
+            steps=[steps[step_type]],
+            output="{x}",
+        )
+        report = run_contract_tests(tool)
+        assert report.outcome == "unsupported"
+
+    def test_schema_mismatch_is_a_failure(self):
+        from cmdforge.tool import Tool
+
+        report = run_contract_tests(
+            Tool(
+                name="mismatch",
+                input_schema={},
+                output_schema={"type": "integer"},
+                output="not-an-integer",
+            )
+        )
+        assert report.outcome == "failed"