M8.0/M8.V1: Add contract inference engine and deterministic conformance testing

tests/test_cli.py

@@ -560,6 +560,35 @@ def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
     assert "official/summarize" in capsys.readouterr().out
 
 
+def test_inspect_shows_contract_proposal_and_passing_conformance(
+    monkeypatch, capsys
+):
+    from cmdforge.cli import cmd_inspect
+
+    step_schema = {
+        "type": "object",
+        "properties": {"answer": {"type": "string"}},
+        "required": ["answer"],
+    }
+    tool = Tool(
+        name="contracted",
+        arguments=[],
+        input_schema={},
+        output_schema=step_schema,
+        steps=[PromptStep(
+            prompt="Answer", provider="paid-provider", output_var="answer",
+            output_schema=step_schema,
+        )],
+        output="{answer}",
+    )
+    monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
+
+    assert cmd_inspect(SimpleNamespace(name="contracted", registry=False)) == 0
+    output = capsys.readouterr().out
+    assert "Contract conformance" in output
+    assert "PASSED" in output
+
+
 def test_switch_to_existing_tool_closes_creation_page_first():
     pytest.importorskip("PySide6")
     from cmdforge.gui.pages.tool_builder_page import _switch_to_existing_tool

tests/test_collection_api.py

@@ -180,6 +180,7 @@ class TestPublishPreflightEndpoint:
         assert response.status_code == 200
         report = response.get_json()["data"]["preflight"]
         assert report["errors"] == []
+        assert report["generated_tests"] == []
 
     def test_invalid_contract_is_rejected(self, client, auth_headers):
         response = client.post(

tests/test_contract_testing.py

@@ -0,0 +1,210 @@
+"""Tests for deterministic contract conformance (M8.V1)."""
+
+import pytest
+from jsonschema import validate
+
+from cmdforge.contract_testing import (
+    ConformanceResult,
+    ConformanceReport,
+    UnsupportedContract,
+    _generate_inputs,
+    _fallback_value,
+    run_contract_tests,
+)
+
+
+class TestInputGeneration:
+    def test_enum_generates_variants(self):
+        schema = {
+            "type": "object",
+            "properties": {
+                "mode": {"type": "string", "enum": ["fast", "accurate", "balanced"]},
+                "limit": {"type": "integer", "default": 10},
+            },
+            "required": ["mode"],
+        }
+        cases = _generate_inputs(schema)
+        assert len(cases) >= 1
+        assert cases[0]["mode"] in ("fast", "accurate", "balanced")
+
+    def test_fallback_string(self):
+        cases = _generate_inputs({
+            "type": "object",
+            "properties": {"name": {"type": "string"}},
+            "required": ["name"],
+        })
+        assert "test-name" in cases[0]["name"]
+
+    def test_fallback_integer(self):
+        cases = _generate_inputs({
+            "type": "object",
+            "properties": {"count": {"type": "integer"}},
+            "required": ["count"],
+        })
+        assert cases[0]["count"] == 0
+
+    def test_default_used(self):
+        cases = _generate_inputs({
+            "type": "object",
+            "properties": {"name": {"type": "string", "default": "World"}},
+            "required": ["name"],
+        })
+        assert cases[0]["name"] == "World"
+
+    def test_scalar_schema_generates_scalar(self):
+        cases = _generate_inputs({"type": "string"})
+        assert cases == ["test-input"]
+
+    def test_generated_value_respects_numeric_bounds(self):
+        schema = {
+            "type": "object",
+            "properties": {
+                "count": {"type": "integer", "minimum": 1, "maximum": 3}
+            },
+            "required": ["count"],
+        }
+        case = _generate_inputs(schema)[0]
+        validate(case, schema)
+        assert 1 <= case["count"] <= 3
+
+    def test_array_and_nested_object_are_supported(self):
+        schema = {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+                "type": "object",
+                "properties": {"name": {"type": "string"}},
+                "required": ["name"],
+            },
+        }
+        value = _generate_inputs(schema)[0]
+        validate(value, schema)
+
+    def test_unsynthesizable_pattern_is_unsupported(self):
+        with pytest.raises(UnsupportedContract):
+            _generate_inputs({"type": "string", "pattern": "^z{20}$"})
+
+    def test_invalid_default_is_unsupported_not_returned(self):
+        with pytest.raises(UnsupportedContract, match="valid input"):
+            _generate_inputs({"type": "integer", "default": "wrong"})
+
+
+class TestFallbackValue:
+    def test_explicit_default(self):
+        assert _fallback_value({"default": "hi"}, "x", []) == "hi"
+
+    def test_examples(self):
+        assert _fallback_value({"examples": ["a", "b"]}, "x", []) == "a"
+
+    def test_enum(self):
+        assert _fallback_value({"enum": ["yes", "no"]}, "x", []) == "yes"
+
+    def test_type_fallback(self):
+        assert _fallback_value({"type": "boolean"}, "flag", []) is True
+
+
+class TestConformanceReport:
+    def test_empty_is_not_run_not_pass(self):
+        report = ConformanceReport()
+        assert report.outcome == "not_run"
+        assert not report.passed
+
+    def test_invalid_state_is_rejected(self):
+        with pytest.raises(ValueError, match="Unknown conformance state"):
+            ConformanceResult("case", "unknown")
+
+    def test_summary_counts(self):
+        report = ConformanceReport(results=[
+            ConformanceResult("a", "passed"),
+            ConformanceResult("b", "passed"),
+            ConformanceResult("c", "failed", "error"),
+        ])
+        assert report.summary == {"passed": 2, "failed": 1, "unsupported": 0, "not_run": 0}
+        assert not report.passed
+
+
+class TestRunContractTests:
+    def test_missing_schemas_returns_not_run(self, tmp_path):
+        from unittest.mock import patch
+        from cmdforge.tool import Tool
+
+        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
+            tool = Tool(name="bare")
+            report = run_contract_tests(tool)
+            assert report.results[0].state == "not_run"
+
+    def test_simple_tool_passes(self, tmp_path):
+        from unittest.mock import patch
+        from cmdforge.tool import Tool, PromptStep
+
+        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
+            tool = Tool(
+                name="greet",
+                input_schema={
+                    "type": "object",
+                    "properties": {
+                        "name": {"type": "string", "default": "World"},
+                    },
+                },
+                output_schema={
+                    "type": "object",
+                    "properties": {"result": {"type": "string"}},
+                    "required": ["result"],
+                },
+                steps=[
+                    PromptStep(
+                        prompt="Greet {name}",
+                        provider="real-provider-that-must-not-run",
+                        output_var="response",
+                        output_schema={
+                            "type": "object",
+                            "properties": {"result": {"type": "string"}},
+                            "required": ["result"],
+                        },
+                    ),
+                ],
+                output="{response}",
+            )
+            report = run_contract_tests(tool)
+            assert report.outcome == "passed"
+            assert report.results[0].state == "passed"
+
+    def test_empty_schemas_are_explicit_and_runnable(self):
+        from cmdforge.tool import Tool
+
+        report = run_contract_tests(
+            Tool(name="anything", input_schema={}, output_schema={}, output="ok")
+        )
+        assert report.outcome == "passed"
+
+    @pytest.mark.parametrize("step_type", ["code", "tool", "mcp"])
+    def test_side_effecting_steps_are_unsupported(self, step_type):
+        from cmdforge.tool import CodeStep, McpStep, Tool, ToolStep
+
+        steps = {
+            "code": CodeStep(code="raise AssertionError('must not run')", output_var="x"),
+            "tool": ToolStep(tool="external", output_var="x"),
+            "mcp": McpStep(server="remote", tool="external", output_var="x"),
+        }
+        tool = Tool(
+            name="unsafe",
+            input_schema={},
+            output_schema={},
+            steps=[steps[step_type]],
+            output="{x}",
+        )
+        report = run_contract_tests(tool)
+        assert report.outcome == "unsupported"
+
+    def test_schema_mismatch_is_a_failure(self):
+        from cmdforge.tool import Tool
+
+        report = run_contract_tests(
+            Tool(
+                name="mismatch",
+                input_schema={},
+                output_schema={"type": "integer"},
+                output="not-an-integer",
+            )
+        )
+        assert report.outcome == "failed"

tests/test_preflight.py

@@ -38,12 +38,22 @@ class TestPreflightReport:
         serialized["errors"].append("another")
         assert report.errors == ["bad"]
 
+    def test_to_dict_copies_contract_proposal(self):
+        report = PreflightReport(
+            contract_proposal={"input_schema": {"type": "string"}}
+        )
+        serialized = report.to_dict()
+        serialized["contract_proposal"]["input_schema"]["type"] = "integer"
+        assert report.contract_proposal["input_schema"]["type"] == "string"
+
 
 class TestAnalyzeTool:
     def test_no_steps_warns(self, tmp_path):
         from cmdforge.tool import Tool, save_tool
 
-        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
+        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"), patch(
+            "cmdforge.tool.BIN_DIR", tmp_path / "bin"
+        ):
             tool = Tool(name="empty-tool")
             save_tool(tool)
 
@@ -134,3 +144,201 @@ class TestAnalyzeTool:
             _check_similar_tools(
                 SimpleNamespace(name="summarize"), PreflightReport(), Client()
             )
+
+
+class TestContractInference:
+    def test_no_arguments_no_input(self):
+        from cmdforge.tool import Tool
+        from cmdforge.preflight import infer_contracts
+
+        tool = Tool(name="empty", output="{response}")
+        result = infer_contracts(tool)
+        assert result["input_schema"] is None
+        assert result["output_schema"] is None
+        assert result["confidence"] == "low"
+
+    def test_argument_becomes_property(self):
+        from cmdforge.tool import Tool, ToolArgument
+        from cmdforge.preflight import infer_contracts
+
+        tool = Tool(
+            name="greet",
+            arguments=[ToolArgument(flag="--name", variable="name", description="Who to greet")],
+            output="Hello",
+        )
+        result = infer_contracts(tool)
+        assert result["input_schema"] is not None
+        assert "name" in result["input_schema"]["properties"]
+        assert result["input_schema"]["properties"]["name"]["description"] == "Who to greet"
+
+    def test_stdin_detected_when_input_used(self):
+        from cmdforge.tool import Tool, PromptStep
+        from cmdforge.preflight import infer_contracts
+
+        tool = Tool(
+            name="echo",
+            steps=[PromptStep(prompt="Repeat: {input}", provider="mock", output_var="response")],
+            output="{response}",
+        )
+        result = infer_contracts(tool)
+        assert result["input_schema"] is not None
+        assert "input" in result["input_schema"]["properties"]
+        assert any("stdin" in e.lower() for e in result["evidence"])
+
+    def test_enum_propagates(self):
+        from cmdforge.tool import Tool, ToolArgument
+        from cmdforge.preflight import infer_contracts
+
+        arg = ToolArgument(flag="--mode", variable="mode")
+        arg.enum = ["fast", "accurate"]
+        tool = Tool(name="tool", arguments=[arg], output="done")
+        result = infer_contracts(tool)
+        assert result["input_schema"]["properties"]["mode"]["enum"] == ["fast", "accurate"]
+
+    def test_explicit_empty_schema_not_overwritten(self):
+        from cmdforge.tool import Tool
+        from cmdforge.preflight import infer_contracts
+
+        tool = Tool(name="tool", input_schema={}, output_schema={})
+        result = infer_contracts(tool)
+        assert result["input_schema"] is None
+        assert result["output_schema"] is None
+
+    def test_output_propagated_from_step_schema(self):
+        from cmdforge.tool import Tool, PromptStep
+        from cmdforge.preflight import infer_contracts
+
+        step = PromptStep(
+            prompt="Analyze: {input}",
+            provider="mock",
+            output_var="analysis",
+        )
+        step.output_schema = {
+            "type": "object",
+            "properties": {"summary": {"type": "string"}},
+        }
+        tool = Tool(name="analyze", steps=[step], output="{analysis}")
+        result = infer_contracts(tool)
+        assert result["output_schema"] is not None
+        assert result["output_schema"]["type"] == "object"
+        assert "summary" in result["output_schema"]["properties"]
+        assert result["confidence"] == "high"
+
+    @pytest.mark.parametrize("schema_type", ["string", "number", "array", "boolean"])
+    def test_non_object_output_schema_is_propagated(self, schema_type):
+        from cmdforge.tool import PromptStep, Tool
+        from cmdforge.preflight import infer_contracts
+
+        schema = {"type": schema_type}
+        step = PromptStep(
+            prompt="Generate", provider="mock", output_var="result",
+            output_schema=schema,
+        )
+        proposal = infer_contracts(
+            Tool(name="typed", steps=[step], output="{result}")
+        )
+        assert proposal["output_schema"] == schema
+
+    def test_proposed_schema_does_not_alias_step_schema(self):
+        from cmdforge.tool import PromptStep, Tool
+        from cmdforge.preflight import infer_contracts
+
+        step = PromptStep(
+            prompt="Generate", provider="mock", output_var="result",
+            output_schema={
+                "type": "object",
+                "properties": {"value": {"type": "string"}},
+            },
+        )
+        proposal = infer_contracts(
+            Tool(name="copy", steps=[step], output="{result}")
+        )
+        proposal["output_schema"]["properties"]["value"]["type"] = "integer"
+        assert step.output_schema["properties"]["value"]["type"] == "string"
+
+    def test_stdin_detected_in_nested_tool_input(self):
+        from cmdforge.tool import Tool, ToolStep
+        from cmdforge.preflight import infer_contracts
+
+        proposal = infer_contracts(Tool(
+            name="nested",
+            steps=[ToolStep(tool="child", output_var="result")],
+            output="{result}",
+        ))
+        assert "input" in proposal["input_schema"]["properties"]
+
+    def test_evidence_uses_real_argument_flag(self):
+        from cmdforge.tool import Tool, ToolArgument
+        from cmdforge.preflight import infer_contracts
+
+        proposal = infer_contracts(Tool(
+            name="flags",
+            arguments=[ToolArgument(flag="-n", variable="name")],
+            output="constant",
+        ))
+        assert any("argument -n" in item for item in proposal["evidence"])
+
+    def test_explicit_contracts_produce_high_confidence(self):
+        from cmdforge.tool import Tool
+        from cmdforge.preflight import infer_contracts
+
+        proposal = infer_contracts(
+            Tool(name="explicit", input_schema={}, output_schema={})
+        )
+        assert proposal["confidence"] == "high"
+
+    def test_bare_variable_output_stays_bare(self):
+        from cmdforge.tool import Tool
+        from cmdforge.preflight import infer_contracts
+
+        tool = Tool(name="passthrough", steps=[], output="{response}")
+        result = infer_contracts(tool)
+        assert result["output_schema"] is None
+        assert any("source not found" in e.lower() for e in result["evidence"])
+
+
+class TestConformanceIntegration:
+    def test_contract_tests_are_opt_in_for_server_side_preflight(self):
+        from cmdforge.tool import Tool
+
+        tool = Tool(name="server-safe", input_schema={}, output_schema={})
+        with patch("cmdforge.contract_testing.run_contract_tests") as runner:
+            analyze_tool(tool)
+        runner.assert_not_called()
+
+    def test_analyze_tool_records_passing_contract_case(self):
+        from cmdforge.tool import PromptStep, Tool
+
+        schema = {
+            "type": "object",
+            "properties": {"answer": {"type": "string"}},
+            "required": ["answer"],
+        }
+        tool = Tool(
+            name="contracted",
+            input_schema={},
+            output_schema=schema,
+            steps=[PromptStep(
+                prompt="Answer", provider="paid-provider", output_var="answer",
+                output_schema=schema,
+            )],
+            output="{answer}",
+        )
+        report = analyze_tool(tool, include_contract_tests=True)
+        assert report.ok
+        assert report.generated_tests[0]["state"] == "passed"
+
+    def test_unsupported_execution_is_advisory_not_failure(self):
+        from cmdforge.tool import CodeStep, Tool
+
+        tool = Tool(
+            name="code-tool",
+            input_schema={},
+            output_schema={},
+            steps=[CodeStep(code="result = 1", output_var="result")],
+            output="{result}",
+        )
+        report = analyze_tool(tool, include_contract_tests=True)
+        assert report.ok
+        assert report.generated_tests[0]["state"] == "unsupported"
+        assert any("unsupported" in warning.lower() for warning in report.warnings)