CmdForge/tests/test_contract_testing.py

"""Tests for deterministic contract conformance (M8.V1)."""

import pytest
from jsonschema import validate

from cmdforge.contract_testing import (
    ConformanceResult,
    ConformanceReport,
    UnsupportedContract,
    _generate_inputs,
    _fallback_value,
    run_contract_tests,
)


class TestInputGeneration:
    def test_enum_generates_variants(self):
        schema = {
            "type": "object",
            "properties": {
                "mode": {"type": "string", "enum": ["fast", "accurate", "balanced"]},
                "limit": {"type": "integer", "default": 10},
            },
            "required": ["mode"],
        }
        cases = _generate_inputs(schema)
        assert len(cases) >= 1
        assert cases[0]["mode"] in ("fast", "accurate", "balanced")

    def test_fallback_string(self):
        cases = _generate_inputs({
            "type": "object",
            "properties": {"name": {"type": "string"}},
            "required": ["name"],
        })
        assert "test-name" in cases[0]["name"]

    def test_fallback_integer(self):
        cases = _generate_inputs({
            "type": "object",
            "properties": {"count": {"type": "integer"}},
            "required": ["count"],
        })
        assert cases[0]["count"] == 0

    def test_default_used(self):
        cases = _generate_inputs({
            "type": "object",
            "properties": {"name": {"type": "string", "default": "World"}},
            "required": ["name"],
        })
        assert cases[0]["name"] == "World"

    def test_scalar_schema_generates_scalar(self):
        cases = _generate_inputs({"type": "string"})
        assert cases == ["test-input"]

    def test_generated_value_respects_numeric_bounds(self):
        schema = {
            "type": "object",
            "properties": {
                "count": {"type": "integer", "minimum": 1, "maximum": 3}
            },
            "required": ["count"],
        }
        case = _generate_inputs(schema)[0]
        validate(case, schema)
        assert 1 <= case["count"] <= 3

    def test_array_and_nested_object_are_supported(self):
        schema = {
            "type": "array",
            "minItems": 1,
            "items": {
                "type": "object",
                "properties": {"name": {"type": "string"}},
                "required": ["name"],
            },
        }
        value = _generate_inputs(schema)[0]
        validate(value, schema)

    def test_unsynthesizable_pattern_is_unsupported(self):
        with pytest.raises(UnsupportedContract):
            _generate_inputs({"type": "string", "pattern": "^z{20}$"})

    def test_invalid_default_is_unsupported_not_returned(self):
        with pytest.raises(UnsupportedContract, match="valid input"):
            _generate_inputs({"type": "integer", "default": "wrong"})


class TestFallbackValue:
    def test_explicit_default(self):
        assert _fallback_value({"default": "hi"}, "x", []) == "hi"

    def test_examples(self):
        assert _fallback_value({"examples": ["a", "b"]}, "x", []) == "a"

    def test_enum(self):
        assert _fallback_value({"enum": ["yes", "no"]}, "x", []) == "yes"

    def test_type_fallback(self):
        assert _fallback_value({"type": "boolean"}, "flag", []) is True


class TestConformanceReport:
    def test_empty_is_not_run_not_pass(self):
        report = ConformanceReport()
        assert report.outcome == "not_run"
        assert not report.passed

    def test_invalid_state_is_rejected(self):
        with pytest.raises(ValueError, match="Unknown conformance state"):
            ConformanceResult("case", "unknown")

    def test_summary_counts(self):
        report = ConformanceReport(results=[
            ConformanceResult("a", "passed"),
            ConformanceResult("b", "passed"),
            ConformanceResult("c", "failed", "error"),
        ])
        assert report.summary == {"passed": 2, "failed": 1, "unsupported": 0, "not_run": 0}
        assert not report.passed


class TestRunContractTests:
    def test_missing_schemas_returns_not_run(self, tmp_path):
        from unittest.mock import patch
        from cmdforge.tool import Tool

        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
            tool = Tool(name="bare")
            report = run_contract_tests(tool)
            assert report.results[0].state == "not_run"

    def test_simple_tool_passes(self, tmp_path):
        from unittest.mock import patch
        from cmdforge.tool import Tool, PromptStep

        with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
            tool = Tool(
                name="greet",
                input_schema={
                    "type": "object",
                    "properties": {
                        "name": {"type": "string", "default": "World"},
                    },
                },
                output_schema={
                    "type": "object",
                    "properties": {"result": {"type": "string"}},
                    "required": ["result"],
                },
                steps=[
                    PromptStep(
                        prompt="Greet {name}",
                        provider="real-provider-that-must-not-run",
                        output_var="response",
                        output_schema={
                            "type": "object",
                            "properties": {"result": {"type": "string"}},
                            "required": ["result"],
                        },
                    ),
                ],
                output="{response}",
            )
            report = run_contract_tests(tool)
            assert report.outcome == "passed"
            assert report.results[0].state == "passed"

    def test_empty_schemas_are_explicit_and_runnable(self):
        from cmdforge.tool import Tool

        report = run_contract_tests(
            Tool(name="anything", input_schema={}, output_schema={}, output="ok")
        )
        assert report.outcome == "passed"

    @pytest.mark.parametrize("step_type", ["code", "tool", "mcp"])
    def test_side_effecting_steps_are_unsupported(self, step_type):
        from cmdforge.tool import CodeStep, McpStep, Tool, ToolStep

        steps = {
            "code": CodeStep(code="raise AssertionError('must not run')", output_var="x"),
            "tool": ToolStep(tool="external", output_var="x"),
            "mcp": McpStep(server="remote", tool="external", output_var="x"),
        }
        tool = Tool(
            name="unsafe",
            input_schema={},
            output_schema={},
            steps=[steps[step_type]],
            output="{x}",
        )
        report = run_contract_tests(tool)
        assert report.outcome == "unsupported"

    def test_schema_mismatch_is_a_failure(self):
        from cmdforge.tool import Tool

        report = run_contract_tests(
            Tool(
                name="mismatch",
                input_schema={},
                output_schema={"type": "integer"},
                output="not-an-integer",
            )
        )
        assert report.outcome == "failed"