"""Tests for deterministic contract conformance (M8.V1).""" import pytest from jsonschema import validate from cmdforge.contract_testing import ( ConformanceResult, ConformanceReport, UnsupportedContract, _generate_inputs, _fallback_value, run_contract_tests, ) class TestInputGeneration: def test_enum_generates_variants(self): schema = { "type": "object", "properties": { "mode": {"type": "string", "enum": ["fast", "accurate", "balanced"]}, "limit": {"type": "integer", "default": 10}, }, "required": ["mode"], } cases = _generate_inputs(schema) assert len(cases) >= 1 assert cases[0]["mode"] in ("fast", "accurate", "balanced") def test_fallback_string(self): cases = _generate_inputs({ "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], }) assert "test-name" in cases[0]["name"] def test_fallback_integer(self): cases = _generate_inputs({ "type": "object", "properties": {"count": {"type": "integer"}}, "required": ["count"], }) assert cases[0]["count"] == 0 def test_default_used(self): cases = _generate_inputs({ "type": "object", "properties": {"name": {"type": "string", "default": "World"}}, "required": ["name"], }) assert cases[0]["name"] == "World" def test_scalar_schema_generates_scalar(self): cases = _generate_inputs({"type": "string"}) assert cases == ["test-input"] def test_generated_value_respects_numeric_bounds(self): schema = { "type": "object", "properties": { "count": {"type": "integer", "minimum": 1, "maximum": 3} }, "required": ["count"], } case = _generate_inputs(schema)[0] validate(case, schema) assert 1 <= case["count"] <= 3 def test_array_and_nested_object_are_supported(self): schema = { "type": "array", "minItems": 1, "items": { "type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"], }, } value = _generate_inputs(schema)[0] validate(value, schema) def test_unsynthesizable_pattern_is_unsupported(self): with pytest.raises(UnsupportedContract): _generate_inputs({"type": "string", "pattern": "^z{20}$"}) def test_invalid_default_is_unsupported_not_returned(self): with pytest.raises(UnsupportedContract, match="valid input"): _generate_inputs({"type": "integer", "default": "wrong"}) class TestFallbackValue: def test_explicit_default(self): assert _fallback_value({"default": "hi"}, "x", []) == "hi" def test_examples(self): assert _fallback_value({"examples": ["a", "b"]}, "x", []) == "a" def test_enum(self): assert _fallback_value({"enum": ["yes", "no"]}, "x", []) == "yes" def test_type_fallback(self): assert _fallback_value({"type": "boolean"}, "flag", []) is True class TestConformanceReport: def test_empty_is_not_run_not_pass(self): report = ConformanceReport() assert report.outcome == "not_run" assert not report.passed def test_invalid_state_is_rejected(self): with pytest.raises(ValueError, match="Unknown conformance state"): ConformanceResult("case", "unknown") def test_summary_counts(self): report = ConformanceReport(results=[ ConformanceResult("a", "passed"), ConformanceResult("b", "passed"), ConformanceResult("c", "failed", "error"), ]) assert report.summary == {"passed": 2, "failed": 1, "unsupported": 0, "not_run": 0} assert not report.passed class TestRunContractTests: def test_missing_schemas_returns_not_run(self, tmp_path): from unittest.mock import patch from cmdforge.tool import Tool with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"): tool = Tool(name="bare") report = run_contract_tests(tool) assert report.results[0].state == "not_run" def test_simple_tool_passes(self, tmp_path): from unittest.mock import patch from cmdforge.tool import Tool, PromptStep with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"): tool = Tool( name="greet", input_schema={ "type": "object", "properties": { "name": {"type": "string", "default": "World"}, }, }, output_schema={ "type": "object", "properties": {"result": {"type": "string"}}, "required": ["result"], }, steps=[ PromptStep( prompt="Greet {name}", provider="real-provider-that-must-not-run", output_var="response", output_schema={ "type": "object", "properties": {"result": {"type": "string"}}, "required": ["result"], }, ), ], output="{response}", ) report = run_contract_tests(tool) assert report.outcome == "passed" assert report.results[0].state == "passed" def test_empty_schemas_are_explicit_and_runnable(self): from cmdforge.tool import Tool report = run_contract_tests( Tool(name="anything", input_schema={}, output_schema={}, output="ok") ) assert report.outcome == "passed" @pytest.mark.parametrize("step_type", ["code", "tool", "mcp"]) def test_side_effecting_steps_are_unsupported(self, step_type): from cmdforge.tool import CodeStep, McpStep, Tool, ToolStep steps = { "code": CodeStep(code="raise AssertionError('must not run')", output_var="x"), "tool": ToolStep(tool="external", output_var="x"), "mcp": McpStep(server="remote", tool="external", output_var="x"), } tool = Tool( name="unsafe", input_schema={}, output_schema={}, steps=[steps[step_type]], output="{x}", ) report = run_contract_tests(tool) assert report.outcome == "unsupported" def test_schema_mismatch_is_a_failure(self): from cmdforge.tool import Tool report = run_contract_tests( Tool( name="mismatch", input_schema={}, output_schema={"type": "integer"}, output="not-an-integer", ) ) assert report.outcome == "failed"