"""Tests for explainable quality scoring (M8.Q).""" from cmdforge.quality import ( CategoryScore, QualityScore, compute_quality, ) from cmdforge.preflight import PreflightReport from cmdforge.tool import Tool class TestCategoryScore: def test_display(self): cs = CategoryScore("Tests", 25, 30) assert cs.display == "25/30" def test_percentage(self): cs = CategoryScore("Tests", 15, 30) assert cs.percentage == 50.0 def test_zero_available(self): cs = CategoryScore("X", 0, 0) assert cs.percentage == 0.0 class TestQualityScore: def test_to_dict(self): qs = QualityScore( tool_name="test", version="1.0.0", headline=85, categories=[CategoryScore("Contracts", 15, 15)], ) d = qs.to_dict() assert d["tool"] == "test" assert d["score"] == 85 assert d["categories"][0]["name"] == "Contracts" def test_str_has_headline(self): qs = QualityScore( tool_name="test", version="1.0.0", headline=87, categories=[CategoryScore("Contracts", 15, 15)], ) assert "Quality 87" in str(qs) class TestComputeQuality: def test_empty_tool_low_score(self): tool = Tool(name="bare") report = PreflightReport() qs = compute_quality(tool, report) assert qs.headline < 50 # Contracts should be not_tested contracts = [c for c in qs.categories if c.name == "Contracts"][0] assert contracts.state == "not_tested" def test_with_contracts_scores_higher(self): tool = Tool( name="contracted", input_schema={"type": "object", "properties": {"input": {"type": "string"}}}, output_schema={"type": "object", "properties": {"result": {"type": "string"}}}, ) report = PreflightReport() qs = compute_quality(tool, report) contracts = [c for c in qs.categories if c.name == "Contracts"][0] assert contracts.earned == 15 assert contracts.state == "checked" def test_with_tests_scores_higher(self): tool = Tool(name="tested") report = PreflightReport( generated_tests=[ {"step": "case-1", "state": "passed", "detail": "ok"}, {"step": "case-2", "state": "passed", "detail": "ok"}, ] ) qs = compute_quality(tool, report) tests = [c for c in qs.categories if c.name == "Deterministic tests"][0] assert tests.earned == 30 assert tests.state == "checked" def test_failed_tests_reduce_score(self): tool = Tool(name="broken") report = PreflightReport( generated_tests=[ {"step": "case-1", "state": "passed", "detail": "ok"}, {"step": "case-2", "state": "failed", "detail": "broken"}, ] ) qs = compute_quality(tool, report) tests = [c for c in qs.categories if c.name == "Deterministic tests"][0] assert tests.earned == 15 def test_regression_no_baseline_is_not_tested(self): tool = Tool(name="test") report = PreflightReport() qs = compute_quality(tool, report) regression = [c for c in qs.categories if c.name == "Regression history"][0] assert regression.state == "not_tested" def test_regression_stable_full_score(self): tool = Tool(name="test") report = PreflightReport( regression={"has_regressions": False, "summary": "no changes"} ) qs = compute_quality(tool, report) regression = [c for c in qs.categories if c.name == "Regression history"][0] assert regression.earned == 20 def test_regression_regressions_zero(self): tool = Tool(name="test") report = PreflightReport( regression={"has_regressions": True, "summary": "1 regression(s)"} ) qs = compute_quality(tool, report) regression = [c for c in qs.categories if c.name == "Regression history"][0] assert regression.earned == 0 def test_secret_warnings_reduce_security(self): tool = Tool(name="leaky") report = PreflightReport( warnings=["Prompt step contains potential secret pattern 'api_key'"] ) qs = compute_quality(tool, report) security = [c for c in qs.categories if c.name == "Security scrutiny"][0] assert security.earned == 15 # 20 - 5 def test_community_not_tested_without_data(self): tool = Tool(name="new") report = PreflightReport() qs = compute_quality(tool, report) community = [c for c in qs.categories if c.name == "Community evidence"][0] assert community.state == "not_tested" def test_community_with_data(self): tool = Tool(name="popular") report = PreflightReport() qs = compute_quality( tool, report, registry_data={ "reviews": [{"rating": 4}, {"rating": 5}], "downloads": 500, "featured": True, } ) community = [c for c in qs.categories if c.name == "Community evidence"][0] assert community.earned == 14 # 7 (reviews) + 4 (downloads) + 3 (featured) assert community.state == "checked"