"""Tests for explainable quality scoring (M8.Q).""" from cmdforge.quality import ( CategoryScore, QualityScore, compute_quality, ) from cmdforge.preflight import PreflightReport from cmdforge.tool import Tool class TestCategoryScore: def test_display(self): cs = CategoryScore("Tests", 25, 30) assert cs.display == "25/30" def test_percentage(self): cs = CategoryScore("Tests", 15, 30) assert cs.percentage == 50.0 def test_zero_available(self): cs = CategoryScore("X", 0, 0) assert cs.percentage == 0.0 def test_display_distinguishes_missing_evidence(self): assert CategoryScore("Tests", 0, 30, "not_tested").display == "not tested" assert CategoryScore("Tests", 0, 30, "not_applicable").display == "n/a" class TestQualityScore: def test_to_dict(self): qs = QualityScore( tool_name="test", version="1.0.0", headline=85, categories=[CategoryScore("Contracts", 15, 15)], ) d = qs.to_dict() assert d["tool"] == "test" assert d["score"] == 85 assert d["categories"][0]["name"] == "Contracts" def test_str_has_headline(self): qs = QualityScore( tool_name="test", version="1.0.0", headline=87, categories=[CategoryScore("Contracts", 15, 15)], ) assert "Quality 87" in str(qs) class TestComputeQuality: def test_empty_tool_low_score(self): tool = Tool(name="bare") report = PreflightReport() qs = compute_quality(tool, report) assert qs.headline < 50 # Contracts should be not_tested contracts = [c for c in qs.categories if c.name == "Contracts"][0] assert contracts.state == "not_tested" assert qs.headline == 0 assert qs.evidence_coverage == 0 def test_missing_categories_do_not_reduce_checked_score(self): tool = Tool(name="contracted", input_schema={}, output_schema={}) report = PreflightReport() qs = compute_quality(tool, report) assert qs.headline == 100 assert qs.evidence_coverage == 15 contracts = [c for c in qs.categories if c.name == "Contracts"][0] assert contracts.state == "checked" def test_unsupported_tests_are_not_scored_as_failures(self): report = PreflightReport(generated_tests=[{ "step": "preflight", "state": "unsupported", "detail": "safe mode" }]) tests = [ c for c in compute_quality(Tool(name="tool"), report).categories if c.name == "Deterministic tests" ][0] assert tests.state == "not_tested" assert tests.display == "not tested" def test_partial_unsupported_tests_reduce_coverage_not_score(self): report = PreflightReport(generated_tests=[ {"step": "case", "state": "passed", "detail": "ok"}, {"step": "generation", "state": "unsupported", "detail": "unknown"}, ]) tests = [ c for c in compute_quality(Tool(name="tool"), report).categories if c.name == "Deterministic tests" ][0] assert tests.earned == tests.available == 15 assert tests.possible == 30 def test_with_contracts_scores_higher(self): tool = Tool( name="contracted", input_schema={"type": "object", "properties": {"input": {"type": "string"}}}, output_schema={"type": "object", "properties": {"result": {"type": "string"}}}, ) report = PreflightReport() qs = compute_quality(tool, report) contracts = [c for c in qs.categories if c.name == "Contracts"][0] assert contracts.earned == 15 assert contracts.state == "checked" def test_with_tests_scores_higher(self): tool = Tool(name="tested") report = PreflightReport( generated_tests=[ {"step": "case-1", "state": "passed", "detail": "ok"}, {"step": "case-2", "state": "passed", "detail": "ok"}, ] ) qs = compute_quality(tool, report) tests = [c for c in qs.categories if c.name == "Deterministic tests"][0] assert tests.earned == 30 assert tests.state == "checked" def test_failed_tests_reduce_score(self): tool = Tool(name="broken") report = PreflightReport( generated_tests=[ {"step": "case-1", "state": "passed", "detail": "ok"}, {"step": "case-2", "state": "failed", "detail": "broken"}, ] ) qs = compute_quality(tool, report) tests = [c for c in qs.categories if c.name == "Deterministic tests"][0] assert tests.earned == 15 def test_regression_no_baseline_is_not_tested(self): tool = Tool(name="test") report = PreflightReport() qs = compute_quality(tool, report) regression = [c for c in qs.categories if c.name == "Regression history"][0] assert regression.state == "not_tested" def test_regression_stable_full_score(self): tool = Tool(name="test") report = PreflightReport( regression={"has_regressions": False, "summary": "no changes"} ) qs = compute_quality(tool, report) regression = [c for c in qs.categories if c.name == "Regression history"][0] assert regression.earned == 20 def test_regression_regressions_zero(self): tool = Tool(name="test") report = PreflightReport( regression={"has_regressions": True, "summary": "1 regression(s)"} ) qs = compute_quality(tool, report) regression = [c for c in qs.categories if c.name == "Regression history"][0] assert regression.earned == 0 def test_secret_warnings_reduce_security(self): tool = Tool(name="leaky") report = PreflightReport( warnings=["Prompt step contains potential secret pattern 'api_key'"] ) qs = compute_quality(tool, report) security = [c for c in qs.categories if c.name == "Security scrutiny"][0] assert security.earned == 15 # 20 - 5 def test_skipped_dependency_check_is_not_claimed_as_security_evidence(self): tool = Tool(name="server") report = PreflightReport(audit_evidence={"checks_run": ["secrets"]}) security = [ c for c in compute_quality(tool, report).categories if c.name == "Security scrutiny" ][0] assert security.earned == 10 assert security.available == 10 assert security.possible == 20 def test_community_not_tested_without_data(self): tool = Tool(name="new") report = PreflightReport() qs = compute_quality(tool, report) community = [c for c in qs.categories if c.name == "Community evidence"][0] assert community.state == "not_tested" def test_community_with_data(self): tool = Tool(name="popular") report = PreflightReport() qs = compute_quality( tool, report, registry_data={ "reviews": [{"rating": 4}, {"rating": 5}], "downloads": 500, "featured": True, } ) community = [c for c in qs.categories if c.name == "Community evidence"][0] assert community.earned == 14 # 7 (reviews) + 4 (downloads) + 3 (featured) assert community.state == "checked"