M8.Q/M8.D1: Add quality scoring engine and tool deprecation fields

2026-07-20 14:37:58 -03:00
parent 501e6cd589
commit bda0c76d5f
4 changed files with 376 additions and 0 deletions
@@ -0,0 +1,150 @@
+"""Tests for explainable quality scoring (M8.Q)."""
+
+from cmdforge.quality import (
+    CategoryScore,
+    QualityScore,
+    compute_quality,
+)
+from cmdforge.preflight import PreflightReport
+from cmdforge.tool import Tool
+
+
+class TestCategoryScore:
+    def test_display(self):
+        cs = CategoryScore("Tests", 25, 30)
+        assert cs.display == "25/30"
+
+    def test_percentage(self):
+        cs = CategoryScore("Tests", 15, 30)
+        assert cs.percentage == 50.0
+
+    def test_zero_available(self):
+        cs = CategoryScore("X", 0, 0)
+        assert cs.percentage == 0.0
+
+
+class TestQualityScore:
+    def test_to_dict(self):
+        qs = QualityScore(
+            tool_name="test",
+            version="1.0.0",
+            headline=85,
+            categories=[CategoryScore("Contracts", 15, 15)],
+        )
+        d = qs.to_dict()
+        assert d["tool"] == "test"
+        assert d["score"] == 85
+        assert d["categories"][0]["name"] == "Contracts"
+
+    def test_str_has_headline(self):
+        qs = QualityScore(
+            tool_name="test",
+            version="1.0.0",
+            headline=87,
+            categories=[CategoryScore("Contracts", 15, 15)],
+        )
+        assert "Quality 87" in str(qs)
+
+
+class TestComputeQuality:
+    def test_empty_tool_low_score(self):
+        tool = Tool(name="bare")
+        report = PreflightReport()
+        qs = compute_quality(tool, report)
+        assert qs.headline < 50
+        # Contracts should be not_tested
+        contracts = [c for c in qs.categories if c.name == "Contracts"][0]
+        assert contracts.state == "not_tested"
+
+    def test_with_contracts_scores_higher(self):
+        tool = Tool(
+            name="contracted",
+            input_schema={"type": "object", "properties": {"input": {"type": "string"}}},
+            output_schema={"type": "object", "properties": {"result": {"type": "string"}}},
+        )
+        report = PreflightReport()
+        qs = compute_quality(tool, report)
+        contracts = [c for c in qs.categories if c.name == "Contracts"][0]
+        assert contracts.earned == 15
+        assert contracts.state == "checked"
+
+    def test_with_tests_scores_higher(self):
+        tool = Tool(name="tested")
+        report = PreflightReport(
+            generated_tests=[
+                {"step": "case-1", "state": "passed", "detail": "ok"},
+                {"step": "case-2", "state": "passed", "detail": "ok"},
+            ]
+        )
+        qs = compute_quality(tool, report)
+        tests = [c for c in qs.categories if c.name == "Deterministic tests"][0]
+        assert tests.earned == 30
+        assert tests.state == "checked"
+
+    def test_failed_tests_reduce_score(self):
+        tool = Tool(name="broken")
+        report = PreflightReport(
+            generated_tests=[
+                {"step": "case-1", "state": "passed", "detail": "ok"},
+                {"step": "case-2", "state": "failed", "detail": "broken"},
+            ]
+        )
+        qs = compute_quality(tool, report)
+        tests = [c for c in qs.categories if c.name == "Deterministic tests"][0]
+        assert tests.earned == 15
+
+    def test_regression_no_baseline_is_not_tested(self):
+        tool = Tool(name="test")
+        report = PreflightReport()
+        qs = compute_quality(tool, report)
+        regression = [c for c in qs.categories if c.name == "Regression history"][0]
+        assert regression.state == "not_tested"
+
+    def test_regression_stable_full_score(self):
+        tool = Tool(name="test")
+        report = PreflightReport(
+            regression={"has_regressions": False, "summary": "no changes"}
+        )
+        qs = compute_quality(tool, report)
+        regression = [c for c in qs.categories if c.name == "Regression history"][0]
+        assert regression.earned == 20
+
+    def test_regression_regressions_zero(self):
+        tool = Tool(name="test")
+        report = PreflightReport(
+            regression={"has_regressions": True, "summary": "1 regression(s)"}
+        )
+        qs = compute_quality(tool, report)
+        regression = [c for c in qs.categories if c.name == "Regression history"][0]
+        assert regression.earned == 0
+
+    def test_secret_warnings_reduce_security(self):
+        tool = Tool(name="leaky")
+        report = PreflightReport(
+            warnings=["Prompt step contains potential secret pattern 'api_key'"]
+        )
+        qs = compute_quality(tool, report)
+        security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
+        assert security.earned == 15  # 20 - 5
+
+    def test_community_not_tested_without_data(self):
+        tool = Tool(name="new")
+        report = PreflightReport()
+        qs = compute_quality(tool, report)
+        community = [c for c in qs.categories if c.name == "Community evidence"][0]
+        assert community.state == "not_tested"
+
+    def test_community_with_data(self):
+        tool = Tool(name="popular")
+        report = PreflightReport()
+        qs = compute_quality(
+            tool, report,
+            registry_data={
+                "reviews": [{"rating": 4}, {"rating": 5}],
+                "downloads": 500,
+                "featured": True,
+            }
+        )
+        community = [c for c in qs.categories if c.name == "Community evidence"][0]
+        assert community.earned == 14  # 7 (reviews) + 4 (downloads) + 3 (featured)
+        assert community.state == "checked"