Harden M8 trust and guidance workflows

tests/test_quality.py

@@ -22,6 +22,10 @@ class TestCategoryScore:
         cs = CategoryScore("X", 0, 0)
         assert cs.percentage == 0.0
 
+    def test_display_distinguishes_missing_evidence(self):
+        assert CategoryScore("Tests", 0, 30, "not_tested").display == "not tested"
+        assert CategoryScore("Tests", 0, 30, "not_applicable").display == "n/a"
+
 
 class TestQualityScore:
     def test_to_dict(self):
@@ -55,6 +59,40 @@ class TestComputeQuality:
         # Contracts should be not_tested
         contracts = [c for c in qs.categories if c.name == "Contracts"][0]
         assert contracts.state == "not_tested"
+        assert qs.headline == 0
+        assert qs.evidence_coverage == 0
+
+    def test_missing_categories_do_not_reduce_checked_score(self):
+        tool = Tool(name="contracted", input_schema={}, output_schema={})
+        report = PreflightReport()
+        qs = compute_quality(tool, report)
+        assert qs.headline == 100
+        assert qs.evidence_coverage == 15
+        contracts = [c for c in qs.categories if c.name == "Contracts"][0]
+        assert contracts.state == "checked"
+
+    def test_unsupported_tests_are_not_scored_as_failures(self):
+        report = PreflightReport(generated_tests=[{
+            "step": "preflight", "state": "unsupported", "detail": "safe mode"
+        }])
+        tests = [
+            c for c in compute_quality(Tool(name="tool"), report).categories
+            if c.name == "Deterministic tests"
+        ][0]
+        assert tests.state == "not_tested"
+        assert tests.display == "not tested"
+
+    def test_partial_unsupported_tests_reduce_coverage_not_score(self):
+        report = PreflightReport(generated_tests=[
+            {"step": "case", "state": "passed", "detail": "ok"},
+            {"step": "generation", "state": "unsupported", "detail": "unknown"},
+        ])
+        tests = [
+            c for c in compute_quality(Tool(name="tool"), report).categories
+            if c.name == "Deterministic tests"
+        ][0]
+        assert tests.earned == tests.available == 15
+        assert tests.possible == 30
 
     def test_with_contracts_scores_higher(self):
         tool = Tool(
@@ -127,6 +165,17 @@ class TestComputeQuality:
         security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
         assert security.earned == 15  # 20 - 5
 
+    def test_skipped_dependency_check_is_not_claimed_as_security_evidence(self):
+        tool = Tool(name="server")
+        report = PreflightReport(audit_evidence={"checks_run": ["secrets"]})
+        security = [
+            c for c in compute_quality(tool, report).categories
+            if c.name == "Security scrutiny"
+        ][0]
+        assert security.earned == 10
+        assert security.available == 10
+        assert security.possible == 20
+
     def test_community_not_tested_without_data(self):
         tool = Tool(name="new")
         report = PreflightReport()