200 lines
7.4 KiB
Python
200 lines
7.4 KiB
Python
"""Tests for explainable quality scoring (M8.Q)."""
|
|
|
|
from cmdforge.quality import (
|
|
CategoryScore,
|
|
QualityScore,
|
|
compute_quality,
|
|
)
|
|
from cmdforge.preflight import PreflightReport
|
|
from cmdforge.tool import Tool
|
|
|
|
|
|
class TestCategoryScore:
|
|
def test_display(self):
|
|
cs = CategoryScore("Tests", 25, 30)
|
|
assert cs.display == "25/30"
|
|
|
|
def test_percentage(self):
|
|
cs = CategoryScore("Tests", 15, 30)
|
|
assert cs.percentage == 50.0
|
|
|
|
def test_zero_available(self):
|
|
cs = CategoryScore("X", 0, 0)
|
|
assert cs.percentage == 0.0
|
|
|
|
def test_display_distinguishes_missing_evidence(self):
|
|
assert CategoryScore("Tests", 0, 30, "not_tested").display == "not tested"
|
|
assert CategoryScore("Tests", 0, 30, "not_applicable").display == "n/a"
|
|
|
|
|
|
class TestQualityScore:
|
|
def test_to_dict(self):
|
|
qs = QualityScore(
|
|
tool_name="test",
|
|
version="1.0.0",
|
|
headline=85,
|
|
categories=[CategoryScore("Contracts", 15, 15)],
|
|
)
|
|
d = qs.to_dict()
|
|
assert d["tool"] == "test"
|
|
assert d["score"] == 85
|
|
assert d["categories"][0]["name"] == "Contracts"
|
|
|
|
def test_str_has_headline(self):
|
|
qs = QualityScore(
|
|
tool_name="test",
|
|
version="1.0.0",
|
|
headline=87,
|
|
categories=[CategoryScore("Contracts", 15, 15)],
|
|
)
|
|
assert "Quality 87" in str(qs)
|
|
|
|
|
|
class TestComputeQuality:
|
|
def test_empty_tool_low_score(self):
|
|
tool = Tool(name="bare")
|
|
report = PreflightReport()
|
|
qs = compute_quality(tool, report)
|
|
assert qs.headline < 50
|
|
# Contracts should be not_tested
|
|
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
|
assert contracts.state == "not_tested"
|
|
assert qs.headline == 0
|
|
assert qs.evidence_coverage == 0
|
|
|
|
def test_missing_categories_do_not_reduce_checked_score(self):
|
|
tool = Tool(name="contracted", input_schema={}, output_schema={})
|
|
report = PreflightReport()
|
|
qs = compute_quality(tool, report)
|
|
assert qs.headline == 100
|
|
assert qs.evidence_coverage == 15
|
|
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
|
assert contracts.state == "checked"
|
|
|
|
def test_unsupported_tests_are_not_scored_as_failures(self):
|
|
report = PreflightReport(generated_tests=[{
|
|
"step": "preflight", "state": "unsupported", "detail": "safe mode"
|
|
}])
|
|
tests = [
|
|
c for c in compute_quality(Tool(name="tool"), report).categories
|
|
if c.name == "Deterministic tests"
|
|
][0]
|
|
assert tests.state == "not_tested"
|
|
assert tests.display == "not tested"
|
|
|
|
def test_partial_unsupported_tests_reduce_coverage_not_score(self):
|
|
report = PreflightReport(generated_tests=[
|
|
{"step": "case", "state": "passed", "detail": "ok"},
|
|
{"step": "generation", "state": "unsupported", "detail": "unknown"},
|
|
])
|
|
tests = [
|
|
c for c in compute_quality(Tool(name="tool"), report).categories
|
|
if c.name == "Deterministic tests"
|
|
][0]
|
|
assert tests.earned == tests.available == 15
|
|
assert tests.possible == 30
|
|
|
|
def test_with_contracts_scores_higher(self):
|
|
tool = Tool(
|
|
name="contracted",
|
|
input_schema={"type": "object", "properties": {"input": {"type": "string"}}},
|
|
output_schema={"type": "object", "properties": {"result": {"type": "string"}}},
|
|
)
|
|
report = PreflightReport()
|
|
qs = compute_quality(tool, report)
|
|
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
|
assert contracts.earned == 15
|
|
assert contracts.state == "checked"
|
|
|
|
def test_with_tests_scores_higher(self):
|
|
tool = Tool(name="tested")
|
|
report = PreflightReport(
|
|
generated_tests=[
|
|
{"step": "case-1", "state": "passed", "detail": "ok"},
|
|
{"step": "case-2", "state": "passed", "detail": "ok"},
|
|
]
|
|
)
|
|
qs = compute_quality(tool, report)
|
|
tests = [c for c in qs.categories if c.name == "Deterministic tests"][0]
|
|
assert tests.earned == 30
|
|
assert tests.state == "checked"
|
|
|
|
def test_failed_tests_reduce_score(self):
|
|
tool = Tool(name="broken")
|
|
report = PreflightReport(
|
|
generated_tests=[
|
|
{"step": "case-1", "state": "passed", "detail": "ok"},
|
|
{"step": "case-2", "state": "failed", "detail": "broken"},
|
|
]
|
|
)
|
|
qs = compute_quality(tool, report)
|
|
tests = [c for c in qs.categories if c.name == "Deterministic tests"][0]
|
|
assert tests.earned == 15
|
|
|
|
def test_regression_no_baseline_is_not_tested(self):
|
|
tool = Tool(name="test")
|
|
report = PreflightReport()
|
|
qs = compute_quality(tool, report)
|
|
regression = [c for c in qs.categories if c.name == "Regression history"][0]
|
|
assert regression.state == "not_tested"
|
|
|
|
def test_regression_stable_full_score(self):
|
|
tool = Tool(name="test")
|
|
report = PreflightReport(
|
|
regression={"has_regressions": False, "summary": "no changes"}
|
|
)
|
|
qs = compute_quality(tool, report)
|
|
regression = [c for c in qs.categories if c.name == "Regression history"][0]
|
|
assert regression.earned == 20
|
|
|
|
def test_regression_regressions_zero(self):
|
|
tool = Tool(name="test")
|
|
report = PreflightReport(
|
|
regression={"has_regressions": True, "summary": "1 regression(s)"}
|
|
)
|
|
qs = compute_quality(tool, report)
|
|
regression = [c for c in qs.categories if c.name == "Regression history"][0]
|
|
assert regression.earned == 0
|
|
|
|
def test_secret_warnings_reduce_security(self):
|
|
tool = Tool(name="leaky")
|
|
report = PreflightReport(
|
|
warnings=["Prompt step contains potential secret pattern 'api_key'"]
|
|
)
|
|
qs = compute_quality(tool, report)
|
|
security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
|
|
assert security.earned == 15 # 20 - 5
|
|
|
|
def test_skipped_dependency_check_is_not_claimed_as_security_evidence(self):
|
|
tool = Tool(name="server")
|
|
report = PreflightReport(audit_evidence={"checks_run": ["secrets"]})
|
|
security = [
|
|
c for c in compute_quality(tool, report).categories
|
|
if c.name == "Security scrutiny"
|
|
][0]
|
|
assert security.earned == 10
|
|
assert security.available == 10
|
|
assert security.possible == 20
|
|
|
|
def test_community_not_tested_without_data(self):
|
|
tool = Tool(name="new")
|
|
report = PreflightReport()
|
|
qs = compute_quality(tool, report)
|
|
community = [c for c in qs.categories if c.name == "Community evidence"][0]
|
|
assert community.state == "not_tested"
|
|
|
|
def test_community_with_data(self):
|
|
tool = Tool(name="popular")
|
|
report = PreflightReport()
|
|
qs = compute_quality(
|
|
tool, report,
|
|
registry_data={
|
|
"reviews": [{"rating": 4}, {"rating": 5}],
|
|
"downloads": 500,
|
|
"featured": True,
|
|
}
|
|
)
|
|
community = [c for c in qs.categories if c.name == "Community evidence"][0]
|
|
assert community.earned == 14 # 7 (reviews) + 4 (downloads) + 3 (featured)
|
|
assert community.state == "checked"
|