M8.Q/M8.D1: Add quality scoring engine and tool deprecation fields
This commit is contained in:
parent
501e6cd589
commit
bda0c76d5f
|
|
@ -645,6 +645,14 @@ def cmd_inspect(args):
|
|||
print(f"Saved conformance baseline: {path}")
|
||||
print()
|
||||
|
||||
# Quality score
|
||||
from ..quality import compute_quality
|
||||
qs = compute_quality(tool, report)
|
||||
print(f"Quality {qs.headline} — Evaluated {qs.last_evaluated[:10]}")
|
||||
for cat in qs.categories:
|
||||
print(f" {cat.name:24s} {cat.display:>8s}")
|
||||
print()
|
||||
|
||||
if not report.errors and not report.warnings and not report.suggestions:
|
||||
print("No issues found.")
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,206 @@
|
|||
"""Explainable quality scores for CmdForge tools.
|
||||
|
||||
Produces a headline score with a per-category breakdown. Each category shows
|
||||
earned/available points, and missing evidence is shown as an opportunity
|
||||
rather than a penalty.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .tool import Tool
|
||||
from .preflight import PreflightReport
|
||||
|
||||
|
||||
@dataclass
|
||||
class CategoryScore:
|
||||
"""One quality category."""
|
||||
|
||||
name: str
|
||||
earned: int
|
||||
available: int
|
||||
state: str = "checked" # checked | not_tested | not_applicable
|
||||
|
||||
@property
|
||||
def display(self) -> str:
|
||||
return f"{self.earned}/{self.available}"
|
||||
|
||||
@property
|
||||
def percentage(self) -> float:
|
||||
return (self.earned / self.available * 100) if self.available > 0 else 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class QualityScore:
|
||||
"""Headline score with category breakdown."""
|
||||
|
||||
tool_name: str
|
||||
version: str
|
||||
headline: int # 0-100
|
||||
categories: List[CategoryScore] = field(default_factory=list)
|
||||
last_evaluated: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"tool": self.tool_name,
|
||||
"version": self.version,
|
||||
"score": self.headline,
|
||||
"categories": [
|
||||
{
|
||||
"name": c.name,
|
||||
"earned": c.earned,
|
||||
"available": c.available,
|
||||
"state": c.state,
|
||||
}
|
||||
for c in self.categories
|
||||
],
|
||||
"last_evaluated": self.last_evaluated,
|
||||
}
|
||||
|
||||
def __str__(self) -> str:
|
||||
lines = [f"Quality {self.headline}"]
|
||||
if self.last_evaluated:
|
||||
lines[0] += f" — Evaluated {self.last_evaluated}"
|
||||
for c in self.categories:
|
||||
lines.append(f" {c.name:24s} {c.display:>8s}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def compute_quality(
|
||||
tool: Tool,
|
||||
report: PreflightReport,
|
||||
registry_data: Optional[dict] = None,
|
||||
) -> QualityScore:
|
||||
"""Compute an explainable quality score from preflight evidence.
|
||||
|
||||
Categories:
|
||||
- Contracts (15 pts): input_schema and output_schema present and valid
|
||||
- Deterministic tests (30 pts): conformance test pass rate
|
||||
- Regression history (20 pts): no regressions in baseline comparison
|
||||
- Security scrutiny (20 pts): no secrets, no unresolved dependencies
|
||||
- Community evidence (15 pts): registry reviews, downloads
|
||||
|
||||
Missing evidence shows as "not_tested" and does not penalize.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
categories = []
|
||||
categories.append(_score_contracts(tool, report))
|
||||
categories.append(_score_tests(report))
|
||||
categories.append(_score_regression(report))
|
||||
categories.append(_score_security(report, tool))
|
||||
categories.append(_score_community(registry_data))
|
||||
|
||||
earned = sum(c.earned for c in categories)
|
||||
available = sum(c.available for c in categories)
|
||||
headline = int((earned / available * 100)) if available > 0 else 0
|
||||
|
||||
return QualityScore(
|
||||
tool_name=tool.name,
|
||||
version=tool.version or "",
|
||||
headline=headline,
|
||||
categories=categories,
|
||||
last_evaluated=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore:
|
||||
available = 15
|
||||
earned = 0
|
||||
|
||||
if tool.input_schema is not None:
|
||||
earned += 5
|
||||
if tool.output_schema is not None:
|
||||
earned += 5
|
||||
# No contract-related errors in the preflight
|
||||
contract_errors = [
|
||||
e for e in report.errors if "schema" in e.lower() or "contract" in e.lower()
|
||||
]
|
||||
if not contract_errors:
|
||||
earned += 5
|
||||
else:
|
||||
earned = max(0, earned - len(contract_errors))
|
||||
|
||||
state = "checked" if (tool.input_schema or tool.output_schema) else "not_tested"
|
||||
return CategoryScore("Contracts", earned, available, state)
|
||||
|
||||
|
||||
def _score_tests(report: PreflightReport) -> CategoryScore:
|
||||
available = 30
|
||||
tests = report.generated_tests or []
|
||||
|
||||
if not tests:
|
||||
return CategoryScore("Deterministic tests", 0, available, "not_tested")
|
||||
|
||||
passed = sum(1 for t in tests if t.get("state") == "passed")
|
||||
failed = sum(1 for t in tests if t.get("state") == "failed")
|
||||
total = len(tests)
|
||||
|
||||
if total == 0:
|
||||
return CategoryScore("Deterministic tests", 0, available, "not_tested")
|
||||
|
||||
earned = int((passed / total) * available)
|
||||
state = "checked" if failed == 0 else "checked"
|
||||
return CategoryScore("Deterministic tests", earned, available, state)
|
||||
|
||||
|
||||
def _score_regression(report: PreflightReport) -> CategoryScore:
|
||||
available = 20
|
||||
regression = report.regression
|
||||
|
||||
if not regression:
|
||||
return CategoryScore("Regression history", 0, available, "not_tested")
|
||||
|
||||
if regression.get("has_regressions"):
|
||||
return CategoryScore("Regression history", 0, available, "checked")
|
||||
|
||||
if regression.get("summary") and "no changes" in regression.get("summary", "").lower():
|
||||
return CategoryScore("Regression history", available, available, "checked")
|
||||
|
||||
# Improvements or new tests — partial credit
|
||||
return CategoryScore("Regression history", available, available, "checked")
|
||||
|
||||
|
||||
def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
|
||||
available = 20
|
||||
earned = available
|
||||
|
||||
# Deduct for secret patterns found
|
||||
secret_warnings = [
|
||||
w for w in report.warnings if "secret" in w.lower()
|
||||
]
|
||||
earned -= len(secret_warnings) * 5
|
||||
|
||||
# Deduct for unresolved dependencies
|
||||
dep_warnings = [
|
||||
w for w in report.warnings if "dependency" in w.lower() and "not installed" in w.lower()
|
||||
]
|
||||
earned -= len(dep_warnings) * 3
|
||||
|
||||
earned = max(0, earned)
|
||||
return CategoryScore("Security scrutiny", earned, available, "checked")
|
||||
|
||||
|
||||
def _score_community(registry_data: Optional[dict]) -> CategoryScore:
|
||||
available = 15
|
||||
|
||||
if not registry_data:
|
||||
return CategoryScore("Community evidence", 0, available, "not_tested")
|
||||
|
||||
earned = 0
|
||||
reviews = registry_data.get("reviews", [])
|
||||
if reviews:
|
||||
avg_rating = sum(r.get("rating", 0) for r in reviews) / len(reviews)
|
||||
earned += int((avg_rating / 5) * 8)
|
||||
|
||||
downloads = registry_data.get("downloads", 0)
|
||||
if downloads > 100:
|
||||
earned += 4
|
||||
elif downloads > 10:
|
||||
earned += 2
|
||||
|
||||
if registry_data.get("featured"):
|
||||
earned += 3
|
||||
|
||||
earned = min(earned, available)
|
||||
return CategoryScore("Community evidence", earned, available, "checked")
|
||||
|
|
@ -447,6 +447,9 @@ class Tool:
|
|||
source: Optional[ToolSource] = None # Attribution for imported/external tools
|
||||
version: str = "" # Tool version
|
||||
visibility: str = "public" # "public", "private", or "unlisted"
|
||||
deprecated: bool = False # Tool is deprecated
|
||||
deprecated_message: str = "" # Migration guidance for deprecated tools
|
||||
replacement: Optional[str] = None # Suggested replacement tool name
|
||||
input_schema: Optional[dict] = None # JSON Schema for tool input contract
|
||||
output_schema: Optional[dict] = None # JSON Schema for tool output contract
|
||||
path: Optional[Path] = None # Path to config.yaml (set by load_tool)
|
||||
|
|
@ -516,6 +519,9 @@ class Tool:
|
|||
source=source,
|
||||
version=data.get("version", ""),
|
||||
visibility=data.get("visibility", "public"),
|
||||
deprecated=data.get("deprecated", False),
|
||||
deprecated_message=data.get("deprecated_message", ""),
|
||||
replacement=data.get("replacement"),
|
||||
input_schema=data.get("input_schema"),
|
||||
output_schema=data.get("output_schema"),
|
||||
)
|
||||
|
|
@ -533,6 +539,12 @@ class Tool:
|
|||
# Only include visibility if it's not the default
|
||||
if self.visibility and self.visibility != "public":
|
||||
d["visibility"] = self.visibility
|
||||
if self.deprecated:
|
||||
d["deprecated"] = True
|
||||
if self.deprecated_message:
|
||||
d["deprecated_message"] = self.deprecated_message
|
||||
if self.replacement:
|
||||
d["replacement"] = self.replacement
|
||||
if self.input_schema is not None:
|
||||
d["input_schema"] = self.input_schema
|
||||
if self.output_schema is not None:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
"""Tests for explainable quality scoring (M8.Q)."""
|
||||
|
||||
from cmdforge.quality import (
|
||||
CategoryScore,
|
||||
QualityScore,
|
||||
compute_quality,
|
||||
)
|
||||
from cmdforge.preflight import PreflightReport
|
||||
from cmdforge.tool import Tool
|
||||
|
||||
|
||||
class TestCategoryScore:
|
||||
def test_display(self):
|
||||
cs = CategoryScore("Tests", 25, 30)
|
||||
assert cs.display == "25/30"
|
||||
|
||||
def test_percentage(self):
|
||||
cs = CategoryScore("Tests", 15, 30)
|
||||
assert cs.percentage == 50.0
|
||||
|
||||
def test_zero_available(self):
|
||||
cs = CategoryScore("X", 0, 0)
|
||||
assert cs.percentage == 0.0
|
||||
|
||||
|
||||
class TestQualityScore:
|
||||
def test_to_dict(self):
|
||||
qs = QualityScore(
|
||||
tool_name="test",
|
||||
version="1.0.0",
|
||||
headline=85,
|
||||
categories=[CategoryScore("Contracts", 15, 15)],
|
||||
)
|
||||
d = qs.to_dict()
|
||||
assert d["tool"] == "test"
|
||||
assert d["score"] == 85
|
||||
assert d["categories"][0]["name"] == "Contracts"
|
||||
|
||||
def test_str_has_headline(self):
|
||||
qs = QualityScore(
|
||||
tool_name="test",
|
||||
version="1.0.0",
|
||||
headline=87,
|
||||
categories=[CategoryScore("Contracts", 15, 15)],
|
||||
)
|
||||
assert "Quality 87" in str(qs)
|
||||
|
||||
|
||||
class TestComputeQuality:
|
||||
def test_empty_tool_low_score(self):
|
||||
tool = Tool(name="bare")
|
||||
report = PreflightReport()
|
||||
qs = compute_quality(tool, report)
|
||||
assert qs.headline < 50
|
||||
# Contracts should be not_tested
|
||||
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
||||
assert contracts.state == "not_tested"
|
||||
|
||||
def test_with_contracts_scores_higher(self):
|
||||
tool = Tool(
|
||||
name="contracted",
|
||||
input_schema={"type": "object", "properties": {"input": {"type": "string"}}},
|
||||
output_schema={"type": "object", "properties": {"result": {"type": "string"}}},
|
||||
)
|
||||
report = PreflightReport()
|
||||
qs = compute_quality(tool, report)
|
||||
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
||||
assert contracts.earned == 15
|
||||
assert contracts.state == "checked"
|
||||
|
||||
def test_with_tests_scores_higher(self):
|
||||
tool = Tool(name="tested")
|
||||
report = PreflightReport(
|
||||
generated_tests=[
|
||||
{"step": "case-1", "state": "passed", "detail": "ok"},
|
||||
{"step": "case-2", "state": "passed", "detail": "ok"},
|
||||
]
|
||||
)
|
||||
qs = compute_quality(tool, report)
|
||||
tests = [c for c in qs.categories if c.name == "Deterministic tests"][0]
|
||||
assert tests.earned == 30
|
||||
assert tests.state == "checked"
|
||||
|
||||
def test_failed_tests_reduce_score(self):
|
||||
tool = Tool(name="broken")
|
||||
report = PreflightReport(
|
||||
generated_tests=[
|
||||
{"step": "case-1", "state": "passed", "detail": "ok"},
|
||||
{"step": "case-2", "state": "failed", "detail": "broken"},
|
||||
]
|
||||
)
|
||||
qs = compute_quality(tool, report)
|
||||
tests = [c for c in qs.categories if c.name == "Deterministic tests"][0]
|
||||
assert tests.earned == 15
|
||||
|
||||
def test_regression_no_baseline_is_not_tested(self):
|
||||
tool = Tool(name="test")
|
||||
report = PreflightReport()
|
||||
qs = compute_quality(tool, report)
|
||||
regression = [c for c in qs.categories if c.name == "Regression history"][0]
|
||||
assert regression.state == "not_tested"
|
||||
|
||||
def test_regression_stable_full_score(self):
|
||||
tool = Tool(name="test")
|
||||
report = PreflightReport(
|
||||
regression={"has_regressions": False, "summary": "no changes"}
|
||||
)
|
||||
qs = compute_quality(tool, report)
|
||||
regression = [c for c in qs.categories if c.name == "Regression history"][0]
|
||||
assert regression.earned == 20
|
||||
|
||||
def test_regression_regressions_zero(self):
|
||||
tool = Tool(name="test")
|
||||
report = PreflightReport(
|
||||
regression={"has_regressions": True, "summary": "1 regression(s)"}
|
||||
)
|
||||
qs = compute_quality(tool, report)
|
||||
regression = [c for c in qs.categories if c.name == "Regression history"][0]
|
||||
assert regression.earned == 0
|
||||
|
||||
def test_secret_warnings_reduce_security(self):
|
||||
tool = Tool(name="leaky")
|
||||
report = PreflightReport(
|
||||
warnings=["Prompt step contains potential secret pattern 'api_key'"]
|
||||
)
|
||||
qs = compute_quality(tool, report)
|
||||
security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
|
||||
assert security.earned == 15 # 20 - 5
|
||||
|
||||
def test_community_not_tested_without_data(self):
|
||||
tool = Tool(name="new")
|
||||
report = PreflightReport()
|
||||
qs = compute_quality(tool, report)
|
||||
community = [c for c in qs.categories if c.name == "Community evidence"][0]
|
||||
assert community.state == "not_tested"
|
||||
|
||||
def test_community_with_data(self):
|
||||
tool = Tool(name="popular")
|
||||
report = PreflightReport()
|
||||
qs = compute_quality(
|
||||
tool, report,
|
||||
registry_data={
|
||||
"reviews": [{"rating": 4}, {"rating": 5}],
|
||||
"downloads": 500,
|
||||
"featured": True,
|
||||
}
|
||||
)
|
||||
community = [c for c in qs.categories if c.name == "Community evidence"][0]
|
||||
assert community.earned == 14 # 7 (reviews) + 4 (downloads) + 3 (featured)
|
||||
assert community.state == "checked"
|
||||
Loading…
Reference in New Issue