From bda0c76d5f1377c1d135e5edf88e6d78cdf4e681 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 20 Jul 2026 14:37:58 -0300 Subject: [PATCH] M8.Q/M8.D1: Add quality scoring engine and tool deprecation fields --- src/cmdforge/cli/__init__.py | 8 ++ src/cmdforge/quality.py | 206 +++++++++++++++++++++++++++++++++++ src/cmdforge/tool.py | 12 ++ tests/test_quality.py | 150 +++++++++++++++++++++++++ 4 files changed, 376 insertions(+) create mode 100644 src/cmdforge/quality.py create mode 100644 tests/test_quality.py diff --git a/src/cmdforge/cli/__init__.py b/src/cmdforge/cli/__init__.py index 4ea0f43..335c1ee 100644 --- a/src/cmdforge/cli/__init__.py +++ b/src/cmdforge/cli/__init__.py @@ -645,6 +645,14 @@ def cmd_inspect(args): print(f"Saved conformance baseline: {path}") print() + # Quality score + from ..quality import compute_quality + qs = compute_quality(tool, report) + print(f"Quality {qs.headline} — Evaluated {qs.last_evaluated[:10]}") + for cat in qs.categories: + print(f" {cat.name:24s} {cat.display:>8s}") + print() + if not report.errors and not report.warnings and not report.suggestions: print("No issues found.") diff --git a/src/cmdforge/quality.py b/src/cmdforge/quality.py new file mode 100644 index 0000000..27fb5a7 --- /dev/null +++ b/src/cmdforge/quality.py @@ -0,0 +1,206 @@ +"""Explainable quality scores for CmdForge tools. + +Produces a headline score with a per-category breakdown. Each category shows +earned/available points, and missing evidence is shown as an opportunity +rather than a penalty. +""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from .tool import Tool +from .preflight import PreflightReport + + +@dataclass +class CategoryScore: + """One quality category.""" + + name: str + earned: int + available: int + state: str = "checked" # checked | not_tested | not_applicable + + @property + def display(self) -> str: + return f"{self.earned}/{self.available}" + + @property + def percentage(self) -> float: + return (self.earned / self.available * 100) if self.available > 0 else 0.0 + + +@dataclass +class QualityScore: + """Headline score with category breakdown.""" + + tool_name: str + version: str + headline: int # 0-100 + categories: List[CategoryScore] = field(default_factory=list) + last_evaluated: str = "" + + def to_dict(self) -> dict: + return { + "tool": self.tool_name, + "version": self.version, + "score": self.headline, + "categories": [ + { + "name": c.name, + "earned": c.earned, + "available": c.available, + "state": c.state, + } + for c in self.categories + ], + "last_evaluated": self.last_evaluated, + } + + def __str__(self) -> str: + lines = [f"Quality {self.headline}"] + if self.last_evaluated: + lines[0] += f" — Evaluated {self.last_evaluated}" + for c in self.categories: + lines.append(f" {c.name:24s} {c.display:>8s}") + return "\n".join(lines) + + +def compute_quality( + tool: Tool, + report: PreflightReport, + registry_data: Optional[dict] = None, +) -> QualityScore: + """Compute an explainable quality score from preflight evidence. + + Categories: + - Contracts (15 pts): input_schema and output_schema present and valid + - Deterministic tests (30 pts): conformance test pass rate + - Regression history (20 pts): no regressions in baseline comparison + - Security scrutiny (20 pts): no secrets, no unresolved dependencies + - Community evidence (15 pts): registry reviews, downloads + + Missing evidence shows as "not_tested" and does not penalize. + """ + from datetime import datetime, timezone + + categories = [] + categories.append(_score_contracts(tool, report)) + categories.append(_score_tests(report)) + categories.append(_score_regression(report)) + categories.append(_score_security(report, tool)) + categories.append(_score_community(registry_data)) + + earned = sum(c.earned for c in categories) + available = sum(c.available for c in categories) + headline = int((earned / available * 100)) if available > 0 else 0 + + return QualityScore( + tool_name=tool.name, + version=tool.version or "", + headline=headline, + categories=categories, + last_evaluated=datetime.now(timezone.utc).isoformat(), + ) + + +def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore: + available = 15 + earned = 0 + + if tool.input_schema is not None: + earned += 5 + if tool.output_schema is not None: + earned += 5 + # No contract-related errors in the preflight + contract_errors = [ + e for e in report.errors if "schema" in e.lower() or "contract" in e.lower() + ] + if not contract_errors: + earned += 5 + else: + earned = max(0, earned - len(contract_errors)) + + state = "checked" if (tool.input_schema or tool.output_schema) else "not_tested" + return CategoryScore("Contracts", earned, available, state) + + +def _score_tests(report: PreflightReport) -> CategoryScore: + available = 30 + tests = report.generated_tests or [] + + if not tests: + return CategoryScore("Deterministic tests", 0, available, "not_tested") + + passed = sum(1 for t in tests if t.get("state") == "passed") + failed = sum(1 for t in tests if t.get("state") == "failed") + total = len(tests) + + if total == 0: + return CategoryScore("Deterministic tests", 0, available, "not_tested") + + earned = int((passed / total) * available) + state = "checked" if failed == 0 else "checked" + return CategoryScore("Deterministic tests", earned, available, state) + + +def _score_regression(report: PreflightReport) -> CategoryScore: + available = 20 + regression = report.regression + + if not regression: + return CategoryScore("Regression history", 0, available, "not_tested") + + if regression.get("has_regressions"): + return CategoryScore("Regression history", 0, available, "checked") + + if regression.get("summary") and "no changes" in regression.get("summary", "").lower(): + return CategoryScore("Regression history", available, available, "checked") + + # Improvements or new tests — partial credit + return CategoryScore("Regression history", available, available, "checked") + + +def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore: + available = 20 + earned = available + + # Deduct for secret patterns found + secret_warnings = [ + w for w in report.warnings if "secret" in w.lower() + ] + earned -= len(secret_warnings) * 5 + + # Deduct for unresolved dependencies + dep_warnings = [ + w for w in report.warnings if "dependency" in w.lower() and "not installed" in w.lower() + ] + earned -= len(dep_warnings) * 3 + + earned = max(0, earned) + return CategoryScore("Security scrutiny", earned, available, "checked") + + +def _score_community(registry_data: Optional[dict]) -> CategoryScore: + available = 15 + + if not registry_data: + return CategoryScore("Community evidence", 0, available, "not_tested") + + earned = 0 + reviews = registry_data.get("reviews", []) + if reviews: + avg_rating = sum(r.get("rating", 0) for r in reviews) / len(reviews) + earned += int((avg_rating / 5) * 8) + + downloads = registry_data.get("downloads", 0) + if downloads > 100: + earned += 4 + elif downloads > 10: + earned += 2 + + if registry_data.get("featured"): + earned += 3 + + earned = min(earned, available) + return CategoryScore("Community evidence", earned, available, "checked") diff --git a/src/cmdforge/tool.py b/src/cmdforge/tool.py index 7b55620..9c85e53 100644 --- a/src/cmdforge/tool.py +++ b/src/cmdforge/tool.py @@ -447,6 +447,9 @@ class Tool: source: Optional[ToolSource] = None # Attribution for imported/external tools version: str = "" # Tool version visibility: str = "public" # "public", "private", or "unlisted" + deprecated: bool = False # Tool is deprecated + deprecated_message: str = "" # Migration guidance for deprecated tools + replacement: Optional[str] = None # Suggested replacement tool name input_schema: Optional[dict] = None # JSON Schema for tool input contract output_schema: Optional[dict] = None # JSON Schema for tool output contract path: Optional[Path] = None # Path to config.yaml (set by load_tool) @@ -516,6 +519,9 @@ class Tool: source=source, version=data.get("version", ""), visibility=data.get("visibility", "public"), + deprecated=data.get("deprecated", False), + deprecated_message=data.get("deprecated_message", ""), + replacement=data.get("replacement"), input_schema=data.get("input_schema"), output_schema=data.get("output_schema"), ) @@ -533,6 +539,12 @@ class Tool: # Only include visibility if it's not the default if self.visibility and self.visibility != "public": d["visibility"] = self.visibility + if self.deprecated: + d["deprecated"] = True + if self.deprecated_message: + d["deprecated_message"] = self.deprecated_message + if self.replacement: + d["replacement"] = self.replacement if self.input_schema is not None: d["input_schema"] = self.input_schema if self.output_schema is not None: diff --git a/tests/test_quality.py b/tests/test_quality.py new file mode 100644 index 0000000..defe60f --- /dev/null +++ b/tests/test_quality.py @@ -0,0 +1,150 @@ +"""Tests for explainable quality scoring (M8.Q).""" + +from cmdforge.quality import ( + CategoryScore, + QualityScore, + compute_quality, +) +from cmdforge.preflight import PreflightReport +from cmdforge.tool import Tool + + +class TestCategoryScore: + def test_display(self): + cs = CategoryScore("Tests", 25, 30) + assert cs.display == "25/30" + + def test_percentage(self): + cs = CategoryScore("Tests", 15, 30) + assert cs.percentage == 50.0 + + def test_zero_available(self): + cs = CategoryScore("X", 0, 0) + assert cs.percentage == 0.0 + + +class TestQualityScore: + def test_to_dict(self): + qs = QualityScore( + tool_name="test", + version="1.0.0", + headline=85, + categories=[CategoryScore("Contracts", 15, 15)], + ) + d = qs.to_dict() + assert d["tool"] == "test" + assert d["score"] == 85 + assert d["categories"][0]["name"] == "Contracts" + + def test_str_has_headline(self): + qs = QualityScore( + tool_name="test", + version="1.0.0", + headline=87, + categories=[CategoryScore("Contracts", 15, 15)], + ) + assert "Quality 87" in str(qs) + + +class TestComputeQuality: + def test_empty_tool_low_score(self): + tool = Tool(name="bare") + report = PreflightReport() + qs = compute_quality(tool, report) + assert qs.headline < 50 + # Contracts should be not_tested + contracts = [c for c in qs.categories if c.name == "Contracts"][0] + assert contracts.state == "not_tested" + + def test_with_contracts_scores_higher(self): + tool = Tool( + name="contracted", + input_schema={"type": "object", "properties": {"input": {"type": "string"}}}, + output_schema={"type": "object", "properties": {"result": {"type": "string"}}}, + ) + report = PreflightReport() + qs = compute_quality(tool, report) + contracts = [c for c in qs.categories if c.name == "Contracts"][0] + assert contracts.earned == 15 + assert contracts.state == "checked" + + def test_with_tests_scores_higher(self): + tool = Tool(name="tested") + report = PreflightReport( + generated_tests=[ + {"step": "case-1", "state": "passed", "detail": "ok"}, + {"step": "case-2", "state": "passed", "detail": "ok"}, + ] + ) + qs = compute_quality(tool, report) + tests = [c for c in qs.categories if c.name == "Deterministic tests"][0] + assert tests.earned == 30 + assert tests.state == "checked" + + def test_failed_tests_reduce_score(self): + tool = Tool(name="broken") + report = PreflightReport( + generated_tests=[ + {"step": "case-1", "state": "passed", "detail": "ok"}, + {"step": "case-2", "state": "failed", "detail": "broken"}, + ] + ) + qs = compute_quality(tool, report) + tests = [c for c in qs.categories if c.name == "Deterministic tests"][0] + assert tests.earned == 15 + + def test_regression_no_baseline_is_not_tested(self): + tool = Tool(name="test") + report = PreflightReport() + qs = compute_quality(tool, report) + regression = [c for c in qs.categories if c.name == "Regression history"][0] + assert regression.state == "not_tested" + + def test_regression_stable_full_score(self): + tool = Tool(name="test") + report = PreflightReport( + regression={"has_regressions": False, "summary": "no changes"} + ) + qs = compute_quality(tool, report) + regression = [c for c in qs.categories if c.name == "Regression history"][0] + assert regression.earned == 20 + + def test_regression_regressions_zero(self): + tool = Tool(name="test") + report = PreflightReport( + regression={"has_regressions": True, "summary": "1 regression(s)"} + ) + qs = compute_quality(tool, report) + regression = [c for c in qs.categories if c.name == "Regression history"][0] + assert regression.earned == 0 + + def test_secret_warnings_reduce_security(self): + tool = Tool(name="leaky") + report = PreflightReport( + warnings=["Prompt step contains potential secret pattern 'api_key'"] + ) + qs = compute_quality(tool, report) + security = [c for c in qs.categories if c.name == "Security scrutiny"][0] + assert security.earned == 15 # 20 - 5 + + def test_community_not_tested_without_data(self): + tool = Tool(name="new") + report = PreflightReport() + qs = compute_quality(tool, report) + community = [c for c in qs.categories if c.name == "Community evidence"][0] + assert community.state == "not_tested" + + def test_community_with_data(self): + tool = Tool(name="popular") + report = PreflightReport() + qs = compute_quality( + tool, report, + registry_data={ + "reviews": [{"rating": 4}, {"rating": 5}], + "downloads": 500, + "featured": True, + } + ) + community = [c for c in qs.categories if c.name == "Community evidence"][0] + assert community.earned == 14 # 7 (reviews) + 4 (downloads) + 3 (featured) + assert community.state == "checked"