Harden M8 trust and guidance workflows

This commit is contained in:
rob
2026-07-20 15:21:50 -03:00
parent d954c1823a
commit 4d22d532a3
24 changed files with 1281 additions and 121 deletions
+4 -1
View File
@@ -648,7 +648,10 @@ def cmd_inspect(args):
# Quality score
from ..quality import compute_quality
qs = compute_quality(tool, report)
print(f"Quality {qs.headline} — Evaluated {qs.last_evaluated[:10]}")
print(
f"Quality {qs.headline} — {qs.evidence_coverage}% evidence coverage — "
f"Evaluated {qs.last_evaluated[:10]}"
)
for cat in qs.categories:
print(f" {cat.name:24s} {cat.display:>8s}")
print()
+92 -30
View File
@@ -2,8 +2,10 @@
import sys
import os
import select
import tty
import termios
from concurrent.futures import ThreadPoolExecutor
from typing import List, Tuple, Optional
from dataclasses import dataclass
@@ -30,6 +32,7 @@ GREEN = "\033[32m"
RESET = "\033[0m"
MAX_VISIBLE = 8 # Show at most 8 items
_registry_cache = {}
# Output stream for UI (stderr when stdout is piped, stdout otherwise)
_ui_out = None
@@ -90,31 +93,60 @@ def search_registry(query: str) -> List[dict]:
"""Search the registry for tools matching the query.
Returns a list of dicts with name, desc, and registry metadata.
Returns empty list if no token configured or search fails.
Registry search is public; authentication is not required.
"""
if not query or len(query) < 2:
return []
normalized = query.strip().lower()
if normalized in _registry_cache:
return [dict(item) for item in _registry_cache[normalized]]
try:
from ..registry_client import get_client
from ..registry_client import RegistryError, get_client
client = get_client()
if not client or not client.token:
return []
results = client.search_tools(query, per_page=5)
items = results.data if hasattr(results, "data") else results
return [
client.timeout = min(getattr(client, "timeout", 3), 3)
client.max_retries = 1
semantic = client.semantic_search(query, limit=5)
if semantic.get("available") and semantic.get("data"):
items = semantic["data"]
else:
results = client.search_tools(query, per_page=5)
items = results.data if hasattr(results, "data") else results
found = [
{
"name": f"{item.get('owner', '')}/{item.get('name', '')}",
"desc": (item.get("description") or "")[:50],
"args": [],
"registry": True,
"downloads": item.get("downloads", 0),
"relevance": item.get("similarity", item.get("score")),
"quality_score": item.get("quality_score"),
"quality_coverage": item.get("quality_coverage"),
}
for item in items
]
except Exception:
_registry_cache[normalized] = found
return [dict(item) for item in found]
except RegistryError:
return []
def _install_registry_selection(tool: dict) -> PickerResult:
"""Install a registry result and retain its qualified identity for execution."""
from ..resolver import install_from_registry
install_from_registry(tool["name"])
return PickerResult(tool["name"], {})
def _local_selection(tool: dict, arguments: Optional[dict] = None) -> PickerResult:
if tool.get("deprecated"):
guidance = tool.get("deprecated_message") or "This tool is no longer maintained."
if tool.get("replacement"):
guidance += f" Use '{tool['replacement']}' instead."
_write(f"{YELLOW}Warning: {guidance}{RESET}\n")
return PickerResult(tool["name"], arguments or {})
class TTYInput:
"""Read from /dev/tty for keyboard input, even when stdin is piped."""
@@ -135,10 +167,14 @@ class TTYInput:
if self.tty:
self.tty.close()
def getch(self):
def getch(self, timeout: Optional[float] = None):
"""Read a single character."""
tty.setraw(self.fd)
try:
if timeout is not None:
ready, _, _ = select.select([self.tty], [], [], timeout)
if not ready:
return None
ch = self.tty.read(1)
if ch == '\x1b':
# Read escape sequence - bytes arrive together from terminal
@@ -164,19 +200,25 @@ def clear_dropdown(n_lines: int):
def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
"""Run inline picker."""
tools = get_tools()
if not tools:
_write("No tools. Create one: cmdforge\n")
return None
query = ""
selected = 0
scroll = 0
last_drawn = 0
registry_results = {}
registry_future = None
registry_future_query = None
registry_executor = ThreadPoolExecutor(max_workers=1)
_write(HIDE_CURSOR)
try:
while True:
if registry_future is not None and registry_future.done():
registry_results[registry_future_query] = registry_future.result()
registry_future = None
registry_future_query = None
# Filter
matches = []
for t in tools:
@@ -190,12 +232,17 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
filtered = [m[0] for m in matches]
# If no local matches, search registry
registry_results = []
if query and len(query) >= 2 and not filtered:
registry_results = search_registry(query)
for rt in registry_results:
normalized_query = query.strip().lower()
found = registry_results.get(normalized_query, [])
if registry_future is None and normalized_query not in registry_results:
registry_future_query = normalized_query
registry_future = registry_executor.submit(
search_registry, query
)
for rt in found:
rt["_registry"] = True
filtered = registry_results
filtered = found
if selected >= len(filtered):
selected = max(0, len(filtered) - 1)
@@ -232,6 +279,12 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
# Add registry marker
if is_registry:
prefix += f" {YELLOW}[registry]{RESET}"
if t.get("quality_score") is not None:
prefix += f" Q{t['quality_score']}"
if t.get("quality_coverage") is not None:
prefix += f"/{t['quality_coverage']}%"
if t.get("relevance") is not None:
prefix += f" R{float(t['relevance']):.2f}"
# Add deprecation marker
if is_deprecated:
prefix += f" {YELLOW}[deprecated]{RESET}"
@@ -251,7 +304,11 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
last_drawn = len(lines)
# Input
ch = tty_input.getch()
ch = tty_input.getch(
timeout=0.1 if registry_future is not None else None
)
if ch is None:
continue
if ch in ('\r', '\n'): # Enter - run
if filtered:
@@ -261,29 +318,32 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
# If it's a registry tool, install it first
if selected_tool.get("_registry") or selected_tool.get("registry"):
_write(f"{YELLOW}Installing {selected_tool['name']}...{RESET}\n")
import subprocess
proc = subprocess.run(
["cmdforge", "registry", "install", selected_tool["name"]],
capture_output=True, text=True
)
if proc.returncode != 0:
_write(f"Install failed: {proc.stderr}\n")
try:
return _install_registry_selection(selected_tool)
except Exception as exc:
_write(f"Install failed: {exc}\n")
return None
# Extract tool name from "owner/name" format
tool_name = selected_tool["name"].split("/")[-1]
return PickerResult(tool_name, {})
return PickerResult(selected_tool["name"], {})
return _local_selection(selected_tool)
elif ch == '\t': # Tab - configure args or run
if filtered:
tool = filtered[selected]
clear_dropdown(last_drawn)
if tool.get("_registry") or tool.get("registry"):
_write(f"{YELLOW}Installing {tool['name']}...{RESET}\n")
try:
_write(SHOW_CURSOR)
return _install_registry_selection(tool)
except Exception as exc:
_write(f"Install failed: {exc}\n")
_write(SHOW_CURSOR)
return None
if tool['args']:
args = pick_args(tty_input, tool)
_write(SHOW_CURSOR)
return PickerResult(tool["name"], args) if args is not None else None
return _local_selection(tool, args) if args is not None else None
_write(SHOW_CURSOR)
return PickerResult(tool["name"], {})
return _local_selection(tool)
elif ch == '\x1b' or ch == '\x03': # Esc or Ctrl+C
clear_dropdown(last_drawn)
@@ -308,6 +368,8 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
except Exception:
_write(SHOW_CURSOR)
raise
finally:
registry_executor.shutdown(wait=False, cancel_futures=True)
def pick_args(tty_input: TTYInput, tool: dict) -> Optional[dict]:
+55
View File
@@ -410,6 +410,22 @@ def _print_preflight_sections(report: dict, prefix: str = "") -> None:
)
def _print_quality_summary(quality: dict | None) -> None:
if not quality:
return
print(
f"Quality {quality.get('score', 0)} — "
f"{quality.get('evidence_coverage', 0)}% evidence coverage"
)
for category in quality.get("categories", []):
state = category.get("state", "not_tested")
value = (
f"{category.get('earned', 0)}/{category.get('available', 0)}"
if state == "checked" else state.replace("_", " ")
)
print(f" {category.get('name', 'Unknown'):24s} {value:>12s}")
def _print_registry_suggestions(suggestions: dict) -> None:
"""Render registry-specific category, similarity, and scrutiny evidence."""
category = suggestions.get("category") or {}
@@ -547,6 +563,7 @@ def _cmd_registry_publish(args):
remote_report = result.get("preflight") or {}
_print_preflight_sections(remote_report, prefix="Registry ")
_print_quality_summary(result.get("quality"))
_print_registry_suggestions(result.get("suggestions") or {})
if not remote_report.get("errors"):
print("Registry preflight passed.")
@@ -691,6 +708,44 @@ def _cmd_registry_publish(args):
except Exception:
pass # Continue with publish
# Stage 1 runs after dependency publication and any automatic version bump,
# so the evidence describes the exact version Stage 2 will mutate.
from ..preflight import analyze_tool
try:
preflight_tool = Tool.from_dict(data)
preflight_tool.path = config_path
local_report = analyze_tool(preflight_tool, include_contract_tests=True)
except (KeyError, TypeError, ValueError) as exc:
print(f"Local preflight error: {exc}", file=sys.stderr)
return 1
print("Preflight stage:")
_print_preflight_sections(local_report.to_dict(), prefix="Local ")
if not local_report.ok:
return 1
try:
preflight_client = get_client()
preflight_result = preflight_client.publish_tool(
config_yaml, readme=readme, defaults=defaults,
owner=getattr(args, "owner", ""), dry_run=True,
)
except RegistryError as exc:
print(f"Registry preflight failed: {exc}", file=sys.stderr)
return 1
remote_report = preflight_result.get("preflight") or {}
_print_preflight_sections(remote_report, prefix="Registry ")
_print_quality_summary(preflight_result.get("quality"))
if remote_report.get("errors"):
return 1
if sys.stdin.isatty() and not getattr(args, "force", False):
try:
if input("Publish this validated version? [y/N] ").strip().lower() != "y":
print("Cancelled.")
return 1
except (EOFError, KeyboardInterrupt):
print("\nCancelled.")
return 1
print(f"Publishing {name}@{version}...")
try:
+4
View File
@@ -187,6 +187,10 @@ def cmd_run(args):
if not tool:
print(f"Error: Tool '{args.name}' not found.", file=sys.stderr)
return 1
from ..tool import deprecation_warning
warning = deprecation_warning(tool)
if warning:
print(warning, file=sys.stderr)
tool_args = list(args.tool_args or [])
if tool_args and tool_args[0] == '--':
+11
View File
@@ -1,5 +1,6 @@
"""Registry page - browse and install tools from registry."""
import html
from PySide6.QtWidgets import (
QWidget, QVBoxLayout, QHBoxLayout, QLineEdit,
QPushButton, QTableWidget, QTableWidgetItem, QLabel,
@@ -877,6 +878,16 @@ class RegistryPage(QWidget):
if tool.get("description"):
lines.append(f"<p style='color: #4a5568;'>{tool.get('description')}</p>")
if tool.get("deprecated"):
guidance = html.escape(
tool.get("deprecated_message") or "This tool is no longer maintained."
)
if tool.get("replacement"):
guidance += f" Use <strong>{html.escape(tool['replacement'])}</strong> instead."
lines.append(
"<p style='background: #fef3c7; color: #92400e; padding: 8px 10px; "
f"border-radius: 4px;'><strong>Deprecated:</strong> {guidance}</p>"
)
# Rating display
rating = tool.get("average_rating", 0) or 0
+42 -3
View File
@@ -1161,18 +1161,57 @@ class ToolBuilderPage(QWidget):
"It will not be applied automatically."
)
msg.setInformativeText(
"Review the proposal below. Contract editing and approval "
"will be added by the guided-creation milestone."
"Review the proposal below. Applying it changes only this "
"in-memory draft and remains reversible until you save."
)
msg.setDetailedText(
yaml.safe_dump(proposed_schemas, sort_keys=False).rstrip()
)
apply_button = msg.addButton(
"Apply Proposal", QMessageBox.AcceptRole
)
save_button = msg.addButton(
"Save Without Applying", QMessageBox.AcceptRole
)
msg.addButton(QMessageBox.Cancel)
msg.exec()
if msg.clickedButton() is not save_button:
if msg.clickedButton() is apply_button:
if proposed_schemas.get("input_schema") is not None:
tool.input_schema = proposed_schemas["input_schema"]
if proposed_schemas.get("output_schema") is not None:
tool.output_schema = proposed_schemas["output_schema"]
elif msg.clickedButton() is not save_button:
return
if preflight.reuse_opportunities:
reuse = QMessageBox(self)
reuse.setIcon(QMessageBox.Information)
reuse.setWindowTitle("Reuse Opportunities")
reuse.setText(
"CmdForge found exact, contract-backed reuse evidence. "
"This is advisory and will not change your draft."
)
reuse.setDetailedText("\n".join(
f"- {item['detail']}\n Evidence: {item.get('evidence', 'exact match')}"
for item in preflight.reuse_opportunities
))
continue_button = reuse.addButton(
"Continue With Draft", QMessageBox.AcceptRole
)
matching = next(
(item.get("tool") for item in preflight.reuse_opportunities if item.get("tool")),
None,
)
open_button = (
reuse.addButton("Open Existing", QMessageBox.ActionRole)
if matching else None
)
reuse.addButton(QMessageBox.Cancel)
reuse.exec()
if open_button and reuse.clickedButton() is open_button:
_switch_to_existing_tool(self.main_window, matching)
return
if reuse.clickedButton() is not continue_button:
return
advisory_count = len(preflight.warnings) + len(preflight.suggestions)
+11
View File
@@ -1,5 +1,6 @@
"""Tools page - main view for managing tools."""
import html
from collections import defaultdict
from pathlib import Path
from typing import Optional, Tuple, Dict, Any
@@ -852,6 +853,16 @@ class ToolsPage(QWidget):
lines.append(f"<h2 style='margin: 0 0 8px 0; color: #2d3748;'>{tool.name}</h2>")
if tool.description:
lines.append(f"<p style='color: #4a5568; margin-bottom: 16px;'>{tool.description}</p>")
if tool.deprecated:
guidance = html.escape(
tool.deprecated_message or "This tool is no longer maintained."
)
if tool.replacement:
guidance += f" Use <strong>{html.escape(tool.replacement)}</strong> instead."
lines.append(
"<p style='background: #fef3c7; color: #92400e; padding: 8px 10px; "
f"border-radius: 4px;'><strong>Deprecated:</strong> {guidance}</p>"
)
# Publish state
state, registry_hash = get_tool_publish_state(qname)
+88 -48
View File
@@ -5,6 +5,8 @@ All checks are deterministic and evidence-based.
"""
import copy
import hashlib
import json
import re
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
@@ -26,9 +28,6 @@ class PreflightReport:
regression: Optional[Dict[str, Any]] = None
reuse_opportunities: List[Dict[str, Any]] = field(default_factory=list)
audit_evidence: Optional[Dict[str, Any]] = None
compatibility: List[Dict[str, Any]] = field(default_factory=list)
contract_proposal: Optional[Dict[str, Any]] = None
regression: Optional[Dict[str, Any]] = None
@property
def ok(self) -> bool:
@@ -69,6 +68,7 @@ def analyze_tool(
*,
check_local_dependencies: bool = True,
include_contract_tests: bool = False,
check_reuse: Optional[bool] = None,
) -> PreflightReport:
"""Run all preflight checks on a tool.
@@ -80,17 +80,27 @@ def analyze_tool(
A PreflightReport with structured findings.
"""
report = PreflightReport()
checks_run = ["config_integrity", "contracts"]
_check_config_integrity(tool, report)
_check_contracts(tool, report)
_add_contract_guidance(tool, report, include_contract_tests)
if include_contract_tests:
checks_run.append("contract_conformance")
_check_secrets(tool, report)
checks_run.append("secrets")
if check_local_dependencies:
_check_toolstep_compatibility(tool, report)
_check_dependencies(tool, report)
_check_reuse_opportunities(tool, report)
_add_audit_evidence(tool, report)
checks_run.extend(["toolstep_compatibility", "dependencies"])
if registry_client:
_check_similar_tools(tool, report, registry_client)
checks_run.append("registry_similarity")
if check_reuse is None:
check_reuse = check_local_dependencies
if check_reuse:
_check_reuse_opportunities(tool, report)
checks_run.append("reuse_opportunities")
_add_audit_evidence(tool, report, checks_run)
return report
@@ -372,21 +382,14 @@ def _is_semver(version: str) -> bool:
def _check_reuse_opportunities(tool: Tool, report: PreflightReport):
"""Detect repeated step patterns that could be extracted into reusable tools."""
"""Report only exact repeated sequences with contract-boundary evidence."""
from .tool import list_tools, load_tool
if len(tool.steps) < 2:
return
# Build a signature of each step for comparison
step_signatures = []
for step in tool.steps:
sig = _step_signature(step)
if sig:
step_signatures.append(sig)
if not step_signatures:
return
step_signatures = [_step_signature(step) for step in tool.steps]
_find_internal_reuse(tool, step_signatures, report)
# Check if this step sequence appears in other local tools
our_sequence = tuple(step_signatures)
@@ -398,55 +401,92 @@ def _check_reuse_opportunities(tool: Tool, report: PreflightReport):
continue
other_sigs = [_step_signature(s) for s in other.steps]
other_sigs = [s for s in other_sigs if s]
if our_sequence == tuple(other_sigs):
if (
our_sequence == tuple(other_sigs)
and _has_tool_contracts(tool)
and _has_tool_contracts(other)
):
report.reuse_opportunities.append({
"type": "duplicate_sequence",
"tool": name,
"detail": f"Step sequence identical to '{name}' — consider extracting shared logic",
"length": len(our_sequence),
"evidence": "exact normalized steps and explicit tool contracts",
"detail": f"Exact contracted step sequence also exists in '{name}'",
})
elif len(our_sequence) >= 2:
# Check for overlapping subsequences
for start in range(len(other_sigs) - 1):
for length in range(2, min(len(our_sequence), len(other_sigs) - start) + 1):
if our_sequence[:length] == tuple(other_sigs[start:start + length]):
report.reuse_opportunities.append({
"type": "shared_subsequence",
"tool": name,
"length": length,
"detail": f"Shares {length} step(s) with '{name}' — possible extraction candidate",
})
break
def _step_signature(step) -> str:
"""Build a comparable signature for a step."""
if hasattr(step, "prompt"):
return f"prompt:{step.provider}"
elif hasattr(step, "code"):
return "code"
elif hasattr(step, "tool"):
return f"tool:{step.tool}"
elif hasattr(step, "server"):
return f"mcp:{step.server}"
return ""
"""Build an exact normalized signature without incidental labels/outputs."""
data = step.to_dict()
data.pop("name", None)
data.pop("output_var", None)
encoded = json.dumps(data, sort_keys=True, separators=(",", ":"), default=str)
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _add_audit_evidence(tool: Tool, report: PreflightReport):
def _has_tool_contracts(tool: Tool) -> bool:
return tool.input_schema is not None and tool.output_schema is not None
def _find_internal_reuse(
tool: Tool, signatures: List[str], report: PreflightReport
) -> None:
"""Find non-overlapping repeated sequences with per-step output evidence."""
from .tool import PromptStep
count = len(signatures)
for length in range(count // 2, 1, -1):
for first in range(0, count - (2 * length) + 1):
candidate = signatures[first:first + length]
for second in range(first + length, count - length + 1):
if candidate != signatures[second:second + length]:
continue
steps = tool.steps[first:first + length]
if not all(
isinstance(step, PromptStep) and step.output_schema is not None
for step in steps
):
continue
report.reuse_opportunities.append({
"type": "repeated_sequence",
"length": length,
"locations": [first + 1, second + 1],
"evidence": "exact normalized prompt steps with output schemas",
"detail": (
f"Steps {first + 1}-{first + length} repeat at "
f"steps {second + 1}-{second + length}"
),
})
return
def _add_audit_evidence(
tool: Tool, report: PreflightReport, checks_run: List[str]
):
"""Attach audit evidence metadata to the report."""
from datetime import datetime, timezone
findings = {
"errors": report.errors,
"warnings": report.warnings,
"suggestions": report.suggestions,
"generated_tests": report.generated_tests,
"regression": report.regression,
"compatibility": report.compatibility,
"similar_tools": report.similar_tools,
"reuse_opportunities": report.reuse_opportunities,
}
findings_hash = hashlib.sha256(
json.dumps(findings, sort_keys=True, separators=(",", ":"), default=str).encode()
).hexdigest()
report.audit_evidence = {
"tool_version": tool.version or "",
"engine_version": "1.0",
"timestamp": datetime.now(timezone.utc).isoformat(),
"checks_run": [
"config_integrity",
"contracts",
"secrets",
"dependencies",
"toolstep_compatibility",
"reuse_opportunities",
],
"checks_run": checks_run,
"findings_hash": findings_hash,
"findings": copy.deepcopy(findings),
"outcome": "failed" if report.errors else "advisory" if report.warnings else "passed",
}
+64 -22
View File
@@ -20,9 +20,14 @@ class CategoryScore:
earned: int
available: int
state: str = "checked" # checked | not_tested | not_applicable
possible: Optional[int] = None # Category maximum when only partial evidence ran
@property
def display(self) -> str:
if self.state == "not_tested":
return "not tested"
if self.state == "not_applicable":
return "n/a"
return f"{self.earned}/{self.available}"
@property
@@ -39,6 +44,7 @@ class QualityScore:
headline: int # 0-100
categories: List[CategoryScore] = field(default_factory=list)
last_evaluated: str = ""
evidence_coverage: int = 0
def to_dict(self) -> dict:
return {
@@ -51,14 +57,16 @@ class QualityScore:
"earned": c.earned,
"available": c.available,
"state": c.state,
"possible": c.possible if c.possible is not None else c.available,
}
for c in self.categories
],
"last_evaluated": self.last_evaluated,
"evidence_coverage": self.evidence_coverage,
}
def __str__(self) -> str:
lines = [f"Quality {self.headline}"]
lines = [f"Quality {self.headline} — {self.evidence_coverage}% evidence coverage"]
if self.last_evaluated:
lines[0] += f" — Evaluated {self.last_evaluated}"
for c in self.categories:
@@ -91,8 +99,13 @@ def compute_quality(
categories.append(_score_security(report, tool))
categories.append(_score_community(registry_data))
earned = sum(c.earned for c in categories)
available = sum(c.available for c in categories)
checked = [c for c in categories if c.state == "checked"]
earned = sum(c.earned for c in checked)
available = sum(c.available for c in checked)
possible = sum(
c.possible if c.possible is not None else c.available
for c in categories if c.state != "not_applicable"
)
headline = int((earned / available * 100)) if available > 0 else 0
return QualityScore(
@@ -101,6 +114,7 @@ def compute_quality(
headline=headline,
categories=categories,
last_evaluated=datetime.now(timezone.utc).isoformat(),
evidence_coverage=int((available / possible * 100)) if possible else 0,
)
@@ -112,6 +126,9 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore:
earned += 5
if tool.output_schema is not None:
earned += 5
if tool.input_schema is None and tool.output_schema is None:
return CategoryScore("Contracts", 0, available, "not_tested")
# No contract-related errors in the preflight
contract_errors = [
e for e in report.errors if "schema" in e.lower() or "contract" in e.lower()
@@ -121,27 +138,26 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore:
else:
earned = max(0, earned - len(contract_errors))
state = "checked" if (tool.input_schema or tool.output_schema) else "not_tested"
return CategoryScore("Contracts", earned, available, state)
return CategoryScore("Contracts", earned, available, "checked")
def _score_tests(report: PreflightReport) -> CategoryScore:
available = 30
maximum = 30
tests = report.generated_tests or []
if not tests:
return CategoryScore("Deterministic tests", 0, available, "not_tested")
passed = sum(1 for t in tests if t.get("state") == "passed")
failed = sum(1 for t in tests if t.get("state") == "failed")
total = len(tests)
if total == 0:
return CategoryScore("Deterministic tests", 0, available, "not_tested")
return CategoryScore("Deterministic tests", 0, maximum, "not_tested")
executed = [t for t in tests if t.get("state") in ("passed", "failed")]
if not executed:
return CategoryScore("Deterministic tests", 0, maximum, "not_tested")
passed = sum(1 for t in executed if t.get("state") == "passed")
total = len(executed)
available = max(1, int(maximum * total / len(tests)))
earned = int((passed / total) * available)
state = "checked" if failed == 0 else "checked"
return CategoryScore("Deterministic tests", earned, available, state)
return CategoryScore(
"Deterministic tests", earned, available, "checked", possible=maximum
)
def _score_regression(report: PreflightReport) -> CategoryScore:
@@ -162,7 +178,23 @@ def _score_regression(report: PreflightReport) -> CategoryScore:
def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
available = 20
maximum = 20
checks = set((report.audit_evidence or {}).get("checks_run", []))
if "security_scrutiny" in checks:
available = maximum
elif checks:
available = (10 if "secrets" in checks else 0) + (
10 if checks.intersection({"dependencies", "registry_dependencies"}) else 0
)
elif report.errors or report.warnings:
# Backward-compatible evidence supplied by callers without audit metadata.
available = maximum
else:
available = 0
if not available:
return CategoryScore(
"Security scrutiny", 0, maximum, "not_tested", possible=maximum
)
earned = available
# Deduct for secret patterns found
@@ -172,13 +204,20 @@ def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
earned -= len(secret_warnings) * 5
# Deduct for unresolved dependencies
dep_warnings = [
w for w in report.warnings if "dependency" in w.lower() and "not installed" in w.lower()
dependency_findings = [
finding for finding in report.warnings + report.errors
if ("dependency" in finding.lower() or "dependencies" in finding.lower())
and any(
marker in finding.lower()
for marker in ("not installed", "missing", "deprecated")
)
]
earned -= len(dep_warnings) * 3
earned -= len(dependency_findings) * 3
earned = max(0, earned)
return CategoryScore("Security scrutiny", earned, available, "checked")
return CategoryScore(
"Security scrutiny", earned, available, "checked", possible=maximum
)
def _score_community(registry_data: Optional[dict]) -> CategoryScore:
@@ -189,8 +228,11 @@ def _score_community(registry_data: Optional[dict]) -> CategoryScore:
earned = 0
reviews = registry_data.get("reviews", [])
if reviews:
rating_count = registry_data.get("rating_count", len(reviews))
avg_rating = registry_data.get("average_rating")
if avg_rating is None and reviews:
avg_rating = sum(r.get("rating", 0) for r in reviews) / len(reviews)
if avg_rating is not None and rating_count:
earned += int((avg_rating / 5) * 8)
downloads = registry_data.get("downloads", 0)
+174 -8
View File
@@ -9,7 +9,7 @@ import os
import re
import secrets
from dataclasses import dataclass
from datetime import date, datetime, timedelta
from datetime import date, datetime, timedelta, timezone
from typing import Any, Dict, Iterable, List, Optional, Tuple
from flask import Flask, Response, g, jsonify, request
@@ -418,7 +418,7 @@ def create_app() -> Flask:
g.db.execute(
"UPDATE api_tokens SET last_used_at = ? WHERE id = ?",
[datetime.utcnow().isoformat(), row["id"]],
[datetime.now(timezone.utc).isoformat(), row["id"]],
)
g.current_publisher = {
"id": row["publisher_id"],
@@ -870,6 +870,14 @@ def create_app() -> Flask:
order_sql = f"{sort} {order_dir}, published_at DESC, f.id DESC"
stats_join = "LEFT JOIN tool_stats ts ON ts.tool_id = f.id"
audit_join = """
LEFT JOIN tool_audits ta ON ta.id = (
SELECT latest_audit.id FROM tool_audits latest_audit
WHERE latest_audit.tool_id = f.id
ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC
LIMIT 1
)
"""
rows = query_all(
g.db,
@@ -897,9 +905,13 @@ def create_app() -> Flask:
GROUP BY owner, name
)
SELECT f.*, COALESCE(ts.average_rating, 0) AS average_rating,
COALESCE(ts.rating_count, 0) AS rating_count
COALESCE(ts.rating_count, 0) AS rating_count,
json_extract(ta.quality_json, '$.score') AS quality_score,
json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage,
ta.evaluated_at AS quality_evaluated_at
FROM filtered f
{stats_join}
{audit_join}
JOIN (
SELECT a.owner, a.name, COALESCE(s.max_id, a.max_id) AS max_id
FROM latest_any a
@@ -948,6 +960,12 @@ def create_app() -> Flask:
"average_rating": row["average_rating"],
"rating_count": row["rating_count"],
"score": score,
"quality_score": row["quality_score"],
"quality_coverage": row["quality_coverage"],
"quality_evaluated_at": row["quality_evaluated_at"],
"deprecated": bool(row["deprecated"]),
"deprecated_message": row["deprecated_message"],
"replacement": row["replacement"],
})
result: dict = {"data": data, "meta": paginate(page, per_page, total)}
@@ -1089,6 +1107,39 @@ def create_app() -> Flask:
if fork_row:
fork_count = fork_row["cnt"]
replacement_chain = []
if row["replacement"]:
try:
replacement_chain = _get_replacement_chain(
row["owner"], row["name"], row["replacement"]
)
except ValueError:
# Preserve access to legacy rows while refusing new invalid chains.
replacement_chain = []
latest_audit = query_one(
g.db,
"""
SELECT engine_version, evidence_json, quality_json, evaluated_at
FROM tool_audits
WHERE tool_id = ?
ORDER BY evaluated_at DESC, id DESC
LIMIT 1
""",
[row["id"]],
)
audit_stale = True
if latest_audit:
try:
evaluated = datetime.fromisoformat(
latest_audit["evaluated_at"].replace("Z", "+00:00")
)
if evaluated.tzinfo is None:
evaluated = evaluated.replace(tzinfo=timezone.utc)
audit_stale = datetime.now(timezone.utc) - evaluated > timedelta(days=30)
except (AttributeError, TypeError, ValueError):
audit_stale = True
payload = {
"owner": row["owner"],
"name": row["name"],
@@ -1100,7 +1151,8 @@ def create_app() -> Flask:
"published_at": row["published_at"],
"deprecated": bool(row["deprecated"]),
"deprecated_message": row["deprecated_message"],
"replacement": row["replacement"],
"replacement": row["replacement"] if replacement_chain else None,
"replacement_chain": replacement_chain,
"config": row["config_yaml"],
"readme": row["readme"],
"defaults": row.get("defaults") or "",
@@ -1108,6 +1160,10 @@ def create_app() -> Flask:
"forked_from": row.get("forked_from"),
"forked_version": row.get("forked_version"),
"fork_count": fork_count,
"audit": json.loads(latest_audit["evidence_json"]) if latest_audit else None,
"quality": json.loads(latest_audit["quality_json"]) if latest_audit else None,
"audit_evaluated_at": latest_audit["evaluated_at"] if latest_audit else None,
"audit_stale": audit_stale,
}
response = jsonify({"data": payload})
response.headers["Cache-Control"] = "max-age=60"
@@ -2496,8 +2552,12 @@ def create_app() -> Flask:
from ..preflight import analyze_tool as run_preflight
from ..tool import Tool
published_tool = Tool.from_dict(data)
preflight_report = run_preflight(
Tool.from_dict(data), check_local_dependencies=False
published_tool,
check_local_dependencies=False,
include_contract_tests=True,
check_reuse=False,
)
except (KeyError, TypeError, ValueError) as exc:
return error_response("INVALID_CONFIG", str(exc), 400)
@@ -2538,6 +2598,44 @@ def create_app() -> Flask:
)
owner = requested_owner
from .auditing import registry_dependency_findings
dependency_findings = registry_dependency_findings(
g.db, published_tool, owner
)
missing_dependencies = dependency_findings["missing"]
preflight_report.warnings.extend(dependency_findings["deprecated"])
from ..preflight import _add_audit_evidence
audit_checks = list(
(preflight_report.audit_evidence or {}).get("checks_run", [])
)
_add_audit_evidence(
published_tool, preflight_report,
audit_checks + ["registry_dependencies"],
)
from ..quality import compute_quality
quality_report = compute_quality(
published_tool, preflight_report
).to_dict()
if missing_dependencies:
message = "Missing registry dependencies: " + ", ".join(
missing_dependencies
)
preflight_report.errors.append(message)
_add_audit_evidence(
published_tool, preflight_report,
audit_checks + ["registry_dependencies"],
)
return error_response(
"PREFLIGHT_FAILED", message, 400,
details={"preflight": preflight_report.to_dict()},
)
if data.get("deprecated") and data.get("replacement"):
try:
_get_replacement_chain(owner, name, data["replacement"])
except ValueError as exc:
return error_response("VALIDATION_ERROR", str(exc), 400)
# Compute config hash early for idempotency check
config_hash = compute_yaml_hash(config_text)
@@ -2650,6 +2748,7 @@ def create_app() -> Flask:
"status": "validated",
"suggestions": suggestions,
"preflight": preflight_report.to_dict(),
"quality": quality_report,
}
})
@@ -2675,7 +2774,7 @@ def create_app() -> Flask:
# Public tools need moderation approval
moderation_status = "pending"
g.db.execute(
insert_cursor = g.db.execute(
"""
INSERT INTO tools (
owner, name, version, description, category, tags, config_yaml, readme,
@@ -2710,7 +2809,25 @@ def create_app() -> Flask:
moderation_status,
forked_from,
forked_version,
datetime.utcnow().isoformat(),
datetime.now(timezone.utc).isoformat(),
],
)
tool_id = insert_cursor.lastrowid
audit = preflight_report.audit_evidence or {}
g.db.execute(
"""
INSERT INTO tool_audits (
tool_id, engine_version, trigger, evidence_json, quality_json,
findings_hash, evaluated_at
) VALUES (?, ?, 'publish', ?, ?, ?, ?)
""",
[
tool_id,
audit.get("engine_version", "1.0"),
json.dumps(audit, sort_keys=True),
json.dumps(quality_report, sort_keys=True),
audit.get("findings_hash", ""),
audit.get("timestamp") or datetime.now(timezone.utc).isoformat(),
],
)
g.db.commit()
@@ -2756,6 +2873,7 @@ def create_app() -> Flask:
"forked_version": forked_version,
"suggestions": suggestions,
"preflight": preflight_report.to_dict(),
"quality": quality_report,
}
})
response.status_code = 201
@@ -2870,6 +2988,47 @@ def create_app() -> Flask:
return jsonify({"data": results})
def _get_replacement_chain(
original_owner: str, original_name: str, replacement: str
) -> List[str]:
"""Validate and resolve a deprecation chain without allowing cycles."""
chain: List[str] = []
visited = {(original_owner, original_name)}
current = replacement
default_owner = original_owner
for _ in range(20):
if "/" in current:
next_owner, next_name = current.split("/", 1)
else:
next_owner, next_name = default_owner, current
if not OWNER_RE.fullmatch(next_owner) or not TOOL_NAME_RE.fullmatch(next_name):
raise ValueError(
"replacement must be a valid tool name or owner/tool reference"
)
identity = (next_owner, next_name)
if identity in visited:
raise ValueError("replacement creates a deprecation cycle")
row = query_one(
g.db,
"""
SELECT deprecated, replacement FROM tools
WHERE owner = ? AND name = ?
ORDER BY id DESC LIMIT 1
""",
[next_owner, next_name],
)
if row is None:
raise ValueError(
f"replacement tool '{next_owner}/{next_name}' does not exist"
)
visited.add(identity)
chain.append(f"{next_owner}/{next_name}")
if not row["deprecated"] or not row["replacement"]:
return chain
current = row["replacement"]
default_owner = next_owner
raise ValueError("replacement chain exceeds 20 tools")
@app.route("/api/v1/tools/<owner>/<name>/deprecate", methods=["POST"])
@require_token
def deprecate_tool(owner: str, name: str) -> Response:
@@ -2883,6 +3042,10 @@ def create_app() -> Flask:
if message and len(message) > 500:
return error_response("VALIDATION_ERROR", "Message too long (max 500)", 400)
try:
chain = _get_replacement_chain(owner, name, replacement) if replacement else []
except ValueError as exc:
return error_response("VALIDATION_ERROR", str(exc), 400)
# Update all versions of the tool
result = g.db.execute(
@@ -2896,7 +3059,10 @@ def create_app() -> Flask:
return error_response("TOOL_NOT_FOUND", f"Tool {owner}/{name} not found", 404)
g.db.commit()
return jsonify({"data": {"status": "deprecated", "owner": owner, "name": name}})
return jsonify({"data": {
"status": "deprecated", "owner": owner, "name": name,
"replacement_chain": chain,
}})
@app.route("/api/v1/tools/<owner>/<name>/undeprecate", methods=["POST"])
@require_token
+210
View File
@@ -0,0 +1,210 @@
"""Immutable publish-time and refresh audits for registry tool versions."""
from __future__ import annotations
import argparse
import json
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List
import yaml
from ..preflight import analyze_tool
from ..quality import compute_quality
from ..tool import Tool
from .db import connect_db, get_db_path, init_db
def registry_dependency_findings(conn, tool: Tool, owner: str) -> Dict[str, List[str]]:
"""Resolve registry dependencies and report missing/deprecated references."""
from ..tool import ToolStep
references = set(tool.dependencies)
references.update(
step.tool for step in tool.steps if isinstance(step, ToolStep)
)
missing: List[str] = []
deprecated: List[str] = []
for reference in sorted(references):
if reference == tool.name or reference == f"{owner}/{tool.name}":
continue
if "/" in reference:
dep_owner, dep_name = reference.split("/", 1)
candidates = [(dep_owner, dep_name)]
else:
candidates = [(owner, reference), ("official", reference)]
row = None
for dep_owner, dep_name in candidates:
row = conn.execute(
"""
SELECT deprecated, replacement FROM tools
WHERE owner = ? AND name = ? ORDER BY id DESC LIMIT 1
""",
[dep_owner, dep_name],
).fetchone()
if row is not None:
break
if row is None:
missing.append(reference)
elif row["deprecated"]:
guidance = f"{reference} is deprecated"
if row["replacement"]:
guidance += f"; use {row['replacement']}"
deprecated.append(guidance)
return {"missing": missing, "deprecated": deprecated}
def audit_tool_version(conn, tool_id: int, trigger: str = "background") -> Dict[str, Any]:
"""Evaluate one immutable tool version and append an audit record."""
row = conn.execute(
"""
SELECT tools.*, COALESCE(tool_stats.average_rating, 0) AS average_rating,
COALESCE(tool_stats.rating_count, 0) AS rating_count
FROM tools
LEFT JOIN tool_stats ON tool_stats.tool_id = tools.id
WHERE tools.id = ?
""",
[tool_id],
).fetchone()
if row is None:
raise ValueError(f"Unknown tool version id {tool_id}")
previous = conn.execute(
"""
SELECT evidence_json, quality_json FROM tool_audits
WHERE tool_id = ? ORDER BY evaluated_at DESC, id DESC LIMIT 1
""",
[tool_id],
).fetchone()
config = yaml.safe_load(row["config_yaml"])
tool = Tool.from_dict(config)
report = analyze_tool(
tool,
check_local_dependencies=False,
include_contract_tests=True,
check_reuse=False,
)
dependency_findings = registry_dependency_findings(conn, tool, row["owner"])
if dependency_findings["missing"]:
report.errors.append(
"Missing registry dependencies: "
+ ", ".join(dependency_findings["missing"])
)
report.warnings.extend(dependency_findings["deprecated"])
from ..preflight import _add_audit_evidence
checks = list((report.audit_evidence or {}).get("checks_run", []))
_add_audit_evidence(tool, report, checks + ["registry_dependencies"])
registry_data = {
"average_rating": row["average_rating"],
"rating_count": row["rating_count"],
"downloads": row["downloads"],
}
quality = compute_quality(tool, report, registry_data=registry_data).to_dict()
evidence = report.audit_evidence or {}
evaluated_at = datetime.now(timezone.utc).isoformat()
conn.execute(
"""
INSERT INTO tool_audits (
tool_id, engine_version, trigger, evidence_json, quality_json,
findings_hash, evaluated_at
) VALUES (?, ?, ?, ?, ?, ?, ?)
""",
[
tool_id,
evidence.get("engine_version", "1.0"),
trigger,
json.dumps(evidence, sort_keys=True),
json.dumps(quality, sort_keys=True),
evidence.get("findings_hash", ""),
evaluated_at,
],
)
quality_change = None
if previous:
previous_quality = json.loads(previous["quality_json"])
quality_change = quality["score"] - previous_quality.get("score", 0)
previous_evidence = json.loads(previous["evidence_json"])
if quality_change < 0 or previous_evidence.get("outcome") != evidence.get("outcome"):
conn.execute(
"""
INSERT INTO audit_log (
action, target_type, target_id, actor_id, details
) VALUES ('automated_audit_changed', 'tool', ?, 'system', ?)
""",
[
str(tool_id),
json.dumps({
"quality_change": quality_change,
"previous_outcome": previous_evidence.get("outcome"),
"current_outcome": evidence.get("outcome"),
}, sort_keys=True),
],
)
conn.commit()
return {
"tool_id": tool_id, "evidence": evidence, "quality": quality,
"quality_change": quality_change,
}
def audit_stale_tools(
conn, *, max_age_days: int = 30, limit: int = 100
) -> List[Dict[str, Any]]:
"""Append refreshed evidence for versions with missing or stale audits."""
if max_age_days < 0:
raise ValueError("max_age_days must be non-negative")
if limit < 1:
raise ValueError("limit must be positive")
cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days)
rows = conn.execute(
"""
SELECT tools.id, MAX(tool_audits.evaluated_at) AS last_evaluated
FROM tools
LEFT JOIN tool_audits ON tool_audits.tool_id = tools.id
GROUP BY tools.id
ORDER BY last_evaluated IS NOT NULL, last_evaluated
"""
).fetchall()
stale_ids = []
for row in rows:
value = row["last_evaluated"]
if value:
try:
evaluated = datetime.fromisoformat(value.replace("Z", "+00:00"))
if evaluated.tzinfo is None:
evaluated = evaluated.replace(tzinfo=timezone.utc)
except ValueError:
evaluated = datetime.min.replace(tzinfo=timezone.utc)
if evaluated >= cutoff:
continue
stale_ids.append(row["id"])
if len(stale_ids) >= limit:
break
results = []
for tool_id in stale_ids:
try:
results.append(audit_tool_version(conn, tool_id))
except Exception as exc:
results.append({"tool_id": tool_id, "error": str(exc)})
return results
def main() -> int:
parser = argparse.ArgumentParser(description="Refresh stale CmdForge registry audits")
parser.add_argument("--max-age-days", type=int, default=30)
parser.add_argument("--limit", type=int, default=100)
args = parser.parse_args()
conn = connect_db(get_db_path())
try:
init_db(conn)
results = audit_stale_tools(
conn, max_age_days=args.max_age_days, limit=args.limit
)
finally:
conn.close()
print(f"Audited {len(results)} tool version(s)")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+14
View File
@@ -80,6 +80,20 @@ CREATE TABLE IF NOT EXISTS download_stats (
UNIQUE(tool_id, client_id, downloaded_at)
);
CREATE TABLE IF NOT EXISTS tool_audits (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tool_id INTEGER NOT NULL REFERENCES tools(id) ON DELETE CASCADE,
engine_version TEXT NOT NULL,
trigger TEXT NOT NULL,
evidence_json TEXT NOT NULL,
quality_json TEXT NOT NULL,
findings_hash TEXT NOT NULL,
evaluated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_tool_audits_tool_time
ON tool_audits(tool_id, evaluated_at DESC, id DESC);
CREATE VIRTUAL TABLE IF NOT EXISTS tools_fts USING fts5(
name, description, tags, readme,
content='tools',
+12 -1
View File
@@ -157,10 +157,19 @@ def semantic_search(
"""
SELECT te.tool_id, te.embedding, te.dimensions, te.model,
t.owner, t.name, t.version, t.description, t.category,
t.tags, t.downloads
t.tags, t.downloads,
json_extract(ta.quality_json, '$.score') AS quality_score,
json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage
FROM tool_embeddings te
JOIN tools t ON t.id = te.tool_id
LEFT JOIN tool_audits ta ON ta.id = (
SELECT latest_audit.id FROM tool_audits latest_audit
WHERE latest_audit.tool_id = t.id
ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC
LIMIT 1
)
WHERE t.visibility = 'public' AND t.moderation_status = 'approved'
AND t.deprecated = 0
""",
).fetchall()
@@ -192,6 +201,8 @@ def semantic_search(
"tags": tags_list,
"downloads": row["downloads"] or 0,
"similarity": round(score, 4),
"quality_score": row["quality_score"],
"quality_coverage": row["quality_coverage"],
})
# Sort by similarity descending
+5
View File
@@ -1173,6 +1173,11 @@ def main():
print(f"Searched: {', '.join(e.searched_paths[:3])}", file=sys.stderr)
sys.exit(1)
from .tool import deprecation_warning
warning = deprecation_warning(tool)
if warning:
print(warning, file=sys.stderr)
# Check for manifest overrides
manifest = load_manifest()
provider_override_from_manifest = None
+28 -3
View File
@@ -1,6 +1,7 @@
"""Tool loading, saving, and management."""
import os
import re
import shutil
import stat
from dataclasses import dataclass, field
@@ -15,6 +16,9 @@ TOOLS_DIR = Path.home() / ".cmdforge"
# Default bin directory for wrapper scripts
BIN_DIR = Path.home() / ".local" / "bin"
_TOOL_REF_RE = re.compile(
r"^(?:[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?/)?[A-Za-z0-9_-]{1,64}$"
)
def _validate_skill_selection(skills: Optional[List[str]]) -> None:
@@ -447,16 +451,27 @@ class Tool:
source: Optional[ToolSource] = None # Attribution for imported/external tools
version: str = "" # Tool version
visibility: str = "public" # "public", "private", or "unlisted"
deprecated: bool = False # Tool is deprecated
deprecated_message: str = "" # Migration guidance for deprecated tools
replacement: Optional[str] = None # Suggested replacement tool name
input_schema: Optional[dict] = None # JSON Schema for tool input contract
output_schema: Optional[dict] = None # JSON Schema for tool output contract
path: Optional[Path] = None # Path to config.yaml (set by load_tool)
deprecated: bool = False # Tool is deprecated
deprecated_message: str = "" # Migration guidance for deprecated tools
replacement: Optional[str] = None # Suggested replacement tool name
def __post_init__(self) -> None:
validate_json_schema(self.input_schema, "input_schema")
validate_json_schema(self.output_schema, "output_schema")
if not isinstance(self.deprecated, bool):
raise ValueError("deprecated must be true or false")
if not isinstance(self.deprecated_message, str):
raise ValueError("deprecated_message must be a string")
if len(self.deprecated_message) > 500:
raise ValueError("deprecated_message must be at most 500 characters")
if self.replacement is not None and (
not isinstance(self.replacement, str)
or not _TOOL_REF_RE.fullmatch(self.replacement)
):
raise ValueError("replacement must be a tool name or owner/tool reference")
@classmethod
def from_dict(cls, data: dict) -> "Tool":
@@ -809,6 +824,16 @@ def tool_exists(name: str) -> bool:
return (get_tools_dir() / name / "config.yaml").exists()
def deprecation_warning(tool: Tool) -> Optional[str]:
"""Return consistent migration guidance for a deprecated tool."""
if not tool.deprecated:
return None
message = tool.deprecated_message or "This tool is no longer maintained."
if tool.replacement:
message += f" Use '{tool.replacement}' instead."
return f"Warning: '{tool.name}' is deprecated. {message}"
def validate_tool_name(name: str) -> tuple[bool, str]:
"""
Validate a tool name.