rob
/
CmdForge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Harden M8 trust and guidance workflows

This commit is contained in:
rob 2026-07-20 15:21:50 -03:00
parent d954c1823a
commit 4d22d532a3
24 changed files with 1281 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CLAUDE.md
View File

@ -79,10 +79,16 @@ Tools are YAML configs with:
- `output`: Template for final output (e.g., `"{response}"`)
- `input_schema`, `output_schema`: Optional tool-level JSON Schema contracts;
schemas are validated when tools are loaded or created
- `deprecated`, `deprecated_message`, `replacement`: Validated migration metadata;
replacements may be local names or `owner/tool` registry references
Run `cmdforge inspect <tool> [--registry]` for the shared local preflight
report. `cmdforge registry publish <path> --dry-run` runs local checks first
and, when authenticated, the registry's publish-time checks without publishing.
Normal publication uses the same checks as Stage 1, asks for confirmation in an
interactive terminal, and then performs the Stage 2 mutation. The registry
reruns deterministic conformance authoritatively and stores immutable audit and
quality evidence for the published version.
Inspect also shows inferred contract proposals and runs deterministic contract
conformance when both tool schemas are explicit. Prompt outputs are synthesized
from their schemas; code, nested-tool, and MCP steps are reported as unsupported
@ -92,6 +98,8 @@ evidence to `conformance.json`. Later inspections and local publish dry-runs
rerun the stored synthetic inputs and report state, coverage, contract, and
normalized output changes. ToolStep preflight also compares the values the
runner supplies with the called tool's input contract.
Quality headlines exclude categories with missing evidence and are accompanied
by evidence coverage plus checked/not-tested category states.
### Step Types

5
README.md
View File

@ -144,7 +144,7 @@ cmdforge docs mytool # View/create tool documentation
# Registry
cmdforge registry search "keyword" # Search for tools
cmdforge registry install owner/tool # Install a tool
cmdforge registry publish mytool # Publish your tool
cmdforge registry publish mytool # Preflight, confirm, then publish
cmdforge registry publish mytool --dry-run # Preflight without publishing
cmdforge registry status # Check moderation status
cmdforge registry my-tools # List your published tools
@ -180,6 +180,9 @@ cmdforge config connect username # Connect to registry account
cmdforge config disconnect # Disconnect from registry
```
`cf` searches the public registry when no local tool matches. Registry results
show available relevance and quality evidence and install on selection.
### Running Tools
Once created, tools work like any Unix command:

5
src/cmdforge/cli/__init__.py
View File

@ -648,7 +648,10 @@ def cmd_inspect(args):
# Quality score
from ..quality import compute_quality
qs = compute_quality(tool, report)
print(f"Quality {qs.headline} — Evaluated {qs.last_evaluated[:10]}")
print(
f"Quality {qs.headline} — {qs.evidence_coverage}% evidence coverage — "
f"Evaluated {qs.last_evaluated[:10]}"
)
for cat in qs.categories:
print(f" {cat.name:24s} {cat.display:>8s}")
print()

118
src/cmdforge/cli/picker.py
View File

@ -2,8 +2,10 @@
import sys
import os
import select
import tty
import termios
from concurrent.futures import ThreadPoolExecutor
from typing import List, Tuple, Optional
from dataclasses import dataclass
@ -30,6 +32,7 @@ GREEN = "\033[32m"
RESET = "\033[0m"
MAX_VISIBLE = 8 # Show at most 8 items
_registry_cache = {}
# Output stream for UI (stderr when stdout is piped, stdout otherwise)
_ui_out = None
@ -90,31 +93,60 @@ def search_registry(query: str) -> List[dict]:
"""Search the registry for tools matching the query.
Returns a list of dicts with name, desc, and registry metadata.
Returns empty list if no token configured or search fails.
Registry search is public; authentication is not required.
"""
if not query or len(query) < 2:
return []
normalized = query.strip().lower()
if normalized in _registry_cache:
return [dict(item) for item in _registry_cache[normalized]]
try:
from ..registry_client import get_client
from ..registry_client import RegistryError, get_client
client = get_client()
if not client or not client.token:
return []
client.timeout = min(getattr(client, "timeout", 3), 3)
client.max_retries = 1
semantic = client.semantic_search(query, limit=5)
if semantic.get("available") and semantic.get("data"):
items = semantic["data"]
else:
results = client.search_tools(query, per_page=5)
items = results.data if hasattr(results, "data") else results
return [
found = [
{
"name": f"{item.get('owner', '')}/{item.get('name', '')}",
"desc": (item.get("description") or "")[:50],
"args": [],
"registry": True,
"downloads": item.get("downloads", 0),
"relevance": item.get("similarity", item.get("score")),
"quality_score": item.get("quality_score"),
"quality_coverage": item.get("quality_coverage"),
}
for item in items
]
except Exception:
_registry_cache[normalized] = found
return [dict(item) for item in found]
except RegistryError:
return []
def _install_registry_selection(tool: dict) -> PickerResult:
"""Install a registry result and retain its qualified identity for execution."""
from ..resolver import install_from_registry
install_from_registry(tool["name"])
return PickerResult(tool["name"], {})
def _local_selection(tool: dict, arguments: Optional[dict] = None) -> PickerResult:
if tool.get("deprecated"):
guidance = tool.get("deprecated_message") or "This tool is no longer maintained."
if tool.get("replacement"):
guidance += f" Use '{tool['replacement']}' instead."
_write(f"{YELLOW}Warning: {guidance}{RESET}\n")
return PickerResult(tool["name"], arguments or {})
class TTYInput:
"""Read from /dev/tty for keyboard input, even when stdin is piped."""
@ -135,10 +167,14 @@ class TTYInput:
if self.tty:
self.tty.close()
def getch(self):
def getch(self, timeout: Optional[float] = None):
"""Read a single character."""
tty.setraw(self.fd)
try:
if timeout is not None:
ready, _, _ = select.select([self.tty], [], [], timeout)
if not ready:
return None
ch = self.tty.read(1)
if ch == '\x1b':
# Read escape sequence - bytes arrive together from terminal
@ -164,19 +200,25 @@ def clear_dropdown(n_lines: int):
def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
"""Run inline picker."""
tools = get_tools()
if not tools:
_write("No tools. Create one: cmdforge\n")
return None
query = ""
selected = 0
scroll = 0
last_drawn = 0
registry_results = {}
registry_future = None
registry_future_query = None
registry_executor = ThreadPoolExecutor(max_workers=1)
_write(HIDE_CURSOR)
try:
while True:
if registry_future is not None and registry_future.done():
registry_results[registry_future_query] = registry_future.result()
registry_future = None
registry_future_query = None
# Filter
matches = []
for t in tools:
@ -190,12 +232,17 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
filtered = [m[0] for m in matches]
# If no local matches, search registry
registry_results = []
if query and len(query) >= 2 and not filtered:
registry_results = search_registry(query)
for rt in registry_results:
normalized_query = query.strip().lower()
found = registry_results.get(normalized_query, [])
if registry_future is None and normalized_query not in registry_results:
registry_future_query = normalized_query
registry_future = registry_executor.submit(
search_registry, query
)
for rt in found:
rt["_registry"] = True
filtered = registry_results
filtered = found
if selected >= len(filtered):
selected = max(0, len(filtered) - 1)
@ -232,6 +279,12 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
# Add registry marker
if is_registry:
prefix += f" {YELLOW}[registry]{RESET}"
if t.get("quality_score") is not None:
prefix += f" Q{t['quality_score']}"
if t.get("quality_coverage") is not None:
prefix += f"/{t['quality_coverage']}%"
if t.get("relevance") is not None:
prefix += f" R{float(t['relevance']):.2f}"
# Add deprecation marker
if is_deprecated:
prefix += f" {YELLOW}[deprecated]{RESET}"
@ -251,7 +304,11 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
last_drawn = len(lines)
# Input
ch = tty_input.getch()
ch = tty_input.getch(
timeout=0.1 if registry_future is not None else None
)
if ch is None:
continue
if ch in ('\r', '\n'): # Enter - run
if filtered:
@ -261,29 +318,32 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
# If it's a registry tool, install it first
if selected_tool.get("_registry") or selected_tool.get("registry"):
_write(f"{YELLOW}Installing {selected_tool['name']}...{RESET}\n")
import subprocess
proc = subprocess.run(
["cmdforge", "registry", "install", selected_tool["name"]],
capture_output=True, text=True
)
if proc.returncode != 0:
_write(f"Install failed: {proc.stderr}\n")
try:
return _install_registry_selection(selected_tool)
except Exception as exc:
_write(f"Install failed: {exc}\n")
return None
# Extract tool name from "owner/name" format
tool_name = selected_tool["name"].split("/")[-1]
return PickerResult(tool_name, {})
return PickerResult(selected_tool["name"], {})
return _local_selection(selected_tool)
elif ch == '\t': # Tab - configure args or run
if filtered:
tool = filtered[selected]
clear_dropdown(last_drawn)
if tool.get("_registry") or tool.get("registry"):
_write(f"{YELLOW}Installing {tool['name']}...{RESET}\n")
try:
_write(SHOW_CURSOR)
return _install_registry_selection(tool)
except Exception as exc:
_write(f"Install failed: {exc}\n")
_write(SHOW_CURSOR)
return None
if tool['args']:
args = pick_args(tty_input, tool)
_write(SHOW_CURSOR)
return PickerResult(tool["name"], args) if args is not None else None
return _local_selection(tool, args) if args is not None else None
_write(SHOW_CURSOR)
return PickerResult(tool["name"], {})
return _local_selection(tool)
elif ch == '\x1b' or ch == '\x03': # Esc or Ctrl+C
clear_dropdown(last_drawn)
@ -308,6 +368,8 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
except Exception:
_write(SHOW_CURSOR)
raise
finally:
registry_executor.shutdown(wait=False, cancel_futures=True)
def pick_args(tty_input: TTYInput, tool: dict) -> Optional[dict]:

55
src/cmdforge/cli/registry_commands.py
View File

@ -410,6 +410,22 @@ def _print_preflight_sections(report: dict, prefix: str = "") -> None:
)
def _print_quality_summary(quality: dict | None) -> None:
if not quality:
return
print(
f"Quality {quality.get('score', 0)} — "
f"{quality.get('evidence_coverage', 0)}% evidence coverage"
)
for category in quality.get("categories", []):
state = category.get("state", "not_tested")
value = (
f"{category.get('earned', 0)}/{category.get('available', 0)}"
if state == "checked" else state.replace("_", " ")
)
print(f" {category.get('name', 'Unknown'):24s} {value:>12s}")
def _print_registry_suggestions(suggestions: dict) -> None:
"""Render registry-specific category, similarity, and scrutiny evidence."""
category = suggestions.get("category") or {}
@ -547,6 +563,7 @@ def _cmd_registry_publish(args):
remote_report = result.get("preflight") or {}
_print_preflight_sections(remote_report, prefix="Registry ")
_print_quality_summary(result.get("quality"))
_print_registry_suggestions(result.get("suggestions") or {})
if not remote_report.get("errors"):
print("Registry preflight passed.")
@ -691,6 +708,44 @@ def _cmd_registry_publish(args):
except Exception:
pass # Continue with publish
# Stage 1 runs after dependency publication and any automatic version bump,
# so the evidence describes the exact version Stage 2 will mutate.
from ..preflight import analyze_tool
try:
preflight_tool = Tool.from_dict(data)
preflight_tool.path = config_path
local_report = analyze_tool(preflight_tool, include_contract_tests=True)
except (KeyError, TypeError, ValueError) as exc:
print(f"Local preflight error: {exc}", file=sys.stderr)
return 1
print("Preflight stage:")
_print_preflight_sections(local_report.to_dict(), prefix="Local ")
if not local_report.ok:
return 1
try:
preflight_client = get_client()
preflight_result = preflight_client.publish_tool(
config_yaml, readme=readme, defaults=defaults,
owner=getattr(args, "owner", ""), dry_run=True,
)
except RegistryError as exc:
print(f"Registry preflight failed: {exc}", file=sys.stderr)
return 1
remote_report = preflight_result.get("preflight") or {}
_print_preflight_sections(remote_report, prefix="Registry ")
_print_quality_summary(preflight_result.get("quality"))
if remote_report.get("errors"):
return 1
if sys.stdin.isatty() and not getattr(args, "force", False):
try:
if input("Publish this validated version? [y/N] ").strip().lower() != "y":
print("Cancelled.")
return 1
except (EOFError, KeyboardInterrupt):
print("\nCancelled.")
return 1
print(f"Publishing {name}@{version}...")
try:

4
src/cmdforge/cli/tool_commands.py
View File

@ -187,6 +187,10 @@ def cmd_run(args):
if not tool:
print(f"Error: Tool '{args.name}' not found.", file=sys.stderr)
return 1
from ..tool import deprecation_warning
warning = deprecation_warning(tool)
if warning:
print(warning, file=sys.stderr)
tool_args = list(args.tool_args or [])
if tool_args and tool_args[0] == '--':

11
src/cmdforge/gui/pages/registry_page.py
View File

@ -1,5 +1,6 @@
"""Registry page - browse and install tools from registry."""
import html
from PySide6.QtWidgets import (
QWidget, QVBoxLayout, QHBoxLayout, QLineEdit,
QPushButton, QTableWidget, QTableWidgetItem, QLabel,
@ -877,6 +878,16 @@ class RegistryPage(QWidget):
if tool.get("description"):
lines.append(f"<p style='color: #4a5568;'>{tool.get('description')}</p>")
if tool.get("deprecated"):
guidance = html.escape(
tool.get("deprecated_message") or "This tool is no longer maintained."
)
if tool.get("replacement"):
guidance += f" Use <strong>{html.escape(tool['replacement'])}</strong> instead."
lines.append(
"<p style='background: #fef3c7; color: #92400e; padding: 8px 10px; "
f"border-radius: 4px;'><strong>Deprecated:</strong> {guidance}</p>"
)
# Rating display
rating = tool.get("average_rating", 0) or 0

45
src/cmdforge/gui/pages/tool_builder_page.py
View File

@ -1161,18 +1161,57 @@ class ToolBuilderPage(QWidget):
"It will not be applied automatically."
)
msg.setInformativeText(
"Review the proposal below. Contract editing and approval "
"will be added by the guided-creation milestone."
"Review the proposal below. Applying it changes only this "
"in-memory draft and remains reversible until you save."
)
msg.setDetailedText(
yaml.safe_dump(proposed_schemas, sort_keys=False).rstrip()
)
apply_button = msg.addButton(
"Apply Proposal", QMessageBox.AcceptRole
)
save_button = msg.addButton(
"Save Without Applying", QMessageBox.AcceptRole
)
msg.addButton(QMessageBox.Cancel)
msg.exec()
if msg.clickedButton() is not save_button:
if msg.clickedButton() is apply_button:
if proposed_schemas.get("input_schema") is not None:
tool.input_schema = proposed_schemas["input_schema"]
if proposed_schemas.get("output_schema") is not None:
tool.output_schema = proposed_schemas["output_schema"]
elif msg.clickedButton() is not save_button:
return
if preflight.reuse_opportunities:
reuse = QMessageBox(self)
reuse.setIcon(QMessageBox.Information)
reuse.setWindowTitle("Reuse Opportunities")
reuse.setText(
"CmdForge found exact, contract-backed reuse evidence. "
"This is advisory and will not change your draft."
)
reuse.setDetailedText("\n".join(
f"- {item['detail']}\n Evidence: {item.get('evidence', 'exact match')}"
for item in preflight.reuse_opportunities
))
continue_button = reuse.addButton(
"Continue With Draft", QMessageBox.AcceptRole
)
matching = next(
(item.get("tool") for item in preflight.reuse_opportunities if item.get("tool")),
None,
)
open_button = (
reuse.addButton("Open Existing", QMessageBox.ActionRole)
if matching else None
)
reuse.addButton(QMessageBox.Cancel)
reuse.exec()
if open_button and reuse.clickedButton() is open_button:
_switch_to_existing_tool(self.main_window, matching)
return
if reuse.clickedButton() is not continue_button:
return
advisory_count = len(preflight.warnings) + len(preflight.suggestions)

11
src/cmdforge/gui/pages/tools_page.py
View File

@ -1,5 +1,6 @@
"""Tools page - main view for managing tools."""
import html
from collections import defaultdict
from pathlib import Path
from typing import Optional, Tuple, Dict, Any
@ -852,6 +853,16 @@ class ToolsPage(QWidget):
lines.append(f"<h2 style='margin: 0 0 8px 0; color: #2d3748;'>{tool.name}</h2>")
if tool.description:
lines.append(f"<p style='color: #4a5568; margin-bottom: 16px;'>{tool.description}</p>")
if tool.deprecated:
guidance = html.escape(
tool.deprecated_message or "This tool is no longer maintained."
)
if tool.replacement:
guidance += f" Use <strong>{html.escape(tool.replacement)}</strong> instead."
lines.append(
"<p style='background: #fef3c7; color: #92400e; padding: 8px 10px; "
f"border-radius: 4px;'><strong>Deprecated:</strong> {guidance}</p>"
)
# Publish state
state, registry_hash = get_tool_publish_state(qname)

136
src/cmdforge/preflight.py
View File

@ -5,6 +5,8 @@ All checks are deterministic and evidence-based.
"""
import copy
import hashlib
import json
import re
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
@ -26,9 +28,6 @@ class PreflightReport:
regression: Optional[Dict[str, Any]] = None
reuse_opportunities: List[Dict[str, Any]] = field(default_factory=list)
audit_evidence: Optional[Dict[str, Any]] = None
compatibility: List[Dict[str, Any]] = field(default_factory=list)
contract_proposal: Optional[Dict[str, Any]] = None
regression: Optional[Dict[str, Any]] = None
@property
def ok(self) -> bool:
@ -69,6 +68,7 @@ def analyze_tool(
*,
check_local_dependencies: bool = True,
include_contract_tests: bool = False,
check_reuse: Optional[bool] = None,
) -> PreflightReport:
"""Run all preflight checks on a tool.
@ -80,17 +80,27 @@ def analyze_tool(
A PreflightReport with structured findings.
"""
report = PreflightReport()
checks_run = ["config_integrity", "contracts"]
_check_config_integrity(tool, report)
_check_contracts(tool, report)
_add_contract_guidance(tool, report, include_contract_tests)
if include_contract_tests:
checks_run.append("contract_conformance")
_check_secrets(tool, report)
checks_run.append("secrets")
if check_local_dependencies:
_check_toolstep_compatibility(tool, report)
_check_dependencies(tool, report)
_check_reuse_opportunities(tool, report)
_add_audit_evidence(tool, report)
checks_run.extend(["toolstep_compatibility", "dependencies"])
if registry_client:
_check_similar_tools(tool, report, registry_client)
checks_run.append("registry_similarity")
if check_reuse is None:
check_reuse = check_local_dependencies
if check_reuse:
_check_reuse_opportunities(tool, report)
checks_run.append("reuse_opportunities")
_add_audit_evidence(tool, report, checks_run)
return report
@ -372,21 +382,14 @@ def _is_semver(version: str) -> bool:
def _check_reuse_opportunities(tool: Tool, report: PreflightReport):
"""Detect repeated step patterns that could be extracted into reusable tools."""
"""Report only exact repeated sequences with contract-boundary evidence."""
from .tool import list_tools, load_tool
if len(tool.steps) < 2:
return
# Build a signature of each step for comparison
step_signatures = []
for step in tool.steps:
sig = _step_signature(step)
if sig:
step_signatures.append(sig)
if not step_signatures:
return
step_signatures = [_step_signature(step) for step in tool.steps]
_find_internal_reuse(tool, step_signatures, report)
# Check if this step sequence appears in other local tools
our_sequence = tuple(step_signatures)
@ -398,55 +401,92 @@ def _check_reuse_opportunities(tool: Tool, report: PreflightReport):
continue
other_sigs = [_step_signature(s) for s in other.steps]
other_sigs = [s for s in other_sigs if s]
if our_sequence == tuple(other_sigs):
if (
our_sequence == tuple(other_sigs)
and _has_tool_contracts(tool)
and _has_tool_contracts(other)
):
report.reuse_opportunities.append({
"type": "duplicate_sequence",
"tool": name,
"detail": f"Step sequence identical to '{name}' — consider extracting shared logic",
"length": len(our_sequence),
"evidence": "exact normalized steps and explicit tool contracts",
"detail": f"Exact contracted step sequence also exists in '{name}'",
})
elif len(our_sequence) >= 2:
# Check for overlapping subsequences
for start in range(len(other_sigs) - 1):
for length in range(2, min(len(our_sequence), len(other_sigs) - start) + 1):
if our_sequence[:length] == tuple(other_sigs[start:start + length]):
report.reuse_opportunities.append({
"type": "shared_subsequence",
"tool": name,
"length": length,
"detail": f"Shares {length} step(s) with '{name}' — possible extraction candidate",
})
break
def _step_signature(step) -> str:
"""Build a comparable signature for a step."""
if hasattr(step, "prompt"):
return f"prompt:{step.provider}"
elif hasattr(step, "code"):
return "code"
elif hasattr(step, "tool"):
return f"tool:{step.tool}"
elif hasattr(step, "server"):
return f"mcp:{step.server}"
return ""
"""Build an exact normalized signature without incidental labels/outputs."""
data = step.to_dict()
data.pop("name", None)
data.pop("output_var", None)
encoded = json.dumps(data, sort_keys=True, separators=(",", ":"), default=str)
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _add_audit_evidence(tool: Tool, report: PreflightReport):
def _has_tool_contracts(tool: Tool) -> bool:
return tool.input_schema is not None and tool.output_schema is not None
def _find_internal_reuse(
tool: Tool, signatures: List[str], report: PreflightReport
) -> None:
"""Find non-overlapping repeated sequences with per-step output evidence."""
from .tool import PromptStep
count = len(signatures)
for length in range(count // 2, 1, -1):
for first in range(0, count - (2 * length) + 1):
candidate = signatures[first:first + length]
for second in range(first + length, count - length + 1):
if candidate != signatures[second:second + length]:
continue
steps = tool.steps[first:first + length]
if not all(
isinstance(step, PromptStep) and step.output_schema is not None
for step in steps
):
continue
report.reuse_opportunities.append({
"type": "repeated_sequence",
"length": length,
"locations": [first + 1, second + 1],
"evidence": "exact normalized prompt steps with output schemas",
"detail": (
f"Steps {first + 1}-{first + length} repeat at "
f"steps {second + 1}-{second + length}"
),
})
return
def _add_audit_evidence(
tool: Tool, report: PreflightReport, checks_run: List[str]
):
"""Attach audit evidence metadata to the report."""
from datetime import datetime, timezone
findings = {
"errors": report.errors,
"warnings": report.warnings,
"suggestions": report.suggestions,
"generated_tests": report.generated_tests,
"regression": report.regression,
"compatibility": report.compatibility,
"similar_tools": report.similar_tools,
"reuse_opportunities": report.reuse_opportunities,
}
findings_hash = hashlib.sha256(
json.dumps(findings, sort_keys=True, separators=(",", ":"), default=str).encode()
).hexdigest()
report.audit_evidence = {
"tool_version": tool.version or "",
"engine_version": "1.0",
"timestamp": datetime.now(timezone.utc).isoformat(),
"checks_run": [
"config_integrity",
"contracts",
"secrets",
"dependencies",
"toolstep_compatibility",
"reuse_opportunities",
],
"checks_run": checks_run,
"findings_hash": findings_hash,
"findings": copy.deepcopy(findings),
"outcome": "failed" if report.errors else "advisory" if report.warnings else "passed",
}

86
src/cmdforge/quality.py
View File

@ -20,9 +20,14 @@ class CategoryScore:
earned: int
available: int
state: str = "checked" # checked | not_tested | not_applicable
possible: Optional[int] = None # Category maximum when only partial evidence ran
@property
def display(self) -> str:
if self.state == "not_tested":
return "not tested"
if self.state == "not_applicable":
return "n/a"
return f"{self.earned}/{self.available}"
@property
@ -39,6 +44,7 @@ class QualityScore:
headline: int # 0-100
categories: List[CategoryScore] = field(default_factory=list)
last_evaluated: str = ""
evidence_coverage: int = 0
def to_dict(self) -> dict:
return {
@ -51,14 +57,16 @@ class QualityScore:
"earned": c.earned,
"available": c.available,
"state": c.state,
"possible": c.possible if c.possible is not None else c.available,
}
for c in self.categories
],
"last_evaluated": self.last_evaluated,
"evidence_coverage": self.evidence_coverage,
}
def __str__(self) -> str:
lines = [f"Quality {self.headline}"]
lines = [f"Quality {self.headline} — {self.evidence_coverage}% evidence coverage"]
if self.last_evaluated:
lines[0] += f" — Evaluated {self.last_evaluated}"
for c in self.categories:
@ -91,8 +99,13 @@ def compute_quality(
categories.append(_score_security(report, tool))
categories.append(_score_community(registry_data))
earned = sum(c.earned for c in categories)
available = sum(c.available for c in categories)
checked = [c for c in categories if c.state == "checked"]
earned = sum(c.earned for c in checked)
available = sum(c.available for c in checked)
possible = sum(
c.possible if c.possible is not None else c.available
for c in categories if c.state != "not_applicable"
)
headline = int((earned / available * 100)) if available > 0 else 0
return QualityScore(
@ -101,6 +114,7 @@ def compute_quality(
headline=headline,
categories=categories,
last_evaluated=datetime.now(timezone.utc).isoformat(),
evidence_coverage=int((available / possible * 100)) if possible else 0,
)
@ -112,6 +126,9 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore:
earned += 5
if tool.output_schema is not None:
earned += 5
if tool.input_schema is None and tool.output_schema is None:
return CategoryScore("Contracts", 0, available, "not_tested")
# No contract-related errors in the preflight
contract_errors = [
e for e in report.errors if "schema" in e.lower() or "contract" in e.lower()
@ -121,27 +138,26 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore:
else:
earned = max(0, earned - len(contract_errors))
state = "checked" if (tool.input_schema or tool.output_schema) else "not_tested"
return CategoryScore("Contracts", earned, available, state)
return CategoryScore("Contracts", earned, available, "checked")
def _score_tests(report: PreflightReport) -> CategoryScore:
available = 30
maximum = 30
tests = report.generated_tests or []
if not tests:
return CategoryScore("Deterministic tests", 0, available, "not_tested")
passed = sum(1 for t in tests if t.get("state") == "passed")
failed = sum(1 for t in tests if t.get("state") == "failed")
total = len(tests)
if total == 0:
return CategoryScore("Deterministic tests", 0, available, "not_tested")
return CategoryScore("Deterministic tests", 0, maximum, "not_tested")
executed = [t for t in tests if t.get("state") in ("passed", "failed")]
if not executed:
return CategoryScore("Deterministic tests", 0, maximum, "not_tested")
passed = sum(1 for t in executed if t.get("state") == "passed")
total = len(executed)
available = max(1, int(maximum * total / len(tests)))
earned = int((passed / total) * available)
state = "checked" if failed == 0 else "checked"
return CategoryScore("Deterministic tests", earned, available, state)
return CategoryScore(
"Deterministic tests", earned, available, "checked", possible=maximum
)
def _score_regression(report: PreflightReport) -> CategoryScore:
@ -162,7 +178,23 @@ def _score_regression(report: PreflightReport) -> CategoryScore:
def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
available = 20
maximum = 20
checks = set((report.audit_evidence or {}).get("checks_run", []))
if "security_scrutiny" in checks:
available = maximum
elif checks:
available = (10 if "secrets" in checks else 0) + (
10 if checks.intersection({"dependencies", "registry_dependencies"}) else 0
)
elif report.errors or report.warnings:
# Backward-compatible evidence supplied by callers without audit metadata.
available = maximum
else:
available = 0
if not available:
return CategoryScore(
"Security scrutiny", 0, maximum, "not_tested", possible=maximum
)
earned = available
# Deduct for secret patterns found
@ -172,13 +204,20 @@ def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
earned -= len(secret_warnings) * 5
# Deduct for unresolved dependencies
dep_warnings = [
w for w in report.warnings if "dependency" in w.lower() and "not installed" in w.lower()
dependency_findings = [
finding for finding in report.warnings + report.errors
if ("dependency" in finding.lower() or "dependencies" in finding.lower())
and any(
marker in finding.lower()
for marker in ("not installed", "missing", "deprecated")
)
]
earned -= len(dep_warnings) * 3
earned -= len(dependency_findings) * 3
earned = max(0, earned)
return CategoryScore("Security scrutiny", earned, available, "checked")
return CategoryScore(
"Security scrutiny", earned, available, "checked", possible=maximum
)
def _score_community(registry_data: Optional[dict]) -> CategoryScore:
@ -189,8 +228,11 @@ def _score_community(registry_data: Optional[dict]) -> CategoryScore:
earned = 0
reviews = registry_data.get("reviews", [])
if reviews:
rating_count = registry_data.get("rating_count", len(reviews))
avg_rating = registry_data.get("average_rating")
if avg_rating is None and reviews:
avg_rating = sum(r.get("rating", 0) for r in reviews) / len(reviews)
if avg_rating is not None and rating_count:
earned += int((avg_rating / 5) * 8)
downloads = registry_data.get("downloads", 0)

182
src/cmdforge/registry/app.py
View File

@ -9,7 +9,7 @@ import os
import re
import secrets
from dataclasses import dataclass
from datetime import date, datetime, timedelta
from datetime import date, datetime, timedelta, timezone
from typing import Any, Dict, Iterable, List, Optional, Tuple
from flask import Flask, Response, g, jsonify, request
@ -418,7 +418,7 @@ def create_app() -> Flask:
g.db.execute(
"UPDATE api_tokens SET last_used_at = ? WHERE id = ?",
[datetime.utcnow().isoformat(), row["id"]],
[datetime.now(timezone.utc).isoformat(), row["id"]],
)
g.current_publisher = {
"id": row["publisher_id"],
@ -870,6 +870,14 @@ def create_app() -> Flask:
order_sql = f"{sort} {order_dir}, published_at DESC, f.id DESC"
stats_join = "LEFT JOIN tool_stats ts ON ts.tool_id = f.id"
audit_join = """
LEFT JOIN tool_audits ta ON ta.id = (
SELECT latest_audit.id FROM tool_audits latest_audit
WHERE latest_audit.tool_id = f.id
ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC
LIMIT 1
)
"""
rows = query_all(
g.db,
@ -897,9 +905,13 @@ def create_app() -> Flask:
GROUP BY owner, name
)
SELECT f.*, COALESCE(ts.average_rating, 0) AS average_rating,
COALESCE(ts.rating_count, 0) AS rating_count
COALESCE(ts.rating_count, 0) AS rating_count,
json_extract(ta.quality_json, '$.score') AS quality_score,
json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage,
ta.evaluated_at AS quality_evaluated_at
FROM filtered f
{stats_join}
{audit_join}
JOIN (
SELECT a.owner, a.name, COALESCE(s.max_id, a.max_id) AS max_id
FROM latest_any a
@ -948,6 +960,12 @@ def create_app() -> Flask:
"average_rating": row["average_rating"],
"rating_count": row["rating_count"],
"score": score,
"quality_score": row["quality_score"],
"quality_coverage": row["quality_coverage"],
"quality_evaluated_at": row["quality_evaluated_at"],
"deprecated": bool(row["deprecated"]),
"deprecated_message": row["deprecated_message"],
"replacement": row["replacement"],
})
result: dict = {"data": data, "meta": paginate(page, per_page, total)}
@ -1089,6 +1107,39 @@ def create_app() -> Flask:
if fork_row:
fork_count = fork_row["cnt"]
replacement_chain = []
if row["replacement"]:
try:
replacement_chain = _get_replacement_chain(
row["owner"], row["name"], row["replacement"]
)
except ValueError:
# Preserve access to legacy rows while refusing new invalid chains.
replacement_chain = []
latest_audit = query_one(
g.db,
"""
SELECT engine_version, evidence_json, quality_json, evaluated_at
FROM tool_audits
WHERE tool_id = ?
ORDER BY evaluated_at DESC, id DESC
LIMIT 1
""",
[row["id"]],
)
audit_stale = True
if latest_audit:
try:
evaluated = datetime.fromisoformat(
latest_audit["evaluated_at"].replace("Z", "+00:00")
)
if evaluated.tzinfo is None:
evaluated = evaluated.replace(tzinfo=timezone.utc)
audit_stale = datetime.now(timezone.utc) - evaluated > timedelta(days=30)
except (AttributeError, TypeError, ValueError):
audit_stale = True
payload = {
"owner": row["owner"],
"name": row["name"],
@ -1100,7 +1151,8 @@ def create_app() -> Flask:
"published_at": row["published_at"],
"deprecated": bool(row["deprecated"]),
"deprecated_message": row["deprecated_message"],
"replacement": row["replacement"],
"replacement": row["replacement"] if replacement_chain else None,
"replacement_chain": replacement_chain,
"config": row["config_yaml"],
"readme": row["readme"],
"defaults": row.get("defaults") or "",
@ -1108,6 +1160,10 @@ def create_app() -> Flask:
"forked_from": row.get("forked_from"),
"forked_version": row.get("forked_version"),
"fork_count": fork_count,
"audit": json.loads(latest_audit["evidence_json"]) if latest_audit else None,
"quality": json.loads(latest_audit["quality_json"]) if latest_audit else None,
"audit_evaluated_at": latest_audit["evaluated_at"] if latest_audit else None,
"audit_stale": audit_stale,
}
response = jsonify({"data": payload})
response.headers["Cache-Control"] = "max-age=60"
@ -2496,8 +2552,12 @@ def create_app() -> Flask:
from ..preflight import analyze_tool as run_preflight
from ..tool import Tool
published_tool = Tool.from_dict(data)
preflight_report = run_preflight(
Tool.from_dict(data), check_local_dependencies=False
published_tool,
check_local_dependencies=False,
include_contract_tests=True,
check_reuse=False,
)
except (KeyError, TypeError, ValueError) as exc:
return error_response("INVALID_CONFIG", str(exc), 400)
@ -2538,6 +2598,44 @@ def create_app() -> Flask:
)
owner = requested_owner
from .auditing import registry_dependency_findings
dependency_findings = registry_dependency_findings(
g.db, published_tool, owner
)
missing_dependencies = dependency_findings["missing"]
preflight_report.warnings.extend(dependency_findings["deprecated"])
from ..preflight import _add_audit_evidence
audit_checks = list(
(preflight_report.audit_evidence or {}).get("checks_run", [])
)
_add_audit_evidence(
published_tool, preflight_report,
audit_checks + ["registry_dependencies"],
)
from ..quality import compute_quality
quality_report = compute_quality(
published_tool, preflight_report
).to_dict()
if missing_dependencies:
message = "Missing registry dependencies: " + ", ".join(
missing_dependencies
)
preflight_report.errors.append(message)
_add_audit_evidence(
published_tool, preflight_report,
audit_checks + ["registry_dependencies"],
)
return error_response(
"PREFLIGHT_FAILED", message, 400,
details={"preflight": preflight_report.to_dict()},
)
if data.get("deprecated") and data.get("replacement"):
try:
_get_replacement_chain(owner, name, data["replacement"])
except ValueError as exc:
return error_response("VALIDATION_ERROR", str(exc), 400)
# Compute config hash early for idempotency check
config_hash = compute_yaml_hash(config_text)
@ -2650,6 +2748,7 @@ def create_app() -> Flask:
"status": "validated",
"suggestions": suggestions,
"preflight": preflight_report.to_dict(),
"quality": quality_report,
}
})
@ -2675,7 +2774,7 @@ def create_app() -> Flask:
# Public tools need moderation approval
moderation_status = "pending"
g.db.execute(
insert_cursor = g.db.execute(
"""
INSERT INTO tools (
owner, name, version, description, category, tags, config_yaml, readme,
@ -2710,7 +2809,25 @@ def create_app() -> Flask:
moderation_status,
forked_from,
forked_version,
datetime.utcnow().isoformat(),
datetime.now(timezone.utc).isoformat(),
],
)
tool_id = insert_cursor.lastrowid
audit = preflight_report.audit_evidence or {}
g.db.execute(
"""
INSERT INTO tool_audits (
tool_id, engine_version, trigger, evidence_json, quality_json,
findings_hash, evaluated_at
) VALUES (?, ?, 'publish', ?, ?, ?, ?)
""",
[
tool_id,
audit.get("engine_version", "1.0"),
json.dumps(audit, sort_keys=True),
json.dumps(quality_report, sort_keys=True),
audit.get("findings_hash", ""),
audit.get("timestamp") or datetime.now(timezone.utc).isoformat(),
],
)
g.db.commit()
@ -2756,6 +2873,7 @@ def create_app() -> Flask:
"forked_version": forked_version,
"suggestions": suggestions,
"preflight": preflight_report.to_dict(),
"quality": quality_report,
}
})
response.status_code = 201
@ -2870,6 +2988,47 @@ def create_app() -> Flask:
return jsonify({"data": results})
def _get_replacement_chain(
original_owner: str, original_name: str, replacement: str
) -> List[str]:
"""Validate and resolve a deprecation chain without allowing cycles."""
chain: List[str] = []
visited = {(original_owner, original_name)}
current = replacement
default_owner = original_owner
for _ in range(20):
if "/" in current:
next_owner, next_name = current.split("/", 1)
else:
next_owner, next_name = default_owner, current
if not OWNER_RE.fullmatch(next_owner) or not TOOL_NAME_RE.fullmatch(next_name):
raise ValueError(
"replacement must be a valid tool name or owner/tool reference"
)
identity = (next_owner, next_name)
if identity in visited:
raise ValueError("replacement creates a deprecation cycle")
row = query_one(
g.db,
"""
SELECT deprecated, replacement FROM tools
WHERE owner = ? AND name = ?
ORDER BY id DESC LIMIT 1
""",
[next_owner, next_name],
)
if row is None:
raise ValueError(
f"replacement tool '{next_owner}/{next_name}' does not exist"
)
visited.add(identity)
chain.append(f"{next_owner}/{next_name}")
if not row["deprecated"] or not row["replacement"]:
return chain
current = row["replacement"]
default_owner = next_owner
raise ValueError("replacement chain exceeds 20 tools")
@app.route("/api/v1/tools/<owner>/<name>/deprecate", methods=["POST"])
@require_token
def deprecate_tool(owner: str, name: str) -> Response:
@ -2883,6 +3042,10 @@ def create_app() -> Flask:
if message and len(message) > 500:
return error_response("VALIDATION_ERROR", "Message too long (max 500)", 400)
try:
chain = _get_replacement_chain(owner, name, replacement) if replacement else []
except ValueError as exc:
return error_response("VALIDATION_ERROR", str(exc), 400)
# Update all versions of the tool
result = g.db.execute(
@ -2896,7 +3059,10 @@ def create_app() -> Flask:
return error_response("TOOL_NOT_FOUND", f"Tool {owner}/{name} not found", 404)
g.db.commit()
return jsonify({"data": {"status": "deprecated", "owner": owner, "name": name}})
return jsonify({"data": {
"status": "deprecated", "owner": owner, "name": name,
"replacement_chain": chain,
}})
@app.route("/api/v1/tools/<owner>/<name>/undeprecate", methods=["POST"])
@require_token

210
src/cmdforge/registry/auditing.py Normal file
View File

@ -0,0 +1,210 @@
"""Immutable publish-time and refresh audits for registry tool versions."""
from __future__ import annotations
import argparse
import json
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List
import yaml
from ..preflight import analyze_tool
from ..quality import compute_quality
from ..tool import Tool
from .db import connect_db, get_db_path, init_db
def registry_dependency_findings(conn, tool: Tool, owner: str) -> Dict[str, List[str]]:
"""Resolve registry dependencies and report missing/deprecated references."""
from ..tool import ToolStep
references = set(tool.dependencies)
references.update(
step.tool for step in tool.steps if isinstance(step, ToolStep)
)
missing: List[str] = []
deprecated: List[str] = []
for reference in sorted(references):
if reference == tool.name or reference == f"{owner}/{tool.name}":
continue
if "/" in reference:
dep_owner, dep_name = reference.split("/", 1)
candidates = [(dep_owner, dep_name)]
else:
candidates = [(owner, reference), ("official", reference)]
row = None
for dep_owner, dep_name in candidates:
row = conn.execute(
"""
SELECT deprecated, replacement FROM tools
WHERE owner = ? AND name = ? ORDER BY id DESC LIMIT 1
""",
[dep_owner, dep_name],
).fetchone()
if row is not None:
break
if row is None:
missing.append(reference)
elif row["deprecated"]:
guidance = f"{reference} is deprecated"
if row["replacement"]:
guidance += f"; use {row['replacement']}"
deprecated.append(guidance)
return {"missing": missing, "deprecated": deprecated}
def audit_tool_version(conn, tool_id: int, trigger: str = "background") -> Dict[str, Any]:
"""Evaluate one immutable tool version and append an audit record."""
row = conn.execute(
"""
SELECT tools.*, COALESCE(tool_stats.average_rating, 0) AS average_rating,
COALESCE(tool_stats.rating_count, 0) AS rating_count
FROM tools
LEFT JOIN tool_stats ON tool_stats.tool_id = tools.id
WHERE tools.id = ?
""",
[tool_id],
).fetchone()
if row is None:
raise ValueError(f"Unknown tool version id {tool_id}")
previous = conn.execute(
"""
SELECT evidence_json, quality_json FROM tool_audits
WHERE tool_id = ? ORDER BY evaluated_at DESC, id DESC LIMIT 1
""",
[tool_id],
).fetchone()
config = yaml.safe_load(row["config_yaml"])
tool = Tool.from_dict(config)
report = analyze_tool(
tool,
check_local_dependencies=False,
include_contract_tests=True,
check_reuse=False,
)
dependency_findings = registry_dependency_findings(conn, tool, row["owner"])
if dependency_findings["missing"]:
report.errors.append(
"Missing registry dependencies: "
+ ", ".join(dependency_findings["missing"])
)
report.warnings.extend(dependency_findings["deprecated"])
from ..preflight import _add_audit_evidence
checks = list((report.audit_evidence or {}).get("checks_run", []))
_add_audit_evidence(tool, report, checks + ["registry_dependencies"])
registry_data = {
"average_rating": row["average_rating"],
"rating_count": row["rating_count"],
"downloads": row["downloads"],
}
quality = compute_quality(tool, report, registry_data=registry_data).to_dict()
evidence = report.audit_evidence or {}
evaluated_at = datetime.now(timezone.utc).isoformat()
conn.execute(
"""
INSERT INTO tool_audits (
tool_id, engine_version, trigger, evidence_json, quality_json,
findings_hash, evaluated_at
) VALUES (?, ?, ?, ?, ?, ?, ?)
""",
[
tool_id,
evidence.get("engine_version", "1.0"),
trigger,
json.dumps(evidence, sort_keys=True),
json.dumps(quality, sort_keys=True),
evidence.get("findings_hash", ""),
evaluated_at,
],
)
quality_change = None
if previous:
previous_quality = json.loads(previous["quality_json"])
quality_change = quality["score"] - previous_quality.get("score", 0)
previous_evidence = json.loads(previous["evidence_json"])
if quality_change < 0 or previous_evidence.get("outcome") != evidence.get("outcome"):
conn.execute(
"""
INSERT INTO audit_log (
action, target_type, target_id, actor_id, details
) VALUES ('automated_audit_changed', 'tool', ?, 'system', ?)
""",
[
str(tool_id),
json.dumps({
"quality_change": quality_change,
"previous_outcome": previous_evidence.get("outcome"),
"current_outcome": evidence.get("outcome"),
}, sort_keys=True),
],
)
conn.commit()
return {
"tool_id": tool_id, "evidence": evidence, "quality": quality,
"quality_change": quality_change,
}
def audit_stale_tools(
conn, *, max_age_days: int = 30, limit: int = 100
) -> List[Dict[str, Any]]:
"""Append refreshed evidence for versions with missing or stale audits."""
if max_age_days < 0:
raise ValueError("max_age_days must be non-negative")
if limit < 1:
raise ValueError("limit must be positive")
cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days)
rows = conn.execute(
"""
SELECT tools.id, MAX(tool_audits.evaluated_at) AS last_evaluated
FROM tools
LEFT JOIN tool_audits ON tool_audits.tool_id = tools.id
GROUP BY tools.id
ORDER BY last_evaluated IS NOT NULL, last_evaluated
"""
).fetchall()
stale_ids = []
for row in rows:
value = row["last_evaluated"]
if value:
try:
evaluated = datetime.fromisoformat(value.replace("Z", "+00:00"))
if evaluated.tzinfo is None:
evaluated = evaluated.replace(tzinfo=timezone.utc)
except ValueError:
evaluated = datetime.min.replace(tzinfo=timezone.utc)
if evaluated >= cutoff:
continue
stale_ids.append(row["id"])
if len(stale_ids) >= limit:
break
results = []
for tool_id in stale_ids:
try:
results.append(audit_tool_version(conn, tool_id))
except Exception as exc:
results.append({"tool_id": tool_id, "error": str(exc)})
return results
def main() -> int:
parser = argparse.ArgumentParser(description="Refresh stale CmdForge registry audits")
parser.add_argument("--max-age-days", type=int, default=30)
parser.add_argument("--limit", type=int, default=100)
args = parser.parse_args()
conn = connect_db(get_db_path())
try:
init_db(conn)
results = audit_stale_tools(
conn, max_age_days=args.max_age_days, limit=args.limit
)
finally:
conn.close()
print(f"Audited {len(results)} tool version(s)")
return 0
if __name__ == "__main__":
raise SystemExit(main())

14
src/cmdforge/registry/db.py
View File

@ -80,6 +80,20 @@ CREATE TABLE IF NOT EXISTS download_stats (
UNIQUE(tool_id, client_id, downloaded_at)
);
CREATE TABLE IF NOT EXISTS tool_audits (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tool_id INTEGER NOT NULL REFERENCES tools(id) ON DELETE CASCADE,
engine_version TEXT NOT NULL,
trigger TEXT NOT NULL,
evidence_json TEXT NOT NULL,
quality_json TEXT NOT NULL,
findings_hash TEXT NOT NULL,
evaluated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_tool_audits_tool_time
ON tool_audits(tool_id, evaluated_at DESC, id DESC);
CREATE VIRTUAL TABLE IF NOT EXISTS tools_fts USING fts5(
name, description, tags, readme,
content='tools',

13
src/cmdforge/registry/embeddings.py
View File

@ -157,10 +157,19 @@ def semantic_search(
"""
SELECT te.tool_id, te.embedding, te.dimensions, te.model,
t.owner, t.name, t.version, t.description, t.category,
t.tags, t.downloads
t.tags, t.downloads,
json_extract(ta.quality_json, '$.score') AS quality_score,
json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage
FROM tool_embeddings te
JOIN tools t ON t.id = te.tool_id
LEFT JOIN tool_audits ta ON ta.id = (
SELECT latest_audit.id FROM tool_audits latest_audit
WHERE latest_audit.tool_id = t.id
ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC
LIMIT 1
)
WHERE t.visibility = 'public' AND t.moderation_status = 'approved'
AND t.deprecated = 0
""",
).fetchall()
@ -192,6 +201,8 @@ def semantic_search(
"tags": tags_list,
"downloads": row["downloads"] or 0,
"similarity": round(score, 4),
"quality_score": row["quality_score"],
"quality_coverage": row["quality_coverage"],
})
# Sort by similarity descending

5
src/cmdforge/runner.py
View File

@ -1173,6 +1173,11 @@ def main():
print(f"Searched: {', '.join(e.searched_paths[:3])}", file=sys.stderr)
sys.exit(1)
from .tool import deprecation_warning
warning = deprecation_warning(tool)
if warning:
print(warning, file=sys.stderr)
# Check for manifest overrides
manifest = load_manifest()
provider_override_from_manifest = None

31
src/cmdforge/tool.py
View File

@ -1,6 +1,7 @@
"""Tool loading, saving, and management."""
import os
import re
import shutil
import stat
from dataclasses import dataclass, field
@ -15,6 +16,9 @@ TOOLS_DIR = Path.home() / ".cmdforge"
# Default bin directory for wrapper scripts
BIN_DIR = Path.home() / ".local" / "bin"
_TOOL_REF_RE = re.compile(
r"^(?:[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?/)?[A-Za-z0-9_-]{1,64}$"
)
def _validate_skill_selection(skills: Optional[List[str]]) -> None:
@ -447,16 +451,27 @@ class Tool:
source: Optional[ToolSource] = None # Attribution for imported/external tools
version: str = "" # Tool version
visibility: str = "public" # "public", "private", or "unlisted"
deprecated: bool = False # Tool is deprecated
deprecated_message: str = "" # Migration guidance for deprecated tools
replacement: Optional[str] = None # Suggested replacement tool name
input_schema: Optional[dict] = None # JSON Schema for tool input contract
output_schema: Optional[dict] = None # JSON Schema for tool output contract
path: Optional[Path] = None # Path to config.yaml (set by load_tool)
deprecated: bool = False # Tool is deprecated
deprecated_message: str = "" # Migration guidance for deprecated tools
replacement: Optional[str] = None # Suggested replacement tool name
def __post_init__(self) -> None:
validate_json_schema(self.input_schema, "input_schema")
validate_json_schema(self.output_schema, "output_schema")
if not isinstance(self.deprecated, bool):
raise ValueError("deprecated must be true or false")
if not isinstance(self.deprecated_message, str):
raise ValueError("deprecated_message must be a string")
if len(self.deprecated_message) > 500:
raise ValueError("deprecated_message must be at most 500 characters")
if self.replacement is not None and (
not isinstance(self.replacement, str)
or not _TOOL_REF_RE.fullmatch(self.replacement)
):
raise ValueError("replacement must be a tool name or owner/tool reference")
@classmethod
def from_dict(cls, data: dict) -> "Tool":
@ -809,6 +824,16 @@ def tool_exists(name: str) -> bool:
return (get_tools_dir() / name / "config.yaml").exists()
def deprecation_warning(tool: Tool) -> Optional[str]:
"""Return consistent migration guidance for a deprecated tool."""
if not tool.deprecated:
return None
message = tool.deprecated_message or "This tool is no longer maintained."
if tool.replacement:
message += f" Use '{tool.replacement}' instead."
return f"Warning: '{tool.name}' is deprecated. {message}"
def validate_tool_name(name: str) -> tuple[bool, str]:
"""
Validate a tool name.

31
tests/test_cli.py
View File

@ -541,6 +541,37 @@ class TestRegistryPublishDryRun:
assert _cmd_registry_publish(self.args(tool_dir)) == 1
assert "YAML mapping" in capsys.readouterr().err
def test_normal_publish_runs_remote_preflight_before_mutation(
self, tool_dir, monkeypatch
):
from cmdforge.cli.registry_commands import _cmd_registry_publish
client = MagicMock()
client.get_me.return_value = {"slug": "testuser"}
client.get_my_tool_status.return_value = {"status": "pending"}
client.publish_tool.side_effect = [
{
"preflight": {"errors": [], "warnings": [], "suggestions": []},
"quality": {"score": 80, "evidence_coverage": 60, "categories": []},
},
{
"owner": "testuser", "name": "dry-run-tool",
"version": "1.0.0", "status": "pending",
},
]
monkeypatch.setattr(
"cmdforge.cli.registry_commands.load_config",
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
)
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
args = SimpleNamespace(
path=str(tool_dir), dry_run=False, force=True, owner=""
)
assert _cmd_registry_publish(args) == 0
assert client.publish_tool.call_count == 2
assert client.publish_tool.call_args_list[0].kwargs["dry_run"] is True
assert "dry_run" not in client.publish_tool.call_args_list[1].kwargs
def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
from cmdforge.cli import cmd_inspect

120
tests/test_collection_api.py
View File

@ -56,7 +56,7 @@ def client(app):
def auth_headers(app):
"""Create auth headers with a valid token."""
import hashlib
from datetime import datetime
from datetime import datetime, timezone
from cmdforge.registry.db import connect_db
token = "test-token"
@ -94,7 +94,7 @@ def auth_headers(app):
INSERT INTO api_tokens (publisher_id, token_hash, name, created_at)
VALUES (?, ?, ?, ?)
""",
[publisher_id, token_hash, "test-token", datetime.utcnow().isoformat()],
[publisher_id, token_hash, "test-token", datetime.now(timezone.utc).isoformat()],
)
# Insert a public approved tool for tests if missing
@ -112,7 +112,7 @@ def auth_headers(app):
[
"testuser", "tool1", "1.0.0", "Test tool", "Other", "[]",
"name: tool1\nversion: 1.0.0\n", "", publisher_id,
"public", "approved", datetime.utcnow().isoformat(),
"public", "approved", datetime.now(timezone.utc).isoformat(),
],
)
@ -158,6 +158,44 @@ class TestToolApprovedEndpoint:
assert data['data']['has_approved_public_version'] is False
@flask_required
class TestDeprecationChains:
def test_replacement_chain_is_validated_and_returned(self, client, auth_headers):
target = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": "name: replacement\nversion: 1.0.0\noutput: ok\n"},
)
assert target.status_code == 201
response = client.post(
"/api/v1/tools/testuser/tool1/deprecate",
headers=auth_headers,
json={"deprecated_message": "Moved", "replacement": "replacement"},
)
assert response.status_code == 200
assert response.get_json()["data"]["replacement_chain"] == [
"testuser/replacement"
]
def test_replacement_cycle_is_rejected(self, client, auth_headers):
target = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": "name: cycle-target\nversion: 1.0.0\noutput: ok\n"},
)
assert target.status_code == 201
assert client.post(
"/api/v1/tools/testuser/cycle-target/deprecate",
headers=auth_headers,
json={"replacement": "tool1"},
).status_code == 200
response = client.post(
"/api/v1/tools/testuser/tool1/deprecate",
headers=auth_headers,
json={"replacement": "cycle-target"},
)
assert response.status_code == 400
assert "cycle" in response.get_json()["error"]["message"]
@flask_required
class TestPublishPreflightEndpoint:
def test_dry_run_includes_shared_preflight(self, client, auth_headers):
@ -180,7 +218,81 @@ class TestPublishPreflightEndpoint:
assert response.status_code == 200
report = response.get_json()["data"]["preflight"]
assert report["errors"] == []
assert report["generated_tests"] == []
assert report["generated_tests"][0]["state"] == "passed"
assert response.get_json()["data"]["quality"]["evidence_coverage"] > 0
def test_real_publish_persists_immutable_audit(self, client, auth_headers):
from cmdforge.registry.db import connect_db
response = client.post(
"/api/v1/tools",
headers=auth_headers,
json={
"config": (
"name: audited-tool\nversion: 1.0.0\noutput: stable\n"
"input_schema: {}\noutput_schema:\n type: string\n"
),
},
)
assert response.status_code == 201
conn = connect_db()
try:
rows = conn.execute(
"""
SELECT tool_audits.trigger, tool_audits.evidence_json
FROM tool_audits JOIN tools ON tools.id = tool_audits.tool_id
WHERE tools.owner = 'testuser' AND tools.name = 'audited-tool'
"""
).fetchall()
conn.execute(
"""
UPDATE tools SET moderation_status = 'approved'
WHERE owner = 'testuser' AND name = 'audited-tool'
"""
)
conn.commit()
finally:
conn.close()
assert len(rows) == 1
assert rows[0]["trigger"] == "publish"
assert "contract_conformance" in rows[0]["evidence_json"]
search = client.get("/api/v1/tools/search?q=audited")
result = search.get_json()["data"][0]
assert isinstance(result["quality_score"], int)
assert isinstance(result["quality_coverage"], int)
assert result["quality_evaluated_at"]
def test_real_publish_blocks_deterministic_contract_failure(
self, client, auth_headers
):
response = client.post(
"/api/v1/tools",
headers=auth_headers,
json={
"config": (
"name: broken-contract\nversion: 1.0.0\noutput: text\n"
"input_schema: {}\noutput_schema:\n type: integer\n"
),
},
)
assert response.status_code == 400
assert response.get_json()["error"]["code"] == "PREFLIGHT_FAILED"
def test_real_publish_blocks_missing_registry_dependency(
self, client, auth_headers
):
response = client.post(
"/api/v1/tools",
headers=auth_headers,
json={
"config": (
"name: missing-dependency\nversion: 1.0.0\n"
"dependencies:\n - absent-tool\n"
),
},
)
assert response.status_code == 400
assert "absent-tool" in response.get_json()["error"]["message"]
def test_invalid_contract_is_rejected(self, client, auth_headers):
response = client.post(

91
tests/test_picker_m8.py Normal file
View File

@ -0,0 +1,91 @@
"""Behavioral coverage for registry-aware picker and deprecation UX."""
from io import StringIO
import time
from unittest.mock import patch
import cmdforge.cli.picker as picker
from cmdforge.cli.picker import PickerResult
class FakeTTY:
def __init__(self, *characters):
self.characters = iter(characters)
def getch(self, timeout=None):
if timeout is not None:
time.sleep(0.01)
return None
return next(self.characters)
def setup_function():
picker._registry_cache.clear()
picker._ui_out = StringIO()
def test_public_registry_search_does_not_require_token():
class Client:
token = ""
def semantic_search(self, query, limit):
return {"available": False, "data": [], "error": None}
def search_tools(self, query, per_page):
return [{
"owner": "official", "name": "weather",
"description": "Weather", "score": 0.8,
"quality_score": 91, "quality_coverage": 70,
}]
with patch("cmdforge.registry_client.get_client", return_value=Client()):
results = picker.search_registry("weather")
assert results[0]["name"] == "official/weather"
assert results[0]["quality_score"] == 91
assert results[0]["quality_coverage"] == 70
def test_picker_can_install_when_no_local_tools():
registry_tool = {
"name": "official/weather", "desc": "Weather", "args": [],
"registry": True, "quality_score": 91, "quality_coverage": 70,
"relevance": 0.8,
}
with patch.object(picker, "get_tools", return_value=[]), patch.object(
picker, "search_registry", return_value=[registry_tool]
), patch.object(
picker, "_install_registry_selection",
return_value=PickerResult("official/weather", {}),
) as install:
result = picker.run_picker(FakeTTY("w", "e", "\n"))
install.assert_called_once()
assert result.tool_name == "official/weather"
assert "Q91/70%" in picker._ui_out.getvalue()
def test_tab_installs_registry_selection_too():
registry_tool = {
"name": "official/weather", "desc": "Weather", "args": [],
"registry": True,
}
with patch.object(picker, "get_tools", return_value=[]), patch.object(
picker, "search_registry", return_value=[registry_tool]
), patch.object(
picker, "_install_registry_selection",
return_value=PickerResult("official/weather", {}),
) as install:
result = picker.run_picker(FakeTTY("w", "e", "\t"))
install.assert_called_once()
assert result.tool_name == "official/weather"
def test_local_deprecation_selection_prints_migration_guidance():
tool = {
"name": "old", "desc": "", "args": [], "deprecated": True,
"deprecated_message": "Moved.", "replacement": "official/new",
}
result = picker._local_selection(tool)
assert result.tool_name == "old"
output = picker._ui_out.getvalue()
assert "Moved." in output
assert "official/new" in output

65
tests/test_preflight.py
View File

@ -126,6 +126,71 @@ class TestAnalyzeTool:
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
assert report.similar_tools[0]["name"] == "official/summary"
def test_server_audit_does_not_claim_skipped_local_checks(self):
from cmdforge.tool import Tool, ToolStep
report = analyze_tool(
Tool(name="server", steps=[ToolStep(tool="child", output_var="out")]),
check_local_dependencies=False,
)
checks = report.audit_evidence["checks_run"]
assert "dependencies" not in checks
assert "toolstep_compatibility" not in checks
assert "reuse_opportunities" not in checks
assert report.compatibility == []
def test_unrelated_code_steps_are_not_reuse_evidence(self):
from cmdforge.tool import CodeStep, Tool
current = Tool(
name="current", input_schema={}, output_schema={},
steps=[
CodeStep(code="x = 1", output_var="x"),
CodeStep(code="y = 2", output_var="y"),
],
)
other = Tool(
name="other", input_schema={}, output_schema={},
steps=[
CodeStep(code="a = 99", output_var="a"),
CodeStep(code="b = 100", output_var="b"),
],
)
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
"cmdforge.tool.load_tool", return_value=other
):
report = analyze_tool(current)
assert report.reuse_opportunities == []
def test_exact_contracted_duplicate_has_evidence(self):
from cmdforge.tool import PromptStep, Tool
def make(name):
return Tool(
name=name, input_schema={}, output_schema={"type": "string"},
steps=[
PromptStep(
prompt="first", provider="mock", output_var="one",
output_schema={"type": "string"},
),
PromptStep(
prompt="second", provider="mock", output_var="two",
output_schema={"type": "string"},
),
],
)
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
"cmdforge.tool.load_tool", return_value=make("other")
):
report = analyze_tool(make("current"))
duplicate = [
item for item in report.reuse_opportunities
if item["type"] == "duplicate_sequence"
]
assert duplicate
assert "explicit tool contracts" in duplicate[0]["evidence"]
def test_registry_error_becomes_warning(self):
class Client:
def search_tools(self, query, per_page):

49
tests/test_quality.py
View File

@ -22,6 +22,10 @@ class TestCategoryScore:
cs = CategoryScore("X", 0, 0)
assert cs.percentage == 0.0
def test_display_distinguishes_missing_evidence(self):
assert CategoryScore("Tests", 0, 30, "not_tested").display == "not tested"
assert CategoryScore("Tests", 0, 30, "not_applicable").display == "n/a"
class TestQualityScore:
def test_to_dict(self):
@ -55,6 +59,40 @@ class TestComputeQuality:
# Contracts should be not_tested
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
assert contracts.state == "not_tested"
assert qs.headline == 0
assert qs.evidence_coverage == 0
def test_missing_categories_do_not_reduce_checked_score(self):
tool = Tool(name="contracted", input_schema={}, output_schema={})
report = PreflightReport()
qs = compute_quality(tool, report)
assert qs.headline == 100
assert qs.evidence_coverage == 15
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
assert contracts.state == "checked"
def test_unsupported_tests_are_not_scored_as_failures(self):
report = PreflightReport(generated_tests=[{
"step": "preflight", "state": "unsupported", "detail": "safe mode"
}])
tests = [
c for c in compute_quality(Tool(name="tool"), report).categories
if c.name == "Deterministic tests"
][0]
assert tests.state == "not_tested"
assert tests.display == "not tested"
def test_partial_unsupported_tests_reduce_coverage_not_score(self):
report = PreflightReport(generated_tests=[
{"step": "case", "state": "passed", "detail": "ok"},
{"step": "generation", "state": "unsupported", "detail": "unknown"},
])
tests = [
c for c in compute_quality(Tool(name="tool"), report).categories
if c.name == "Deterministic tests"
][0]
assert tests.earned == tests.available == 15
assert tests.possible == 30
def test_with_contracts_scores_higher(self):
tool = Tool(
@ -127,6 +165,17 @@ class TestComputeQuality:
security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
assert security.earned == 15 # 20 - 5
def test_skipped_dependency_check_is_not_claimed_as_security_evidence(self):
tool = Tool(name="server")
report = PreflightReport(audit_evidence={"checks_run": ["secrets"]})
security = [
c for c in compute_quality(tool, report).categories
if c.name == "Security scrutiny"
][0]
assert security.earned == 10
assert security.available == 10
assert security.possible == 20
def test_community_not_tested_without_data(self):
tool = Tool(name="new")
report = PreflightReport()

76
tests/test_registry_auditing.py Normal file
View File

@ -0,0 +1,76 @@
"""Tests for immutable registry audit refreshes."""
from datetime import datetime, timedelta, timezone
import pytest
from cmdforge.registry.auditing import audit_stale_tools, audit_tool_version
from cmdforge.registry.db import connect_db, init_db
@pytest.fixture
def audit_db(tmp_path):
conn = connect_db(tmp_path / "registry.db")
init_db(conn)
cursor = conn.execute(
"""
INSERT INTO publishers (email, password_hash, slug, display_name)
VALUES ('audit@example.com', 'x', 'auditor', 'Auditor')
"""
)
publisher_id = cursor.lastrowid
tool = conn.execute(
"""
INSERT INTO tools (
owner, name, version, config_yaml, publisher_id,
visibility, moderation_status
) VALUES ('auditor', 'checked', '1.0.0', ?, ?, 'public', 'approved')
""",
[
"name: checked\nversion: 1.0.0\noutput: stable\n"
"input_schema: {}\noutput_schema:\n type: string\n",
publisher_id,
],
)
conn.commit()
yield conn, tool.lastrowid
conn.close()
def test_background_audits_append_instead_of_overwriting(audit_db):
conn, tool_id = audit_db
first = audit_tool_version(conn, tool_id)
second = audit_tool_version(conn, tool_id)
count = conn.execute(
"SELECT COUNT(*) FROM tool_audits WHERE tool_id = ?", [tool_id]
).fetchone()[0]
assert count == 2
assert first["evidence"]["findings_hash"] == second["evidence"]["findings_hash"]
assert "generated_tests" in first["evidence"]["findings"]
assert "dependencies" not in first["evidence"]["checks_run"]
def test_stale_audit_refresh_adds_new_evidence(audit_db):
conn, tool_id = audit_db
audit_tool_version(conn, tool_id)
old = (datetime.now(timezone.utc) - timedelta(days=60)).isoformat()
conn.execute("UPDATE tool_audits SET evaluated_at = ?", [old])
conn.commit()
results = audit_stale_tools(conn, max_age_days=30)
assert [item["tool_id"] for item in results] == [tool_id]
assert conn.execute("SELECT COUNT(*) FROM tool_audits").fetchone()[0] == 2
def test_degraded_background_score_is_logged(audit_db):
conn, tool_id = audit_db
conn.execute("UPDATE tools SET downloads = 500 WHERE id = ?", [tool_id])
conn.commit()
audit_tool_version(conn, tool_id)
conn.execute("UPDATE tools SET downloads = 0 WHERE id = ?", [tool_id])
conn.commit()
result = audit_tool_version(conn, tool_id)
assert result["quality_change"] < 0
log = conn.execute(
"SELECT action FROM audit_log WHERE target_id = ?", [str(tool_id)]
).fetchone()
assert log["action"] == "automated_audit_changed"

27
tests/test_tool.py
View File

@ -685,3 +685,30 @@ class TestAgentContext:
)
restored = PromptStep.from_dict(step.to_dict())
assert restored.skills == ["python"]
class TestDeprecationFields:
def test_round_trip_and_warning(self):
from cmdforge.tool import deprecation_warning
tool = Tool(
name="old", deprecated=True, deprecated_message="Moved.",
replacement="official/new",
)
restored = Tool.from_dict(tool.to_dict())
assert restored.deprecated is True
assert restored.replacement == "official/new"
assert "official/new" in deprecation_warning(restored)
@pytest.mark.parametrize(
"values",
[
{"deprecated": "false"},
{"deprecated_message": 123},
{"replacement": "../../escape"},
{"replacement": "owner/name/extra"},
],
)
def test_invalid_deprecation_metadata_is_rejected(self, values):
with pytest.raises(ValueError):
Tool(name="old", **values)