diff --git a/CLAUDE.md b/CLAUDE.md index c20e804..cf3ec44 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,10 +79,16 @@ Tools are YAML configs with: - `output`: Template for final output (e.g., `"{response}"`) - `input_schema`, `output_schema`: Optional tool-level JSON Schema contracts; schemas are validated when tools are loaded or created +- `deprecated`, `deprecated_message`, `replacement`: Validated migration metadata; + replacements may be local names or `owner/tool` registry references Run `cmdforge inspect [--registry]` for the shared local preflight report. `cmdforge registry publish --dry-run` runs local checks first and, when authenticated, the registry's publish-time checks without publishing. +Normal publication uses the same checks as Stage 1, asks for confirmation in an +interactive terminal, and then performs the Stage 2 mutation. The registry +reruns deterministic conformance authoritatively and stores immutable audit and +quality evidence for the published version. Inspect also shows inferred contract proposals and runs deterministic contract conformance when both tool schemas are explicit. Prompt outputs are synthesized from their schemas; code, nested-tool, and MCP steps are reported as unsupported @@ -92,6 +98,8 @@ evidence to `conformance.json`. Later inspections and local publish dry-runs rerun the stored synthetic inputs and report state, coverage, contract, and normalized output changes. ToolStep preflight also compares the values the runner supplies with the called tool's input contract. +Quality headlines exclude categories with missing evidence and are accompanied +by evidence coverage plus checked/not-tested category states. ### Step Types diff --git a/README.md b/README.md index b75c775..5c1fcf0 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ cmdforge docs mytool # View/create tool documentation # Registry cmdforge registry search "keyword" # Search for tools cmdforge registry install owner/tool # Install a tool -cmdforge registry publish mytool # Publish your tool +cmdforge registry publish mytool # Preflight, confirm, then publish cmdforge registry publish mytool --dry-run # Preflight without publishing cmdforge registry status # Check moderation status cmdforge registry my-tools # List your published tools @@ -180,6 +180,9 @@ cmdforge config connect username # Connect to registry account cmdforge config disconnect # Disconnect from registry ``` +`cf` searches the public registry when no local tool matches. Registry results +show available relevance and quality evidence and install on selection. + ### Running Tools Once created, tools work like any Unix command: diff --git a/src/cmdforge/cli/__init__.py b/src/cmdforge/cli/__init__.py index a92f8a6..74864a0 100644 --- a/src/cmdforge/cli/__init__.py +++ b/src/cmdforge/cli/__init__.py @@ -648,7 +648,10 @@ def cmd_inspect(args): # Quality score from ..quality import compute_quality qs = compute_quality(tool, report) - print(f"Quality {qs.headline} — Evaluated {qs.last_evaluated[:10]}") + print( + f"Quality {qs.headline} — {qs.evidence_coverage}% evidence coverage — " + f"Evaluated {qs.last_evaluated[:10]}" + ) for cat in qs.categories: print(f" {cat.name:24s} {cat.display:>8s}") print() diff --git a/src/cmdforge/cli/picker.py b/src/cmdforge/cli/picker.py index 7651b92..4e7b54c 100644 --- a/src/cmdforge/cli/picker.py +++ b/src/cmdforge/cli/picker.py @@ -2,8 +2,10 @@ import sys import os +import select import tty import termios +from concurrent.futures import ThreadPoolExecutor from typing import List, Tuple, Optional from dataclasses import dataclass @@ -30,6 +32,7 @@ GREEN = "\033[32m" RESET = "\033[0m" MAX_VISIBLE = 8 # Show at most 8 items +_registry_cache = {} # Output stream for UI (stderr when stdout is piped, stdout otherwise) _ui_out = None @@ -90,31 +93,60 @@ def search_registry(query: str) -> List[dict]: """Search the registry for tools matching the query. Returns a list of dicts with name, desc, and registry metadata. - Returns empty list if no token configured or search fails. + Registry search is public; authentication is not required. """ if not query or len(query) < 2: return [] + normalized = query.strip().lower() + if normalized in _registry_cache: + return [dict(item) for item in _registry_cache[normalized]] try: - from ..registry_client import get_client + from ..registry_client import RegistryError, get_client client = get_client() - if not client or not client.token: - return [] - results = client.search_tools(query, per_page=5) - items = results.data if hasattr(results, "data") else results - return [ + client.timeout = min(getattr(client, "timeout", 3), 3) + client.max_retries = 1 + semantic = client.semantic_search(query, limit=5) + if semantic.get("available") and semantic.get("data"): + items = semantic["data"] + else: + results = client.search_tools(query, per_page=5) + items = results.data if hasattr(results, "data") else results + found = [ { "name": f"{item.get('owner', '')}/{item.get('name', '')}", "desc": (item.get("description") or "")[:50], "args": [], "registry": True, "downloads": item.get("downloads", 0), + "relevance": item.get("similarity", item.get("score")), + "quality_score": item.get("quality_score"), + "quality_coverage": item.get("quality_coverage"), } for item in items ] - except Exception: + _registry_cache[normalized] = found + return [dict(item) for item in found] + except RegistryError: return [] +def _install_registry_selection(tool: dict) -> PickerResult: + """Install a registry result and retain its qualified identity for execution.""" + from ..resolver import install_from_registry + + install_from_registry(tool["name"]) + return PickerResult(tool["name"], {}) + + +def _local_selection(tool: dict, arguments: Optional[dict] = None) -> PickerResult: + if tool.get("deprecated"): + guidance = tool.get("deprecated_message") or "This tool is no longer maintained." + if tool.get("replacement"): + guidance += f" Use '{tool['replacement']}' instead." + _write(f"{YELLOW}Warning: {guidance}{RESET}\n") + return PickerResult(tool["name"], arguments or {}) + + class TTYInput: """Read from /dev/tty for keyboard input, even when stdin is piped.""" @@ -135,10 +167,14 @@ class TTYInput: if self.tty: self.tty.close() - def getch(self): + def getch(self, timeout: Optional[float] = None): """Read a single character.""" tty.setraw(self.fd) try: + if timeout is not None: + ready, _, _ = select.select([self.tty], [], [], timeout) + if not ready: + return None ch = self.tty.read(1) if ch == '\x1b': # Read escape sequence - bytes arrive together from terminal @@ -164,19 +200,25 @@ def clear_dropdown(n_lines: int): def run_picker(tty_input: TTYInput) -> Optional[PickerResult]: """Run inline picker.""" tools = get_tools() - if not tools: - _write("No tools. Create one: cmdforge\n") - return None query = "" selected = 0 scroll = 0 last_drawn = 0 + registry_results = {} + registry_future = None + registry_future_query = None + registry_executor = ThreadPoolExecutor(max_workers=1) _write(HIDE_CURSOR) try: while True: + if registry_future is not None and registry_future.done(): + registry_results[registry_future_query] = registry_future.result() + registry_future = None + registry_future_query = None + # Filter matches = [] for t in tools: @@ -190,12 +232,17 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]: filtered = [m[0] for m in matches] # If no local matches, search registry - registry_results = [] if query and len(query) >= 2 and not filtered: - registry_results = search_registry(query) - for rt in registry_results: + normalized_query = query.strip().lower() + found = registry_results.get(normalized_query, []) + if registry_future is None and normalized_query not in registry_results: + registry_future_query = normalized_query + registry_future = registry_executor.submit( + search_registry, query + ) + for rt in found: rt["_registry"] = True - filtered = registry_results + filtered = found if selected >= len(filtered): selected = max(0, len(filtered) - 1) @@ -232,6 +279,12 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]: # Add registry marker if is_registry: prefix += f" {YELLOW}[registry]{RESET}" + if t.get("quality_score") is not None: + prefix += f" Q{t['quality_score']}" + if t.get("quality_coverage") is not None: + prefix += f"/{t['quality_coverage']}%" + if t.get("relevance") is not None: + prefix += f" R{float(t['relevance']):.2f}" # Add deprecation marker if is_deprecated: prefix += f" {YELLOW}[deprecated]{RESET}" @@ -251,7 +304,11 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]: last_drawn = len(lines) # Input - ch = tty_input.getch() + ch = tty_input.getch( + timeout=0.1 if registry_future is not None else None + ) + if ch is None: + continue if ch in ('\r', '\n'): # Enter - run if filtered: @@ -261,29 +318,32 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]: # If it's a registry tool, install it first if selected_tool.get("_registry") or selected_tool.get("registry"): _write(f"{YELLOW}Installing {selected_tool['name']}...{RESET}\n") - import subprocess - proc = subprocess.run( - ["cmdforge", "registry", "install", selected_tool["name"]], - capture_output=True, text=True - ) - if proc.returncode != 0: - _write(f"Install failed: {proc.stderr}\n") + try: + return _install_registry_selection(selected_tool) + except Exception as exc: + _write(f"Install failed: {exc}\n") return None - # Extract tool name from "owner/name" format - tool_name = selected_tool["name"].split("/")[-1] - return PickerResult(tool_name, {}) - return PickerResult(selected_tool["name"], {}) + return _local_selection(selected_tool) elif ch == '\t': # Tab - configure args or run if filtered: tool = filtered[selected] clear_dropdown(last_drawn) + if tool.get("_registry") or tool.get("registry"): + _write(f"{YELLOW}Installing {tool['name']}...{RESET}\n") + try: + _write(SHOW_CURSOR) + return _install_registry_selection(tool) + except Exception as exc: + _write(f"Install failed: {exc}\n") + _write(SHOW_CURSOR) + return None if tool['args']: args = pick_args(tty_input, tool) _write(SHOW_CURSOR) - return PickerResult(tool["name"], args) if args is not None else None + return _local_selection(tool, args) if args is not None else None _write(SHOW_CURSOR) - return PickerResult(tool["name"], {}) + return _local_selection(tool) elif ch == '\x1b' or ch == '\x03': # Esc or Ctrl+C clear_dropdown(last_drawn) @@ -308,6 +368,8 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]: except Exception: _write(SHOW_CURSOR) raise + finally: + registry_executor.shutdown(wait=False, cancel_futures=True) def pick_args(tty_input: TTYInput, tool: dict) -> Optional[dict]: diff --git a/src/cmdforge/cli/registry_commands.py b/src/cmdforge/cli/registry_commands.py index 4c3d255..b329ab4 100644 --- a/src/cmdforge/cli/registry_commands.py +++ b/src/cmdforge/cli/registry_commands.py @@ -410,6 +410,22 @@ def _print_preflight_sections(report: dict, prefix: str = "") -> None: ) +def _print_quality_summary(quality: dict | None) -> None: + if not quality: + return + print( + f"Quality {quality.get('score', 0)} — " + f"{quality.get('evidence_coverage', 0)}% evidence coverage" + ) + for category in quality.get("categories", []): + state = category.get("state", "not_tested") + value = ( + f"{category.get('earned', 0)}/{category.get('available', 0)}" + if state == "checked" else state.replace("_", " ") + ) + print(f" {category.get('name', 'Unknown'):24s} {value:>12s}") + + def _print_registry_suggestions(suggestions: dict) -> None: """Render registry-specific category, similarity, and scrutiny evidence.""" category = suggestions.get("category") or {} @@ -547,6 +563,7 @@ def _cmd_registry_publish(args): remote_report = result.get("preflight") or {} _print_preflight_sections(remote_report, prefix="Registry ") + _print_quality_summary(result.get("quality")) _print_registry_suggestions(result.get("suggestions") or {}) if not remote_report.get("errors"): print("Registry preflight passed.") @@ -691,6 +708,44 @@ def _cmd_registry_publish(args): except Exception: pass # Continue with publish + # Stage 1 runs after dependency publication and any automatic version bump, + # so the evidence describes the exact version Stage 2 will mutate. + from ..preflight import analyze_tool + try: + preflight_tool = Tool.from_dict(data) + preflight_tool.path = config_path + local_report = analyze_tool(preflight_tool, include_contract_tests=True) + except (KeyError, TypeError, ValueError) as exc: + print(f"Local preflight error: {exc}", file=sys.stderr) + return 1 + print("Preflight stage:") + _print_preflight_sections(local_report.to_dict(), prefix="Local ") + if not local_report.ok: + return 1 + + try: + preflight_client = get_client() + preflight_result = preflight_client.publish_tool( + config_yaml, readme=readme, defaults=defaults, + owner=getattr(args, "owner", ""), dry_run=True, + ) + except RegistryError as exc: + print(f"Registry preflight failed: {exc}", file=sys.stderr) + return 1 + remote_report = preflight_result.get("preflight") or {} + _print_preflight_sections(remote_report, prefix="Registry ") + _print_quality_summary(preflight_result.get("quality")) + if remote_report.get("errors"): + return 1 + if sys.stdin.isatty() and not getattr(args, "force", False): + try: + if input("Publish this validated version? [y/N] ").strip().lower() != "y": + print("Cancelled.") + return 1 + except (EOFError, KeyboardInterrupt): + print("\nCancelled.") + return 1 + print(f"Publishing {name}@{version}...") try: diff --git a/src/cmdforge/cli/tool_commands.py b/src/cmdforge/cli/tool_commands.py index 8754135..74c7386 100644 --- a/src/cmdforge/cli/tool_commands.py +++ b/src/cmdforge/cli/tool_commands.py @@ -187,6 +187,10 @@ def cmd_run(args): if not tool: print(f"Error: Tool '{args.name}' not found.", file=sys.stderr) return 1 + from ..tool import deprecation_warning + warning = deprecation_warning(tool) + if warning: + print(warning, file=sys.stderr) tool_args = list(args.tool_args or []) if tool_args and tool_args[0] == '--': diff --git a/src/cmdforge/gui/pages/registry_page.py b/src/cmdforge/gui/pages/registry_page.py index fed6479..3ebe8ed 100644 --- a/src/cmdforge/gui/pages/registry_page.py +++ b/src/cmdforge/gui/pages/registry_page.py @@ -1,5 +1,6 @@ """Registry page - browse and install tools from registry.""" +import html from PySide6.QtWidgets import ( QWidget, QVBoxLayout, QHBoxLayout, QLineEdit, QPushButton, QTableWidget, QTableWidgetItem, QLabel, @@ -877,6 +878,16 @@ class RegistryPage(QWidget): if tool.get("description"): lines.append(f"

{tool.get('description')}

") + if tool.get("deprecated"): + guidance = html.escape( + tool.get("deprecated_message") or "This tool is no longer maintained." + ) + if tool.get("replacement"): + guidance += f" Use {html.escape(tool['replacement'])} instead." + lines.append( + "

Deprecated: {guidance}

" + ) # Rating display rating = tool.get("average_rating", 0) or 0 diff --git a/src/cmdforge/gui/pages/tool_builder_page.py b/src/cmdforge/gui/pages/tool_builder_page.py index b8dba07..758a879 100644 --- a/src/cmdforge/gui/pages/tool_builder_page.py +++ b/src/cmdforge/gui/pages/tool_builder_page.py @@ -1161,18 +1161,57 @@ class ToolBuilderPage(QWidget): "It will not be applied automatically." ) msg.setInformativeText( - "Review the proposal below. Contract editing and approval " - "will be added by the guided-creation milestone." + "Review the proposal below. Applying it changes only this " + "in-memory draft and remains reversible until you save." ) msg.setDetailedText( yaml.safe_dump(proposed_schemas, sort_keys=False).rstrip() ) + apply_button = msg.addButton( + "Apply Proposal", QMessageBox.AcceptRole + ) save_button = msg.addButton( "Save Without Applying", QMessageBox.AcceptRole ) msg.addButton(QMessageBox.Cancel) msg.exec() - if msg.clickedButton() is not save_button: + if msg.clickedButton() is apply_button: + if proposed_schemas.get("input_schema") is not None: + tool.input_schema = proposed_schemas["input_schema"] + if proposed_schemas.get("output_schema") is not None: + tool.output_schema = proposed_schemas["output_schema"] + elif msg.clickedButton() is not save_button: + return + + if preflight.reuse_opportunities: + reuse = QMessageBox(self) + reuse.setIcon(QMessageBox.Information) + reuse.setWindowTitle("Reuse Opportunities") + reuse.setText( + "CmdForge found exact, contract-backed reuse evidence. " + "This is advisory and will not change your draft." + ) + reuse.setDetailedText("\n".join( + f"- {item['detail']}\n Evidence: {item.get('evidence', 'exact match')}" + for item in preflight.reuse_opportunities + )) + continue_button = reuse.addButton( + "Continue With Draft", QMessageBox.AcceptRole + ) + matching = next( + (item.get("tool") for item in preflight.reuse_opportunities if item.get("tool")), + None, + ) + open_button = ( + reuse.addButton("Open Existing", QMessageBox.ActionRole) + if matching else None + ) + reuse.addButton(QMessageBox.Cancel) + reuse.exec() + if open_button and reuse.clickedButton() is open_button: + _switch_to_existing_tool(self.main_window, matching) + return + if reuse.clickedButton() is not continue_button: return advisory_count = len(preflight.warnings) + len(preflight.suggestions) diff --git a/src/cmdforge/gui/pages/tools_page.py b/src/cmdforge/gui/pages/tools_page.py index d4824b0..b7c8f40 100644 --- a/src/cmdforge/gui/pages/tools_page.py +++ b/src/cmdforge/gui/pages/tools_page.py @@ -1,5 +1,6 @@ """Tools page - main view for managing tools.""" +import html from collections import defaultdict from pathlib import Path from typing import Optional, Tuple, Dict, Any @@ -852,6 +853,16 @@ class ToolsPage(QWidget): lines.append(f"

{tool.name}

") if tool.description: lines.append(f"

{tool.description}

") + if tool.deprecated: + guidance = html.escape( + tool.deprecated_message or "This tool is no longer maintained." + ) + if tool.replacement: + guidance += f" Use {html.escape(tool.replacement)} instead." + lines.append( + "

Deprecated: {guidance}

" + ) # Publish state state, registry_hash = get_tool_publish_state(qname) diff --git a/src/cmdforge/preflight.py b/src/cmdforge/preflight.py index a62575f..39ac33e 100644 --- a/src/cmdforge/preflight.py +++ b/src/cmdforge/preflight.py @@ -5,6 +5,8 @@ All checks are deterministic and evidence-based. """ import copy +import hashlib +import json import re from dataclasses import dataclass, field from typing import Any, Dict, List, Optional @@ -26,9 +28,6 @@ class PreflightReport: regression: Optional[Dict[str, Any]] = None reuse_opportunities: List[Dict[str, Any]] = field(default_factory=list) audit_evidence: Optional[Dict[str, Any]] = None - compatibility: List[Dict[str, Any]] = field(default_factory=list) - contract_proposal: Optional[Dict[str, Any]] = None - regression: Optional[Dict[str, Any]] = None @property def ok(self) -> bool: @@ -69,6 +68,7 @@ def analyze_tool( *, check_local_dependencies: bool = True, include_contract_tests: bool = False, + check_reuse: Optional[bool] = None, ) -> PreflightReport: """Run all preflight checks on a tool. @@ -80,17 +80,27 @@ def analyze_tool( A PreflightReport with structured findings. """ report = PreflightReport() + checks_run = ["config_integrity", "contracts"] _check_config_integrity(tool, report) _check_contracts(tool, report) _add_contract_guidance(tool, report, include_contract_tests) + if include_contract_tests: + checks_run.append("contract_conformance") _check_secrets(tool, report) + checks_run.append("secrets") if check_local_dependencies: _check_toolstep_compatibility(tool, report) _check_dependencies(tool, report) - _check_reuse_opportunities(tool, report) - _add_audit_evidence(tool, report) + checks_run.extend(["toolstep_compatibility", "dependencies"]) if registry_client: _check_similar_tools(tool, report, registry_client) + checks_run.append("registry_similarity") + if check_reuse is None: + check_reuse = check_local_dependencies + if check_reuse: + _check_reuse_opportunities(tool, report) + checks_run.append("reuse_opportunities") + _add_audit_evidence(tool, report, checks_run) return report @@ -372,21 +382,14 @@ def _is_semver(version: str) -> bool: def _check_reuse_opportunities(tool: Tool, report: PreflightReport): - """Detect repeated step patterns that could be extracted into reusable tools.""" + """Report only exact repeated sequences with contract-boundary evidence.""" from .tool import list_tools, load_tool if len(tool.steps) < 2: return - # Build a signature of each step for comparison - step_signatures = [] - for step in tool.steps: - sig = _step_signature(step) - if sig: - step_signatures.append(sig) - - if not step_signatures: - return + step_signatures = [_step_signature(step) for step in tool.steps] + _find_internal_reuse(tool, step_signatures, report) # Check if this step sequence appears in other local tools our_sequence = tuple(step_signatures) @@ -398,55 +401,92 @@ def _check_reuse_opportunities(tool: Tool, report: PreflightReport): continue other_sigs = [_step_signature(s) for s in other.steps] other_sigs = [s for s in other_sigs if s] - if our_sequence == tuple(other_sigs): + if ( + our_sequence == tuple(other_sigs) + and _has_tool_contracts(tool) + and _has_tool_contracts(other) + ): report.reuse_opportunities.append({ "type": "duplicate_sequence", "tool": name, - "detail": f"Step sequence identical to '{name}' — consider extracting shared logic", + "length": len(our_sequence), + "evidence": "exact normalized steps and explicit tool contracts", + "detail": f"Exact contracted step sequence also exists in '{name}'", }) - elif len(our_sequence) >= 2: - # Check for overlapping subsequences - for start in range(len(other_sigs) - 1): - for length in range(2, min(len(our_sequence), len(other_sigs) - start) + 1): - if our_sequence[:length] == tuple(other_sigs[start:start + length]): - report.reuse_opportunities.append({ - "type": "shared_subsequence", - "tool": name, - "length": length, - "detail": f"Shares {length} step(s) with '{name}' — possible extraction candidate", - }) - break def _step_signature(step) -> str: - """Build a comparable signature for a step.""" - if hasattr(step, "prompt"): - return f"prompt:{step.provider}" - elif hasattr(step, "code"): - return "code" - elif hasattr(step, "tool"): - return f"tool:{step.tool}" - elif hasattr(step, "server"): - return f"mcp:{step.server}" - return "" + """Build an exact normalized signature without incidental labels/outputs.""" + data = step.to_dict() + data.pop("name", None) + data.pop("output_var", None) + encoded = json.dumps(data, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() -def _add_audit_evidence(tool: Tool, report: PreflightReport): +def _has_tool_contracts(tool: Tool) -> bool: + return tool.input_schema is not None and tool.output_schema is not None + + +def _find_internal_reuse( + tool: Tool, signatures: List[str], report: PreflightReport +) -> None: + """Find non-overlapping repeated sequences with per-step output evidence.""" + from .tool import PromptStep + + count = len(signatures) + for length in range(count // 2, 1, -1): + for first in range(0, count - (2 * length) + 1): + candidate = signatures[first:first + length] + for second in range(first + length, count - length + 1): + if candidate != signatures[second:second + length]: + continue + steps = tool.steps[first:first + length] + if not all( + isinstance(step, PromptStep) and step.output_schema is not None + for step in steps + ): + continue + report.reuse_opportunities.append({ + "type": "repeated_sequence", + "length": length, + "locations": [first + 1, second + 1], + "evidence": "exact normalized prompt steps with output schemas", + "detail": ( + f"Steps {first + 1}-{first + length} repeat at " + f"steps {second + 1}-{second + length}" + ), + }) + return + + +def _add_audit_evidence( + tool: Tool, report: PreflightReport, checks_run: List[str] +): """Attach audit evidence metadata to the report.""" from datetime import datetime, timezone + findings = { + "errors": report.errors, + "warnings": report.warnings, + "suggestions": report.suggestions, + "generated_tests": report.generated_tests, + "regression": report.regression, + "compatibility": report.compatibility, + "similar_tools": report.similar_tools, + "reuse_opportunities": report.reuse_opportunities, + } + findings_hash = hashlib.sha256( + json.dumps(findings, sort_keys=True, separators=(",", ":"), default=str).encode() + ).hexdigest() report.audit_evidence = { "tool_version": tool.version or "", "engine_version": "1.0", "timestamp": datetime.now(timezone.utc).isoformat(), - "checks_run": [ - "config_integrity", - "contracts", - "secrets", - "dependencies", - "toolstep_compatibility", - "reuse_opportunities", - ], + "checks_run": checks_run, + "findings_hash": findings_hash, + "findings": copy.deepcopy(findings), + "outcome": "failed" if report.errors else "advisory" if report.warnings else "passed", } diff --git a/src/cmdforge/quality.py b/src/cmdforge/quality.py index 27fb5a7..a06e305 100644 --- a/src/cmdforge/quality.py +++ b/src/cmdforge/quality.py @@ -20,9 +20,14 @@ class CategoryScore: earned: int available: int state: str = "checked" # checked | not_tested | not_applicable + possible: Optional[int] = None # Category maximum when only partial evidence ran @property def display(self) -> str: + if self.state == "not_tested": + return "not tested" + if self.state == "not_applicable": + return "n/a" return f"{self.earned}/{self.available}" @property @@ -39,6 +44,7 @@ class QualityScore: headline: int # 0-100 categories: List[CategoryScore] = field(default_factory=list) last_evaluated: str = "" + evidence_coverage: int = 0 def to_dict(self) -> dict: return { @@ -51,14 +57,16 @@ class QualityScore: "earned": c.earned, "available": c.available, "state": c.state, + "possible": c.possible if c.possible is not None else c.available, } for c in self.categories ], "last_evaluated": self.last_evaluated, + "evidence_coverage": self.evidence_coverage, } def __str__(self) -> str: - lines = [f"Quality {self.headline}"] + lines = [f"Quality {self.headline} — {self.evidence_coverage}% evidence coverage"] if self.last_evaluated: lines[0] += f" — Evaluated {self.last_evaluated}" for c in self.categories: @@ -91,8 +99,13 @@ def compute_quality( categories.append(_score_security(report, tool)) categories.append(_score_community(registry_data)) - earned = sum(c.earned for c in categories) - available = sum(c.available for c in categories) + checked = [c for c in categories if c.state == "checked"] + earned = sum(c.earned for c in checked) + available = sum(c.available for c in checked) + possible = sum( + c.possible if c.possible is not None else c.available + for c in categories if c.state != "not_applicable" + ) headline = int((earned / available * 100)) if available > 0 else 0 return QualityScore( @@ -101,6 +114,7 @@ def compute_quality( headline=headline, categories=categories, last_evaluated=datetime.now(timezone.utc).isoformat(), + evidence_coverage=int((available / possible * 100)) if possible else 0, ) @@ -112,6 +126,9 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore: earned += 5 if tool.output_schema is not None: earned += 5 + if tool.input_schema is None and tool.output_schema is None: + return CategoryScore("Contracts", 0, available, "not_tested") + # No contract-related errors in the preflight contract_errors = [ e for e in report.errors if "schema" in e.lower() or "contract" in e.lower() @@ -121,27 +138,26 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore: else: earned = max(0, earned - len(contract_errors)) - state = "checked" if (tool.input_schema or tool.output_schema) else "not_tested" - return CategoryScore("Contracts", earned, available, state) + return CategoryScore("Contracts", earned, available, "checked") def _score_tests(report: PreflightReport) -> CategoryScore: - available = 30 + maximum = 30 tests = report.generated_tests or [] if not tests: - return CategoryScore("Deterministic tests", 0, available, "not_tested") - - passed = sum(1 for t in tests if t.get("state") == "passed") - failed = sum(1 for t in tests if t.get("state") == "failed") - total = len(tests) - - if total == 0: - return CategoryScore("Deterministic tests", 0, available, "not_tested") + return CategoryScore("Deterministic tests", 0, maximum, "not_tested") + executed = [t for t in tests if t.get("state") in ("passed", "failed")] + if not executed: + return CategoryScore("Deterministic tests", 0, maximum, "not_tested") + passed = sum(1 for t in executed if t.get("state") == "passed") + total = len(executed) + available = max(1, int(maximum * total / len(tests))) earned = int((passed / total) * available) - state = "checked" if failed == 0 else "checked" - return CategoryScore("Deterministic tests", earned, available, state) + return CategoryScore( + "Deterministic tests", earned, available, "checked", possible=maximum + ) def _score_regression(report: PreflightReport) -> CategoryScore: @@ -162,7 +178,23 @@ def _score_regression(report: PreflightReport) -> CategoryScore: def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore: - available = 20 + maximum = 20 + checks = set((report.audit_evidence or {}).get("checks_run", [])) + if "security_scrutiny" in checks: + available = maximum + elif checks: + available = (10 if "secrets" in checks else 0) + ( + 10 if checks.intersection({"dependencies", "registry_dependencies"}) else 0 + ) + elif report.errors or report.warnings: + # Backward-compatible evidence supplied by callers without audit metadata. + available = maximum + else: + available = 0 + if not available: + return CategoryScore( + "Security scrutiny", 0, maximum, "not_tested", possible=maximum + ) earned = available # Deduct for secret patterns found @@ -172,13 +204,20 @@ def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore: earned -= len(secret_warnings) * 5 # Deduct for unresolved dependencies - dep_warnings = [ - w for w in report.warnings if "dependency" in w.lower() and "not installed" in w.lower() + dependency_findings = [ + finding for finding in report.warnings + report.errors + if ("dependency" in finding.lower() or "dependencies" in finding.lower()) + and any( + marker in finding.lower() + for marker in ("not installed", "missing", "deprecated") + ) ] - earned -= len(dep_warnings) * 3 + earned -= len(dependency_findings) * 3 earned = max(0, earned) - return CategoryScore("Security scrutiny", earned, available, "checked") + return CategoryScore( + "Security scrutiny", earned, available, "checked", possible=maximum + ) def _score_community(registry_data: Optional[dict]) -> CategoryScore: @@ -189,8 +228,11 @@ def _score_community(registry_data: Optional[dict]) -> CategoryScore: earned = 0 reviews = registry_data.get("reviews", []) - if reviews: + rating_count = registry_data.get("rating_count", len(reviews)) + avg_rating = registry_data.get("average_rating") + if avg_rating is None and reviews: avg_rating = sum(r.get("rating", 0) for r in reviews) / len(reviews) + if avg_rating is not None and rating_count: earned += int((avg_rating / 5) * 8) downloads = registry_data.get("downloads", 0) diff --git a/src/cmdforge/registry/app.py b/src/cmdforge/registry/app.py index 5e2508d..a0e481f 100644 --- a/src/cmdforge/registry/app.py +++ b/src/cmdforge/registry/app.py @@ -9,7 +9,7 @@ import os import re import secrets from dataclasses import dataclass -from datetime import date, datetime, timedelta +from datetime import date, datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Optional, Tuple from flask import Flask, Response, g, jsonify, request @@ -418,7 +418,7 @@ def create_app() -> Flask: g.db.execute( "UPDATE api_tokens SET last_used_at = ? WHERE id = ?", - [datetime.utcnow().isoformat(), row["id"]], + [datetime.now(timezone.utc).isoformat(), row["id"]], ) g.current_publisher = { "id": row["publisher_id"], @@ -870,6 +870,14 @@ def create_app() -> Flask: order_sql = f"{sort} {order_dir}, published_at DESC, f.id DESC" stats_join = "LEFT JOIN tool_stats ts ON ts.tool_id = f.id" + audit_join = """ + LEFT JOIN tool_audits ta ON ta.id = ( + SELECT latest_audit.id FROM tool_audits latest_audit + WHERE latest_audit.tool_id = f.id + ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC + LIMIT 1 + ) + """ rows = query_all( g.db, @@ -897,9 +905,13 @@ def create_app() -> Flask: GROUP BY owner, name ) SELECT f.*, COALESCE(ts.average_rating, 0) AS average_rating, - COALESCE(ts.rating_count, 0) AS rating_count + COALESCE(ts.rating_count, 0) AS rating_count, + json_extract(ta.quality_json, '$.score') AS quality_score, + json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage, + ta.evaluated_at AS quality_evaluated_at FROM filtered f {stats_join} + {audit_join} JOIN ( SELECT a.owner, a.name, COALESCE(s.max_id, a.max_id) AS max_id FROM latest_any a @@ -948,6 +960,12 @@ def create_app() -> Flask: "average_rating": row["average_rating"], "rating_count": row["rating_count"], "score": score, + "quality_score": row["quality_score"], + "quality_coverage": row["quality_coverage"], + "quality_evaluated_at": row["quality_evaluated_at"], + "deprecated": bool(row["deprecated"]), + "deprecated_message": row["deprecated_message"], + "replacement": row["replacement"], }) result: dict = {"data": data, "meta": paginate(page, per_page, total)} @@ -1089,6 +1107,39 @@ def create_app() -> Flask: if fork_row: fork_count = fork_row["cnt"] + replacement_chain = [] + if row["replacement"]: + try: + replacement_chain = _get_replacement_chain( + row["owner"], row["name"], row["replacement"] + ) + except ValueError: + # Preserve access to legacy rows while refusing new invalid chains. + replacement_chain = [] + + latest_audit = query_one( + g.db, + """ + SELECT engine_version, evidence_json, quality_json, evaluated_at + FROM tool_audits + WHERE tool_id = ? + ORDER BY evaluated_at DESC, id DESC + LIMIT 1 + """, + [row["id"]], + ) + audit_stale = True + if latest_audit: + try: + evaluated = datetime.fromisoformat( + latest_audit["evaluated_at"].replace("Z", "+00:00") + ) + if evaluated.tzinfo is None: + evaluated = evaluated.replace(tzinfo=timezone.utc) + audit_stale = datetime.now(timezone.utc) - evaluated > timedelta(days=30) + except (AttributeError, TypeError, ValueError): + audit_stale = True + payload = { "owner": row["owner"], "name": row["name"], @@ -1100,7 +1151,8 @@ def create_app() -> Flask: "published_at": row["published_at"], "deprecated": bool(row["deprecated"]), "deprecated_message": row["deprecated_message"], - "replacement": row["replacement"], + "replacement": row["replacement"] if replacement_chain else None, + "replacement_chain": replacement_chain, "config": row["config_yaml"], "readme": row["readme"], "defaults": row.get("defaults") or "", @@ -1108,6 +1160,10 @@ def create_app() -> Flask: "forked_from": row.get("forked_from"), "forked_version": row.get("forked_version"), "fork_count": fork_count, + "audit": json.loads(latest_audit["evidence_json"]) if latest_audit else None, + "quality": json.loads(latest_audit["quality_json"]) if latest_audit else None, + "audit_evaluated_at": latest_audit["evaluated_at"] if latest_audit else None, + "audit_stale": audit_stale, } response = jsonify({"data": payload}) response.headers["Cache-Control"] = "max-age=60" @@ -2496,8 +2552,12 @@ def create_app() -> Flask: from ..preflight import analyze_tool as run_preflight from ..tool import Tool + published_tool = Tool.from_dict(data) preflight_report = run_preflight( - Tool.from_dict(data), check_local_dependencies=False + published_tool, + check_local_dependencies=False, + include_contract_tests=True, + check_reuse=False, ) except (KeyError, TypeError, ValueError) as exc: return error_response("INVALID_CONFIG", str(exc), 400) @@ -2538,6 +2598,44 @@ def create_app() -> Flask: ) owner = requested_owner + from .auditing import registry_dependency_findings + dependency_findings = registry_dependency_findings( + g.db, published_tool, owner + ) + missing_dependencies = dependency_findings["missing"] + preflight_report.warnings.extend(dependency_findings["deprecated"]) + from ..preflight import _add_audit_evidence + audit_checks = list( + (preflight_report.audit_evidence or {}).get("checks_run", []) + ) + _add_audit_evidence( + published_tool, preflight_report, + audit_checks + ["registry_dependencies"], + ) + from ..quality import compute_quality + quality_report = compute_quality( + published_tool, preflight_report + ).to_dict() + if missing_dependencies: + message = "Missing registry dependencies: " + ", ".join( + missing_dependencies + ) + preflight_report.errors.append(message) + _add_audit_evidence( + published_tool, preflight_report, + audit_checks + ["registry_dependencies"], + ) + return error_response( + "PREFLIGHT_FAILED", message, 400, + details={"preflight": preflight_report.to_dict()}, + ) + + if data.get("deprecated") and data.get("replacement"): + try: + _get_replacement_chain(owner, name, data["replacement"]) + except ValueError as exc: + return error_response("VALIDATION_ERROR", str(exc), 400) + # Compute config hash early for idempotency check config_hash = compute_yaml_hash(config_text) @@ -2650,6 +2748,7 @@ def create_app() -> Flask: "status": "validated", "suggestions": suggestions, "preflight": preflight_report.to_dict(), + "quality": quality_report, } }) @@ -2675,7 +2774,7 @@ def create_app() -> Flask: # Public tools need moderation approval moderation_status = "pending" - g.db.execute( + insert_cursor = g.db.execute( """ INSERT INTO tools ( owner, name, version, description, category, tags, config_yaml, readme, @@ -2710,7 +2809,25 @@ def create_app() -> Flask: moderation_status, forked_from, forked_version, - datetime.utcnow().isoformat(), + datetime.now(timezone.utc).isoformat(), + ], + ) + tool_id = insert_cursor.lastrowid + audit = preflight_report.audit_evidence or {} + g.db.execute( + """ + INSERT INTO tool_audits ( + tool_id, engine_version, trigger, evidence_json, quality_json, + findings_hash, evaluated_at + ) VALUES (?, ?, 'publish', ?, ?, ?, ?) + """, + [ + tool_id, + audit.get("engine_version", "1.0"), + json.dumps(audit, sort_keys=True), + json.dumps(quality_report, sort_keys=True), + audit.get("findings_hash", ""), + audit.get("timestamp") or datetime.now(timezone.utc).isoformat(), ], ) g.db.commit() @@ -2756,6 +2873,7 @@ def create_app() -> Flask: "forked_version": forked_version, "suggestions": suggestions, "preflight": preflight_report.to_dict(), + "quality": quality_report, } }) response.status_code = 201 @@ -2870,6 +2988,47 @@ def create_app() -> Flask: return jsonify({"data": results}) + def _get_replacement_chain( + original_owner: str, original_name: str, replacement: str + ) -> List[str]: + """Validate and resolve a deprecation chain without allowing cycles.""" + chain: List[str] = [] + visited = {(original_owner, original_name)} + current = replacement + default_owner = original_owner + for _ in range(20): + if "/" in current: + next_owner, next_name = current.split("/", 1) + else: + next_owner, next_name = default_owner, current + if not OWNER_RE.fullmatch(next_owner) or not TOOL_NAME_RE.fullmatch(next_name): + raise ValueError( + "replacement must be a valid tool name or owner/tool reference" + ) + identity = (next_owner, next_name) + if identity in visited: + raise ValueError("replacement creates a deprecation cycle") + row = query_one( + g.db, + """ + SELECT deprecated, replacement FROM tools + WHERE owner = ? AND name = ? + ORDER BY id DESC LIMIT 1 + """, + [next_owner, next_name], + ) + if row is None: + raise ValueError( + f"replacement tool '{next_owner}/{next_name}' does not exist" + ) + visited.add(identity) + chain.append(f"{next_owner}/{next_name}") + if not row["deprecated"] or not row["replacement"]: + return chain + current = row["replacement"] + default_owner = next_owner + raise ValueError("replacement chain exceeds 20 tools") + @app.route("/api/v1/tools///deprecate", methods=["POST"]) @require_token def deprecate_tool(owner: str, name: str) -> Response: @@ -2883,6 +3042,10 @@ def create_app() -> Flask: if message and len(message) > 500: return error_response("VALIDATION_ERROR", "Message too long (max 500)", 400) + try: + chain = _get_replacement_chain(owner, name, replacement) if replacement else [] + except ValueError as exc: + return error_response("VALIDATION_ERROR", str(exc), 400) # Update all versions of the tool result = g.db.execute( @@ -2896,7 +3059,10 @@ def create_app() -> Flask: return error_response("TOOL_NOT_FOUND", f"Tool {owner}/{name} not found", 404) g.db.commit() - return jsonify({"data": {"status": "deprecated", "owner": owner, "name": name}}) + return jsonify({"data": { + "status": "deprecated", "owner": owner, "name": name, + "replacement_chain": chain, + }}) @app.route("/api/v1/tools///undeprecate", methods=["POST"]) @require_token diff --git a/src/cmdforge/registry/auditing.py b/src/cmdforge/registry/auditing.py new file mode 100644 index 0000000..2d0f567 --- /dev/null +++ b/src/cmdforge/registry/auditing.py @@ -0,0 +1,210 @@ +"""Immutable publish-time and refresh audits for registry tool versions.""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List + +import yaml + +from ..preflight import analyze_tool +from ..quality import compute_quality +from ..tool import Tool +from .db import connect_db, get_db_path, init_db + + +def registry_dependency_findings(conn, tool: Tool, owner: str) -> Dict[str, List[str]]: + """Resolve registry dependencies and report missing/deprecated references.""" + from ..tool import ToolStep + + references = set(tool.dependencies) + references.update( + step.tool for step in tool.steps if isinstance(step, ToolStep) + ) + missing: List[str] = [] + deprecated: List[str] = [] + for reference in sorted(references): + if reference == tool.name or reference == f"{owner}/{tool.name}": + continue + if "/" in reference: + dep_owner, dep_name = reference.split("/", 1) + candidates = [(dep_owner, dep_name)] + else: + candidates = [(owner, reference), ("official", reference)] + row = None + for dep_owner, dep_name in candidates: + row = conn.execute( + """ + SELECT deprecated, replacement FROM tools + WHERE owner = ? AND name = ? ORDER BY id DESC LIMIT 1 + """, + [dep_owner, dep_name], + ).fetchone() + if row is not None: + break + if row is None: + missing.append(reference) + elif row["deprecated"]: + guidance = f"{reference} is deprecated" + if row["replacement"]: + guidance += f"; use {row['replacement']}" + deprecated.append(guidance) + return {"missing": missing, "deprecated": deprecated} + + +def audit_tool_version(conn, tool_id: int, trigger: str = "background") -> Dict[str, Any]: + """Evaluate one immutable tool version and append an audit record.""" + row = conn.execute( + """ + SELECT tools.*, COALESCE(tool_stats.average_rating, 0) AS average_rating, + COALESCE(tool_stats.rating_count, 0) AS rating_count + FROM tools + LEFT JOIN tool_stats ON tool_stats.tool_id = tools.id + WHERE tools.id = ? + """, + [tool_id], + ).fetchone() + if row is None: + raise ValueError(f"Unknown tool version id {tool_id}") + + previous = conn.execute( + """ + SELECT evidence_json, quality_json FROM tool_audits + WHERE tool_id = ? ORDER BY evaluated_at DESC, id DESC LIMIT 1 + """, + [tool_id], + ).fetchone() + config = yaml.safe_load(row["config_yaml"]) + tool = Tool.from_dict(config) + report = analyze_tool( + tool, + check_local_dependencies=False, + include_contract_tests=True, + check_reuse=False, + ) + dependency_findings = registry_dependency_findings(conn, tool, row["owner"]) + if dependency_findings["missing"]: + report.errors.append( + "Missing registry dependencies: " + + ", ".join(dependency_findings["missing"]) + ) + report.warnings.extend(dependency_findings["deprecated"]) + from ..preflight import _add_audit_evidence + checks = list((report.audit_evidence or {}).get("checks_run", [])) + _add_audit_evidence(tool, report, checks + ["registry_dependencies"]) + registry_data = { + "average_rating": row["average_rating"], + "rating_count": row["rating_count"], + "downloads": row["downloads"], + } + quality = compute_quality(tool, report, registry_data=registry_data).to_dict() + evidence = report.audit_evidence or {} + evaluated_at = datetime.now(timezone.utc).isoformat() + conn.execute( + """ + INSERT INTO tool_audits ( + tool_id, engine_version, trigger, evidence_json, quality_json, + findings_hash, evaluated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [ + tool_id, + evidence.get("engine_version", "1.0"), + trigger, + json.dumps(evidence, sort_keys=True), + json.dumps(quality, sort_keys=True), + evidence.get("findings_hash", ""), + evaluated_at, + ], + ) + quality_change = None + if previous: + previous_quality = json.loads(previous["quality_json"]) + quality_change = quality["score"] - previous_quality.get("score", 0) + previous_evidence = json.loads(previous["evidence_json"]) + if quality_change < 0 or previous_evidence.get("outcome") != evidence.get("outcome"): + conn.execute( + """ + INSERT INTO audit_log ( + action, target_type, target_id, actor_id, details + ) VALUES ('automated_audit_changed', 'tool', ?, 'system', ?) + """, + [ + str(tool_id), + json.dumps({ + "quality_change": quality_change, + "previous_outcome": previous_evidence.get("outcome"), + "current_outcome": evidence.get("outcome"), + }, sort_keys=True), + ], + ) + conn.commit() + return { + "tool_id": tool_id, "evidence": evidence, "quality": quality, + "quality_change": quality_change, + } + + +def audit_stale_tools( + conn, *, max_age_days: int = 30, limit: int = 100 +) -> List[Dict[str, Any]]: + """Append refreshed evidence for versions with missing or stale audits.""" + if max_age_days < 0: + raise ValueError("max_age_days must be non-negative") + if limit < 1: + raise ValueError("limit must be positive") + cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days) + rows = conn.execute( + """ + SELECT tools.id, MAX(tool_audits.evaluated_at) AS last_evaluated + FROM tools + LEFT JOIN tool_audits ON tool_audits.tool_id = tools.id + GROUP BY tools.id + ORDER BY last_evaluated IS NOT NULL, last_evaluated + """ + ).fetchall() + stale_ids = [] + for row in rows: + value = row["last_evaluated"] + if value: + try: + evaluated = datetime.fromisoformat(value.replace("Z", "+00:00")) + if evaluated.tzinfo is None: + evaluated = evaluated.replace(tzinfo=timezone.utc) + except ValueError: + evaluated = datetime.min.replace(tzinfo=timezone.utc) + if evaluated >= cutoff: + continue + stale_ids.append(row["id"]) + if len(stale_ids) >= limit: + break + results = [] + for tool_id in stale_ids: + try: + results.append(audit_tool_version(conn, tool_id)) + except Exception as exc: + results.append({"tool_id": tool_id, "error": str(exc)}) + return results + + +def main() -> int: + parser = argparse.ArgumentParser(description="Refresh stale CmdForge registry audits") + parser.add_argument("--max-age-days", type=int, default=30) + parser.add_argument("--limit", type=int, default=100) + args = parser.parse_args() + conn = connect_db(get_db_path()) + try: + init_db(conn) + results = audit_stale_tools( + conn, max_age_days=args.max_age_days, limit=args.limit + ) + finally: + conn.close() + print(f"Audited {len(results)} tool version(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cmdforge/registry/db.py b/src/cmdforge/registry/db.py index b55335f..a442d06 100644 --- a/src/cmdforge/registry/db.py +++ b/src/cmdforge/registry/db.py @@ -80,6 +80,20 @@ CREATE TABLE IF NOT EXISTS download_stats ( UNIQUE(tool_id, client_id, downloaded_at) ); +CREATE TABLE IF NOT EXISTS tool_audits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tool_id INTEGER NOT NULL REFERENCES tools(id) ON DELETE CASCADE, + engine_version TEXT NOT NULL, + trigger TEXT NOT NULL, + evidence_json TEXT NOT NULL, + quality_json TEXT NOT NULL, + findings_hash TEXT NOT NULL, + evaluated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_tool_audits_tool_time +ON tool_audits(tool_id, evaluated_at DESC, id DESC); + CREATE VIRTUAL TABLE IF NOT EXISTS tools_fts USING fts5( name, description, tags, readme, content='tools', diff --git a/src/cmdforge/registry/embeddings.py b/src/cmdforge/registry/embeddings.py index 6f99685..4a34c27 100644 --- a/src/cmdforge/registry/embeddings.py +++ b/src/cmdforge/registry/embeddings.py @@ -157,10 +157,19 @@ def semantic_search( """ SELECT te.tool_id, te.embedding, te.dimensions, te.model, t.owner, t.name, t.version, t.description, t.category, - t.tags, t.downloads + t.tags, t.downloads, + json_extract(ta.quality_json, '$.score') AS quality_score, + json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage FROM tool_embeddings te JOIN tools t ON t.id = te.tool_id + LEFT JOIN tool_audits ta ON ta.id = ( + SELECT latest_audit.id FROM tool_audits latest_audit + WHERE latest_audit.tool_id = t.id + ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC + LIMIT 1 + ) WHERE t.visibility = 'public' AND t.moderation_status = 'approved' + AND t.deprecated = 0 """, ).fetchall() @@ -192,6 +201,8 @@ def semantic_search( "tags": tags_list, "downloads": row["downloads"] or 0, "similarity": round(score, 4), + "quality_score": row["quality_score"], + "quality_coverage": row["quality_coverage"], }) # Sort by similarity descending diff --git a/src/cmdforge/runner.py b/src/cmdforge/runner.py index af97e43..fa23efd 100644 --- a/src/cmdforge/runner.py +++ b/src/cmdforge/runner.py @@ -1173,6 +1173,11 @@ def main(): print(f"Searched: {', '.join(e.searched_paths[:3])}", file=sys.stderr) sys.exit(1) + from .tool import deprecation_warning + warning = deprecation_warning(tool) + if warning: + print(warning, file=sys.stderr) + # Check for manifest overrides manifest = load_manifest() provider_override_from_manifest = None diff --git a/src/cmdforge/tool.py b/src/cmdforge/tool.py index 9c85e53..97148af 100644 --- a/src/cmdforge/tool.py +++ b/src/cmdforge/tool.py @@ -1,6 +1,7 @@ """Tool loading, saving, and management.""" import os +import re import shutil import stat from dataclasses import dataclass, field @@ -15,6 +16,9 @@ TOOLS_DIR = Path.home() / ".cmdforge" # Default bin directory for wrapper scripts BIN_DIR = Path.home() / ".local" / "bin" +_TOOL_REF_RE = re.compile( + r"^(?:[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?/)?[A-Za-z0-9_-]{1,64}$" +) def _validate_skill_selection(skills: Optional[List[str]]) -> None: @@ -447,16 +451,27 @@ class Tool: source: Optional[ToolSource] = None # Attribution for imported/external tools version: str = "" # Tool version visibility: str = "public" # "public", "private", or "unlisted" - deprecated: bool = False # Tool is deprecated - deprecated_message: str = "" # Migration guidance for deprecated tools - replacement: Optional[str] = None # Suggested replacement tool name input_schema: Optional[dict] = None # JSON Schema for tool input contract output_schema: Optional[dict] = None # JSON Schema for tool output contract path: Optional[Path] = None # Path to config.yaml (set by load_tool) + deprecated: bool = False # Tool is deprecated + deprecated_message: str = "" # Migration guidance for deprecated tools + replacement: Optional[str] = None # Suggested replacement tool name def __post_init__(self) -> None: validate_json_schema(self.input_schema, "input_schema") validate_json_schema(self.output_schema, "output_schema") + if not isinstance(self.deprecated, bool): + raise ValueError("deprecated must be true or false") + if not isinstance(self.deprecated_message, str): + raise ValueError("deprecated_message must be a string") + if len(self.deprecated_message) > 500: + raise ValueError("deprecated_message must be at most 500 characters") + if self.replacement is not None and ( + not isinstance(self.replacement, str) + or not _TOOL_REF_RE.fullmatch(self.replacement) + ): + raise ValueError("replacement must be a tool name or owner/tool reference") @classmethod def from_dict(cls, data: dict) -> "Tool": @@ -809,6 +824,16 @@ def tool_exists(name: str) -> bool: return (get_tools_dir() / name / "config.yaml").exists() +def deprecation_warning(tool: Tool) -> Optional[str]: + """Return consistent migration guidance for a deprecated tool.""" + if not tool.deprecated: + return None + message = tool.deprecated_message or "This tool is no longer maintained." + if tool.replacement: + message += f" Use '{tool.replacement}' instead." + return f"Warning: '{tool.name}' is deprecated. {message}" + + def validate_tool_name(name: str) -> tuple[bool, str]: """ Validate a tool name. diff --git a/tests/test_cli.py b/tests/test_cli.py index bdecfcc..922c0c0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -541,6 +541,37 @@ class TestRegistryPublishDryRun: assert _cmd_registry_publish(self.args(tool_dir)) == 1 assert "YAML mapping" in capsys.readouterr().err + def test_normal_publish_runs_remote_preflight_before_mutation( + self, tool_dir, monkeypatch + ): + from cmdforge.cli.registry_commands import _cmd_registry_publish + + client = MagicMock() + client.get_me.return_value = {"slug": "testuser"} + client.get_my_tool_status.return_value = {"status": "pending"} + client.publish_tool.side_effect = [ + { + "preflight": {"errors": [], "warnings": [], "suggestions": []}, + "quality": {"score": 80, "evidence_coverage": 60, "categories": []}, + }, + { + "owner": "testuser", "name": "dry-run-tool", + "version": "1.0.0", "status": "pending", + }, + ] + monkeypatch.setattr( + "cmdforge.cli.registry_commands.load_config", + lambda: SimpleNamespace(registry=SimpleNamespace(token="token")), + ) + monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client) + args = SimpleNamespace( + path=str(tool_dir), dry_run=False, force=True, owner="" + ) + assert _cmd_registry_publish(args) == 0 + assert client.publish_tool.call_count == 2 + assert client.publish_tool.call_args_list[0].kwargs["dry_run"] is True + assert "dry_run" not in client.publish_tool.call_args_list[1].kwargs + def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys): from cmdforge.cli import cmd_inspect diff --git a/tests/test_collection_api.py b/tests/test_collection_api.py index 809a762..3bcf08b 100644 --- a/tests/test_collection_api.py +++ b/tests/test_collection_api.py @@ -56,7 +56,7 @@ def client(app): def auth_headers(app): """Create auth headers with a valid token.""" import hashlib - from datetime import datetime + from datetime import datetime, timezone from cmdforge.registry.db import connect_db token = "test-token" @@ -94,7 +94,7 @@ def auth_headers(app): INSERT INTO api_tokens (publisher_id, token_hash, name, created_at) VALUES (?, ?, ?, ?) """, - [publisher_id, token_hash, "test-token", datetime.utcnow().isoformat()], + [publisher_id, token_hash, "test-token", datetime.now(timezone.utc).isoformat()], ) # Insert a public approved tool for tests if missing @@ -112,7 +112,7 @@ def auth_headers(app): [ "testuser", "tool1", "1.0.0", "Test tool", "Other", "[]", "name: tool1\nversion: 1.0.0\n", "", publisher_id, - "public", "approved", datetime.utcnow().isoformat(), + "public", "approved", datetime.now(timezone.utc).isoformat(), ], ) @@ -158,6 +158,44 @@ class TestToolApprovedEndpoint: assert data['data']['has_approved_public_version'] is False +@flask_required +class TestDeprecationChains: + def test_replacement_chain_is_validated_and_returned(self, client, auth_headers): + target = client.post( + "/api/v1/tools", headers=auth_headers, + json={"config": "name: replacement\nversion: 1.0.0\noutput: ok\n"}, + ) + assert target.status_code == 201 + response = client.post( + "/api/v1/tools/testuser/tool1/deprecate", + headers=auth_headers, + json={"deprecated_message": "Moved", "replacement": "replacement"}, + ) + assert response.status_code == 200 + assert response.get_json()["data"]["replacement_chain"] == [ + "testuser/replacement" + ] + + def test_replacement_cycle_is_rejected(self, client, auth_headers): + target = client.post( + "/api/v1/tools", headers=auth_headers, + json={"config": "name: cycle-target\nversion: 1.0.0\noutput: ok\n"}, + ) + assert target.status_code == 201 + assert client.post( + "/api/v1/tools/testuser/cycle-target/deprecate", + headers=auth_headers, + json={"replacement": "tool1"}, + ).status_code == 200 + response = client.post( + "/api/v1/tools/testuser/tool1/deprecate", + headers=auth_headers, + json={"replacement": "cycle-target"}, + ) + assert response.status_code == 400 + assert "cycle" in response.get_json()["error"]["message"] + + @flask_required class TestPublishPreflightEndpoint: def test_dry_run_includes_shared_preflight(self, client, auth_headers): @@ -180,7 +218,81 @@ class TestPublishPreflightEndpoint: assert response.status_code == 200 report = response.get_json()["data"]["preflight"] assert report["errors"] == [] - assert report["generated_tests"] == [] + assert report["generated_tests"][0]["state"] == "passed" + assert response.get_json()["data"]["quality"]["evidence_coverage"] > 0 + + def test_real_publish_persists_immutable_audit(self, client, auth_headers): + from cmdforge.registry.db import connect_db + + response = client.post( + "/api/v1/tools", + headers=auth_headers, + json={ + "config": ( + "name: audited-tool\nversion: 1.0.0\noutput: stable\n" + "input_schema: {}\noutput_schema:\n type: string\n" + ), + }, + ) + assert response.status_code == 201 + conn = connect_db() + try: + rows = conn.execute( + """ + SELECT tool_audits.trigger, tool_audits.evidence_json + FROM tool_audits JOIN tools ON tools.id = tool_audits.tool_id + WHERE tools.owner = 'testuser' AND tools.name = 'audited-tool' + """ + ).fetchall() + conn.execute( + """ + UPDATE tools SET moderation_status = 'approved' + WHERE owner = 'testuser' AND name = 'audited-tool' + """ + ) + conn.commit() + finally: + conn.close() + assert len(rows) == 1 + assert rows[0]["trigger"] == "publish" + assert "contract_conformance" in rows[0]["evidence_json"] + search = client.get("/api/v1/tools/search?q=audited") + result = search.get_json()["data"][0] + assert isinstance(result["quality_score"], int) + assert isinstance(result["quality_coverage"], int) + assert result["quality_evaluated_at"] + + def test_real_publish_blocks_deterministic_contract_failure( + self, client, auth_headers + ): + response = client.post( + "/api/v1/tools", + headers=auth_headers, + json={ + "config": ( + "name: broken-contract\nversion: 1.0.0\noutput: text\n" + "input_schema: {}\noutput_schema:\n type: integer\n" + ), + }, + ) + assert response.status_code == 400 + assert response.get_json()["error"]["code"] == "PREFLIGHT_FAILED" + + def test_real_publish_blocks_missing_registry_dependency( + self, client, auth_headers + ): + response = client.post( + "/api/v1/tools", + headers=auth_headers, + json={ + "config": ( + "name: missing-dependency\nversion: 1.0.0\n" + "dependencies:\n - absent-tool\n" + ), + }, + ) + assert response.status_code == 400 + assert "absent-tool" in response.get_json()["error"]["message"] def test_invalid_contract_is_rejected(self, client, auth_headers): response = client.post( diff --git a/tests/test_picker_m8.py b/tests/test_picker_m8.py new file mode 100644 index 0000000..cef8cc7 --- /dev/null +++ b/tests/test_picker_m8.py @@ -0,0 +1,91 @@ +"""Behavioral coverage for registry-aware picker and deprecation UX.""" + +from io import StringIO +import time +from unittest.mock import patch + +import cmdforge.cli.picker as picker +from cmdforge.cli.picker import PickerResult + + +class FakeTTY: + def __init__(self, *characters): + self.characters = iter(characters) + + def getch(self, timeout=None): + if timeout is not None: + time.sleep(0.01) + return None + return next(self.characters) + + +def setup_function(): + picker._registry_cache.clear() + picker._ui_out = StringIO() + + +def test_public_registry_search_does_not_require_token(): + class Client: + token = "" + + def semantic_search(self, query, limit): + return {"available": False, "data": [], "error": None} + + def search_tools(self, query, per_page): + return [{ + "owner": "official", "name": "weather", + "description": "Weather", "score": 0.8, + "quality_score": 91, "quality_coverage": 70, + }] + + with patch("cmdforge.registry_client.get_client", return_value=Client()): + results = picker.search_registry("weather") + assert results[0]["name"] == "official/weather" + assert results[0]["quality_score"] == 91 + assert results[0]["quality_coverage"] == 70 + + +def test_picker_can_install_when_no_local_tools(): + registry_tool = { + "name": "official/weather", "desc": "Weather", "args": [], + "registry": True, "quality_score": 91, "quality_coverage": 70, + "relevance": 0.8, + } + with patch.object(picker, "get_tools", return_value=[]), patch.object( + picker, "search_registry", return_value=[registry_tool] + ), patch.object( + picker, "_install_registry_selection", + return_value=PickerResult("official/weather", {}), + ) as install: + result = picker.run_picker(FakeTTY("w", "e", "\n")) + install.assert_called_once() + assert result.tool_name == "official/weather" + assert "Q91/70%" in picker._ui_out.getvalue() + + +def test_tab_installs_registry_selection_too(): + registry_tool = { + "name": "official/weather", "desc": "Weather", "args": [], + "registry": True, + } + with patch.object(picker, "get_tools", return_value=[]), patch.object( + picker, "search_registry", return_value=[registry_tool] + ), patch.object( + picker, "_install_registry_selection", + return_value=PickerResult("official/weather", {}), + ) as install: + result = picker.run_picker(FakeTTY("w", "e", "\t")) + install.assert_called_once() + assert result.tool_name == "official/weather" + + +def test_local_deprecation_selection_prints_migration_guidance(): + tool = { + "name": "old", "desc": "", "args": [], "deprecated": True, + "deprecated_message": "Moved.", "replacement": "official/new", + } + result = picker._local_selection(tool) + assert result.tool_name == "old" + output = picker._ui_out.getvalue() + assert "Moved." in output + assert "official/new" in output diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 509e878..519db5b 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -126,6 +126,71 @@ class TestAnalyzeTool: _check_similar_tools(SimpleNamespace(name="summarize"), report, Client()) assert report.similar_tools[0]["name"] == "official/summary" + def test_server_audit_does_not_claim_skipped_local_checks(self): + from cmdforge.tool import Tool, ToolStep + + report = analyze_tool( + Tool(name="server", steps=[ToolStep(tool="child", output_var="out")]), + check_local_dependencies=False, + ) + checks = report.audit_evidence["checks_run"] + assert "dependencies" not in checks + assert "toolstep_compatibility" not in checks + assert "reuse_opportunities" not in checks + assert report.compatibility == [] + + def test_unrelated_code_steps_are_not_reuse_evidence(self): + from cmdforge.tool import CodeStep, Tool + + current = Tool( + name="current", input_schema={}, output_schema={}, + steps=[ + CodeStep(code="x = 1", output_var="x"), + CodeStep(code="y = 2", output_var="y"), + ], + ) + other = Tool( + name="other", input_schema={}, output_schema={}, + steps=[ + CodeStep(code="a = 99", output_var="a"), + CodeStep(code="b = 100", output_var="b"), + ], + ) + with patch("cmdforge.tool.list_tools", return_value=["other"]), patch( + "cmdforge.tool.load_tool", return_value=other + ): + report = analyze_tool(current) + assert report.reuse_opportunities == [] + + def test_exact_contracted_duplicate_has_evidence(self): + from cmdforge.tool import PromptStep, Tool + + def make(name): + return Tool( + name=name, input_schema={}, output_schema={"type": "string"}, + steps=[ + PromptStep( + prompt="first", provider="mock", output_var="one", + output_schema={"type": "string"}, + ), + PromptStep( + prompt="second", provider="mock", output_var="two", + output_schema={"type": "string"}, + ), + ], + ) + + with patch("cmdforge.tool.list_tools", return_value=["other"]), patch( + "cmdforge.tool.load_tool", return_value=make("other") + ): + report = analyze_tool(make("current")) + duplicate = [ + item for item in report.reuse_opportunities + if item["type"] == "duplicate_sequence" + ] + assert duplicate + assert "explicit tool contracts" in duplicate[0]["evidence"] + def test_registry_error_becomes_warning(self): class Client: def search_tools(self, query, per_page): diff --git a/tests/test_quality.py b/tests/test_quality.py index defe60f..8acde15 100644 --- a/tests/test_quality.py +++ b/tests/test_quality.py @@ -22,6 +22,10 @@ class TestCategoryScore: cs = CategoryScore("X", 0, 0) assert cs.percentage == 0.0 + def test_display_distinguishes_missing_evidence(self): + assert CategoryScore("Tests", 0, 30, "not_tested").display == "not tested" + assert CategoryScore("Tests", 0, 30, "not_applicable").display == "n/a" + class TestQualityScore: def test_to_dict(self): @@ -55,6 +59,40 @@ class TestComputeQuality: # Contracts should be not_tested contracts = [c for c in qs.categories if c.name == "Contracts"][0] assert contracts.state == "not_tested" + assert qs.headline == 0 + assert qs.evidence_coverage == 0 + + def test_missing_categories_do_not_reduce_checked_score(self): + tool = Tool(name="contracted", input_schema={}, output_schema={}) + report = PreflightReport() + qs = compute_quality(tool, report) + assert qs.headline == 100 + assert qs.evidence_coverage == 15 + contracts = [c for c in qs.categories if c.name == "Contracts"][0] + assert contracts.state == "checked" + + def test_unsupported_tests_are_not_scored_as_failures(self): + report = PreflightReport(generated_tests=[{ + "step": "preflight", "state": "unsupported", "detail": "safe mode" + }]) + tests = [ + c for c in compute_quality(Tool(name="tool"), report).categories + if c.name == "Deterministic tests" + ][0] + assert tests.state == "not_tested" + assert tests.display == "not tested" + + def test_partial_unsupported_tests_reduce_coverage_not_score(self): + report = PreflightReport(generated_tests=[ + {"step": "case", "state": "passed", "detail": "ok"}, + {"step": "generation", "state": "unsupported", "detail": "unknown"}, + ]) + tests = [ + c for c in compute_quality(Tool(name="tool"), report).categories + if c.name == "Deterministic tests" + ][0] + assert tests.earned == tests.available == 15 + assert tests.possible == 30 def test_with_contracts_scores_higher(self): tool = Tool( @@ -127,6 +165,17 @@ class TestComputeQuality: security = [c for c in qs.categories if c.name == "Security scrutiny"][0] assert security.earned == 15 # 20 - 5 + def test_skipped_dependency_check_is_not_claimed_as_security_evidence(self): + tool = Tool(name="server") + report = PreflightReport(audit_evidence={"checks_run": ["secrets"]}) + security = [ + c for c in compute_quality(tool, report).categories + if c.name == "Security scrutiny" + ][0] + assert security.earned == 10 + assert security.available == 10 + assert security.possible == 20 + def test_community_not_tested_without_data(self): tool = Tool(name="new") report = PreflightReport() diff --git a/tests/test_registry_auditing.py b/tests/test_registry_auditing.py new file mode 100644 index 0000000..e8b88fc --- /dev/null +++ b/tests/test_registry_auditing.py @@ -0,0 +1,76 @@ +"""Tests for immutable registry audit refreshes.""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from cmdforge.registry.auditing import audit_stale_tools, audit_tool_version +from cmdforge.registry.db import connect_db, init_db + + +@pytest.fixture +def audit_db(tmp_path): + conn = connect_db(tmp_path / "registry.db") + init_db(conn) + cursor = conn.execute( + """ + INSERT INTO publishers (email, password_hash, slug, display_name) + VALUES ('audit@example.com', 'x', 'auditor', 'Auditor') + """ + ) + publisher_id = cursor.lastrowid + tool = conn.execute( + """ + INSERT INTO tools ( + owner, name, version, config_yaml, publisher_id, + visibility, moderation_status + ) VALUES ('auditor', 'checked', '1.0.0', ?, ?, 'public', 'approved') + """, + [ + "name: checked\nversion: 1.0.0\noutput: stable\n" + "input_schema: {}\noutput_schema:\n type: string\n", + publisher_id, + ], + ) + conn.commit() + yield conn, tool.lastrowid + conn.close() + + +def test_background_audits_append_instead_of_overwriting(audit_db): + conn, tool_id = audit_db + first = audit_tool_version(conn, tool_id) + second = audit_tool_version(conn, tool_id) + count = conn.execute( + "SELECT COUNT(*) FROM tool_audits WHERE tool_id = ?", [tool_id] + ).fetchone()[0] + assert count == 2 + assert first["evidence"]["findings_hash"] == second["evidence"]["findings_hash"] + assert "generated_tests" in first["evidence"]["findings"] + assert "dependencies" not in first["evidence"]["checks_run"] + + +def test_stale_audit_refresh_adds_new_evidence(audit_db): + conn, tool_id = audit_db + audit_tool_version(conn, tool_id) + old = (datetime.now(timezone.utc) - timedelta(days=60)).isoformat() + conn.execute("UPDATE tool_audits SET evaluated_at = ?", [old]) + conn.commit() + results = audit_stale_tools(conn, max_age_days=30) + assert [item["tool_id"] for item in results] == [tool_id] + assert conn.execute("SELECT COUNT(*) FROM tool_audits").fetchone()[0] == 2 + + +def test_degraded_background_score_is_logged(audit_db): + conn, tool_id = audit_db + conn.execute("UPDATE tools SET downloads = 500 WHERE id = ?", [tool_id]) + conn.commit() + audit_tool_version(conn, tool_id) + conn.execute("UPDATE tools SET downloads = 0 WHERE id = ?", [tool_id]) + conn.commit() + result = audit_tool_version(conn, tool_id) + assert result["quality_change"] < 0 + log = conn.execute( + "SELECT action FROM audit_log WHERE target_id = ?", [str(tool_id)] + ).fetchone() + assert log["action"] == "automated_audit_changed" diff --git a/tests/test_tool.py b/tests/test_tool.py index f3594ab..b2282b7 100644 --- a/tests/test_tool.py +++ b/tests/test_tool.py @@ -685,3 +685,30 @@ class TestAgentContext: ) restored = PromptStep.from_dict(step.to_dict()) assert restored.skills == ["python"] + + +class TestDeprecationFields: + def test_round_trip_and_warning(self): + from cmdforge.tool import deprecation_warning + + tool = Tool( + name="old", deprecated=True, deprecated_message="Moved.", + replacement="official/new", + ) + restored = Tool.from_dict(tool.to_dict()) + assert restored.deprecated is True + assert restored.replacement == "official/new" + assert "official/new" in deprecation_warning(restored) + + @pytest.mark.parametrize( + "values", + [ + {"deprecated": "false"}, + {"deprecated_message": 123}, + {"replacement": "../../escape"}, + {"replacement": "owner/name/extra"}, + ], + ) + def test_invalid_deprecation_metadata_is_rejected(self, values): + with pytest.raises(ValueError): + Tool(name="old", **values)