Harden M8 trust and guidance workflows
This commit is contained in:
parent
d954c1823a
commit
4d22d532a3
|
|
@ -79,10 +79,16 @@ Tools are YAML configs with:
|
||||||
- `output`: Template for final output (e.g., `"{response}"`)
|
- `output`: Template for final output (e.g., `"{response}"`)
|
||||||
- `input_schema`, `output_schema`: Optional tool-level JSON Schema contracts;
|
- `input_schema`, `output_schema`: Optional tool-level JSON Schema contracts;
|
||||||
schemas are validated when tools are loaded or created
|
schemas are validated when tools are loaded or created
|
||||||
|
- `deprecated`, `deprecated_message`, `replacement`: Validated migration metadata;
|
||||||
|
replacements may be local names or `owner/tool` registry references
|
||||||
|
|
||||||
Run `cmdforge inspect <tool> [--registry]` for the shared local preflight
|
Run `cmdforge inspect <tool> [--registry]` for the shared local preflight
|
||||||
report. `cmdforge registry publish <path> --dry-run` runs local checks first
|
report. `cmdforge registry publish <path> --dry-run` runs local checks first
|
||||||
and, when authenticated, the registry's publish-time checks without publishing.
|
and, when authenticated, the registry's publish-time checks without publishing.
|
||||||
|
Normal publication uses the same checks as Stage 1, asks for confirmation in an
|
||||||
|
interactive terminal, and then performs the Stage 2 mutation. The registry
|
||||||
|
reruns deterministic conformance authoritatively and stores immutable audit and
|
||||||
|
quality evidence for the published version.
|
||||||
Inspect also shows inferred contract proposals and runs deterministic contract
|
Inspect also shows inferred contract proposals and runs deterministic contract
|
||||||
conformance when both tool schemas are explicit. Prompt outputs are synthesized
|
conformance when both tool schemas are explicit. Prompt outputs are synthesized
|
||||||
from their schemas; code, nested-tool, and MCP steps are reported as unsupported
|
from their schemas; code, nested-tool, and MCP steps are reported as unsupported
|
||||||
|
|
@ -92,6 +98,8 @@ evidence to `conformance.json`. Later inspections and local publish dry-runs
|
||||||
rerun the stored synthetic inputs and report state, coverage, contract, and
|
rerun the stored synthetic inputs and report state, coverage, contract, and
|
||||||
normalized output changes. ToolStep preflight also compares the values the
|
normalized output changes. ToolStep preflight also compares the values the
|
||||||
runner supplies with the called tool's input contract.
|
runner supplies with the called tool's input contract.
|
||||||
|
Quality headlines exclude categories with missing evidence and are accompanied
|
||||||
|
by evidence coverage plus checked/not-tested category states.
|
||||||
|
|
||||||
### Step Types
|
### Step Types
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -144,7 +144,7 @@ cmdforge docs mytool # View/create tool documentation
|
||||||
# Registry
|
# Registry
|
||||||
cmdforge registry search "keyword" # Search for tools
|
cmdforge registry search "keyword" # Search for tools
|
||||||
cmdforge registry install owner/tool # Install a tool
|
cmdforge registry install owner/tool # Install a tool
|
||||||
cmdforge registry publish mytool # Publish your tool
|
cmdforge registry publish mytool # Preflight, confirm, then publish
|
||||||
cmdforge registry publish mytool --dry-run # Preflight without publishing
|
cmdforge registry publish mytool --dry-run # Preflight without publishing
|
||||||
cmdforge registry status # Check moderation status
|
cmdforge registry status # Check moderation status
|
||||||
cmdforge registry my-tools # List your published tools
|
cmdforge registry my-tools # List your published tools
|
||||||
|
|
@ -180,6 +180,9 @@ cmdforge config connect username # Connect to registry account
|
||||||
cmdforge config disconnect # Disconnect from registry
|
cmdforge config disconnect # Disconnect from registry
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`cf` searches the public registry when no local tool matches. Registry results
|
||||||
|
show available relevance and quality evidence and install on selection.
|
||||||
|
|
||||||
### Running Tools
|
### Running Tools
|
||||||
|
|
||||||
Once created, tools work like any Unix command:
|
Once created, tools work like any Unix command:
|
||||||
|
|
|
||||||
|
|
@ -648,7 +648,10 @@ def cmd_inspect(args):
|
||||||
# Quality score
|
# Quality score
|
||||||
from ..quality import compute_quality
|
from ..quality import compute_quality
|
||||||
qs = compute_quality(tool, report)
|
qs = compute_quality(tool, report)
|
||||||
print(f"Quality {qs.headline} — Evaluated {qs.last_evaluated[:10]}")
|
print(
|
||||||
|
f"Quality {qs.headline} — {qs.evidence_coverage}% evidence coverage — "
|
||||||
|
f"Evaluated {qs.last_evaluated[:10]}"
|
||||||
|
)
|
||||||
for cat in qs.categories:
|
for cat in qs.categories:
|
||||||
print(f" {cat.name:24s} {cat.display:>8s}")
|
print(f" {cat.name:24s} {cat.display:>8s}")
|
||||||
print()
|
print()
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,10 @@
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
|
import select
|
||||||
import tty
|
import tty
|
||||||
import termios
|
import termios
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from typing import List, Tuple, Optional
|
from typing import List, Tuple, Optional
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
@ -30,6 +32,7 @@ GREEN = "\033[32m"
|
||||||
RESET = "\033[0m"
|
RESET = "\033[0m"
|
||||||
|
|
||||||
MAX_VISIBLE = 8 # Show at most 8 items
|
MAX_VISIBLE = 8 # Show at most 8 items
|
||||||
|
_registry_cache = {}
|
||||||
|
|
||||||
# Output stream for UI (stderr when stdout is piped, stdout otherwise)
|
# Output stream for UI (stderr when stdout is piped, stdout otherwise)
|
||||||
_ui_out = None
|
_ui_out = None
|
||||||
|
|
@ -90,31 +93,60 @@ def search_registry(query: str) -> List[dict]:
|
||||||
"""Search the registry for tools matching the query.
|
"""Search the registry for tools matching the query.
|
||||||
|
|
||||||
Returns a list of dicts with name, desc, and registry metadata.
|
Returns a list of dicts with name, desc, and registry metadata.
|
||||||
Returns empty list if no token configured or search fails.
|
Registry search is public; authentication is not required.
|
||||||
"""
|
"""
|
||||||
if not query or len(query) < 2:
|
if not query or len(query) < 2:
|
||||||
return []
|
return []
|
||||||
|
normalized = query.strip().lower()
|
||||||
|
if normalized in _registry_cache:
|
||||||
|
return [dict(item) for item in _registry_cache[normalized]]
|
||||||
try:
|
try:
|
||||||
from ..registry_client import get_client
|
from ..registry_client import RegistryError, get_client
|
||||||
client = get_client()
|
client = get_client()
|
||||||
if not client or not client.token:
|
client.timeout = min(getattr(client, "timeout", 3), 3)
|
||||||
return []
|
client.max_retries = 1
|
||||||
|
semantic = client.semantic_search(query, limit=5)
|
||||||
|
if semantic.get("available") and semantic.get("data"):
|
||||||
|
items = semantic["data"]
|
||||||
|
else:
|
||||||
results = client.search_tools(query, per_page=5)
|
results = client.search_tools(query, per_page=5)
|
||||||
items = results.data if hasattr(results, "data") else results
|
items = results.data if hasattr(results, "data") else results
|
||||||
return [
|
found = [
|
||||||
{
|
{
|
||||||
"name": f"{item.get('owner', '')}/{item.get('name', '')}",
|
"name": f"{item.get('owner', '')}/{item.get('name', '')}",
|
||||||
"desc": (item.get("description") or "")[:50],
|
"desc": (item.get("description") or "")[:50],
|
||||||
"args": [],
|
"args": [],
|
||||||
"registry": True,
|
"registry": True,
|
||||||
"downloads": item.get("downloads", 0),
|
"downloads": item.get("downloads", 0),
|
||||||
|
"relevance": item.get("similarity", item.get("score")),
|
||||||
|
"quality_score": item.get("quality_score"),
|
||||||
|
"quality_coverage": item.get("quality_coverage"),
|
||||||
}
|
}
|
||||||
for item in items
|
for item in items
|
||||||
]
|
]
|
||||||
except Exception:
|
_registry_cache[normalized] = found
|
||||||
|
return [dict(item) for item in found]
|
||||||
|
except RegistryError:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _install_registry_selection(tool: dict) -> PickerResult:
|
||||||
|
"""Install a registry result and retain its qualified identity for execution."""
|
||||||
|
from ..resolver import install_from_registry
|
||||||
|
|
||||||
|
install_from_registry(tool["name"])
|
||||||
|
return PickerResult(tool["name"], {})
|
||||||
|
|
||||||
|
|
||||||
|
def _local_selection(tool: dict, arguments: Optional[dict] = None) -> PickerResult:
|
||||||
|
if tool.get("deprecated"):
|
||||||
|
guidance = tool.get("deprecated_message") or "This tool is no longer maintained."
|
||||||
|
if tool.get("replacement"):
|
||||||
|
guidance += f" Use '{tool['replacement']}' instead."
|
||||||
|
_write(f"{YELLOW}Warning: {guidance}{RESET}\n")
|
||||||
|
return PickerResult(tool["name"], arguments or {})
|
||||||
|
|
||||||
|
|
||||||
class TTYInput:
|
class TTYInput:
|
||||||
"""Read from /dev/tty for keyboard input, even when stdin is piped."""
|
"""Read from /dev/tty for keyboard input, even when stdin is piped."""
|
||||||
|
|
||||||
|
|
@ -135,10 +167,14 @@ class TTYInput:
|
||||||
if self.tty:
|
if self.tty:
|
||||||
self.tty.close()
|
self.tty.close()
|
||||||
|
|
||||||
def getch(self):
|
def getch(self, timeout: Optional[float] = None):
|
||||||
"""Read a single character."""
|
"""Read a single character."""
|
||||||
tty.setraw(self.fd)
|
tty.setraw(self.fd)
|
||||||
try:
|
try:
|
||||||
|
if timeout is not None:
|
||||||
|
ready, _, _ = select.select([self.tty], [], [], timeout)
|
||||||
|
if not ready:
|
||||||
|
return None
|
||||||
ch = self.tty.read(1)
|
ch = self.tty.read(1)
|
||||||
if ch == '\x1b':
|
if ch == '\x1b':
|
||||||
# Read escape sequence - bytes arrive together from terminal
|
# Read escape sequence - bytes arrive together from terminal
|
||||||
|
|
@ -164,19 +200,25 @@ def clear_dropdown(n_lines: int):
|
||||||
def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
|
def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
|
||||||
"""Run inline picker."""
|
"""Run inline picker."""
|
||||||
tools = get_tools()
|
tools = get_tools()
|
||||||
if not tools:
|
|
||||||
_write("No tools. Create one: cmdforge\n")
|
|
||||||
return None
|
|
||||||
|
|
||||||
query = ""
|
query = ""
|
||||||
selected = 0
|
selected = 0
|
||||||
scroll = 0
|
scroll = 0
|
||||||
last_drawn = 0
|
last_drawn = 0
|
||||||
|
registry_results = {}
|
||||||
|
registry_future = None
|
||||||
|
registry_future_query = None
|
||||||
|
registry_executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
|
||||||
_write(HIDE_CURSOR)
|
_write(HIDE_CURSOR)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
|
if registry_future is not None and registry_future.done():
|
||||||
|
registry_results[registry_future_query] = registry_future.result()
|
||||||
|
registry_future = None
|
||||||
|
registry_future_query = None
|
||||||
|
|
||||||
# Filter
|
# Filter
|
||||||
matches = []
|
matches = []
|
||||||
for t in tools:
|
for t in tools:
|
||||||
|
|
@ -190,12 +232,17 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
|
||||||
filtered = [m[0] for m in matches]
|
filtered = [m[0] for m in matches]
|
||||||
|
|
||||||
# If no local matches, search registry
|
# If no local matches, search registry
|
||||||
registry_results = []
|
|
||||||
if query and len(query) >= 2 and not filtered:
|
if query and len(query) >= 2 and not filtered:
|
||||||
registry_results = search_registry(query)
|
normalized_query = query.strip().lower()
|
||||||
for rt in registry_results:
|
found = registry_results.get(normalized_query, [])
|
||||||
|
if registry_future is None and normalized_query not in registry_results:
|
||||||
|
registry_future_query = normalized_query
|
||||||
|
registry_future = registry_executor.submit(
|
||||||
|
search_registry, query
|
||||||
|
)
|
||||||
|
for rt in found:
|
||||||
rt["_registry"] = True
|
rt["_registry"] = True
|
||||||
filtered = registry_results
|
filtered = found
|
||||||
|
|
||||||
if selected >= len(filtered):
|
if selected >= len(filtered):
|
||||||
selected = max(0, len(filtered) - 1)
|
selected = max(0, len(filtered) - 1)
|
||||||
|
|
@ -232,6 +279,12 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
|
||||||
# Add registry marker
|
# Add registry marker
|
||||||
if is_registry:
|
if is_registry:
|
||||||
prefix += f" {YELLOW}[registry]{RESET}"
|
prefix += f" {YELLOW}[registry]{RESET}"
|
||||||
|
if t.get("quality_score") is not None:
|
||||||
|
prefix += f" Q{t['quality_score']}"
|
||||||
|
if t.get("quality_coverage") is not None:
|
||||||
|
prefix += f"/{t['quality_coverage']}%"
|
||||||
|
if t.get("relevance") is not None:
|
||||||
|
prefix += f" R{float(t['relevance']):.2f}"
|
||||||
# Add deprecation marker
|
# Add deprecation marker
|
||||||
if is_deprecated:
|
if is_deprecated:
|
||||||
prefix += f" {YELLOW}[deprecated]{RESET}"
|
prefix += f" {YELLOW}[deprecated]{RESET}"
|
||||||
|
|
@ -251,7 +304,11 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
|
||||||
last_drawn = len(lines)
|
last_drawn = len(lines)
|
||||||
|
|
||||||
# Input
|
# Input
|
||||||
ch = tty_input.getch()
|
ch = tty_input.getch(
|
||||||
|
timeout=0.1 if registry_future is not None else None
|
||||||
|
)
|
||||||
|
if ch is None:
|
||||||
|
continue
|
||||||
|
|
||||||
if ch in ('\r', '\n'): # Enter - run
|
if ch in ('\r', '\n'): # Enter - run
|
||||||
if filtered:
|
if filtered:
|
||||||
|
|
@ -261,29 +318,32 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
|
||||||
# If it's a registry tool, install it first
|
# If it's a registry tool, install it first
|
||||||
if selected_tool.get("_registry") or selected_tool.get("registry"):
|
if selected_tool.get("_registry") or selected_tool.get("registry"):
|
||||||
_write(f"{YELLOW}Installing {selected_tool['name']}...{RESET}\n")
|
_write(f"{YELLOW}Installing {selected_tool['name']}...{RESET}\n")
|
||||||
import subprocess
|
try:
|
||||||
proc = subprocess.run(
|
return _install_registry_selection(selected_tool)
|
||||||
["cmdforge", "registry", "install", selected_tool["name"]],
|
except Exception as exc:
|
||||||
capture_output=True, text=True
|
_write(f"Install failed: {exc}\n")
|
||||||
)
|
|
||||||
if proc.returncode != 0:
|
|
||||||
_write(f"Install failed: {proc.stderr}\n")
|
|
||||||
return None
|
return None
|
||||||
# Extract tool name from "owner/name" format
|
return _local_selection(selected_tool)
|
||||||
tool_name = selected_tool["name"].split("/")[-1]
|
|
||||||
return PickerResult(tool_name, {})
|
|
||||||
return PickerResult(selected_tool["name"], {})
|
|
||||||
|
|
||||||
elif ch == '\t': # Tab - configure args or run
|
elif ch == '\t': # Tab - configure args or run
|
||||||
if filtered:
|
if filtered:
|
||||||
tool = filtered[selected]
|
tool = filtered[selected]
|
||||||
clear_dropdown(last_drawn)
|
clear_dropdown(last_drawn)
|
||||||
|
if tool.get("_registry") or tool.get("registry"):
|
||||||
|
_write(f"{YELLOW}Installing {tool['name']}...{RESET}\n")
|
||||||
|
try:
|
||||||
|
_write(SHOW_CURSOR)
|
||||||
|
return _install_registry_selection(tool)
|
||||||
|
except Exception as exc:
|
||||||
|
_write(f"Install failed: {exc}\n")
|
||||||
|
_write(SHOW_CURSOR)
|
||||||
|
return None
|
||||||
if tool['args']:
|
if tool['args']:
|
||||||
args = pick_args(tty_input, tool)
|
args = pick_args(tty_input, tool)
|
||||||
_write(SHOW_CURSOR)
|
_write(SHOW_CURSOR)
|
||||||
return PickerResult(tool["name"], args) if args is not None else None
|
return _local_selection(tool, args) if args is not None else None
|
||||||
_write(SHOW_CURSOR)
|
_write(SHOW_CURSOR)
|
||||||
return PickerResult(tool["name"], {})
|
return _local_selection(tool)
|
||||||
|
|
||||||
elif ch == '\x1b' or ch == '\x03': # Esc or Ctrl+C
|
elif ch == '\x1b' or ch == '\x03': # Esc or Ctrl+C
|
||||||
clear_dropdown(last_drawn)
|
clear_dropdown(last_drawn)
|
||||||
|
|
@ -308,6 +368,8 @@ def run_picker(tty_input: TTYInput) -> Optional[PickerResult]:
|
||||||
except Exception:
|
except Exception:
|
||||||
_write(SHOW_CURSOR)
|
_write(SHOW_CURSOR)
|
||||||
raise
|
raise
|
||||||
|
finally:
|
||||||
|
registry_executor.shutdown(wait=False, cancel_futures=True)
|
||||||
|
|
||||||
|
|
||||||
def pick_args(tty_input: TTYInput, tool: dict) -> Optional[dict]:
|
def pick_args(tty_input: TTYInput, tool: dict) -> Optional[dict]:
|
||||||
|
|
|
||||||
|
|
@ -410,6 +410,22 @@ def _print_preflight_sections(report: dict, prefix: str = "") -> None:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _print_quality_summary(quality: dict | None) -> None:
|
||||||
|
if not quality:
|
||||||
|
return
|
||||||
|
print(
|
||||||
|
f"Quality {quality.get('score', 0)} — "
|
||||||
|
f"{quality.get('evidence_coverage', 0)}% evidence coverage"
|
||||||
|
)
|
||||||
|
for category in quality.get("categories", []):
|
||||||
|
state = category.get("state", "not_tested")
|
||||||
|
value = (
|
||||||
|
f"{category.get('earned', 0)}/{category.get('available', 0)}"
|
||||||
|
if state == "checked" else state.replace("_", " ")
|
||||||
|
)
|
||||||
|
print(f" {category.get('name', 'Unknown'):24s} {value:>12s}")
|
||||||
|
|
||||||
|
|
||||||
def _print_registry_suggestions(suggestions: dict) -> None:
|
def _print_registry_suggestions(suggestions: dict) -> None:
|
||||||
"""Render registry-specific category, similarity, and scrutiny evidence."""
|
"""Render registry-specific category, similarity, and scrutiny evidence."""
|
||||||
category = suggestions.get("category") or {}
|
category = suggestions.get("category") or {}
|
||||||
|
|
@ -547,6 +563,7 @@ def _cmd_registry_publish(args):
|
||||||
|
|
||||||
remote_report = result.get("preflight") or {}
|
remote_report = result.get("preflight") or {}
|
||||||
_print_preflight_sections(remote_report, prefix="Registry ")
|
_print_preflight_sections(remote_report, prefix="Registry ")
|
||||||
|
_print_quality_summary(result.get("quality"))
|
||||||
_print_registry_suggestions(result.get("suggestions") or {})
|
_print_registry_suggestions(result.get("suggestions") or {})
|
||||||
if not remote_report.get("errors"):
|
if not remote_report.get("errors"):
|
||||||
print("Registry preflight passed.")
|
print("Registry preflight passed.")
|
||||||
|
|
@ -691,6 +708,44 @@ def _cmd_registry_publish(args):
|
||||||
except Exception:
|
except Exception:
|
||||||
pass # Continue with publish
|
pass # Continue with publish
|
||||||
|
|
||||||
|
# Stage 1 runs after dependency publication and any automatic version bump,
|
||||||
|
# so the evidence describes the exact version Stage 2 will mutate.
|
||||||
|
from ..preflight import analyze_tool
|
||||||
|
try:
|
||||||
|
preflight_tool = Tool.from_dict(data)
|
||||||
|
preflight_tool.path = config_path
|
||||||
|
local_report = analyze_tool(preflight_tool, include_contract_tests=True)
|
||||||
|
except (KeyError, TypeError, ValueError) as exc:
|
||||||
|
print(f"Local preflight error: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print("Preflight stage:")
|
||||||
|
_print_preflight_sections(local_report.to_dict(), prefix="Local ")
|
||||||
|
if not local_report.ok:
|
||||||
|
return 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
preflight_client = get_client()
|
||||||
|
preflight_result = preflight_client.publish_tool(
|
||||||
|
config_yaml, readme=readme, defaults=defaults,
|
||||||
|
owner=getattr(args, "owner", ""), dry_run=True,
|
||||||
|
)
|
||||||
|
except RegistryError as exc:
|
||||||
|
print(f"Registry preflight failed: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
remote_report = preflight_result.get("preflight") or {}
|
||||||
|
_print_preflight_sections(remote_report, prefix="Registry ")
|
||||||
|
_print_quality_summary(preflight_result.get("quality"))
|
||||||
|
if remote_report.get("errors"):
|
||||||
|
return 1
|
||||||
|
if sys.stdin.isatty() and not getattr(args, "force", False):
|
||||||
|
try:
|
||||||
|
if input("Publish this validated version? [y/N] ").strip().lower() != "y":
|
||||||
|
print("Cancelled.")
|
||||||
|
return 1
|
||||||
|
except (EOFError, KeyboardInterrupt):
|
||||||
|
print("\nCancelled.")
|
||||||
|
return 1
|
||||||
|
|
||||||
print(f"Publishing {name}@{version}...")
|
print(f"Publishing {name}@{version}...")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -187,6 +187,10 @@ def cmd_run(args):
|
||||||
if not tool:
|
if not tool:
|
||||||
print(f"Error: Tool '{args.name}' not found.", file=sys.stderr)
|
print(f"Error: Tool '{args.name}' not found.", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
|
from ..tool import deprecation_warning
|
||||||
|
warning = deprecation_warning(tool)
|
||||||
|
if warning:
|
||||||
|
print(warning, file=sys.stderr)
|
||||||
|
|
||||||
tool_args = list(args.tool_args or [])
|
tool_args = list(args.tool_args or [])
|
||||||
if tool_args and tool_args[0] == '--':
|
if tool_args and tool_args[0] == '--':
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
"""Registry page - browse and install tools from registry."""
|
"""Registry page - browse and install tools from registry."""
|
||||||
|
|
||||||
|
import html
|
||||||
from PySide6.QtWidgets import (
|
from PySide6.QtWidgets import (
|
||||||
QWidget, QVBoxLayout, QHBoxLayout, QLineEdit,
|
QWidget, QVBoxLayout, QHBoxLayout, QLineEdit,
|
||||||
QPushButton, QTableWidget, QTableWidgetItem, QLabel,
|
QPushButton, QTableWidget, QTableWidgetItem, QLabel,
|
||||||
|
|
@ -877,6 +878,16 @@ class RegistryPage(QWidget):
|
||||||
|
|
||||||
if tool.get("description"):
|
if tool.get("description"):
|
||||||
lines.append(f"<p style='color: #4a5568;'>{tool.get('description')}</p>")
|
lines.append(f"<p style='color: #4a5568;'>{tool.get('description')}</p>")
|
||||||
|
if tool.get("deprecated"):
|
||||||
|
guidance = html.escape(
|
||||||
|
tool.get("deprecated_message") or "This tool is no longer maintained."
|
||||||
|
)
|
||||||
|
if tool.get("replacement"):
|
||||||
|
guidance += f" Use <strong>{html.escape(tool['replacement'])}</strong> instead."
|
||||||
|
lines.append(
|
||||||
|
"<p style='background: #fef3c7; color: #92400e; padding: 8px 10px; "
|
||||||
|
f"border-radius: 4px;'><strong>Deprecated:</strong> {guidance}</p>"
|
||||||
|
)
|
||||||
|
|
||||||
# Rating display
|
# Rating display
|
||||||
rating = tool.get("average_rating", 0) or 0
|
rating = tool.get("average_rating", 0) or 0
|
||||||
|
|
|
||||||
|
|
@ -1161,18 +1161,57 @@ class ToolBuilderPage(QWidget):
|
||||||
"It will not be applied automatically."
|
"It will not be applied automatically."
|
||||||
)
|
)
|
||||||
msg.setInformativeText(
|
msg.setInformativeText(
|
||||||
"Review the proposal below. Contract editing and approval "
|
"Review the proposal below. Applying it changes only this "
|
||||||
"will be added by the guided-creation milestone."
|
"in-memory draft and remains reversible until you save."
|
||||||
)
|
)
|
||||||
msg.setDetailedText(
|
msg.setDetailedText(
|
||||||
yaml.safe_dump(proposed_schemas, sort_keys=False).rstrip()
|
yaml.safe_dump(proposed_schemas, sort_keys=False).rstrip()
|
||||||
)
|
)
|
||||||
|
apply_button = msg.addButton(
|
||||||
|
"Apply Proposal", QMessageBox.AcceptRole
|
||||||
|
)
|
||||||
save_button = msg.addButton(
|
save_button = msg.addButton(
|
||||||
"Save Without Applying", QMessageBox.AcceptRole
|
"Save Without Applying", QMessageBox.AcceptRole
|
||||||
)
|
)
|
||||||
msg.addButton(QMessageBox.Cancel)
|
msg.addButton(QMessageBox.Cancel)
|
||||||
msg.exec()
|
msg.exec()
|
||||||
if msg.clickedButton() is not save_button:
|
if msg.clickedButton() is apply_button:
|
||||||
|
if proposed_schemas.get("input_schema") is not None:
|
||||||
|
tool.input_schema = proposed_schemas["input_schema"]
|
||||||
|
if proposed_schemas.get("output_schema") is not None:
|
||||||
|
tool.output_schema = proposed_schemas["output_schema"]
|
||||||
|
elif msg.clickedButton() is not save_button:
|
||||||
|
return
|
||||||
|
|
||||||
|
if preflight.reuse_opportunities:
|
||||||
|
reuse = QMessageBox(self)
|
||||||
|
reuse.setIcon(QMessageBox.Information)
|
||||||
|
reuse.setWindowTitle("Reuse Opportunities")
|
||||||
|
reuse.setText(
|
||||||
|
"CmdForge found exact, contract-backed reuse evidence. "
|
||||||
|
"This is advisory and will not change your draft."
|
||||||
|
)
|
||||||
|
reuse.setDetailedText("\n".join(
|
||||||
|
f"- {item['detail']}\n Evidence: {item.get('evidence', 'exact match')}"
|
||||||
|
for item in preflight.reuse_opportunities
|
||||||
|
))
|
||||||
|
continue_button = reuse.addButton(
|
||||||
|
"Continue With Draft", QMessageBox.AcceptRole
|
||||||
|
)
|
||||||
|
matching = next(
|
||||||
|
(item.get("tool") for item in preflight.reuse_opportunities if item.get("tool")),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
open_button = (
|
||||||
|
reuse.addButton("Open Existing", QMessageBox.ActionRole)
|
||||||
|
if matching else None
|
||||||
|
)
|
||||||
|
reuse.addButton(QMessageBox.Cancel)
|
||||||
|
reuse.exec()
|
||||||
|
if open_button and reuse.clickedButton() is open_button:
|
||||||
|
_switch_to_existing_tool(self.main_window, matching)
|
||||||
|
return
|
||||||
|
if reuse.clickedButton() is not continue_button:
|
||||||
return
|
return
|
||||||
advisory_count = len(preflight.warnings) + len(preflight.suggestions)
|
advisory_count = len(preflight.warnings) + len(preflight.suggestions)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
"""Tools page - main view for managing tools."""
|
"""Tools page - main view for managing tools."""
|
||||||
|
|
||||||
|
import html
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional, Tuple, Dict, Any
|
from typing import Optional, Tuple, Dict, Any
|
||||||
|
|
@ -852,6 +853,16 @@ class ToolsPage(QWidget):
|
||||||
lines.append(f"<h2 style='margin: 0 0 8px 0; color: #2d3748;'>{tool.name}</h2>")
|
lines.append(f"<h2 style='margin: 0 0 8px 0; color: #2d3748;'>{tool.name}</h2>")
|
||||||
if tool.description:
|
if tool.description:
|
||||||
lines.append(f"<p style='color: #4a5568; margin-bottom: 16px;'>{tool.description}</p>")
|
lines.append(f"<p style='color: #4a5568; margin-bottom: 16px;'>{tool.description}</p>")
|
||||||
|
if tool.deprecated:
|
||||||
|
guidance = html.escape(
|
||||||
|
tool.deprecated_message or "This tool is no longer maintained."
|
||||||
|
)
|
||||||
|
if tool.replacement:
|
||||||
|
guidance += f" Use <strong>{html.escape(tool.replacement)}</strong> instead."
|
||||||
|
lines.append(
|
||||||
|
"<p style='background: #fef3c7; color: #92400e; padding: 8px 10px; "
|
||||||
|
f"border-radius: 4px;'><strong>Deprecated:</strong> {guidance}</p>"
|
||||||
|
)
|
||||||
|
|
||||||
# Publish state
|
# Publish state
|
||||||
state, registry_hash = get_tool_publish_state(qname)
|
state, registry_hash = get_tool_publish_state(qname)
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,8 @@ All checks are deterministic and evidence-based.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
import re
|
import re
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
@ -26,9 +28,6 @@ class PreflightReport:
|
||||||
regression: Optional[Dict[str, Any]] = None
|
regression: Optional[Dict[str, Any]] = None
|
||||||
reuse_opportunities: List[Dict[str, Any]] = field(default_factory=list)
|
reuse_opportunities: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
audit_evidence: Optional[Dict[str, Any]] = None
|
audit_evidence: Optional[Dict[str, Any]] = None
|
||||||
compatibility: List[Dict[str, Any]] = field(default_factory=list)
|
|
||||||
contract_proposal: Optional[Dict[str, Any]] = None
|
|
||||||
regression: Optional[Dict[str, Any]] = None
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def ok(self) -> bool:
|
def ok(self) -> bool:
|
||||||
|
|
@ -69,6 +68,7 @@ def analyze_tool(
|
||||||
*,
|
*,
|
||||||
check_local_dependencies: bool = True,
|
check_local_dependencies: bool = True,
|
||||||
include_contract_tests: bool = False,
|
include_contract_tests: bool = False,
|
||||||
|
check_reuse: Optional[bool] = None,
|
||||||
) -> PreflightReport:
|
) -> PreflightReport:
|
||||||
"""Run all preflight checks on a tool.
|
"""Run all preflight checks on a tool.
|
||||||
|
|
||||||
|
|
@ -80,17 +80,27 @@ def analyze_tool(
|
||||||
A PreflightReport with structured findings.
|
A PreflightReport with structured findings.
|
||||||
"""
|
"""
|
||||||
report = PreflightReport()
|
report = PreflightReport()
|
||||||
|
checks_run = ["config_integrity", "contracts"]
|
||||||
_check_config_integrity(tool, report)
|
_check_config_integrity(tool, report)
|
||||||
_check_contracts(tool, report)
|
_check_contracts(tool, report)
|
||||||
_add_contract_guidance(tool, report, include_contract_tests)
|
_add_contract_guidance(tool, report, include_contract_tests)
|
||||||
|
if include_contract_tests:
|
||||||
|
checks_run.append("contract_conformance")
|
||||||
_check_secrets(tool, report)
|
_check_secrets(tool, report)
|
||||||
|
checks_run.append("secrets")
|
||||||
if check_local_dependencies:
|
if check_local_dependencies:
|
||||||
_check_toolstep_compatibility(tool, report)
|
_check_toolstep_compatibility(tool, report)
|
||||||
_check_dependencies(tool, report)
|
_check_dependencies(tool, report)
|
||||||
_check_reuse_opportunities(tool, report)
|
checks_run.extend(["toolstep_compatibility", "dependencies"])
|
||||||
_add_audit_evidence(tool, report)
|
|
||||||
if registry_client:
|
if registry_client:
|
||||||
_check_similar_tools(tool, report, registry_client)
|
_check_similar_tools(tool, report, registry_client)
|
||||||
|
checks_run.append("registry_similarity")
|
||||||
|
if check_reuse is None:
|
||||||
|
check_reuse = check_local_dependencies
|
||||||
|
if check_reuse:
|
||||||
|
_check_reuse_opportunities(tool, report)
|
||||||
|
checks_run.append("reuse_opportunities")
|
||||||
|
_add_audit_evidence(tool, report, checks_run)
|
||||||
return report
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -372,21 +382,14 @@ def _is_semver(version: str) -> bool:
|
||||||
|
|
||||||
|
|
||||||
def _check_reuse_opportunities(tool: Tool, report: PreflightReport):
|
def _check_reuse_opportunities(tool: Tool, report: PreflightReport):
|
||||||
"""Detect repeated step patterns that could be extracted into reusable tools."""
|
"""Report only exact repeated sequences with contract-boundary evidence."""
|
||||||
from .tool import list_tools, load_tool
|
from .tool import list_tools, load_tool
|
||||||
|
|
||||||
if len(tool.steps) < 2:
|
if len(tool.steps) < 2:
|
||||||
return
|
return
|
||||||
|
|
||||||
# Build a signature of each step for comparison
|
step_signatures = [_step_signature(step) for step in tool.steps]
|
||||||
step_signatures = []
|
_find_internal_reuse(tool, step_signatures, report)
|
||||||
for step in tool.steps:
|
|
||||||
sig = _step_signature(step)
|
|
||||||
if sig:
|
|
||||||
step_signatures.append(sig)
|
|
||||||
|
|
||||||
if not step_signatures:
|
|
||||||
return
|
|
||||||
|
|
||||||
# Check if this step sequence appears in other local tools
|
# Check if this step sequence appears in other local tools
|
||||||
our_sequence = tuple(step_signatures)
|
our_sequence = tuple(step_signatures)
|
||||||
|
|
@ -398,55 +401,92 @@ def _check_reuse_opportunities(tool: Tool, report: PreflightReport):
|
||||||
continue
|
continue
|
||||||
other_sigs = [_step_signature(s) for s in other.steps]
|
other_sigs = [_step_signature(s) for s in other.steps]
|
||||||
other_sigs = [s for s in other_sigs if s]
|
other_sigs = [s for s in other_sigs if s]
|
||||||
if our_sequence == tuple(other_sigs):
|
if (
|
||||||
|
our_sequence == tuple(other_sigs)
|
||||||
|
and _has_tool_contracts(tool)
|
||||||
|
and _has_tool_contracts(other)
|
||||||
|
):
|
||||||
report.reuse_opportunities.append({
|
report.reuse_opportunities.append({
|
||||||
"type": "duplicate_sequence",
|
"type": "duplicate_sequence",
|
||||||
"tool": name,
|
"tool": name,
|
||||||
"detail": f"Step sequence identical to '{name}' — consider extracting shared logic",
|
"length": len(our_sequence),
|
||||||
|
"evidence": "exact normalized steps and explicit tool contracts",
|
||||||
|
"detail": f"Exact contracted step sequence also exists in '{name}'",
|
||||||
})
|
})
|
||||||
elif len(our_sequence) >= 2:
|
|
||||||
# Check for overlapping subsequences
|
|
||||||
for start in range(len(other_sigs) - 1):
|
|
||||||
for length in range(2, min(len(our_sequence), len(other_sigs) - start) + 1):
|
|
||||||
if our_sequence[:length] == tuple(other_sigs[start:start + length]):
|
|
||||||
report.reuse_opportunities.append({
|
|
||||||
"type": "shared_subsequence",
|
|
||||||
"tool": name,
|
|
||||||
"length": length,
|
|
||||||
"detail": f"Shares {length} step(s) with '{name}' — possible extraction candidate",
|
|
||||||
})
|
|
||||||
break
|
|
||||||
|
|
||||||
|
|
||||||
def _step_signature(step) -> str:
|
def _step_signature(step) -> str:
|
||||||
"""Build a comparable signature for a step."""
|
"""Build an exact normalized signature without incidental labels/outputs."""
|
||||||
if hasattr(step, "prompt"):
|
data = step.to_dict()
|
||||||
return f"prompt:{step.provider}"
|
data.pop("name", None)
|
||||||
elif hasattr(step, "code"):
|
data.pop("output_var", None)
|
||||||
return "code"
|
encoded = json.dumps(data, sort_keys=True, separators=(",", ":"), default=str)
|
||||||
elif hasattr(step, "tool"):
|
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||||
return f"tool:{step.tool}"
|
|
||||||
elif hasattr(step, "server"):
|
|
||||||
return f"mcp:{step.server}"
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def _add_audit_evidence(tool: Tool, report: PreflightReport):
|
def _has_tool_contracts(tool: Tool) -> bool:
|
||||||
|
return tool.input_schema is not None and tool.output_schema is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _find_internal_reuse(
|
||||||
|
tool: Tool, signatures: List[str], report: PreflightReport
|
||||||
|
) -> None:
|
||||||
|
"""Find non-overlapping repeated sequences with per-step output evidence."""
|
||||||
|
from .tool import PromptStep
|
||||||
|
|
||||||
|
count = len(signatures)
|
||||||
|
for length in range(count // 2, 1, -1):
|
||||||
|
for first in range(0, count - (2 * length) + 1):
|
||||||
|
candidate = signatures[first:first + length]
|
||||||
|
for second in range(first + length, count - length + 1):
|
||||||
|
if candidate != signatures[second:second + length]:
|
||||||
|
continue
|
||||||
|
steps = tool.steps[first:first + length]
|
||||||
|
if not all(
|
||||||
|
isinstance(step, PromptStep) and step.output_schema is not None
|
||||||
|
for step in steps
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
report.reuse_opportunities.append({
|
||||||
|
"type": "repeated_sequence",
|
||||||
|
"length": length,
|
||||||
|
"locations": [first + 1, second + 1],
|
||||||
|
"evidence": "exact normalized prompt steps with output schemas",
|
||||||
|
"detail": (
|
||||||
|
f"Steps {first + 1}-{first + length} repeat at "
|
||||||
|
f"steps {second + 1}-{second + length}"
|
||||||
|
),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _add_audit_evidence(
|
||||||
|
tool: Tool, report: PreflightReport, checks_run: List[str]
|
||||||
|
):
|
||||||
"""Attach audit evidence metadata to the report."""
|
"""Attach audit evidence metadata to the report."""
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
findings = {
|
||||||
|
"errors": report.errors,
|
||||||
|
"warnings": report.warnings,
|
||||||
|
"suggestions": report.suggestions,
|
||||||
|
"generated_tests": report.generated_tests,
|
||||||
|
"regression": report.regression,
|
||||||
|
"compatibility": report.compatibility,
|
||||||
|
"similar_tools": report.similar_tools,
|
||||||
|
"reuse_opportunities": report.reuse_opportunities,
|
||||||
|
}
|
||||||
|
findings_hash = hashlib.sha256(
|
||||||
|
json.dumps(findings, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||||
|
).hexdigest()
|
||||||
report.audit_evidence = {
|
report.audit_evidence = {
|
||||||
"tool_version": tool.version or "",
|
"tool_version": tool.version or "",
|
||||||
"engine_version": "1.0",
|
"engine_version": "1.0",
|
||||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
"checks_run": [
|
"checks_run": checks_run,
|
||||||
"config_integrity",
|
"findings_hash": findings_hash,
|
||||||
"contracts",
|
"findings": copy.deepcopy(findings),
|
||||||
"secrets",
|
"outcome": "failed" if report.errors else "advisory" if report.warnings else "passed",
|
||||||
"dependencies",
|
|
||||||
"toolstep_compatibility",
|
|
||||||
"reuse_opportunities",
|
|
||||||
],
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -20,9 +20,14 @@ class CategoryScore:
|
||||||
earned: int
|
earned: int
|
||||||
available: int
|
available: int
|
||||||
state: str = "checked" # checked | not_tested | not_applicable
|
state: str = "checked" # checked | not_tested | not_applicable
|
||||||
|
possible: Optional[int] = None # Category maximum when only partial evidence ran
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def display(self) -> str:
|
def display(self) -> str:
|
||||||
|
if self.state == "not_tested":
|
||||||
|
return "not tested"
|
||||||
|
if self.state == "not_applicable":
|
||||||
|
return "n/a"
|
||||||
return f"{self.earned}/{self.available}"
|
return f"{self.earned}/{self.available}"
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|
@ -39,6 +44,7 @@ class QualityScore:
|
||||||
headline: int # 0-100
|
headline: int # 0-100
|
||||||
categories: List[CategoryScore] = field(default_factory=list)
|
categories: List[CategoryScore] = field(default_factory=list)
|
||||||
last_evaluated: str = ""
|
last_evaluated: str = ""
|
||||||
|
evidence_coverage: int = 0
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
def to_dict(self) -> dict:
|
||||||
return {
|
return {
|
||||||
|
|
@ -51,14 +57,16 @@ class QualityScore:
|
||||||
"earned": c.earned,
|
"earned": c.earned,
|
||||||
"available": c.available,
|
"available": c.available,
|
||||||
"state": c.state,
|
"state": c.state,
|
||||||
|
"possible": c.possible if c.possible is not None else c.available,
|
||||||
}
|
}
|
||||||
for c in self.categories
|
for c in self.categories
|
||||||
],
|
],
|
||||||
"last_evaluated": self.last_evaluated,
|
"last_evaluated": self.last_evaluated,
|
||||||
|
"evidence_coverage": self.evidence_coverage,
|
||||||
}
|
}
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
lines = [f"Quality {self.headline}"]
|
lines = [f"Quality {self.headline} — {self.evidence_coverage}% evidence coverage"]
|
||||||
if self.last_evaluated:
|
if self.last_evaluated:
|
||||||
lines[0] += f" — Evaluated {self.last_evaluated}"
|
lines[0] += f" — Evaluated {self.last_evaluated}"
|
||||||
for c in self.categories:
|
for c in self.categories:
|
||||||
|
|
@ -91,8 +99,13 @@ def compute_quality(
|
||||||
categories.append(_score_security(report, tool))
|
categories.append(_score_security(report, tool))
|
||||||
categories.append(_score_community(registry_data))
|
categories.append(_score_community(registry_data))
|
||||||
|
|
||||||
earned = sum(c.earned for c in categories)
|
checked = [c for c in categories if c.state == "checked"]
|
||||||
available = sum(c.available for c in categories)
|
earned = sum(c.earned for c in checked)
|
||||||
|
available = sum(c.available for c in checked)
|
||||||
|
possible = sum(
|
||||||
|
c.possible if c.possible is not None else c.available
|
||||||
|
for c in categories if c.state != "not_applicable"
|
||||||
|
)
|
||||||
headline = int((earned / available * 100)) if available > 0 else 0
|
headline = int((earned / available * 100)) if available > 0 else 0
|
||||||
|
|
||||||
return QualityScore(
|
return QualityScore(
|
||||||
|
|
@ -101,6 +114,7 @@ def compute_quality(
|
||||||
headline=headline,
|
headline=headline,
|
||||||
categories=categories,
|
categories=categories,
|
||||||
last_evaluated=datetime.now(timezone.utc).isoformat(),
|
last_evaluated=datetime.now(timezone.utc).isoformat(),
|
||||||
|
evidence_coverage=int((available / possible * 100)) if possible else 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -112,6 +126,9 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore:
|
||||||
earned += 5
|
earned += 5
|
||||||
if tool.output_schema is not None:
|
if tool.output_schema is not None:
|
||||||
earned += 5
|
earned += 5
|
||||||
|
if tool.input_schema is None and tool.output_schema is None:
|
||||||
|
return CategoryScore("Contracts", 0, available, "not_tested")
|
||||||
|
|
||||||
# No contract-related errors in the preflight
|
# No contract-related errors in the preflight
|
||||||
contract_errors = [
|
contract_errors = [
|
||||||
e for e in report.errors if "schema" in e.lower() or "contract" in e.lower()
|
e for e in report.errors if "schema" in e.lower() or "contract" in e.lower()
|
||||||
|
|
@ -121,27 +138,26 @@ def _score_contracts(tool: Tool, report: PreflightReport) -> CategoryScore:
|
||||||
else:
|
else:
|
||||||
earned = max(0, earned - len(contract_errors))
|
earned = max(0, earned - len(contract_errors))
|
||||||
|
|
||||||
state = "checked" if (tool.input_schema or tool.output_schema) else "not_tested"
|
return CategoryScore("Contracts", earned, available, "checked")
|
||||||
return CategoryScore("Contracts", earned, available, state)
|
|
||||||
|
|
||||||
|
|
||||||
def _score_tests(report: PreflightReport) -> CategoryScore:
|
def _score_tests(report: PreflightReport) -> CategoryScore:
|
||||||
available = 30
|
maximum = 30
|
||||||
tests = report.generated_tests or []
|
tests = report.generated_tests or []
|
||||||
|
|
||||||
if not tests:
|
if not tests:
|
||||||
return CategoryScore("Deterministic tests", 0, available, "not_tested")
|
return CategoryScore("Deterministic tests", 0, maximum, "not_tested")
|
||||||
|
|
||||||
passed = sum(1 for t in tests if t.get("state") == "passed")
|
|
||||||
failed = sum(1 for t in tests if t.get("state") == "failed")
|
|
||||||
total = len(tests)
|
|
||||||
|
|
||||||
if total == 0:
|
|
||||||
return CategoryScore("Deterministic tests", 0, available, "not_tested")
|
|
||||||
|
|
||||||
|
executed = [t for t in tests if t.get("state") in ("passed", "failed")]
|
||||||
|
if not executed:
|
||||||
|
return CategoryScore("Deterministic tests", 0, maximum, "not_tested")
|
||||||
|
passed = sum(1 for t in executed if t.get("state") == "passed")
|
||||||
|
total = len(executed)
|
||||||
|
available = max(1, int(maximum * total / len(tests)))
|
||||||
earned = int((passed / total) * available)
|
earned = int((passed / total) * available)
|
||||||
state = "checked" if failed == 0 else "checked"
|
return CategoryScore(
|
||||||
return CategoryScore("Deterministic tests", earned, available, state)
|
"Deterministic tests", earned, available, "checked", possible=maximum
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _score_regression(report: PreflightReport) -> CategoryScore:
|
def _score_regression(report: PreflightReport) -> CategoryScore:
|
||||||
|
|
@ -162,7 +178,23 @@ def _score_regression(report: PreflightReport) -> CategoryScore:
|
||||||
|
|
||||||
|
|
||||||
def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
|
def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
|
||||||
available = 20
|
maximum = 20
|
||||||
|
checks = set((report.audit_evidence or {}).get("checks_run", []))
|
||||||
|
if "security_scrutiny" in checks:
|
||||||
|
available = maximum
|
||||||
|
elif checks:
|
||||||
|
available = (10 if "secrets" in checks else 0) + (
|
||||||
|
10 if checks.intersection({"dependencies", "registry_dependencies"}) else 0
|
||||||
|
)
|
||||||
|
elif report.errors or report.warnings:
|
||||||
|
# Backward-compatible evidence supplied by callers without audit metadata.
|
||||||
|
available = maximum
|
||||||
|
else:
|
||||||
|
available = 0
|
||||||
|
if not available:
|
||||||
|
return CategoryScore(
|
||||||
|
"Security scrutiny", 0, maximum, "not_tested", possible=maximum
|
||||||
|
)
|
||||||
earned = available
|
earned = available
|
||||||
|
|
||||||
# Deduct for secret patterns found
|
# Deduct for secret patterns found
|
||||||
|
|
@ -172,13 +204,20 @@ def _score_security(report: PreflightReport, tool: Tool) -> CategoryScore:
|
||||||
earned -= len(secret_warnings) * 5
|
earned -= len(secret_warnings) * 5
|
||||||
|
|
||||||
# Deduct for unresolved dependencies
|
# Deduct for unresolved dependencies
|
||||||
dep_warnings = [
|
dependency_findings = [
|
||||||
w for w in report.warnings if "dependency" in w.lower() and "not installed" in w.lower()
|
finding for finding in report.warnings + report.errors
|
||||||
|
if ("dependency" in finding.lower() or "dependencies" in finding.lower())
|
||||||
|
and any(
|
||||||
|
marker in finding.lower()
|
||||||
|
for marker in ("not installed", "missing", "deprecated")
|
||||||
|
)
|
||||||
]
|
]
|
||||||
earned -= len(dep_warnings) * 3
|
earned -= len(dependency_findings) * 3
|
||||||
|
|
||||||
earned = max(0, earned)
|
earned = max(0, earned)
|
||||||
return CategoryScore("Security scrutiny", earned, available, "checked")
|
return CategoryScore(
|
||||||
|
"Security scrutiny", earned, available, "checked", possible=maximum
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _score_community(registry_data: Optional[dict]) -> CategoryScore:
|
def _score_community(registry_data: Optional[dict]) -> CategoryScore:
|
||||||
|
|
@ -189,8 +228,11 @@ def _score_community(registry_data: Optional[dict]) -> CategoryScore:
|
||||||
|
|
||||||
earned = 0
|
earned = 0
|
||||||
reviews = registry_data.get("reviews", [])
|
reviews = registry_data.get("reviews", [])
|
||||||
if reviews:
|
rating_count = registry_data.get("rating_count", len(reviews))
|
||||||
|
avg_rating = registry_data.get("average_rating")
|
||||||
|
if avg_rating is None and reviews:
|
||||||
avg_rating = sum(r.get("rating", 0) for r in reviews) / len(reviews)
|
avg_rating = sum(r.get("rating", 0) for r in reviews) / len(reviews)
|
||||||
|
if avg_rating is not None and rating_count:
|
||||||
earned += int((avg_rating / 5) * 8)
|
earned += int((avg_rating / 5) * 8)
|
||||||
|
|
||||||
downloads = registry_data.get("downloads", 0)
|
downloads = registry_data.get("downloads", 0)
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ import os
|
||||||
import re
|
import re
|
||||||
import secrets
|
import secrets
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import date, datetime, timedelta
|
from datetime import date, datetime, timedelta, timezone
|
||||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||||
|
|
||||||
from flask import Flask, Response, g, jsonify, request
|
from flask import Flask, Response, g, jsonify, request
|
||||||
|
|
@ -418,7 +418,7 @@ def create_app() -> Flask:
|
||||||
|
|
||||||
g.db.execute(
|
g.db.execute(
|
||||||
"UPDATE api_tokens SET last_used_at = ? WHERE id = ?",
|
"UPDATE api_tokens SET last_used_at = ? WHERE id = ?",
|
||||||
[datetime.utcnow().isoformat(), row["id"]],
|
[datetime.now(timezone.utc).isoformat(), row["id"]],
|
||||||
)
|
)
|
||||||
g.current_publisher = {
|
g.current_publisher = {
|
||||||
"id": row["publisher_id"],
|
"id": row["publisher_id"],
|
||||||
|
|
@ -870,6 +870,14 @@ def create_app() -> Flask:
|
||||||
order_sql = f"{sort} {order_dir}, published_at DESC, f.id DESC"
|
order_sql = f"{sort} {order_dir}, published_at DESC, f.id DESC"
|
||||||
|
|
||||||
stats_join = "LEFT JOIN tool_stats ts ON ts.tool_id = f.id"
|
stats_join = "LEFT JOIN tool_stats ts ON ts.tool_id = f.id"
|
||||||
|
audit_join = """
|
||||||
|
LEFT JOIN tool_audits ta ON ta.id = (
|
||||||
|
SELECT latest_audit.id FROM tool_audits latest_audit
|
||||||
|
WHERE latest_audit.tool_id = f.id
|
||||||
|
ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC
|
||||||
|
LIMIT 1
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
rows = query_all(
|
rows = query_all(
|
||||||
g.db,
|
g.db,
|
||||||
|
|
@ -897,9 +905,13 @@ def create_app() -> Flask:
|
||||||
GROUP BY owner, name
|
GROUP BY owner, name
|
||||||
)
|
)
|
||||||
SELECT f.*, COALESCE(ts.average_rating, 0) AS average_rating,
|
SELECT f.*, COALESCE(ts.average_rating, 0) AS average_rating,
|
||||||
COALESCE(ts.rating_count, 0) AS rating_count
|
COALESCE(ts.rating_count, 0) AS rating_count,
|
||||||
|
json_extract(ta.quality_json, '$.score') AS quality_score,
|
||||||
|
json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage,
|
||||||
|
ta.evaluated_at AS quality_evaluated_at
|
||||||
FROM filtered f
|
FROM filtered f
|
||||||
{stats_join}
|
{stats_join}
|
||||||
|
{audit_join}
|
||||||
JOIN (
|
JOIN (
|
||||||
SELECT a.owner, a.name, COALESCE(s.max_id, a.max_id) AS max_id
|
SELECT a.owner, a.name, COALESCE(s.max_id, a.max_id) AS max_id
|
||||||
FROM latest_any a
|
FROM latest_any a
|
||||||
|
|
@ -948,6 +960,12 @@ def create_app() -> Flask:
|
||||||
"average_rating": row["average_rating"],
|
"average_rating": row["average_rating"],
|
||||||
"rating_count": row["rating_count"],
|
"rating_count": row["rating_count"],
|
||||||
"score": score,
|
"score": score,
|
||||||
|
"quality_score": row["quality_score"],
|
||||||
|
"quality_coverage": row["quality_coverage"],
|
||||||
|
"quality_evaluated_at": row["quality_evaluated_at"],
|
||||||
|
"deprecated": bool(row["deprecated"]),
|
||||||
|
"deprecated_message": row["deprecated_message"],
|
||||||
|
"replacement": row["replacement"],
|
||||||
})
|
})
|
||||||
|
|
||||||
result: dict = {"data": data, "meta": paginate(page, per_page, total)}
|
result: dict = {"data": data, "meta": paginate(page, per_page, total)}
|
||||||
|
|
@ -1089,6 +1107,39 @@ def create_app() -> Flask:
|
||||||
if fork_row:
|
if fork_row:
|
||||||
fork_count = fork_row["cnt"]
|
fork_count = fork_row["cnt"]
|
||||||
|
|
||||||
|
replacement_chain = []
|
||||||
|
if row["replacement"]:
|
||||||
|
try:
|
||||||
|
replacement_chain = _get_replacement_chain(
|
||||||
|
row["owner"], row["name"], row["replacement"]
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
# Preserve access to legacy rows while refusing new invalid chains.
|
||||||
|
replacement_chain = []
|
||||||
|
|
||||||
|
latest_audit = query_one(
|
||||||
|
g.db,
|
||||||
|
"""
|
||||||
|
SELECT engine_version, evidence_json, quality_json, evaluated_at
|
||||||
|
FROM tool_audits
|
||||||
|
WHERE tool_id = ?
|
||||||
|
ORDER BY evaluated_at DESC, id DESC
|
||||||
|
LIMIT 1
|
||||||
|
""",
|
||||||
|
[row["id"]],
|
||||||
|
)
|
||||||
|
audit_stale = True
|
||||||
|
if latest_audit:
|
||||||
|
try:
|
||||||
|
evaluated = datetime.fromisoformat(
|
||||||
|
latest_audit["evaluated_at"].replace("Z", "+00:00")
|
||||||
|
)
|
||||||
|
if evaluated.tzinfo is None:
|
||||||
|
evaluated = evaluated.replace(tzinfo=timezone.utc)
|
||||||
|
audit_stale = datetime.now(timezone.utc) - evaluated > timedelta(days=30)
|
||||||
|
except (AttributeError, TypeError, ValueError):
|
||||||
|
audit_stale = True
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"owner": row["owner"],
|
"owner": row["owner"],
|
||||||
"name": row["name"],
|
"name": row["name"],
|
||||||
|
|
@ -1100,7 +1151,8 @@ def create_app() -> Flask:
|
||||||
"published_at": row["published_at"],
|
"published_at": row["published_at"],
|
||||||
"deprecated": bool(row["deprecated"]),
|
"deprecated": bool(row["deprecated"]),
|
||||||
"deprecated_message": row["deprecated_message"],
|
"deprecated_message": row["deprecated_message"],
|
||||||
"replacement": row["replacement"],
|
"replacement": row["replacement"] if replacement_chain else None,
|
||||||
|
"replacement_chain": replacement_chain,
|
||||||
"config": row["config_yaml"],
|
"config": row["config_yaml"],
|
||||||
"readme": row["readme"],
|
"readme": row["readme"],
|
||||||
"defaults": row.get("defaults") or "",
|
"defaults": row.get("defaults") or "",
|
||||||
|
|
@ -1108,6 +1160,10 @@ def create_app() -> Flask:
|
||||||
"forked_from": row.get("forked_from"),
|
"forked_from": row.get("forked_from"),
|
||||||
"forked_version": row.get("forked_version"),
|
"forked_version": row.get("forked_version"),
|
||||||
"fork_count": fork_count,
|
"fork_count": fork_count,
|
||||||
|
"audit": json.loads(latest_audit["evidence_json"]) if latest_audit else None,
|
||||||
|
"quality": json.loads(latest_audit["quality_json"]) if latest_audit else None,
|
||||||
|
"audit_evaluated_at": latest_audit["evaluated_at"] if latest_audit else None,
|
||||||
|
"audit_stale": audit_stale,
|
||||||
}
|
}
|
||||||
response = jsonify({"data": payload})
|
response = jsonify({"data": payload})
|
||||||
response.headers["Cache-Control"] = "max-age=60"
|
response.headers["Cache-Control"] = "max-age=60"
|
||||||
|
|
@ -2496,8 +2552,12 @@ def create_app() -> Flask:
|
||||||
from ..preflight import analyze_tool as run_preflight
|
from ..preflight import analyze_tool as run_preflight
|
||||||
from ..tool import Tool
|
from ..tool import Tool
|
||||||
|
|
||||||
|
published_tool = Tool.from_dict(data)
|
||||||
preflight_report = run_preflight(
|
preflight_report = run_preflight(
|
||||||
Tool.from_dict(data), check_local_dependencies=False
|
published_tool,
|
||||||
|
check_local_dependencies=False,
|
||||||
|
include_contract_tests=True,
|
||||||
|
check_reuse=False,
|
||||||
)
|
)
|
||||||
except (KeyError, TypeError, ValueError) as exc:
|
except (KeyError, TypeError, ValueError) as exc:
|
||||||
return error_response("INVALID_CONFIG", str(exc), 400)
|
return error_response("INVALID_CONFIG", str(exc), 400)
|
||||||
|
|
@ -2538,6 +2598,44 @@ def create_app() -> Flask:
|
||||||
)
|
)
|
||||||
owner = requested_owner
|
owner = requested_owner
|
||||||
|
|
||||||
|
from .auditing import registry_dependency_findings
|
||||||
|
dependency_findings = registry_dependency_findings(
|
||||||
|
g.db, published_tool, owner
|
||||||
|
)
|
||||||
|
missing_dependencies = dependency_findings["missing"]
|
||||||
|
preflight_report.warnings.extend(dependency_findings["deprecated"])
|
||||||
|
from ..preflight import _add_audit_evidence
|
||||||
|
audit_checks = list(
|
||||||
|
(preflight_report.audit_evidence or {}).get("checks_run", [])
|
||||||
|
)
|
||||||
|
_add_audit_evidence(
|
||||||
|
published_tool, preflight_report,
|
||||||
|
audit_checks + ["registry_dependencies"],
|
||||||
|
)
|
||||||
|
from ..quality import compute_quality
|
||||||
|
quality_report = compute_quality(
|
||||||
|
published_tool, preflight_report
|
||||||
|
).to_dict()
|
||||||
|
if missing_dependencies:
|
||||||
|
message = "Missing registry dependencies: " + ", ".join(
|
||||||
|
missing_dependencies
|
||||||
|
)
|
||||||
|
preflight_report.errors.append(message)
|
||||||
|
_add_audit_evidence(
|
||||||
|
published_tool, preflight_report,
|
||||||
|
audit_checks + ["registry_dependencies"],
|
||||||
|
)
|
||||||
|
return error_response(
|
||||||
|
"PREFLIGHT_FAILED", message, 400,
|
||||||
|
details={"preflight": preflight_report.to_dict()},
|
||||||
|
)
|
||||||
|
|
||||||
|
if data.get("deprecated") and data.get("replacement"):
|
||||||
|
try:
|
||||||
|
_get_replacement_chain(owner, name, data["replacement"])
|
||||||
|
except ValueError as exc:
|
||||||
|
return error_response("VALIDATION_ERROR", str(exc), 400)
|
||||||
|
|
||||||
# Compute config hash early for idempotency check
|
# Compute config hash early for idempotency check
|
||||||
config_hash = compute_yaml_hash(config_text)
|
config_hash = compute_yaml_hash(config_text)
|
||||||
|
|
||||||
|
|
@ -2650,6 +2748,7 @@ def create_app() -> Flask:
|
||||||
"status": "validated",
|
"status": "validated",
|
||||||
"suggestions": suggestions,
|
"suggestions": suggestions,
|
||||||
"preflight": preflight_report.to_dict(),
|
"preflight": preflight_report.to_dict(),
|
||||||
|
"quality": quality_report,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -2675,7 +2774,7 @@ def create_app() -> Flask:
|
||||||
# Public tools need moderation approval
|
# Public tools need moderation approval
|
||||||
moderation_status = "pending"
|
moderation_status = "pending"
|
||||||
|
|
||||||
g.db.execute(
|
insert_cursor = g.db.execute(
|
||||||
"""
|
"""
|
||||||
INSERT INTO tools (
|
INSERT INTO tools (
|
||||||
owner, name, version, description, category, tags, config_yaml, readme,
|
owner, name, version, description, category, tags, config_yaml, readme,
|
||||||
|
|
@ -2710,7 +2809,25 @@ def create_app() -> Flask:
|
||||||
moderation_status,
|
moderation_status,
|
||||||
forked_from,
|
forked_from,
|
||||||
forked_version,
|
forked_version,
|
||||||
datetime.utcnow().isoformat(),
|
datetime.now(timezone.utc).isoformat(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
tool_id = insert_cursor.lastrowid
|
||||||
|
audit = preflight_report.audit_evidence or {}
|
||||||
|
g.db.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO tool_audits (
|
||||||
|
tool_id, engine_version, trigger, evidence_json, quality_json,
|
||||||
|
findings_hash, evaluated_at
|
||||||
|
) VALUES (?, ?, 'publish', ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
[
|
||||||
|
tool_id,
|
||||||
|
audit.get("engine_version", "1.0"),
|
||||||
|
json.dumps(audit, sort_keys=True),
|
||||||
|
json.dumps(quality_report, sort_keys=True),
|
||||||
|
audit.get("findings_hash", ""),
|
||||||
|
audit.get("timestamp") or datetime.now(timezone.utc).isoformat(),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
g.db.commit()
|
g.db.commit()
|
||||||
|
|
@ -2756,6 +2873,7 @@ def create_app() -> Flask:
|
||||||
"forked_version": forked_version,
|
"forked_version": forked_version,
|
||||||
"suggestions": suggestions,
|
"suggestions": suggestions,
|
||||||
"preflight": preflight_report.to_dict(),
|
"preflight": preflight_report.to_dict(),
|
||||||
|
"quality": quality_report,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
response.status_code = 201
|
response.status_code = 201
|
||||||
|
|
@ -2870,6 +2988,47 @@ def create_app() -> Flask:
|
||||||
|
|
||||||
return jsonify({"data": results})
|
return jsonify({"data": results})
|
||||||
|
|
||||||
|
def _get_replacement_chain(
|
||||||
|
original_owner: str, original_name: str, replacement: str
|
||||||
|
) -> List[str]:
|
||||||
|
"""Validate and resolve a deprecation chain without allowing cycles."""
|
||||||
|
chain: List[str] = []
|
||||||
|
visited = {(original_owner, original_name)}
|
||||||
|
current = replacement
|
||||||
|
default_owner = original_owner
|
||||||
|
for _ in range(20):
|
||||||
|
if "/" in current:
|
||||||
|
next_owner, next_name = current.split("/", 1)
|
||||||
|
else:
|
||||||
|
next_owner, next_name = default_owner, current
|
||||||
|
if not OWNER_RE.fullmatch(next_owner) or not TOOL_NAME_RE.fullmatch(next_name):
|
||||||
|
raise ValueError(
|
||||||
|
"replacement must be a valid tool name or owner/tool reference"
|
||||||
|
)
|
||||||
|
identity = (next_owner, next_name)
|
||||||
|
if identity in visited:
|
||||||
|
raise ValueError("replacement creates a deprecation cycle")
|
||||||
|
row = query_one(
|
||||||
|
g.db,
|
||||||
|
"""
|
||||||
|
SELECT deprecated, replacement FROM tools
|
||||||
|
WHERE owner = ? AND name = ?
|
||||||
|
ORDER BY id DESC LIMIT 1
|
||||||
|
""",
|
||||||
|
[next_owner, next_name],
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
raise ValueError(
|
||||||
|
f"replacement tool '{next_owner}/{next_name}' does not exist"
|
||||||
|
)
|
||||||
|
visited.add(identity)
|
||||||
|
chain.append(f"{next_owner}/{next_name}")
|
||||||
|
if not row["deprecated"] or not row["replacement"]:
|
||||||
|
return chain
|
||||||
|
current = row["replacement"]
|
||||||
|
default_owner = next_owner
|
||||||
|
raise ValueError("replacement chain exceeds 20 tools")
|
||||||
|
|
||||||
@app.route("/api/v1/tools/<owner>/<name>/deprecate", methods=["POST"])
|
@app.route("/api/v1/tools/<owner>/<name>/deprecate", methods=["POST"])
|
||||||
@require_token
|
@require_token
|
||||||
def deprecate_tool(owner: str, name: str) -> Response:
|
def deprecate_tool(owner: str, name: str) -> Response:
|
||||||
|
|
@ -2883,6 +3042,10 @@ def create_app() -> Flask:
|
||||||
|
|
||||||
if message and len(message) > 500:
|
if message and len(message) > 500:
|
||||||
return error_response("VALIDATION_ERROR", "Message too long (max 500)", 400)
|
return error_response("VALIDATION_ERROR", "Message too long (max 500)", 400)
|
||||||
|
try:
|
||||||
|
chain = _get_replacement_chain(owner, name, replacement) if replacement else []
|
||||||
|
except ValueError as exc:
|
||||||
|
return error_response("VALIDATION_ERROR", str(exc), 400)
|
||||||
|
|
||||||
# Update all versions of the tool
|
# Update all versions of the tool
|
||||||
result = g.db.execute(
|
result = g.db.execute(
|
||||||
|
|
@ -2896,7 +3059,10 @@ def create_app() -> Flask:
|
||||||
return error_response("TOOL_NOT_FOUND", f"Tool {owner}/{name} not found", 404)
|
return error_response("TOOL_NOT_FOUND", f"Tool {owner}/{name} not found", 404)
|
||||||
g.db.commit()
|
g.db.commit()
|
||||||
|
|
||||||
return jsonify({"data": {"status": "deprecated", "owner": owner, "name": name}})
|
return jsonify({"data": {
|
||||||
|
"status": "deprecated", "owner": owner, "name": name,
|
||||||
|
"replacement_chain": chain,
|
||||||
|
}})
|
||||||
|
|
||||||
@app.route("/api/v1/tools/<owner>/<name>/undeprecate", methods=["POST"])
|
@app.route("/api/v1/tools/<owner>/<name>/undeprecate", methods=["POST"])
|
||||||
@require_token
|
@require_token
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,210 @@
|
||||||
|
"""Immutable publish-time and refresh audits for registry tool versions."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
from ..preflight import analyze_tool
|
||||||
|
from ..quality import compute_quality
|
||||||
|
from ..tool import Tool
|
||||||
|
from .db import connect_db, get_db_path, init_db
|
||||||
|
|
||||||
|
|
||||||
|
def registry_dependency_findings(conn, tool: Tool, owner: str) -> Dict[str, List[str]]:
|
||||||
|
"""Resolve registry dependencies and report missing/deprecated references."""
|
||||||
|
from ..tool import ToolStep
|
||||||
|
|
||||||
|
references = set(tool.dependencies)
|
||||||
|
references.update(
|
||||||
|
step.tool for step in tool.steps if isinstance(step, ToolStep)
|
||||||
|
)
|
||||||
|
missing: List[str] = []
|
||||||
|
deprecated: List[str] = []
|
||||||
|
for reference in sorted(references):
|
||||||
|
if reference == tool.name or reference == f"{owner}/{tool.name}":
|
||||||
|
continue
|
||||||
|
if "/" in reference:
|
||||||
|
dep_owner, dep_name = reference.split("/", 1)
|
||||||
|
candidates = [(dep_owner, dep_name)]
|
||||||
|
else:
|
||||||
|
candidates = [(owner, reference), ("official", reference)]
|
||||||
|
row = None
|
||||||
|
for dep_owner, dep_name in candidates:
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT deprecated, replacement FROM tools
|
||||||
|
WHERE owner = ? AND name = ? ORDER BY id DESC LIMIT 1
|
||||||
|
""",
|
||||||
|
[dep_owner, dep_name],
|
||||||
|
).fetchone()
|
||||||
|
if row is not None:
|
||||||
|
break
|
||||||
|
if row is None:
|
||||||
|
missing.append(reference)
|
||||||
|
elif row["deprecated"]:
|
||||||
|
guidance = f"{reference} is deprecated"
|
||||||
|
if row["replacement"]:
|
||||||
|
guidance += f"; use {row['replacement']}"
|
||||||
|
deprecated.append(guidance)
|
||||||
|
return {"missing": missing, "deprecated": deprecated}
|
||||||
|
|
||||||
|
|
||||||
|
def audit_tool_version(conn, tool_id: int, trigger: str = "background") -> Dict[str, Any]:
|
||||||
|
"""Evaluate one immutable tool version and append an audit record."""
|
||||||
|
row = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT tools.*, COALESCE(tool_stats.average_rating, 0) AS average_rating,
|
||||||
|
COALESCE(tool_stats.rating_count, 0) AS rating_count
|
||||||
|
FROM tools
|
||||||
|
LEFT JOIN tool_stats ON tool_stats.tool_id = tools.id
|
||||||
|
WHERE tools.id = ?
|
||||||
|
""",
|
||||||
|
[tool_id],
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise ValueError(f"Unknown tool version id {tool_id}")
|
||||||
|
|
||||||
|
previous = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT evidence_json, quality_json FROM tool_audits
|
||||||
|
WHERE tool_id = ? ORDER BY evaluated_at DESC, id DESC LIMIT 1
|
||||||
|
""",
|
||||||
|
[tool_id],
|
||||||
|
).fetchone()
|
||||||
|
config = yaml.safe_load(row["config_yaml"])
|
||||||
|
tool = Tool.from_dict(config)
|
||||||
|
report = analyze_tool(
|
||||||
|
tool,
|
||||||
|
check_local_dependencies=False,
|
||||||
|
include_contract_tests=True,
|
||||||
|
check_reuse=False,
|
||||||
|
)
|
||||||
|
dependency_findings = registry_dependency_findings(conn, tool, row["owner"])
|
||||||
|
if dependency_findings["missing"]:
|
||||||
|
report.errors.append(
|
||||||
|
"Missing registry dependencies: "
|
||||||
|
+ ", ".join(dependency_findings["missing"])
|
||||||
|
)
|
||||||
|
report.warnings.extend(dependency_findings["deprecated"])
|
||||||
|
from ..preflight import _add_audit_evidence
|
||||||
|
checks = list((report.audit_evidence or {}).get("checks_run", []))
|
||||||
|
_add_audit_evidence(tool, report, checks + ["registry_dependencies"])
|
||||||
|
registry_data = {
|
||||||
|
"average_rating": row["average_rating"],
|
||||||
|
"rating_count": row["rating_count"],
|
||||||
|
"downloads": row["downloads"],
|
||||||
|
}
|
||||||
|
quality = compute_quality(tool, report, registry_data=registry_data).to_dict()
|
||||||
|
evidence = report.audit_evidence or {}
|
||||||
|
evaluated_at = datetime.now(timezone.utc).isoformat()
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO tool_audits (
|
||||||
|
tool_id, engine_version, trigger, evidence_json, quality_json,
|
||||||
|
findings_hash, evaluated_at
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
[
|
||||||
|
tool_id,
|
||||||
|
evidence.get("engine_version", "1.0"),
|
||||||
|
trigger,
|
||||||
|
json.dumps(evidence, sort_keys=True),
|
||||||
|
json.dumps(quality, sort_keys=True),
|
||||||
|
evidence.get("findings_hash", ""),
|
||||||
|
evaluated_at,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
quality_change = None
|
||||||
|
if previous:
|
||||||
|
previous_quality = json.loads(previous["quality_json"])
|
||||||
|
quality_change = quality["score"] - previous_quality.get("score", 0)
|
||||||
|
previous_evidence = json.loads(previous["evidence_json"])
|
||||||
|
if quality_change < 0 or previous_evidence.get("outcome") != evidence.get("outcome"):
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO audit_log (
|
||||||
|
action, target_type, target_id, actor_id, details
|
||||||
|
) VALUES ('automated_audit_changed', 'tool', ?, 'system', ?)
|
||||||
|
""",
|
||||||
|
[
|
||||||
|
str(tool_id),
|
||||||
|
json.dumps({
|
||||||
|
"quality_change": quality_change,
|
||||||
|
"previous_outcome": previous_evidence.get("outcome"),
|
||||||
|
"current_outcome": evidence.get("outcome"),
|
||||||
|
}, sort_keys=True),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return {
|
||||||
|
"tool_id": tool_id, "evidence": evidence, "quality": quality,
|
||||||
|
"quality_change": quality_change,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def audit_stale_tools(
|
||||||
|
conn, *, max_age_days: int = 30, limit: int = 100
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Append refreshed evidence for versions with missing or stale audits."""
|
||||||
|
if max_age_days < 0:
|
||||||
|
raise ValueError("max_age_days must be non-negative")
|
||||||
|
if limit < 1:
|
||||||
|
raise ValueError("limit must be positive")
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days)
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT tools.id, MAX(tool_audits.evaluated_at) AS last_evaluated
|
||||||
|
FROM tools
|
||||||
|
LEFT JOIN tool_audits ON tool_audits.tool_id = tools.id
|
||||||
|
GROUP BY tools.id
|
||||||
|
ORDER BY last_evaluated IS NOT NULL, last_evaluated
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
stale_ids = []
|
||||||
|
for row in rows:
|
||||||
|
value = row["last_evaluated"]
|
||||||
|
if value:
|
||||||
|
try:
|
||||||
|
evaluated = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
if evaluated.tzinfo is None:
|
||||||
|
evaluated = evaluated.replace(tzinfo=timezone.utc)
|
||||||
|
except ValueError:
|
||||||
|
evaluated = datetime.min.replace(tzinfo=timezone.utc)
|
||||||
|
if evaluated >= cutoff:
|
||||||
|
continue
|
||||||
|
stale_ids.append(row["id"])
|
||||||
|
if len(stale_ids) >= limit:
|
||||||
|
break
|
||||||
|
results = []
|
||||||
|
for tool_id in stale_ids:
|
||||||
|
try:
|
||||||
|
results.append(audit_tool_version(conn, tool_id))
|
||||||
|
except Exception as exc:
|
||||||
|
results.append({"tool_id": tool_id, "error": str(exc)})
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Refresh stale CmdForge registry audits")
|
||||||
|
parser.add_argument("--max-age-days", type=int, default=30)
|
||||||
|
parser.add_argument("--limit", type=int, default=100)
|
||||||
|
args = parser.parse_args()
|
||||||
|
conn = connect_db(get_db_path())
|
||||||
|
try:
|
||||||
|
init_db(conn)
|
||||||
|
results = audit_stale_tools(
|
||||||
|
conn, max_age_days=args.max_age_days, limit=args.limit
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
print(f"Audited {len(results)} tool version(s)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -80,6 +80,20 @@ CREATE TABLE IF NOT EXISTS download_stats (
|
||||||
UNIQUE(tool_id, client_id, downloaded_at)
|
UNIQUE(tool_id, client_id, downloaded_at)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tool_audits (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
tool_id INTEGER NOT NULL REFERENCES tools(id) ON DELETE CASCADE,
|
||||||
|
engine_version TEXT NOT NULL,
|
||||||
|
trigger TEXT NOT NULL,
|
||||||
|
evidence_json TEXT NOT NULL,
|
||||||
|
quality_json TEXT NOT NULL,
|
||||||
|
findings_hash TEXT NOT NULL,
|
||||||
|
evaluated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tool_audits_tool_time
|
||||||
|
ON tool_audits(tool_id, evaluated_at DESC, id DESC);
|
||||||
|
|
||||||
CREATE VIRTUAL TABLE IF NOT EXISTS tools_fts USING fts5(
|
CREATE VIRTUAL TABLE IF NOT EXISTS tools_fts USING fts5(
|
||||||
name, description, tags, readme,
|
name, description, tags, readme,
|
||||||
content='tools',
|
content='tools',
|
||||||
|
|
|
||||||
|
|
@ -157,10 +157,19 @@ def semantic_search(
|
||||||
"""
|
"""
|
||||||
SELECT te.tool_id, te.embedding, te.dimensions, te.model,
|
SELECT te.tool_id, te.embedding, te.dimensions, te.model,
|
||||||
t.owner, t.name, t.version, t.description, t.category,
|
t.owner, t.name, t.version, t.description, t.category,
|
||||||
t.tags, t.downloads
|
t.tags, t.downloads,
|
||||||
|
json_extract(ta.quality_json, '$.score') AS quality_score,
|
||||||
|
json_extract(ta.quality_json, '$.evidence_coverage') AS quality_coverage
|
||||||
FROM tool_embeddings te
|
FROM tool_embeddings te
|
||||||
JOIN tools t ON t.id = te.tool_id
|
JOIN tools t ON t.id = te.tool_id
|
||||||
|
LEFT JOIN tool_audits ta ON ta.id = (
|
||||||
|
SELECT latest_audit.id FROM tool_audits latest_audit
|
||||||
|
WHERE latest_audit.tool_id = t.id
|
||||||
|
ORDER BY latest_audit.evaluated_at DESC, latest_audit.id DESC
|
||||||
|
LIMIT 1
|
||||||
|
)
|
||||||
WHERE t.visibility = 'public' AND t.moderation_status = 'approved'
|
WHERE t.visibility = 'public' AND t.moderation_status = 'approved'
|
||||||
|
AND t.deprecated = 0
|
||||||
""",
|
""",
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
|
||||||
|
|
@ -192,6 +201,8 @@ def semantic_search(
|
||||||
"tags": tags_list,
|
"tags": tags_list,
|
||||||
"downloads": row["downloads"] or 0,
|
"downloads": row["downloads"] or 0,
|
||||||
"similarity": round(score, 4),
|
"similarity": round(score, 4),
|
||||||
|
"quality_score": row["quality_score"],
|
||||||
|
"quality_coverage": row["quality_coverage"],
|
||||||
})
|
})
|
||||||
|
|
||||||
# Sort by similarity descending
|
# Sort by similarity descending
|
||||||
|
|
|
||||||
|
|
@ -1173,6 +1173,11 @@ def main():
|
||||||
print(f"Searched: {', '.join(e.searched_paths[:3])}", file=sys.stderr)
|
print(f"Searched: {', '.join(e.searched_paths[:3])}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
from .tool import deprecation_warning
|
||||||
|
warning = deprecation_warning(tool)
|
||||||
|
if warning:
|
||||||
|
print(warning, file=sys.stderr)
|
||||||
|
|
||||||
# Check for manifest overrides
|
# Check for manifest overrides
|
||||||
manifest = load_manifest()
|
manifest = load_manifest()
|
||||||
provider_override_from_manifest = None
|
provider_override_from_manifest = None
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
"""Tool loading, saving, and management."""
|
"""Tool loading, saving, and management."""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import stat
|
import stat
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
@ -15,6 +16,9 @@ TOOLS_DIR = Path.home() / ".cmdforge"
|
||||||
|
|
||||||
# Default bin directory for wrapper scripts
|
# Default bin directory for wrapper scripts
|
||||||
BIN_DIR = Path.home() / ".local" / "bin"
|
BIN_DIR = Path.home() / ".local" / "bin"
|
||||||
|
_TOOL_REF_RE = re.compile(
|
||||||
|
r"^(?:[a-z0-9](?:[a-z0-9-]{0,37}[a-z0-9])?/)?[A-Za-z0-9_-]{1,64}$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _validate_skill_selection(skills: Optional[List[str]]) -> None:
|
def _validate_skill_selection(skills: Optional[List[str]]) -> None:
|
||||||
|
|
@ -447,16 +451,27 @@ class Tool:
|
||||||
source: Optional[ToolSource] = None # Attribution for imported/external tools
|
source: Optional[ToolSource] = None # Attribution for imported/external tools
|
||||||
version: str = "" # Tool version
|
version: str = "" # Tool version
|
||||||
visibility: str = "public" # "public", "private", or "unlisted"
|
visibility: str = "public" # "public", "private", or "unlisted"
|
||||||
deprecated: bool = False # Tool is deprecated
|
|
||||||
deprecated_message: str = "" # Migration guidance for deprecated tools
|
|
||||||
replacement: Optional[str] = None # Suggested replacement tool name
|
|
||||||
input_schema: Optional[dict] = None # JSON Schema for tool input contract
|
input_schema: Optional[dict] = None # JSON Schema for tool input contract
|
||||||
output_schema: Optional[dict] = None # JSON Schema for tool output contract
|
output_schema: Optional[dict] = None # JSON Schema for tool output contract
|
||||||
path: Optional[Path] = None # Path to config.yaml (set by load_tool)
|
path: Optional[Path] = None # Path to config.yaml (set by load_tool)
|
||||||
|
deprecated: bool = False # Tool is deprecated
|
||||||
|
deprecated_message: str = "" # Migration guidance for deprecated tools
|
||||||
|
replacement: Optional[str] = None # Suggested replacement tool name
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
def __post_init__(self) -> None:
|
||||||
validate_json_schema(self.input_schema, "input_schema")
|
validate_json_schema(self.input_schema, "input_schema")
|
||||||
validate_json_schema(self.output_schema, "output_schema")
|
validate_json_schema(self.output_schema, "output_schema")
|
||||||
|
if not isinstance(self.deprecated, bool):
|
||||||
|
raise ValueError("deprecated must be true or false")
|
||||||
|
if not isinstance(self.deprecated_message, str):
|
||||||
|
raise ValueError("deprecated_message must be a string")
|
||||||
|
if len(self.deprecated_message) > 500:
|
||||||
|
raise ValueError("deprecated_message must be at most 500 characters")
|
||||||
|
if self.replacement is not None and (
|
||||||
|
not isinstance(self.replacement, str)
|
||||||
|
or not _TOOL_REF_RE.fullmatch(self.replacement)
|
||||||
|
):
|
||||||
|
raise ValueError("replacement must be a tool name or owner/tool reference")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, data: dict) -> "Tool":
|
def from_dict(cls, data: dict) -> "Tool":
|
||||||
|
|
@ -809,6 +824,16 @@ def tool_exists(name: str) -> bool:
|
||||||
return (get_tools_dir() / name / "config.yaml").exists()
|
return (get_tools_dir() / name / "config.yaml").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def deprecation_warning(tool: Tool) -> Optional[str]:
|
||||||
|
"""Return consistent migration guidance for a deprecated tool."""
|
||||||
|
if not tool.deprecated:
|
||||||
|
return None
|
||||||
|
message = tool.deprecated_message or "This tool is no longer maintained."
|
||||||
|
if tool.replacement:
|
||||||
|
message += f" Use '{tool.replacement}' instead."
|
||||||
|
return f"Warning: '{tool.name}' is deprecated. {message}"
|
||||||
|
|
||||||
|
|
||||||
def validate_tool_name(name: str) -> tuple[bool, str]:
|
def validate_tool_name(name: str) -> tuple[bool, str]:
|
||||||
"""
|
"""
|
||||||
Validate a tool name.
|
Validate a tool name.
|
||||||
|
|
|
||||||
|
|
@ -541,6 +541,37 @@ class TestRegistryPublishDryRun:
|
||||||
assert _cmd_registry_publish(self.args(tool_dir)) == 1
|
assert _cmd_registry_publish(self.args(tool_dir)) == 1
|
||||||
assert "YAML mapping" in capsys.readouterr().err
|
assert "YAML mapping" in capsys.readouterr().err
|
||||||
|
|
||||||
|
def test_normal_publish_runs_remote_preflight_before_mutation(
|
||||||
|
self, tool_dir, monkeypatch
|
||||||
|
):
|
||||||
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
||||||
|
|
||||||
|
client = MagicMock()
|
||||||
|
client.get_me.return_value = {"slug": "testuser"}
|
||||||
|
client.get_my_tool_status.return_value = {"status": "pending"}
|
||||||
|
client.publish_tool.side_effect = [
|
||||||
|
{
|
||||||
|
"preflight": {"errors": [], "warnings": [], "suggestions": []},
|
||||||
|
"quality": {"score": 80, "evidence_coverage": 60, "categories": []},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"owner": "testuser", "name": "dry-run-tool",
|
||||||
|
"version": "1.0.0", "status": "pending",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"cmdforge.cli.registry_commands.load_config",
|
||||||
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
||||||
|
args = SimpleNamespace(
|
||||||
|
path=str(tool_dir), dry_run=False, force=True, owner=""
|
||||||
|
)
|
||||||
|
assert _cmd_registry_publish(args) == 0
|
||||||
|
assert client.publish_tool.call_count == 2
|
||||||
|
assert client.publish_tool.call_args_list[0].kwargs["dry_run"] is True
|
||||||
|
assert "dry_run" not in client.publish_tool.call_args_list[1].kwargs
|
||||||
|
|
||||||
|
|
||||||
def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
|
def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
|
||||||
from cmdforge.cli import cmd_inspect
|
from cmdforge.cli import cmd_inspect
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ def client(app):
|
||||||
def auth_headers(app):
|
def auth_headers(app):
|
||||||
"""Create auth headers with a valid token."""
|
"""Create auth headers with a valid token."""
|
||||||
import hashlib
|
import hashlib
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from cmdforge.registry.db import connect_db
|
from cmdforge.registry.db import connect_db
|
||||||
|
|
||||||
token = "test-token"
|
token = "test-token"
|
||||||
|
|
@ -94,7 +94,7 @@ def auth_headers(app):
|
||||||
INSERT INTO api_tokens (publisher_id, token_hash, name, created_at)
|
INSERT INTO api_tokens (publisher_id, token_hash, name, created_at)
|
||||||
VALUES (?, ?, ?, ?)
|
VALUES (?, ?, ?, ?)
|
||||||
""",
|
""",
|
||||||
[publisher_id, token_hash, "test-token", datetime.utcnow().isoformat()],
|
[publisher_id, token_hash, "test-token", datetime.now(timezone.utc).isoformat()],
|
||||||
)
|
)
|
||||||
|
|
||||||
# Insert a public approved tool for tests if missing
|
# Insert a public approved tool for tests if missing
|
||||||
|
|
@ -112,7 +112,7 @@ def auth_headers(app):
|
||||||
[
|
[
|
||||||
"testuser", "tool1", "1.0.0", "Test tool", "Other", "[]",
|
"testuser", "tool1", "1.0.0", "Test tool", "Other", "[]",
|
||||||
"name: tool1\nversion: 1.0.0\n", "", publisher_id,
|
"name: tool1\nversion: 1.0.0\n", "", publisher_id,
|
||||||
"public", "approved", datetime.utcnow().isoformat(),
|
"public", "approved", datetime.now(timezone.utc).isoformat(),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -158,6 +158,44 @@ class TestToolApprovedEndpoint:
|
||||||
assert data['data']['has_approved_public_version'] is False
|
assert data['data']['has_approved_public_version'] is False
|
||||||
|
|
||||||
|
|
||||||
|
@flask_required
|
||||||
|
class TestDeprecationChains:
|
||||||
|
def test_replacement_chain_is_validated_and_returned(self, client, auth_headers):
|
||||||
|
target = client.post(
|
||||||
|
"/api/v1/tools", headers=auth_headers,
|
||||||
|
json={"config": "name: replacement\nversion: 1.0.0\noutput: ok\n"},
|
||||||
|
)
|
||||||
|
assert target.status_code == 201
|
||||||
|
response = client.post(
|
||||||
|
"/api/v1/tools/testuser/tool1/deprecate",
|
||||||
|
headers=auth_headers,
|
||||||
|
json={"deprecated_message": "Moved", "replacement": "replacement"},
|
||||||
|
)
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.get_json()["data"]["replacement_chain"] == [
|
||||||
|
"testuser/replacement"
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_replacement_cycle_is_rejected(self, client, auth_headers):
|
||||||
|
target = client.post(
|
||||||
|
"/api/v1/tools", headers=auth_headers,
|
||||||
|
json={"config": "name: cycle-target\nversion: 1.0.0\noutput: ok\n"},
|
||||||
|
)
|
||||||
|
assert target.status_code == 201
|
||||||
|
assert client.post(
|
||||||
|
"/api/v1/tools/testuser/cycle-target/deprecate",
|
||||||
|
headers=auth_headers,
|
||||||
|
json={"replacement": "tool1"},
|
||||||
|
).status_code == 200
|
||||||
|
response = client.post(
|
||||||
|
"/api/v1/tools/testuser/tool1/deprecate",
|
||||||
|
headers=auth_headers,
|
||||||
|
json={"replacement": "cycle-target"},
|
||||||
|
)
|
||||||
|
assert response.status_code == 400
|
||||||
|
assert "cycle" in response.get_json()["error"]["message"]
|
||||||
|
|
||||||
|
|
||||||
@flask_required
|
@flask_required
|
||||||
class TestPublishPreflightEndpoint:
|
class TestPublishPreflightEndpoint:
|
||||||
def test_dry_run_includes_shared_preflight(self, client, auth_headers):
|
def test_dry_run_includes_shared_preflight(self, client, auth_headers):
|
||||||
|
|
@ -180,7 +218,81 @@ class TestPublishPreflightEndpoint:
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
report = response.get_json()["data"]["preflight"]
|
report = response.get_json()["data"]["preflight"]
|
||||||
assert report["errors"] == []
|
assert report["errors"] == []
|
||||||
assert report["generated_tests"] == []
|
assert report["generated_tests"][0]["state"] == "passed"
|
||||||
|
assert response.get_json()["data"]["quality"]["evidence_coverage"] > 0
|
||||||
|
|
||||||
|
def test_real_publish_persists_immutable_audit(self, client, auth_headers):
|
||||||
|
from cmdforge.registry.db import connect_db
|
||||||
|
|
||||||
|
response = client.post(
|
||||||
|
"/api/v1/tools",
|
||||||
|
headers=auth_headers,
|
||||||
|
json={
|
||||||
|
"config": (
|
||||||
|
"name: audited-tool\nversion: 1.0.0\noutput: stable\n"
|
||||||
|
"input_schema: {}\noutput_schema:\n type: string\n"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert response.status_code == 201
|
||||||
|
conn = connect_db()
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"""
|
||||||
|
SELECT tool_audits.trigger, tool_audits.evidence_json
|
||||||
|
FROM tool_audits JOIN tools ON tools.id = tool_audits.tool_id
|
||||||
|
WHERE tools.owner = 'testuser' AND tools.name = 'audited-tool'
|
||||||
|
"""
|
||||||
|
).fetchall()
|
||||||
|
conn.execute(
|
||||||
|
"""
|
||||||
|
UPDATE tools SET moderation_status = 'approved'
|
||||||
|
WHERE owner = 'testuser' AND name = 'audited-tool'
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0]["trigger"] == "publish"
|
||||||
|
assert "contract_conformance" in rows[0]["evidence_json"]
|
||||||
|
search = client.get("/api/v1/tools/search?q=audited")
|
||||||
|
result = search.get_json()["data"][0]
|
||||||
|
assert isinstance(result["quality_score"], int)
|
||||||
|
assert isinstance(result["quality_coverage"], int)
|
||||||
|
assert result["quality_evaluated_at"]
|
||||||
|
|
||||||
|
def test_real_publish_blocks_deterministic_contract_failure(
|
||||||
|
self, client, auth_headers
|
||||||
|
):
|
||||||
|
response = client.post(
|
||||||
|
"/api/v1/tools",
|
||||||
|
headers=auth_headers,
|
||||||
|
json={
|
||||||
|
"config": (
|
||||||
|
"name: broken-contract\nversion: 1.0.0\noutput: text\n"
|
||||||
|
"input_schema: {}\noutput_schema:\n type: integer\n"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert response.status_code == 400
|
||||||
|
assert response.get_json()["error"]["code"] == "PREFLIGHT_FAILED"
|
||||||
|
|
||||||
|
def test_real_publish_blocks_missing_registry_dependency(
|
||||||
|
self, client, auth_headers
|
||||||
|
):
|
||||||
|
response = client.post(
|
||||||
|
"/api/v1/tools",
|
||||||
|
headers=auth_headers,
|
||||||
|
json={
|
||||||
|
"config": (
|
||||||
|
"name: missing-dependency\nversion: 1.0.0\n"
|
||||||
|
"dependencies:\n - absent-tool\n"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert response.status_code == 400
|
||||||
|
assert "absent-tool" in response.get_json()["error"]["message"]
|
||||||
|
|
||||||
def test_invalid_contract_is_rejected(self, client, auth_headers):
|
def test_invalid_contract_is_rejected(self, client, auth_headers):
|
||||||
response = client.post(
|
response = client.post(
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,91 @@
|
||||||
|
"""Behavioral coverage for registry-aware picker and deprecation UX."""
|
||||||
|
|
||||||
|
from io import StringIO
|
||||||
|
import time
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import cmdforge.cli.picker as picker
|
||||||
|
from cmdforge.cli.picker import PickerResult
|
||||||
|
|
||||||
|
|
||||||
|
class FakeTTY:
|
||||||
|
def __init__(self, *characters):
|
||||||
|
self.characters = iter(characters)
|
||||||
|
|
||||||
|
def getch(self, timeout=None):
|
||||||
|
if timeout is not None:
|
||||||
|
time.sleep(0.01)
|
||||||
|
return None
|
||||||
|
return next(self.characters)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_function():
|
||||||
|
picker._registry_cache.clear()
|
||||||
|
picker._ui_out = StringIO()
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_registry_search_does_not_require_token():
|
||||||
|
class Client:
|
||||||
|
token = ""
|
||||||
|
|
||||||
|
def semantic_search(self, query, limit):
|
||||||
|
return {"available": False, "data": [], "error": None}
|
||||||
|
|
||||||
|
def search_tools(self, query, per_page):
|
||||||
|
return [{
|
||||||
|
"owner": "official", "name": "weather",
|
||||||
|
"description": "Weather", "score": 0.8,
|
||||||
|
"quality_score": 91, "quality_coverage": 70,
|
||||||
|
}]
|
||||||
|
|
||||||
|
with patch("cmdforge.registry_client.get_client", return_value=Client()):
|
||||||
|
results = picker.search_registry("weather")
|
||||||
|
assert results[0]["name"] == "official/weather"
|
||||||
|
assert results[0]["quality_score"] == 91
|
||||||
|
assert results[0]["quality_coverage"] == 70
|
||||||
|
|
||||||
|
|
||||||
|
def test_picker_can_install_when_no_local_tools():
|
||||||
|
registry_tool = {
|
||||||
|
"name": "official/weather", "desc": "Weather", "args": [],
|
||||||
|
"registry": True, "quality_score": 91, "quality_coverage": 70,
|
||||||
|
"relevance": 0.8,
|
||||||
|
}
|
||||||
|
with patch.object(picker, "get_tools", return_value=[]), patch.object(
|
||||||
|
picker, "search_registry", return_value=[registry_tool]
|
||||||
|
), patch.object(
|
||||||
|
picker, "_install_registry_selection",
|
||||||
|
return_value=PickerResult("official/weather", {}),
|
||||||
|
) as install:
|
||||||
|
result = picker.run_picker(FakeTTY("w", "e", "\n"))
|
||||||
|
install.assert_called_once()
|
||||||
|
assert result.tool_name == "official/weather"
|
||||||
|
assert "Q91/70%" in picker._ui_out.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def test_tab_installs_registry_selection_too():
|
||||||
|
registry_tool = {
|
||||||
|
"name": "official/weather", "desc": "Weather", "args": [],
|
||||||
|
"registry": True,
|
||||||
|
}
|
||||||
|
with patch.object(picker, "get_tools", return_value=[]), patch.object(
|
||||||
|
picker, "search_registry", return_value=[registry_tool]
|
||||||
|
), patch.object(
|
||||||
|
picker, "_install_registry_selection",
|
||||||
|
return_value=PickerResult("official/weather", {}),
|
||||||
|
) as install:
|
||||||
|
result = picker.run_picker(FakeTTY("w", "e", "\t"))
|
||||||
|
install.assert_called_once()
|
||||||
|
assert result.tool_name == "official/weather"
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_deprecation_selection_prints_migration_guidance():
|
||||||
|
tool = {
|
||||||
|
"name": "old", "desc": "", "args": [], "deprecated": True,
|
||||||
|
"deprecated_message": "Moved.", "replacement": "official/new",
|
||||||
|
}
|
||||||
|
result = picker._local_selection(tool)
|
||||||
|
assert result.tool_name == "old"
|
||||||
|
output = picker._ui_out.getvalue()
|
||||||
|
assert "Moved." in output
|
||||||
|
assert "official/new" in output
|
||||||
|
|
@ -126,6 +126,71 @@ class TestAnalyzeTool:
|
||||||
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
|
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
|
||||||
assert report.similar_tools[0]["name"] == "official/summary"
|
assert report.similar_tools[0]["name"] == "official/summary"
|
||||||
|
|
||||||
|
def test_server_audit_does_not_claim_skipped_local_checks(self):
|
||||||
|
from cmdforge.tool import Tool, ToolStep
|
||||||
|
|
||||||
|
report = analyze_tool(
|
||||||
|
Tool(name="server", steps=[ToolStep(tool="child", output_var="out")]),
|
||||||
|
check_local_dependencies=False,
|
||||||
|
)
|
||||||
|
checks = report.audit_evidence["checks_run"]
|
||||||
|
assert "dependencies" not in checks
|
||||||
|
assert "toolstep_compatibility" not in checks
|
||||||
|
assert "reuse_opportunities" not in checks
|
||||||
|
assert report.compatibility == []
|
||||||
|
|
||||||
|
def test_unrelated_code_steps_are_not_reuse_evidence(self):
|
||||||
|
from cmdforge.tool import CodeStep, Tool
|
||||||
|
|
||||||
|
current = Tool(
|
||||||
|
name="current", input_schema={}, output_schema={},
|
||||||
|
steps=[
|
||||||
|
CodeStep(code="x = 1", output_var="x"),
|
||||||
|
CodeStep(code="y = 2", output_var="y"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
other = Tool(
|
||||||
|
name="other", input_schema={}, output_schema={},
|
||||||
|
steps=[
|
||||||
|
CodeStep(code="a = 99", output_var="a"),
|
||||||
|
CodeStep(code="b = 100", output_var="b"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
|
||||||
|
"cmdforge.tool.load_tool", return_value=other
|
||||||
|
):
|
||||||
|
report = analyze_tool(current)
|
||||||
|
assert report.reuse_opportunities == []
|
||||||
|
|
||||||
|
def test_exact_contracted_duplicate_has_evidence(self):
|
||||||
|
from cmdforge.tool import PromptStep, Tool
|
||||||
|
|
||||||
|
def make(name):
|
||||||
|
return Tool(
|
||||||
|
name=name, input_schema={}, output_schema={"type": "string"},
|
||||||
|
steps=[
|
||||||
|
PromptStep(
|
||||||
|
prompt="first", provider="mock", output_var="one",
|
||||||
|
output_schema={"type": "string"},
|
||||||
|
),
|
||||||
|
PromptStep(
|
||||||
|
prompt="second", provider="mock", output_var="two",
|
||||||
|
output_schema={"type": "string"},
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
|
||||||
|
"cmdforge.tool.load_tool", return_value=make("other")
|
||||||
|
):
|
||||||
|
report = analyze_tool(make("current"))
|
||||||
|
duplicate = [
|
||||||
|
item for item in report.reuse_opportunities
|
||||||
|
if item["type"] == "duplicate_sequence"
|
||||||
|
]
|
||||||
|
assert duplicate
|
||||||
|
assert "explicit tool contracts" in duplicate[0]["evidence"]
|
||||||
|
|
||||||
def test_registry_error_becomes_warning(self):
|
def test_registry_error_becomes_warning(self):
|
||||||
class Client:
|
class Client:
|
||||||
def search_tools(self, query, per_page):
|
def search_tools(self, query, per_page):
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,10 @@ class TestCategoryScore:
|
||||||
cs = CategoryScore("X", 0, 0)
|
cs = CategoryScore("X", 0, 0)
|
||||||
assert cs.percentage == 0.0
|
assert cs.percentage == 0.0
|
||||||
|
|
||||||
|
def test_display_distinguishes_missing_evidence(self):
|
||||||
|
assert CategoryScore("Tests", 0, 30, "not_tested").display == "not tested"
|
||||||
|
assert CategoryScore("Tests", 0, 30, "not_applicable").display == "n/a"
|
||||||
|
|
||||||
|
|
||||||
class TestQualityScore:
|
class TestQualityScore:
|
||||||
def test_to_dict(self):
|
def test_to_dict(self):
|
||||||
|
|
@ -55,6 +59,40 @@ class TestComputeQuality:
|
||||||
# Contracts should be not_tested
|
# Contracts should be not_tested
|
||||||
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
||||||
assert contracts.state == "not_tested"
|
assert contracts.state == "not_tested"
|
||||||
|
assert qs.headline == 0
|
||||||
|
assert qs.evidence_coverage == 0
|
||||||
|
|
||||||
|
def test_missing_categories_do_not_reduce_checked_score(self):
|
||||||
|
tool = Tool(name="contracted", input_schema={}, output_schema={})
|
||||||
|
report = PreflightReport()
|
||||||
|
qs = compute_quality(tool, report)
|
||||||
|
assert qs.headline == 100
|
||||||
|
assert qs.evidence_coverage == 15
|
||||||
|
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
|
||||||
|
assert contracts.state == "checked"
|
||||||
|
|
||||||
|
def test_unsupported_tests_are_not_scored_as_failures(self):
|
||||||
|
report = PreflightReport(generated_tests=[{
|
||||||
|
"step": "preflight", "state": "unsupported", "detail": "safe mode"
|
||||||
|
}])
|
||||||
|
tests = [
|
||||||
|
c for c in compute_quality(Tool(name="tool"), report).categories
|
||||||
|
if c.name == "Deterministic tests"
|
||||||
|
][0]
|
||||||
|
assert tests.state == "not_tested"
|
||||||
|
assert tests.display == "not tested"
|
||||||
|
|
||||||
|
def test_partial_unsupported_tests_reduce_coverage_not_score(self):
|
||||||
|
report = PreflightReport(generated_tests=[
|
||||||
|
{"step": "case", "state": "passed", "detail": "ok"},
|
||||||
|
{"step": "generation", "state": "unsupported", "detail": "unknown"},
|
||||||
|
])
|
||||||
|
tests = [
|
||||||
|
c for c in compute_quality(Tool(name="tool"), report).categories
|
||||||
|
if c.name == "Deterministic tests"
|
||||||
|
][0]
|
||||||
|
assert tests.earned == tests.available == 15
|
||||||
|
assert tests.possible == 30
|
||||||
|
|
||||||
def test_with_contracts_scores_higher(self):
|
def test_with_contracts_scores_higher(self):
|
||||||
tool = Tool(
|
tool = Tool(
|
||||||
|
|
@ -127,6 +165,17 @@ class TestComputeQuality:
|
||||||
security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
|
security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
|
||||||
assert security.earned == 15 # 20 - 5
|
assert security.earned == 15 # 20 - 5
|
||||||
|
|
||||||
|
def test_skipped_dependency_check_is_not_claimed_as_security_evidence(self):
|
||||||
|
tool = Tool(name="server")
|
||||||
|
report = PreflightReport(audit_evidence={"checks_run": ["secrets"]})
|
||||||
|
security = [
|
||||||
|
c for c in compute_quality(tool, report).categories
|
||||||
|
if c.name == "Security scrutiny"
|
||||||
|
][0]
|
||||||
|
assert security.earned == 10
|
||||||
|
assert security.available == 10
|
||||||
|
assert security.possible == 20
|
||||||
|
|
||||||
def test_community_not_tested_without_data(self):
|
def test_community_not_tested_without_data(self):
|
||||||
tool = Tool(name="new")
|
tool = Tool(name="new")
|
||||||
report = PreflightReport()
|
report = PreflightReport()
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
"""Tests for immutable registry audit refreshes."""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from cmdforge.registry.auditing import audit_stale_tools, audit_tool_version
|
||||||
|
from cmdforge.registry.db import connect_db, init_db
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def audit_db(tmp_path):
|
||||||
|
conn = connect_db(tmp_path / "registry.db")
|
||||||
|
init_db(conn)
|
||||||
|
cursor = conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO publishers (email, password_hash, slug, display_name)
|
||||||
|
VALUES ('audit@example.com', 'x', 'auditor', 'Auditor')
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
publisher_id = cursor.lastrowid
|
||||||
|
tool = conn.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO tools (
|
||||||
|
owner, name, version, config_yaml, publisher_id,
|
||||||
|
visibility, moderation_status
|
||||||
|
) VALUES ('auditor', 'checked', '1.0.0', ?, ?, 'public', 'approved')
|
||||||
|
""",
|
||||||
|
[
|
||||||
|
"name: checked\nversion: 1.0.0\noutput: stable\n"
|
||||||
|
"input_schema: {}\noutput_schema:\n type: string\n",
|
||||||
|
publisher_id,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
yield conn, tool.lastrowid
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_background_audits_append_instead_of_overwriting(audit_db):
|
||||||
|
conn, tool_id = audit_db
|
||||||
|
first = audit_tool_version(conn, tool_id)
|
||||||
|
second = audit_tool_version(conn, tool_id)
|
||||||
|
count = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM tool_audits WHERE tool_id = ?", [tool_id]
|
||||||
|
).fetchone()[0]
|
||||||
|
assert count == 2
|
||||||
|
assert first["evidence"]["findings_hash"] == second["evidence"]["findings_hash"]
|
||||||
|
assert "generated_tests" in first["evidence"]["findings"]
|
||||||
|
assert "dependencies" not in first["evidence"]["checks_run"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_audit_refresh_adds_new_evidence(audit_db):
|
||||||
|
conn, tool_id = audit_db
|
||||||
|
audit_tool_version(conn, tool_id)
|
||||||
|
old = (datetime.now(timezone.utc) - timedelta(days=60)).isoformat()
|
||||||
|
conn.execute("UPDATE tool_audits SET evaluated_at = ?", [old])
|
||||||
|
conn.commit()
|
||||||
|
results = audit_stale_tools(conn, max_age_days=30)
|
||||||
|
assert [item["tool_id"] for item in results] == [tool_id]
|
||||||
|
assert conn.execute("SELECT COUNT(*) FROM tool_audits").fetchone()[0] == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_degraded_background_score_is_logged(audit_db):
|
||||||
|
conn, tool_id = audit_db
|
||||||
|
conn.execute("UPDATE tools SET downloads = 500 WHERE id = ?", [tool_id])
|
||||||
|
conn.commit()
|
||||||
|
audit_tool_version(conn, tool_id)
|
||||||
|
conn.execute("UPDATE tools SET downloads = 0 WHERE id = ?", [tool_id])
|
||||||
|
conn.commit()
|
||||||
|
result = audit_tool_version(conn, tool_id)
|
||||||
|
assert result["quality_change"] < 0
|
||||||
|
log = conn.execute(
|
||||||
|
"SELECT action FROM audit_log WHERE target_id = ?", [str(tool_id)]
|
||||||
|
).fetchone()
|
||||||
|
assert log["action"] == "automated_audit_changed"
|
||||||
|
|
@ -685,3 +685,30 @@ class TestAgentContext:
|
||||||
)
|
)
|
||||||
restored = PromptStep.from_dict(step.to_dict())
|
restored = PromptStep.from_dict(step.to_dict())
|
||||||
assert restored.skills == ["python"]
|
assert restored.skills == ["python"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeprecationFields:
|
||||||
|
def test_round_trip_and_warning(self):
|
||||||
|
from cmdforge.tool import deprecation_warning
|
||||||
|
|
||||||
|
tool = Tool(
|
||||||
|
name="old", deprecated=True, deprecated_message="Moved.",
|
||||||
|
replacement="official/new",
|
||||||
|
)
|
||||||
|
restored = Tool.from_dict(tool.to_dict())
|
||||||
|
assert restored.deprecated is True
|
||||||
|
assert restored.replacement == "official/new"
|
||||||
|
assert "official/new" in deprecation_warning(restored)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"values",
|
||||||
|
[
|
||||||
|
{"deprecated": "false"},
|
||||||
|
{"deprecated_message": 123},
|
||||||
|
{"replacement": "../../escape"},
|
||||||
|
{"replacement": "owner/name/extra"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_invalid_deprecation_metadata_is_rejected(self, values):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
Tool(name="old", **values)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue