Rename project from SmartTools to CmdForge
Major rename operation: - Rename Python package: smarttools -> cmdforge - Update CLI entry point: smarttools -> cmdforge - Update all imports and module references - Update pyproject.toml with new name, URLs, entry point - Update all HTML templates with new branding - Update documentation (CLAUDE.md, README.md, docs/*, wiki/*) - Update environment variables: - SMARTTOOLS_ENV -> CMDFORGE_ENV - SMARTTOOLS_REGISTRY_DB -> CMDFORGE_REGISTRY_DB - SMARTTOOLS_TOKEN -> CMDFORGE_TOKEN - SMARTTOOLS_REGISTRY_* -> CMDFORGE_REGISTRY_* - Update Dockerfile and docker-compose.yml - Update tests to use new package name - Update scripts and examples - Update package.json and tailwind.config.js All 158 unit tests pass. The CLI is working correctly with the new cmdforge command. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
Claude Opus 4.5
@@ -0,0 +1,3 @@
|
||||
"""Registry API server package."""
|
||||
|
||||
__all__ = ["app"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
"""Category suggestion helpers for registry tools."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def load_categories(categories_path: Path) -> List[Dict]:
|
||||
data = yaml.safe_load(categories_path.read_text(encoding="utf-8")) or {}
|
||||
return data.get("categories", [])
|
||||
|
||||
|
||||
def suggest_categories(
|
||||
name: str,
|
||||
description: str,
|
||||
tags: List[str],
|
||||
categories_path: Path,
|
||||
) -> List[Tuple[str, float]]:
|
||||
"""Suggest categories ranked by confidence.
|
||||
|
||||
Uses keyword matching against name/description/tags.
|
||||
Returns a list of (category_name, confidence).
|
||||
"""
|
||||
categories = load_categories(categories_path)
|
||||
text = f"{name} {description} {' '.join(tags)}".lower()
|
||||
suggestions: List[Tuple[str, float]] = []
|
||||
|
||||
for cat in categories:
|
||||
cat_name = cat.get("name")
|
||||
keywords = [str(k).lower() for k in cat.get("keywords", []) if k]
|
||||
if not cat_name or not keywords:
|
||||
continue
|
||||
hits = sum(1 for k in keywords if k in text)
|
||||
confidence = hits / max(len(keywords), 1)
|
||||
if hits:
|
||||
suggestions.append((cat_name, round(confidence, 3)))
|
||||
|
||||
suggestions.sort(key=lambda item: item[1], reverse=True)
|
||||
return suggestions
|
||||
@@ -0,0 +1,276 @@
|
||||
"""SQLite storage and schema setup for the registry API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS publishers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email TEXT UNIQUE NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
slug TEXT UNIQUE NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
bio TEXT,
|
||||
website TEXT,
|
||||
verified BOOLEAN DEFAULT FALSE,
|
||||
locked_until TIMESTAMP,
|
||||
failed_login_attempts INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS api_tokens (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
publisher_id INTEGER NOT NULL REFERENCES publishers(id),
|
||||
token_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
last_used_at TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
revoked_at TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tools (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
owner TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
description TEXT,
|
||||
category TEXT,
|
||||
tags TEXT,
|
||||
config_yaml TEXT NOT NULL,
|
||||
readme TEXT,
|
||||
publisher_id INTEGER NOT NULL REFERENCES publishers(id),
|
||||
deprecated BOOLEAN DEFAULT FALSE,
|
||||
deprecated_message TEXT,
|
||||
replacement TEXT,
|
||||
downloads INTEGER DEFAULT 0,
|
||||
scrutiny_status TEXT DEFAULT 'pending',
|
||||
scrutiny_report TEXT,
|
||||
published_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(owner, name, version)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS download_stats (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tool_id INTEGER NOT NULL REFERENCES tools(id),
|
||||
client_id TEXT NOT NULL,
|
||||
downloaded_at DATE NOT NULL,
|
||||
UNIQUE(tool_id, client_id, downloaded_at)
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS tools_fts USING fts5(
|
||||
name, description, tags, readme,
|
||||
content='tools',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS tools_ai AFTER INSERT ON tools BEGIN
|
||||
INSERT INTO tools_fts(rowid, name, description, tags, readme)
|
||||
VALUES (new.id, new.name, new.description, new.tags, new.readme);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS tools_ad AFTER DELETE ON tools BEGIN
|
||||
INSERT INTO tools_fts(tools_fts, rowid, name, description, tags, readme)
|
||||
VALUES ('delete', old.id, old.name, old.description, old.tags, old.readme);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS tools_au AFTER UPDATE ON tools BEGIN
|
||||
INSERT INTO tools_fts(tools_fts, rowid, name, description, tags, readme)
|
||||
VALUES ('delete', old.id, old.name, old.description, old.tags, old.readme);
|
||||
INSERT INTO tools_fts(rowid, name, description, tags, readme)
|
||||
VALUES (new.id, new.name, new.description, new.tags, new.readme);
|
||||
END;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pending_prs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
publisher_id INTEGER NOT NULL REFERENCES publishers(id),
|
||||
owner TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
pr_number INTEGER NOT NULL,
|
||||
pr_url TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(owner, name, version)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS webhook_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
delivery_id TEXT UNIQUE NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS web_sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT UNIQUE NOT NULL,
|
||||
publisher_id INTEGER REFERENCES publishers(id),
|
||||
data TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at TIMESTAMP NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tools_owner_name ON tools(owner, name);
|
||||
CREATE INDEX IF NOT EXISTS idx_tools_category ON tools(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_tools_published_at ON tools(published_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_tools_downloads ON tools(downloads DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_tools_owner_name_version ON tools(owner, name, version);
|
||||
CREATE INDEX IF NOT EXISTS idx_tools_sort_stable ON tools(downloads DESC, published_at DESC, id DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_publishers_slug ON publishers(slug);
|
||||
CREATE INDEX IF NOT EXISTS idx_publishers_email ON publishers(email);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_api_tokens_hash ON api_tokens(token_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_api_tokens_publisher ON api_tokens(publisher_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_web_sessions_id ON web_sessions(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_web_sessions_expires ON web_sessions(expires_at);
|
||||
|
||||
-- Web UI tables (Phase 7)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS announcements (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT NOT NULL,
|
||||
body TEXT NOT NULL,
|
||||
published BOOLEAN DEFAULT FALSE,
|
||||
published_at TIMESTAMP,
|
||||
created_by INTEGER REFERENCES publishers(id),
|
||||
updated_by INTEGER REFERENCES publishers(id),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS featured_tools (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tool_id INTEGER NOT NULL REFERENCES tools(id),
|
||||
placement TEXT NOT NULL DEFAULT 'homepage',
|
||||
priority INTEGER DEFAULT 0,
|
||||
start_at TIMESTAMP,
|
||||
end_at TIMESTAMP,
|
||||
status TEXT DEFAULT 'active',
|
||||
created_by INTEGER REFERENCES publishers(id),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS featured_contributors (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
publisher_id INTEGER NOT NULL REFERENCES publishers(id),
|
||||
bio_override TEXT,
|
||||
placement TEXT NOT NULL DEFAULT 'homepage',
|
||||
start_at TIMESTAMP,
|
||||
end_at TIMESTAMP,
|
||||
status TEXT DEFAULT 'active',
|
||||
created_by INTEGER REFERENCES publishers(id),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS reports (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tool_id INTEGER NOT NULL REFERENCES tools(id),
|
||||
reporter_id INTEGER REFERENCES publishers(id),
|
||||
reporter_ip TEXT,
|
||||
reason TEXT NOT NULL,
|
||||
details TEXT,
|
||||
status TEXT DEFAULT 'pending',
|
||||
resolved_by INTEGER REFERENCES publishers(id),
|
||||
resolved_at TIMESTAMP,
|
||||
resolution_note TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS consents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
client_id TEXT,
|
||||
publisher_id INTEGER REFERENCES publishers(id),
|
||||
analytics_consent BOOLEAN DEFAULT FALSE,
|
||||
ads_consent BOOLEAN DEFAULT FALSE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(client_id),
|
||||
UNIQUE(publisher_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS content_pages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
slug TEXT UNIQUE NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
content_type TEXT NOT NULL DEFAULT 'doc',
|
||||
body TEXT,
|
||||
published BOOLEAN DEFAULT FALSE,
|
||||
published_at TIMESTAMP,
|
||||
created_by INTEGER REFERENCES publishers(id),
|
||||
updated_by INTEGER REFERENCES publishers(id),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_announcements_published ON announcements(published, published_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_featured_tools_placement ON featured_tools(placement, status, priority DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_featured_contributors_placement ON featured_contributors(placement, status);
|
||||
CREATE INDEX IF NOT EXISTS idx_reports_status ON reports(status, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_content_pages_type ON content_pages(content_type, published);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pageviews (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
path TEXT NOT NULL,
|
||||
referrer TEXT,
|
||||
user_agent TEXT,
|
||||
ip_hash TEXT,
|
||||
viewed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pageviews_path ON pageviews(path, viewed_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_pageviews_date ON pageviews(date(viewed_at), path);
|
||||
"""
|
||||
|
||||
|
||||
def get_db_path() -> Path:
|
||||
default_path = Path.home() / ".cmdforge" / "registry" / "registry.db"
|
||||
return Path(os.environ.get("CMDFORGE_REGISTRY_DB", default_path))
|
||||
|
||||
|
||||
def ensure_db_directory(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def connect_db(path: Path | None = None) -> sqlite3.Connection:
|
||||
db_path = path or get_db_path()
|
||||
ensure_db_directory(db_path)
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
# WAL mode provides better concurrency but may not work on all filesystems
|
||||
# (e.g., some FUSE filesystems like mergerfs). Fall back gracefully.
|
||||
try:
|
||||
conn.execute("PRAGMA journal_mode=WAL;")
|
||||
except sqlite3.OperationalError:
|
||||
# Fall back to DELETE journal mode which works everywhere
|
||||
conn.execute("PRAGMA journal_mode=DELETE;")
|
||||
conn.execute("PRAGMA foreign_keys=ON;")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def query_one(conn: sqlite3.Connection, sql: str, params: Iterable | None = None):
|
||||
cur = conn.execute(sql, params or [])
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def query_all(conn: sqlite3.Connection, sql: str, params: Iterable | None = None):
|
||||
cur = conn.execute(sql, params or [])
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
def execute(conn: sqlite3.Connection, sql: str, params: Iterable | None = None) -> None:
|
||||
conn.execute(sql, params or [])
|
||||
conn.commit()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""In-memory rate limiting for the registry API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Tuple
|
||||
|
||||
|
||||
@dataclass
|
||||
class RateLimitState:
|
||||
reset_at: float
|
||||
count: int
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self) -> None:
|
||||
self._state: Dict[Tuple[str, str], RateLimitState] = {}
|
||||
|
||||
def check(self, scope_key: str, limit: int, window_seconds: int) -> Tuple[bool, RateLimitState]:
|
||||
now = time.time()
|
||||
key = (scope_key, str(window_seconds))
|
||||
state = self._state.get(key)
|
||||
if not state or now >= state.reset_at:
|
||||
state = RateLimitState(reset_at=now + window_seconds, count=0)
|
||||
self._state[key] = state
|
||||
state.count += 1
|
||||
allowed = state.count <= limit
|
||||
return allowed, state
|
||||
@@ -0,0 +1,588 @@
|
||||
"""Tool scrutiny system for automated review of published tools.
|
||||
|
||||
This module analyzes tools on publish to verify:
|
||||
1. Honesty - Does the tool do what its description claims?
|
||||
2. Transparency - Is behavior visible or hidden/obfuscated?
|
||||
3. Scope - Does the code stay within expected bounds?
|
||||
4. Efficiency - Are AI calls necessary or wasteful?
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class CheckResult(Enum):
|
||||
PASS = "pass"
|
||||
WARNING = "warning"
|
||||
FAIL = "fail"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Finding:
|
||||
"""A single finding from scrutiny analysis."""
|
||||
check: str
|
||||
result: CheckResult
|
||||
message: str
|
||||
suggestion: Optional[str] = None
|
||||
location: Optional[str] = None # e.g., "step 2", "code block"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScrutinyReport:
|
||||
"""Complete scrutiny report for a tool."""
|
||||
tool_name: str
|
||||
findings: List[Finding] = field(default_factory=list)
|
||||
optimizations: List[Any] = field(default_factory=list) # OptimizationSuggestion list
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
"""Tool passes if no FAIL results."""
|
||||
return not any(f.result == CheckResult.FAIL for f in self.findings)
|
||||
|
||||
@property
|
||||
def has_warnings(self) -> bool:
|
||||
return any(f.result == CheckResult.WARNING for f in self.findings)
|
||||
|
||||
@property
|
||||
def decision(self) -> str:
|
||||
"""Auto-approve, flag for review, or reject."""
|
||||
if any(f.result == CheckResult.FAIL for f in self.findings):
|
||||
return "reject"
|
||||
if any(f.result == CheckResult.WARNING for f in self.findings):
|
||||
return "review"
|
||||
return "approve"
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
result = {
|
||||
"tool_name": self.tool_name,
|
||||
"decision": self.decision,
|
||||
"passed": self.passed,
|
||||
"has_warnings": self.has_warnings,
|
||||
"findings": [
|
||||
{
|
||||
"check": f.check,
|
||||
"result": f.result.value,
|
||||
"message": f.message,
|
||||
"suggestion": f.suggestion,
|
||||
"location": f.location,
|
||||
}
|
||||
for f in self.findings
|
||||
],
|
||||
}
|
||||
|
||||
# Add optimization suggestions if any
|
||||
if self.optimizations:
|
||||
result["optimizations"] = [
|
||||
{
|
||||
"operation": opt.operation,
|
||||
"current_prompt": opt.current_prompt,
|
||||
"optimized_code": opt.optimized_code,
|
||||
"tradeoffs": opt.tradeoffs,
|
||||
"location": opt.location,
|
||||
"action": "optional",
|
||||
}
|
||||
for opt in self.optimizations
|
||||
]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# Keywords that indicate specific behaviors
|
||||
BEHAVIOR_KEYWORDS = {
|
||||
"summarize": ["summary", "summarize", "condense", "brief", "shorten", "tldr"],
|
||||
"translate": ["translate", "translation", "language", "convert to"],
|
||||
"explain": ["explain", "clarify", "describe", "what is", "how does"],
|
||||
"fix": ["fix", "correct", "repair", "grammar", "spelling"],
|
||||
"generate": ["generate", "create", "write", "produce", "make"],
|
||||
"extract": ["extract", "pull", "find", "identify", "parse"],
|
||||
"analyze": ["analyze", "review", "examine", "check", "audit"],
|
||||
"convert": ["convert", "transform", "format", "change to"],
|
||||
}
|
||||
|
||||
# Patterns that suggest obfuscation or suspicious behavior
|
||||
SUSPICIOUS_PATTERNS = [
|
||||
(r'base64\.(b64decode|decodebytes)', "Base64 decoding detected"),
|
||||
(r'exec\s*\(', "Dynamic code execution with exec()"),
|
||||
(r'eval\s*\(', "Dynamic code execution with eval()"),
|
||||
(r'__import__\s*\(', "Dynamic import detected"),
|
||||
(r'subprocess\.(run|call|Popen)', "Subprocess execution detected"),
|
||||
(r'os\.(system|popen|exec)', "OS command execution detected"),
|
||||
(r'requests\.(get|post|put|delete)', "Network request detected"),
|
||||
(r'urllib', "URL library usage detected"),
|
||||
(r'socket\.', "Raw socket usage detected"),
|
||||
(r'\\x[0-9a-fA-F]{2}', "Hex-encoded strings detected"),
|
||||
]
|
||||
|
||||
# File/system access patterns
|
||||
SCOPE_PATTERNS = [
|
||||
(r'open\s*\([^)]*["\']/', "Absolute path file access"),
|
||||
(r'open\s*\([^)]*~', "Home directory file access"),
|
||||
(r'\.ssh', "SSH directory access"),
|
||||
(r'\.aws', "AWS credentials access"),
|
||||
(r'\.env', "Environment file access"),
|
||||
(r'/etc/', "System config access"),
|
||||
(r'os\.environ', "Environment variable access"),
|
||||
(r'keyring|password|credential', "Credential-related access"),
|
||||
]
|
||||
|
||||
# Operations that could be done with code (with tradeoffs)
|
||||
# Format: (prompt_pattern, operation_name, code_template, code_benefit, ai_benefit)
|
||||
CODE_OPTIMIZATIONS = [
|
||||
(
|
||||
r'count (the |how many )?words',
|
||||
"Word counting",
|
||||
'word_count = len(input.split())',
|
||||
"Faster, no API cost, deterministic",
|
||||
"Handles edge cases (hyphenated words, contractions, numbers as words)",
|
||||
),
|
||||
(
|
||||
r'count (the |how many )?(lines|line)',
|
||||
"Line counting",
|
||||
'line_count = len(input.strip().split("\\n"))',
|
||||
"Instant, no API cost",
|
||||
"Could interpret 'meaningful lines' vs blank lines",
|
||||
),
|
||||
(
|
||||
r'count (the |how many )?(characters?|chars?)',
|
||||
"Character counting",
|
||||
'char_count = len(input)',
|
||||
"Instant, no API cost",
|
||||
"Could exclude whitespace or count 'visible' characters",
|
||||
),
|
||||
(
|
||||
r'convert.*(to|into) json',
|
||||
"JSON conversion",
|
||||
'import json\nresult = json.dumps(data, indent=2)',
|
||||
"Reliable, no API cost, handles valid input perfectly",
|
||||
"Can interpret messy/partial data and fix formatting issues",
|
||||
),
|
||||
(
|
||||
r'parse.*(json|the json)',
|
||||
"JSON parsing",
|
||||
'import json\nresult = json.loads(input)',
|
||||
"Fast, no API cost, strict validation",
|
||||
"Can fix malformed JSON, handle comments, trailing commas",
|
||||
),
|
||||
(
|
||||
r'convert.*(to|into) csv',
|
||||
"CSV conversion",
|
||||
'import csv\nimport io\n# ... csv.writer conversion',
|
||||
"Reliable for structured data",
|
||||
"Can infer structure from unstructured text",
|
||||
),
|
||||
(
|
||||
r'(extract|find|get) (all )?(emails?|email addresses)',
|
||||
"Email extraction",
|
||||
'import re\nemails = re.findall(r"[\\w.+-]+@[\\w.-]+\\.[a-zA-Z]{2,}", input)',
|
||||
"Fast, consistent pattern matching",
|
||||
"Handles obfuscated emails (user [at] domain), context-aware",
|
||||
),
|
||||
(
|
||||
r'(extract|find|get) (all )?(urls?|links?)',
|
||||
"URL extraction",
|
||||
'import re\nurls = re.findall(r"https?://[^\\s<>\"]+", input)',
|
||||
"Fast, consistent pattern matching",
|
||||
"Handles partial URLs, context-aware link detection",
|
||||
),
|
||||
(
|
||||
r'(uppercase|to upper|make upper)',
|
||||
"Uppercase conversion",
|
||||
'result = input.upper()',
|
||||
"Instant, no API cost",
|
||||
"Language-aware (Turkish İ/i, Greek σ/ς edge cases)",
|
||||
),
|
||||
(
|
||||
r'(lowercase|to lower|make lower)',
|
||||
"Lowercase conversion",
|
||||
'result = input.lower()',
|
||||
"Instant, no API cost",
|
||||
"Language-aware case handling",
|
||||
),
|
||||
(
|
||||
r'(capitalize|title case)',
|
||||
"Title case conversion",
|
||||
'result = input.title()',
|
||||
"Instant, no API cost",
|
||||
"Knows style rules (articles, prepositions)",
|
||||
),
|
||||
(
|
||||
r'(reverse|reversed?) (the |this )?(text|string|order)',
|
||||
"Text reversal",
|
||||
'result = input[::-1]',
|
||||
"Instant, no API cost",
|
||||
"Could reverse by word, sentence, or paragraph intelligently",
|
||||
),
|
||||
(
|
||||
r'(sort|sorted?) (the |these |this )?(lines?|items?|list)',
|
||||
"Sorting",
|
||||
'result = "\\n".join(sorted(input.strip().split("\\n")))',
|
||||
"Fast, deterministic alphabetical sort",
|
||||
"Can sort by meaning ('sort by importance', 'by date mentioned')",
|
||||
),
|
||||
(
|
||||
r'remove (duplicate|duplicated?) (lines?|items?)',
|
||||
"Deduplication",
|
||||
'seen = set()\nresult = "\\n".join(x for x in input.split("\\n") if not (x in seen or seen.add(x)))',
|
||||
"Fast, exact matching",
|
||||
"Can detect semantic duplicates ('USA' vs 'United States')",
|
||||
),
|
||||
(
|
||||
r'(remove|strip|trim) (whitespace|spaces|blank)',
|
||||
"Whitespace removal",
|
||||
'result = " ".join(input.split())',
|
||||
"Instant, predictable",
|
||||
"Could preserve meaningful spacing, format-aware",
|
||||
),
|
||||
(
|
||||
r'split (by|on|into)',
|
||||
"Text splitting",
|
||||
'# result = input.split(delimiter)',
|
||||
"Fast, exact delimiter matching",
|
||||
"Can split by meaning ('split into paragraphs about each topic')",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def analyze_tool(config_yaml: str, description: str) -> ScrutinyReport:
|
||||
"""Run full scrutiny analysis on a tool.
|
||||
|
||||
Args:
|
||||
config_yaml: The tool's YAML configuration
|
||||
description: The tool's description
|
||||
|
||||
Returns:
|
||||
ScrutinyReport with all findings
|
||||
"""
|
||||
try:
|
||||
config = yaml.safe_load(config_yaml) or {}
|
||||
except yaml.YAMLError:
|
||||
return ScrutinyReport(
|
||||
tool_name="unknown",
|
||||
findings=[Finding(
|
||||
check="parse",
|
||||
result=CheckResult.FAIL,
|
||||
message="Invalid YAML configuration",
|
||||
)]
|
||||
)
|
||||
|
||||
tool_name = config.get("name", "unknown")
|
||||
report = ScrutinyReport(tool_name=tool_name)
|
||||
|
||||
# Run all checks
|
||||
report.findings.extend(check_honesty(config, description))
|
||||
report.findings.extend(check_transparency(config))
|
||||
report.findings.extend(check_scope(config))
|
||||
|
||||
# Efficiency check returns findings and optimizations
|
||||
efficiency_findings, optimizations = check_efficiency(config)
|
||||
report.findings.extend(efficiency_findings)
|
||||
report.optimizations = optimizations
|
||||
|
||||
# If no findings, add a pass
|
||||
if not report.findings:
|
||||
report.findings.append(Finding(
|
||||
check="overall",
|
||||
result=CheckResult.PASS,
|
||||
message="All checks passed",
|
||||
))
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def check_honesty(config: Dict[str, Any], description: str) -> List[Finding]:
|
||||
"""Check if tool behavior matches its description.
|
||||
|
||||
Analyzes prompts and compares against description claims.
|
||||
"""
|
||||
findings = []
|
||||
steps = config.get("steps", [])
|
||||
description_lower = description.lower()
|
||||
|
||||
# Extract all prompt text
|
||||
all_prompts = ""
|
||||
for step in steps:
|
||||
if step.get("type") == "prompt":
|
||||
all_prompts += " " + (step.get("prompt") or "").lower()
|
||||
|
||||
# Detect what the description claims
|
||||
claimed_behaviors = set()
|
||||
for behavior, keywords in BEHAVIOR_KEYWORDS.items():
|
||||
if any(kw in description_lower for kw in keywords):
|
||||
claimed_behaviors.add(behavior)
|
||||
|
||||
# Detect what prompts actually do
|
||||
actual_behaviors = set()
|
||||
for behavior, keywords in BEHAVIOR_KEYWORDS.items():
|
||||
if any(kw in all_prompts for kw in keywords):
|
||||
actual_behaviors.add(behavior)
|
||||
|
||||
# Check for mismatches
|
||||
claimed_not_done = claimed_behaviors - actual_behaviors
|
||||
done_not_claimed = actual_behaviors - claimed_behaviors
|
||||
|
||||
if claimed_not_done:
|
||||
findings.append(Finding(
|
||||
check="honesty",
|
||||
result=CheckResult.WARNING,
|
||||
message=f"Description claims '{', '.join(claimed_not_done)}' but prompts don't reflect this",
|
||||
suggestion="Update description to accurately reflect tool behavior",
|
||||
))
|
||||
|
||||
if done_not_claimed and claimed_behaviors:
|
||||
# Only warn if tool claims to do something specific but also does other things
|
||||
findings.append(Finding(
|
||||
check="honesty",
|
||||
result=CheckResult.WARNING,
|
||||
message=f"Prompts include '{', '.join(done_not_claimed)}' not mentioned in description",
|
||||
suggestion="Update description to include all tool capabilities",
|
||||
))
|
||||
|
||||
# Check for empty or minimal prompts
|
||||
for i, step in enumerate(steps):
|
||||
if step.get("type") == "prompt":
|
||||
prompt = step.get("prompt", "").strip()
|
||||
if len(prompt) < 10:
|
||||
findings.append(Finding(
|
||||
check="honesty",
|
||||
result=CheckResult.WARNING,
|
||||
message="Prompt is unusually short",
|
||||
location=f"step {i + 1}",
|
||||
))
|
||||
|
||||
if not findings:
|
||||
findings.append(Finding(
|
||||
check="honesty",
|
||||
result=CheckResult.PASS,
|
||||
message="Description matches observed behavior",
|
||||
))
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def check_transparency(config: Dict[str, Any]) -> List[Finding]:
|
||||
"""Check for obfuscated or hidden behavior.
|
||||
|
||||
Looks for encoded strings, dynamic execution, etc.
|
||||
"""
|
||||
findings = []
|
||||
steps = config.get("steps", [])
|
||||
|
||||
for i, step in enumerate(steps):
|
||||
if step.get("type") == "code":
|
||||
code = step.get("code", "")
|
||||
location = f"step {i + 1}"
|
||||
|
||||
# Check for suspicious patterns
|
||||
for pattern, description in SUSPICIOUS_PATTERNS:
|
||||
if re.search(pattern, code, re.IGNORECASE):
|
||||
# Determine severity
|
||||
if "exec" in pattern or "eval" in pattern:
|
||||
result = CheckResult.FAIL
|
||||
msg = f"{description} - potential code injection risk"
|
||||
elif "socket" in pattern or "subprocess" in pattern:
|
||||
result = CheckResult.WARNING
|
||||
msg = f"{description} - flagged for manual review"
|
||||
else:
|
||||
result = CheckResult.WARNING
|
||||
msg = description
|
||||
|
||||
findings.append(Finding(
|
||||
check="transparency",
|
||||
result=result,
|
||||
message=msg,
|
||||
location=location,
|
||||
))
|
||||
|
||||
# Check for obfuscated variable names
|
||||
obfuscated = re.findall(r'\b([a-z])\1{2,}\b|\b[a-z]{1,2}\d+\b', code)
|
||||
if len(obfuscated) > 3:
|
||||
findings.append(Finding(
|
||||
check="transparency",
|
||||
result=CheckResult.WARNING,
|
||||
message="Multiple obfuscated variable names detected",
|
||||
location=location,
|
||||
suggestion="Use descriptive variable names",
|
||||
))
|
||||
|
||||
# Check for very long single lines (possible obfuscation)
|
||||
for line_num, line in enumerate(code.split('\n'), 1):
|
||||
if len(line) > 500:
|
||||
findings.append(Finding(
|
||||
check="transparency",
|
||||
result=CheckResult.WARNING,
|
||||
message=f"Unusually long code line ({len(line)} chars)",
|
||||
location=f"{location}, line {line_num}",
|
||||
))
|
||||
|
||||
if not findings:
|
||||
findings.append(Finding(
|
||||
check="transparency",
|
||||
result=CheckResult.PASS,
|
||||
message="No obfuscation detected",
|
||||
))
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def check_scope(config: Dict[str, Any]) -> List[Finding]:
|
||||
"""Check if code accesses unexpected resources.
|
||||
|
||||
Flags file system, network, or credential access.
|
||||
"""
|
||||
findings = []
|
||||
steps = config.get("steps", [])
|
||||
tool_name = config.get("name", "").lower()
|
||||
description = config.get("description", "").lower()
|
||||
|
||||
for i, step in enumerate(steps):
|
||||
if step.get("type") == "code":
|
||||
code = step.get("code", "")
|
||||
location = f"step {i + 1}"
|
||||
|
||||
for pattern, description_text in SCOPE_PATTERNS:
|
||||
if re.search(pattern, code, re.IGNORECASE):
|
||||
# Check if this access seems expected based on tool name/description
|
||||
is_expected = False
|
||||
if "ssh" in pattern and ("ssh" in tool_name or "ssh" in description):
|
||||
is_expected = True
|
||||
if "env" in pattern and ("env" in tool_name or "environment" in description):
|
||||
is_expected = True
|
||||
if "file" in description_text.lower() and ("file" in tool_name or "file" in description):
|
||||
is_expected = True
|
||||
|
||||
if not is_expected:
|
||||
# Credential access is always flagged
|
||||
if "credential" in pattern or ".ssh" in pattern or ".aws" in pattern:
|
||||
result = CheckResult.FAIL
|
||||
suggestion = "Remove credential access or justify in description"
|
||||
else:
|
||||
result = CheckResult.WARNING
|
||||
suggestion = "Document this access in tool description"
|
||||
|
||||
findings.append(Finding(
|
||||
check="scope",
|
||||
result=result,
|
||||
message=description_text,
|
||||
location=location,
|
||||
suggestion=suggestion,
|
||||
))
|
||||
|
||||
if not findings:
|
||||
findings.append(Finding(
|
||||
check="scope",
|
||||
result=CheckResult.PASS,
|
||||
message="No unexpected resource access",
|
||||
))
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
@dataclass
|
||||
class OptimizationSuggestion:
|
||||
"""A code optimization suggestion for an AI prompt."""
|
||||
operation: str
|
||||
current_prompt: str
|
||||
optimized_code: str
|
||||
tradeoffs: Dict[str, str]
|
||||
location: str
|
||||
|
||||
|
||||
def check_efficiency(config: Dict[str, Any]) -> List[Finding]:
|
||||
"""Check for AI calls that could optionally be pure code.
|
||||
|
||||
Generates optimization suggestions with tradeoffs - these are
|
||||
informational, not penalties. Authors can choose to keep AI
|
||||
for valid reasons.
|
||||
"""
|
||||
findings = []
|
||||
steps = config.get("steps", [])
|
||||
optimizations = []
|
||||
|
||||
for i, step in enumerate(steps):
|
||||
if step.get("type") == "prompt":
|
||||
prompt = step.get("prompt", "")
|
||||
prompt_lower = prompt.lower()
|
||||
location = f"step {i + 1}"
|
||||
|
||||
# Check against known optimizable patterns
|
||||
for pattern, operation, code, code_benefit, ai_benefit in CODE_OPTIMIZATIONS:
|
||||
if re.search(pattern, prompt_lower):
|
||||
optimizations.append(OptimizationSuggestion(
|
||||
operation=operation,
|
||||
current_prompt=prompt.strip()[:200], # Truncate for display
|
||||
optimized_code=code,
|
||||
tradeoffs={
|
||||
"code": code_benefit,
|
||||
"ai": ai_benefit,
|
||||
},
|
||||
location=location,
|
||||
))
|
||||
break # One suggestion per step
|
||||
|
||||
# Add optimization suggestions as findings (not warnings - optional)
|
||||
for opt in optimizations:
|
||||
findings.append(Finding(
|
||||
check="efficiency",
|
||||
result=CheckResult.PASS, # Not a warning - it's a suggestion
|
||||
message=f"Optional optimization: {opt.operation} could use code",
|
||||
location=opt.location,
|
||||
suggestion=f"Code: {opt.optimized_code}",
|
||||
))
|
||||
|
||||
# Check for all-AI tools (informational, not a warning)
|
||||
code_steps = [s for s in steps if s.get("type") == "code"]
|
||||
prompt_steps = [s for s in steps if s.get("type") == "prompt"]
|
||||
|
||||
if prompt_steps and not code_steps and len(prompt_steps) > 3:
|
||||
findings.append(Finding(
|
||||
check="efficiency",
|
||||
result=CheckResult.PASS, # Informational
|
||||
message=f"Tool has {len(prompt_steps)} AI steps - consider if any could be code",
|
||||
suggestion="Mixing AI and code steps can reduce API costs",
|
||||
))
|
||||
|
||||
if not findings:
|
||||
findings.append(Finding(
|
||||
check="efficiency",
|
||||
result=CheckResult.PASS,
|
||||
message="AI usage appears appropriate",
|
||||
))
|
||||
|
||||
return findings, optimizations
|
||||
|
||||
|
||||
def get_optimization_details(optimizations: List[OptimizationSuggestion]) -> List[Dict[str, Any]]:
|
||||
"""Convert optimization suggestions to serializable format."""
|
||||
return [
|
||||
{
|
||||
"operation": opt.operation,
|
||||
"current_prompt": opt.current_prompt,
|
||||
"optimized_code": opt.optimized_code,
|
||||
"tradeoffs": opt.tradeoffs,
|
||||
"location": opt.location,
|
||||
"action": "optional", # Author chooses
|
||||
}
|
||||
for opt in optimizations
|
||||
]
|
||||
|
||||
|
||||
def scrutinize_tool(config_yaml: str, description: str, readme: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Main entry point for tool scrutiny.
|
||||
|
||||
Args:
|
||||
config_yaml: The tool's YAML configuration
|
||||
description: The tool's description
|
||||
readme: Optional README content (for additional context)
|
||||
|
||||
Returns:
|
||||
Dictionary with scrutiny results
|
||||
"""
|
||||
report = analyze_tool(config_yaml, description)
|
||||
return report.to_dict()
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Similarity detection for registry tools."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
|
||||
def _tokenize(text: str) -> List[str]:
|
||||
return [t for t in re_split_nonword(text.lower()) if t]
|
||||
|
||||
|
||||
def re_split_nonword(text: str) -> List[str]:
|
||||
token = ""
|
||||
tokens = []
|
||||
for ch in text:
|
||||
if ch.isalnum():
|
||||
token += ch
|
||||
else:
|
||||
if token:
|
||||
tokens.append(token)
|
||||
token = ""
|
||||
if token:
|
||||
tokens.append(token)
|
||||
return tokens
|
||||
|
||||
|
||||
def jaccard(a: List[str], b: List[str]) -> float:
|
||||
set_a = set(a)
|
||||
set_b = set(b)
|
||||
if not set_a and not set_b:
|
||||
return 0.0
|
||||
return len(set_a & set_b) / max(len(set_a | set_b), 1)
|
||||
|
||||
|
||||
def name_similarity(name_a: str, name_b: str) -> float:
|
||||
return SequenceMatcher(None, name_a.lower(), name_b.lower()).ratio()
|
||||
|
||||
|
||||
def description_similarity(desc_a: str, desc_b: str) -> float:
|
||||
return jaccard(_tokenize(desc_a), _tokenize(desc_b))
|
||||
|
||||
|
||||
def tags_similarity(tags_a: List[str], tags_b: List[str]) -> float:
|
||||
return jaccard([t.lower() for t in tags_a], [t.lower() for t in tags_b])
|
||||
|
||||
|
||||
def score_similarity(
|
||||
candidate: Dict,
|
||||
name: str,
|
||||
description: str,
|
||||
tags: List[str],
|
||||
category: str | None,
|
||||
) -> float:
|
||||
name_score = name_similarity(name, candidate.get("name", ""))
|
||||
desc_score = description_similarity(description, candidate.get("description", ""))
|
||||
tags_score = tags_similarity(tags, candidate.get("tags", []))
|
||||
category_bonus = 0.1 if category and candidate.get("category") == category else 0.0
|
||||
score = (0.5 * name_score) + (0.3 * desc_score) + (0.2 * tags_score) + category_bonus
|
||||
return min(score, 1.0)
|
||||
|
||||
|
||||
def find_similar_tools(
|
||||
tools: List[Dict],
|
||||
name: str,
|
||||
description: str,
|
||||
tags: List[str],
|
||||
category: str | None,
|
||||
threshold: float = 0.6,
|
||||
) -> List[Tuple[Dict, float]]:
|
||||
results = []
|
||||
for tool in tools:
|
||||
score = score_similarity(tool, name, description, tags, category)
|
||||
if score >= threshold:
|
||||
results.append((tool, round(score, 3)))
|
||||
results.sort(key=lambda item: item[1], reverse=True)
|
||||
return results
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Registry repository sync and webhook processing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, Tuple
|
||||
|
||||
import yaml
|
||||
|
||||
from .db import connect_db, query_one
|
||||
|
||||
|
||||
def get_repo_dir() -> Path:
|
||||
default_dir = Path.home() / ".cmdforge" / "registry" / "repo"
|
||||
return Path(os.environ.get("CMDFORGE_REGISTRY_REPO_DIR", default_dir))
|
||||
|
||||
|
||||
def get_repo_url() -> str:
|
||||
return os.environ.get("CMDFORGE_REGISTRY_REPO_URL", "https://gitea.brrd.tech/rob/CmdForge-Registry.git")
|
||||
|
||||
|
||||
def get_repo_branch() -> str:
|
||||
return os.environ.get("CMDFORGE_REGISTRY_REPO_BRANCH", "main")
|
||||
|
||||
|
||||
def get_categories_cache_path() -> Path:
|
||||
return Path(os.environ.get(
|
||||
"CMDFORGE_REGISTRY_CATEGORIES_CACHE",
|
||||
Path.home() / ".cmdforge" / "registry" / "categories_cache.json",
|
||||
))
|
||||
|
||||
|
||||
def verify_hmac(body: bytes, signature: str | None, secret: str) -> bool:
|
||||
if not signature:
|
||||
return False
|
||||
expected = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
|
||||
return hmac.compare_digest(signature, expected)
|
||||
|
||||
|
||||
def clone_or_update_repo(repo_dir: Path) -> None:
|
||||
repo_dir.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not repo_dir.exists():
|
||||
subprocess.run(
|
||||
["git", "clone", "--depth", "1", "--branch", get_repo_branch(), get_repo_url(), str(repo_dir)],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
return
|
||||
|
||||
subprocess.run(["git", "-C", str(repo_dir), "fetch", "origin"], check=True)
|
||||
subprocess.run(
|
||||
["git", "-C", str(repo_dir), "reset", "--hard", f"origin/{get_repo_branch()}"]
|
||||
, check=True
|
||||
)
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> Dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return yaml.safe_load(handle) or {}
|
||||
|
||||
|
||||
def ensure_publisher(conn, owner: str) -> int:
|
||||
row = query_one(conn, "SELECT id FROM publishers WHERE slug = ?", [owner])
|
||||
if row:
|
||||
return int(row["id"])
|
||||
placeholder_email = f"{owner}@registry.local"
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO publishers (email, password_hash, slug, display_name, verified)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
[placeholder_email, "", owner, owner, False],
|
||||
)
|
||||
return int(conn.execute("SELECT last_insert_rowid() AS id").fetchone()["id"])
|
||||
|
||||
|
||||
def normalize_tags(tags_value: Any) -> str:
|
||||
if not tags_value:
|
||||
return "[]"
|
||||
if isinstance(tags_value, list):
|
||||
return json.dumps(tags_value)
|
||||
return json.dumps([str(tags_value)])
|
||||
|
||||
|
||||
def upsert_tool(conn, owner: str, name: str, data: Dict[str, Any], config_text: str, readme_text: str | None) -> None:
|
||||
version = data.get("version")
|
||||
if not version:
|
||||
return
|
||||
publisher_id = ensure_publisher(conn, owner)
|
||||
registry_meta = data.get("registry", {}) or {}
|
||||
|
||||
tags = normalize_tags(data.get("tags"))
|
||||
description = data.get("description")
|
||||
category = data.get("category")
|
||||
deprecated = bool(data.get("deprecated", False))
|
||||
deprecated_message = data.get("deprecated_message")
|
||||
replacement = data.get("replacement")
|
||||
downloads = registry_meta.get("downloads")
|
||||
published_at = registry_meta.get("published_at")
|
||||
|
||||
existing = query_one(
|
||||
conn,
|
||||
"SELECT id FROM tools WHERE owner = ? AND name = ? AND version = ?",
|
||||
[owner, name, version],
|
||||
)
|
||||
|
||||
if existing:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE tools
|
||||
SET description = ?, category = ?, tags = ?, config_yaml = ?, readme = ?,
|
||||
deprecated = ?, deprecated_message = ?, replacement = ?,
|
||||
downloads = COALESCE(?, downloads), published_at = COALESCE(?, published_at)
|
||||
WHERE id = ?
|
||||
""",
|
||||
[
|
||||
description,
|
||||
category,
|
||||
tags,
|
||||
config_text,
|
||||
readme_text,
|
||||
int(deprecated),
|
||||
deprecated_message,
|
||||
replacement,
|
||||
downloads,
|
||||
published_at,
|
||||
existing["id"],
|
||||
],
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO tools (
|
||||
owner, name, version, description, category, tags, config_yaml, readme,
|
||||
publisher_id, deprecated, deprecated_message, replacement, downloads, published_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
owner,
|
||||
name,
|
||||
version,
|
||||
description,
|
||||
category,
|
||||
tags,
|
||||
config_text,
|
||||
readme_text,
|
||||
publisher_id,
|
||||
int(deprecated),
|
||||
deprecated_message,
|
||||
replacement,
|
||||
downloads or 0,
|
||||
published_at,
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def sync_categories(repo_dir: Path) -> None:
|
||||
categories_path = repo_dir / "categories" / "categories.yaml"
|
||||
if not categories_path.exists():
|
||||
return
|
||||
payload = load_yaml(categories_path)
|
||||
cache_path = get_categories_cache_path()
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def sync_from_repo() -> Tuple[bool, str]:
|
||||
repo_dir = get_repo_dir()
|
||||
clone_or_update_repo(repo_dir)
|
||||
|
||||
tools_root = repo_dir / "tools"
|
||||
if not tools_root.exists():
|
||||
return False, "missing_tools_dir"
|
||||
|
||||
conn = connect_db()
|
||||
try:
|
||||
conn.execute("BEGIN")
|
||||
for config_path in tools_root.glob("*/*/config.yaml"):
|
||||
owner = config_path.parent.parent.name
|
||||
name = config_path.parent.name
|
||||
try:
|
||||
config_text = config_path.read_text(encoding="utf-8")
|
||||
data = yaml.safe_load(config_text) or {}
|
||||
readme_path = config_path.parent / "README.md"
|
||||
readme_text = readme_path.read_text(encoding="utf-8") if readme_path.exists() else None
|
||||
upsert_tool(conn, owner, name, data, config_text, readme_text)
|
||||
except Exception:
|
||||
continue
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
sync_categories(repo_dir)
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def record_webhook_delivery(conn, delivery_id: str, event_type: str) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO webhook_log (delivery_id, event_type, processed_at) VALUES (?, ?, ?)",
|
||||
[delivery_id, event_type, datetime.utcnow().isoformat()],
|
||||
)
|
||||
|
||||
|
||||
def is_delivery_processed(conn, delivery_id: str) -> bool:
|
||||
row = query_one(conn, "SELECT 1 FROM webhook_log WHERE delivery_id = ?", [delivery_id])
|
||||
return bool(row)
|
||||
|
||||
|
||||
def acquire_lock(lock_path: Path, timeout: int) -> bool:
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
os.close(fd)
|
||||
return True
|
||||
except FileExistsError:
|
||||
time.sleep(0.1)
|
||||
return False
|
||||
|
||||
|
||||
def release_lock(lock_path: Path) -> None:
|
||||
if lock_path.exists():
|
||||
lock_path.unlink()
|
||||
|
||||
|
||||
def process_webhook(body: bytes, headers: Dict[str, str], secret: str, timeout: int = 300) -> Tuple[int, Dict[str, Any]]:
|
||||
delivery_id = headers.get("X-Gitea-Delivery")
|
||||
signature = headers.get("X-Gitea-Signature")
|
||||
event_type = headers.get("X-Gitea-Event", "unknown")
|
||||
|
||||
if not delivery_id:
|
||||
return 400, {"error": {"code": "VALIDATION_ERROR", "message": "Missing X-Gitea-Delivery"}}
|
||||
|
||||
if not verify_hmac(body, signature, secret):
|
||||
return 401, {"error": {"code": "UNAUTHORIZED", "message": "Invalid webhook signature"}}
|
||||
|
||||
conn = connect_db()
|
||||
try:
|
||||
if is_delivery_processed(conn, delivery_id):
|
||||
return 200, {"data": {"status": "already_processed"}}
|
||||
|
||||
lock_path = Path.home() / ".cmdforge" / "registry" / "locks" / "webhook.lock"
|
||||
if not acquire_lock(lock_path, timeout):
|
||||
return 200, {"data": {"status": "skipped", "reason": "sync_in_progress"}}
|
||||
|
||||
try:
|
||||
if is_delivery_processed(conn, delivery_id):
|
||||
return 200, {"data": {"status": "already_processed"}}
|
||||
ok, reason = sync_from_repo()
|
||||
if ok:
|
||||
record_webhook_delivery(conn, delivery_id, event_type)
|
||||
conn.commit()
|
||||
return 200, {"data": {"status": "processed"}}
|
||||
return 500, {"error": {"code": "SERVER_ERROR", "message": f"Sync failed: {reason}"}}
|
||||
finally:
|
||||
release_lock(lock_path)
|
||||
finally:
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user