Add admin configuration system and M3 content automation

Admin Configuration System: - Add registry_settings table to database schema - Create settings.py module with 17 configurable settings across 5 categories (vetting, similarity, sync, moderation, rate_limits) - Add admin API endpoints for settings management (list/get/set/reset) - Create admin settings web UI page with tabbed categories - Add CLI commands: cmdforge registry config list/get/set - Wire settings page link into admin index M3 Content Automation Scripts: - scrutiny.py: Quality scoring with 5 weighted checks - similarity.py: TF-IDF based duplicate detection - vet_pipeline.py: Combined vetting with auto-decision logic - fabric_sync.py: Scheduled Fabric pattern synchronization - fabric-sync.service/timer: systemd units for scheduled sync Settings available: - Vetting thresholds (approve/reject) and check weights - Similarity detection thresholds (duplicate/similar/related) - Fabric sync configuration (enabled, interval, auto-approve) - Moderation settings (require review, auto-approve private) - Rate limiting configuration Co-Authored-By: Claude Opus 4.5 <[email protected]>
2026-01-15 15:20:21 -04:00
parent d1eb4231a7
commit a5e0948881
14 changed files with 2958 additions and 0 deletions
@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+"""
+Auto-vetting pipeline for CmdForge tool submissions.
+
+Combines scrutiny (quality checks) and similarity (duplicate detection)
+to automatically triage incoming tools:
+- Auto-approve: High quality, no duplicates
+- Auto-reject: Low quality or exact duplicates
+- Review queue: Needs human review
+
+Usage:
+    # Vet a single tool
+    python scripts/vet_pipeline.py path/to/tool/config.yaml
+
+    # Vet with custom thresholds
+    python scripts/vet_pipeline.py --approve-threshold 0.85 path/to/tool/config.yaml
+
+    # Process all tools in import directory
+    python scripts/vet_pipeline.py --batch /tmp/fabric-import/
+
+    # Output detailed JSON report
+    python scripts/vet_pipeline.py --json path/to/tool/config.yaml
+"""
+
+import argparse
+import json
+import sys
+from dataclasses import dataclass, asdict
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+# Import our vetting modules
+from scrutiny import (
+    VetResult as ScrutinyResult,
+    VetReport,
+    load_tool_config,
+    vet_tool,
+)
+from similarity import (
+    ToolText,
+    SimilarityMatch,
+    extract_tool_text,
+    load_corpus,
+    compute_tfidf,
+    find_similar,
+    DUPLICATE_THRESHOLD,
+    SIMILAR_THRESHOLD,
+)
+
+
+class PipelineDecision(Enum):
+    """Final pipeline decision."""
+    AUTO_APPROVE = "auto_approve"
+    AUTO_REJECT = "auto_reject"
+    NEEDS_REVIEW = "needs_review"
+    ERROR = "error"
+
+
+@dataclass
+class PipelineResult:
+    """Complete pipeline result for a tool."""
+    tool_name: str
+    tool_path: str
+    decision: PipelineDecision
+    reason: str
+    scrutiny_report: Optional[VetReport] = None
+    similarity_matches: list = None
+    suggestions: list = None
+
+    def __post_init__(self):
+        if self.similarity_matches is None:
+            self.similarity_matches = []
+        if self.suggestions is None:
+            self.suggestions = []
+
+    def to_dict(self) -> dict:
+        d = {
+            "tool_name": self.tool_name,
+            "tool_path": self.tool_path,
+            "decision": self.decision.value,
+            "reason": self.reason,
+            "suggestions": self.suggestions,
+        }
+        if self.scrutiny_report:
+            d["scrutiny"] = self.scrutiny_report.to_dict()
+        if self.similarity_matches:
+            d["similar_tools"] = [m.to_dict() for m in self.similarity_matches]
+        return d
+
+
+# Pipeline thresholds
+DEFAULT_APPROVE_THRESHOLD = 0.8
+DEFAULT_REJECT_THRESHOLD = 0.3
+
+
+def run_pipeline(
+    tool_path: Path,
+    corpus_dir: Path,
+    approve_threshold: float = DEFAULT_APPROVE_THRESHOLD,
+    reject_threshold: float = DEFAULT_REJECT_THRESHOLD,
+) -> PipelineResult:
+    """Run the complete vetting pipeline on a tool."""
+
+    # Load tool config
+    config = load_tool_config(tool_path)
+    if not config:
+        return PipelineResult(
+            tool_name="unknown",
+            tool_path=str(tool_path),
+            decision=PipelineDecision.ERROR,
+            reason=f"Could not load tool config from {tool_path}",
+        )
+
+    tool_name = config.get("name", "unknown")
+
+    # Phase 1: Scrutiny (quality checks)
+    scrutiny_report = vet_tool(config, str(tool_path))
+
+    # Phase 2: Similarity check
+    tool_text = extract_tool_text(config, str(tool_path))
+    corpus = load_corpus(corpus_dir) if corpus_dir.exists() else []
+
+    similarity_matches = []
+    if corpus:
+        all_tools = corpus + [tool_text]
+        compute_tfidf(all_tools)
+        similarity_matches = find_similar(tool_text, corpus, threshold=0.5)
+
+    # Phase 3: Decision logic
+    decision, reason, suggestions = make_decision(
+        scrutiny_report,
+        similarity_matches,
+        approve_threshold,
+        reject_threshold,
+    )
+
+    return PipelineResult(
+        tool_name=tool_name,
+        tool_path=str(tool_path),
+        decision=decision,
+        reason=reason,
+        scrutiny_report=scrutiny_report,
+        similarity_matches=similarity_matches,
+        suggestions=suggestions,
+    )
+
+
+def make_decision(
+    scrutiny: VetReport,
+    similarity_matches: list[SimilarityMatch],
+    approve_threshold: float,
+    reject_threshold: float,
+) -> tuple[PipelineDecision, str, list[str]]:
+    """Make final decision based on scrutiny and similarity results."""
+
+    suggestions = list(scrutiny.suggestions) if scrutiny.suggestions else []
+    score = scrutiny.overall_score
+
+    # Check for exact duplicates first
+    duplicates = [m for m in similarity_matches if m.match_type == "duplicate"]
+    if duplicates:
+        dupe = duplicates[0]
+        return (
+            PipelineDecision.AUTO_REJECT,
+            f"Duplicate of existing tool '{dupe.tool2_name}' ({dupe.similarity:.0%} match)",
+            suggestions + [f"Consider updating {dupe.tool2_name} instead of creating a new tool"],
+        )
+
+    # Check for very similar tools
+    similar = [m for m in similarity_matches if m.match_type == "similar"]
+    if similar:
+        # If high quality + similar, might be an improved version - needs review
+        if score >= approve_threshold:
+            sim = similar[0]
+            return (
+                PipelineDecision.NEEDS_REVIEW,
+                f"High quality but similar to '{sim.tool2_name}' ({sim.similarity:.0%})",
+                suggestions + [f"Review whether this improves on {sim.tool2_name}"],
+            )
+        else:
+            sim = similar[0]
+            return (
+                PipelineDecision.NEEDS_REVIEW,
+                f"Similar to existing tool '{sim.tool2_name}' ({sim.similarity:.0%})",
+                suggestions + [f"Consider if this duplicates {sim.tool2_name}"],
+            )
+
+    # No duplicates or very similar tools - decide based on quality
+    if scrutiny.result == ScrutinyResult.REJECT or score < reject_threshold:
+        return (
+            PipelineDecision.AUTO_REJECT,
+            f"Quality score too low ({score:.2f} < {reject_threshold})",
+            suggestions,
+        )
+
+    if scrutiny.result == ScrutinyResult.APPROVE and score >= approve_threshold:
+        return (
+            PipelineDecision.AUTO_APPROVE,
+            f"High quality score ({score:.2f}) with no similar tools",
+            suggestions,
+        )
+
+    # Middle ground - needs review
+    return (
+        PipelineDecision.NEEDS_REVIEW,
+        f"Quality score {score:.2f} - needs human review",
+        suggestions,
+    )
+
+
+def print_result(result: PipelineResult, verbose: bool = False):
+    """Print pipeline result to console."""
+    decision_emoji = {
+        PipelineDecision.AUTO_APPROVE: "✅",
+        PipelineDecision.AUTO_REJECT: "❌",
+        PipelineDecision.NEEDS_REVIEW: "⚠️",
+        PipelineDecision.ERROR: "💥",
+    }
+
+    decision_color = {
+        PipelineDecision.AUTO_APPROVE: "\033[92m",  # Green
+        PipelineDecision.AUTO_REJECT: "\033[91m",   # Red
+        PipelineDecision.NEEDS_REVIEW: "\033[93m",  # Yellow
+        PipelineDecision.ERROR: "\033[91m",         # Red
+    }
+
+    reset = "\033[0m"
+    emoji = decision_emoji.get(result.decision, "❓")
+    color = decision_color.get(result.decision, "")
+
+    print(f"\n{emoji} {color}{result.tool_name}: {result.decision.value.upper()}{reset}")
+    print(f"   {result.reason}")
+
+    if verbose:
+        print(f"   Path: {result.tool_path}")
+
+        if result.scrutiny_report:
+            print(f"\n   Quality Scores:")
+            for score in result.scrutiny_report.scores:
+                bar = "█" * int(score.score * 10) + "░" * (10 - int(score.score * 10))
+                print(f"      {score.criterion:12} [{bar}] {score.score:.2f}")
+
+        if result.similarity_matches:
+            print(f"\n   Similar Tools:")
+            for match in result.similarity_matches[:3]:
+                print(f"      • {match.tool2_name} ({match.similarity:.0%} {match.match_type})")
+
+        if result.suggestions:
+            print(f"\n   Suggestions:")
+            for suggestion in result.suggestions[:5]:
+                print(f"      • {suggestion}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Run auto-vetting pipeline on CmdForge tools",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__
+    )
+
+    parser.add_argument(
+        "path",
+        type=Path,
+        nargs="?",
+        help="Tool config file or directory to vet"
+    )
+    parser.add_argument(
+        "--batch",
+        type=Path,
+        metavar="DIR",
+        help="Process all tools in directory"
+    )
+    parser.add_argument(
+        "--corpus",
+        type=Path,
+        default=Path.home() / ".cmdforge",
+        help="Directory of existing tools for comparison (default: ~/.cmdforge)"
+    )
+    parser.add_argument(
+        "--approve-threshold",
+        type=float,
+        default=DEFAULT_APPROVE_THRESHOLD,
+        help=f"Score threshold for auto-approve (default: {DEFAULT_APPROVE_THRESHOLD})"
+    )
+    parser.add_argument(
+        "--reject-threshold",
+        type=float,
+        default=DEFAULT_REJECT_THRESHOLD,
+        help=f"Score threshold for auto-reject (default: {DEFAULT_REJECT_THRESHOLD})"
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output as JSON"
+    )
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        help="Show detailed output"
+    )
+
+    args = parser.parse_args()
+
+    if not args.path and not args.batch:
+        parser.error("Specify a tool path or use --batch")
+
+    results = []
+
+    if args.batch:
+        # Batch mode
+        batch_dir = args.batch
+        if not batch_dir.exists():
+            print(f"Error: Directory {batch_dir} does not exist", file=sys.stderr)
+            return 1
+
+        print(f"Processing tools in {batch_dir}...")
+
+        for entry in sorted(batch_dir.iterdir()):
+            config_file = None
+            if entry.is_dir():
+                config_file = entry / "config.yaml"
+            elif entry.suffix in [".yaml", ".yml"]:
+                config_file = entry
+
+            if config_file and config_file.exists():
+                result = run_pipeline(
+                    config_file,
+                    args.corpus,
+                    args.approve_threshold,
+                    args.reject_threshold,
+                )
+                results.append(result)
+    else:
+        # Single tool mode
+        result = run_pipeline(
+            args.path,
+            args.corpus,
+            args.approve_threshold,
+            args.reject_threshold,
+        )
+        results.append(result)
+
+    # Output
+    if args.json:
+        output = [r.to_dict() for r in results]
+        print(json.dumps(output, indent=2))
+    else:
+        for result in results:
+            print_result(result, args.verbose)
+
+        # Summary
+        if len(results) > 1:
+            approved = sum(1 for r in results if r.decision == PipelineDecision.AUTO_APPROVE)
+            rejected = sum(1 for r in results if r.decision == PipelineDecision.AUTO_REJECT)
+            review = sum(1 for r in results if r.decision == PipelineDecision.NEEDS_REVIEW)
+            errors = sum(1 for r in results if r.decision == PipelineDecision.ERROR)
+
+            print(f"\n{'═' * 50}")
+            print(f"Pipeline Summary: {len(results)} tools processed")
+            print(f"  ✅ Auto-approved: {approved}")
+            print(f"  ⚠️  Needs review: {review}")
+            print(f"  ❌ Auto-rejected: {rejected}")
+            if errors:
+                print(f"  💥 Errors: {errors}")
+
+    # Return codes
+    has_rejected = any(r.decision == PipelineDecision.AUTO_REJECT for r in results)
+    has_review = any(r.decision == PipelineDecision.NEEDS_REVIEW for r in results)
+
+    if has_rejected:
+        return 2
+    elif has_review:
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())