Add semantic search (RAG) for registry tool discovery

Users can now find tools by describing what they need in natural language.
Uses Ollama embeddings (nomic-embed-text) on AI-Server for vector similarity
search. Available via CLI (registry describe), GUI (AI search row), and API.

New files:
- registry/embeddings.py: core embedding logic (Ollama API, cosine similarity,
  pack/unpack vectors, backfill)
- tests/test_embeddings.py: 16 unit tests

Modified:
- registry/db.py: tool_embeddings table (schema + migration)
- registry/settings.py: embeddings.* settings (ollama_url, model, enabled, min_score)
- registry/app.py: semantic-search endpoint, publish hook, admin backfill/status
- registry_client.py: semantic_search() method with error surfacing
- gui/pages/registry_page.py: AI search row with SemanticSearchWorker
- cli/__init__.py + registry_commands.py: registry describe subcommand

Backfill required after deploy: POST /api/v1/admin/embeddings/backfill

Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
rob
2026-02-02 01:52:27 -04:00
co-authored by Claude Opus 4.5
parent f24450c19c
commit 5159450feb
9 changed files with 897 additions and 0 deletions
+64
View File
@@ -37,6 +37,8 @@ def cmd_registry(args):
return _cmd_registry_status(args)
elif args.registry_cmd == "browse":
return _cmd_registry_browse(args)
elif args.registry_cmd == "describe":
return _cmd_registry_describe(args)
elif args.registry_cmd == "config":
return _cmd_registry_config(args)
else:
@@ -53,6 +55,7 @@ def cmd_registry(args):
print(" my-tools List your published tools")
print(" status <tool> Check moderation status of a tool")
print(" browse Browse tools (GUI)")
print(" describe <query> Find tools by describing what you need (AI)")
print(" config [action] Manage registry settings (admin)")
return 0
@@ -894,6 +897,67 @@ def _cmd_registry_status(args):
return 0
def _cmd_registry_describe(args):
"""Find tools by describing what you need (semantic search)."""
from ..registry_client import RegistryError, get_client
query = args.query
limit = getattr(args, 'limit', 20)
try:
client = get_client()
result = client.semantic_search(query, limit=limit)
# JSON output
if getattr(args, 'json', False):
print(json.dumps(result, indent=2))
return 0
available = result.get("available", False)
tools = result.get("data", [])
error = result.get("error")
if error:
print(f"Error: {error}", file=sys.stderr)
return 1
if not available:
print("AI search is not available on this registry.")
print("The Ollama embedding service may be offline or the feature is disabled.")
return 1
if not tools:
print(f'No tools found matching: "{query}"')
print("Try different phrasing or use keyword search: cmdforge registry search <query>")
return 0
print(f'Found {len(tools)} tools matching: "{query}"\n')
for i, tool in enumerate(tools, 1):
similarity = tool.get("similarity", 0)
pct = f"{similarity * 100:.0f}%"
owner = tool.get("owner", "")
name = tool.get("name", "")
desc = tool.get("description", "")
print(f" {i}. {owner}/{name} ({pct} match)")
if desc:
print(f" {desc[:70]}{'...' if len(desc) > 70 else ''}")
print()
except RegistryError as e:
if e.code == "CONNECTION_ERROR":
print("Could not connect to the registry.", file=sys.stderr)
else:
print(f"Error: {e.message}", file=sys.stderr)
return 1
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
return 1
return 0
def _cmd_registry_browse(args):
"""Browse tools (GUI)."""
from ..gui import run_gui