603 lines
20 KiB
Python
603 lines
20 KiB
Python
"""Tests for CLI commands."""
|
|
|
|
import tempfile
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import call, patch, MagicMock
|
|
from io import StringIO
|
|
|
|
import pytest
|
|
|
|
from cmdforge.cli import main
|
|
from cmdforge.tool import Tool, ToolArgument, PromptStep
|
|
|
|
|
|
class TestCLIBasics:
|
|
"""Basic CLI tests."""
|
|
|
|
def test_help_flag(self, capsys):
|
|
"""--help should show usage."""
|
|
with pytest.raises(SystemExit) as exc_info:
|
|
with patch('sys.argv', ['cmdforge', '--help']):
|
|
main()
|
|
|
|
assert exc_info.value.code == 0
|
|
captured = capsys.readouterr()
|
|
assert 'usage' in captured.out.lower() or 'cmdforge' in captured.out.lower()
|
|
|
|
def test_version_flag(self, capsys):
|
|
"""--version should show version."""
|
|
with pytest.raises(SystemExit) as exc_info:
|
|
with patch('sys.argv', ['cmdforge', '--version']):
|
|
main()
|
|
|
|
assert exc_info.value.code == 0
|
|
|
|
|
|
class TestListCommand:
|
|
"""Tests for 'cmdforge list' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_list_empty(self, temp_tools_dir, capsys):
|
|
"""List with no tools should show message."""
|
|
with patch('sys.argv', ['cmdforge', 'list']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'no tools' in captured.out.lower() or '0' in captured.out
|
|
|
|
def test_list_with_tools(self, temp_tools_dir, capsys):
|
|
"""List should show available tools."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(name="test-tool", description="A test tool"))
|
|
save_tool(Tool(name="another-tool", description="Another tool"))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'test-tool' in captured.out
|
|
assert 'another-tool' in captured.out
|
|
|
|
|
|
class TestCreateCommand:
|
|
"""Tests for 'cmdforge create' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_create_minimal(self, temp_tools_dir, capsys):
|
|
"""Create a minimal tool."""
|
|
with patch('sys.argv', ['cmdforge', 'create', 'my-tool']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.tool import tool_exists
|
|
assert tool_exists('my-tool')
|
|
|
|
def test_create_with_description(self, temp_tools_dir):
|
|
"""Create tool with description."""
|
|
with patch('sys.argv', ['cmdforge', 'create', 'described-tool',
|
|
'-d', 'A helpful description']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.tool import load_tool
|
|
tool = load_tool('described-tool')
|
|
assert tool.description == 'A helpful description'
|
|
|
|
def test_create_duplicate_fails(self, temp_tools_dir, capsys):
|
|
"""Creating duplicate tool should fail."""
|
|
from cmdforge.tool import save_tool
|
|
save_tool(Tool(name="existing"))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'create', 'existing']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
captured = capsys.readouterr()
|
|
assert 'exists' in captured.err.lower() or 'exists' in captured.out.lower()
|
|
|
|
def test_create_invalid_name(self, temp_tools_dir, capsys):
|
|
"""Invalid tool name should fail."""
|
|
with patch('sys.argv', ['cmdforge', 'create', 'invalid/name']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
captured = capsys.readouterr()
|
|
assert 'invalid' in captured.out.lower() or 'invalid' in captured.err.lower()
|
|
|
|
|
|
class TestDeleteCommand:
|
|
"""Tests for 'cmdforge delete' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_delete_existing(self, temp_tools_dir, capsys):
|
|
"""Delete an existing tool."""
|
|
from cmdforge.tool import save_tool, tool_exists
|
|
save_tool(Tool(name="to-delete"))
|
|
assert tool_exists("to-delete")
|
|
|
|
with patch('sys.argv', ['cmdforge', 'delete', 'to-delete', '-f']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
assert not tool_exists("to-delete")
|
|
|
|
def test_delete_nonexistent(self, temp_tools_dir, capsys):
|
|
"""Deleting nonexistent tool should fail."""
|
|
with patch('sys.argv', ['cmdforge', 'delete', 'nonexistent', '-f']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
|
|
|
|
class TestRunCommand:
|
|
"""Tests for 'cmdforge run' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_run_simple_tool(self, temp_tools_dir, capsys):
|
|
"""Run a simple tool without AI calls."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="echo",
|
|
output="Echo: {input}"
|
|
)
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'run', 'echo']):
|
|
with patch('sys.stdin', StringIO("Hello")):
|
|
with patch('sys.stdin.isatty', return_value=False):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'Echo: Hello' in captured.out
|
|
|
|
def test_run_with_mock_provider(self, temp_tools_dir, capsys):
|
|
"""Run tool with mock provider."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="summarize",
|
|
steps=[
|
|
PromptStep(prompt="Summarize: {input}", provider="mock", output_var="summary", plain_text=True)
|
|
],
|
|
output="{summary}"
|
|
)
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'run', 'summarize']):
|
|
with patch('sys.stdin', StringIO("Some text")):
|
|
with patch('sys.stdin.isatty', return_value=False):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'MOCK' in captured.out
|
|
|
|
def test_run_with_tool_specific_args_after_separator(self, temp_tools_dir, capsys):
|
|
"""cmdforge run should parse tool args with the wrapper parser."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="greet",
|
|
arguments=[
|
|
ToolArgument(flag="--name", variable="name", default="World")
|
|
],
|
|
output="Hello, {name}!"
|
|
)
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'run', 'greet', '--', '--name', 'Alice']):
|
|
with patch('sys.stdin', StringIO("")):
|
|
with patch('sys.stdin.isatty', return_value=True):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'Hello, Alice!' in captured.out
|
|
|
|
def test_run_nonexistent_tool(self, temp_tools_dir, capsys):
|
|
"""Running nonexistent tool should fail."""
|
|
with patch('sys.argv', ['cmdforge', 'run', 'nonexistent']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
|
|
|
|
class TestTestCommand:
|
|
"""Tests for 'cmdforge test' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_test_tool(self, temp_tools_dir, capsys):
|
|
"""Test command should run with mock provider."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="test-me",
|
|
steps=[
|
|
PromptStep(prompt="Test: {input}", provider="claude", output_var="result", plain_text=True)
|
|
],
|
|
output="{result}"
|
|
)
|
|
save_tool(tool)
|
|
|
|
# Provide stdin input for the test command
|
|
with patch('sys.argv', ['cmdforge', 'test', 'test-me']):
|
|
with patch('sys.stdin', StringIO("test input")):
|
|
result = main()
|
|
|
|
# Test command uses mock, so should succeed
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'MOCK' in captured.out or 'mock' in captured.out.lower()
|
|
|
|
|
|
class TestProvidersCommand:
|
|
"""Tests for 'cmdforge providers' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_providers_file(self, tmp_path):
|
|
providers_file = tmp_path / ".cmdforge" / "providers.yaml"
|
|
with patch('cmdforge.providers.PROVIDERS_FILE', providers_file):
|
|
yield providers_file
|
|
|
|
def test_providers_list(self, temp_providers_file, capsys):
|
|
"""List providers."""
|
|
with patch('sys.argv', ['cmdforge', 'providers']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
# Should show some default providers
|
|
assert 'mock' in captured.out.lower() or 'claude' in captured.out.lower()
|
|
|
|
def test_providers_add(self, temp_providers_file, capsys):
|
|
"""Add a custom provider."""
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'add',
|
|
'custom', 'my-ai --prompt']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.providers import get_provider
|
|
provider = get_provider('custom')
|
|
assert provider is not None
|
|
assert provider.command == 'my-ai --prompt'
|
|
|
|
def test_providers_add_api_configuration(self, temp_providers_file, capsys):
|
|
with patch('sys.argv', [
|
|
'cmdforge', 'providers', 'add', 'custom-api',
|
|
'https://example.test/v1',
|
|
'--type', 'api',
|
|
'--model', 'example/model',
|
|
'--api-key-env', 'CUSTOM_API_KEY',
|
|
'--tag', 'api',
|
|
'--fallback-chain', 'free',
|
|
]):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
from cmdforge.providers import PRESET_CHAINS, get_provider
|
|
provider = get_provider('custom-api')
|
|
assert provider.type == 'api'
|
|
assert provider.model == 'example/model'
|
|
assert provider.api_key_env == 'CUSTOM_API_KEY'
|
|
assert provider.tags == ['api']
|
|
assert provider.fallback_chain == PRESET_CHAINS['free']
|
|
|
|
def test_providers_list_reports_missing_api_key(self, temp_providers_file, capsys):
|
|
from cmdforge.providers import Provider, save_providers
|
|
|
|
save_providers([
|
|
Provider(
|
|
'custom-api',
|
|
'https://example.test/v1',
|
|
type='api',
|
|
model='example/model',
|
|
api_key_env='MISSING_CUSTOM_API_KEY',
|
|
)
|
|
])
|
|
|
|
with patch.dict('os.environ', {'MISSING_CUSTOM_API_KEY': ''}):
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'list']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'API KEY NOT SET (MISSING_CUSTOM_API_KEY)' in captured.out
|
|
assert 'NOT FOUND (https://' not in captured.out
|
|
|
|
def test_providers_discover_adds_new_provider(self, temp_providers_file, capsys):
|
|
discovered = [{
|
|
'source': 'cli',
|
|
'binary': 'example-ai',
|
|
'path': '/usr/bin/example-ai',
|
|
'name': 'example-ai',
|
|
'command': 'example-ai --print',
|
|
'description': 'Example provider',
|
|
'tags': ['test'],
|
|
}]
|
|
|
|
with patch('cmdforge.providers.discover_installed_providers', return_value=discovered):
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'discover', '--add']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
from cmdforge.providers import get_provider
|
|
provider = get_provider('example-ai')
|
|
assert provider is not None
|
|
assert provider.command == 'example-ai --print'
|
|
assert provider.tags == ['test']
|
|
|
|
def test_providers_remove(self, temp_providers_file, capsys):
|
|
"""Remove a provider."""
|
|
from cmdforge.providers import add_provider, Provider
|
|
add_provider(Provider('removeme', 'cmd'))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'remove', 'removeme']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.providers import get_provider
|
|
assert get_provider('removeme') is None
|
|
|
|
|
|
class TestRefreshCommand:
|
|
"""Tests for 'cmdforge refresh' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_refresh_creates_wrappers(self, temp_tools_dir, capsys):
|
|
"""Refresh should create wrapper scripts."""
|
|
from cmdforge.tool import save_tool, get_bin_dir
|
|
|
|
save_tool(Tool(name="wrapper-test"))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'refresh']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
wrapper = get_bin_dir() / "wrapper-test"
|
|
assert wrapper.exists()
|
|
|
|
|
|
class TestDocsCommand:
|
|
"""Tests for 'cmdforge docs' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_docs_for_tool_with_readme(self, temp_tools_dir, capsys):
|
|
"""Docs should show README content when it exists."""
|
|
from cmdforge.tool import save_tool, get_tools_dir
|
|
|
|
tool = Tool(
|
|
name="documented",
|
|
description="A well-documented tool",
|
|
)
|
|
save_tool(tool)
|
|
|
|
# Create a README.md for the tool
|
|
readme_path = get_tools_dir() / "documented" / "README.md"
|
|
readme_path.write_text("# Documented Tool\n\nThis is the documentation.")
|
|
|
|
with patch('sys.argv', ['cmdforge', 'docs', 'documented']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'Documented Tool' in captured.out
|
|
assert 'documentation' in captured.out.lower()
|
|
|
|
def test_docs_no_readme(self, temp_tools_dir, capsys):
|
|
"""Docs without README should prompt to create one."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(name="no-docs")
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'docs', 'no-docs']):
|
|
result = main()
|
|
|
|
assert result == 1 # Returns 1 when no README
|
|
captured = capsys.readouterr()
|
|
assert 'No documentation' in captured.out or '--edit' in captured.out
|
|
|
|
|
|
class TestRegistryPublishDryRun:
|
|
@pytest.fixture
|
|
def tool_dir(self, tmp_path):
|
|
directory = tmp_path / "dry-run-tool"
|
|
directory.mkdir()
|
|
(directory / "config.yaml").write_text(
|
|
"name: dry-run-tool\n"
|
|
"version: 1.0.0\n"
|
|
"description: Test dry-run behavior\n"
|
|
"output: constant\n"
|
|
)
|
|
return directory
|
|
|
|
@staticmethod
|
|
def args(tool_dir):
|
|
return SimpleNamespace(
|
|
path=str(tool_dir), dry_run=True, force=False, owner=""
|
|
)
|
|
|
|
def test_without_token_stops_after_local_preflight(
|
|
self, tool_dir, monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
|
|
get_client = MagicMock()
|
|
monkeypatch.setattr(
|
|
"cmdforge.cli.registry_commands.load_config",
|
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="")),
|
|
)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", get_client)
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 0
|
|
get_client.assert_not_called()
|
|
assert "local preflight only" in capsys.readouterr().out
|
|
|
|
def test_renders_structured_registry_evidence(
|
|
self, tool_dir, monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
|
|
client = MagicMock()
|
|
client.publish_tool.return_value = {
|
|
"preflight": {
|
|
"errors": [],
|
|
"warnings": ["remote warning"],
|
|
"suggestions": ["remote hint"],
|
|
},
|
|
"suggestions": {
|
|
"similar_tools": [{
|
|
"name": "official/similar",
|
|
"similarity": 0.8,
|
|
}],
|
|
"scrutiny": {
|
|
"findings": [{"result": "warning", "message": "review me"}]
|
|
},
|
|
},
|
|
}
|
|
monkeypatch.setattr(
|
|
"cmdforge.cli.registry_commands.load_config",
|
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
|
|
)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 0
|
|
output = capsys.readouterr().out
|
|
assert "remote warning" in output
|
|
assert "official/similar" in output
|
|
assert "review me" in output
|
|
client.publish_tool.assert_called_once()
|
|
|
|
def test_registry_rejection_returns_failure(
|
|
self, tool_dir, monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
from cmdforge.registry_client import RegistryError
|
|
|
|
client = MagicMock()
|
|
client.publish_tool.side_effect = RegistryError(
|
|
"SCRUTINY_FAILED", "Tool rejected"
|
|
)
|
|
monkeypatch.setattr(
|
|
"cmdforge.cli.registry_commands.load_config",
|
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
|
|
)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 1
|
|
assert "Tool rejected" in capsys.readouterr().err
|
|
|
|
def test_non_mapping_config_returns_failure(self, tool_dir, capsys):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
|
|
(tool_dir / "config.yaml").write_text("- not\n- a\n- tool\n")
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 1
|
|
assert "YAML mapping" in capsys.readouterr().err
|
|
|
|
|
|
def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
|
|
from cmdforge.cli import cmd_inspect
|
|
from cmdforge.registry_client import PaginatedResponse
|
|
|
|
tool = Tool(name="summary", output="constant")
|
|
client = MagicMock()
|
|
client.search_tools.return_value = PaginatedResponse(data=[{
|
|
"owner": "official",
|
|
"name": "summarize",
|
|
"description": "Summarize text",
|
|
}])
|
|
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
|
|
|
assert cmd_inspect(SimpleNamespace(name="summary", registry=True)) == 0
|
|
assert "official/summarize" in capsys.readouterr().out
|
|
|
|
|
|
def test_inspect_shows_contract_proposal_and_passing_conformance(
|
|
monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli import cmd_inspect
|
|
|
|
step_schema = {
|
|
"type": "object",
|
|
"properties": {"answer": {"type": "string"}},
|
|
"required": ["answer"],
|
|
}
|
|
tool = Tool(
|
|
name="contracted",
|
|
arguments=[],
|
|
input_schema={},
|
|
output_schema=step_schema,
|
|
steps=[PromptStep(
|
|
prompt="Answer", provider="paid-provider", output_var="answer",
|
|
output_schema=step_schema,
|
|
)],
|
|
output="{answer}",
|
|
)
|
|
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
|
|
|
|
assert cmd_inspect(SimpleNamespace(name="contracted", registry=False)) == 0
|
|
output = capsys.readouterr().out
|
|
assert "Contract conformance" in output
|
|
assert "PASSED" in output
|
|
|
|
|
|
def test_switch_to_existing_tool_closes_creation_page_first():
|
|
pytest.importorskip("PySide6")
|
|
from cmdforge.gui.pages.tool_builder_page import _switch_to_existing_tool
|
|
|
|
main_window = MagicMock()
|
|
_switch_to_existing_tool(main_window, "existing")
|
|
|
|
assert main_window.method_calls == [
|
|
call.close_tool_builder(),
|
|
call.open_tool_builder("existing"),
|
|
]
|