1121 lines
39 KiB
Python
1121 lines
39 KiB
Python
"""Tests for CLI commands."""
|
|
|
|
import json
|
|
import tempfile
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import call, patch, MagicMock
|
|
from io import StringIO
|
|
|
|
import pytest
|
|
|
|
from cmdforge.cli import main
|
|
from cmdforge.tool import Tool, ToolArgument, PromptStep
|
|
|
|
|
|
class TestCLIBasics:
|
|
"""Basic CLI tests."""
|
|
|
|
def test_help_flag(self, capsys):
|
|
"""--help should show usage."""
|
|
with pytest.raises(SystemExit) as exc_info:
|
|
with patch('sys.argv', ['cmdforge', '--help']):
|
|
main()
|
|
|
|
assert exc_info.value.code == 0
|
|
captured = capsys.readouterr()
|
|
assert 'usage' in captured.out.lower() or 'cmdforge' in captured.out.lower()
|
|
|
|
def test_version_flag(self, capsys):
|
|
"""--version should show version."""
|
|
with pytest.raises(SystemExit) as exc_info:
|
|
with patch('sys.argv', ['cmdforge', '--version']):
|
|
main()
|
|
|
|
assert exc_info.value.code == 0
|
|
|
|
def test_mcp_configure_dispatches_host_options(self):
|
|
with (
|
|
patch(
|
|
"cmdforge.agent_setup.configure_agent_host",
|
|
return_value=0,
|
|
) as configure,
|
|
patch("sys.argv", [
|
|
"cmdforge", "mcp", "configure", "claude-code",
|
|
"--scope", "project", "--dry-run",
|
|
]),
|
|
):
|
|
assert main() == 0
|
|
|
|
configure.assert_called_once_with(
|
|
"claude-code",
|
|
server_name="cmdforge",
|
|
scope="project",
|
|
install_policy=True,
|
|
dry_run=True,
|
|
force=False,
|
|
)
|
|
|
|
|
|
class TestListCommand:
|
|
"""Tests for 'cmdforge list' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_list_empty(self, temp_tools_dir, capsys):
|
|
"""List with no tools should show message."""
|
|
with patch('sys.argv', ['cmdforge', 'list']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'no tools' in captured.out.lower() or '0' in captured.out
|
|
|
|
def test_list_with_tools(self, temp_tools_dir, capsys):
|
|
"""List should show available tools."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(name="test-tool", description="A test tool"))
|
|
save_tool(Tool(name="another-tool", description="Another tool"))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'test-tool' in captured.out
|
|
assert 'another-tool' in captured.out
|
|
|
|
def test_json_is_compact_catalog_unless_full_requested(
|
|
self, temp_tools_dir, capsys
|
|
):
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(
|
|
name="catalog-tool",
|
|
description="Catalog entry",
|
|
input_schema={"type": "string"},
|
|
output_schema={"type": "string"},
|
|
steps=[PromptStep(
|
|
prompt="Private implementation {input}",
|
|
provider="mock",
|
|
output_var="result",
|
|
plain_text=True,
|
|
)],
|
|
output="{result}",
|
|
))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list', '--json']):
|
|
assert main() == 0
|
|
compact = json.loads(capsys.readouterr().out)
|
|
assert compact[0]["name"] == "catalog-tool"
|
|
assert compact[0]["input_schema"] == {"type": "string"}
|
|
assert "steps" not in compact[0]
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list', '--json', '--full']):
|
|
assert main() == 0
|
|
full = json.loads(capsys.readouterr().out)
|
|
assert full[0]["steps"][0]["prompt"] == "Private implementation {input}"
|
|
|
|
def test_filter_reports_filtered_count_and_empty_result(
|
|
self, temp_tools_dir, capsys
|
|
):
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(name="alpha", description="First"))
|
|
save_tool(Tool(name="beta", description="Second"))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list', '--filter', 'alpha']):
|
|
assert main() == 0
|
|
output = capsys.readouterr().out
|
|
assert "Available tools (1)" in output
|
|
assert "alpha" in output
|
|
assert "beta" not in output
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list', '--filter', 'missing']):
|
|
assert main() == 0
|
|
assert "No tools match filter 'missing'" in capsys.readouterr().out
|
|
|
|
def test_json_limit_and_invalid_limit(self, temp_tools_dir, capsys):
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(name="alpha"))
|
|
save_tool(Tool(name="beta"))
|
|
with patch('sys.argv', ['cmdforge', 'list', '--json', '--limit', '1']):
|
|
assert main() == 0
|
|
assert len(json.loads(capsys.readouterr().out)) == 1
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list', '--limit', '0']):
|
|
assert main() == 1
|
|
assert "at least 1" in capsys.readouterr().err
|
|
|
|
with patch('sys.argv', ['cmdforge', 'list', '--full']):
|
|
assert main() == 1
|
|
assert "--full requires --json" in capsys.readouterr().err
|
|
|
|
def test_list_includes_project_tools_and_prefers_project_shadow(
|
|
self, temp_tools_dir, tmp_path, monkeypatch, capsys
|
|
):
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(name="shadowed", description="global"))
|
|
project = tmp_path / "project"
|
|
project.mkdir()
|
|
save_tool(
|
|
Tool(name="shadowed", description="project"),
|
|
tools_dir=project / ".cmdforge",
|
|
)
|
|
save_tool(
|
|
Tool(name="project-only", description="local"),
|
|
tools_dir=project / ".cmdforge",
|
|
)
|
|
monkeypatch.chdir(project)
|
|
|
|
with patch("sys.argv", ["cmdforge", "list", "--json"]):
|
|
assert main() == 0
|
|
|
|
payload = json.loads(capsys.readouterr().out)
|
|
by_name = {item["name"]: item for item in payload}
|
|
assert by_name["shadowed"]["description"] == "project"
|
|
assert "project-only" in by_name
|
|
assert [item["name"] for item in payload].count("shadowed") == 1
|
|
|
|
|
|
class TestRunOnceCommand:
|
|
def test_combines_instruction_with_piped_input(self, capsys):
|
|
result = SimpleNamespace(success=True, text="summary", error=None)
|
|
with (
|
|
patch('sys.argv', [
|
|
'cmdforge', 'run-once', 'Summarize:\n{input}',
|
|
'--provider', 'mock',
|
|
]),
|
|
patch('sys.stdin', StringIO('document body')),
|
|
patch('cmdforge.providers.call_provider', return_value=result) as call_provider,
|
|
):
|
|
assert main() == 0
|
|
|
|
assert call_provider.call_args.args[:2] == (
|
|
'mock', 'Summarize:\ndocument body'
|
|
)
|
|
assert capsys.readouterr().out == "summary\n"
|
|
|
|
def test_appends_piped_input_when_template_has_no_placeholder(self):
|
|
result = SimpleNamespace(success=True, text="ok", error=None)
|
|
with (
|
|
patch('sys.argv', [
|
|
'cmdforge', 'run-once', 'Summarize this', '--provider', 'mock',
|
|
]),
|
|
patch('sys.stdin', StringIO('document body')),
|
|
patch('cmdforge.providers.call_provider', return_value=result) as call_provider,
|
|
):
|
|
assert main() == 0
|
|
|
|
assert call_provider.call_args.args[1] == "Summarize this\n\ndocument body"
|
|
|
|
def test_uses_configured_default_provider(self):
|
|
result = SimpleNamespace(success=True, text="ok", error=None)
|
|
config = SimpleNamespace(default_provider="configured-provider")
|
|
stdin = MagicMock()
|
|
stdin.isatty.return_value = True
|
|
with (
|
|
patch('sys.argv', ['cmdforge', 'run-once', 'hello']),
|
|
patch('sys.stdin', stdin),
|
|
patch('cmdforge.config.load_config', return_value=config),
|
|
patch('cmdforge.providers.call_provider', return_value=result) as call_provider,
|
|
):
|
|
assert main() == 0
|
|
assert call_provider.call_args.args[0] == "configured-provider"
|
|
|
|
def test_selects_first_available_balanced_provider(self):
|
|
from cmdforge.cli.tool_commands import _resolve_one_shot_provider
|
|
from cmdforge.providers import Provider
|
|
|
|
config = SimpleNamespace(default_provider=None)
|
|
providers = [
|
|
Provider("opencode-pickle", "missing-opencode"),
|
|
Provider("codex", "codex exec -"),
|
|
]
|
|
with (
|
|
patch('cmdforge.config.load_config', return_value=config),
|
|
patch('cmdforge.providers.load_providers', return_value=providers),
|
|
patch(
|
|
'cmdforge.cli.tool_commands._provider_is_available',
|
|
side_effect=lambda provider: provider.name == "codex",
|
|
),
|
|
):
|
|
assert _resolve_one_shot_provider() == "codex"
|
|
|
|
@pytest.mark.parametrize(
|
|
"arguments,message",
|
|
[
|
|
(["--timeout", "0"], "--timeout"),
|
|
(["--max-tokens", "-1"], "--max-tokens"),
|
|
],
|
|
)
|
|
def test_rejects_invalid_limits_before_provider_call(
|
|
self, arguments, message, capsys
|
|
):
|
|
with (
|
|
patch('sys.argv', [
|
|
'cmdforge', 'run-once', 'hello', '--provider', 'mock', *arguments,
|
|
]),
|
|
patch('sys.stdin', StringIO()),
|
|
patch('cmdforge.providers.call_provider') as call_provider,
|
|
):
|
|
assert main() == 1
|
|
call_provider.assert_not_called()
|
|
assert message in capsys.readouterr().err
|
|
|
|
def test_json_envelope_contains_runtime_provenance(self, capsys):
|
|
from cmdforge.providers import ProviderResult
|
|
|
|
result = ProviderResult(
|
|
text="answer", success=True,
|
|
requested_provider="local", actual_provider="local",
|
|
attempted_providers=["local"], locality="local",
|
|
)
|
|
with (
|
|
patch("sys.argv", [
|
|
"cmdforge", "run-once", "hello", "--provider", "local",
|
|
"--no-fallback", "--result-envelope", "json",
|
|
]),
|
|
patch("sys.stdin", StringIO()),
|
|
patch("cmdforge.providers.call_provider", return_value=result),
|
|
):
|
|
assert main() == 0
|
|
|
|
payload = json.loads(capsys.readouterr().out)
|
|
call = payload["execution"]["provider_calls"][0]
|
|
assert call["actual_provider"] == "local"
|
|
assert call["fallback_used"] is False
|
|
|
|
|
|
class TestCreateCommand:
|
|
"""Tests for 'cmdforge create' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_create_minimal(self, temp_tools_dir, capsys):
|
|
"""Create a minimal tool."""
|
|
with patch('sys.argv', ['cmdforge', 'create', 'my-tool']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.tool import tool_exists
|
|
assert tool_exists('my-tool')
|
|
|
|
def test_create_with_description(self, temp_tools_dir):
|
|
"""Create tool with description."""
|
|
with patch('sys.argv', ['cmdforge', 'create', 'described-tool',
|
|
'-d', 'A helpful description']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.tool import load_tool
|
|
tool = load_tool('described-tool')
|
|
assert tool.description == 'A helpful description'
|
|
|
|
def test_create_duplicate_fails(self, temp_tools_dir, capsys):
|
|
"""Creating duplicate tool should fail."""
|
|
from cmdforge.tool import save_tool
|
|
save_tool(Tool(name="existing"))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'create', 'existing']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
captured = capsys.readouterr()
|
|
assert 'exists' in captured.err.lower() or 'exists' in captured.out.lower()
|
|
|
|
def test_create_invalid_name(self, temp_tools_dir, capsys):
|
|
"""Invalid tool name should fail."""
|
|
with patch('sys.argv', ['cmdforge', 'create', 'invalid/name']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
captured = capsys.readouterr()
|
|
assert 'invalid' in captured.out.lower() or 'invalid' in captured.err.lower()
|
|
|
|
def test_create_project_tool_does_not_create_global_wrapper(
|
|
self, temp_tools_dir, tmp_path, monkeypatch, capsys
|
|
):
|
|
project = tmp_path / "project"
|
|
project.mkdir()
|
|
monkeypatch.chdir(project)
|
|
|
|
with patch("sys.argv", [
|
|
"cmdforge", "create", "project-echo", "--project",
|
|
"--prompt", "Echo {input}", "--provider", "mock",
|
|
]):
|
|
assert main() == 0
|
|
|
|
config = project / ".cmdforge" / "project-echo" / "config.yaml"
|
|
assert config.exists()
|
|
assert not (tmp_path / ".cmdforge" / "project-echo" / "config.yaml").exists()
|
|
assert not (tmp_path / ".local" / "bin" / "project-echo").exists()
|
|
assert "cmdforge run project-echo" in capsys.readouterr().out
|
|
|
|
def test_create_output_dir_uses_explicit_tool_root(
|
|
self, temp_tools_dir, tmp_path
|
|
):
|
|
target = tmp_path / "portable-tools"
|
|
with patch("sys.argv", [
|
|
"cmdforge", "create", "portable", "--output-dir", str(target),
|
|
]):
|
|
assert main() == 0
|
|
|
|
assert (target / "portable" / "config.yaml").exists()
|
|
|
|
|
|
class TestDeleteCommand:
|
|
"""Tests for 'cmdforge delete' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_delete_existing(self, temp_tools_dir, capsys):
|
|
"""Delete an existing tool."""
|
|
from cmdforge.tool import save_tool, tool_exists
|
|
save_tool(Tool(name="to-delete"))
|
|
assert tool_exists("to-delete")
|
|
|
|
with patch('sys.argv', ['cmdforge', 'delete', 'to-delete', '-f']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
assert not tool_exists("to-delete")
|
|
|
|
def test_delete_nonexistent(self, temp_tools_dir, capsys):
|
|
"""Deleting nonexistent tool should fail."""
|
|
with patch('sys.argv', ['cmdforge', 'delete', 'nonexistent', '-f']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
|
|
|
|
class TestRunCommand:
|
|
"""Tests for 'cmdforge run' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_run_simple_tool(self, temp_tools_dir, capsys):
|
|
"""Run a simple tool without AI calls."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="echo",
|
|
output="Echo: {input}"
|
|
)
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'run', 'echo']):
|
|
with patch('sys.stdin', StringIO("Hello")):
|
|
with patch('sys.stdin.isatty', return_value=False):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'Echo: Hello' in captured.out
|
|
|
|
def test_run_with_mock_provider(self, temp_tools_dir, capsys):
|
|
"""Run tool with mock provider."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="summarize",
|
|
steps=[
|
|
PromptStep(prompt="Summarize: {input}", provider="mock", output_var="summary", plain_text=True)
|
|
],
|
|
output="{summary}"
|
|
)
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'run', 'summarize']):
|
|
with patch('sys.stdin', StringIO("Some text")):
|
|
with patch('sys.stdin.isatty', return_value=False):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'MOCK' in captured.out
|
|
|
|
def test_run_with_tool_specific_args_after_separator(self, temp_tools_dir, capsys):
|
|
"""cmdforge run should parse tool args with the wrapper parser."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="greet",
|
|
arguments=[
|
|
ToolArgument(flag="--name", variable="name", default="World")
|
|
],
|
|
output="Hello, {name}!"
|
|
)
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'run', 'greet', '--', '--name', 'Alice']):
|
|
with patch('sys.stdin', StringIO("")):
|
|
with patch('sys.stdin.isatty', return_value=True):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'Hello, Alice!' in captured.out
|
|
|
|
def test_run_nonexistent_tool(self, temp_tools_dir, capsys):
|
|
"""Running nonexistent tool should fail."""
|
|
with patch('sys.argv', ['cmdforge', 'run', 'nonexistent']):
|
|
result = main()
|
|
|
|
assert result != 0
|
|
|
|
def test_run_prefers_project_local_tool(
|
|
self, temp_tools_dir, tmp_path, monkeypatch, capsys
|
|
):
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(name="shadowed", output="global"))
|
|
project = tmp_path / "project"
|
|
project.mkdir()
|
|
save_tool(
|
|
Tool(name="shadowed", output="project"),
|
|
tools_dir=project / ".cmdforge",
|
|
)
|
|
monkeypatch.chdir(project)
|
|
|
|
with (
|
|
patch("sys.argv", ["cmdforge", "run", "shadowed"]),
|
|
patch("sys.stdin", StringIO("")),
|
|
patch("sys.stdin.isatty", return_value=True),
|
|
):
|
|
assert main() == 0
|
|
|
|
assert capsys.readouterr().out.strip() == "project"
|
|
|
|
def test_run_json_envelope_preserves_output_and_provenance(
|
|
self, temp_tools_dir, capsys
|
|
):
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(
|
|
name="audited",
|
|
steps=[PromptStep(
|
|
prompt="Echo {input}", provider="mock",
|
|
output_var="answer", plain_text=True,
|
|
)],
|
|
output="{answer}",
|
|
))
|
|
|
|
with (
|
|
patch("sys.argv", [
|
|
"cmdforge", "run", "audited", "--result-envelope", "json",
|
|
]),
|
|
patch("sys.stdin", StringIO("hello")),
|
|
patch("sys.stdin.isatty", return_value=False),
|
|
):
|
|
assert main() == 0
|
|
|
|
payload = json.loads(capsys.readouterr().out)
|
|
assert "MOCK" in payload["output"]
|
|
assert payload["execution"]["actual_providers"] == ["mock"]
|
|
assert payload["execution"]["provider_calls"][0]["locality"] == "local"
|
|
|
|
def test_failed_run_still_emits_audit_envelope(
|
|
self, temp_tools_dir, capsys
|
|
):
|
|
from cmdforge.providers import ProviderResult
|
|
from cmdforge.tool import save_tool
|
|
|
|
save_tool(Tool(
|
|
name="strict",
|
|
steps=[PromptStep(
|
|
prompt="Private", provider="primary",
|
|
output_var="answer", plain_text=True,
|
|
)],
|
|
output="{answer}",
|
|
))
|
|
failure = ProviderResult(
|
|
text="", success=False, error="offline",
|
|
requested_provider="primary", attempted_providers=["primary"],
|
|
)
|
|
with (
|
|
patch("sys.argv", [
|
|
"cmdforge", "run", "strict", "--no-fallback",
|
|
"--result-envelope", "json",
|
|
]),
|
|
patch("sys.stdin", StringIO("private")),
|
|
patch("sys.stdin.isatty", return_value=False),
|
|
patch("cmdforge.runner.call_provider", return_value=failure),
|
|
):
|
|
assert main() == 2
|
|
|
|
payload = json.loads(capsys.readouterr().out)
|
|
assert payload["output"] == ""
|
|
assert payload["execution"]["exit_code"] == 2
|
|
assert payload["execution"]["provider_calls"][0]["error"] == "offline"
|
|
|
|
|
|
class TestTestCommand:
|
|
"""Tests for 'cmdforge test' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_test_tool(self, temp_tools_dir, capsys):
|
|
"""Test command should run with mock provider."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(
|
|
name="test-me",
|
|
steps=[
|
|
PromptStep(prompt="Test: {input}", provider="claude", output_var="result", plain_text=True)
|
|
],
|
|
output="{result}"
|
|
)
|
|
save_tool(tool)
|
|
|
|
# Provide stdin input for the test command
|
|
with patch('sys.argv', ['cmdforge', 'test', 'test-me']):
|
|
with patch('sys.stdin', StringIO("test input")):
|
|
result = main()
|
|
|
|
# Test command uses mock, so should succeed
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'MOCK' in captured.out or 'mock' in captured.out.lower()
|
|
|
|
|
|
class TestProvidersCommand:
|
|
"""Tests for 'cmdforge providers' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_providers_file(self, tmp_path):
|
|
providers_file = tmp_path / ".cmdforge" / "providers.yaml"
|
|
with patch('cmdforge.providers.PROVIDERS_FILE', providers_file):
|
|
yield providers_file
|
|
|
|
def test_providers_list(self, temp_providers_file, capsys):
|
|
"""List providers."""
|
|
with patch('sys.argv', ['cmdforge', 'providers']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
# Should show some default providers
|
|
assert 'mock' in captured.out.lower() or 'claude' in captured.out.lower()
|
|
|
|
def test_providers_list_json_exposes_routing_metadata(
|
|
self, temp_providers_file, capsys
|
|
):
|
|
from cmdforge.providers import Provider, save_providers
|
|
|
|
save_providers([Provider(
|
|
"safe", "safe-cli", locality="local",
|
|
capabilities=["structured-json"], data_policy="private",
|
|
max_context_tokens=8192,
|
|
)])
|
|
with (
|
|
patch("sys.argv", ["cmdforge", "providers", "list", "--json"]),
|
|
patch("cmdforge.cli.provider_commands._provider_status", return_value=(True, "Ready")),
|
|
):
|
|
assert main() == 0
|
|
|
|
payload = json.loads(capsys.readouterr().out)
|
|
assert payload[0]["locality"] == "local"
|
|
assert payload[0]["capabilities"] == ["structured-json"]
|
|
assert payload[0]["data_policy"] == "private"
|
|
assert payload[0]["max_context_tokens"] == 8192
|
|
assert payload[0]["available"] is True
|
|
|
|
def test_providers_add(self, temp_providers_file, capsys):
|
|
"""Add a custom provider."""
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'add',
|
|
'custom', 'my-ai --prompt']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.providers import get_provider
|
|
provider = get_provider('custom')
|
|
assert provider is not None
|
|
assert provider.command == 'my-ai --prompt'
|
|
|
|
def test_providers_add_api_configuration(self, temp_providers_file, capsys):
|
|
with patch('sys.argv', [
|
|
'cmdforge', 'providers', 'add', 'custom-api',
|
|
'https://example.test/v1',
|
|
'--type', 'api',
|
|
'--model', 'example/model',
|
|
'--api-key-env', 'CUSTOM_API_KEY',
|
|
'--tag', 'api',
|
|
'--locality', 'remote',
|
|
'--capability', 'structured-json',
|
|
'--model-digest', 'sha256:abc',
|
|
'--cost-class', 'low',
|
|
'--latency-class', 'fast',
|
|
'--max-context-tokens', '128000',
|
|
'--data-policy', 'internal',
|
|
'--fallback-chain', 'free',
|
|
]):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
from cmdforge.providers import PRESET_CHAINS, get_provider
|
|
provider = get_provider('custom-api')
|
|
assert provider.type == 'api'
|
|
assert provider.model == 'example/model'
|
|
assert provider.api_key_env == 'CUSTOM_API_KEY'
|
|
assert provider.tags == ['api']
|
|
assert provider.fallback_chain == PRESET_CHAINS['free']
|
|
assert provider.locality == 'remote'
|
|
assert provider.capabilities == ['structured-json']
|
|
assert provider.model_digest == 'sha256:abc'
|
|
assert provider.cost_class == 'low'
|
|
assert provider.latency_class == 'fast'
|
|
assert provider.max_context_tokens == 128000
|
|
assert provider.data_policy == 'internal'
|
|
|
|
def test_providers_list_reports_missing_api_key(self, temp_providers_file, capsys):
|
|
from cmdforge.providers import Provider, save_providers
|
|
|
|
save_providers([
|
|
Provider(
|
|
'custom-api',
|
|
'https://example.test/v1',
|
|
type='api',
|
|
model='example/model',
|
|
api_key_env='MISSING_CUSTOM_API_KEY',
|
|
)
|
|
])
|
|
|
|
with patch.dict('os.environ', {'MISSING_CUSTOM_API_KEY': ''}):
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'list']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'API KEY NOT SET (MISSING_CUSTOM_API_KEY)' in captured.out
|
|
assert 'NOT FOUND (https://' not in captured.out
|
|
|
|
def test_providers_discover_adds_new_provider(self, temp_providers_file, capsys):
|
|
discovered = [{
|
|
'source': 'cli',
|
|
'binary': 'example-ai',
|
|
'path': '/usr/bin/example-ai',
|
|
'name': 'example-ai',
|
|
'command': 'example-ai --print',
|
|
'description': 'Example provider',
|
|
'tags': ['test'],
|
|
}]
|
|
|
|
with patch('cmdforge.providers.discover_installed_providers', return_value=discovered):
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'discover', '--add']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
from cmdforge.providers import get_provider
|
|
provider = get_provider('example-ai')
|
|
assert provider is not None
|
|
assert provider.command == 'example-ai --print'
|
|
assert provider.tags == ['test']
|
|
|
|
def test_providers_remove(self, temp_providers_file, capsys):
|
|
"""Remove a provider."""
|
|
from cmdforge.providers import add_provider, Provider
|
|
add_provider(Provider('removeme', 'cmd'))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'providers', 'remove', 'removeme']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
from cmdforge.providers import get_provider
|
|
assert get_provider('removeme') is None
|
|
|
|
|
|
class TestRefreshCommand:
|
|
"""Tests for 'cmdforge refresh' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_refresh_creates_wrappers(self, temp_tools_dir, capsys):
|
|
"""Refresh should create wrapper scripts."""
|
|
from cmdforge.tool import save_tool, get_bin_dir
|
|
|
|
save_tool(Tool(name="wrapper-test"))
|
|
|
|
with patch('sys.argv', ['cmdforge', 'refresh']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
|
|
wrapper = get_bin_dir() / "wrapper-test"
|
|
assert wrapper.exists()
|
|
|
|
|
|
class TestDocsCommand:
|
|
"""Tests for 'cmdforge docs' command."""
|
|
|
|
@pytest.fixture
|
|
def temp_tools_dir(self, tmp_path):
|
|
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
|
|
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
|
|
yield tmp_path
|
|
|
|
def test_docs_for_tool_with_readme(self, temp_tools_dir, capsys):
|
|
"""Docs should show README content when it exists."""
|
|
from cmdforge.tool import save_tool, get_tools_dir
|
|
|
|
tool = Tool(
|
|
name="documented",
|
|
description="A well-documented tool",
|
|
)
|
|
save_tool(tool)
|
|
|
|
# Create a README.md for the tool
|
|
readme_path = get_tools_dir() / "documented" / "README.md"
|
|
readme_path.write_text("# Documented Tool\n\nThis is the documentation.")
|
|
|
|
with patch('sys.argv', ['cmdforge', 'docs', 'documented']):
|
|
result = main()
|
|
|
|
assert result == 0
|
|
captured = capsys.readouterr()
|
|
assert 'Documented Tool' in captured.out
|
|
assert 'documentation' in captured.out.lower()
|
|
|
|
def test_docs_no_readme(self, temp_tools_dir, capsys):
|
|
"""Docs without README should prompt to create one."""
|
|
from cmdforge.tool import save_tool
|
|
|
|
tool = Tool(name="no-docs")
|
|
save_tool(tool)
|
|
|
|
with patch('sys.argv', ['cmdforge', 'docs', 'no-docs']):
|
|
result = main()
|
|
|
|
assert result == 1 # Returns 1 when no README
|
|
captured = capsys.readouterr()
|
|
assert 'No documentation' in captured.out or '--edit' in captured.out
|
|
|
|
|
|
class TestRegistryPublishDryRun:
|
|
@pytest.fixture
|
|
def tool_dir(self, tmp_path):
|
|
directory = tmp_path / "dry-run-tool"
|
|
directory.mkdir()
|
|
(directory / "config.yaml").write_text(
|
|
"name: dry-run-tool\n"
|
|
"version: 1.0.0\n"
|
|
"description: Test dry-run behavior\n"
|
|
"output: constant\n"
|
|
)
|
|
return directory
|
|
|
|
@staticmethod
|
|
def args(tool_dir):
|
|
return SimpleNamespace(
|
|
path=str(tool_dir), dry_run=True, force=False, owner=""
|
|
)
|
|
|
|
def test_without_token_stops_after_local_preflight(
|
|
self, tool_dir, monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
|
|
get_client = MagicMock()
|
|
monkeypatch.setattr(
|
|
"cmdforge.cli.registry_commands.load_config",
|
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="")),
|
|
)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", get_client)
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 0
|
|
get_client.assert_not_called()
|
|
assert "local preflight only" in capsys.readouterr().out
|
|
|
|
def test_renders_structured_registry_evidence(
|
|
self, tool_dir, monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
|
|
client = MagicMock()
|
|
client.publish_tool.return_value = {
|
|
"preflight": {
|
|
"errors": [],
|
|
"warnings": ["remote warning"],
|
|
"suggestions": ["remote hint"],
|
|
},
|
|
"suggestions": {
|
|
"similar_tools": [{
|
|
"name": "official/similar",
|
|
"similarity": 0.8,
|
|
}],
|
|
"scrutiny": {
|
|
"findings": [{"result": "warning", "message": "review me"}]
|
|
},
|
|
},
|
|
}
|
|
monkeypatch.setattr(
|
|
"cmdforge.cli.registry_commands.load_config",
|
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
|
|
)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 0
|
|
output = capsys.readouterr().out
|
|
assert "remote warning" in output
|
|
assert "official/similar" in output
|
|
assert "review me" in output
|
|
client.publish_tool.assert_called_once()
|
|
|
|
def test_registry_rejection_returns_failure(
|
|
self, tool_dir, monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
from cmdforge.registry_client import RegistryError
|
|
|
|
client = MagicMock()
|
|
client.publish_tool.side_effect = RegistryError(
|
|
"SCRUTINY_FAILED", "Tool rejected"
|
|
)
|
|
monkeypatch.setattr(
|
|
"cmdforge.cli.registry_commands.load_config",
|
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
|
|
)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 1
|
|
assert "Tool rejected" in capsys.readouterr().err
|
|
|
|
def test_non_mapping_config_returns_failure(self, tool_dir, capsys):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
|
|
(tool_dir / "config.yaml").write_text("- not\n- a\n- tool\n")
|
|
|
|
assert _cmd_registry_publish(self.args(tool_dir)) == 1
|
|
assert "YAML mapping" in capsys.readouterr().err
|
|
|
|
def test_normal_publish_runs_remote_preflight_before_mutation(
|
|
self, tool_dir, monkeypatch
|
|
):
|
|
from cmdforge.cli.registry_commands import _cmd_registry_publish
|
|
|
|
client = MagicMock()
|
|
client.get_me.return_value = {"slug": "testuser"}
|
|
client.get_my_tool_status.return_value = {"status": "pending"}
|
|
client.publish_tool.side_effect = [
|
|
{
|
|
"preflight": {"errors": [], "warnings": [], "suggestions": []},
|
|
"quality": {"score": 80, "evidence_coverage": 60, "categories": []},
|
|
},
|
|
{
|
|
"owner": "testuser", "name": "dry-run-tool",
|
|
"version": "1.0.0", "status": "pending",
|
|
},
|
|
]
|
|
monkeypatch.setattr(
|
|
"cmdforge.cli.registry_commands.load_config",
|
|
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
|
|
)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
|
args = SimpleNamespace(
|
|
path=str(tool_dir), dry_run=False, force=True, owner=""
|
|
)
|
|
assert _cmd_registry_publish(args) == 0
|
|
assert client.publish_tool.call_count == 2
|
|
assert client.publish_tool.call_args_list[0].kwargs["dry_run"] is True
|
|
assert "dry_run" not in client.publish_tool.call_args_list[1].kwargs
|
|
|
|
|
|
def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
|
|
from cmdforge.cli import cmd_inspect
|
|
from cmdforge.registry_client import PaginatedResponse
|
|
|
|
tool = Tool(name="summary", output="constant")
|
|
client = MagicMock()
|
|
client.search_tools.return_value = PaginatedResponse(data=[{
|
|
"owner": "official",
|
|
"name": "summarize",
|
|
"description": "Summarize text",
|
|
}])
|
|
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
|
|
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
|
|
|
|
assert cmd_inspect(SimpleNamespace(name="summary", registry=True)) == 0
|
|
assert "official/summarize" in capsys.readouterr().out
|
|
|
|
|
|
def test_inspect_shows_contract_proposal_and_passing_conformance(
|
|
monkeypatch, capsys
|
|
):
|
|
from cmdforge.cli import cmd_inspect
|
|
|
|
step_schema = {
|
|
"type": "object",
|
|
"properties": {"answer": {"type": "string"}},
|
|
"required": ["answer"],
|
|
}
|
|
tool = Tool(
|
|
name="contracted",
|
|
arguments=[],
|
|
input_schema={},
|
|
output_schema=step_schema,
|
|
steps=[PromptStep(
|
|
prompt="Answer", provider="paid-provider", output_var="answer",
|
|
output_schema=step_schema,
|
|
)],
|
|
output="{answer}",
|
|
)
|
|
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
|
|
|
|
assert cmd_inspect(SimpleNamespace(name="contracted", registry=False)) == 0
|
|
output = capsys.readouterr().out
|
|
assert "Contract conformance" in output
|
|
assert "PASSED" in output
|
|
|
|
|
|
def test_inspect_can_save_passing_baseline(tmp_path, monkeypatch, capsys):
|
|
from cmdforge.cli import cmd_inspect
|
|
from cmdforge.contract_testing import CONFORMANCE_FILE
|
|
|
|
tool = Tool(
|
|
name="baseline", version="1.0.0", input_schema={},
|
|
output_schema={"type": "string"}, output="stable",
|
|
path=tmp_path / "config.yaml",
|
|
)
|
|
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
|
|
|
|
args = SimpleNamespace(
|
|
name="baseline", registry=False, save_baseline=True
|
|
)
|
|
assert cmd_inspect(args) == 0
|
|
assert (tmp_path / CONFORMANCE_FILE).exists()
|
|
assert "Saved conformance baseline" in capsys.readouterr().out
|
|
|
|
|
|
def test_switch_to_existing_tool_closes_creation_page_first():
|
|
pytest.importorskip("PySide6")
|
|
from cmdforge.gui.pages.tool_builder_page import _switch_to_existing_tool
|
|
|
|
main_window = MagicMock()
|
|
_switch_to_existing_tool(main_window, "existing")
|
|
|
|
assert main_window.method_calls == [
|
|
call.close_tool_builder(),
|
|
call.open_tool_builder("existing"),
|
|
]
|
|
|
|
|
|
def test_tool_builder_save_scope_prefers_recognized_projects(tmp_path):
|
|
pytest.importorskip("PySide6")
|
|
from cmdforge.gui.pages.tool_builder_page import default_save_scope
|
|
|
|
assert default_save_scope(tmp_path) == "global"
|
|
(tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n")
|
|
assert default_save_scope(tmp_path) == "project"
|
|
|
|
|
|
def test_tool_builder_existing_tool_root_does_not_nest_tool_name(tmp_path):
|
|
pytest.importorskip("PySide6")
|
|
from cmdforge.gui.pages.tool_builder_page import existing_tool_root
|
|
|
|
config = tmp_path / ".cmdforge" / "demo" / "config.yaml"
|
|
assert existing_tool_root(config) == (tmp_path / ".cmdforge").resolve()
|
|
|
|
|
|
def test_guided_extraction_builds_new_tool_without_mutating_draft():
|
|
pytest.importorskip("PySide6")
|
|
from cmdforge.gui.pages.tool_builder_page import (
|
|
build_extracted_tool,
|
|
extraction_diff,
|
|
)
|
|
|
|
schema = {"type": "string"}
|
|
repeated = [
|
|
PromptStep(
|
|
prompt="First {input}", provider="mock", output_var="first",
|
|
output_schema=schema,
|
|
),
|
|
PromptStep(
|
|
prompt="Second {first}", provider="mock", output_var="second",
|
|
output_schema=schema,
|
|
),
|
|
]
|
|
draft = Tool(
|
|
name="draft", steps=repeated + [
|
|
PromptStep(
|
|
prompt="First {input}", provider="mock", output_var="first_copy",
|
|
output_schema=schema,
|
|
),
|
|
PromptStep(
|
|
prompt="Second {first}", provider="mock", output_var="second_copy",
|
|
output_schema=schema,
|
|
),
|
|
], input_schema=schema,
|
|
)
|
|
before = draft.to_dict()
|
|
|
|
extracted = build_extracted_tool(
|
|
draft,
|
|
{"type": "repeated_sequence", "locations": [1, 3], "length": 2},
|
|
"draft-shared",
|
|
)
|
|
|
|
assert len(extracted.steps) == 2
|
|
assert extracted.output == "{second}"
|
|
assert extracted.output_schema == schema
|
|
assert draft.to_dict() == before
|
|
assert "+++ draft-shared/config.yaml" in extraction_diff(extracted)
|
|
|
|
|
|
def test_guided_extraction_rejects_hidden_external_step_dependency():
|
|
pytest.importorskip("PySide6")
|
|
from cmdforge.gui.pages.tool_builder_page import build_extracted_tool
|
|
|
|
schema = {"type": "string"}
|
|
draft = Tool(name="draft", steps=[
|
|
PromptStep(
|
|
prompt="Produce", provider="mock", output_var="earlier",
|
|
output_schema=schema,
|
|
),
|
|
PromptStep(
|
|
prompt="Use {earlier}", provider="mock", output_var="one",
|
|
output_schema=schema,
|
|
),
|
|
PromptStep(
|
|
prompt="Finish {one}", provider="mock", output_var="two",
|
|
output_schema=schema,
|
|
),
|
|
])
|
|
|
|
with pytest.raises(ValueError, match="outside its boundary"):
|
|
build_extracted_tool(
|
|
draft,
|
|
{"type": "repeated_sequence", "locations": [2, 4], "length": 2},
|
|
"draft-shared",
|
|
)
|