CmdForge/tests/test_cli.py

1153 lines
40 KiB
Python

"""Tests for CLI commands."""
import json
import tempfile
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import call, patch, MagicMock
from io import StringIO
import pytest
from cmdforge.cli import main
from cmdforge.tool import Tool, ToolArgument, PromptStep
class TestCLIBasics:
"""Basic CLI tests."""
def test_help_flag(self, capsys):
"""--help should show usage."""
with pytest.raises(SystemExit) as exc_info:
with patch('sys.argv', ['cmdforge', '--help']):
main()
assert exc_info.value.code == 0
captured = capsys.readouterr()
assert 'usage' in captured.out.lower() or 'cmdforge' in captured.out.lower()
def test_version_flag(self, capsys):
"""--version should show version."""
with pytest.raises(SystemExit) as exc_info:
with patch('sys.argv', ['cmdforge', '--version']):
main()
assert exc_info.value.code == 0
def test_mcp_configure_dispatches_host_options(self):
with (
patch(
"cmdforge.agent_setup.configure_agent_host",
return_value=0,
) as configure,
patch("sys.argv", [
"cmdforge", "mcp", "configure", "claude-code",
"--scope", "project", "--dry-run",
]),
):
assert main() == 0
configure.assert_called_once_with(
"claude-code",
server_name="cmdforge",
scope="project",
install_policy=True,
dry_run=True,
force=False,
)
class TestListCommand:
"""Tests for 'cmdforge list' command."""
@pytest.fixture
def temp_tools_dir(self, tmp_path):
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
yield tmp_path
def test_list_empty(self, temp_tools_dir, capsys):
"""List with no tools should show message."""
with patch('sys.argv', ['cmdforge', 'list']):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'no tools' in captured.out.lower() or '0' in captured.out
def test_list_with_tools(self, temp_tools_dir, capsys):
"""List should show available tools."""
from cmdforge.tool import save_tool
save_tool(Tool(name="test-tool", description="A test tool"))
save_tool(Tool(name="another-tool", description="Another tool"))
with patch('sys.argv', ['cmdforge', 'list']):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'test-tool' in captured.out
assert 'another-tool' in captured.out
def test_json_is_compact_catalog_unless_full_requested(
self, temp_tools_dir, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(
name="catalog-tool",
description="Catalog entry",
input_schema={"type": "string"},
output_schema={"type": "string"},
steps=[PromptStep(
prompt="Private implementation {input}",
provider="mock",
output_var="result",
plain_text=True,
)],
output="{result}",
))
with patch('sys.argv', ['cmdforge', 'list', '--json']):
assert main() == 0
compact = json.loads(capsys.readouterr().out)
assert compact[0]["name"] == "catalog-tool"
assert compact[0]["input_schema"] == {"type": "string"}
assert "steps" not in compact[0]
with patch('sys.argv', ['cmdforge', 'list', '--json', '--full']):
assert main() == 0
full = json.loads(capsys.readouterr().out)
assert full[0]["steps"][0]["prompt"] == "Private implementation {input}"
def test_filter_reports_filtered_count_and_empty_result(
self, temp_tools_dir, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(name="alpha", description="First"))
save_tool(Tool(name="beta", description="Second"))
with patch('sys.argv', ['cmdforge', 'list', '--filter', 'alpha']):
assert main() == 0
output = capsys.readouterr().out
assert "Available tools (1)" in output
assert "alpha" in output
assert "beta" not in output
with patch('sys.argv', ['cmdforge', 'list', '--filter', 'missing']):
assert main() == 0
assert "No tools match filter 'missing'" in capsys.readouterr().out
def test_json_limit_and_invalid_limit(self, temp_tools_dir, capsys):
from cmdforge.tool import save_tool
save_tool(Tool(name="alpha"))
save_tool(Tool(name="beta"))
with patch('sys.argv', ['cmdforge', 'list', '--json', '--limit', '1']):
assert main() == 0
assert len(json.loads(capsys.readouterr().out)) == 1
with patch('sys.argv', ['cmdforge', 'list', '--limit', '0']):
assert main() == 1
assert "at least 1" in capsys.readouterr().err
with patch('sys.argv', ['cmdforge', 'list', '--full']):
assert main() == 1
assert "--full requires --json" in capsys.readouterr().err
def test_list_includes_project_tools_and_prefers_project_shadow(
self, temp_tools_dir, tmp_path, monkeypatch, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(name="shadowed", description="global"))
project = tmp_path / "project"
project.mkdir()
save_tool(
Tool(name="shadowed", description="project"),
tools_dir=project / ".cmdforge",
)
save_tool(
Tool(name="project-only", description="local"),
tools_dir=project / ".cmdforge",
)
monkeypatch.chdir(project)
with patch("sys.argv", ["cmdforge", "list", "--json"]):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
by_name = {item["name"]: item for item in payload}
assert by_name["shadowed"]["description"] == "project"
assert "project-only" in by_name
assert [item["name"] for item in payload].count("shadowed") == 1
class TestRunOnceCommand:
def test_combines_instruction_with_piped_input(self, capsys):
result = SimpleNamespace(success=True, text="summary", error=None)
with (
patch('sys.argv', [
'cmdforge', 'run-once', 'Summarize:\n{input}',
'--provider', 'mock',
]),
patch('sys.stdin', StringIO('document body')),
patch('cmdforge.providers.call_provider', return_value=result) as call_provider,
):
assert main() == 0
assert call_provider.call_args.args[:2] == (
'mock', 'Summarize:\ndocument body'
)
assert capsys.readouterr().out == "summary\n"
def test_appends_piped_input_when_template_has_no_placeholder(self):
result = SimpleNamespace(success=True, text="ok", error=None)
with (
patch('sys.argv', [
'cmdforge', 'run-once', 'Summarize this', '--provider', 'mock',
]),
patch('sys.stdin', StringIO('document body')),
patch('cmdforge.providers.call_provider', return_value=result) as call_provider,
):
assert main() == 0
assert call_provider.call_args.args[1] == "Summarize this\n\ndocument body"
def test_uses_configured_default_provider(self):
result = SimpleNamespace(success=True, text="ok", error=None)
config = SimpleNamespace(default_provider="configured-provider")
stdin = MagicMock()
stdin.isatty.return_value = True
with (
patch('sys.argv', ['cmdforge', 'run-once', 'hello']),
patch('sys.stdin', stdin),
patch('cmdforge.config.load_config', return_value=config),
patch('cmdforge.providers.call_provider', return_value=result) as call_provider,
):
assert main() == 0
assert call_provider.call_args.args[0] == "configured-provider"
def test_selects_first_available_balanced_provider(self):
from cmdforge.cli.tool_commands import _resolve_one_shot_provider
from cmdforge.providers import Provider
config = SimpleNamespace(default_provider=None)
providers = [
Provider("opencode-pickle", "missing-opencode"),
Provider("codex", "codex exec -"),
]
with (
patch('cmdforge.config.load_config', return_value=config),
patch('cmdforge.providers.load_providers', return_value=providers),
patch(
'cmdforge.cli.tool_commands._provider_is_available',
side_effect=lambda provider: provider.name == "codex",
),
):
assert _resolve_one_shot_provider() == "codex"
@pytest.mark.parametrize(
"arguments,message",
[
(["--timeout", "0"], "--timeout"),
(["--max-tokens", "-1"], "--max-tokens"),
],
)
def test_rejects_invalid_limits_before_provider_call(
self, arguments, message, capsys
):
with (
patch('sys.argv', [
'cmdforge', 'run-once', 'hello', '--provider', 'mock', *arguments,
]),
patch('sys.stdin', StringIO()),
patch('cmdforge.providers.call_provider') as call_provider,
):
assert main() == 1
call_provider.assert_not_called()
assert message in capsys.readouterr().err
def test_json_envelope_contains_runtime_provenance(self, capsys):
from cmdforge.providers import ProviderResult
result = ProviderResult(
text="answer", success=True,
requested_provider="local", actual_provider="local",
attempted_providers=["local"], locality="local",
)
with (
patch("sys.argv", [
"cmdforge", "run-once", "hello", "--provider", "local",
"--no-fallback", "--result-envelope", "json",
]),
patch("sys.stdin", StringIO()),
patch("cmdforge.providers.call_provider", return_value=result),
):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
call = payload["execution"]["provider_calls"][0]
assert call["actual_provider"] == "local"
assert call["fallback_used"] is False
class TestCreateCommand:
"""Tests for 'cmdforge create' command."""
@pytest.fixture
def temp_tools_dir(self, tmp_path):
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
yield tmp_path
def test_create_minimal(self, temp_tools_dir, capsys):
"""Create a minimal tool."""
with patch('sys.argv', ['cmdforge', 'create', 'my-tool']):
result = main()
assert result == 0
from cmdforge.tool import tool_exists
assert tool_exists('my-tool')
def test_create_with_description(self, temp_tools_dir):
"""Create tool with description."""
with patch('sys.argv', ['cmdforge', 'create', 'described-tool',
'-d', 'A helpful description']):
result = main()
assert result == 0
from cmdforge.tool import load_tool
tool = load_tool('described-tool')
assert tool.description == 'A helpful description'
def test_create_duplicate_fails(self, temp_tools_dir, capsys):
"""Creating duplicate tool should fail."""
from cmdforge.tool import save_tool
save_tool(Tool(name="existing"))
with patch('sys.argv', ['cmdforge', 'create', 'existing']):
result = main()
assert result != 0
captured = capsys.readouterr()
assert 'exists' in captured.err.lower() or 'exists' in captured.out.lower()
def test_create_invalid_name(self, temp_tools_dir, capsys):
"""Invalid tool name should fail."""
with patch('sys.argv', ['cmdforge', 'create', 'invalid/name']):
result = main()
assert result != 0
captured = capsys.readouterr()
assert 'invalid' in captured.out.lower() or 'invalid' in captured.err.lower()
def test_create_project_tool_does_not_create_global_wrapper(
self, temp_tools_dir, tmp_path, monkeypatch, capsys
):
project = tmp_path / "project"
project.mkdir()
monkeypatch.chdir(project)
with patch("sys.argv", [
"cmdforge", "create", "project-echo", "--project",
"--prompt", "Echo {input}", "--provider", "mock",
]):
assert main() == 0
config = project / ".cmdforge" / "project-echo" / "config.yaml"
assert config.exists()
assert not (tmp_path / ".cmdforge" / "project-echo" / "config.yaml").exists()
assert not (tmp_path / ".local" / "bin" / "project-echo").exists()
assert "cmdforge run project-echo" in capsys.readouterr().out
def test_create_output_dir_uses_explicit_tool_root(
self, temp_tools_dir, tmp_path
):
target = tmp_path / "portable-tools"
with patch("sys.argv", [
"cmdforge", "create", "portable", "--output-dir", str(target),
]):
assert main() == 0
assert (target / "portable" / "config.yaml").exists()
class TestDeleteCommand:
"""Tests for 'cmdforge delete' command."""
@pytest.fixture
def temp_tools_dir(self, tmp_path):
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
yield tmp_path
def test_delete_existing(self, temp_tools_dir, capsys):
"""Delete an existing tool."""
from cmdforge.tool import save_tool, tool_exists
save_tool(Tool(name="to-delete"))
assert tool_exists("to-delete")
with patch('sys.argv', ['cmdforge', 'delete', 'to-delete', '-f']):
result = main()
assert result == 0
assert not tool_exists("to-delete")
def test_delete_nonexistent(self, temp_tools_dir, capsys):
"""Deleting nonexistent tool should fail."""
with patch('sys.argv', ['cmdforge', 'delete', 'nonexistent', '-f']):
result = main()
assert result != 0
class TestRunCommand:
"""Tests for 'cmdforge run' command."""
@pytest.fixture
def temp_tools_dir(self, tmp_path):
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
yield tmp_path
def test_run_simple_tool(self, temp_tools_dir, capsys):
"""Run a simple tool without AI calls."""
from cmdforge.tool import save_tool
tool = Tool(
name="echo",
output="Echo: {input}"
)
save_tool(tool)
with patch('sys.argv', ['cmdforge', 'run', 'echo']):
with patch('sys.stdin', StringIO("Hello")):
with patch('sys.stdin.isatty', return_value=False):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'Echo: Hello' in captured.out
def test_run_with_mock_provider(self, temp_tools_dir, capsys):
"""Run tool with mock provider."""
from cmdforge.tool import save_tool
tool = Tool(
name="summarize",
steps=[
PromptStep(prompt="Summarize: {input}", provider="mock", output_var="summary", plain_text=True)
],
output="{summary}"
)
save_tool(tool)
with patch('sys.argv', ['cmdforge', 'run', 'summarize']):
with patch('sys.stdin', StringIO("Some text")):
with patch('sys.stdin.isatty', return_value=False):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'MOCK' in captured.out
def test_run_with_tool_specific_args_after_separator(self, temp_tools_dir, capsys):
"""cmdforge run should parse tool args with the wrapper parser."""
from cmdforge.tool import save_tool
tool = Tool(
name="greet",
arguments=[
ToolArgument(flag="--name", variable="name", default="World")
],
output="Hello, {name}!"
)
save_tool(tool)
with patch('sys.argv', ['cmdforge', 'run', 'greet', '--', '--name', 'Alice']):
with patch('sys.stdin', StringIO("")):
with patch('sys.stdin.isatty', return_value=True):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'Hello, Alice!' in captured.out
def test_run_nonexistent_tool(self, temp_tools_dir, capsys):
"""Running nonexistent tool should fail."""
with patch('sys.argv', ['cmdforge', 'run', 'nonexistent']):
result = main()
assert result != 0
def test_run_prefers_project_local_tool(
self, temp_tools_dir, tmp_path, monkeypatch, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(name="shadowed", output="global"))
project = tmp_path / "project"
project.mkdir()
save_tool(
Tool(name="shadowed", output="project"),
tools_dir=project / ".cmdforge",
)
monkeypatch.chdir(project)
with (
patch("sys.argv", ["cmdforge", "run", "shadowed"]),
patch("sys.stdin", StringIO("")),
patch("sys.stdin.isatty", return_value=True),
):
assert main() == 0
assert capsys.readouterr().out.strip() == "project"
def test_run_json_envelope_preserves_output_and_provenance(
self, temp_tools_dir, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(
name="audited",
steps=[PromptStep(
prompt="Echo {input}", provider="mock",
output_var="answer", plain_text=True,
)],
output="{answer}",
))
with (
patch("sys.argv", [
"cmdforge", "run", "audited", "--result-envelope", "json",
]),
patch("sys.stdin", StringIO("hello")),
patch("sys.stdin.isatty", return_value=False),
):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
assert "MOCK" in payload["output"]
assert payload["execution"]["actual_providers"] == ["mock"]
assert payload["execution"]["provider_calls"][0]["locality"] == "local"
def test_failed_run_still_emits_audit_envelope(
self, temp_tools_dir, capsys
):
from cmdforge.providers import ProviderResult
from cmdforge.tool import save_tool
save_tool(Tool(
name="strict",
steps=[PromptStep(
prompt="Private", provider="primary",
output_var="answer", plain_text=True,
)],
output="{answer}",
))
failure = ProviderResult(
text="", success=False, error="offline",
requested_provider="primary", attempted_providers=["primary"],
)
with (
patch("sys.argv", [
"cmdforge", "run", "strict", "--no-fallback",
"--result-envelope", "json",
]),
patch("sys.stdin", StringIO("private")),
patch("sys.stdin.isatty", return_value=False),
patch("cmdforge.runner.call_provider", return_value=failure),
):
assert main() == 2
payload = json.loads(capsys.readouterr().out)
assert payload["output"] == ""
assert payload["execution"]["exit_code"] == 2
assert payload["execution"]["provider_calls"][0]["error"] == "offline"
class TestTestCommand:
"""Tests for 'cmdforge test' command."""
@pytest.fixture
def temp_tools_dir(self, tmp_path):
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
yield tmp_path
def test_test_tool(self, temp_tools_dir, capsys):
"""Test command should run with mock provider."""
from cmdforge.tool import save_tool
tool = Tool(
name="test-me",
steps=[
PromptStep(prompt="Test: {input}", provider="claude", output_var="result", plain_text=True)
],
output="{result}"
)
save_tool(tool)
# Provide stdin input for the test command
with patch('sys.argv', ['cmdforge', 'test', 'test-me']):
with patch('sys.stdin', StringIO("test input")):
result = main()
# Test command uses mock, so should succeed
assert result == 0
captured = capsys.readouterr()
assert 'MOCK' in captured.out or 'mock' in captured.out.lower()
class TestProvidersCommand:
"""Tests for 'cmdforge providers' command."""
@pytest.fixture
def temp_providers_file(self, tmp_path):
providers_file = tmp_path / ".cmdforge" / "providers.yaml"
with patch('cmdforge.providers.PROVIDERS_FILE', providers_file):
yield providers_file
def test_providers_list(self, temp_providers_file, capsys):
"""List providers."""
with patch('sys.argv', ['cmdforge', 'providers']):
result = main()
assert result == 0
captured = capsys.readouterr()
# Should show some default providers
assert 'mock' in captured.out.lower() or 'claude' in captured.out.lower()
def test_providers_list_json_exposes_routing_metadata(
self, temp_providers_file, capsys
):
from cmdforge.providers import Provider, save_providers
save_providers([Provider(
"safe", "safe-cli", locality="local",
capabilities=["structured-json"], data_policy="private",
max_context_tokens=8192,
)])
with (
patch("sys.argv", ["cmdforge", "providers", "list", "--json"]),
patch("cmdforge.cli.provider_commands._provider_status", return_value=(True, "Ready")),
):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
assert payload[0]["locality"] == "local"
assert payload[0]["capabilities"] == ["structured-json"]
assert payload[0]["data_policy"] == "private"
assert payload[0]["max_context_tokens"] == 8192
assert payload[0]["available"] is True
def test_providers_add(self, temp_providers_file, capsys):
"""Add a custom provider."""
with patch('sys.argv', ['cmdforge', 'providers', 'add',
'custom', 'my-ai --prompt']):
result = main()
assert result == 0
from cmdforge.providers import get_provider
provider = get_provider('custom')
assert provider is not None
assert provider.command == 'my-ai --prompt'
def test_providers_add_api_configuration(self, temp_providers_file, capsys):
with patch('sys.argv', [
'cmdforge', 'providers', 'add', 'custom-api',
'https://example.test/v1',
'--type', 'api',
'--model', 'example/model',
'--api-key-env', 'CUSTOM_API_KEY',
'--tag', 'api',
'--locality', 'remote',
'--capability', 'structured-json',
'--model-digest', 'sha256:abc',
'--cost-class', 'low',
'--latency-class', 'fast',
'--max-context-tokens', '128000',
'--data-policy', 'internal',
'--fallback-chain', 'free',
]):
result = main()
assert result == 0
from cmdforge.providers import PRESET_CHAINS, get_provider
provider = get_provider('custom-api')
assert provider.type == 'api'
assert provider.model == 'example/model'
assert provider.api_key_env == 'CUSTOM_API_KEY'
assert provider.tags == ['api']
assert provider.fallback_chain == PRESET_CHAINS['free']
assert provider.locality == 'remote'
assert provider.capabilities == ['structured-json']
assert provider.model_digest == 'sha256:abc'
assert provider.cost_class == 'low'
assert provider.latency_class == 'fast'
assert provider.max_context_tokens == 128000
assert provider.data_policy == 'internal'
def test_providers_list_reports_missing_api_key(self, temp_providers_file, capsys):
from cmdforge.providers import Provider, save_providers
save_providers([
Provider(
'custom-api',
'https://example.test/v1',
type='api',
model='example/model',
api_key_env='MISSING_CUSTOM_API_KEY',
)
])
with patch.dict('os.environ', {'MISSING_CUSTOM_API_KEY': ''}):
with patch('sys.argv', ['cmdforge', 'providers', 'list']):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'API KEY NOT SET (MISSING_CUSTOM_API_KEY)' in captured.out
assert 'NOT FOUND (https://' not in captured.out
def test_providers_discover_adds_new_provider(self, temp_providers_file, capsys):
discovered = [{
'source': 'cli',
'binary': 'example-ai',
'path': '/usr/bin/example-ai',
'name': 'example-ai',
'command': 'example-ai --print',
'description': 'Example provider',
'tags': ['test'],
}]
with patch('cmdforge.providers.discover_installed_providers', return_value=discovered):
with patch('sys.argv', ['cmdforge', 'providers', 'discover', '--add']):
result = main()
assert result == 0
from cmdforge.providers import get_provider
provider = get_provider('example-ai')
assert provider is not None
assert provider.command == 'example-ai --print'
assert provider.tags == ['test']
def test_providers_remove(self, temp_providers_file, capsys):
"""Remove a provider."""
from cmdforge.providers import add_provider, Provider
add_provider(Provider('removeme', 'cmd'))
with patch('sys.argv', ['cmdforge', 'providers', 'remove', 'removeme']):
result = main()
assert result == 0
from cmdforge.providers import get_provider
assert get_provider('removeme') is None
class TestRefreshCommand:
"""Tests for 'cmdforge refresh' command."""
@pytest.fixture
def temp_tools_dir(self, tmp_path):
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
yield tmp_path
def test_refresh_creates_wrappers(self, temp_tools_dir, capsys):
"""Refresh should create wrapper scripts."""
from cmdforge.tool import save_tool, get_bin_dir
save_tool(Tool(name="wrapper-test"))
with patch('sys.argv', ['cmdforge', 'refresh']):
result = main()
assert result == 0
wrapper = get_bin_dir() / "wrapper-test"
assert wrapper.exists()
class TestDocsCommand:
"""Tests for 'cmdforge docs' command."""
@pytest.fixture
def temp_tools_dir(self, tmp_path):
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
with patch('cmdforge.tool.BIN_DIR', tmp_path / ".local" / "bin"):
yield tmp_path
def test_docs_for_tool_with_readme(self, temp_tools_dir, capsys):
"""Docs should show README content when it exists."""
from cmdforge.tool import save_tool, get_tools_dir
tool = Tool(
name="documented",
description="A well-documented tool",
)
save_tool(tool)
# Create a README.md for the tool
readme_path = get_tools_dir() / "documented" / "README.md"
readme_path.write_text("# Documented Tool\n\nThis is the documentation.")
with patch('sys.argv', ['cmdforge', 'docs', 'documented']):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'Documented Tool' in captured.out
assert 'documentation' in captured.out.lower()
def test_docs_no_readme(self, temp_tools_dir, capsys):
"""Docs without README should prompt to create one."""
from cmdforge.tool import save_tool
tool = Tool(name="no-docs")
save_tool(tool)
with patch('sys.argv', ['cmdforge', 'docs', 'no-docs']):
result = main()
assert result == 1 # Returns 1 when no README
captured = capsys.readouterr()
assert 'No documentation' in captured.out or '--edit' in captured.out
class TestRegistryPublishDryRun:
@pytest.fixture
def tool_dir(self, tmp_path):
directory = tmp_path / "dry-run-tool"
directory.mkdir()
(directory / "config.yaml").write_text(
"name: dry-run-tool\n"
"version: 1.0.0\n"
"description: Test dry-run behavior\n"
"output: constant\n"
)
return directory
@staticmethod
def args(tool_dir):
return SimpleNamespace(
path=str(tool_dir), dry_run=True, force=False, owner=""
)
def test_without_token_stops_after_local_preflight(
self, tool_dir, monkeypatch, capsys
):
from cmdforge.cli.registry_commands import _cmd_registry_publish
get_client = MagicMock()
monkeypatch.setattr(
"cmdforge.cli.registry_commands.load_config",
lambda: SimpleNamespace(registry=SimpleNamespace(token="")),
)
monkeypatch.setattr("cmdforge.registry_client.get_client", get_client)
assert _cmd_registry_publish(self.args(tool_dir)) == 0
get_client.assert_not_called()
assert "local preflight only" in capsys.readouterr().out
def test_renders_structured_registry_evidence(
self, tool_dir, monkeypatch, capsys
):
from cmdforge.cli.registry_commands import _cmd_registry_publish
client = MagicMock()
client.publish_tool.return_value = {
"preflight": {
"errors": [],
"warnings": ["remote warning"],
"suggestions": ["remote hint"],
},
"suggestions": {
"similar_tools": [{
"name": "official/similar",
"similarity": 0.8,
}],
"scrutiny": {
"findings": [{"result": "warning", "message": "review me"}]
},
},
}
monkeypatch.setattr(
"cmdforge.cli.registry_commands.load_config",
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
)
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
assert _cmd_registry_publish(self.args(tool_dir)) == 0
output = capsys.readouterr().out
assert "remote warning" in output
assert "official/similar" in output
assert "review me" in output
client.publish_tool.assert_called_once()
def test_registry_rejection_returns_failure(
self, tool_dir, monkeypatch, capsys
):
from cmdforge.cli.registry_commands import _cmd_registry_publish
from cmdforge.registry_client import RegistryError
client = MagicMock()
client.publish_tool.side_effect = RegistryError(
"SCRUTINY_FAILED", "Tool rejected"
)
monkeypatch.setattr(
"cmdforge.cli.registry_commands.load_config",
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
)
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
assert _cmd_registry_publish(self.args(tool_dir)) == 1
assert "Tool rejected" in capsys.readouterr().err
def test_strips_local_registry_metadata_from_payload(
self, tool_dir, monkeypatch
):
import yaml
from cmdforge.cli.registry_commands import _cmd_registry_publish
config_path = tool_dir / "config.yaml"
config_path.write_text(
config_path.read_text()
+ "registry_hash: sha256:stale\n"
+ "registry_status: approved\n"
+ "registry_owner: previous-owner\n"
+ "registry_feedback: old feedback\n"
)
client = MagicMock()
client.publish_tool.return_value = {"preflight": {"errors": []}}
monkeypatch.setattr(
"cmdforge.cli.registry_commands.load_config",
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
)
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
assert _cmd_registry_publish(self.args(tool_dir)) == 0
payload = yaml.safe_load(client.publish_tool.call_args.args[0])
assert not {
"registry_hash",
"registry_status",
"registry_owner",
"registry_feedback",
}.intersection(payload)
def test_non_mapping_config_returns_failure(self, tool_dir, capsys):
from cmdforge.cli.registry_commands import _cmd_registry_publish
(tool_dir / "config.yaml").write_text("- not\n- a\n- tool\n")
assert _cmd_registry_publish(self.args(tool_dir)) == 1
assert "YAML mapping" in capsys.readouterr().err
def test_normal_publish_runs_remote_preflight_before_mutation(
self, tool_dir, monkeypatch
):
from cmdforge.cli.registry_commands import _cmd_registry_publish
client = MagicMock()
client.get_me.return_value = {"slug": "testuser"}
client.get_my_tool_status.return_value = {"status": "pending"}
client.publish_tool.side_effect = [
{
"preflight": {"errors": [], "warnings": [], "suggestions": []},
"quality": {"score": 80, "evidence_coverage": 60, "categories": []},
},
{
"owner": "testuser", "name": "dry-run-tool",
"version": "1.0.0", "status": "pending",
},
]
monkeypatch.setattr(
"cmdforge.cli.registry_commands.load_config",
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
)
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
args = SimpleNamespace(
path=str(tool_dir), dry_run=False, force=True, owner=""
)
assert _cmd_registry_publish(args) == 0
assert client.publish_tool.call_count == 2
assert client.publish_tool.call_args_list[0].kwargs["dry_run"] is True
assert "dry_run" not in client.publish_tool.call_args_list[1].kwargs
def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
from cmdforge.cli import cmd_inspect
from cmdforge.registry_client import PaginatedResponse
tool = Tool(name="summary", output="constant")
client = MagicMock()
client.search_tools.return_value = PaginatedResponse(data=[{
"owner": "official",
"name": "summarize",
"description": "Summarize text",
}])
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
assert cmd_inspect(SimpleNamespace(name="summary", registry=True)) == 0
assert "official/summarize" in capsys.readouterr().out
def test_inspect_shows_contract_proposal_and_passing_conformance(
monkeypatch, capsys
):
from cmdforge.cli import cmd_inspect
step_schema = {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"],
}
tool = Tool(
name="contracted",
arguments=[],
input_schema={},
output_schema=step_schema,
steps=[PromptStep(
prompt="Answer", provider="paid-provider", output_var="answer",
output_schema=step_schema,
)],
output="{answer}",
)
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
assert cmd_inspect(SimpleNamespace(name="contracted", registry=False)) == 0
output = capsys.readouterr().out
assert "Contract conformance" in output
assert "PASSED" in output
def test_inspect_can_save_passing_baseline(tmp_path, monkeypatch, capsys):
from cmdforge.cli import cmd_inspect
from cmdforge.contract_testing import CONFORMANCE_FILE
tool = Tool(
name="baseline", version="1.0.0", input_schema={},
output_schema={"type": "string"}, output="stable",
path=tmp_path / "config.yaml",
)
monkeypatch.setattr("cmdforge.tool.load_tool", lambda name: tool)
args = SimpleNamespace(
name="baseline", registry=False, save_baseline=True
)
assert cmd_inspect(args) == 0
assert (tmp_path / CONFORMANCE_FILE).exists()
assert "Saved conformance baseline" in capsys.readouterr().out
def test_switch_to_existing_tool_closes_creation_page_first():
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import _switch_to_existing_tool
main_window = MagicMock()
_switch_to_existing_tool(main_window, "existing")
assert main_window.method_calls == [
call.close_tool_builder(),
call.open_tool_builder("existing"),
]
def test_tool_builder_save_scope_prefers_recognized_projects(tmp_path):
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import default_save_scope
assert default_save_scope(tmp_path) == "global"
(tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n")
assert default_save_scope(tmp_path) == "project"
def test_tool_builder_existing_tool_root_does_not_nest_tool_name(tmp_path):
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import existing_tool_root
config = tmp_path / ".cmdforge" / "demo" / "config.yaml"
assert existing_tool_root(config) == (tmp_path / ".cmdforge").resolve()
def test_guided_extraction_builds_new_tool_without_mutating_draft():
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import (
build_extracted_tool,
extraction_diff,
)
schema = {"type": "string"}
repeated = [
PromptStep(
prompt="First {input}", provider="mock", output_var="first",
output_schema=schema,
),
PromptStep(
prompt="Second {first}", provider="mock", output_var="second",
output_schema=schema,
),
]
draft = Tool(
name="draft", steps=repeated + [
PromptStep(
prompt="First {input}", provider="mock", output_var="first_copy",
output_schema=schema,
),
PromptStep(
prompt="Second {first}", provider="mock", output_var="second_copy",
output_schema=schema,
),
], input_schema=schema,
)
before = draft.to_dict()
extracted = build_extracted_tool(
draft,
{"type": "repeated_sequence", "locations": [1, 3], "length": 2},
"draft-shared",
)
assert len(extracted.steps) == 2
assert extracted.output == "{second}"
assert extracted.output_schema == schema
assert draft.to_dict() == before
assert "+++ draft-shared/config.yaml" in extraction_diff(extracted)
def test_guided_extraction_rejects_hidden_external_step_dependency():
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import build_extracted_tool
schema = {"type": "string"}
draft = Tool(name="draft", steps=[
PromptStep(
prompt="Produce", provider="mock", output_var="earlier",
output_schema=schema,
),
PromptStep(
prompt="Use {earlier}", provider="mock", output_var="one",
output_schema=schema,
),
PromptStep(
prompt="Finish {one}", provider="mock", output_var="two",
output_schema=schema,
),
])
with pytest.raises(ValueError, match="outside its boundary"):
build_extracted_tool(
draft,
{"type": "repeated_sequence", "locations": [2, 4], "length": 2},
"draft-shared",
)