rob/CmdForge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modernize provider support

Browse Source
This commit is contained in:
rob
2026-07-19 20:58:04 -03:00
parent 15943f251c
commit 6908d8ee6f
25 changed files with 1749 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+87
View File
@@ -199,6 +199,28 @@ class TestRunCommand:
captured = capsys.readouterr()
assert 'MOCK' in captured.out
def test_run_with_tool_specific_args_after_separator(self, temp_tools_dir, capsys):
"""cmdforge run should parse tool args with the wrapper parser."""
from cmdforge.tool import save_tool
tool = Tool(
name="greet",
arguments=[
ToolArgument(flag="--name", variable="name", default="World")
],
output="Hello, {name}!"
)
save_tool(tool)
with patch('sys.argv', ['cmdforge', 'run', 'greet', '--', '--name', 'Alice']):
with patch('sys.stdin', StringIO("")):
with patch('sys.stdin.isatty', return_value=True):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'Hello, Alice!' in captured.out
def test_run_nonexistent_tool(self, temp_tools_dir, capsys):
"""Running nonexistent tool should fail."""
with patch('sys.argv', ['cmdforge', 'run', 'nonexistent']):
@@ -272,6 +294,71 @@ class TestProvidersCommand:
assert provider is not None
assert provider.command == 'my-ai --prompt'
def test_providers_add_api_configuration(self, temp_providers_file, capsys):
with patch('sys.argv', [
'cmdforge', 'providers', 'add', 'custom-api',
'https://example.test/v1',
'--type', 'api',
'--model', 'example/model',
'--api-key-env', 'CUSTOM_API_KEY',
'--tag', 'api',
'--fallback-chain', 'free',
]):
result = main()
assert result == 0
from cmdforge.providers import PRESET_CHAINS, get_provider
provider = get_provider('custom-api')
assert provider.type == 'api'
assert provider.model == 'example/model'
assert provider.api_key_env == 'CUSTOM_API_KEY'
assert provider.tags == ['api']
assert provider.fallback_chain == PRESET_CHAINS['free']
def test_providers_list_reports_missing_api_key(self, temp_providers_file, capsys):
from cmdforge.providers import Provider, save_providers
save_providers([
Provider(
'custom-api',
'https://example.test/v1',
type='api',
model='example/model',
api_key_env='MISSING_CUSTOM_API_KEY',
)
])
with patch.dict('os.environ', {'MISSING_CUSTOM_API_KEY': ''}):
with patch('sys.argv', ['cmdforge', 'providers', 'list']):
result = main()
assert result == 0
captured = capsys.readouterr()
assert 'API KEY NOT SET (MISSING_CUSTOM_API_KEY)' in captured.out
assert 'NOT FOUND (https://' not in captured.out
def test_providers_discover_adds_new_provider(self, temp_providers_file, capsys):
discovered = [{
'source': 'cli',
'binary': 'example-ai',
'path': '/usr/bin/example-ai',
'name': 'example-ai',
'command': 'example-ai --print',
'description': 'Example provider',
'tags': ['test'],
}]
with patch('cmdforge.providers.discover_installed_providers', return_value=discovered):
with patch('sys.argv', ['cmdforge', 'providers', 'discover', '--add']):
result = main()
assert result == 0
from cmdforge.providers import get_provider
provider = get_provider('example-ai')
assert provider is not None
assert provider.command == 'example-ai --print'
assert provider.tags == ['test']
def test_providers_remove(self, temp_providers_file, capsys):
"""Remove a provider."""
from cmdforge.providers import add_provider, Provider
tests/test_collection.py
+19
View File
@@ -269,6 +269,25 @@ class TestResolveToolReferences:
assert ("official/unapproved", "no approved public version") in result.registry_tool_issues
def test_get_visibility_uses_owner_path(self, tmp_path):
from cmdforge.collection import _get_tool_visibility_from_yaml
from cmdforge.tool import get_tools_dir
with patch('cmdforge.tool.TOOLS_DIR', tmp_path / ".cmdforge"):
tools_dir = get_tools_dir()
owned = tools_dir / "myuser" / "private-tool"
owned.mkdir(parents=True)
(owned / "config.yaml").write_text(
yaml.safe_dump({
"name": "private-tool",
"visibility": "private",
})
)
result = _get_tool_visibility_from_yaml("private-tool", my_owner="myuser")
assert result == "private"
class TestToolResolutionResult:
"""Tests for ToolResolutionResult dataclass."""
tests/test_packaging.py
+20
View File
@@ -0,0 +1,20 @@
"""Tests for packaging metadata."""
import sys
import pytest
if sys.version_info >= (3, 11):
import tomllib
else:
tomllib = pytest.importorskip("tomli")
def test_web_templates_and_static_are_declared_as_package_data():
with open("pyproject.toml", "rb") as f:
data = tomllib.load(f)
package_data = data["tool"]["setuptools"]["package-data"]["cmdforge.web"]
assert "templates/**/*.html" in package_data
assert "static/**/*" in package_data
tests/test_providers.py
+159 -1
View File
@@ -11,7 +11,7 @@ from cmdforge.providers import (
Provider, ProviderResult,
load_providers, save_providers, get_provider,
add_provider, delete_provider,
call_provider, mock_provider,
call_provider, discover_installed_providers, mock_provider,
DEFAULT_PROVIDERS, strip_ansi
)
@@ -71,6 +71,29 @@ class TestProvider:
assert restored.command == original.command
assert restored.description == original.description
def test_legacy_positional_fallback_is_preserved(self):
provider = Provider("primary", "primary-cmd", "Primary", "backup")
assert provider.fallback == "backup"
assert provider.type == "subprocess"
def test_extended_fields_roundtrip(self):
original = Provider(
name="api-test",
command="https://example.test/v1",
description="Test API",
fallback="mock",
type="api",
model="test/model",
tags=["api", "test"],
install={"cost": "test"},
fallback_chain=["backup", "mock"],
api_key_env="TEST_API_KEY",
pty_config={"prompt_pattern": ">"},
)
assert Provider.from_dict(original.to_dict()) == original
class TestProviderResult:
"""Tests for ProviderResult dataclass."""
@@ -144,6 +167,34 @@ class TestProviderPersistence:
assert loaded[0].name == "test1"
assert loaded[1].name == "test2"
def test_save_providers_uses_private_file_permissions(self, temp_providers_file):
save_providers([Provider("test", "cmd")])
assert oct(temp_providers_file.stat().st_mode & 0o777) == "0o600"
def test_legacy_config_gains_missing_defaults_without_losing_custom_provider(self, temp_providers_file):
temp_providers_file.parent.mkdir(parents=True)
temp_providers_file.write_text(yaml.safe_dump({
"providers": [
{
"name": "custom",
"command": "custom-command",
"description": "Keep me",
"fallback": "mock",
}
]
}))
loaded = load_providers()
custom = next(provider for provider in loaded if provider.name == "custom")
assert custom.command == "custom-command"
assert custom.fallback == "mock"
assert any(provider.name == "opencode-free" for provider in loaded)
assert any(provider.name == "openrouter" for provider in loaded)
saved = yaml.safe_load(temp_providers_file.read_text())
assert saved["version"] == 2
def test_get_provider_exists(self, temp_providers_file):
providers = [
Provider("target", "target-cmd", "Target provider")
@@ -246,6 +297,16 @@ class TestCallProvider:
assert result.success is False
assert "not found" in result.error.lower()
@patch('subprocess.run')
def test_call_provider_rejects_non_integer_max_tokens(self, mock_run, temp_providers_file):
save_providers([Provider("claude", "claude -p")])
result = call_provider("claude", "Test", max_tokens="1; touch /tmp/pwned")
assert result.success is False
assert "max_tokens" in result.error
mock_run.assert_not_called()
@patch('subprocess.run')
@patch('shutil.which')
def test_call_real_provider_success(self, mock_which, mock_run, temp_providers_file):
@@ -456,6 +517,18 @@ class TestProviderFallback:
assert result.success is True
assert "[MOCK]" in result.text
def test_fallback_chain_continues_after_failed_candidate(self, temp_providers_file):
save_providers([
Provider("primary", "missing-primary", fallback_chain=["secondary", "mock"]),
Provider("secondary", "missing-secondary"),
Provider("mock", "mock"),
])
result = call_provider("primary", "Test prompt")
assert result.success is True
assert "[MOCK]" in result.text
def test_fallback_prevents_infinite_loop(self, temp_providers_file):
"""Circular fallback references should not cause infinite loop."""
save_providers([
@@ -626,3 +699,88 @@ class TestMaxTokens:
assert "--max-tokens 4096" in calls[0][0][0]
assert "--max-tokens 4096" in calls[1][0][0]
class TestApiProviders:
@pytest.fixture
def temp_providers_file(self, tmp_path):
providers_file = tmp_path / ".cmdforge" / "providers.yaml"
with patch('cmdforge.providers.PROVIDERS_FILE', providers_file):
yield providers_file
@patch('requests.post')
def test_openai_compatible_api_dispatch(self, mock_post, temp_providers_file):
response = MagicMock(status_code=200)
response.json.return_value = {
"choices": [{"message": {"content": "API response"}}]
}
mock_post.return_value = response
save_providers([
Provider(
"test-api",
"https://example.test/v1",
type="api",
model="example/model",
api_key_env="TEST_API_KEY",
)
])
with patch.dict('os.environ', {"TEST_API_KEY": "secret"}):
result = call_provider("test-api", "Prompt", max_tokens=128)
assert result.success is True
assert result.text == "API response"
mock_post.assert_called_once_with(
"https://example.test/v1/chat/completions",
json={
"model": "example/model",
"messages": [{"role": "user", "content": "Prompt"}],
"max_tokens": 128,
},
headers={
"Authorization": "Bearer secret",
"Content-Type": "application/json",
},
timeout=300,
)
def test_api_provider_requires_configured_key(self, temp_providers_file):
save_providers([
Provider(
"test-api",
"https://example.test/v1",
type="api",
model="example/model",
api_key_env="MISSING_TEST_API_KEY",
)
])
with patch.dict('os.environ', {"MISSING_TEST_API_KEY": ""}):
result = call_provider("test-api", "Prompt")
assert result.success is False
assert "MISSING_TEST_API_KEY" in result.error
class TestProviderDiscovery:
def test_ollama_model_names_include_tags_and_are_unique(self):
ollama_list = MagicMock(
returncode=0,
stdout=(
"NAME ID SIZE MODIFIED\n"
"hermes4.3:latest abc 1 GB now\n"
"hermes4.3:q4_k_m def 1 GB now\n"
),
)
def which(binary):
return "/usr/bin/ollama" if binary == "ollama" else None
with patch('cmdforge.providers.shutil.which', side_effect=which):
with patch('cmdforge.providers.subprocess.run', return_value=ollama_list):
found = discover_installed_providers()
models = [item for item in found if item["source"] == "ollama-model"]
assert len(models) == 2
assert len({item["name"] for item in models}) == 2
assert models[0]["name"] == "ollama-hermes4-3-latest"
assert models[1]["name"] == "ollama-hermes4-3-q4-k-m"
tests/test_registry_integration.py
+7
View File
@@ -202,6 +202,13 @@ class TestConfig:
assert restored.auto_fetch_from_registry is False
assert restored.default_provider == "claude"
def test_save_config_uses_private_permissions(self, tmp_path):
with patch("cmdforge.config.CONFIG_DIR", tmp_path / ".cmdforge"):
path = save_config(Config(registry=RegistryConfig(token="test_token")))
assert oct(path.parent.stat().st_mode & 0o777) == "0o700"
assert oct(path.stat().st_mode & 0o777) == "0o600"
class TestRegistryClient:
"""Tests for the registry client (mocked)."""
tests/test_runner.py
+91 -1
View File
@@ -8,7 +8,8 @@ from cmdforge.runner import (
execute_prompt_step,
execute_code_step,
run_tool,
create_argument_parser
create_argument_parser,
collect_custom_args
)
from cmdforge.tool import Tool, ToolArgument, PromptStep, CodeStep
from cmdforge.providers import ProviderResult
@@ -246,6 +247,47 @@ class TestExecutePromptStep:
# Should use override, not step's provider
assert mock_call.call_args[0][0] == "gpt4"
@patch('cmdforge.runner.call_provider')
def test_prompt_file_loaded_relative_to_tool_dir(self, mock_call, tmp_path):
mock_call.return_value = ProviderResult(text="response", success=True)
tool_dir = tmp_path / "tool"
tool_dir.mkdir()
(tool_dir / "prompt.txt").write_text("Summarize: {input}")
step = PromptStep(
prompt="",
provider="claude",
output_var="out",
prompt_file="prompt.txt",
plain_text=True,
)
output, success = execute_prompt_step(
step,
{"input": "file text"},
base_dir=tool_dir,
)
assert success is True
assert output == "response"
assert mock_call.call_args[0][1] == "Summarize: file text"
def test_prompt_file_rejects_path_traversal(self, tmp_path):
tool_dir = tmp_path / "tool"
tool_dir.mkdir()
(tmp_path / "outside.txt").write_text("Nope")
step = PromptStep(
prompt="",
provider="mock",
output_var="out",
prompt_file="../outside.txt",
plain_text=True,
)
output, success = execute_prompt_step(step, {"input": ""}, base_dir=tool_dir)
assert success is False
assert output == ""
class TestStructuredOutput:
"""Tests for structured output enforcement."""
@@ -685,6 +727,38 @@ result = multiply_all([1, 2, 3])
assert success is True
assert outputs["result"] == [10, 20, 30]
def test_code_file_loaded_relative_to_tool_dir(self, tmp_path):
tool_dir = tmp_path / "tool"
tool_dir.mkdir()
(tool_dir / "process.py").write_text("result = input.upper()")
step = CodeStep(code="", code_file="process.py", output_var="result")
outputs, success = execute_code_step(
step,
{"input": "hello"},
step_num=1,
base_dir=tool_dir,
)
assert success is True
assert outputs["result"] == "HELLO"
def test_code_file_rejects_path_traversal(self, tmp_path):
tool_dir = tmp_path / "tool"
tool_dir.mkdir()
(tmp_path / "outside.py").write_text("result = 'bad'")
step = CodeStep(code="", code_file="../outside.py", output_var="result")
outputs, success = execute_code_step(
step,
{"input": ""},
step_num=1,
base_dir=tool_dir,
)
assert success is False
assert outputs == {}
class TestRunTool:
"""Tests for run_tool function."""
@@ -920,3 +994,19 @@ class TestCreateArgumentParser:
assert args.input_file == "input.txt"
assert args.output_file == "output.txt"
def test_collect_custom_args_uses_parser_namespace(self):
tool = Tool(
name="test",
arguments=[
ToolArgument(flag="--max-size", variable="max_size", default="100"),
ToolArgument(flag="-f", variable="format"),
]
)
parser = create_argument_parser(tool)
args = parser.parse_args(["--max-size", "50", "-f", "json"])
assert collect_custom_args(tool, args) == {
"max_size": "50",
"format": "json",
}
tests/test_tool.py
+38
View File
@@ -142,6 +142,28 @@ class TestPromptStep:
assert restored.output_var == original.output_var
assert restored.prompt_file == original.prompt_file
def test_from_dict_rejects_non_integer_max_tokens(self):
data = {
"type": "prompt",
"prompt": "Test",
"provider": "claude",
"output_var": "result",
"max_tokens": "100; touch /tmp/pwned",
}
with pytest.raises(ValueError, match="max_tokens"):
PromptStep.from_dict(data)
def test_from_dict_rejects_out_of_range_max_tokens(self):
data = {
"type": "prompt",
"prompt": "Test",
"provider": "claude",
"output_var": "result",
"max_tokens": 0,
}
with pytest.raises(ValueError, match="max_tokens"):
PromptStep.from_dict(data)
class TestCodeStep:
"""Tests for CodeStep dataclass."""
@@ -507,6 +529,22 @@ class TestLegacyFormat:
assert len(tool.arguments) == 1
assert tool.arguments[0].variable == "max_size"
def test_legacy_format_preserves_visibility(self, temp_tools_dir):
tool_dir = temp_tools_dir / "legacy-private"
tool_dir.mkdir(parents=True)
legacy_config = {
"name": "legacy-private",
"prompt": "Process: {input}",
"provider": "mock",
"visibility": "private",
}
(tool_dir / "config.yaml").write_text(yaml.dump(legacy_config))
tool = load_tool("legacy-private")
assert tool is not None
assert tool.visibility == "private"
class TestDefaultCategories:
"""Tests for default categories."""
tests/test_version.py
+14
View File
@@ -216,3 +216,17 @@ class TestCompareVersions:
compare_versions("invalid", "1.0.0")
with pytest.raises(ValueError):
compare_versions("1.0.0", "invalid")
class TestRegexEdgeCases:
"""Tests for semver regex edge cases."""
def test_dot_separators_required(self):
assert Version.parse("1x2x3") is None
assert Version.parse("1-2-3") is None
def test_double_dot_rejected(self):
assert Version.parse("1..2.3") is None
def test_trailing_dot_rejected(self):
assert Version.parse("1.2.3.") is None