Add provider fallback, visual schema builder, and UI improvements
Provider fallback system: - Add optional fallback field to Provider dataclass - Automatically try fallback provider when primary fails (timeout, error, offline) - Prevent infinite loops with circular fallback detection - Add fallback column and dropdown in providers UI Visual schema builder for prompt steps: - Add SchemaBuilderDialog with table-based field editor - Support field name, type, description, and required flag - Live JSON schema preview - Integrate into PromptStepDialog with Edit Schema button - Fix output_schema being lost when editing prompt steps Other improvements: - Add search box to tools page - Improve schema example generation for structured output - Update CLAUDE.md documentation Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
+135
-4
@@ -94,12 +94,18 @@ class TestMockProvider:
|
||||
def test_mock_returns_success(self):
|
||||
result = mock_provider("Test prompt")
|
||||
assert result.success is True
|
||||
assert "[MOCK RESPONSE]" in result.text
|
||||
# Mock returns structured JSON with output and reasoning
|
||||
import json
|
||||
parsed = json.loads(result.text)
|
||||
assert "[MOCK]" in parsed["output"]
|
||||
|
||||
def test_mock_includes_prompt_info(self):
|
||||
result = mock_provider("This is a test prompt")
|
||||
assert "Prompt length:" in result.text
|
||||
assert "chars" in result.text
|
||||
import json
|
||||
parsed = json.loads(result.text)
|
||||
# Prompt info is in reasoning field
|
||||
assert "chars" in parsed["reasoning"]
|
||||
assert "lines" in parsed["reasoning"]
|
||||
|
||||
def test_mock_shows_first_line_preview(self):
|
||||
result = mock_provider("First line here\nSecond line")
|
||||
@@ -228,7 +234,10 @@ class TestCallProvider:
|
||||
"""Mock provider should work without subprocess."""
|
||||
result = call_provider("mock", "Test prompt")
|
||||
assert result.success is True
|
||||
assert "[MOCK RESPONSE]" in result.text
|
||||
# Mock returns structured JSON
|
||||
import json
|
||||
parsed = json.loads(result.text)
|
||||
assert "[MOCK]" in parsed["output"]
|
||||
|
||||
def test_call_nonexistent_provider(self, temp_providers_file):
|
||||
save_providers([])
|
||||
@@ -376,3 +385,125 @@ class TestProviderCommandParsing:
|
||||
# Error should show expanded path
|
||||
home = os.environ.get("HOME", "")
|
||||
assert "$HOME" not in result.error or home in result.error
|
||||
|
||||
|
||||
class TestProviderFallback:
|
||||
"""Tests for provider fallback functionality."""
|
||||
|
||||
@pytest.fixture
|
||||
def temp_providers_file(self, tmp_path):
|
||||
providers_file = tmp_path / ".cmdforge" / "providers.yaml"
|
||||
with patch('cmdforge.providers.PROVIDERS_FILE', providers_file):
|
||||
yield providers_file
|
||||
|
||||
def test_provider_with_fallback_to_dict(self):
|
||||
"""Provider with fallback should include it in dict."""
|
||||
provider = Provider(
|
||||
name="cloud",
|
||||
command="cloud-ai",
|
||||
description="Cloud AI",
|
||||
fallback="local"
|
||||
)
|
||||
d = provider.to_dict()
|
||||
assert d["fallback"] == "local"
|
||||
|
||||
def test_provider_without_fallback_to_dict(self):
|
||||
"""Provider without fallback should not include fallback key."""
|
||||
provider = Provider(name="simple", command="simple-ai")
|
||||
d = provider.to_dict()
|
||||
assert "fallback" not in d
|
||||
|
||||
def test_provider_from_dict_with_fallback(self):
|
||||
"""Provider.from_dict should handle fallback field."""
|
||||
data = {
|
||||
"name": "cloud",
|
||||
"command": "cloud-ai",
|
||||
"fallback": "local"
|
||||
}
|
||||
provider = Provider.from_dict(data)
|
||||
assert provider.fallback == "local"
|
||||
|
||||
def test_provider_from_dict_without_fallback(self):
|
||||
"""Provider.from_dict should handle missing fallback."""
|
||||
data = {"name": "simple", "command": "simple-ai"}
|
||||
provider = Provider.from_dict(data)
|
||||
assert provider.fallback is None
|
||||
|
||||
def test_fallback_triggers_on_failure(self, temp_providers_file):
|
||||
"""When primary provider fails, fallback should be tried."""
|
||||
save_providers([
|
||||
Provider("primary", "nonexistent-cmd", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("primary", "Test prompt")
|
||||
|
||||
# Fallback to mock should succeed
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
def test_fallback_chain(self, temp_providers_file):
|
||||
"""Fallback can chain to another provider with fallback."""
|
||||
save_providers([
|
||||
Provider("first", "nonexistent1", fallback="second"),
|
||||
Provider("second", "nonexistent2", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("first", "Test prompt")
|
||||
|
||||
# Should chain: first -> second -> mock
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
def test_fallback_prevents_infinite_loop(self, temp_providers_file):
|
||||
"""Circular fallback references should not cause infinite loop."""
|
||||
save_providers([
|
||||
Provider("a", "nonexistent-a", fallback="b"),
|
||||
Provider("b", "nonexistent-b", fallback="a"),
|
||||
])
|
||||
|
||||
result = call_provider("a", "Test prompt")
|
||||
|
||||
# Should fail gracefully, not loop forever
|
||||
assert result.success is False
|
||||
assert "not found" in result.error.lower()
|
||||
|
||||
@patch('subprocess.run')
|
||||
@patch('shutil.which')
|
||||
def test_fallback_on_timeout(self, mock_which, mock_run, temp_providers_file):
|
||||
"""Fallback should trigger on timeout."""
|
||||
import subprocess
|
||||
mock_which.return_value = "/usr/bin/slow-ai"
|
||||
mock_run.side_effect = subprocess.TimeoutExpired("slow-ai", 5)
|
||||
|
||||
save_providers([
|
||||
Provider("slow", "slow-ai", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("slow", "Test prompt", timeout=5)
|
||||
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
@patch('subprocess.run')
|
||||
@patch('shutil.which')
|
||||
def test_fallback_on_nonzero_exit(self, mock_which, mock_run, temp_providers_file):
|
||||
"""Fallback should trigger on non-zero exit code."""
|
||||
mock_which.return_value = "/usr/bin/failing-ai"
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr="API error"
|
||||
)
|
||||
|
||||
save_providers([
|
||||
Provider("failing", "failing-ai", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("failing", "Test prompt")
|
||||
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
Reference in New Issue
Block a user