Add provider fallback, visual schema builder, and UI improvements

Provider fallback system:
- Add optional fallback field to Provider dataclass
- Automatically try fallback provider when primary fails (timeout, error, offline)
- Prevent infinite loops with circular fallback detection
- Add fallback column and dropdown in providers UI

Visual schema builder for prompt steps:
- Add SchemaBuilderDialog with table-based field editor
- Support field name, type, description, and required flag
- Live JSON schema preview
- Integrate into PromptStepDialog with Edit Schema button
- Fix output_schema being lost when editing prompt steps

Other improvements:
- Add search box to tools page
- Improve schema example generation for structured output
- Update CLAUDE.md documentation

Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
rob
2026-02-21 00:50:18 -04:00
co-authored by Claude Opus 4.5
parent 8581775002
commit 71c6358b9d
9 changed files with 892 additions and 99 deletions
+135 -4
View File
@@ -94,12 +94,18 @@ class TestMockProvider:
def test_mock_returns_success(self):
result = mock_provider("Test prompt")
assert result.success is True
assert "[MOCK RESPONSE]" in result.text
# Mock returns structured JSON with output and reasoning
import json
parsed = json.loads(result.text)
assert "[MOCK]" in parsed["output"]
def test_mock_includes_prompt_info(self):
result = mock_provider("This is a test prompt")
assert "Prompt length:" in result.text
assert "chars" in result.text
import json
parsed = json.loads(result.text)
# Prompt info is in reasoning field
assert "chars" in parsed["reasoning"]
assert "lines" in parsed["reasoning"]
def test_mock_shows_first_line_preview(self):
result = mock_provider("First line here\nSecond line")
@@ -228,7 +234,10 @@ class TestCallProvider:
"""Mock provider should work without subprocess."""
result = call_provider("mock", "Test prompt")
assert result.success is True
assert "[MOCK RESPONSE]" in result.text
# Mock returns structured JSON
import json
parsed = json.loads(result.text)
assert "[MOCK]" in parsed["output"]
def test_call_nonexistent_provider(self, temp_providers_file):
save_providers([])
@@ -376,3 +385,125 @@ class TestProviderCommandParsing:
# Error should show expanded path
home = os.environ.get("HOME", "")
assert "$HOME" not in result.error or home in result.error
class TestProviderFallback:
"""Tests for provider fallback functionality."""
@pytest.fixture
def temp_providers_file(self, tmp_path):
providers_file = tmp_path / ".cmdforge" / "providers.yaml"
with patch('cmdforge.providers.PROVIDERS_FILE', providers_file):
yield providers_file
def test_provider_with_fallback_to_dict(self):
"""Provider with fallback should include it in dict."""
provider = Provider(
name="cloud",
command="cloud-ai",
description="Cloud AI",
fallback="local"
)
d = provider.to_dict()
assert d["fallback"] == "local"
def test_provider_without_fallback_to_dict(self):
"""Provider without fallback should not include fallback key."""
provider = Provider(name="simple", command="simple-ai")
d = provider.to_dict()
assert "fallback" not in d
def test_provider_from_dict_with_fallback(self):
"""Provider.from_dict should handle fallback field."""
data = {
"name": "cloud",
"command": "cloud-ai",
"fallback": "local"
}
provider = Provider.from_dict(data)
assert provider.fallback == "local"
def test_provider_from_dict_without_fallback(self):
"""Provider.from_dict should handle missing fallback."""
data = {"name": "simple", "command": "simple-ai"}
provider = Provider.from_dict(data)
assert provider.fallback is None
def test_fallback_triggers_on_failure(self, temp_providers_file):
"""When primary provider fails, fallback should be tried."""
save_providers([
Provider("primary", "nonexistent-cmd", fallback="mock"),
Provider("mock", "mock")
])
result = call_provider("primary", "Test prompt")
# Fallback to mock should succeed
assert result.success is True
assert "[MOCK]" in result.text
def test_fallback_chain(self, temp_providers_file):
"""Fallback can chain to another provider with fallback."""
save_providers([
Provider("first", "nonexistent1", fallback="second"),
Provider("second", "nonexistent2", fallback="mock"),
Provider("mock", "mock")
])
result = call_provider("first", "Test prompt")
# Should chain: first -> second -> mock
assert result.success is True
assert "[MOCK]" in result.text
def test_fallback_prevents_infinite_loop(self, temp_providers_file):
"""Circular fallback references should not cause infinite loop."""
save_providers([
Provider("a", "nonexistent-a", fallback="b"),
Provider("b", "nonexistent-b", fallback="a"),
])
result = call_provider("a", "Test prompt")
# Should fail gracefully, not loop forever
assert result.success is False
assert "not found" in result.error.lower()
@patch('subprocess.run')
@patch('shutil.which')
def test_fallback_on_timeout(self, mock_which, mock_run, temp_providers_file):
"""Fallback should trigger on timeout."""
import subprocess
mock_which.return_value = "/usr/bin/slow-ai"
mock_run.side_effect = subprocess.TimeoutExpired("slow-ai", 5)
save_providers([
Provider("slow", "slow-ai", fallback="mock"),
Provider("mock", "mock")
])
result = call_provider("slow", "Test prompt", timeout=5)
assert result.success is True
assert "[MOCK]" in result.text
@patch('subprocess.run')
@patch('shutil.which')
def test_fallback_on_nonzero_exit(self, mock_which, mock_run, temp_providers_file):
"""Fallback should trigger on non-zero exit code."""
mock_which.return_value = "/usr/bin/failing-ai"
mock_run.return_value = MagicMock(
returncode=1,
stdout="",
stderr="API error"
)
save_providers([
Provider("failing", "failing-ai", fallback="mock"),
Provider("mock", "mock")
])
result = call_provider("failing", "Test prompt")
assert result.success is True
assert "[MOCK]" in result.text