Add provider fallback, visual schema builder, and UI improvements
Provider fallback system: - Add optional fallback field to Provider dataclass - Automatically try fallback provider when primary fails (timeout, error, offline) - Prevent infinite loops with circular fallback detection - Add fallback column and dropdown in providers UI Visual schema builder for prompt steps: - Add SchemaBuilderDialog with table-based field editor - Support field name, type, description, and required flag - Live JSON schema preview - Integrate into PromptStepDialog with Edit Schema button - Fix output_schema being lost when editing prompt steps Other improvements: - Add search box to tools page - Improve schema example generation for structured output - Update CLAUDE.md documentation Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
Claude Opus 4.5
+135
-4
@@ -94,12 +94,18 @@ class TestMockProvider:
|
||||
def test_mock_returns_success(self):
|
||||
result = mock_provider("Test prompt")
|
||||
assert result.success is True
|
||||
assert "[MOCK RESPONSE]" in result.text
|
||||
# Mock returns structured JSON with output and reasoning
|
||||
import json
|
||||
parsed = json.loads(result.text)
|
||||
assert "[MOCK]" in parsed["output"]
|
||||
|
||||
def test_mock_includes_prompt_info(self):
|
||||
result = mock_provider("This is a test prompt")
|
||||
assert "Prompt length:" in result.text
|
||||
assert "chars" in result.text
|
||||
import json
|
||||
parsed = json.loads(result.text)
|
||||
# Prompt info is in reasoning field
|
||||
assert "chars" in parsed["reasoning"]
|
||||
assert "lines" in parsed["reasoning"]
|
||||
|
||||
def test_mock_shows_first_line_preview(self):
|
||||
result = mock_provider("First line here\nSecond line")
|
||||
@@ -228,7 +234,10 @@ class TestCallProvider:
|
||||
"""Mock provider should work without subprocess."""
|
||||
result = call_provider("mock", "Test prompt")
|
||||
assert result.success is True
|
||||
assert "[MOCK RESPONSE]" in result.text
|
||||
# Mock returns structured JSON
|
||||
import json
|
||||
parsed = json.loads(result.text)
|
||||
assert "[MOCK]" in parsed["output"]
|
||||
|
||||
def test_call_nonexistent_provider(self, temp_providers_file):
|
||||
save_providers([])
|
||||
@@ -376,3 +385,125 @@ class TestProviderCommandParsing:
|
||||
# Error should show expanded path
|
||||
home = os.environ.get("HOME", "")
|
||||
assert "$HOME" not in result.error or home in result.error
|
||||
|
||||
|
||||
class TestProviderFallback:
|
||||
"""Tests for provider fallback functionality."""
|
||||
|
||||
@pytest.fixture
|
||||
def temp_providers_file(self, tmp_path):
|
||||
providers_file = tmp_path / ".cmdforge" / "providers.yaml"
|
||||
with patch('cmdforge.providers.PROVIDERS_FILE', providers_file):
|
||||
yield providers_file
|
||||
|
||||
def test_provider_with_fallback_to_dict(self):
|
||||
"""Provider with fallback should include it in dict."""
|
||||
provider = Provider(
|
||||
name="cloud",
|
||||
command="cloud-ai",
|
||||
description="Cloud AI",
|
||||
fallback="local"
|
||||
)
|
||||
d = provider.to_dict()
|
||||
assert d["fallback"] == "local"
|
||||
|
||||
def test_provider_without_fallback_to_dict(self):
|
||||
"""Provider without fallback should not include fallback key."""
|
||||
provider = Provider(name="simple", command="simple-ai")
|
||||
d = provider.to_dict()
|
||||
assert "fallback" not in d
|
||||
|
||||
def test_provider_from_dict_with_fallback(self):
|
||||
"""Provider.from_dict should handle fallback field."""
|
||||
data = {
|
||||
"name": "cloud",
|
||||
"command": "cloud-ai",
|
||||
"fallback": "local"
|
||||
}
|
||||
provider = Provider.from_dict(data)
|
||||
assert provider.fallback == "local"
|
||||
|
||||
def test_provider_from_dict_without_fallback(self):
|
||||
"""Provider.from_dict should handle missing fallback."""
|
||||
data = {"name": "simple", "command": "simple-ai"}
|
||||
provider = Provider.from_dict(data)
|
||||
assert provider.fallback is None
|
||||
|
||||
def test_fallback_triggers_on_failure(self, temp_providers_file):
|
||||
"""When primary provider fails, fallback should be tried."""
|
||||
save_providers([
|
||||
Provider("primary", "nonexistent-cmd", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("primary", "Test prompt")
|
||||
|
||||
# Fallback to mock should succeed
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
def test_fallback_chain(self, temp_providers_file):
|
||||
"""Fallback can chain to another provider with fallback."""
|
||||
save_providers([
|
||||
Provider("first", "nonexistent1", fallback="second"),
|
||||
Provider("second", "nonexistent2", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("first", "Test prompt")
|
||||
|
||||
# Should chain: first -> second -> mock
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
def test_fallback_prevents_infinite_loop(self, temp_providers_file):
|
||||
"""Circular fallback references should not cause infinite loop."""
|
||||
save_providers([
|
||||
Provider("a", "nonexistent-a", fallback="b"),
|
||||
Provider("b", "nonexistent-b", fallback="a"),
|
||||
])
|
||||
|
||||
result = call_provider("a", "Test prompt")
|
||||
|
||||
# Should fail gracefully, not loop forever
|
||||
assert result.success is False
|
||||
assert "not found" in result.error.lower()
|
||||
|
||||
@patch('subprocess.run')
|
||||
@patch('shutil.which')
|
||||
def test_fallback_on_timeout(self, mock_which, mock_run, temp_providers_file):
|
||||
"""Fallback should trigger on timeout."""
|
||||
import subprocess
|
||||
mock_which.return_value = "/usr/bin/slow-ai"
|
||||
mock_run.side_effect = subprocess.TimeoutExpired("slow-ai", 5)
|
||||
|
||||
save_providers([
|
||||
Provider("slow", "slow-ai", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("slow", "Test prompt", timeout=5)
|
||||
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
@patch('subprocess.run')
|
||||
@patch('shutil.which')
|
||||
def test_fallback_on_nonzero_exit(self, mock_which, mock_run, temp_providers_file):
|
||||
"""Fallback should trigger on non-zero exit code."""
|
||||
mock_which.return_value = "/usr/bin/failing-ai"
|
||||
mock_run.return_value = MagicMock(
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr="API error"
|
||||
)
|
||||
|
||||
save_providers([
|
||||
Provider("failing", "failing-ai", fallback="mock"),
|
||||
Provider("mock", "mock")
|
||||
])
|
||||
|
||||
result = call_provider("failing", "Test prompt")
|
||||
|
||||
assert result.success is True
|
||||
assert "[MOCK]" in result.text
|
||||
|
||||
+107
-11
@@ -268,11 +268,10 @@ class TestStructuredOutput:
|
||||
output, success = execute_prompt_step(step, {"input": ""})
|
||||
|
||||
assert success is True
|
||||
# Output should be normalized JSON
|
||||
import json
|
||||
parsed = json.loads(output)
|
||||
assert parsed["output"] == "Hello, world!"
|
||||
assert parsed["reasoning"] == "Greeting requested"
|
||||
# Output should be a parsed dict (for code step compatibility)
|
||||
assert isinstance(output, dict)
|
||||
assert output["output"] == "Hello, world!"
|
||||
assert output["reasoning"] == "Greeting requested"
|
||||
|
||||
@patch('cmdforge.runner.call_provider')
|
||||
def test_structured_output_strips_markdown_fences(self, mock_call):
|
||||
@@ -291,9 +290,8 @@ class TestStructuredOutput:
|
||||
output, success = execute_prompt_step(step, {"input": ""})
|
||||
|
||||
assert success is True
|
||||
import json
|
||||
parsed = json.loads(output)
|
||||
assert parsed["output"] == "Hello"
|
||||
assert isinstance(output, dict)
|
||||
assert output["output"] == "Hello"
|
||||
|
||||
@patch('cmdforge.runner.call_provider')
|
||||
def test_structured_output_retry_on_invalid_json(self, mock_call):
|
||||
@@ -366,9 +364,8 @@ class TestStructuredOutput:
|
||||
output, success = execute_prompt_step(step, {"input": ""})
|
||||
|
||||
assert success is True
|
||||
import json
|
||||
parsed = json.loads(output)
|
||||
assert parsed["score"] == 0.8
|
||||
assert isinstance(output, dict)
|
||||
assert output["score"] == 0.8
|
||||
|
||||
@patch('cmdforge.runner.call_provider')
|
||||
def test_structured_output_schema_violation_triggers_retry(self, mock_call):
|
||||
@@ -414,6 +411,105 @@ class TestStructuredOutput:
|
||||
assert mock_call.call_count == 1
|
||||
|
||||
|
||||
class TestSchemaInstructions:
|
||||
"""Tests for schema instruction generation."""
|
||||
|
||||
def test_generate_schema_example_default(self):
|
||||
"""Default schema should generate output/reasoning example."""
|
||||
from cmdforge.runner import generate_schema_example, DEFAULT_OUTPUT_SCHEMA
|
||||
import json
|
||||
|
||||
example = generate_schema_example(DEFAULT_OUTPUT_SCHEMA)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert "output" in parsed
|
||||
assert "reasoning" in parsed
|
||||
assert "<" in parsed["output"] # Placeholder format
|
||||
|
||||
def test_generate_schema_example_with_enum(self):
|
||||
"""Enum fields should use first enum value."""
|
||||
from cmdforge.runner import generate_schema_example
|
||||
import json
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"intent": {"type": "string", "enum": ["question", "task", "chat"]}
|
||||
}
|
||||
}
|
||||
|
||||
example = generate_schema_example(schema)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert parsed["intent"] == "question" # First enum value
|
||||
|
||||
def test_generate_schema_example_with_array(self):
|
||||
"""Array fields should generate array example."""
|
||||
from cmdforge.runner import generate_schema_example
|
||||
import json
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {"type": "array", "items": {"type": "string"}}
|
||||
}
|
||||
}
|
||||
|
||||
example = generate_schema_example(schema)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert isinstance(parsed["items"], list)
|
||||
assert len(parsed["items"]) == 2
|
||||
|
||||
def test_generate_schema_example_nested_object(self):
|
||||
"""Nested objects should be recursively generated."""
|
||||
from cmdforge.runner import generate_schema_example
|
||||
import json
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"scores": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"accuracy": {"type": "number"},
|
||||
"completeness": {"type": "number"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
example = generate_schema_example(schema)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert isinstance(parsed["scores"], dict)
|
||||
assert parsed["scores"]["accuracy"] == 0.0
|
||||
assert parsed["scores"]["completeness"] == 0.0
|
||||
|
||||
def test_prepend_schema_instructions_format(self):
|
||||
"""Instructions should be prepended, not appended."""
|
||||
from cmdforge.runner import prepend_schema_instructions, DEFAULT_OUTPUT_SCHEMA
|
||||
|
||||
prompt = "What is 2+2?"
|
||||
result = prepend_schema_instructions(prompt, DEFAULT_OUTPUT_SCHEMA)
|
||||
|
||||
# Instructions should come before the prompt
|
||||
assert result.startswith("Respond with ONLY valid JSON")
|
||||
assert prompt in result
|
||||
# Prompt should be after the separator
|
||||
assert result.index("---") < result.index(prompt)
|
||||
|
||||
def test_prepend_schema_instructions_includes_guidance(self):
|
||||
"""Should include field guidance for output/reasoning schemas."""
|
||||
from cmdforge.runner import prepend_schema_instructions, DEFAULT_OUTPUT_SCHEMA
|
||||
|
||||
result = prepend_schema_instructions("Test", DEFAULT_OUTPUT_SCHEMA)
|
||||
|
||||
assert "reasoning" in result.lower()
|
||||
assert "output" in result.lower()
|
||||
assert "Required:" in result
|
||||
|
||||
|
||||
class TestExecuteCodeStep:
|
||||
"""Tests for code step execution."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user