Add provider fallback, visual schema builder, and UI improvements
Provider fallback system: - Add optional fallback field to Provider dataclass - Automatically try fallback provider when primary fails (timeout, error, offline) - Prevent infinite loops with circular fallback detection - Add fallback column and dropdown in providers UI Visual schema builder for prompt steps: - Add SchemaBuilderDialog with table-based field editor - Support field name, type, description, and required flag - Live JSON schema preview - Integrate into PromptStepDialog with Edit Schema button - Fix output_schema being lost when editing prompt steps Other improvements: - Add search box to tools page - Improve schema example generation for structured output - Update CLAUDE.md documentation Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
+107
-11
@@ -268,11 +268,10 @@ class TestStructuredOutput:
|
||||
output, success = execute_prompt_step(step, {"input": ""})
|
||||
|
||||
assert success is True
|
||||
# Output should be normalized JSON
|
||||
import json
|
||||
parsed = json.loads(output)
|
||||
assert parsed["output"] == "Hello, world!"
|
||||
assert parsed["reasoning"] == "Greeting requested"
|
||||
# Output should be a parsed dict (for code step compatibility)
|
||||
assert isinstance(output, dict)
|
||||
assert output["output"] == "Hello, world!"
|
||||
assert output["reasoning"] == "Greeting requested"
|
||||
|
||||
@patch('cmdforge.runner.call_provider')
|
||||
def test_structured_output_strips_markdown_fences(self, mock_call):
|
||||
@@ -291,9 +290,8 @@ class TestStructuredOutput:
|
||||
output, success = execute_prompt_step(step, {"input": ""})
|
||||
|
||||
assert success is True
|
||||
import json
|
||||
parsed = json.loads(output)
|
||||
assert parsed["output"] == "Hello"
|
||||
assert isinstance(output, dict)
|
||||
assert output["output"] == "Hello"
|
||||
|
||||
@patch('cmdforge.runner.call_provider')
|
||||
def test_structured_output_retry_on_invalid_json(self, mock_call):
|
||||
@@ -366,9 +364,8 @@ class TestStructuredOutput:
|
||||
output, success = execute_prompt_step(step, {"input": ""})
|
||||
|
||||
assert success is True
|
||||
import json
|
||||
parsed = json.loads(output)
|
||||
assert parsed["score"] == 0.8
|
||||
assert isinstance(output, dict)
|
||||
assert output["score"] == 0.8
|
||||
|
||||
@patch('cmdforge.runner.call_provider')
|
||||
def test_structured_output_schema_violation_triggers_retry(self, mock_call):
|
||||
@@ -414,6 +411,105 @@ class TestStructuredOutput:
|
||||
assert mock_call.call_count == 1
|
||||
|
||||
|
||||
class TestSchemaInstructions:
|
||||
"""Tests for schema instruction generation."""
|
||||
|
||||
def test_generate_schema_example_default(self):
|
||||
"""Default schema should generate output/reasoning example."""
|
||||
from cmdforge.runner import generate_schema_example, DEFAULT_OUTPUT_SCHEMA
|
||||
import json
|
||||
|
||||
example = generate_schema_example(DEFAULT_OUTPUT_SCHEMA)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert "output" in parsed
|
||||
assert "reasoning" in parsed
|
||||
assert "<" in parsed["output"] # Placeholder format
|
||||
|
||||
def test_generate_schema_example_with_enum(self):
|
||||
"""Enum fields should use first enum value."""
|
||||
from cmdforge.runner import generate_schema_example
|
||||
import json
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"intent": {"type": "string", "enum": ["question", "task", "chat"]}
|
||||
}
|
||||
}
|
||||
|
||||
example = generate_schema_example(schema)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert parsed["intent"] == "question" # First enum value
|
||||
|
||||
def test_generate_schema_example_with_array(self):
|
||||
"""Array fields should generate array example."""
|
||||
from cmdforge.runner import generate_schema_example
|
||||
import json
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"items": {"type": "array", "items": {"type": "string"}}
|
||||
}
|
||||
}
|
||||
|
||||
example = generate_schema_example(schema)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert isinstance(parsed["items"], list)
|
||||
assert len(parsed["items"]) == 2
|
||||
|
||||
def test_generate_schema_example_nested_object(self):
|
||||
"""Nested objects should be recursively generated."""
|
||||
from cmdforge.runner import generate_schema_example
|
||||
import json
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"scores": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"accuracy": {"type": "number"},
|
||||
"completeness": {"type": "number"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
example = generate_schema_example(schema)
|
||||
parsed = json.loads(example)
|
||||
|
||||
assert isinstance(parsed["scores"], dict)
|
||||
assert parsed["scores"]["accuracy"] == 0.0
|
||||
assert parsed["scores"]["completeness"] == 0.0
|
||||
|
||||
def test_prepend_schema_instructions_format(self):
|
||||
"""Instructions should be prepended, not appended."""
|
||||
from cmdforge.runner import prepend_schema_instructions, DEFAULT_OUTPUT_SCHEMA
|
||||
|
||||
prompt = "What is 2+2?"
|
||||
result = prepend_schema_instructions(prompt, DEFAULT_OUTPUT_SCHEMA)
|
||||
|
||||
# Instructions should come before the prompt
|
||||
assert result.startswith("Respond with ONLY valid JSON")
|
||||
assert prompt in result
|
||||
# Prompt should be after the separator
|
||||
assert result.index("---") < result.index(prompt)
|
||||
|
||||
def test_prepend_schema_instructions_includes_guidance(self):
|
||||
"""Should include field guidance for output/reasoning schemas."""
|
||||
from cmdforge.runner import prepend_schema_instructions, DEFAULT_OUTPUT_SCHEMA
|
||||
|
||||
result = prepend_schema_instructions("Test", DEFAULT_OUTPUT_SCHEMA)
|
||||
|
||||
assert "reasoning" in result.lower()
|
||||
assert "output" in result.lower()
|
||||
assert "Required:" in result
|
||||
|
||||
|
||||
class TestExecuteCodeStep:
|
||||
"""Tests for code step execution."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user