110 lines
4.1 KiB
Python
110 lines
4.1 KiB
Python
"""Tests for scrutiny-driven improvement pipeline (M9.2)."""
|
|
|
|
from cmdforge.improvement import (
|
|
ImprovementSuggestion,
|
|
ImprovementReport,
|
|
generate_improvements,
|
|
)
|
|
from cmdforge.tool import Tool, PromptStep, CodeStep
|
|
|
|
|
|
class TestImprovementReport:
|
|
def test_empty_report(self):
|
|
report = ImprovementReport(tool_name="test")
|
|
assert not report.has_high_severity
|
|
assert report.by_category == {}
|
|
|
|
def test_has_high_severity(self):
|
|
report = ImprovementReport(
|
|
tool_name="test",
|
|
suggestions=[
|
|
ImprovementSuggestion("honesty", "high", "Bad", "desc"),
|
|
ImprovementSuggestion("efficiency", "low", "Minor", "desc"),
|
|
],
|
|
)
|
|
assert report.has_high_severity
|
|
|
|
def test_by_category(self):
|
|
report = ImprovementReport(
|
|
tool_name="test",
|
|
suggestions=[
|
|
ImprovementSuggestion("honesty", "high", "A", "d"),
|
|
ImprovementSuggestion("honesty", "low", "B", "d"),
|
|
ImprovementSuggestion("efficiency", "medium", "C", "d"),
|
|
],
|
|
)
|
|
assert len(report.by_category["honesty"]) == 2
|
|
assert len(report.by_category["efficiency"]) == 1
|
|
|
|
|
|
class TestGenerateImprovements:
|
|
def test_missing_description(self):
|
|
tool = Tool(name="nodesc")
|
|
report = generate_improvements(tool)
|
|
honesty = [s for s in report.suggestions if s.category == "honesty"]
|
|
assert any("Missing description" in s.title for s in honesty)
|
|
|
|
def test_short_description(self):
|
|
tool = Tool(name="short", description="Does stuff")
|
|
report = generate_improvements(tool)
|
|
honesty = [s for s in report.suggestions if s.category == "honesty"]
|
|
assert any("short" in s.title.lower() for s in honesty)
|
|
|
|
def test_echo_prompt_detected(self):
|
|
tool = Tool(
|
|
name="echo",
|
|
steps=[PromptStep(prompt="{input}", provider="mock", output_var="out")],
|
|
output="{out}",
|
|
)
|
|
report = generate_improvements(tool)
|
|
efficiency = [s for s in report.suggestions if s.category == "efficiency"]
|
|
assert any("echo" in s.title.lower() for s in efficiency)
|
|
|
|
def test_multiple_same_provider_prompts(self):
|
|
tool = Tool(
|
|
name="multi",
|
|
steps=[
|
|
PromptStep(prompt="Step 1: {input}", provider="claude", output_var="a"),
|
|
PromptStep(prompt="Step 2: {a}", provider="claude", output_var="b"),
|
|
],
|
|
output="{b}",
|
|
)
|
|
report = generate_improvements(tool)
|
|
efficiency = [s for s in report.suggestions if s.category == "efficiency"]
|
|
assert any("same provider" in s.title.lower() for s in efficiency)
|
|
|
|
def test_exec_detected(self):
|
|
tool = Tool(
|
|
name="dangerous",
|
|
steps=[CodeStep(code="exec('print(1)')", output_var="x")],
|
|
output="{x}",
|
|
)
|
|
report = generate_improvements(tool)
|
|
transparency = [s for s in report.suggestions if s.category == "transparency"]
|
|
assert any("exec" in s.title.lower() for s in transparency)
|
|
|
|
def test_short_prompt_detected(self):
|
|
tool = Tool(
|
|
name="vague",
|
|
steps=[PromptStep(prompt="Do it", provider="mock", output_var="x")],
|
|
output="{x}",
|
|
)
|
|
report = generate_improvements(tool)
|
|
transparency = [s for s in report.suggestions if s.category == "transparency"]
|
|
assert any("short" in s.title.lower() for s in transparency)
|
|
|
|
def test_clean_tool_no_suggestions(self):
|
|
tool = Tool(
|
|
name="clean",
|
|
description="A well-documented tool that summarizes text input concisely",
|
|
steps=[PromptStep(
|
|
prompt="Summarize the following text in 2-3 sentences. Focus on key points.\n\nText: {input}",
|
|
provider="mock",
|
|
output_var="summary",
|
|
)],
|
|
output="{summary}",
|
|
)
|
|
report = generate_improvements(tool)
|
|
high = [s for s in report.suggestions if s.severity == "high"]
|
|
assert len(high) == 0
|