"""Tests for scrutiny-driven improvement pipeline (M9.2).""" from cmdforge.improvement import ( ImprovementSuggestion, ImprovementReport, generate_improvements, ) from cmdforge.tool import Tool, PromptStep, CodeStep class TestImprovementReport: def test_empty_report(self): report = ImprovementReport(tool_name="test") assert not report.has_high_severity assert report.by_category == {} def test_has_high_severity(self): report = ImprovementReport( tool_name="test", suggestions=[ ImprovementSuggestion("honesty", "high", "Bad", "desc"), ImprovementSuggestion("efficiency", "low", "Minor", "desc"), ], ) assert report.has_high_severity def test_by_category(self): report = ImprovementReport( tool_name="test", suggestions=[ ImprovementSuggestion("honesty", "high", "A", "d"), ImprovementSuggestion("honesty", "low", "B", "d"), ImprovementSuggestion("efficiency", "medium", "C", "d"), ], ) assert len(report.by_category["honesty"]) == 2 assert len(report.by_category["efficiency"]) == 1 class TestGenerateImprovements: def test_missing_description(self): tool = Tool(name="nodesc") report = generate_improvements(tool) honesty = [s for s in report.suggestions if s.category == "honesty"] assert any("Missing description" in s.title for s in honesty) def test_short_description(self): tool = Tool(name="short", description="Does stuff") report = generate_improvements(tool) honesty = [s for s in report.suggestions if s.category == "honesty"] assert any("short" in s.title.lower() for s in honesty) def test_echo_prompt_detected(self): tool = Tool( name="echo", steps=[PromptStep(prompt="{input}", provider="mock", output_var="out")], output="{out}", ) report = generate_improvements(tool) efficiency = [s for s in report.suggestions if s.category == "efficiency"] assert any("echo" in s.title.lower() for s in efficiency) def test_multiple_same_provider_prompts(self): tool = Tool( name="multi", steps=[ PromptStep(prompt="Step 1: {input}", provider="claude", output_var="a"), PromptStep(prompt="Step 2: {a}", provider="claude", output_var="b"), ], output="{b}", ) report = generate_improvements(tool) efficiency = [s for s in report.suggestions if s.category == "efficiency"] assert any("same provider" in s.title.lower() for s in efficiency) def test_exec_detected(self): tool = Tool( name="dangerous", steps=[CodeStep(code="exec('print(1)')", output_var="x")], output="{x}", ) report = generate_improvements(tool) transparency = [s for s in report.suggestions if s.category == "transparency"] assert any("exec" in s.title.lower() for s in transparency) def test_short_prompt_detected(self): tool = Tool( name="vague", steps=[PromptStep(prompt="Do it", provider="mock", output_var="x")], output="{x}", ) report = generate_improvements(tool) transparency = [s for s in report.suggestions if s.category == "transparency"] assert any("short" in s.title.lower() for s in transparency) def test_clean_tool_no_suggestions(self): tool = Tool( name="clean", description="A well-documented tool that summarizes text input concisely", steps=[PromptStep( prompt="Summarize the following text in 2-3 sentences. Focus on key points.\n\nText: {input}", provider="mock", output_var="summary", )], output="{summary}", ) report = generate_improvements(tool) high = [s for s in report.suggestions if s.severity == "high"] assert len(high) == 0