M9.2-M9.5: Add improvement pipeline, community workflow, transitive integrity, and supply chain attestation

tests/test_improvement.py

@@ -0,0 +1,109 @@
+"""Tests for scrutiny-driven improvement pipeline (M9.2)."""
+
+from cmdforge.improvement import (
+    ImprovementSuggestion,
+    ImprovementReport,
+    generate_improvements,
+)
+from cmdforge.tool import Tool, PromptStep, CodeStep
+
+
+class TestImprovementReport:
+    def test_empty_report(self):
+        report = ImprovementReport(tool_name="test")
+        assert not report.has_high_severity
+        assert report.by_category == {}
+
+    def test_has_high_severity(self):
+        report = ImprovementReport(
+            tool_name="test",
+            suggestions=[
+                ImprovementSuggestion("honesty", "high", "Bad", "desc"),
+                ImprovementSuggestion("efficiency", "low", "Minor", "desc"),
+            ],
+        )
+        assert report.has_high_severity
+
+    def test_by_category(self):
+        report = ImprovementReport(
+            tool_name="test",
+            suggestions=[
+                ImprovementSuggestion("honesty", "high", "A", "d"),
+                ImprovementSuggestion("honesty", "low", "B", "d"),
+                ImprovementSuggestion("efficiency", "medium", "C", "d"),
+            ],
+        )
+        assert len(report.by_category["honesty"]) == 2
+        assert len(report.by_category["efficiency"]) == 1
+
+
+class TestGenerateImprovements:
+    def test_missing_description(self):
+        tool = Tool(name="nodesc")
+        report = generate_improvements(tool)
+        honesty = [s for s in report.suggestions if s.category == "honesty"]
+        assert any("Missing description" in s.title for s in honesty)
+
+    def test_short_description(self):
+        tool = Tool(name="short", description="Does stuff")
+        report = generate_improvements(tool)
+        honesty = [s for s in report.suggestions if s.category == "honesty"]
+        assert any("short" in s.title.lower() for s in honesty)
+
+    def test_echo_prompt_detected(self):
+        tool = Tool(
+            name="echo",
+            steps=[PromptStep(prompt="{input}", provider="mock", output_var="out")],
+            output="{out}",
+        )
+        report = generate_improvements(tool)
+        efficiency = [s for s in report.suggestions if s.category == "efficiency"]
+        assert any("echo" in s.title.lower() for s in efficiency)
+
+    def test_multiple_same_provider_prompts(self):
+        tool = Tool(
+            name="multi",
+            steps=[
+                PromptStep(prompt="Step 1: {input}", provider="claude", output_var="a"),
+                PromptStep(prompt="Step 2: {a}", provider="claude", output_var="b"),
+            ],
+            output="{b}",
+        )
+        report = generate_improvements(tool)
+        efficiency = [s for s in report.suggestions if s.category == "efficiency"]
+        assert any("same provider" in s.title.lower() for s in efficiency)
+
+    def test_exec_detected(self):
+        tool = Tool(
+            name="dangerous",
+            steps=[CodeStep(code="exec('print(1)')", output_var="x")],
+            output="{x}",
+        )
+        report = generate_improvements(tool)
+        transparency = [s for s in report.suggestions if s.category == "transparency"]
+        assert any("exec" in s.title.lower() for s in transparency)
+
+    def test_short_prompt_detected(self):
+        tool = Tool(
+            name="vague",
+            steps=[PromptStep(prompt="Do it", provider="mock", output_var="x")],
+            output="{x}",
+        )
+        report = generate_improvements(tool)
+        transparency = [s for s in report.suggestions if s.category == "transparency"]
+        assert any("short" in s.title.lower() for s in transparency)
+
+    def test_clean_tool_no_suggestions(self):
+        tool = Tool(
+            name="clean",
+            description="A well-documented tool that summarizes text input concisely",
+            steps=[PromptStep(
+                prompt="Summarize the following text in 2-3 sentences. Focus on key points.\n\nText: {input}",
+                provider="mock",
+                output_var="summary",
+            )],
+            output="{summary}",
+        )
+        report = generate_improvements(tool)
+        high = [s for s in report.suggestions if s.severity == "high"]
+        assert len(high) == 0