137 lines
4.5 KiB
Python
137 lines
4.5 KiB
Python
"""Tests for preflight analysis engine."""
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from cmdforge.preflight import (
|
|
PreflightReport,
|
|
analyze_tool,
|
|
_check_config_integrity,
|
|
_check_similar_tools,
|
|
_is_semver,
|
|
)
|
|
from cmdforge.registry_client import PaginatedResponse, RegistryError
|
|
|
|
|
|
class TestPreflightReport:
|
|
def test_empty_report_is_ok(self):
|
|
report = PreflightReport()
|
|
assert report.ok
|
|
|
|
def test_errors_make_not_ok(self):
|
|
report = PreflightReport(errors=["bad"])
|
|
assert not report.ok
|
|
|
|
def test_merge(self):
|
|
a = PreflightReport(errors=["e1"], warnings=["w1"])
|
|
b = PreflightReport(errors=["e2"], suggestions=["s1"])
|
|
c = a.merge(b)
|
|
assert c.errors == ["e1", "e2"]
|
|
assert c.warnings == ["w1"]
|
|
assert c.suggestions == ["s1"]
|
|
|
|
def test_to_dict_returns_independent_lists(self):
|
|
report = PreflightReport(errors=["bad"])
|
|
serialized = report.to_dict()
|
|
serialized["errors"].append("another")
|
|
assert report.errors == ["bad"]
|
|
|
|
|
|
class TestAnalyzeTool:
|
|
def test_no_steps_warns(self, tmp_path):
|
|
from cmdforge.tool import Tool, save_tool
|
|
|
|
with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
|
|
tool = Tool(name="empty-tool")
|
|
save_tool(tool)
|
|
|
|
report = analyze_tool(tool)
|
|
assert report.ok
|
|
assert any("no steps" in w.lower() for w in report.warnings)
|
|
|
|
def test_missing_name(self, tmp_path):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = PreflightReport()
|
|
tool = Tool(name="")
|
|
_check_config_integrity(tool, report)
|
|
assert not report.ok
|
|
|
|
def test_invalid_version(self, tmp_path):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = PreflightReport()
|
|
tool = Tool(name="test", version="not-semver")
|
|
_check_config_integrity(tool, report)
|
|
assert not report.ok
|
|
|
|
def test_ok_tool(self, tmp_path):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = PreflightReport()
|
|
tool = Tool(name="ok-tool", version="1.0.0")
|
|
_check_config_integrity(tool, report)
|
|
assert report.ok
|
|
|
|
def test_constant_tool_without_steps_is_not_passthrough_warning(self):
|
|
from cmdforge.tool import Tool
|
|
|
|
report = analyze_tool(Tool(name="constant", output="hello"))
|
|
assert not any("passthrough" in warning for warning in report.warnings)
|
|
|
|
def test_secret_pattern_in_prompt(self, tmp_path):
|
|
from cmdforge.tool import Tool, PromptStep
|
|
|
|
tool = Tool(
|
|
name="leaky",
|
|
steps=[PromptStep(
|
|
prompt="Use api_key: abc123 to auth",
|
|
provider="mock",
|
|
output_var="out",
|
|
)],
|
|
)
|
|
report = PreflightReport()
|
|
from cmdforge.preflight import _check_secrets
|
|
_check_secrets(tool, report)
|
|
assert any("secret" in w.lower() for w in report.warnings)
|
|
|
|
@pytest.mark.parametrize("version", ["1.2.3garbage", "1.2.3.4", "1.2"])
|
|
def test_semver_rejects_trailing_or_incomplete_values(self, version):
|
|
assert _is_semver(version) is False
|
|
|
|
def test_registry_similarity_uses_paginated_data(self):
|
|
class Client:
|
|
def search_tools(self, query, per_page):
|
|
assert (query, per_page) == ("summarize", 5)
|
|
return PaginatedResponse(data=[{
|
|
"owner": "official",
|
|
"name": "summary",
|
|
"description": "Summarize text",
|
|
"downloads": 10,
|
|
}])
|
|
|
|
report = PreflightReport()
|
|
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
|
|
assert report.similar_tools[0]["name"] == "official/summary"
|
|
|
|
def test_registry_error_becomes_warning(self):
|
|
class Client:
|
|
def search_tools(self, query, per_page):
|
|
raise RegistryError("CONNECTION_ERROR", "offline")
|
|
|
|
report = PreflightReport()
|
|
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
|
|
assert report.warnings == ["Could not search registry for similar tools"]
|
|
|
|
def test_programming_error_is_not_swallowed(self):
|
|
class Client:
|
|
def search_tools(self, query, per_page):
|
|
raise TypeError("broken adapter")
|
|
|
|
with pytest.raises(TypeError, match="broken adapter"):
|
|
_check_similar_tools(
|
|
SimpleNamespace(name="summarize"), PreflightReport(), Client()
|
|
)
|