rob/CmdForge
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Harden M8 trust and guidance workflows

Browse Source
This commit is contained in:
rob
2026-07-20 15:21:50 -03:00
parent d954c1823a
commit 4d22d532a3
24 changed files with 1281 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_cli.py
+31
View File
@@ -541,6 +541,37 @@ class TestRegistryPublishDryRun:
assert _cmd_registry_publish(self.args(tool_dir)) == 1
assert "YAML mapping" in capsys.readouterr().err
def test_normal_publish_runs_remote_preflight_before_mutation(
self, tool_dir, monkeypatch
):
from cmdforge.cli.registry_commands import _cmd_registry_publish
client = MagicMock()
client.get_me.return_value = {"slug": "testuser"}
client.get_my_tool_status.return_value = {"status": "pending"}
client.publish_tool.side_effect = [
{
"preflight": {"errors": [], "warnings": [], "suggestions": []},
"quality": {"score": 80, "evidence_coverage": 60, "categories": []},
},
{
"owner": "testuser", "name": "dry-run-tool",
"version": "1.0.0", "status": "pending",
},
]
monkeypatch.setattr(
"cmdforge.cli.registry_commands.load_config",
lambda: SimpleNamespace(registry=SimpleNamespace(token="token")),
)
monkeypatch.setattr("cmdforge.registry_client.get_client", lambda: client)
args = SimpleNamespace(
path=str(tool_dir), dry_run=False, force=True, owner=""
)
assert _cmd_registry_publish(args) == 0
assert client.publish_tool.call_count == 2
assert client.publish_tool.call_args_list[0].kwargs["dry_run"] is True
assert "dry_run" not in client.publish_tool.call_args_list[1].kwargs
def test_inspect_with_registry_uses_similarity_results(monkeypatch, capsys):
from cmdforge.cli import cmd_inspect
tests/test_collection_api.py
+116 -4
View File
@@ -56,7 +56,7 @@ def client(app):
def auth_headers(app):
"""Create auth headers with a valid token."""
import hashlib
from datetime import datetime
from datetime import datetime, timezone
from cmdforge.registry.db import connect_db
token = "test-token"
@@ -94,7 +94,7 @@ def auth_headers(app):
INSERT INTO api_tokens (publisher_id, token_hash, name, created_at)
VALUES (?, ?, ?, ?)
""",
[publisher_id, token_hash, "test-token", datetime.utcnow().isoformat()],
[publisher_id, token_hash, "test-token", datetime.now(timezone.utc).isoformat()],
)
# Insert a public approved tool for tests if missing
@@ -112,7 +112,7 @@ def auth_headers(app):
[
"testuser", "tool1", "1.0.0", "Test tool", "Other", "[]",
"name: tool1\nversion: 1.0.0\n", "", publisher_id,
"public", "approved", datetime.utcnow().isoformat(),
"public", "approved", datetime.now(timezone.utc).isoformat(),
],
)
@@ -158,6 +158,44 @@ class TestToolApprovedEndpoint:
assert data['data']['has_approved_public_version'] is False
@flask_required
class TestDeprecationChains:
def test_replacement_chain_is_validated_and_returned(self, client, auth_headers):
target = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": "name: replacement\nversion: 1.0.0\noutput: ok\n"},
)
assert target.status_code == 201
response = client.post(
"/api/v1/tools/testuser/tool1/deprecate",
headers=auth_headers,
json={"deprecated_message": "Moved", "replacement": "replacement"},
)
assert response.status_code == 200
assert response.get_json()["data"]["replacement_chain"] == [
"testuser/replacement"
]
def test_replacement_cycle_is_rejected(self, client, auth_headers):
target = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": "name: cycle-target\nversion: 1.0.0\noutput: ok\n"},
)
assert target.status_code == 201
assert client.post(
"/api/v1/tools/testuser/cycle-target/deprecate",
headers=auth_headers,
json={"replacement": "tool1"},
).status_code == 200
response = client.post(
"/api/v1/tools/testuser/tool1/deprecate",
headers=auth_headers,
json={"replacement": "cycle-target"},
)
assert response.status_code == 400
assert "cycle" in response.get_json()["error"]["message"]
@flask_required
class TestPublishPreflightEndpoint:
def test_dry_run_includes_shared_preflight(self, client, auth_headers):
@@ -180,7 +218,81 @@ class TestPublishPreflightEndpoint:
assert response.status_code == 200
report = response.get_json()["data"]["preflight"]
assert report["errors"] == []
assert report["generated_tests"] == []
assert report["generated_tests"][0]["state"] == "passed"
assert response.get_json()["data"]["quality"]["evidence_coverage"] > 0
def test_real_publish_persists_immutable_audit(self, client, auth_headers):
from cmdforge.registry.db import connect_db
response = client.post(
"/api/v1/tools",
headers=auth_headers,
json={
"config": (
"name: audited-tool\nversion: 1.0.0\noutput: stable\n"
"input_schema: {}\noutput_schema:\n type: string\n"
),
},
)
assert response.status_code == 201
conn = connect_db()
try:
rows = conn.execute(
"""
SELECT tool_audits.trigger, tool_audits.evidence_json
FROM tool_audits JOIN tools ON tools.id = tool_audits.tool_id
WHERE tools.owner = 'testuser' AND tools.name = 'audited-tool'
"""
).fetchall()
conn.execute(
"""
UPDATE tools SET moderation_status = 'approved'
WHERE owner = 'testuser' AND name = 'audited-tool'
"""
)
conn.commit()
finally:
conn.close()
assert len(rows) == 1
assert rows[0]["trigger"] == "publish"
assert "contract_conformance" in rows[0]["evidence_json"]
search = client.get("/api/v1/tools/search?q=audited")
result = search.get_json()["data"][0]
assert isinstance(result["quality_score"], int)
assert isinstance(result["quality_coverage"], int)
assert result["quality_evaluated_at"]
def test_real_publish_blocks_deterministic_contract_failure(
self, client, auth_headers
):
response = client.post(
"/api/v1/tools",
headers=auth_headers,
json={
"config": (
"name: broken-contract\nversion: 1.0.0\noutput: text\n"
"input_schema: {}\noutput_schema:\n type: integer\n"
),
},
)
assert response.status_code == 400
assert response.get_json()["error"]["code"] == "PREFLIGHT_FAILED"
def test_real_publish_blocks_missing_registry_dependency(
self, client, auth_headers
):
response = client.post(
"/api/v1/tools",
headers=auth_headers,
json={
"config": (
"name: missing-dependency\nversion: 1.0.0\n"
"dependencies:\n - absent-tool\n"
),
},
)
assert response.status_code == 400
assert "absent-tool" in response.get_json()["error"]["message"]
def test_invalid_contract_is_rejected(self, client, auth_headers):
response = client.post(
tests/test_picker_m8.py
+91
View File
@@ -0,0 +1,91 @@
"""Behavioral coverage for registry-aware picker and deprecation UX."""
from io import StringIO
import time
from unittest.mock import patch
import cmdforge.cli.picker as picker
from cmdforge.cli.picker import PickerResult
class FakeTTY:
def __init__(self, *characters):
self.characters = iter(characters)
def getch(self, timeout=None):
if timeout is not None:
time.sleep(0.01)
return None
return next(self.characters)
def setup_function():
picker._registry_cache.clear()
picker._ui_out = StringIO()
def test_public_registry_search_does_not_require_token():
class Client:
token = ""
def semantic_search(self, query, limit):
return {"available": False, "data": [], "error": None}
def search_tools(self, query, per_page):
return [{
"owner": "official", "name": "weather",
"description": "Weather", "score": 0.8,
"quality_score": 91, "quality_coverage": 70,
}]
with patch("cmdforge.registry_client.get_client", return_value=Client()):
results = picker.search_registry("weather")
assert results[0]["name"] == "official/weather"
assert results[0]["quality_score"] == 91
assert results[0]["quality_coverage"] == 70
def test_picker_can_install_when_no_local_tools():
registry_tool = {
"name": "official/weather", "desc": "Weather", "args": [],
"registry": True, "quality_score": 91, "quality_coverage": 70,
"relevance": 0.8,
}
with patch.object(picker, "get_tools", return_value=[]), patch.object(
picker, "search_registry", return_value=[registry_tool]
), patch.object(
picker, "_install_registry_selection",
return_value=PickerResult("official/weather", {}),
) as install:
result = picker.run_picker(FakeTTY("w", "e", "\n"))
install.assert_called_once()
assert result.tool_name == "official/weather"
assert "Q91/70%" in picker._ui_out.getvalue()
def test_tab_installs_registry_selection_too():
registry_tool = {
"name": "official/weather", "desc": "Weather", "args": [],
"registry": True,
}
with patch.object(picker, "get_tools", return_value=[]), patch.object(
picker, "search_registry", return_value=[registry_tool]
), patch.object(
picker, "_install_registry_selection",
return_value=PickerResult("official/weather", {}),
) as install:
result = picker.run_picker(FakeTTY("w", "e", "\t"))
install.assert_called_once()
assert result.tool_name == "official/weather"
def test_local_deprecation_selection_prints_migration_guidance():
tool = {
"name": "old", "desc": "", "args": [], "deprecated": True,
"deprecated_message": "Moved.", "replacement": "official/new",
}
result = picker._local_selection(tool)
assert result.tool_name == "old"
output = picker._ui_out.getvalue()
assert "Moved." in output
assert "official/new" in output
tests/test_preflight.py
+65
View File
@@ -126,6 +126,71 @@ class TestAnalyzeTool:
_check_similar_tools(SimpleNamespace(name="summarize"), report, Client())
assert report.similar_tools[0]["name"] == "official/summary"
def test_server_audit_does_not_claim_skipped_local_checks(self):
from cmdforge.tool import Tool, ToolStep
report = analyze_tool(
Tool(name="server", steps=[ToolStep(tool="child", output_var="out")]),
check_local_dependencies=False,
)
checks = report.audit_evidence["checks_run"]
assert "dependencies" not in checks
assert "toolstep_compatibility" not in checks
assert "reuse_opportunities" not in checks
assert report.compatibility == []
def test_unrelated_code_steps_are_not_reuse_evidence(self):
from cmdforge.tool import CodeStep, Tool
current = Tool(
name="current", input_schema={}, output_schema={},
steps=[
CodeStep(code="x = 1", output_var="x"),
CodeStep(code="y = 2", output_var="y"),
],
)
other = Tool(
name="other", input_schema={}, output_schema={},
steps=[
CodeStep(code="a = 99", output_var="a"),
CodeStep(code="b = 100", output_var="b"),
],
)
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
"cmdforge.tool.load_tool", return_value=other
):
report = analyze_tool(current)
assert report.reuse_opportunities == []
def test_exact_contracted_duplicate_has_evidence(self):
from cmdforge.tool import PromptStep, Tool
def make(name):
return Tool(
name=name, input_schema={}, output_schema={"type": "string"},
steps=[
PromptStep(
prompt="first", provider="mock", output_var="one",
output_schema={"type": "string"},
),
PromptStep(
prompt="second", provider="mock", output_var="two",
output_schema={"type": "string"},
),
],
)
with patch("cmdforge.tool.list_tools", return_value=["other"]), patch(
"cmdforge.tool.load_tool", return_value=make("other")
):
report = analyze_tool(make("current"))
duplicate = [
item for item in report.reuse_opportunities
if item["type"] == "duplicate_sequence"
]
assert duplicate
assert "explicit tool contracts" in duplicate[0]["evidence"]
def test_registry_error_becomes_warning(self):
class Client:
def search_tools(self, query, per_page):
tests/test_quality.py
+49
View File
@@ -22,6 +22,10 @@ class TestCategoryScore:
cs = CategoryScore("X", 0, 0)
assert cs.percentage == 0.0
def test_display_distinguishes_missing_evidence(self):
assert CategoryScore("Tests", 0, 30, "not_tested").display == "not tested"
assert CategoryScore("Tests", 0, 30, "not_applicable").display == "n/a"
class TestQualityScore:
def test_to_dict(self):
@@ -55,6 +59,40 @@ class TestComputeQuality:
# Contracts should be not_tested
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
assert contracts.state == "not_tested"
assert qs.headline == 0
assert qs.evidence_coverage == 0
def test_missing_categories_do_not_reduce_checked_score(self):
tool = Tool(name="contracted", input_schema={}, output_schema={})
report = PreflightReport()
qs = compute_quality(tool, report)
assert qs.headline == 100
assert qs.evidence_coverage == 15
contracts = [c for c in qs.categories if c.name == "Contracts"][0]
assert contracts.state == "checked"
def test_unsupported_tests_are_not_scored_as_failures(self):
report = PreflightReport(generated_tests=[{
"step": "preflight", "state": "unsupported", "detail": "safe mode"
}])
tests = [
c for c in compute_quality(Tool(name="tool"), report).categories
if c.name == "Deterministic tests"
][0]
assert tests.state == "not_tested"
assert tests.display == "not tested"
def test_partial_unsupported_tests_reduce_coverage_not_score(self):
report = PreflightReport(generated_tests=[
{"step": "case", "state": "passed", "detail": "ok"},
{"step": "generation", "state": "unsupported", "detail": "unknown"},
])
tests = [
c for c in compute_quality(Tool(name="tool"), report).categories
if c.name == "Deterministic tests"
][0]
assert tests.earned == tests.available == 15
assert tests.possible == 30
def test_with_contracts_scores_higher(self):
tool = Tool(
@@ -127,6 +165,17 @@ class TestComputeQuality:
security = [c for c in qs.categories if c.name == "Security scrutiny"][0]
assert security.earned == 15 # 20 - 5
def test_skipped_dependency_check_is_not_claimed_as_security_evidence(self):
tool = Tool(name="server")
report = PreflightReport(audit_evidence={"checks_run": ["secrets"]})
security = [
c for c in compute_quality(tool, report).categories
if c.name == "Security scrutiny"
][0]
assert security.earned == 10
assert security.available == 10
assert security.possible == 20
def test_community_not_tested_without_data(self):
tool = Tool(name="new")
report = PreflightReport()
tests/test_registry_auditing.py
+76
View File
@@ -0,0 +1,76 @@
"""Tests for immutable registry audit refreshes."""
from datetime import datetime, timedelta, timezone
import pytest
from cmdforge.registry.auditing import audit_stale_tools, audit_tool_version
from cmdforge.registry.db import connect_db, init_db
@pytest.fixture
def audit_db(tmp_path):
conn = connect_db(tmp_path / "registry.db")
init_db(conn)
cursor = conn.execute(
"""
INSERT INTO publishers (email, password_hash, slug, display_name)
VALUES ('[email protected]', 'x', 'auditor', 'Auditor')
"""
)
publisher_id = cursor.lastrowid
tool = conn.execute(
"""
INSERT INTO tools (
owner, name, version, config_yaml, publisher_id,
visibility, moderation_status
) VALUES ('auditor', 'checked', '1.0.0', ?, ?, 'public', 'approved')
""",
[
"name: checked\nversion: 1.0.0\noutput: stable\n"
"input_schema: {}\noutput_schema:\n type: string\n",
publisher_id,
],
)
conn.commit()
yield conn, tool.lastrowid
conn.close()
def test_background_audits_append_instead_of_overwriting(audit_db):
conn, tool_id = audit_db
first = audit_tool_version(conn, tool_id)
second = audit_tool_version(conn, tool_id)
count = conn.execute(
"SELECT COUNT(*) FROM tool_audits WHERE tool_id = ?", [tool_id]
).fetchone()[0]
assert count == 2
assert first["evidence"]["findings_hash"] == second["evidence"]["findings_hash"]
assert "generated_tests" in first["evidence"]["findings"]
assert "dependencies" not in first["evidence"]["checks_run"]
def test_stale_audit_refresh_adds_new_evidence(audit_db):
conn, tool_id = audit_db
audit_tool_version(conn, tool_id)
old = (datetime.now(timezone.utc) - timedelta(days=60)).isoformat()
conn.execute("UPDATE tool_audits SET evaluated_at = ?", [old])
conn.commit()
results = audit_stale_tools(conn, max_age_days=30)
assert [item["tool_id"] for item in results] == [tool_id]
assert conn.execute("SELECT COUNT(*) FROM tool_audits").fetchone()[0] == 2
def test_degraded_background_score_is_logged(audit_db):
conn, tool_id = audit_db
conn.execute("UPDATE tools SET downloads = 500 WHERE id = ?", [tool_id])
conn.commit()
audit_tool_version(conn, tool_id)
conn.execute("UPDATE tools SET downloads = 0 WHERE id = ?", [tool_id])
conn.commit()
result = audit_tool_version(conn, tool_id)
assert result["quality_change"] < 0
log = conn.execute(
"SELECT action FROM audit_log WHERE target_id = ?", [str(tool_id)]
).fetchone()
assert log["action"] == "automated_audit_changed"
tests/test_tool.py
+27
View File
@@ -685,3 +685,30 @@ class TestAgentContext:
)
restored = PromptStep.from_dict(step.to_dict())
assert restored.skills == ["python"]
class TestDeprecationFields:
def test_round_trip_and_warning(self):
from cmdforge.tool import deprecation_warning
tool = Tool(
name="old", deprecated=True, deprecated_message="Moved.",
replacement="official/new",
)
restored = Tool.from_dict(tool.to_dict())
assert restored.deprecated is True
assert restored.replacement == "official/new"
assert "official/new" in deprecation_warning(restored)
@pytest.mark.parametrize(
"values",
[
{"deprecated": "false"},
{"deprecated_message": 123},
{"replacement": "../../escape"},
{"replacement": "owner/name/extra"},
],
)
def test_invalid_deprecation_metadata_is_rejected(self, values):
with pytest.raises(ValueError):
Tool(name="old", **values)