Harden M9: Ed25519 attestation, full SHA-256 integrity, locked-by-default, prompt improvements, registry verification

This commit is contained in:
rob
2026-07-20 16:50:49 -03:00
parent 04153a4bbd
commit 56cb34276e
34 changed files with 2549 additions and 257 deletions
+41 -15
View File
@@ -2,44 +2,70 @@
from cmdforge.attestation import (
Attestation,
generate_keypair,
sign_tool,
verify_attestation,
verify_content_hash,
verify_trusted_attestation,
)
from cmdforge.tool import Tool
class TestSignTool:
def test_creates_attestation(self):
att = sign_tool("mytool", "1.0.0", "abc123", "alice", "secret-key")
private, public = generate_keypair()
att = sign_tool("mytool", "1.0.0", "abc123", "alice", private)
assert att.tool_name == "mytool"
assert att.version == "1.0.0"
assert att.content_hash == "abc123"
assert att.signer == "alice"
assert len(att.signature) == 64 # SHA256 hex
assert att.algorithm == "hmac-sha256"
assert len(att.signature) == 88
assert att.algorithm == "ed25519"
assert verify_attestation(att, public)
def test_different_keys_different_signatures(self):
att1 = sign_tool("tool", "1.0.0", "hash", "alice", "key1")
att2 = sign_tool("tool", "1.0.0", "hash", "alice", "key2")
key1, _ = generate_keypair()
key2, _ = generate_keypair()
att1 = sign_tool("tool", "1.0.0", "hash", "alice", key1)
att2 = sign_tool("tool", "1.0.0", "hash", "alice", key2)
assert att1.signature != att2.signature
class TestVerifyAttestation:
def test_valid_signature(self):
att = sign_tool("mytool", "1.0.0", "abc123", "alice", "secret-key")
assert verify_attestation(att, "secret-key")
private, public = generate_keypair()
att = sign_tool("mytool", "1.0.0", "abc123", "alice", private)
assert verify_attestation(att, public)
def test_wrong_key_fails(self):
att = sign_tool("mytool", "1.0.0", "abc123", "alice", "secret-key")
assert not verify_attestation(att, "wrong-key")
private, _ = generate_keypair()
_, wrong_public = generate_keypair()
att = sign_tool("mytool", "1.0.0", "abc123", "alice", private)
assert not verify_attestation(att, wrong_public)
def test_tampered_content_fails(self):
att = sign_tool("mytool", "1.0.0", "abc123", "alice", "secret-key")
att.content_hash = "tampered"
assert not verify_attestation(att, "secret-key")
from dataclasses import replace
private, public = generate_keypair()
att = sign_tool("mytool", "1.0.0", "abc123", "alice", private)
att = replace(att, content_hash="tampered")
assert not verify_attestation(att, public)
def test_tampered_signer_fails(self):
att = sign_tool("mytool", "1.0.0", "abc123", "alice", "secret-key")
att.signer = "eve"
assert not verify_attestation(att, "secret-key")
from dataclasses import replace
private, public = generate_keypair()
att = sign_tool("mytool", "1.0.0", "abc123", "alice", private)
att = replace(att, signer="eve")
assert not verify_attestation(att, public)
def test_timestamp_and_algorithm_are_signed(self):
from dataclasses import replace
private, public = generate_keypair()
att = sign_tool("tool", "1.0.0", "hash", "alice", private)
assert not verify_attestation(replace(att, signed_at="tomorrow"), public)
assert not verify_attestation(replace(att, algorithm="hmac-sha256"), public)
def test_requires_trusted_publisher_mapping(self):
private, public = generate_keypair()
att = sign_tool("tool", "1.0.0", "hash", "alice", private)
assert verify_trusted_attestation(att, {"alice": public})
assert not verify_trusted_attestation(att, {"mallory": public})
+73
View File
@@ -650,3 +650,76 @@ def test_switch_to_existing_tool_closes_creation_page_first():
call.close_tool_builder(),
call.open_tool_builder("existing"),
]
def test_guided_extraction_builds_new_tool_without_mutating_draft():
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import (
build_extracted_tool,
extraction_diff,
)
schema = {"type": "string"}
repeated = [
PromptStep(
prompt="First {input}", provider="mock", output_var="first",
output_schema=schema,
),
PromptStep(
prompt="Second {first}", provider="mock", output_var="second",
output_schema=schema,
),
]
draft = Tool(
name="draft", steps=repeated + [
PromptStep(
prompt="First {input}", provider="mock", output_var="first_copy",
output_schema=schema,
),
PromptStep(
prompt="Second {first}", provider="mock", output_var="second_copy",
output_schema=schema,
),
], input_schema=schema,
)
before = draft.to_dict()
extracted = build_extracted_tool(
draft,
{"type": "repeated_sequence", "locations": [1, 3], "length": 2},
"draft-shared",
)
assert len(extracted.steps) == 2
assert extracted.output == "{second}"
assert extracted.output_schema == schema
assert draft.to_dict() == before
assert "+++ draft-shared/config.yaml" in extraction_diff(extracted)
def test_guided_extraction_rejects_hidden_external_step_dependency():
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import build_extracted_tool
schema = {"type": "string"}
draft = Tool(name="draft", steps=[
PromptStep(
prompt="Produce", provider="mock", output_var="earlier",
output_schema=schema,
),
PromptStep(
prompt="Use {earlier}", provider="mock", output_var="one",
output_schema=schema,
),
PromptStep(
prompt="Finish {one}", provider="mock", output_var="two",
output_schema=schema,
),
])
with pytest.raises(ValueError, match="outside its boundary"):
build_extracted_tool(
draft,
{"type": "repeated_sequence", "locations": [2, 4], "length": 2},
"draft-shared",
)
+140
View File
@@ -339,6 +339,146 @@ class TestPublishPreflightEndpoint:
assert response.status_code == 200
@flask_required
class TestM9RegistryTrustAndCommunity:
def test_signed_publish_download_and_content_lookup(
self, client, auth_headers
):
from cmdforge.attestation import (
Attestation, generate_keypair, sign_tool, verify_attestation,
)
from cmdforge.registry.db import connect_db
private_key, public_key = generate_keypair()
response = client.put(
"/api/v1/me/signing-key", headers=auth_headers,
json={"public_key": public_key},
)
assert response.status_code == 200
config = "name: signed-tool\nversion: 1.0.0\noutput: stable\n"
preflight = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": config, "dry_run": True},
)
assert preflight.status_code == 200
content_hash = preflight.get_json()["data"]["content_hash"]
attestation = sign_tool(
"signed-tool", "1.0.0", content_hash, "testuser", private_key
)
published = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": config, "attestation": attestation.to_dict()},
)
assert published.status_code == 201
assert published.get_json()["data"]["content_hash"] == content_hash
conn = connect_db()
try:
conn.execute(
"UPDATE tools SET moderation_status = 'approved' "
"WHERE owner = 'testuser' AND name = 'signed-tool'"
)
conn.commit()
finally:
conn.close()
downloaded = client.get(
"/api/v1/tools/testuser/signed-tool/download?install=false"
)
assert downloaded.status_code == 200
data = downloaded.get_json()["data"]
assert data["content_hash"] == content_hash
assert verify_attestation(Attestation.from_dict(data["attestation"]), public_key)
lookup = client.get(f"/api/v1/tools/by-content-hash/{content_hash}")
assert lookup.status_code == 200
assert lookup.get_json()["data"]["name"] == "signed-tool"
def test_registered_key_requires_valid_signature(self, client, auth_headers):
from cmdforge.attestation import generate_keypair
_, public_key = generate_keypair()
assert client.put(
"/api/v1/me/signing-key", headers=auth_headers,
json={"public_key": public_key},
).status_code == 200
response = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": "name: unsigned-tool\nversion: 1.0.0\n"},
)
assert response.status_code == 400
assert response.get_json()["error"]["code"] == "ATTESTATION_REQUIRED"
def test_improvement_is_tested_reviewed_and_credited(
self, client, auth_headers
):
from cmdforge.registry.db import connect_db
config = (
"name: improvable\nversion: 1.0.0\noutput: '{result}'\n"
"input_schema:\n type: string\noutput_schema:\n type: string\n"
"steps:\n - type: prompt\n prompt: 'Summarize: {input}'\n"
" provider: mock\n output_var: result\n"
)
published = client.post(
"/api/v1/tools", headers=auth_headers, json={"config": config}
)
assert published.status_code == 201
conn = connect_db()
try:
conn.execute(
"UPDATE tools SET moderation_status = 'approved' "
"WHERE owner = 'testuser' AND name = 'improvable'"
)
conn.commit()
finally:
conn.close()
submitted = client.post(
"/api/v1/tools/testuser/improvable/1.0.0/improvements",
headers=auth_headers,
json={
"step_index": 0,
"proposed": "Summarize the input accurately and concisely: {input}",
"rationale": "Clearer expected behavior",
},
)
assert submitted.status_code == 201
submission_id = submitted.get_json()["data"]["id"]
reviewed = client.patch(
f"/api/v1/improvements/{submission_id}", headers=auth_headers,
json={"decision": "approve", "notes": "Validated"},
)
assert reviewed.status_code == 200
assert reviewed.get_json()["data"]["ready_to_apply"] is True
improved_config = config.replace("version: 1.0.0", "version: 1.0.1").replace(
"Summarize: {input}",
"Summarize the input accurately and concisely: {input}",
)
applied = client.post(
"/api/v1/tools", headers=auth_headers,
json={"config": improved_config, "improvement_id": submission_id},
)
assert applied.status_code == 201
conn = connect_db()
try:
conn.execute(
"UPDATE tools SET moderation_status = 'approved' "
"WHERE owner = 'testuser' AND name = 'improvable' "
"AND version = '1.0.1'"
)
conn.commit()
credited = conn.execute(
"SELECT COUNT(*) AS count FROM tool_contributors"
).fetchone()["count"]
finally:
conn.close()
assert credited == 1
detail = client.get("/api/v1/tools/testuser/improvable?version=1.0.1")
assert detail.status_code == 200
assert detail.get_json()["data"]["badges"] == [
"optimized", "community-reviewed"
]
@flask_required
class TestPostCollectionsEndpoint:
"""Tests for POST /api/v1/collections endpoint."""
+2 -2
View File
@@ -49,11 +49,11 @@ class TestReviewSubmission:
sub = ImprovementSubmission(
tool_name="test", tool_version="1.0.0", submitter="alice",
step_index=0, step_type="prompt", original="a", proposed="b",
status="tested",
status="tested", test_result={"passed_for_review": True},
)
review = review_submission(sub, "approve", "admin", "Good improvement")
assert review.decision == "approve"
assert sub.status == "approve"
assert sub.status == "approved"
def test_review_pending_rejected(self):
sub = ImprovementSubmission(
+31 -7
View File
@@ -44,12 +44,14 @@ class TestIntegrityChain:
)
chain = build_integrity_chain(tool)
assert "missing-dep" not in chain.nodes
# Chain is still valid because the dependency hash is "unresolved"
# and there's no node to compare against
assert chain.is_valid
assert not chain.is_valid
assert chain.errors == ["Unresolved dependency: missing-dep"]
def test_chain_valid_with_resolved_dep(self, tmp_path):
with patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"):
with (
patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"),
patch("cmdforge.tool.BIN_DIR", tmp_path / "bin"),
):
from cmdforge.tool import save_tool
child = Tool(name="child", version="1.0.0")
@@ -67,13 +69,35 @@ class TestIntegrityChain:
assert "child" in chain.nodes
assert chain.is_valid
def test_child_change_changes_root_identity(self, tmp_path):
with (
patch("cmdforge.tool.TOOLS_DIR", tmp_path / ".cmdforge"),
patch("cmdforge.tool.BIN_DIR", tmp_path / "bin"),
):
from cmdforge.tool import save_tool
child = Tool(name="child", version="1.0.0", output="one")
save_tool(child)
parent = Tool(
name="parent", version="1.0.0",
steps=[ToolStep(tool="child", output_var="x")], output="{x}",
)
first = build_integrity_chain(parent).root.content_hash
child.output = "two"
save_tool(child)
second = build_integrity_chain(parent).root.content_hash
assert first != second
class TestVerifyIntegrity:
def test_valid_chain(self):
chain = IntegrityChain(
root=IntegrityNode(name="root", content_hash="abc"),
nodes={"root": IntegrityNode(name="root", content_hash="abc")},
from cmdforge.integrity import compute_content_identity
definition_hash = "sha256:" + "a" * 64
content_hash = compute_content_identity(definition_hash, {})
node = IntegrityNode(
name="root", definition_hash=definition_hash,
content_hash=content_hash,
)
chain = IntegrityChain(root=node, nodes={"root": node})
assert verify_integrity(chain)
def test_tampered_chain(self):
+13 -1
View File
@@ -259,6 +259,12 @@ class TestGenerateLockfile:
assert "official/text-utils" in lock.packages
assert lock.packages["official/summarize"].direct is True
assert lock.packages["official/text-utils"].direct is False
child_identity = lock.packages["official/text-utils"].content_hash
assert child_identity.startswith("sha256:")
assert lock.packages["official/summarize"].dependency_hashes == {
"official/text-utils": child_identity
}
assert lock.packages["official/summarize"].content_hash.startswith("sha256:")
class TestVerifyLockfile:
@@ -319,7 +325,9 @@ class TestLockfileRoundTrip:
integrity="sha256:abc123",
source="registry",
direct=True,
required_by=[]
required_by=[],
content_hash="sha256:root",
dependency_hashes={"official/text-utils": "sha256:child"},
),
"official/text-utils": LockedPackage(
name="official/text-utils",
@@ -364,6 +372,10 @@ class TestLockfileRoundTrip:
assert summarize.integrity == "sha256:abc123"
assert summarize.source == "registry"
assert summarize.direct is True
assert summarize.content_hash == "sha256:root"
assert summarize.dependency_hashes == {
"official/text-utils": "sha256:child"
}
text_utils = loaded.packages["official/text-utils"]
assert text_utils.direct is False
+90 -2
View File
@@ -12,6 +12,7 @@ from cmdforge.mcp_client import (
McpServerConfig,
McpClientManager,
_build_server_env,
_build_http_headers,
_normalize_result,
_sanitize,
_fingerprint,
@@ -34,14 +35,15 @@ class TestMcpServerConfig:
@pytest.mark.parametrize(
"change, message",
[
({"transport": "streamable-http"}, "Unsupported MCP transport"),
({"transport": "streamable-http"}, "requires a url"),
({"args": "-y"}, "list of strings"),
({"timeout": 0}, "greater than 0"),
({"timeout": True}, "must be a number"),
],
)
def test_validation_rejects_invalid_config(self, change, message):
cfg = McpServerConfig(name="test", command="server", **change)
command = None if change.get("transport") == "streamable-http" else "server"
cfg = McpServerConfig(name="test", command=command, **change)
with pytest.raises(ValueError, match=message):
cfg.validate()
@@ -55,6 +57,31 @@ class TestMcpServerConfig:
b = McpServerConfig(name="b", command="cmd", args=["-v"])
assert _fingerprint(a) == _fingerprint(b)
def test_streamable_http_accepts_https(self):
cfg = McpServerConfig(
name="remote", transport="streamable-http",
url="https://example.com/mcp", approved=True,
)
cfg.validate()
def test_streamable_http_allows_loopback_http_only(self):
McpServerConfig(
name="local", transport="streamable-http",
url="http://127.0.0.1:8000/mcp",
).validate()
with pytest.raises(ValueError, match="must use HTTPS"):
McpServerConfig(
name="remote", transport="streamable-http",
url="http://example.com/mcp",
).validate()
def test_streamable_http_rejects_url_credentials(self):
with pytest.raises(ValueError, match="must not contain credentials"):
McpServerConfig(
name="remote", transport="streamable-http",
url="https://user:[email protected]/mcp",
).validate()
class TestSanitize:
def test_redacts_bearer(self):
@@ -85,6 +112,15 @@ class TestEnvironmentIsolation:
)
assert _build_server_env(cfg)["SERVER_TOKEN"] == "allowed"
def test_http_header_reference_is_resolved(self, monkeypatch):
monkeypatch.setenv("MCP_TOKEN", "secret")
cfg = McpServerConfig(
name="remote", transport="streamable-http",
url="https://example.com/mcp",
headers={"Authorization": "Bearer ${MCP_TOKEN}"},
)
assert _build_http_headers(cfg)["Authorization"] == "Bearer secret"
def test_missing_environment_reference_fails(self, monkeypatch):
monkeypatch.delenv("CMDFORGE_MISSING", raising=False)
cfg = McpServerConfig(
@@ -122,6 +158,19 @@ class TestMcpConfigPersistence:
def test_load_no_file(self, temp_mcp_file):
assert load_mcp_config() == []
def test_streamable_http_round_trip(self, temp_mcp_file):
save_mcp_config([McpServerConfig(
name="remote", transport="streamable-http",
url="https://example.com/mcp",
headers={"Authorization": "Bearer ${MCP_TOKEN}"},
approved=True,
)])
loaded = load_mcp_config()[0]
assert loaded.transport == "streamable-http"
assert loaded.command is None
assert loaded.url == "https://example.com/mcp"
assert loaded.headers == {"Authorization": "Bearer ${MCP_TOKEN}"}
def test_save_creates_0600_permissions(self, temp_mcp_file):
save_mcp_config([McpServerConfig(name="test", command="echo")])
perms = oct(temp_mcp_file.stat().st_mode & 0o777)
@@ -651,6 +700,45 @@ class TestRegisteredMcpTool:
serve()
class TestStreamableHttpServerSafety:
def test_defaults_are_loopback_with_origin_allowlist(self):
from cmdforge.mcp_server import _validate_http_server_options
origins, token = _validate_http_server_options(
"127.0.0.1", 8000, [], None, None
)
assert token is None
assert origins == [
"http://localhost:8000", "http://127.0.0.1:8000"
]
def test_nonlocal_requires_https_and_auth(self):
from cmdforge.mcp_server import _validate_http_server_options
with pytest.raises(ValueError, match="external-url"):
_validate_http_server_options("0.0.0.0", 8000, [], "secret", None)
with pytest.raises(ValueError, match="auth-token"):
_validate_http_server_options(
"0.0.0.0", 8000, [], None, "https://mcp.example.com"
)
def test_nonlocal_infers_https_origin_and_expands_token(self, monkeypatch):
from cmdforge.mcp_server import _validate_http_server_options
monkeypatch.setenv("MCP_AUTH", "secret-value")
origins, token = _validate_http_server_options(
"0.0.0.0", 8000, [], "${MCP_AUTH}",
"https://mcp.example.com",
)
assert origins == ["https://mcp.example.com"]
assert token == "secret-value"
def test_nonlocal_rejects_insecure_origin(self):
from cmdforge.mcp_server import _validate_http_server_options
with pytest.raises(ValueError, match="origins must use HTTPS"):
_validate_http_server_options(
"0.0.0.0", 8000, ["http://example.com"], "secret",
"https://mcp.example.com",
)
class TestCmdForgeMcpServerEndToEnd:
def test_stdio_discovery_and_invocation(self, tmp_path, monkeypatch):
pytest.importorskip("mcp")
+38
View File
@@ -23,6 +23,21 @@ class TestTruncate:
class TestGenerateVariations:
def test_mock_provider_never_calls_external_provider(self):
with patch("cmdforge.providers.call_provider") as call:
_get_strategies("Summarize the text", 2, provider="mock")
call.assert_not_called()
def test_selected_provider_is_honored(self):
from cmdforge.providers import ProviderResult
with patch(
"cmdforge.providers.call_provider",
return_value=ProviderResult(text="A materially clearer instruction", success=True),
) as call:
result = _get_strategies("Summarize the text", 1, provider="chosen")
assert result[0][1] == "A materially clearer instruction"
assert call.call_args.args[0] == "chosen"
def test_generates_for_prompt_steps(self, tmp_path):
from cmdforge.tool import Tool, PromptStep
@@ -75,6 +90,29 @@ class TestGetStrategies:
class TestOptimizeTool:
def test_structural_results_do_not_claim_a_best_prompt(self):
from cmdforge.tool import Tool, PromptStep
tool = Tool(
name="structural", steps=[PromptStep("Summarize", "mock", "out")],
output="{out}", input_schema={"type": "string"},
output_schema={"type": "string"},
)
result = optimize_tool(tool, count=2)
assert result.best is None
assert "cannot compare prompt semantics" in result.note
def test_behavioral_evaluator_selects_only_an_improvement(self):
from cmdforge.tool import Tool, PromptStep
tool = Tool(
name="behavioral", steps=[PromptStep("Do it", "mock", "out")],
output="{out}",
)
result = optimize_tool(
tool, count=3,
evaluator=lambda candidate: len(candidate.steps[0].prompt),
)
assert result.best is not None
assert result.scores[result.best_index] > result.baseline_score
def test_baseline_score_recorded(self, tmp_path):
from cmdforge.tool import Tool, PromptStep
+11
View File
@@ -0,0 +1,11 @@
import stat
from cmdforge.signing import initialize_signing_key, load_signing_key
def test_signing_key_is_persistent_and_private(tmp_path):
path = tmp_path / "release-key.json"
created = initialize_signing_key(path)
assert load_signing_key(path) == created
assert stat.S_IMODE(path.stat().st_mode) == 0o600
assert initialize_signing_key(path) == created
+67
View File
@@ -0,0 +1,67 @@
import json
import stat
from cmdforge.usage import (
build_composite_tool,
clear_usage,
get_suggestions,
is_enabled,
record_invocation,
set_enabled,
)
class Stream:
def __init__(self, inode=None):
self.inode = inode
def configure_paths(tmp_path, monkeypatch):
monkeypatch.setattr("cmdforge.usage.CONFIG_DIR", tmp_path)
monkeypatch.setattr("cmdforge.usage.USAGE_FILE", tmp_path / "usage.json")
monkeypatch.setattr("cmdforge.usage.USAGE_LOCK_FILE", tmp_path / ".usage.lock")
def test_tracking_is_disabled_by_default(tmp_path, monkeypatch):
configure_paths(tmp_path, monkeypatch)
assert not is_enabled()
monkeypatch.setattr("cmdforge.usage._pipe_inode", lambda stream: stream.inode)
record_invocation("one", Stream(), Stream())
assert not (tmp_path / "usage.json").exists()
def test_detects_only_pipe_link_and_stores_no_content(tmp_path, monkeypatch):
configure_paths(tmp_path, monkeypatch)
set_enabled(True)
monkeypatch.setattr("cmdforge.usage._pipe_inode", lambda stream: stream.inode)
for inode in range(44, 47):
record_invocation("tool-a", Stream(), Stream(inode))
record_invocation("tool-b", Stream(inode), Stream())
assert get_suggestions() == [{
"tools": ["tool-a", "tool-b"],
"count": 3,
"last_seen": get_suggestions()[0]["last_seen"],
}]
data = json.loads((tmp_path / "usage.json").read_text())
serialized = json.dumps(data)
for forbidden in ("input_text", "output", "arguments", "cwd", "environment"):
assert forbidden not in serialized
assert stat.S_IMODE((tmp_path / "usage.json").stat().st_mode) == 0o600
def test_disable_and_clear_preserve_consent_state(tmp_path, monkeypatch):
configure_paths(tmp_path, monkeypatch)
set_enabled(True)
clear_usage()
assert is_enabled()
set_enabled(False)
assert not is_enabled()
def test_build_composite_tool_preserves_pipeline_order():
tool = build_composite_tool("combined", ["tool-a", "tool-b"])
assert tool.dependencies == ["tool-a", "tool-b"]
assert tool.steps[0].input_template == "{input}"
assert tool.steps[1].input_template == "{pipeline_1}"
assert tool.output == "{pipeline_2}"