M9.2-M9.5: Add improvement pipeline, community workflow, transitive integrity, and supply chain attestation
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
"""Supply chain attestation (M9.5).
|
||||
|
||||
Tool publishers sign releases. Registry verifies signatures before
|
||||
accepting publish. Clients verify signatures before installing.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class Attestation:
|
||||
"""A signed attestation for a tool release."""
|
||||
|
||||
tool_name: str
|
||||
version: str
|
||||
content_hash: str
|
||||
signer: str # publisher identity
|
||||
signature: str
|
||||
signed_at: str = ""
|
||||
algorithm: str = "hmac-sha256"
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"tool_name": self.tool_name,
|
||||
"version": self.version,
|
||||
"content_hash": self.content_hash,
|
||||
"signer": self.signer,
|
||||
"signature": self.signature,
|
||||
"signed_at": self.signed_at,
|
||||
"algorithm": self.algorithm,
|
||||
}
|
||||
|
||||
|
||||
def sign_tool(
|
||||
tool_name: str,
|
||||
version: str,
|
||||
content_hash: str,
|
||||
signer: str,
|
||||
secret_key: str,
|
||||
) -> Attestation:
|
||||
"""Sign a tool release with HMAC-SHA256.
|
||||
|
||||
Args:
|
||||
tool_name: Tool name
|
||||
version: Tool version
|
||||
content_hash: Content hash from integrity module
|
||||
signer: Publisher identity (username)
|
||||
secret_key: Signing key (from config or keyring)
|
||||
|
||||
Returns:
|
||||
Attestation with signature
|
||||
"""
|
||||
payload = f"{tool_name}:{version}:{content_hash}:{signer}"
|
||||
signature = hmac.new(
|
||||
secret_key.encode(),
|
||||
payload.encode(),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
return Attestation(
|
||||
tool_name=tool_name,
|
||||
version=version,
|
||||
content_hash=content_hash,
|
||||
signer=signer,
|
||||
signature=signature,
|
||||
signed_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def verify_attestation(attestation: Attestation, secret_key: str) -> bool:
|
||||
"""Verify a tool attestation signature.
|
||||
|
||||
Args:
|
||||
attestation: The attestation to verify
|
||||
secret_key: The signing key to verify against
|
||||
|
||||
Returns:
|
||||
True if signature is valid
|
||||
"""
|
||||
payload = f"{attestation.tool_name}:{attestation.version}:{attestation.content_hash}:{attestation.signer}"
|
||||
expected = hmac.new(
|
||||
secret_key.encode(),
|
||||
payload.encode(),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return hmac.compare_digest(expected, attestation.signature)
|
||||
|
||||
|
||||
def verify_content_hash(content_hash: str, tool_dict: dict) -> bool:
|
||||
"""Verify that a content hash matches the tool definition.
|
||||
|
||||
Args:
|
||||
content_hash: The hash to verify
|
||||
tool_dict: Tool dictionary (from to_dict())
|
||||
|
||||
Returns:
|
||||
True if hash matches
|
||||
"""
|
||||
from .integrity import compute_tool_hash
|
||||
# Reconstruct what the hash should be
|
||||
tool_dict_copy = dict(tool_dict)
|
||||
tool_dict_copy.pop("path", None)
|
||||
content = json.dumps(tool_dict_copy, sort_keys=True)
|
||||
expected = hashlib.sha256(content.encode()).hexdigest()[:16]
|
||||
return expected == content_hash
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Community improvement workflow (M9.3).
|
||||
|
||||
Allows community members to submit suggested prompt/code improvements.
|
||||
Submissions are auto-tested before review. Successful improvements
|
||||
credit both original and improving author.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .tool import Tool, PromptStep, CodeStep
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImprovementSubmission:
|
||||
"""A community-submitted improvement for a tool."""
|
||||
|
||||
tool_name: str
|
||||
tool_version: str
|
||||
submitter: str
|
||||
step_index: int
|
||||
step_type: str # "prompt" | "code"
|
||||
original: str
|
||||
proposed: str
|
||||
rationale: str = ""
|
||||
submitted_at: str = ""
|
||||
status: str = "pending" # pending | tested | approved | rejected
|
||||
test_result: Optional[dict] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"tool_name": self.tool_name,
|
||||
"tool_version": self.tool_version,
|
||||
"submitter": self.submitter,
|
||||
"step_index": self.step_index,
|
||||
"step_type": self.step_type,
|
||||
"original": self.original[:200],
|
||||
"proposed": self.proposed[:200],
|
||||
"rationale": self.rationale,
|
||||
"submitted_at": self.submitted_at,
|
||||
"status": self.status,
|
||||
"test_result": self.test_result,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SubmissionReview:
|
||||
"""Review result for an improvement submission."""
|
||||
|
||||
submission: ImprovementSubmission
|
||||
decision: str # "approve" | "reject" | "request_changes"
|
||||
reviewer: str = ""
|
||||
notes: str = ""
|
||||
reviewed_at: str = ""
|
||||
|
||||
|
||||
def create_submission(
|
||||
tool: Tool,
|
||||
step_index: int,
|
||||
proposed: str,
|
||||
submitter: str,
|
||||
rationale: str = "",
|
||||
) -> ImprovementSubmission:
|
||||
"""Create a new improvement submission."""
|
||||
if step_index < 0 or step_index >= len(tool.steps):
|
||||
raise ValueError(f"Step index {step_index} out of range (0-{len(tool.steps)-1})")
|
||||
|
||||
step = tool.steps[step_index]
|
||||
if isinstance(step, PromptStep):
|
||||
original = step.prompt
|
||||
step_type = "prompt"
|
||||
elif isinstance(step, CodeStep):
|
||||
original = step.code
|
||||
step_type = "code"
|
||||
else:
|
||||
raise ValueError(f"Step {step_index} is not a prompt or code step")
|
||||
|
||||
return ImprovementSubmission(
|
||||
tool_name=tool.name,
|
||||
tool_version=tool.version or "",
|
||||
submitter=submitter,
|
||||
step_index=step_index,
|
||||
step_type=step_type,
|
||||
original=original,
|
||||
proposed=proposed,
|
||||
rationale=rationale,
|
||||
submitted_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def run_submission_tests(submission: ImprovementSubmission, tool: Tool) -> dict:
|
||||
"""Run contract tests on the tool with the proposed change applied.
|
||||
|
||||
Returns a dict with test results comparing baseline vs proposed.
|
||||
"""
|
||||
import copy
|
||||
|
||||
from .contract_testing import run_contract_tests
|
||||
|
||||
# Baseline
|
||||
baseline = run_contract_tests(tool)
|
||||
baseline_passed = sum(
|
||||
1 for r in baseline.results if r.state == "passed"
|
||||
)
|
||||
|
||||
# Apply proposed change
|
||||
tool_copy = copy.deepcopy(tool)
|
||||
if submission.step_index >= len(tool_copy.steps):
|
||||
return {"error": "Step index out of range", "baseline_passed": baseline_passed}
|
||||
|
||||
step = tool_copy.steps[submission.step_index]
|
||||
if isinstance(step, PromptStep) and submission.step_type == "prompt":
|
||||
step.prompt = submission.proposed
|
||||
elif isinstance(step, CodeStep) and submission.step_type == "code":
|
||||
step.code = submission.proposed
|
||||
else:
|
||||
return {"error": "Step type mismatch", "baseline_passed": baseline_passed}
|
||||
|
||||
# Test with proposed change
|
||||
proposed_result = run_contract_tests(tool_copy)
|
||||
proposed_passed = sum(
|
||||
1 for r in proposed_result.results if r.state == "passed"
|
||||
)
|
||||
|
||||
result = {
|
||||
"baseline_passed": baseline_passed,
|
||||
"proposed_passed": proposed_passed,
|
||||
"improvement": proposed_passed - baseline_passed,
|
||||
"regressed": proposed_passed < baseline_passed,
|
||||
"details": [r.to_dict() for r in proposed_result.results],
|
||||
}
|
||||
|
||||
submission.status = "tested"
|
||||
submission.test_result = result
|
||||
return result
|
||||
|
||||
|
||||
def review_submission(
|
||||
submission: ImprovementSubmission,
|
||||
decision: str,
|
||||
reviewer: str,
|
||||
notes: str = "",
|
||||
) -> SubmissionReview:
|
||||
"""Review a tested submission."""
|
||||
if decision not in ("approve", "reject", "request_changes"):
|
||||
raise ValueError("Decision must be: approve, reject, or request_changes")
|
||||
|
||||
if submission.status == "pending":
|
||||
raise ValueError("Submission must be tested before review")
|
||||
|
||||
submission.status = decision
|
||||
return SubmissionReview(
|
||||
submission=submission,
|
||||
decision=decision,
|
||||
reviewer=reviewer,
|
||||
notes=notes,
|
||||
reviewed_at=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Scrutiny-driven improvement pipeline (M9.2).
|
||||
|
||||
Takes a tool with low scrutiny scores and generates actionable improvement
|
||||
suggestions for honesty, efficiency, and transparency.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .tool import Tool, PromptStep, CodeStep
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImprovementSuggestion:
|
||||
"""One actionable improvement suggestion."""
|
||||
|
||||
category: str # "honesty" | "efficiency" | "transparency"
|
||||
severity: str # "high" | "medium" | "low"
|
||||
title: str
|
||||
description: str
|
||||
location: str = "" # step index or "tool"
|
||||
current: str = "" # current code/prompt text
|
||||
suggested: str = "" # suggested replacement
|
||||
auto_applicable: bool = False
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"category": self.category,
|
||||
"severity": self.severity,
|
||||
"title": self.title,
|
||||
"description": self.description,
|
||||
"location": self.location,
|
||||
"current": self.current[:200],
|
||||
"suggested": self.suggested[:200],
|
||||
"auto_applicable": self.auto_applicable,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImprovementReport:
|
||||
"""Collection of improvement suggestions for a tool."""
|
||||
|
||||
tool_name: str
|
||||
suggestions: List[ImprovementSuggestion] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def has_high_severity(self) -> bool:
|
||||
return any(s.severity == "high" for s in self.suggestions)
|
||||
|
||||
@property
|
||||
def by_category(self) -> Dict[str, List[ImprovementSuggestion]]:
|
||||
result: Dict[str, List[ImprovementSuggestion]] = {}
|
||||
for s in self.suggestions:
|
||||
result.setdefault(s.category, []).append(s)
|
||||
return result
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"tool": self.tool_name,
|
||||
"suggestion_count": len(self.suggestions),
|
||||
"has_high_severity": self.has_high_severity,
|
||||
"suggestions": [s.to_dict() for s in self.suggestions],
|
||||
}
|
||||
|
||||
|
||||
def generate_improvements(tool: Tool) -> ImprovementReport:
|
||||
"""Analyze a tool and generate improvement suggestions.
|
||||
|
||||
Categories:
|
||||
- Honesty: description doesn't match behavior
|
||||
- Efficiency: unnecessary AI calls, redundant steps
|
||||
- Transparency: obfuscated code, unclear prompts
|
||||
"""
|
||||
report = ImprovementReport(tool_name=tool.name)
|
||||
|
||||
_check_honesty(tool, report)
|
||||
_check_efficiency(tool, report)
|
||||
_check_transparency(tool, report)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def _check_honesty(tool: Tool, report: ImprovementReport):
|
||||
"""Check description-to-behavior alignment."""
|
||||
if not tool.description:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="honesty",
|
||||
severity="high",
|
||||
title="Missing description",
|
||||
description="Tool has no description. Users cannot know what it does without reading the config.",
|
||||
location="tool",
|
||||
auto_applicable=False,
|
||||
))
|
||||
return
|
||||
|
||||
# Check if description mentions capabilities not in steps
|
||||
desc_lower = tool.description.lower()
|
||||
has_prompt = any(isinstance(s, PromptStep) for s in tool.steps)
|
||||
has_code = any(isinstance(s, CodeStep) for s in tool.steps)
|
||||
|
||||
if "ai" in desc_lower or "generate" in desc_lower or "analyze" in desc_lower:
|
||||
if not has_prompt:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="honesty",
|
||||
severity="medium",
|
||||
title="Description implies AI but no prompt step found",
|
||||
description="The description mentions AI/generation/analysis but the tool has no PromptStep.",
|
||||
location="tool",
|
||||
auto_applicable=False,
|
||||
))
|
||||
|
||||
# Check if description is too vague
|
||||
if len(tool.description) < 20:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="honesty",
|
||||
severity="low",
|
||||
title="Description is very short",
|
||||
description="A more detailed description helps users find and trust this tool.",
|
||||
location="tool",
|
||||
auto_applicable=False,
|
||||
))
|
||||
|
||||
|
||||
def _check_efficiency(tool: Tool, report: ImprovementReport):
|
||||
"""Check for unnecessary AI calls and redundant steps."""
|
||||
prompt_steps = [s for s in tool.steps if isinstance(s, PromptStep)]
|
||||
|
||||
# Multiple prompt steps with same provider — could be combined
|
||||
if len(prompt_steps) > 1:
|
||||
providers = [s.provider for s in prompt_steps]
|
||||
if len(set(providers)) == 1:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="efficiency",
|
||||
severity="medium",
|
||||
title="Multiple prompt steps with same provider",
|
||||
description=f"Found {len(prompt_steps)} prompt steps all using '{providers[0]}'. "
|
||||
"Consider combining them into a single step to reduce API calls.",
|
||||
location=f"steps 0-{len(prompt_steps)-1}",
|
||||
auto_applicable=False,
|
||||
))
|
||||
|
||||
# Prompt step that just passes input through
|
||||
for i, step in enumerate(tool.steps):
|
||||
if isinstance(step, PromptStep):
|
||||
if step.prompt.strip() in ("{input}", "Repeat: {input}", "Echo: {input}"):
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="efficiency",
|
||||
severity="high",
|
||||
title="Prompt step just echoes input",
|
||||
description="This prompt step appears to just pass input through without transformation. "
|
||||
"Remove it and use the input directly.",
|
||||
location=f"step {i}",
|
||||
current=step.prompt,
|
||||
suggested="(remove this step — use {input} directly)",
|
||||
auto_applicable=False,
|
||||
))
|
||||
|
||||
# No steps at all — output is just a template
|
||||
if not tool.steps and "{input}" in tool.output:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="efficiency",
|
||||
severity="low",
|
||||
title="Tool has no steps",
|
||||
description="This tool only does variable substitution. Consider if it needs to be a CmdForge tool "
|
||||
"or if a simple shell alias would suffice.",
|
||||
location="tool",
|
||||
auto_applicable=False,
|
||||
))
|
||||
|
||||
|
||||
def _check_transparency(tool: Tool, report: ImprovementReport):
|
||||
"""Check for obfuscated code and unclear prompts."""
|
||||
for i, step in enumerate(tool.steps):
|
||||
if isinstance(step, CodeStep):
|
||||
# Check for overly complex code
|
||||
lines = step.code.strip().split("\n")
|
||||
if len(lines) > 20:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="transparency",
|
||||
severity="medium",
|
||||
title=f"Code step {i} is complex ({len(lines)} lines)",
|
||||
description="Long code steps are harder to audit. Consider extracting into a separate "
|
||||
"tool or adding comments.",
|
||||
location=f"step {i}",
|
||||
auto_applicable=False,
|
||||
))
|
||||
|
||||
# Check for exec/eval
|
||||
if "exec(" in step.code or "eval(" in step.code:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="transparency",
|
||||
severity="high",
|
||||
title=f"Code step {i} uses exec/eval",
|
||||
description="Using exec() or eval() in code steps makes the tool harder to audit and "
|
||||
"poses security risks. Consider restructuring.",
|
||||
location=f"step {i}",
|
||||
auto_applicable=False,
|
||||
))
|
||||
|
||||
if isinstance(step, PromptStep):
|
||||
# Check for very short prompts
|
||||
if len(step.prompt.strip()) < 20:
|
||||
report.suggestions.append(ImprovementSuggestion(
|
||||
category="transparency",
|
||||
severity="medium",
|
||||
title=f"Prompt step {i} is very short",
|
||||
description="Short prompts give the AI model less context, leading to unpredictable output. "
|
||||
"Add more detail about the expected format and content.",
|
||||
location=f"step {i}",
|
||||
current=step.prompt,
|
||||
auto_applicable=False,
|
||||
))
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Transitive integrity verification (M9.4).
|
||||
|
||||
Content-addressable tool identity: tool = hash(tool definition + all dep hashes).
|
||||
Extends lockfile to include transitive integrity chain.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .tool import Tool, load_tool
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntegrityNode:
|
||||
"""One node in the integrity chain."""
|
||||
|
||||
name: str
|
||||
version: str = ""
|
||||
content_hash: str = ""
|
||||
dependencies: List[str] = field(default_factory=list)
|
||||
dependency_hashes: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"name": self.name,
|
||||
"version": self.version,
|
||||
"hash": self.content_hash,
|
||||
"dependencies": self.dependencies,
|
||||
"dependency_hashes": self.dependency_hashes,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class IntegrityChain:
|
||||
"""Full transitive integrity chain for a tool."""
|
||||
|
||||
root: IntegrityNode
|
||||
nodes: Dict[str, IntegrityNode] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def is_valid(self) -> bool:
|
||||
"""Check all dependency hashes match."""
|
||||
for name, node in self.nodes.items():
|
||||
for dep_name in node.dependencies:
|
||||
expected_hash = node.dependency_hashes.get(dep_name)
|
||||
if expected_hash == "unresolved":
|
||||
# Unresolved dependencies don't invalidate the chain
|
||||
continue
|
||||
if dep_name not in self.nodes:
|
||||
return False
|
||||
dep_node = self.nodes[dep_name]
|
||||
if expected_hash and expected_hash != dep_node.content_hash:
|
||||
return False
|
||||
return True
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"root": self.root.to_dict(),
|
||||
"nodes": {k: v.to_dict() for k, v in self.nodes.items()},
|
||||
"valid": self.is_valid,
|
||||
}
|
||||
|
||||
|
||||
def compute_tool_hash(tool: Tool) -> str:
|
||||
"""Compute a content hash for a tool definition."""
|
||||
tool_dict = tool.to_dict()
|
||||
# Remove path (not part of content identity)
|
||||
tool_dict.pop("path", None)
|
||||
content = json.dumps(tool_dict, sort_keys=True)
|
||||
return hashlib.sha256(content.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def build_integrity_chain(tool: Tool, max_depth: int = 10) -> IntegrityChain:
|
||||
"""Build a transitive integrity chain for a tool.
|
||||
|
||||
Traverses all ToolStep dependencies and computes hashes for each.
|
||||
"""
|
||||
chain = IntegrityChain(root=_build_node(tool))
|
||||
chain.nodes[tool.name] = chain.root
|
||||
_traverse_deps(tool, chain, depth=0, max_depth=max_depth)
|
||||
return chain
|
||||
|
||||
|
||||
def _build_node(tool: Tool) -> IntegrityNode:
|
||||
"""Build an integrity node from a tool."""
|
||||
deps = []
|
||||
dep_hashes = {}
|
||||
|
||||
for step in tool.steps:
|
||||
if hasattr(step, "tool") and step.tool:
|
||||
deps.append(step.tool)
|
||||
|
||||
# Resolve dependency hashes
|
||||
for dep_name in deps:
|
||||
dep_tool = load_tool(dep_name)
|
||||
if dep_tool:
|
||||
dep_hashes[dep_name] = compute_tool_hash(dep_tool)
|
||||
else:
|
||||
dep_hashes[dep_name] = "unresolved"
|
||||
|
||||
return IntegrityNode(
|
||||
name=tool.name,
|
||||
version=tool.version or "",
|
||||
content_hash=compute_tool_hash(tool),
|
||||
dependencies=deps,
|
||||
dependency_hashes=dep_hashes,
|
||||
)
|
||||
|
||||
|
||||
def _traverse_deps(tool: Tool, chain: IntegrityChain, depth: int, max_depth: int):
|
||||
"""Recursively traverse dependencies and add to chain."""
|
||||
if depth >= max_depth:
|
||||
return
|
||||
|
||||
for step in tool.steps:
|
||||
if hasattr(step, "tool") and step.tool:
|
||||
dep_name = step.tool
|
||||
if dep_name in chain.nodes:
|
||||
continue
|
||||
dep_tool = load_tool(dep_name)
|
||||
if not dep_tool:
|
||||
continue
|
||||
node = _build_node(dep_tool)
|
||||
chain.nodes[dep_name] = node
|
||||
_traverse_deps(dep_tool, chain, depth + 1, max_depth)
|
||||
|
||||
|
||||
def verify_integrity(chain: IntegrityChain) -> bool:
|
||||
"""Verify that all hashes in the chain are consistent."""
|
||||
return chain.is_valid
|
||||
Reference in New Issue
Block a user