195 lines
6.2 KiB
Python
195 lines
6.2 KiB
Python
"""Community improvement workflow (M9.3).
|
|
|
|
Allows community members to submit suggested prompt/code improvements.
|
|
Submissions are auto-tested before review. Successful improvements
|
|
credit both original and improving author.
|
|
"""
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from .tool import Tool, PromptStep, CodeStep
|
|
|
|
|
|
@dataclass
|
|
class ImprovementSubmission:
|
|
"""A community-submitted improvement for a tool."""
|
|
|
|
tool_name: str
|
|
tool_version: str
|
|
submitter: str
|
|
step_index: int
|
|
step_type: str # "prompt" | "code"
|
|
original: str
|
|
proposed: str
|
|
rationale: str = ""
|
|
submitted_at: str = ""
|
|
status: str = "pending" # pending | tested | approved | rejected
|
|
test_result: Optional[dict] = None
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"tool_name": self.tool_name,
|
|
"tool_version": self.tool_version,
|
|
"submitter": self.submitter,
|
|
"step_index": self.step_index,
|
|
"step_type": self.step_type,
|
|
"original": self.original[:200],
|
|
"proposed": self.proposed[:200],
|
|
"rationale": self.rationale,
|
|
"submitted_at": self.submitted_at,
|
|
"status": self.status,
|
|
"test_result": self.test_result,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class SubmissionReview:
|
|
"""Review result for an improvement submission."""
|
|
|
|
submission: ImprovementSubmission
|
|
decision: str # "approve" | "reject" | "request_changes"
|
|
reviewer: str = ""
|
|
notes: str = ""
|
|
reviewed_at: str = ""
|
|
|
|
|
|
def create_submission(
|
|
tool: Tool,
|
|
step_index: int,
|
|
proposed: str,
|
|
submitter: str,
|
|
rationale: str = "",
|
|
) -> ImprovementSubmission:
|
|
"""Create a new improvement submission."""
|
|
if step_index < 0 or step_index >= len(tool.steps):
|
|
raise ValueError(f"Step index {step_index} out of range (0-{len(tool.steps)-1})")
|
|
|
|
step = tool.steps[step_index]
|
|
if isinstance(step, PromptStep):
|
|
original = step.prompt
|
|
step_type = "prompt"
|
|
elif isinstance(step, CodeStep):
|
|
original = step.code
|
|
step_type = "code"
|
|
else:
|
|
raise ValueError(f"Step {step_index} is not a prompt or code step")
|
|
|
|
return ImprovementSubmission(
|
|
tool_name=tool.name,
|
|
tool_version=tool.version or "",
|
|
submitter=submitter,
|
|
step_index=step_index,
|
|
step_type=step_type,
|
|
original=original,
|
|
proposed=proposed,
|
|
rationale=rationale,
|
|
submitted_at=datetime.now(timezone.utc).isoformat(),
|
|
)
|
|
|
|
|
|
def run_submission_tests(submission: ImprovementSubmission, tool: Tool) -> dict:
|
|
"""Run contract tests on the tool with the proposed change applied.
|
|
|
|
Returns a dict with test results comparing baseline vs proposed.
|
|
"""
|
|
import copy
|
|
|
|
from .contract_testing import run_contract_tests
|
|
|
|
if submission.tool_name != tool.name or submission.tool_version != (tool.version or ""):
|
|
raise ValueError("Submission targets a different tool or version")
|
|
|
|
# Baseline
|
|
baseline = run_contract_tests(tool)
|
|
baseline_passed = sum(
|
|
1 for r in baseline.results if r.state == "passed"
|
|
)
|
|
|
|
# Apply proposed change
|
|
tool_copy = copy.deepcopy(tool)
|
|
if submission.step_index >= len(tool_copy.steps):
|
|
return {"error": "Step index out of range", "baseline_passed": baseline_passed}
|
|
|
|
step = tool_copy.steps[submission.step_index]
|
|
current = step.prompt if isinstance(step, PromptStep) else (
|
|
step.code if isinstance(step, CodeStep) else None
|
|
)
|
|
if current != submission.original:
|
|
return {
|
|
"error": "Original step has changed since submission",
|
|
"baseline_passed": baseline_passed,
|
|
}
|
|
if isinstance(step, PromptStep) and submission.step_type == "prompt":
|
|
step.prompt = submission.proposed
|
|
elif isinstance(step, CodeStep) and submission.step_type == "code":
|
|
try:
|
|
compile(submission.proposed, f"<{tool.name}:step-{submission.step_index}>", "exec")
|
|
except SyntaxError as exc:
|
|
result = {
|
|
"error": f"Proposed code has invalid syntax: {exc.msg}",
|
|
"baseline_passed": baseline_passed,
|
|
"passed_for_review": False,
|
|
}
|
|
submission.test_result = result
|
|
return result
|
|
step.code = submission.proposed
|
|
else:
|
|
return {"error": "Step type mismatch", "baseline_passed": baseline_passed}
|
|
|
|
# Test with proposed change
|
|
proposed_result = run_contract_tests(tool_copy)
|
|
proposed_passed = sum(
|
|
1 for r in proposed_result.results if r.state == "passed"
|
|
)
|
|
|
|
result = {
|
|
"baseline_passed": baseline_passed,
|
|
"proposed_passed": proposed_passed,
|
|
"improvement": proposed_passed - baseline_passed,
|
|
"regressed": proposed_passed < baseline_passed,
|
|
"baseline_outcome": baseline.outcome,
|
|
"proposed_outcome": proposed_result.outcome,
|
|
"passed_for_review": (
|
|
(
|
|
proposed_result.outcome == "passed"
|
|
and proposed_passed >= baseline_passed
|
|
)
|
|
or (
|
|
submission.step_type == "code"
|
|
and proposed_result.outcome == "unsupported"
|
|
)
|
|
),
|
|
"details": [r.to_dict() for r in proposed_result.results],
|
|
}
|
|
|
|
submission.status = "tested"
|
|
submission.test_result = result
|
|
return result
|
|
|
|
|
|
def review_submission(
|
|
submission: ImprovementSubmission,
|
|
decision: str,
|
|
reviewer: str,
|
|
notes: str = "",
|
|
) -> SubmissionReview:
|
|
"""Review a tested submission."""
|
|
if decision not in ("approve", "reject", "request_changes"):
|
|
raise ValueError("Decision must be: approve, reject, or request_changes")
|
|
|
|
if submission.status != "tested" or not (
|
|
submission.test_result or {}
|
|
).get("passed_for_review"):
|
|
raise ValueError("Submission must be tested before review")
|
|
|
|
submission.status = "approved" if decision == "approve" else decision
|
|
return SubmissionReview(
|
|
submission=submission,
|
|
decision=decision,
|
|
reviewer=reviewer,
|
|
notes=notes,
|
|
reviewed_at=datetime.now(timezone.utc).isoformat(),
|
|
)
|