M8.0/M8.V1: Add contract inference engine and deterministic conformance testing
This commit is contained in:
@@ -538,7 +538,11 @@ def cmd_inspect(args):
|
||||
if getattr(args, "registry", False):
|
||||
from ..registry_client import get_client
|
||||
registry_client = get_client()
|
||||
report = analyze_tool(tool, registry_client=registry_client)
|
||||
report = analyze_tool(
|
||||
tool,
|
||||
registry_client=registry_client,
|
||||
include_contract_tests=True,
|
||||
)
|
||||
|
||||
if report.errors:
|
||||
print(f"Errors ({len(report.errors)}):")
|
||||
@@ -564,6 +568,31 @@ def cmd_inspect(args):
|
||||
print(f" - {t['name']}: {t.get('description', '')[:80]}")
|
||||
print()
|
||||
|
||||
proposal = report.contract_proposal or {}
|
||||
proposed_schemas = {
|
||||
key: proposal.get(key)
|
||||
for key in ("input_schema", "output_schema")
|
||||
if proposal.get(key) is not None
|
||||
}
|
||||
if proposed_schemas:
|
||||
import yaml
|
||||
|
||||
print(f"Contract proposal ({proposal.get('confidence', 'low')} confidence):")
|
||||
print(yaml.safe_dump(proposed_schemas, sort_keys=False).rstrip())
|
||||
for evidence in proposal.get("evidence", []):
|
||||
print(f" EVIDENCE: {evidence}")
|
||||
print(" Contracts are suggestions only and were not saved.")
|
||||
print()
|
||||
|
||||
if report.generated_tests:
|
||||
print(f"Contract conformance ({len(report.generated_tests)} case(s)):")
|
||||
for result in report.generated_tests:
|
||||
print(
|
||||
f" {result['state'].upper()}: {result['step']} — "
|
||||
f"{result.get('detail', '')}"
|
||||
)
|
||||
print()
|
||||
|
||||
if not report.errors and not report.warnings and not report.suggestions:
|
||||
print("No issues found.")
|
||||
|
||||
|
||||
@@ -365,6 +365,28 @@ def _print_preflight_sections(report: dict, prefix: str = "") -> None:
|
||||
for value in values:
|
||||
print(f" {marker}: {value}")
|
||||
|
||||
proposal = report.get("contract_proposal") or {}
|
||||
proposed = {
|
||||
key: proposal.get(key)
|
||||
for key in ("input_schema", "output_schema")
|
||||
if proposal.get(key) is not None
|
||||
}
|
||||
if proposed:
|
||||
print(
|
||||
f"{prefix}Contract proposal "
|
||||
f"({proposal.get('confidence', 'low')} confidence; not saved):"
|
||||
)
|
||||
print(yaml.safe_dump(proposed, sort_keys=False).rstrip())
|
||||
|
||||
generated = report.get("generated_tests") or []
|
||||
if generated:
|
||||
print(f"{prefix}Contract conformance ({len(generated)} case(s)):")
|
||||
for result in generated:
|
||||
print(
|
||||
f" {str(result.get('state', 'not_run')).upper()}: "
|
||||
f"{result.get('step', 'unknown')} — {result.get('detail', '')}"
|
||||
)
|
||||
|
||||
|
||||
def _print_registry_suggestions(suggestions: dict) -> None:
|
||||
"""Render registry-specific category, similarity, and scrutiny evidence."""
|
||||
@@ -472,7 +494,9 @@ def _cmd_registry_publish(args):
|
||||
from ..preflight import analyze_tool
|
||||
from ..tool import Tool
|
||||
try:
|
||||
local_report = analyze_tool(Tool.from_dict(data))
|
||||
local_report = analyze_tool(
|
||||
Tool.from_dict(data), include_contract_tests=True
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
print(f"Local preflight error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
@@ -0,0 +1,477 @@
|
||||
"""Deterministic, side-effect-safe contract conformance verification.
|
||||
|
||||
The verifier generates inputs that are validated against ``input_schema``,
|
||||
replaces prompt steps with schema-derived deterministic values, executes the
|
||||
remaining safe pipeline, and validates its final result. It never calls an AI
|
||||
provider. Steps that could execute user code or contact another tool/server are
|
||||
reported as unsupported instead of being run implicitly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Any, Dict, List, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from .tool import CodeStep, McpStep, PromptStep, Tool, ToolStep
|
||||
|
||||
|
||||
ConformanceState = Literal["passed", "failed", "unsupported", "not_run"]
|
||||
_VALID_STATES = ("passed", "failed", "unsupported", "not_run")
|
||||
|
||||
|
||||
class UnsupportedContract(ValueError):
|
||||
"""Raised when a valid deterministic value cannot be produced safely."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConformanceResult:
|
||||
step: str
|
||||
state: ConformanceState
|
||||
detail: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.state not in _VALID_STATES:
|
||||
raise ValueError(f"Unknown conformance state: {self.state}")
|
||||
|
||||
def is_pass(self) -> bool:
|
||||
return self.state == "passed"
|
||||
|
||||
def to_dict(self) -> Dict[str, str]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConformanceReport:
|
||||
results: List[ConformanceResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def outcome(self) -> ConformanceState:
|
||||
states = {result.state for result in self.results}
|
||||
if "failed" in states:
|
||||
return "failed"
|
||||
if "passed" in states:
|
||||
return "passed"
|
||||
if "unsupported" in states:
|
||||
return "unsupported"
|
||||
return "not_run"
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return self.outcome == "passed"
|
||||
|
||||
@property
|
||||
def failed(self) -> bool:
|
||||
return self.outcome == "failed"
|
||||
|
||||
@property
|
||||
def summary(self) -> Dict[str, int]:
|
||||
counts = {state: 0 for state in _VALID_STATES}
|
||||
for result in self.results:
|
||||
counts[result.state] += 1
|
||||
return counts
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"outcome": self.outcome,
|
||||
"summary": self.summary,
|
||||
"results": [result.to_dict() for result in self.results],
|
||||
}
|
||||
|
||||
|
||||
def run_contract_tests(tool: Tool) -> ConformanceReport:
|
||||
"""Run safe structural conformance tests for a tool.
|
||||
|
||||
This is deliberately not a semantic quality test. Original CodeStep,
|
||||
ToolStep, and McpStep instances are not executed because they may have side
|
||||
effects, make network calls, or invoke real providers.
|
||||
"""
|
||||
if tool.input_schema is None or tool.output_schema is None:
|
||||
return _single_result(
|
||||
"preflight",
|
||||
"not_run",
|
||||
"Both input_schema and output_schema are required",
|
||||
)
|
||||
|
||||
unsafe = next(
|
||||
(
|
||||
type(step).__name__
|
||||
for step in tool.steps
|
||||
if isinstance(step, (CodeStep, ToolStep, McpStep))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if unsafe:
|
||||
return _single_result(
|
||||
"preflight",
|
||||
"unsupported",
|
||||
f"Safe conformance mode does not execute {unsafe}",
|
||||
)
|
||||
|
||||
try:
|
||||
test_cases = _generate_inputs(tool.input_schema)
|
||||
except UnsupportedContract as exc:
|
||||
return _single_result("input-generation", "unsupported", str(exc))
|
||||
|
||||
if not test_cases:
|
||||
return _single_result(
|
||||
"input-generation", "not_run", "No deterministic inputs generated"
|
||||
)
|
||||
|
||||
report = ConformanceReport()
|
||||
for index, test_input in enumerate(test_cases, start=1):
|
||||
label = f"case-{index}"
|
||||
try:
|
||||
result = _run_with_stub(tool, test_input)
|
||||
except UnsupportedContract as exc:
|
||||
report.results.append(
|
||||
ConformanceResult(label, "unsupported", str(exc))
|
||||
)
|
||||
except Exception as exc:
|
||||
report.results.append(
|
||||
ConformanceResult(label, "failed", f"Execution error: {exc}")
|
||||
)
|
||||
else:
|
||||
try:
|
||||
_validate_instance(result, tool.output_schema, "output")
|
||||
except UnsupportedContract as exc:
|
||||
report.results.append(
|
||||
ConformanceResult(label, "unsupported", str(exc))
|
||||
)
|
||||
except ValueError as exc:
|
||||
report.results.append(ConformanceResult(label, "failed", str(exc)))
|
||||
else:
|
||||
report.results.append(
|
||||
ConformanceResult(
|
||||
label, "passed", "Output conforms to output_schema"
|
||||
)
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def _single_result(
|
||||
step: str, state: ConformanceState, detail: str
|
||||
) -> ConformanceReport:
|
||||
return ConformanceReport([ConformanceResult(step, state, detail)])
|
||||
|
||||
|
||||
def _generate_inputs(schema: dict) -> List[Any]:
|
||||
"""Generate deterministic values and validate every returned candidate."""
|
||||
candidates: List[Any] = []
|
||||
schema_type = _schema_type(schema)
|
||||
|
||||
if schema_type == "object":
|
||||
enum_properties = {
|
||||
name: spec["enum"]
|
||||
for name, spec in schema.get("properties", {}).items()
|
||||
if isinstance(spec, dict) and spec.get("enum")
|
||||
}
|
||||
variant_count = min(max((len(v) for v in enum_properties.values()), default=1), 3)
|
||||
for index in range(variant_count):
|
||||
overrides = {
|
||||
name: values[index % len(values)]
|
||||
for name, values in enum_properties.items()
|
||||
}
|
||||
candidates.append(_object_value(schema, overrides))
|
||||
else:
|
||||
candidates.append(_value_for_schema(schema, "input"))
|
||||
|
||||
unique: List[Any] = []
|
||||
seen = set()
|
||||
for candidate in candidates:
|
||||
try:
|
||||
_validate_instance(candidate, schema, "generated input")
|
||||
except ValueError as exc:
|
||||
raise UnsupportedContract(
|
||||
f"Could not generate a valid input: {exc}"
|
||||
) from exc
|
||||
marker = json.dumps(candidate, sort_keys=True, default=str)
|
||||
if marker not in seen:
|
||||
seen.add(marker)
|
||||
unique.append(candidate)
|
||||
return unique
|
||||
|
||||
|
||||
def _schema_type(schema: dict) -> str | None:
|
||||
schema_type = schema.get("type")
|
||||
if isinstance(schema_type, list):
|
||||
return next((value for value in schema_type if value != "null"), "null")
|
||||
if schema_type is None and ("properties" in schema or "required" in schema):
|
||||
return "object"
|
||||
return schema_type
|
||||
|
||||
|
||||
def _value_for_schema(schema: dict, name: str) -> Any:
|
||||
for keyword in ("const", "default"):
|
||||
if keyword in schema:
|
||||
return copy.deepcopy(schema[keyword])
|
||||
if schema.get("examples"):
|
||||
return copy.deepcopy(schema["examples"][0])
|
||||
if schema.get("enum"):
|
||||
return copy.deepcopy(schema["enum"][0])
|
||||
|
||||
alternatives = schema.get("oneOf") or schema.get("anyOf")
|
||||
if alternatives:
|
||||
for alternative in alternatives:
|
||||
try:
|
||||
candidate = _value_for_schema(alternative, name)
|
||||
_validate_instance(candidate, schema, name)
|
||||
return candidate
|
||||
except (UnsupportedContract, ValueError):
|
||||
continue
|
||||
raise UnsupportedContract(f"Cannot synthesize {name} from alternatives")
|
||||
|
||||
if schema.get("allOf"):
|
||||
merged = _merge_all_of(schema)
|
||||
candidate = _value_for_schema(merged, name)
|
||||
_validate_instance(candidate, schema, name)
|
||||
return candidate
|
||||
|
||||
schema_type = _schema_type(schema)
|
||||
if schema_type == "object":
|
||||
return _object_value(schema, {})
|
||||
if schema_type == "array":
|
||||
item_schema = schema.get("items", {})
|
||||
count = max(0, schema.get("minItems", 0))
|
||||
if count == 0 and schema.get("contains"):
|
||||
return [_value_for_schema(schema["contains"], f"{name}-item")]
|
||||
return [
|
||||
_vary_value(_value_for_schema(item_schema, f"{name}-item"), index)
|
||||
for index in range(count)
|
||||
]
|
||||
if schema_type == "string":
|
||||
return _string_value(schema, name)
|
||||
if schema_type in ("integer", "number"):
|
||||
return _number_value(schema, integer=schema_type == "integer")
|
||||
if schema_type == "boolean":
|
||||
return True
|
||||
if schema_type == "null":
|
||||
return None
|
||||
if schema_type is None and not schema:
|
||||
return {}
|
||||
if schema_type is None:
|
||||
candidate = {}
|
||||
try:
|
||||
_validate_instance(candidate, schema, name)
|
||||
return candidate
|
||||
except ValueError as exc:
|
||||
raise UnsupportedContract(str(exc)) from exc
|
||||
raise UnsupportedContract(f"Unsupported JSON Schema type for {name}: {schema_type}")
|
||||
|
||||
|
||||
def _object_value(schema: dict, overrides: Dict[str, Any]) -> Dict[str, Any]:
|
||||
properties = schema.get("properties", {})
|
||||
required = list(schema.get("required", []))
|
||||
result: Dict[str, Any] = {}
|
||||
|
||||
names = list(required)
|
||||
for name, spec in properties.items():
|
||||
if name in overrides or any(
|
||||
keyword in spec for keyword in ("const", "default", "examples", "enum")
|
||||
):
|
||||
if name not in names:
|
||||
names.append(name)
|
||||
|
||||
minimum = schema.get("minProperties", 0)
|
||||
for name in properties:
|
||||
if len(names) >= minimum:
|
||||
break
|
||||
if name not in names:
|
||||
names.append(name)
|
||||
|
||||
for name in names:
|
||||
if name in overrides:
|
||||
result[name] = copy.deepcopy(overrides[name])
|
||||
else:
|
||||
result[name] = _value_for_schema(properties.get(name, {}), name)
|
||||
|
||||
for trigger, dependents in schema.get("dependentRequired", {}).items():
|
||||
if trigger in result:
|
||||
for dependent in dependents:
|
||||
if dependent not in result:
|
||||
result[dependent] = _value_for_schema(
|
||||
properties.get(dependent, {}), dependent
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _string_value(schema: dict, name: str) -> str:
|
||||
formats = {
|
||||
"date": date(2020, 1, 2).isoformat(),
|
||||
"date-time": datetime(2020, 1, 2, 3, 4, 5, tzinfo=timezone.utc).isoformat(),
|
||||
"email": "[email protected]",
|
||||
"hostname": "example.com",
|
||||
"ipv4": "192.0.2.1",
|
||||
"uri": "https://example.com/",
|
||||
"uuid": str(UUID(int=0)),
|
||||
}
|
||||
minimum = max(0, schema.get("minLength", 0))
|
||||
maximum = schema.get("maxLength")
|
||||
seeds = [formats.get(schema.get("format")), f"test-{name}", "a", "0", ""]
|
||||
pattern = schema.get("pattern")
|
||||
for seed in seeds:
|
||||
if seed is None:
|
||||
continue
|
||||
candidate = seed
|
||||
if len(candidate) < minimum:
|
||||
candidate += "a" * (minimum - len(candidate))
|
||||
if maximum is not None:
|
||||
candidate = candidate[:maximum]
|
||||
if pattern and re.search(pattern, candidate) is None:
|
||||
continue
|
||||
return candidate
|
||||
raise UnsupportedContract(f"Cannot synthesize string '{name}' for pattern/bounds")
|
||||
|
||||
|
||||
def _number_value(schema: dict, *, integer: bool) -> int | float:
|
||||
lower = schema.get("minimum")
|
||||
if "exclusiveMinimum" in schema:
|
||||
exclusive_lower = schema["exclusiveMinimum"] + (1 if integer else 0.5)
|
||||
lower = exclusive_lower if lower is None else max(lower, exclusive_lower)
|
||||
upper = schema.get("maximum")
|
||||
if "exclusiveMaximum" in schema:
|
||||
exclusive_upper = schema["exclusiveMaximum"] - (1 if integer else 0.5)
|
||||
upper = exclusive_upper if upper is None else min(upper, exclusive_upper)
|
||||
value = 0
|
||||
if lower is not None:
|
||||
value = max(value, lower)
|
||||
if upper is not None:
|
||||
value = min(value, upper)
|
||||
multiple = schema.get("multipleOf")
|
||||
if multiple:
|
||||
if lower is not None:
|
||||
value = math.ceil(value / multiple) * multiple
|
||||
else:
|
||||
value = math.floor(value / multiple) * multiple
|
||||
if (lower is not None and value < lower) or (upper is not None and value > upper):
|
||||
raise UnsupportedContract("Numeric bounds do not admit a deterministic value")
|
||||
return int(value) if integer else float(value)
|
||||
|
||||
|
||||
def _vary_value(value: Any, index: int) -> Any:
|
||||
if index == 0:
|
||||
return value
|
||||
if isinstance(value, bool) or value is None:
|
||||
return value
|
||||
if isinstance(value, int):
|
||||
return value + index
|
||||
if isinstance(value, float):
|
||||
return value + float(index)
|
||||
if isinstance(value, str):
|
||||
return f"{value}{index}"
|
||||
return value
|
||||
|
||||
|
||||
def _merge_all_of(schema: dict) -> dict:
|
||||
merged = {key: copy.deepcopy(value) for key, value in schema.items() if key != "allOf"}
|
||||
properties = copy.deepcopy(merged.get("properties", {}))
|
||||
required = list(merged.get("required", []))
|
||||
for part in schema["allOf"]:
|
||||
properties.update(copy.deepcopy(part.get("properties", {})))
|
||||
for name in part.get("required", []):
|
||||
if name not in required:
|
||||
required.append(name)
|
||||
for key, value in part.items():
|
||||
if key not in ("properties", "required"):
|
||||
merged.setdefault(key, copy.deepcopy(value))
|
||||
if properties:
|
||||
merged["properties"] = properties
|
||||
if required:
|
||||
merged["required"] = required
|
||||
return merged
|
||||
|
||||
|
||||
def _validate_instance(instance: Any, schema: dict, label: str) -> None:
|
||||
try:
|
||||
from jsonschema import FormatChecker
|
||||
from jsonschema.exceptions import ValidationError
|
||||
from jsonschema.validators import validator_for
|
||||
except ImportError as exc:
|
||||
raise UnsupportedContract("jsonschema is required for conformance testing") from exc
|
||||
|
||||
validator = validator_for(schema)(schema, format_checker=FormatChecker())
|
||||
try:
|
||||
validator.validate(instance)
|
||||
except ValidationError as exc:
|
||||
path = ".".join(str(part) for part in exc.absolute_path) or "root"
|
||||
raise ValueError(f"{label} does not conform at {path}: {exc.message}") from exc
|
||||
|
||||
|
||||
def _run_with_stub(tool: Tool, inputs: Any) -> Any:
|
||||
"""Execute only a prompt/no-step pipeline with generated prompt outputs."""
|
||||
from .runner import DEFAULT_OUTPUT_SCHEMA, run_tool
|
||||
|
||||
tool_copy = copy.deepcopy(tool)
|
||||
transformed = []
|
||||
final_variable = _single_output_variable(tool.output)
|
||||
for step in tool_copy.steps:
|
||||
if not isinstance(step, PromptStep):
|
||||
raise UnsupportedContract(
|
||||
f"Safe conformance mode does not execute {type(step).__name__}"
|
||||
)
|
||||
schema = step.output_schema
|
||||
if schema is None and step.output_var == final_variable:
|
||||
schema = tool.output_schema
|
||||
if schema is None:
|
||||
schema = DEFAULT_OUTPUT_SCHEMA
|
||||
value = _value_for_schema(schema, step.output_var)
|
||||
try:
|
||||
_validate_instance(value, schema, f"stub output '{step.output_var}'")
|
||||
except ValueError as exc:
|
||||
raise UnsupportedContract(
|
||||
f"Could not synthesize '{step.output_var}': {exc}"
|
||||
) from exc
|
||||
transformed.append(
|
||||
CodeStep(
|
||||
code=f"globals()[{step.output_var!r}] = {value!r}",
|
||||
output_var=step.output_var,
|
||||
name=f"Contract stub for {step.name or step.output_var}",
|
||||
)
|
||||
)
|
||||
tool_copy.steps = transformed
|
||||
|
||||
if isinstance(inputs, dict):
|
||||
custom_args = {key: value for key, value in inputs.items() if key != "input"}
|
||||
input_text = inputs.get("input", "")
|
||||
else:
|
||||
custom_args = {}
|
||||
input_text = inputs
|
||||
|
||||
output, exit_code = run_tool(
|
||||
tool_copy,
|
||||
input_text="" if input_text is None else str(input_text),
|
||||
custom_args=custom_args,
|
||||
provider_override=None,
|
||||
dry_run=False,
|
||||
verbose=False,
|
||||
)
|
||||
if exit_code != 0:
|
||||
raise RuntimeError(f"Tool execution failed with exit code {exit_code}")
|
||||
try:
|
||||
return json.loads(output)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return output
|
||||
|
||||
|
||||
def _single_output_variable(output: str) -> str | None:
|
||||
match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", output.strip())
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def _fallback_value(prop: dict, name: str, required: list | None = None) -> Any:
|
||||
"""Backward-compatible helper used by callers and focused tests."""
|
||||
return _value_for_schema(prop, name)
|
||||
|
||||
|
||||
def _validate_output(output: Any, schema: dict) -> bool:
|
||||
"""Backward-compatible boolean output validator."""
|
||||
try:
|
||||
_validate_instance(output, schema, "output")
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
@@ -1135,7 +1135,7 @@ class ToolBuilderPage(QWidget):
|
||||
)
|
||||
|
||||
from ...preflight import analyze_tool
|
||||
preflight = analyze_tool(tool)
|
||||
preflight = analyze_tool(tool, include_contract_tests=True)
|
||||
if preflight.errors:
|
||||
QMessageBox.warning(
|
||||
self,
|
||||
@@ -1143,6 +1143,35 @@ class ToolBuilderPage(QWidget):
|
||||
"\n".join(preflight.errors),
|
||||
)
|
||||
return
|
||||
|
||||
proposal = preflight.contract_proposal or {}
|
||||
proposed_schemas = {
|
||||
key: proposal.get(key)
|
||||
for key in ("input_schema", "output_schema")
|
||||
if proposal.get(key) is not None
|
||||
}
|
||||
if proposed_schemas:
|
||||
msg = QMessageBox(self)
|
||||
msg.setIcon(QMessageBox.Information)
|
||||
msg.setWindowTitle("Contract Proposal")
|
||||
msg.setText(
|
||||
"CmdForge inferred a possible tool contract. "
|
||||
"It will not be applied automatically."
|
||||
)
|
||||
msg.setInformativeText(
|
||||
"Review the proposal below. Contract editing and approval "
|
||||
"will be added by the guided-creation milestone."
|
||||
)
|
||||
msg.setDetailedText(
|
||||
yaml.safe_dump(proposed_schemas, sort_keys=False).rstrip()
|
||||
)
|
||||
save_button = msg.addButton(
|
||||
"Save Without Applying", QMessageBox.AcceptRole
|
||||
)
|
||||
msg.addButton(QMessageBox.Cancel)
|
||||
msg.exec()
|
||||
if msg.clickedButton() is not save_button:
|
||||
return
|
||||
advisory_count = len(preflight.warnings) + len(preflight.suggestions)
|
||||
|
||||
# Preserve source if editing
|
||||
|
||||
@@ -4,6 +4,8 @@ Produces a PreflightReport used by the GUI, CLI, and registry workflow.
|
||||
All checks are deterministic and evidence-based.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
@@ -20,6 +22,7 @@ class PreflightReport:
|
||||
similar_tools: List[Dict[str, Any]] = field(default_factory=list)
|
||||
generated_tests: List[Dict[str, Any]] = field(default_factory=list)
|
||||
compatibility: List[Dict[str, Any]] = field(default_factory=list)
|
||||
contract_proposal: Optional[Dict[str, Any]] = None
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
@@ -33,6 +36,7 @@ class PreflightReport:
|
||||
similar_tools=self.similar_tools + other.similar_tools,
|
||||
generated_tests=self.generated_tests + other.generated_tests,
|
||||
compatibility=self.compatibility + other.compatibility,
|
||||
contract_proposal=self.contract_proposal or other.contract_proposal,
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
@@ -43,6 +47,7 @@ class PreflightReport:
|
||||
"similar_tools": list(self.similar_tools),
|
||||
"generated_tests": list(self.generated_tests),
|
||||
"compatibility": list(self.compatibility),
|
||||
"contract_proposal": copy.deepcopy(self.contract_proposal),
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +56,7 @@ def analyze_tool(
|
||||
registry_client=None,
|
||||
*,
|
||||
check_local_dependencies: bool = True,
|
||||
include_contract_tests: bool = False,
|
||||
) -> PreflightReport:
|
||||
"""Run all preflight checks on a tool.
|
||||
|
||||
@@ -64,6 +70,7 @@ def analyze_tool(
|
||||
report = PreflightReport()
|
||||
_check_config_integrity(tool, report)
|
||||
_check_contracts(tool, report)
|
||||
_add_contract_guidance(tool, report, include_contract_tests)
|
||||
_check_secrets(tool, report)
|
||||
if check_local_dependencies:
|
||||
_check_dependencies(tool, report)
|
||||
@@ -72,6 +79,44 @@ def analyze_tool(
|
||||
return report
|
||||
|
||||
|
||||
def _add_contract_guidance(
|
||||
tool: Tool, report: PreflightReport, include_contract_tests: bool
|
||||
) -> None:
|
||||
proposal = infer_contracts(tool)
|
||||
report.contract_proposal = proposal
|
||||
inferred = [
|
||||
label
|
||||
for label in ("input", "output")
|
||||
if proposal[f"{label}_source"] == "inferred"
|
||||
]
|
||||
if inferred:
|
||||
report.suggestions.append(
|
||||
"Review inferred " + " and ".join(inferred) + " contract proposal"
|
||||
)
|
||||
|
||||
if not include_contract_tests:
|
||||
return
|
||||
if tool.input_schema is None or tool.output_schema is None:
|
||||
return
|
||||
|
||||
from .contract_testing import run_contract_tests
|
||||
|
||||
conformance = run_contract_tests(tool)
|
||||
report.generated_tests = [result.to_dict() for result in conformance.results]
|
||||
if conformance.outcome == "failed":
|
||||
details = "; ".join(
|
||||
result.detail for result in conformance.results if result.state == "failed"
|
||||
)
|
||||
report.errors.append(f"Contract conformance failed: {details}")
|
||||
elif conformance.outcome == "unsupported":
|
||||
details = "; ".join(
|
||||
result.detail
|
||||
for result in conformance.results
|
||||
if result.state == "unsupported"
|
||||
)
|
||||
report.warnings.append(f"Contract conformance unsupported: {details}")
|
||||
|
||||
|
||||
def _check_config_integrity(tool: Tool, report: PreflightReport):
|
||||
if not tool.name:
|
||||
report.errors.append("Tool name is required")
|
||||
@@ -143,3 +188,132 @@ def _check_similar_tools(tool: Tool, report: PreflightReport, client):
|
||||
def _is_semver(version: str) -> bool:
|
||||
from .semver import Version
|
||||
return Version.parse(version) is not None
|
||||
|
||||
|
||||
def infer_contracts(tool: Tool) -> Dict[str, Any]:
|
||||
"""Propose input/output schemas without mutating the tool.
|
||||
|
||||
Returns a dict with:
|
||||
- input_schema: proposed JSON Schema (or None)
|
||||
- output_schema: proposed JSON Schema (or None)
|
||||
- evidence: list of human-readable provenance strings
|
||||
- confidence: "high" / "medium" / "low"
|
||||
|
||||
Never overwrites an explicit schema. ``{}`` counts as explicit.
|
||||
"""
|
||||
evidence = []
|
||||
proposed_input = None
|
||||
proposed_output = None
|
||||
|
||||
if tool.input_schema is None:
|
||||
proposed_input, input_evidence = _infer_input_schema(tool)
|
||||
evidence.extend(input_evidence)
|
||||
|
||||
if tool.output_schema is None:
|
||||
proposed_output, output_evidence = _infer_output_schema(tool)
|
||||
evidence.extend(output_evidence)
|
||||
|
||||
effective_input = tool.input_schema if tool.input_schema is not None else proposed_input
|
||||
effective_output = tool.output_schema if tool.output_schema is not None else proposed_output
|
||||
confidence = _confidence(effective_input, effective_output)
|
||||
|
||||
return {
|
||||
"input_schema": proposed_input,
|
||||
"output_schema": proposed_output,
|
||||
"input_source": "explicit" if tool.input_schema is not None else (
|
||||
"inferred" if proposed_input is not None else "unavailable"
|
||||
),
|
||||
"output_source": "explicit" if tool.output_schema is not None else (
|
||||
"inferred" if proposed_output is not None else "unavailable"
|
||||
),
|
||||
"evidence": evidence,
|
||||
"confidence": confidence,
|
||||
}
|
||||
|
||||
|
||||
def _infer_input_schema(tool: Tool) -> tuple:
|
||||
props = {}
|
||||
required = []
|
||||
evidence = []
|
||||
has_stdin = _contains_input_reference(tool.output) or any(
|
||||
_contains_input_reference(vars(step)) for step in tool.steps
|
||||
)
|
||||
|
||||
if has_stdin:
|
||||
props["input"] = {"type": "string", "description": "Standard input text"}
|
||||
evidence.append("stdin detected from {input} usage")
|
||||
|
||||
for arg in tool.arguments:
|
||||
prop = {"type": arg.type or "string"}
|
||||
if arg.description:
|
||||
prop["description"] = arg.description
|
||||
if arg.default is not None:
|
||||
prop["default"] = arg.default
|
||||
if arg.enum:
|
||||
prop["enum"] = arg.enum
|
||||
props[arg.variable] = prop
|
||||
if arg.required:
|
||||
required.append(arg.variable)
|
||||
evidence.append(f"argument {arg.flag} ({arg.type or 'string'})")
|
||||
|
||||
if not props:
|
||||
return None, []
|
||||
|
||||
schema = {"type": "object", "properties": props}
|
||||
if required:
|
||||
schema["required"] = required
|
||||
return schema, evidence
|
||||
|
||||
|
||||
def _infer_output_schema(tool: Tool) -> tuple:
|
||||
evidence = []
|
||||
|
||||
output = tool.output.strip()
|
||||
if not output or output == "{input}":
|
||||
return None, evidence
|
||||
|
||||
var_match = None
|
||||
m = re.match(r"^\{([a-zA-Z_][a-zA-Z0-9_]*)\}$", output)
|
||||
if m:
|
||||
var_match = m.group(1)
|
||||
|
||||
if not var_match:
|
||||
evidence.append("output template is a composite expression — schema must be defined manually")
|
||||
return None, evidence
|
||||
|
||||
# Look for the step that produces this variable
|
||||
for step in reversed(tool.steps):
|
||||
if hasattr(step, "output_var") and step.output_var == var_match:
|
||||
schema = getattr(step, "output_schema", None)
|
||||
if schema is not None:
|
||||
evidence.append(
|
||||
f"output variable '{var_match}' has an explicit step schema — using as tool output schema"
|
||||
)
|
||||
return copy.deepcopy(schema), evidence
|
||||
elif hasattr(step, "provider"):
|
||||
evidence.append(
|
||||
f"output variable '{var_match}' comes from a provider step with no schema — define output_schema on the step"
|
||||
)
|
||||
return None, evidence
|
||||
break
|
||||
|
||||
evidence.append(f"output variable '{var_match}' source not found")
|
||||
return None, evidence
|
||||
|
||||
|
||||
def _contains_input_reference(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return "{input}" in value or "{input." in value
|
||||
if isinstance(value, dict):
|
||||
return any(_contains_input_reference(item) for item in value.values())
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(_contains_input_reference(item) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def _confidence(input_schema, output_schema) -> str:
|
||||
if input_schema is not None and output_schema is not None:
|
||||
return "high"
|
||||
if input_schema is not None or output_schema is not None:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
Reference in New Issue
Block a user