Add project tools and strict provider execution

This commit is contained in:
rob 2026-07-20 22:24:44 -03:00
parent c289e812fb
commit 6a88fe2e0f
19 changed files with 1504 additions and 84 deletions

View File

@ -45,6 +45,8 @@ cf # Interactive tool picker
- Use `cmdforge run-once "Instruction {input}"` with piped input for ad-hoc AI work; create a permanent tool only when the workflow is reusable.
- Prefer CmdForge for local automation and composable workflows. Direct SDK/API integration is appropriate when the external API is a runtime dependency of the product itself.
- Bootstrap supported coding hosts with `cmdforge mcp configure codex` or `cmdforge mcp configure claude-code`; preview changes with `--dry-run`. This updates only a marked policy block and never expands `server.expose`.
- Create project-owned tools with `cmdforge create NAME --project` or `forge-tool --name NAME --project`; do not edit CmdForge's source repository to add a consumer project's tools.
- For sensitive inputs, use `--no-fallback`, explicit provider constraints, and `--result-envelope json`; provenance is attached by CmdForge and must not be generated by the model.
## Testing

View File

@ -45,6 +45,7 @@ python -m cmdforge.cli # Alternative CLI invocation
- **resolver.py**: Tool resolution. `resolve_tool()` searches: project manifest → local tools → owner/name → registry. Returns `ResolvedTool` with path info
- **collection.py**: Collection management (`Collection` dataclass, `resolve_tool_references()`, `classify_tool_reference()`), local collection storage in `~/.cmdforge/collections/`
- **providers.py**: Provider abstraction. Supports subprocess CLI tools, OpenAI-compatible HTTP APIs, experimental PTY wrappers, fallback chains, and tool/MCP-server allowlists. Auto-discovers installed providers on first run. Config in `~/.cmdforge/providers.yaml` with versioned migration.
- **Provider execution contract**: Providers declare locality, typed capabilities, model identity, context/cost/latency hints, and an approved data classification. `ProviderExecutionPolicy` filters every fallback candidate; `ProviderResult` records runtime-owned provider/model provenance.
- **skills.py**: Validated Agent Skills loader for per-provider `SKILL.md` context under `~/.cmdforge/providers/<name>/skills/`
- **mcp_client.py / mcp_server.py**: Stdio and Streamable HTTP MCP integration, configuration, schema discovery, transport security, and exposure policy
- **profiles.py**: AI persona profiles with system prompts, stored in `~/.cmdforge/profiles/`
@ -115,6 +116,11 @@ Use `cmdforge mcp configure codex` or
`cmdforge mcp configure claude-code --scope project` to register the stdio MCP
server and install a marked project-policy block. Preview with `--dry-run`.
This bootstrap never adds tools to the MCP `server.expose` allowlist.
Project-owned tools belong under `./.cmdforge/`, created with
`cmdforge create NAME --project` or `forge-tool --name NAME --project`. Do not
edit CmdForge core merely to add another project's tools. Sensitive runs should
use `--no-fallback` and `--result-envelope json`; provenance must come from the
runtime rather than model-generated fields.
### Step Types

View File

@ -124,6 +124,9 @@ Opens the graphical interface where you can create and manage tools visually. Fe
- **Registry** - Search and install community tools from the CmdForge registry
- **Providers** - Manage AI provider configurations
- **Reuse & Discovery** - Compare similar tools and preview safe extraction of reusable steps
- **Scoped saving** - Save a new tool to the current project or your global tool collection
![Tool Builder save-location selector](assets/screenshots/tool-builder-save-location.png)
### CLI Mode
@ -132,11 +135,15 @@ Opens the graphical interface where you can create and manage tools visually. Fe
cmdforge list --filter summarize # Filter installed tools for humans
cmdforge list --json --filter summarize --limit 10 # Compact agent catalog
cmdforge list --json --full --limit 1 # Explicitly include prompts/code
cmdforge create mytool # Create new tool
cmdforge create mytool # Create a global tool
cmdforge create mytool --project # Versionable ./.cmdforge tool
echo "Build a bounded classifier" | forge-tool --name classify --project
cmdforge edit mytool # Edit in $EDITOR
cmdforge delete mytool # Delete tool
cmdforge run mytool # Run a tool
cat document.txt | cmdforge run-once "Summarize this:\n\n{input}"
cmdforge run private-tool --provider ollama --no-fallback \
--require-local --data-classification private --result-envelope json
cmdforge test mytool # Test with mock provider
cmdforge check mytool # Check dependencies (meta-tools)
cmdforge inspect mytool # Preflight, contract proposals, safe conformance tests

Binary file not shown.

After

Width:  |  Height:  |  Size: 82 KiB

View File

@ -2,6 +2,7 @@
import argparse
import sys
from pathlib import Path
from .. import __version__
@ -50,6 +51,15 @@ def main():
p_create.add_argument("-p", "--prompt", help="Prompt template")
p_create.add_argument("--provider", help="AI provider (default: mock)")
p_create.add_argument("-f", "--force", action="store_true", help="Overwrite existing")
create_location = p_create.add_mutually_exclusive_group()
create_location.add_argument(
"--project", action="store_true",
help="Save under ./.cmdforge instead of ~/.cmdforge",
)
create_location.add_argument(
"--output-dir", type=Path,
help="Save under an explicit tool root directory",
)
p_create.set_defaults(func=cmd_create)
# 'edit' command
@ -81,6 +91,13 @@ def main():
p_run.add_argument("--show-prompt", action="store_true", help="Show prompts in addition to output")
p_run.add_argument("-v", "--verbose", action="store_true", help="Show debug information")
p_run.add_argument("--auto-install", action="store_true", help="Automatically install missing tool dependencies")
p_run.add_argument("--no-fallback", action="store_true", help="Fail instead of trying configured fallback providers")
p_run.add_argument("--require-local", action="store_true", help="Allow only providers declared local")
p_run.add_argument("--require-capability", action="append", default=[], help="Required provider capability (repeatable)")
p_run.add_argument("--data-classification", choices=["public", "internal", "private"], help="Input data classification")
p_run.add_argument("--require-model-identity", action="store_true")
p_run.add_argument("--require-model-digest", action="store_true")
p_run.add_argument("--result-envelope", choices=["json"], help="Wrap output with verified execution provenance")
p_run.add_argument("tool_args", nargs=argparse.REMAINDER, help="Additional tool-specific arguments (use -- to separate)")
p_run.set_defaults(func=cmd_run)
@ -100,6 +117,13 @@ def main():
)
p_run_once.add_argument("--max-tokens", type=int, help="Max output tokens")
p_run_once.add_argument("--timeout", type=int, default=60, help="Timeout in seconds")
p_run_once.add_argument("--no-fallback", action="store_true")
p_run_once.add_argument("--require-local", action="store_true")
p_run_once.add_argument("--require-capability", action="append", default=[])
p_run_once.add_argument("--data-classification", choices=["public", "internal", "private"])
p_run_once.add_argument("--require-model-identity", action="store_true")
p_run_once.add_argument("--require-model-digest", action="store_true")
p_run_once.add_argument("--result-envelope", choices=["json"])
p_run_once.set_defaults(func=cmd_run_once)
# 'ui' command (explicit)
@ -157,6 +181,7 @@ def main():
# providers list
p_prov_list = providers_sub.add_parser("list", help="List all providers and their status")
p_prov_list.add_argument("--json", action="store_true")
p_prov_list.set_defaults(func=cmd_providers)
# providers check
@ -181,6 +206,27 @@ def main():
p_prov_add.add_argument("--model", help="Model ID for API providers")
p_prov_add.add_argument("--api-key-env", help="Environment variable containing the API key")
p_prov_add.add_argument("--tag", action="append", dest="tags", help="Provider tag (repeatable)")
p_prov_add.add_argument(
"--locality", choices=["local", "remote", "unknown"],
help="Where inference runs",
)
p_prov_add.add_argument(
"--capability", action="append", dest="capabilities",
help="Typed capability such as structured-json (repeatable)",
)
p_prov_add.add_argument("--model-digest", help="Verified model content digest")
p_prov_add.add_argument(
"--cost-class", choices=["free", "low", "medium", "high"]
)
p_prov_add.add_argument(
"--latency-class", choices=["fast", "standard", "slow"]
)
p_prov_add.add_argument("--max-context-tokens", type=int)
p_prov_add.add_argument(
"--data-policy",
choices=["unspecified", "public", "internal", "private"],
help="Maximum approved input classification",
)
p_prov_add.add_argument("--fallback", help="Fallback provider name; pass an empty value to clear")
p_prov_add.add_argument(
"--fallback-chain",
@ -667,9 +713,9 @@ def main():
def cmd_inspect(args):
"""Run preflight analysis on a tool."""
from ..preflight import analyze_tool
from ..tool import load_tool
from .tool_commands import _resolve_existing_tool
tool = load_tool(args.name)
tool = _resolve_existing_tool(args.name)
if not tool:
print(f"Error: Tool '{args.name}' not found.", file=sys.stderr)
return 1

View File

@ -353,6 +353,17 @@ def _cmd_providers_discover(args):
def _cmd_providers_list(args):
"""List all providers and their status."""
providers = load_providers()
if getattr(args, "json", False):
import json
payload = []
for provider in providers:
available, status = _provider_status(provider)
item = provider.to_dict()
item["available"] = available
item["status"] = status
payload.append(item)
print(json.dumps(payload, indent=2))
return 0
print(f"Configured providers ({len(providers)}):\n")
for p in providers:
exists, status = _provider_status(p)
@ -365,6 +376,15 @@ def _cmd_providers_list(args):
print(f" Status: {status}")
if p.description:
print(f" Info: {p.description}")
print(f" Locality: {p.locality}")
print(f" Data policy: {p.data_policy}")
if p.capabilities:
print(f" Capabilities: {', '.join(p.capabilities)}")
if p.model:
identity = p.model
if p.model_digest:
identity += f" ({p.model_digest})"
print(f" Model: {identity}")
print()
return 0
@ -379,6 +399,17 @@ def _cmd_providers_add(args):
model = args.model if args.model is not None else (existing.model if existing else None)
api_key_env = args.api_key_env if args.api_key_env is not None else (existing.api_key_env if existing else None)
tags = args.tags if args.tags is not None else (existing.tags if existing else [])
locality = args.locality if args.locality is not None else (existing.locality if existing else "unknown")
capabilities = args.capabilities if args.capabilities is not None else (existing.capabilities if existing else [])
model_digest = args.model_digest if args.model_digest is not None else (existing.model_digest if existing else None)
cost_class = args.cost_class if args.cost_class is not None else (existing.cost_class if existing else None)
latency_class = args.latency_class if args.latency_class is not None else (existing.latency_class if existing else None)
max_context_tokens = (
args.max_context_tokens
if args.max_context_tokens is not None
else existing.max_context_tokens if existing else None
)
data_policy = args.data_policy if args.data_policy is not None else (existing.data_policy if existing else "unspecified")
fallback = args.fallback if args.fallback is not None else (existing.fallback if existing else None)
fallback = fallback or None
@ -406,6 +437,15 @@ def _cmd_providers_add(args):
fallback_chain=fallback_chain,
api_key_env=api_key_env,
pty_config=existing.pty_config if existing else None,
tools=existing.tools if existing else None,
mcp_servers=existing.mcp_servers if existing else None,
locality=locality,
capabilities=capabilities,
model_digest=model_digest,
cost_class=cost_class,
latency_class=latency_class,
max_context_tokens=max_context_tokens,
data_policy=data_policy,
)
add_provider(provider)
print(f"Provider '{name}' added/updated.")

View File

@ -5,11 +5,105 @@ from pathlib import Path
from ..tool import (
list_tools, load_tool, save_tool, delete_tool, get_tools_dir,
Tool, ToolArgument, PromptStep, CodeStep, ToolStep, validate_tool_name
get_project_tools_dir, Tool, ToolArgument, PromptStep, CodeStep, ToolStep,
validate_tool_name
)
from ..gui import run_gui
def _resolve_existing_tool(name: str):
"""Resolve project-local before global without registry auto-fetch."""
from ..resolver import find_tool
project_candidate = any(
(Path.cwd() / root / name / "config.yaml").exists()
for root in (".cmdforge", "cmdforge")
)
if project_candidate:
resolved = find_tool(name)
if resolved:
return resolved.tool
from .. import tool as tool_module
return tool_module.load_tool(name)
def _project_tool_names(project_dir=None) -> list[str]:
"""List project-local references using the same two resolver roots."""
project_dir = (project_dir or Path.cwd()).resolve()
names = []
for root_name in (".cmdforge", "cmdforge"):
root = project_dir / root_name
if not root.is_dir():
continue
for item in sorted(root.iterdir()):
if not item.is_dir() or item.name.startswith("."):
continue
if (item / "config.yaml").is_file():
names.append(item.name)
continue
for child in sorted(item.iterdir()):
if child.is_dir() and (child / "config.yaml").is_file():
names.append(f"{item.name}/{child.name}")
return list(dict.fromkeys(names))
def _execution_policy_from_args(*namespaces):
"""Combine top-level and wrapper-compatible execution policy flags."""
from ..providers import ProviderExecutionPolicy
capabilities = []
for namespace in namespaces:
capabilities.extend(
getattr(namespace, "require_capability", None) or []
)
data_classification = next((
getattr(namespace, "data_classification", None)
for namespace in namespaces
if getattr(namespace, "data_classification", None)
), "public")
return ProviderExecutionPolicy(
allow_fallback=not any(
getattr(namespace, "no_fallback", False) for namespace in namespaces
),
required_locality=(
"local" if any(
getattr(namespace, "require_local", False)
for namespace in namespaces
) else None
),
required_capabilities=tuple(dict.fromkeys(capabilities)),
data_classification=data_classification,
require_model_identity=any(
getattr(namespace, "require_model_identity", False)
for namespace in namespaces
),
require_model_digest=any(
getattr(namespace, "require_model_digest", False)
for namespace in namespaces
),
)
def _result_envelope(
output: str, provider_calls: list[dict], exit_code: int = 0
) -> str:
"""Build a stable machine-readable envelope without parsing tool output."""
import json
execution = {
"exit_code": exit_code,
"provider_calls": provider_calls,
"fallback_used": any(
call.get("fallback_used", False) for call in provider_calls
),
"actual_providers": list(dict.fromkeys(
call["actual_provider"] for call in provider_calls
if call.get("actual_provider")
)),
}
return json.dumps({"output": output, "execution": execution}, indent=2)
def cmd_list(args):
"""List all tools."""
import json
@ -24,8 +118,10 @@ def cmd_list(args):
return 1
matches = []
for name in sorted(list_tools()):
tool = load_tool(name)
project_names = _project_tool_names()
all_names = list(dict.fromkeys([*project_names, *sorted(list_tools())]))
for name in all_names:
tool = _resolve_existing_tool(name)
if not tool:
continue
if filter_text and (
@ -151,14 +247,21 @@ def cmd_run_once(args):
return 1
print(f"[run-once] Provider: {provider}", file=sys.stderr)
policy = _execution_policy_from_args(args)
result = call_provider(
provider, prompt, timeout=timeout, max_tokens=max_tokens
provider, prompt, timeout=timeout, max_tokens=max_tokens,
execution_policy=policy,
)
if not result.success:
if getattr(args, "result_envelope", None) == "json":
print(_result_envelope("", [result.provenance()], exit_code=1))
print(f"Error: {result.error}", file=sys.stderr)
return 1
if getattr(args, "result_envelope", None) == "json":
print(_result_envelope(result.text, [result.provenance()]))
else:
print(result.text)
return 0
@ -242,10 +345,22 @@ def cmd_create(args):
print(f"Error: Invalid tool name '{name}': {error_msg}")
return 1
# Check if already exists
existing = load_tool(name)
if existing and not args.force:
print(f"Error: Tool '{name}' already exists. Use --force to overwrite.")
output_dir = getattr(args, "output_dir", None)
project = getattr(args, "project", False)
target_root = (
Path(output_dir).resolve()
if output_dir is not None
else get_project_tools_dir() if project else get_tools_dir()
)
# Check only the requested scope. A project tool may intentionally shadow
# a global tool with the same name.
config_path = target_root / name / "config.yaml"
if config_path.exists() and not args.force:
print(
f"Error: Tool '{name}' already exists at {config_path}. "
"Use --force to overwrite."
)
return 1
# Create a tool with a single prompt step
@ -266,10 +381,18 @@ def cmd_create(args):
output="{response}" if steps else "{input}"
)
path = save_tool(tool)
path = save_tool(tool, tools_dir=target_root)
print(f"Created tool '{name}'")
print(f"Config: {path}")
print(f"\nUse 'cmdforge ui' to add arguments, steps, and customize.")
print("\nUse 'cmdforge ui' to add arguments, steps, and customize.")
if project:
print(f"Run from its project: cmdforge run {name} < input.txt")
elif output_dir is not None:
print(
"Saved under the requested tool root. CmdForge automatically "
"discovers roots named .cmdforge or cmdforge."
)
else:
print(f"Or run: {name} < input.txt")
return 0
@ -323,7 +446,7 @@ def cmd_delete(args):
def cmd_test(args):
"""Test a tool with mock provider."""
tool = load_tool(args.name)
tool = _resolve_existing_tool(args.name)
if not tool:
print(f"Error: Tool '{args.name}' not found.")
return 1
@ -360,7 +483,7 @@ def cmd_run(args):
"""Run a tool."""
from ..runner import collect_custom_args, create_argument_parser, run_tool
tool = load_tool(args.name)
tool = _resolve_existing_tool(args.name)
if not tool:
print(f"Error: Tool '{args.name}' not found.", file=sys.stderr)
return 1
@ -404,6 +527,12 @@ def cmd_run(args):
show_prompt = args.show_prompt or parsed_tool_args.show_prompt
verbose = args.verbose or parsed_tool_args.verbose
auto_install = args.auto_install or parsed_tool_args.auto_install
execution_policy = _execution_policy_from_args(args, parsed_tool_args)
result_envelope = (
getattr(args, "result_envelope", None)
or getattr(parsed_tool_args, "result_envelope", None)
)
provider_calls = []
# Run tool
output, code = run_tool(
@ -414,16 +543,22 @@ def cmd_run(args):
dry_run=dry_run,
show_prompt=show_prompt,
verbose=verbose,
auto_install=auto_install
auto_install=auto_install,
execution_policy=execution_policy,
provenance_log=provider_calls,
)
# Write output
if code == 0 and output:
if result_envelope == "json" or (code == 0 and output):
rendered_output = (
_result_envelope(output, provider_calls, exit_code=code)
if result_envelope == "json" else output
)
output_file = args.output or parsed_tool_args.output_file
if output_file:
Path(output_file).write_text(output)
Path(output_file).write_text(rendered_output)
else:
print(output)
print(rendered_output)
return code

View File

@ -44,6 +44,44 @@ class ProviderDialog(QDialog):
self.desc_input.setPlaceholderText("Claude AI via claude-cli")
form.addRow("Description:", self.desc_input)
self.locality_combo = QComboBox()
for value in ("unknown", "local", "remote"):
self.locality_combo.addItem(value.title(), value)
form.addRow("Locality:", self.locality_combo)
self.data_policy_combo = QComboBox()
for value in ("unspecified", "public", "internal", "private"):
self.data_policy_combo.addItem(value.title(), value)
form.addRow("Data policy:", self.data_policy_combo)
self.capabilities_input = QLineEdit()
self.capabilities_input.setPlaceholderText("text, structured-json, reasoning")
form.addRow("Capabilities:", self.capabilities_input)
self.model_input = QLineEdit()
self.model_input.setPlaceholderText("Model ID, if known")
form.addRow("Model:", self.model_input)
self.model_digest_input = QLineEdit()
self.model_digest_input.setPlaceholderText("sha256:... (verified local models)")
form.addRow("Model digest:", self.model_digest_input)
self.max_context_input = QLineEdit()
self.max_context_input.setPlaceholderText("Maximum context tokens")
form.addRow("Context tokens:", self.max_context_input)
self.cost_class_combo = QComboBox()
self.cost_class_combo.addItem("Unspecified", None)
for value in ("free", "low", "medium", "high"):
self.cost_class_combo.addItem(value.title(), value)
form.addRow("Cost class:", self.cost_class_combo)
self.latency_class_combo = QComboBox()
self.latency_class_combo.addItem("Unspecified", None)
for value in ("fast", "standard", "slow"):
self.latency_class_combo.addItem(value.title(), value)
form.addRow("Latency class:", self.latency_class_combo)
self.fallback_combo = QComboBox()
self.fallback_combo.addItem("(none)", None)
# Populate with available providers
@ -86,6 +124,24 @@ class ProviderDialog(QDialog):
self.name_input.setEnabled(False) # Can't rename
self.cmd_input.setText(provider.command)
self.desc_input.setText(provider.description or "")
self.locality_combo.setCurrentIndex(
max(self.locality_combo.findData(provider.locality), 0)
)
self.data_policy_combo.setCurrentIndex(
max(self.data_policy_combo.findData(provider.data_policy), 0)
)
self.capabilities_input.setText(", ".join(provider.capabilities))
self.model_input.setText(provider.model or "")
self.model_digest_input.setText(provider.model_digest or "")
self.max_context_input.setText(
str(provider.max_context_tokens) if provider.max_context_tokens else ""
)
self.cost_class_combo.setCurrentIndex(
max(self.cost_class_combo.findData(provider.cost_class), 0)
)
self.latency_class_combo.setCurrentIndex(
max(self.latency_class_combo.findData(provider.latency_class), 0)
)
# Set fallback selection
if provider.fallback:
idx = self.fallback_combo.findData(provider.fallback)
@ -107,6 +163,21 @@ class ProviderDialog(QDialog):
description = self.desc_input.text().strip()
fallback = self.fallback_combo.currentData()
capabilities = [
value.strip() for value in self.capabilities_input.text().split(",")
if value.strip()
]
try:
max_context_tokens = (
int(self.max_context_input.text().strip())
if self.max_context_input.text().strip() else None
)
except ValueError:
from PySide6.QtWidgets import QMessageBox
QMessageBox.warning(
self, "Invalid", "Context tokens must be a positive integer."
)
return
# Prevent self-referential fallback
if fallback == name:
@ -122,12 +193,21 @@ class ProviderDialog(QDialog):
description=description,
fallback=fallback,
type=existing.type if existing else "subprocess",
model=existing.model if existing else None,
model=self.model_input.text().strip() or None,
tags=existing.tags if existing else [],
install=existing.install if existing else None,
fallback_chain=existing.fallback_chain if existing else None,
api_key_env=existing.api_key_env if existing else None,
pty_config=existing.pty_config if existing else None,
tools=existing.tools if existing else None,
mcp_servers=existing.mcp_servers if existing else None,
locality=self.locality_combo.currentData(),
capabilities=capabilities,
model_digest=self.model_digest_input.text().strip() or None,
cost_class=self.cost_class_combo.currentData(),
latency_class=self.latency_class_combo.currentData(),
max_context_tokens=max_context_tokens,
data_policy=self.data_policy_combo.currentData(),
))
self.accept()
except Exception as e:

View File

@ -321,6 +321,15 @@ class PromptStepDialog(QDialog):
self.strip_fences_check = QCheckBox("Strip markdown code fences from output")
form.addRow("", self.strip_fences_check)
self.allow_fallback_check = QCheckBox(
"Allow configured fallback providers if this provider fails"
)
self.allow_fallback_check.setChecked(True)
self.allow_fallback_check.setToolTip(
"Disable for privacy-sensitive or reproducible steps that must fail closed."
)
form.addRow("", self.allow_fallback_check)
# Structured output options
form.addRow(QLabel("")) # Spacer
structured_label = QLabel("<b>Structured Output</b>")
@ -455,6 +464,7 @@ class PromptStepDialog(QDialog):
self.output_input.setText(step.output_var)
self.prompt_input.setPlainText(step.prompt)
self.strip_fences_check.setChecked(step.strip_fences)
self.allow_fallback_check.setChecked(step.fallback_policy != "deny")
# Structured output fields
self.plain_text_check.setChecked(step.plain_text)
@ -497,7 +507,10 @@ class PromptStepDialog(QDialog):
strip_fences=self.strip_fences_check.isChecked(),
output_schema=self._output_schema,
plain_text=self.plain_text_check.isChecked(),
max_retries=self.retries_spin.value()
max_retries=self.retries_spin.value(),
fallback_policy=(
"allow" if self.allow_fallback_check.isChecked() else "deny"
),
)

View File

@ -3,6 +3,7 @@
import copy
import difflib
import re
from pathlib import Path
import yaml
@ -17,8 +18,8 @@ from PySide6.QtCore import Qt, QThread, Signal, QTimer
from ...tool import (
Tool, ToolArgument, PromptStep, CodeStep, ToolStep,
load_tool, save_tool, tool_exists, validate_tool_name, get_all_categories,
ensure_settings
load_tool, save_tool, validate_tool_name, get_all_categories,
ensure_settings, get_project_tools_dir, get_tools_dir
)
from ..widgets.icons import get_prompt_icon, get_code_icon, get_tool_icon
@ -107,6 +108,21 @@ def _switch_to_existing_tool(main_window, name: str) -> None:
main_window.open_tool_builder(name)
def default_save_scope(project_dir: Path) -> str:
"""Prefer project scope only when the cwd is recognizably a project."""
project_dir = Path(project_dir)
if any((project_dir / marker).exists() for marker in (
"cmdforge.yaml", ".cmdforge", ".git", "pyproject.toml", "package.json",
)):
return "project"
return "global"
def existing_tool_root(config_path: Path) -> Path:
"""Return the tool-root directory containing an existing named tool."""
return Path(config_path).resolve().parent.parent
class ToolBuilderPage(QWidget):
"""Tool builder/editor page."""
@ -185,6 +201,23 @@ class ToolBuilderPage(QWidget):
self.category_combo.addItem(cat)
info_layout.addRow("Category:", self.category_combo)
self.save_location_combo = QComboBox()
project_root = get_project_tools_dir()
self.save_location_combo.addItem(
f"This project ({project_root})", "project"
)
self.save_location_combo.addItem(
f"My tools ({get_tools_dir()})", "global"
)
preferred_scope = default_save_scope(Path.cwd())
preferred_index = self.save_location_combo.findData(preferred_scope)
self.save_location_combo.setCurrentIndex(max(preferred_index, 0))
self.save_location_combo.setToolTip(
"Project tools are versionable and resolve only from this project; "
"My tools are available globally and receive a shell wrapper."
)
info_layout.addRow("Save tool:", self.save_location_combo)
left_layout.addWidget(info_box)
# Reuse guidance is advisory and never changes a draft automatically.
@ -658,7 +691,8 @@ class ToolBuilderPage(QWidget):
if not valid:
QMessageBox.warning(self, "Validation", error)
return
if tool_exists(name):
extraction_root = self._selected_tools_dir()
if (extraction_root / name / "config.yaml").exists():
QMessageBox.warning(self, "Tool Already Exists", f"'{name}' already exists.")
return
try:
@ -684,7 +718,7 @@ class ToolBuilderPage(QWidget):
if preview.clickedButton() is not create_button:
return
try:
save_tool(extracted)
save_tool(extracted, tools_dir=extraction_root)
except (OSError, ValueError) as exc:
QMessageBox.critical(self, "Extraction Failed", str(exc))
return
@ -807,6 +841,17 @@ class ToolBuilderPage(QWidget):
self._tool = tool
self.name_input.setText(tool.name)
self.name_input.setEnabled(False) # Can't rename
if tool.path:
project_root = get_project_tools_dir()
scope = (
"project"
if existing_tool_root(tool.path) == project_root.resolve()
else "global"
)
index = self.save_location_combo.findData(scope)
if index >= 0:
self.save_location_combo.setCurrentIndex(index)
self.save_location_combo.setEnabled(False)
self.desc_input.setText(tool.description or "")
# Set category
@ -1371,9 +1416,13 @@ class ToolBuilderPage(QWidget):
QMessageBox.warning(self, "Validation", error)
return
# Block silent overwrite when creating a new tool
if (not self.editing or name != self.original_name) and tool_exists(name):
existing = load_tool(name)
target_root = self._selected_tools_dir()
# Block silent overwrite in the selected scope. A project-local tool
# may intentionally shadow a global tool with the same name.
target_config = target_root / name / "config.yaml"
if (not self.editing or name != self.original_name) and target_config.exists():
existing = load_tool(name) if self.save_location_combo.currentData() == "global" else None
existing_desc = f" ({existing.description})" if existing and existing.description else ""
msg = QMessageBox(self)
msg.setIcon(QMessageBox.Warning)
@ -1393,7 +1442,7 @@ class ToolBuilderPage(QWidget):
return
elif clicked is btn_copy:
suffix = 2
while tool_exists(f"{name}-{suffix}"):
while (target_root / f"{name}-{suffix}" / "config.yaml").exists():
suffix += 1
copy_name = f"{name}-{suffix}"
self.name_input.setText(copy_name)
@ -1514,7 +1563,7 @@ class ToolBuilderPage(QWidget):
tool.version = self._tool.version
try:
config_path = save_tool(tool)
config_path = save_tool(tool, tools_dir=target_root)
tool_dir = config_path.parent
# Save defaults if provided
@ -1551,6 +1600,14 @@ class ToolBuilderPage(QWidget):
"""Cancel and return to tools page."""
self.main_window.close_tool_builder()
def _selected_tools_dir(self) -> Path:
"""Return the save root selected by the author."""
if self.editing and self._tool and self._tool.path:
return existing_tool_root(self._tool.path)
if self.save_location_combo.currentData() == "project":
return get_project_tools_dir()
return get_tools_dir().resolve()
def save_tool(self):
"""Public method for keyboard shortcut to save the tool."""
self._save()

View File

@ -55,6 +55,9 @@ KNOWN_PROVIDER_CLIS = {
"command": "opencode run --model opencode/big-pickle",
"description": "OpenCode - Big Pickle (free general model)",
"tags": ["free", "code", "general"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
"install_group": "opencode",
},
"agy": {
@ -62,6 +65,9 @@ KNOWN_PROVIDER_CLIS = {
"command": "agy -p",
"description": "Antigravity - Google free tier, Gemini models",
"tags": ["free-tier", "code", "large-context"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
"install_group": "agy",
},
"codex": {
@ -69,6 +75,9 @@ KNOWN_PROVIDER_CLIS = {
"command": "codex exec -",
"description": "Codex CLI - OpenAI free tier available",
"tags": ["free-tier", "code", "general"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
"install_group": "codex",
},
"claude": {
@ -76,6 +85,9 @@ KNOWN_PROVIDER_CLIS = {
"command": "claude -p",
"description": "Claude Code - auto-routes to best model",
"tags": ["paid", "subscription", "code"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
"install_group": "claude",
},
"crush": {
@ -83,6 +95,9 @@ KNOWN_PROVIDER_CLIS = {
"command": "crush run --quiet",
"description": "Crush - multi-model via Hyper credits or API keys",
"tags": ["free-tier", "multi", "code"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
"install_group": "crush",
},
"ollama": {
@ -90,6 +105,10 @@ KNOWN_PROVIDER_CLIS = {
"command": "ollama run llama3.2",
"description": "Ollama - local, private, free",
"tags": ["free", "local", "private"],
"model": "llama3.2",
"locality": "local",
"capabilities": ["text", "structured-json"],
"data_policy": "private",
"install_group": "ollama",
},
}
@ -102,6 +121,9 @@ KNOWN_API_KEYS = {
"model": "openrouter/auto-beta",
"description": "OpenRouter - 300+ models, one API key",
"tags": ["api", "per-token", "multi"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
},
"DEEPSEEK_API_KEY": {
"name": "deepseek-api",
@ -109,6 +131,9 @@ KNOWN_API_KEYS = {
"model": "deepseek-chat",
"description": "DeepSeek API - inexpensive per-token access",
"tags": ["api", "per-token", "cheap"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
},
"OPENAI_API_KEY": {
"name": "openai-api",
@ -116,6 +141,9 @@ KNOWN_API_KEYS = {
"model": "gpt-4o",
"description": "OpenAI API - direct GPT access",
"tags": ["api", "per-token", "code"],
"locality": "remote",
"capabilities": ["text", "structured-json", "reasoning"],
"data_policy": "public",
},
}
@ -181,6 +209,10 @@ def discover_installed_providers() -> List[dict]:
"command": f"ollama run {model_name}",
"description": f"Ollama local model: {model_name}",
"tags": ["free", "local", "private"],
"model": model_name,
"locality": "local",
"capabilities": ["text", "structured-json"],
"data_policy": "private",
})
except (subprocess.TimeoutExpired, FileNotFoundError):
pass
@ -188,6 +220,26 @@ def discover_installed_providers() -> List[dict]:
return found
def _provider_from_discovery(data: dict) -> "Provider":
"""Convert trusted discovery facts without persisting scanner-only keys."""
return Provider(
name=data["name"],
command=data["command"],
description=data.get("description", ""),
type=data.get("type", "subprocess"),
model=data.get("model"),
tags=data.get("tags", []),
api_key_env=data.get("env_var"),
locality=data.get("locality", "unknown"),
capabilities=data.get("capabilities", []),
model_digest=data.get("model_digest"),
cost_class=data.get("cost_class"),
latency_class=data.get("latency_class"),
max_context_tokens=data.get("max_context_tokens"),
data_policy=data.get("data_policy", "unspecified"),
)
@dataclass
class Provider:
"""Definition of an AI provider.
@ -215,6 +267,13 @@ class Provider:
pty_config: Optional[dict] = None # Patterns for pty-type providers
tools: Optional[List[str]] = None # Allowed CmdForge tools (None = all)
mcp_servers: Optional[List[str]] = None # MCP servers available to this provider
locality: str = "unknown" # "local" | "remote" | "unknown"
capabilities: List[str] = field(default_factory=list)
model_digest: Optional[str] = None
cost_class: Optional[str] = None # "free" | "low" | "medium" | "high"
latency_class: Optional[str] = None # "fast" | "standard" | "slow"
max_context_tokens: Optional[int] = None
data_policy: str = "unspecified" # max: public | internal | private
def __post_init__(self) -> None:
for field_name, values in (
@ -228,6 +287,30 @@ class Provider:
raise ValueError(
f"Provider {field_name} must be a list of non-empty strings or null"
)
if self.locality not in {"local", "remote", "unknown"}:
raise ValueError("Provider locality must be local, remote, or unknown")
if (
not isinstance(self.capabilities, list)
or not all(isinstance(value, str) and value for value in self.capabilities)
):
raise ValueError("Provider capabilities must be a list of non-empty strings")
if self.cost_class not in {None, "free", "low", "medium", "high"}:
raise ValueError("Provider cost_class must be free, low, medium, or high")
if self.latency_class not in {None, "fast", "standard", "slow"}:
raise ValueError("Provider latency_class must be fast, standard, or slow")
if self.data_policy not in {"unspecified", "public", "internal", "private"}:
raise ValueError(
"Provider data_policy must be unspecified, public, internal, or private"
)
if (
self.max_context_tokens is not None
and (
not isinstance(self.max_context_tokens, int)
or isinstance(self.max_context_tokens, bool)
or self.max_context_tokens <= 0
)
):
raise ValueError("Provider max_context_tokens must be a positive integer")
def to_dict(self) -> dict:
d = {
@ -256,6 +339,20 @@ class Provider:
d["tools"] = self.tools
if self.mcp_servers is not None:
d["mcp_servers"] = self.mcp_servers
if self.locality != "unknown":
d["locality"] = self.locality
if self.capabilities:
d["capabilities"] = self.capabilities
if self.model_digest:
d["model_digest"] = self.model_digest
if self.cost_class:
d["cost_class"] = self.cost_class
if self.latency_class:
d["latency_class"] = self.latency_class
if self.max_context_tokens is not None:
d["max_context_tokens"] = self.max_context_tokens
if self.data_policy != "unspecified":
d["data_policy"] = self.data_policy
return d
@classmethod
@ -265,7 +362,7 @@ class Provider:
command=data["command"],
description=data.get("description", ""),
type=data.get("type", "subprocess"),
model=data.get("model"),
model=data.get("model", data.get("model_id")),
fallback=data.get("fallback"),
tags=data.get("tags", []) or [],
install=data.get("install"),
@ -274,6 +371,15 @@ class Provider:
pty_config=data.get("pty_config"),
tools=data.get("tools"),
mcp_servers=data.get("mcp_servers"),
locality=data.get("locality", "unknown"),
capabilities=data.get("capabilities", []) or [],
model_digest=data.get("model_digest"),
cost_class=data.get("cost_class"),
latency_class=data.get("latency_class"),
max_context_tokens=data.get(
"max_context_tokens", data.get("max_input_tokens")
),
data_policy=data.get("data_policy", "unspecified"),
)
@ -283,6 +389,164 @@ class ProviderResult:
text: str
success: bool
error: Optional[str] = None
requested_provider: Optional[str] = None
actual_provider: Optional[str] = None
model: Optional[str] = None
model_digest: Optional[str] = None
attempted_providers: List[str] = field(default_factory=list)
fallback_used: bool = False
locality: str = "unknown"
model_identity_source: str = "unknown"
def provenance(self) -> dict:
"""Return runtime-owned execution facts safe for result envelopes."""
data = {
"success": self.success,
"requested_provider": self.requested_provider,
"actual_provider": self.actual_provider,
"attempted_providers": list(self.attempted_providers),
"fallback_used": self.fallback_used,
"model": self.model,
"model_digest": self.model_digest,
"locality": self.locality,
"model_identity_source": self.model_identity_source,
}
if self.error:
data["error"] = self.error
return data
@dataclass(frozen=True)
class ProviderExecutionPolicy:
"""Caller-supplied, fail-closed constraints for provider execution."""
allow_fallback: bool = True
required_locality: Optional[str] = None
required_capabilities: tuple[str, ...] = ()
data_classification: str = "public"
require_model_identity: bool = False
require_model_digest: bool = False
def __post_init__(self) -> None:
if self.required_locality not in {None, "local", "remote"}:
raise ValueError("required_locality must be local, remote, or null")
if self.data_classification not in {"public", "internal", "private"}:
raise ValueError(
"data_classification must be public, internal, or private"
)
_DATA_CLASS_LEVEL = {"public": 0, "internal": 1, "private": 2}
def effective_provider_locality(provider: Provider) -> str:
"""Account for Ollama clients redirected to a non-loopback host."""
try:
executable = shlex.split(os.path.expandvars(provider.command))[0]
except (IndexError, ValueError):
executable = ""
if Path(executable).name != "ollama":
return provider.locality
ollama_host = os.environ.get("OLLAMA_HOST", "").strip()
if not ollama_host:
return provider.locality
from urllib.parse import urlparse
parsed = urlparse(
ollama_host if "://" in ollama_host else f"//{ollama_host}"
)
hostname = (parsed.hostname or "").lower()
if hostname in {"localhost", "127.0.0.1", "::1"}:
return "local"
return "remote"
def runtime_provider_identity(provider: Provider) -> tuple[Optional[str], Optional[str], str]:
"""Return observed model identity when supported, else configured facts."""
configured = (
provider.model,
provider.model_digest,
"provider-config" if provider.model or provider.model_digest else "unknown",
)
try:
parts = shlex.split(os.path.expandvars(provider.command))
except ValueError:
return configured
if not parts or Path(parts[0]).name != "ollama":
return configured
if effective_provider_locality(provider) != "local":
return configured
try:
run_index = parts.index("run")
model = provider.model or parts[run_index + 1]
except (ValueError, IndexError):
return configured
host = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434").strip()
if "://" not in host:
host = f"http://{host}"
from urllib.request import Request, urlopen
import json
request = Request(host.rstrip("/") + "/api/tags")
api_key = os.environ.get("OLLAMA_API_KEY")
if api_key:
request.add_header("Authorization", f"Bearer {api_key}")
try:
with urlopen(request, timeout=2) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception:
return configured
wanted = {model, f"{model}:latest"}
for item in payload.get("models", []):
names = {item.get("name"), item.get("model")}
if names & wanted or any(
value and value.removesuffix(":latest") == model.removesuffix(":latest")
for value in names
):
digest = item.get("digest") or provider.model_digest
if digest and not digest.startswith("sha256:"):
digest = f"sha256:{digest}"
return item.get("model") or item.get("name") or model, digest, "ollama-api"
return configured
def provider_policy_error(
provider: Provider,
policy: ProviderExecutionPolicy,
runtime_identity: Optional[tuple[Optional[str], Optional[str], str]] = None,
) -> Optional[str]:
"""Return why a provider is ineligible, or None when it is allowed."""
if (
policy.required_locality is not None
and effective_provider_locality(provider) != policy.required_locality
):
return (
f"Provider '{provider.name}' locality is "
f"{effective_provider_locality(provider)}; "
f"{policy.required_locality} is required"
)
missing = sorted(set(policy.required_capabilities) - set(provider.capabilities))
if missing:
return (
f"Provider '{provider.name}' lacks required capabilities: "
+ ", ".join(missing)
)
if policy.data_classification != "public":
allowed = _DATA_CLASS_LEVEL.get(provider.data_policy, -1)
needed = _DATA_CLASS_LEVEL[policy.data_classification]
if allowed < needed:
return (
f"Provider '{provider.name}' data_policy is {provider.data_policy}; "
f"{policy.data_classification} data requires an explicit compatible policy"
)
model, digest, _ = runtime_identity or (
provider.model, provider.model_digest, "provider-config"
)
if policy.require_model_identity and not model:
return f"Provider '{provider.name}' does not declare a model identity"
if policy.require_model_digest and not digest:
return f"Provider '{provider.name}' does not declare a model digest"
return None
# Default providers that come pre-configured
@ -334,7 +598,12 @@ DEFAULT_PROVIDERS = [
# LOCAL MODELS
Provider("ollama", "ollama run llama3.2",
"Ollama - local, private, free (GPU recommended)",
tags=["free", "local", "private"]),
model="llama3.2",
tags=["free", "local", "private"],
locality="local",
capabilities=["text", "structured-json"],
cost_class="free",
data_policy="private"),
# API-TYPE PROVIDERS (pay-per-token, fallback when no CLI covers model)
Provider("openrouter",
@ -343,18 +612,26 @@ DEFAULT_PROVIDERS = [
type="api",
model="openrouter/auto-beta",
api_key_env="OPENROUTER_API_KEY",
tags=["api", "per-token", "multi", "fallback"]),
tags=["api", "per-token", "multi", "fallback"],
locality="remote",
capabilities=["text", "structured-json", "reasoning"],
data_policy="public"),
Provider("deepseek-api",
"https://api.deepseek.com/v1",
"DeepSeek API - inexpensive per-token access",
type="api",
model="deepseek-chat",
api_key_env="DEEPSEEK_API_KEY",
tags=["api", "per-token", "cheap", "reasoning"]),
tags=["api", "per-token", "cheap", "reasoning"],
locality="remote",
capabilities=["text", "structured-json", "reasoning"],
data_policy="public"),
# Mock for testing
Provider("mock", "mock", "Mock provider for testing",
tags=["testing"]),
model="mock", tags=["testing"], locality="local",
capabilities=["text", "structured-json"],
data_policy="private"),
]
@ -379,10 +656,7 @@ def get_providers_file() -> Path:
for d in cli_found:
name = d["name"]
desc = d.get("description", "")
selected.append(Provider(
name=name, command=d["command"], description=desc,
tags=d.get("tags", []),
))
selected.append(_provider_from_discovery(d))
print(f" [+] {name:20s} {desc}", file=sys.stderr)
if api_found:
@ -392,11 +666,7 @@ def get_providers_file() -> Path:
desc = d.get("description", "")
model = d.get("model", "auto")
env_var = d.get("env_var", "")
selected.append(Provider(
name=name, command=d["command"], description=desc,
type="api", model=model, api_key_env=env_var,
tags=d.get("tags", []),
))
selected.append(_provider_from_discovery(d))
print(f" [+] {name:20s} ({model})", file=sys.stderr)
if ollama_found:
@ -404,10 +674,7 @@ def get_providers_file() -> Path:
for d in ollama_found[:10]:
name = d["name"]
desc = d.get("description", "")
selected.append(Provider(
name=name, command=d["command"], description=desc,
tags=d.get("tags", []),
))
selected.append(_provider_from_discovery(d))
print(f" [+] {name:30s} {d['command']}", file=sys.stderr)
if len(ollama_found) > 10:
print(f" ... and {len(ollama_found) - 10} more", file=sys.stderr)
@ -514,7 +781,17 @@ def delete_provider(name: str) -> bool:
return False
def call_provider(provider_name: str, prompt: str, timeout: int = 300, max_tokens: Optional[int] = None, _tried: Optional[set] = None) -> ProviderResult:
def call_provider(
provider_name: str,
prompt: str,
timeout: int = 300,
max_tokens: Optional[int] = None,
_tried: Optional[set] = None,
*,
execution_policy: Optional[ProviderExecutionPolicy] = None,
_requested_provider: Optional[str] = None,
_attempted_providers: Optional[List[str]] = None,
) -> ProviderResult:
"""
Call an AI provider with the given prompt.
@ -528,59 +805,110 @@ def call_provider(provider_name: str, prompt: str, timeout: int = 300, max_token
timeout: Maximum execution time in seconds
max_tokens: Optional max output tokens (appends provider-specific flag)
_tried: Internal set of already-tried providers (prevents infinite loops)
execution_policy: Optional fallback and eligibility constraints
Returns:
ProviderResult with the response text or error
"""
policy = execution_policy or ProviderExecutionPolicy()
requested_provider = _requested_provider or provider_name
attempted = _attempted_providers if _attempted_providers is not None else []
if provider_name not in attempted:
attempted.append(provider_name)
observed_identity = ("mock", None, "built-in")
# Track which providers we've tried to prevent infinite fallback loops
if _tried is None:
_tried = set()
_tried.add(provider_name)
def finish(
result: ProviderResult,
provider: Optional[Provider] = None,
) -> ProviderResult:
"""Attach facts observed by CmdForge, never values generated by the model."""
result.requested_provider = requested_provider
result.attempted_providers = list(attempted)
if result.success:
result.actual_provider = provider_name
(
result.model,
result.model_digest,
result.model_identity_source,
) = observed_identity
result.locality = (
effective_provider_locality(provider) if provider else "local"
)
result.fallback_used = provider_name != requested_provider
return result
# Handle mock provider specially
if provider_name.lower() == "mock":
return mock_provider(prompt)
return finish(mock_provider(prompt))
# Look up provider
provider = get_provider(provider_name)
if not provider:
return ProviderResult(
return finish(ProviderResult(
text="",
success=False,
error=f"Provider '{provider_name}' not found. Use 'cmdforge providers' to manage providers."
)
))
observed_identity = runtime_provider_identity(provider)
if max_tokens is not None:
try:
max_tokens = int(max_tokens)
except (TypeError, ValueError):
return ProviderResult(text="", success=False, error="max_tokens must be an integer")
return finish(ProviderResult(text="", success=False, error="max_tokens must be an integer"), provider)
if max_tokens <= 0 or max_tokens > 1_000_000:
return ProviderResult(
return finish(ProviderResult(
text="", success=False, error="max_tokens must be between 1 and 1000000"
)
), provider)
# Helper to try fallback provider(s) if available
def try_fallback(error_msg: str) -> ProviderResult:
import sys
last_error = error_msg
if not policy.allow_fallback:
return finish(
ProviderResult(text="", success=False, error=last_error), provider
)
# Walk fallback_chain first (ordered, multi-step)
if provider.fallback_chain:
for fb in provider.fallback_chain:
if fb not in _tried:
print(f"[fallback] {provider_name} failed, trying {fb}...", file=sys.stderr)
result = call_provider(fb, prompt, timeout, max_tokens, _tried)
result = call_provider(
fb, prompt, timeout, max_tokens, _tried,
execution_policy=policy,
_requested_provider=requested_provider,
_attempted_providers=attempted,
)
if result.success:
return result
last_error = result.error or last_error
# Fall back to single fallback (backward-compat)
if provider.fallback and provider.fallback not in _tried:
print(f"[fallback] {provider_name} failed, trying {provider.fallback}...", file=sys.stderr)
result = call_provider(provider.fallback, prompt, timeout, max_tokens, _tried)
result = call_provider(
provider.fallback, prompt, timeout, max_tokens, _tried,
execution_policy=policy,
_requested_provider=requested_provider,
_attempted_providers=attempted,
)
if result.success:
return result
last_error = result.error or last_error
return ProviderResult(text="", success=False, error=last_error)
return finish(
ProviderResult(text="", success=False, error=last_error), provider
)
eligibility_error = provider_policy_error(
provider, policy, observed_identity
)
if eligibility_error:
return try_fallback(eligibility_error)
# Dispatch by provider type
ptype = getattr(provider, "type", None) or "subprocess"
@ -595,7 +923,7 @@ def call_provider(provider_name: str, prompt: str, timeout: int = 300, max_token
return try_fallback(f"Unknown provider type: {ptype}")
if result.success:
return result
return finish(result, provider)
return try_fallback(result.error or "Provider call failed")
except Exception as e:
return try_fallback(f"Provider error: {str(e)}")

View File

@ -16,7 +16,7 @@ from typing import Optional, Tuple
import yaml
from .tool import Tool, TOOLS_DIR, get_bin_dir, BIN_DIR
from .tool import Tool, TOOLS_DIR, get_tools_dir, get_bin_dir, BIN_DIR
from .config import is_auto_fetch_enabled, load_config
from .manifest import load_manifest
@ -243,7 +243,7 @@ class ToolResolver:
searched_paths: list
) -> Optional[ResolvedTool]:
"""Search for tool in global user directory."""
global_dir = TOOLS_DIR
global_dir = get_tools_dir()
if not global_dir.exists():
return None

View File

@ -12,7 +12,9 @@ from typing import Optional, List
import yaml
from .tool import Tool, PromptStep, CodeStep, ToolStep, McpStep
from .providers import call_provider, mock_provider
from .providers import (
call_provider, mock_provider, ProviderExecutionPolicy,
)
from .resolver import resolve_tool, ToolNotFoundError, ToolSpec, install_from_registry
from .manifest import load_manifest
from .profiles import load_profile
@ -508,7 +510,9 @@ def execute_prompt_step(
variables: dict,
provider_override: str = None,
verbose: bool = False,
base_dir: Optional[Path] = None
base_dir: Optional[Path] = None,
execution_policy: Optional[ProviderExecutionPolicy] = None,
provenance_log: Optional[list[dict]] = None,
) -> tuple[str, bool]:
"""
Execute a prompt step.
@ -525,6 +529,24 @@ def execute_prompt_step(
Otherwise, enforces structured JSON output with schema validation.
"""
import re
from dataclasses import replace
effective_policy = execution_policy or ProviderExecutionPolicy()
if step.fallback_policy == "deny" and effective_policy.allow_fallback:
effective_policy = replace(effective_policy, allow_fallback=False)
def record_result(result, selected_provider: str) -> None:
# Mock calls bypass call_provider for deterministic tests, so attach
# the same runtime-owned identity here.
if selected_provider.lower() == "mock" and not result.actual_provider:
result.requested_provider = "mock"
result.actual_provider = "mock"
result.attempted_providers = ["mock"]
result.model = "mock"
result.locality = "local"
result.model_identity_source = "built-in"
if provenance_log is not None:
provenance_log.append(result.provenance())
# Build prompt with variable substitution
try:
@ -557,7 +579,12 @@ def execute_prompt_step(
if provider.lower() == "mock":
result = mock_provider(prompt)
else:
result = call_provider(provider, prompt, max_tokens=step.max_tokens)
result = call_provider(
provider, prompt, max_tokens=step.max_tokens,
execution_policy=effective_policy,
)
record_result(result, provider)
if not result.success:
print(f"Error in prompt step: {result.error}", file=sys.stderr)
@ -601,7 +628,12 @@ Please try again with valid JSON matching the schema exactly."""
if provider.lower() == "mock":
result = mock_provider(current_prompt)
else:
result = call_provider(provider, current_prompt, max_tokens=step.max_tokens)
result = call_provider(
provider, current_prompt, max_tokens=step.max_tokens,
execution_policy=effective_policy,
)
record_result(result, provider)
if not result.success:
print(f"Error in prompt step: {result.error}", file=sys.stderr)
@ -743,6 +775,8 @@ def execute_tool_step(
agent_profile: Optional[str] = None,
agent_skills: Optional[List[str]] = None,
agent_tool_policies: Optional[List[List[str]]] = None,
execution_policy: Optional[ProviderExecutionPolicy] = None,
provenance_log: Optional[list[dict]] = None,
) -> tuple[str, bool]:
"""
Execute a tool step by calling another tool.
@ -815,6 +849,8 @@ def execute_tool_step(
agent_tool_policies=_extend_agent_tool_policies(
agent_tool_policies, step.tools
),
execution_policy=execution_policy,
provenance_log=provenance_log,
)
return output, exit_code == 0
@ -834,6 +870,8 @@ def run_tool(
agent_profile: Optional[str] = None,
agent_skills: Optional[List[str]] = None,
agent_tool_policies: Optional[List[List[str]]] = None,
execution_policy: Optional[ProviderExecutionPolicy] = None,
provenance_log: Optional[list[dict]] = None,
) -> tuple[str, int]:
"""
Execute a tool.
@ -969,7 +1007,9 @@ def run_tool(
variables,
provider_override,
verbose=verbose,
base_dir=tool_base_dir
base_dir=tool_base_dir,
execution_policy=execution_policy,
provenance_log=provenance_log,
)
if not success:
return "", 2
@ -1018,6 +1058,8 @@ def run_tool(
agent_profile=agent_profile,
agent_skills=agent_skills,
agent_tool_policies=agent_tool_policies,
execution_policy=execution_policy,
provenance_log=provenance_log,
)
if not success:
return "", 3
@ -1133,14 +1175,65 @@ def create_argument_parser(tool: Tool) -> argparse.ArgumentParser:
help="Show debug information")
parser.add_argument("--auto-install", action="store_true",
help="Automatically install missing tool dependencies")
parser.add_argument("--no-fallback", action="store_true",
help="Fail instead of trying configured fallbacks")
parser.add_argument("--require-local", action="store_true",
help="Allow only providers declared local")
parser.add_argument("--require-capability", action="append", default=[],
help="Required provider capability (repeatable)")
parser.add_argument("--data-classification",
choices=["public", "internal", "private"])
parser.add_argument("--require-model-identity", action="store_true")
parser.add_argument("--require-model-digest", action="store_true")
parser.add_argument("--result-envelope", choices=["json"])
# Tool-specific flags from arguments
for arg in tool.arguments:
existing_action = parser._option_string_actions.get(arg.flag)
if existing_action is not None:
# Preserve the legacy and useful case where a boolean tool option
# intentionally shares a universal flag (for example --verbose).
# Both consumers then read the same parsed boolean.
if arg.type == "boolean" and existing_action.dest == arg.variable:
continue
raise ValueError(
f"Tool argument {arg.flag!r} conflicts with CmdForge's "
"universal run options"
)
def parse_boolean(value):
if isinstance(value, bool):
return value
normalized = str(value).strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
raise argparse.ArgumentTypeError(
"expected true/false, yes/no, on/off, or 1/0"
)
value_type = {
"string": str,
"integer": int,
"number": float,
"boolean": parse_boolean,
}[arg.type]
options = {
"dest": arg.variable,
"default": arg.default,
"help": arg.description or f"{arg.variable} (default: {arg.default})",
"required": arg.required,
"type": value_type,
}
if arg.enum is not None:
options["choices"] = [value_type(value) for value in arg.enum]
if arg.type == "boolean":
options["nargs"] = "?"
options["const"] = True
parser.add_argument(
arg.flag,
dest=arg.variable,
default=arg.default,
help=arg.description or f"{arg.variable} (default: {arg.default})"
**options,
)
return parser
@ -1215,6 +1308,16 @@ def main():
# Determine provider override (CLI flag takes precedence over manifest)
effective_provider = args.provider or provider_override_from_manifest
execution_policy = ProviderExecutionPolicy(
allow_fallback=not args.no_fallback,
required_locality="local" if args.require_local else None,
required_capabilities=tuple(args.require_capability or []),
data_classification=args.data_classification or "public",
require_model_identity=args.require_model_identity,
require_model_digest=args.require_model_digest,
)
provider_calls = []
# Run tool
output, exit_code = run_tool(
tool=tool,
@ -1224,15 +1327,36 @@ def main():
dry_run=args.dry_run,
show_prompt=args.show_prompt,
verbose=args.verbose,
auto_install=args.auto_install
auto_install=args.auto_install,
execution_policy=execution_policy,
provenance_log=provider_calls,
)
# Write output
if exit_code == 0 and output:
if args.output_file:
Path(args.output_file).write_text(output)
if args.result_envelope == "json" or (exit_code == 0 and output):
if args.result_envelope == "json":
import json
rendered_output = json.dumps({
"output": output,
"execution": {
"exit_code": exit_code,
"provider_calls": provider_calls,
"fallback_used": any(
item.get("fallback_used", False)
for item in provider_calls
),
"actual_providers": list(dict.fromkeys(
item["actual_provider"] for item in provider_calls
if item.get("actual_provider")
)),
},
}, indent=2)
else:
print(output)
rendered_output = output
if args.output_file:
Path(args.output_file).write_text(rendered_output)
else:
print(rendered_output)
if exit_code == 0 and not args.dry_run:
try:

View File

@ -124,6 +124,15 @@ class ToolArgument:
required: bool = False
def __post_init__(self) -> None:
# Older CmdForge tools used Python-style scalar names. Normalize them
# on load so typed parsing does not break existing user tools.
legacy_type_aliases = {
"str": "string",
"int": "integer",
"float": "number",
"bool": "boolean",
}
self.type = legacy_type_aliases.get(self.type, self.type)
if self.type not in ("string", "integer", "number", "boolean"):
raise ValueError(
"ToolArgument type must be string, integer, number, or boolean"
@ -178,9 +187,12 @@ class PromptStep:
plain_text: bool = False # Bypass structured output enforcement
max_tokens: Optional[int] = None # Max output tokens (provider-dependent)
skills: Optional[List[str]] = None # Skill names to enable for this step
fallback_policy: str = "allow" # "allow" | "deny"
def __post_init__(self) -> None:
_validate_skill_selection(self.skills)
if self.fallback_policy not in {"allow", "deny"}:
raise ValueError("PromptStep fallback_policy must be allow or deny")
def to_dict(self) -> dict:
d = {
@ -207,6 +219,8 @@ class PromptStep:
d["max_tokens"] = self.max_tokens
if self.skills is not None:
d["skills"] = self.skills
if self.fallback_policy != "allow":
d["fallback_policy"] = self.fallback_policy
return d
@classmethod
@ -233,6 +247,7 @@ class PromptStep:
plain_text=data.get("plain_text", False),
max_tokens=max_tokens,
skills=data.get("skills"),
fallback_policy=data.get("fallback_policy", "allow"),
)
@ -598,6 +613,11 @@ def get_tools_dir() -> Path:
return TOOLS_DIR
def get_project_tools_dir(project_dir: Optional[Path] = None) -> Path:
"""Return the canonical project-local tool root without creating it."""
return (project_dir or Path.cwd()).resolve() / ".cmdforge"
def ensure_settings(tool_dir: Path) -> Optional[Path]:
"""Ensure settings.yaml exists if defaults.yaml exists.
@ -747,12 +767,32 @@ def load_tool(name: str) -> Optional[Tool]:
_REGISTRY_FIELDS = ("registry_hash", "registry_status", "registry_owner", "registry_feedback")
def save_tool(tool: Tool) -> Path:
"""Save a tool to disk, preserving registry metadata from existing config."""
tool_dir = get_tools_dir() / tool.name
def save_tool(
tool: Tool,
*,
tools_dir: Optional[Path] = None,
create_wrapper: Optional[bool] = None,
) -> Path:
"""Save a tool globally or under an explicit project-local tool root.
Global saves keep the historical wrapper behavior. Project-local saves are
invoked through ``cmdforge run`` so a global wrapper cannot accidentally
resolve a different project tool with the same name.
"""
valid, error = validate_tool_name(tool.name)
if not valid:
raise ValueError(error)
global_tools_dir = get_tools_dir()
target_root = Path(tools_dir).resolve() if tools_dir is not None else global_tools_dir
target_root.mkdir(parents=True, exist_ok=True)
tool_dir = target_root / tool.name
if tool_dir.is_symlink():
raise ValueError(f"Refusing to save through symlinked tool directory: {tool_dir}")
tool_dir.mkdir(parents=True, exist_ok=True)
config_path = tool_dir / "config.yaml"
if config_path.is_symlink():
raise ValueError(f"Refusing to overwrite symlinked tool config: {config_path}")
# Preserve registry fields from existing config (not part of Tool model)
preserved = {}
@ -769,7 +809,11 @@ def save_tool(tool: Tool) -> Path:
new_data.update(preserved)
config_path.write_text(yaml.dump(new_data, default_flow_style=False, sort_keys=False))
# Create wrapper script
# Global tools retain convenient shell wrappers. Project tools are
# intentionally cwd-scoped and must be resolved by ``cmdforge run``.
if create_wrapper is None:
create_wrapper = target_root == global_tools_dir.resolve()
if create_wrapper:
create_wrapper_script(tool.name)
# Ensure settings.yaml exists if defaults.yaml exists

View File

@ -157,6 +157,33 @@ class TestListCommand:
assert main() == 1
assert "--full requires --json" in capsys.readouterr().err
def test_list_includes_project_tools_and_prefers_project_shadow(
self, temp_tools_dir, tmp_path, monkeypatch, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(name="shadowed", description="global"))
project = tmp_path / "project"
project.mkdir()
save_tool(
Tool(name="shadowed", description="project"),
tools_dir=project / ".cmdforge",
)
save_tool(
Tool(name="project-only", description="local"),
tools_dir=project / ".cmdforge",
)
monkeypatch.chdir(project)
with patch("sys.argv", ["cmdforge", "list", "--json"]):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
by_name = {item["name"]: item for item in payload}
assert by_name["shadowed"]["description"] == "project"
assert "project-only" in by_name
assert [item["name"] for item in payload].count("shadowed") == 1
class TestRunOnceCommand:
def test_combines_instruction_with_piped_input(self, capsys):
@ -243,6 +270,29 @@ class TestRunOnceCommand:
call_provider.assert_not_called()
assert message in capsys.readouterr().err
def test_json_envelope_contains_runtime_provenance(self, capsys):
from cmdforge.providers import ProviderResult
result = ProviderResult(
text="answer", success=True,
requested_provider="local", actual_provider="local",
attempted_providers=["local"], locality="local",
)
with (
patch("sys.argv", [
"cmdforge", "run-once", "hello", "--provider", "local",
"--no-fallback", "--result-envelope", "json",
]),
patch("sys.stdin", StringIO()),
patch("cmdforge.providers.call_provider", return_value=result),
):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
call = payload["execution"]["provider_calls"][0]
assert call["actual_provider"] == "local"
assert call["fallback_used"] is False
class TestCreateCommand:
"""Tests for 'cmdforge create' command."""
@ -296,6 +346,36 @@ class TestCreateCommand:
captured = capsys.readouterr()
assert 'invalid' in captured.out.lower() or 'invalid' in captured.err.lower()
def test_create_project_tool_does_not_create_global_wrapper(
self, temp_tools_dir, tmp_path, monkeypatch, capsys
):
project = tmp_path / "project"
project.mkdir()
monkeypatch.chdir(project)
with patch("sys.argv", [
"cmdforge", "create", "project-echo", "--project",
"--prompt", "Echo {input}", "--provider", "mock",
]):
assert main() == 0
config = project / ".cmdforge" / "project-echo" / "config.yaml"
assert config.exists()
assert not (tmp_path / ".cmdforge" / "project-echo" / "config.yaml").exists()
assert not (tmp_path / ".local" / "bin" / "project-echo").exists()
assert "cmdforge run project-echo" in capsys.readouterr().out
def test_create_output_dir_uses_explicit_tool_root(
self, temp_tools_dir, tmp_path
):
target = tmp_path / "portable-tools"
with patch("sys.argv", [
"cmdforge", "create", "portable", "--output-dir", str(target),
]):
assert main() == 0
assert (target / "portable" / "config.yaml").exists()
class TestDeleteCommand:
"""Tests for 'cmdforge delete' command."""
@ -405,6 +485,91 @@ class TestRunCommand:
assert result != 0
def test_run_prefers_project_local_tool(
self, temp_tools_dir, tmp_path, monkeypatch, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(name="shadowed", output="global"))
project = tmp_path / "project"
project.mkdir()
save_tool(
Tool(name="shadowed", output="project"),
tools_dir=project / ".cmdforge",
)
monkeypatch.chdir(project)
with (
patch("sys.argv", ["cmdforge", "run", "shadowed"]),
patch("sys.stdin", StringIO("")),
patch("sys.stdin.isatty", return_value=True),
):
assert main() == 0
assert capsys.readouterr().out.strip() == "project"
def test_run_json_envelope_preserves_output_and_provenance(
self, temp_tools_dir, capsys
):
from cmdforge.tool import save_tool
save_tool(Tool(
name="audited",
steps=[PromptStep(
prompt="Echo {input}", provider="mock",
output_var="answer", plain_text=True,
)],
output="{answer}",
))
with (
patch("sys.argv", [
"cmdforge", "run", "audited", "--result-envelope", "json",
]),
patch("sys.stdin", StringIO("hello")),
patch("sys.stdin.isatty", return_value=False),
):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
assert "MOCK" in payload["output"]
assert payload["execution"]["actual_providers"] == ["mock"]
assert payload["execution"]["provider_calls"][0]["locality"] == "local"
def test_failed_run_still_emits_audit_envelope(
self, temp_tools_dir, capsys
):
from cmdforge.providers import ProviderResult
from cmdforge.tool import save_tool
save_tool(Tool(
name="strict",
steps=[PromptStep(
prompt="Private", provider="primary",
output_var="answer", plain_text=True,
)],
output="{answer}",
))
failure = ProviderResult(
text="", success=False, error="offline",
requested_provider="primary", attempted_providers=["primary"],
)
with (
patch("sys.argv", [
"cmdforge", "run", "strict", "--no-fallback",
"--result-envelope", "json",
]),
patch("sys.stdin", StringIO("private")),
patch("sys.stdin.isatty", return_value=False),
patch("cmdforge.runner.call_provider", return_value=failure),
):
assert main() == 2
payload = json.loads(capsys.readouterr().out)
assert payload["output"] == ""
assert payload["execution"]["exit_code"] == 2
assert payload["execution"]["provider_calls"][0]["error"] == "offline"
class TestTestCommand:
"""Tests for 'cmdforge test' command."""
@ -458,6 +623,29 @@ class TestProvidersCommand:
# Should show some default providers
assert 'mock' in captured.out.lower() or 'claude' in captured.out.lower()
def test_providers_list_json_exposes_routing_metadata(
self, temp_providers_file, capsys
):
from cmdforge.providers import Provider, save_providers
save_providers([Provider(
"safe", "safe-cli", locality="local",
capabilities=["structured-json"], data_policy="private",
max_context_tokens=8192,
)])
with (
patch("sys.argv", ["cmdforge", "providers", "list", "--json"]),
patch("cmdforge.cli.provider_commands._provider_status", return_value=(True, "Ready")),
):
assert main() == 0
payload = json.loads(capsys.readouterr().out)
assert payload[0]["locality"] == "local"
assert payload[0]["capabilities"] == ["structured-json"]
assert payload[0]["data_policy"] == "private"
assert payload[0]["max_context_tokens"] == 8192
assert payload[0]["available"] is True
def test_providers_add(self, temp_providers_file, capsys):
"""Add a custom provider."""
with patch('sys.argv', ['cmdforge', 'providers', 'add',
@ -479,6 +667,13 @@ class TestProvidersCommand:
'--model', 'example/model',
'--api-key-env', 'CUSTOM_API_KEY',
'--tag', 'api',
'--locality', 'remote',
'--capability', 'structured-json',
'--model-digest', 'sha256:abc',
'--cost-class', 'low',
'--latency-class', 'fast',
'--max-context-tokens', '128000',
'--data-policy', 'internal',
'--fallback-chain', 'free',
]):
result = main()
@ -491,6 +686,13 @@ class TestProvidersCommand:
assert provider.api_key_env == 'CUSTOM_API_KEY'
assert provider.tags == ['api']
assert provider.fallback_chain == PRESET_CHAINS['free']
assert provider.locality == 'remote'
assert provider.capabilities == ['structured-json']
assert provider.model_digest == 'sha256:abc'
assert provider.cost_class == 'low'
assert provider.latency_class == 'fast'
assert provider.max_context_tokens == 128000
assert provider.data_policy == 'internal'
def test_providers_list_reports_missing_api_key(self, temp_providers_file, capsys):
from cmdforge.providers import Provider, save_providers
@ -828,6 +1030,23 @@ def test_switch_to_existing_tool_closes_creation_page_first():
]
def test_tool_builder_save_scope_prefers_recognized_projects(tmp_path):
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import default_save_scope
assert default_save_scope(tmp_path) == "global"
(tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n")
assert default_save_scope(tmp_path) == "project"
def test_tool_builder_existing_tool_root_does_not_nest_tool_name(tmp_path):
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import existing_tool_root
config = tmp_path / ".cmdforge" / "demo" / "config.yaml"
assert existing_tool_root(config) == (tmp_path / ".cmdforge").resolve()
def test_guided_extraction_builds_new_tool_without_mutating_draft():
pytest.importorskip("PySide6")
from cmdforge.gui.pages.tool_builder_page import (

View File

@ -8,7 +8,7 @@ import pytest
import yaml
from cmdforge.providers import (
Provider, ProviderResult,
Provider, ProviderResult, ProviderExecutionPolicy,
load_providers, save_providers, get_provider,
add_provider, delete_provider,
call_provider, discover_installed_providers, mock_provider,
@ -60,6 +60,19 @@ class TestProvider:
provider = Provider.from_dict(data)
assert provider.description == ""
def test_legacy_policy_field_aliases_are_canonicalized(self):
provider = Provider.from_dict({
"name": "legacy-policy",
"command": "provider",
"model_id": "model-a",
"max_input_tokens": 8192,
})
assert provider.model == "model-a"
assert provider.max_context_tokens == 8192
assert provider.to_dict()["model"] == "model-a"
assert provider.to_dict()["max_context_tokens"] == 8192
def test_roundtrip(self):
original = Provider(
name="custom",
@ -90,6 +103,13 @@ class TestProvider:
fallback_chain=["backup", "mock"],
api_key_env="TEST_API_KEY",
pty_config={"prompt_pattern": ">"},
locality="remote",
capabilities=["text", "structured-json"],
model_digest="sha256:abc",
cost_class="low",
latency_class="fast",
max_context_tokens=131072,
data_policy="internal",
)
assert Provider.from_dict(original.to_dict()) == original
@ -516,6 +536,200 @@ class TestProviderFallback:
# Fallback to mock should succeed
assert result.success is True
assert "[MOCK]" in result.text
assert result.requested_provider == "primary"
assert result.actual_provider == "mock"
assert result.attempted_providers == ["primary", "mock"]
assert result.fallback_used is True
def test_no_fallback_fails_closed(self, temp_providers_file):
save_providers([
Provider("primary", "nonexistent-cmd", fallback="mock"),
Provider("mock", "mock"),
])
result = call_provider(
"primary", "Test prompt",
execution_policy=ProviderExecutionPolicy(allow_fallback=False),
)
assert result.success is False
assert result.requested_provider == "primary"
assert result.actual_provider is None
assert result.attempted_providers == ["primary"]
assert result.fallback_used is False
def test_private_data_requires_explicit_compatible_policy(
self, temp_providers_file
):
save_providers([
Provider(
"local", "unused", locality="local",
capabilities=["structured-json"],
),
])
result = call_provider(
"local", "private packet",
execution_policy=ProviderExecutionPolicy(
allow_fallback=False,
required_locality="local",
required_capabilities=("structured-json",),
data_classification="private",
),
)
assert result.success is False
assert "data_policy is unspecified" in result.error
def test_require_local_rejects_ollama_redirected_to_remote_host(
self, temp_providers_file, monkeypatch
):
save_providers([
Provider(
"ollama-private", "ollama run safe", locality="local",
data_policy="private",
),
])
monkeypatch.setenv("OLLAMA_HOST", "gpu.example.test:11434")
result = call_provider(
"ollama-private", "private packet",
execution_policy=ProviderExecutionPolicy(
allow_fallback=False,
required_locality="local",
data_classification="private",
),
)
assert result.success is False
assert "locality is remote" in result.error
def test_ollama_digest_is_observed_from_local_api(
self, temp_providers_file, monkeypatch
):
save_providers([
Provider(
"ollama-safe", "ollama run safe:latest", model="safe:latest",
locality="local", data_policy="private",
),
])
monkeypatch.delenv("OLLAMA_HOST", raising=False)
class Response:
def __enter__(self):
return self
def __exit__(self, *args):
return False
def read(self):
return (
b'{"models":[{"name":"safe:latest","model":"safe:latest",'
b'"digest":"abcdef"}]}'
)
with (
patch("urllib.request.urlopen", return_value=Response()),
patch(
"cmdforge.providers.call_provider_subprocess",
return_value=ProviderResult(text="safe", success=True),
),
):
result = call_provider(
"ollama-safe", "private",
execution_policy=ProviderExecutionPolicy(
allow_fallback=False,
required_locality="local",
data_classification="private",
require_model_identity=True,
require_model_digest=True,
),
)
assert result.model == "safe:latest"
assert result.model_digest == "sha256:abcdef"
assert result.model_identity_source == "ollama-api"
def test_policy_eligible_provider_reports_verified_metadata(
self, temp_providers_file
):
provider = Provider(
"local", "local-ai", model="model-a",
model_digest="sha256:123", locality="local",
capabilities=["structured-json"], data_policy="private",
)
save_providers([provider])
low_level = ProviderResult(text='{"ok": true}', success=True)
with patch(
"cmdforge.providers.call_provider_subprocess",
return_value=low_level,
):
result = call_provider(
"local", "private packet",
execution_policy=ProviderExecutionPolicy(
allow_fallback=False,
required_locality="local",
required_capabilities=("structured-json",),
data_classification="private",
require_model_identity=True,
require_model_digest=True,
),
)
assert result.success is True
assert result.provenance() == {
"success": True,
"requested_provider": "local",
"actual_provider": "local",
"attempted_providers": ["local"],
"fallback_used": False,
"model": "model-a",
"model_digest": "sha256:123",
"locality": "local",
"model_identity_source": "provider-config",
}
def test_fallback_chain_skips_policy_ineligible_provider(
self, temp_providers_file
):
save_providers([
Provider(
"primary", "primary", fallback_chain=["remote", "local"],
locality="local", data_policy="private",
),
Provider(
"remote", "remote", locality="remote", data_policy="private",
),
Provider(
"local", "local", locality="local", data_policy="private",
model="safe-model",
),
])
def invoke(provider, prompt, timeout, max_tokens):
if provider.name == "primary":
return ProviderResult(text="", success=False, error="offline")
assert provider.name == "local"
return ProviderResult(text="safe", success=True)
with patch(
"cmdforge.providers.call_provider_subprocess", side_effect=invoke
) as low_level:
result = call_provider(
"primary", "private packet",
execution_policy=ProviderExecutionPolicy(
required_locality="local",
data_classification="private",
),
)
assert result.success is True
assert result.actual_provider == "local"
assert result.attempted_providers == ["primary", "remote", "local"]
assert [call.args[0].name for call in low_level.call_args_list] == [
"primary", "local"
]
def test_fallback_chain(self, temp_providers_file):
"""Fallback can chain to another provider with fallback."""

View File

@ -229,6 +229,23 @@ class TestExecutePromptStep:
assert success is False
assert output == ""
@patch('cmdforge.runner.call_provider')
def test_step_can_deny_fallback_independently_of_caller(self, mock_call):
mock_call.return_value = ProviderResult(
text="private", success=True,
)
step = PromptStep(
prompt="Private", provider="local", output_var="out",
plain_text=True, fallback_policy="deny",
)
output, success = execute_prompt_step(step, {"input": ""})
assert success is True
assert output == "private"
policy = mock_call.call_args.kwargs["execution_policy"]
assert policy.allow_fallback is False
@patch('cmdforge.runner.mock_provider')
def test_mock_provider_used(self, mock_mock):
mock_mock.return_value = ProviderResult(text="mock response", success=True)
@ -988,6 +1005,58 @@ class TestCreateArgumentParser:
assert args.count == "10"
def test_parser_enforces_typed_boolean_integer_and_enum_arguments(self):
tool = Tool(
name="typed",
arguments=[
ToolArgument(
flag="--project", variable="project",
type="boolean", default=False,
),
ToolArgument(
flag="--count", variable="count",
type="integer", required=True,
),
ToolArgument(
flag="--mode", variable="mode",
enum=["safe", "fast"], default="safe",
),
],
)
parser = create_argument_parser(tool)
args = parser.parse_args(["--project", "--count", "3", "--mode", "fast"])
assert args.project is True
assert args.count == 3
assert args.mode == "fast"
def test_parser_reuses_matching_legacy_boolean_universal_flag(self):
tool = Tool(
name="legacy",
arguments=[
ToolArgument(
flag="--verbose", variable="verbose", type="bool"
)
],
)
args = create_argument_parser(tool).parse_args(["--verbose"])
assert args.verbose is True
assert collect_custom_args(tool, args) == {"verbose": True}
def test_parser_rejects_incompatible_universal_flag_collision(self):
tool = Tool(
name="conflict",
arguments=[
ToolArgument(flag="--provider", variable="destination")
],
)
with pytest.raises(ValueError, match="universal run options"):
create_argument_parser(tool)
def test_parser_input_output_flags(self):
tool = Tool(name="test")
parser = create_argument_parser(tool)

View File

@ -378,7 +378,7 @@ class TestRunnerSkillIntegration:
):
captured = {}
def fake_call(provider, prompt, max_tokens=None):
def fake_call(provider, prompt, max_tokens=None, **kwargs):
captured.update(provider=provider, prompt=prompt)
return ProviderResult(text="done", success=True)
@ -412,7 +412,7 @@ class TestRunnerSkillIntegration:
):
captured = {}
def fake_call(provider, prompt, max_tokens=None):
def fake_call(provider, prompt, max_tokens=None, **kwargs):
captured.update(provider=provider, prompt=prompt)
return ProviderResult(text='{"output": "done"}', success=True)

View File

@ -105,6 +105,25 @@ class TestToolArgument:
with pytest.raises(ValueError):
ToolArgument(flag="--value", variable="value", **values)
@pytest.mark.parametrize(
("legacy_type", "canonical_type"),
[
("str", "string"),
("int", "integer"),
("float", "number"),
("bool", "boolean"),
],
)
def test_legacy_scalar_type_aliases_are_normalized(
self, legacy_type, canonical_type
):
argument = ToolArgument(
flag="--value", variable="value", type=legacy_type
)
assert argument.type == canonical_type
assert argument.to_dict().get("type", "string") == canonical_type
class TestPromptStep:
"""Tests for PromptStep dataclass."""
@ -686,6 +705,23 @@ class TestAgentContext:
restored = PromptStep.from_dict(step.to_dict())
assert restored.skills == ["python"]
def test_promptstep_deny_fallback_roundtrip(self):
step = PromptStep(
prompt="Private", provider="local", output_var="out",
fallback_policy="deny",
)
restored = PromptStep.from_dict(step.to_dict())
assert restored.fallback_policy == "deny"
def test_promptstep_rejects_invalid_fallback_policy(self):
with pytest.raises(ValueError, match="fallback_policy"):
PromptStep(
prompt="Test", provider="local", output_var="out",
fallback_policy="sometimes",
)
class TestDeprecationFields:
def test_round_trip_and_warning(self):