Add project tools and strict provider execution

tests/test_cli.py

@@ -157,6 +157,33 @@ class TestListCommand:
             assert main() == 1
         assert "--full requires --json" in capsys.readouterr().err
 
+    def test_list_includes_project_tools_and_prefers_project_shadow(
+        self, temp_tools_dir, tmp_path, monkeypatch, capsys
+    ):
+        from cmdforge.tool import save_tool
+
+        save_tool(Tool(name="shadowed", description="global"))
+        project = tmp_path / "project"
+        project.mkdir()
+        save_tool(
+            Tool(name="shadowed", description="project"),
+            tools_dir=project / ".cmdforge",
+        )
+        save_tool(
+            Tool(name="project-only", description="local"),
+            tools_dir=project / ".cmdforge",
+        )
+        monkeypatch.chdir(project)
+
+        with patch("sys.argv", ["cmdforge", "list", "--json"]):
+            assert main() == 0
+
+        payload = json.loads(capsys.readouterr().out)
+        by_name = {item["name"]: item for item in payload}
+        assert by_name["shadowed"]["description"] == "project"
+        assert "project-only" in by_name
+        assert [item["name"] for item in payload].count("shadowed") == 1
+
 
 class TestRunOnceCommand:
     def test_combines_instruction_with_piped_input(self, capsys):
@@ -243,6 +270,29 @@ class TestRunOnceCommand:
         call_provider.assert_not_called()
         assert message in capsys.readouterr().err
 
+    def test_json_envelope_contains_runtime_provenance(self, capsys):
+        from cmdforge.providers import ProviderResult
+
+        result = ProviderResult(
+            text="answer", success=True,
+            requested_provider="local", actual_provider="local",
+            attempted_providers=["local"], locality="local",
+        )
+        with (
+            patch("sys.argv", [
+                "cmdforge", "run-once", "hello", "--provider", "local",
+                "--no-fallback", "--result-envelope", "json",
+            ]),
+            patch("sys.stdin", StringIO()),
+            patch("cmdforge.providers.call_provider", return_value=result),
+        ):
+            assert main() == 0
+
+        payload = json.loads(capsys.readouterr().out)
+        call = payload["execution"]["provider_calls"][0]
+        assert call["actual_provider"] == "local"
+        assert call["fallback_used"] is False
+
 
 class TestCreateCommand:
     """Tests for 'cmdforge create' command."""
@@ -296,6 +346,36 @@ class TestCreateCommand:
         captured = capsys.readouterr()
         assert 'invalid' in captured.out.lower() or 'invalid' in captured.err.lower()
 
+    def test_create_project_tool_does_not_create_global_wrapper(
+        self, temp_tools_dir, tmp_path, monkeypatch, capsys
+    ):
+        project = tmp_path / "project"
+        project.mkdir()
+        monkeypatch.chdir(project)
+
+        with patch("sys.argv", [
+            "cmdforge", "create", "project-echo", "--project",
+            "--prompt", "Echo {input}", "--provider", "mock",
+        ]):
+            assert main() == 0
+
+        config = project / ".cmdforge" / "project-echo" / "config.yaml"
+        assert config.exists()
+        assert not (tmp_path / ".cmdforge" / "project-echo" / "config.yaml").exists()
+        assert not (tmp_path / ".local" / "bin" / "project-echo").exists()
+        assert "cmdforge run project-echo" in capsys.readouterr().out
+
+    def test_create_output_dir_uses_explicit_tool_root(
+        self, temp_tools_dir, tmp_path
+    ):
+        target = tmp_path / "portable-tools"
+        with patch("sys.argv", [
+            "cmdforge", "create", "portable", "--output-dir", str(target),
+        ]):
+            assert main() == 0
+
+        assert (target / "portable" / "config.yaml").exists()
+
 
 class TestDeleteCommand:
     """Tests for 'cmdforge delete' command."""
@@ -405,6 +485,91 @@ class TestRunCommand:
 
         assert result != 0
 
+    def test_run_prefers_project_local_tool(
+        self, temp_tools_dir, tmp_path, monkeypatch, capsys
+    ):
+        from cmdforge.tool import save_tool
+
+        save_tool(Tool(name="shadowed", output="global"))
+        project = tmp_path / "project"
+        project.mkdir()
+        save_tool(
+            Tool(name="shadowed", output="project"),
+            tools_dir=project / ".cmdforge",
+        )
+        monkeypatch.chdir(project)
+
+        with (
+            patch("sys.argv", ["cmdforge", "run", "shadowed"]),
+            patch("sys.stdin", StringIO("")),
+            patch("sys.stdin.isatty", return_value=True),
+        ):
+            assert main() == 0
+
+        assert capsys.readouterr().out.strip() == "project"
+
+    def test_run_json_envelope_preserves_output_and_provenance(
+        self, temp_tools_dir, capsys
+    ):
+        from cmdforge.tool import save_tool
+
+        save_tool(Tool(
+            name="audited",
+            steps=[PromptStep(
+                prompt="Echo {input}", provider="mock",
+                output_var="answer", plain_text=True,
+            )],
+            output="{answer}",
+        ))
+
+        with (
+            patch("sys.argv", [
+                "cmdforge", "run", "audited", "--result-envelope", "json",
+            ]),
+            patch("sys.stdin", StringIO("hello")),
+            patch("sys.stdin.isatty", return_value=False),
+        ):
+            assert main() == 0
+
+        payload = json.loads(capsys.readouterr().out)
+        assert "MOCK" in payload["output"]
+        assert payload["execution"]["actual_providers"] == ["mock"]
+        assert payload["execution"]["provider_calls"][0]["locality"] == "local"
+
+    def test_failed_run_still_emits_audit_envelope(
+        self, temp_tools_dir, capsys
+    ):
+        from cmdforge.providers import ProviderResult
+        from cmdforge.tool import save_tool
+
+        save_tool(Tool(
+            name="strict",
+            steps=[PromptStep(
+                prompt="Private", provider="primary",
+                output_var="answer", plain_text=True,
+            )],
+            output="{answer}",
+        ))
+        failure = ProviderResult(
+            text="", success=False, error="offline",
+            requested_provider="primary", attempted_providers=["primary"],
+        )
+        with (
+            patch("sys.argv", [
+                "cmdforge", "run", "strict", "--no-fallback",
+                "--result-envelope", "json",
+            ]),
+            patch("sys.stdin", StringIO("private")),
+            patch("sys.stdin.isatty", return_value=False),
+            patch("cmdforge.runner.call_provider", return_value=failure),
+        ):
+            assert main() == 2
+
+        payload = json.loads(capsys.readouterr().out)
+        assert payload["output"] == ""
+        assert payload["execution"]["exit_code"] == 2
+        assert payload["execution"]["provider_calls"][0]["error"] == "offline"
+
 
 class TestTestCommand:
     """Tests for 'cmdforge test' command."""
@@ -458,6 +623,29 @@ class TestProvidersCommand:
         # Should show some default providers
         assert 'mock' in captured.out.lower() or 'claude' in captured.out.lower()
 
+    def test_providers_list_json_exposes_routing_metadata(
+        self, temp_providers_file, capsys
+    ):
+        from cmdforge.providers import Provider, save_providers
+
+        save_providers([Provider(
+            "safe", "safe-cli", locality="local",
+            capabilities=["structured-json"], data_policy="private",
+            max_context_tokens=8192,
+        )])
+        with (
+            patch("sys.argv", ["cmdforge", "providers", "list", "--json"]),
+            patch("cmdforge.cli.provider_commands._provider_status", return_value=(True, "Ready")),
+        ):
+            assert main() == 0
+
+        payload = json.loads(capsys.readouterr().out)
+        assert payload[0]["locality"] == "local"
+        assert payload[0]["capabilities"] == ["structured-json"]
+        assert payload[0]["data_policy"] == "private"
+        assert payload[0]["max_context_tokens"] == 8192
+        assert payload[0]["available"] is True
+
     def test_providers_add(self, temp_providers_file, capsys):
         """Add a custom provider."""
         with patch('sys.argv', ['cmdforge', 'providers', 'add',
@@ -479,6 +667,13 @@ class TestProvidersCommand:
             '--model', 'example/model',
             '--api-key-env', 'CUSTOM_API_KEY',
             '--tag', 'api',
+            '--locality', 'remote',
+            '--capability', 'structured-json',
+            '--model-digest', 'sha256:abc',
+            '--cost-class', 'low',
+            '--latency-class', 'fast',
+            '--max-context-tokens', '128000',
+            '--data-policy', 'internal',
             '--fallback-chain', 'free',
         ]):
             result = main()
@@ -491,6 +686,13 @@ class TestProvidersCommand:
         assert provider.api_key_env == 'CUSTOM_API_KEY'
         assert provider.tags == ['api']
         assert provider.fallback_chain == PRESET_CHAINS['free']
+        assert provider.locality == 'remote'
+        assert provider.capabilities == ['structured-json']
+        assert provider.model_digest == 'sha256:abc'
+        assert provider.cost_class == 'low'
+        assert provider.latency_class == 'fast'
+        assert provider.max_context_tokens == 128000
+        assert provider.data_policy == 'internal'
 
     def test_providers_list_reports_missing_api_key(self, temp_providers_file, capsys):
         from cmdforge.providers import Provider, save_providers
@@ -828,6 +1030,23 @@ def test_switch_to_existing_tool_closes_creation_page_first():
     ]
 
 
+def test_tool_builder_save_scope_prefers_recognized_projects(tmp_path):
+    pytest.importorskip("PySide6")
+    from cmdforge.gui.pages.tool_builder_page import default_save_scope
+
+    assert default_save_scope(tmp_path) == "global"
+    (tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n")
+    assert default_save_scope(tmp_path) == "project"
+
+
+def test_tool_builder_existing_tool_root_does_not_nest_tool_name(tmp_path):
+    pytest.importorskip("PySide6")
+    from cmdforge.gui.pages.tool_builder_page import existing_tool_root
+
+    config = tmp_path / ".cmdforge" / "demo" / "config.yaml"
+    assert existing_tool_root(config) == (tmp_path / ".cmdforge").resolve()
+
+
 def test_guided_extraction_builds_new_tool_without_mutating_draft():
     pytest.importorskip("PySide6")
     from cmdforge.gui.pages.tool_builder_page import (

tests/test_providers.py

@@ -8,7 +8,7 @@ import pytest
 import yaml
 
 from cmdforge.providers import (
-    Provider, ProviderResult,
+    Provider, ProviderResult, ProviderExecutionPolicy,
     load_providers, save_providers, get_provider,
     add_provider, delete_provider,
     call_provider, discover_installed_providers, mock_provider,
@@ -60,6 +60,19 @@ class TestProvider:
         provider = Provider.from_dict(data)
         assert provider.description == ""
 
+    def test_legacy_policy_field_aliases_are_canonicalized(self):
+        provider = Provider.from_dict({
+            "name": "legacy-policy",
+            "command": "provider",
+            "model_id": "model-a",
+            "max_input_tokens": 8192,
+        })
+
+        assert provider.model == "model-a"
+        assert provider.max_context_tokens == 8192
+        assert provider.to_dict()["model"] == "model-a"
+        assert provider.to_dict()["max_context_tokens"] == 8192
+
     def test_roundtrip(self):
         original = Provider(
             name="custom",
@@ -90,6 +103,13 @@ class TestProvider:
             fallback_chain=["backup", "mock"],
             api_key_env="TEST_API_KEY",
             pty_config={"prompt_pattern": ">"},
+            locality="remote",
+            capabilities=["text", "structured-json"],
+            model_digest="sha256:abc",
+            cost_class="low",
+            latency_class="fast",
+            max_context_tokens=131072,
+            data_policy="internal",
         )
 
         assert Provider.from_dict(original.to_dict()) == original
@@ -516,6 +536,200 @@ class TestProviderFallback:
         # Fallback to mock should succeed
         assert result.success is True
         assert "[MOCK]" in result.text
+        assert result.requested_provider == "primary"
+        assert result.actual_provider == "mock"
+        assert result.attempted_providers == ["primary", "mock"]
+        assert result.fallback_used is True
+
+    def test_no_fallback_fails_closed(self, temp_providers_file):
+        save_providers([
+            Provider("primary", "nonexistent-cmd", fallback="mock"),
+            Provider("mock", "mock"),
+        ])
+
+        result = call_provider(
+            "primary", "Test prompt",
+            execution_policy=ProviderExecutionPolicy(allow_fallback=False),
+        )
+
+        assert result.success is False
+        assert result.requested_provider == "primary"
+        assert result.actual_provider is None
+        assert result.attempted_providers == ["primary"]
+        assert result.fallback_used is False
+
+    def test_private_data_requires_explicit_compatible_policy(
+        self, temp_providers_file
+    ):
+        save_providers([
+            Provider(
+                "local", "unused", locality="local",
+                capabilities=["structured-json"],
+            ),
+        ])
+
+        result = call_provider(
+            "local", "private packet",
+            execution_policy=ProviderExecutionPolicy(
+                allow_fallback=False,
+                required_locality="local",
+                required_capabilities=("structured-json",),
+                data_classification="private",
+            ),
+        )
+
+        assert result.success is False
+        assert "data_policy is unspecified" in result.error
+
+    def test_require_local_rejects_ollama_redirected_to_remote_host(
+        self, temp_providers_file, monkeypatch
+    ):
+        save_providers([
+            Provider(
+                "ollama-private", "ollama run safe", locality="local",
+                data_policy="private",
+            ),
+        ])
+        monkeypatch.setenv("OLLAMA_HOST", "gpu.example.test:11434")
+
+        result = call_provider(
+            "ollama-private", "private packet",
+            execution_policy=ProviderExecutionPolicy(
+                allow_fallback=False,
+                required_locality="local",
+                data_classification="private",
+            ),
+        )
+
+        assert result.success is False
+        assert "locality is remote" in result.error
+
+    def test_ollama_digest_is_observed_from_local_api(
+        self, temp_providers_file, monkeypatch
+    ):
+        save_providers([
+            Provider(
+                "ollama-safe", "ollama run safe:latest", model="safe:latest",
+                locality="local", data_policy="private",
+            ),
+        ])
+        monkeypatch.delenv("OLLAMA_HOST", raising=False)
+
+        class Response:
+            def __enter__(self):
+                return self
+
+            def __exit__(self, *args):
+                return False
+
+            def read(self):
+                return (
+                    b'{"models":[{"name":"safe:latest","model":"safe:latest",'
+                    b'"digest":"abcdef"}]}'
+                )
+
+        with (
+            patch("urllib.request.urlopen", return_value=Response()),
+            patch(
+                "cmdforge.providers.call_provider_subprocess",
+                return_value=ProviderResult(text="safe", success=True),
+            ),
+        ):
+            result = call_provider(
+                "ollama-safe", "private",
+                execution_policy=ProviderExecutionPolicy(
+                    allow_fallback=False,
+                    required_locality="local",
+                    data_classification="private",
+                    require_model_identity=True,
+                    require_model_digest=True,
+                ),
+            )
+
+        assert result.model == "safe:latest"
+        assert result.model_digest == "sha256:abcdef"
+        assert result.model_identity_source == "ollama-api"
+
+    def test_policy_eligible_provider_reports_verified_metadata(
+        self, temp_providers_file
+    ):
+        provider = Provider(
+            "local", "local-ai", model="model-a",
+            model_digest="sha256:123", locality="local",
+            capabilities=["structured-json"], data_policy="private",
+        )
+        save_providers([provider])
+        low_level = ProviderResult(text='{"ok": true}', success=True)
+
+        with patch(
+            "cmdforge.providers.call_provider_subprocess",
+            return_value=low_level,
+        ):
+            result = call_provider(
+                "local", "private packet",
+                execution_policy=ProviderExecutionPolicy(
+                    allow_fallback=False,
+                    required_locality="local",
+                    required_capabilities=("structured-json",),
+                    data_classification="private",
+                    require_model_identity=True,
+                    require_model_digest=True,
+                ),
+            )
+
+        assert result.success is True
+        assert result.provenance() == {
+            "success": True,
+            "requested_provider": "local",
+            "actual_provider": "local",
+            "attempted_providers": ["local"],
+            "fallback_used": False,
+            "model": "model-a",
+            "model_digest": "sha256:123",
+            "locality": "local",
+            "model_identity_source": "provider-config",
+        }
+
+    def test_fallback_chain_skips_policy_ineligible_provider(
+        self, temp_providers_file
+    ):
+        save_providers([
+            Provider(
+                "primary", "primary", fallback_chain=["remote", "local"],
+                locality="local", data_policy="private",
+            ),
+            Provider(
+                "remote", "remote", locality="remote", data_policy="private",
+            ),
+            Provider(
+                "local", "local", locality="local", data_policy="private",
+                model="safe-model",
+            ),
+        ])
+
+        def invoke(provider, prompt, timeout, max_tokens):
+            if provider.name == "primary":
+                return ProviderResult(text="", success=False, error="offline")
+            assert provider.name == "local"
+            return ProviderResult(text="safe", success=True)
+
+        with patch(
+            "cmdforge.providers.call_provider_subprocess", side_effect=invoke
+        ) as low_level:
+            result = call_provider(
+                "primary", "private packet",
+                execution_policy=ProviderExecutionPolicy(
+                    required_locality="local",
+                    data_classification="private",
+                ),
+            )
+
+        assert result.success is True
+        assert result.actual_provider == "local"
+        assert result.attempted_providers == ["primary", "remote", "local"]
+        assert [call.args[0].name for call in low_level.call_args_list] == [
+            "primary", "local"
+        ]
 
     def test_fallback_chain(self, temp_providers_file):
         """Fallback can chain to another provider with fallback."""

tests/test_runner.py

@@ -229,6 +229,23 @@ class TestExecutePromptStep:
         assert success is False
         assert output == ""
 
+    @patch('cmdforge.runner.call_provider')
+    def test_step_can_deny_fallback_independently_of_caller(self, mock_call):
+        mock_call.return_value = ProviderResult(
+            text="private", success=True,
+        )
+        step = PromptStep(
+            prompt="Private", provider="local", output_var="out",
+            plain_text=True, fallback_policy="deny",
+        )
+
+        output, success = execute_prompt_step(step, {"input": ""})
+
+        assert success is True
+        assert output == "private"
+        policy = mock_call.call_args.kwargs["execution_policy"]
+        assert policy.allow_fallback is False
+
     @patch('cmdforge.runner.mock_provider')
     def test_mock_provider_used(self, mock_mock):
         mock_mock.return_value = ProviderResult(text="mock response", success=True)
@@ -988,6 +1005,58 @@ class TestCreateArgumentParser:
 
         assert args.count == "10"
 
+    def test_parser_enforces_typed_boolean_integer_and_enum_arguments(self):
+        tool = Tool(
+            name="typed",
+            arguments=[
+                ToolArgument(
+                    flag="--project", variable="project",
+                    type="boolean", default=False,
+                ),
+                ToolArgument(
+                    flag="--count", variable="count",
+                    type="integer", required=True,
+                ),
+                ToolArgument(
+                    flag="--mode", variable="mode",
+                    enum=["safe", "fast"], default="safe",
+                ),
+            ],
+        )
+        parser = create_argument_parser(tool)
+
+        args = parser.parse_args(["--project", "--count", "3", "--mode", "fast"])
+
+        assert args.project is True
+        assert args.count == 3
+        assert args.mode == "fast"
+
+    def test_parser_reuses_matching_legacy_boolean_universal_flag(self):
+        tool = Tool(
+            name="legacy",
+            arguments=[
+                ToolArgument(
+                    flag="--verbose", variable="verbose", type="bool"
+                )
+            ],
+        )
+
+        args = create_argument_parser(tool).parse_args(["--verbose"])
+
+        assert args.verbose is True
+        assert collect_custom_args(tool, args) == {"verbose": True}
+
+    def test_parser_rejects_incompatible_universal_flag_collision(self):
+        tool = Tool(
+            name="conflict",
+            arguments=[
+                ToolArgument(flag="--provider", variable="destination")
+            ],
+        )
+
+        with pytest.raises(ValueError, match="universal run options"):
+            create_argument_parser(tool)
+
     def test_parser_input_output_flags(self):
         tool = Tool(name="test")
         parser = create_argument_parser(tool)

tests/test_skills.py

@@ -378,7 +378,7 @@ class TestRunnerSkillIntegration:
     ):
         captured = {}
 
-        def fake_call(provider, prompt, max_tokens=None):
+        def fake_call(provider, prompt, max_tokens=None, **kwargs):
             captured.update(provider=provider, prompt=prompt)
             return ProviderResult(text="done", success=True)
 
@@ -412,7 +412,7 @@ class TestRunnerSkillIntegration:
     ):
         captured = {}
 
-        def fake_call(provider, prompt, max_tokens=None):
+        def fake_call(provider, prompt, max_tokens=None, **kwargs):
             captured.update(provider=provider, prompt=prompt)
             return ProviderResult(text='{"output": "done"}', success=True)
 

tests/test_tool.py

@@ -105,6 +105,25 @@ class TestToolArgument:
         with pytest.raises(ValueError):
             ToolArgument(flag="--value", variable="value", **values)
 
+    @pytest.mark.parametrize(
+        ("legacy_type", "canonical_type"),
+        [
+            ("str", "string"),
+            ("int", "integer"),
+            ("float", "number"),
+            ("bool", "boolean"),
+        ],
+    )
+    def test_legacy_scalar_type_aliases_are_normalized(
+        self, legacy_type, canonical_type
+    ):
+        argument = ToolArgument(
+            flag="--value", variable="value", type=legacy_type
+        )
+
+        assert argument.type == canonical_type
+        assert argument.to_dict().get("type", "string") == canonical_type
+
 
 class TestPromptStep:
     """Tests for PromptStep dataclass."""
@@ -686,6 +705,23 @@ class TestAgentContext:
         restored = PromptStep.from_dict(step.to_dict())
         assert restored.skills == ["python"]
 
+    def test_promptstep_deny_fallback_roundtrip(self):
+        step = PromptStep(
+            prompt="Private", provider="local", output_var="out",
+            fallback_policy="deny",
+        )
+
+        restored = PromptStep.from_dict(step.to_dict())
+
+        assert restored.fallback_policy == "deny"
+
+    def test_promptstep_rejects_invalid_fallback_policy(self):
+        with pytest.raises(ValueError, match="fallback_policy"):
+            PromptStep(
+                prompt="Test", provider="local", output_var="out",
+                fallback_policy="sometimes",
+            )
+
 
 class TestDeprecationFields:
     def test_round_trip_and_warning(self):