test_mcp_safety.py

Go to the documentation of this file.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# tests/tests_mcp/test_mcp_safety.py
 
"""
Purpose
-------
Regression tests for :mod:`oct.mcp.safety` — ``SafetyGate`` HITL
enforcement.
 
Responsibilities
----------------
- Verify ``SafetyGate.check()`` HITL rules (rules 1-5).
- Verify oct_format double guard (apply + confirm).
- Verify non-destructive tools always proceed; destructive tools
  require confirmation.
- Verify ``_format_confirmation_request`` sentinel and content.
- Verify T-08: no stateful replay surface (each call independent).
 
Diagnostics
-----------
Domain: MCP-TESTS
Levels:
    L2 — test lifecycle
    L3 — assertion details
    L4 — deep tracing
 
Contracts
---------
- Tests are in-memory safety checks; no subprocess or filesystem writes.
"""
 
from __future__ import annotations
 
import pytest
 
from oct.mcp.config import McpConfig
from oct.mcp.policy import TOOL_MANIFEST, ToolSpec
from oct.mcp.safety import (
    HITL_SENTINEL,
    SafetyDecision,
    SafetyGate,
    _format_confirmation_request,
    _TOOL_RISK_DESCRIPTIONS,
)
 
 
# -------------------------------------------------------------------
# Fixtures
# -------------------------------------------------------------------
 
@pytest.fixture
def config():
    return McpConfig()
 
 
@pytest.fixture
def gate(config):
    return SafetyGate(config)
 
 
def _nondestructive_spec():
    return ToolSpec(cli="lint", permissions=frozenset({"read"}), destructive=False, auto_approve=True)
 
 
def _write_spec(tool: str = "oct_format"):
    return TOOL_MANIFEST[tool]
 
 
# -------------------------------------------------------------------
# Rule 1: non-destructive tools always proceed
# -------------------------------------------------------------------
 
class TestNonDestructiveTools:
    def test_non_destructive_proceeds(self, gate):
        spec = _nondestructive_spec()
        d = gate.check("oct_lint", spec, {})
        assert d.proceed is True
        assert d.requires_confirmation is False
        assert d.dry_run is False
 
    def test_non_destructive_ignores_confirm_false(self, gate):
        spec = _nondestructive_spec()
        d = gate.check("oct_lint", spec, {"confirm": False})
        assert d.proceed is True
 
    def test_non_destructive_ignores_confirm_true(self, gate):
        spec = _nondestructive_spec()
        d = gate.check("oct_health", spec, {"confirm": True})
        assert d.proceed is True
 
    def test_all_phase5a_tools_proceed(self, gate):
        phase5a = [
            "oct_lint", "oct_health", "oct_skeleton", "oct_deps",
            "oct_test", "oct_typecheck", "oct_diag", "oct_git_status", "oct_git_check",
        ]
        for tool_name in phase5a:
            spec = TOOL_MANIFEST[tool_name]
            d = gate.check(tool_name, spec, {})
            assert d.proceed is True, f"{tool_name} should proceed"
            assert d.dry_run is False
 
 
# -------------------------------------------------------------------
# Rule 2: destructive + confirm=False → requires_confirmation
# -------------------------------------------------------------------
 
class TestDestructiveRequiresConfirmation:
    @pytest.mark.parametrize("tool_name", [
        "oct_format", "oct_docs", "oct_export_source", "oct_new",
        "oct_scaffold", "oct_clean", "oct_install_hooks",
        "oct_git_commit", "oct_git_hooks_install", "oct_git_init",
        "oct_git_changelog",
    ])
    def test_all_write_tools_require_confirmation(self, gate, tool_name):
        spec = TOOL_MANIFEST[tool_name]
        args = {"confirm": False}
        # Provide required fields for each tool
        if tool_name == "oct_new":
            args["path"] = "src/foo.py"
        if tool_name == "oct_scaffold":
            args["name"] = "myproject"
        if tool_name == "oct_git_commit":
            args["message"] = "feat: add feature"
        d = gate.check(tool_name, spec, args)
        assert d.proceed is False, f"{tool_name} should not proceed without confirm"
        assert d.requires_confirmation is True
        assert d.dry_run is False
 
    def test_confirm_false_is_default(self, gate):
        spec = TOOL_MANIFEST["oct_clean"]
        d = gate.check("oct_clean", spec, {})  # no confirm key
        assert d.proceed is False
        assert d.requires_confirmation is True
 
    def test_confirm_false_explicit(self, gate):
        spec = TOOL_MANIFEST["oct_git_init"]
        d = gate.check("oct_git_init", spec, {"confirm": False})
        assert d.proceed is False
        assert d.requires_confirmation is True
 
 
# -------------------------------------------------------------------
# Rule 5: destructive + confirm=True → proceed (non-format tools)
# -------------------------------------------------------------------
 
class TestDestructiveConfirmed:
    @pytest.mark.parametrize("tool_name", [
        "oct_docs", "oct_export_source", "oct_new",
        "oct_scaffold", "oct_clean", "oct_install_hooks",
        "oct_git_commit", "oct_git_hooks_install", "oct_git_init",
        "oct_git_changelog",
    ])
    def test_write_tools_proceed_with_confirm(self, gate, tool_name):
        spec = TOOL_MANIFEST[tool_name]
        args: dict = {"confirm": True}
        if tool_name == "oct_new":
            args["path"] = "src/foo.py"
        if tool_name == "oct_scaffold":
            args["name"] = "myproject"
        if tool_name == "oct_git_commit":
            args["message"] = "feat: add feature"
        d = gate.check(tool_name, spec, args)
        assert d.proceed is True, f"{tool_name} should proceed with confirm=True"
        assert d.requires_confirmation is False
        assert d.dry_run is False
 
 
# -------------------------------------------------------------------
# Rules 3 & 4: oct_format double guard
# -------------------------------------------------------------------
 
class TestFormatDoubleGuard:
    def test_format_confirm_false_requires_confirmation(self, gate):
        spec = TOOL_MANIFEST["oct_format"]
        d = gate.check("oct_format", spec, {"confirm": False, "apply": False})
        assert d.proceed is False
        assert d.requires_confirmation is True
 
    def test_format_confirm_true_apply_false_dry_run(self, gate):
        spec = TOOL_MANIFEST["oct_format"]
        d = gate.check("oct_format", spec, {"confirm": True, "apply": False})
        assert d.proceed is True
        assert d.dry_run is True
        assert d.requires_confirmation is False
 
    def test_format_confirm_true_apply_true_no_dry_run(self, gate):
        spec = TOOL_MANIFEST["oct_format"]
        d = gate.check("oct_format", spec, {"confirm": True, "apply": True})
        assert d.proceed is True
        assert d.dry_run is False
        assert d.requires_confirmation is False
 
    def test_format_confirm_true_apply_missing_defaults_to_false(self, gate):
        spec = TOOL_MANIFEST["oct_format"]
        # apply not provided → defaults to False → dry_run
        d = gate.check("oct_format", spec, {"confirm": True})
        assert d.proceed is True
        assert d.dry_run is True
 
 
# -------------------------------------------------------------------
# _format_confirmation_request
# -------------------------------------------------------------------
 
class TestFormatConfirmationRequest:
    def test_starts_with_sentinel(self):
        spec = TOOL_MANIFEST["oct_format"]
        result = _format_confirmation_request("oct_format", spec, {"apply": False, "confirm": False})
        assert result.startswith(HITL_SENTINEL)
 
    def test_contains_tool_name(self):
        spec = TOOL_MANIFEST["oct_clean"]
        result = _format_confirmation_request("oct_clean", spec, {"dry_run": True})
        assert "oct_clean" in result
 
    def test_contains_re_call_instruction(self):
        spec = TOOL_MANIFEST["oct_scaffold"]
        result = _format_confirmation_request("oct_scaffold", spec, {"name": "myproject"})
        assert "confirm=True" in result
 
    def test_risk_description_present(self):
        spec = TOOL_MANIFEST["oct_git_commit"]
        result = _format_confirmation_request(
            "oct_git_commit", spec, {"message": "feat: test"}
        )
        assert "commit" in result.lower()
 
    def test_confirm_key_excluded_from_args_display(self):
        spec = TOOL_MANIFEST["oct_format"]
        result = _format_confirmation_request(
            "oct_format", spec, {"confirm": False, "apply": False, "paths": []}
        )
        # confirm itself should not appear in the args display section
        # (the sentinel and re-call instruction contain "confirm" but the args section should not)
        # At minimum, result must start with sentinel
        assert result.startswith(HITL_SENTINEL)
 
    def test_all_write_tools_have_risk_description(self):
        for tool_name in _TOOL_RISK_DESCRIPTIONS:
            assert _TOOL_RISK_DESCRIPTIONS[tool_name], f"{tool_name} risk desc should not be empty"
 
    def test_unknown_tool_fallback(self):
        spec = ToolSpec(cli="unknown", permissions=frozenset({"write"}), destructive=True)
        result = _format_confirmation_request("oct_mystery", spec, {})
        assert result.startswith(HITL_SENTINEL)
        assert "oct_mystery" in result
 
 
# -------------------------------------------------------------------
# SafetyDecision dataclass
# -------------------------------------------------------------------
 
class TestSafetyDecision:
    def test_fields_present(self):
        d = SafetyDecision(proceed=True, requires_confirmation=False, dry_run=False, reason="ok")
        assert d.proceed is True
        assert d.requires_confirmation is False
        assert d.dry_run is False
        assert d.reason == "ok"
 
    def test_fields_false(self):
        d = SafetyDecision(proceed=False, requires_confirmation=True, dry_run=False, reason="need confirm")
        assert d.proceed is False
        assert d.requires_confirmation is True
 
 
# -------------------------------------------------------------------
# T-08: no stateful replay surface
# -------------------------------------------------------------------
 
class TestNoReplayState:
    def test_each_call_independent(self, gate):
        """T-08: successive calls with same args produce same result (no state)."""
        spec = TOOL_MANIFEST["oct_clean"]
        args = {"confirm": True, "dry_run": True}
        d1 = gate.check("oct_clean", spec, args)
        d2 = gate.check("oct_clean", spec, args)
        assert d1.proceed == d2.proceed
        assert d1.requires_confirmation == d2.requires_confirmation
 
    def test_first_false_second_true_independent(self, gate):
        """Calling with confirm=False then confirm=True works correctly (no stale state)."""
        spec = TOOL_MANIFEST["oct_git_init"]
        d1 = gate.check("oct_git_init", spec, {"confirm": False})
        assert d1.proceed is False
        d2 = gate.check("oct_git_init", spec, {"confirm": True})
        assert d2.proceed is True
 
    def test_multiple_gate_instances_independent(self):
        """Two SafetyGate instances with the same config are both stateless."""
        cfg = McpConfig()
        gate1 = SafetyGate(cfg)
        gate2 = SafetyGate(cfg)
        spec = TOOL_MANIFEST["oct_scaffold"]
        args = {"confirm": True, "name": "myproject"}
        d1 = gate1.check("oct_scaffold", spec, args)
        d2 = gate2.check("oct_scaffold", spec, args)
        assert d1.proceed == d2.proceed