diff --git a/Makefile b/Makefile index db82844..4d4af7c 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ BUNDLE_NAME = mcp-example VERSION ?= 0.1.0 -.PHONY: help install dev-install format format-check lint lint-fix typecheck test test-cov clean run run-http check all bump bundle +.PHONY: help install dev-install format format-check lint lint-fix typecheck test test-integration test-llm test-cov clean run run-http check all bump bundle help: ## Show this help message @echo 'Usage: make [target]' @@ -34,6 +34,12 @@ typecheck: ## Type check code with ty test: ## Run tests with pytest uv run pytest tests/ -v +test-integration: ## Run integration tests (requires EXAMPLE_API_KEY) + uv run pytest tests-integration/ -v --ignore=tests-integration/test_skill_llm.py + +test-llm: ## Run LLM smoke tests (requires EXAMPLE_API_KEY + ANTHROPIC_API_KEY) + uv run pytest tests-integration/test_skill_llm.py -v + test-cov: ## Run tests with coverage uv run pytest tests/ -v --cov=src/mcp_example --cov-report=term-missing diff --git a/src/mcp_example/SKILL.md b/src/mcp_example/SKILL.md new file mode 100644 index 0000000..be43ee9 --- /dev/null +++ b/src/mcp_example/SKILL.md @@ -0,0 +1,18 @@ +# Example MCP Server — Skill Guide + +## Tools + +| Tool | Use when... | +|------|-------------| +| `list_items` | You need to browse or search items | +| `get_item` | You have an item ID and need full details | + +## Context Reuse + +- Use the `id` from `list_items` results when calling `get_item` + +## Workflows + +### 1. Browse and Inspect +1. `list_items` with a limit to get an overview +2. For interesting items: `get_item` to get full details diff --git a/src/mcp_example/api_client.py b/src/mcp_example/api_client.py index 0aedb54..ff14791 100644 --- a/src/mcp_example/api_client.py +++ b/src/mcp_example/api_client.py @@ -102,7 +102,7 @@ async def _request( raise ExampleAPIError(response.status, error_msg, result) - return result # type: ignore[no-any-return] + return result except ClientError as e: raise ExampleAPIError(500, f"Network error: {str(e)}") from e diff --git a/src/mcp_example/server.py b/src/mcp_example/server.py index c18d408..f3db9f6 100644 --- a/src/mcp_example/server.py +++ b/src/mcp_example/server.py @@ -14,6 +14,7 @@ import logging import os import sys +from importlib.resources import files from fastmcp import Context, FastMCP from starlette.requests import Request @@ -31,8 +32,16 @@ logger.info("Example server module loading...") +SKILL_CONTENT = files("mcp_example").joinpath("SKILL.md").read_text() + # Create MCP server -mcp = FastMCP("Example") +mcp = FastMCP( + "Example", + instructions=( + "Before using tools, read the skill://example/usage resource " + "for tool selection guidance and workflow patterns." + ), +) # Global client instance (lazy initialization) _client: ExampleClient | None = None @@ -110,6 +119,17 @@ async def get_item( raise +# ============================================================================ +# Resources +# ============================================================================ + + +@mcp.resource("skill://example/usage") +def skill_usage() -> str: + """Usage guide for the Example MCP server tools.""" + return SKILL_CONTENT + + # ============================================================================ # Entrypoints # ============================================================================ diff --git a/tests-integration/__init__.py b/tests-integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests-integration/conftest.py b/tests-integration/conftest.py new file mode 100644 index 0000000..31d7393 --- /dev/null +++ b/tests-integration/conftest.py @@ -0,0 +1,47 @@ +""" +Shared fixtures and configuration for integration tests. + +These tests require a valid EXAMPLE_API_KEY environment variable. +They make real API calls and should not be run in CI without proper setup. +""" + +import os + +import pytest +import pytest_asyncio + +from mcp_example.api_client import ExampleClient + + +def pytest_configure(config): + """Check for required environment variables before running tests.""" + if not os.environ.get("EXAMPLE_API_KEY"): + pytest.exit( + "ERROR: EXAMPLE_API_KEY environment variable is required.\n" + "Set it before running integration tests:\n" + " export EXAMPLE_API_KEY=your_key_here\n" + " make test-integration" + ) + + +@pytest.fixture +def api_key() -> str: + """Get the API key from environment.""" + key = os.environ.get("EXAMPLE_API_KEY") + if not key: + pytest.skip("EXAMPLE_API_KEY not set") + return key + + +@pytest_asyncio.fixture +async def client(api_key: str) -> ExampleClient: + """Create a client for testing.""" + client = ExampleClient(api_key=api_key) + yield client + await client.close() + + +# TODO: Add well-known test data constants +# class TestData: +# """Well-known test data for integration tests.""" +# KNOWN_ID = "abc123" diff --git a/tests-integration/test_core_tools.py b/tests-integration/test_core_tools.py new file mode 100644 index 0000000..b974567 --- /dev/null +++ b/tests-integration/test_core_tools.py @@ -0,0 +1,35 @@ +""" +Core tools integration tests. + +Tests basic API functionality with real API calls. +Replace with your actual endpoints and assertions. +""" + +# import pytest +# from mcp_example.api_client import ExampleAPIError, ExampleClient + + +# TODO: Add integration tests for each tool group. Example: +# +# class TestListItems: +# """Test list items endpoint.""" +# +# @pytest.mark.asyncio +# async def test_list_items(self, client: ExampleClient): +# """Test listing items.""" +# result = await client.list_items(limit=5) +# assert isinstance(result, list) +# print(f"Found {len(result)} items") +# +# +# For tier-gated endpoints, add a helper: +# +# async def has_premium_access(client: ExampleClient) -> bool: +# """Check if the plan supports premium endpoints.""" +# try: +# await client.premium_method() +# return True +# except ExampleAPIError as e: +# if e.status in (400, 401, 403): +# return False +# raise diff --git a/tests-integration/test_skill_llm.py b/tests-integration/test_skill_llm.py new file mode 100644 index 0000000..f4d622a --- /dev/null +++ b/tests-integration/test_skill_llm.py @@ -0,0 +1,86 @@ +""" +Smoke test: verify the LLM reads the skill resource and selects the correct tool. + +Requires ANTHROPIC_API_KEY and EXAMPLE_API_KEY in environment. +""" + +import os + +import anthropic +import pytest +from fastmcp import Client + +from mcp_example.server import mcp + + +def get_anthropic_client() -> anthropic.Anthropic: + token = os.environ.get("ANTHROPIC_API_KEY") + if not token: + pytest.skip("ANTHROPIC_API_KEY not set") + return anthropic.Anthropic(api_key=token) + + +async def get_server_context() -> dict: + """Extract instructions, skill content, and tool definitions from the MCP server.""" + async with Client(mcp) as client: + init = await client.initialize() + instructions = init.instructions + + resources = await client.list_resources() + skill_text = "" + for r in resources: + if "skill://" in str(r.uri): + contents = await client.read_resource(str(r.uri)) + skill_text = contents[0].text if hasattr(contents[0], "text") else str(contents[0]) + + tools_list = await client.list_tools() + tools = [] + for t in tools_list: + tool_def = { + "name": t.name, + "description": t.description or "", + "input_schema": t.inputSchema, + } + tools.append(tool_def) + + return { + "instructions": instructions, + "skill": skill_text, + "tools": tools, + } + + +class TestSkillLLMInvocation: + """Test that an LLM reads the skill and makes correct tool choices. + + TODO: Replace with tests specific to your server's tools and skill. + + Each test should: + 1. Send a user prompt that maps to a specific tool per the SKILL.md + 2. Assert the LLM calls the expected tool (not a similar one) + """ + + # @pytest.mark.asyncio + # async def test_query_selects_correct_tool(self): + # """When asked to X, the LLM should call tool_name.""" + # ctx = await get_server_context() + # client = get_anthropic_client() + # + # system = ( + # f"You are an assistant.\n\n" + # f"## Server Instructions\n{ctx['instructions']}\n\n" + # f"## Skill Resource\n{ctx['skill']}" + # ) + # + # response = client.messages.create( + # model="claude-haiku-4-5-20251001", + # max_tokens=1024, + # system=system, + # messages=[{"role": "user", "content": "Your test prompt here"}], + # tools=[{"type": "custom", **t} for t in ctx["tools"]], + # ) + # + # tool_calls = [b for b in response.content if b.type == "tool_use"] + # assert len(tool_calls) > 0, "LLM did not call any tool" + # assert tool_calls[0].name == "expected_tool_name" + pass diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..feb2172 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,33 @@ +"""Shared fixtures for unit tests.""" + +from unittest.mock import AsyncMock + +import pytest + +from mcp_example.server import mcp + + +@pytest.fixture +def mcp_server(): + """Return the MCP server instance.""" + return mcp + + +@pytest.fixture +def mock_client(): + """Create a mock API client.""" + client = AsyncMock() + client.list_items = AsyncMock( + return_value=[ + {"id": "1", "name": "Item 1"}, + {"id": "2", "name": "Item 2"}, + ] + ) + client.get_item = AsyncMock( + return_value={ + "id": "1", + "name": "Item 1", + "description": "Test item", + } + ) + return client diff --git a/tests/test_api_client.py b/tests/test_api_client.py new file mode 100644 index 0000000..5e39711 --- /dev/null +++ b/tests/test_api_client.py @@ -0,0 +1,123 @@ +"""Unit tests for the Example API client.""" + +import os +from unittest.mock import AsyncMock, patch + +import pytest +import pytest_asyncio + +from mcp_example.api_client import ExampleAPIError, ExampleClient + + +@pytest_asyncio.fixture +async def mock_client(): + """Create an ExampleClient with mocked session.""" + client = ExampleClient(api_key="test_key") + client._session = AsyncMock() + yield client + await client.close() + + +class TestClientInitialization: + """Test client creation and configuration.""" + + def test_init_with_explicit_key(self): + """Client accepts an explicit API key.""" + client = ExampleClient(api_key="explicit_key") + assert client.api_key == "explicit_key" + + def test_init_with_env_var(self): + """Client falls back to EXAMPLE_API_KEY env var.""" + os.environ["EXAMPLE_API_KEY"] = "env_key" + try: + client = ExampleClient() + assert client.api_key == "env_key" + finally: + del os.environ["EXAMPLE_API_KEY"] + + def test_init_without_key_raises(self): + """Client raises ValueError when no key is available.""" + with patch.dict(os.environ, {}, clear=True): + os.environ.pop("EXAMPLE_API_KEY", None) + with pytest.raises(ValueError, match="EXAMPLE_API_KEY is required"): + ExampleClient() + + def test_custom_timeout(self): + """Client accepts a custom timeout.""" + client = ExampleClient(api_key="key", timeout=60.0) + assert client.timeout == 60.0 + + @pytest.mark.asyncio + async def test_context_manager(self): + """Client works as an async context manager.""" + async with ExampleClient(api_key="test") as client: + assert client._session is not None + assert client._session is None + + +class TestClientMethods: + """Test API client methods with mocked responses.""" + + @pytest.mark.asyncio + async def test_list_items(self, mock_client): + """Test list items endpoint.""" + mock_response = {"items": [{"id": "1", "name": "Item 1"}, {"id": "2", "name": "Item 2"}]} + with patch.object(mock_client, "_request", return_value=mock_response): + result = await mock_client.list_items(limit=10) + assert len(result) == 2 + + @pytest.mark.asyncio + async def test_get_item(self, mock_client): + """Test get item endpoint.""" + mock_response = {"id": "1", "name": "Item 1", "description": "Test"} + with patch.object(mock_client, "_request", return_value=mock_response): + result = await mock_client.get_item("1") + assert result["id"] == "1" + + +class TestErrorHandling: + """Test error handling for API errors.""" + + @pytest.mark.asyncio + async def test_401_unauthorized(self, mock_client): + """Test handling of unauthorized errors.""" + with patch.object( + mock_client, + "_request", + side_effect=ExampleAPIError(401, "Invalid API key"), + ): + with pytest.raises(ExampleAPIError) as exc_info: + await mock_client.list_items() + assert exc_info.value.status == 401 + + @pytest.mark.asyncio + async def test_429_rate_limit(self, mock_client): + """Test handling of rate limit errors.""" + with patch.object( + mock_client, + "_request", + side_effect=ExampleAPIError(429, "Rate limit exceeded"), + ): + with pytest.raises(ExampleAPIError) as exc_info: + await mock_client.list_items() + assert exc_info.value.status == 429 + + @pytest.mark.asyncio + async def test_network_error(self, mock_client): + """Test handling of network errors.""" + with patch.object( + mock_client, + "_request", + side_effect=ExampleAPIError(500, "Network error: Connection failed"), + ): + with pytest.raises(ExampleAPIError) as exc_info: + await mock_client.list_items() + assert exc_info.value.status == 500 + assert "Network error" in exc_info.value.message + + def test_error_string_representation(self): + """Test error string format.""" + err = ExampleAPIError(401, "Unauthorized", {"id": "auth_error"}) + assert "401" in str(err) + assert "Unauthorized" in str(err) + assert err.details == {"id": "auth_error"} diff --git a/tests/test_server.py b/tests/test_server.py index 164ad4f..3c0dd32 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,60 +1,91 @@ -"""Tests for Example MCP Server tools.""" +"""Tests for Example MCP Server tools and skill resource.""" -from unittest.mock import AsyncMock, patch +from unittest.mock import patch import pytest +from fastmcp import Client +from fastmcp.exceptions import ToolError from mcp_example.api_client import ExampleAPIError +from mcp_example.server import SKILL_CONTENT -@pytest.fixture -def mock_client(): - """Create a mock API client.""" - client = AsyncMock() - client.list_items = AsyncMock( - return_value=[ - {"id": "1", "name": "Item 1"}, - {"id": "2", "name": "Item 2"}, - ] - ) - client.get_item = AsyncMock( - return_value={ - "id": "1", - "name": "Item 1", - "description": "Test item", - } - ) - return client - - -@pytest.mark.asyncio -async def test_list_items(mock_client): - """Test list_items tool.""" - with patch("mcp_example.server.get_client", return_value=mock_client): - from mcp_example.server import list_items - - result = await list_items(limit=10) - assert len(result) == 2 - mock_client.list_items.assert_called_once_with(limit=10) - - -@pytest.mark.asyncio -async def test_get_item(mock_client): - """Test get_item tool.""" - with patch("mcp_example.server.get_client", return_value=mock_client): - from mcp_example.server import get_item - - result = await get_item(item_id="1") - assert result["id"] == "1" - mock_client.get_item.assert_called_once_with("1") - - -@pytest.mark.asyncio -async def test_list_items_api_error(mock_client): - """Test list_items handles API errors.""" - mock_client.list_items = AsyncMock(side_effect=ExampleAPIError(401, "Unauthorized")) - with patch("mcp_example.server.get_client", return_value=mock_client): - from mcp_example.server import list_items - - with pytest.raises(ExampleAPIError): - await list_items() +class TestSkillResource: + """Test the skill resource and server instructions.""" + + @pytest.mark.asyncio + async def test_initialize_returns_instructions(self, mcp_server): + """Server instructions reference the skill resource.""" + async with Client(mcp_server) as client: + result = await client.initialize() + assert result.instructions is not None + assert "skill://example/usage" in result.instructions + + @pytest.mark.asyncio + async def test_skill_resource_listed(self, mcp_server): + """skill://example/usage appears in resource listing.""" + async with Client(mcp_server) as client: + resources = await client.list_resources() + uris = [str(r.uri) for r in resources] + assert "skill://example/usage" in uris + + @pytest.mark.asyncio + async def test_skill_resource_readable(self, mcp_server): + """Reading the skill resource returns the full skill content.""" + async with Client(mcp_server) as client: + contents = await client.read_resource("skill://example/usage") + text = contents[0].text if hasattr(contents[0], "text") else str(contents[0]) + assert "list_items" in text + assert "get_item" in text + + @pytest.mark.asyncio + async def test_skill_content_matches_constant(self, mcp_server): + """Resource content matches the SKILL_CONTENT constant.""" + async with Client(mcp_server) as client: + contents = await client.read_resource("skill://example/usage") + text = contents[0].text if hasattr(contents[0], "text") else str(contents[0]) + assert text == SKILL_CONTENT + + +class TestToolListing: + """Test that all tools are registered and discoverable.""" + + @pytest.mark.asyncio + async def test_all_tools_listed(self, mcp_server): + """All expected tools appear in tool listing.""" + async with Client(mcp_server) as client: + tools = await client.list_tools() + names = {t.name for t in tools} + expected = {"list_items", "get_item"} + assert expected == names + + +class TestMCPTools: + """Test the MCP server tools via FastMCP Client.""" + + @pytest.mark.asyncio + async def test_list_items(self, mcp_server, mock_client): + """Test list_items tool.""" + with patch("mcp_example.server.get_client", return_value=mock_client): + async with Client(mcp_server) as client: + result = await client.call_tool("list_items", {"limit": 10}) + assert result is not None + mock_client.list_items.assert_called_once_with(limit=10) + + @pytest.mark.asyncio + async def test_get_item(self, mcp_server, mock_client): + """Test get_item tool.""" + with patch("mcp_example.server.get_client", return_value=mock_client): + async with Client(mcp_server) as client: + result = await client.call_tool("get_item", {"item_id": "1"}) + assert result is not None + mock_client.get_item.assert_called_once_with("1") + + @pytest.mark.asyncio + async def test_list_items_api_error(self, mcp_server, mock_client): + """Test list_items handles API errors.""" + mock_client.list_items.side_effect = ExampleAPIError(401, "Unauthorized") + with patch("mcp_example.server.get_client", return_value=mock_client): + async with Client(mcp_server) as client: + with pytest.raises(ToolError, match="401"): + await client.call_tool("list_items", {})