mongodb-js · kmruiz · Jun 18, 2025 · Jun 20, 2025
@@ -29,7 +29,10 @@
     "check:types": "tsc --noEmit --project tsconfig.json",
     "reformat": "prettier --write .",
     "generate": "./scripts/generate.sh",
-    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage"
+    "test": "npm run test:unit && npm run test:integration",
+    "test:accuracy": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --testPathPattern=tests/accuracy",
+    "test:unit": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --testPathPattern=tests/unit",
+    "test:integration": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --testPathPattern=tests/integration"
   },
   "license": "Apache-2.0",
   "devDependencies": {
@@ -57,7 +60,8 @@
     "tsx": "^4.19.3",
     "typescript": "^5.8.2",
     "typescript-eslint": "^8.29.1",
-    "yaml": "^2.7.1"
+    "yaml": "^2.7.1",
+    "zod-to-json-schema": "^3.24.5"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.11.2",

@@ -0,0 +1,11 @@
+import { describeAccuracyTest } from "../test-sdk.js";
+
+describeAccuracyTest("1 step delete queries", ({ prompt }) => {
+    prompt("delete all disabled users (disabled = true) in database 'my' and collection 'users'", (tool) => {
+        tool("delete-many").verifyCalled({
+            database: "my",
+            collection: "users",
+            filter: { disabled: true },
+        });
+    });
+});
@@ -0,0 +1,20 @@
+import { describeAccuracyTest } from "../test-sdk.js";
+
+describeAccuracyTest("1 step find queries", ({ prompt }) => {
+    prompt("find all users in database 'my' and collection 'users'", (tool) => {
+        tool("find").verifyCalled({ database: "my", collection: "users", limit: 10 });
+    });
+
+    prompt("find all red cars in database 'production' and collection 'cars'", (tool) => {
+        tool("find").verifyCalled({ filter: { color: "red" }, database: "production", collection: "cars", limit: 10 });
+    });
+
+    prompt("get 100 books in database 'prod' and collection 'books' where the author is J.R.R Tolkien", (tool) => {
+        tool("find").verifyCalled({
+            filter: { author: "J.R.R Tolkien" },
+            database: "prod",
+            collection: "books",
+            limit: 100,
+        });
+    });
+});
@@ -0,0 +1,12 @@
+import { describeAccuracyTest } from "../test-sdk.js";
+
+describeAccuracyTest("1 step update queries", ({ prompt }) => {
+    prompt("set all users with an empty email to disabled in database 'my' and collection 'users'", (tool) => {
+        tool("update-many").verifyCalled({
+            database: "my",
+            collection: "users",
+            filter: { email: "" },
+            update: { $set: { disabled: true } },
+        });
+    });
+});
@@ -0,0 +1,32 @@
+import { describeAccuracyTest } from "../test-sdk.js";
+
+describeAccuracyTest("2 step create collection", ({ prompt }) => {
+    prompt(
+        `
+        create a new collection named 'users' in database 'my' and afterwards create a sample document with the following data:
+        - username: "john_doe"
+        - email: test@mongodb.com
+        - password: "password123"
+        - disabled: false
+    `,
+        (tool) => {
+            tool("create-collection").verifyCalled({
+                database: "my",
+                collection: "users",
+            });
+
+            tool("insert-many").verifyCalled({
+                database: "my",
+                collection: "users",
+                documents: [
+                    {
+                        username: "john_doe",
+                        email: "test@mongodb.com",
+                        password: "password123",
+                        disabled: false,
+                    },
+                ],
+            });
+        }
+    );
+});
@@ -0,0 +1,147 @@
+import { ModelFacade, ToolCall, ToolDefinition } from "./model.js";
+
+type GeminiModel = "gemini-2.0-flash" | "gemini-1.5-flash";
+
+export class GeminiModelFacade implements ModelFacade {
+    readonly name: GeminiModel;
+
+    constructor(modelName: GeminiModel) {
+        this.name = modelName;
+    }
+
+    available(): boolean {
+        return process.env.MONGODB_MCP_TEST_GEMINI_API_KEY !== undefined;
+    }
+
+    async generatePlan(prompt: string, tools: ToolDefinition[]): Promise<string[]> {
+        const planPrompt = `You are an expert MongoDB developer. Create a plan for the following task: \n ${prompt} \n Return the plan as a list of steps, as a JSON array. For example: [ "Step 1: ...", "Step 2: ...", "Step 3: ..." ]. Only return the JSON array, nothing else. Do not include any wrapper markdown or anything, just the plain JSON array.`;
+        const chatHistory = [{ role: "user", parts: [{ text: planPrompt }] }];
+
+        const apiKey = process.env.MONGODB_MCP_TEST_GEMINI_API_KEY;
+        const apiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${this.name}:generateContent?key=${apiKey}`;
+
+        const toolDefinitions = tools.map((tool) => ({
+            name: tool.name,
+            description: tool.description,
+            parameters: tool.parameters || {},
+        }));
+
+        const payload = {
+            contents: chatHistory,
+            tools: {
+                function_declarations: [toolDefinitions],
+            },
+        };
+
+        try {
+            const response = await fetch(apiUrl, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify(payload),
+            });
+
+            if (!response.ok) {
+                const errorData = await response.text();
+                console.error(`[Gemini API Error] HTTP error! status: ${response.status}, data: ${errorData}`);
+                return [];
+            }
+
+            const result = (await response.json()) as {
+                candidates: Array<{
+                    content: {
+                        parts: Array<{
+                            text?: string;
+                            functionCall?: {
+                                name: string;
+                                args: Record<string, unknown>;
+                            };
+                        }>;
+                    };
+                }>;
+            };
+
+            const responseString = result.candidates
+                .flatMap((candidate) => candidate.content.parts.map((part) => part.text || ""))
+                .join("")
+                .replace("```json", "")
+                .replace("```", "");
+
+            try {
+                return JSON.parse(responseString) as string[];
+            } catch (parseError) {
+                console.error("[Gemini API JSON.parse Error]", responseString, parseError);
+            }
+            return [];
+        } catch (error: unknown) {
+            console.error("[Gemini API Fetch Error]", error);
+            return [];
+        }
+    }
+
+    async generateContent(parts: string[], tools: ToolDefinition[]): Promise<{ toolCall: ToolCall[]; text?: string }> {
+        const toolDefinitions = tools.map((tool) => ({
+            name: tool.name,
+            description: tool.description,
+            parameters: tool.parameters || {},
+        }));
+
+        const chatHistory = [{ role: "user", parts: parts.map((part) => ({ text: part })) }];
+        const payload = {
+            contents: chatHistory,
+            tools: {
+                function_declarations: [toolDefinitions],
+            },
+        };
+
+        const apiKey = process.env.MONGODB_MCP_TEST_GEMINI_API_KEY;
+        const apiUrl = `https://generativelanguage.googleapis.com/v1beta/models/${this.name}:generateContent?key=${apiKey}`;
+
+        try {
+            const response = await fetch(apiUrl, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify(payload),
+            });
+
+            if (!response.ok) {
+                const errorData = await response.text();
+                console.error(`[Gemini API Error] HTTP error! status: ${response.status}, data: ${errorData}`);
+                return { toolCall: [], text: `Gemini API error: ${response.status}` };
+            }
+
+            const result = (await response.json()) as {
+                candidates: Array<{
+                    content: {
+                        parts: Array<{
+                            text?: string;
+                            functionCall?: {
+                                name: string;
+                                args: Record<string, unknown>;
+                            };
+                        }>;
+                    };
+                }>;
+            };
+
+            if (result.candidates && result.candidates.length > 0) {
+                const firstPart = result.candidates[0]?.content.parts[0];
+                if (firstPart?.functionCall) {
+                    return {
+                        toolCall: [
+                            {
+                                name: firstPart.functionCall.name,
+                                args: firstPart.functionCall.args,
+                            },
+                        ],
+                    };
+                } else if (firstPart?.text) {
+                    return { toolCall: [], text: firstPart.text };
+                }
+            }
+            return { toolCall: [], text: "Gemini response was empty or unexpected." };
+        } catch (error: unknown) {
+            console.error("[Gemini API Fetch Error]", error);
+            return { toolCall: [], text: `Error contacting Gemini LLM.` };
+        }
+    }
+}
@@ -0,0 +1,11 @@
+import { ModelFacade } from "./model.js";
+import { GeminiModelFacade } from "./gemini.js";
+
+const ALL_MODELS: ModelFacade[] = [
+    new GeminiModelFacade("gemini-2.0-flash"),
+    new GeminiModelFacade("gemini-1.5-flash"),
+];
+
+export function availableModels(): ModelFacade[] {
+    return ALL_MODELS.filter((model) => model.available());
+}
@@ -0,0 +1,14 @@
+export type ToolCall = { name: string; args: Record<string, unknown> };
+export type ToolDefinition = {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+};
+
+export interface ModelFacade {
+    name: string;
+    available(): boolean;
+
+    generatePlan(prompt: string, tools: ToolDefinition[]): Promise<string[]>;
+    generateContent(parts: string[], tools: ToolDefinition[]): Promise<{ toolCall: ToolCall[]; text?: string }>;
+}