diff --git a/evals/calendar-all-day.eval.ts b/evals/calendar-all-day.eval.ts new file mode 100644 index 0000000000..e29a779391 --- /dev/null +++ b/evals/calendar-all-day.eval.ts @@ -0,0 +1,33 @@ +import { evalTest } from './test-helper.js'; +import { expect } from 'vitest'; + +evalTest('USUALLY_PASSES', { + suiteName: 'default', + suiteType: 'behavioral', + name: 'should create an all-day event using the optional date field', + prompt: + 'Create an all-day event for 2026-05-20 titled "Company Retreat". Do not use a specific time.', + setup: async (rig) => { + rig.addTestMcpServer('workspace-server', 'google-workspace'); + }, + assert: async (rig) => { + const toolLogs = rig.readToolLogs(); + console.log('TOOL LOGS:', JSON.stringify(toolLogs, null, 2)); + + const createEventCall = toolLogs.find( + (log) => + log.toolRequest.name === 'mcp_workspace-server_calendar.createEvent', + ); + + expect(createEventCall).toBeDefined(); + + const args = JSON.parse(createEventCall!.toolRequest.args); + + expect(args?.start).toHaveProperty('date'); + expect(args?.start).not.toHaveProperty('dateTime'); + expect(args?.start?.date).toBe('2026-05-20'); + + expect(args?.end).toHaveProperty('date'); + expect(args?.end).not.toHaveProperty('dateTime'); + }, +}); diff --git a/packages/test-utils/assets/test-servers/google-workspace.json b/packages/test-utils/assets/test-servers/google-workspace.json index ceb46c0671..106be5ecca 100644 --- a/packages/test-utils/assets/test-servers/google-workspace.json +++ b/packages/test-utils/assets/test-servers/google-workspace.json @@ -684,9 +684,12 @@ "dateTime": { "type": "string", "description": "The start time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T10:30:00Z or 2024-01-15T10:30:00-05:00)." + }, + "date": { + "type": "string", + "description": "The date, in the format \"yyyy-mm-dd\", for an all-day event." } - }, - "required": ["dateTime"] + } }, "end": { "type": "object", @@ -694,9 +697,12 @@ "dateTime": { "type": "string", "description": "The end time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T11:30:00Z or 2024-01-15T11:30:00-05:00)." + }, + "date": { + "type": "string", + "description": "The date, in the format \"yyyy-mm-dd\", for an all-day event." } - }, - "required": ["dateTime"] + } }, "attendees": { "description": "The email addresses of the attendees.", @@ -881,9 +887,12 @@ "dateTime": { "type": "string", "description": "The new start time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T10:30:00Z or 2024-01-15T10:30:00-05:00)." + }, + "date": { + "type": "string", + "description": "The new date, in the format \"yyyy-mm-dd\", for an all-day event." } - }, - "required": ["dateTime"] + } }, "end": { "type": "object", @@ -891,9 +900,13 @@ "dateTime": { "type": "string", "description": "The new end time in strict ISO 8601 format with seconds and timezone (e.g., 2024-01-15T11:30:00Z or 2024-01-15T11:30:00-05:00)." + }, + "date": { + "type": "string", + "description": "The new date, in the format \"yyyy-mm-dd\", for an all-day event." } }, - "required": ["dateTime"] + "anyOf": [{ "required": ["dateTime"] }, { "required": ["date"] }] }, "attendees": { "description": "The new list of attendees for the event.",