fix: use web standard event apis for twilio websocket

.changeset/shiny-berries-kiss.md

@@ -0,0 +1,7 @@
+---
+'@openai/agents': patch
+'@openai/agents-extensions': patch
+'@openai/agents-realtime': patch
+---
+
+fix: use web standard event apis for twilio websocket

examples/docs/extensions/twilio-basic.ts

@@ -9,7 +9,7 @@ const agent = new RealtimeAgent({
 // the OpenAI Realtime API.
 const twilioTransport = new TwilioRealtimeTransportLayer({
   // @ts-expect-error - this is not defined
-  twilioWebSocket: websoketConnection,
+  twilioWebSocket: websocketConnection,
 });
 
 const session = new RealtimeSession(agent, {

packages/agents-extensions/src/TwilioRealtimeTransport.ts

@@ -7,7 +7,13 @@ import {
   RealtimeSessionConfig,
 } from '@openai/agents/realtime';
 import { getLogger } from '@openai/agents';
-import type { WebSocket, MessageEvent } from 'ws';
+import type {
+  WebSocket as NodeWebSocket,
+  MessageEvent as NodeMessageEvent,
+  ErrorEvent as NodeErrorEvent,
+} from 'ws';
+
+import type { ErrorEvent } from 'undici-types';
 
 /**
  * The options for the Twilio Realtime Transport Layer.
@@ -18,7 +24,7 @@ export type TwilioRealtimeTransportLayerOptions =
      * The websocket that is receiving messages from Twilio's Media Streams API. Typically the
      * connection gets passed into your request handler when running your WebSocket server.
      */
-    twilioWebSocket: WebSocket;
+    twilioWebSocket: WebSocket | NodeWebSocket;
   };
 
 /**
@@ -48,7 +54,7 @@ export type TwilioRealtimeTransportLayerOptions =
  * ```
  */
 export class TwilioRealtimeTransportLayer extends OpenAIRealtimeWebSocket {
-  #twilioWebSocket: WebSocket;
+  #twilioWebSocket: WebSocket | NodeWebSocket;
   #streamSid: string | null = null;
   #audioChunkCount: number = 0;
   #lastPlayedChunkCount: number = 0;
@@ -82,74 +88,80 @@ export class TwilioRealtimeTransportLayer extends OpenAIRealtimeWebSocket {
       options.initialSessionConfig,
     );
     // listen to Twilio messages as quickly as possible
-    this.#twilioWebSocket.on('message', (message: MessageEvent) => {
-      try {
-        const data = JSON.parse(message.toString());
-        if (this.#logger.dontLogModelData) {
-          this.#logger.debug('Twilio message:', data.event);
-        } else {
-          this.#logger.debug('Twilio message:', data);
-        }
-        this.emit('*', {
-          type: 'twilio_message',
-          message: data,
-        });
-        switch (data.event) {
-          case 'media':
-            if (this.status === 'connected') {
-              this.sendAudio(utils.base64ToArrayBuffer(data.media.payload));
-            }
-            break;
-          case 'mark':
-            if (
-              !data.mark.name.startsWith('done:') &&
-              data.mark.name.includes(':')
-            ) {
-              // keeping track of what the last chunk was that the user heard fully
-              const count = Number(data.mark.name.split(':')[1]);
-              if (Number.isFinite(count)) {
-                this.#lastPlayedChunkCount = count;
-              } else {
-                this.#logger.warn(
-                  'Invalid mark name received:',
-                  data.mark.name,
-                );
+    this.#twilioWebSocket.addEventListener(
+      'message',
+      (message: MessageEvent | NodeMessageEvent) => {
+        try {
+          const data = JSON.parse(message.data.toString());
+          if (this.#logger.dontLogModelData) {
+            this.#logger.debug('Twilio message:', data.event);
+          } else {
+            this.#logger.debug('Twilio message:', data);
+          }
+          this.emit('*', {
+            type: 'twilio_message',
+            message: data,
+          });
+          switch (data.event) {
+            case 'media':
+              if (this.status === 'connected') {
+                this.sendAudio(utils.base64ToArrayBuffer(data.media.payload));
               }
-            } else if (data.mark.name.startsWith('done:')) {
-              this.#lastPlayedChunkCount = 0;
-            }
-            break;
-          case 'start':
-            this.#streamSid = data.start.streamSid;
-            break;
-          default:
-            break;
+              break;
+            case 'mark':
+              if (
+                !data.mark.name.startsWith('done:') &&
+                data.mark.name.includes(':')
+              ) {
+                // keeping track of what the last chunk was that the user heard fully
+                const count = Number(data.mark.name.split(':')[1]);
+                if (Number.isFinite(count)) {
+                  this.#lastPlayedChunkCount = count;
+                } else {
+                  this.#logger.warn(
+                    'Invalid mark name received:',
+                    data.mark.name,
+                  );
+                }
+              } else if (data.mark.name.startsWith('done:')) {
+                this.#lastPlayedChunkCount = 0;
+              }
+              break;
+            case 'start':
+              this.#streamSid = data.start.streamSid;
+              break;
+            default:
+              break;
+          }
+        } catch (error) {
+          this.#logger.error(
+            'Error parsing message:',
+            error,
+            'Message:',
+            message,
+          );
+          this.emit('error', {
+            type: 'error',
+            error,
+          });
         }
-      } catch (error) {
-        this.#logger.error(
-          'Error parsing message:',
-          error,
-          'Message:',
-          message,
-        );
+      },
+    );
+    this.#twilioWebSocket.addEventListener('close', () => {
+      if (this.status !== 'disconnected') {
+        this.close();
+      }
+    });
+    this.#twilioWebSocket.addEventListener(
+      'error',
+      (error: ErrorEvent | NodeErrorEvent) => {
         this.emit('error', {
           type: 'error',
           error,
         });
-      }
-    });
-    this.#twilioWebSocket.on('close', () => {
-      if (this.status !== 'disconnected') {
         this.close();
-      }
-    });
-    this.#twilioWebSocket.on('error', (error) => {
-      this.emit('error', {
-        type: 'error',
-        error,
-      });
-      this.close();
-    });
+      },
+    );
     this.on('audio_done', () => {
       this.#twilioWebSocket.send(
         JSON.stringify({

packages/agents-extensions/test/TwilioRealtimeTransport.test.ts

@@ -2,6 +2,9 @@ import { describe, test, expect, vi, beforeEach } from 'vitest';
 import { EventEmitter } from 'events';
 import { TwilioRealtimeTransportLayer } from '../src/TwilioRealtimeTransport';
 
+import type { MessageEvent as NodeMessageEvent } from 'ws';
+import type { MessageEvent } from 'undici-types';
+
 vi.mock('@openai/agents/realtime', () => {
   // eslint-disable-next-line @typescript-eslint/no-require-imports
   const { EventEmitter } = require('events');
@@ -31,6 +34,14 @@ class FakeTwilioWebSocket extends EventEmitter {
   close = vi.fn();
 }
 
+// @ts-expect-error - we're making the node event emitter compatible with the browser event emitter
+FakeTwilioWebSocket.prototype.addEventListener = function (
+  type: string,
+  listener: (evt: MessageEvent | NodeMessageEvent) => void,
+) {
+  this.on(type, (evt) => listener(type === 'message' ? { data: evt } : evt));
+};
+
 const base64 = (data: string) => Buffer.from(data).toString('base64');
 
 describe('TwilioRealtimeTransportLayer', () => {

packages/agents-extensions/test/index.test.ts

@@ -1,6 +1,7 @@
 import { describe, test, expect, vi } from 'vitest';
 import { EventEmitter } from 'events';
 import { TwilioRealtimeTransportLayer } from '../src';
+import type { MessageEvent as NodeMessageEvent } from 'ws';
 
 vi.mock('ws', () => {
   class FakeWebSocket {
@@ -30,6 +31,14 @@ class FakeTwilioWebSocket extends EventEmitter {
   close = vi.fn();
 }
 
+// @ts-expect-error - we're making the node event emitter compatible with the browser event emitter
+FakeTwilioWebSocket.prototype.addEventListener = function (
+  type: string,
+  listener: (evt: MessageEvent | NodeMessageEvent) => void,
+) {
+  this.on(type, (evt) => listener(type === 'message' ? { data: evt } : evt));
+};
+
 describe('TwilioRealtimeTransportLayer', () => {
   test('should be available', () => {
     const transport = new TwilioRealtimeTransportLayer({

packages/agents-realtime/package.json

@@ -23,6 +23,11 @@
       "default": "./dist/index.mjs"
     },
     "./_shims": {
+      "workerd": {
+        "require": "./dist/shims/shims-workerd.js",
+        "types": "./dist/shims/shims-workerd.d.ts",
+        "default": "./dist/shims/shims-workerd.mjs"
+      },
       "browser": {
         "require": "./dist/shims/shims-browser.js",
         "types": "./dist/shims/shims-browser.d.ts",

packages/agents-realtime/src/openaiRealtimeWebsocket.ts

@@ -1,5 +1,6 @@
 import {
   isBrowserEnvironment,
+  useWebSocketProtocols,
   WebSocket,
 } from '@openai/agents-realtime/_shims';
 import {
@@ -149,7 +150,8 @@ export class OpenAIRealtimeWebSocket
       );
     }
 
-    const websocketArguments = isBrowserEnvironment()
+    // browsers and workerd should use the protocols argument, node should use the headers argument
+    const websocketArguments = useWebSocketProtocols
       ? [
           'realtime',
           // Auth

packages/agents-realtime/src/shims/shims-browser.ts

@@ -4,3 +4,4 @@ export const WebSocket = globalThis.WebSocket;
 export function isBrowserEnvironment(): boolean {
   return true;
 }
+export const useWebSocketProtocols = true;

packages/agents-realtime/src/shims/shims-node.ts

@@ -2,3 +2,4 @@ export { WebSocket } from 'ws';
 export function isBrowserEnvironment(): boolean {
   return false;
 }
+export const useWebSocketProtocols = false;

packages/agents-realtime/src/shims/shims-workerd.ts

@@ -0,0 +1,5 @@
+export const WebSocket = globalThis.WebSocket;
+export function isBrowserEnvironment(): boolean {
+  return false;
+}
+export const useWebSocketProtocols = true;

pnpm-workspace.yaml

@@ -2,7 +2,10 @@ packages:
   - packages/*
   - examples/*
   - docs
+
 onlyBuiltDependencies:
+  - '@tailwindcss/oxide'
   - esbuild
   - sharp
+
 publishBranch: main