|
6 | 6 | from random import randint |
7 | 7 |
|
8 | 8 | from samples.concepts.realtime.utils import AudioPlayerWebRTC, AudioRecorderWebRTC, check_audio_devices |
9 | | -from semantic_kernel import Kernel |
10 | 9 | from semantic_kernel.connectors.ai import FunctionChoiceBehavior |
11 | 10 | from semantic_kernel.connectors.ai.open_ai import ( |
12 | 11 | ListenEvents, |
13 | 12 | OpenAIRealtimeExecutionSettings, |
14 | 13 | OpenAIRealtimeWebRTC, |
15 | 14 | TurnDetection, |
16 | 15 | ) |
17 | | -from semantic_kernel.contents import ChatHistory |
18 | | -from semantic_kernel.contents.realtime_events import RealtimeTextEvent |
| 16 | +from semantic_kernel.contents import ChatHistory, RealtimeTextEvent |
19 | 17 | from semantic_kernel.functions import kernel_function |
20 | 18 |
|
21 | 19 | logging.basicConfig(level=logging.WARNING) |
|
26 | 24 |
|
27 | 25 | """ |
28 | 26 | This simple sample demonstrates how to use the OpenAI Realtime API to create |
29 | | -a chat bot that can listen and respond directly through audio. |
| 27 | +a agent that can listen and respond directly through audio. |
30 | 28 | It requires installing: |
31 | 29 | - semantic-kernel[realtime] |
32 | 30 | - pyaudio |
33 | 31 | - sounddevice |
34 | 32 | - pydub |
35 | | -e.g. pip install pyaudio sounddevice pydub semantic_kernel[realtime] |
| 33 | +e.g. pip install pyaudio sounddevice pydub semantic-kernel[realtime] |
36 | 34 |
|
37 | 35 | For more details of the exact setup, see the README.md in the realtime folder. |
38 | 36 | """ |
|
46 | 44 | check_audio_devices() |
47 | 45 |
|
48 | 46 |
|
49 | | -@kernel_function |
50 | | -def get_weather(location: str) -> str: |
51 | | - """Get the weather for a location.""" |
52 | | - weather_conditions = ("sunny", "hot", "cloudy", "raining", "freezing", "snowing") |
53 | | - weather = weather_conditions[randint(0, len(weather_conditions) - 1)] # nosec |
54 | | - logger.info(f"@ Getting weather for {location}: {weather}") |
55 | | - return f"The weather in {location} is {weather}." |
| 47 | +class Helpers: |
| 48 | + """A set of helper functions for the Realtime Agent.""" |
56 | 49 |
|
| 50 | + @kernel_function |
| 51 | + def get_weather(self, location: str) -> str: |
| 52 | + """Get the weather for a location.""" |
| 53 | + weather_conditions = ("sunny", "hot", "cloudy", "raining", "freezing", "snowing") |
| 54 | + weather = weather_conditions[randint(0, len(weather_conditions) - 1)] # nosec |
| 55 | + logger.info(f"@ Getting weather for {location}: {weather}") |
| 56 | + return f"The weather in {location} is {weather}." |
57 | 57 |
|
58 | | -@kernel_function |
59 | | -def get_date_time() -> str: |
60 | | - """Get the current date and time.""" |
61 | | - logger.info("@ Getting current datetime") |
62 | | - return f"The current date and time is {datetime.now().isoformat()}." |
| 58 | + @kernel_function |
| 59 | + def get_date_time(self) -> str: |
| 60 | + """Get the current date and time.""" |
| 61 | + logger.info("@ Getting current datetime") |
| 62 | + return f"The current date and time is {datetime.now().isoformat()}." |
63 | 63 |
|
64 | | - |
65 | | -@kernel_function |
66 | | -def goodbye(): |
67 | | - """When the user is done, say goodbye and then call this function.""" |
68 | | - logger.info("@ Goodbye has been called!") |
69 | | - raise KeyboardInterrupt |
| 64 | + @kernel_function |
| 65 | + def goodbye(self): |
| 66 | + """When the user is done, say goodbye and then call this function.""" |
| 67 | + logger.info("@ Goodbye has been called!") |
| 68 | + raise KeyboardInterrupt |
70 | 69 |
|
71 | 70 |
|
72 | 71 | async def main() -> None: |
73 | 72 | print_transcript = True |
74 | | - # create the Kernel and add a simple function for function calling. |
75 | | - kernel = Kernel() |
76 | | - kernel.add_functions(plugin_name="helpers", functions=[goodbye, get_weather, get_date_time]) |
77 | | - |
78 | 73 | # create the audio player and audio track |
79 | | - # both take a device_id parameter, which is the index of the device to use, if None the default device is used |
| 74 | + # both take a device_id parameter, which is the index of the device to use, |
| 75 | + # if None the default device will be used |
80 | 76 | audio_player = AudioPlayerWebRTC() |
81 | | - # create the realtime client and optionally add the audio output function, this is optional |
| 77 | + |
| 78 | + # create the realtime agent and optionally add the audio output function, this is optional |
82 | 79 | # and can also be passed in the receive method |
83 | | - realtime_client = OpenAIRealtimeWebRTC(audio_track=AudioRecorderWebRTC()) |
| 80 | + # You can also pass in kernel, plugins, chat_history or settings here. |
| 81 | + # For WebRTC the audio_track is required |
| 82 | + realtime_agent = OpenAIRealtimeWebRTC(audio_track=AudioRecorderWebRTC(), plugins=[Helpers()]) |
84 | 83 |
|
85 | 84 | # Create the settings for the session |
86 | 85 | # The realtime api, does not use a system message, but takes instructions as a parameter for a session |
@@ -111,14 +110,13 @@ async def main() -> None: |
111 | 110 | # the context manager calls the create_session method on the client and starts listening to the audio stream |
112 | 111 | async with ( |
113 | 112 | audio_player, |
114 | | - realtime_client( |
| 113 | + realtime_agent( |
115 | 114 | settings=settings, |
116 | 115 | chat_history=chat_history, |
117 | | - kernel=kernel, |
118 | 116 | create_response=True, |
119 | 117 | ), |
120 | 118 | ): |
121 | | - async for event in realtime_client.receive(audio_output_callback=audio_player.client_callback): |
| 119 | + async for event in realtime_agent.receive(audio_output_callback=audio_player.client_callback): |
122 | 120 | match event: |
123 | 121 | case RealtimeTextEvent(): |
124 | 122 | if print_transcript: |
|
0 commit comments