b60439152b
- harness/es.py: append-only event store + проекции (образец — кооп-портал) - harness/levels.py: уровни 1с/1м/1ч/1д, контекст ∝ масштабу - harness/scheduler.py: уровневые потоки, идемпотентные тики, absorbed-стабы - harness/isolated_runner.py + _llm_worker.py: LLM-тик в сабпроцессе (глобальный asyncio-рантайм Strands не переживает 2+ create_harness) - cli.py: init/task/tasks/status/events + requeue (написан самим минутным агентом) - 9router: отдельный ключ sk-simbiot, модель auto (glm-5.3 первым)
940 lines
41 KiB
Python
940 lines
41 KiB
Python
"""OpenAI Realtime API provider for Strands bidirectional streaming.
|
|
|
|
Provides real-time audio and text communication through OpenAI's Realtime API
|
|
with WebSocket connections, voice activity detection, and function calling.
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import copy
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
import uuid
|
|
from collections.abc import AsyncGenerator
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, cast
|
|
|
|
import websockets
|
|
from typing_extensions import Unpack, override
|
|
from websockets import ClientConnection
|
|
|
|
from ....types._events import ToolUseStreamEvent
|
|
from ....types.content import Messages, TextBlock
|
|
from ....types.media import ImageBlock
|
|
from ....types.tools import ToolResultBlock, ToolSpec, ToolUse
|
|
from .._async import stop_all
|
|
from ..types.content import BidiContentBlock, BidiContentDelta
|
|
from ..types.events import (
|
|
BidiAudioDeltaEvent,
|
|
BidiAudioStartEvent,
|
|
BidiAudioStopEvent,
|
|
BidiBargeInEvent,
|
|
BidiConnectionStartEvent,
|
|
BidiOutputEvent,
|
|
BidiResponseStartEvent,
|
|
BidiResponseStopEvent,
|
|
BidiTranscriptDeltaEvent,
|
|
BidiTranscriptStartEvent,
|
|
BidiTranscriptStopEvent,
|
|
BidiUsageEvent,
|
|
ModalityUsage,
|
|
Role,
|
|
)
|
|
from ..types.media import AudioDelta
|
|
from .configs import (
|
|
AudioConfig,
|
|
AudioStreamConfig,
|
|
ConnectionConfig,
|
|
ModelConfig,
|
|
ModelUpdateConfig,
|
|
_merge_config,
|
|
_validate_audio_config,
|
|
_validate_model_config,
|
|
)
|
|
from .model import AudioCapable, BidiModel, ConnectionTimeoutError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Test idle_timeout_ms
|
|
|
|
# OpenAI Realtime API configuration
|
|
OPENAI_MAX_TIMEOUT_S = 3000 # 50 minutes
|
|
"""Max timeout before closing connection.
|
|
|
|
OpenAI documents a 60 minute limit on realtime sessions
|
|
([docs](https://platform.openai.com/docs/guides/realtime-conversations#session-lifecycle-events)). However, OpenAI does
|
|
not emit any warnings when approaching the limit. As a workaround, we configure a max timeout client side to gracefully
|
|
handle the connection closure. We set the max to 50 minutes to provide enough buffer before hitting the real limit.
|
|
"""
|
|
# Proactive reconnect fires this many seconds below the reader's reactive timeout, leaving room for
|
|
# the turn-boundary alignment wait so a mid-turn swap stays graceful instead of being preempted by
|
|
# the reactive timeout firing at the same instant.
|
|
OPENAI_PROACTIVE_RECONNECT_MARGIN_S = 300
|
|
OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime"
|
|
DEFAULT_SAMPLE_RATE = 24000
|
|
|
|
DEFAULT_SESSION_CONFIG = {
|
|
"type": "realtime",
|
|
"instructions": "You are a helpful assistant. Please speak in English and keep your responses clear and concise.",
|
|
"output_modalities": ["audio"],
|
|
"audio": {
|
|
"input": {
|
|
"format": {"type": "audio/pcm", "rate": DEFAULT_SAMPLE_RATE},
|
|
"turn_detection": {
|
|
"type": "server_vad",
|
|
"threshold": 0.5,
|
|
"prefix_padding_ms": 300,
|
|
"silence_duration_ms": 500,
|
|
},
|
|
},
|
|
"output": {"format": {"type": "audio/pcm", "rate": DEFAULT_SAMPLE_RATE}},
|
|
},
|
|
}
|
|
|
|
# Sent as a system message after a restart. Replay restores the conversation text but not the
|
|
# live audio state, so the fresh session can drift language or re-introduce itself; this steers it
|
|
# to continue seamlessly.
|
|
_RESTART_INSTRUCTION = (
|
|
"The connection was re-established mid-conversation. Continue seamlessly from the prior "
|
|
"context: do not greet or re-introduce yourself, and keep replying in the language already in use."
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class _SessionState:
|
|
"""Connection-local transcript identities and response creation state."""
|
|
|
|
started_transcripts: set[str] = field(default_factory=set)
|
|
assistant_parts: dict[str, tuple[str, int]] = field(default_factory=dict)
|
|
audio_responses: set[str | None] = field(default_factory=set)
|
|
active_responses: set[str] = field(default_factory=set)
|
|
pending_tools: set[str] = field(default_factory=set)
|
|
response_pending: bool = False
|
|
response_requested: bool = False
|
|
transcription_enabled: bool = True
|
|
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
|
|
|
def start_audio(self, response_id: str | None) -> list[BidiOutputEvent]:
|
|
"""Open an audio stream before its first chunk."""
|
|
if response_id in self.audio_responses:
|
|
return []
|
|
self.audio_responses.add(response_id)
|
|
return [BidiAudioStartEvent()]
|
|
|
|
def stop_audio(self, response_id: str | None) -> list[BidiOutputEvent]:
|
|
"""Close an open audio stream once."""
|
|
if response_id not in self.audio_responses:
|
|
return []
|
|
self.audio_responses.remove(response_id)
|
|
return [BidiAudioStopEvent()]
|
|
|
|
def start_transcript(self, role: Role, content_id: str) -> list[BidiOutputEvent]:
|
|
"""Open a transcript once, when speech or its first text arrives."""
|
|
if content_id in self.started_transcripts:
|
|
return []
|
|
self.started_transcripts.add(content_id)
|
|
return [BidiTranscriptStartEvent(role, content_id=content_id)]
|
|
|
|
def transcript_events(self, event: BidiTranscriptDeltaEvent | BidiTranscriptStopEvent) -> list[BidiOutputEvent]:
|
|
"""Ensure the transcript starts before emitting its delta or stop."""
|
|
events = [*self.start_transcript(event.role, event.content_id), event]
|
|
if isinstance(event, BidiTranscriptStopEvent):
|
|
self.started_transcripts.remove(event.content_id)
|
|
return events
|
|
|
|
|
|
class OpenAIRealtimeModel(BidiModel, AudioCapable):
|
|
"""OpenAI Realtime API implementation for bidirectional streaming.
|
|
|
|
Combines model configuration and connection state in a single class.
|
|
Manages WebSocket connection to OpenAI's Realtime API with automatic VAD,
|
|
function calling, and event conversion to Strands format.
|
|
"""
|
|
|
|
_websocket: ClientConnection
|
|
_start_time: int
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
transcription_model_id: str | None,
|
|
api_key: str | None = None,
|
|
organization: str | None = None,
|
|
project: str | None = None,
|
|
timeout_s: int = OPENAI_MAX_TIMEOUT_S,
|
|
voice: str = "alloy",
|
|
**model_config: Unpack[ModelConfig],
|
|
) -> None:
|
|
"""Initialize OpenAI Realtime bidirectional model.
|
|
|
|
Args:
|
|
transcription_model_id: Input transcription model identifier. Pass ``None`` to disable user transcription.
|
|
api_key: OpenAI API key. Defaults to ``OPENAI_API_KEY``.
|
|
organization: OpenAI organization. Defaults to ``OPENAI_ORGANIZATION``.
|
|
project: OpenAI project. Defaults to ``OPENAI_PROJECT``.
|
|
timeout_s: Maximum connection duration in seconds.
|
|
voice: Output voice identifier. Defaults to ``alloy``.
|
|
**model_config: Model configuration.
|
|
|
|
Raises:
|
|
ValueError: If any of the following conditions apply:
|
|
|
|
- Required model configuration fields are missing.
|
|
- ``model_id`` is not a non-empty string.
|
|
- The API key is missing.
|
|
- ``timeout_s`` exceeds the maximum.
|
|
- The configured audio formats are unsupported.
|
|
"""
|
|
_validate_model_config(model_config)
|
|
self._config = ModelConfig(**model_config)
|
|
self._config["params"] = dict(self._config.get("params") or {})
|
|
|
|
# OpenAI reports per-response token usage on response.done, not cumulative session totals.
|
|
self.usage_is_cumulative = False
|
|
|
|
self.api_key = api_key if api_key is not None else os.getenv("OPENAI_API_KEY")
|
|
if not self.api_key:
|
|
raise ValueError(
|
|
"OpenAI API key is required. Provide via api_key or set OPENAI_API_KEY environment variable."
|
|
)
|
|
|
|
self.organization = organization if organization is not None else os.getenv("OPENAI_ORGANIZATION")
|
|
self.project = project if project is not None else os.getenv("OPENAI_PROJECT")
|
|
self.timeout_s = timeout_s
|
|
if timeout_s > OPENAI_MAX_TIMEOUT_S:
|
|
raise ValueError(
|
|
f"timeout_s=<{timeout_s}>, max_timeout_s=<{OPENAI_MAX_TIMEOUT_S}> | timeout exceeds max limit"
|
|
)
|
|
|
|
# OpenAI emits no approaching-limit warning, so reconnect proactively a margin below the
|
|
# reader's reactive timeout: the swap can then align to a turn boundary before the reactive
|
|
# path fires. Deriving from timeout_s keeps that headroom when a caller lowers it.
|
|
self._config["connection"] = ConnectionConfig(
|
|
**{
|
|
"restart_after_s": timeout_s - OPENAI_PROACTIVE_RECONNECT_MARGIN_S,
|
|
**self._config.get("connection", {}),
|
|
}
|
|
)
|
|
|
|
self._transcription_model_id = transcription_model_id
|
|
self._voice = voice
|
|
self._resolve_audio_config(self._config.get("params"))
|
|
|
|
# Connection state (initialized in start())
|
|
self._connection_id: str | None = None
|
|
|
|
self._function_call_buffer: dict[str, Any] = {}
|
|
self._session_state = _SessionState()
|
|
|
|
logger.debug("model=<%s> | openai realtime model initialized", self._config["model_id"])
|
|
|
|
@override
|
|
def update_config(self, **model_config: Unpack[ModelUpdateConfig]) -> None: # type: ignore[override]
|
|
"""Update the model configuration with the provided arguments.
|
|
|
|
Args:
|
|
**model_config: Configuration overrides.
|
|
|
|
Raises:
|
|
ValueError: If any of the following conditions apply:
|
|
|
|
- The resulting configuration is missing required fields.
|
|
- ``model_id`` is not a non-empty string.
|
|
- The configured audio formats are unsupported.
|
|
"""
|
|
_validate_model_config(self._config | model_config)
|
|
if "params" in model_config:
|
|
self._resolve_audio_config(model_config["params"])
|
|
self._config.update(model_config)
|
|
|
|
@override
|
|
def get_config(self) -> ModelConfig:
|
|
"""Return the model configuration by reference."""
|
|
return self._config
|
|
|
|
@override
|
|
def get_audio_config(self) -> AudioConfig:
|
|
"""Get the resolved audio configuration."""
|
|
return self._audio_config
|
|
|
|
def _resolve_audio_config(self, params: dict[str, Any] | None) -> None:
|
|
"""Resolve audio settings and validate native format overrides."""
|
|
audio = (params or {}).get("audio", {})
|
|
for direction in ("input", "output"):
|
|
stream = audio.get(direction, {})
|
|
audio_format = stream.get("format", {})
|
|
|
|
format_type = audio_format.get("type", "audio/pcm")
|
|
if format_type != "audio/pcm":
|
|
raise ValueError(f"Unsupported audio format: {format_type}. Expected audio/pcm.")
|
|
|
|
sample_rate = audio_format.get("rate", DEFAULT_SAMPLE_RATE)
|
|
if sample_rate != DEFAULT_SAMPLE_RATE:
|
|
raise ValueError(f"Unsupported sample rate: {sample_rate}. Expected {DEFAULT_SAMPLE_RATE}.")
|
|
|
|
self._audio_config = AudioConfig(
|
|
input=AudioStreamConfig(sample_rate=DEFAULT_SAMPLE_RATE, channels=1, format="pcm"),
|
|
output=AudioStreamConfig(sample_rate=DEFAULT_SAMPLE_RATE, channels=1, format="pcm"),
|
|
)
|
|
_validate_audio_config(self._audio_config)
|
|
|
|
async def start(
|
|
self,
|
|
system_prompt: str | None = None,
|
|
tools: list[ToolSpec] | None = None,
|
|
messages: Messages | None = None,
|
|
**kwargs: Any,
|
|
) -> None:
|
|
"""Establish bidirectional connection to OpenAI Realtime API.
|
|
|
|
Args:
|
|
system_prompt: System instructions for the model.
|
|
tools: List of tools available to the model.
|
|
messages: Conversation history to initialize with.
|
|
**kwargs: Additional configuration options.
|
|
"""
|
|
if self._connection_id:
|
|
raise RuntimeError("model already started | call stop before starting again")
|
|
|
|
logger.debug("openai realtime connection starting")
|
|
|
|
# Initialize connection state
|
|
self._connection_id = str(uuid.uuid4())
|
|
self._start_time = int(time.time())
|
|
|
|
self._function_call_buffer = {}
|
|
self._session_state = _SessionState()
|
|
|
|
# Establish WebSocket connection
|
|
url = f"{OPENAI_REALTIME_URL}?model={self._config['model_id']}"
|
|
|
|
headers = [("Authorization", f"Bearer {self.api_key}")]
|
|
if self.organization:
|
|
headers.append(("OpenAI-Organization", self.organization))
|
|
if self.project:
|
|
headers.append(("OpenAI-Project", self.project))
|
|
|
|
self._websocket = await websockets.connect(url, additional_headers=headers)
|
|
logger.debug("connection_id=<%s> | websocket connected successfully", self._connection_id)
|
|
|
|
# Configure session
|
|
session_config = self._build_session_config(system_prompt, tools)
|
|
self._session_state.transcription_enabled = session_config["audio"]["input"].get("transcription") is not None
|
|
await self._send_event({"type": "session.update", "session": session_config})
|
|
|
|
# Add conversation history if provided
|
|
if messages:
|
|
await self._add_conversation_history(messages)
|
|
|
|
def _build_session_config(self, system_prompt: str | None, tools: list[ToolSpec] | None) -> dict[str, Any]:
|
|
"""Build session configuration for OpenAI Realtime API.
|
|
|
|
Model params recursively override defaults and directly supplied options.
|
|
"""
|
|
config: dict[str, Any] = copy.deepcopy(DEFAULT_SESSION_CONFIG)
|
|
|
|
if system_prompt:
|
|
config["instructions"] = system_prompt
|
|
|
|
if tools:
|
|
config["tools"] = self._convert_tools_to_openai_format(tools)
|
|
|
|
config["audio"]["input"]["transcription"] = (
|
|
{"model": self._transcription_model_id} if self._transcription_model_id is not None else None
|
|
)
|
|
config["audio"]["output"]["voice"] = self._voice
|
|
|
|
return _merge_config(config, self._config.get("params") or {})
|
|
|
|
def _convert_tools_to_openai_format(self, tools: list[ToolSpec]) -> list[dict]:
|
|
"""Convert Strands tool specifications to OpenAI Realtime API format."""
|
|
openai_tools = []
|
|
|
|
for tool in tools:
|
|
input_schema = tool["inputSchema"]
|
|
if "json" in input_schema:
|
|
schema = (
|
|
json.loads(input_schema["json"]) if isinstance(input_schema["json"], str) else input_schema["json"]
|
|
)
|
|
else:
|
|
schema = input_schema
|
|
|
|
# OpenAI Realtime API expects flat structure, not nested under "function"
|
|
openai_tool = {
|
|
"type": "function",
|
|
"name": tool["name"],
|
|
"description": tool["description"],
|
|
"parameters": schema,
|
|
}
|
|
openai_tools.append(openai_tool)
|
|
|
|
return openai_tools
|
|
|
|
async def _add_conversation_history(self, messages: Messages) -> None:
|
|
"""Add conversation history to the session.
|
|
|
|
Converts agent message history to OpenAI Realtime API format using
|
|
conversation.item.create events for each message.
|
|
|
|
Note: OpenAI Realtime API has a 32-character limit on call_id, so we truncate
|
|
UUIDs consistently to ensure tool calls and their results match.
|
|
|
|
Args:
|
|
messages: List of conversation messages with role and content.
|
|
"""
|
|
# Track tool call IDs to ensure consistency between calls and results
|
|
call_id_map: dict[str, str] = {}
|
|
|
|
# First pass: collect all tool call IDs
|
|
for message in messages:
|
|
for block in message.get("content", []):
|
|
if "toolUse" in block:
|
|
tool_use = block["toolUse"]
|
|
original_id = tool_use["toolUseId"]
|
|
call_id = original_id[:32]
|
|
call_id_map[original_id] = call_id
|
|
|
|
# Second pass: send messages
|
|
for message in messages:
|
|
role = message["role"]
|
|
content_blocks = message.get("content", [])
|
|
|
|
# Build content array for OpenAI format
|
|
openai_content = []
|
|
|
|
for block in content_blocks:
|
|
if "text" in block:
|
|
# Text content - use appropriate type based on role
|
|
# User messages use "input_text", assistant messages use "output_text"
|
|
if role == "user":
|
|
openai_content.append({"type": "input_text", "text": block["text"]})
|
|
else: # assistant
|
|
openai_content.append({"type": "output_text", "text": block["text"]})
|
|
elif "toolUse" in block:
|
|
# Tool use - create as function_call item
|
|
tool_use = block["toolUse"]
|
|
original_id = tool_use["toolUseId"]
|
|
# Use pre-mapped call_id
|
|
call_id = call_id_map[original_id]
|
|
|
|
tool_item = {
|
|
"type": "conversation.item.create",
|
|
"item": {
|
|
"type": "function_call",
|
|
"call_id": call_id,
|
|
"name": tool_use["name"],
|
|
"arguments": json.dumps(tool_use["input"]),
|
|
},
|
|
}
|
|
await self._send_event(tool_item)
|
|
continue # Tool use is sent separately, not in message content
|
|
elif "toolResult" in block:
|
|
# Tool result - create as function_call_output item
|
|
tool_result = block["toolResult"]
|
|
original_id = tool_result["toolUseId"]
|
|
|
|
# Validate content types and serialize, preserving structure
|
|
result_output = ""
|
|
if "content" in tool_result:
|
|
# First validate all content types are supported
|
|
for result_block in tool_result["content"]:
|
|
if "text" not in result_block and "json" not in result_block:
|
|
# Unsupported content type - raise error
|
|
raise ValueError(
|
|
f"tool_use_id=<{original_id}>, content_types=<{list(result_block.keys())}> | "
|
|
f"Content type not supported by OpenAI Realtime API"
|
|
)
|
|
|
|
# Preserve structure by JSON-dumping the entire content array
|
|
result_output = json.dumps(tool_result["content"])
|
|
|
|
# Use mapped call_id if available, otherwise skip orphaned result
|
|
if original_id not in call_id_map:
|
|
continue # Skip this tool result since we don't have the call
|
|
|
|
call_id = call_id_map[original_id]
|
|
|
|
result_item = {
|
|
"type": "conversation.item.create",
|
|
"item": {
|
|
"type": "function_call_output",
|
|
"call_id": call_id,
|
|
"output": result_output,
|
|
},
|
|
}
|
|
await self._send_event(result_item)
|
|
continue # Tool result is sent separately, not in message content
|
|
|
|
# Only create message item if there's text content
|
|
if openai_content:
|
|
conversation_item = {
|
|
"type": "conversation.item.create",
|
|
"item": {"type": "message", "role": role, "content": openai_content},
|
|
}
|
|
await self._send_event(conversation_item)
|
|
|
|
logger.debug("message_count=<%d> | conversation history added to openai session", len(messages))
|
|
|
|
async def receive(self) -> AsyncGenerator[BidiOutputEvent, None]:
|
|
"""Receive OpenAI events and convert to Strands TypedEvent format."""
|
|
if not self._connection_id:
|
|
raise RuntimeError("model not started | call start before receiving")
|
|
|
|
yield BidiConnectionStartEvent(connection_id=self._connection_id, model=self._config["model_id"])
|
|
|
|
# Bind this reader to the connection it started on. After a reconnect swaps self._websocket,
|
|
# a still-draining superseded reader keeps reading its own (now-closed) socket rather than
|
|
# stealing messages from the connection that replaced it.
|
|
websocket = self._websocket
|
|
start_time = self._start_time
|
|
state = self._session_state
|
|
|
|
while True:
|
|
duration = time.time() - start_time
|
|
if duration >= self.timeout_s:
|
|
raise ConnectionTimeoutError(f"timeout_s=<{self.timeout_s}>")
|
|
|
|
try:
|
|
message = await asyncio.wait_for(websocket.recv(), timeout=10)
|
|
except asyncio.TimeoutError:
|
|
continue
|
|
|
|
openai_event = json.loads(message)
|
|
event_type = openai_event.get("type")
|
|
if event_type == "response.created":
|
|
response_id = openai_event["response"]["id"]
|
|
if response_id in state.active_responses:
|
|
continue
|
|
state.active_responses.add(response_id)
|
|
state.response_requested = False
|
|
elif event_type == "response.done":
|
|
response_id = openai_event["response"]["id"]
|
|
if response_id not in state.active_responses:
|
|
continue
|
|
state.active_responses.remove(response_id)
|
|
|
|
if event_type == "error" and openai_event.get("error", {}).get("code") == (
|
|
"conversation_already_has_active_response"
|
|
):
|
|
state.response_requested = False
|
|
state.response_pending = True
|
|
continue
|
|
|
|
for event in self._convert_openai_event(openai_event, state) or []:
|
|
if isinstance(event, ToolUseStreamEvent):
|
|
state.pending_tools.add(event["current_tool_use"]["toolUseId"])
|
|
yield event
|
|
if state is self._session_state and event_type == "response.done":
|
|
await self._flush_response_request(state)
|
|
|
|
def _convert_openai_event(
|
|
self, openai_event: dict[str, Any], state: _SessionState | None = None
|
|
) -> list[BidiOutputEvent] | None:
|
|
"""Convert OpenAI events to Strands TypedEvent format."""
|
|
event_type = openai_event.get("type")
|
|
state = state if state is not None else self._session_state
|
|
|
|
if event_type == "input_audio_buffer.speech_started":
|
|
events: list[BidiOutputEvent] = [BidiBargeInEvent(reason="user_speech")]
|
|
if state.transcription_enabled:
|
|
events.extend(state.start_transcript("user", openai_event["item_id"]))
|
|
return events
|
|
|
|
input_id = None
|
|
if event_type == "input_audio_buffer.committed" and state.transcription_enabled:
|
|
input_id = openai_event.get("item_id")
|
|
elif event_type == "conversation.item.added" and openai_event.get("item", {}).get("role") == "user":
|
|
item = openai_event["item"]
|
|
is_audio = any(content.get("type") == "input_audio" for content in item.get("content", []))
|
|
if state.transcription_enabled and is_audio:
|
|
input_id = item["id"]
|
|
if input_id is not None:
|
|
return state.start_transcript("user", input_id) or None
|
|
|
|
if event_type == "response.created":
|
|
response = openai_event.get("response", {})
|
|
response_id = response.get("id", str(uuid.uuid4()))
|
|
return [BidiResponseStartEvent(response_id=response_id)]
|
|
|
|
if event_type == "response.content_part.added" and openai_event.get("part", {}).get("type") == "audio":
|
|
return state.start_audio(openai_event.get("response_id")) or None
|
|
|
|
if event_type == "response.output_audio.delta":
|
|
return [
|
|
*state.start_audio(openai_event.get("response_id")),
|
|
BidiAudioDeltaEvent(
|
|
audio=openai_event["delta"],
|
|
**self._audio_config["output"],
|
|
),
|
|
]
|
|
|
|
if event_type == "response.output_audio.done":
|
|
return state.stop_audio(openai_event.get("response_id")) or None
|
|
|
|
if event_type in ("response.output_text.delta", "response.output_audio_transcript.delta"):
|
|
text = openai_event.get("delta", "")
|
|
if not text:
|
|
return None
|
|
response_id = openai_event["response_id"]
|
|
part_id = (openai_event["item_id"], openai_event["content_index"])
|
|
previous_part = state.assistant_parts.get(response_id)
|
|
if previous_part is not None and previous_part != part_id:
|
|
text = "\n\n" + text
|
|
state.assistant_parts[response_id] = part_id
|
|
return state.transcript_events(BidiTranscriptDeltaEvent(text, "assistant", content_id=response_id))
|
|
|
|
if event_type in ("response.output_text.done", "response.output_audio_transcript.done"):
|
|
# Response completion closes the combined assistant transcript.
|
|
return None
|
|
|
|
if event_type in (
|
|
"conversation.item.input_audio_transcription.delta",
|
|
"conversation.item.input_audio_transcription.segment",
|
|
):
|
|
if event_type == "conversation.item.input_audio_transcription.segment":
|
|
segment = openai_event.get("segment", {})
|
|
text = segment.get("text", "")
|
|
role = cast(Role, segment.get("role", "user"))
|
|
else:
|
|
text = openai_event.get("delta", "")
|
|
role = "user"
|
|
if not text:
|
|
return None
|
|
return state.transcript_events(BidiTranscriptDeltaEvent(text, role, content_id=openai_event["item_id"]))
|
|
|
|
if event_type == "conversation.item.input_audio_transcription.completed":
|
|
return state.transcript_events(
|
|
BidiTranscriptStopEvent(openai_event["transcript"], "user", content_id=openai_event["item_id"])
|
|
)
|
|
|
|
if event_type == "conversation.item.input_audio_transcription.failed":
|
|
error_info = openai_event.get("error", {})
|
|
raise RuntimeError(error_info.get("message", "Transcription failed."))
|
|
|
|
if event_type == "response.function_call_arguments.delta":
|
|
call_id = openai_event.get("call_id")
|
|
delta = openai_event.get("delta", "")
|
|
if call_id:
|
|
if call_id not in self._function_call_buffer:
|
|
self._function_call_buffer[call_id] = {"call_id": call_id, "name": "", "arguments": delta}
|
|
else:
|
|
self._function_call_buffer[call_id]["arguments"] += delta
|
|
return None
|
|
|
|
if event_type == "response.function_call_arguments.done":
|
|
call_id = openai_event.get("call_id")
|
|
if call_id and call_id in self._function_call_buffer:
|
|
function_call = self._function_call_buffer[call_id]
|
|
try:
|
|
tool_use: ToolUse = {
|
|
"toolUseId": call_id,
|
|
"name": function_call["name"],
|
|
"input": json.loads(function_call["arguments"]) if function_call["arguments"] else {},
|
|
}
|
|
del self._function_call_buffer[call_id]
|
|
# Return ToolUseStreamEvent for consistency with standard agent
|
|
return [
|
|
ToolUseStreamEvent(
|
|
delta={
|
|
"toolUse": {
|
|
"toolUseId": tool_use["toolUseId"],
|
|
"name": tool_use["name"],
|
|
"input": json.dumps(tool_use["input"]),
|
|
}
|
|
},
|
|
current_tool_use=dict(tool_use),
|
|
)
|
|
]
|
|
except (json.JSONDecodeError, KeyError) as error:
|
|
logger.warning("call_id=<%s>, error=<%s> | error parsing function arguments", call_id, error)
|
|
del self._function_call_buffer[call_id]
|
|
return None
|
|
|
|
if event_type == "response.done":
|
|
return self._complete_response(openai_event.get("response", {}), state)
|
|
|
|
if event_type in ("conversation.item.retrieve", "conversation.item.added"):
|
|
item = openai_event.get("item", {})
|
|
action = "retrieved" if "retrieve" in event_type else "added"
|
|
logger.debug("action=<%s>, item_id=<%s> | openai conversation item event", action, item.get("id"))
|
|
return None
|
|
|
|
if event_type == "conversation.item.done":
|
|
logger.debug("item_id=<%s> | openai conversation item done", openai_event.get("item", {}).get("id"))
|
|
return None
|
|
|
|
if event_type in (
|
|
"response.output_item.added",
|
|
"response.output_item.done",
|
|
"response.content_part.added",
|
|
"response.content_part.done",
|
|
):
|
|
item_data = openai_event.get("item") or openai_event.get("part")
|
|
logger.debug(
|
|
"event_type=<%s>, item_id=<%s> | openai output event",
|
|
event_type,
|
|
item_data.get("id") if item_data else "unknown",
|
|
)
|
|
|
|
# Track function call names from response.output_item.added
|
|
if event_type == "response.output_item.added":
|
|
item = openai_event.get("item", {})
|
|
if item.get("type") == "function_call":
|
|
call_id = item.get("call_id")
|
|
function_name = item.get("name")
|
|
if call_id and function_name:
|
|
if call_id not in self._function_call_buffer:
|
|
self._function_call_buffer[call_id] = {
|
|
"call_id": call_id,
|
|
"name": function_name,
|
|
"arguments": "",
|
|
}
|
|
else:
|
|
self._function_call_buffer[call_id]["name"] = function_name
|
|
return None
|
|
|
|
if event_type in (
|
|
"input_audio_buffer.committed",
|
|
"input_audio_buffer.cleared",
|
|
"session.created",
|
|
"session.updated",
|
|
):
|
|
logger.debug("event_type=<%s> | openai event received", event_type)
|
|
return None
|
|
|
|
if event_type == "error":
|
|
error_data = openai_event.get("error", {})
|
|
error_code = error_data.get("code", "")
|
|
|
|
# Suppress expected errors that don't affect session state
|
|
if error_code == "response_cancel_not_active":
|
|
# This happens when trying to cancel a response that's not active
|
|
# It's safe to ignore as the session remains functional
|
|
logger.debug("openai response cancel attempted when no response active")
|
|
return None
|
|
|
|
# Log other errors
|
|
logger.error("error=<%s> | openai realtime error", error_data)
|
|
return None
|
|
|
|
logger.debug("event_type=<%s> | unhandled openai event type", event_type)
|
|
return None
|
|
|
|
def _complete_response(self, response: dict[str, Any], state: _SessionState) -> list[BidiOutputEvent]:
|
|
"""Close audio and the combined assistant transcript before stopping the response."""
|
|
response_id = response.get("id", "unknown")
|
|
output = response.get("output", [])
|
|
events = state.stop_audio(response.get("id"))
|
|
transcript_parts = [
|
|
part.get("transcript", part.get("text", ""))
|
|
for item in output
|
|
if item.get("type") == "message" and item.get("role") == "assistant"
|
|
for part in item.get("content", [])
|
|
if part.get("type") in ("output_audio", "output_text")
|
|
]
|
|
if transcript_parts or response_id in state.assistant_parts:
|
|
events.extend(
|
|
state.transcript_events(
|
|
BidiTranscriptStopEvent("\n\n".join(transcript_parts), "assistant", content_id=response_id)
|
|
)
|
|
)
|
|
state.assistant_parts.pop(response_id, None)
|
|
|
|
events.append(BidiResponseStopEvent(response_id=response_id))
|
|
|
|
if usage := response.get("usage"):
|
|
events.append(self._convert_usage_metadata(usage))
|
|
return events
|
|
|
|
def _convert_usage_metadata(self, usage: dict[str, Any]) -> BidiUsageEvent:
|
|
"""Convert response token counts and modality details into a usage event."""
|
|
input_details = usage.get("input_token_details", {})
|
|
output_details = usage.get("output_token_details", {})
|
|
modality_details: list[dict[str, Any]] = []
|
|
for modality in ("text", "audio"):
|
|
input_tokens = input_details.get(f"{modality}_tokens", 0)
|
|
output_tokens = output_details.get(f"{modality}_tokens", 0)
|
|
if input_tokens > 0 or output_tokens > 0:
|
|
modality_details.append(
|
|
{"modality": modality, "input_tokens": input_tokens, "output_tokens": output_tokens}
|
|
)
|
|
|
|
image_tokens = input_details.get("image_tokens", 0)
|
|
if image_tokens > 0:
|
|
modality_details.append({"modality": "image", "input_tokens": image_tokens, "output_tokens": 0})
|
|
|
|
cached_tokens = input_details.get("cached_tokens", 0)
|
|
return BidiUsageEvent(
|
|
input_tokens=usage.get("input_tokens", 0),
|
|
output_tokens=usage.get("output_tokens", 0),
|
|
total_tokens=usage.get("total_tokens", 0),
|
|
modality_details=cast(list[ModalityUsage], modality_details) if modality_details else None,
|
|
cache_read_input_tokens=cached_tokens if cached_tokens > 0 else None,
|
|
)
|
|
|
|
async def send(
|
|
self,
|
|
content: BidiContentBlock | BidiContentDelta | ToolResultBlock,
|
|
) -> None:
|
|
"""Unified send method for all content types. Sends the given content to OpenAI.
|
|
|
|
Dispatches to appropriate internal handler based on content type.
|
|
|
|
Args:
|
|
content: A TextBlock, AudioDelta, ImageBlock, or ToolResultBlock.
|
|
|
|
Raises:
|
|
ValueError: If content type not supported.
|
|
"""
|
|
if not self._connection_id:
|
|
raise RuntimeError("model not started | call start before sending")
|
|
|
|
if isinstance(content, TextBlock):
|
|
await self._send_text_content(content.text)
|
|
elif isinstance(content, AudioDelta):
|
|
await self._send_audio_content(content)
|
|
elif isinstance(content, ImageBlock):
|
|
await self._send_image_content(content)
|
|
elif isinstance(content, ToolResultBlock):
|
|
await self._send_tool_result(content)
|
|
else:
|
|
raise ValueError(f"content_type={type(content)} | content not supported")
|
|
|
|
async def _send_audio_content(self, audio_input: AudioDelta) -> None:
|
|
"""Internal: Send audio content to OpenAI for processing."""
|
|
audio_bytes = audio_input.source.get("bytes")
|
|
if audio_bytes is None:
|
|
raise ValueError("audio source must contain bytes for OpenAI Realtime")
|
|
audio = base64.b64encode(audio_bytes).decode("utf-8")
|
|
await self._send_event({"type": "input_audio_buffer.append", "audio": audio})
|
|
|
|
async def _send_image_content(self, image_input: ImageBlock) -> None:
|
|
"""Internal: Send image content to OpenAI for processing.
|
|
|
|
Image data is encoded as a ``data:`` URL using the image format and base64
|
|
payload, matching OpenAI's Realtime API image input format.
|
|
"""
|
|
image_bytes = image_input.source.get("bytes")
|
|
if image_bytes is None:
|
|
raise ValueError("image source must contain bytes for OpenAI Realtime")
|
|
image = base64.b64encode(image_bytes).decode("utf-8")
|
|
data_url = f"data:image/{image_input.format};base64,{image}"
|
|
item_data = {
|
|
"type": "message",
|
|
"role": "user",
|
|
"content": [{"type": "input_image", "image_url": data_url}],
|
|
}
|
|
await self._send_event({"type": "conversation.item.create", "item": item_data})
|
|
|
|
async def _send_text_content(self, text: str) -> None:
|
|
"""Internal: Send text content to OpenAI for processing."""
|
|
item_data = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}
|
|
await self._send_event({"type": "conversation.item.create", "item": item_data})
|
|
await self._request_response()
|
|
|
|
async def _send_tool_result(self, tool_result: ToolResultBlock) -> None:
|
|
"""Internal: Send tool result back to OpenAI."""
|
|
tool_use_id = tool_result.tool_use_id
|
|
|
|
logger.debug("tool_use_id=<%s> | sending openai tool result", tool_use_id)
|
|
|
|
# Validate content types and serialize, preserving structure
|
|
for block in tool_result.content:
|
|
if "text" not in block and "json" not in block:
|
|
# Unsupported content type - raise error
|
|
raise ValueError(
|
|
f"tool_use_id=<{tool_use_id}>, content_types=<{list(block.keys())}> | "
|
|
f"Content type not supported by OpenAI Realtime API"
|
|
)
|
|
|
|
# Preserve structure by JSON-dumping the entire content array
|
|
result_output = json.dumps(tool_result.content)
|
|
|
|
item_data = {"type": "function_call_output", "call_id": tool_use_id, "output": result_output}
|
|
await self._send_event({"type": "conversation.item.create", "item": item_data})
|
|
self._session_state.pending_tools.discard(tool_use_id)
|
|
await self._request_response()
|
|
|
|
async def _request_response(self) -> None:
|
|
"""Coalesce continuation requests while a native response or tool group is active."""
|
|
state = self._session_state
|
|
state.response_pending = True
|
|
await self._flush_response_request(state)
|
|
|
|
async def _flush_response_request(self, state: _SessionState) -> None:
|
|
async with state.lock:
|
|
if not state.response_pending or state.response_requested or state.active_responses or state.pending_tools:
|
|
return
|
|
state.response_pending = False
|
|
state.response_requested = True
|
|
try:
|
|
await self._send_event({"type": "response.create"})
|
|
except BaseException:
|
|
state.response_requested = False
|
|
state.response_pending = True
|
|
raise
|
|
|
|
async def stop(self) -> None:
|
|
"""Close session and cleanup resources."""
|
|
logger.debug("openai realtime connection cleanup starting")
|
|
|
|
async def stop_websocket() -> None:
|
|
if not hasattr(self, "_websocket"):
|
|
return
|
|
|
|
await self._websocket.close()
|
|
|
|
async def stop_connection() -> None:
|
|
self._connection_id = None
|
|
|
|
await stop_all(stop_websocket, stop_connection)
|
|
|
|
logger.debug("openai realtime connection closed")
|
|
|
|
async def restart(
|
|
self,
|
|
system_prompt: str | None = None,
|
|
tools: list[ToolSpec] | None = None,
|
|
messages: Messages | None = None,
|
|
**restart_kwargs: Any,
|
|
) -> None:
|
|
"""Restart by closing the connection and starting a new one, replaying history.
|
|
|
|
OpenAI's Realtime API exposes no server-side resume handle, so a restart re-establishes
|
|
the session and replays the accumulated conversation history to preserve context across the
|
|
swap.
|
|
|
|
Args:
|
|
system_prompt: System instructions for the new connection.
|
|
tools: Tool specifications for the new connection.
|
|
messages: Conversation history to replay into the new connection.
|
|
**restart_kwargs: Reserved for provider-specific restart options.
|
|
"""
|
|
logger.debug("openai realtime restart starting")
|
|
await self.stop()
|
|
await self.start(system_prompt, tools, messages, **restart_kwargs)
|
|
# Re-anchor the fresh session so it continues the conversation rather than drifting. This is
|
|
# a best-effort nudge: if the send fails, the connection is still healthy, so log and move on
|
|
# rather than let a failed nudge tear down the session.
|
|
try:
|
|
await self._send_event(
|
|
{
|
|
"type": "conversation.item.create",
|
|
"item": {
|
|
"type": "message",
|
|
"role": "system",
|
|
"content": [{"type": "input_text", "text": _RESTART_INSTRUCTION}],
|
|
},
|
|
}
|
|
)
|
|
except Exception as error:
|
|
logger.warning("error=<%s> | failed to send restart re-anchor message | continuing", error)
|
|
logger.debug("connection_id=<%s> | openai realtime restart complete", self._connection_id)
|
|
|
|
async def _send_event(self, event: dict[str, Any]) -> None:
|
|
"""Send event to OpenAI via WebSocket."""
|
|
message = json.dumps(event)
|
|
await self._websocket.send(message)
|
|
logger.debug("event_type=<%s> | openai event sent", event.get("type"))
|