"""Send and receive audio data from devices. Reads user audio from input device and sends agent audio to output device using PyAudio. If a user barges in, the output buffer is cleared to stop playback. Audio configuration is provided by models that implement ``AudioCapable``. Optional microphone audio processing (acoustic echo cancellation, noise suppression, and automatic gain control) is enabled by passing ``audio_processor=True`` or an ``AudioProcessorConfig`` to ``AudioIO``. It requires pywebrtc-audio (pip install strands-agents[bidi-aec]). """ import asyncio import base64 import logging from typing import TYPE_CHECKING, Any import pyaudio from typing_extensions import Unpack from .._audio.buffer import AudioBuffer from ..models.configs import AudioStreamConfig from ..models.model import AudioCapable from ..types.events import ( BidiAudioDeltaEvent, BidiBargeInEvent, BidiOutputEvent, ) from ..types.io import InputStream, OutputStream from ..types.media import AudioDelta from .configs import AudioIOConfig, AudioProcessorConfig from .transcript import _TranscriptOutputStream if TYPE_CHECKING: from .._audio.processor import AudioProcessor from ..agent.agent import BidiAgent logger = logging.getLogger(__name__) class _AudioInputStream(InputStream): """Handle audio input from user. Attributes: _audio: PyAudio instance for audio system access. _stream: Audio input stream. _buffer: Buffer for sharing audio data between agent and PyAudio. """ _audio: pyaudio.PyAudio _stream: pyaudio.Stream _BUFFER_SIZE = None _DEVICE_INDEX = None _FRAMES_PER_BUFFER = 512 def __init__( self, config: AudioIOConfig, *, audio_processor: "AudioProcessor | None", ) -> None: """Initialize input settings. Args: config: Audio device configuration. audio_processor: Shared microphone audio processor. """ self._buffer_size = config.get("input_buffer_size", _AudioInputStream._BUFFER_SIZE) self._device_index = config.get("input_device_index", _AudioInputStream._DEVICE_INDEX) self._frames_per_buffer = config.get("input_frames_per_buffer", _AudioInputStream._FRAMES_PER_BUFFER) self._audio_processor = audio_processor self._buffer = AudioBuffer(self._buffer_size) async def start(self, agent: "BidiAgent") -> None: """Start input stream. Args: agent: The BidiAgent instance, providing access to model configuration. Raises: ValueError: If the audio format or audio processing configuration is unsupported. """ logger.debug("starting audio input stream") if not isinstance(agent.model, AudioCapable): raise TypeError("AudioIO requires a model that implements AudioCapable") audio_config = agent.model.get_audio_config() self._audio_config = audio_config["input"] self._validate_audio_config(self._audio_config) if self._audio_processor is not None: output_config = audio_config["output"] if self._audio_processor.echo_cancellation_enabled: self._validate_audio_config(output_config) if self._audio_config["channels"] != output_config["channels"]: raise ValueError("Echo cancellation requires matching input and output channel counts") self._audio_processor.start( input_rate=self._audio_config["sample_rate"], output_rate=output_config["sample_rate"], num_channels=self._audio_config["channels"], ) if self._audio_processor.echo_cancellation_enabled: self._frames_per_buffer = self._audio_processor.frames_per_buffer(self._audio_config["sample_rate"]) self._buffer.start() self._audio = pyaudio.PyAudio() self._stream = self._audio.open( channels=self._audio_config["channels"], format=pyaudio.paInt16, frames_per_buffer=self._frames_per_buffer, input=True, input_device_index=self._device_index, rate=self._audio_config["sample_rate"], stream_callback=self._callback, ) logger.debug("audio input stream started") async def stop(self) -> None: """Stop input stream.""" logger.debug("stopping audio input stream") if hasattr(self, "_stream"): self._stream.close() if hasattr(self, "_audio"): self._audio.terminate() if hasattr(self, "_buffer"): self._buffer.stop() logger.debug("audio input stream stopped") async def __call__(self) -> AudioDelta: """Read audio from input stream, applying echo cancellation if enabled.""" data = await asyncio.to_thread(self._buffer.get) if self._audio_processor is not None: data = await asyncio.to_thread(self._audio_processor.process, data) return AudioDelta(format=self._audio_config["format"], source={"bytes": data}) def _callback( self, in_data: bytes | None, *_: Any, ) -> tuple[None, int]: """Callback to receive audio data from PyAudio.""" self._buffer.put(in_data or b"") return (None, pyaudio.paContinue) @staticmethod def _validate_audio_config(config: AudioStreamConfig) -> None: """Require PCM for microphone and echo cancellation reference audio.""" if config["format"] != "pcm": raise ValueError(f"AudioIO requires signed 16-bit PCM, received {config['format']}") class _AudioOutputStream(OutputStream): """Handle audio output from bidi agent. Attributes: _audio: PyAudio instance for audio system access. _stream: Audio output stream. _buffer: Buffer for sharing audio data between agent and PyAudio. """ _audio: pyaudio.PyAudio _stream: pyaudio.Stream _BUFFER_SIZE = None _DEVICE_INDEX = None _FRAMES_PER_BUFFER = 512 def __init__( self, config: AudioIOConfig, *, audio_processor: "AudioProcessor | None", ) -> None: """Initialize output settings. Args: config: Audio device configuration. audio_processor: Shared audio processor that receives played audio for echo cancellation. """ self._buffer_size = config.get("output_buffer_size", _AudioOutputStream._BUFFER_SIZE) self._device_index = config.get("output_device_index", _AudioOutputStream._DEVICE_INDEX) self._frames_per_buffer = config.get("output_frames_per_buffer", _AudioOutputStream._FRAMES_PER_BUFFER) self._audio_processor = audio_processor self._buffer = AudioBuffer(self._buffer_size) self._transcript_output = _TranscriptOutputStream() async def start(self, agent: "BidiAgent") -> None: """Start output stream. Args: agent: The BidiAgent instance, providing access to model configuration. Raises: ValueError: If the model's output encoding is unsupported. """ logger.debug("starting audio output stream") if not isinstance(agent.model, AudioCapable): raise TypeError("AudioIO requires a model that implements AudioCapable") self._audio_config = agent.model.get_audio_config()["output"] self._validate_audio_config(self._audio_config) if self._audio_processor is not None: self._frames_per_buffer = self._audio_processor.frames_per_buffer(self._audio_config["sample_rate"]) self._buffer.start() self._audio = pyaudio.PyAudio() self._stream = self._audio.open( channels=self._audio_config["channels"], format=pyaudio.paInt16, frames_per_buffer=self._frames_per_buffer, output=True, output_device_index=self._device_index, rate=self._audio_config["sample_rate"], stream_callback=self._callback, ) await self._transcript_output.start(agent) logger.debug("audio output stream started") async def stop(self) -> None: """Stop output stream.""" logger.debug("stopping audio output stream") await self._transcript_output.stop() if hasattr(self, "_stream"): self._stream.close() if hasattr(self, "_audio"): self._audio.terminate() if hasattr(self, "_buffer"): self._buffer.stop() logger.debug("audio output stream stopped") async def __call__(self, event: BidiOutputEvent) -> None: """Send audio to output stream. Raises: ValueError: If the audio encoding, rate, or channels differ from the playback stream. """ await self._transcript_output(event) if isinstance(event, BidiAudioDeltaEvent): self._validate_audio_event(event, self._audio_config) data = base64.b64decode(event["audio"]) self._buffer.put(data) logger.debug("audio_bytes=<%d> | audio chunk buffered for playback", len(data)) elif isinstance(event, BidiBargeInEvent): logger.debug("reason=<%s> | clearing audio buffer due to barge-in", event["reason"]) self._buffer.clear() if self._audio_processor is not None: self._audio_processor.clear_far_data() def _callback( self, _in_data: bytes | None, frame_count: int, *_: Any, ) -> tuple[bytes, int]: """Callback to send audio data to PyAudio. When echo cancellation is enabled, records played audio as the reference at the moment it exits the speaker — the correct temporal alignment point for echo cancellation. """ byte_count = frame_count * self._audio_config["channels"] * pyaudio.get_sample_size(pyaudio.paInt16) data = self._buffer.get(byte_count) if self._audio_processor is not None: self._audio_processor.add_far_data(data) return (data, pyaudio.paContinue) @staticmethod def _validate_audio_config(config: AudioStreamConfig) -> None: """Require PCM for speaker audio.""" if config["format"] != "pcm": raise ValueError(f"AudioIO requires signed 16-bit PCM, received {config['format']}") @staticmethod def _validate_audio_event(event: BidiAudioDeltaEvent, config: AudioStreamConfig) -> None: """Require audio to match the playback format.""" if (event.format, event.sample_rate, event.channels) != ( config["format"], config["sample_rate"], config["channels"], ): raise ValueError("Audio output does not match the playback format. Restart audio I/O to reconfigure it.") class AudioIO: """Send and receive audio data from devices using PyAudio. Reads microphone audio via ``input()``, plays agent audio via ``output()``, and displays user and assistant transcripts. Barge-ins clear the playback buffer to stop the agent mid-response. When ``audio_processor=True`` or an ``AudioProcessorConfig`` is passed, the microphone signal gets audio processing and, when echo cancellation is enabled, the agent's speaker output is used as a reference to cancel echo from the mic input. A shared processor coordinates the input and output channels, so echo cancellation only works when both come from the *same* ``AudioIO`` instance. Audio processing requires pywebrtc-audio (``pip install strands-agents[bidi-aec]``) and a microphone sample rate of 16000, 32000, or 48000 Hz (set via the model's audio config). Device audio requires PyAudio. Install the PortAudio system library, then install ``strands-agents[bidi-pyaudio]``. Example: ```python from strands.experimental.bidi.io import AudioIO, AudioProcessorConfig # Plain mic/speaker, no processing (a headset is recommended to avoid echo): audio_io = AudioIO() await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()]) # Full processing with defaults: echo cancellation, noise suppression, and auto gain control: audio_io = AudioIO(audio_processor=True) await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()]) # Noise suppression and auto gain control without echo cancellation (e.g. headset users): audio_io = AudioIO(audio_processor=AudioProcessorConfig(echo_cancellation=False)) await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()]) # Processing on a specific input device: audio_io = AudioIO(audio_processor=AudioProcessorConfig(), input_device_index=1) await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()]) ``` """ _audio_processor: "AudioProcessor | None" _audio_processor_config: AudioProcessorConfig | None def __init__(self, **config: Unpack[AudioIOConfig]) -> None: """Initialize audio devices. Args: **config: Optional configuration: - audio_processor (bool | AudioProcessorConfig): Set to True to enable microphone audio processing with defaults, or supply a configuration for custom options. False and None disable processing. - input_buffer_size (int): Maximum input buffer size (default: None). Must be between 1 and 100 when echo cancellation is on; defaults to 100 so the mic and reference buffers remain aligned. - input_device_index (int): Specific input device (default: None = system default) - input_frames_per_buffer (int): Input buffer size (default: 512). Must not be provided when echo cancellation is on because it is calculated from the model's input rate. - output_buffer_size (int): Maximum output buffer size (default: None) - output_device_index (int): Specific output device (default: None = system default) - output_frames_per_buffer (int): Output buffer size (default: 512). Must not be provided when echo cancellation is on because it is calculated from the model's output rate. Raises: ImportError: If audio processing is configured but its optional dependencies are unavailable. ValueError: If the configuration is invalid. """ self._config = config audio_processor_config = self._config.get("audio_processor") if isinstance(audio_processor_config, dict): self._audio_processor_config = AudioProcessorConfig(**audio_processor_config) else: self._audio_processor_config = AudioProcessorConfig() if audio_processor_config else None self._config["audio_processor"] = self._audio_processor_config self._validate_config_echo_cancellation() self._validate_config_buffer_size() self._validate_config_frames_per_buffer() self._import_audio_processor() def _import_audio_processor(self) -> None: """Import and initialize the optional audio processor implementation.""" if self._audio_processor_config is None: self._audio_processor = None return try: from .._audio.processor import AudioProcessor except ImportError as error: raise ImportError( f"{error}. Audio processing requires this optional dependency. " "Install it with: pip install 'strands-agents[bidi-aec]'." ) from error self._audio_processor = AudioProcessor( echo_cancellation=self._audio_processor_config["echo_cancellation"], stream_delay_ms=self._audio_processor_config["stream_delay_ms"], far_buffer_size=self._config.get("input_buffer_size"), ) def _validate_config_echo_cancellation(self) -> None: """Validate and normalize echo cancellation configuration.""" if self._audio_processor_config is None: return echo_cancellation = self._audio_processor_config.get("echo_cancellation", True) stream_delay_ms = self._audio_processor_config.get("stream_delay_ms", 0) if not 0 <= stream_delay_ms <= 1000: raise ValueError(f"stream_delay_ms=<{stream_delay_ms}> | must be between 0 and 1000") if not echo_cancellation and stream_delay_ms: raise ValueError("echo_cancellation= | stream_delay_ms requires echo cancellation") self._audio_processor_config["echo_cancellation"] = echo_cancellation self._audio_processor_config["stream_delay_ms"] = stream_delay_ms def _validate_config_buffer_size(self) -> None: """Validate the configured input buffer size for echo cancellation.""" if self._audio_processor_config is None or not self._audio_processor_config["echo_cancellation"]: return size = self._config.get("input_buffer_size") if size is not None and not 1 <= size <= 100: raise ValueError(f"input_buffer_size=<{size}> | must be between 1 and 100") self._config["input_buffer_size"] = size or 100 def _validate_config_frames_per_buffer(self) -> None: """Reject frame sizes that are calculated automatically for echo cancellation.""" if self._audio_processor_config is None or not self._audio_processor_config["echo_cancellation"]: return configured_fields = [ field for field in ("input_frames_per_buffer", "output_frames_per_buffer") if field in self._config ] if configured_fields: fields = ", ".join(configured_fields) raise ValueError( f"{fields} cannot be provided when echo cancellation is enabled; " "frames per buffer are calculated automatically" ) def input(self) -> _AudioInputStream: """Return the microphone input stream.""" return _AudioInputStream( self._config, audio_processor=self._audio_processor, ) def output(self) -> _AudioOutputStream: """Return the speaker and transcript output stream.""" return _AudioOutputStream( self._config, audio_processor=( self._audio_processor if self._audio_processor and self._audio_processor.echo_cancellation_enabled else None ), )