b60439152b
- harness/es.py: append-only event store + проекции (образец — кооп-портал) - harness/levels.py: уровни 1с/1м/1ч/1д, контекст ∝ масштабу - harness/scheduler.py: уровневые потоки, идемпотентные тики, absorbed-стабы - harness/isolated_runner.py + _llm_worker.py: LLM-тик в сабпроцессе (глобальный asyncio-рантайм Strands не переживает 2+ create_harness) - cli.py: init/task/tasks/status/events + requeue (написан самим минутным агентом) - 9router: отдельный ключ sk-simbiot, модель auto (glm-5.3 первым)
458 lines
18 KiB
Python
458 lines
18 KiB
Python
"""Send and receive audio data from devices.
|
|
|
|
Reads user audio from input device and sends agent audio to output device using PyAudio. If a user barges in,
|
|
the output buffer is cleared to stop playback.
|
|
|
|
Audio configuration is provided by models that implement ``AudioCapable``.
|
|
|
|
Optional microphone audio processing (acoustic echo cancellation, noise suppression, and automatic gain
|
|
control) is enabled by passing ``audio_processor=True`` or an ``AudioProcessorConfig`` to ``AudioIO``. It
|
|
requires pywebrtc-audio (pip install strands-agents[bidi-aec]).
|
|
"""
|
|
|
|
import asyncio
|
|
import base64
|
|
import logging
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
import pyaudio
|
|
from typing_extensions import Unpack
|
|
|
|
from .._audio.buffer import AudioBuffer
|
|
from ..models.configs import AudioStreamConfig
|
|
from ..models.model import AudioCapable
|
|
from ..types.events import (
|
|
BidiAudioDeltaEvent,
|
|
BidiBargeInEvent,
|
|
BidiOutputEvent,
|
|
)
|
|
from ..types.io import InputStream, OutputStream
|
|
from ..types.media import AudioDelta
|
|
from .configs import AudioIOConfig, AudioProcessorConfig
|
|
from .transcript import _TranscriptOutputStream
|
|
|
|
if TYPE_CHECKING:
|
|
from .._audio.processor import AudioProcessor
|
|
from ..agent.agent import BidiAgent
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class _AudioInputStream(InputStream):
|
|
"""Handle audio input from user.
|
|
|
|
Attributes:
|
|
_audio: PyAudio instance for audio system access.
|
|
_stream: Audio input stream.
|
|
_buffer: Buffer for sharing audio data between agent and PyAudio.
|
|
"""
|
|
|
|
_audio: pyaudio.PyAudio
|
|
_stream: pyaudio.Stream
|
|
|
|
_BUFFER_SIZE = None
|
|
_DEVICE_INDEX = None
|
|
_FRAMES_PER_BUFFER = 512
|
|
|
|
def __init__(
|
|
self,
|
|
config: AudioIOConfig,
|
|
*,
|
|
audio_processor: "AudioProcessor | None",
|
|
) -> None:
|
|
"""Initialize input settings.
|
|
|
|
Args:
|
|
config: Audio device configuration.
|
|
audio_processor: Shared microphone audio processor.
|
|
"""
|
|
self._buffer_size = config.get("input_buffer_size", _AudioInputStream._BUFFER_SIZE)
|
|
self._device_index = config.get("input_device_index", _AudioInputStream._DEVICE_INDEX)
|
|
self._frames_per_buffer = config.get("input_frames_per_buffer", _AudioInputStream._FRAMES_PER_BUFFER)
|
|
|
|
self._audio_processor = audio_processor
|
|
self._buffer = AudioBuffer(self._buffer_size)
|
|
|
|
async def start(self, agent: "BidiAgent") -> None:
|
|
"""Start input stream.
|
|
|
|
Args:
|
|
agent: The BidiAgent instance, providing access to model configuration.
|
|
|
|
Raises:
|
|
ValueError: If the audio format or audio processing configuration is unsupported.
|
|
"""
|
|
logger.debug("starting audio input stream")
|
|
|
|
if not isinstance(agent.model, AudioCapable):
|
|
raise TypeError("AudioIO requires a model that implements AudioCapable")
|
|
|
|
audio_config = agent.model.get_audio_config()
|
|
self._audio_config = audio_config["input"]
|
|
self._validate_audio_config(self._audio_config)
|
|
|
|
if self._audio_processor is not None:
|
|
output_config = audio_config["output"]
|
|
if self._audio_processor.echo_cancellation_enabled:
|
|
self._validate_audio_config(output_config)
|
|
if self._audio_config["channels"] != output_config["channels"]:
|
|
raise ValueError("Echo cancellation requires matching input and output channel counts")
|
|
|
|
self._audio_processor.start(
|
|
input_rate=self._audio_config["sample_rate"],
|
|
output_rate=output_config["sample_rate"],
|
|
num_channels=self._audio_config["channels"],
|
|
)
|
|
if self._audio_processor.echo_cancellation_enabled:
|
|
self._frames_per_buffer = self._audio_processor.frames_per_buffer(self._audio_config["sample_rate"])
|
|
|
|
self._buffer.start()
|
|
self._audio = pyaudio.PyAudio()
|
|
self._stream = self._audio.open(
|
|
channels=self._audio_config["channels"],
|
|
format=pyaudio.paInt16,
|
|
frames_per_buffer=self._frames_per_buffer,
|
|
input=True,
|
|
input_device_index=self._device_index,
|
|
rate=self._audio_config["sample_rate"],
|
|
stream_callback=self._callback,
|
|
)
|
|
|
|
logger.debug("audio input stream started")
|
|
|
|
async def stop(self) -> None:
|
|
"""Stop input stream."""
|
|
logger.debug("stopping audio input stream")
|
|
|
|
if hasattr(self, "_stream"):
|
|
self._stream.close()
|
|
if hasattr(self, "_audio"):
|
|
self._audio.terminate()
|
|
if hasattr(self, "_buffer"):
|
|
self._buffer.stop()
|
|
|
|
logger.debug("audio input stream stopped")
|
|
|
|
async def __call__(self) -> AudioDelta:
|
|
"""Read audio from input stream, applying echo cancellation if enabled."""
|
|
data = await asyncio.to_thread(self._buffer.get)
|
|
|
|
if self._audio_processor is not None:
|
|
data = await asyncio.to_thread(self._audio_processor.process, data)
|
|
|
|
return AudioDelta(format=self._audio_config["format"], source={"bytes": data})
|
|
|
|
def _callback(
|
|
self,
|
|
in_data: bytes | None,
|
|
*_: Any,
|
|
) -> tuple[None, int]:
|
|
"""Callback to receive audio data from PyAudio."""
|
|
self._buffer.put(in_data or b"")
|
|
return (None, pyaudio.paContinue)
|
|
|
|
@staticmethod
|
|
def _validate_audio_config(config: AudioStreamConfig) -> None:
|
|
"""Require PCM for microphone and echo cancellation reference audio."""
|
|
if config["format"] != "pcm":
|
|
raise ValueError(f"AudioIO requires signed 16-bit PCM, received {config['format']}")
|
|
|
|
|
|
class _AudioOutputStream(OutputStream):
|
|
"""Handle audio output from bidi agent.
|
|
|
|
Attributes:
|
|
_audio: PyAudio instance for audio system access.
|
|
_stream: Audio output stream.
|
|
_buffer: Buffer for sharing audio data between agent and PyAudio.
|
|
"""
|
|
|
|
_audio: pyaudio.PyAudio
|
|
_stream: pyaudio.Stream
|
|
|
|
_BUFFER_SIZE = None
|
|
_DEVICE_INDEX = None
|
|
_FRAMES_PER_BUFFER = 512
|
|
|
|
def __init__(
|
|
self,
|
|
config: AudioIOConfig,
|
|
*,
|
|
audio_processor: "AudioProcessor | None",
|
|
) -> None:
|
|
"""Initialize output settings.
|
|
|
|
Args:
|
|
config: Audio device configuration.
|
|
audio_processor: Shared audio processor that receives played audio for echo cancellation.
|
|
"""
|
|
self._buffer_size = config.get("output_buffer_size", _AudioOutputStream._BUFFER_SIZE)
|
|
self._device_index = config.get("output_device_index", _AudioOutputStream._DEVICE_INDEX)
|
|
self._frames_per_buffer = config.get("output_frames_per_buffer", _AudioOutputStream._FRAMES_PER_BUFFER)
|
|
|
|
self._audio_processor = audio_processor
|
|
self._buffer = AudioBuffer(self._buffer_size)
|
|
self._transcript_output = _TranscriptOutputStream()
|
|
|
|
async def start(self, agent: "BidiAgent") -> None:
|
|
"""Start output stream.
|
|
|
|
Args:
|
|
agent: The BidiAgent instance, providing access to model configuration.
|
|
|
|
Raises:
|
|
ValueError: If the model's output encoding is unsupported.
|
|
"""
|
|
logger.debug("starting audio output stream")
|
|
|
|
if not isinstance(agent.model, AudioCapable):
|
|
raise TypeError("AudioIO requires a model that implements AudioCapable")
|
|
|
|
self._audio_config = agent.model.get_audio_config()["output"]
|
|
self._validate_audio_config(self._audio_config)
|
|
|
|
if self._audio_processor is not None:
|
|
self._frames_per_buffer = self._audio_processor.frames_per_buffer(self._audio_config["sample_rate"])
|
|
|
|
self._buffer.start()
|
|
self._audio = pyaudio.PyAudio()
|
|
self._stream = self._audio.open(
|
|
channels=self._audio_config["channels"],
|
|
format=pyaudio.paInt16,
|
|
frames_per_buffer=self._frames_per_buffer,
|
|
output=True,
|
|
output_device_index=self._device_index,
|
|
rate=self._audio_config["sample_rate"],
|
|
stream_callback=self._callback,
|
|
)
|
|
await self._transcript_output.start(agent)
|
|
|
|
logger.debug("audio output stream started")
|
|
|
|
async def stop(self) -> None:
|
|
"""Stop output stream."""
|
|
logger.debug("stopping audio output stream")
|
|
|
|
await self._transcript_output.stop()
|
|
|
|
if hasattr(self, "_stream"):
|
|
self._stream.close()
|
|
if hasattr(self, "_audio"):
|
|
self._audio.terminate()
|
|
if hasattr(self, "_buffer"):
|
|
self._buffer.stop()
|
|
|
|
logger.debug("audio output stream stopped")
|
|
|
|
async def __call__(self, event: BidiOutputEvent) -> None:
|
|
"""Send audio to output stream.
|
|
|
|
Raises:
|
|
ValueError: If the audio encoding, rate, or channels differ from the playback stream.
|
|
"""
|
|
await self._transcript_output(event)
|
|
|
|
if isinstance(event, BidiAudioDeltaEvent):
|
|
self._validate_audio_event(event, self._audio_config)
|
|
|
|
data = base64.b64decode(event["audio"])
|
|
self._buffer.put(data)
|
|
logger.debug("audio_bytes=<%d> | audio chunk buffered for playback", len(data))
|
|
|
|
elif isinstance(event, BidiBargeInEvent):
|
|
logger.debug("reason=<%s> | clearing audio buffer due to barge-in", event["reason"])
|
|
self._buffer.clear()
|
|
if self._audio_processor is not None:
|
|
self._audio_processor.clear_far_data()
|
|
|
|
def _callback(
|
|
self,
|
|
_in_data: bytes | None,
|
|
frame_count: int,
|
|
*_: Any,
|
|
) -> tuple[bytes, int]:
|
|
"""Callback to send audio data to PyAudio.
|
|
|
|
When echo cancellation is enabled, records played audio as the reference at the moment it exits the
|
|
speaker — the correct temporal alignment point for echo cancellation.
|
|
"""
|
|
byte_count = frame_count * self._audio_config["channels"] * pyaudio.get_sample_size(pyaudio.paInt16)
|
|
data = self._buffer.get(byte_count)
|
|
|
|
if self._audio_processor is not None:
|
|
self._audio_processor.add_far_data(data)
|
|
|
|
return (data, pyaudio.paContinue)
|
|
|
|
@staticmethod
|
|
def _validate_audio_config(config: AudioStreamConfig) -> None:
|
|
"""Require PCM for speaker audio."""
|
|
if config["format"] != "pcm":
|
|
raise ValueError(f"AudioIO requires signed 16-bit PCM, received {config['format']}")
|
|
|
|
@staticmethod
|
|
def _validate_audio_event(event: BidiAudioDeltaEvent, config: AudioStreamConfig) -> None:
|
|
"""Require audio to match the playback format."""
|
|
if (event.format, event.sample_rate, event.channels) != (
|
|
config["format"],
|
|
config["sample_rate"],
|
|
config["channels"],
|
|
):
|
|
raise ValueError("Audio output does not match the playback format. Restart audio I/O to reconfigure it.")
|
|
|
|
|
|
class AudioIO:
|
|
"""Send and receive audio data from devices using PyAudio.
|
|
|
|
Reads microphone audio via ``input()``, plays agent audio via ``output()``, and displays user and assistant
|
|
transcripts. Barge-ins clear the playback buffer to stop the agent mid-response.
|
|
|
|
When ``audio_processor=True`` or an ``AudioProcessorConfig`` is passed, the microphone signal gets audio
|
|
processing and, when echo cancellation is enabled, the agent's speaker output is used as a reference to
|
|
cancel echo from the mic input. A shared processor coordinates the input and output channels, so echo
|
|
cancellation only works when both come from the *same* ``AudioIO`` instance.
|
|
|
|
Audio processing requires pywebrtc-audio (``pip install strands-agents[bidi-aec]``) and a microphone
|
|
sample rate of 16000, 32000, or 48000 Hz (set via the model's audio config).
|
|
|
|
Device audio requires PyAudio. Install the PortAudio system library, then install
|
|
``strands-agents[bidi-pyaudio]``.
|
|
|
|
Example:
|
|
```python
|
|
from strands.experimental.bidi.io import AudioIO, AudioProcessorConfig
|
|
|
|
# Plain mic/speaker, no processing (a headset is recommended to avoid echo):
|
|
audio_io = AudioIO()
|
|
await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()])
|
|
|
|
# Full processing with defaults: echo cancellation, noise suppression, and auto gain control:
|
|
audio_io = AudioIO(audio_processor=True)
|
|
await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()])
|
|
|
|
# Noise suppression and auto gain control without echo cancellation (e.g. headset users):
|
|
audio_io = AudioIO(audio_processor=AudioProcessorConfig(echo_cancellation=False))
|
|
await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()])
|
|
|
|
# Processing on a specific input device:
|
|
audio_io = AudioIO(audio_processor=AudioProcessorConfig(), input_device_index=1)
|
|
await agent.run(inputs=[audio_io.input()], outputs=[audio_io.output()])
|
|
```
|
|
"""
|
|
|
|
_audio_processor: "AudioProcessor | None"
|
|
_audio_processor_config: AudioProcessorConfig | None
|
|
|
|
def __init__(self, **config: Unpack[AudioIOConfig]) -> None:
|
|
"""Initialize audio devices.
|
|
|
|
Args:
|
|
**config: Optional configuration:
|
|
|
|
- audio_processor (bool | AudioProcessorConfig): Set to True to enable microphone audio processing
|
|
with defaults, or supply a configuration for custom options. False and None disable processing.
|
|
- input_buffer_size (int): Maximum input buffer size (default: None). Must be between 1 and 100
|
|
when echo cancellation is on; defaults to 100 so the mic and reference buffers remain aligned.
|
|
- input_device_index (int): Specific input device (default: None = system default)
|
|
- input_frames_per_buffer (int): Input buffer size (default: 512). Must not be provided when echo
|
|
cancellation is on because it is calculated from the model's input rate.
|
|
- output_buffer_size (int): Maximum output buffer size (default: None)
|
|
- output_device_index (int): Specific output device (default: None = system default)
|
|
- output_frames_per_buffer (int): Output buffer size (default: 512). Must not be provided when
|
|
echo cancellation is on because it is calculated from the model's output rate.
|
|
|
|
Raises:
|
|
ImportError: If audio processing is configured but its optional dependencies are unavailable.
|
|
ValueError: If the configuration is invalid.
|
|
"""
|
|
self._config = config
|
|
audio_processor_config = self._config.get("audio_processor")
|
|
if isinstance(audio_processor_config, dict):
|
|
self._audio_processor_config = AudioProcessorConfig(**audio_processor_config)
|
|
else:
|
|
self._audio_processor_config = AudioProcessorConfig() if audio_processor_config else None
|
|
self._config["audio_processor"] = self._audio_processor_config
|
|
|
|
self._validate_config_echo_cancellation()
|
|
self._validate_config_buffer_size()
|
|
self._validate_config_frames_per_buffer()
|
|
|
|
self._import_audio_processor()
|
|
|
|
def _import_audio_processor(self) -> None:
|
|
"""Import and initialize the optional audio processor implementation."""
|
|
if self._audio_processor_config is None:
|
|
self._audio_processor = None
|
|
return
|
|
|
|
try:
|
|
from .._audio.processor import AudioProcessor
|
|
except ImportError as error:
|
|
raise ImportError(
|
|
f"{error}. Audio processing requires this optional dependency. "
|
|
"Install it with: pip install 'strands-agents[bidi-aec]'."
|
|
) from error
|
|
|
|
self._audio_processor = AudioProcessor(
|
|
echo_cancellation=self._audio_processor_config["echo_cancellation"],
|
|
stream_delay_ms=self._audio_processor_config["stream_delay_ms"],
|
|
far_buffer_size=self._config.get("input_buffer_size"),
|
|
)
|
|
|
|
def _validate_config_echo_cancellation(self) -> None:
|
|
"""Validate and normalize echo cancellation configuration."""
|
|
if self._audio_processor_config is None:
|
|
return
|
|
|
|
echo_cancellation = self._audio_processor_config.get("echo_cancellation", True)
|
|
stream_delay_ms = self._audio_processor_config.get("stream_delay_ms", 0)
|
|
if not 0 <= stream_delay_ms <= 1000:
|
|
raise ValueError(f"stream_delay_ms=<{stream_delay_ms}> | must be between 0 and 1000")
|
|
if not echo_cancellation and stream_delay_ms:
|
|
raise ValueError("echo_cancellation=<False> | stream_delay_ms requires echo cancellation")
|
|
|
|
self._audio_processor_config["echo_cancellation"] = echo_cancellation
|
|
self._audio_processor_config["stream_delay_ms"] = stream_delay_ms
|
|
|
|
def _validate_config_buffer_size(self) -> None:
|
|
"""Validate the configured input buffer size for echo cancellation."""
|
|
if self._audio_processor_config is None or not self._audio_processor_config["echo_cancellation"]:
|
|
return
|
|
|
|
size = self._config.get("input_buffer_size")
|
|
if size is not None and not 1 <= size <= 100:
|
|
raise ValueError(f"input_buffer_size=<{size}> | must be between 1 and 100")
|
|
self._config["input_buffer_size"] = size or 100
|
|
|
|
def _validate_config_frames_per_buffer(self) -> None:
|
|
"""Reject frame sizes that are calculated automatically for echo cancellation."""
|
|
if self._audio_processor_config is None or not self._audio_processor_config["echo_cancellation"]:
|
|
return
|
|
|
|
configured_fields = [
|
|
field for field in ("input_frames_per_buffer", "output_frames_per_buffer") if field in self._config
|
|
]
|
|
if configured_fields:
|
|
fields = ", ".join(configured_fields)
|
|
raise ValueError(
|
|
f"{fields} cannot be provided when echo cancellation is enabled; "
|
|
"frames per buffer are calculated automatically"
|
|
)
|
|
|
|
def input(self) -> _AudioInputStream:
|
|
"""Return the microphone input stream."""
|
|
return _AudioInputStream(
|
|
self._config,
|
|
audio_processor=self._audio_processor,
|
|
)
|
|
|
|
def output(self) -> _AudioOutputStream:
|
|
"""Return the speaker and transcript output stream."""
|
|
return _AudioOutputStream(
|
|
self._config,
|
|
audio_processor=(
|
|
self._audio_processor
|
|
if self._audio_processor and self._audio_processor.echo_cancellation_enabled
|
|
else None
|
|
),
|
|
)
|