#include "multimodal/audio_pipeline.h" #include #include #include #include #include #include "audio_codec_ctrl_if.h" #include "audio_codec_data_if.h" #include "audio_codec_gpio_if.h" #include "audio_codec_if.h" #include "es8311_codec.h" #include "esp_codec_dev.h" #include "esp_codec_dev_defaults.h" #include "esp_audio_dec.h" #include "esp_audio_enc.h" #include "esp_afe_aec.h" #include "esp_err.h" #include "esp_heap_caps.h" #include "esp_log.h" #include "esp_timer.h" #include "esp_opus_dec.h" #include "esp_opus_enc.h" #include "freertos/FreeRTOS.h" #include "freertos/idf_additions.h" #include "freertos/queue.h" #include "freertos/semphr.h" #include "freertos/task.h" #include "soc/soc_caps.h" #include "sdkconfig.h" #include "driver/i2s_std.h" #include "driver/i2c_master.h" /* * Kconfig 兼容默认值:仅在配置项缺失时生效。后面的编译期约束仍强制语音链路使用 * 16 kHz、双物理时隙和16位 PCM,因此默认值不等于所有板型都可直接运行。 * These are compatibility fallbacks for missing Kconfig values; compile-time * checks below still enforce the audio format required by voice/AEC processing. */ #ifndef CONFIG_MULTIMODAL_AUDIO_ENABLE_CODEC_PATH #define CONFIG_MULTIMODAL_AUDIO_ENABLE_CODEC_PATH 1 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE #define CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE 16000 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS #define CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS 1 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE #define CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE 16 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2C_PORT #define CONFIG_MULTIMODAL_AUDIO_I2C_PORT 0 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2C_SCL_GPIO #error "CONFIG_MULTIMODAL_AUDIO_I2C_SCL_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2C_SDA_GPIO #error "CONFIG_MULTIMODAL_AUDIO_I2C_SDA_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2C_ADDR #define CONFIG_MULTIMODAL_AUDIO_I2C_ADDR 0x18 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_PORT #define CONFIG_MULTIMODAL_AUDIO_I2S_PORT 0 #endif /* I2C/I2S GPIO 必须由板级 Kconfig 明确提供,避免错误默认引脚驱动真实硬件。 */ #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_MCLK_GPIO #error "CONFIG_MULTIMODAL_AUDIO_I2S_MCLK_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_BCLK_GPIO #error "CONFIG_MULTIMODAL_AUDIO_I2S_BCLK_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_WS_GPIO #error "CONFIG_MULTIMODAL_AUDIO_I2S_WS_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DOUT_GPIO #error "CONFIG_MULTIMODAL_AUDIO_I2S_DOUT_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DIN_GPIO #error "CONFIG_MULTIMODAL_AUDIO_I2S_DIN_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_USE_MCLK #define CONFIG_MULTIMODAL_AUDIO_USE_MCLK 1 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_PA_GPIO #error "CONFIG_MULTIMODAL_AUDIO_PA_GPIO must be defined in Kconfig" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DMA_DESC_NUM /* * I2S DMA 默认4个描述符、每描述符160帧:增加可提高抗调度抖动能力,但会占用更多 * DMA 内存并增加排队延迟;减少则降低内存占用,但更容易出现采集/播放 underrun。 */ #define CONFIG_MULTIMODAL_AUDIO_I2S_DMA_DESC_NUM 4 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DMA_FRAME_NUM #define CONFIG_MULTIMODAL_AUDIO_I2S_DMA_FRAME_NUM 160 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_OUT_VOLUME /* codec 初始增益属于硬件安全基线;调整前需同时检查削波、啸叫和扬声器功率。 */ #define CONFIG_MULTIMODAL_AUDIO_OUT_VOLUME 60 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_IN_GAIN_DB #define CONFIG_MULTIMODAL_AUDIO_IN_GAIN_DB 10 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL /* * 双时隙输入中,麦克风与 AEC 回采必须使用不同槽位;滤波长度增大可覆盖更长回声, * 但会增加运算量和工作内存。编译期检查防止两个通道配置为同一槽位。 */ #define CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL 0 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL #define CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL 1 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH #define CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH 4 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS #define CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS 1 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_IDLE_MUTE_MS /* * 下行播放时序权衡:首次积累 PREBUFFER 后开声;断流超过 REBUFFER_GAP 后重新缓冲, * 最大积累量受 MAX_REBUFFER 限制以避免延迟持续增长;正常排空后等待 IDLE_MUTE * 再静音以减少爆音,活动响应长时间无数据则按 ACTIVE_STALL 判定卡顿。 */ #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_IDLE_MUTE_MS 250 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_ACTIVE_STALL_MUTE_MS #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_ACTIVE_STALL_MUTE_MS 3000 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PREBUFFER_MS #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PREBUFFER_MS 160 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_REBUFFER_GAP_MS #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_REBUFFER_GAP_MS 120 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_MAX_REBUFFER_MS #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_MAX_REBUFFER_MS 640 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_DIAGNOSTICS_ENABLE #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_DIAGNOSTICS_ENABLE 0 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK /* * 下行任务承担 Opus 解码和 codec 写入,栈下限保护最深调用链。队列长度和单包容量 * 共同决定最坏内存占用;配置值会被归一化到 MIN/CAP 范围,避免过小不可运行或 * 过大耗尽 PSRAM。修改时应同时评估端到端延迟和网络突发大小。 */ #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK 12288 #endif #define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK_MIN_BYTES 8192U #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_LEN #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_LEN 1024 #endif #define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_MIN_LEN 16U #define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_CAP_LEN 1024U #ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_MAX_PACKET_BYTES #define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_MAX_PACKET_BYTES 2048 #endif #define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_PACKET_MIN_BYTES 256U #define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_PACKET_CAP_BYTES 8192U #ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_TASK_STACK /* * 上行采集栈包含 I2S、AEC 和 Opus 编码调用链,因此基线大于纯发送任务。发送队列 * 将网络抖动与实时采集隔离;5ms 入队等待刻意选择“及时丢包”而非阻塞采集。 */ #define CONFIG_MULTIMODAL_AUDIO_UPLINK_TASK_STACK 32768 #endif #define MULTIMODAL_AUDIO_UPLINK_TASK_STACK_MIN_BYTES 32768U #ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK #define CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK 12288 #endif #define MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK_MIN_BYTES 8192U #ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_LEN #define CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_LEN 24 #endif #define MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_MIN_LEN 24U #ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TIMEOUT_MS #define CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TIMEOUT_MS 5 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES #define CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES 1024 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_INIT_RETRY_COUNT /* * 初始化重试前先检查最大连续 DMA/内部内存块,避免在堆碎片严重时构建出半初始化 * codec。阈值是创建 I2S/codec 控制对象所需的安全水位,不是总空闲内存要求。 */ #define CONFIG_MULTIMODAL_AUDIO_INIT_RETRY_COUNT 1 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_DMA #define CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_DMA 4096 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_INTERNAL #define CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_INTERNAL 6144 #endif #ifndef CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS /* * 服务端协议要求 60ms raw-Opus 帧。16kbps、低复杂度和关闭 VBR/DTX/FEC 是当前 * 实时性与 MCU 负载基线;改变任一项需同步验证服务端解码、包容量和网络带宽。 */ #define CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS 60 #endif #ifndef CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS #define CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS 16 #endif #ifndef CONFIG_MULTIMODAL_OPUS_COMPLEXITY #define CONFIG_MULTIMODAL_OPUS_COMPLEXITY 0 #endif #ifndef CONFIG_MULTIMODAL_OPUS_ENABLE_VBR #define CONFIG_MULTIMODAL_OPUS_ENABLE_VBR 0 #endif #ifndef CONFIG_MULTIMODAL_OPUS_ENABLE_DTX #define CONFIG_MULTIMODAL_OPUS_ENABLE_DTX 0 #endif #ifndef CONFIG_MULTIMODAL_OPUS_ENABLE_FEC #define CONFIG_MULTIMODAL_OPUS_ENABLE_FEC 0 #endif #if CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE != 16000 #error "Multimodal voice requires 16 kHz audio" #endif #if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2 #error "ES8311 local wake AEC requires two physical codec slots" #endif #if CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE != 16 #error "ESP AEC requires signed 16-bit PCM" #endif #if CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL == CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL #error "Microphone and AEC reference must use distinct codec slots" #endif #if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60 #error "Multimodal voice requires 60 ms raw-opus frames" #endif #ifndef CONFIG_MULTIMODAL_OPUS_DEC_SELF_DELIMITED #define CONFIG_MULTIMODAL_OPUS_DEC_SELF_DELIMITED 0 #endif #ifndef CONFIG_MULTIMODAL_AUDIO_TASK_CORE /* 音频任务默认固定到本地实时核心;负数表示由调度器决定,需重新验证抖动。 */ #define CONFIG_MULTIMODAL_AUDIO_TASK_CORE 1 #endif #if CONFIG_MULTIMODAL_AUDIO_TASK_CORE < 0 #define MULTIMODAL_AUDIO_CORE_ID tskNO_AFFINITY #else #define MULTIMODAL_AUDIO_CORE_ID ((BaseType_t)CONFIG_MULTIMODAL_AUDIO_TASK_CORE) #endif /* * 解码缓冲包含最小工作区和保护余量;TX bounce 缓冲用于处理 codec 短写。 * AEC 缓冲按16字节对齐以满足 DSP/SIMD 访问要求。 */ #define MULTIMODAL_PCM_SAMPLE_BYTES 2U #define MULTIMODAL_MIN_DECODER_PCM_BUFFER_BYTES 2048U #define MULTIMODAL_DECODER_BUFFER_GUARD_BYTES 512U #define MULTIMODAL_AUDIO_DOWNSTREAM_TX_BOUNCE_BYTES 3200U #define MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL ((uint8_t)CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL) #define MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL \ ((uint8_t)CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL) #define MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT 16U static const char *TAG = "multimodal_audio"; static portMUX_TYPE s_audio_runtime_lock = portMUX_INITIALIZER_UNLOCKED; /* 音频上下行共享 codec/I2S 资源,因此所有生命周期状态集中保存在单一上下文中。 */ typedef struct { const audio_codec_data_if_t *data_if; const audio_codec_ctrl_if_t *ctrl_if; const audio_codec_gpio_if_t *gpio_if; const audio_codec_if_t *codec_if; esp_codec_dev_handle_t codec_dev; esp_codec_dev_sample_info_t fs; i2s_chan_handle_t tx_handle; i2s_chan_handle_t rx_handle; i2c_master_bus_handle_t i2c_bus; uint8_t i2c_probe_addr_7bit; uint8_t i2c_ctrl_addr_8bit; esp_audio_enc_handle_t opus_enc; esp_audio_dec_handle_t opus_dec; afe_aec_handle_t *afe_aec_handle; int opus_enc_in_bytes; int opus_enc_out_bytes; uint32_t opus_sample_rate; uint32_t opus_frame_duration_ms; uint32_t opus_bitrate_bps; uint8_t *uplink_codec_buf; int16_t *uplink_mono_buf; int16_t *uplink_aec_clean_buf; uint8_t *uplink_opus_buf; size_t uplink_codec_read_bytes; size_t uplink_aec_frame_bytes; size_t uplink_aec_frame_samples; size_t uplink_mono_bytes; size_t uplink_mono_buf_capacity_bytes; size_t uplink_mono_fill_bytes; size_t uplink_opus_buf_bytes; uint8_t *downstream_pcm_buf; size_t downstream_pcm_buf_bytes; int16_t *downstream_codec_out_buf; size_t downstream_codec_out_buf_bytes; uint8_t *downstream_tx_bounce_buf; size_t downstream_tx_bounce_buf_bytes; uint32_t downstream_playback_generation; volatile bool downstream_decoder_reset_pending; bool downstream_playback_in_progress; bool downstream_output_muted; bool downstream_output_mute_state_known; bool downstream_output_primed; bool downstream_response_active; bool downstream_response_end_received; TickType_t downstream_last_audio_tick; TickType_t downstream_last_response_activity_tick; esp_timer_handle_t downstream_idle_mute_timer; QueueHandle_t downstream_playback_queue; bool downstream_playback_queue_with_caps; TaskHandle_t downstream_playback_task; bool downstream_playback_starting; volatile bool downstream_playback_stop_requested; bool downstream_playback_stopping; bool downstream_playback_keep_stopping_on_exit; bool audio_deinit_in_progress; uint8_t mic_capture_channel; uint8_t aec_reference_channel; QueueHandle_t uplink_send_queue; bool uplink_send_queue_with_caps; TaskHandle_t uplink_task; TaskHandle_t uplink_send_task; volatile bool uplink_recording_active; volatile bool uplink_task_exit_requested; bool uplink_send_in_flight; bool uplink_tail_flush_requested; bool uplink_stop_incomplete; esp_err_t uplink_tail_flush_error; bool uplink_first_network_pcm_pending; bool uplink_first_network_packet_pending; TickType_t uplink_network_start_tick; multimodal_audio_uplink_cb_t uplink_cb; void *uplink_cb_ctx; bool opus_ready; bool initialized; } multimodal_audio_ctx_t; static multimodal_audio_ctx_t s_audio; /* 下行播放以 generation 标识会话,终止事件只唤醒等待同一代的调用方。 */ static multimodal_audio_downstream_playback_callback_t s_downstream_playback_cb; static void *s_downstream_playback_cb_ctx; static SemaphoreHandle_t s_downstream_playback_completion_sem; #if ( configSUPPORT_STATIC_ALLOCATION == 1 ) static StaticSemaphore_t s_downstream_playback_completion_sem_storage; #endif static bool s_downstream_playback_wait_registered; static uint32_t s_downstream_playback_wait_generation; static bool s_downstream_last_terminal_valid; static uint32_t s_downstream_last_terminal_generation; static multimodal_audio_downstream_playback_event_type_t s_downstream_last_terminal_type; #define MULTIMODAL_AUDIO_PCM_OBSERVER_SLOTS 2U typedef struct { multimodal_audio_pcm_observer_t callback; void *ctx; uint32_t in_flight; uint32_t registration; bool enabled; } audio_pcm_observer_slot_t; static audio_pcm_observer_slot_t s_pcm_observers[MULTIMODAL_AUDIO_PCM_OBSERVER_SLOTS]; static uint32_t s_pcm_observer_registration; static StaticSemaphore_t s_uplink_capture_mutex_storage; static SemaphoreHandle_t s_uplink_capture_mutex; static StaticSemaphore_t s_uplink_lifecycle_mutex_storage; static SemaphoreHandle_t s_uplink_lifecycle_mutex; static bool s_uplink_capture_mutex_initializing; static portMUX_TYPE s_pcm_observer_lock = portMUX_INITIALIZER_UNLOCKED; static bool s_uplink_stack_warned; static bool s_uplink_send_stack_warned; static bool s_uplink_send_queue_warned; static bool s_downstream_playback_stack_warned; static bool s_downstream_playback_queue_warned; static bool s_downstream_playback_packet_warned; static uint32_t s_uplink_overflow_count; static TickType_t s_uplink_overflow_last_log_tick; static uint32_t s_downstream_overflow_count; static TickType_t s_downstream_overflow_last_log_tick; static uint32_t s_downstream_queued_packet_total; static uint32_t s_downstream_played_packet_total; #if CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_DIAGNOSTICS_ENABLE typedef struct { TickType_t response_begin_tick; TickType_t arrival_last_tick; TickType_t first_arrival_tick; TickType_t first_playback_tick; TickType_t last_playback_tick; TickType_t queue_empty_since_tick; uint32_t arrival_gap_min_ms; uint32_t arrival_gap_max_ms; uint64_t arrival_gap_total_ms; uint32_t arrival_gap_samples; UBaseType_t queue_depth_min; UBaseType_t queue_depth_max; uint32_t queue_empty_events; uint32_t queue_empty_max_ms; uint32_t enqueue_failures; uint32_t playback_failures; bool summary_logged; } downstream_diagnostics_t; static downstream_diagnostics_t s_downstream_diagnostics; #endif static SemaphoreHandle_t s_downstream_playback_queue_mutex; static bool s_downstream_playback_queue_mutex_initializing; #if ( configSUPPORT_STATIC_ALLOCATION == 1 ) static StaticSemaphore_t s_downstream_playback_queue_mutex_buf; #endif typedef struct { uint16_t len; uint8_t data[CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES]; } uplink_packet_t; typedef struct { size_t len; uint32_t playback_generation; uint8_t *data; } downstream_packet_t; /* * 实现文件拆分规则: * - 下列 *_impl.c 不能单独加入 CMake 或独立编译,只能由本文件按当前顺序包含; * - 它们共享本文件上方定义的 static 类型、状态、宏和前置声明; * - 拆分仅为了按职责降低阅读长度,不形成新的模块或链接边界; * - include 顺序有意为 helpers -> codec -> debug -> runtime,后面的实现可以使用 * 前面已经定义的内部辅助函数,调整顺序前必须检查可见声明和依赖方向。 * * Included *_impl.c files are not standalone translation units. They share this * file's static state and are split only for readability. Their include order is * dependency-sensitive and must not be changed without reviewing symbol visibility. */ #include "audio_pipeline/audio_pipeline_helpers_impl.c" #include "audio_pipeline/audio_pipeline_codec_impl.c" #include "audio_pipeline/audio_debug_dump_impl.c" #include "audio_pipeline/audio_pipeline_runtime_impl.c"