chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(voice): add on-device Miao Zhang Gui wake word

Browse Source 
Add an optional local wake response prompt configuration. The prompt asset is intentionally not included yet.
This commit is contained in:
chenjintao
2026-08-10 16:54:42 +08:00
parent d9660164a6
commit e5c1564130
21 changed files with 2010 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/platform/src/multimodal/audio_pipeline.c
+31 -2
View File
@@ -15,6 +15,7 @@
#include "esp_codec_dev_defaults.h"
#include "esp_audio_dec.h"
#include "esp_audio_enc.h"
#include "esp_afe_aec.h"
#include "esp_err.h"
#include "esp_heap_caps.h"
#include "esp_log.h"
@@ -114,6 +115,14 @@
#define CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL 0
#endif
#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL
#define CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL 1
#endif
#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH
#define CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH 4
#endif
#ifndef CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
#define CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS 1
#endif
@@ -208,8 +217,16 @@
#error "Multimodal voice requires 16 kHz audio"
#endif
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1
#error "Multimodal voice requires mono audio"
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2
#error "ES8311 local wake AEC requires two physical codec slots"
#endif
#if CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE != 16
#error "ESP AEC requires signed 16-bit PCM"
#endif
#if CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL == CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL
#error "Microphone and AEC reference must use distinct codec slots"
#endif
#if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60
@@ -235,6 +252,9 @@
#define MULTIMODAL_DECODER_BUFFER_GUARD_BYTES 512U
#define MULTIMODAL_AUDIO_DOWNSTREAM_TX_BOUNCE_BYTES 3200U
#define MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL ((uint8_t)CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL)
#define MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL \
((uint8_t)CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL)
#define MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT 16U
static const char *TAG = "multimodal_audio";
static portMUX_TYPE s_audio_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
@@ -253,6 +273,7 @@ typedef struct {
uint8_t i2c_ctrl_addr_8bit;
esp_audio_enc_handle_t opus_enc;
esp_audio_dec_handle_t opus_dec;
afe_aec_handle_t *afe_aec_handle;
int opus_enc_in_bytes;
int opus_enc_out_bytes;
uint32_t opus_sample_rate;
@@ -260,8 +281,11 @@ typedef struct {
uint32_t opus_bitrate_bps;
uint8_t *uplink_codec_buf;
int16_t *uplink_mono_buf;
int16_t *uplink_aec_clean_buf;
uint8_t *uplink_opus_buf;
size_t uplink_codec_read_bytes;
size_t uplink_aec_frame_bytes;
size_t uplink_aec_frame_samples;
size_t uplink_mono_bytes;
size_t uplink_mono_buf_capacity_bytes;
size_t uplink_mono_fill_bytes;
@@ -290,6 +314,7 @@ typedef struct {
bool downstream_playback_keep_stopping_on_exit;
bool audio_deinit_in_progress;
uint8_t mic_capture_channel;
uint8_t aec_reference_channel;
QueueHandle_t uplink_send_queue;
bool uplink_send_queue_with_caps;
TaskHandle_t uplink_task;
@@ -306,6 +331,10 @@ typedef struct {
static multimodal_audio_ctx_t s_audio;
static multimodal_audio_downstream_playback_callback_t s_downstream_playback_cb;
static void *s_downstream_playback_cb_ctx;
static multimodal_audio_pcm_observer_t s_pcm_observer;
static void *s_pcm_observer_ctx;
static uint32_t s_pcm_observer_in_flight;
static portMUX_TYPE s_pcm_observer_lock = portMUX_INITIALIZER_UNLOCKED;
static bool s_uplink_stack_warned;
static bool s_uplink_send_stack_warned;
static bool s_uplink_send_queue_warned;