chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(voice): gate local wake prompt on network readiness

Browse Source
This commit is contained in:
chenjintao
2026-08-12 09:47:55 +08:00
parent 004400fee7
commit 7a4699ea68
15 changed files with 323 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/platform/CMakeLists.txt
+3
View File
@@ -82,6 +82,9 @@ set(platform_lcd_png_assets
jichubiaoqing0
jichubiaoqing1
jichubiaoqing2
listening0
listening1
listening2
muyu0
muyu1
muyu2
components/platform/assets/lcd/listening0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB

components/platform/assets/lcd/listening1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 15 KiB

components/platform/assets/lcd/listening2.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

components/platform/src/lcd_direct_assets.c
+12
View File
@@ -20,6 +20,12 @@ static const lcd_direct_image_t s_basic_face_frames[] = {
},
};
static const lcd_direct_image_t s_listening_frames[] = {
{.path = LCD_ASSET_BASE_PATH "listening0.png"},
{.path = LCD_ASSET_BASE_PATH "listening1.png"},
{.path = LCD_ASSET_BASE_PATH "listening2.png"},
};
static const lcd_direct_image_t s_muyu_frames[] = {
{
.path = LCD_ASSET_BASE_PATH "muyu0.png",
@@ -236,6 +242,12 @@ const lcd_direct_image_t *lcd_direct_asset_frame(const char *asset_dir, uint8_t
frame_index);
}
if (strcmp(asset_dir, "listening") == 0) {
return lcd_direct_frame_from_array(s_listening_frames,
sizeof(s_listening_frames) / sizeof(s_listening_frames[0]),
frame_index);
}
if (strcmp(asset_dir, "muyu") == 0) {
return lcd_direct_frame_from_array(s_muyu_frames,
sizeof(s_muyu_frames) / sizeof(s_muyu_frames[0]),
components/platform/src/multimodal/audio_pipeline.c
+3
View File
@@ -344,6 +344,9 @@ typedef struct {
volatile bool uplink_recording_active;
volatile bool uplink_task_exit_requested;
volatile bool uplink_send_in_flight;
bool uplink_first_network_pcm_pending;
bool uplink_first_network_packet_pending;
TickType_t uplink_network_start_tick;
multimodal_audio_uplink_cb_t uplink_cb;
void *uplink_cb_ctx;
bool opus_ready;
components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c
+65
View File
@@ -157,6 +157,42 @@ static void audio_set_uplink_callback(multimodal_audio_ctx_t *ctx,
portEXIT_CRITICAL(&s_audio_runtime_lock);
}
static bool audio_consume_first_network_pcm(multimodal_audio_ctx_t *ctx,
uint32_t *out_elapsed_ms)
{
bool pending = false;
TickType_t started = 0U;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (ctx != NULL && ctx->uplink_first_network_pcm_pending) {
ctx->uplink_first_network_pcm_pending = false;
started = ctx->uplink_network_start_tick;
pending = true;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
if (pending && out_elapsed_ms != NULL) {
*out_elapsed_ms = (uint32_t)pdTICKS_TO_MS(xTaskGetTickCount() - started);
}
return pending;
}
static bool audio_consume_first_network_packet(multimodal_audio_ctx_t *ctx,
uint32_t *out_elapsed_ms)
{
bool pending = false;
TickType_t started = 0U;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (ctx != NULL && ctx->uplink_first_network_packet_pending) {
ctx->uplink_first_network_packet_pending = false;
started = ctx->uplink_network_start_tick;
pending = true;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
if (pending && out_elapsed_ms != NULL) {
*out_elapsed_ms = (uint32_t)pdTICKS_TO_MS(xTaskGetTickCount() - started);
}
return pending;
}
static void notify_pcm_observer(const int16_t *pcm, size_t samples)
{
multimodal_audio_pcm_observer_t observer = NULL;
@@ -988,6 +1024,14 @@ static void uplink_send_task(void *arg)
ctx->uplink_send_in_flight = true;
esp_err_t send_err = cb(packet.data, packet.len, cb_ctx);
ctx->uplink_send_in_flight = false;
uint32_t first_packet_elapsed_ms = 0U;
if (send_err == ESP_OK &&
audio_consume_first_network_packet(ctx, &first_packet_elapsed_ms)) {
ESP_LOGI(TAG,
"Mic uplink first packet sent: activation_to_send_ms=%u bytes=%u",
(unsigned)first_packet_elapsed_ms,
(unsigned)packet.len);
}
track_uplink_send_result(send_err, packet.len);
if (send_err != ESP_OK && send_err != ESP_ERR_INVALID_STATE) {
if (send_err != ESP_ERR_TIMEOUT) {
@@ -1068,6 +1112,13 @@ static void uplink_capture_task(void *arg)
continue;
}
size_t uplink_frames = ctx->uplink_aec_frame_samples;
uint32_t first_pcm_elapsed_ms = 0U;
if (audio_consume_first_network_pcm(ctx, &first_pcm_elapsed_ms)) {
ESP_LOGI(TAG,
"Mic uplink first PCM captured: activation_to_pcm_ms=%u samples=%u",
(unsigned)first_pcm_elapsed_ms,
(unsigned)uplink_frames);
}
notify_pcm_observer(ctx->uplink_aec_clean_buf, uplink_frames);
audio_debug_dump_pcm(ctx->uplink_aec_clean_buf, clean_bytes);
multimodal_audio_uplink_cb_t uplink_cb = NULL;
@@ -1410,6 +1461,7 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
}
bool already_active = audio_get_uplink_recording_active(&s_audio);
bool network_uplink = cb != NULL;
audio_set_uplink_callback(&s_audio, cb, cb_ctx);
if (already_active) {
return ESP_OK;
@@ -1426,6 +1478,11 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
}
reset_uplink_send_queue(&s_audio);
portENTER_CRITICAL(&s_audio_runtime_lock);
s_audio.uplink_network_start_tick = xTaskGetTickCount();
s_audio.uplink_first_network_pcm_pending = network_uplink;
s_audio.uplink_first_network_packet_pending = network_uplink;
portEXIT_CRITICAL(&s_audio_runtime_lock);
audio_set_uplink_recording_active(&s_audio, true);
TaskHandle_t capture_task = NULL;
audio_snapshot_uplink_tasks(&s_audio, &capture_task, NULL);
@@ -1433,6 +1490,10 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
xTaskNotifyGive(capture_task);
}
log_heap_snapshot("after_uplink_start");
ESP_LOGI(TAG,
"Mic uplink activated: mode=%s sample_rate=%u channels=1",
network_uplink ? "network" : "local_observer_only",
(unsigned)CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE);
return ESP_OK;
}
@@ -1443,6 +1504,10 @@ esp_err_t multimodal_audio_stop_uplink(int timeout_ms)
}
audio_set_uplink_recording_active(&s_audio, false);
portENTER_CRITICAL(&s_audio_runtime_lock);
s_audio.uplink_first_network_pcm_pending = false;
s_audio.uplink_first_network_packet_pending = false;
portEXIT_CRITICAL(&s_audio_runtime_lock);
TickType_t deadline = xTaskGetTickCount() + pdMS_TO_TICKS((uint32_t)timeout_ms);
while (((s_audio.uplink_send_queue != NULL &&