chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(voice): gate local wake prompt on network readiness

Browse Source
This commit is contained in:
chenjintao
2026-08-12 09:47:55 +08:00
parent 004400fee7
commit 7a4699ea68
15 changed files with 323 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/control_plane/internal/control_plane_internal.h
+2
View File
@@ -137,6 +137,8 @@ esp_err_t control_plane_local_wake_start(void);
esp_err_t control_plane_local_wake_stop(int timeout_ms);
void control_plane_local_wake_pause_for_foreground(void);
void control_plane_local_wake_resume_after_foreground(void);
bool control_plane_local_wake_prompt_gate_active(void);
void control_plane_ptt_reset_capture_window(void);
bool control_plane_voice_turn_is_idle(void);
esp_err_t control_plane_start_system_event_task(void);
components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c
+150 -59
View File
@@ -25,6 +25,10 @@
#define CONFIG_TQ_LOCAL_WAKE_RETRY_MS 5000
#endif
#ifndef CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS
#define CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS 10000
#endif
#ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS
#define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000
#endif
@@ -43,8 +47,6 @@
#define CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN 4U
#define CONTROL_PLANE_LOCAL_WAKE_POLL_MS 100U
#define CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_WAIT_MS 2000U
#define CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_POLL_MS 20U
#if CONFIG_TQ_LOCAL_WAKE_TASK_CORE < 0
#define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE tskNO_AFFINITY
@@ -59,6 +61,11 @@ typedef struct {
bool started;
bool paused_for_foreground;
bool capture_owned;
bool prompt_pending;
bool prompt_playing;
bool listening_ui_active;
TickType_t prompt_wait_started_tick;
uint32_t detection_generation;
} local_wake_state_t;
static const char *TAG = "cp_local_wake";
@@ -81,6 +88,31 @@ static void local_wake_set_capture_owned(bool owned)
portEXIT_CRITICAL(&s_local_wake_lock);
}
static void local_wake_request_expression(domain_expression_id_t id)
{
const domain_expression_request_t request = {
.id = id,
.trigger = DOMAIN_EXPRESSION_TRIGGER_LOCAL_WAKE,
};
esp_err_t err = domain_runtime_request_expression(&request);
if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
ESP_LOGW(TAG,
"Local wake expression request failed: id=%d err=%s",
(int)id,
esp_err_to_name(err));
}
}
bool control_plane_local_wake_prompt_gate_active(void)
{
#if CONFIG_TQ_LOCAL_WAKE_ENABLE && CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
local_wake_state_t state = local_wake_snapshot();
return state.prompt_pending || state.prompt_playing;
#else
return false;
#endif
}
static void local_wake_domain_event(
const domain_local_command_event_t *event,
void *user_data)
@@ -153,63 +185,107 @@ static esp_err_t local_wake_start_listening(void)
return ESP_OK;
}
#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
static bool local_wake_prepare_prompt_playback(void)
static bool local_wake_runtime_ready_for_prompt(
control_plane_multimodal_runtime_state_t *out_state)
{
control_plane_multimodal_runtime_state_t state = {0};
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
if (state_err != ESP_OK) {
ESP_LOGW(TAG,
"Skip local wake prompt: runtime state unavailable: %s",
esp_err_to_name(state_err));
return false;
}
if (out_state != NULL) {
*out_state = state;
}
return state.connected &&
state.has_active_task &&
state.server_listening &&
!state.remote_responding &&
!state.request_to_speak_pending &&
!state.client_speech_active &&
!state.ptt_release_required;
}
if (!state.remote_responding && !state.request_to_speak_pending) {
return true;
static void local_wake_service_ready_prompt(void)
{
#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
local_wake_state_t state = local_wake_snapshot();
if (!state.prompt_pending || state.prompt_playing || state.stop_requested) {
return;
}
esp_err_t request_err = control_plane_multimodal_send_request_to_speak();
if (request_err != ESP_OK && request_err != ESP_ERR_INVALID_STATE) {
ESP_LOGW(TAG,
"Skip local wake prompt: barge-in request failed: %s",
esp_err_to_name(request_err));
return false;
}
TickType_t started = xTaskGetTickCount();
TickType_t timeout_ticks = pdMS_TO_TICKS(
CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_WAIT_MS);
TickType_t now = xTaskGetTickCount();
TickType_t timeout_ticks = pdMS_TO_TICKS(CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS);
if (timeout_ticks == 0U) {
timeout_ticks = 1U;
}
while ((xTaskGetTickCount() - started) < timeout_ticks) {
local_wake_state_t wake_state = local_wake_snapshot();
if (wake_state.stop_requested) {
return false;
}
state_err = control_plane_multimodal_get_runtime_state(&state);
if (state_err == ESP_OK) {
if (!state.connected || !state.has_active_task) {
return false;
}
if (!state.remote_responding &&
!state.request_to_speak_pending &&
state.server_listening) {
return true;
}
}
vTaskDelay(pdMS_TO_TICKS(CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_POLL_MS));
control_plane_multimodal_runtime_state_t runtime = {0};
bool runtime_ready = local_wake_runtime_ready_for_prompt(&runtime);
if (state.prompt_wait_started_tick != 0U &&
(now - state.prompt_wait_started_tick) >= timeout_ticks) {
ESP_LOGW(TAG,
"Local wake network readiness timed out: generation=%u timeout_ms=%u ws=%d task=%d listening=%d responding=%d request_pending=%d speech=%d release_required=%d",
(unsigned)state.detection_generation,
(unsigned)CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS,
runtime.connected ? 1 : 0,
runtime.has_active_task ? 1 : 0,
runtime.server_listening ? 1 : 0,
runtime.remote_responding ? 1 : 0,
runtime.request_to_speak_pending ? 1 : 0,
runtime.client_speech_active ? 1 : 0,
runtime.ptt_release_required ? 1 : 0);
control_plane_set_ptt_active(false);
return;
}
ESP_LOGW(TAG,
"Skip local wake prompt: barge-in confirmation timed out after %ums",
(unsigned)CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_WAIT_MS);
return false;
}
if (!runtime_ready) {
return;
}
portENTER_CRITICAL(&s_local_wake_lock);
if (!s_local_wake.prompt_pending || s_local_wake.stop_requested) {
portEXIT_CRITICAL(&s_local_wake_lock);
return;
}
s_local_wake.prompt_pending = false;
s_local_wake.prompt_playing = true;
portEXIT_CRITICAL(&s_local_wake_lock);
const uint32_t ready_wait_ms = state.prompt_wait_started_tick != 0U ?
(uint32_t)pdTICKS_TO_MS(now - state.prompt_wait_started_tick) : 0U;
ESP_LOGI(TAG,
"Local wake network ready: generation=%u wait_ms=%u ws=1 task=1 listening=1",
(unsigned)state.detection_generation,
(unsigned)ready_wait_ms);
TickType_t prompt_started_tick = xTaskGetTickCount();
ESP_LOGI(TAG,
"Local wake response prompt started: generation=%u file=%s",
(unsigned)state.detection_generation,
CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE);
esp_err_t prompt_err = domain_runtime_behavior_play_prompt_file(
CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE);
const uint32_t prompt_elapsed_ms = (uint32_t)pdTICKS_TO_MS(
xTaskGetTickCount() - prompt_started_tick);
portENTER_CRITICAL(&s_local_wake_lock);
s_local_wake.prompt_playing = false;
s_local_wake.prompt_wait_started_tick = 0U;
portEXIT_CRITICAL(&s_local_wake_lock);
if (prompt_err != ESP_OK) {
ESP_LOGW(TAG,
"Local wake response prompt failed: generation=%u elapsed_ms=%u err=%s",
(unsigned)state.detection_generation,
(unsigned)prompt_elapsed_ms,
esp_err_to_name(prompt_err));
} else {
ESP_LOGI(TAG,
"Local wake response prompt completed: generation=%u elapsed_ms=%u",
(unsigned)state.detection_generation,
(unsigned)prompt_elapsed_ms);
}
control_plane_ptt_reset_capture_window();
#endif
}
static void local_wake_handle_detection(
const domain_local_command_event_t *event)
@@ -226,31 +302,31 @@ static void local_wake_handle_detection(
return;
}
s_local_wake.paused_for_foreground = true;
s_local_wake.listening_ui_active = true;
s_local_wake.detection_generation = event->detection_generation;
#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
s_local_wake.prompt_pending = CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE[0] != '\0';
s_local_wake.prompt_playing = false;
s_local_wake.prompt_wait_started_tick = xTaskGetTickCount();
#endif
portEXIT_CRITICAL(&s_local_wake_lock);
local_wake_request_expression(DOMAIN_EXPRESSION_LISTENING);
ESP_LOGI(TAG,
"Local wake foreground entered: generation=%u listening_ui=1 prompt_gate=%d ready_timeout_ms=%u",
(unsigned)event->detection_generation,
control_plane_local_wake_prompt_gate_active() ? 1 : 0,
(unsigned)CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS);
local_wake_stop_owned_capture();
(void)domain_runtime_local_command_suspend(
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
ESP_LOGI(TAG,
"Local wake accepted: phrase=喵掌柜 confidence_x10000=%u generation=%u",
"Local wake accepted: phrase=喵掌柜 confidence_x10000=%u generation=%u; network preparation requested",
(unsigned)event->confidence_x10000,
(unsigned)event->detection_generation);
#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
bool prompt_playback_ready = local_wake_prepare_prompt_playback();
if (prompt_playback_ready &&
CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE[0] != '\0') {
esp_err_t prompt_err = domain_runtime_behavior_play_prompt_file(
CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE);
if (prompt_err != ESP_OK) {
ESP_LOGW(TAG,
"Local wake response prompt failed: %s",
esp_err_to_name(prompt_err));
}
}
#endif
control_plane_set_ptt_active(true);
}
@@ -264,6 +340,7 @@ static void local_wake_task(void *arg)
while (!local_wake_snapshot().stop_requested) {
local_wake_state_t state = local_wake_snapshot();
TickType_t now = xTaskGetTickCount();
local_wake_service_ready_prompt();
if (!state.paused_for_foreground &&
control_plane_voice_turn_is_idle() &&
(retry_not_before == 0U || now >= retry_not_before)) {
@@ -342,6 +419,11 @@ esp_err_t control_plane_local_wake_start(void)
s_local_wake.stop_requested = false;
s_local_wake.paused_for_foreground = false;
s_local_wake.capture_owned = false;
s_local_wake.prompt_pending = false;
s_local_wake.prompt_playing = false;
s_local_wake.listening_ui_active = false;
s_local_wake.prompt_wait_started_tick = 0U;
s_local_wake.detection_generation = 0U;
s_local_wake.started = true;
portEXIT_CRITICAL(&s_local_wake_lock);
@@ -434,10 +516,19 @@ void control_plane_local_wake_pause_for_foreground(void)
void control_plane_local_wake_resume_after_foreground(void)
{
#if CONFIG_TQ_LOCAL_WAKE_ENABLE
bool restore_idle_ui = false;
portENTER_CRITICAL(&s_local_wake_lock);
s_local_wake.paused_for_foreground = false;
s_local_wake.prompt_pending = false;
s_local_wake.prompt_playing = false;
s_local_wake.prompt_wait_started_tick = 0U;
restore_idle_ui = s_local_wake.listening_ui_active;
s_local_wake.listening_ui_active = false;
TaskHandle_t task = s_local_wake.task;
portEXIT_CRITICAL(&s_local_wake_lock);
if (restore_idle_ui) {
local_wake_request_expression(DOMAIN_EXPRESSION_IDLE);
}
if (task != NULL) {
xTaskNotifyGive(task);
}
components/control_plane/src/runtime/control_plane_runtime_workers.c
+40 -3
View File
@@ -152,6 +152,7 @@ typedef enum {
PTT_STATE_WAIT_WS,
PTT_STATE_WAIT_START_SENT,
PTT_STATE_WAIT_LISTENING,
PTT_STATE_WAIT_WAKE_PROMPT,
PTT_STATE_WAIT_SPEECH_STARTED,
PTT_STATE_STREAMING,
PTT_STATE_TEARDOWN_LOCAL,
@@ -227,6 +228,8 @@ static const char *ptt_state_name(ptt_state_t state)
return "WAIT_START_SENT";
case PTT_STATE_WAIT_LISTENING:
return "WAIT_LISTENING";
case PTT_STATE_WAIT_WAKE_PROMPT:
return "WAIT_WAKE_PROMPT";
case PTT_STATE_WAIT_SPEECH_STARTED:
return "WAIT_SPEECH_STARTED";
case PTT_STATE_STREAMING:
@@ -349,6 +352,7 @@ void control_plane_set_ptt_active(bool active)
s_ptt_runtime.state == PTT_STATE_WAIT_WS ||
s_ptt_runtime.state == PTT_STATE_WAIT_START_SENT ||
s_ptt_runtime.state == PTT_STATE_WAIT_LISTENING ||
s_ptt_runtime.state == PTT_STATE_WAIT_WAKE_PROMPT ||
s_ptt_runtime.state == PTT_STATE_WAIT_SPEECH_STARTED ||
s_ptt_runtime.state == PTT_STATE_DENIED) {
control_plane_ptt_transition_locked(PTT_STATE_IDLE, "release_before_streaming");
@@ -368,6 +372,30 @@ void control_plane_set_ptt_active(bool active)
}
}
void control_plane_ptt_reset_capture_window(void)
{
bool active = false;
uint32_t generation = 0U;
portENTER_CRITICAL(&s_runtime_lock);
active = s_ptt_runtime.active;
if (active) {
s_ptt_runtime.press_tick = xTaskGetTickCount();
generation = s_ptt_runtime.generation;
}
portEXIT_CRITICAL(&s_runtime_lock);
if (!active) {
ESP_LOGW(TAG, "Skip PTT capture window reset: turn is no longer active");
return;
}
ESP_LOGI(TAG,
"PTT capture window armed after local wake prompt: generation=%u timeout_ms=%u",
(unsigned)generation,
(unsigned)CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS);
control_plane_notify_wifi_state_task();
}
bool control_plane_voice_turn_is_idle(void)
{
bool idle = false;
@@ -1218,12 +1246,14 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte
}
TickType_t now = xTaskGetTickCount();
bool local_wake_prompt_gate = control_plane_local_wake_prompt_gate_active();
#if CONFIG_TQ_PTT_MODE_CLICK_TO_TALK
TickType_t capture_timeout_ticks = pdMS_TO_TICKS(CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS);
if (capture_timeout_ticks == 0) {
capture_timeout_ticks = 1;
}
if (ptt.press_tick != 0 &&
if (!local_wake_prompt_gate &&
ptt.press_tick != 0 &&
(now - ptt.press_tick) >= capture_timeout_ticks) {
control_plane_set_ptt_active(false);
return;
@@ -1234,7 +1264,8 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte
if (denied_ticks == 0) {
denied_ticks = 1;
}
if (ptt.press_tick != 0 &&
if (!local_wake_prompt_gate &&
ptt.press_tick != 0 &&
(now - ptt.press_tick) >= denied_ticks &&
ptt.state != PTT_STATE_STREAMING &&
ptt.state != PTT_STATE_DENIED) {
@@ -1287,6 +1318,8 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte
control_plane_ptt_set_state(PTT_STATE_WAIT_START_SENT, "send_start");
} else if (!state.server_listening) {
control_plane_ptt_set_state(PTT_STATE_WAIT_LISTENING, "wait_listening");
} else if (local_wake_prompt_gate) {
control_plane_ptt_set_state(PTT_STATE_WAIT_WAKE_PROMPT, "wait_local_wake_prompt");
} else if (!state.client_speech_active) {
control_plane_ptt_set_state(PTT_STATE_WAIT_SPEECH_STARTED, "send_speech");
} else if (control_plane_multimodal_is_uplink_running()) {
@@ -1296,7 +1329,8 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte
}
ptt = control_plane_ptt_snapshot();
if ((ptt.state == PTT_STATE_WAIT_START_SENT ||
if (!local_wake_prompt_gate &&
(ptt.state == PTT_STATE_WAIT_START_SENT ||
ptt.state == PTT_STATE_WAIT_LISTENING) &&
(now - ptt.state_enter_tick) >=
pdMS_TO_TICKS(CONTROL_PLANE_PTT_LISTENING_TIMEOUT_MS)) {
@@ -1327,6 +1361,9 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte
if (ptt.state == PTT_STATE_WAIT_SPEECH_STARTED && ptt.speech_action_sent) {
should_run_press_step = false;
}
if (ptt.state == PTT_STATE_WAIT_WAKE_PROMPT) {
should_run_press_step = false;
}
if (!should_run_press_step) {
return;
}
components/domain/include/domain.h
+2
View File
@@ -118,6 +118,7 @@ typedef enum {
DOMAIN_EXPRESSION_MUYU,
DOMAIN_EXPRESSION_CHOUQIAN,
DOMAIN_EXPRESSION_WUXING,
DOMAIN_EXPRESSION_LISTENING,
} domain_expression_id_t;
typedef enum {
@@ -132,6 +133,7 @@ typedef enum {
DOMAIN_EXPRESSION_TRIGGER_PTT_DOUBLE_CLICK,
DOMAIN_EXPRESSION_TRIGGER_SW2_SINGLE_CLICK,
DOMAIN_EXPRESSION_TRIGGER_SW2_DOUBLE_CLICK,
DOMAIN_EXPRESSION_TRIGGER_LOCAL_WAKE,
} domain_expression_trigger_t;
typedef struct {
components/domain/src/runtime/domain_runtime_adapters.c
+16
View File
@@ -78,6 +78,7 @@ static const uint8_t s_server_error_frame_order[] = {0, 1, 2, 1, 0};
static const uint8_t s_wifi_disconnected_frame_order[] = {0, 1, 2, 3, 2, 1, 0};
static const uint8_t s_basic_face_frame_order[] = {0, 1, 2};
static const uint8_t s_muyu_frame_order[] = {0, 1, 2, 1};
static const uint8_t s_listening_frame_order[] = {0, 1, 2};
static const char *const s_chouqian_result_asset_dirs[] = {
"chouqian_buyun",
"chouqian_heping",
@@ -227,6 +228,21 @@ static const domain_expression_catalog_entry_t s_expression_catalog[] = {
.duration_ms = 0,
},
},
{
.id = DOMAIN_EXPRESSION_LISTENING,
.platform_request = {
.kind = PLATFORM_LCD_PRESENTATION_ASSET_SEQUENCE,
.priority = 130,
.interrupt_current = true,
.return_to_idle = false,
.asset_dir = "listening",
.frame_order = s_listening_frame_order,
.frame_order_count = sizeof(s_listening_frame_order) /
sizeof(s_listening_frame_order[0]),
.repeat_count = 0,
.frame_ms = 500,
},
},
};
static void snapshot_domain_wifi_behavior_hooks(domain_wifi_behavior_hooks_t *out_hooks)
components/platform/CMakeLists.txt
+3
View File
@@ -82,6 +82,9 @@ set(platform_lcd_png_assets
jichubiaoqing0
jichubiaoqing1
jichubiaoqing2
listening0
listening1
listening2
muyu0
muyu1
muyu2
components/platform/assets/lcd/listening0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB

components/platform/assets/lcd/listening1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 15 KiB

components/platform/assets/lcd/listening2.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

components/platform/src/lcd_direct_assets.c
+12
View File
@@ -20,6 +20,12 @@ static const lcd_direct_image_t s_basic_face_frames[] = {
},
};
static const lcd_direct_image_t s_listening_frames[] = {
{.path = LCD_ASSET_BASE_PATH "listening0.png"},
{.path = LCD_ASSET_BASE_PATH "listening1.png"},
{.path = LCD_ASSET_BASE_PATH "listening2.png"},
};
static const lcd_direct_image_t s_muyu_frames[] = {
{
.path = LCD_ASSET_BASE_PATH "muyu0.png",
@@ -236,6 +242,12 @@ const lcd_direct_image_t *lcd_direct_asset_frame(const char *asset_dir, uint8_t
frame_index);
}
if (strcmp(asset_dir, "listening") == 0) {
return lcd_direct_frame_from_array(s_listening_frames,
sizeof(s_listening_frames) / sizeof(s_listening_frames[0]),
frame_index);
}
if (strcmp(asset_dir, "muyu") == 0) {
return lcd_direct_frame_from_array(s_muyu_frames,
sizeof(s_muyu_frames) / sizeof(s_muyu_frames[0]),
components/platform/src/multimodal/audio_pipeline.c
+3
View File
@@ -344,6 +344,9 @@ typedef struct {
volatile bool uplink_recording_active;
volatile bool uplink_task_exit_requested;
volatile bool uplink_send_in_flight;
bool uplink_first_network_pcm_pending;
bool uplink_first_network_packet_pending;
TickType_t uplink_network_start_tick;
multimodal_audio_uplink_cb_t uplink_cb;
void *uplink_cb_ctx;
bool opus_ready;
components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c
+65
View File
@@ -157,6 +157,42 @@ static void audio_set_uplink_callback(multimodal_audio_ctx_t *ctx,
portEXIT_CRITICAL(&s_audio_runtime_lock);
}
static bool audio_consume_first_network_pcm(multimodal_audio_ctx_t *ctx,
uint32_t *out_elapsed_ms)
{
bool pending = false;
TickType_t started = 0U;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (ctx != NULL && ctx->uplink_first_network_pcm_pending) {
ctx->uplink_first_network_pcm_pending = false;
started = ctx->uplink_network_start_tick;
pending = true;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
if (pending && out_elapsed_ms != NULL) {
*out_elapsed_ms = (uint32_t)pdTICKS_TO_MS(xTaskGetTickCount() - started);
}
return pending;
}
static bool audio_consume_first_network_packet(multimodal_audio_ctx_t *ctx,
uint32_t *out_elapsed_ms)
{
bool pending = false;
TickType_t started = 0U;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (ctx != NULL && ctx->uplink_first_network_packet_pending) {
ctx->uplink_first_network_packet_pending = false;
started = ctx->uplink_network_start_tick;
pending = true;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
if (pending && out_elapsed_ms != NULL) {
*out_elapsed_ms = (uint32_t)pdTICKS_TO_MS(xTaskGetTickCount() - started);
}
return pending;
}
static void notify_pcm_observer(const int16_t *pcm, size_t samples)
{
multimodal_audio_pcm_observer_t observer = NULL;
@@ -988,6 +1024,14 @@ static void uplink_send_task(void *arg)
ctx->uplink_send_in_flight = true;
esp_err_t send_err = cb(packet.data, packet.len, cb_ctx);
ctx->uplink_send_in_flight = false;
uint32_t first_packet_elapsed_ms = 0U;
if (send_err == ESP_OK &&
audio_consume_first_network_packet(ctx, &first_packet_elapsed_ms)) {
ESP_LOGI(TAG,
"Mic uplink first packet sent: activation_to_send_ms=%u bytes=%u",
(unsigned)first_packet_elapsed_ms,
(unsigned)packet.len);
}
track_uplink_send_result(send_err, packet.len);
if (send_err != ESP_OK && send_err != ESP_ERR_INVALID_STATE) {
if (send_err != ESP_ERR_TIMEOUT) {
@@ -1068,6 +1112,13 @@ static void uplink_capture_task(void *arg)
continue;
}
size_t uplink_frames = ctx->uplink_aec_frame_samples;
uint32_t first_pcm_elapsed_ms = 0U;
if (audio_consume_first_network_pcm(ctx, &first_pcm_elapsed_ms)) {
ESP_LOGI(TAG,
"Mic uplink first PCM captured: activation_to_pcm_ms=%u samples=%u",
(unsigned)first_pcm_elapsed_ms,
(unsigned)uplink_frames);
}
notify_pcm_observer(ctx->uplink_aec_clean_buf, uplink_frames);
audio_debug_dump_pcm(ctx->uplink_aec_clean_buf, clean_bytes);
multimodal_audio_uplink_cb_t uplink_cb = NULL;
@@ -1410,6 +1461,7 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
}
bool already_active = audio_get_uplink_recording_active(&s_audio);
bool network_uplink = cb != NULL;
audio_set_uplink_callback(&s_audio, cb, cb_ctx);
if (already_active) {
return ESP_OK;
@@ -1426,6 +1478,11 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
}
reset_uplink_send_queue(&s_audio);
portENTER_CRITICAL(&s_audio_runtime_lock);
s_audio.uplink_network_start_tick = xTaskGetTickCount();
s_audio.uplink_first_network_pcm_pending = network_uplink;
s_audio.uplink_first_network_packet_pending = network_uplink;
portEXIT_CRITICAL(&s_audio_runtime_lock);
audio_set_uplink_recording_active(&s_audio, true);
TaskHandle_t capture_task = NULL;
audio_snapshot_uplink_tasks(&s_audio, &capture_task, NULL);
@@ -1433,6 +1490,10 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
xTaskNotifyGive(capture_task);
}
log_heap_snapshot("after_uplink_start");
ESP_LOGI(TAG,
"Mic uplink activated: mode=%s sample_rate=%u channels=1",
network_uplink ? "network" : "local_observer_only",
(unsigned)CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE);
return ESP_OK;
}
@@ -1443,6 +1504,10 @@ esp_err_t multimodal_audio_stop_uplink(int timeout_ms)
}
audio_set_uplink_recording_active(&s_audio, false);
portENTER_CRITICAL(&s_audio_runtime_lock);
s_audio.uplink_first_network_pcm_pending = false;
s_audio.uplink_first_network_packet_pending = false;
portEXIT_CRITICAL(&s_audio_runtime_lock);
TickType_t deadline = xTaskGetTickCount() + pdMS_TO_TICKS((uint32_t)timeout_ms);
while (((s_audio.uplink_send_queue != NULL &&