diff --git a/components/control_plane/internal/control_plane_internal.h b/components/control_plane/internal/control_plane_internal.h index 3b3f9b9..b0932ef 100644 --- a/components/control_plane/internal/control_plane_internal.h +++ b/components/control_plane/internal/control_plane_internal.h @@ -137,6 +137,8 @@ esp_err_t control_plane_local_wake_start(void); esp_err_t control_plane_local_wake_stop(int timeout_ms); void control_plane_local_wake_pause_for_foreground(void); void control_plane_local_wake_resume_after_foreground(void); +bool control_plane_local_wake_prompt_gate_active(void); +void control_plane_ptt_reset_capture_window(void); bool control_plane_voice_turn_is_idle(void); esp_err_t control_plane_start_system_event_task(void); diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c index 78cf954..bcc7d85 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c @@ -25,6 +25,10 @@ #define CONFIG_TQ_LOCAL_WAKE_RETRY_MS 5000 #endif +#ifndef CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS +#define CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS 10000 +#endif + #ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS #define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000 #endif @@ -43,8 +47,6 @@ #define CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN 4U #define CONTROL_PLANE_LOCAL_WAKE_POLL_MS 100U -#define CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_WAIT_MS 2000U -#define CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_POLL_MS 20U #if CONFIG_TQ_LOCAL_WAKE_TASK_CORE < 0 #define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE tskNO_AFFINITY @@ -59,6 +61,11 @@ typedef struct { bool started; bool paused_for_foreground; bool capture_owned; + bool prompt_pending; + bool prompt_playing; + bool listening_ui_active; + TickType_t prompt_wait_started_tick; + uint32_t detection_generation; } local_wake_state_t; static const char *TAG = "cp_local_wake"; @@ -81,6 +88,31 @@ static void local_wake_set_capture_owned(bool owned) portEXIT_CRITICAL(&s_local_wake_lock); } +static void local_wake_request_expression(domain_expression_id_t id) +{ + const domain_expression_request_t request = { + .id = id, + .trigger = DOMAIN_EXPRESSION_TRIGGER_LOCAL_WAKE, + }; + esp_err_t err = domain_runtime_request_expression(&request); + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, + "Local wake expression request failed: id=%d err=%s", + (int)id, + esp_err_to_name(err)); + } +} + +bool control_plane_local_wake_prompt_gate_active(void) +{ +#if CONFIG_TQ_LOCAL_WAKE_ENABLE && CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE + local_wake_state_t state = local_wake_snapshot(); + return state.prompt_pending || state.prompt_playing; +#else + return false; +#endif +} + static void local_wake_domain_event( const domain_local_command_event_t *event, void *user_data) @@ -153,63 +185,107 @@ static esp_err_t local_wake_start_listening(void) return ESP_OK; } -#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE -static bool local_wake_prepare_prompt_playback(void) +static bool local_wake_runtime_ready_for_prompt( + control_plane_multimodal_runtime_state_t *out_state) { control_plane_multimodal_runtime_state_t state = {0}; esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state); if (state_err != ESP_OK) { - ESP_LOGW(TAG, - "Skip local wake prompt: runtime state unavailable: %s", - esp_err_to_name(state_err)); return false; } + if (out_state != NULL) { + *out_state = state; + } + return state.connected && + state.has_active_task && + state.server_listening && + !state.remote_responding && + !state.request_to_speak_pending && + !state.client_speech_active && + !state.ptt_release_required; +} - if (!state.remote_responding && !state.request_to_speak_pending) { - return true; +static void local_wake_service_ready_prompt(void) +{ +#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE + local_wake_state_t state = local_wake_snapshot(); + if (!state.prompt_pending || state.prompt_playing || state.stop_requested) { + return; } - esp_err_t request_err = control_plane_multimodal_send_request_to_speak(); - if (request_err != ESP_OK && request_err != ESP_ERR_INVALID_STATE) { - ESP_LOGW(TAG, - "Skip local wake prompt: barge-in request failed: %s", - esp_err_to_name(request_err)); - return false; - } - - TickType_t started = xTaskGetTickCount(); - TickType_t timeout_ticks = pdMS_TO_TICKS( - CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_WAIT_MS); + TickType_t now = xTaskGetTickCount(); + TickType_t timeout_ticks = pdMS_TO_TICKS(CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS); if (timeout_ticks == 0U) { timeout_ticks = 1U; } - - while ((xTaskGetTickCount() - started) < timeout_ticks) { - local_wake_state_t wake_state = local_wake_snapshot(); - if (wake_state.stop_requested) { - return false; - } - - state_err = control_plane_multimodal_get_runtime_state(&state); - if (state_err == ESP_OK) { - if (!state.connected || !state.has_active_task) { - return false; - } - if (!state.remote_responding && - !state.request_to_speak_pending && - state.server_listening) { - return true; - } - } - vTaskDelay(pdMS_TO_TICKS(CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_POLL_MS)); + control_plane_multimodal_runtime_state_t runtime = {0}; + bool runtime_ready = local_wake_runtime_ready_for_prompt(&runtime); + if (state.prompt_wait_started_tick != 0U && + (now - state.prompt_wait_started_tick) >= timeout_ticks) { + ESP_LOGW(TAG, + "Local wake network readiness timed out: generation=%u timeout_ms=%u ws=%d task=%d listening=%d responding=%d request_pending=%d speech=%d release_required=%d", + (unsigned)state.detection_generation, + (unsigned)CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS, + runtime.connected ? 1 : 0, + runtime.has_active_task ? 1 : 0, + runtime.server_listening ? 1 : 0, + runtime.remote_responding ? 1 : 0, + runtime.request_to_speak_pending ? 1 : 0, + runtime.client_speech_active ? 1 : 0, + runtime.ptt_release_required ? 1 : 0); + control_plane_set_ptt_active(false); + return; } - ESP_LOGW(TAG, - "Skip local wake prompt: barge-in confirmation timed out after %ums", - (unsigned)CONTROL_PLANE_LOCAL_WAKE_BARGE_IN_WAIT_MS); - return false; -} + if (!runtime_ready) { + return; + } + + portENTER_CRITICAL(&s_local_wake_lock); + if (!s_local_wake.prompt_pending || s_local_wake.stop_requested) { + portEXIT_CRITICAL(&s_local_wake_lock); + return; + } + s_local_wake.prompt_pending = false; + s_local_wake.prompt_playing = true; + portEXIT_CRITICAL(&s_local_wake_lock); + + const uint32_t ready_wait_ms = state.prompt_wait_started_tick != 0U ? + (uint32_t)pdTICKS_TO_MS(now - state.prompt_wait_started_tick) : 0U; + ESP_LOGI(TAG, + "Local wake network ready: generation=%u wait_ms=%u ws=1 task=1 listening=1", + (unsigned)state.detection_generation, + (unsigned)ready_wait_ms); + TickType_t prompt_started_tick = xTaskGetTickCount(); + ESP_LOGI(TAG, + "Local wake response prompt started: generation=%u file=%s", + (unsigned)state.detection_generation, + CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE); + esp_err_t prompt_err = domain_runtime_behavior_play_prompt_file( + CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE); + const uint32_t prompt_elapsed_ms = (uint32_t)pdTICKS_TO_MS( + xTaskGetTickCount() - prompt_started_tick); + + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.prompt_playing = false; + s_local_wake.prompt_wait_started_tick = 0U; + portEXIT_CRITICAL(&s_local_wake_lock); + + if (prompt_err != ESP_OK) { + ESP_LOGW(TAG, + "Local wake response prompt failed: generation=%u elapsed_ms=%u err=%s", + (unsigned)state.detection_generation, + (unsigned)prompt_elapsed_ms, + esp_err_to_name(prompt_err)); + } else { + ESP_LOGI(TAG, + "Local wake response prompt completed: generation=%u elapsed_ms=%u", + (unsigned)state.detection_generation, + (unsigned)prompt_elapsed_ms); + } + control_plane_ptt_reset_capture_window(); #endif +} static void local_wake_handle_detection( const domain_local_command_event_t *event) @@ -226,31 +302,31 @@ static void local_wake_handle_detection( return; } s_local_wake.paused_for_foreground = true; + s_local_wake.listening_ui_active = true; + s_local_wake.detection_generation = event->detection_generation; +#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE + s_local_wake.prompt_pending = CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE[0] != '\0'; + s_local_wake.prompt_playing = false; + s_local_wake.prompt_wait_started_tick = xTaskGetTickCount(); +#endif portEXIT_CRITICAL(&s_local_wake_lock); + local_wake_request_expression(DOMAIN_EXPRESSION_LISTENING); + ESP_LOGI(TAG, + "Local wake foreground entered: generation=%u listening_ui=1 prompt_gate=%d ready_timeout_ms=%u", + (unsigned)event->detection_generation, + control_plane_local_wake_prompt_gate_active() ? 1 : 0, + (unsigned)CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS); + local_wake_stop_owned_capture(); (void)domain_runtime_local_command_suspend( CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); ESP_LOGI(TAG, - "Local wake accepted: phrase=喵掌柜 confidence_x10000=%u generation=%u", + "Local wake accepted: phrase=喵掌柜 confidence_x10000=%u generation=%u; network preparation requested", (unsigned)event->confidence_x10000, (unsigned)event->detection_generation); -#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE - bool prompt_playback_ready = local_wake_prepare_prompt_playback(); - if (prompt_playback_ready && - CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE[0] != '\0') { - esp_err_t prompt_err = domain_runtime_behavior_play_prompt_file( - CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE); - if (prompt_err != ESP_OK) { - ESP_LOGW(TAG, - "Local wake response prompt failed: %s", - esp_err_to_name(prompt_err)); - } - } -#endif - control_plane_set_ptt_active(true); } @@ -264,6 +340,7 @@ static void local_wake_task(void *arg) while (!local_wake_snapshot().stop_requested) { local_wake_state_t state = local_wake_snapshot(); TickType_t now = xTaskGetTickCount(); + local_wake_service_ready_prompt(); if (!state.paused_for_foreground && control_plane_voice_turn_is_idle() && (retry_not_before == 0U || now >= retry_not_before)) { @@ -342,6 +419,11 @@ esp_err_t control_plane_local_wake_start(void) s_local_wake.stop_requested = false; s_local_wake.paused_for_foreground = false; s_local_wake.capture_owned = false; + s_local_wake.prompt_pending = false; + s_local_wake.prompt_playing = false; + s_local_wake.listening_ui_active = false; + s_local_wake.prompt_wait_started_tick = 0U; + s_local_wake.detection_generation = 0U; s_local_wake.started = true; portEXIT_CRITICAL(&s_local_wake_lock); @@ -434,10 +516,19 @@ void control_plane_local_wake_pause_for_foreground(void) void control_plane_local_wake_resume_after_foreground(void) { #if CONFIG_TQ_LOCAL_WAKE_ENABLE + bool restore_idle_ui = false; portENTER_CRITICAL(&s_local_wake_lock); s_local_wake.paused_for_foreground = false; + s_local_wake.prompt_pending = false; + s_local_wake.prompt_playing = false; + s_local_wake.prompt_wait_started_tick = 0U; + restore_idle_ui = s_local_wake.listening_ui_active; + s_local_wake.listening_ui_active = false; TaskHandle_t task = s_local_wake.task; portEXIT_CRITICAL(&s_local_wake_lock); + if (restore_idle_ui) { + local_wake_request_expression(DOMAIN_EXPRESSION_IDLE); + } if (task != NULL) { xTaskNotifyGive(task); } diff --git a/components/control_plane/src/runtime/control_plane_runtime_workers.c b/components/control_plane/src/runtime/control_plane_runtime_workers.c index a3765f6..1118878 100644 --- a/components/control_plane/src/runtime/control_plane_runtime_workers.c +++ b/components/control_plane/src/runtime/control_plane_runtime_workers.c @@ -152,6 +152,7 @@ typedef enum { PTT_STATE_WAIT_WS, PTT_STATE_WAIT_START_SENT, PTT_STATE_WAIT_LISTENING, + PTT_STATE_WAIT_WAKE_PROMPT, PTT_STATE_WAIT_SPEECH_STARTED, PTT_STATE_STREAMING, PTT_STATE_TEARDOWN_LOCAL, @@ -227,6 +228,8 @@ static const char *ptt_state_name(ptt_state_t state) return "WAIT_START_SENT"; case PTT_STATE_WAIT_LISTENING: return "WAIT_LISTENING"; + case PTT_STATE_WAIT_WAKE_PROMPT: + return "WAIT_WAKE_PROMPT"; case PTT_STATE_WAIT_SPEECH_STARTED: return "WAIT_SPEECH_STARTED"; case PTT_STATE_STREAMING: @@ -349,6 +352,7 @@ void control_plane_set_ptt_active(bool active) s_ptt_runtime.state == PTT_STATE_WAIT_WS || s_ptt_runtime.state == PTT_STATE_WAIT_START_SENT || s_ptt_runtime.state == PTT_STATE_WAIT_LISTENING || + s_ptt_runtime.state == PTT_STATE_WAIT_WAKE_PROMPT || s_ptt_runtime.state == PTT_STATE_WAIT_SPEECH_STARTED || s_ptt_runtime.state == PTT_STATE_DENIED) { control_plane_ptt_transition_locked(PTT_STATE_IDLE, "release_before_streaming"); @@ -368,6 +372,30 @@ void control_plane_set_ptt_active(bool active) } } +void control_plane_ptt_reset_capture_window(void) +{ + bool active = false; + uint32_t generation = 0U; + portENTER_CRITICAL(&s_runtime_lock); + active = s_ptt_runtime.active; + if (active) { + s_ptt_runtime.press_tick = xTaskGetTickCount(); + generation = s_ptt_runtime.generation; + } + portEXIT_CRITICAL(&s_runtime_lock); + + if (!active) { + ESP_LOGW(TAG, "Skip PTT capture window reset: turn is no longer active"); + return; + } + + ESP_LOGI(TAG, + "PTT capture window armed after local wake prompt: generation=%u timeout_ms=%u", + (unsigned)generation, + (unsigned)CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS); + control_plane_notify_wifi_state_task(); +} + bool control_plane_voice_turn_is_idle(void) { bool idle = false; @@ -1218,12 +1246,14 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte } TickType_t now = xTaskGetTickCount(); + bool local_wake_prompt_gate = control_plane_local_wake_prompt_gate_active(); #if CONFIG_TQ_PTT_MODE_CLICK_TO_TALK TickType_t capture_timeout_ticks = pdMS_TO_TICKS(CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS); if (capture_timeout_ticks == 0) { capture_timeout_ticks = 1; } - if (ptt.press_tick != 0 && + if (!local_wake_prompt_gate && + ptt.press_tick != 0 && (now - ptt.press_tick) >= capture_timeout_ticks) { control_plane_set_ptt_active(false); return; @@ -1234,7 +1264,8 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte if (denied_ticks == 0) { denied_ticks = 1; } - if (ptt.press_tick != 0 && + if (!local_wake_prompt_gate && + ptt.press_tick != 0 && (now - ptt.press_tick) >= denied_ticks && ptt.state != PTT_STATE_STREAMING && ptt.state != PTT_STATE_DENIED) { @@ -1287,6 +1318,8 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte control_plane_ptt_set_state(PTT_STATE_WAIT_START_SENT, "send_start"); } else if (!state.server_listening) { control_plane_ptt_set_state(PTT_STATE_WAIT_LISTENING, "wait_listening"); + } else if (local_wake_prompt_gate) { + control_plane_ptt_set_state(PTT_STATE_WAIT_WAKE_PROMPT, "wait_local_wake_prompt"); } else if (!state.client_speech_active) { control_plane_ptt_set_state(PTT_STATE_WAIT_SPEECH_STARTED, "send_speech"); } else if (control_plane_multimodal_is_uplink_running()) { @@ -1296,7 +1329,8 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte } ptt = control_plane_ptt_snapshot(); - if ((ptt.state == PTT_STATE_WAIT_START_SENT || + if (!local_wake_prompt_gate && + (ptt.state == PTT_STATE_WAIT_START_SENT || ptt.state == PTT_STATE_WAIT_LISTENING) && (now - ptt.state_enter_tick) >= pdMS_TO_TICKS(CONTROL_PLANE_PTT_LISTENING_TIMEOUT_MS)) { @@ -1327,6 +1361,9 @@ static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_atte if (ptt.state == PTT_STATE_WAIT_SPEECH_STARTED && ptt.speech_action_sent) { should_run_press_step = false; } + if (ptt.state == PTT_STATE_WAIT_WAKE_PROMPT) { + should_run_press_step = false; + } if (!should_run_press_step) { return; } diff --git a/components/domain/include/domain.h b/components/domain/include/domain.h index a76a912..cfd0ad5 100644 --- a/components/domain/include/domain.h +++ b/components/domain/include/domain.h @@ -118,6 +118,7 @@ typedef enum { DOMAIN_EXPRESSION_MUYU, DOMAIN_EXPRESSION_CHOUQIAN, DOMAIN_EXPRESSION_WUXING, + DOMAIN_EXPRESSION_LISTENING, } domain_expression_id_t; typedef enum { @@ -132,6 +133,7 @@ typedef enum { DOMAIN_EXPRESSION_TRIGGER_PTT_DOUBLE_CLICK, DOMAIN_EXPRESSION_TRIGGER_SW2_SINGLE_CLICK, DOMAIN_EXPRESSION_TRIGGER_SW2_DOUBLE_CLICK, + DOMAIN_EXPRESSION_TRIGGER_LOCAL_WAKE, } domain_expression_trigger_t; typedef struct { diff --git a/components/domain/src/runtime/domain_runtime_adapters.c b/components/domain/src/runtime/domain_runtime_adapters.c index 7acb13b..a3ea016 100644 --- a/components/domain/src/runtime/domain_runtime_adapters.c +++ b/components/domain/src/runtime/domain_runtime_adapters.c @@ -78,6 +78,7 @@ static const uint8_t s_server_error_frame_order[] = {0, 1, 2, 1, 0}; static const uint8_t s_wifi_disconnected_frame_order[] = {0, 1, 2, 3, 2, 1, 0}; static const uint8_t s_basic_face_frame_order[] = {0, 1, 2}; static const uint8_t s_muyu_frame_order[] = {0, 1, 2, 1}; +static const uint8_t s_listening_frame_order[] = {0, 1, 2}; static const char *const s_chouqian_result_asset_dirs[] = { "chouqian_buyun", "chouqian_heping", @@ -227,6 +228,21 @@ static const domain_expression_catalog_entry_t s_expression_catalog[] = { .duration_ms = 0, }, }, + { + .id = DOMAIN_EXPRESSION_LISTENING, + .platform_request = { + .kind = PLATFORM_LCD_PRESENTATION_ASSET_SEQUENCE, + .priority = 130, + .interrupt_current = true, + .return_to_idle = false, + .asset_dir = "listening", + .frame_order = s_listening_frame_order, + .frame_order_count = sizeof(s_listening_frame_order) / + sizeof(s_listening_frame_order[0]), + .repeat_count = 0, + .frame_ms = 500, + }, + }, }; static void snapshot_domain_wifi_behavior_hooks(domain_wifi_behavior_hooks_t *out_hooks) diff --git a/components/platform/CMakeLists.txt b/components/platform/CMakeLists.txt index 8a8e525..4b1a626 100644 --- a/components/platform/CMakeLists.txt +++ b/components/platform/CMakeLists.txt @@ -82,6 +82,9 @@ set(platform_lcd_png_assets jichubiaoqing0 jichubiaoqing1 jichubiaoqing2 + listening0 + listening1 + listening2 muyu0 muyu1 muyu2 diff --git a/components/platform/assets/lcd/listening0.png b/components/platform/assets/lcd/listening0.png new file mode 100644 index 0000000..0891c1c Binary files /dev/null and b/components/platform/assets/lcd/listening0.png differ diff --git a/components/platform/assets/lcd/listening1.png b/components/platform/assets/lcd/listening1.png new file mode 100644 index 0000000..3a873a8 Binary files /dev/null and b/components/platform/assets/lcd/listening1.png differ diff --git a/components/platform/assets/lcd/listening2.png b/components/platform/assets/lcd/listening2.png new file mode 100644 index 0000000..0dc3866 Binary files /dev/null and b/components/platform/assets/lcd/listening2.png differ diff --git a/components/platform/src/lcd_direct_assets.c b/components/platform/src/lcd_direct_assets.c index bfb8ae4..812ca64 100644 --- a/components/platform/src/lcd_direct_assets.c +++ b/components/platform/src/lcd_direct_assets.c @@ -20,6 +20,12 @@ static const lcd_direct_image_t s_basic_face_frames[] = { }, }; +static const lcd_direct_image_t s_listening_frames[] = { + {.path = LCD_ASSET_BASE_PATH "listening0.png"}, + {.path = LCD_ASSET_BASE_PATH "listening1.png"}, + {.path = LCD_ASSET_BASE_PATH "listening2.png"}, +}; + static const lcd_direct_image_t s_muyu_frames[] = { { .path = LCD_ASSET_BASE_PATH "muyu0.png", @@ -236,6 +242,12 @@ const lcd_direct_image_t *lcd_direct_asset_frame(const char *asset_dir, uint8_t frame_index); } + if (strcmp(asset_dir, "listening") == 0) { + return lcd_direct_frame_from_array(s_listening_frames, + sizeof(s_listening_frames) / sizeof(s_listening_frames[0]), + frame_index); + } + if (strcmp(asset_dir, "muyu") == 0) { return lcd_direct_frame_from_array(s_muyu_frames, sizeof(s_muyu_frames) / sizeof(s_muyu_frames[0]), diff --git a/components/platform/src/multimodal/audio_pipeline.c b/components/platform/src/multimodal/audio_pipeline.c index cdfed89..83caba1 100644 --- a/components/platform/src/multimodal/audio_pipeline.c +++ b/components/platform/src/multimodal/audio_pipeline.c @@ -344,6 +344,9 @@ typedef struct { volatile bool uplink_recording_active; volatile bool uplink_task_exit_requested; volatile bool uplink_send_in_flight; + bool uplink_first_network_pcm_pending; + bool uplink_first_network_packet_pending; + TickType_t uplink_network_start_tick; multimodal_audio_uplink_cb_t uplink_cb; void *uplink_cb_ctx; bool opus_ready; diff --git a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c index c359902..07a8377 100644 --- a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c +++ b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c @@ -157,6 +157,42 @@ static void audio_set_uplink_callback(multimodal_audio_ctx_t *ctx, portEXIT_CRITICAL(&s_audio_runtime_lock); } +static bool audio_consume_first_network_pcm(multimodal_audio_ctx_t *ctx, + uint32_t *out_elapsed_ms) +{ + bool pending = false; + TickType_t started = 0U; + portENTER_CRITICAL(&s_audio_runtime_lock); + if (ctx != NULL && ctx->uplink_first_network_pcm_pending) { + ctx->uplink_first_network_pcm_pending = false; + started = ctx->uplink_network_start_tick; + pending = true; + } + portEXIT_CRITICAL(&s_audio_runtime_lock); + if (pending && out_elapsed_ms != NULL) { + *out_elapsed_ms = (uint32_t)pdTICKS_TO_MS(xTaskGetTickCount() - started); + } + return pending; +} + +static bool audio_consume_first_network_packet(multimodal_audio_ctx_t *ctx, + uint32_t *out_elapsed_ms) +{ + bool pending = false; + TickType_t started = 0U; + portENTER_CRITICAL(&s_audio_runtime_lock); + if (ctx != NULL && ctx->uplink_first_network_packet_pending) { + ctx->uplink_first_network_packet_pending = false; + started = ctx->uplink_network_start_tick; + pending = true; + } + portEXIT_CRITICAL(&s_audio_runtime_lock); + if (pending && out_elapsed_ms != NULL) { + *out_elapsed_ms = (uint32_t)pdTICKS_TO_MS(xTaskGetTickCount() - started); + } + return pending; +} + static void notify_pcm_observer(const int16_t *pcm, size_t samples) { multimodal_audio_pcm_observer_t observer = NULL; @@ -988,6 +1024,14 @@ static void uplink_send_task(void *arg) ctx->uplink_send_in_flight = true; esp_err_t send_err = cb(packet.data, packet.len, cb_ctx); ctx->uplink_send_in_flight = false; + uint32_t first_packet_elapsed_ms = 0U; + if (send_err == ESP_OK && + audio_consume_first_network_packet(ctx, &first_packet_elapsed_ms)) { + ESP_LOGI(TAG, + "Mic uplink first packet sent: activation_to_send_ms=%u bytes=%u", + (unsigned)first_packet_elapsed_ms, + (unsigned)packet.len); + } track_uplink_send_result(send_err, packet.len); if (send_err != ESP_OK && send_err != ESP_ERR_INVALID_STATE) { if (send_err != ESP_ERR_TIMEOUT) { @@ -1068,6 +1112,13 @@ static void uplink_capture_task(void *arg) continue; } size_t uplink_frames = ctx->uplink_aec_frame_samples; + uint32_t first_pcm_elapsed_ms = 0U; + if (audio_consume_first_network_pcm(ctx, &first_pcm_elapsed_ms)) { + ESP_LOGI(TAG, + "Mic uplink first PCM captured: activation_to_pcm_ms=%u samples=%u", + (unsigned)first_pcm_elapsed_ms, + (unsigned)uplink_frames); + } notify_pcm_observer(ctx->uplink_aec_clean_buf, uplink_frames); audio_debug_dump_pcm(ctx->uplink_aec_clean_buf, clean_bytes); multimodal_audio_uplink_cb_t uplink_cb = NULL; @@ -1410,6 +1461,7 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c } bool already_active = audio_get_uplink_recording_active(&s_audio); + bool network_uplink = cb != NULL; audio_set_uplink_callback(&s_audio, cb, cb_ctx); if (already_active) { return ESP_OK; @@ -1426,6 +1478,11 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c } reset_uplink_send_queue(&s_audio); + portENTER_CRITICAL(&s_audio_runtime_lock); + s_audio.uplink_network_start_tick = xTaskGetTickCount(); + s_audio.uplink_first_network_pcm_pending = network_uplink; + s_audio.uplink_first_network_packet_pending = network_uplink; + portEXIT_CRITICAL(&s_audio_runtime_lock); audio_set_uplink_recording_active(&s_audio, true); TaskHandle_t capture_task = NULL; audio_snapshot_uplink_tasks(&s_audio, &capture_task, NULL); @@ -1433,6 +1490,10 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c xTaskNotifyGive(capture_task); } log_heap_snapshot("after_uplink_start"); + ESP_LOGI(TAG, + "Mic uplink activated: mode=%s sample_rate=%u channels=1", + network_uplink ? "network" : "local_observer_only", + (unsigned)CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE); return ESP_OK; } @@ -1443,6 +1504,10 @@ esp_err_t multimodal_audio_stop_uplink(int timeout_ms) } audio_set_uplink_recording_active(&s_audio, false); + portENTER_CRITICAL(&s_audio_runtime_lock); + s_audio.uplink_first_network_pcm_pending = false; + s_audio.uplink_first_network_packet_pending = false; + portEXIT_CRITICAL(&s_audio_runtime_lock); TickType_t deadline = xTaskGetTickCount() + pdMS_TO_TICKS((uint32_t)timeout_ms); while (((s_audio.uplink_send_queue != NULL && diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index f208574..042a592 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -908,6 +908,16 @@ menu "Example Configuration" default 5000 range 500 60000 + config TQ_LOCAL_WAKE_READY_TIMEOUT_MS + int "Local wake network readiness timeout (ms)" + depends on TQ_LOCAL_WAKE_ENABLE + default 10000 + range 1000 30000 + help + Maximum time to prepare the multimodal transport and reach the + server Listening state before the local wake turn is aborted. + The acknowledgement prompt is played only after readiness. + config TQ_LOCAL_WAKE_STOP_WAIT_MS int "Local wake lifecycle wait (ms)" depends on TQ_LOCAL_WAKE_ENABLE diff --git a/sdkconfig.defaults b/sdkconfig.defaults index 5780c38..db63ba1 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -63,6 +63,7 @@ CONFIG_TQ_LOCAL_WAKE_ENABLE=y CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE=y CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE="local_wake_response.opus" CONFIG_TQ_LOCAL_WAKE_RETRY_MS=5000 +CONFIG_TQ_LOCAL_WAKE_READY_TIMEOUT_MS=10000 CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS=1000 CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES=6144 CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY=4 diff --git a/tools/check_architecture_guards.sh b/tools/check_architecture_guards.sh index 1d36286..5e3b436 100644 --- a/tools/check_architecture_guards.sh +++ b/tools/check_architecture_guards.sh @@ -50,6 +50,10 @@ fi if ! rg -n 'TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE' \ main/Kconfig.projbuild \ components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1 || + ! rg -n 'TQ_LOCAL_WAKE_READY_TIMEOUT_MS' \ + main/Kconfig.projbuild \ + sdkconfig.defaults \ + components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1 || ! rg -n 'CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE=y' \ sdkconfig.defaults >/dev/null 2>&1 || ! rg -n 'local_wake_response\.opus' \ @@ -59,6 +63,21 @@ if ! rg -n 'TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE' \ [[ ! -f components/platform/assets/audio_prompts/local_wake_response.opus ]]; then fail "Local wake response prompt must remain enabled with its packaged asset contract" fi +if ! rg -n 'DOMAIN_EXPRESSION_LISTENING|DOMAIN_EXPRESSION_TRIGGER_LOCAL_WAKE' \ + components/domain/include/domain.h \ + components/domain/src/runtime/domain_runtime_adapters.c \ + components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1 || + ! rg -n 'PTT_STATE_WAIT_WAKE_PROMPT|control_plane_local_wake_prompt_gate_active' \ + components/control_plane/src/runtime/control_plane_runtime_workers.c >/dev/null 2>&1 || + ! rg -n '!local_wake_prompt_gate.*&&' \ + components/control_plane/src/runtime/control_plane_runtime_workers.c >/dev/null 2>&1 || + ! rg -n '^ listening[012]$' components/platform/CMakeLists.txt >/dev/null 2>&1 || + ! rg -n 'listening[012]\.png' components/platform/src/lcd_direct_assets.c >/dev/null 2>&1 || + [[ ! -f components/platform/assets/lcd/listening0.png ]] || + [[ ! -f components/platform/assets/lcd/listening1.png ]] || + [[ ! -f components/platform/assets/lcd/listening2.png ]]; then + fail "Local wake must show its packaged listening animation while gating speech uplink until the ready prompt finishes" +fi if ! rg -n '#include "esp_afe_aec.h"' components/platform/src/multimodal/audio_pipeline.c >/dev/null 2>&1 || ! rg -n 'afe_aec_create\(' components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c >/dev/null 2>&1 || ! rg -n 'afe_aec_process\(' components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c >/dev/null 2>&1; then