From 609fe978f1fd832061b62a27c2314068c4e501ae Mon Sep 17 00:00:00 2001 From: chenjintao Date: Fri, 14 Aug 2026 18:23:46 +0800 Subject: [PATCH] fix(multimodal): protect voice startup from background uploads --- .../internal/control_plane_internal.h | 1 + .../control_plane_multimodal_local_wake.c | 13 ++++++ ...control_plane_multimodal_session_service.c | 43 +++++++++++++++++-- .../control_plane_remote_log_service.c | 34 ++++++++++++++- 4 files changed, 85 insertions(+), 6 deletions(-) diff --git a/components/control_plane/internal/control_plane_internal.h b/components/control_plane/internal/control_plane_internal.h index f1b3801..e1f472d 100644 --- a/components/control_plane/internal/control_plane_internal.h +++ b/components/control_plane/internal/control_plane_internal.h @@ -77,6 +77,7 @@ typedef struct { bool request_to_speak_pending; bool client_speech_active; bool ptt_release_required; + bool response_pending; bool local_responding_started; bool received_downstream_audio; uint32_t last_heartbeat_ms; diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c index babc072..9acef34 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c @@ -233,6 +233,19 @@ static void local_wake_service_ready_prompt(void) runtime.request_to_speak_pending ? 1 : 0, runtime.client_speech_active ? 1 : 0, runtime.ptt_release_required ? 1 : 0); + bool incomplete_task_start = runtime.connected && + runtime.has_active_task && + !runtime.server_listening && + !runtime.remote_responding && + !runtime.request_to_speak_pending && + !runtime.client_speech_active; + if (incomplete_task_start) { + esp_err_t stop_err = control_plane_multimodal_send_stop( + "LocalWakeListeningTimeout"); + ESP_LOGW(TAG, + "Local wake timed-out task cleanup: result=%s", + esp_err_to_name(stop_err)); + } control_plane_set_ptt_active(false); return; } diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c index e474ef8..bb7838d 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c @@ -148,6 +148,7 @@ static bool s_remote_responding_ended; static bool s_request_to_speak_pending; static bool s_client_speech_active; static bool s_ptt_release_required; +static bool s_response_pending; static bool s_local_responding_started; static bool s_local_responding_ended_sent; static bool s_downstream_playback_completed; @@ -437,6 +438,7 @@ static void reset_session_state_locked(void) s_request_to_speak_pending = false; s_client_speech_active = false; s_ptt_release_required = false; + s_response_pending = false; s_local_responding_started = false; s_local_responding_ended_sent = false; s_downstream_playback_completed = false; @@ -1307,6 +1309,8 @@ static void process_text_json(const char *json_text) cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header"); cJSON *event = header ? cJSON_GetObjectItemCaseSensitive(header, "event") : NULL; + cJSON *message_task_id = header ? + cJSON_GetObjectItemCaseSensitive(header, "task_id") : NULL; cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload"); cJSON *output = payload ? cJSON_GetObjectItemCaseSensitive(payload, "output") : NULL; cJSON *output_event = output ? cJSON_GetObjectItemCaseSensitive(output, "event") : NULL; @@ -1390,23 +1394,42 @@ static void process_text_json(const char *json_text) } notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED); } else if (strcmp(event->valuestring, "task-started") == 0) { + ESP_LOGI(TAG, "Cloud task started"); notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_TASK_STARTED); } } if (cJSON_IsString(output_event) && output_event->valuestring != NULL) { if (strcmp(output_event->valuestring, "Started") == 0) { + ESP_LOGI(TAG, "Cloud task started (output event)"); notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_TASK_STARTED); } else if (strcmp(output_event->valuestring, "DialogStateChanged") == 0 && cJSON_IsString(state) && state->valuestring != NULL) { if (strcmp(state->valuestring, "Listening") == 0) { + bool accepted = false; if (lock_state()) { - s_server_listening = true; - s_remote_responding = false; - s_remote_responding_ended = false; - s_request_to_speak_pending = false; + accepted = s_task_id[0] != '\0' && + (!cJSON_IsString(message_task_id) || + message_task_id->valuestring == NULL || + strcmp(s_task_id, message_task_id->valuestring) == 0); + if (accepted) { + s_server_listening = true; + s_remote_responding = false; + s_remote_responding_ended = false; + s_request_to_speak_pending = false; + } unlock_state(); } + if (!accepted) { + ESP_LOGW(TAG, + "Ignore stale Listening event for task_id=%s", + cJSON_IsString(message_task_id) && + message_task_id->valuestring != NULL ? + message_task_id->valuestring : "-"); + cJSON_Delete(root); + return; + } + ESP_LOGI(TAG, "Cloud dialog entered Listening"); notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_LISTENING); } else { if (lock_state()) { @@ -1424,6 +1447,7 @@ static void process_text_json(const char *json_text) s_request_to_speak_pending = false; s_client_speech_active = false; s_ptt_release_required = false; + s_response_pending = false; s_drop_downstream_audio = false; unlock_state(); } @@ -1431,6 +1455,7 @@ static void process_text_json(const char *json_text) notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_LISTENING); } else if (strcmp(output_event->valuestring, "RespondingStarted") == 0) { if (lock_state()) { + s_response_pending = false; s_remote_responding = true; s_remote_responding_ended = false; s_local_responding_ended_sent = false; @@ -1517,6 +1542,7 @@ static void process_text_json(const char *json_text) s_request_to_speak_pending = false; s_client_speech_active = false; s_ptt_release_required = false; + s_response_pending = false; s_local_responding_started = false; s_downstream_playback_completed = false; s_drop_downstream_audio = false; @@ -1530,6 +1556,7 @@ static void process_text_json(const char *json_text) notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_NOT_LISTENING); } else if (strcmp(output_event->valuestring, "SpeechStarted") == 0) { if (lock_state()) { + s_response_pending = false; s_last_asr_text_len = 0; unlock_state(); } @@ -1553,6 +1580,7 @@ static void process_text_json(const char *json_text) control_plane_multimodal_stop_uplink(); if (lock_state()) { s_ptt_release_required = s_client_speech_active; + s_response_pending = true; unlock_state(); } ESP_LOGI(TAG, "ASR ended"); @@ -2026,6 +2054,7 @@ esp_err_t control_plane_multimodal_send_start(void) s_request_to_speak_pending = false; s_client_speech_active = false; s_ptt_release_required = false; + s_response_pending = false; unlock_state(); cJSON *root = cJSON_CreateObject(); @@ -2276,6 +2305,7 @@ esp_err_t control_plane_multimodal_send_speech_start(void) if (err == ESP_OK && lock_state()) { s_client_speech_active = true; s_ptt_release_required = false; + s_response_pending = false; s_request_to_speak_pending = false; unlock_state(); } @@ -2305,6 +2335,9 @@ esp_err_t control_plane_multimodal_send_speech_stop(void) if (lock_state()) { s_client_speech_active = false; s_ptt_release_required = false; + if (err == ESP_OK && has_active_task) { + s_response_pending = true; + } s_request_to_speak_pending = false; unlock_state(); } @@ -2339,6 +2372,7 @@ esp_err_t control_plane_multimodal_send_cancel_speech(void) if (err == ESP_OK && lock_state()) { s_client_speech_active = false; s_ptt_release_required = false; + s_response_pending = false; s_request_to_speak_pending = false; unlock_state(); } @@ -2520,6 +2554,7 @@ esp_err_t control_plane_multimodal_get_runtime_state(control_plane_multimodal_ru out_state->request_to_speak_pending = s_request_to_speak_pending; out_state->client_speech_active = s_client_speech_active; out_state->ptt_release_required = s_ptt_release_required; + out_state->response_pending = s_response_pending; out_state->local_responding_started = s_local_responding_started; out_state->received_downstream_audio = s_received_downstream_audio; out_state->last_heartbeat_ms = tick_to_ms(s_last_heartbeat_tick); diff --git a/components/control_plane/src/runtime/control_plane_remote_log_service.c b/components/control_plane/src/runtime/control_plane_remote_log_service.c index 665f9ee..622a4ad 100644 --- a/components/control_plane/src/runtime/control_plane_remote_log_service.c +++ b/components/control_plane/src/runtime/control_plane_remote_log_service.c @@ -88,6 +88,7 @@ #define REMOTE_LOG_UPLOAD_BODY_MAX (CONFIG_TQ_REMOTE_LOG_FILE_MAX_BYTES + 4096) #define REMOTE_LOG_BACKOFF_MAX_MS 300000 #define REMOTE_LOG_POLL_MS 200 +#define REMOTE_LOG_VOICE_STARTUP_GRACE_MS 60000 #if CONFIG_TQ_REMOTE_LOG_ENABLE @@ -600,20 +601,47 @@ static void drain_memory_queue_to_flash(void) static bool realtime_path_busy(void) { +#if CONFIG_TQ_LOCAL_WAKE_ENABLE + if (!control_plane_ota_voice_services_allowed() || + !control_plane_voice_turn_is_idle()) { + return true; + } +#endif #if CONFIG_ENABLE_MULTIMODAL_INTERACTION control_plane_multimodal_runtime_state_t state = {0}; if (control_plane_multimodal_get_runtime_state(&state) != ESP_OK) { return false; } - return state.client_speech_active || + return state.starting || + (state.connected && state.has_active_task && !state.server_listening) || + state.client_speech_active || + state.response_pending || state.remote_responding || state.request_to_speak_pending || + state.local_responding_started || state.received_downstream_audio; #else return false; #endif } +static bool voice_startup_upload_gate_open(TickType_t startup_tick) +{ +#if CONFIG_TQ_LOCAL_WAKE_ENABLE && CONFIG_ENABLE_MULTIMODAL_INTERACTION + control_plane_multimodal_runtime_state_t state = {0}; + if (control_plane_multimodal_get_runtime_state(&state) == ESP_OK && + state.connected && state.has_active_task && state.server_listening) { + return true; + } + + return (xTaskGetTickCount() - startup_tick) >= + pdMS_TO_TICKS(REMOTE_LOG_VOICE_STARTUP_GRACE_MS); +#else + (void)startup_tick; + return true; +#endif +} + static const char *reset_reason_name(esp_reset_reason_t reason) { switch (reason) { @@ -1065,6 +1093,7 @@ static void remote_log_task(void *arg) log_startup_self_check(); TickType_t last_flush = xTaskGetTickCount(); + TickType_t startup_tick = last_flush; uint32_t backoff_ms = 0U; TickType_t next_upload_tick = xTaskGetTickCount(); @@ -1075,7 +1104,8 @@ static void remote_log_task(void *arg) last_flush = now; } - if (backoff_ms == 0U || now >= next_upload_tick) { + if (voice_startup_upload_gate_open(startup_tick) && + (backoff_ms == 0U || now >= next_upload_tick)) { drain_memory_queue_to_flash(); (void)upload_coredump_report(); bool ok = upload_pending_files();