chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(multimodal): harden playback and client lifecycle handling

Browse Source 
- retry the server connection prompt after transient playback failures
- serialize multimodal client and session start-stop operations
- tag downstream responses to ignore stale packets and playback events
- complete bounded teardown and clean callbacks after partial startup
This commit is contained in:
chenjintao
2026-08-14 15:24:29 +08:00
parent c56c65462e
commit 0cfb10252c
12 changed files with 436 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/control_plane/include/control_plane.h
+2 -1
View File
@@ -154,8 +154,9 @@ esp_err_t control_plane_start_multimodal_client(void);
* Semantics:
* - Idempotent and safe to call repeatedly.
* - Requests cooperative stop first, then applies bounded waits for worker teardown.
* - Returns an explicit error if teardown does not complete.
*/
void control_plane_stop_multimodal_client(void);
esp_err_t control_plane_stop_multimodal_client(void);
esp_err_t control_plane_register_model_command_callback(control_plane_model_command_callback_t cb,
void *user_data);
components/control_plane/internal/control_plane_internal.h
+1 -1
View File
@@ -96,7 +96,7 @@ typedef enum {
} control_plane_transport_loss_reason_t;
esp_err_t control_plane_multimodal_start(void);
void control_plane_multimodal_stop(void);
esp_err_t control_plane_multimodal_stop(void);
bool control_plane_multimodal_is_connected(void);
esp_err_t control_plane_multimodal_send_start(void);
esp_err_t control_plane_multimodal_send_stop(const char *reason);
components/control_plane/src/multimodal/control_plane_multimodal_runtime.c
+138 -14
View File
@@ -11,6 +11,7 @@
#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "sdkconfig.h"
@@ -40,6 +41,9 @@
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus"
#define CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS 30000U
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS 1000U
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS 3U
#ifndef CONFIG_TQ_NETWORK_TASK_CORE
#define CONFIG_TQ_NETWORK_TASK_CORE 0
@@ -57,6 +61,10 @@ static bool s_multimodal_watchdog_stop_requested;
static bool s_multimodal_watchdog_task_starting;
static bool s_model_command_dispatched;
static bool s_server_success_prompt_played;
static bool s_server_success_prompt_pending;
static bool s_server_success_prompt_in_progress;
static uint8_t s_server_success_prompt_attempts;
static uint32_t s_server_success_prompt_next_attempt_ms;
static bool s_cloud_chouqian_pending;
static bool s_downstream_stall_recovery_pending;
static bool s_downstream_stall_stop_sent;
@@ -64,6 +72,45 @@ static uint32_t s_downstream_stall_recovery_started_ms;
static control_plane_model_command_callback_t s_model_command_cb;
static void *s_model_command_cb_user_data;
static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
static SemaphoreHandle_t s_client_lifecycle_mutex;
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
static StaticSemaphore_t s_client_lifecycle_mutex_buf;
#endif
static portMUX_TYPE s_client_lifecycle_init_lock = portMUX_INITIALIZER_UNLOCKED;
static bool lock_client_lifecycle(void)
{
portENTER_CRITICAL(&s_client_lifecycle_init_lock);
if (s_client_lifecycle_mutex == NULL) {
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
s_client_lifecycle_mutex =
xSemaphoreCreateMutexStatic(&s_client_lifecycle_mutex_buf);
#else
s_client_lifecycle_mutex = xSemaphoreCreateMutex();
#endif
}
SemaphoreHandle_t mutex = s_client_lifecycle_mutex;
portEXIT_CRITICAL(&s_client_lifecycle_init_lock);
if (mutex == NULL) {
return false;
}
TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS);
if (timeout_ticks == 0) {
timeout_ticks = 1;
}
if (xSemaphoreTake(mutex, timeout_ticks) != pdTRUE) {
ESP_LOGW(TAG, "Timeout acquiring multimodal client lifecycle lock");
return false;
}
return true;
}
static void unlock_client_lifecycle(void)
{
if (s_client_lifecycle_mutex != NULL) {
(void)xSemaphoreGive(s_client_lifecycle_mutex);
}
}
static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot)
{
@@ -171,6 +218,74 @@ static void control_plane_clear_downstream_stall_recovery(void)
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
}
static void control_plane_schedule_server_success_prompt(void)
{
uint32_t now_ms = (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS);
portENTER_CRITICAL(&s_multimodal_runtime_lock);
if (!s_server_success_prompt_played &&
s_server_success_prompt_attempts <
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) {
s_server_success_prompt_pending = true;
s_server_success_prompt_next_attempt_ms = now_ms;
}
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
}
static void control_plane_try_server_success_prompt(
const control_plane_multimodal_runtime_state_t *state,
uint32_t now_ms)
{
if (state == NULL || !state->connected || state->has_active_task) {
return;
}
bool should_attempt = false;
uint8_t attempt = 0U;
portENTER_CRITICAL(&s_multimodal_runtime_lock);
if (s_server_success_prompt_pending &&
!s_server_success_prompt_played &&
!s_server_success_prompt_in_progress &&
s_server_success_prompt_attempts <
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS &&
(int32_t)(now_ms - s_server_success_prompt_next_attempt_ms) >= 0) {
s_server_success_prompt_in_progress = true;
s_server_success_prompt_attempts++;
attempt = s_server_success_prompt_attempts;
should_attempt = true;
}
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
if (!should_attempt) {
return;
}
esp_err_t err = domain_runtime_behavior_play_prompt_file(
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE);
portENTER_CRITICAL(&s_multimodal_runtime_lock);
s_server_success_prompt_in_progress = false;
if (err == ESP_OK) {
s_server_success_prompt_played = true;
s_server_success_prompt_pending = false;
} else if (s_server_success_prompt_attempts >=
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) {
s_server_success_prompt_pending = false;
} else {
s_server_success_prompt_next_attempt_ms =
now_ms + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS;
}
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
if (err == ESP_OK) {
ESP_LOGI(TAG, "Server success prompt played on attempt %u", (unsigned)attempt);
} else {
ESP_LOGW(TAG,
"Server success prompt attempt %u/%u failed: %s",
(unsigned)attempt,
(unsigned)CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS,
esp_err_to_name(err));
}
}
static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag)
{
bool can_start = false;
@@ -507,6 +622,7 @@ static void multimodal_watchdog_task(void *arg)
}
uint32_t now_ms = control_plane_now_ms();
control_plane_try_server_success_prompt(&state, now_ms);
bool stall_stop_sent = false;
uint32_t stall_recovery_started_ms = 0U;
if (control_plane_get_downstream_stall_recovery(&stall_stop_sent,
@@ -559,7 +675,7 @@ static void multimodal_watchdog_task(void *arg)
"Idle multimodal transport timeout, stopping client after %u ms",
(unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS);
control_plane_multimodal_stop_uplink();
control_plane_multimodal_stop();
(void)control_plane_multimodal_stop();
idle_transport_started_ms = 0U;
}
continue;
@@ -627,16 +743,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
switch (event) {
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED: {
bool should_play_prompt = false;
portENTER_CRITICAL(&s_multimodal_runtime_lock);
if (!s_server_success_prompt_played) {
s_server_success_prompt_played = true;
should_play_prompt = true;
}
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
if (should_play_prompt) {
(void)domain_runtime_behavior_play_prompt_file(CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE);
}
control_plane_schedule_server_success_prompt();
break;
}
@@ -718,7 +825,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
}
}
esp_err_t control_plane_start_multimodal_client(void)
static esp_err_t control_plane_start_multimodal_client_locked(void)
{
control_plane_clear_downstream_stall_recovery();
@@ -768,18 +875,35 @@ esp_err_t control_plane_start_multimodal_client(void)
esp_err_t err = control_plane_multimodal_start();
if (err != ESP_OK) {
stop_watchdog_task();
(void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL);
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
return err;
}
return ESP_OK;
}
void control_plane_stop_multimodal_client(void)
esp_err_t control_plane_start_multimodal_client(void)
{
if (!lock_client_lifecycle()) {
return ESP_ERR_TIMEOUT;
}
esp_err_t err = control_plane_start_multimodal_client_locked();
unlock_client_lifecycle();
return err;
}
esp_err_t control_plane_stop_multimodal_client(void)
{
if (!lock_client_lifecycle()) {
return ESP_ERR_TIMEOUT;
}
control_plane_clear_downstream_stall_recovery();
control_plane_clear_cloud_chouqian_pending("multimodal client stopped");
stop_watchdog_task();
control_plane_multimodal_stop();
esp_err_t err = control_plane_multimodal_stop();
unlock_client_lifecycle();
return err;
}
esp_err_t control_plane_register_model_command_callback(
components/control_plane/src/multimodal/control_plane_multimodal_session_service.c
+135 -35
View File
@@ -88,6 +88,7 @@
#define CONTROL_PLANE_UPLINK_READY_WAIT_MS 6000U
#define CONTROL_PLANE_AUDIO_SEND_FAILURE_WINDOW_MS 100U
#define CONTROL_PLANE_AUDIO_SEND_FAILURE_THRESHOLD 2U
#define CONTROL_PLANE_MULTIMODAL_LIFECYCLE_WAIT_MS 30000U
#ifndef CONFIG_TQ_LOCAL_TASK_CORE
#define CONFIG_TQ_LOCAL_TASK_CORE 1
@@ -154,6 +155,7 @@ static bool s_downstream_stall_stop_pending;
static bool s_drop_downstream_audio;
static bool s_received_downstream_audio;
static bool s_current_response_received_audio;
static uint32_t s_current_playback_generation;
static uint32_t s_audio_send_failure_count;
static TickType_t s_audio_send_failure_window_start_tick;
static size_t s_last_asr_text_len;
@@ -171,9 +173,12 @@ static control_plane_multimodal_downstream_audio_callback_t s_downstream_audio_c
static void *s_downstream_audio_cb_user_data;
static SemaphoreHandle_t s_state_mutex;
static SemaphoreHandle_t s_lifecycle_mutex;
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
static StaticSemaphore_t s_state_mutex_buf;
static StaticSemaphore_t s_lifecycle_mutex_buf;
#endif
static portMUX_TYPE s_mutex_init_lock = portMUX_INITIALIZER_UNLOCKED;
typedef enum {
LOCAL_RESPONDING_EVENT_STARTED = 0,
@@ -299,23 +304,60 @@ static void log_downstream_binary_frame(size_t len, bool drop_audio, bool has_ca
}
}
static bool ensure_session_mutexes(void)
{
portENTER_CRITICAL(&s_mutex_init_lock);
if (s_state_mutex == NULL) {
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
s_state_mutex = xSemaphoreCreateMutexStatic(&s_state_mutex_buf);
#else
s_state_mutex = xSemaphoreCreateMutex();
#endif
}
if (s_lifecycle_mutex == NULL) {
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
s_lifecycle_mutex = xSemaphoreCreateMutexStatic(&s_lifecycle_mutex_buf);
#else
s_lifecycle_mutex = xSemaphoreCreateMutex();
#endif
}
bool ready = s_state_mutex != NULL && s_lifecycle_mutex != NULL;
portEXIT_CRITICAL(&s_mutex_init_lock);
return ready;
}
static bool ensure_state_mutex(void)
{
if (s_state_mutex != NULL) {
return true;
}
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
s_state_mutex = xSemaphoreCreateMutexStatic(&s_state_mutex_buf);
#else
s_state_mutex = xSemaphoreCreateMutex();
#endif
if (s_state_mutex == NULL) {
ESP_LOGE(TAG, "Failed to create multimodal state mutex");
if (!ensure_session_mutexes()) {
ESP_LOGE(TAG, "Failed to create multimodal lifecycle mutexes");
return false;
}
return true;
}
static bool lock_lifecycle(void)
{
if (!ensure_session_mutexes()) {
return false;
}
TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_MULTIMODAL_LIFECYCLE_WAIT_MS);
if (timeout_ticks == 0) {
timeout_ticks = 1;
}
if (xSemaphoreTake(s_lifecycle_mutex, timeout_ticks) != pdTRUE) {
ESP_LOGW(TAG, "Timeout acquiring multimodal lifecycle lock");
return false;
}
return true;
}
static void unlock_lifecycle(void)
{
if (s_lifecycle_mutex != NULL) {
(void)xSemaphoreGive(s_lifecycle_mutex);
}
}
static bool lock_state(void)
{
if (!ensure_state_mutex()) {
@@ -402,6 +444,7 @@ static void reset_session_state_locked(void)
s_drop_downstream_audio = false;
s_received_downstream_audio = false;
s_current_response_received_audio = false;
s_current_playback_generation = 0U;
s_audio_send_failure_count = 0;
s_audio_send_failure_window_start_tick = 0;
s_local_responding_started_send_pending = false;
@@ -1403,7 +1446,13 @@ static void process_text_json(const char *json_text)
unlock_state();
}
ESP_LOGI(TAG, "TTS responding started");
esp_err_t begin_err = domain_runtime_multimodal_audio_begin_downstream_response();
uint32_t playback_generation = 0U;
esp_err_t begin_err = domain_runtime_multimodal_audio_begin_downstream_response(
&playback_generation);
if (begin_err == ESP_OK && lock_state()) {
s_current_playback_generation = playback_generation;
unlock_state();
}
if (begin_err != ESP_OK && begin_err != ESP_ERR_INVALID_STATE) {
ESP_LOGW(TAG,
"Begin downstream response failed: %s",
@@ -1636,6 +1685,21 @@ static void downstream_playback_event_bridge(
return;
}
bool generation_matches = false;
if (lock_state()) {
generation_matches = s_current_playback_generation != 0U &&
s_current_playback_generation ==
event->playback_generation;
unlock_state();
}
if (!generation_matches) {
ESP_LOGD(TAG,
"Ignore stale or local playback event type=%d generation=%u",
(int)event->type,
(unsigned)event->playback_generation);
return;
}
switch (event->type) {
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED: {
bool should_send_started = false;
@@ -1656,12 +1720,17 @@ static void downstream_playback_event_bridge(
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED:
if (lock_state()) {
s_downstream_playback_completed = true;
s_current_playback_generation = 0U;
unlock_state();
}
maybe_send_local_responding_ended_if_ready();
break;
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED:
if (lock_state()) {
s_current_playback_generation = 0U;
unlock_state();
}
send_local_responding_ended_after_playback_stop();
break;
@@ -1671,6 +1740,7 @@ static void downstream_playback_event_bridge(
cloud_response_active = s_task_id[0] != '\0' &&
(s_remote_responding ||
s_current_response_received_audio);
s_current_playback_generation = 0U;
unlock_state();
}
@@ -1697,7 +1767,20 @@ static esp_err_t control_plane_audio_uplink_send_cb(const uint8_t *data, size_t
return control_plane_multimodal_send_audio_realtime(data, len);
}
static esp_err_t control_plane_multimodal_start_locked(void);
static esp_err_t control_plane_multimodal_stop_locked(void);
esp_err_t control_plane_multimodal_start(void)
{
if (!lock_lifecycle()) {
return ESP_ERR_TIMEOUT;
}
esp_err_t err = control_plane_multimodal_start_locked();
unlock_lifecycle();
return err;
}
static esp_err_t control_plane_multimodal_start_locked(void)
{
#if !CONFIG_ENABLE_MULTIMODAL_INTERACTION
ESP_LOGW(TAG, "Multimodal interaction is disabled by config");
@@ -1707,25 +1790,26 @@ esp_err_t control_plane_multimodal_start(void)
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
uint32_t session_generation = 0U;
#endif
if (lock_state()) {
bool already_started = s_started || s_auth.token[0] != '\0';
bool starting = s_starting;
bool stopping = s_stopping;
if (already_started) {
unlock_state();
return ESP_OK;
}
if (starting || stopping) {
unlock_state();
return ESP_ERR_INVALID_STATE;
}
s_starting = true;
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
s_session_generation++;
session_generation = s_session_generation;
#endif
unlock_state();
if (!lock_state()) {
return ESP_ERR_TIMEOUT;
}
bool already_started = s_started || s_auth.token[0] != '\0';
bool starting = s_starting;
bool stopping = s_stopping;
if (already_started) {
unlock_state();
return ESP_OK;
}
if (starting || stopping) {
unlock_state();
return ESP_ERR_INVALID_STATE;
}
s_starting = true;
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
s_session_generation++;
session_generation = s_session_generation;
#endif
unlock_state();
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
ESP_LOGI(TAG,
@@ -1804,14 +1888,28 @@ esp_err_t control_plane_multimodal_start(void)
return err;
}
if (lock_state()) {
s_auth = auth;
unlock_state();
if (!lock_state()) {
(void)domain_runtime_multimodal_transport_stop(2000);
(void)domain_runtime_multimodal_audio_register_downstream_playback_callback(NULL, NULL);
stop_local_responding_event_worker();
return ESP_ERR_TIMEOUT;
}
s_auth = auth;
unlock_state();
return ESP_OK;
}
void control_plane_multimodal_stop(void)
esp_err_t control_plane_multimodal_stop(void)
{
if (!lock_lifecycle()) {
return ESP_ERR_TIMEOUT;
}
esp_err_t err = control_plane_multimodal_stop_locked();
unlock_lifecycle();
return err;
}
static esp_err_t control_plane_multimodal_stop_locked(void)
{
bool already_stopping = false;
if (lock_state()) {
@@ -1823,8 +1921,7 @@ void control_plane_multimodal_stop(void)
unlock_state();
}
if (already_stopping) {
ESP_LOGI(TAG, "Multimodal stop already in progress");
return;
ESP_LOGI(TAG, "Completing previously started multimodal teardown");
}
control_plane_multimodal_stop_uplink();
@@ -1846,7 +1943,10 @@ void control_plane_multimodal_stop(void)
memset(&s_auth, 0, sizeof(s_auth));
reset_session_state_locked();
unlock_state();
} else if (transport_stop_err == ESP_OK) {
transport_stop_err = ESP_ERR_TIMEOUT;
}
return transport_stop_err;
}
bool control_plane_multimodal_is_connected(void)
components/control_plane/src/runtime/control_plane_runtime_workers.c
+2 -2
View File
@@ -1777,7 +1777,7 @@ static void control_plane_wifi_state_task(void *arg)
if (!disconnected_expression_active) {
ESP_LOGI(TAG, "Wi-Fi STA disconnect persisted; stopping multimodal client");
control_plane_stop_multimodal_client();
(void)control_plane_stop_multimodal_client();
control_plane_stop_user_config_server();
}
@@ -2455,7 +2455,7 @@ void control_plane_stop_runtime_workers(void)
portEXIT_CRITICAL(&s_runtime_lock);
control_plane_delete_model_cmd_queue(queue, queue_with_caps);
control_plane_stop_multimodal_client();
(void)control_plane_stop_multimodal_client();
control_plane_set_runtime_stop_requested(false);
}