From 0cfb10252c2cc94a06258beb66c6c60ab9485f67 Mon Sep 17 00:00:00 2001 From: chenjintao Date: Fri, 14 Aug 2026 15:24:29 +0800 Subject: [PATCH] fix(multimodal): harden playback and client lifecycle handling - retry the server connection prompt after transient playback failures - serialize multimodal client and session start-stop operations - tag downstream responses to ignore stale packets and playback events - complete bounded teardown and clean callbacks after partial startup --- .../control_plane/include/control_plane.h | 3 +- .../internal/control_plane_internal.h | 2 +- .../control_plane_multimodal_runtime.c | 152 ++++++++++-- ...control_plane_multimodal_session_service.c | 170 ++++++++++--- .../runtime/control_plane_runtime_workers.c | 4 +- components/domain/include/domain.h | 4 +- .../src/runtime/domain_runtime_adapters.c | 34 ++- components/platform/include/platform.h | 10 +- .../internal/multimodal/audio_pipeline.h | 3 +- .../audio_pipeline_runtime_impl.c | 226 +++++++++--------- .../platform/src/platform_behavior_stubs.c | 3 +- .../src/runtime/platform_multimodal_runtime.c | 13 +- 12 files changed, 436 insertions(+), 188 deletions(-) diff --git a/components/control_plane/include/control_plane.h b/components/control_plane/include/control_plane.h index 82c2bfe..2313c55 100644 --- a/components/control_plane/include/control_plane.h +++ b/components/control_plane/include/control_plane.h @@ -154,8 +154,9 @@ esp_err_t control_plane_start_multimodal_client(void); * Semantics: * - Idempotent and safe to call repeatedly. * - Requests cooperative stop first, then applies bounded waits for worker teardown. + * - Returns an explicit error if teardown does not complete. */ -void control_plane_stop_multimodal_client(void); +esp_err_t control_plane_stop_multimodal_client(void); esp_err_t control_plane_register_model_command_callback(control_plane_model_command_callback_t cb, void *user_data); diff --git a/components/control_plane/internal/control_plane_internal.h b/components/control_plane/internal/control_plane_internal.h index 3f7d62d..37723ae 100644 --- a/components/control_plane/internal/control_plane_internal.h +++ b/components/control_plane/internal/control_plane_internal.h @@ -96,7 +96,7 @@ typedef enum { } control_plane_transport_loss_reason_t; esp_err_t control_plane_multimodal_start(void); -void control_plane_multimodal_stop(void); +esp_err_t control_plane_multimodal_stop(void); bool control_plane_multimodal_is_connected(void); esp_err_t control_plane_multimodal_send_start(void); esp_err_t control_plane_multimodal_send_stop(const char *reason); diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c index 1a7e15b..e161a32 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c @@ -11,6 +11,7 @@ #include #include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" #include "freertos/task.h" #include "esp_log.h" #include "sdkconfig.h" @@ -40,6 +41,9 @@ #define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000 #define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144 #define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus" +#define CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS 30000U +#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS 1000U +#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS 3U #ifndef CONFIG_TQ_NETWORK_TASK_CORE #define CONFIG_TQ_NETWORK_TASK_CORE 0 @@ -57,6 +61,10 @@ static bool s_multimodal_watchdog_stop_requested; static bool s_multimodal_watchdog_task_starting; static bool s_model_command_dispatched; static bool s_server_success_prompt_played; +static bool s_server_success_prompt_pending; +static bool s_server_success_prompt_in_progress; +static uint8_t s_server_success_prompt_attempts; +static uint32_t s_server_success_prompt_next_attempt_ms; static bool s_cloud_chouqian_pending; static bool s_downstream_stall_recovery_pending; static bool s_downstream_stall_stop_sent; @@ -64,6 +72,45 @@ static uint32_t s_downstream_stall_recovery_started_ms; static control_plane_model_command_callback_t s_model_command_cb; static void *s_model_command_cb_user_data; static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED; +static SemaphoreHandle_t s_client_lifecycle_mutex; +#if ( configSUPPORT_STATIC_ALLOCATION == 1 ) +static StaticSemaphore_t s_client_lifecycle_mutex_buf; +#endif +static portMUX_TYPE s_client_lifecycle_init_lock = portMUX_INITIALIZER_UNLOCKED; + +static bool lock_client_lifecycle(void) +{ + portENTER_CRITICAL(&s_client_lifecycle_init_lock); + if (s_client_lifecycle_mutex == NULL) { +#if ( configSUPPORT_STATIC_ALLOCATION == 1 ) + s_client_lifecycle_mutex = + xSemaphoreCreateMutexStatic(&s_client_lifecycle_mutex_buf); +#else + s_client_lifecycle_mutex = xSemaphoreCreateMutex(); +#endif + } + SemaphoreHandle_t mutex = s_client_lifecycle_mutex; + portEXIT_CRITICAL(&s_client_lifecycle_init_lock); + if (mutex == NULL) { + return false; + } + TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS); + if (timeout_ticks == 0) { + timeout_ticks = 1; + } + if (xSemaphoreTake(mutex, timeout_ticks) != pdTRUE) { + ESP_LOGW(TAG, "Timeout acquiring multimodal client lifecycle lock"); + return false; + } + return true; +} + +static void unlock_client_lifecycle(void) +{ + if (s_client_lifecycle_mutex != NULL) { + (void)xSemaphoreGive(s_client_lifecycle_mutex); + } +} static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot) { @@ -171,6 +218,74 @@ static void control_plane_clear_downstream_stall_recovery(void) portEXIT_CRITICAL(&s_multimodal_runtime_lock); } +static void control_plane_schedule_server_success_prompt(void) +{ + uint32_t now_ms = (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS); + portENTER_CRITICAL(&s_multimodal_runtime_lock); + if (!s_server_success_prompt_played && + s_server_success_prompt_attempts < + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) { + s_server_success_prompt_pending = true; + s_server_success_prompt_next_attempt_ms = now_ms; + } + portEXIT_CRITICAL(&s_multimodal_runtime_lock); +} + +static void control_plane_try_server_success_prompt( + const control_plane_multimodal_runtime_state_t *state, + uint32_t now_ms) +{ + if (state == NULL || !state->connected || state->has_active_task) { + return; + } + + bool should_attempt = false; + uint8_t attempt = 0U; + portENTER_CRITICAL(&s_multimodal_runtime_lock); + if (s_server_success_prompt_pending && + !s_server_success_prompt_played && + !s_server_success_prompt_in_progress && + s_server_success_prompt_attempts < + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS && + (int32_t)(now_ms - s_server_success_prompt_next_attempt_ms) >= 0) { + s_server_success_prompt_in_progress = true; + s_server_success_prompt_attempts++; + attempt = s_server_success_prompt_attempts; + should_attempt = true; + } + portEXIT_CRITICAL(&s_multimodal_runtime_lock); + if (!should_attempt) { + return; + } + + esp_err_t err = domain_runtime_behavior_play_prompt_file( + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE); + + portENTER_CRITICAL(&s_multimodal_runtime_lock); + s_server_success_prompt_in_progress = false; + if (err == ESP_OK) { + s_server_success_prompt_played = true; + s_server_success_prompt_pending = false; + } else if (s_server_success_prompt_attempts >= + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) { + s_server_success_prompt_pending = false; + } else { + s_server_success_prompt_next_attempt_ms = + now_ms + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS; + } + portEXIT_CRITICAL(&s_multimodal_runtime_lock); + + if (err == ESP_OK) { + ESP_LOGI(TAG, "Server success prompt played on attempt %u", (unsigned)attempt); + } else { + ESP_LOGW(TAG, + "Server success prompt attempt %u/%u failed: %s", + (unsigned)attempt, + (unsigned)CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS, + esp_err_to_name(err)); + } +} + static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag) { bool can_start = false; @@ -507,6 +622,7 @@ static void multimodal_watchdog_task(void *arg) } uint32_t now_ms = control_plane_now_ms(); + control_plane_try_server_success_prompt(&state, now_ms); bool stall_stop_sent = false; uint32_t stall_recovery_started_ms = 0U; if (control_plane_get_downstream_stall_recovery(&stall_stop_sent, @@ -559,7 +675,7 @@ static void multimodal_watchdog_task(void *arg) "Idle multimodal transport timeout, stopping client after %u ms", (unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS); control_plane_multimodal_stop_uplink(); - control_plane_multimodal_stop(); + (void)control_plane_multimodal_stop(); idle_transport_started_ms = 0U; } continue; @@ -627,16 +743,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event switch (event) { case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED: { - bool should_play_prompt = false; - portENTER_CRITICAL(&s_multimodal_runtime_lock); - if (!s_server_success_prompt_played) { - s_server_success_prompt_played = true; - should_play_prompt = true; - } - portEXIT_CRITICAL(&s_multimodal_runtime_lock); - if (should_play_prompt) { - (void)domain_runtime_behavior_play_prompt_file(CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE); - } + control_plane_schedule_server_success_prompt(); break; } @@ -718,7 +825,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event } } -esp_err_t control_plane_start_multimodal_client(void) +static esp_err_t control_plane_start_multimodal_client_locked(void) { control_plane_clear_downstream_stall_recovery(); @@ -768,18 +875,35 @@ esp_err_t control_plane_start_multimodal_client(void) esp_err_t err = control_plane_multimodal_start(); if (err != ESP_OK) { stop_watchdog_task(); + (void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL); + (void)control_plane_multimodal_register_event_callback(NULL, NULL); return err; } return ESP_OK; } -void control_plane_stop_multimodal_client(void) +esp_err_t control_plane_start_multimodal_client(void) { + if (!lock_client_lifecycle()) { + return ESP_ERR_TIMEOUT; + } + esp_err_t err = control_plane_start_multimodal_client_locked(); + unlock_client_lifecycle(); + return err; +} + +esp_err_t control_plane_stop_multimodal_client(void) +{ + if (!lock_client_lifecycle()) { + return ESP_ERR_TIMEOUT; + } control_plane_clear_downstream_stall_recovery(); control_plane_clear_cloud_chouqian_pending("multimodal client stopped"); stop_watchdog_task(); - control_plane_multimodal_stop(); + esp_err_t err = control_plane_multimodal_stop(); + unlock_client_lifecycle(); + return err; } esp_err_t control_plane_register_model_command_callback( diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c index 4c031af..e474ef8 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c @@ -88,6 +88,7 @@ #define CONTROL_PLANE_UPLINK_READY_WAIT_MS 6000U #define CONTROL_PLANE_AUDIO_SEND_FAILURE_WINDOW_MS 100U #define CONTROL_PLANE_AUDIO_SEND_FAILURE_THRESHOLD 2U +#define CONTROL_PLANE_MULTIMODAL_LIFECYCLE_WAIT_MS 30000U #ifndef CONFIG_TQ_LOCAL_TASK_CORE #define CONFIG_TQ_LOCAL_TASK_CORE 1 @@ -154,6 +155,7 @@ static bool s_downstream_stall_stop_pending; static bool s_drop_downstream_audio; static bool s_received_downstream_audio; static bool s_current_response_received_audio; +static uint32_t s_current_playback_generation; static uint32_t s_audio_send_failure_count; static TickType_t s_audio_send_failure_window_start_tick; static size_t s_last_asr_text_len; @@ -171,9 +173,12 @@ static control_plane_multimodal_downstream_audio_callback_t s_downstream_audio_c static void *s_downstream_audio_cb_user_data; static SemaphoreHandle_t s_state_mutex; +static SemaphoreHandle_t s_lifecycle_mutex; #if ( configSUPPORT_STATIC_ALLOCATION == 1 ) static StaticSemaphore_t s_state_mutex_buf; +static StaticSemaphore_t s_lifecycle_mutex_buf; #endif +static portMUX_TYPE s_mutex_init_lock = portMUX_INITIALIZER_UNLOCKED; typedef enum { LOCAL_RESPONDING_EVENT_STARTED = 0, @@ -299,23 +304,60 @@ static void log_downstream_binary_frame(size_t len, bool drop_audio, bool has_ca } } +static bool ensure_session_mutexes(void) +{ + portENTER_CRITICAL(&s_mutex_init_lock); + if (s_state_mutex == NULL) { +#if ( configSUPPORT_STATIC_ALLOCATION == 1 ) + s_state_mutex = xSemaphoreCreateMutexStatic(&s_state_mutex_buf); +#else + s_state_mutex = xSemaphoreCreateMutex(); +#endif + } + if (s_lifecycle_mutex == NULL) { +#if ( configSUPPORT_STATIC_ALLOCATION == 1 ) + s_lifecycle_mutex = xSemaphoreCreateMutexStatic(&s_lifecycle_mutex_buf); +#else + s_lifecycle_mutex = xSemaphoreCreateMutex(); +#endif + } + bool ready = s_state_mutex != NULL && s_lifecycle_mutex != NULL; + portEXIT_CRITICAL(&s_mutex_init_lock); + return ready; +} + static bool ensure_state_mutex(void) { - if (s_state_mutex != NULL) { - return true; - } -#if ( configSUPPORT_STATIC_ALLOCATION == 1 ) - s_state_mutex = xSemaphoreCreateMutexStatic(&s_state_mutex_buf); -#else - s_state_mutex = xSemaphoreCreateMutex(); -#endif - if (s_state_mutex == NULL) { - ESP_LOGE(TAG, "Failed to create multimodal state mutex"); + if (!ensure_session_mutexes()) { + ESP_LOGE(TAG, "Failed to create multimodal lifecycle mutexes"); return false; } return true; } +static bool lock_lifecycle(void) +{ + if (!ensure_session_mutexes()) { + return false; + } + TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_MULTIMODAL_LIFECYCLE_WAIT_MS); + if (timeout_ticks == 0) { + timeout_ticks = 1; + } + if (xSemaphoreTake(s_lifecycle_mutex, timeout_ticks) != pdTRUE) { + ESP_LOGW(TAG, "Timeout acquiring multimodal lifecycle lock"); + return false; + } + return true; +} + +static void unlock_lifecycle(void) +{ + if (s_lifecycle_mutex != NULL) { + (void)xSemaphoreGive(s_lifecycle_mutex); + } +} + static bool lock_state(void) { if (!ensure_state_mutex()) { @@ -402,6 +444,7 @@ static void reset_session_state_locked(void) s_drop_downstream_audio = false; s_received_downstream_audio = false; s_current_response_received_audio = false; + s_current_playback_generation = 0U; s_audio_send_failure_count = 0; s_audio_send_failure_window_start_tick = 0; s_local_responding_started_send_pending = false; @@ -1403,7 +1446,13 @@ static void process_text_json(const char *json_text) unlock_state(); } ESP_LOGI(TAG, "TTS responding started"); - esp_err_t begin_err = domain_runtime_multimodal_audio_begin_downstream_response(); + uint32_t playback_generation = 0U; + esp_err_t begin_err = domain_runtime_multimodal_audio_begin_downstream_response( + &playback_generation); + if (begin_err == ESP_OK && lock_state()) { + s_current_playback_generation = playback_generation; + unlock_state(); + } if (begin_err != ESP_OK && begin_err != ESP_ERR_INVALID_STATE) { ESP_LOGW(TAG, "Begin downstream response failed: %s", @@ -1636,6 +1685,21 @@ static void downstream_playback_event_bridge( return; } + bool generation_matches = false; + if (lock_state()) { + generation_matches = s_current_playback_generation != 0U && + s_current_playback_generation == + event->playback_generation; + unlock_state(); + } + if (!generation_matches) { + ESP_LOGD(TAG, + "Ignore stale or local playback event type=%d generation=%u", + (int)event->type, + (unsigned)event->playback_generation); + return; + } + switch (event->type) { case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED: { bool should_send_started = false; @@ -1656,12 +1720,17 @@ static void downstream_playback_event_bridge( case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED: if (lock_state()) { s_downstream_playback_completed = true; + s_current_playback_generation = 0U; unlock_state(); } maybe_send_local_responding_ended_if_ready(); break; case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED: + if (lock_state()) { + s_current_playback_generation = 0U; + unlock_state(); + } send_local_responding_ended_after_playback_stop(); break; @@ -1671,6 +1740,7 @@ static void downstream_playback_event_bridge( cloud_response_active = s_task_id[0] != '\0' && (s_remote_responding || s_current_response_received_audio); + s_current_playback_generation = 0U; unlock_state(); } @@ -1697,7 +1767,20 @@ static esp_err_t control_plane_audio_uplink_send_cb(const uint8_t *data, size_t return control_plane_multimodal_send_audio_realtime(data, len); } +static esp_err_t control_plane_multimodal_start_locked(void); +static esp_err_t control_plane_multimodal_stop_locked(void); + esp_err_t control_plane_multimodal_start(void) +{ + if (!lock_lifecycle()) { + return ESP_ERR_TIMEOUT; + } + esp_err_t err = control_plane_multimodal_start_locked(); + unlock_lifecycle(); + return err; +} + +static esp_err_t control_plane_multimodal_start_locked(void) { #if !CONFIG_ENABLE_MULTIMODAL_INTERACTION ESP_LOGW(TAG, "Multimodal interaction is disabled by config"); @@ -1707,25 +1790,26 @@ esp_err_t control_plane_multimodal_start(void) #if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE uint32_t session_generation = 0U; #endif - if (lock_state()) { - bool already_started = s_started || s_auth.token[0] != '\0'; - bool starting = s_starting; - bool stopping = s_stopping; - if (already_started) { - unlock_state(); - return ESP_OK; - } - if (starting || stopping) { - unlock_state(); - return ESP_ERR_INVALID_STATE; - } - s_starting = true; -#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE - s_session_generation++; - session_generation = s_session_generation; -#endif - unlock_state(); + if (!lock_state()) { + return ESP_ERR_TIMEOUT; } + bool already_started = s_started || s_auth.token[0] != '\0'; + bool starting = s_starting; + bool stopping = s_stopping; + if (already_started) { + unlock_state(); + return ESP_OK; + } + if (starting || stopping) { + unlock_state(); + return ESP_ERR_INVALID_STATE; + } + s_starting = true; +#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE + s_session_generation++; + session_generation = s_session_generation; +#endif + unlock_state(); #if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE ESP_LOGI(TAG, @@ -1804,14 +1888,28 @@ esp_err_t control_plane_multimodal_start(void) return err; } - if (lock_state()) { - s_auth = auth; - unlock_state(); + if (!lock_state()) { + (void)domain_runtime_multimodal_transport_stop(2000); + (void)domain_runtime_multimodal_audio_register_downstream_playback_callback(NULL, NULL); + stop_local_responding_event_worker(); + return ESP_ERR_TIMEOUT; } + s_auth = auth; + unlock_state(); return ESP_OK; } -void control_plane_multimodal_stop(void) +esp_err_t control_plane_multimodal_stop(void) +{ + if (!lock_lifecycle()) { + return ESP_ERR_TIMEOUT; + } + esp_err_t err = control_plane_multimodal_stop_locked(); + unlock_lifecycle(); + return err; +} + +static esp_err_t control_plane_multimodal_stop_locked(void) { bool already_stopping = false; if (lock_state()) { @@ -1823,8 +1921,7 @@ void control_plane_multimodal_stop(void) unlock_state(); } if (already_stopping) { - ESP_LOGI(TAG, "Multimodal stop already in progress"); - return; + ESP_LOGI(TAG, "Completing previously started multimodal teardown"); } control_plane_multimodal_stop_uplink(); @@ -1846,7 +1943,10 @@ void control_plane_multimodal_stop(void) memset(&s_auth, 0, sizeof(s_auth)); reset_session_state_locked(); unlock_state(); + } else if (transport_stop_err == ESP_OK) { + transport_stop_err = ESP_ERR_TIMEOUT; } + return transport_stop_err; } bool control_plane_multimodal_is_connected(void) diff --git a/components/control_plane/src/runtime/control_plane_runtime_workers.c b/components/control_plane/src/runtime/control_plane_runtime_workers.c index 5744fa9..7479c60 100644 --- a/components/control_plane/src/runtime/control_plane_runtime_workers.c +++ b/components/control_plane/src/runtime/control_plane_runtime_workers.c @@ -1777,7 +1777,7 @@ static void control_plane_wifi_state_task(void *arg) if (!disconnected_expression_active) { ESP_LOGI(TAG, "Wi-Fi STA disconnect persisted; stopping multimodal client"); - control_plane_stop_multimodal_client(); + (void)control_plane_stop_multimodal_client(); control_plane_stop_user_config_server(); } @@ -2455,7 +2455,7 @@ void control_plane_stop_runtime_workers(void) portEXIT_CRITICAL(&s_runtime_lock); control_plane_delete_model_cmd_queue(queue, queue_with_caps); - control_plane_stop_multimodal_client(); + (void)control_plane_stop_multimodal_client(); control_plane_set_runtime_stop_requested(false); } diff --git a/components/domain/include/domain.h b/components/domain/include/domain.h index 4888a45..a53fc1e 100644 --- a/components/domain/include/domain.h +++ b/components/domain/include/domain.h @@ -103,6 +103,7 @@ typedef enum { typedef struct { domain_multimodal_downstream_playback_event_type_t type; + uint32_t playback_generation; } domain_multimodal_downstream_playback_event_t; typedef void (*domain_multimodal_downstream_playback_callback_t)( @@ -228,7 +229,8 @@ esp_err_t domain_runtime_local_command_stop(int timeout_ms); esp_err_t domain_runtime_local_command_get_snapshot( domain_local_command_snapshot_t *out_snapshot); esp_err_t domain_runtime_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len); -esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(void); +esp_err_t domain_runtime_multimodal_audio_begin_downstream_response( + uint32_t *out_playback_generation); esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void); esp_err_t domain_runtime_multimodal_audio_stop_downstream_playback(void); esp_err_t domain_runtime_multimodal_audio_register_downstream_playback_callback( diff --git a/components/domain/src/runtime/domain_runtime_adapters.c b/components/domain/src/runtime/domain_runtime_adapters.c index 4adefec..20cf680 100644 --- a/components/domain/src/runtime/domain_runtime_adapters.c +++ b/components/domain/src/runtime/domain_runtime_adapters.c @@ -435,7 +435,9 @@ static const char *domain_wifi_prompt_file_path(domain_wifi_prompt_id_t prompt) } } -static void domain_downstream_playback_dispatch(domain_multimodal_downstream_playback_event_type_t type) +static void domain_downstream_playback_dispatch( + domain_multimodal_downstream_playback_event_type_t type, + uint32_t playback_generation) { domain_multimodal_downstream_playback_callback_t cb = NULL; void *cb_user_data = NULL; @@ -446,32 +448,41 @@ static void domain_downstream_playback_dispatch(domain_multimodal_downstream_pla const domain_multimodal_downstream_playback_event_t event = { .type = type, + .playback_generation = playback_generation, }; cb(&event, cb_user_data); } -static void domain_downstream_playback_started_adapter(void *user_data) +static void domain_downstream_playback_started_adapter(uint32_t playback_generation, + void *user_data) { (void)user_data; - domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED); + domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED, + playback_generation); } -static void domain_downstream_playback_completed_adapter(void *user_data) +static void domain_downstream_playback_completed_adapter(uint32_t playback_generation, + void *user_data) { (void)user_data; - domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED); + domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED, + playback_generation); } -static void domain_downstream_playback_stopped_adapter(void *user_data) +static void domain_downstream_playback_stopped_adapter(uint32_t playback_generation, + void *user_data) { (void)user_data; - domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED); + domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED, + playback_generation); } -static void domain_downstream_playback_stalled_adapter(void *user_data) +static void domain_downstream_playback_stalled_adapter(uint32_t playback_generation, + void *user_data) { (void)user_data; - domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STALLED); + domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STALLED, + playback_generation); } static void domain_wifi_set_keep_awake_adapter(bool keep_awake, void *user_data) @@ -687,9 +698,10 @@ esp_err_t domain_runtime_multimodal_audio_play_downstream_opus(const uint8_t *da return platform_multimodal_audio_play_downstream_opus(data, len); } -esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(void) +esp_err_t domain_runtime_multimodal_audio_begin_downstream_response( + uint32_t *out_playback_generation) { - return platform_multimodal_audio_begin_downstream_response(); + return platform_multimodal_audio_begin_downstream_response(out_playback_generation); } esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void) diff --git a/components/platform/include/platform.h b/components/platform/include/platform.h index fa69639..23d93a0 100644 --- a/components/platform/include/platform.h +++ b/components/platform/include/platform.h @@ -43,10 +43,10 @@ typedef struct { } platform_multimodal_transport_callbacks_t; typedef struct { - void (*on_started)(void *user_data); - void (*on_completed)(void *user_data); - void (*on_stopped)(void *user_data); - void (*on_stalled)(void *user_data); + void (*on_started)(uint32_t playback_generation, void *user_data); + void (*on_completed)(uint32_t playback_generation, void *user_data); + void (*on_stopped)(uint32_t playback_generation, void *user_data); + void (*on_stalled)(uint32_t playback_generation, void *user_data); void *user_data; } platform_multimodal_downstream_playback_callbacks_t; @@ -399,7 +399,7 @@ esp_err_t platform_multimodal_audio_unregister_pcm_observer( void *user_data, int timeout_ms); esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len); -esp_err_t platform_multimodal_audio_begin_downstream_response(void); +esp_err_t platform_multimodal_audio_begin_downstream_response(uint32_t *out_playback_generation); esp_err_t platform_multimodal_audio_finish_downstream_response(void); esp_err_t platform_multimodal_audio_stop_downstream_playback(void); esp_err_t platform_multimodal_audio_register_downstream_playback_callbacks( diff --git a/components/platform/internal/multimodal/audio_pipeline.h b/components/platform/internal/multimodal/audio_pipeline.h index 444fa2b..0f4bd59 100644 --- a/components/platform/internal/multimodal/audio_pipeline.h +++ b/components/platform/internal/multimodal/audio_pipeline.h @@ -25,6 +25,7 @@ typedef enum { typedef struct { multimodal_audio_downstream_playback_event_type_t type; + uint32_t playback_generation; } multimodal_audio_downstream_playback_event_t; typedef void (*multimodal_audio_downstream_playback_callback_t)( @@ -47,7 +48,7 @@ esp_err_t multimodal_audio_unregister_pcm_observer( int timeout_ms); esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t len); -esp_err_t multimodal_audio_begin_downstream_response(void); +esp_err_t multimodal_audio_begin_downstream_response(uint32_t *out_playback_generation); esp_err_t multimodal_audio_finish_downstream_response(void); esp_err_t multimodal_audio_stop_downstream_playback(void); esp_err_t multimodal_audio_register_downstream_playback_callback( diff --git a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c index a48e8bb..20b69fc 100644 --- a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c +++ b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c @@ -461,16 +461,6 @@ static void drain_downstream_playback_queue(multimodal_audio_ctx_t *ctx) unlock_downstream_playback_queue(); } -static void drain_downstream_playback_queue_now(multimodal_audio_ctx_t *ctx) -{ - if (ctx == NULL || !lock_downstream_playback_queue(0)) { - return; - } - QueueHandle_t queue = ctx->downstream_playback_queue; - drain_downstream_playback_queue_handle(queue); - unlock_downstream_playback_queue(); -} - static void track_uplink_packet_queued(size_t len) { #if CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS @@ -581,10 +571,11 @@ static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ct static esp_err_t ensure_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx); static void notify_downstream_audio_active(multimodal_audio_ctx_t *ctx); static void notify_downstream_playback_started(uint32_t expected_generation); -static void notify_downstream_playback_completed(void); -static void notify_downstream_playback_stopped(void); +static void notify_downstream_playback_completed(uint32_t expected_generation); static esp_err_t prime_downstream_output_path(multimodal_audio_ctx_t *ctx); -static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len); +static esp_err_t handle_downstream_opus_sync(const uint8_t *data, + size_t len, + uint32_t expected_generation); static esp_err_t ensure_downstream_playback_task(multimodal_audio_ctx_t *ctx); static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx); static esp_err_t stop_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx, int timeout_ms); @@ -1563,20 +1554,44 @@ static void snapshot_downstream_playback_state(uint32_t *out_generation, portEXIT_CRITICAL(&s_audio_runtime_lock); } -static uint32_t advance_downstream_playback_generation(void) +static uint32_t advance_downstream_playback_generation_locked(void) { - uint32_t generation = 0U; - - portENTER_CRITICAL(&s_audio_runtime_lock); s_audio.downstream_playback_generation++; if (s_audio.downstream_playback_generation == 0U) { s_audio.downstream_playback_generation = 1U; } s_audio.downstream_decoder_reset_pending = true; - generation = s_audio.downstream_playback_generation; - portEXIT_CRITICAL(&s_audio_runtime_lock); + return s_audio.downstream_playback_generation; +} - return generation; +static bool claim_downstream_response_terminal(uint32_t expected_generation, + bool invalidate_generation, + uint32_t *out_terminal_generation) +{ + if (out_terminal_generation == NULL) { + return false; + } + + portENTER_CRITICAL(&s_audio_runtime_lock); + bool generation_matches = expected_generation == 0U || + s_audio.downstream_playback_generation == + expected_generation; + bool should_notify = generation_matches && + (s_audio.downstream_response_active || + s_audio.downstream_response_end_received || + s_audio.downstream_playback_in_progress); + *out_terminal_generation = s_audio.downstream_playback_generation; + if (generation_matches && invalidate_generation && + (should_notify || expected_generation == 0U)) { + (void)advance_downstream_playback_generation_locked(); + } + if (should_notify) { + s_audio.downstream_response_active = false; + s_audio.downstream_response_end_received = false; + s_audio.downstream_playback_in_progress = false; + } + portEXIT_CRITICAL(&s_audio_runtime_lock); + return should_notify; } static bool clear_downstream_decoder_reset_pending(uint32_t expected_generation) @@ -1628,7 +1643,8 @@ static void snapshot_downstream_playback_callback( } static void dispatch_downstream_playback_event( - multimodal_audio_downstream_playback_event_type_t type) + multimodal_audio_downstream_playback_event_type_t type, + uint32_t playback_generation) { multimodal_audio_downstream_playback_callback_t cb = NULL; void *cb_ctx = NULL; @@ -1639,6 +1655,7 @@ static void dispatch_downstream_playback_event( multimodal_audio_downstream_playback_event_t event = { .type = type, + .playback_generation = playback_generation, }; cb(&event, cb_ctx); } @@ -1656,51 +1673,32 @@ static void notify_downstream_playback_started(uint32_t expected_generation) portEXIT_CRITICAL(&s_audio_runtime_lock); if (should_notify) { - dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED); + dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED, + expected_generation); } } -static void notify_downstream_playback_finished( - multimodal_audio_downstream_playback_event_type_t type) +static void notify_downstream_playback_completed(uint32_t expected_generation) { - bool should_notify = false; - - portENTER_CRITICAL(&s_audio_runtime_lock); - if (s_audio.downstream_playback_in_progress) { - s_audio.downstream_playback_in_progress = false; - should_notify = true; - } - portEXIT_CRITICAL(&s_audio_runtime_lock); - - if (should_notify) { - dispatch_downstream_playback_event(type); + uint32_t playback_generation = 0U; + if (claim_downstream_response_terminal(expected_generation, + false, + &playback_generation)) { + dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED, + playback_generation); } } -static void notify_downstream_playback_completed(void) -{ - bool should_notify = false; - - portENTER_CRITICAL(&s_audio_runtime_lock); - if (s_audio.downstream_playback_in_progress) { - s_audio.downstream_playback_in_progress = false; - should_notify = true; - } else if (s_audio.downstream_response_end_received) { - should_notify = true; - } - portEXIT_CRITICAL(&s_audio_runtime_lock); - - if (should_notify) { - dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED); - } -} - -static bool snapshot_downstream_response_end_received(void) +static bool snapshot_downstream_response_end_received( + uint32_t *out_playback_generation) { bool end_received = false; portENTER_CRITICAL(&s_audio_runtime_lock); end_received = s_audio.downstream_response_end_received; + if (out_playback_generation != NULL) { + *out_playback_generation = s_audio.downstream_playback_generation; + } portEXIT_CRITICAL(&s_audio_runtime_lock); return end_received; @@ -1717,27 +1715,24 @@ static bool snapshot_downstream_response_active(void) return active; } -static TickType_t snapshot_downstream_last_audio_tick(const multimodal_audio_ctx_t *ctx) +static bool snapshot_downstream_response_watchdog( + const multimodal_audio_ctx_t *ctx, + uint32_t *out_playback_generation, + TickType_t *out_last_audio_tick, + TickType_t *out_last_response_activity_tick) { - TickType_t tick = 0; - portENTER_CRITICAL(&s_audio_runtime_lock); - if (ctx != NULL) { - tick = ctx->downstream_last_audio_tick; + if (ctx == NULL || out_playback_generation == NULL || + out_last_audio_tick == NULL || out_last_response_activity_tick == NULL) { + return false; } - portEXIT_CRITICAL(&s_audio_runtime_lock); - return tick; -} -static TickType_t snapshot_downstream_last_response_activity_tick( - const multimodal_audio_ctx_t *ctx) -{ - TickType_t tick = 0; portENTER_CRITICAL(&s_audio_runtime_lock); - if (ctx != NULL) { - tick = ctx->downstream_last_response_activity_tick; - } + bool active = s_audio.downstream_response_active; + *out_playback_generation = s_audio.downstream_playback_generation; + *out_last_audio_tick = ctx->downstream_last_audio_tick; + *out_last_response_activity_tick = ctx->downstream_last_response_activity_tick; portEXIT_CRITICAL(&s_audio_runtime_lock); - return tick; + return active; } static void update_downstream_last_audio_tick(multimodal_audio_ctx_t *ctx, @@ -1769,37 +1764,28 @@ static void update_downstream_response_state(bool active, bool end_received) portEXIT_CRITICAL(&s_audio_runtime_lock); } -static void notify_downstream_playback_stopped(void) -{ - notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED); -} - -static void notify_downstream_playback_stalled(void) -{ - notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED); -} - static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t *ctx, - uint32_t idle_ms) + uint32_t idle_ms, + uint32_t expected_generation) { + (void)ctx; + uint32_t stopped_generation = 0U; + if (!claim_downstream_response_terminal(expected_generation, + true, + &stopped_generation)) { + return; + } ESP_LOGW(TAG, "Downstream protection watchdog: abort stalled response idle_ms=%u", (unsigned)idle_ms); - (void)advance_downstream_playback_generation(); - notify_downstream_playback_stalled(); - update_downstream_response_state(false, false); - if (audio_get_downstream_playback_task(ctx) != NULL || - audio_get_downstream_playback_starting(ctx)) { - audio_set_downstream_playback_stopping(ctx, true); - audio_set_downstream_playback_keep_stopping_on_exit(ctx, false); - audio_set_downstream_playback_stop_requested(ctx, true); - } - drain_downstream_playback_queue_now(ctx); + dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED, + stopped_generation); } static void maybe_notify_downstream_playback_completed_after_drain(const char *reason) { - if (!snapshot_downstream_response_end_received()) { + uint32_t playback_generation = 0U; + if (!snapshot_downstream_response_end_received(&playback_generation)) { return; } @@ -1820,8 +1806,7 @@ static void maybe_notify_downstream_playback_completed_after_drain(const char *r "Downstream playback completed after response end (%s)", reason ? reason : "drained"); log_downstream_diagnostics("playback_complete", true); - notify_downstream_playback_completed(); - update_downstream_response_state(false, false); + notify_downstream_playback_completed(playback_generation); } static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ctx, @@ -1933,11 +1918,17 @@ static void downstream_idle_mute_timer_cb(void *arg) return; } - const bool response_active = snapshot_downstream_response_active(); + uint32_t playback_generation = 0U; + TickType_t last_audio_tick = 0; + TickType_t last_response_activity_tick = 0; + const bool response_active = snapshot_downstream_response_watchdog( + ctx, + &playback_generation, + &last_audio_tick, + &last_response_activity_tick); if (response_active) { log_downstream_active_diagnostics(); TickType_t now = xTaskGetTickCount(); - TickType_t last_audio_tick = snapshot_downstream_last_audio_tick(ctx); uint32_t idle_ms = last_audio_tick != 0U ? (uint32_t)pdTICKS_TO_MS(now - last_audio_tick) : 0U; uint32_t idle_timeout_ms = get_downstream_idle_mute_timeout_ms(); @@ -1960,14 +1951,15 @@ static void downstream_idle_mute_timer_cb(void *arg) return; } - TickType_t last_activity_tick = snapshot_downstream_last_response_activity_tick(ctx); - uint32_t response_idle_ms = last_activity_tick != 0U ? - (uint32_t)pdTICKS_TO_MS(now - last_activity_tick) : idle_ms; + uint32_t response_idle_ms = last_response_activity_tick != 0U ? + (uint32_t)pdTICKS_TO_MS(now - last_response_activity_tick) : idle_ms; if (response_idle_ms < stall_timeout_ms) { restart_downstream_idle_mute_timer(ctx, stall_timeout_ms - response_idle_ms); return; } - abort_downstream_response_after_active_stall(ctx, idle_ms); + abort_downstream_response_after_active_stall(ctx, + idle_ms, + playback_generation); return; } @@ -2221,7 +2213,7 @@ static void downstream_playback_task(void *arg) if (generation_started && queued_packets > 0U && first_recovery_packet_tick == 0U) { first_recovery_packet_tick = now; } - bool response_ended = snapshot_downstream_response_end_received(); + bool response_ended = snapshot_downstream_response_end_received(NULL); if (queued_packets >= target_packets || (response_ended && queued_packets > 0U)) { playback_buffer_ready = true; uint32_t buffered_ms = (uint32_t)queued_packets * ctx->opus_frame_duration_ms; @@ -2288,7 +2280,7 @@ static void downstream_playback_task(void *arg) } if (received != pdTRUE || packet.data == NULL || packet.len == 0U) { - if (snapshot_downstream_response_end_received()) { + if (snapshot_downstream_response_end_received(NULL)) { maybe_notify_downstream_playback_completed_after_drain("queue_empty"); } else if (playback_buffer_ready && generation_started && snapshot_downstream_response_active()) { @@ -2338,7 +2330,9 @@ static void downstream_playback_task(void *arg) } if (!audio_get_downstream_playback_stop_requested(ctx)) { - esp_err_t err = handle_downstream_opus_sync(packet.data, packet.len); + esp_err_t err = handle_downstream_opus_sync(packet.data, + packet.len, + packet.playback_generation); if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { note_downstream_playback_failure(); ESP_LOGW(TAG, "Downstream playback packet failed: %s", esp_err_to_name(err)); @@ -2586,9 +2580,11 @@ esp_err_t multimodal_audio_stop_downstream_playback(void) { multimodal_audio_ctx_t *ctx = &s_audio; - update_downstream_response_state(false, false); - (void)advance_downstream_playback_generation(); - notify_downstream_playback_stopped(); + uint32_t stopped_generation = 0U; + if (claim_downstream_response_terminal(0U, true, &stopped_generation)) { + dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED, + stopped_generation); + } if (audio_get_deinit_in_progress(ctx)) { esp_err_t task_stop_err = stop_downstream_playback_task(ctx, 1000, true); @@ -2608,7 +2604,9 @@ esp_err_t multimodal_audio_stop_downstream_playback(void) return set_downstream_output_mute_if_needed(ctx, true, "Mute"); } -static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len) +static esp_err_t handle_downstream_opus_sync(const uint8_t *data, + size_t len, + uint32_t expected_generation) { multimodal_audio_ctx_t *ctx = &s_audio; if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_dec) { @@ -2635,6 +2633,9 @@ static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len) uint32_t playback_generation = 0U; bool decoder_reset_pending = false; snapshot_downstream_playback_state(&playback_generation, &decoder_reset_pending); + if (playback_generation != expected_generation) { + return ESP_ERR_INVALID_STATE; + } if (decoder_reset_pending) { esp_err_t reset_err = reset_downstream_decoder(ctx); if (reset_err != ESP_OK) { @@ -2929,17 +2930,21 @@ esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t le return ESP_OK; } -esp_err_t multimodal_audio_begin_downstream_response(void) +esp_err_t multimodal_audio_begin_downstream_response( + uint32_t *out_playback_generation) { multimodal_audio_ctx_t *ctx = &s_audio; + if (out_playback_generation == NULL) { + return ESP_ERR_INVALID_ARG; + } + *out_playback_generation = 0U; if (!ctx->initialized || ctx->codec_dev == NULL || ctx->opus_dec == NULL) { return ESP_ERR_INVALID_STATE; } - uint32_t playback_generation = advance_downstream_playback_generation(); - TickType_t begin_tick = xTaskGetTickCount(); portENTER_CRITICAL(&s_audio_runtime_lock); + uint32_t playback_generation = advance_downstream_playback_generation_locked(); ctx->downstream_response_active = true; ctx->downstream_response_end_received = false; ctx->downstream_output_primed = false; @@ -2947,6 +2952,7 @@ esp_err_t multimodal_audio_begin_downstream_response(void) ctx->downstream_last_audio_tick = begin_tick; ctx->downstream_last_response_activity_tick = begin_tick; portEXIT_CRITICAL(&s_audio_runtime_lock); + *out_playback_generation = playback_generation; s_downstream_queued_packet_total = 0U; s_downstream_played_packet_total = 0U; diff --git a/components/platform/src/platform_behavior_stubs.c b/components/platform/src/platform_behavior_stubs.c index 40a990d..db0bd76 100644 --- a/components/platform/src/platform_behavior_stubs.c +++ b/components/platform/src/platform_behavior_stubs.c @@ -235,7 +235,8 @@ esp_err_t voice_prompt_play_file(const char *path) return ESP_ERR_NOT_FOUND; } - esp_err_t err = multimodal_audio_begin_downstream_response(); + uint32_t playback_generation = 0U; + esp_err_t err = multimodal_audio_begin_downstream_response(&playback_generation); if (err != ESP_OK) { ESP_LOGW(VOICE_TAG, "Begin prompt playback failed: %s", esp_err_to_name(err)); fclose(file); diff --git a/components/platform/src/runtime/platform_multimodal_runtime.c b/components/platform/src/runtime/platform_multimodal_runtime.c index b6d8d69..e0d0262 100644 --- a/components/platform/src/runtime/platform_multimodal_runtime.c +++ b/components/platform/src/runtime/platform_multimodal_runtime.c @@ -200,9 +200,10 @@ esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, si return multimodal_audio_handle_downstream_opus(data, len); } -esp_err_t platform_multimodal_audio_begin_downstream_response(void) +esp_err_t platform_multimodal_audio_begin_downstream_response( + uint32_t *out_playback_generation) { - return multimodal_audio_begin_downstream_response(); + return multimodal_audio_begin_downstream_response(out_playback_generation); } esp_err_t platform_multimodal_audio_finish_downstream_response(void) @@ -230,22 +231,22 @@ static void platform_multimodal_downstream_playback_bridge( switch (event->type) { case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED: if (callbacks.on_started != NULL) { - callbacks.on_started(callbacks.user_data); + callbacks.on_started(event->playback_generation, callbacks.user_data); } break; case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED: if (callbacks.on_completed != NULL) { - callbacks.on_completed(callbacks.user_data); + callbacks.on_completed(event->playback_generation, callbacks.user_data); } break; case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED: if (callbacks.on_stopped != NULL) { - callbacks.on_stopped(callbacks.user_data); + callbacks.on_stopped(event->playback_generation, callbacks.user_data); } break; case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED: if (callbacks.on_stalled != NULL) { - callbacks.on_stalled(callbacks.user_data); + callbacks.on_stalled(event->playback_generation, callbacks.user_data); } break; default: