fix(multimodal): harden playback and client lifecycle handling

- retry the server connection prompt after transient playback failures
- serialize multimodal client and session start-stop operations
- tag downstream responses to ignore stale packets and playback events
- complete bounded teardown and clean callbacks after partial startup

components/platform/include/platform.h

@@ -43,10 +43,10 @@ typedef struct {
 } platform_multimodal_transport_callbacks_t;
 
 typedef struct {
-    void (*on_started)(void *user_data);
-    void (*on_completed)(void *user_data);
-    void (*on_stopped)(void *user_data);
-    void (*on_stalled)(void *user_data);
+    void (*on_started)(uint32_t playback_generation, void *user_data);
+    void (*on_completed)(uint32_t playback_generation, void *user_data);
+    void (*on_stopped)(uint32_t playback_generation, void *user_data);
+    void (*on_stalled)(uint32_t playback_generation, void *user_data);
     void *user_data;
 } platform_multimodal_downstream_playback_callbacks_t;
 
@@ -399,7 +399,7 @@ esp_err_t platform_multimodal_audio_unregister_pcm_observer(
         void *user_data,
         int timeout_ms);
 esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
-esp_err_t platform_multimodal_audio_begin_downstream_response(void);
+esp_err_t platform_multimodal_audio_begin_downstream_response(uint32_t *out_playback_generation);
 esp_err_t platform_multimodal_audio_finish_downstream_response(void);
 esp_err_t platform_multimodal_audio_stop_downstream_playback(void);
 esp_err_t platform_multimodal_audio_register_downstream_playback_callbacks(

components/platform/internal/multimodal/audio_pipeline.h

@@ -25,6 +25,7 @@ typedef enum {
 
 typedef struct {
     multimodal_audio_downstream_playback_event_type_t type;
+    uint32_t playback_generation;
 } multimodal_audio_downstream_playback_event_t;
 
 typedef void (*multimodal_audio_downstream_playback_callback_t)(
@@ -47,7 +48,7 @@ esp_err_t multimodal_audio_unregister_pcm_observer(
         int timeout_ms);
 
 esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t len);
-esp_err_t multimodal_audio_begin_downstream_response(void);
+esp_err_t multimodal_audio_begin_downstream_response(uint32_t *out_playback_generation);
 esp_err_t multimodal_audio_finish_downstream_response(void);
 esp_err_t multimodal_audio_stop_downstream_playback(void);
 esp_err_t multimodal_audio_register_downstream_playback_callback(

components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c

@@ -461,16 +461,6 @@ static void drain_downstream_playback_queue(multimodal_audio_ctx_t *ctx)
     unlock_downstream_playback_queue();
 }
 
-static void drain_downstream_playback_queue_now(multimodal_audio_ctx_t *ctx)
-{
-    if (ctx == NULL || !lock_downstream_playback_queue(0)) {
-        return;
-    }
-    QueueHandle_t queue = ctx->downstream_playback_queue;
-    drain_downstream_playback_queue_handle(queue);
-    unlock_downstream_playback_queue();
-}
-
 static void track_uplink_packet_queued(size_t len)
 {
 #if CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
@@ -581,10 +571,11 @@ static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ct
 static esp_err_t ensure_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx);
 static void notify_downstream_audio_active(multimodal_audio_ctx_t *ctx);
 static void notify_downstream_playback_started(uint32_t expected_generation);
-static void notify_downstream_playback_completed(void);
-static void notify_downstream_playback_stopped(void);
+static void notify_downstream_playback_completed(uint32_t expected_generation);
 static esp_err_t prime_downstream_output_path(multimodal_audio_ctx_t *ctx);
-static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len);
+static esp_err_t handle_downstream_opus_sync(const uint8_t *data,
+                                             size_t len,
+                                             uint32_t expected_generation);
 static esp_err_t ensure_downstream_playback_task(multimodal_audio_ctx_t *ctx);
 static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx);
 static esp_err_t stop_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx, int timeout_ms);
@@ -1563,20 +1554,44 @@ static void snapshot_downstream_playback_state(uint32_t *out_generation,
     portEXIT_CRITICAL(&s_audio_runtime_lock);
 }
 
-static uint32_t advance_downstream_playback_generation(void)
+static uint32_t advance_downstream_playback_generation_locked(void)
 {
-    uint32_t generation = 0U;
-
-    portENTER_CRITICAL(&s_audio_runtime_lock);
     s_audio.downstream_playback_generation++;
     if (s_audio.downstream_playback_generation == 0U) {
         s_audio.downstream_playback_generation = 1U;
     }
     s_audio.downstream_decoder_reset_pending = true;
-    generation = s_audio.downstream_playback_generation;
-    portEXIT_CRITICAL(&s_audio_runtime_lock);
+    return s_audio.downstream_playback_generation;
+}
 
-    return generation;
+static bool claim_downstream_response_terminal(uint32_t expected_generation,
+                                               bool invalidate_generation,
+                                               uint32_t *out_terminal_generation)
+{
+    if (out_terminal_generation == NULL) {
+        return false;
+    }
+
+    portENTER_CRITICAL(&s_audio_runtime_lock);
+    bool generation_matches = expected_generation == 0U ||
+                              s_audio.downstream_playback_generation ==
+                                      expected_generation;
+    bool should_notify = generation_matches &&
+            (s_audio.downstream_response_active ||
+             s_audio.downstream_response_end_received ||
+             s_audio.downstream_playback_in_progress);
+    *out_terminal_generation = s_audio.downstream_playback_generation;
+    if (generation_matches && invalidate_generation &&
+        (should_notify || expected_generation == 0U)) {
+        (void)advance_downstream_playback_generation_locked();
+    }
+    if (should_notify) {
+        s_audio.downstream_response_active = false;
+        s_audio.downstream_response_end_received = false;
+        s_audio.downstream_playback_in_progress = false;
+    }
+    portEXIT_CRITICAL(&s_audio_runtime_lock);
+    return should_notify;
 }
 
 static bool clear_downstream_decoder_reset_pending(uint32_t expected_generation)
@@ -1628,7 +1643,8 @@ static void snapshot_downstream_playback_callback(
 }
 
 static void dispatch_downstream_playback_event(
-        multimodal_audio_downstream_playback_event_type_t type)
+        multimodal_audio_downstream_playback_event_type_t type,
+        uint32_t playback_generation)
 {
     multimodal_audio_downstream_playback_callback_t cb = NULL;
     void *cb_ctx = NULL;
@@ -1639,6 +1655,7 @@ static void dispatch_downstream_playback_event(
 
     multimodal_audio_downstream_playback_event_t event = {
         .type = type,
+        .playback_generation = playback_generation,
     };
     cb(&event, cb_ctx);
 }
@@ -1656,51 +1673,32 @@ static void notify_downstream_playback_started(uint32_t expected_generation)
     portEXIT_CRITICAL(&s_audio_runtime_lock);
 
     if (should_notify) {
-        dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED);
+        dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED,
+                                           expected_generation);
     }
 }
 
-static void notify_downstream_playback_finished(
-        multimodal_audio_downstream_playback_event_type_t type)
+static void notify_downstream_playback_completed(uint32_t expected_generation)
 {
-    bool should_notify = false;
-
-    portENTER_CRITICAL(&s_audio_runtime_lock);
-    if (s_audio.downstream_playback_in_progress) {
-        s_audio.downstream_playback_in_progress = false;
-        should_notify = true;
-    }
-    portEXIT_CRITICAL(&s_audio_runtime_lock);
-
-    if (should_notify) {
-        dispatch_downstream_playback_event(type);
+    uint32_t playback_generation = 0U;
+    if (claim_downstream_response_terminal(expected_generation,
+                                           false,
+                                           &playback_generation)) {
+        dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED,
+                                           playback_generation);
     }
 }
 
-static void notify_downstream_playback_completed(void)
-{
-    bool should_notify = false;
-
-    portENTER_CRITICAL(&s_audio_runtime_lock);
-    if (s_audio.downstream_playback_in_progress) {
-        s_audio.downstream_playback_in_progress = false;
-        should_notify = true;
-    } else if (s_audio.downstream_response_end_received) {
-        should_notify = true;
-    }
-    portEXIT_CRITICAL(&s_audio_runtime_lock);
-
-    if (should_notify) {
-        dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED);
-    }
-}
-
-static bool snapshot_downstream_response_end_received(void)
+static bool snapshot_downstream_response_end_received(
+        uint32_t *out_playback_generation)
 {
     bool end_received = false;
 
     portENTER_CRITICAL(&s_audio_runtime_lock);
     end_received = s_audio.downstream_response_end_received;
+    if (out_playback_generation != NULL) {
+        *out_playback_generation = s_audio.downstream_playback_generation;
+    }
     portEXIT_CRITICAL(&s_audio_runtime_lock);
 
     return end_received;
@@ -1717,27 +1715,24 @@ static bool snapshot_downstream_response_active(void)
     return active;
 }
 
-static TickType_t snapshot_downstream_last_audio_tick(const multimodal_audio_ctx_t *ctx)
+static bool snapshot_downstream_response_watchdog(
+        const multimodal_audio_ctx_t *ctx,
+        uint32_t *out_playback_generation,
+        TickType_t *out_last_audio_tick,
+        TickType_t *out_last_response_activity_tick)
 {
-    TickType_t tick = 0;
-    portENTER_CRITICAL(&s_audio_runtime_lock);
-    if (ctx != NULL) {
-        tick = ctx->downstream_last_audio_tick;
+    if (ctx == NULL || out_playback_generation == NULL ||
+        out_last_audio_tick == NULL || out_last_response_activity_tick == NULL) {
+        return false;
     }
-    portEXIT_CRITICAL(&s_audio_runtime_lock);
-    return tick;
-}
 
-static TickType_t snapshot_downstream_last_response_activity_tick(
-        const multimodal_audio_ctx_t *ctx)
-{
-    TickType_t tick = 0;
     portENTER_CRITICAL(&s_audio_runtime_lock);
-    if (ctx != NULL) {
-        tick = ctx->downstream_last_response_activity_tick;
-    }
+    bool active = s_audio.downstream_response_active;
+    *out_playback_generation = s_audio.downstream_playback_generation;
+    *out_last_audio_tick = ctx->downstream_last_audio_tick;
+    *out_last_response_activity_tick = ctx->downstream_last_response_activity_tick;
     portEXIT_CRITICAL(&s_audio_runtime_lock);
-    return tick;
+    return active;
 }
 
 static void update_downstream_last_audio_tick(multimodal_audio_ctx_t *ctx,
@@ -1769,37 +1764,28 @@ static void update_downstream_response_state(bool active, bool end_received)
     portEXIT_CRITICAL(&s_audio_runtime_lock);
 }
 
-static void notify_downstream_playback_stopped(void)
-{
-    notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED);
-}
-
-static void notify_downstream_playback_stalled(void)
-{
-    notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED);
-}
-
 static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t *ctx,
-                                                         uint32_t idle_ms)
+                                                         uint32_t idle_ms,
+                                                         uint32_t expected_generation)
 {
+    (void)ctx;
+    uint32_t stopped_generation = 0U;
+    if (!claim_downstream_response_terminal(expected_generation,
+                                            true,
+                                            &stopped_generation)) {
+        return;
+    }
     ESP_LOGW(TAG,
              "Downstream protection watchdog: abort stalled response idle_ms=%u",
              (unsigned)idle_ms);
-    (void)advance_downstream_playback_generation();
-    notify_downstream_playback_stalled();
-    update_downstream_response_state(false, false);
-    if (audio_get_downstream_playback_task(ctx) != NULL ||
-        audio_get_downstream_playback_starting(ctx)) {
-        audio_set_downstream_playback_stopping(ctx, true);
-        audio_set_downstream_playback_keep_stopping_on_exit(ctx, false);
-        audio_set_downstream_playback_stop_requested(ctx, true);
-    }
-    drain_downstream_playback_queue_now(ctx);
+    dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED,
+                                       stopped_generation);
 }
 
 static void maybe_notify_downstream_playback_completed_after_drain(const char *reason)
 {
-    if (!snapshot_downstream_response_end_received()) {
+    uint32_t playback_generation = 0U;
+    if (!snapshot_downstream_response_end_received(&playback_generation)) {
         return;
     }
 
@@ -1820,8 +1806,7 @@ static void maybe_notify_downstream_playback_completed_after_drain(const char *r
              "Downstream playback completed after response end (%s)",
              reason ? reason : "drained");
     log_downstream_diagnostics("playback_complete", true);
-    notify_downstream_playback_completed();
-    update_downstream_response_state(false, false);
+    notify_downstream_playback_completed(playback_generation);
 }
 
 static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ctx,
@@ -1933,11 +1918,17 @@ static void downstream_idle_mute_timer_cb(void *arg)
         return;
     }
 
-    const bool response_active = snapshot_downstream_response_active();
+    uint32_t playback_generation = 0U;
+    TickType_t last_audio_tick = 0;
+    TickType_t last_response_activity_tick = 0;
+    const bool response_active = snapshot_downstream_response_watchdog(
+            ctx,
+            &playback_generation,
+            &last_audio_tick,
+            &last_response_activity_tick);
     if (response_active) {
         log_downstream_active_diagnostics();
         TickType_t now = xTaskGetTickCount();
-        TickType_t last_audio_tick = snapshot_downstream_last_audio_tick(ctx);
         uint32_t idle_ms = last_audio_tick != 0U ?
                 (uint32_t)pdTICKS_TO_MS(now - last_audio_tick) : 0U;
         uint32_t idle_timeout_ms = get_downstream_idle_mute_timeout_ms();
@@ -1960,14 +1951,15 @@ static void downstream_idle_mute_timer_cb(void *arg)
             return;
         }
 
-        TickType_t last_activity_tick = snapshot_downstream_last_response_activity_tick(ctx);
-        uint32_t response_idle_ms = last_activity_tick != 0U ?
-                (uint32_t)pdTICKS_TO_MS(now - last_activity_tick) : idle_ms;
+        uint32_t response_idle_ms = last_response_activity_tick != 0U ?
+                (uint32_t)pdTICKS_TO_MS(now - last_response_activity_tick) : idle_ms;
         if (response_idle_ms < stall_timeout_ms) {
             restart_downstream_idle_mute_timer(ctx, stall_timeout_ms - response_idle_ms);
             return;
         }
-        abort_downstream_response_after_active_stall(ctx, idle_ms);
+        abort_downstream_response_after_active_stall(ctx,
+                                                     idle_ms,
+                                                     playback_generation);
         return;
     }
 
@@ -2221,7 +2213,7 @@ static void downstream_playback_task(void *arg)
             if (generation_started && queued_packets > 0U && first_recovery_packet_tick == 0U) {
                 first_recovery_packet_tick = now;
             }
-            bool response_ended = snapshot_downstream_response_end_received();
+            bool response_ended = snapshot_downstream_response_end_received(NULL);
             if (queued_packets >= target_packets || (response_ended && queued_packets > 0U)) {
                 playback_buffer_ready = true;
                 uint32_t buffered_ms = (uint32_t)queued_packets * ctx->opus_frame_duration_ms;
@@ -2288,7 +2280,7 @@ static void downstream_playback_task(void *arg)
         }
 
         if (received != pdTRUE || packet.data == NULL || packet.len == 0U) {
-            if (snapshot_downstream_response_end_received()) {
+            if (snapshot_downstream_response_end_received(NULL)) {
                 maybe_notify_downstream_playback_completed_after_drain("queue_empty");
             } else if (playback_buffer_ready && generation_started &&
                        snapshot_downstream_response_active()) {
@@ -2338,7 +2330,9 @@ static void downstream_playback_task(void *arg)
         }
 
         if (!audio_get_downstream_playback_stop_requested(ctx)) {
-            esp_err_t err = handle_downstream_opus_sync(packet.data, packet.len);
+            esp_err_t err = handle_downstream_opus_sync(packet.data,
+                                                        packet.len,
+                                                        packet.playback_generation);
             if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
                 note_downstream_playback_failure();
                 ESP_LOGW(TAG, "Downstream playback packet failed: %s", esp_err_to_name(err));
@@ -2586,9 +2580,11 @@ esp_err_t multimodal_audio_stop_downstream_playback(void)
 {
     multimodal_audio_ctx_t *ctx = &s_audio;
 
-    update_downstream_response_state(false, false);
-    (void)advance_downstream_playback_generation();
-    notify_downstream_playback_stopped();
+    uint32_t stopped_generation = 0U;
+    if (claim_downstream_response_terminal(0U, true, &stopped_generation)) {
+        dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED,
+                                           stopped_generation);
+    }
 
     if (audio_get_deinit_in_progress(ctx)) {
         esp_err_t task_stop_err = stop_downstream_playback_task(ctx, 1000, true);
@@ -2608,7 +2604,9 @@ esp_err_t multimodal_audio_stop_downstream_playback(void)
     return set_downstream_output_mute_if_needed(ctx, true, "Mute");
 }
 
-static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len)
+static esp_err_t handle_downstream_opus_sync(const uint8_t *data,
+                                             size_t len,
+                                             uint32_t expected_generation)
 {
     multimodal_audio_ctx_t *ctx = &s_audio;
     if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_dec) {
@@ -2635,6 +2633,9 @@ static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len)
     uint32_t playback_generation = 0U;
     bool decoder_reset_pending = false;
     snapshot_downstream_playback_state(&playback_generation, &decoder_reset_pending);
+    if (playback_generation != expected_generation) {
+        return ESP_ERR_INVALID_STATE;
+    }
     if (decoder_reset_pending) {
         esp_err_t reset_err = reset_downstream_decoder(ctx);
         if (reset_err != ESP_OK) {
@@ -2929,17 +2930,21 @@ esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t le
     return ESP_OK;
 }
 
-esp_err_t multimodal_audio_begin_downstream_response(void)
+esp_err_t multimodal_audio_begin_downstream_response(
+        uint32_t *out_playback_generation)
 {
     multimodal_audio_ctx_t *ctx = &s_audio;
+    if (out_playback_generation == NULL) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    *out_playback_generation = 0U;
     if (!ctx->initialized || ctx->codec_dev == NULL || ctx->opus_dec == NULL) {
         return ESP_ERR_INVALID_STATE;
     }
 
-    uint32_t playback_generation = advance_downstream_playback_generation();
-
     TickType_t begin_tick = xTaskGetTickCount();
     portENTER_CRITICAL(&s_audio_runtime_lock);
+    uint32_t playback_generation = advance_downstream_playback_generation_locked();
     ctx->downstream_response_active = true;
     ctx->downstream_response_end_received = false;
     ctx->downstream_output_primed = false;
@@ -2947,6 +2952,7 @@ esp_err_t multimodal_audio_begin_downstream_response(void)
     ctx->downstream_last_audio_tick = begin_tick;
     ctx->downstream_last_response_activity_tick = begin_tick;
     portEXIT_CRITICAL(&s_audio_runtime_lock);
+    *out_playback_generation = playback_generation;
 
     s_downstream_queued_packet_total = 0U;
     s_downstream_played_packet_total = 0U;

components/platform/src/platform_behavior_stubs.c

@@ -235,7 +235,8 @@ esp_err_t voice_prompt_play_file(const char *path)
         return ESP_ERR_NOT_FOUND;
     }
 
-    esp_err_t err = multimodal_audio_begin_downstream_response();
+    uint32_t playback_generation = 0U;
+    esp_err_t err = multimodal_audio_begin_downstream_response(&playback_generation);
     if (err != ESP_OK) {
         ESP_LOGW(VOICE_TAG, "Begin prompt playback failed: %s", esp_err_to_name(err));
         fclose(file);

components/platform/src/runtime/platform_multimodal_runtime.c

@@ -200,9 +200,10 @@ esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, si
     return multimodal_audio_handle_downstream_opus(data, len);
 }
 
-esp_err_t platform_multimodal_audio_begin_downstream_response(void)
+esp_err_t platform_multimodal_audio_begin_downstream_response(
+        uint32_t *out_playback_generation)
 {
-    return multimodal_audio_begin_downstream_response();
+    return multimodal_audio_begin_downstream_response(out_playback_generation);
 }
 
 esp_err_t platform_multimodal_audio_finish_downstream_response(void)
@@ -230,22 +231,22 @@ static void platform_multimodal_downstream_playback_bridge(
     switch (event->type) {
         case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED:
             if (callbacks.on_started != NULL) {
-                callbacks.on_started(callbacks.user_data);
+                callbacks.on_started(event->playback_generation, callbacks.user_data);
             }
             break;
         case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED:
             if (callbacks.on_completed != NULL) {
-                callbacks.on_completed(callbacks.user_data);
+                callbacks.on_completed(event->playback_generation, callbacks.user_data);
             }
             break;
         case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED:
             if (callbacks.on_stopped != NULL) {
-                callbacks.on_stopped(callbacks.user_data);
+                callbacks.on_stopped(event->playback_generation, callbacks.user_data);
             }
             break;
         case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED:
             if (callbacks.on_stalled != NULL) {
-                callbacks.on_stalled(callbacks.user_data);
+                callbacks.on_stalled(event->playback_generation, callbacks.user_data);
             }
             break;
         default: