chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(multimodal): harden playback and client lifecycle handling

Browse Source 
- retry the server connection prompt after transient playback failures
- serialize multimodal client and session start-stop operations
- tag downstream responses to ignore stale packets and playback events
- complete bounded teardown and clean callbacks after partial startup
This commit is contained in:
chenjintao
2026-08-14 15:24:29 +08:00
parent c56c65462e
commit 0cfb10252c
12 changed files with 436 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c
+116 -110
View File
@@ -461,16 +461,6 @@ static void drain_downstream_playback_queue(multimodal_audio_ctx_t *ctx)
unlock_downstream_playback_queue();
}
static void drain_downstream_playback_queue_now(multimodal_audio_ctx_t *ctx)
{
if (ctx == NULL || !lock_downstream_playback_queue(0)) {
return;
}
QueueHandle_t queue = ctx->downstream_playback_queue;
drain_downstream_playback_queue_handle(queue);
unlock_downstream_playback_queue();
}
static void track_uplink_packet_queued(size_t len)
{
#if CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
@@ -581,10 +571,11 @@ static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ct
static esp_err_t ensure_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx);
static void notify_downstream_audio_active(multimodal_audio_ctx_t *ctx);
static void notify_downstream_playback_started(uint32_t expected_generation);
static void notify_downstream_playback_completed(void);
static void notify_downstream_playback_stopped(void);
static void notify_downstream_playback_completed(uint32_t expected_generation);
static esp_err_t prime_downstream_output_path(multimodal_audio_ctx_t *ctx);
static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len);
static esp_err_t handle_downstream_opus_sync(const uint8_t *data,
size_t len,
uint32_t expected_generation);
static esp_err_t ensure_downstream_playback_task(multimodal_audio_ctx_t *ctx);
static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx);
static esp_err_t stop_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx, int timeout_ms);
@@ -1563,20 +1554,44 @@ static void snapshot_downstream_playback_state(uint32_t *out_generation,
portEXIT_CRITICAL(&s_audio_runtime_lock);
}
static uint32_t advance_downstream_playback_generation(void)
static uint32_t advance_downstream_playback_generation_locked(void)
{
uint32_t generation = 0U;
portENTER_CRITICAL(&s_audio_runtime_lock);
s_audio.downstream_playback_generation++;
if (s_audio.downstream_playback_generation == 0U) {
s_audio.downstream_playback_generation = 1U;
}
s_audio.downstream_decoder_reset_pending = true;
generation = s_audio.downstream_playback_generation;
portEXIT_CRITICAL(&s_audio_runtime_lock);
return s_audio.downstream_playback_generation;
}
return generation;
static bool claim_downstream_response_terminal(uint32_t expected_generation,
bool invalidate_generation,
uint32_t *out_terminal_generation)
{
if (out_terminal_generation == NULL) {
return false;
}
portENTER_CRITICAL(&s_audio_runtime_lock);
bool generation_matches = expected_generation == 0U ||
s_audio.downstream_playback_generation ==
expected_generation;
bool should_notify = generation_matches &&
(s_audio.downstream_response_active ||
s_audio.downstream_response_end_received ||
s_audio.downstream_playback_in_progress);
*out_terminal_generation = s_audio.downstream_playback_generation;
if (generation_matches && invalidate_generation &&
(should_notify || expected_generation == 0U)) {
(void)advance_downstream_playback_generation_locked();
}
if (should_notify) {
s_audio.downstream_response_active = false;
s_audio.downstream_response_end_received = false;
s_audio.downstream_playback_in_progress = false;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
return should_notify;
}
static bool clear_downstream_decoder_reset_pending(uint32_t expected_generation)
@@ -1628,7 +1643,8 @@ static void snapshot_downstream_playback_callback(
}
static void dispatch_downstream_playback_event(
multimodal_audio_downstream_playback_event_type_t type)
multimodal_audio_downstream_playback_event_type_t type,
uint32_t playback_generation)
{
multimodal_audio_downstream_playback_callback_t cb = NULL;
void *cb_ctx = NULL;
@@ -1639,6 +1655,7 @@ static void dispatch_downstream_playback_event(
multimodal_audio_downstream_playback_event_t event = {
.type = type,
.playback_generation = playback_generation,
};
cb(&event, cb_ctx);
}
@@ -1656,51 +1673,32 @@ static void notify_downstream_playback_started(uint32_t expected_generation)
portEXIT_CRITICAL(&s_audio_runtime_lock);
if (should_notify) {
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED);
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED,
expected_generation);
}
}
static void notify_downstream_playback_finished(
multimodal_audio_downstream_playback_event_type_t type)
static void notify_downstream_playback_completed(uint32_t expected_generation)
{
bool should_notify = false;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (s_audio.downstream_playback_in_progress) {
s_audio.downstream_playback_in_progress = false;
should_notify = true;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
if (should_notify) {
dispatch_downstream_playback_event(type);
uint32_t playback_generation = 0U;
if (claim_downstream_response_terminal(expected_generation,
false,
&playback_generation)) {
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED,
playback_generation);
}
}
static void notify_downstream_playback_completed(void)
{
bool should_notify = false;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (s_audio.downstream_playback_in_progress) {
s_audio.downstream_playback_in_progress = false;
should_notify = true;
} else if (s_audio.downstream_response_end_received) {
should_notify = true;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
if (should_notify) {
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED);
}
}
static bool snapshot_downstream_response_end_received(void)
static bool snapshot_downstream_response_end_received(
uint32_t *out_playback_generation)
{
bool end_received = false;
portENTER_CRITICAL(&s_audio_runtime_lock);
end_received = s_audio.downstream_response_end_received;
if (out_playback_generation != NULL) {
*out_playback_generation = s_audio.downstream_playback_generation;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
return end_received;
@@ -1717,27 +1715,24 @@ static bool snapshot_downstream_response_active(void)
return active;
}
static TickType_t snapshot_downstream_last_audio_tick(const multimodal_audio_ctx_t *ctx)
static bool snapshot_downstream_response_watchdog(
const multimodal_audio_ctx_t *ctx,
uint32_t *out_playback_generation,
TickType_t *out_last_audio_tick,
TickType_t *out_last_response_activity_tick)
{
TickType_t tick = 0;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (ctx != NULL) {
tick = ctx->downstream_last_audio_tick;
if (ctx == NULL || out_playback_generation == NULL ||
out_last_audio_tick == NULL || out_last_response_activity_tick == NULL) {
return false;
}
portEXIT_CRITICAL(&s_audio_runtime_lock);
return tick;
}
static TickType_t snapshot_downstream_last_response_activity_tick(
const multimodal_audio_ctx_t *ctx)
{
TickType_t tick = 0;
portENTER_CRITICAL(&s_audio_runtime_lock);
if (ctx != NULL) {
tick = ctx->downstream_last_response_activity_tick;
}
bool active = s_audio.downstream_response_active;
*out_playback_generation = s_audio.downstream_playback_generation;
*out_last_audio_tick = ctx->downstream_last_audio_tick;
*out_last_response_activity_tick = ctx->downstream_last_response_activity_tick;
portEXIT_CRITICAL(&s_audio_runtime_lock);
return tick;
return active;
}
static void update_downstream_last_audio_tick(multimodal_audio_ctx_t *ctx,
@@ -1769,37 +1764,28 @@ static void update_downstream_response_state(bool active, bool end_received)
portEXIT_CRITICAL(&s_audio_runtime_lock);
}
static void notify_downstream_playback_stopped(void)
{
notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED);
}
static void notify_downstream_playback_stalled(void)
{
notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED);
}
static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t *ctx,
uint32_t idle_ms)
uint32_t idle_ms,
uint32_t expected_generation)
{
(void)ctx;
uint32_t stopped_generation = 0U;
if (!claim_downstream_response_terminal(expected_generation,
true,
&stopped_generation)) {
return;
}
ESP_LOGW(TAG,
"Downstream protection watchdog: abort stalled response idle_ms=%u",
(unsigned)idle_ms);
(void)advance_downstream_playback_generation();
notify_downstream_playback_stalled();
update_downstream_response_state(false, false);
if (audio_get_downstream_playback_task(ctx) != NULL ||
audio_get_downstream_playback_starting(ctx)) {
audio_set_downstream_playback_stopping(ctx, true);
audio_set_downstream_playback_keep_stopping_on_exit(ctx, false);
audio_set_downstream_playback_stop_requested(ctx, true);
}
drain_downstream_playback_queue_now(ctx);
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED,
stopped_generation);
}
static void maybe_notify_downstream_playback_completed_after_drain(const char *reason)
{
if (!snapshot_downstream_response_end_received()) {
uint32_t playback_generation = 0U;
if (!snapshot_downstream_response_end_received(&playback_generation)) {
return;
}
@@ -1820,8 +1806,7 @@ static void maybe_notify_downstream_playback_completed_after_drain(const char *r
"Downstream playback completed after response end (%s)",
reason ? reason : "drained");
log_downstream_diagnostics("playback_complete", true);
notify_downstream_playback_completed();
update_downstream_response_state(false, false);
notify_downstream_playback_completed(playback_generation);
}
static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ctx,
@@ -1933,11 +1918,17 @@ static void downstream_idle_mute_timer_cb(void *arg)
return;
}
const bool response_active = snapshot_downstream_response_active();
uint32_t playback_generation = 0U;
TickType_t last_audio_tick = 0;
TickType_t last_response_activity_tick = 0;
const bool response_active = snapshot_downstream_response_watchdog(
ctx,
&playback_generation,
&last_audio_tick,
&last_response_activity_tick);
if (response_active) {
log_downstream_active_diagnostics();
TickType_t now = xTaskGetTickCount();
TickType_t last_audio_tick = snapshot_downstream_last_audio_tick(ctx);
uint32_t idle_ms = last_audio_tick != 0U ?
(uint32_t)pdTICKS_TO_MS(now - last_audio_tick) : 0U;
uint32_t idle_timeout_ms = get_downstream_idle_mute_timeout_ms();
@@ -1960,14 +1951,15 @@ static void downstream_idle_mute_timer_cb(void *arg)
return;
}
TickType_t last_activity_tick = snapshot_downstream_last_response_activity_tick(ctx);
uint32_t response_idle_ms = last_activity_tick != 0U ?
(uint32_t)pdTICKS_TO_MS(now - last_activity_tick) : idle_ms;
uint32_t response_idle_ms = last_response_activity_tick != 0U ?
(uint32_t)pdTICKS_TO_MS(now - last_response_activity_tick) : idle_ms;
if (response_idle_ms < stall_timeout_ms) {
restart_downstream_idle_mute_timer(ctx, stall_timeout_ms - response_idle_ms);
return;
}
abort_downstream_response_after_active_stall(ctx, idle_ms);
abort_downstream_response_after_active_stall(ctx,
idle_ms,
playback_generation);
return;
}
@@ -2221,7 +2213,7 @@ static void downstream_playback_task(void *arg)
if (generation_started && queued_packets > 0U && first_recovery_packet_tick == 0U) {
first_recovery_packet_tick = now;
}
bool response_ended = snapshot_downstream_response_end_received();
bool response_ended = snapshot_downstream_response_end_received(NULL);
if (queued_packets >= target_packets || (response_ended && queued_packets > 0U)) {
playback_buffer_ready = true;
uint32_t buffered_ms = (uint32_t)queued_packets * ctx->opus_frame_duration_ms;
@@ -2288,7 +2280,7 @@ static void downstream_playback_task(void *arg)
}
if (received != pdTRUE || packet.data == NULL || packet.len == 0U) {
if (snapshot_downstream_response_end_received()) {
if (snapshot_downstream_response_end_received(NULL)) {
maybe_notify_downstream_playback_completed_after_drain("queue_empty");
} else if (playback_buffer_ready && generation_started &&
snapshot_downstream_response_active()) {
@@ -2338,7 +2330,9 @@ static void downstream_playback_task(void *arg)
}
if (!audio_get_downstream_playback_stop_requested(ctx)) {
esp_err_t err = handle_downstream_opus_sync(packet.data, packet.len);
esp_err_t err = handle_downstream_opus_sync(packet.data,
packet.len,
packet.playback_generation);
if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
note_downstream_playback_failure();
ESP_LOGW(TAG, "Downstream playback packet failed: %s", esp_err_to_name(err));
@@ -2586,9 +2580,11 @@ esp_err_t multimodal_audio_stop_downstream_playback(void)
{
multimodal_audio_ctx_t *ctx = &s_audio;
update_downstream_response_state(false, false);
(void)advance_downstream_playback_generation();
notify_downstream_playback_stopped();
uint32_t stopped_generation = 0U;
if (claim_downstream_response_terminal(0U, true, &stopped_generation)) {
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED,
stopped_generation);
}
if (audio_get_deinit_in_progress(ctx)) {
esp_err_t task_stop_err = stop_downstream_playback_task(ctx, 1000, true);
@@ -2608,7 +2604,9 @@ esp_err_t multimodal_audio_stop_downstream_playback(void)
return set_downstream_output_mute_if_needed(ctx, true, "Mute");
}
static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len)
static esp_err_t handle_downstream_opus_sync(const uint8_t *data,
size_t len,
uint32_t expected_generation)
{
multimodal_audio_ctx_t *ctx = &s_audio;
if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_dec) {
@@ -2635,6 +2633,9 @@ static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len)
uint32_t playback_generation = 0U;
bool decoder_reset_pending = false;
snapshot_downstream_playback_state(&playback_generation, &decoder_reset_pending);
if (playback_generation != expected_generation) {
return ESP_ERR_INVALID_STATE;
}
if (decoder_reset_pending) {
esp_err_t reset_err = reset_downstream_decoder(ctx);
if (reset_err != ESP_OK) {
@@ -2929,17 +2930,21 @@ esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t le
return ESP_OK;
}
esp_err_t multimodal_audio_begin_downstream_response(void)
esp_err_t multimodal_audio_begin_downstream_response(
uint32_t *out_playback_generation)
{
multimodal_audio_ctx_t *ctx = &s_audio;
if (out_playback_generation == NULL) {
return ESP_ERR_INVALID_ARG;
}
*out_playback_generation = 0U;
if (!ctx->initialized || ctx->codec_dev == NULL || ctx->opus_dec == NULL) {
return ESP_ERR_INVALID_STATE;
}
uint32_t playback_generation = advance_downstream_playback_generation();
TickType_t begin_tick = xTaskGetTickCount();
portENTER_CRITICAL(&s_audio_runtime_lock);
uint32_t playback_generation = advance_downstream_playback_generation_locked();
ctx->downstream_response_active = true;
ctx->downstream_response_end_received = false;
ctx->downstream_output_primed = false;
@@ -2947,6 +2952,7 @@ esp_err_t multimodal_audio_begin_downstream_response(void)
ctx->downstream_last_audio_tick = begin_tick;
ctx->downstream_last_response_activity_tick = begin_tick;
portEXIT_CRITICAL(&s_audio_runtime_lock);
*out_playback_generation = playback_generation;
s_downstream_queued_packet_total = 0U;
s_downstream_played_packet_total = 0U;
components/platform/src/platform_behavior_stubs.c
+2 -1
View File
@@ -235,7 +235,8 @@ esp_err_t voice_prompt_play_file(const char *path)
return ESP_ERR_NOT_FOUND;
}
esp_err_t err = multimodal_audio_begin_downstream_response();
uint32_t playback_generation = 0U;
esp_err_t err = multimodal_audio_begin_downstream_response(&playback_generation);
if (err != ESP_OK) {
ESP_LOGW(VOICE_TAG, "Begin prompt playback failed: %s", esp_err_to_name(err));
fclose(file);
components/platform/src/runtime/platform_multimodal_runtime.c
+7 -6
View File
@@ -200,9 +200,10 @@ esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, si
return multimodal_audio_handle_downstream_opus(data, len);
}
esp_err_t platform_multimodal_audio_begin_downstream_response(void)
esp_err_t platform_multimodal_audio_begin_downstream_response(
uint32_t *out_playback_generation)
{
return multimodal_audio_begin_downstream_response();
return multimodal_audio_begin_downstream_response(out_playback_generation);
}
esp_err_t platform_multimodal_audio_finish_downstream_response(void)
@@ -230,22 +231,22 @@ static void platform_multimodal_downstream_playback_bridge(
switch (event->type) {
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED:
if (callbacks.on_started != NULL) {
callbacks.on_started(callbacks.user_data);
callbacks.on_started(event->playback_generation, callbacks.user_data);
}
break;
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED:
if (callbacks.on_completed != NULL) {
callbacks.on_completed(callbacks.user_data);
callbacks.on_completed(event->playback_generation, callbacks.user_data);
}
break;
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED:
if (callbacks.on_stopped != NULL) {
callbacks.on_stopped(callbacks.user_data);
callbacks.on_stopped(event->playback_generation, callbacks.user_data);
}
break;
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED:
if (callbacks.on_stalled != NULL) {
callbacks.on_stalled(callbacks.user_data);
callbacks.on_stalled(event->playback_generation, callbacks.user_data);
}
break;
default: