fix(multimodal): recover stalled downstream sessions

This commit is contained in:
chenjintao
2026-08-12 16:32:25 +08:00
parent 4e7fadde56
commit a94df9545d
9 changed files with 272 additions and 12 deletions
@@ -56,7 +56,8 @@ typedef enum {
CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_CONTENT, CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_CONTENT,
CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_ENDED, CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_ENDED,
CONTROL_PLANE_MULTIMODAL_EVENT_OUTPUT_ERROR, CONTROL_PLANE_MULTIMODAL_EVENT_OUTPUT_ERROR,
CONTROL_PLANE_MULTIMODAL_EVENT_DOWNSTREAM_AUDIO CONTROL_PLANE_MULTIMODAL_EVENT_DOWNSTREAM_AUDIO,
CONTROL_PLANE_MULTIMODAL_EVENT_DOWNSTREAM_STALLED,
} control_plane_multimodal_event_t; } control_plane_multimodal_event_t;
typedef void (*control_plane_multimodal_event_callback_t)(control_plane_multimodal_event_t event, typedef void (*control_plane_multimodal_event_callback_t)(control_plane_multimodal_event_t event,
@@ -90,6 +91,7 @@ typedef enum {
CONTROL_PLANE_TRANSPORT_LOSS_SEND_BINARY_FAILURE, CONTROL_PLANE_TRANSPORT_LOSS_SEND_BINARY_FAILURE,
CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT, CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT,
CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT, CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT,
CONTROL_PLANE_TRANSPORT_LOSS_DOWNSTREAM_STALL,
CONTROL_PLANE_TRANSPORT_LOSS_PROTOCOL_ERROR, CONTROL_PLANE_TRANSPORT_LOSS_PROTOCOL_ERROR,
} control_plane_transport_loss_reason_t; } control_plane_transport_loss_reason_t;
@@ -98,6 +100,7 @@ void control_plane_multimodal_stop(void);
bool control_plane_multimodal_is_connected(void); bool control_plane_multimodal_is_connected(void);
esp_err_t control_plane_multimodal_send_start(void); esp_err_t control_plane_multimodal_send_start(void);
esp_err_t control_plane_multimodal_send_stop(const char *reason); esp_err_t control_plane_multimodal_send_stop(const char *reason);
esp_err_t control_plane_multimodal_request_downstream_stall_stop(void);
esp_err_t control_plane_multimodal_send_request_to_speak(void); esp_err_t control_plane_multimodal_send_request_to_speak(void);
esp_err_t control_plane_multimodal_send_speech_start(void); esp_err_t control_plane_multimodal_send_speech_start(void);
esp_err_t control_plane_multimodal_send_speech_stop(void); esp_err_t control_plane_multimodal_send_speech_stop(void);
@@ -36,6 +36,7 @@
#endif #endif
#define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220 #define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220
#define CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS 2000
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000 #define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144 #define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus" #define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus"
@@ -57,6 +58,9 @@ static bool s_multimodal_watchdog_task_starting;
static bool s_model_command_dispatched; static bool s_model_command_dispatched;
static bool s_server_success_prompt_played; static bool s_server_success_prompt_played;
static bool s_cloud_chouqian_pending; static bool s_cloud_chouqian_pending;
static bool s_downstream_stall_recovery_pending;
static bool s_downstream_stall_stop_sent;
static uint32_t s_downstream_stall_recovery_started_ms;
static control_plane_model_command_callback_t s_model_command_cb; static control_plane_model_command_callback_t s_model_command_cb;
static void *s_model_command_cb_user_data; static void *s_model_command_cb_user_data;
static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED; static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
@@ -118,6 +122,55 @@ static void set_multimodal_start_flag(bool *flag, bool value)
portEXIT_CRITICAL(&s_multimodal_runtime_lock); portEXIT_CRITICAL(&s_multimodal_runtime_lock);
} }
static bool control_plane_begin_downstream_stall_recovery(void)
{
bool started = false;
portENTER_CRITICAL(&s_multimodal_runtime_lock);
if (!s_downstream_stall_recovery_pending) {
s_downstream_stall_recovery_pending = true;
s_downstream_stall_stop_sent = false;
s_downstream_stall_recovery_started_ms = 0U;
started = true;
}
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
return started;
}
static bool control_plane_get_downstream_stall_recovery(bool *out_stop_sent,
uint32_t *out_started_ms)
{
bool pending = false;
portENTER_CRITICAL(&s_multimodal_runtime_lock);
pending = s_downstream_stall_recovery_pending;
if (out_stop_sent != NULL) {
*out_stop_sent = s_downstream_stall_stop_sent;
}
if (out_started_ms != NULL) {
*out_started_ms = s_downstream_stall_recovery_started_ms;
}
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
return pending;
}
static void control_plane_mark_downstream_stall_stop_sent(uint32_t now_ms)
{
portENTER_CRITICAL(&s_multimodal_runtime_lock);
if (s_downstream_stall_recovery_pending) {
s_downstream_stall_stop_sent = true;
s_downstream_stall_recovery_started_ms = now_ms;
}
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
}
static void control_plane_clear_downstream_stall_recovery(void)
{
portENTER_CRITICAL(&s_multimodal_runtime_lock);
s_downstream_stall_recovery_pending = false;
s_downstream_stall_stop_sent = false;
s_downstream_stall_recovery_started_ms = 0U;
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
}
static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag) static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag)
{ {
bool can_start = false; bool can_start = false;
@@ -454,6 +507,49 @@ static void multimodal_watchdog_task(void *arg)
} }
uint32_t now_ms = control_plane_now_ms(); uint32_t now_ms = control_plane_now_ms();
bool stall_stop_sent = false;
uint32_t stall_recovery_started_ms = 0U;
if (control_plane_get_downstream_stall_recovery(&stall_stop_sent,
&stall_recovery_started_ms)) {
if (!state.connected) {
control_plane_clear_downstream_stall_recovery();
} else if (!stall_stop_sent) {
ESP_LOGW(TAG,
"Downstream stall recovery started: request Stop and wait up to %u ms",
(unsigned)CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS);
esp_err_t stop_err =
control_plane_multimodal_request_downstream_stall_stop();
if (stop_err == ESP_OK) {
control_plane_mark_downstream_stall_stop_sent(now_ms);
} else {
ESP_LOGW(TAG,
"Downstream stall Stop request failed: %s; retire transport now",
esp_err_to_name(stop_err));
control_plane_clear_downstream_stall_recovery();
control_plane_multimodal_mark_transport_unavailable(
CONTROL_PLANE_TRANSPORT_LOSS_DOWNSTREAM_STALL,
"downstream_stall_stop_failed");
(void)control_plane_schedule_websocket_wait_task();
}
continue;
} else if (!state.has_active_task) {
ESP_LOGI(TAG,
"Downstream stall recovered by server Stop acknowledgement; keep WebSocket");
control_plane_clear_downstream_stall_recovery();
} else if ((now_ms - stall_recovery_started_ms) >=
CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS) {
ESP_LOGW(TAG,
"Downstream stall Stop acknowledgement timed out after %u ms; retire transport",
(unsigned)CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS);
control_plane_clear_downstream_stall_recovery();
control_plane_multimodal_mark_transport_unavailable(
CONTROL_PLANE_TRANSPORT_LOSS_DOWNSTREAM_STALL,
"downstream_stall_recovery");
(void)control_plane_schedule_websocket_wait_task();
continue;
}
}
if (state.connected && !state.has_active_task) { if (state.connected && !state.has_active_task) {
if (idle_transport_started_ms == 0U) { if (idle_transport_started_ms == 0U) {
idle_transport_started_ms = now_ms; idle_transport_started_ms = now_ms;
@@ -545,6 +641,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
} }
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED: case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED:
control_plane_clear_downstream_stall_recovery();
control_plane_clear_cloud_chouqian_pending("transport disconnected"); control_plane_clear_cloud_chouqian_pending("transport disconnected");
control_plane_multimodal_stop_uplink(); control_plane_multimodal_stop_uplink();
ESP_LOGI(TAG, "Schedule fresh-auth reconnect after transport disconnect"); ESP_LOGI(TAG, "Schedule fresh-auth reconnect after transport disconnect");
@@ -552,6 +649,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
break; break;
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_ERROR: case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_ERROR:
control_plane_clear_downstream_stall_recovery();
control_plane_clear_cloud_chouqian_pending("transport error"); control_plane_clear_cloud_chouqian_pending("transport error");
control_plane_multimodal_stop_uplink(); control_plane_multimodal_stop_uplink();
ESP_LOGI(TAG, "Schedule fresh-auth reconnect after transport error"); ESP_LOGI(TAG, "Schedule fresh-auth reconnect after transport error");
@@ -559,6 +657,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
break; break;
case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED: case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED:
control_plane_clear_downstream_stall_recovery();
control_plane_clear_cloud_chouqian_pending("task failed"); control_plane_clear_cloud_chouqian_pending("task failed");
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_TASK_FAILED); control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_TASK_FAILED);
control_plane_multimodal_stop_uplink(); control_plane_multimodal_stop_uplink();
@@ -605,6 +704,15 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_OUTPUT_ERROR); control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_OUTPUT_ERROR);
break; break;
case CONTROL_PLANE_MULTIMODAL_EVENT_DOWNSTREAM_STALLED:
if (control_plane_begin_downstream_stall_recovery()) {
ESP_LOGW(TAG,
"Downstream stall recovery queued for watchdog worker");
} else {
ESP_LOGI(TAG, "Downstream stall recovery already pending");
}
break;
default: default:
break; break;
} }
@@ -612,6 +720,8 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
esp_err_t control_plane_start_multimodal_client(void) esp_err_t control_plane_start_multimodal_client(void)
{ {
control_plane_clear_downstream_stall_recovery();
esp_err_t prepare_err = control_plane_prepare_for_multimodal_start(); esp_err_t prepare_err = control_plane_prepare_for_multimodal_start();
if (prepare_err != ESP_OK) { if (prepare_err != ESP_OK) {
ESP_LOGW(TAG, "Prepare multimodal start failed: %s", esp_err_to_name(prepare_err)); ESP_LOGW(TAG, "Prepare multimodal start failed: %s", esp_err_to_name(prepare_err));
@@ -666,6 +776,7 @@ esp_err_t control_plane_start_multimodal_client(void)
void control_plane_stop_multimodal_client(void) void control_plane_stop_multimodal_client(void)
{ {
control_plane_clear_downstream_stall_recovery();
control_plane_clear_cloud_chouqian_pending("multimodal client stopped"); control_plane_clear_cloud_chouqian_pending("multimodal client stopped");
stop_watchdog_task(); stop_watchdog_task();
control_plane_multimodal_stop(); control_plane_multimodal_stop();
@@ -93,6 +93,10 @@
#define CONFIG_TQ_LOCAL_TASK_CORE 1 #define CONFIG_TQ_LOCAL_TASK_CORE 1
#endif #endif
#ifndef CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
#define CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE 0
#endif
#if CONFIG_TQ_LOCAL_TASK_CORE < 0 #if CONFIG_TQ_LOCAL_TASK_CORE < 0
#define CONTROL_PLANE_LOCAL_RESPONDING_TASK_CORE tskNO_AFFINITY #define CONTROL_PLANE_LOCAL_RESPONDING_TASK_CORE tskNO_AFFINITY
#else #else
@@ -122,6 +126,9 @@ static const char *TAG = "cp_mm_session";
static bool s_started; static bool s_started;
static bool s_starting; static bool s_starting;
static bool s_stopping; static bool s_stopping;
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
static uint32_t s_session_generation;
#endif
static control_plane_multimodal_auth_info_t s_auth; static control_plane_multimodal_auth_info_t s_auth;
static char s_task_id[40]; static char s_task_id[40];
@@ -143,6 +150,7 @@ static bool s_ptt_release_required;
static bool s_local_responding_started; static bool s_local_responding_started;
static bool s_local_responding_ended_sent; static bool s_local_responding_ended_sent;
static bool s_downstream_playback_completed; static bool s_downstream_playback_completed;
static bool s_downstream_stall_stop_pending;
static bool s_drop_downstream_audio; static bool s_drop_downstream_audio;
static bool s_received_downstream_audio; static bool s_received_downstream_audio;
static bool s_current_response_received_audio; static bool s_current_response_received_audio;
@@ -381,6 +389,7 @@ static void reset_session_state_locked(void)
s_local_responding_started = false; s_local_responding_started = false;
s_local_responding_ended_sent = false; s_local_responding_ended_sent = false;
s_downstream_playback_completed = false; s_downstream_playback_completed = false;
s_downstream_stall_stop_pending = false;
s_drop_downstream_audio = false; s_drop_downstream_audio = false;
s_received_downstream_audio = false; s_received_downstream_audio = false;
s_current_response_received_audio = false; s_current_response_received_audio = false;
@@ -422,6 +431,8 @@ static const char *transport_loss_reason_name(control_plane_transport_loss_reaso
return "listening_timeout"; return "listening_timeout";
case CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT: case CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT:
return "no_response_timeout"; return "no_response_timeout";
case CONTROL_PLANE_TRANSPORT_LOSS_DOWNSTREAM_STALL:
return "downstream_stall";
case CONTROL_PLANE_TRANSPORT_LOSS_PROTOCOL_ERROR: case CONTROL_PLANE_TRANSPORT_LOSS_PROTOCOL_ERROR:
return "protocol_error"; return "protocol_error";
default: default:
@@ -443,10 +454,23 @@ void control_plane_multimodal_mark_transport_unavailable(
control_plane_transport_loss_reason_t reason, control_plane_transport_loss_reason_t reason,
const char *caller) const char *caller)
{ {
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
uint32_t session_generation = 0U;
if (lock_state()) {
session_generation = s_session_generation;
unlock_state();
}
ESP_LOGW(TAG,
"Transport unavailable: session_generation=%u reason=%s caller=%s",
(unsigned)session_generation,
transport_loss_reason_name(reason),
caller != NULL ? caller : "-");
#else
ESP_LOGW(TAG, ESP_LOGW(TAG,
"Transport unavailable: reason=%s caller=%s", "Transport unavailable: reason=%s caller=%s",
transport_loss_reason_name(reason), transport_loss_reason_name(reason),
caller != NULL ? caller : "-"); caller != NULL ? caller : "-");
#endif
if (lock_state()) { if (lock_state()) {
s_started = false; s_started = false;
@@ -465,7 +489,8 @@ void control_plane_multimodal_mark_transport_unavailable(
reason == CONTROL_PLANE_TRANSPORT_LOSS_WS_DISCONNECTED || reason == CONTROL_PLANE_TRANSPORT_LOSS_WS_DISCONNECTED ||
reason == CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE || reason == CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE ||
reason == CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT || reason == CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT ||
reason == CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT; reason == CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT ||
reason == CONTROL_PLANE_TRANSPORT_LOSS_DOWNSTREAM_STALL;
if (close_transport) { if (close_transport) {
ESP_LOGI(TAG, ESP_LOGI(TAG,
"Retire unavailable transport: reason=%s; next start requires fresh auth", "Retire unavailable transport: reason=%s; next start requires fresh auth",
@@ -1422,18 +1447,27 @@ static void process_text_json(const char *json_text)
notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_ENDED); notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_ENDED);
maybe_send_local_responding_ended_if_ready(); maybe_send_local_responding_ended_if_ready();
} else if (strcmp(output_event->valuestring, "Stopped") == 0) { } else if (strcmp(output_event->valuestring, "Stopped") == 0) {
bool downstream_stall_stop_acknowledged = false;
if (lock_state()) { if (lock_state()) {
s_server_listening = false; downstream_stall_stop_acknowledged = s_downstream_stall_stop_pending;
s_remote_responding = false; if (downstream_stall_stop_acknowledged) {
s_remote_responding_ended = false; reset_session_state_locked();
s_request_to_speak_pending = false; } else {
s_client_speech_active = false; s_server_listening = false;
s_ptt_release_required = false; s_remote_responding = false;
s_local_responding_started = false; s_remote_responding_ended = false;
s_downstream_playback_completed = false; s_request_to_speak_pending = false;
s_drop_downstream_audio = false; s_client_speech_active = false;
s_ptt_release_required = false;
s_local_responding_started = false;
s_downstream_playback_completed = false;
s_drop_downstream_audio = false;
}
unlock_state(); unlock_state();
} }
if (downstream_stall_stop_acknowledged) {
ESP_LOGI(TAG, "Downstream stall Stop acknowledged; transport remains reusable");
}
stop_local_downstream_playback(); stop_local_downstream_playback();
notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_NOT_LISTENING); notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_NOT_LISTENING);
} else if (strcmp(output_event->valuestring, "SpeechStarted") == 0) { } else if (strcmp(output_event->valuestring, "SpeechStarted") == 0) {
@@ -1622,6 +1656,27 @@ static void downstream_playback_event_bridge(
send_local_responding_ended_after_playback_stop(); send_local_responding_ended_after_playback_stop();
break; break;
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STALLED: {
bool cloud_response_active = false;
if (lock_state()) {
cloud_response_active = s_task_id[0] != '\0' &&
(s_remote_responding ||
s_current_response_received_audio);
unlock_state();
}
send_local_responding_ended_after_playback_stop();
if (cloud_response_active) {
ESP_LOGW(TAG,
"Cloud downstream playback stalled; request bounded session recovery");
notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_DOWNSTREAM_STALLED);
} else {
ESP_LOGW(TAG,
"Local downstream playback stalled; keep cloud transport unchanged");
}
break;
}
default: default:
break; break;
} }
@@ -1640,6 +1695,9 @@ esp_err_t control_plane_multimodal_start(void)
return ESP_ERR_NOT_SUPPORTED; return ESP_ERR_NOT_SUPPORTED;
#endif #endif
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
uint32_t session_generation = 0U;
#endif
if (lock_state()) { if (lock_state()) {
bool already_started = s_started || s_auth.token[0] != '\0'; bool already_started = s_started || s_auth.token[0] != '\0';
bool starting = s_starting; bool starting = s_starting;
@@ -1653,9 +1711,42 @@ esp_err_t control_plane_multimodal_start(void)
return ESP_ERR_INVALID_STATE; return ESP_ERR_INVALID_STATE;
} }
s_starting = true; s_starting = true;
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
s_session_generation++;
session_generation = s_session_generation;
#endif
unlock_state(); unlock_state();
} }
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
ESP_LOGI(TAG,
"Multimodal start generation=%u: wait for previous transport cleanup",
(unsigned)session_generation);
#endif
esp_err_t cleanup_err = domain_runtime_multimodal_transport_stop(2000);
if (cleanup_err != ESP_OK) {
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
ESP_LOGW(TAG,
"Multimodal start generation=%u blocked by previous transport cleanup: %s",
(unsigned)session_generation,
esp_err_to_name(cleanup_err));
#else
ESP_LOGW(TAG,
"Multimodal start blocked by previous transport cleanup: %s",
esp_err_to_name(cleanup_err));
#endif
if (lock_state()) {
s_starting = false;
unlock_state();
}
return cleanup_err;
}
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
ESP_LOGI(TAG,
"Multimodal start generation=%u: previous transport cleanup complete; begin auth",
(unsigned)session_generation);
#endif
control_plane_multimodal_auth_info_t auth = {0}; control_plane_multimodal_auth_info_t auth = {0};
esp_err_t err = control_plane_multimodal_auth_fetch(&auth); esp_err_t err = control_plane_multimodal_auth_fetch(&auth);
if (err != ESP_OK) { if (err != ESP_OK) {
@@ -1981,6 +2072,41 @@ esp_err_t control_plane_multimodal_send_stop(const char *reason)
return err; return err;
} }
esp_err_t control_plane_multimodal_request_downstream_stall_stop(void)
{
control_plane_multimodal_stop_uplink();
bool already_pending = false;
bool cloud_response_active = false;
if (!lock_state()) {
return ESP_ERR_TIMEOUT;
}
already_pending = s_downstream_stall_stop_pending;
cloud_response_active = s_task_id[0] != '\0' &&
(s_remote_responding || s_current_response_received_audio);
if (!already_pending && cloud_response_active) {
s_downstream_stall_stop_pending = true;
s_request_to_speak_pending = false;
s_drop_downstream_audio = true;
}
unlock_state();
if (already_pending) {
return ESP_OK;
}
if (!cloud_response_active) {
return ESP_ERR_INVALID_STATE;
}
ESP_LOGW(TAG, "Request Stop for stalled downstream response");
esp_err_t err = send_directive("finish-task", "Stop", true, false);
if (err != ESP_OK && lock_state()) {
s_downstream_stall_stop_pending = false;
unlock_state();
}
return err;
}
esp_err_t control_plane_multimodal_send_request_to_speak(void) esp_err_t control_plane_multimodal_send_request_to_speak(void)
{ {
if (!transport_available()) { if (!transport_available()) {
+1
View File
@@ -98,6 +98,7 @@ typedef enum {
DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED = 0, DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED = 0,
DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED, DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED,
DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED, DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED,
DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STALLED,
} domain_multimodal_downstream_playback_event_type_t; } domain_multimodal_downstream_playback_event_type_t;
typedef struct { typedef struct {
@@ -452,6 +452,12 @@ static void domain_downstream_playback_stopped_adapter(void *user_data)
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED); domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED);
} }
static void domain_downstream_playback_stalled_adapter(void *user_data)
{
(void)user_data;
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STALLED);
}
static void domain_wifi_set_keep_awake_adapter(bool keep_awake, void *user_data) static void domain_wifi_set_keep_awake_adapter(bool keep_awake, void *user_data)
{ {
(void)user_data; (void)user_data;
@@ -688,6 +694,7 @@ esp_err_t domain_runtime_multimodal_audio_register_downstream_playback_callback(
.on_started = domain_downstream_playback_started_adapter, .on_started = domain_downstream_playback_started_adapter,
.on_completed = domain_downstream_playback_completed_adapter, .on_completed = domain_downstream_playback_completed_adapter,
.on_stopped = domain_downstream_playback_stopped_adapter, .on_stopped = domain_downstream_playback_stopped_adapter,
.on_stalled = domain_downstream_playback_stalled_adapter,
.user_data = NULL, .user_data = NULL,
}; };
+1
View File
@@ -46,6 +46,7 @@ typedef struct {
void (*on_started)(void *user_data); void (*on_started)(void *user_data);
void (*on_completed)(void *user_data); void (*on_completed)(void *user_data);
void (*on_stopped)(void *user_data); void (*on_stopped)(void *user_data);
void (*on_stalled)(void *user_data);
void *user_data; void *user_data;
} platform_multimodal_downstream_playback_callbacks_t; } platform_multimodal_downstream_playback_callbacks_t;
@@ -20,6 +20,7 @@ typedef enum {
MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED = 0, MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED = 0,
MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED, MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED,
MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED, MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED,
MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED,
} multimodal_audio_downstream_playback_event_type_t; } multimodal_audio_downstream_playback_event_type_t;
typedef struct { typedef struct {
@@ -1774,6 +1774,11 @@ static void notify_downstream_playback_stopped(void)
notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED); notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED);
} }
static void notify_downstream_playback_stalled(void)
{
notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED);
}
static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t *ctx, static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t *ctx,
uint32_t idle_ms) uint32_t idle_ms)
{ {
@@ -1781,7 +1786,7 @@ static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t
"Downstream protection watchdog: abort stalled response idle_ms=%u", "Downstream protection watchdog: abort stalled response idle_ms=%u",
(unsigned)idle_ms); (unsigned)idle_ms);
(void)advance_downstream_playback_generation(); (void)advance_downstream_playback_generation();
notify_downstream_playback_stopped(); notify_downstream_playback_stalled();
update_downstream_response_state(false, false); update_downstream_response_state(false, false);
if (audio_get_downstream_playback_task(ctx) != NULL || if (audio_get_downstream_playback_task(ctx) != NULL ||
audio_get_downstream_playback_starting(ctx)) { audio_get_downstream_playback_starting(ctx)) {
@@ -243,6 +243,11 @@ static void platform_multimodal_downstream_playback_bridge(
callbacks.on_stopped(callbacks.user_data); callbacks.on_stopped(callbacks.user_data);
} }
break; break;
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED:
if (callbacks.on_stalled != NULL) {
callbacks.on_stalled(callbacks.user_data);
}
break;
default: default:
break; break;
} }