fix(multimodal): harden playback and client lifecycle handling
- retry the server connection prompt after transient playback failures - serialize multimodal client and session start-stop operations - tag downstream responses to ignore stale packets and playback events - complete bounded teardown and clean callbacks after partial startup
This commit is contained in:
@@ -154,8 +154,9 @@ esp_err_t control_plane_start_multimodal_client(void);
|
|||||||
* Semantics:
|
* Semantics:
|
||||||
* - Idempotent and safe to call repeatedly.
|
* - Idempotent and safe to call repeatedly.
|
||||||
* - Requests cooperative stop first, then applies bounded waits for worker teardown.
|
* - Requests cooperative stop first, then applies bounded waits for worker teardown.
|
||||||
|
* - Returns an explicit error if teardown does not complete.
|
||||||
*/
|
*/
|
||||||
void control_plane_stop_multimodal_client(void);
|
esp_err_t control_plane_stop_multimodal_client(void);
|
||||||
|
|
||||||
esp_err_t control_plane_register_model_command_callback(control_plane_model_command_callback_t cb,
|
esp_err_t control_plane_register_model_command_callback(control_plane_model_command_callback_t cb,
|
||||||
void *user_data);
|
void *user_data);
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ typedef enum {
|
|||||||
} control_plane_transport_loss_reason_t;
|
} control_plane_transport_loss_reason_t;
|
||||||
|
|
||||||
esp_err_t control_plane_multimodal_start(void);
|
esp_err_t control_plane_multimodal_start(void);
|
||||||
void control_plane_multimodal_stop(void);
|
esp_err_t control_plane_multimodal_stop(void);
|
||||||
bool control_plane_multimodal_is_connected(void);
|
bool control_plane_multimodal_is_connected(void);
|
||||||
esp_err_t control_plane_multimodal_send_start(void);
|
esp_err_t control_plane_multimodal_send_start(void);
|
||||||
esp_err_t control_plane_multimodal_send_stop(const char *reason);
|
esp_err_t control_plane_multimodal_send_stop(const char *reason);
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#include "freertos/FreeRTOS.h"
|
#include "freertos/FreeRTOS.h"
|
||||||
|
#include "freertos/semphr.h"
|
||||||
#include "freertos/task.h"
|
#include "freertos/task.h"
|
||||||
#include "esp_log.h"
|
#include "esp_log.h"
|
||||||
#include "sdkconfig.h"
|
#include "sdkconfig.h"
|
||||||
@@ -40,6 +41,9 @@
|
|||||||
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
|
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
|
||||||
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144
|
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144
|
||||||
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus"
|
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus"
|
||||||
|
#define CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS 30000U
|
||||||
|
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS 1000U
|
||||||
|
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS 3U
|
||||||
|
|
||||||
#ifndef CONFIG_TQ_NETWORK_TASK_CORE
|
#ifndef CONFIG_TQ_NETWORK_TASK_CORE
|
||||||
#define CONFIG_TQ_NETWORK_TASK_CORE 0
|
#define CONFIG_TQ_NETWORK_TASK_CORE 0
|
||||||
@@ -57,6 +61,10 @@ static bool s_multimodal_watchdog_stop_requested;
|
|||||||
static bool s_multimodal_watchdog_task_starting;
|
static bool s_multimodal_watchdog_task_starting;
|
||||||
static bool s_model_command_dispatched;
|
static bool s_model_command_dispatched;
|
||||||
static bool s_server_success_prompt_played;
|
static bool s_server_success_prompt_played;
|
||||||
|
static bool s_server_success_prompt_pending;
|
||||||
|
static bool s_server_success_prompt_in_progress;
|
||||||
|
static uint8_t s_server_success_prompt_attempts;
|
||||||
|
static uint32_t s_server_success_prompt_next_attempt_ms;
|
||||||
static bool s_cloud_chouqian_pending;
|
static bool s_cloud_chouqian_pending;
|
||||||
static bool s_downstream_stall_recovery_pending;
|
static bool s_downstream_stall_recovery_pending;
|
||||||
static bool s_downstream_stall_stop_sent;
|
static bool s_downstream_stall_stop_sent;
|
||||||
@@ -64,6 +72,45 @@ static uint32_t s_downstream_stall_recovery_started_ms;
|
|||||||
static control_plane_model_command_callback_t s_model_command_cb;
|
static control_plane_model_command_callback_t s_model_command_cb;
|
||||||
static void *s_model_command_cb_user_data;
|
static void *s_model_command_cb_user_data;
|
||||||
static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||||
|
static SemaphoreHandle_t s_client_lifecycle_mutex;
|
||||||
|
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
||||||
|
static StaticSemaphore_t s_client_lifecycle_mutex_buf;
|
||||||
|
#endif
|
||||||
|
static portMUX_TYPE s_client_lifecycle_init_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||||
|
|
||||||
|
static bool lock_client_lifecycle(void)
|
||||||
|
{
|
||||||
|
portENTER_CRITICAL(&s_client_lifecycle_init_lock);
|
||||||
|
if (s_client_lifecycle_mutex == NULL) {
|
||||||
|
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
||||||
|
s_client_lifecycle_mutex =
|
||||||
|
xSemaphoreCreateMutexStatic(&s_client_lifecycle_mutex_buf);
|
||||||
|
#else
|
||||||
|
s_client_lifecycle_mutex = xSemaphoreCreateMutex();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
SemaphoreHandle_t mutex = s_client_lifecycle_mutex;
|
||||||
|
portEXIT_CRITICAL(&s_client_lifecycle_init_lock);
|
||||||
|
if (mutex == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS);
|
||||||
|
if (timeout_ticks == 0) {
|
||||||
|
timeout_ticks = 1;
|
||||||
|
}
|
||||||
|
if (xSemaphoreTake(mutex, timeout_ticks) != pdTRUE) {
|
||||||
|
ESP_LOGW(TAG, "Timeout acquiring multimodal client lifecycle lock");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void unlock_client_lifecycle(void)
|
||||||
|
{
|
||||||
|
if (s_client_lifecycle_mutex != NULL) {
|
||||||
|
(void)xSemaphoreGive(s_client_lifecycle_mutex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot)
|
static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot)
|
||||||
{
|
{
|
||||||
@@ -171,6 +218,74 @@ static void control_plane_clear_downstream_stall_recovery(void)
|
|||||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void control_plane_schedule_server_success_prompt(void)
|
||||||
|
{
|
||||||
|
uint32_t now_ms = (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS);
|
||||||
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||||
|
if (!s_server_success_prompt_played &&
|
||||||
|
s_server_success_prompt_attempts <
|
||||||
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) {
|
||||||
|
s_server_success_prompt_pending = true;
|
||||||
|
s_server_success_prompt_next_attempt_ms = now_ms;
|
||||||
|
}
|
||||||
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void control_plane_try_server_success_prompt(
|
||||||
|
const control_plane_multimodal_runtime_state_t *state,
|
||||||
|
uint32_t now_ms)
|
||||||
|
{
|
||||||
|
if (state == NULL || !state->connected || state->has_active_task) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool should_attempt = false;
|
||||||
|
uint8_t attempt = 0U;
|
||||||
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||||
|
if (s_server_success_prompt_pending &&
|
||||||
|
!s_server_success_prompt_played &&
|
||||||
|
!s_server_success_prompt_in_progress &&
|
||||||
|
s_server_success_prompt_attempts <
|
||||||
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS &&
|
||||||
|
(int32_t)(now_ms - s_server_success_prompt_next_attempt_ms) >= 0) {
|
||||||
|
s_server_success_prompt_in_progress = true;
|
||||||
|
s_server_success_prompt_attempts++;
|
||||||
|
attempt = s_server_success_prompt_attempts;
|
||||||
|
should_attempt = true;
|
||||||
|
}
|
||||||
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||||
|
if (!should_attempt) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
esp_err_t err = domain_runtime_behavior_play_prompt_file(
|
||||||
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE);
|
||||||
|
|
||||||
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||||
|
s_server_success_prompt_in_progress = false;
|
||||||
|
if (err == ESP_OK) {
|
||||||
|
s_server_success_prompt_played = true;
|
||||||
|
s_server_success_prompt_pending = false;
|
||||||
|
} else if (s_server_success_prompt_attempts >=
|
||||||
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) {
|
||||||
|
s_server_success_prompt_pending = false;
|
||||||
|
} else {
|
||||||
|
s_server_success_prompt_next_attempt_ms =
|
||||||
|
now_ms + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS;
|
||||||
|
}
|
||||||
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||||
|
|
||||||
|
if (err == ESP_OK) {
|
||||||
|
ESP_LOGI(TAG, "Server success prompt played on attempt %u", (unsigned)attempt);
|
||||||
|
} else {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Server success prompt attempt %u/%u failed: %s",
|
||||||
|
(unsigned)attempt,
|
||||||
|
(unsigned)CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS,
|
||||||
|
esp_err_to_name(err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag)
|
static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag)
|
||||||
{
|
{
|
||||||
bool can_start = false;
|
bool can_start = false;
|
||||||
@@ -507,6 +622,7 @@ static void multimodal_watchdog_task(void *arg)
|
|||||||
}
|
}
|
||||||
|
|
||||||
uint32_t now_ms = control_plane_now_ms();
|
uint32_t now_ms = control_plane_now_ms();
|
||||||
|
control_plane_try_server_success_prompt(&state, now_ms);
|
||||||
bool stall_stop_sent = false;
|
bool stall_stop_sent = false;
|
||||||
uint32_t stall_recovery_started_ms = 0U;
|
uint32_t stall_recovery_started_ms = 0U;
|
||||||
if (control_plane_get_downstream_stall_recovery(&stall_stop_sent,
|
if (control_plane_get_downstream_stall_recovery(&stall_stop_sent,
|
||||||
@@ -559,7 +675,7 @@ static void multimodal_watchdog_task(void *arg)
|
|||||||
"Idle multimodal transport timeout, stopping client after %u ms",
|
"Idle multimodal transport timeout, stopping client after %u ms",
|
||||||
(unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS);
|
(unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS);
|
||||||
control_plane_multimodal_stop_uplink();
|
control_plane_multimodal_stop_uplink();
|
||||||
control_plane_multimodal_stop();
|
(void)control_plane_multimodal_stop();
|
||||||
idle_transport_started_ms = 0U;
|
idle_transport_started_ms = 0U;
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
@@ -627,16 +743,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
|
|||||||
|
|
||||||
switch (event) {
|
switch (event) {
|
||||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED: {
|
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED: {
|
||||||
bool should_play_prompt = false;
|
control_plane_schedule_server_success_prompt();
|
||||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
||||||
if (!s_server_success_prompt_played) {
|
|
||||||
s_server_success_prompt_played = true;
|
|
||||||
should_play_prompt = true;
|
|
||||||
}
|
|
||||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
||||||
if (should_play_prompt) {
|
|
||||||
(void)domain_runtime_behavior_play_prompt_file(CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE);
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -718,7 +825,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t control_plane_start_multimodal_client(void)
|
static esp_err_t control_plane_start_multimodal_client_locked(void)
|
||||||
{
|
{
|
||||||
control_plane_clear_downstream_stall_recovery();
|
control_plane_clear_downstream_stall_recovery();
|
||||||
|
|
||||||
@@ -768,18 +875,35 @@ esp_err_t control_plane_start_multimodal_client(void)
|
|||||||
esp_err_t err = control_plane_multimodal_start();
|
esp_err_t err = control_plane_multimodal_start();
|
||||||
if (err != ESP_OK) {
|
if (err != ESP_OK) {
|
||||||
stop_watchdog_task();
|
stop_watchdog_task();
|
||||||
|
(void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL);
|
||||||
|
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
return ESP_OK;
|
return ESP_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
void control_plane_stop_multimodal_client(void)
|
esp_err_t control_plane_start_multimodal_client(void)
|
||||||
{
|
{
|
||||||
|
if (!lock_client_lifecycle()) {
|
||||||
|
return ESP_ERR_TIMEOUT;
|
||||||
|
}
|
||||||
|
esp_err_t err = control_plane_start_multimodal_client_locked();
|
||||||
|
unlock_client_lifecycle();
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
esp_err_t control_plane_stop_multimodal_client(void)
|
||||||
|
{
|
||||||
|
if (!lock_client_lifecycle()) {
|
||||||
|
return ESP_ERR_TIMEOUT;
|
||||||
|
}
|
||||||
control_plane_clear_downstream_stall_recovery();
|
control_plane_clear_downstream_stall_recovery();
|
||||||
control_plane_clear_cloud_chouqian_pending("multimodal client stopped");
|
control_plane_clear_cloud_chouqian_pending("multimodal client stopped");
|
||||||
stop_watchdog_task();
|
stop_watchdog_task();
|
||||||
control_plane_multimodal_stop();
|
esp_err_t err = control_plane_multimodal_stop();
|
||||||
|
unlock_client_lifecycle();
|
||||||
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t control_plane_register_model_command_callback(
|
esp_err_t control_plane_register_model_command_callback(
|
||||||
|
|||||||
+135
-35
@@ -88,6 +88,7 @@
|
|||||||
#define CONTROL_PLANE_UPLINK_READY_WAIT_MS 6000U
|
#define CONTROL_PLANE_UPLINK_READY_WAIT_MS 6000U
|
||||||
#define CONTROL_PLANE_AUDIO_SEND_FAILURE_WINDOW_MS 100U
|
#define CONTROL_PLANE_AUDIO_SEND_FAILURE_WINDOW_MS 100U
|
||||||
#define CONTROL_PLANE_AUDIO_SEND_FAILURE_THRESHOLD 2U
|
#define CONTROL_PLANE_AUDIO_SEND_FAILURE_THRESHOLD 2U
|
||||||
|
#define CONTROL_PLANE_MULTIMODAL_LIFECYCLE_WAIT_MS 30000U
|
||||||
|
|
||||||
#ifndef CONFIG_TQ_LOCAL_TASK_CORE
|
#ifndef CONFIG_TQ_LOCAL_TASK_CORE
|
||||||
#define CONFIG_TQ_LOCAL_TASK_CORE 1
|
#define CONFIG_TQ_LOCAL_TASK_CORE 1
|
||||||
@@ -154,6 +155,7 @@ static bool s_downstream_stall_stop_pending;
|
|||||||
static bool s_drop_downstream_audio;
|
static bool s_drop_downstream_audio;
|
||||||
static bool s_received_downstream_audio;
|
static bool s_received_downstream_audio;
|
||||||
static bool s_current_response_received_audio;
|
static bool s_current_response_received_audio;
|
||||||
|
static uint32_t s_current_playback_generation;
|
||||||
static uint32_t s_audio_send_failure_count;
|
static uint32_t s_audio_send_failure_count;
|
||||||
static TickType_t s_audio_send_failure_window_start_tick;
|
static TickType_t s_audio_send_failure_window_start_tick;
|
||||||
static size_t s_last_asr_text_len;
|
static size_t s_last_asr_text_len;
|
||||||
@@ -171,9 +173,12 @@ static control_plane_multimodal_downstream_audio_callback_t s_downstream_audio_c
|
|||||||
static void *s_downstream_audio_cb_user_data;
|
static void *s_downstream_audio_cb_user_data;
|
||||||
|
|
||||||
static SemaphoreHandle_t s_state_mutex;
|
static SemaphoreHandle_t s_state_mutex;
|
||||||
|
static SemaphoreHandle_t s_lifecycle_mutex;
|
||||||
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
||||||
static StaticSemaphore_t s_state_mutex_buf;
|
static StaticSemaphore_t s_state_mutex_buf;
|
||||||
|
static StaticSemaphore_t s_lifecycle_mutex_buf;
|
||||||
#endif
|
#endif
|
||||||
|
static portMUX_TYPE s_mutex_init_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
LOCAL_RESPONDING_EVENT_STARTED = 0,
|
LOCAL_RESPONDING_EVENT_STARTED = 0,
|
||||||
@@ -299,23 +304,60 @@ static void log_downstream_binary_frame(size_t len, bool drop_audio, bool has_ca
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool ensure_session_mutexes(void)
|
||||||
|
{
|
||||||
|
portENTER_CRITICAL(&s_mutex_init_lock);
|
||||||
|
if (s_state_mutex == NULL) {
|
||||||
|
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
||||||
|
s_state_mutex = xSemaphoreCreateMutexStatic(&s_state_mutex_buf);
|
||||||
|
#else
|
||||||
|
s_state_mutex = xSemaphoreCreateMutex();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
if (s_lifecycle_mutex == NULL) {
|
||||||
|
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
||||||
|
s_lifecycle_mutex = xSemaphoreCreateMutexStatic(&s_lifecycle_mutex_buf);
|
||||||
|
#else
|
||||||
|
s_lifecycle_mutex = xSemaphoreCreateMutex();
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
bool ready = s_state_mutex != NULL && s_lifecycle_mutex != NULL;
|
||||||
|
portEXIT_CRITICAL(&s_mutex_init_lock);
|
||||||
|
return ready;
|
||||||
|
}
|
||||||
|
|
||||||
static bool ensure_state_mutex(void)
|
static bool ensure_state_mutex(void)
|
||||||
{
|
{
|
||||||
if (s_state_mutex != NULL) {
|
if (!ensure_session_mutexes()) {
|
||||||
return true;
|
ESP_LOGE(TAG, "Failed to create multimodal lifecycle mutexes");
|
||||||
}
|
|
||||||
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
|
||||||
s_state_mutex = xSemaphoreCreateMutexStatic(&s_state_mutex_buf);
|
|
||||||
#else
|
|
||||||
s_state_mutex = xSemaphoreCreateMutex();
|
|
||||||
#endif
|
|
||||||
if (s_state_mutex == NULL) {
|
|
||||||
ESP_LOGE(TAG, "Failed to create multimodal state mutex");
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool lock_lifecycle(void)
|
||||||
|
{
|
||||||
|
if (!ensure_session_mutexes()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_MULTIMODAL_LIFECYCLE_WAIT_MS);
|
||||||
|
if (timeout_ticks == 0) {
|
||||||
|
timeout_ticks = 1;
|
||||||
|
}
|
||||||
|
if (xSemaphoreTake(s_lifecycle_mutex, timeout_ticks) != pdTRUE) {
|
||||||
|
ESP_LOGW(TAG, "Timeout acquiring multimodal lifecycle lock");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void unlock_lifecycle(void)
|
||||||
|
{
|
||||||
|
if (s_lifecycle_mutex != NULL) {
|
||||||
|
(void)xSemaphoreGive(s_lifecycle_mutex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static bool lock_state(void)
|
static bool lock_state(void)
|
||||||
{
|
{
|
||||||
if (!ensure_state_mutex()) {
|
if (!ensure_state_mutex()) {
|
||||||
@@ -402,6 +444,7 @@ static void reset_session_state_locked(void)
|
|||||||
s_drop_downstream_audio = false;
|
s_drop_downstream_audio = false;
|
||||||
s_received_downstream_audio = false;
|
s_received_downstream_audio = false;
|
||||||
s_current_response_received_audio = false;
|
s_current_response_received_audio = false;
|
||||||
|
s_current_playback_generation = 0U;
|
||||||
s_audio_send_failure_count = 0;
|
s_audio_send_failure_count = 0;
|
||||||
s_audio_send_failure_window_start_tick = 0;
|
s_audio_send_failure_window_start_tick = 0;
|
||||||
s_local_responding_started_send_pending = false;
|
s_local_responding_started_send_pending = false;
|
||||||
@@ -1403,7 +1446,13 @@ static void process_text_json(const char *json_text)
|
|||||||
unlock_state();
|
unlock_state();
|
||||||
}
|
}
|
||||||
ESP_LOGI(TAG, "TTS responding started");
|
ESP_LOGI(TAG, "TTS responding started");
|
||||||
esp_err_t begin_err = domain_runtime_multimodal_audio_begin_downstream_response();
|
uint32_t playback_generation = 0U;
|
||||||
|
esp_err_t begin_err = domain_runtime_multimodal_audio_begin_downstream_response(
|
||||||
|
&playback_generation);
|
||||||
|
if (begin_err == ESP_OK && lock_state()) {
|
||||||
|
s_current_playback_generation = playback_generation;
|
||||||
|
unlock_state();
|
||||||
|
}
|
||||||
if (begin_err != ESP_OK && begin_err != ESP_ERR_INVALID_STATE) {
|
if (begin_err != ESP_OK && begin_err != ESP_ERR_INVALID_STATE) {
|
||||||
ESP_LOGW(TAG,
|
ESP_LOGW(TAG,
|
||||||
"Begin downstream response failed: %s",
|
"Begin downstream response failed: %s",
|
||||||
@@ -1636,6 +1685,21 @@ static void downstream_playback_event_bridge(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool generation_matches = false;
|
||||||
|
if (lock_state()) {
|
||||||
|
generation_matches = s_current_playback_generation != 0U &&
|
||||||
|
s_current_playback_generation ==
|
||||||
|
event->playback_generation;
|
||||||
|
unlock_state();
|
||||||
|
}
|
||||||
|
if (!generation_matches) {
|
||||||
|
ESP_LOGD(TAG,
|
||||||
|
"Ignore stale or local playback event type=%d generation=%u",
|
||||||
|
(int)event->type,
|
||||||
|
(unsigned)event->playback_generation);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
switch (event->type) {
|
switch (event->type) {
|
||||||
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED: {
|
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED: {
|
||||||
bool should_send_started = false;
|
bool should_send_started = false;
|
||||||
@@ -1656,12 +1720,17 @@ static void downstream_playback_event_bridge(
|
|||||||
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED:
|
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED:
|
||||||
if (lock_state()) {
|
if (lock_state()) {
|
||||||
s_downstream_playback_completed = true;
|
s_downstream_playback_completed = true;
|
||||||
|
s_current_playback_generation = 0U;
|
||||||
unlock_state();
|
unlock_state();
|
||||||
}
|
}
|
||||||
maybe_send_local_responding_ended_if_ready();
|
maybe_send_local_responding_ended_if_ready();
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED:
|
case DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED:
|
||||||
|
if (lock_state()) {
|
||||||
|
s_current_playback_generation = 0U;
|
||||||
|
unlock_state();
|
||||||
|
}
|
||||||
send_local_responding_ended_after_playback_stop();
|
send_local_responding_ended_after_playback_stop();
|
||||||
break;
|
break;
|
||||||
|
|
||||||
@@ -1671,6 +1740,7 @@ static void downstream_playback_event_bridge(
|
|||||||
cloud_response_active = s_task_id[0] != '\0' &&
|
cloud_response_active = s_task_id[0] != '\0' &&
|
||||||
(s_remote_responding ||
|
(s_remote_responding ||
|
||||||
s_current_response_received_audio);
|
s_current_response_received_audio);
|
||||||
|
s_current_playback_generation = 0U;
|
||||||
unlock_state();
|
unlock_state();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1697,7 +1767,20 @@ static esp_err_t control_plane_audio_uplink_send_cb(const uint8_t *data, size_t
|
|||||||
return control_plane_multimodal_send_audio_realtime(data, len);
|
return control_plane_multimodal_send_audio_realtime(data, len);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static esp_err_t control_plane_multimodal_start_locked(void);
|
||||||
|
static esp_err_t control_plane_multimodal_stop_locked(void);
|
||||||
|
|
||||||
esp_err_t control_plane_multimodal_start(void)
|
esp_err_t control_plane_multimodal_start(void)
|
||||||
|
{
|
||||||
|
if (!lock_lifecycle()) {
|
||||||
|
return ESP_ERR_TIMEOUT;
|
||||||
|
}
|
||||||
|
esp_err_t err = control_plane_multimodal_start_locked();
|
||||||
|
unlock_lifecycle();
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
static esp_err_t control_plane_multimodal_start_locked(void)
|
||||||
{
|
{
|
||||||
#if !CONFIG_ENABLE_MULTIMODAL_INTERACTION
|
#if !CONFIG_ENABLE_MULTIMODAL_INTERACTION
|
||||||
ESP_LOGW(TAG, "Multimodal interaction is disabled by config");
|
ESP_LOGW(TAG, "Multimodal interaction is disabled by config");
|
||||||
@@ -1707,25 +1790,26 @@ esp_err_t control_plane_multimodal_start(void)
|
|||||||
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
||||||
uint32_t session_generation = 0U;
|
uint32_t session_generation = 0U;
|
||||||
#endif
|
#endif
|
||||||
if (lock_state()) {
|
if (!lock_state()) {
|
||||||
bool already_started = s_started || s_auth.token[0] != '\0';
|
return ESP_ERR_TIMEOUT;
|
||||||
bool starting = s_starting;
|
|
||||||
bool stopping = s_stopping;
|
|
||||||
if (already_started) {
|
|
||||||
unlock_state();
|
|
||||||
return ESP_OK;
|
|
||||||
}
|
|
||||||
if (starting || stopping) {
|
|
||||||
unlock_state();
|
|
||||||
return ESP_ERR_INVALID_STATE;
|
|
||||||
}
|
|
||||||
s_starting = true;
|
|
||||||
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
|
||||||
s_session_generation++;
|
|
||||||
session_generation = s_session_generation;
|
|
||||||
#endif
|
|
||||||
unlock_state();
|
|
||||||
}
|
}
|
||||||
|
bool already_started = s_started || s_auth.token[0] != '\0';
|
||||||
|
bool starting = s_starting;
|
||||||
|
bool stopping = s_stopping;
|
||||||
|
if (already_started) {
|
||||||
|
unlock_state();
|
||||||
|
return ESP_OK;
|
||||||
|
}
|
||||||
|
if (starting || stopping) {
|
||||||
|
unlock_state();
|
||||||
|
return ESP_ERR_INVALID_STATE;
|
||||||
|
}
|
||||||
|
s_starting = true;
|
||||||
|
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
||||||
|
s_session_generation++;
|
||||||
|
session_generation = s_session_generation;
|
||||||
|
#endif
|
||||||
|
unlock_state();
|
||||||
|
|
||||||
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
||||||
ESP_LOGI(TAG,
|
ESP_LOGI(TAG,
|
||||||
@@ -1804,14 +1888,28 @@ esp_err_t control_plane_multimodal_start(void)
|
|||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (lock_state()) {
|
if (!lock_state()) {
|
||||||
s_auth = auth;
|
(void)domain_runtime_multimodal_transport_stop(2000);
|
||||||
unlock_state();
|
(void)domain_runtime_multimodal_audio_register_downstream_playback_callback(NULL, NULL);
|
||||||
|
stop_local_responding_event_worker();
|
||||||
|
return ESP_ERR_TIMEOUT;
|
||||||
}
|
}
|
||||||
|
s_auth = auth;
|
||||||
|
unlock_state();
|
||||||
return ESP_OK;
|
return ESP_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
void control_plane_multimodal_stop(void)
|
esp_err_t control_plane_multimodal_stop(void)
|
||||||
|
{
|
||||||
|
if (!lock_lifecycle()) {
|
||||||
|
return ESP_ERR_TIMEOUT;
|
||||||
|
}
|
||||||
|
esp_err_t err = control_plane_multimodal_stop_locked();
|
||||||
|
unlock_lifecycle();
|
||||||
|
return err;
|
||||||
|
}
|
||||||
|
|
||||||
|
static esp_err_t control_plane_multimodal_stop_locked(void)
|
||||||
{
|
{
|
||||||
bool already_stopping = false;
|
bool already_stopping = false;
|
||||||
if (lock_state()) {
|
if (lock_state()) {
|
||||||
@@ -1823,8 +1921,7 @@ void control_plane_multimodal_stop(void)
|
|||||||
unlock_state();
|
unlock_state();
|
||||||
}
|
}
|
||||||
if (already_stopping) {
|
if (already_stopping) {
|
||||||
ESP_LOGI(TAG, "Multimodal stop already in progress");
|
ESP_LOGI(TAG, "Completing previously started multimodal teardown");
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
control_plane_multimodal_stop_uplink();
|
control_plane_multimodal_stop_uplink();
|
||||||
@@ -1846,7 +1943,10 @@ void control_plane_multimodal_stop(void)
|
|||||||
memset(&s_auth, 0, sizeof(s_auth));
|
memset(&s_auth, 0, sizeof(s_auth));
|
||||||
reset_session_state_locked();
|
reset_session_state_locked();
|
||||||
unlock_state();
|
unlock_state();
|
||||||
|
} else if (transport_stop_err == ESP_OK) {
|
||||||
|
transport_stop_err = ESP_ERR_TIMEOUT;
|
||||||
}
|
}
|
||||||
|
return transport_stop_err;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool control_plane_multimodal_is_connected(void)
|
bool control_plane_multimodal_is_connected(void)
|
||||||
|
|||||||
@@ -1777,7 +1777,7 @@ static void control_plane_wifi_state_task(void *arg)
|
|||||||
|
|
||||||
if (!disconnected_expression_active) {
|
if (!disconnected_expression_active) {
|
||||||
ESP_LOGI(TAG, "Wi-Fi STA disconnect persisted; stopping multimodal client");
|
ESP_LOGI(TAG, "Wi-Fi STA disconnect persisted; stopping multimodal client");
|
||||||
control_plane_stop_multimodal_client();
|
(void)control_plane_stop_multimodal_client();
|
||||||
control_plane_stop_user_config_server();
|
control_plane_stop_user_config_server();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2455,7 +2455,7 @@ void control_plane_stop_runtime_workers(void)
|
|||||||
portEXIT_CRITICAL(&s_runtime_lock);
|
portEXIT_CRITICAL(&s_runtime_lock);
|
||||||
control_plane_delete_model_cmd_queue(queue, queue_with_caps);
|
control_plane_delete_model_cmd_queue(queue, queue_with_caps);
|
||||||
|
|
||||||
control_plane_stop_multimodal_client();
|
(void)control_plane_stop_multimodal_client();
|
||||||
control_plane_set_runtime_stop_requested(false);
|
control_plane_set_runtime_stop_requested(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -103,6 +103,7 @@ typedef enum {
|
|||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
domain_multimodal_downstream_playback_event_type_t type;
|
domain_multimodal_downstream_playback_event_type_t type;
|
||||||
|
uint32_t playback_generation;
|
||||||
} domain_multimodal_downstream_playback_event_t;
|
} domain_multimodal_downstream_playback_event_t;
|
||||||
|
|
||||||
typedef void (*domain_multimodal_downstream_playback_callback_t)(
|
typedef void (*domain_multimodal_downstream_playback_callback_t)(
|
||||||
@@ -228,7 +229,8 @@ esp_err_t domain_runtime_local_command_stop(int timeout_ms);
|
|||||||
esp_err_t domain_runtime_local_command_get_snapshot(
|
esp_err_t domain_runtime_local_command_get_snapshot(
|
||||||
domain_local_command_snapshot_t *out_snapshot);
|
domain_local_command_snapshot_t *out_snapshot);
|
||||||
esp_err_t domain_runtime_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
|
esp_err_t domain_runtime_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
|
||||||
esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(void);
|
esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(
|
||||||
|
uint32_t *out_playback_generation);
|
||||||
esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void);
|
esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void);
|
||||||
esp_err_t domain_runtime_multimodal_audio_stop_downstream_playback(void);
|
esp_err_t domain_runtime_multimodal_audio_stop_downstream_playback(void);
|
||||||
esp_err_t domain_runtime_multimodal_audio_register_downstream_playback_callback(
|
esp_err_t domain_runtime_multimodal_audio_register_downstream_playback_callback(
|
||||||
|
|||||||
@@ -435,7 +435,9 @@ static const char *domain_wifi_prompt_file_path(domain_wifi_prompt_id_t prompt)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void domain_downstream_playback_dispatch(domain_multimodal_downstream_playback_event_type_t type)
|
static void domain_downstream_playback_dispatch(
|
||||||
|
domain_multimodal_downstream_playback_event_type_t type,
|
||||||
|
uint32_t playback_generation)
|
||||||
{
|
{
|
||||||
domain_multimodal_downstream_playback_callback_t cb = NULL;
|
domain_multimodal_downstream_playback_callback_t cb = NULL;
|
||||||
void *cb_user_data = NULL;
|
void *cb_user_data = NULL;
|
||||||
@@ -446,32 +448,41 @@ static void domain_downstream_playback_dispatch(domain_multimodal_downstream_pla
|
|||||||
|
|
||||||
const domain_multimodal_downstream_playback_event_t event = {
|
const domain_multimodal_downstream_playback_event_t event = {
|
||||||
.type = type,
|
.type = type,
|
||||||
|
.playback_generation = playback_generation,
|
||||||
};
|
};
|
||||||
cb(&event, cb_user_data);
|
cb(&event, cb_user_data);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void domain_downstream_playback_started_adapter(void *user_data)
|
static void domain_downstream_playback_started_adapter(uint32_t playback_generation,
|
||||||
|
void *user_data)
|
||||||
{
|
{
|
||||||
(void)user_data;
|
(void)user_data;
|
||||||
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED);
|
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED,
|
||||||
|
playback_generation);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void domain_downstream_playback_completed_adapter(void *user_data)
|
static void domain_downstream_playback_completed_adapter(uint32_t playback_generation,
|
||||||
|
void *user_data)
|
||||||
{
|
{
|
||||||
(void)user_data;
|
(void)user_data;
|
||||||
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED);
|
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED,
|
||||||
|
playback_generation);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void domain_downstream_playback_stopped_adapter(void *user_data)
|
static void domain_downstream_playback_stopped_adapter(uint32_t playback_generation,
|
||||||
|
void *user_data)
|
||||||
{
|
{
|
||||||
(void)user_data;
|
(void)user_data;
|
||||||
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED);
|
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STOPPED,
|
||||||
|
playback_generation);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void domain_downstream_playback_stalled_adapter(void *user_data)
|
static void domain_downstream_playback_stalled_adapter(uint32_t playback_generation,
|
||||||
|
void *user_data)
|
||||||
{
|
{
|
||||||
(void)user_data;
|
(void)user_data;
|
||||||
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STALLED);
|
domain_downstream_playback_dispatch(DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STALLED,
|
||||||
|
playback_generation);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void domain_wifi_set_keep_awake_adapter(bool keep_awake, void *user_data)
|
static void domain_wifi_set_keep_awake_adapter(bool keep_awake, void *user_data)
|
||||||
@@ -687,9 +698,10 @@ esp_err_t domain_runtime_multimodal_audio_play_downstream_opus(const uint8_t *da
|
|||||||
return platform_multimodal_audio_play_downstream_opus(data, len);
|
return platform_multimodal_audio_play_downstream_opus(data, len);
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(void)
|
esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(
|
||||||
|
uint32_t *out_playback_generation)
|
||||||
{
|
{
|
||||||
return platform_multimodal_audio_begin_downstream_response();
|
return platform_multimodal_audio_begin_downstream_response(out_playback_generation);
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void)
|
esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void)
|
||||||
|
|||||||
@@ -43,10 +43,10 @@ typedef struct {
|
|||||||
} platform_multimodal_transport_callbacks_t;
|
} platform_multimodal_transport_callbacks_t;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
void (*on_started)(void *user_data);
|
void (*on_started)(uint32_t playback_generation, void *user_data);
|
||||||
void (*on_completed)(void *user_data);
|
void (*on_completed)(uint32_t playback_generation, void *user_data);
|
||||||
void (*on_stopped)(void *user_data);
|
void (*on_stopped)(uint32_t playback_generation, void *user_data);
|
||||||
void (*on_stalled)(void *user_data);
|
void (*on_stalled)(uint32_t playback_generation, void *user_data);
|
||||||
void *user_data;
|
void *user_data;
|
||||||
} platform_multimodal_downstream_playback_callbacks_t;
|
} platform_multimodal_downstream_playback_callbacks_t;
|
||||||
|
|
||||||
@@ -399,7 +399,7 @@ esp_err_t platform_multimodal_audio_unregister_pcm_observer(
|
|||||||
void *user_data,
|
void *user_data,
|
||||||
int timeout_ms);
|
int timeout_ms);
|
||||||
esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
|
esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
|
||||||
esp_err_t platform_multimodal_audio_begin_downstream_response(void);
|
esp_err_t platform_multimodal_audio_begin_downstream_response(uint32_t *out_playback_generation);
|
||||||
esp_err_t platform_multimodal_audio_finish_downstream_response(void);
|
esp_err_t platform_multimodal_audio_finish_downstream_response(void);
|
||||||
esp_err_t platform_multimodal_audio_stop_downstream_playback(void);
|
esp_err_t platform_multimodal_audio_stop_downstream_playback(void);
|
||||||
esp_err_t platform_multimodal_audio_register_downstream_playback_callbacks(
|
esp_err_t platform_multimodal_audio_register_downstream_playback_callbacks(
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ typedef enum {
|
|||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
multimodal_audio_downstream_playback_event_type_t type;
|
multimodal_audio_downstream_playback_event_type_t type;
|
||||||
|
uint32_t playback_generation;
|
||||||
} multimodal_audio_downstream_playback_event_t;
|
} multimodal_audio_downstream_playback_event_t;
|
||||||
|
|
||||||
typedef void (*multimodal_audio_downstream_playback_callback_t)(
|
typedef void (*multimodal_audio_downstream_playback_callback_t)(
|
||||||
@@ -47,7 +48,7 @@ esp_err_t multimodal_audio_unregister_pcm_observer(
|
|||||||
int timeout_ms);
|
int timeout_ms);
|
||||||
|
|
||||||
esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t len);
|
esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t len);
|
||||||
esp_err_t multimodal_audio_begin_downstream_response(void);
|
esp_err_t multimodal_audio_begin_downstream_response(uint32_t *out_playback_generation);
|
||||||
esp_err_t multimodal_audio_finish_downstream_response(void);
|
esp_err_t multimodal_audio_finish_downstream_response(void);
|
||||||
esp_err_t multimodal_audio_stop_downstream_playback(void);
|
esp_err_t multimodal_audio_stop_downstream_playback(void);
|
||||||
esp_err_t multimodal_audio_register_downstream_playback_callback(
|
esp_err_t multimodal_audio_register_downstream_playback_callback(
|
||||||
|
|||||||
@@ -461,16 +461,6 @@ static void drain_downstream_playback_queue(multimodal_audio_ctx_t *ctx)
|
|||||||
unlock_downstream_playback_queue();
|
unlock_downstream_playback_queue();
|
||||||
}
|
}
|
||||||
|
|
||||||
static void drain_downstream_playback_queue_now(multimodal_audio_ctx_t *ctx)
|
|
||||||
{
|
|
||||||
if (ctx == NULL || !lock_downstream_playback_queue(0)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
QueueHandle_t queue = ctx->downstream_playback_queue;
|
|
||||||
drain_downstream_playback_queue_handle(queue);
|
|
||||||
unlock_downstream_playback_queue();
|
|
||||||
}
|
|
||||||
|
|
||||||
static void track_uplink_packet_queued(size_t len)
|
static void track_uplink_packet_queued(size_t len)
|
||||||
{
|
{
|
||||||
#if CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
|
#if CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
|
||||||
@@ -581,10 +571,11 @@ static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ct
|
|||||||
static esp_err_t ensure_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx);
|
static esp_err_t ensure_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx);
|
||||||
static void notify_downstream_audio_active(multimodal_audio_ctx_t *ctx);
|
static void notify_downstream_audio_active(multimodal_audio_ctx_t *ctx);
|
||||||
static void notify_downstream_playback_started(uint32_t expected_generation);
|
static void notify_downstream_playback_started(uint32_t expected_generation);
|
||||||
static void notify_downstream_playback_completed(void);
|
static void notify_downstream_playback_completed(uint32_t expected_generation);
|
||||||
static void notify_downstream_playback_stopped(void);
|
|
||||||
static esp_err_t prime_downstream_output_path(multimodal_audio_ctx_t *ctx);
|
static esp_err_t prime_downstream_output_path(multimodal_audio_ctx_t *ctx);
|
||||||
static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len);
|
static esp_err_t handle_downstream_opus_sync(const uint8_t *data,
|
||||||
|
size_t len,
|
||||||
|
uint32_t expected_generation);
|
||||||
static esp_err_t ensure_downstream_playback_task(multimodal_audio_ctx_t *ctx);
|
static esp_err_t ensure_downstream_playback_task(multimodal_audio_ctx_t *ctx);
|
||||||
static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx);
|
static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx);
|
||||||
static esp_err_t stop_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx, int timeout_ms);
|
static esp_err_t stop_downstream_idle_mute_task(multimodal_audio_ctx_t *ctx, int timeout_ms);
|
||||||
@@ -1563,20 +1554,44 @@ static void snapshot_downstream_playback_state(uint32_t *out_generation,
|
|||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
static uint32_t advance_downstream_playback_generation(void)
|
static uint32_t advance_downstream_playback_generation_locked(void)
|
||||||
{
|
{
|
||||||
uint32_t generation = 0U;
|
|
||||||
|
|
||||||
portENTER_CRITICAL(&s_audio_runtime_lock);
|
|
||||||
s_audio.downstream_playback_generation++;
|
s_audio.downstream_playback_generation++;
|
||||||
if (s_audio.downstream_playback_generation == 0U) {
|
if (s_audio.downstream_playback_generation == 0U) {
|
||||||
s_audio.downstream_playback_generation = 1U;
|
s_audio.downstream_playback_generation = 1U;
|
||||||
}
|
}
|
||||||
s_audio.downstream_decoder_reset_pending = true;
|
s_audio.downstream_decoder_reset_pending = true;
|
||||||
generation = s_audio.downstream_playback_generation;
|
return s_audio.downstream_playback_generation;
|
||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
}
|
||||||
|
|
||||||
return generation;
|
static bool claim_downstream_response_terminal(uint32_t expected_generation,
|
||||||
|
bool invalidate_generation,
|
||||||
|
uint32_t *out_terminal_generation)
|
||||||
|
{
|
||||||
|
if (out_terminal_generation == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
portENTER_CRITICAL(&s_audio_runtime_lock);
|
||||||
|
bool generation_matches = expected_generation == 0U ||
|
||||||
|
s_audio.downstream_playback_generation ==
|
||||||
|
expected_generation;
|
||||||
|
bool should_notify = generation_matches &&
|
||||||
|
(s_audio.downstream_response_active ||
|
||||||
|
s_audio.downstream_response_end_received ||
|
||||||
|
s_audio.downstream_playback_in_progress);
|
||||||
|
*out_terminal_generation = s_audio.downstream_playback_generation;
|
||||||
|
if (generation_matches && invalidate_generation &&
|
||||||
|
(should_notify || expected_generation == 0U)) {
|
||||||
|
(void)advance_downstream_playback_generation_locked();
|
||||||
|
}
|
||||||
|
if (should_notify) {
|
||||||
|
s_audio.downstream_response_active = false;
|
||||||
|
s_audio.downstream_response_end_received = false;
|
||||||
|
s_audio.downstream_playback_in_progress = false;
|
||||||
|
}
|
||||||
|
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||||
|
return should_notify;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool clear_downstream_decoder_reset_pending(uint32_t expected_generation)
|
static bool clear_downstream_decoder_reset_pending(uint32_t expected_generation)
|
||||||
@@ -1628,7 +1643,8 @@ static void snapshot_downstream_playback_callback(
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void dispatch_downstream_playback_event(
|
static void dispatch_downstream_playback_event(
|
||||||
multimodal_audio_downstream_playback_event_type_t type)
|
multimodal_audio_downstream_playback_event_type_t type,
|
||||||
|
uint32_t playback_generation)
|
||||||
{
|
{
|
||||||
multimodal_audio_downstream_playback_callback_t cb = NULL;
|
multimodal_audio_downstream_playback_callback_t cb = NULL;
|
||||||
void *cb_ctx = NULL;
|
void *cb_ctx = NULL;
|
||||||
@@ -1639,6 +1655,7 @@ static void dispatch_downstream_playback_event(
|
|||||||
|
|
||||||
multimodal_audio_downstream_playback_event_t event = {
|
multimodal_audio_downstream_playback_event_t event = {
|
||||||
.type = type,
|
.type = type,
|
||||||
|
.playback_generation = playback_generation,
|
||||||
};
|
};
|
||||||
cb(&event, cb_ctx);
|
cb(&event, cb_ctx);
|
||||||
}
|
}
|
||||||
@@ -1656,51 +1673,32 @@ static void notify_downstream_playback_started(uint32_t expected_generation)
|
|||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||||
|
|
||||||
if (should_notify) {
|
if (should_notify) {
|
||||||
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED);
|
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED,
|
||||||
|
expected_generation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void notify_downstream_playback_finished(
|
static void notify_downstream_playback_completed(uint32_t expected_generation)
|
||||||
multimodal_audio_downstream_playback_event_type_t type)
|
|
||||||
{
|
{
|
||||||
bool should_notify = false;
|
uint32_t playback_generation = 0U;
|
||||||
|
if (claim_downstream_response_terminal(expected_generation,
|
||||||
portENTER_CRITICAL(&s_audio_runtime_lock);
|
false,
|
||||||
if (s_audio.downstream_playback_in_progress) {
|
&playback_generation)) {
|
||||||
s_audio.downstream_playback_in_progress = false;
|
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED,
|
||||||
should_notify = true;
|
playback_generation);
|
||||||
}
|
|
||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
|
||||||
|
|
||||||
if (should_notify) {
|
|
||||||
dispatch_downstream_playback_event(type);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void notify_downstream_playback_completed(void)
|
static bool snapshot_downstream_response_end_received(
|
||||||
{
|
uint32_t *out_playback_generation)
|
||||||
bool should_notify = false;
|
|
||||||
|
|
||||||
portENTER_CRITICAL(&s_audio_runtime_lock);
|
|
||||||
if (s_audio.downstream_playback_in_progress) {
|
|
||||||
s_audio.downstream_playback_in_progress = false;
|
|
||||||
should_notify = true;
|
|
||||||
} else if (s_audio.downstream_response_end_received) {
|
|
||||||
should_notify = true;
|
|
||||||
}
|
|
||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
|
||||||
|
|
||||||
if (should_notify) {
|
|
||||||
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool snapshot_downstream_response_end_received(void)
|
|
||||||
{
|
{
|
||||||
bool end_received = false;
|
bool end_received = false;
|
||||||
|
|
||||||
portENTER_CRITICAL(&s_audio_runtime_lock);
|
portENTER_CRITICAL(&s_audio_runtime_lock);
|
||||||
end_received = s_audio.downstream_response_end_received;
|
end_received = s_audio.downstream_response_end_received;
|
||||||
|
if (out_playback_generation != NULL) {
|
||||||
|
*out_playback_generation = s_audio.downstream_playback_generation;
|
||||||
|
}
|
||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||||
|
|
||||||
return end_received;
|
return end_received;
|
||||||
@@ -1717,27 +1715,24 @@ static bool snapshot_downstream_response_active(void)
|
|||||||
return active;
|
return active;
|
||||||
}
|
}
|
||||||
|
|
||||||
static TickType_t snapshot_downstream_last_audio_tick(const multimodal_audio_ctx_t *ctx)
|
static bool snapshot_downstream_response_watchdog(
|
||||||
|
const multimodal_audio_ctx_t *ctx,
|
||||||
|
uint32_t *out_playback_generation,
|
||||||
|
TickType_t *out_last_audio_tick,
|
||||||
|
TickType_t *out_last_response_activity_tick)
|
||||||
{
|
{
|
||||||
TickType_t tick = 0;
|
if (ctx == NULL || out_playback_generation == NULL ||
|
||||||
portENTER_CRITICAL(&s_audio_runtime_lock);
|
out_last_audio_tick == NULL || out_last_response_activity_tick == NULL) {
|
||||||
if (ctx != NULL) {
|
return false;
|
||||||
tick = ctx->downstream_last_audio_tick;
|
|
||||||
}
|
}
|
||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
|
||||||
return tick;
|
|
||||||
}
|
|
||||||
|
|
||||||
static TickType_t snapshot_downstream_last_response_activity_tick(
|
|
||||||
const multimodal_audio_ctx_t *ctx)
|
|
||||||
{
|
|
||||||
TickType_t tick = 0;
|
|
||||||
portENTER_CRITICAL(&s_audio_runtime_lock);
|
portENTER_CRITICAL(&s_audio_runtime_lock);
|
||||||
if (ctx != NULL) {
|
bool active = s_audio.downstream_response_active;
|
||||||
tick = ctx->downstream_last_response_activity_tick;
|
*out_playback_generation = s_audio.downstream_playback_generation;
|
||||||
}
|
*out_last_audio_tick = ctx->downstream_last_audio_tick;
|
||||||
|
*out_last_response_activity_tick = ctx->downstream_last_response_activity_tick;
|
||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||||
return tick;
|
return active;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void update_downstream_last_audio_tick(multimodal_audio_ctx_t *ctx,
|
static void update_downstream_last_audio_tick(multimodal_audio_ctx_t *ctx,
|
||||||
@@ -1769,37 +1764,28 @@ static void update_downstream_response_state(bool active, bool end_received)
|
|||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void notify_downstream_playback_stopped(void)
|
|
||||||
{
|
|
||||||
notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void notify_downstream_playback_stalled(void)
|
|
||||||
{
|
|
||||||
notify_downstream_playback_finished(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t *ctx,
|
static void abort_downstream_response_after_active_stall(multimodal_audio_ctx_t *ctx,
|
||||||
uint32_t idle_ms)
|
uint32_t idle_ms,
|
||||||
|
uint32_t expected_generation)
|
||||||
{
|
{
|
||||||
|
(void)ctx;
|
||||||
|
uint32_t stopped_generation = 0U;
|
||||||
|
if (!claim_downstream_response_terminal(expected_generation,
|
||||||
|
true,
|
||||||
|
&stopped_generation)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
ESP_LOGW(TAG,
|
ESP_LOGW(TAG,
|
||||||
"Downstream protection watchdog: abort stalled response idle_ms=%u",
|
"Downstream protection watchdog: abort stalled response idle_ms=%u",
|
||||||
(unsigned)idle_ms);
|
(unsigned)idle_ms);
|
||||||
(void)advance_downstream_playback_generation();
|
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED,
|
||||||
notify_downstream_playback_stalled();
|
stopped_generation);
|
||||||
update_downstream_response_state(false, false);
|
|
||||||
if (audio_get_downstream_playback_task(ctx) != NULL ||
|
|
||||||
audio_get_downstream_playback_starting(ctx)) {
|
|
||||||
audio_set_downstream_playback_stopping(ctx, true);
|
|
||||||
audio_set_downstream_playback_keep_stopping_on_exit(ctx, false);
|
|
||||||
audio_set_downstream_playback_stop_requested(ctx, true);
|
|
||||||
}
|
|
||||||
drain_downstream_playback_queue_now(ctx);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void maybe_notify_downstream_playback_completed_after_drain(const char *reason)
|
static void maybe_notify_downstream_playback_completed_after_drain(const char *reason)
|
||||||
{
|
{
|
||||||
if (!snapshot_downstream_response_end_received()) {
|
uint32_t playback_generation = 0U;
|
||||||
|
if (!snapshot_downstream_response_end_received(&playback_generation)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1820,8 +1806,7 @@ static void maybe_notify_downstream_playback_completed_after_drain(const char *r
|
|||||||
"Downstream playback completed after response end (%s)",
|
"Downstream playback completed after response end (%s)",
|
||||||
reason ? reason : "drained");
|
reason ? reason : "drained");
|
||||||
log_downstream_diagnostics("playback_complete", true);
|
log_downstream_diagnostics("playback_complete", true);
|
||||||
notify_downstream_playback_completed();
|
notify_downstream_playback_completed(playback_generation);
|
||||||
update_downstream_response_state(false, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ctx,
|
static esp_err_t set_downstream_output_mute_if_needed(multimodal_audio_ctx_t *ctx,
|
||||||
@@ -1933,11 +1918,17 @@ static void downstream_idle_mute_timer_cb(void *arg)
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const bool response_active = snapshot_downstream_response_active();
|
uint32_t playback_generation = 0U;
|
||||||
|
TickType_t last_audio_tick = 0;
|
||||||
|
TickType_t last_response_activity_tick = 0;
|
||||||
|
const bool response_active = snapshot_downstream_response_watchdog(
|
||||||
|
ctx,
|
||||||
|
&playback_generation,
|
||||||
|
&last_audio_tick,
|
||||||
|
&last_response_activity_tick);
|
||||||
if (response_active) {
|
if (response_active) {
|
||||||
log_downstream_active_diagnostics();
|
log_downstream_active_diagnostics();
|
||||||
TickType_t now = xTaskGetTickCount();
|
TickType_t now = xTaskGetTickCount();
|
||||||
TickType_t last_audio_tick = snapshot_downstream_last_audio_tick(ctx);
|
|
||||||
uint32_t idle_ms = last_audio_tick != 0U ?
|
uint32_t idle_ms = last_audio_tick != 0U ?
|
||||||
(uint32_t)pdTICKS_TO_MS(now - last_audio_tick) : 0U;
|
(uint32_t)pdTICKS_TO_MS(now - last_audio_tick) : 0U;
|
||||||
uint32_t idle_timeout_ms = get_downstream_idle_mute_timeout_ms();
|
uint32_t idle_timeout_ms = get_downstream_idle_mute_timeout_ms();
|
||||||
@@ -1960,14 +1951,15 @@ static void downstream_idle_mute_timer_cb(void *arg)
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
TickType_t last_activity_tick = snapshot_downstream_last_response_activity_tick(ctx);
|
uint32_t response_idle_ms = last_response_activity_tick != 0U ?
|
||||||
uint32_t response_idle_ms = last_activity_tick != 0U ?
|
(uint32_t)pdTICKS_TO_MS(now - last_response_activity_tick) : idle_ms;
|
||||||
(uint32_t)pdTICKS_TO_MS(now - last_activity_tick) : idle_ms;
|
|
||||||
if (response_idle_ms < stall_timeout_ms) {
|
if (response_idle_ms < stall_timeout_ms) {
|
||||||
restart_downstream_idle_mute_timer(ctx, stall_timeout_ms - response_idle_ms);
|
restart_downstream_idle_mute_timer(ctx, stall_timeout_ms - response_idle_ms);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
abort_downstream_response_after_active_stall(ctx, idle_ms);
|
abort_downstream_response_after_active_stall(ctx,
|
||||||
|
idle_ms,
|
||||||
|
playback_generation);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2221,7 +2213,7 @@ static void downstream_playback_task(void *arg)
|
|||||||
if (generation_started && queued_packets > 0U && first_recovery_packet_tick == 0U) {
|
if (generation_started && queued_packets > 0U && first_recovery_packet_tick == 0U) {
|
||||||
first_recovery_packet_tick = now;
|
first_recovery_packet_tick = now;
|
||||||
}
|
}
|
||||||
bool response_ended = snapshot_downstream_response_end_received();
|
bool response_ended = snapshot_downstream_response_end_received(NULL);
|
||||||
if (queued_packets >= target_packets || (response_ended && queued_packets > 0U)) {
|
if (queued_packets >= target_packets || (response_ended && queued_packets > 0U)) {
|
||||||
playback_buffer_ready = true;
|
playback_buffer_ready = true;
|
||||||
uint32_t buffered_ms = (uint32_t)queued_packets * ctx->opus_frame_duration_ms;
|
uint32_t buffered_ms = (uint32_t)queued_packets * ctx->opus_frame_duration_ms;
|
||||||
@@ -2288,7 +2280,7 @@ static void downstream_playback_task(void *arg)
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (received != pdTRUE || packet.data == NULL || packet.len == 0U) {
|
if (received != pdTRUE || packet.data == NULL || packet.len == 0U) {
|
||||||
if (snapshot_downstream_response_end_received()) {
|
if (snapshot_downstream_response_end_received(NULL)) {
|
||||||
maybe_notify_downstream_playback_completed_after_drain("queue_empty");
|
maybe_notify_downstream_playback_completed_after_drain("queue_empty");
|
||||||
} else if (playback_buffer_ready && generation_started &&
|
} else if (playback_buffer_ready && generation_started &&
|
||||||
snapshot_downstream_response_active()) {
|
snapshot_downstream_response_active()) {
|
||||||
@@ -2338,7 +2330,9 @@ static void downstream_playback_task(void *arg)
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!audio_get_downstream_playback_stop_requested(ctx)) {
|
if (!audio_get_downstream_playback_stop_requested(ctx)) {
|
||||||
esp_err_t err = handle_downstream_opus_sync(packet.data, packet.len);
|
esp_err_t err = handle_downstream_opus_sync(packet.data,
|
||||||
|
packet.len,
|
||||||
|
packet.playback_generation);
|
||||||
if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
|
if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
|
||||||
note_downstream_playback_failure();
|
note_downstream_playback_failure();
|
||||||
ESP_LOGW(TAG, "Downstream playback packet failed: %s", esp_err_to_name(err));
|
ESP_LOGW(TAG, "Downstream playback packet failed: %s", esp_err_to_name(err));
|
||||||
@@ -2586,9 +2580,11 @@ esp_err_t multimodal_audio_stop_downstream_playback(void)
|
|||||||
{
|
{
|
||||||
multimodal_audio_ctx_t *ctx = &s_audio;
|
multimodal_audio_ctx_t *ctx = &s_audio;
|
||||||
|
|
||||||
update_downstream_response_state(false, false);
|
uint32_t stopped_generation = 0U;
|
||||||
(void)advance_downstream_playback_generation();
|
if (claim_downstream_response_terminal(0U, true, &stopped_generation)) {
|
||||||
notify_downstream_playback_stopped();
|
dispatch_downstream_playback_event(MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED,
|
||||||
|
stopped_generation);
|
||||||
|
}
|
||||||
|
|
||||||
if (audio_get_deinit_in_progress(ctx)) {
|
if (audio_get_deinit_in_progress(ctx)) {
|
||||||
esp_err_t task_stop_err = stop_downstream_playback_task(ctx, 1000, true);
|
esp_err_t task_stop_err = stop_downstream_playback_task(ctx, 1000, true);
|
||||||
@@ -2608,7 +2604,9 @@ esp_err_t multimodal_audio_stop_downstream_playback(void)
|
|||||||
return set_downstream_output_mute_if_needed(ctx, true, "Mute");
|
return set_downstream_output_mute_if_needed(ctx, true, "Mute");
|
||||||
}
|
}
|
||||||
|
|
||||||
static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len)
|
static esp_err_t handle_downstream_opus_sync(const uint8_t *data,
|
||||||
|
size_t len,
|
||||||
|
uint32_t expected_generation)
|
||||||
{
|
{
|
||||||
multimodal_audio_ctx_t *ctx = &s_audio;
|
multimodal_audio_ctx_t *ctx = &s_audio;
|
||||||
if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_dec) {
|
if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_dec) {
|
||||||
@@ -2635,6 +2633,9 @@ static esp_err_t handle_downstream_opus_sync(const uint8_t *data, size_t len)
|
|||||||
uint32_t playback_generation = 0U;
|
uint32_t playback_generation = 0U;
|
||||||
bool decoder_reset_pending = false;
|
bool decoder_reset_pending = false;
|
||||||
snapshot_downstream_playback_state(&playback_generation, &decoder_reset_pending);
|
snapshot_downstream_playback_state(&playback_generation, &decoder_reset_pending);
|
||||||
|
if (playback_generation != expected_generation) {
|
||||||
|
return ESP_ERR_INVALID_STATE;
|
||||||
|
}
|
||||||
if (decoder_reset_pending) {
|
if (decoder_reset_pending) {
|
||||||
esp_err_t reset_err = reset_downstream_decoder(ctx);
|
esp_err_t reset_err = reset_downstream_decoder(ctx);
|
||||||
if (reset_err != ESP_OK) {
|
if (reset_err != ESP_OK) {
|
||||||
@@ -2929,17 +2930,21 @@ esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t le
|
|||||||
return ESP_OK;
|
return ESP_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t multimodal_audio_begin_downstream_response(void)
|
esp_err_t multimodal_audio_begin_downstream_response(
|
||||||
|
uint32_t *out_playback_generation)
|
||||||
{
|
{
|
||||||
multimodal_audio_ctx_t *ctx = &s_audio;
|
multimodal_audio_ctx_t *ctx = &s_audio;
|
||||||
|
if (out_playback_generation == NULL) {
|
||||||
|
return ESP_ERR_INVALID_ARG;
|
||||||
|
}
|
||||||
|
*out_playback_generation = 0U;
|
||||||
if (!ctx->initialized || ctx->codec_dev == NULL || ctx->opus_dec == NULL) {
|
if (!ctx->initialized || ctx->codec_dev == NULL || ctx->opus_dec == NULL) {
|
||||||
return ESP_ERR_INVALID_STATE;
|
return ESP_ERR_INVALID_STATE;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint32_t playback_generation = advance_downstream_playback_generation();
|
|
||||||
|
|
||||||
TickType_t begin_tick = xTaskGetTickCount();
|
TickType_t begin_tick = xTaskGetTickCount();
|
||||||
portENTER_CRITICAL(&s_audio_runtime_lock);
|
portENTER_CRITICAL(&s_audio_runtime_lock);
|
||||||
|
uint32_t playback_generation = advance_downstream_playback_generation_locked();
|
||||||
ctx->downstream_response_active = true;
|
ctx->downstream_response_active = true;
|
||||||
ctx->downstream_response_end_received = false;
|
ctx->downstream_response_end_received = false;
|
||||||
ctx->downstream_output_primed = false;
|
ctx->downstream_output_primed = false;
|
||||||
@@ -2947,6 +2952,7 @@ esp_err_t multimodal_audio_begin_downstream_response(void)
|
|||||||
ctx->downstream_last_audio_tick = begin_tick;
|
ctx->downstream_last_audio_tick = begin_tick;
|
||||||
ctx->downstream_last_response_activity_tick = begin_tick;
|
ctx->downstream_last_response_activity_tick = begin_tick;
|
||||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||||
|
*out_playback_generation = playback_generation;
|
||||||
|
|
||||||
s_downstream_queued_packet_total = 0U;
|
s_downstream_queued_packet_total = 0U;
|
||||||
s_downstream_played_packet_total = 0U;
|
s_downstream_played_packet_total = 0U;
|
||||||
|
|||||||
@@ -235,7 +235,8 @@ esp_err_t voice_prompt_play_file(const char *path)
|
|||||||
return ESP_ERR_NOT_FOUND;
|
return ESP_ERR_NOT_FOUND;
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t err = multimodal_audio_begin_downstream_response();
|
uint32_t playback_generation = 0U;
|
||||||
|
esp_err_t err = multimodal_audio_begin_downstream_response(&playback_generation);
|
||||||
if (err != ESP_OK) {
|
if (err != ESP_OK) {
|
||||||
ESP_LOGW(VOICE_TAG, "Begin prompt playback failed: %s", esp_err_to_name(err));
|
ESP_LOGW(VOICE_TAG, "Begin prompt playback failed: %s", esp_err_to_name(err));
|
||||||
fclose(file);
|
fclose(file);
|
||||||
|
|||||||
@@ -200,9 +200,10 @@ esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, si
|
|||||||
return multimodal_audio_handle_downstream_opus(data, len);
|
return multimodal_audio_handle_downstream_opus(data, len);
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t platform_multimodal_audio_begin_downstream_response(void)
|
esp_err_t platform_multimodal_audio_begin_downstream_response(
|
||||||
|
uint32_t *out_playback_generation)
|
||||||
{
|
{
|
||||||
return multimodal_audio_begin_downstream_response();
|
return multimodal_audio_begin_downstream_response(out_playback_generation);
|
||||||
}
|
}
|
||||||
|
|
||||||
esp_err_t platform_multimodal_audio_finish_downstream_response(void)
|
esp_err_t platform_multimodal_audio_finish_downstream_response(void)
|
||||||
@@ -230,22 +231,22 @@ static void platform_multimodal_downstream_playback_bridge(
|
|||||||
switch (event->type) {
|
switch (event->type) {
|
||||||
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED:
|
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED:
|
||||||
if (callbacks.on_started != NULL) {
|
if (callbacks.on_started != NULL) {
|
||||||
callbacks.on_started(callbacks.user_data);
|
callbacks.on_started(event->playback_generation, callbacks.user_data);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED:
|
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_COMPLETED:
|
||||||
if (callbacks.on_completed != NULL) {
|
if (callbacks.on_completed != NULL) {
|
||||||
callbacks.on_completed(callbacks.user_data);
|
callbacks.on_completed(event->playback_generation, callbacks.user_data);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED:
|
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STOPPED:
|
||||||
if (callbacks.on_stopped != NULL) {
|
if (callbacks.on_stopped != NULL) {
|
||||||
callbacks.on_stopped(callbacks.user_data);
|
callbacks.on_stopped(event->playback_generation, callbacks.user_data);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED:
|
case MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STALLED:
|
||||||
if (callbacks.on_stalled != NULL) {
|
if (callbacks.on_stalled != NULL) {
|
||||||
callbacks.on_stalled(callbacks.user_data);
|
callbacks.on_stalled(event->playback_generation, callbacks.user_data);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
|
|||||||
Reference in New Issue
Block a user