1050 lines
36 KiB
C
1050 lines
36 KiB
C
/*
|
|
* SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
#include "control_plane.h"
|
|
|
|
#include "control_plane_internal.h"
|
|
|
|
#include <string.h>
|
|
|
|
#include "freertos/FreeRTOS.h"
|
|
#include "freertos/semphr.h"
|
|
#include "freertos/task.h"
|
|
#include "esp_log.h"
|
|
#include "sdkconfig.h"
|
|
|
|
#ifndef CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS
|
|
#define CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS 50000
|
|
#endif
|
|
|
|
#ifndef CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS
|
|
#define CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS 60000
|
|
#endif
|
|
|
|
#ifndef CONFIG_MULTIMODAL_RECONNECT_DELAY_MS
|
|
#define CONFIG_MULTIMODAL_RECONNECT_DELAY_MS 2000
|
|
#endif
|
|
|
|
#ifndef CONFIG_MULTIMODAL_UPSTREAM_MODE
|
|
#define CONFIG_MULTIMODAL_UPSTREAM_MODE "push2talk"
|
|
#endif
|
|
|
|
#ifndef CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS
|
|
#define CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS 1
|
|
#endif
|
|
|
|
#define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220
|
|
#define CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS 2000
|
|
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
|
|
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 8192
|
|
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus"
|
|
#define CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS 30000U
|
|
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS 1000U
|
|
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS 3U
|
|
#define CONTROL_PLANE_CLOUD_CHOUQIAN_PREPARE_TIMEOUT_MS 10000U
|
|
#define CONTROL_PLANE_CLOUD_CHOUQIAN_RESPONSE_TIMEOUT_MS 60000U
|
|
|
|
#ifndef CONFIG_TQ_NETWORK_TASK_CORE
|
|
#define CONFIG_TQ_NETWORK_TASK_CORE 0
|
|
#endif
|
|
|
|
#if CONFIG_TQ_NETWORK_TASK_CORE < 0
|
|
#define CONTROL_PLANE_MULTIMODAL_NETWORK_TASK_CORE tskNO_AFFINITY
|
|
#else
|
|
#define CONTROL_PLANE_MULTIMODAL_NETWORK_TASK_CORE ((BaseType_t)CONFIG_TQ_NETWORK_TASK_CORE)
|
|
#endif
|
|
|
|
static const char *TAG = "control_plane_mm_rt";
|
|
static TaskHandle_t s_multimodal_watchdog_task;
|
|
static bool s_multimodal_watchdog_stop_requested;
|
|
static bool s_multimodal_watchdog_task_starting;
|
|
static bool s_model_command_dispatched;
|
|
static bool s_server_success_prompt_played;
|
|
static bool s_server_success_prompt_pending;
|
|
static bool s_server_success_prompt_in_progress;
|
|
static uint8_t s_server_success_prompt_attempts;
|
|
static uint32_t s_server_success_prompt_next_attempt_ms;
|
|
static bool s_cloud_chouqian_pending;
|
|
static bool s_cloud_chouqian_action_in_progress;
|
|
static bool s_cloud_chouqian_prompt_sent;
|
|
static uint32_t s_cloud_chouqian_requested_ms;
|
|
static bool s_downstream_stall_recovery_pending;
|
|
static bool s_downstream_stall_stop_sent;
|
|
static uint32_t s_downstream_stall_recovery_started_ms;
|
|
static control_plane_model_command_callback_t s_model_command_cb;
|
|
static void *s_model_command_cb_user_data;
|
|
static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
|
static SemaphoreHandle_t s_client_lifecycle_mutex;
|
|
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
|
static StaticSemaphore_t s_client_lifecycle_mutex_buf;
|
|
#endif
|
|
static portMUX_TYPE s_client_lifecycle_init_lock = portMUX_INITIALIZER_UNLOCKED;
|
|
|
|
static bool lock_client_lifecycle(void)
|
|
{
|
|
portENTER_CRITICAL(&s_client_lifecycle_init_lock);
|
|
if (s_client_lifecycle_mutex == NULL) {
|
|
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
|
s_client_lifecycle_mutex =
|
|
xSemaphoreCreateMutexStatic(&s_client_lifecycle_mutex_buf);
|
|
#else
|
|
s_client_lifecycle_mutex = xSemaphoreCreateMutex();
|
|
#endif
|
|
}
|
|
SemaphoreHandle_t mutex = s_client_lifecycle_mutex;
|
|
portEXIT_CRITICAL(&s_client_lifecycle_init_lock);
|
|
if (mutex == NULL) {
|
|
return false;
|
|
}
|
|
TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS);
|
|
if (timeout_ticks == 0) {
|
|
timeout_ticks = 1;
|
|
}
|
|
if (xSemaphoreTake(mutex, timeout_ticks) != pdTRUE) {
|
|
ESP_LOGW(TAG, "Timeout acquiring multimodal client lifecycle lock");
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static void unlock_client_lifecycle(void)
|
|
{
|
|
if (s_client_lifecycle_mutex != NULL) {
|
|
(void)xSemaphoreGive(s_client_lifecycle_mutex);
|
|
}
|
|
}
|
|
|
|
static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot)
|
|
{
|
|
TaskHandle_t task = NULL;
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
task = *slot;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return task;
|
|
}
|
|
|
|
static void set_multimodal_task_handle(TaskHandle_t *slot, TaskHandle_t task)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
*slot = task;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static void control_plane_wake_multimodal_watchdog(void)
|
|
{
|
|
TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_watchdog_task);
|
|
if (task != NULL) {
|
|
xTaskNotifyGive(task);
|
|
}
|
|
}
|
|
|
|
static void clear_multimodal_task_handle_if_match(TaskHandle_t *slot, TaskHandle_t expected_task)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (*slot == expected_task) {
|
|
*slot = NULL;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static bool get_multimodal_stop_flag(bool *flag)
|
|
{
|
|
bool value = false;
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
value = *flag;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return value;
|
|
}
|
|
|
|
static void set_multimodal_stop_flag(bool *flag, bool value)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
*flag = value;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static bool get_multimodal_start_flag(bool *flag)
|
|
{
|
|
bool value = false;
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
value = *flag;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return value;
|
|
}
|
|
|
|
static void set_multimodal_start_flag(bool *flag, bool value)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
*flag = value;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static bool control_plane_begin_downstream_stall_recovery(void)
|
|
{
|
|
bool started = false;
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (!s_downstream_stall_recovery_pending) {
|
|
s_downstream_stall_recovery_pending = true;
|
|
s_downstream_stall_stop_sent = false;
|
|
s_downstream_stall_recovery_started_ms = 0U;
|
|
started = true;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return started;
|
|
}
|
|
|
|
static bool control_plane_get_downstream_stall_recovery(bool *out_stop_sent,
|
|
uint32_t *out_started_ms)
|
|
{
|
|
bool pending = false;
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
pending = s_downstream_stall_recovery_pending;
|
|
if (out_stop_sent != NULL) {
|
|
*out_stop_sent = s_downstream_stall_stop_sent;
|
|
}
|
|
if (out_started_ms != NULL) {
|
|
*out_started_ms = s_downstream_stall_recovery_started_ms;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return pending;
|
|
}
|
|
|
|
static void control_plane_mark_downstream_stall_stop_sent(uint32_t now_ms)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (s_downstream_stall_recovery_pending) {
|
|
s_downstream_stall_stop_sent = true;
|
|
s_downstream_stall_recovery_started_ms = now_ms;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static void control_plane_clear_downstream_stall_recovery(void)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
s_downstream_stall_recovery_pending = false;
|
|
s_downstream_stall_stop_sent = false;
|
|
s_downstream_stall_recovery_started_ms = 0U;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static void control_plane_schedule_server_success_prompt(void)
|
|
{
|
|
uint32_t now_ms = (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS);
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (!s_server_success_prompt_played &&
|
|
s_server_success_prompt_attempts <
|
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) {
|
|
s_server_success_prompt_pending = true;
|
|
s_server_success_prompt_next_attempt_ms = now_ms;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static void control_plane_try_server_success_prompt(
|
|
const control_plane_multimodal_runtime_state_t *state,
|
|
uint32_t now_ms)
|
|
{
|
|
if (state == NULL || !state->connected || state->has_active_task) {
|
|
return;
|
|
}
|
|
|
|
bool should_attempt = false;
|
|
uint8_t attempt = 0U;
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (s_server_success_prompt_pending &&
|
|
!s_server_success_prompt_played &&
|
|
!s_server_success_prompt_in_progress &&
|
|
s_server_success_prompt_attempts <
|
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS &&
|
|
(int32_t)(now_ms - s_server_success_prompt_next_attempt_ms) >= 0) {
|
|
s_server_success_prompt_in_progress = true;
|
|
s_server_success_prompt_attempts++;
|
|
attempt = s_server_success_prompt_attempts;
|
|
should_attempt = true;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (!should_attempt) {
|
|
return;
|
|
}
|
|
|
|
esp_err_t err = domain_runtime_behavior_play_prompt_file(
|
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE);
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
s_server_success_prompt_in_progress = false;
|
|
if (err == ESP_OK) {
|
|
s_server_success_prompt_played = true;
|
|
s_server_success_prompt_pending = false;
|
|
} else if (s_server_success_prompt_attempts >=
|
|
CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS) {
|
|
s_server_success_prompt_pending = false;
|
|
} else {
|
|
s_server_success_prompt_next_attempt_ms =
|
|
now_ms + CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (err == ESP_OK) {
|
|
ESP_LOGI(TAG, "Server success prompt played on attempt %u", (unsigned)attempt);
|
|
} else {
|
|
ESP_LOGW(TAG,
|
|
"Server success prompt attempt %u/%u failed: %s",
|
|
(unsigned)attempt,
|
|
(unsigned)CONTROL_PLANE_SERVER_SUCCESS_PROMPT_MAX_ATTEMPTS,
|
|
esp_err_to_name(err));
|
|
}
|
|
}
|
|
|
|
static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag)
|
|
{
|
|
bool can_start = false;
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (*slot == NULL && !*start_flag) {
|
|
*start_flag = true;
|
|
can_start = true;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return can_start;
|
|
}
|
|
|
|
static bool wait_task_exit(TaskHandle_t *task_handle, bool *start_flag, int timeout_ms)
|
|
{
|
|
if (task_handle == NULL ||
|
|
(get_multimodal_task_handle(task_handle) == NULL &&
|
|
(start_flag == NULL || !get_multimodal_start_flag(start_flag)))) {
|
|
return true;
|
|
}
|
|
|
|
TickType_t timeout_ticks = pdMS_TO_TICKS(timeout_ms > 0 ? timeout_ms : 1);
|
|
if (timeout_ticks == 0) {
|
|
timeout_ticks = 1;
|
|
}
|
|
TickType_t start_tick = xTaskGetTickCount();
|
|
while ((get_multimodal_task_handle(task_handle) != NULL ||
|
|
(start_flag != NULL && get_multimodal_start_flag(start_flag))) &&
|
|
(xTaskGetTickCount() - start_tick) < timeout_ticks) {
|
|
vTaskDelay(pdMS_TO_TICKS(20));
|
|
}
|
|
return (get_multimodal_task_handle(task_handle) == NULL &&
|
|
(start_flag == NULL || !get_multimodal_start_flag(start_flag)));
|
|
}
|
|
|
|
static esp_err_t control_plane_prepare_for_multimodal_start(void);
|
|
|
|
static void control_plane_set_model_command_dispatched(bool dispatched)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
s_model_command_dispatched = dispatched;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static void control_plane_clear_cloud_chouqian_pending(const char *reason)
|
|
{
|
|
bool was_pending = false;
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
was_pending = s_cloud_chouqian_pending;
|
|
s_cloud_chouqian_pending = false;
|
|
s_cloud_chouqian_action_in_progress = false;
|
|
s_cloud_chouqian_prompt_sent = false;
|
|
s_cloud_chouqian_requested_ms = 0U;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (was_pending) {
|
|
ESP_LOGI(TAG,
|
|
"Cloud chouqian request completed: %s",
|
|
reason != NULL ? reason : "unknown");
|
|
}
|
|
}
|
|
|
|
static void control_plane_clear_interrupted_cloud_chouqian(void)
|
|
{
|
|
bool was_interrupted = false;
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (s_cloud_chouqian_pending &&
|
|
s_cloud_chouqian_prompt_sent &&
|
|
!s_cloud_chouqian_action_in_progress) {
|
|
was_interrupted = true;
|
|
s_cloud_chouqian_pending = false;
|
|
s_cloud_chouqian_action_in_progress = false;
|
|
s_cloud_chouqian_prompt_sent = false;
|
|
s_cloud_chouqian_requested_ms = 0U;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (was_interrupted) {
|
|
ESP_LOGI(TAG, "Cloud chouqian request completed: response interrupted");
|
|
}
|
|
}
|
|
|
|
static void control_plane_set_model_command_callback(control_plane_model_command_callback_t cb,
|
|
void *user_data)
|
|
{
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
s_model_command_cb = cb;
|
|
s_model_command_cb_user_data = user_data;
|
|
s_model_command_dispatched = false;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
}
|
|
|
|
static void control_plane_responding_text_bridge(const char *text,
|
|
bool finished,
|
|
void *user_data)
|
|
{
|
|
(void)finished;
|
|
(void)user_data;
|
|
|
|
control_plane_model_command_callback_t cb = NULL;
|
|
void *cb_user_data = NULL;
|
|
bool dispatched = false;
|
|
|
|
if (text == NULL) {
|
|
return;
|
|
}
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
cb = s_model_command_cb;
|
|
cb_user_data = s_model_command_cb_user_data;
|
|
dispatched = s_model_command_dispatched;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (cb == NULL || dispatched) {
|
|
return;
|
|
}
|
|
|
|
control_plane_model_command_type_t command_type = CONTROL_PLANE_MODEL_COMMAND_UNKNOWN;
|
|
char value[32] = {0};
|
|
size_t value_index = 0U;
|
|
bool value_index_valid = false;
|
|
if (!control_plane_extract_model_command(text,
|
|
&command_type,
|
|
value,
|
|
sizeof(value),
|
|
&value_index,
|
|
&value_index_valid)) {
|
|
return;
|
|
}
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (s_model_command_dispatched) {
|
|
dispatched = true;
|
|
} else {
|
|
s_model_command_dispatched = true;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (dispatched) {
|
|
return;
|
|
}
|
|
|
|
ESP_LOGI(TAG,
|
|
"Device command: type=%d value=%s index=%u valid=%d",
|
|
(int)command_type,
|
|
value,
|
|
(unsigned int)value_index,
|
|
value_index_valid ? 1 : 0);
|
|
if (command_type == CONTROL_PLANE_MODEL_COMMAND_CHOUQIAN) {
|
|
ESP_LOGI(TAG, "Cloud chouqian result received: %s", value);
|
|
}
|
|
cb(command_type, value, value_index, value_index_valid, cb_user_data);
|
|
}
|
|
|
|
static esp_err_t control_plane_prepare_for_multimodal_start(void)
|
|
{
|
|
#if CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS
|
|
if (!domain_runtime_multimodal_audio_is_initialized() &&
|
|
!domain_runtime_multimodal_audio_is_uplink_running()) {
|
|
esp_err_t audio_err = domain_runtime_multimodal_audio_init();
|
|
if (audio_err != ESP_OK) {
|
|
ESP_LOGW(TAG, "Audio prewarm before WebSocket connect failed: %s", esp_err_to_name(audio_err));
|
|
return audio_err;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
return ESP_OK;
|
|
}
|
|
|
|
static uint32_t control_plane_now_ms(void)
|
|
{
|
|
return (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS);
|
|
}
|
|
|
|
static void control_plane_try_cloud_chouqian_request(void)
|
|
{
|
|
bool pending = false;
|
|
bool prompt_sent = false;
|
|
uint32_t requested_ms = 0U;
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
pending = s_cloud_chouqian_pending;
|
|
prompt_sent = s_cloud_chouqian_prompt_sent;
|
|
requested_ms = s_cloud_chouqian_requested_ms;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (!pending) {
|
|
return;
|
|
}
|
|
|
|
uint32_t now_ms = control_plane_now_ms();
|
|
uint32_t timeout_ms = prompt_sent ?
|
|
CONTROL_PLANE_CLOUD_CHOUQIAN_RESPONSE_TIMEOUT_MS :
|
|
CONTROL_PLANE_CLOUD_CHOUQIAN_PREPARE_TIMEOUT_MS;
|
|
if ((now_ms - requested_ms) >= timeout_ms) {
|
|
control_plane_clear_cloud_chouqian_pending(
|
|
prompt_sent ? "response timeout" : "session prepare timeout");
|
|
return;
|
|
}
|
|
|
|
if (prompt_sent) {
|
|
return;
|
|
}
|
|
|
|
control_plane_multimodal_runtime_state_t state = {0};
|
|
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
|
|
if (state_err != ESP_OK) {
|
|
return;
|
|
}
|
|
|
|
if (!state.connected) {
|
|
(void)control_plane_schedule_websocket_wait_task();
|
|
return;
|
|
}
|
|
|
|
bool should_send_start = !state.has_active_task;
|
|
bool should_send_prompt = state.has_active_task && state.server_listening;
|
|
if (!should_send_start && !should_send_prompt) {
|
|
return;
|
|
}
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (!s_cloud_chouqian_pending || s_cloud_chouqian_action_in_progress) {
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return;
|
|
}
|
|
s_cloud_chouqian_action_in_progress = true;
|
|
if (should_send_prompt) {
|
|
s_cloud_chouqian_prompt_sent = true;
|
|
s_cloud_chouqian_requested_ms = now_ms;
|
|
}
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (should_send_start) {
|
|
esp_err_t start_err = control_plane_multimodal_send_start();
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
s_cloud_chouqian_action_in_progress = false;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (start_err == ESP_OK) {
|
|
ESP_LOGI(TAG, "Cloud chouqian waiting for multimodal session");
|
|
} else if (start_err != ESP_ERR_INVALID_STATE) {
|
|
ESP_LOGW(TAG,
|
|
"Cloud chouqian session start failed: %s",
|
|
esp_err_to_name(start_err));
|
|
control_plane_clear_cloud_chouqian_pending("session start failed");
|
|
}
|
|
return;
|
|
}
|
|
|
|
const control_plane_multimodal_respond_request_t request = {
|
|
.type = CONTROL_PLANE_MULTIMODAL_RESPOND_TYPE_PROMPT,
|
|
.text = "我要抽签",
|
|
.parameters_json = NULL,
|
|
};
|
|
esp_err_t request_err = control_plane_multimodal_send_request_to_respond(&request);
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
s_cloud_chouqian_action_in_progress = false;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
if (request_err != ESP_OK) {
|
|
control_plane_clear_cloud_chouqian_pending("request send failed");
|
|
return;
|
|
}
|
|
|
|
ESP_LOGI(TAG, "Cloud chouqian prompt sent");
|
|
}
|
|
|
|
static bool control_plane_is_within_downstream_guard(const control_plane_multimodal_runtime_state_t *state,
|
|
uint32_t guard_ms)
|
|
{
|
|
if (state == NULL || state->last_downstream_audio_ms == 0U || guard_ms == 0U) {
|
|
return false;
|
|
}
|
|
|
|
uint32_t now_ms = control_plane_now_ms();
|
|
return (now_ms - state->last_downstream_audio_ms) < guard_ms;
|
|
}
|
|
|
|
static esp_err_t control_plane_start_uplink_if_ready(void)
|
|
{
|
|
control_plane_multimodal_runtime_state_t state = {0};
|
|
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
|
|
if (state_err != ESP_OK) {
|
|
return state_err;
|
|
}
|
|
|
|
if (!state.connected || !state.has_active_task) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
/* DashScope multimodal protocol requires waiting for DialogStateChanged(Listening)
|
|
* before sending any upstream audio in every upstream mode.
|
|
*/
|
|
if (!state.server_listening) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (!state.client_speech_active || state.ptt_release_required) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (state.remote_responding) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (control_plane_is_within_downstream_guard(&state,
|
|
CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS)) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
return control_plane_multimodal_start_uplink();
|
|
}
|
|
|
|
esp_err_t control_plane_multimodal_handle_ptt_press_step(void)
|
|
{
|
|
control_plane_multimodal_runtime_state_t state = {0};
|
|
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
|
|
if (state_err != ESP_OK) {
|
|
return state_err;
|
|
}
|
|
|
|
if (!state.connected) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (!state.has_active_task) {
|
|
return control_plane_multimodal_send_start();
|
|
}
|
|
|
|
if (state.ptt_release_required) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (state.remote_responding) {
|
|
if (!state.request_to_speak_pending) {
|
|
return control_plane_multimodal_send_request_to_speak();
|
|
}
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (!state.server_listening) {
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (!state.client_speech_active) {
|
|
esp_err_t speech_err = control_plane_multimodal_send_speech_start();
|
|
if (speech_err != ESP_OK) {
|
|
return speech_err;
|
|
}
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
if (control_plane_multimodal_is_uplink_running()) {
|
|
return ESP_OK;
|
|
}
|
|
|
|
return control_plane_start_uplink_if_ready();
|
|
}
|
|
|
|
esp_err_t control_plane_multimodal_handle_ptt_release(void)
|
|
{
|
|
control_plane_multimodal_stop_uplink();
|
|
return control_plane_multimodal_send_speech_stop();
|
|
}
|
|
|
|
esp_err_t control_plane_multimodal_cancel_speech(void)
|
|
{
|
|
return control_plane_multimodal_send_cancel_speech();
|
|
}
|
|
|
|
esp_err_t control_plane_multimodal_request_to_respond(
|
|
const control_plane_multimodal_respond_request_t *request)
|
|
{
|
|
return control_plane_multimodal_send_request_to_respond(request);
|
|
}
|
|
|
|
esp_err_t control_plane_request_cloud_chouqian(void)
|
|
{
|
|
uint32_t requested_ms = control_plane_now_ms();
|
|
|
|
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
|
if (s_cloud_chouqian_pending) {
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
s_cloud_chouqian_pending = true;
|
|
s_cloud_chouqian_action_in_progress = false;
|
|
s_cloud_chouqian_prompt_sent = false;
|
|
s_cloud_chouqian_requested_ms = requested_ms;
|
|
s_model_command_dispatched = false;
|
|
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
|
|
|
ESP_LOGI(TAG, "Cloud chouqian request queued");
|
|
control_plane_wake_multimodal_watchdog();
|
|
return ESP_OK;
|
|
}
|
|
|
|
esp_err_t control_plane_multimodal_update_info(
|
|
const control_plane_multimodal_update_info_request_t *request)
|
|
{
|
|
return control_plane_multimodal_send_update_info(request);
|
|
}
|
|
|
|
static void stop_watchdog_task(void)
|
|
{
|
|
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, true);
|
|
if (get_multimodal_task_handle(&s_multimodal_watchdog_task) != NULL ||
|
|
get_multimodal_start_flag(&s_multimodal_watchdog_task_starting)) {
|
|
if (!wait_task_exit(&s_multimodal_watchdog_task,
|
|
&s_multimodal_watchdog_task_starting,
|
|
1200)) {
|
|
TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_watchdog_task);
|
|
if (task != NULL) {
|
|
vTaskDelete(task);
|
|
}
|
|
set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL);
|
|
}
|
|
}
|
|
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
|
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false);
|
|
}
|
|
|
|
static void multimodal_watchdog_task(void *arg)
|
|
{
|
|
(void)arg;
|
|
TaskHandle_t self = xTaskGetCurrentTaskHandle();
|
|
|
|
const TickType_t tick_interval = pdMS_TO_TICKS(1000);
|
|
uint32_t heartbeat_ms = CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS > 0 ?
|
|
(uint32_t)CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS :
|
|
50000U;
|
|
uint32_t timeout_ms = CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS > 0 ?
|
|
(uint32_t)CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS :
|
|
0U;
|
|
uint32_t idle_transport_started_ms = 0U;
|
|
|
|
while (!get_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested)) {
|
|
(void)ulTaskNotifyTake(pdTRUE, tick_interval);
|
|
|
|
control_plane_multimodal_runtime_state_t state = {0};
|
|
if (control_plane_multimodal_get_runtime_state(&state) != ESP_OK) {
|
|
idle_transport_started_ms = 0U;
|
|
continue;
|
|
}
|
|
|
|
uint32_t now_ms = control_plane_now_ms();
|
|
control_plane_try_cloud_chouqian_request();
|
|
control_plane_try_server_success_prompt(&state, now_ms);
|
|
bool stall_stop_sent = false;
|
|
uint32_t stall_recovery_started_ms = 0U;
|
|
if (control_plane_get_downstream_stall_recovery(&stall_stop_sent,
|
|
&stall_recovery_started_ms)) {
|
|
if (!state.connected) {
|
|
control_plane_clear_downstream_stall_recovery();
|
|
} else if (!stall_stop_sent) {
|
|
ESP_LOGW(TAG,
|
|
"Downstream stall recovery started: request Stop and wait up to %u ms",
|
|
(unsigned)CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS);
|
|
esp_err_t stop_err =
|
|
control_plane_multimodal_request_downstream_stall_stop();
|
|
if (stop_err == ESP_OK) {
|
|
control_plane_mark_downstream_stall_stop_sent(now_ms);
|
|
} else {
|
|
ESP_LOGW(TAG,
|
|
"Downstream stall Stop request failed: %s; retire transport now",
|
|
esp_err_to_name(stop_err));
|
|
control_plane_clear_downstream_stall_recovery();
|
|
control_plane_multimodal_mark_transport_unavailable(
|
|
CONTROL_PLANE_TRANSPORT_LOSS_DOWNSTREAM_STALL,
|
|
"downstream_stall_stop_failed");
|
|
(void)control_plane_schedule_websocket_wait_task();
|
|
}
|
|
continue;
|
|
} else if (!state.has_active_task) {
|
|
ESP_LOGI(TAG,
|
|
"Downstream stall recovered by server Stop acknowledgement; keep WebSocket");
|
|
control_plane_clear_downstream_stall_recovery();
|
|
} else if ((now_ms - stall_recovery_started_ms) >=
|
|
CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS) {
|
|
ESP_LOGW(TAG,
|
|
"Downstream stall Stop acknowledgement timed out after %u ms; retire transport",
|
|
(unsigned)CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS);
|
|
control_plane_clear_downstream_stall_recovery();
|
|
control_plane_multimodal_mark_transport_unavailable(
|
|
CONTROL_PLANE_TRANSPORT_LOSS_DOWNSTREAM_STALL,
|
|
"downstream_stall_recovery");
|
|
(void)control_plane_schedule_websocket_wait_task();
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (state.connected && !state.has_active_task) {
|
|
if (idle_transport_started_ms == 0U) {
|
|
idle_transport_started_ms = now_ms;
|
|
} else if ((now_ms - idle_transport_started_ms) >=
|
|
CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS) {
|
|
ESP_LOGI(TAG,
|
|
"Idle multimodal transport timeout, stopping client after %u ms",
|
|
(unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS);
|
|
control_plane_multimodal_stop_uplink();
|
|
(void)control_plane_multimodal_stop();
|
|
idle_transport_started_ms = 0U;
|
|
}
|
|
continue;
|
|
}
|
|
idle_transport_started_ms = 0U;
|
|
|
|
if (!state.connected || !state.has_active_task) {
|
|
continue;
|
|
}
|
|
|
|
if (state.last_heartbeat_ms > 0U &&
|
|
(now_ms - state.last_heartbeat_ms) >= heartbeat_ms) {
|
|
(void)control_plane_multimodal_send_heartbeat();
|
|
}
|
|
|
|
if (timeout_ms > 0U &&
|
|
state.last_downstream_ms > 0U &&
|
|
(now_ms - state.last_downstream_ms) >= timeout_ms) {
|
|
ESP_LOGW(TAG, "No downstream response timeout, sending Stop");
|
|
(void)control_plane_multimodal_send_stop("NoResponseTimeout");
|
|
control_plane_multimodal_stop_uplink();
|
|
}
|
|
}
|
|
|
|
clear_multimodal_task_handle_if_match(&s_multimodal_watchdog_task, self);
|
|
vTaskDelete(NULL);
|
|
}
|
|
|
|
static void control_plane_downstream_audio_bridge(const uint8_t *data,
|
|
size_t len,
|
|
void *user_data)
|
|
{
|
|
(void)user_data;
|
|
|
|
esp_err_t err = control_plane_multimodal_play_downstream_audio(data, len);
|
|
if (err != ESP_OK) {
|
|
ESP_LOGW(TAG, "Downstream audio handle failed: %s", esp_err_to_name(err));
|
|
}
|
|
}
|
|
|
|
static void control_plane_request_server_error_expression(domain_expression_trigger_t trigger)
|
|
{
|
|
if (!control_plane_is_sta_connected()) {
|
|
ESP_LOGI(TAG,
|
|
"Skip server error expression while Wi-Fi STA is disconnected (trigger=%d)",
|
|
(int)trigger);
|
|
return;
|
|
}
|
|
|
|
const domain_expression_request_t request = {
|
|
.id = DOMAIN_EXPRESSION_SERVER_ERROR,
|
|
.trigger = trigger,
|
|
};
|
|
|
|
esp_err_t err = domain_runtime_request_expression(&request);
|
|
if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
|
|
ESP_LOGW(TAG, "Server error expression request failed: %s", esp_err_to_name(err));
|
|
}
|
|
}
|
|
|
|
static void control_plane_multimodal_event_bridge(control_plane_multimodal_event_t event,
|
|
void *user_data)
|
|
{
|
|
(void)user_data;
|
|
|
|
switch (event) {
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED: {
|
|
control_plane_schedule_server_success_prompt();
|
|
break;
|
|
}
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED:
|
|
control_plane_clear_downstream_stall_recovery();
|
|
control_plane_clear_cloud_chouqian_pending("transport disconnected");
|
|
control_plane_multimodal_stop_uplink();
|
|
ESP_LOGI(TAG, "Schedule fresh-auth reconnect after transport disconnect");
|
|
(void)control_plane_schedule_websocket_wait_task();
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_ERROR:
|
|
control_plane_clear_downstream_stall_recovery();
|
|
control_plane_clear_cloud_chouqian_pending("transport error");
|
|
control_plane_multimodal_stop_uplink();
|
|
ESP_LOGI(TAG, "Schedule fresh-auth reconnect after transport error");
|
|
(void)control_plane_schedule_websocket_wait_task();
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED:
|
|
control_plane_clear_downstream_stall_recovery();
|
|
control_plane_clear_cloud_chouqian_pending("task failed");
|
|
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_TASK_FAILED);
|
|
control_plane_multimodal_stop_uplink();
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_STARTED:
|
|
control_plane_wake_multimodal_watchdog();
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_LISTENING:
|
|
control_plane_clear_interrupted_cloud_chouqian();
|
|
control_plane_wake_multimodal_watchdog();
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_NOT_LISTENING:
|
|
control_plane_multimodal_stop_uplink();
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_STARTED: {
|
|
if (control_plane_multimodal_is_uplink_running()) {
|
|
ESP_LOGI(TAG, "Pause uplink during TTS responding");
|
|
control_plane_multimodal_stop_uplink();
|
|
}
|
|
#if CONFIG_TQ_PTT_MODE_CLICK_TO_TALK
|
|
if (!control_plane_voice_turn_is_idle()) {
|
|
/*
|
|
* The server response closes this one-shot voice turn. Once
|
|
* teardown reaches IDLE, the local listener can resume with
|
|
* AEC-clean PCM and accept 喵掌柜 while TTS is playing.
|
|
*/
|
|
control_plane_set_ptt_active(false);
|
|
}
|
|
#endif
|
|
break;
|
|
}
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_ENDED:
|
|
control_plane_clear_cloud_chouqian_pending("response ended");
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_STARTED:
|
|
control_plane_set_model_command_dispatched(false);
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_OUTPUT_ERROR:
|
|
control_plane_clear_cloud_chouqian_pending("output error");
|
|
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_OUTPUT_ERROR);
|
|
break;
|
|
|
|
case CONTROL_PLANE_MULTIMODAL_EVENT_DOWNSTREAM_STALLED:
|
|
if (control_plane_begin_downstream_stall_recovery()) {
|
|
ESP_LOGW(TAG,
|
|
"Downstream stall recovery queued for watchdog worker");
|
|
} else {
|
|
ESP_LOGI(TAG, "Downstream stall recovery already pending");
|
|
}
|
|
break;
|
|
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
static esp_err_t control_plane_start_multimodal_client_locked(void)
|
|
{
|
|
control_plane_clear_downstream_stall_recovery();
|
|
|
|
esp_err_t prepare_err = control_plane_prepare_for_multimodal_start();
|
|
if (prepare_err != ESP_OK) {
|
|
ESP_LOGW(TAG, "Prepare multimodal start failed: %s", esp_err_to_name(prepare_err));
|
|
}
|
|
|
|
esp_err_t callback_err = control_plane_multimodal_register_event_callback(
|
|
control_plane_multimodal_event_bridge,
|
|
NULL);
|
|
if (callback_err != ESP_OK) {
|
|
return callback_err;
|
|
}
|
|
|
|
callback_err = control_plane_multimodal_register_downstream_audio_callback(
|
|
control_plane_downstream_audio_bridge,
|
|
NULL);
|
|
if (callback_err != ESP_OK) {
|
|
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
|
return callback_err;
|
|
}
|
|
|
|
if (get_multimodal_task_handle(&s_multimodal_watchdog_task) == NULL &&
|
|
!get_multimodal_start_flag(&s_multimodal_watchdog_task_starting) &&
|
|
try_begin_multimodal_task_start(&s_multimodal_watchdog_task,
|
|
&s_multimodal_watchdog_task_starting)) {
|
|
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false);
|
|
if (xTaskCreatePinnedToCore(multimodal_watchdog_task,
|
|
"mm_watchdog",
|
|
CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE,
|
|
NULL,
|
|
4,
|
|
&s_multimodal_watchdog_task,
|
|
CONTROL_PLANE_MULTIMODAL_NETWORK_TASK_CORE) != pdPASS) {
|
|
ESP_LOGE(TAG, "Watchdog task create failed");
|
|
set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL);
|
|
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
|
(void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL);
|
|
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
|
return ESP_ERR_NO_MEM;
|
|
} else {
|
|
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
|
}
|
|
}
|
|
|
|
esp_err_t err = control_plane_multimodal_start();
|
|
if (err != ESP_OK) {
|
|
stop_watchdog_task();
|
|
(void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL);
|
|
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
|
return err;
|
|
}
|
|
|
|
return ESP_OK;
|
|
}
|
|
|
|
esp_err_t control_plane_start_multimodal_client(void)
|
|
{
|
|
if (!lock_client_lifecycle()) {
|
|
return ESP_ERR_TIMEOUT;
|
|
}
|
|
esp_err_t err = control_plane_start_multimodal_client_locked();
|
|
unlock_client_lifecycle();
|
|
return err;
|
|
}
|
|
|
|
esp_err_t control_plane_stop_multimodal_client(void)
|
|
{
|
|
if (!lock_client_lifecycle()) {
|
|
return ESP_ERR_TIMEOUT;
|
|
}
|
|
control_plane_clear_downstream_stall_recovery();
|
|
control_plane_clear_cloud_chouqian_pending("multimodal client stopped");
|
|
stop_watchdog_task();
|
|
esp_err_t err = control_plane_multimodal_stop();
|
|
unlock_client_lifecycle();
|
|
return err;
|
|
}
|
|
|
|
esp_err_t control_plane_register_model_command_callback(
|
|
control_plane_model_command_callback_t cb,
|
|
void *user_data)
|
|
{
|
|
control_plane_set_model_command_callback(cb, user_data);
|
|
return control_plane_multimodal_register_responding_text_callback(
|
|
(cb != NULL) ? control_plane_responding_text_bridge : NULL,
|
|
NULL);
|
|
}
|