/* * SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ #include "control_plane.h" #include "control_plane_internal.h" #include #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "esp_log.h" #include "sdkconfig.h" #ifndef CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS #define CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS 50000 #endif #ifndef CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS #define CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS 60000 #endif #ifndef CONFIG_MULTIMODAL_RECONNECT_DELAY_MS #define CONFIG_MULTIMODAL_RECONNECT_DELAY_MS 2000 #endif #ifndef CONFIG_MULTIMODAL_UPSTREAM_MODE #define CONFIG_MULTIMODAL_UPSTREAM_MODE "push2talk" #endif #ifndef CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS #define CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS 1 #endif #if defined(CONFIG_MULTIMODAL_AUTO_START_ON_WAKEUP) #define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG CONFIG_MULTIMODAL_AUTO_START_ON_WAKEUP #elif defined(CONFIG_MULTIMODAL_WS_AUTO_START) #define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG CONFIG_MULTIMODAL_WS_AUTO_START #else #define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG 0 #endif #define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220 #define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000 #define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144 #ifndef CONFIG_TQ_NETWORK_TASK_CORE #define CONFIG_TQ_NETWORK_TASK_CORE 0 #endif #if CONFIG_TQ_NETWORK_TASK_CORE < 0 #define CONTROL_PLANE_MULTIMODAL_NETWORK_TASK_CORE tskNO_AFFINITY #else #define CONTROL_PLANE_MULTIMODAL_NETWORK_TASK_CORE ((BaseType_t)CONFIG_TQ_NETWORK_TASK_CORE) #endif static const char *TAG = "control_plane_mm_rt"; static TaskHandle_t s_multimodal_watchdog_task; static TaskHandle_t s_multimodal_autostart_task; static bool s_multimodal_watchdog_stop_requested; static bool s_multimodal_autostart_stop_requested; static bool s_multimodal_watchdog_task_starting; static bool s_multimodal_autostart_task_starting; static bool s_model_command_dispatched; static control_plane_model_command_callback_t s_model_command_cb; static void *s_model_command_cb_user_data; static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED; static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot) { TaskHandle_t task = NULL; portENTER_CRITICAL(&s_multimodal_runtime_lock); task = *slot; portEXIT_CRITICAL(&s_multimodal_runtime_lock); return task; } static void set_multimodal_task_handle(TaskHandle_t *slot, TaskHandle_t task) { portENTER_CRITICAL(&s_multimodal_runtime_lock); *slot = task; portEXIT_CRITICAL(&s_multimodal_runtime_lock); } static void clear_multimodal_task_handle_if_match(TaskHandle_t *slot, TaskHandle_t expected_task) { portENTER_CRITICAL(&s_multimodal_runtime_lock); if (*slot == expected_task) { *slot = NULL; } portEXIT_CRITICAL(&s_multimodal_runtime_lock); } static bool get_multimodal_stop_flag(bool *flag) { bool value = false; portENTER_CRITICAL(&s_multimodal_runtime_lock); value = *flag; portEXIT_CRITICAL(&s_multimodal_runtime_lock); return value; } static void set_multimodal_stop_flag(bool *flag, bool value) { portENTER_CRITICAL(&s_multimodal_runtime_lock); *flag = value; portEXIT_CRITICAL(&s_multimodal_runtime_lock); } static bool get_multimodal_start_flag(bool *flag) { bool value = false; portENTER_CRITICAL(&s_multimodal_runtime_lock); value = *flag; portEXIT_CRITICAL(&s_multimodal_runtime_lock); return value; } static void set_multimodal_start_flag(bool *flag, bool value) { portENTER_CRITICAL(&s_multimodal_runtime_lock); *flag = value; portEXIT_CRITICAL(&s_multimodal_runtime_lock); } static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag) { bool can_start = false; portENTER_CRITICAL(&s_multimodal_runtime_lock); if (*slot == NULL && !*start_flag) { *start_flag = true; can_start = true; } portEXIT_CRITICAL(&s_multimodal_runtime_lock); return can_start; } static bool wait_task_exit(TaskHandle_t *task_handle, bool *start_flag, int timeout_ms) { if (task_handle == NULL || (get_multimodal_task_handle(task_handle) == NULL && (start_flag == NULL || !get_multimodal_start_flag(start_flag)))) { return true; } TickType_t timeout_ticks = pdMS_TO_TICKS(timeout_ms > 0 ? timeout_ms : 1); if (timeout_ticks == 0) { timeout_ticks = 1; } TickType_t start_tick = xTaskGetTickCount(); while ((get_multimodal_task_handle(task_handle) != NULL || (start_flag != NULL && get_multimodal_start_flag(start_flag))) && (xTaskGetTickCount() - start_tick) < timeout_ticks) { vTaskDelay(pdMS_TO_TICKS(20)); } return (get_multimodal_task_handle(task_handle) == NULL && (start_flag == NULL || !get_multimodal_start_flag(start_flag))); } static esp_err_t control_plane_prepare_for_multimodal_start(void); static void control_plane_set_model_command_dispatched(bool dispatched) { portENTER_CRITICAL(&s_multimodal_runtime_lock); s_model_command_dispatched = dispatched; portEXIT_CRITICAL(&s_multimodal_runtime_lock); } static void control_plane_set_model_command_callback(control_plane_model_command_callback_t cb, void *user_data) { portENTER_CRITICAL(&s_multimodal_runtime_lock); s_model_command_cb = cb; s_model_command_cb_user_data = user_data; s_model_command_dispatched = false; portEXIT_CRITICAL(&s_multimodal_runtime_lock); } static void control_plane_responding_text_bridge(const char *text, bool finished, void *user_data) { (void)finished; (void)user_data; control_plane_model_command_callback_t cb = NULL; void *cb_user_data = NULL; bool dispatched = false; if (text == NULL) { return; } portENTER_CRITICAL(&s_multimodal_runtime_lock); cb = s_model_command_cb; cb_user_data = s_model_command_cb_user_data; dispatched = s_model_command_dispatched; portEXIT_CRITICAL(&s_multimodal_runtime_lock); if (cb == NULL || dispatched) { return; } control_plane_model_command_type_t command_type = CONTROL_PLANE_MODEL_COMMAND_UNKNOWN; char value[32] = {0}; if (!control_plane_extract_model_command(text, &command_type, value, sizeof(value))) { return; } portENTER_CRITICAL(&s_multimodal_runtime_lock); if (s_model_command_dispatched) { dispatched = true; } else { s_model_command_dispatched = true; } portEXIT_CRITICAL(&s_multimodal_runtime_lock); if (dispatched) { return; } ESP_LOGI(TAG, "Device command: type=%d value=%s", (int)command_type, value); cb(command_type, value, cb_user_data); } static esp_err_t control_plane_prepare_for_multimodal_start(void) { #if CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS if (!domain_runtime_multimodal_audio_is_initialized() && !domain_runtime_multimodal_audio_is_uplink_running()) { esp_err_t audio_err = domain_runtime_multimodal_audio_init(); if (audio_err != ESP_OK) { ESP_LOGW(TAG, "Audio prewarm before WebSocket connect failed: %s", esp_err_to_name(audio_err)); return audio_err; } } #endif return ESP_OK; } static uint32_t control_plane_now_ms(void) { return (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS); } static bool control_plane_is_within_downstream_guard(const control_plane_multimodal_runtime_state_t *state, uint32_t guard_ms) { if (state == NULL || state->last_downstream_audio_ms == 0U || guard_ms == 0U) { return false; } uint32_t now_ms = control_plane_now_ms(); return (now_ms - state->last_downstream_audio_ms) < guard_ms; } static esp_err_t control_plane_start_uplink_if_ready(void) { control_plane_multimodal_runtime_state_t state = {0}; esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state); if (state_err != ESP_OK) { return state_err; } if (!state.connected || !state.has_active_task) { return ESP_ERR_INVALID_STATE; } /* DashScope multimodal protocol requires waiting for DialogStateChanged(Listening) * before sending any upstream audio in every upstream mode. */ if (!state.server_listening) { return ESP_ERR_INVALID_STATE; } if (!state.client_speech_active || state.ptt_release_required) { return ESP_ERR_INVALID_STATE; } if (state.remote_responding) { return ESP_ERR_INVALID_STATE; } if (control_plane_is_within_downstream_guard(&state, CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS)) { return ESP_ERR_INVALID_STATE; } return control_plane_multimodal_start_uplink(); } esp_err_t control_plane_multimodal_handle_ptt_press_step(void) { control_plane_multimodal_runtime_state_t state = {0}; esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state); if (state_err != ESP_OK) { return state_err; } if (!state.connected) { return ESP_ERR_INVALID_STATE; } if (!state.has_active_task) { return control_plane_multimodal_send_start(); } if (state.ptt_release_required) { return ESP_ERR_INVALID_STATE; } if (state.remote_responding) { if (!state.request_to_speak_pending) { return control_plane_multimodal_send_request_to_speak(); } return ESP_ERR_INVALID_STATE; } if (!state.server_listening) { return ESP_ERR_INVALID_STATE; } if (!state.client_speech_active) { esp_err_t speech_err = control_plane_multimodal_send_speech_start(); if (speech_err != ESP_OK) { return speech_err; } } if (control_plane_multimodal_is_uplink_running()) { return ESP_OK; } return control_plane_start_uplink_if_ready(); } esp_err_t control_plane_multimodal_handle_ptt_release(void) { control_plane_multimodal_stop_uplink(); return control_plane_multimodal_send_speech_stop(); } esp_err_t control_plane_multimodal_cancel_speech(void) { return control_plane_multimodal_send_cancel_speech(); } esp_err_t control_plane_multimodal_request_to_respond( const control_plane_multimodal_respond_request_t *request) { return control_plane_multimodal_send_request_to_respond(request); } esp_err_t control_plane_multimodal_update_info( const control_plane_multimodal_update_info_request_t *request) { return control_plane_multimodal_send_update_info(request); } static void multimodal_autostart_task(void *arg) { (void)arg; TaskHandle_t self = xTaskGetCurrentTaskHandle(); TickType_t wait_step = pdMS_TO_TICKS(200); TickType_t wait_timeout = pdMS_TO_TICKS(CONFIG_MULTIMODAL_RECONNECT_DELAY_MS + 3000); if (wait_timeout < pdMS_TO_TICKS(3000)) { wait_timeout = pdMS_TO_TICKS(3000); } TickType_t start_tick = xTaskGetTickCount(); while (!get_multimodal_stop_flag(&s_multimodal_autostart_stop_requested) && !control_plane_multimodal_is_connected()) { if ((xTaskGetTickCount() - start_tick) >= wait_timeout) { ESP_LOGW(TAG, "Auto Start aborted: transport not ready"); clear_multimodal_task_handle_if_match(&s_multimodal_autostart_task, self); vTaskDelete(NULL); return; } vTaskDelay(wait_step); } for (int attempt = 1; !get_multimodal_stop_flag(&s_multimodal_autostart_stop_requested) && attempt <= 5 && control_plane_multimodal_is_connected(); ++attempt) { esp_err_t err = control_plane_multimodal_send_start(); if (err == ESP_OK) { break; } if (attempt < 5 && control_plane_multimodal_is_connected()) { ESP_LOGW(TAG, "Auto Start send failed (%d/5): %s", attempt, esp_err_to_name(err)); vTaskDelay(pdMS_TO_TICKS(300)); } else { ESP_LOGW(TAG, "Auto Start send failed: %s", esp_err_to_name(err)); } } clear_multimodal_task_handle_if_match(&s_multimodal_autostart_task, self); vTaskDelete(NULL); } static void stop_autostart_task(void) { set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, true); if (get_multimodal_task_handle(&s_multimodal_autostart_task) != NULL || get_multimodal_start_flag(&s_multimodal_autostart_task_starting)) { if (!wait_task_exit(&s_multimodal_autostart_task, &s_multimodal_autostart_task_starting, 800)) { TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_autostart_task); if (task != NULL) { vTaskDelete(task); } set_multimodal_task_handle(&s_multimodal_autostart_task, NULL); } } set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false); set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, false); } static void schedule_autostart_if_enabled(void) { if (!CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG) { return; } if (get_multimodal_task_handle(&s_multimodal_autostart_task) != NULL || get_multimodal_start_flag(&s_multimodal_autostart_task_starting)) { return; } if (!try_begin_multimodal_task_start(&s_multimodal_autostart_task, &s_multimodal_autostart_task_starting)) { return; } set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, false); if (xTaskCreatePinnedToCore(multimodal_autostart_task, "mm_autostart", 6144, NULL, 4, &s_multimodal_autostart_task, CONTROL_PLANE_MULTIMODAL_NETWORK_TASK_CORE) != pdPASS) { set_multimodal_task_handle(&s_multimodal_autostart_task, NULL); set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false); ESP_LOGW(TAG, "Auto Start task create failed"); return; } set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false); } static void stop_watchdog_task(void) { set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, true); if (get_multimodal_task_handle(&s_multimodal_watchdog_task) != NULL || get_multimodal_start_flag(&s_multimodal_watchdog_task_starting)) { if (!wait_task_exit(&s_multimodal_watchdog_task, &s_multimodal_watchdog_task_starting, 1200)) { TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_watchdog_task); if (task != NULL) { vTaskDelete(task); } set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL); } } set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false); set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false); } static void multimodal_watchdog_task(void *arg) { (void)arg; TaskHandle_t self = xTaskGetCurrentTaskHandle(); const TickType_t tick_interval = pdMS_TO_TICKS(1000); uint32_t heartbeat_ms = CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS > 0 ? (uint32_t)CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS : 50000U; uint32_t timeout_ms = CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS > 0 ? (uint32_t)CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS : 0U; uint32_t idle_transport_started_ms = 0U; while (!get_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested)) { vTaskDelay(tick_interval); control_plane_multimodal_runtime_state_t state = {0}; if (control_plane_multimodal_get_runtime_state(&state) != ESP_OK) { idle_transport_started_ms = 0U; continue; } uint32_t now_ms = control_plane_now_ms(); if (state.connected && !state.has_active_task) { if (idle_transport_started_ms == 0U) { idle_transport_started_ms = now_ms; } else if ((now_ms - idle_transport_started_ms) >= CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS) { ESP_LOGI(TAG, "Idle multimodal transport timeout, stopping client after %u ms", (unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS); control_plane_multimodal_stop_uplink(); stop_autostart_task(); control_plane_multimodal_stop(); idle_transport_started_ms = 0U; } continue; } idle_transport_started_ms = 0U; if (!state.connected || !state.has_active_task) { continue; } if (state.last_heartbeat_ms > 0U && (now_ms - state.last_heartbeat_ms) >= heartbeat_ms) { (void)control_plane_multimodal_send_heartbeat(); } if (timeout_ms > 0U && state.last_downstream_ms > 0U && (now_ms - state.last_downstream_ms) >= timeout_ms) { ESP_LOGW(TAG, "No downstream response timeout, sending Stop"); (void)control_plane_multimodal_send_stop("NoResponseTimeout"); control_plane_multimodal_stop_uplink(); } } clear_multimodal_task_handle_if_match(&s_multimodal_watchdog_task, self); vTaskDelete(NULL); } static void control_plane_downstream_audio_bridge(const uint8_t *data, size_t len, void *user_data) { (void)user_data; esp_err_t err = control_plane_multimodal_play_downstream_audio(data, len); if (err != ESP_OK) { ESP_LOGW(TAG, "Downstream audio handle failed: %s", esp_err_to_name(err)); } } static void control_plane_request_server_error_expression(domain_expression_trigger_t trigger) { if (!control_plane_is_sta_connected()) { ESP_LOGI(TAG, "Skip server error expression while Wi-Fi STA is disconnected (trigger=%d)", (int)trigger); return; } const domain_expression_request_t request = { .id = DOMAIN_EXPRESSION_SERVER_ERROR, .trigger = trigger, }; esp_err_t err = domain_runtime_request_expression(&request); if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { ESP_LOGW(TAG, "Server error expression request failed: %s", esp_err_to_name(err)); } } static void control_plane_multimodal_event_bridge(control_plane_multimodal_event_t event, void *user_data) { (void)user_data; switch (event) { case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED: if (CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG) { schedule_autostart_if_enabled(); } break; case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED: control_plane_multimodal_stop_uplink(); stop_autostart_task(); break; case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED: control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_TASK_FAILED); control_plane_multimodal_stop_uplink(); schedule_autostart_if_enabled(); break; case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_STARTED: break; case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_LISTENING: break; case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_NOT_LISTENING: control_plane_multimodal_stop_uplink(); break; case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_STARTED: { if (control_plane_multimodal_is_uplink_running()) { ESP_LOGI(TAG, "Pause uplink during TTS responding"); control_plane_multimodal_stop_uplink(); } break; } case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_ENDED: break; case CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_STARTED: control_plane_set_model_command_dispatched(false); break; case CONTROL_PLANE_MULTIMODAL_EVENT_OUTPUT_ERROR: control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_OUTPUT_ERROR); break; default: break; } } esp_err_t control_plane_start_multimodal_client(void) { esp_err_t prepare_err = control_plane_prepare_for_multimodal_start(); if (prepare_err != ESP_OK) { ESP_LOGW(TAG, "Prepare multimodal start failed: %s", esp_err_to_name(prepare_err)); } esp_err_t callback_err = control_plane_multimodal_register_event_callback( control_plane_multimodal_event_bridge, NULL); if (callback_err != ESP_OK) { return callback_err; } callback_err = control_plane_multimodal_register_downstream_audio_callback( control_plane_downstream_audio_bridge, NULL); if (callback_err != ESP_OK) { (void)control_plane_multimodal_register_event_callback(NULL, NULL); return callback_err; } if (get_multimodal_task_handle(&s_multimodal_watchdog_task) == NULL && !get_multimodal_start_flag(&s_multimodal_watchdog_task_starting) && try_begin_multimodal_task_start(&s_multimodal_watchdog_task, &s_multimodal_watchdog_task_starting)) { set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false); if (xTaskCreatePinnedToCore(multimodal_watchdog_task, "mm_watchdog", CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE, NULL, 4, &s_multimodal_watchdog_task, CONTROL_PLANE_MULTIMODAL_NETWORK_TASK_CORE) != pdPASS) { ESP_LOGE(TAG, "Watchdog task create failed"); set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL); set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false); (void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL); (void)control_plane_multimodal_register_event_callback(NULL, NULL); return ESP_ERR_NO_MEM; } else { set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false); } } esp_err_t err = control_plane_multimodal_start(); if (err != ESP_OK) { stop_watchdog_task(); return err; } if (CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG && control_plane_multimodal_is_connected()) { schedule_autostart_if_enabled(); } return ESP_OK; } void control_plane_stop_multimodal_client(void) { stop_autostart_task(); stop_watchdog_task(); control_plane_multimodal_stop(); } esp_err_t control_plane_register_model_command_callback( control_plane_model_command_callback_t cb, void *user_data) { control_plane_set_model_command_callback(cb, user_data); return control_plane_multimodal_register_responding_text_callback( (cb != NULL) ? control_plane_responding_text_bridge : NULL, NULL); }