chore: initial commit
This commit is contained in:
CSBOOK\44744
@@ -0,0 +1,694 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
*
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
#include "control_plane.h"
|
||||
|
||||
#include "control_plane_internal.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/task.h"
|
||||
#include "esp_log.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS
|
||||
#define CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS 50000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS
|
||||
#define CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS 60000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_RECONNECT_DELAY_MS
|
||||
#define CONFIG_MULTIMODAL_RECONNECT_DELAY_MS 2000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_UPSTREAM_MODE
|
||||
#define CONFIG_MULTIMODAL_UPSTREAM_MODE "push2talk"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS
|
||||
#define CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS 1
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_MULTIMODAL_AUTO_START_ON_WAKEUP)
|
||||
#define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG CONFIG_MULTIMODAL_AUTO_START_ON_WAKEUP
|
||||
#elif defined(CONFIG_MULTIMODAL_WS_AUTO_START)
|
||||
#define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG CONFIG_MULTIMODAL_WS_AUTO_START
|
||||
#else
|
||||
#define CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG 0
|
||||
#endif
|
||||
|
||||
#define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220
|
||||
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
|
||||
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144
|
||||
|
||||
static const char *TAG = "control_plane_mm_rt";
|
||||
static TaskHandle_t s_multimodal_watchdog_task;
|
||||
static TaskHandle_t s_multimodal_autostart_task;
|
||||
static bool s_multimodal_watchdog_stop_requested;
|
||||
static bool s_multimodal_autostart_stop_requested;
|
||||
static bool s_multimodal_watchdog_task_starting;
|
||||
static bool s_multimodal_autostart_task_starting;
|
||||
static bool s_model_command_dispatched;
|
||||
static control_plane_model_command_callback_t s_model_command_cb;
|
||||
static void *s_model_command_cb_user_data;
|
||||
static portMUX_TYPE s_multimodal_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||
|
||||
static TaskHandle_t get_multimodal_task_handle(TaskHandle_t *slot)
|
||||
{
|
||||
TaskHandle_t task = NULL;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
task = *slot;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return task;
|
||||
}
|
||||
|
||||
static void set_multimodal_task_handle(TaskHandle_t *slot, TaskHandle_t task)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
*slot = task;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static void clear_multimodal_task_handle_if_match(TaskHandle_t *slot, TaskHandle_t expected_task)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
if (*slot == expected_task) {
|
||||
*slot = NULL;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static bool get_multimodal_stop_flag(bool *flag)
|
||||
{
|
||||
bool value = false;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
value = *flag;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return value;
|
||||
}
|
||||
|
||||
static void set_multimodal_stop_flag(bool *flag, bool value)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
*flag = value;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static bool get_multimodal_start_flag(bool *flag)
|
||||
{
|
||||
bool value = false;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
value = *flag;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return value;
|
||||
}
|
||||
|
||||
static void set_multimodal_start_flag(bool *flag, bool value)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
*flag = value;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static bool try_begin_multimodal_task_start(TaskHandle_t *slot, bool *start_flag)
|
||||
{
|
||||
bool can_start = false;
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
if (*slot == NULL && !*start_flag) {
|
||||
*start_flag = true;
|
||||
can_start = true;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
return can_start;
|
||||
}
|
||||
|
||||
static bool wait_task_exit(TaskHandle_t *task_handle, bool *start_flag, int timeout_ms)
|
||||
{
|
||||
if (task_handle == NULL ||
|
||||
(get_multimodal_task_handle(task_handle) == NULL &&
|
||||
(start_flag == NULL || !get_multimodal_start_flag(start_flag)))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
TickType_t timeout_ticks = pdMS_TO_TICKS(timeout_ms > 0 ? timeout_ms : 1);
|
||||
if (timeout_ticks == 0) {
|
||||
timeout_ticks = 1;
|
||||
}
|
||||
TickType_t start_tick = xTaskGetTickCount();
|
||||
while ((get_multimodal_task_handle(task_handle) != NULL ||
|
||||
(start_flag != NULL && get_multimodal_start_flag(start_flag))) &&
|
||||
(xTaskGetTickCount() - start_tick) < timeout_ticks) {
|
||||
vTaskDelay(pdMS_TO_TICKS(20));
|
||||
}
|
||||
return (get_multimodal_task_handle(task_handle) == NULL &&
|
||||
(start_flag == NULL || !get_multimodal_start_flag(start_flag)));
|
||||
}
|
||||
|
||||
static esp_err_t control_plane_prepare_for_multimodal_start(void);
|
||||
|
||||
static void control_plane_set_model_command_dispatched(bool dispatched)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
s_model_command_dispatched = dispatched;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static void control_plane_set_model_command_callback(control_plane_model_command_callback_t cb,
|
||||
void *user_data)
|
||||
{
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
s_model_command_cb = cb;
|
||||
s_model_command_cb_user_data = user_data;
|
||||
s_model_command_dispatched = false;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
}
|
||||
|
||||
static void control_plane_responding_text_bridge(const char *text,
|
||||
bool finished,
|
||||
void *user_data)
|
||||
{
|
||||
(void)finished;
|
||||
(void)user_data;
|
||||
|
||||
control_plane_model_command_callback_t cb = NULL;
|
||||
void *cb_user_data = NULL;
|
||||
bool dispatched = false;
|
||||
|
||||
if (text == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
cb = s_model_command_cb;
|
||||
cb_user_data = s_model_command_cb_user_data;
|
||||
dispatched = s_model_command_dispatched;
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
|
||||
if (cb == NULL || dispatched) {
|
||||
return;
|
||||
}
|
||||
|
||||
control_plane_model_command_type_t command_type = CONTROL_PLANE_MODEL_COMMAND_UNKNOWN;
|
||||
char value[32] = {0};
|
||||
if (!control_plane_extract_model_command(text, &command_type, value, sizeof(value))) {
|
||||
return;
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_multimodal_runtime_lock);
|
||||
if (s_model_command_dispatched) {
|
||||
dispatched = true;
|
||||
} else {
|
||||
s_model_command_dispatched = true;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_multimodal_runtime_lock);
|
||||
|
||||
if (dispatched) {
|
||||
return;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "Device command: type=%d value=%s", (int)command_type, value);
|
||||
cb(command_type, value, cb_user_data);
|
||||
}
|
||||
|
||||
static esp_err_t control_plane_prepare_for_multimodal_start(void)
|
||||
{
|
||||
#if CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS
|
||||
if (!domain_runtime_multimodal_audio_is_initialized() &&
|
||||
!domain_runtime_multimodal_audio_is_uplink_running()) {
|
||||
esp_err_t audio_err = domain_runtime_multimodal_audio_init();
|
||||
if (audio_err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "Audio prewarm before WebSocket connect failed: %s", esp_err_to_name(audio_err));
|
||||
return audio_err;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static uint32_t control_plane_now_ms(void)
|
||||
{
|
||||
return (uint32_t)(xTaskGetTickCount() * portTICK_PERIOD_MS);
|
||||
}
|
||||
|
||||
static bool control_plane_is_within_downstream_guard(const control_plane_multimodal_runtime_state_t *state,
|
||||
uint32_t guard_ms)
|
||||
{
|
||||
if (state == NULL || state->last_downstream_audio_ms == 0U || guard_ms == 0U) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t now_ms = control_plane_now_ms();
|
||||
return (now_ms - state->last_downstream_audio_ms) < guard_ms;
|
||||
}
|
||||
|
||||
static esp_err_t control_plane_start_uplink_if_ready(void)
|
||||
{
|
||||
control_plane_multimodal_runtime_state_t state = {0};
|
||||
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
|
||||
if (state_err != ESP_OK) {
|
||||
return state_err;
|
||||
}
|
||||
|
||||
if (!state.connected || !state.has_active_task) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
/* DashScope multimodal protocol requires waiting for DialogStateChanged(Listening)
|
||||
* before sending any upstream audio in every upstream mode.
|
||||
*/
|
||||
if (!state.server_listening) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.client_speech_active || state.ptt_release_required) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (state.remote_responding) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (control_plane_is_within_downstream_guard(&state,
|
||||
CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS)) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
return control_plane_multimodal_start_uplink();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_handle_ptt_press_step(void)
|
||||
{
|
||||
control_plane_multimodal_runtime_state_t state = {0};
|
||||
esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state);
|
||||
if (state_err != ESP_OK) {
|
||||
return state_err;
|
||||
}
|
||||
|
||||
if (!state.connected) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.has_active_task) {
|
||||
return control_plane_multimodal_send_start();
|
||||
}
|
||||
|
||||
if (state.ptt_release_required) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (state.remote_responding) {
|
||||
if (!state.request_to_speak_pending) {
|
||||
return control_plane_multimodal_send_request_to_speak();
|
||||
}
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.server_listening) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!state.client_speech_active) {
|
||||
esp_err_t speech_err = control_plane_multimodal_send_speech_start();
|
||||
if (speech_err != ESP_OK) {
|
||||
return speech_err;
|
||||
}
|
||||
}
|
||||
|
||||
if (control_plane_multimodal_is_uplink_running()) {
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
return control_plane_start_uplink_if_ready();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_handle_ptt_release(void)
|
||||
{
|
||||
control_plane_multimodal_stop_uplink();
|
||||
return control_plane_multimodal_send_speech_stop();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_cancel_speech(void)
|
||||
{
|
||||
return control_plane_multimodal_send_cancel_speech();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_request_to_respond(
|
||||
const control_plane_multimodal_respond_request_t *request)
|
||||
{
|
||||
return control_plane_multimodal_send_request_to_respond(request);
|
||||
}
|
||||
|
||||
esp_err_t control_plane_multimodal_update_info(
|
||||
const control_plane_multimodal_update_info_request_t *request)
|
||||
{
|
||||
return control_plane_multimodal_send_update_info(request);
|
||||
}
|
||||
|
||||
static void multimodal_autostart_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
TaskHandle_t self = xTaskGetCurrentTaskHandle();
|
||||
|
||||
TickType_t wait_step = pdMS_TO_TICKS(200);
|
||||
TickType_t wait_timeout = pdMS_TO_TICKS(CONFIG_MULTIMODAL_RECONNECT_DELAY_MS + 3000);
|
||||
if (wait_timeout < pdMS_TO_TICKS(3000)) {
|
||||
wait_timeout = pdMS_TO_TICKS(3000);
|
||||
}
|
||||
|
||||
TickType_t start_tick = xTaskGetTickCount();
|
||||
while (!get_multimodal_stop_flag(&s_multimodal_autostart_stop_requested) &&
|
||||
!control_plane_multimodal_is_connected()) {
|
||||
if ((xTaskGetTickCount() - start_tick) >= wait_timeout) {
|
||||
ESP_LOGW(TAG, "Auto Start aborted: transport not ready");
|
||||
clear_multimodal_task_handle_if_match(&s_multimodal_autostart_task, self);
|
||||
vTaskDelete(NULL);
|
||||
return;
|
||||
}
|
||||
vTaskDelay(wait_step);
|
||||
}
|
||||
|
||||
for (int attempt = 1;
|
||||
!get_multimodal_stop_flag(&s_multimodal_autostart_stop_requested) && attempt <= 5 &&
|
||||
control_plane_multimodal_is_connected();
|
||||
++attempt) {
|
||||
esp_err_t err = control_plane_multimodal_send_start();
|
||||
if (err == ESP_OK) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (attempt < 5 && control_plane_multimodal_is_connected()) {
|
||||
ESP_LOGW(TAG,
|
||||
"Auto Start send failed (%d/5): %s",
|
||||
attempt,
|
||||
esp_err_to_name(err));
|
||||
vTaskDelay(pdMS_TO_TICKS(300));
|
||||
} else {
|
||||
ESP_LOGW(TAG, "Auto Start send failed: %s", esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
clear_multimodal_task_handle_if_match(&s_multimodal_autostart_task, self);
|
||||
vTaskDelete(NULL);
|
||||
}
|
||||
|
||||
static void stop_autostart_task(void)
|
||||
{
|
||||
set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, true);
|
||||
if (get_multimodal_task_handle(&s_multimodal_autostart_task) != NULL ||
|
||||
get_multimodal_start_flag(&s_multimodal_autostart_task_starting)) {
|
||||
if (!wait_task_exit(&s_multimodal_autostart_task,
|
||||
&s_multimodal_autostart_task_starting,
|
||||
800)) {
|
||||
TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_autostart_task);
|
||||
if (task != NULL) {
|
||||
vTaskDelete(task);
|
||||
}
|
||||
set_multimodal_task_handle(&s_multimodal_autostart_task, NULL);
|
||||
}
|
||||
}
|
||||
set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false);
|
||||
set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, false);
|
||||
}
|
||||
|
||||
static void schedule_autostart_if_enabled(void)
|
||||
{
|
||||
if (!CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (get_multimodal_task_handle(&s_multimodal_autostart_task) != NULL ||
|
||||
get_multimodal_start_flag(&s_multimodal_autostart_task_starting)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!try_begin_multimodal_task_start(&s_multimodal_autostart_task,
|
||||
&s_multimodal_autostart_task_starting)) {
|
||||
return;
|
||||
}
|
||||
|
||||
set_multimodal_stop_flag(&s_multimodal_autostart_stop_requested, false);
|
||||
if (xTaskCreate(multimodal_autostart_task,
|
||||
"mm_autostart",
|
||||
6144,
|
||||
NULL,
|
||||
4,
|
||||
&s_multimodal_autostart_task) != pdPASS) {
|
||||
set_multimodal_task_handle(&s_multimodal_autostart_task, NULL);
|
||||
set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false);
|
||||
ESP_LOGW(TAG, "Auto Start task create failed");
|
||||
return;
|
||||
}
|
||||
set_multimodal_start_flag(&s_multimodal_autostart_task_starting, false);
|
||||
}
|
||||
|
||||
static void stop_watchdog_task(void)
|
||||
{
|
||||
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, true);
|
||||
if (get_multimodal_task_handle(&s_multimodal_watchdog_task) != NULL ||
|
||||
get_multimodal_start_flag(&s_multimodal_watchdog_task_starting)) {
|
||||
if (!wait_task_exit(&s_multimodal_watchdog_task,
|
||||
&s_multimodal_watchdog_task_starting,
|
||||
1200)) {
|
||||
TaskHandle_t task = get_multimodal_task_handle(&s_multimodal_watchdog_task);
|
||||
if (task != NULL) {
|
||||
vTaskDelete(task);
|
||||
}
|
||||
set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL);
|
||||
}
|
||||
}
|
||||
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
||||
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false);
|
||||
}
|
||||
|
||||
static void multimodal_watchdog_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
TaskHandle_t self = xTaskGetCurrentTaskHandle();
|
||||
|
||||
const TickType_t tick_interval = pdMS_TO_TICKS(1000);
|
||||
uint32_t heartbeat_ms = CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS > 0 ?
|
||||
(uint32_t)CONFIG_MULTIMODAL_HEARTBEAT_INTERVAL_MS :
|
||||
50000U;
|
||||
uint32_t timeout_ms = CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS > 0 ?
|
||||
(uint32_t)CONFIG_MULTIMODAL_NO_RESPONSE_TIMEOUT_MS :
|
||||
0U;
|
||||
uint32_t idle_transport_started_ms = 0U;
|
||||
|
||||
while (!get_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested)) {
|
||||
vTaskDelay(tick_interval);
|
||||
|
||||
control_plane_multimodal_runtime_state_t state = {0};
|
||||
if (control_plane_multimodal_get_runtime_state(&state) != ESP_OK) {
|
||||
idle_transport_started_ms = 0U;
|
||||
continue;
|
||||
}
|
||||
|
||||
uint32_t now_ms = control_plane_now_ms();
|
||||
if (state.connected && !state.has_active_task) {
|
||||
if (idle_transport_started_ms == 0U) {
|
||||
idle_transport_started_ms = now_ms;
|
||||
} else if ((now_ms - idle_transport_started_ms) >=
|
||||
CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS) {
|
||||
ESP_LOGI(TAG,
|
||||
"Idle multimodal transport timeout, stopping client after %u ms",
|
||||
(unsigned)CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS);
|
||||
control_plane_multimodal_stop_uplink();
|
||||
stop_autostart_task();
|
||||
control_plane_multimodal_stop();
|
||||
idle_transport_started_ms = 0U;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
idle_transport_started_ms = 0U;
|
||||
|
||||
if (!state.connected || !state.has_active_task) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (state.last_heartbeat_ms > 0U &&
|
||||
(now_ms - state.last_heartbeat_ms) >= heartbeat_ms) {
|
||||
(void)control_plane_multimodal_send_heartbeat();
|
||||
}
|
||||
|
||||
if (timeout_ms > 0U &&
|
||||
state.last_downstream_ms > 0U &&
|
||||
(now_ms - state.last_downstream_ms) >= timeout_ms) {
|
||||
ESP_LOGW(TAG, "No downstream response timeout, sending Stop");
|
||||
(void)control_plane_multimodal_send_stop("NoResponseTimeout");
|
||||
control_plane_multimodal_stop_uplink();
|
||||
}
|
||||
}
|
||||
|
||||
clear_multimodal_task_handle_if_match(&s_multimodal_watchdog_task, self);
|
||||
vTaskDelete(NULL);
|
||||
}
|
||||
|
||||
static void control_plane_downstream_audio_bridge(const uint8_t *data,
|
||||
size_t len,
|
||||
void *user_data)
|
||||
{
|
||||
(void)user_data;
|
||||
|
||||
esp_err_t err = control_plane_multimodal_play_downstream_audio(data, len);
|
||||
if (err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "Downstream audio handle failed: %s", esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
static void control_plane_request_server_error_expression(domain_expression_trigger_t trigger)
|
||||
{
|
||||
if (!control_plane_is_sta_connected()) {
|
||||
ESP_LOGI(TAG,
|
||||
"Skip server error expression while Wi-Fi STA is disconnected (trigger=%d)",
|
||||
(int)trigger);
|
||||
return;
|
||||
}
|
||||
|
||||
const domain_expression_request_t request = {
|
||||
.id = DOMAIN_EXPRESSION_SERVER_ERROR,
|
||||
.trigger = trigger,
|
||||
};
|
||||
|
||||
esp_err_t err = domain_runtime_request_expression(&request);
|
||||
if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) {
|
||||
ESP_LOGW(TAG, "Server error expression request failed: %s", esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
static void control_plane_multimodal_event_bridge(control_plane_multimodal_event_t event,
|
||||
void *user_data)
|
||||
{
|
||||
(void)user_data;
|
||||
|
||||
switch (event) {
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED:
|
||||
if (CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG) {
|
||||
schedule_autostart_if_enabled();
|
||||
}
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED:
|
||||
control_plane_multimodal_stop_uplink();
|
||||
stop_autostart_task();
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED:
|
||||
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_TASK_FAILED);
|
||||
control_plane_multimodal_stop_uplink();
|
||||
schedule_autostart_if_enabled();
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_STARTED:
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_LISTENING:
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_SERVER_NOT_LISTENING:
|
||||
control_plane_multimodal_stop_uplink();
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_STARTED: {
|
||||
if (control_plane_multimodal_is_uplink_running()) {
|
||||
ESP_LOGI(TAG, "Pause uplink during TTS responding");
|
||||
control_plane_multimodal_stop_uplink();
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_RESPONDING_ENDED:
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_SPEECH_STARTED:
|
||||
control_plane_set_model_command_dispatched(false);
|
||||
break;
|
||||
|
||||
case CONTROL_PLANE_MULTIMODAL_EVENT_OUTPUT_ERROR:
|
||||
control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_OUTPUT_ERROR);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t control_plane_start_multimodal_client(void)
|
||||
{
|
||||
esp_err_t prepare_err = control_plane_prepare_for_multimodal_start();
|
||||
if (prepare_err != ESP_OK) {
|
||||
ESP_LOGW(TAG, "Prepare multimodal start failed: %s", esp_err_to_name(prepare_err));
|
||||
}
|
||||
|
||||
esp_err_t callback_err = control_plane_multimodal_register_event_callback(
|
||||
control_plane_multimodal_event_bridge,
|
||||
NULL);
|
||||
if (callback_err != ESP_OK) {
|
||||
return callback_err;
|
||||
}
|
||||
|
||||
callback_err = control_plane_multimodal_register_downstream_audio_callback(
|
||||
control_plane_downstream_audio_bridge,
|
||||
NULL);
|
||||
if (callback_err != ESP_OK) {
|
||||
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
||||
return callback_err;
|
||||
}
|
||||
|
||||
if (get_multimodal_task_handle(&s_multimodal_watchdog_task) == NULL &&
|
||||
!get_multimodal_start_flag(&s_multimodal_watchdog_task_starting) &&
|
||||
try_begin_multimodal_task_start(&s_multimodal_watchdog_task,
|
||||
&s_multimodal_watchdog_task_starting)) {
|
||||
set_multimodal_stop_flag(&s_multimodal_watchdog_stop_requested, false);
|
||||
if (xTaskCreate(multimodal_watchdog_task,
|
||||
"mm_watchdog",
|
||||
CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE,
|
||||
NULL,
|
||||
4,
|
||||
&s_multimodal_watchdog_task) != pdPASS) {
|
||||
ESP_LOGE(TAG, "Watchdog task create failed");
|
||||
set_multimodal_task_handle(&s_multimodal_watchdog_task, NULL);
|
||||
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
||||
(void)control_plane_multimodal_register_downstream_audio_callback(NULL, NULL);
|
||||
(void)control_plane_multimodal_register_event_callback(NULL, NULL);
|
||||
return ESP_ERR_NO_MEM;
|
||||
} else {
|
||||
set_multimodal_start_flag(&s_multimodal_watchdog_task_starting, false);
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t err = control_plane_multimodal_start();
|
||||
if (err != ESP_OK) {
|
||||
stop_watchdog_task();
|
||||
return err;
|
||||
}
|
||||
|
||||
if (CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG && control_plane_multimodal_is_connected()) {
|
||||
schedule_autostart_if_enabled();
|
||||
}
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
void control_plane_stop_multimodal_client(void)
|
||||
{
|
||||
stop_autostart_task();
|
||||
stop_watchdog_task();
|
||||
control_plane_multimodal_stop();
|
||||
}
|
||||
|
||||
esp_err_t control_plane_register_model_command_callback(
|
||||
control_plane_model_command_callback_t cb,
|
||||
void *user_data)
|
||||
{
|
||||
control_plane_set_model_command_callback(cb, user_data);
|
||||
return control_plane_multimodal_register_responding_text_callback(
|
||||
(cb != NULL) ? control_plane_responding_text_bridge : NULL,
|
||||
NULL);
|
||||
}
|
||||
Reference in New Issue
Block a user