From 63c4078d0521a19a833c5eb6a356dcdb51c3fdba Mon Sep 17 00:00:00 2001 From: admin Date: Wed, 25 Feb 2026 12:53:02 +0800 Subject: [PATCH] refactor(domain): split voice interaction implementation --- main/CMakeLists.txt | 3 + .../internal/voice_interaction_internal.h | 125 ++ main/domain/src/voice_interaction.c | 1194 +---------------- main/domain/src/voice_interaction_common.c | 485 +++++++ main/domain/src/voice_interaction_tasks.c | 148 ++ main/domain/src/voice_interaction_ws.c | 570 ++++++++ 6 files changed, 1374 insertions(+), 1151 deletions(-) create mode 100644 main/domain/internal/voice_interaction_internal.h create mode 100644 main/domain/src/voice_interaction_common.c create mode 100644 main/domain/src/voice_interaction_tasks.c create mode 100644 main/domain/src/voice_interaction_ws.c diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index a044e17..0817a3b 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -15,6 +15,9 @@ idf_component_register( "domain/src/image_generation.c" "domain/src/system_runtime.c" "domain/src/voice_interaction.c" + "domain/src/voice_interaction_common.c" + "domain/src/voice_interaction_ws.c" + "domain/src/voice_interaction_tasks.c" "platform/src/platform_bootstrap.c" "platform/src/wifi_manager.c" "platform/src/ble_printer_client.c" diff --git a/main/domain/internal/voice_interaction_internal.h b/main/domain/internal/voice_interaction_internal.h new file mode 100644 index 0000000..4bee06e --- /dev/null +++ b/main/domain/internal/voice_interaction_internal.h @@ -0,0 +1,125 @@ +#pragma once + +#include +#include +#include + +#include "esp_opus_dec.h" +#include "esp_opus_enc.h" +#include "esp_websocket_client.h" +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" +#include "freertos/task.h" +#include "voice_interaction.h" + +#define VOICE_STREAMING_MODE "duplex" +#define VOICE_TASK_GROUP "aigc" +#define VOICE_TASK_NAME "multimodal-generation" +#define VOICE_FUNCTION_NAME "generation" +#define VOICE_MODEL_NAME "multimodal-dialog" + +#define VOICE_SAMPLE_BYTES 2 +#define VOICE_STATUS_LOCK_TIMEOUT_MS 1000 +#define VOICE_WS_SEND_TIMEOUT_MS 1000 +#define VOICE_WS_TASK_STACK 24576 +#define VOICE_UPLINK_TASK_STACK 32768 +#define VOICE_HEARTBEAT_TASK_STACK 4096 +#define VOICE_START_CONNECT_TIMEOUT_MS 8000 +#define VOICE_AUDIO_IO_TIMEOUT_MS 1200 +#define VOICE_AUDIO_DRAIN_MARGIN_MS 120 +#define VOICE_AUDIO_DRAIN_QUIET_MS 180 +#define VOICE_AUDIO_DRAIN_WAIT_MS 2200 +#define VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS 3500 + +typedef struct { + SemaphoreHandle_t lock; + esp_websocket_client_handle_t ws; + + TaskHandle_t uplink_task; + TaskHandle_t heartbeat_task; + bool uplink_task_with_caps; + bool heartbeat_task_with_caps; + + void *opus_enc; + void *opus_dec; + int opus_enc_in_size; + int opus_enc_out_size; + + uint8_t *opus_tx_buf; + int16_t *pcm_tx_buf; + + uint8_t *pcm_rx_buf; + size_t pcm_rx_buf_size; + + uint8_t *text_agg; + size_t text_agg_size; + + uint8_t *bin_agg; + size_t bin_agg_size; + + bool ready; + bool session_active; + bool ws_connected; + bool started; + bool tap_active; + bool stop_requested; + bool ws_low_stack_warned; + + voice_dialog_state_t dialog_state; + + char task_id[40]; + char dialog_id[40]; + char device_uuid[40]; + char last_error[128]; + + int64_t last_event_ms; + int64_t last_downstream_ms; + int64_t playback_deadline_ms; + uint32_t upstream_packets; + uint32_t downstream_packets; +} voice_context_t; + +extern voice_context_t s_voice; + +int64_t voice_now_ms(void); +void voice_fill_err(char *err, size_t err_len, const char *msg); +void voice_set_last_error_locked(const char *msg); +bool voice_lock(uint32_t timeout_ms); +void voice_unlock(void); +void voice_log_status_snapshot(const char *stage); +void voice_log_heap_snapshot(const char *stage); +void voice_wait_audio_playback_done(uint32_t timeout_ms, const char *stage); +void voice_close_audio_with_drain(uint32_t timeout_ms, const char *stage); + +void *voice_malloc_prefer_psram(size_t size); +void *voice_calloc_prefer_psram(size_t n, size_t size); +void *voice_realloc_prefer_psram(void *ptr, size_t size); + +BaseType_t voice_create_task_prefer_psram(TaskFunction_t task_fn, + const char *name, + uint32_t stack_size, + UBaseType_t priority, + TaskHandle_t *out_task, + bool *out_with_caps); +void voice_delete_task(TaskHandle_t task, bool with_caps); +void voice_delete_self_task(bool with_caps); + +void voice_make_uuid(char out[40]); +void voice_make_device_uuid(char out[40]); +const char *voice_safe_user_id(void); + +esp_err_t voice_setup_opus(char *err, size_t err_len); +esp_err_t voice_open_audio_for_dialog(char *err, size_t err_len); +void voice_release_runtime_resources(void); + +esp_err_t voice_send_directive(const char *action, + const char *directive, + bool with_dialog_id); + +void voice_websocket_event_handler(void *handler_args, + esp_event_base_t base, + int32_t event_id, + void *event_data); + +void voice_uplink_task(void *arg); +void voice_heartbeat_task(void *arg); diff --git a/main/domain/src/voice_interaction.c b/main/domain/src/voice_interaction.c index aa92fbc..eec83cc 100644 --- a/main/domain/src/voice_interaction.c +++ b/main/domain/src/voice_interaction.c @@ -1,361 +1,15 @@ -#include "voice_interaction.h" +#include "voice_interaction_internal.h" -#include #include #include #include -#include "cJSON.h" -#include "esp_audio_dec.h" -#include "esp_audio_enc.h" -#include "esp_check.h" #include "esp_crt_bundle.h" #include "esp_log.h" -#include "esp_mac.h" -#include "esp_random.h" -#include "esp_heap_caps.h" -#include "esp_system.h" -#include "esp_timer.h" -#include "esp_websocket_client.h" -#include "esp_opus_dec.h" -#include "esp_opus_enc.h" -#include "freertos/FreeRTOS.h" -#include "freertos/idf_additions.h" -#include "freertos/semphr.h" -#include "freertos/task.h" #include "voice_audio.h" -#define VOICE_STREAMING_MODE "duplex" -#define VOICE_TASK_GROUP "aigc" -#define VOICE_TASK_NAME "multimodal-generation" -#define VOICE_FUNCTION_NAME "generation" -#define VOICE_MODEL_NAME "multimodal-dialog" - -#define VOICE_SAMPLE_BYTES 2 -#define VOICE_STATUS_LOCK_TIMEOUT_MS 1000 -#define VOICE_WS_SEND_TIMEOUT_MS 1000 -#define VOICE_WS_TASK_STACK 24576 -#define VOICE_UPLINK_TASK_STACK 32768 -#define VOICE_HEARTBEAT_TASK_STACK 4096 -#define VOICE_START_CONNECT_TIMEOUT_MS 8000 -#define VOICE_AUDIO_IO_TIMEOUT_MS 1200 - static const char *TAG = "voice_interaction"; -typedef struct { - SemaphoreHandle_t lock; - esp_websocket_client_handle_t ws; - - TaskHandle_t uplink_task; - TaskHandle_t heartbeat_task; - bool uplink_task_with_caps; - bool heartbeat_task_with_caps; - - void *opus_enc; - void *opus_dec; - int opus_enc_in_size; - int opus_enc_out_size; - - uint8_t *opus_tx_buf; - int16_t *pcm_tx_buf; - - uint8_t *pcm_rx_buf; - size_t pcm_rx_buf_size; - - uint8_t *text_agg; - size_t text_agg_size; - - uint8_t *bin_agg; - size_t bin_agg_size; - - bool ready; - bool session_active; - bool ws_connected; - bool started; - bool tap_active; - bool stop_requested; - bool ws_low_stack_warned; - - voice_dialog_state_t dialog_state; - - char task_id[40]; - char dialog_id[40]; - char device_uuid[40]; - char last_error[128]; - - int64_t last_event_ms; - uint32_t upstream_packets; - uint32_t downstream_packets; -} voice_context_t; - -static voice_context_t s_voice; - -static int64_t voice_now_ms(void) { - return esp_timer_get_time() / 1000; -} - -static void voice_fill_err(char *err, size_t err_len, const char *msg) { - if (err != NULL && err_len > 0) { - snprintf(err, err_len, "%s", (msg != NULL) ? msg : "unknown error"); - } -} - -static void voice_set_last_error_locked(const char *msg) { - if (msg == NULL) { - s_voice.last_error[0] = '\0'; - return; - } - strlcpy(s_voice.last_error, msg, sizeof(s_voice.last_error)); -} - -static bool voice_lock(uint32_t timeout_ms) { - return s_voice.lock != NULL && xSemaphoreTake(s_voice.lock, pdMS_TO_TICKS(timeout_ms)) == pdTRUE; -} - -static void voice_unlock(void) { - if (s_voice.lock != NULL) { - xSemaphoreGive(s_voice.lock); - } -} - -static void voice_log_status_snapshot(const char *stage) { - if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - ESP_LOGW(TAG, "[stage] %s: status snapshot lock timeout", stage); - return; - } - - ESP_LOGI(TAG, - "[stage] %s: active=%d ws=%d started=%d tap=%d state=%s up=%" PRIu32 " down=%" PRIu32 " err=%s", - stage, - s_voice.session_active, - s_voice.ws_connected, - s_voice.started, - s_voice.tap_active, - voice_interaction_dialog_state_str(s_voice.dialog_state), - s_voice.upstream_packets, - s_voice.downstream_packets, - s_voice.last_error[0] != '\0' ? s_voice.last_error : "-"); - voice_unlock(); -} - -static void voice_log_heap_snapshot(const char *stage) { - size_t free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); - size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - size_t free_spiram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - size_t largest_8bit = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT); - size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); - size_t largest_spiram = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - ESP_LOGI(TAG, - "[stage] %s heap: free8=%u free_internal=%u free_spiram=%u largest8=%u largest_internal=%u largest_spiram=%u", - stage, - (unsigned)free_8bit, - (unsigned)free_internal, - (unsigned)free_spiram, - (unsigned)largest_8bit, - (unsigned)largest_internal, - (unsigned)largest_spiram); -} - -static void *voice_malloc_prefer_psram(size_t size) { - void *ptr = heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - if (ptr == NULL) { - ptr = malloc(size); - } - return ptr; -} - -static void *voice_calloc_prefer_psram(size_t n, size_t size) { - void *ptr = heap_caps_calloc(n, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - if (ptr == NULL) { - ptr = calloc(n, size); - } - return ptr; -} - -static void *voice_realloc_prefer_psram(void *ptr, size_t size) { - void *new_ptr = heap_caps_realloc(ptr, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - if (new_ptr == NULL) { - new_ptr = realloc(ptr, size); - } - return new_ptr; -} - -static BaseType_t voice_create_task_prefer_psram(TaskFunction_t task_fn, - const char *name, - uint32_t stack_size, - UBaseType_t priority, - TaskHandle_t *out_task, - bool *out_with_caps) { - if (out_with_caps != NULL) { - *out_with_caps = false; - } - - BaseType_t rc = pdFAIL; - -#if defined(CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY) && \ - defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && \ - (configSUPPORT_STATIC_ALLOCATION == 1) - rc = xTaskCreatePinnedToCoreWithCaps(task_fn, - name, - (configSTACK_DEPTH_TYPE)stack_size, - NULL, - priority, - out_task, - tskNO_AFFINITY, - MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - if (rc == pdPASS) { - if (out_with_caps != NULL) { - *out_with_caps = true; - } - ESP_LOGI(TAG, "[stage] task_create_psram_ok: name=%s stack=%u", name, (unsigned)stack_size); - return pdPASS; - } - ESP_LOGW(TAG, "[stage] task_create_psram_failed: name=%s stack=%u", name, (unsigned)stack_size); -#endif - - rc = xTaskCreate(task_fn, - name, - (configSTACK_DEPTH_TYPE)stack_size, - NULL, - priority, - out_task); - if (rc == pdPASS) { - ESP_LOGI(TAG, "[stage] task_create_internal_ok: name=%s stack=%u", name, (unsigned)stack_size); - } else { - ESP_LOGE(TAG, "[stage] task_create_internal_failed: name=%s stack=%u", name, (unsigned)stack_size); - } - return rc; -} - -static void voice_delete_task(TaskHandle_t task, bool with_caps) { - if (task == NULL) { - return; - } - -#if (configSUPPORT_STATIC_ALLOCATION == 1) - if (with_caps) { - vTaskDeleteWithCaps(task); - return; - } -#else - (void)with_caps; -#endif - vTaskDelete(task); -} - -static void voice_delete_self_task(bool with_caps) { -#if (configSUPPORT_STATIC_ALLOCATION == 1) - if (with_caps) { - vTaskDeleteWithCaps(NULL); - return; - } -#else - (void)with_caps; -#endif - vTaskDelete(NULL); -} - -static void voice_make_uuid(char out[40]) { - uint8_t raw[16]; - esp_fill_random(raw, sizeof(raw)); - - raw[6] = (uint8_t)((raw[6] & 0x0F) | 0x40); - raw[8] = (uint8_t)((raw[8] & 0x3F) | 0x80); - - snprintf(out, - 40, - "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", - raw[0], - raw[1], - raw[2], - raw[3], - raw[4], - raw[5], - raw[6], - raw[7], - raw[8], - raw[9], - raw[10], - raw[11], - raw[12], - raw[13], - raw[14], - raw[15]); -} - -static void voice_make_device_uuid(char out[40]) { - if (strlen(CONFIG_TQ_VOICE_DEVICE_UUID) > 0) { - strlcpy(out, CONFIG_TQ_VOICE_DEVICE_UUID, 40); - return; - } - - uint8_t mac[6] = {0}; - esp_read_mac(mac, ESP_MAC_WIFI_STA); - - uint32_t r = esp_random(); - snprintf(out, - 40, - "%02x%02x%02x%02x-%02x%02x-4%03x-%04x-%012" PRIx32, - mac[0], - mac[1], - mac[2], - mac[3], - mac[4], - mac[5], - (unsigned)(r & 0x0FFF), - (unsigned)((r >> 12) & 0xFFFF), - esp_random()); -} - -static const char *voice_safe_user_id(void) { - if (strlen(CONFIG_TQ_VOICE_USER_ID) > 0) { - return CONFIG_TQ_VOICE_USER_ID; - } - return "esp32-user"; -} - -static esp_opus_enc_frame_duration_t voice_to_enc_frame_duration(int frame_ms) { - switch (frame_ms) { - case 10: - return ESP_OPUS_ENC_FRAME_DURATION_10_MS; - case 20: - return ESP_OPUS_ENC_FRAME_DURATION_20_MS; - case 40: - return ESP_OPUS_ENC_FRAME_DURATION_40_MS; - case 60: - return ESP_OPUS_ENC_FRAME_DURATION_60_MS; - case 80: - return ESP_OPUS_ENC_FRAME_DURATION_80_MS; - case 100: - return ESP_OPUS_ENC_FRAME_DURATION_100_MS; - case 120: - return ESP_OPUS_ENC_FRAME_DURATION_120_MS; - default: - return ESP_OPUS_ENC_FRAME_DURATION_ARG; - } -} - -static esp_opus_dec_frame_duration_t voice_to_dec_frame_duration(int frame_ms) { - switch (frame_ms) { - case 10: - return ESP_OPUS_DEC_FRAME_DURATION_10_MS; - case 20: - return ESP_OPUS_DEC_FRAME_DURATION_20_MS; - case 40: - return ESP_OPUS_DEC_FRAME_DURATION_40_MS; - case 60: - return ESP_OPUS_DEC_FRAME_DURATION_60_MS; - case 80: - return ESP_OPUS_DEC_FRAME_DURATION_80_MS; - case 100: - return ESP_OPUS_DEC_FRAME_DURATION_100_MS; - case 120: - return ESP_OPUS_DEC_FRAME_DURATION_120_MS; - default: - return ESP_OPUS_DEC_FRAME_DURATION_INVALID; - } -} - const char *voice_interaction_dialog_state_str(voice_dialog_state_t state) { switch (state) { case VOICE_DIALOG_STATE_IDLE: @@ -371,790 +25,6 @@ const char *voice_interaction_dialog_state_str(voice_dialog_state_t state) { } } -static char *voice_build_directive_message(const char *action, - const char *directive, - bool with_dialog_id) { - cJSON *root = cJSON_CreateObject(); - if (root == NULL) { - return NULL; - } - - cJSON *header = cJSON_AddObjectToObject(root, "header"); - cJSON_AddStringToObject(header, "action", action); - cJSON_AddStringToObject(header, "task_id", s_voice.task_id); - cJSON_AddStringToObject(header, "streaming", VOICE_STREAMING_MODE); - - cJSON *payload = cJSON_AddObjectToObject(root, "payload"); - cJSON *input = cJSON_AddObjectToObject(payload, "input"); - cJSON_AddStringToObject(input, "directive", directive); - if (with_dialog_id && s_voice.dialog_id[0] != '\0') { - cJSON_AddStringToObject(input, "dialog_id", s_voice.dialog_id); - } - - char *text = cJSON_PrintUnformatted(root); - cJSON_Delete(root); - return text; -} - -static char *voice_build_start_message(void) { - cJSON *root = cJSON_CreateObject(); - if (root == NULL) { - return NULL; - } - - cJSON *header = cJSON_AddObjectToObject(root, "header"); - cJSON_AddStringToObject(header, "action", "run-task"); - cJSON_AddStringToObject(header, "task_id", s_voice.task_id); - cJSON_AddStringToObject(header, "streaming", VOICE_STREAMING_MODE); - - cJSON *payload = cJSON_AddObjectToObject(root, "payload"); - cJSON_AddStringToObject(payload, "task_group", VOICE_TASK_GROUP); - cJSON_AddStringToObject(payload, "task", VOICE_TASK_NAME); - cJSON_AddStringToObject(payload, "function", VOICE_FUNCTION_NAME); - cJSON_AddStringToObject(payload, "model", VOICE_MODEL_NAME); - - cJSON *input = cJSON_AddObjectToObject(payload, "input"); - cJSON_AddStringToObject(input, "directive", "Start"); - cJSON_AddStringToObject(input, "workspace_id", CONFIG_TQ_VOICE_WORKSPACE_ID); - cJSON_AddStringToObject(input, "app_id", CONFIG_TQ_VOICE_APP_ID); - - cJSON *parameters = cJSON_AddObjectToObject(payload, "parameters"); - cJSON *upstream = cJSON_AddObjectToObject(parameters, "upstream"); - cJSON_AddStringToObject(upstream, "type", "AudioOnly"); - cJSON_AddStringToObject(upstream, "mode", "tap2talk"); - cJSON_AddStringToObject(upstream, "audio_format", "raw-opus"); - cJSON_AddNumberToObject(upstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE); - - cJSON *downstream = cJSON_AddObjectToObject(parameters, "downstream"); - cJSON_AddNumberToObject(downstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE); - cJSON_AddStringToObject(downstream, "audio_format", "raw-opus"); - cJSON_AddNumberToObject(downstream, "frame_size", CONFIG_TQ_VOICE_OPUS_FRAME_MS); - cJSON_AddNumberToObject(downstream, - "bit_rate", - CONFIG_TQ_VOICE_OPUS_BITRATE_KBPS); - if (strlen(CONFIG_TQ_VOICE_TTS_VOICE) > 0) { - cJSON_AddStringToObject(downstream, "voice", CONFIG_TQ_VOICE_TTS_VOICE); - } - - cJSON *client_info = cJSON_AddObjectToObject(parameters, "client_info"); - cJSON_AddStringToObject(client_info, "user_id", voice_safe_user_id()); - cJSON *device = cJSON_AddObjectToObject(client_info, "device"); - cJSON_AddStringToObject(device, "uuid", s_voice.device_uuid); - - char *text = cJSON_PrintUnformatted(root); - cJSON_Delete(root); - return text; -} - -static esp_err_t voice_send_text(const char *text, int len) { - if (text == NULL || len <= 0) { - return ESP_ERR_INVALID_ARG; - } - - esp_websocket_client_handle_t ws = NULL; - if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - return ESP_ERR_TIMEOUT; - } - ws = s_voice.ws; - bool connected = s_voice.ws_connected; - voice_unlock(); - - if (ws == NULL || !connected) { - ESP_LOGW(TAG, "[stage] ws_send_text skipped: ws=%p connected=%d len=%d", (void *)ws, connected, len); - return ESP_ERR_INVALID_STATE; - } - - int ret = esp_websocket_client_send_text(ws, - text, - len, - pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS)); - if (ret < 0) { - ESP_LOGW(TAG, "[stage] ws_send_text failed: len=%d", len); - return ESP_FAIL; - } - ESP_LOGI(TAG, "[stage] ws_send_text ok: len=%d", len); - return ESP_OK; -} - -static esp_err_t voice_send_directive(const char *action, - const char *directive, - bool with_dialog_id) { - char *text = NULL; - if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - return ESP_ERR_TIMEOUT; - } - text = voice_build_directive_message(action, directive, with_dialog_id); - voice_unlock(); - - if (text == NULL) { - return ESP_ERR_NO_MEM; - } - - esp_err_t err = voice_send_text(text, (int)strlen(text)); - ESP_LOGI(TAG, - "[stage] directive: action=%s directive=%s with_dialog_id=%d result=%s", - action, - directive, - with_dialog_id, - esp_err_to_name(err)); - cJSON_free(text); - return err; -} - -static void voice_handle_downstream_packet(const uint8_t *data, size_t len) { - if (data == NULL || len == 0) { - return; - } - - if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - return; - } - - void *opus_dec = s_voice.opus_dec; - uint8_t *pcm_buf = s_voice.pcm_rx_buf; - size_t pcm_buf_size = s_voice.pcm_rx_buf_size; - voice_unlock(); - - if (opus_dec == NULL || pcm_buf == NULL || pcm_buf_size == 0) { - return; - } - - esp_audio_dec_in_raw_t raw = { - .buffer = (uint8_t *)data, - .len = (uint32_t)len, - .consumed = 0, - }; - - while (raw.len > 0) { - esp_audio_dec_out_frame_t out = { - .buffer = pcm_buf, - .len = (uint32_t)pcm_buf_size, - .needed_size = 0, - .decoded_size = 0, - }; - esp_audio_dec_info_t dec_info = {0}; - - esp_audio_err_t ret = esp_opus_dec_decode(opus_dec, &raw, &out, &dec_info); - if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) { - if (out.needed_size > pcm_buf_size) { - uint8_t *new_buf = (uint8_t *)voice_realloc_prefer_psram(pcm_buf, out.needed_size); - if (new_buf == NULL) { - ESP_LOGE(TAG, "No memory to extend pcm rx buffer to %u", out.needed_size); - break; - } - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - s_voice.pcm_rx_buf = new_buf; - s_voice.pcm_rx_buf_size = out.needed_size; - voice_unlock(); - } - pcm_buf = new_buf; - pcm_buf_size = out.needed_size; - continue; - } - break; - } - - if (ret != ESP_AUDIO_ERR_OK) { - ESP_LOGW(TAG, "opus decode failed: %d", ret); - break; - } - - if (out.decoded_size > 0) { - char audio_err[64] = {0}; - (void)voice_audio_write_pcm((const int16_t *)out.buffer, - out.decoded_size / VOICE_SAMPLE_BYTES, - VOICE_AUDIO_IO_TIMEOUT_MS, - audio_err, - sizeof(audio_err)); - } - - if (raw.consumed == 0 || raw.consumed > raw.len) { - break; - } - raw.buffer += raw.consumed; - raw.len -= raw.consumed; - } - - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - s_voice.downstream_packets++; - s_voice.last_event_ms = voice_now_ms(); - voice_unlock(); - } -} - -static const char *voice_get_json_str(cJSON *obj, const char *name) { - if (obj == NULL || name == NULL) { - return NULL; - } - cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); - if (cJSON_IsString(item) && item->valuestring != NULL) { - return item->valuestring; - } - return NULL; -} - -static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) { - if (obj == NULL || name == NULL || out_value == NULL) { - return false; - } - cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); - if (!cJSON_IsBool(item)) { - return false; - } - *out_value = cJSON_IsTrue(item); - return true; -} - -static void voice_log_final_text(const char *stage, const char *text) { - if (stage == NULL || text == NULL || text[0] == '\0') { - return; - } - size_t len = strlen(text); - const int preview = 240; - ESP_LOGI(TAG, - "[stage] %s: len=%u text=%.*s%s", - stage, - (unsigned)len, - preview, - text, - (len > (size_t)preview) ? "..." : ""); -} - -static void voice_handle_output_event(cJSON *output) { - const char *event_name = voice_get_json_str(output, "event"); - if (event_name == NULL) { - return; - } - - const char *dialog_id = voice_get_json_str(output, "dialog_id"); - const char *state = NULL; - if (strcmp(event_name, "DialogStateChanged") == 0) { - state = voice_get_json_str(output, "state"); - } - ESP_LOGI(TAG, - "[stage] ws_event_output: event=%s dialog_id=%s state=%s", - event_name, - dialog_id != NULL ? dialog_id : "-", - state != NULL ? state : "-"); - - bool finished = false; - bool has_finished = voice_get_json_bool(output, "finished", &finished); - if (has_finished && finished) { - if (strcmp(event_name, "SpeechContent") == 0) { - voice_log_final_text("asr_final_text", voice_get_json_str(output, "text")); - } else if (strcmp(event_name, "RespondingContent") == 0) { - const char *final_text = voice_get_json_str(output, "text"); - if (final_text == NULL || final_text[0] == '\0') { - final_text = voice_get_json_str(output, "spoken"); - } - voice_log_final_text("response_final_text", final_text); - } - } - - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - if (dialog_id != NULL) { - strlcpy(s_voice.dialog_id, dialog_id, sizeof(s_voice.dialog_id)); - } - s_voice.last_event_ms = voice_now_ms(); - - if (strcmp(event_name, "Started") == 0) { - s_voice.started = true; - s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; - ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id); - } else if (strcmp(event_name, "DialogStateChanged") == 0) { - if (state != NULL) { - if (strcmp(state, "Listening") == 0) { - s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING; - } else if (strcmp(state, "Thinking") == 0) { - s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING; - } else if (strcmp(state, "Responding") == 0) { - s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING; - } - } - ESP_LOGI(TAG, - "[stage] dialog_state_changed: state=%s current=%s", - state != NULL ? state : "-", - voice_interaction_dialog_state_str(s_voice.dialog_state)); - } else if (strcmp(event_name, "SpeechEnded") == 0) { - ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active); - } else if (strcmp(event_name, "Stopped") == 0) { - s_voice.started = false; - s_voice.tap_active = false; - s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; - ESP_LOGI(TAG, "[stage] session_stopped_by_server"); - } else if (strcmp(event_name, "Error") == 0) { - const char *error_msg = voice_get_json_str(output, "error_message"); - if (error_msg != NULL) { - voice_set_last_error_locked(error_msg); - ESP_LOGW(TAG, "[stage] server_error: %s", error_msg); - } - s_voice.tap_active = false; - } - voice_unlock(); - } - - if (strcmp(event_name, "RespondingStarted") == 0) { - (void)voice_send_directive("continue-task", "LocalRespondingStarted", true); - } else if (strcmp(event_name, "RespondingEnded") == 0) { - (void)voice_send_directive("continue-task", "LocalRespondingEnded", true); - } -} - -static void voice_handle_text_message(const char *text, size_t len) { - cJSON *root = cJSON_ParseWithLength(text, len); - if (root == NULL) { - ESP_LOGW(TAG, "Invalid ws text payload"); - return; - } - - cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header"); - const char *header_event = voice_get_json_str(header, "event"); - - cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload"); - cJSON *output = NULL; - if (payload != NULL) { - output = cJSON_GetObjectItemCaseSensitive(payload, "output"); - } - - if (header != NULL) { - cJSON *status_code = cJSON_GetObjectItemCaseSensitive(header, "status_code"); - cJSON *status_msg = cJSON_GetObjectItemCaseSensitive(header, "status_message"); - if (cJSON_IsNumber(status_code) && status_code->valueint >= 400) { - ESP_LOGW(TAG, - "[stage] ws_header_error: status_code=%d status_message=%s", - status_code->valueint, - (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) ? status_msg->valuestring : "-"); - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - if (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) { - voice_set_last_error_locked(status_msg->valuestring); - } else { - voice_set_last_error_locked("websocket status error"); - } - voice_unlock(); - } - } - } - - if (output != NULL) { - voice_handle_output_event(output); - } else if (header_event != NULL && strcmp(header_event, "task-failed") == 0) { - const char *error_msg = voice_get_json_str(header, "error_message"); - ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed"); - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed"); - voice_unlock(); - } - } - - cJSON_Delete(root); -} - -static void voice_handle_text_chunk(esp_websocket_event_data_t *data) { - if (data->payload_offset == 0) { - free(s_voice.text_agg); - s_voice.text_agg = NULL; - s_voice.text_agg_size = 0; - - if (data->payload_len <= 0 || data->payload_len > 4096) { - return; - } - - s_voice.text_agg = (uint8_t *)voice_calloc_prefer_psram(1, (size_t)data->payload_len + 1); - if (s_voice.text_agg == NULL) { - ESP_LOGW(TAG, "[stage] text_agg_alloc_failed: len=%d", data->payload_len); - return; - } - s_voice.text_agg_size = (size_t)data->payload_len; - } - - if (s_voice.text_agg == NULL || s_voice.text_agg_size == 0) { - return; - } - - if ((size_t)data->payload_offset + (size_t)data->data_len > s_voice.text_agg_size) { - free(s_voice.text_agg); - s_voice.text_agg = NULL; - s_voice.text_agg_size = 0; - return; - } - - memcpy(s_voice.text_agg + data->payload_offset, data->data_ptr, (size_t)data->data_len); - - bool complete = data->fin && - ((size_t)data->payload_offset + (size_t)data->data_len == s_voice.text_agg_size); - if (!complete) { - return; - } - - voice_handle_text_message((const char *)s_voice.text_agg, s_voice.text_agg_size); - free(s_voice.text_agg); - s_voice.text_agg = NULL; - s_voice.text_agg_size = 0; -} - -static void voice_handle_binary_chunk(esp_websocket_event_data_t *data) { - if (data->payload_offset == 0) { - free(s_voice.bin_agg); - s_voice.bin_agg = NULL; - s_voice.bin_agg_size = 0; - - if (data->payload_len <= 0 || data->payload_len > 16384) { - return; - } - - s_voice.bin_agg = (uint8_t *)voice_malloc_prefer_psram((size_t)data->payload_len); - if (s_voice.bin_agg == NULL) { - ESP_LOGW(TAG, "[stage] bin_agg_alloc_failed: len=%d", data->payload_len); - return; - } - s_voice.bin_agg_size = (size_t)data->payload_len; - } - - if (s_voice.bin_agg == NULL || s_voice.bin_agg_size == 0) { - return; - } - - if ((size_t)data->payload_offset + (size_t)data->data_len > s_voice.bin_agg_size) { - free(s_voice.bin_agg); - s_voice.bin_agg = NULL; - s_voice.bin_agg_size = 0; - return; - } - - memcpy(s_voice.bin_agg + data->payload_offset, data->data_ptr, (size_t)data->data_len); - - bool complete = data->fin && - ((size_t)data->payload_offset + (size_t)data->data_len == s_voice.bin_agg_size); - if (!complete) { - return; - } - - voice_handle_downstream_packet(s_voice.bin_agg, s_voice.bin_agg_size); - free(s_voice.bin_agg); - s_voice.bin_agg = NULL; - s_voice.bin_agg_size = 0; -} - -static void voice_websocket_event_handler(void *handler_args, - esp_event_base_t base, - int32_t event_id, - void *event_data) { - (void)handler_args; - (void)base; - - UBaseType_t ws_hwm = uxTaskGetStackHighWaterMark(NULL); - if (!s_voice.ws_low_stack_warned && ws_hwm < 256) { - s_voice.ws_low_stack_warned = true; - ESP_LOGW(TAG, "[stage] ws_task_low_stack: hwm_words=%u", (unsigned)ws_hwm); - } - - esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; - switch ((esp_websocket_event_id_t)event_id) { - case WEBSOCKET_EVENT_CONNECTED: { - ESP_LOGI(TAG, "[stage] ws_connected"); - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - s_voice.ws_connected = true; - s_voice.last_event_ms = voice_now_ms(); - voice_unlock(); - } - - char *start_msg = NULL; - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - start_msg = voice_build_start_message(); - voice_unlock(); - } - if (start_msg != NULL) { - esp_err_t send_err = voice_send_text(start_msg, (int)strlen(start_msg)); - ESP_LOGI(TAG, "[stage] start_message_sent: result=%s", esp_err_to_name(send_err)); - cJSON_free(start_msg); - } else { - ESP_LOGE(TAG, "[stage] start_message_build_failed"); - } - break; - } - case WEBSOCKET_EVENT_DISCONNECTED: - ESP_LOGW(TAG, "[stage] ws_disconnected"); - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - s_voice.ws_connected = false; - s_voice.started = false; - s_voice.tap_active = false; - s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; - s_voice.last_event_ms = voice_now_ms(); - voice_unlock(); - } - break; - case WEBSOCKET_EVENT_DATA: - if (data == NULL || data->data_ptr == NULL || data->data_len <= 0) { - break; - } - if (data->op_code == WS_TRANSPORT_OPCODES_TEXT) { - voice_handle_text_chunk(data); - } else if (data->op_code == WS_TRANSPORT_OPCODES_BINARY) { - voice_handle_binary_chunk(data); - } - break; - case WEBSOCKET_EVENT_ERROR: - ESP_LOGW(TAG, "[stage] ws_error"); - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - voice_set_last_error_locked("websocket error"); - s_voice.last_event_ms = voice_now_ms(); - voice_unlock(); - } - break; - default: - break; - } -} - -static void voice_uplink_task(void *arg) { - (void)arg; - - const int frame_ms = CONFIG_TQ_VOICE_OPUS_FRAME_MS; - bool low_stack_warned = false; - UBaseType_t start_hwm = uxTaskGetStackHighWaterMark(NULL); - ESP_LOGI(TAG, "[stage] uplink_task_started: stack_hwm_words=%u", (unsigned)start_hwm); - - while (true) { - if (!low_stack_warned) { - UBaseType_t hwm = uxTaskGetStackHighWaterMark(NULL); - if (hwm < 256) { - low_stack_warned = true; - ESP_LOGW(TAG, "[stage] uplink_task_low_stack: hwm_words=%u", (unsigned)hwm); - } - } - - bool active = false; - bool connected = false; - bool started = false; - bool tap_active = false; - bool listening = false; - - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - active = s_voice.session_active && !s_voice.stop_requested; - connected = s_voice.ws_connected; - started = s_voice.started; - tap_active = s_voice.tap_active; - listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING); - voice_unlock(); - } - - if (!active) { - break; - } - - if (!(connected && started && listening)) { - vTaskDelay(pdMS_TO_TICKS(20)); - continue; - } - - if (tap_active) { - char read_err[64] = {0}; - esp_err_t rc = voice_audio_read_pcm(s_voice.pcm_tx_buf, - (size_t)s_voice.opus_enc_in_size / VOICE_SAMPLE_BYTES, - VOICE_AUDIO_IO_TIMEOUT_MS, - read_err, - sizeof(read_err)); - if (rc != ESP_OK) { - memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size); - } - } else { - memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size); - vTaskDelay(pdMS_TO_TICKS(frame_ms)); - } - - esp_audio_enc_in_frame_t in_frame = { - .buffer = (uint8_t *)s_voice.pcm_tx_buf, - .len = (uint32_t)s_voice.opus_enc_in_size, - }; - esp_audio_enc_out_frame_t out_frame = { - .buffer = s_voice.opus_tx_buf, - .len = (uint32_t)s_voice.opus_enc_out_size, - .encoded_bytes = 0, - }; - - esp_audio_err_t enc_err = esp_opus_enc_process(s_voice.opus_enc, &in_frame, &out_frame); - if (enc_err != ESP_AUDIO_ERR_OK || out_frame.encoded_bytes == 0) { - continue; - } - - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - esp_websocket_client_handle_t ws = s_voice.ws; - bool can_send = s_voice.ws_connected; - voice_unlock(); - - if (ws != NULL && can_send) { - int ret = esp_websocket_client_send_bin(ws, - (const char *)s_voice.opus_tx_buf, - (int)out_frame.encoded_bytes, - pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS)); - if (ret >= 0) { - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - s_voice.upstream_packets++; - s_voice.last_event_ms = voice_now_ms(); - voice_unlock(); - } - } - } - } - } - - bool delete_with_caps = false; - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - delete_with_caps = s_voice.uplink_task_with_caps; - s_voice.uplink_task = NULL; - s_voice.uplink_task_with_caps = false; - voice_unlock(); - } - voice_delete_self_task(delete_with_caps); -} - -static void voice_heartbeat_task(void *arg) { - (void)arg; - - while (true) { - bool active = false; - bool connected = false; - bool started = false; - - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - active = s_voice.session_active && !s_voice.stop_requested; - connected = s_voice.ws_connected; - started = s_voice.started; - voice_unlock(); - } - - if (!active) { - break; - } - - if (connected && started) { - ESP_LOGI(TAG, "[stage] heartbeat_send"); - (void)voice_send_directive("continue-task", "HeartBeat", true); - } - - vTaskDelay(pdMS_TO_TICKS(CONFIG_TQ_VOICE_HEARTBEAT_SEC * 1000)); - } - - bool delete_with_caps = false; - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - delete_with_caps = s_voice.heartbeat_task_with_caps; - s_voice.heartbeat_task = NULL; - s_voice.heartbeat_task_with_caps = false; - voice_unlock(); - } - voice_delete_self_task(delete_with_caps); -} - -static esp_err_t voice_setup_opus(char *err, size_t err_len) { - esp_opus_enc_frame_duration_t enc_duration = - voice_to_enc_frame_duration(CONFIG_TQ_VOICE_OPUS_FRAME_MS); - esp_opus_dec_frame_duration_t dec_duration = - voice_to_dec_frame_duration(CONFIG_TQ_VOICE_OPUS_FRAME_MS); - - if (enc_duration == ESP_OPUS_ENC_FRAME_DURATION_ARG || - dec_duration == ESP_OPUS_DEC_FRAME_DURATION_INVALID) { - voice_fill_err(err, err_len, "invalid opus frame duration"); - return ESP_ERR_INVALID_ARG; - } - - esp_opus_enc_config_t enc_cfg = ESP_OPUS_ENC_CONFIG_DEFAULT(); - enc_cfg.sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE; - enc_cfg.channel = ESP_AUDIO_MONO; - enc_cfg.bits_per_sample = ESP_AUDIO_BIT16; - enc_cfg.frame_duration = enc_duration; - enc_cfg.bitrate = CONFIG_TQ_VOICE_OPUS_BITRATE_KBPS * 1000; - enc_cfg.application_mode = ESP_OPUS_ENC_APPLICATION_VOIP; - enc_cfg.enable_vbr = true; - - esp_audio_err_t enc_err = esp_opus_enc_open(&enc_cfg, - sizeof(enc_cfg), - &s_voice.opus_enc); - if (enc_err != ESP_AUDIO_ERR_OK || s_voice.opus_enc == NULL) { - voice_fill_err(err, err_len, "opus encoder open failed"); - return ESP_FAIL; - } - - enc_err = esp_opus_enc_get_frame_size(s_voice.opus_enc, - &s_voice.opus_enc_in_size, - &s_voice.opus_enc_out_size); - if (enc_err != ESP_AUDIO_ERR_OK || - s_voice.opus_enc_in_size <= 0 || - s_voice.opus_enc_out_size <= 0) { - voice_fill_err(err, err_len, "opus frame size query failed"); - return ESP_FAIL; - } - - s_voice.pcm_tx_buf = (int16_t *)voice_malloc_prefer_psram((size_t)s_voice.opus_enc_in_size); - s_voice.opus_tx_buf = (uint8_t *)voice_malloc_prefer_psram((size_t)s_voice.opus_enc_out_size); - if (s_voice.pcm_tx_buf == NULL || s_voice.opus_tx_buf == NULL) { - voice_fill_err(err, err_len, "no memory for opus tx buffers"); - return ESP_ERR_NO_MEM; - } - - esp_opus_dec_cfg_t dec_cfg = ESP_OPUS_DEC_CONFIG_DEFAULT(); - dec_cfg.sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE; - dec_cfg.channel = ESP_AUDIO_MONO; - dec_cfg.frame_duration = dec_duration; - dec_cfg.self_delimited = false; - - esp_audio_err_t dec_err = esp_opus_dec_open(&dec_cfg, - sizeof(dec_cfg), - &s_voice.opus_dec); - if (dec_err != ESP_AUDIO_ERR_OK || s_voice.opus_dec == NULL) { - voice_fill_err(err, err_len, "opus decoder open failed"); - return ESP_FAIL; - } - - size_t rx_pcm_size = (size_t)CONFIG_TQ_VOICE_SAMPLE_RATE * - VOICE_SAMPLE_BYTES * - (size_t)CONFIG_TQ_VOICE_OPUS_FRAME_MS * - 2 / 1000; - if (rx_pcm_size < 2048) { - rx_pcm_size = 2048; - } - - s_voice.pcm_rx_buf = (uint8_t *)voice_malloc_prefer_psram(rx_pcm_size); - if (s_voice.pcm_rx_buf == NULL) { - voice_fill_err(err, err_len, "no memory for opus rx buffer"); - return ESP_ERR_NO_MEM; - } - s_voice.pcm_rx_buf_size = rx_pcm_size; - - return ESP_OK; -} - -static void voice_release_runtime_resources(void) { - if (s_voice.opus_enc != NULL) { - esp_opus_enc_close(s_voice.opus_enc); - s_voice.opus_enc = NULL; - } - if (s_voice.opus_dec != NULL) { - (void)esp_opus_dec_close(s_voice.opus_dec); - s_voice.opus_dec = NULL; - } - - free(s_voice.pcm_tx_buf); - s_voice.pcm_tx_buf = NULL; - - free(s_voice.opus_tx_buf); - s_voice.opus_tx_buf = NULL; - - free(s_voice.pcm_rx_buf); - s_voice.pcm_rx_buf = NULL; - s_voice.pcm_rx_buf_size = 0; - - free(s_voice.text_agg); - s_voice.text_agg = NULL; - s_voice.text_agg_size = 0; - - free(s_voice.bin_agg); - s_voice.bin_agg = NULL; - s_voice.bin_agg_size = 0; - - s_voice.opus_enc_in_size = 0; - s_voice.opus_enc_out_size = 0; -} - esp_err_t voice_interaction_init(void) { if (s_voice.ready) { return ESP_OK; @@ -1210,17 +80,6 @@ esp_err_t voice_interaction_start(char *err, size_t err_len) { return ESP_OK; } - voice_audio_open_config_t audio_cfg = { - .sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE, - .channels = 1, - .bits_per_sample = 16, - .output_volume = CONFIG_TQ_VOICE_CODEC_OUT_VOL, - .input_gain_db = (float)CONFIG_TQ_VOICE_CODEC_MIC_GAIN_DB, - }; - - ESP_LOGI(TAG, "[stage] audio_open_begin"); - ESP_RETURN_ON_ERROR(voice_audio_open(&audio_cfg, err, err_len), TAG, "voice audio open failed"); - ESP_LOGI(TAG, "[stage] audio_open_ok"); ESP_LOGI(TAG, "[stage] opus_setup_begin"); esp_err_t opus_setup_err = voice_setup_opus(err, err_len); if (opus_setup_err != ESP_OK) { @@ -1236,6 +95,8 @@ esp_err_t voice_interaction_start(char *err, size_t err_len) { s_voice.last_error[0] = '\0'; s_voice.upstream_packets = 0; s_voice.downstream_packets = 0; + s_voice.last_downstream_ms = 0; + s_voice.playback_deadline_ms = 0; s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; s_voice.stop_requested = false; s_voice.tap_active = false; @@ -1328,11 +189,11 @@ esp_err_t voice_interaction_start(char *err, size_t err_len) { ESP_LOGI(TAG, "[stage] uplink_task_create_begin"); BaseType_t ok = voice_create_task_prefer_psram(voice_uplink_task, - "voice_uplink", - VOICE_UPLINK_TASK_STACK, - 5, - &s_voice.uplink_task, - &s_voice.uplink_task_with_caps); + "voice_uplink", + VOICE_UPLINK_TASK_STACK, + 5, + &s_voice.uplink_task, + &s_voice.uplink_task_with_caps); if (ok != pdPASS) { voice_fill_err(err, err_len, "create uplink task failed"); ESP_LOGE(TAG, "[stage] uplink_task_create_failed"); @@ -1397,7 +258,7 @@ esp_err_t voice_interaction_start(char *err, size_t err_len) { exit_err: ESP_LOGE(TAG, "[stage] session_start_exit_err: %s", (err != NULL && err[0] != '\0') ? err : "unknown"); voice_release_runtime_resources(); - voice_audio_close(); + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "session_start_exit_err_close"); return ESP_FAIL; } @@ -1418,6 +279,7 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) { if (!s_voice.session_active) { voice_unlock(); + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "session_stop_skip_close"); ESP_LOGI(TAG, "[stage] session_stop_skip: not active"); return ESP_OK; } @@ -1437,6 +299,7 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) { if (ws_connected) { (void)voice_send_directive("finish-task", "Stop", true); } + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS, "session_stop_close"); voice_delete_task(uplink_task, uplink_task_with_caps); voice_delete_task(heartbeat_task, heartbeat_task_with_caps); @@ -1463,12 +326,14 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) { s_voice.stop_requested = false; s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; s_voice.dialog_id[0] = '\0'; + s_voice.last_downstream_ms = 0; + s_voice.playback_deadline_ms = 0; voice_unlock(); } voice_log_status_snapshot("session_stop_cleared"); voice_release_runtime_resources(); - voice_audio_close(); + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "session_stop_exit_close"); ESP_LOGI(TAG, "[stage] session_stop_exit_ok"); return ESP_OK; } @@ -1492,6 +357,32 @@ esp_err_t voice_interaction_tap_start(char *err, size_t err_len) { return ESP_ERR_INVALID_STATE; } + voice_unlock(); + + esp_err_t audio_err = voice_open_audio_for_dialog(err, err_len); + if (audio_err != ESP_OK) { + return audio_err; + } + + if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + voice_fill_err(err, err_len, "voice lock timeout"); + voice_audio_close(); + ESP_LOGW(TAG, "[stage] tap_start_abort: lock timeout after audio open"); + return ESP_ERR_TIMEOUT; + } + + if (!s_voice.session_active || !s_voice.ws_connected || !s_voice.started) { + ESP_LOGW(TAG, + "[stage] tap_start_abort_after_audio_open: session_active=%d ws_connected=%d started=%d", + s_voice.session_active, + s_voice.ws_connected, + s_voice.started); + voice_unlock(); + voice_audio_close(); + voice_fill_err(err, err_len, "voice session not ready"); + return ESP_ERR_INVALID_STATE; + } + bool listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING); s_voice.tap_active = true; voice_unlock(); @@ -1527,8 +418,9 @@ esp_err_t voice_interaction_tap_cancel(char *err, size_t err_len) { s_voice.tap_active = false; voice_unlock(); voice_log_status_snapshot("tap_cancel_marked_inactive"); - - return voice_send_directive("continue-task", "CancelSpeech", true); + esp_err_t cancel_err = voice_send_directive("continue-task", "CancelSpeech", true); + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS, "tap_cancel_close"); + return cancel_err; } void voice_interaction_get_status(voice_interaction_status_t *out_status) { diff --git a/main/domain/src/voice_interaction_common.c b/main/domain/src/voice_interaction_common.c new file mode 100644 index 0000000..b3fcf68 --- /dev/null +++ b/main/domain/src/voice_interaction_common.c @@ -0,0 +1,485 @@ +#include "voice_interaction_internal.h" + +#include +#include +#include +#include + +#include "esp_audio_dec.h" +#include "esp_audio_enc.h" +#include "esp_heap_caps.h" +#include "esp_log.h" +#include "esp_mac.h" +#include "esp_random.h" +#include "esp_system.h" +#include "esp_timer.h" +#include "freertos/idf_additions.h" +#include "voice_audio.h" + +static const char *TAG = "voice_interaction"; + +voice_context_t s_voice; + +int64_t voice_now_ms(void) { + return esp_timer_get_time() / 1000; +} + +void voice_fill_err(char *err, size_t err_len, const char *msg) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "%s", (msg != NULL) ? msg : "unknown error"); + } +} + +void voice_set_last_error_locked(const char *msg) { + if (msg == NULL) { + s_voice.last_error[0] = '\0'; + return; + } + strlcpy(s_voice.last_error, msg, sizeof(s_voice.last_error)); +} + +bool voice_lock(uint32_t timeout_ms) { + return s_voice.lock != NULL && xSemaphoreTake(s_voice.lock, pdMS_TO_TICKS(timeout_ms)) == pdTRUE; +} + +void voice_unlock(void) { + if (s_voice.lock != NULL) { + xSemaphoreGive(s_voice.lock); + } +} + +void voice_log_status_snapshot(const char *stage) { + if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + ESP_LOGW(TAG, "[stage] %s: status snapshot lock timeout", stage); + return; + } + + ESP_LOGI(TAG, + "[stage] %s: active=%d ws=%d started=%d tap=%d state=%s up=%" PRIu32 " down=%" PRIu32 " err=%s", + stage, + s_voice.session_active, + s_voice.ws_connected, + s_voice.started, + s_voice.tap_active, + voice_interaction_dialog_state_str(s_voice.dialog_state), + s_voice.upstream_packets, + s_voice.downstream_packets, + s_voice.last_error[0] != '\0' ? s_voice.last_error : "-"); + voice_unlock(); +} + +void voice_log_heap_snapshot(const char *stage) { + size_t free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT); + size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + size_t free_spiram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + size_t largest_8bit = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT); + size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + size_t largest_spiram = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + ESP_LOGI(TAG, + "[stage] %s heap: free8=%u free_internal=%u free_spiram=%u largest8=%u largest_internal=%u largest_spiram=%u", + stage, + (unsigned)free_8bit, + (unsigned)free_internal, + (unsigned)free_spiram, + (unsigned)largest_8bit, + (unsigned)largest_internal, + (unsigned)largest_spiram); +} + +void voice_wait_audio_playback_done(uint32_t timeout_ms, const char *stage) { + if (!voice_audio_is_open() || timeout_ms == 0) { + return; + } + + int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms; + while (voice_now_ms() < deadline_ms) { + int64_t playback_deadline_ms = 0; + int64_t last_downstream_ms = 0; + voice_dialog_state_t dialog_state = VOICE_DIALOG_STATE_IDLE; + bool ws_connected = false; + bool started = false; + + if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + break; + } + playback_deadline_ms = s_voice.playback_deadline_ms; + last_downstream_ms = s_voice.last_downstream_ms; + dialog_state = s_voice.dialog_state; + ws_connected = s_voice.ws_connected; + started = s_voice.started; + voice_unlock(); + + int64_t now_ms = voice_now_ms(); + bool codec_tail_done = now_ms >= playback_deadline_ms; + bool downstream_quiet = (last_downstream_ms == 0) || + ((now_ms - last_downstream_ms) >= VOICE_AUDIO_DRAIN_QUIET_MS); + bool remote_not_responding = !ws_connected || + !started || + dialog_state != VOICE_DIALOG_STATE_RESPONDING; + if (codec_tail_done && downstream_quiet && remote_not_responding) { + ESP_LOGI(TAG, + "[stage] %s: audio_drain_done now=%" PRId64 " play_deadline=%" PRId64 " last_down=%" PRId64, + stage != NULL ? stage : "audio_drain", + now_ms, + playback_deadline_ms, + last_downstream_ms); + return; + } + + vTaskDelay(pdMS_TO_TICKS(30)); + } + + ESP_LOGI(TAG, + "[stage] %s: audio_drain_timeout timeout_ms=%u", + stage != NULL ? stage : "audio_drain", + (unsigned)timeout_ms); +} + +void voice_close_audio_with_drain(uint32_t timeout_ms, const char *stage) { + if (!voice_audio_is_open()) { + return; + } + + voice_wait_audio_playback_done(timeout_ms, stage); + voice_audio_close(); + + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.last_downstream_ms = 0; + s_voice.playback_deadline_ms = 0; + voice_unlock(); + } +} + +void *voice_malloc_prefer_psram(size_t size) { + void *ptr = heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (ptr == NULL) { + ptr = malloc(size); + } + return ptr; +} + +void *voice_calloc_prefer_psram(size_t n, size_t size) { + void *ptr = heap_caps_calloc(n, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (ptr == NULL) { + ptr = calloc(n, size); + } + return ptr; +} + +void *voice_realloc_prefer_psram(void *ptr, size_t size) { + void *new_ptr = heap_caps_realloc(ptr, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (new_ptr == NULL) { + new_ptr = realloc(ptr, size); + } + return new_ptr; +} + +BaseType_t voice_create_task_prefer_psram(TaskFunction_t task_fn, + const char *name, + uint32_t stack_size, + UBaseType_t priority, + TaskHandle_t *out_task, + bool *out_with_caps) { + if (out_with_caps != NULL) { + *out_with_caps = false; + } + + BaseType_t rc = pdFAIL; + +#if defined(CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY) && \ + defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && \ + (configSUPPORT_STATIC_ALLOCATION == 1) + rc = xTaskCreatePinnedToCoreWithCaps(task_fn, + name, + (configSTACK_DEPTH_TYPE)stack_size, + NULL, + priority, + out_task, + tskNO_AFFINITY, + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (rc == pdPASS) { + if (out_with_caps != NULL) { + *out_with_caps = true; + } + ESP_LOGI(TAG, "[stage] task_create_psram_ok: name=%s stack=%u", name, (unsigned)stack_size); + return pdPASS; + } + ESP_LOGW(TAG, "[stage] task_create_psram_failed: name=%s stack=%u", name, (unsigned)stack_size); +#endif + + rc = xTaskCreate(task_fn, + name, + (configSTACK_DEPTH_TYPE)stack_size, + NULL, + priority, + out_task); + if (rc == pdPASS) { + ESP_LOGI(TAG, "[stage] task_create_internal_ok: name=%s stack=%u", name, (unsigned)stack_size); + } else { + ESP_LOGE(TAG, "[stage] task_create_internal_failed: name=%s stack=%u", name, (unsigned)stack_size); + } + return rc; +} + +void voice_delete_task(TaskHandle_t task, bool with_caps) { + if (task == NULL) { + return; + } + +#if (configSUPPORT_STATIC_ALLOCATION == 1) + if (with_caps) { + vTaskDeleteWithCaps(task); + return; + } +#else + (void)with_caps; +#endif + vTaskDelete(task); +} + +void voice_delete_self_task(bool with_caps) { +#if (configSUPPORT_STATIC_ALLOCATION == 1) + if (with_caps) { + vTaskDeleteWithCaps(NULL); + return; + } +#else + (void)with_caps; +#endif + vTaskDelete(NULL); +} + +void voice_make_uuid(char out[40]) { + uint8_t raw[16]; + esp_fill_random(raw, sizeof(raw)); + + raw[6] = (uint8_t)((raw[6] & 0x0F) | 0x40); + raw[8] = (uint8_t)((raw[8] & 0x3F) | 0x80); + + snprintf(out, + 40, + "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + raw[0], + raw[1], + raw[2], + raw[3], + raw[4], + raw[5], + raw[6], + raw[7], + raw[8], + raw[9], + raw[10], + raw[11], + raw[12], + raw[13], + raw[14], + raw[15]); +} + +void voice_make_device_uuid(char out[40]) { + if (strlen(CONFIG_TQ_VOICE_DEVICE_UUID) > 0) { + strlcpy(out, CONFIG_TQ_VOICE_DEVICE_UUID, 40); + return; + } + + uint8_t mac[6] = {0}; + esp_read_mac(mac, ESP_MAC_WIFI_STA); + + uint32_t r = esp_random(); + snprintf(out, + 40, + "%02x%02x%02x%02x-%02x%02x-4%03x-%04x-%012" PRIx32, + mac[0], + mac[1], + mac[2], + mac[3], + mac[4], + mac[5], + (unsigned)(r & 0x0FFF), + (unsigned)((r >> 12) & 0xFFFF), + esp_random()); +} + +const char *voice_safe_user_id(void) { + if (strlen(CONFIG_TQ_VOICE_USER_ID) > 0) { + return CONFIG_TQ_VOICE_USER_ID; + } + return "esp32-user"; +} + +static esp_opus_enc_frame_duration_t voice_to_enc_frame_duration(int frame_ms) { + switch (frame_ms) { + case 10: + return ESP_OPUS_ENC_FRAME_DURATION_10_MS; + case 20: + return ESP_OPUS_ENC_FRAME_DURATION_20_MS; + case 40: + return ESP_OPUS_ENC_FRAME_DURATION_40_MS; + case 60: + return ESP_OPUS_ENC_FRAME_DURATION_60_MS; + case 80: + return ESP_OPUS_ENC_FRAME_DURATION_80_MS; + case 100: + return ESP_OPUS_ENC_FRAME_DURATION_100_MS; + case 120: + return ESP_OPUS_ENC_FRAME_DURATION_120_MS; + default: + return ESP_OPUS_ENC_FRAME_DURATION_ARG; + } +} + +static esp_opus_dec_frame_duration_t voice_to_dec_frame_duration(int frame_ms) { + switch (frame_ms) { + case 10: + return ESP_OPUS_DEC_FRAME_DURATION_10_MS; + case 20: + return ESP_OPUS_DEC_FRAME_DURATION_20_MS; + case 40: + return ESP_OPUS_DEC_FRAME_DURATION_40_MS; + case 60: + return ESP_OPUS_DEC_FRAME_DURATION_60_MS; + case 80: + return ESP_OPUS_DEC_FRAME_DURATION_80_MS; + case 100: + return ESP_OPUS_DEC_FRAME_DURATION_100_MS; + case 120: + return ESP_OPUS_DEC_FRAME_DURATION_120_MS; + default: + return ESP_OPUS_DEC_FRAME_DURATION_INVALID; + } +} + +esp_err_t voice_setup_opus(char *err, size_t err_len) { + esp_opus_enc_frame_duration_t enc_duration = + voice_to_enc_frame_duration(CONFIG_TQ_VOICE_OPUS_FRAME_MS); + esp_opus_dec_frame_duration_t dec_duration = + voice_to_dec_frame_duration(CONFIG_TQ_VOICE_OPUS_FRAME_MS); + + if (enc_duration == ESP_OPUS_ENC_FRAME_DURATION_ARG || + dec_duration == ESP_OPUS_DEC_FRAME_DURATION_INVALID) { + voice_fill_err(err, err_len, "invalid opus frame duration"); + return ESP_ERR_INVALID_ARG; + } + + esp_opus_enc_config_t enc_cfg = ESP_OPUS_ENC_CONFIG_DEFAULT(); + enc_cfg.sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE; + enc_cfg.channel = ESP_AUDIO_MONO; + enc_cfg.bits_per_sample = ESP_AUDIO_BIT16; + enc_cfg.frame_duration = enc_duration; + enc_cfg.bitrate = CONFIG_TQ_VOICE_OPUS_BITRATE_KBPS * 1000; + enc_cfg.application_mode = ESP_OPUS_ENC_APPLICATION_VOIP; + enc_cfg.enable_vbr = true; + + esp_audio_err_t enc_err = esp_opus_enc_open(&enc_cfg, + sizeof(enc_cfg), + &s_voice.opus_enc); + if (enc_err != ESP_AUDIO_ERR_OK || s_voice.opus_enc == NULL) { + voice_fill_err(err, err_len, "opus encoder open failed"); + return ESP_FAIL; + } + + enc_err = esp_opus_enc_get_frame_size(s_voice.opus_enc, + &s_voice.opus_enc_in_size, + &s_voice.opus_enc_out_size); + if (enc_err != ESP_AUDIO_ERR_OK || + s_voice.opus_enc_in_size <= 0 || + s_voice.opus_enc_out_size <= 0) { + voice_fill_err(err, err_len, "opus frame size query failed"); + return ESP_FAIL; + } + + s_voice.pcm_tx_buf = (int16_t *)voice_malloc_prefer_psram((size_t)s_voice.opus_enc_in_size); + s_voice.opus_tx_buf = (uint8_t *)voice_malloc_prefer_psram((size_t)s_voice.opus_enc_out_size); + if (s_voice.pcm_tx_buf == NULL || s_voice.opus_tx_buf == NULL) { + voice_fill_err(err, err_len, "no memory for opus tx buffers"); + return ESP_ERR_NO_MEM; + } + + esp_opus_dec_cfg_t dec_cfg = ESP_OPUS_DEC_CONFIG_DEFAULT(); + dec_cfg.sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE; + dec_cfg.channel = ESP_AUDIO_MONO; + dec_cfg.frame_duration = dec_duration; + dec_cfg.self_delimited = false; + + esp_audio_err_t dec_err = esp_opus_dec_open(&dec_cfg, + sizeof(dec_cfg), + &s_voice.opus_dec); + if (dec_err != ESP_AUDIO_ERR_OK || s_voice.opus_dec == NULL) { + voice_fill_err(err, err_len, "opus decoder open failed"); + return ESP_FAIL; + } + + size_t rx_pcm_size = (size_t)CONFIG_TQ_VOICE_SAMPLE_RATE * + VOICE_SAMPLE_BYTES * + (size_t)CONFIG_TQ_VOICE_OPUS_FRAME_MS * + 2 / 1000; + if (rx_pcm_size < 2048) { + rx_pcm_size = 2048; + } + + s_voice.pcm_rx_buf = (uint8_t *)voice_malloc_prefer_psram(rx_pcm_size); + if (s_voice.pcm_rx_buf == NULL) { + voice_fill_err(err, err_len, "no memory for opus rx buffer"); + return ESP_ERR_NO_MEM; + } + s_voice.pcm_rx_buf_size = rx_pcm_size; + + return ESP_OK; +} + +esp_err_t voice_open_audio_for_dialog(char *err, size_t err_len) { + if (voice_audio_is_open()) { + return ESP_OK; + } + + voice_audio_open_config_t audio_cfg = { + .sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE, + .channels = 1, + .bits_per_sample = 16, + .output_volume = CONFIG_TQ_VOICE_CODEC_OUT_VOL, + .input_gain_db = (float)CONFIG_TQ_VOICE_CODEC_MIC_GAIN_DB, + }; + + ESP_LOGI(TAG, "[stage] audio_open_begin"); + esp_err_t open_err = voice_audio_open(&audio_cfg, err, err_len); + if (open_err != ESP_OK) { + ESP_LOGE(TAG, "[stage] audio_open_failed: %s", esp_err_to_name(open_err)); + return open_err; + } + + ESP_LOGI(TAG, "[stage] audio_open_ok"); + return ESP_OK; +} + +void voice_release_runtime_resources(void) { + if (s_voice.opus_enc != NULL) { + esp_opus_enc_close(s_voice.opus_enc); + s_voice.opus_enc = NULL; + } + if (s_voice.opus_dec != NULL) { + (void)esp_opus_dec_close(s_voice.opus_dec); + s_voice.opus_dec = NULL; + } + + free(s_voice.pcm_tx_buf); + s_voice.pcm_tx_buf = NULL; + + free(s_voice.opus_tx_buf); + s_voice.opus_tx_buf = NULL; + + free(s_voice.pcm_rx_buf); + s_voice.pcm_rx_buf = NULL; + s_voice.pcm_rx_buf_size = 0; + + free(s_voice.text_agg); + s_voice.text_agg = NULL; + s_voice.text_agg_size = 0; + + free(s_voice.bin_agg); + s_voice.bin_agg = NULL; + s_voice.bin_agg_size = 0; + + s_voice.opus_enc_in_size = 0; + s_voice.opus_enc_out_size = 0; +} diff --git a/main/domain/src/voice_interaction_tasks.c b/main/domain/src/voice_interaction_tasks.c new file mode 100644 index 0000000..b568f11 --- /dev/null +++ b/main/domain/src/voice_interaction_tasks.c @@ -0,0 +1,148 @@ +#include "voice_interaction_internal.h" + +#include + +#include "esp_audio_enc.h" +#include "esp_log.h" +#include "voice_audio.h" + +static const char *TAG = "voice_interaction"; + +void voice_uplink_task(void *arg) { + (void)arg; + + const int frame_ms = CONFIG_TQ_VOICE_OPUS_FRAME_MS; + bool low_stack_warned = false; + UBaseType_t start_hwm = uxTaskGetStackHighWaterMark(NULL); + ESP_LOGI(TAG, "[stage] uplink_task_started: stack_hwm_words=%u", (unsigned)start_hwm); + + while (true) { + if (!low_stack_warned) { + UBaseType_t hwm = uxTaskGetStackHighWaterMark(NULL); + if (hwm < 256) { + low_stack_warned = true; + ESP_LOGW(TAG, "[stage] uplink_task_low_stack: hwm_words=%u", (unsigned)hwm); + } + } + + bool active = false; + bool connected = false; + bool started = false; + bool tap_active = false; + bool listening = false; + + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + active = s_voice.session_active && !s_voice.stop_requested; + connected = s_voice.ws_connected; + started = s_voice.started; + tap_active = s_voice.tap_active; + listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING); + voice_unlock(); + } + + if (!active) { + break; + } + + if (!(connected && started && listening)) { + vTaskDelay(pdMS_TO_TICKS(20)); + continue; + } + + if (tap_active) { + char read_err[64] = {0}; + esp_err_t rc = voice_audio_read_pcm(s_voice.pcm_tx_buf, + (size_t)s_voice.opus_enc_in_size / VOICE_SAMPLE_BYTES, + VOICE_AUDIO_IO_TIMEOUT_MS, + read_err, + sizeof(read_err)); + if (rc != ESP_OK) { + memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size); + } + } else { + memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size); + vTaskDelay(pdMS_TO_TICKS(frame_ms)); + } + + esp_audio_enc_in_frame_t in_frame = { + .buffer = (uint8_t *)s_voice.pcm_tx_buf, + .len = (uint32_t)s_voice.opus_enc_in_size, + }; + esp_audio_enc_out_frame_t out_frame = { + .buffer = s_voice.opus_tx_buf, + .len = (uint32_t)s_voice.opus_enc_out_size, + .encoded_bytes = 0, + }; + + esp_audio_err_t enc_err = esp_opus_enc_process(s_voice.opus_enc, &in_frame, &out_frame); + if (enc_err != ESP_AUDIO_ERR_OK || out_frame.encoded_bytes == 0) { + continue; + } + + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + esp_websocket_client_handle_t ws = s_voice.ws; + bool can_send = s_voice.ws_connected; + voice_unlock(); + + if (ws != NULL && can_send) { + int ret = esp_websocket_client_send_bin(ws, + (const char *)s_voice.opus_tx_buf, + (int)out_frame.encoded_bytes, + pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS)); + if (ret >= 0) { + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.upstream_packets++; + s_voice.last_event_ms = voice_now_ms(); + voice_unlock(); + } + } + } + } + } + + bool delete_with_caps = false; + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + delete_with_caps = s_voice.uplink_task_with_caps; + s_voice.uplink_task = NULL; + s_voice.uplink_task_with_caps = false; + voice_unlock(); + } + voice_delete_self_task(delete_with_caps); +} + +void voice_heartbeat_task(void *arg) { + (void)arg; + + while (true) { + bool active = false; + bool connected = false; + bool started = false; + + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + active = s_voice.session_active && !s_voice.stop_requested; + connected = s_voice.ws_connected; + started = s_voice.started; + voice_unlock(); + } + + if (!active) { + break; + } + + if (connected && started) { + ESP_LOGI(TAG, "[stage] heartbeat_send"); + (void)voice_send_directive("continue-task", "HeartBeat", true); + } + + vTaskDelay(pdMS_TO_TICKS(CONFIG_TQ_VOICE_HEARTBEAT_SEC * 1000)); + } + + bool delete_with_caps = false; + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + delete_with_caps = s_voice.heartbeat_task_with_caps; + s_voice.heartbeat_task = NULL; + s_voice.heartbeat_task_with_caps = false; + voice_unlock(); + } + voice_delete_self_task(delete_with_caps); +} diff --git a/main/domain/src/voice_interaction_ws.c b/main/domain/src/voice_interaction_ws.c new file mode 100644 index 0000000..a261a89 --- /dev/null +++ b/main/domain/src/voice_interaction_ws.c @@ -0,0 +1,570 @@ +#include "voice_interaction_internal.h" + +#include +#include +#include +#include + +#include "cJSON.h" +#include "esp_audio_dec.h" +#include "esp_audio_enc.h" +#include "esp_log.h" +#include "voice_audio.h" + +static const char *TAG = "voice_interaction"; + +static char *voice_build_directive_message(const char *action, + const char *directive, + bool with_dialog_id) { + cJSON *root = cJSON_CreateObject(); + if (root == NULL) { + return NULL; + } + + cJSON *header = cJSON_AddObjectToObject(root, "header"); + cJSON_AddStringToObject(header, "action", action); + cJSON_AddStringToObject(header, "task_id", s_voice.task_id); + cJSON_AddStringToObject(header, "streaming", VOICE_STREAMING_MODE); + + cJSON *payload = cJSON_AddObjectToObject(root, "payload"); + cJSON *input = cJSON_AddObjectToObject(payload, "input"); + cJSON_AddStringToObject(input, "directive", directive); + if (with_dialog_id && s_voice.dialog_id[0] != '\0') { + cJSON_AddStringToObject(input, "dialog_id", s_voice.dialog_id); + } + + char *text = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + return text; +} + +static char *voice_build_start_message(void) { + cJSON *root = cJSON_CreateObject(); + if (root == NULL) { + return NULL; + } + + cJSON *header = cJSON_AddObjectToObject(root, "header"); + cJSON_AddStringToObject(header, "action", "run-task"); + cJSON_AddStringToObject(header, "task_id", s_voice.task_id); + cJSON_AddStringToObject(header, "streaming", VOICE_STREAMING_MODE); + + cJSON *payload = cJSON_AddObjectToObject(root, "payload"); + cJSON_AddStringToObject(payload, "task_group", VOICE_TASK_GROUP); + cJSON_AddStringToObject(payload, "task", VOICE_TASK_NAME); + cJSON_AddStringToObject(payload, "function", VOICE_FUNCTION_NAME); + cJSON_AddStringToObject(payload, "model", VOICE_MODEL_NAME); + + cJSON *input = cJSON_AddObjectToObject(payload, "input"); + cJSON_AddStringToObject(input, "directive", "Start"); + cJSON_AddStringToObject(input, "workspace_id", CONFIG_TQ_VOICE_WORKSPACE_ID); + cJSON_AddStringToObject(input, "app_id", CONFIG_TQ_VOICE_APP_ID); + + cJSON *parameters = cJSON_AddObjectToObject(payload, "parameters"); + cJSON *upstream = cJSON_AddObjectToObject(parameters, "upstream"); + cJSON_AddStringToObject(upstream, "type", "AudioOnly"); + cJSON_AddStringToObject(upstream, "mode", "tap2talk"); + cJSON_AddStringToObject(upstream, "audio_format", "raw-opus"); + cJSON_AddNumberToObject(upstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE); + + cJSON *downstream = cJSON_AddObjectToObject(parameters, "downstream"); + cJSON_AddNumberToObject(downstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE); + cJSON_AddStringToObject(downstream, "audio_format", "raw-opus"); + cJSON_AddNumberToObject(downstream, "frame_size", CONFIG_TQ_VOICE_OPUS_FRAME_MS); + cJSON_AddNumberToObject(downstream, + "bit_rate", + CONFIG_TQ_VOICE_OPUS_BITRATE_KBPS); + if (strlen(CONFIG_TQ_VOICE_TTS_VOICE) > 0) { + cJSON_AddStringToObject(downstream, "voice", CONFIG_TQ_VOICE_TTS_VOICE); + } + + cJSON *client_info = cJSON_AddObjectToObject(parameters, "client_info"); + cJSON_AddStringToObject(client_info, "user_id", voice_safe_user_id()); + cJSON *device = cJSON_AddObjectToObject(client_info, "device"); + cJSON_AddStringToObject(device, "uuid", s_voice.device_uuid); + + char *text = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + return text; +} + +static esp_err_t voice_send_text(const char *text, int len) { + if (text == NULL || len <= 0) { + return ESP_ERR_INVALID_ARG; + } + + esp_websocket_client_handle_t ws = NULL; + if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + return ESP_ERR_TIMEOUT; + } + ws = s_voice.ws; + bool connected = s_voice.ws_connected; + voice_unlock(); + + if (ws == NULL || !connected) { + ESP_LOGW(TAG, "[stage] ws_send_text skipped: ws=%p connected=%d len=%d", (void *)ws, connected, len); + return ESP_ERR_INVALID_STATE; + } + + int ret = esp_websocket_client_send_text(ws, + text, + len, + pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS)); + if (ret < 0) { + ESP_LOGW(TAG, "[stage] ws_send_text failed: len=%d", len); + return ESP_FAIL; + } + ESP_LOGI(TAG, "[stage] ws_send_text ok: len=%d", len); + return ESP_OK; +} + +esp_err_t voice_send_directive(const char *action, + const char *directive, + bool with_dialog_id) { + char *text = NULL; + if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + return ESP_ERR_TIMEOUT; + } + text = voice_build_directive_message(action, directive, with_dialog_id); + voice_unlock(); + + if (text == NULL) { + return ESP_ERR_NO_MEM; + } + + esp_err_t err = voice_send_text(text, (int)strlen(text)); + ESP_LOGI(TAG, + "[stage] directive: action=%s directive=%s with_dialog_id=%d result=%s", + action, + directive, + with_dialog_id, + esp_err_to_name(err)); + cJSON_free(text); + return err; +} + +static void voice_handle_downstream_packet(const uint8_t *data, size_t len) { + if (data == NULL || len == 0) { + return; + } + + if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + return; + } + + void *opus_dec = s_voice.opus_dec; + uint8_t *pcm_buf = s_voice.pcm_rx_buf; + size_t pcm_buf_size = s_voice.pcm_rx_buf_size; + voice_unlock(); + + if (opus_dec == NULL || pcm_buf == NULL || pcm_buf_size == 0) { + return; + } + + esp_audio_dec_in_raw_t raw = { + .buffer = (uint8_t *)data, + .len = (uint32_t)len, + .consumed = 0, + }; + + while (raw.len > 0) { + esp_audio_dec_out_frame_t out = { + .buffer = pcm_buf, + .len = (uint32_t)pcm_buf_size, + .needed_size = 0, + .decoded_size = 0, + }; + esp_audio_dec_info_t dec_info = {0}; + + esp_audio_err_t ret = esp_opus_dec_decode(opus_dec, &raw, &out, &dec_info); + if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) { + if (out.needed_size > pcm_buf_size) { + uint8_t *new_buf = (uint8_t *)voice_realloc_prefer_psram(pcm_buf, out.needed_size); + if (new_buf == NULL) { + ESP_LOGE(TAG, "No memory to extend pcm rx buffer to %u", out.needed_size); + break; + } + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.pcm_rx_buf = new_buf; + s_voice.pcm_rx_buf_size = out.needed_size; + voice_unlock(); + } + pcm_buf = new_buf; + pcm_buf_size = out.needed_size; + continue; + } + break; + } + + if (ret != ESP_AUDIO_ERR_OK) { + ESP_LOGW(TAG, "opus decode failed: %d", ret); + break; + } + + if (out.decoded_size > 0) { + char audio_err[64] = {0}; + size_t samples = (size_t)out.decoded_size / VOICE_SAMPLE_BYTES; + esp_err_t write_err = voice_audio_write_pcm((const int16_t *)out.buffer, + samples, + VOICE_AUDIO_IO_TIMEOUT_MS, + audio_err, + sizeof(audio_err)); + if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + int64_t now_ms = voice_now_ms(); + int64_t pcm_ms = ((int64_t)samples * 1000 + + (int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) / + (int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE; + int64_t base_ms = s_voice.playback_deadline_ms > now_ms ? + s_voice.playback_deadline_ms : now_ms; + s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS; + s_voice.last_downstream_ms = now_ms; + voice_unlock(); + } + } + + if (raw.consumed == 0 || raw.consumed > raw.len) { + break; + } + raw.buffer += raw.consumed; + raw.len -= raw.consumed; + } + + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.downstream_packets++; + s_voice.last_event_ms = voice_now_ms(); + voice_unlock(); + } +} + +static const char *voice_get_json_str(cJSON *obj, const char *name) { + if (obj == NULL || name == NULL) { + return NULL; + } + cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); + if (cJSON_IsString(item) && item->valuestring != NULL) { + return item->valuestring; + } + return NULL; +} + +static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) { + if (obj == NULL || name == NULL || out_value == NULL) { + return false; + } + cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); + if (!cJSON_IsBool(item)) { + return false; + } + *out_value = cJSON_IsTrue(item); + return true; +} + +static void voice_log_final_text(const char *stage, const char *text) { + if (stage == NULL || text == NULL || text[0] == '\0') { + return; + } + size_t len = strlen(text); + const int preview = 240; + ESP_LOGI(TAG, + "[stage] %s: len=%u text=%.*s%s", + stage, + (unsigned)len, + preview, + text, + (len > (size_t)preview) ? "..." : ""); +} + +static void voice_handle_output_event(cJSON *output) { + const char *event_name = voice_get_json_str(output, "event"); + if (event_name == NULL) { + return; + } + + const char *dialog_id = voice_get_json_str(output, "dialog_id"); + const char *state = NULL; + if (strcmp(event_name, "DialogStateChanged") == 0) { + state = voice_get_json_str(output, "state"); + } + ESP_LOGI(TAG, + "[stage] ws_event_output: event=%s dialog_id=%s state=%s", + event_name, + dialog_id != NULL ? dialog_id : "-", + state != NULL ? state : "-"); + + bool finished = false; + bool has_finished = voice_get_json_bool(output, "finished", &finished); + if (has_finished && finished) { + if (strcmp(event_name, "SpeechContent") == 0) { + voice_log_final_text("asr_final_text", voice_get_json_str(output, "text")); + } else if (strcmp(event_name, "RespondingContent") == 0) { + const char *final_text = voice_get_json_str(output, "text"); + if (final_text == NULL || final_text[0] == '\0') { + final_text = voice_get_json_str(output, "spoken"); + } + voice_log_final_text("response_final_text", final_text); + } + } + + bool should_close_audio = false; + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + if (dialog_id != NULL) { + strlcpy(s_voice.dialog_id, dialog_id, sizeof(s_voice.dialog_id)); + } + s_voice.last_event_ms = voice_now_ms(); + + if (strcmp(event_name, "Started") == 0) { + s_voice.started = true; + s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id); + } else if (strcmp(event_name, "DialogStateChanged") == 0) { + if (state != NULL) { + if (strcmp(state, "Listening") == 0) { + s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING; + } else if (strcmp(state, "Thinking") == 0) { + s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING; + } else if (strcmp(state, "Responding") == 0) { + s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING; + } + } + ESP_LOGI(TAG, + "[stage] dialog_state_changed: state=%s current=%s", + state != NULL ? state : "-", + voice_interaction_dialog_state_str(s_voice.dialog_state)); + } else if (strcmp(event_name, "SpeechEnded") == 0) { + ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active); + } else if (strcmp(event_name, "Stopped") == 0) { + s_voice.started = false; + s_voice.tap_active = false; + s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + should_close_audio = true; + ESP_LOGI(TAG, "[stage] session_stopped_by_server"); + } else if (strcmp(event_name, "Error") == 0) { + const char *error_msg = voice_get_json_str(output, "error_message"); + if (error_msg != NULL) { + voice_set_last_error_locked(error_msg); + ESP_LOGW(TAG, "[stage] server_error: %s", error_msg); + } + s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + s_voice.tap_active = false; + should_close_audio = true; + } + voice_unlock(); + } + + if (should_close_audio) { + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close"); + } + + if (strcmp(event_name, "RespondingStarted") == 0) { + (void)voice_send_directive("continue-task", "LocalRespondingStarted", true); + } else if (strcmp(event_name, "RespondingEnded") == 0) { + (void)voice_send_directive("continue-task", "LocalRespondingEnded", true); + } +} + +static void voice_handle_text_message(const char *text, size_t len) { + cJSON *root = cJSON_ParseWithLength(text, len); + if (root == NULL) { + ESP_LOGW(TAG, "Invalid ws text payload"); + return; + } + + cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header"); + const char *header_event = voice_get_json_str(header, "event"); + + cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload"); + cJSON *output = NULL; + if (payload != NULL) { + output = cJSON_GetObjectItemCaseSensitive(payload, "output"); + } + + if (header != NULL) { + cJSON *status_code = cJSON_GetObjectItemCaseSensitive(header, "status_code"); + cJSON *status_msg = cJSON_GetObjectItemCaseSensitive(header, "status_message"); + if (cJSON_IsNumber(status_code) && status_code->valueint >= 400) { + ESP_LOGW(TAG, + "[stage] ws_header_error: status_code=%d status_message=%s", + status_code->valueint, + (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) ? status_msg->valuestring : "-"); + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + if (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) { + voice_set_last_error_locked(status_msg->valuestring); + } else { + voice_set_last_error_locked("websocket status error"); + } + voice_unlock(); + } + } + } + + if (output != NULL) { + voice_handle_output_event(output); + } else if (header_event != NULL && strcmp(header_event, "task-failed") == 0) { + const char *error_msg = voice_get_json_str(header, "error_message"); + ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed"); + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed"); + voice_unlock(); + } + } + + cJSON_Delete(root); +} + +static void voice_handle_text_chunk(esp_websocket_event_data_t *data) { + if (data->payload_offset == 0) { + free(s_voice.text_agg); + s_voice.text_agg = NULL; + s_voice.text_agg_size = 0; + + if (data->payload_len <= 0 || data->payload_len > 4096) { + return; + } + + s_voice.text_agg = (uint8_t *)voice_calloc_prefer_psram(1, (size_t)data->payload_len + 1); + if (s_voice.text_agg == NULL) { + ESP_LOGW(TAG, "[stage] text_agg_alloc_failed: len=%d", data->payload_len); + return; + } + s_voice.text_agg_size = (size_t)data->payload_len; + } + + if (s_voice.text_agg == NULL || s_voice.text_agg_size == 0) { + return; + } + + if ((size_t)data->payload_offset + (size_t)data->data_len > s_voice.text_agg_size) { + free(s_voice.text_agg); + s_voice.text_agg = NULL; + s_voice.text_agg_size = 0; + return; + } + + memcpy(s_voice.text_agg + data->payload_offset, data->data_ptr, (size_t)data->data_len); + + bool complete = data->fin && + ((size_t)data->payload_offset + (size_t)data->data_len == s_voice.text_agg_size); + if (!complete) { + return; + } + + voice_handle_text_message((const char *)s_voice.text_agg, s_voice.text_agg_size); + free(s_voice.text_agg); + s_voice.text_agg = NULL; + s_voice.text_agg_size = 0; +} + +static void voice_handle_binary_chunk(esp_websocket_event_data_t *data) { + if (data->payload_offset == 0) { + free(s_voice.bin_agg); + s_voice.bin_agg = NULL; + s_voice.bin_agg_size = 0; + + if (data->payload_len <= 0 || data->payload_len > 16384) { + return; + } + + s_voice.bin_agg = (uint8_t *)voice_malloc_prefer_psram((size_t)data->payload_len); + if (s_voice.bin_agg == NULL) { + ESP_LOGW(TAG, "[stage] bin_agg_alloc_failed: len=%d", data->payload_len); + return; + } + s_voice.bin_agg_size = (size_t)data->payload_len; + } + + if (s_voice.bin_agg == NULL || s_voice.bin_agg_size == 0) { + return; + } + + if ((size_t)data->payload_offset + (size_t)data->data_len > s_voice.bin_agg_size) { + free(s_voice.bin_agg); + s_voice.bin_agg = NULL; + s_voice.bin_agg_size = 0; + return; + } + + memcpy(s_voice.bin_agg + data->payload_offset, data->data_ptr, (size_t)data->data_len); + + bool complete = data->fin && + ((size_t)data->payload_offset + (size_t)data->data_len == s_voice.bin_agg_size); + if (!complete) { + return; + } + + voice_handle_downstream_packet(s_voice.bin_agg, s_voice.bin_agg_size); + free(s_voice.bin_agg); + s_voice.bin_agg = NULL; + s_voice.bin_agg_size = 0; +} + +void voice_websocket_event_handler(void *handler_args, + esp_event_base_t base, + int32_t event_id, + void *event_data) { + (void)handler_args; + (void)base; + + UBaseType_t ws_hwm = uxTaskGetStackHighWaterMark(NULL); + if (!s_voice.ws_low_stack_warned && ws_hwm < 256) { + s_voice.ws_low_stack_warned = true; + ESP_LOGW(TAG, "[stage] ws_task_low_stack: hwm_words=%u", (unsigned)ws_hwm); + } + + esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data; + switch ((esp_websocket_event_id_t)event_id) { + case WEBSOCKET_EVENT_CONNECTED: { + ESP_LOGI(TAG, "[stage] ws_connected"); + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.ws_connected = true; + s_voice.last_event_ms = voice_now_ms(); + voice_unlock(); + } + + char *start_msg = NULL; + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + start_msg = voice_build_start_message(); + voice_unlock(); + } + if (start_msg != NULL) { + esp_err_t send_err = voice_send_text(start_msg, (int)strlen(start_msg)); + ESP_LOGI(TAG, "[stage] start_message_sent: result=%s", esp_err_to_name(send_err)); + cJSON_free(start_msg); + } else { + ESP_LOGE(TAG, "[stage] start_message_build_failed"); + } + break; + } + case WEBSOCKET_EVENT_DISCONNECTED: + ESP_LOGW(TAG, "[stage] ws_disconnected"); + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.ws_connected = false; + s_voice.started = false; + s_voice.tap_active = false; + s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + s_voice.last_event_ms = voice_now_ms(); + voice_unlock(); + } + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "ws_disconnected_close"); + break; + case WEBSOCKET_EVENT_DATA: + if (data == NULL || data->data_ptr == NULL || data->data_len <= 0) { + break; + } + if (data->op_code == WS_TRANSPORT_OPCODES_TEXT) { + voice_handle_text_chunk(data); + } else if (data->op_code == WS_TRANSPORT_OPCODES_BINARY) { + voice_handle_binary_chunk(data); + } + break; + case WEBSOCKET_EVENT_ERROR: + ESP_LOGW(TAG, "[stage] ws_error"); + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + voice_set_last_error_locked("websocket error"); + s_voice.last_event_ms = voice_now_ms(); + voice_unlock(); + } + break; + default: + break; + } +}