refactor(domain): split voice interaction implementation

This commit is contained in:
admin
2026-02-25 12:53:02 +08:00
parent 46783bf672
commit 63c4078d05
6 changed files with 1374 additions and 1151 deletions

View File

@@ -0,0 +1,125 @@
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include "esp_opus_dec.h"
#include "esp_opus_enc.h"
#include "esp_websocket_client.h"
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#include "voice_interaction.h"
#define VOICE_STREAMING_MODE "duplex"
#define VOICE_TASK_GROUP "aigc"
#define VOICE_TASK_NAME "multimodal-generation"
#define VOICE_FUNCTION_NAME "generation"
#define VOICE_MODEL_NAME "multimodal-dialog"
#define VOICE_SAMPLE_BYTES 2
#define VOICE_STATUS_LOCK_TIMEOUT_MS 1000
#define VOICE_WS_SEND_TIMEOUT_MS 1000
#define VOICE_WS_TASK_STACK 24576
#define VOICE_UPLINK_TASK_STACK 32768
#define VOICE_HEARTBEAT_TASK_STACK 4096
#define VOICE_START_CONNECT_TIMEOUT_MS 8000
#define VOICE_AUDIO_IO_TIMEOUT_MS 1200
#define VOICE_AUDIO_DRAIN_MARGIN_MS 120
#define VOICE_AUDIO_DRAIN_QUIET_MS 180
#define VOICE_AUDIO_DRAIN_WAIT_MS 2200
#define VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS 3500
typedef struct {
SemaphoreHandle_t lock;
esp_websocket_client_handle_t ws;
TaskHandle_t uplink_task;
TaskHandle_t heartbeat_task;
bool uplink_task_with_caps;
bool heartbeat_task_with_caps;
void *opus_enc;
void *opus_dec;
int opus_enc_in_size;
int opus_enc_out_size;
uint8_t *opus_tx_buf;
int16_t *pcm_tx_buf;
uint8_t *pcm_rx_buf;
size_t pcm_rx_buf_size;
uint8_t *text_agg;
size_t text_agg_size;
uint8_t *bin_agg;
size_t bin_agg_size;
bool ready;
bool session_active;
bool ws_connected;
bool started;
bool tap_active;
bool stop_requested;
bool ws_low_stack_warned;
voice_dialog_state_t dialog_state;
char task_id[40];
char dialog_id[40];
char device_uuid[40];
char last_error[128];
int64_t last_event_ms;
int64_t last_downstream_ms;
int64_t playback_deadline_ms;
uint32_t upstream_packets;
uint32_t downstream_packets;
} voice_context_t;
extern voice_context_t s_voice;
int64_t voice_now_ms(void);
void voice_fill_err(char *err, size_t err_len, const char *msg);
void voice_set_last_error_locked(const char *msg);
bool voice_lock(uint32_t timeout_ms);
void voice_unlock(void);
void voice_log_status_snapshot(const char *stage);
void voice_log_heap_snapshot(const char *stage);
void voice_wait_audio_playback_done(uint32_t timeout_ms, const char *stage);
void voice_close_audio_with_drain(uint32_t timeout_ms, const char *stage);
void *voice_malloc_prefer_psram(size_t size);
void *voice_calloc_prefer_psram(size_t n, size_t size);
void *voice_realloc_prefer_psram(void *ptr, size_t size);
BaseType_t voice_create_task_prefer_psram(TaskFunction_t task_fn,
const char *name,
uint32_t stack_size,
UBaseType_t priority,
TaskHandle_t *out_task,
bool *out_with_caps);
void voice_delete_task(TaskHandle_t task, bool with_caps);
void voice_delete_self_task(bool with_caps);
void voice_make_uuid(char out[40]);
void voice_make_device_uuid(char out[40]);
const char *voice_safe_user_id(void);
esp_err_t voice_setup_opus(char *err, size_t err_len);
esp_err_t voice_open_audio_for_dialog(char *err, size_t err_len);
void voice_release_runtime_resources(void);
esp_err_t voice_send_directive(const char *action,
const char *directive,
bool with_dialog_id);
void voice_websocket_event_handler(void *handler_args,
esp_event_base_t base,
int32_t event_id,
void *event_data);
void voice_uplink_task(void *arg);
void voice_heartbeat_task(void *arg);

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,485 @@
#include "voice_interaction_internal.h"
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "esp_audio_dec.h"
#include "esp_audio_enc.h"
#include "esp_heap_caps.h"
#include "esp_log.h"
#include "esp_mac.h"
#include "esp_random.h"
#include "esp_system.h"
#include "esp_timer.h"
#include "freertos/idf_additions.h"
#include "voice_audio.h"
static const char *TAG = "voice_interaction";
voice_context_t s_voice;
int64_t voice_now_ms(void) {
return esp_timer_get_time() / 1000;
}
void voice_fill_err(char *err, size_t err_len, const char *msg) {
if (err != NULL && err_len > 0) {
snprintf(err, err_len, "%s", (msg != NULL) ? msg : "unknown error");
}
}
void voice_set_last_error_locked(const char *msg) {
if (msg == NULL) {
s_voice.last_error[0] = '\0';
return;
}
strlcpy(s_voice.last_error, msg, sizeof(s_voice.last_error));
}
bool voice_lock(uint32_t timeout_ms) {
return s_voice.lock != NULL && xSemaphoreTake(s_voice.lock, pdMS_TO_TICKS(timeout_ms)) == pdTRUE;
}
void voice_unlock(void) {
if (s_voice.lock != NULL) {
xSemaphoreGive(s_voice.lock);
}
}
void voice_log_status_snapshot(const char *stage) {
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
ESP_LOGW(TAG, "[stage] %s: status snapshot lock timeout", stage);
return;
}
ESP_LOGI(TAG,
"[stage] %s: active=%d ws=%d started=%d tap=%d state=%s up=%" PRIu32 " down=%" PRIu32 " err=%s",
stage,
s_voice.session_active,
s_voice.ws_connected,
s_voice.started,
s_voice.tap_active,
voice_interaction_dialog_state_str(s_voice.dialog_state),
s_voice.upstream_packets,
s_voice.downstream_packets,
s_voice.last_error[0] != '\0' ? s_voice.last_error : "-");
voice_unlock();
}
void voice_log_heap_snapshot(const char *stage) {
size_t free_8bit = heap_caps_get_free_size(MALLOC_CAP_8BIT);
size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
size_t free_spiram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
size_t largest_8bit = heap_caps_get_largest_free_block(MALLOC_CAP_8BIT);
size_t largest_internal = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
size_t largest_spiram = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
ESP_LOGI(TAG,
"[stage] %s heap: free8=%u free_internal=%u free_spiram=%u largest8=%u largest_internal=%u largest_spiram=%u",
stage,
(unsigned)free_8bit,
(unsigned)free_internal,
(unsigned)free_spiram,
(unsigned)largest_8bit,
(unsigned)largest_internal,
(unsigned)largest_spiram);
}
void voice_wait_audio_playback_done(uint32_t timeout_ms, const char *stage) {
if (!voice_audio_is_open() || timeout_ms == 0) {
return;
}
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
int64_t playback_deadline_ms = 0;
int64_t last_downstream_ms = 0;
voice_dialog_state_t dialog_state = VOICE_DIALOG_STATE_IDLE;
bool ws_connected = false;
bool started = false;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
break;
}
playback_deadline_ms = s_voice.playback_deadline_ms;
last_downstream_ms = s_voice.last_downstream_ms;
dialog_state = s_voice.dialog_state;
ws_connected = s_voice.ws_connected;
started = s_voice.started;
voice_unlock();
int64_t now_ms = voice_now_ms();
bool codec_tail_done = now_ms >= playback_deadline_ms;
bool downstream_quiet = (last_downstream_ms == 0) ||
((now_ms - last_downstream_ms) >= VOICE_AUDIO_DRAIN_QUIET_MS);
bool remote_not_responding = !ws_connected ||
!started ||
dialog_state != VOICE_DIALOG_STATE_RESPONDING;
if (codec_tail_done && downstream_quiet && remote_not_responding) {
ESP_LOGI(TAG,
"[stage] %s: audio_drain_done now=%" PRId64 " play_deadline=%" PRId64 " last_down=%" PRId64,
stage != NULL ? stage : "audio_drain",
now_ms,
playback_deadline_ms,
last_downstream_ms);
return;
}
vTaskDelay(pdMS_TO_TICKS(30));
}
ESP_LOGI(TAG,
"[stage] %s: audio_drain_timeout timeout_ms=%u",
stage != NULL ? stage : "audio_drain",
(unsigned)timeout_ms);
}
void voice_close_audio_with_drain(uint32_t timeout_ms, const char *stage) {
if (!voice_audio_is_open()) {
return;
}
voice_wait_audio_playback_done(timeout_ms, stage);
voice_audio_close();
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.last_downstream_ms = 0;
s_voice.playback_deadline_ms = 0;
voice_unlock();
}
}
void *voice_malloc_prefer_psram(size_t size) {
void *ptr = heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (ptr == NULL) {
ptr = malloc(size);
}
return ptr;
}
void *voice_calloc_prefer_psram(size_t n, size_t size) {
void *ptr = heap_caps_calloc(n, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (ptr == NULL) {
ptr = calloc(n, size);
}
return ptr;
}
void *voice_realloc_prefer_psram(void *ptr, size_t size) {
void *new_ptr = heap_caps_realloc(ptr, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (new_ptr == NULL) {
new_ptr = realloc(ptr, size);
}
return new_ptr;
}
BaseType_t voice_create_task_prefer_psram(TaskFunction_t task_fn,
const char *name,
uint32_t stack_size,
UBaseType_t priority,
TaskHandle_t *out_task,
bool *out_with_caps) {
if (out_with_caps != NULL) {
*out_with_caps = false;
}
BaseType_t rc = pdFAIL;
#if defined(CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY) && \
defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && \
(configSUPPORT_STATIC_ALLOCATION == 1)
rc = xTaskCreatePinnedToCoreWithCaps(task_fn,
name,
(configSTACK_DEPTH_TYPE)stack_size,
NULL,
priority,
out_task,
tskNO_AFFINITY,
MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (rc == pdPASS) {
if (out_with_caps != NULL) {
*out_with_caps = true;
}
ESP_LOGI(TAG, "[stage] task_create_psram_ok: name=%s stack=%u", name, (unsigned)stack_size);
return pdPASS;
}
ESP_LOGW(TAG, "[stage] task_create_psram_failed: name=%s stack=%u", name, (unsigned)stack_size);
#endif
rc = xTaskCreate(task_fn,
name,
(configSTACK_DEPTH_TYPE)stack_size,
NULL,
priority,
out_task);
if (rc == pdPASS) {
ESP_LOGI(TAG, "[stage] task_create_internal_ok: name=%s stack=%u", name, (unsigned)stack_size);
} else {
ESP_LOGE(TAG, "[stage] task_create_internal_failed: name=%s stack=%u", name, (unsigned)stack_size);
}
return rc;
}
void voice_delete_task(TaskHandle_t task, bool with_caps) {
if (task == NULL) {
return;
}
#if (configSUPPORT_STATIC_ALLOCATION == 1)
if (with_caps) {
vTaskDeleteWithCaps(task);
return;
}
#else
(void)with_caps;
#endif
vTaskDelete(task);
}
void voice_delete_self_task(bool with_caps) {
#if (configSUPPORT_STATIC_ALLOCATION == 1)
if (with_caps) {
vTaskDeleteWithCaps(NULL);
return;
}
#else
(void)with_caps;
#endif
vTaskDelete(NULL);
}
void voice_make_uuid(char out[40]) {
uint8_t raw[16];
esp_fill_random(raw, sizeof(raw));
raw[6] = (uint8_t)((raw[6] & 0x0F) | 0x40);
raw[8] = (uint8_t)((raw[8] & 0x3F) | 0x80);
snprintf(out,
40,
"%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",
raw[0],
raw[1],
raw[2],
raw[3],
raw[4],
raw[5],
raw[6],
raw[7],
raw[8],
raw[9],
raw[10],
raw[11],
raw[12],
raw[13],
raw[14],
raw[15]);
}
void voice_make_device_uuid(char out[40]) {
if (strlen(CONFIG_TQ_VOICE_DEVICE_UUID) > 0) {
strlcpy(out, CONFIG_TQ_VOICE_DEVICE_UUID, 40);
return;
}
uint8_t mac[6] = {0};
esp_read_mac(mac, ESP_MAC_WIFI_STA);
uint32_t r = esp_random();
snprintf(out,
40,
"%02x%02x%02x%02x-%02x%02x-4%03x-%04x-%012" PRIx32,
mac[0],
mac[1],
mac[2],
mac[3],
mac[4],
mac[5],
(unsigned)(r & 0x0FFF),
(unsigned)((r >> 12) & 0xFFFF),
esp_random());
}
const char *voice_safe_user_id(void) {
if (strlen(CONFIG_TQ_VOICE_USER_ID) > 0) {
return CONFIG_TQ_VOICE_USER_ID;
}
return "esp32-user";
}
static esp_opus_enc_frame_duration_t voice_to_enc_frame_duration(int frame_ms) {
switch (frame_ms) {
case 10:
return ESP_OPUS_ENC_FRAME_DURATION_10_MS;
case 20:
return ESP_OPUS_ENC_FRAME_DURATION_20_MS;
case 40:
return ESP_OPUS_ENC_FRAME_DURATION_40_MS;
case 60:
return ESP_OPUS_ENC_FRAME_DURATION_60_MS;
case 80:
return ESP_OPUS_ENC_FRAME_DURATION_80_MS;
case 100:
return ESP_OPUS_ENC_FRAME_DURATION_100_MS;
case 120:
return ESP_OPUS_ENC_FRAME_DURATION_120_MS;
default:
return ESP_OPUS_ENC_FRAME_DURATION_ARG;
}
}
static esp_opus_dec_frame_duration_t voice_to_dec_frame_duration(int frame_ms) {
switch (frame_ms) {
case 10:
return ESP_OPUS_DEC_FRAME_DURATION_10_MS;
case 20:
return ESP_OPUS_DEC_FRAME_DURATION_20_MS;
case 40:
return ESP_OPUS_DEC_FRAME_DURATION_40_MS;
case 60:
return ESP_OPUS_DEC_FRAME_DURATION_60_MS;
case 80:
return ESP_OPUS_DEC_FRAME_DURATION_80_MS;
case 100:
return ESP_OPUS_DEC_FRAME_DURATION_100_MS;
case 120:
return ESP_OPUS_DEC_FRAME_DURATION_120_MS;
default:
return ESP_OPUS_DEC_FRAME_DURATION_INVALID;
}
}
esp_err_t voice_setup_opus(char *err, size_t err_len) {
esp_opus_enc_frame_duration_t enc_duration =
voice_to_enc_frame_duration(CONFIG_TQ_VOICE_OPUS_FRAME_MS);
esp_opus_dec_frame_duration_t dec_duration =
voice_to_dec_frame_duration(CONFIG_TQ_VOICE_OPUS_FRAME_MS);
if (enc_duration == ESP_OPUS_ENC_FRAME_DURATION_ARG ||
dec_duration == ESP_OPUS_DEC_FRAME_DURATION_INVALID) {
voice_fill_err(err, err_len, "invalid opus frame duration");
return ESP_ERR_INVALID_ARG;
}
esp_opus_enc_config_t enc_cfg = ESP_OPUS_ENC_CONFIG_DEFAULT();
enc_cfg.sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE;
enc_cfg.channel = ESP_AUDIO_MONO;
enc_cfg.bits_per_sample = ESP_AUDIO_BIT16;
enc_cfg.frame_duration = enc_duration;
enc_cfg.bitrate = CONFIG_TQ_VOICE_OPUS_BITRATE_KBPS * 1000;
enc_cfg.application_mode = ESP_OPUS_ENC_APPLICATION_VOIP;
enc_cfg.enable_vbr = true;
esp_audio_err_t enc_err = esp_opus_enc_open(&enc_cfg,
sizeof(enc_cfg),
&s_voice.opus_enc);
if (enc_err != ESP_AUDIO_ERR_OK || s_voice.opus_enc == NULL) {
voice_fill_err(err, err_len, "opus encoder open failed");
return ESP_FAIL;
}
enc_err = esp_opus_enc_get_frame_size(s_voice.opus_enc,
&s_voice.opus_enc_in_size,
&s_voice.opus_enc_out_size);
if (enc_err != ESP_AUDIO_ERR_OK ||
s_voice.opus_enc_in_size <= 0 ||
s_voice.opus_enc_out_size <= 0) {
voice_fill_err(err, err_len, "opus frame size query failed");
return ESP_FAIL;
}
s_voice.pcm_tx_buf = (int16_t *)voice_malloc_prefer_psram((size_t)s_voice.opus_enc_in_size);
s_voice.opus_tx_buf = (uint8_t *)voice_malloc_prefer_psram((size_t)s_voice.opus_enc_out_size);
if (s_voice.pcm_tx_buf == NULL || s_voice.opus_tx_buf == NULL) {
voice_fill_err(err, err_len, "no memory for opus tx buffers");
return ESP_ERR_NO_MEM;
}
esp_opus_dec_cfg_t dec_cfg = ESP_OPUS_DEC_CONFIG_DEFAULT();
dec_cfg.sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE;
dec_cfg.channel = ESP_AUDIO_MONO;
dec_cfg.frame_duration = dec_duration;
dec_cfg.self_delimited = false;
esp_audio_err_t dec_err = esp_opus_dec_open(&dec_cfg,
sizeof(dec_cfg),
&s_voice.opus_dec);
if (dec_err != ESP_AUDIO_ERR_OK || s_voice.opus_dec == NULL) {
voice_fill_err(err, err_len, "opus decoder open failed");
return ESP_FAIL;
}
size_t rx_pcm_size = (size_t)CONFIG_TQ_VOICE_SAMPLE_RATE *
VOICE_SAMPLE_BYTES *
(size_t)CONFIG_TQ_VOICE_OPUS_FRAME_MS *
2 / 1000;
if (rx_pcm_size < 2048) {
rx_pcm_size = 2048;
}
s_voice.pcm_rx_buf = (uint8_t *)voice_malloc_prefer_psram(rx_pcm_size);
if (s_voice.pcm_rx_buf == NULL) {
voice_fill_err(err, err_len, "no memory for opus rx buffer");
return ESP_ERR_NO_MEM;
}
s_voice.pcm_rx_buf_size = rx_pcm_size;
return ESP_OK;
}
esp_err_t voice_open_audio_for_dialog(char *err, size_t err_len) {
if (voice_audio_is_open()) {
return ESP_OK;
}
voice_audio_open_config_t audio_cfg = {
.sample_rate = CONFIG_TQ_VOICE_SAMPLE_RATE,
.channels = 1,
.bits_per_sample = 16,
.output_volume = CONFIG_TQ_VOICE_CODEC_OUT_VOL,
.input_gain_db = (float)CONFIG_TQ_VOICE_CODEC_MIC_GAIN_DB,
};
ESP_LOGI(TAG, "[stage] audio_open_begin");
esp_err_t open_err = voice_audio_open(&audio_cfg, err, err_len);
if (open_err != ESP_OK) {
ESP_LOGE(TAG, "[stage] audio_open_failed: %s", esp_err_to_name(open_err));
return open_err;
}
ESP_LOGI(TAG, "[stage] audio_open_ok");
return ESP_OK;
}
void voice_release_runtime_resources(void) {
if (s_voice.opus_enc != NULL) {
esp_opus_enc_close(s_voice.opus_enc);
s_voice.opus_enc = NULL;
}
if (s_voice.opus_dec != NULL) {
(void)esp_opus_dec_close(s_voice.opus_dec);
s_voice.opus_dec = NULL;
}
free(s_voice.pcm_tx_buf);
s_voice.pcm_tx_buf = NULL;
free(s_voice.opus_tx_buf);
s_voice.opus_tx_buf = NULL;
free(s_voice.pcm_rx_buf);
s_voice.pcm_rx_buf = NULL;
s_voice.pcm_rx_buf_size = 0;
free(s_voice.text_agg);
s_voice.text_agg = NULL;
s_voice.text_agg_size = 0;
free(s_voice.bin_agg);
s_voice.bin_agg = NULL;
s_voice.bin_agg_size = 0;
s_voice.opus_enc_in_size = 0;
s_voice.opus_enc_out_size = 0;
}

View File

@@ -0,0 +1,148 @@
#include "voice_interaction_internal.h"
#include <string.h>
#include "esp_audio_enc.h"
#include "esp_log.h"
#include "voice_audio.h"
static const char *TAG = "voice_interaction";
void voice_uplink_task(void *arg) {
(void)arg;
const int frame_ms = CONFIG_TQ_VOICE_OPUS_FRAME_MS;
bool low_stack_warned = false;
UBaseType_t start_hwm = uxTaskGetStackHighWaterMark(NULL);
ESP_LOGI(TAG, "[stage] uplink_task_started: stack_hwm_words=%u", (unsigned)start_hwm);
while (true) {
if (!low_stack_warned) {
UBaseType_t hwm = uxTaskGetStackHighWaterMark(NULL);
if (hwm < 256) {
low_stack_warned = true;
ESP_LOGW(TAG, "[stage] uplink_task_low_stack: hwm_words=%u", (unsigned)hwm);
}
}
bool active = false;
bool connected = false;
bool started = false;
bool tap_active = false;
bool listening = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
active = s_voice.session_active && !s_voice.stop_requested;
connected = s_voice.ws_connected;
started = s_voice.started;
tap_active = s_voice.tap_active;
listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING);
voice_unlock();
}
if (!active) {
break;
}
if (!(connected && started && listening)) {
vTaskDelay(pdMS_TO_TICKS(20));
continue;
}
if (tap_active) {
char read_err[64] = {0};
esp_err_t rc = voice_audio_read_pcm(s_voice.pcm_tx_buf,
(size_t)s_voice.opus_enc_in_size / VOICE_SAMPLE_BYTES,
VOICE_AUDIO_IO_TIMEOUT_MS,
read_err,
sizeof(read_err));
if (rc != ESP_OK) {
memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size);
}
} else {
memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size);
vTaskDelay(pdMS_TO_TICKS(frame_ms));
}
esp_audio_enc_in_frame_t in_frame = {
.buffer = (uint8_t *)s_voice.pcm_tx_buf,
.len = (uint32_t)s_voice.opus_enc_in_size,
};
esp_audio_enc_out_frame_t out_frame = {
.buffer = s_voice.opus_tx_buf,
.len = (uint32_t)s_voice.opus_enc_out_size,
.encoded_bytes = 0,
};
esp_audio_err_t enc_err = esp_opus_enc_process(s_voice.opus_enc, &in_frame, &out_frame);
if (enc_err != ESP_AUDIO_ERR_OK || out_frame.encoded_bytes == 0) {
continue;
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
esp_websocket_client_handle_t ws = s_voice.ws;
bool can_send = s_voice.ws_connected;
voice_unlock();
if (ws != NULL && can_send) {
int ret = esp_websocket_client_send_bin(ws,
(const char *)s_voice.opus_tx_buf,
(int)out_frame.encoded_bytes,
pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS));
if (ret >= 0) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.upstream_packets++;
s_voice.last_event_ms = voice_now_ms();
voice_unlock();
}
}
}
}
}
bool delete_with_caps = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
delete_with_caps = s_voice.uplink_task_with_caps;
s_voice.uplink_task = NULL;
s_voice.uplink_task_with_caps = false;
voice_unlock();
}
voice_delete_self_task(delete_with_caps);
}
void voice_heartbeat_task(void *arg) {
(void)arg;
while (true) {
bool active = false;
bool connected = false;
bool started = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
active = s_voice.session_active && !s_voice.stop_requested;
connected = s_voice.ws_connected;
started = s_voice.started;
voice_unlock();
}
if (!active) {
break;
}
if (connected && started) {
ESP_LOGI(TAG, "[stage] heartbeat_send");
(void)voice_send_directive("continue-task", "HeartBeat", true);
}
vTaskDelay(pdMS_TO_TICKS(CONFIG_TQ_VOICE_HEARTBEAT_SEC * 1000));
}
bool delete_with_caps = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
delete_with_caps = s_voice.heartbeat_task_with_caps;
s_voice.heartbeat_task = NULL;
s_voice.heartbeat_task_with_caps = false;
voice_unlock();
}
voice_delete_self_task(delete_with_caps);
}

View File

@@ -0,0 +1,570 @@
#include "voice_interaction_internal.h"
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "cJSON.h"
#include "esp_audio_dec.h"
#include "esp_audio_enc.h"
#include "esp_log.h"
#include "voice_audio.h"
static const char *TAG = "voice_interaction";
static char *voice_build_directive_message(const char *action,
const char *directive,
bool with_dialog_id) {
cJSON *root = cJSON_CreateObject();
if (root == NULL) {
return NULL;
}
cJSON *header = cJSON_AddObjectToObject(root, "header");
cJSON_AddStringToObject(header, "action", action);
cJSON_AddStringToObject(header, "task_id", s_voice.task_id);
cJSON_AddStringToObject(header, "streaming", VOICE_STREAMING_MODE);
cJSON *payload = cJSON_AddObjectToObject(root, "payload");
cJSON *input = cJSON_AddObjectToObject(payload, "input");
cJSON_AddStringToObject(input, "directive", directive);
if (with_dialog_id && s_voice.dialog_id[0] != '\0') {
cJSON_AddStringToObject(input, "dialog_id", s_voice.dialog_id);
}
char *text = cJSON_PrintUnformatted(root);
cJSON_Delete(root);
return text;
}
static char *voice_build_start_message(void) {
cJSON *root = cJSON_CreateObject();
if (root == NULL) {
return NULL;
}
cJSON *header = cJSON_AddObjectToObject(root, "header");
cJSON_AddStringToObject(header, "action", "run-task");
cJSON_AddStringToObject(header, "task_id", s_voice.task_id);
cJSON_AddStringToObject(header, "streaming", VOICE_STREAMING_MODE);
cJSON *payload = cJSON_AddObjectToObject(root, "payload");
cJSON_AddStringToObject(payload, "task_group", VOICE_TASK_GROUP);
cJSON_AddStringToObject(payload, "task", VOICE_TASK_NAME);
cJSON_AddStringToObject(payload, "function", VOICE_FUNCTION_NAME);
cJSON_AddStringToObject(payload, "model", VOICE_MODEL_NAME);
cJSON *input = cJSON_AddObjectToObject(payload, "input");
cJSON_AddStringToObject(input, "directive", "Start");
cJSON_AddStringToObject(input, "workspace_id", CONFIG_TQ_VOICE_WORKSPACE_ID);
cJSON_AddStringToObject(input, "app_id", CONFIG_TQ_VOICE_APP_ID);
cJSON *parameters = cJSON_AddObjectToObject(payload, "parameters");
cJSON *upstream = cJSON_AddObjectToObject(parameters, "upstream");
cJSON_AddStringToObject(upstream, "type", "AudioOnly");
cJSON_AddStringToObject(upstream, "mode", "tap2talk");
cJSON_AddStringToObject(upstream, "audio_format", "raw-opus");
cJSON_AddNumberToObject(upstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE);
cJSON *downstream = cJSON_AddObjectToObject(parameters, "downstream");
cJSON_AddNumberToObject(downstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE);
cJSON_AddStringToObject(downstream, "audio_format", "raw-opus");
cJSON_AddNumberToObject(downstream, "frame_size", CONFIG_TQ_VOICE_OPUS_FRAME_MS);
cJSON_AddNumberToObject(downstream,
"bit_rate",
CONFIG_TQ_VOICE_OPUS_BITRATE_KBPS);
if (strlen(CONFIG_TQ_VOICE_TTS_VOICE) > 0) {
cJSON_AddStringToObject(downstream, "voice", CONFIG_TQ_VOICE_TTS_VOICE);
}
cJSON *client_info = cJSON_AddObjectToObject(parameters, "client_info");
cJSON_AddStringToObject(client_info, "user_id", voice_safe_user_id());
cJSON *device = cJSON_AddObjectToObject(client_info, "device");
cJSON_AddStringToObject(device, "uuid", s_voice.device_uuid);
char *text = cJSON_PrintUnformatted(root);
cJSON_Delete(root);
return text;
}
static esp_err_t voice_send_text(const char *text, int len) {
if (text == NULL || len <= 0) {
return ESP_ERR_INVALID_ARG;
}
esp_websocket_client_handle_t ws = NULL;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
return ESP_ERR_TIMEOUT;
}
ws = s_voice.ws;
bool connected = s_voice.ws_connected;
voice_unlock();
if (ws == NULL || !connected) {
ESP_LOGW(TAG, "[stage] ws_send_text skipped: ws=%p connected=%d len=%d", (void *)ws, connected, len);
return ESP_ERR_INVALID_STATE;
}
int ret = esp_websocket_client_send_text(ws,
text,
len,
pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS));
if (ret < 0) {
ESP_LOGW(TAG, "[stage] ws_send_text failed: len=%d", len);
return ESP_FAIL;
}
ESP_LOGI(TAG, "[stage] ws_send_text ok: len=%d", len);
return ESP_OK;
}
esp_err_t voice_send_directive(const char *action,
const char *directive,
bool with_dialog_id) {
char *text = NULL;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
return ESP_ERR_TIMEOUT;
}
text = voice_build_directive_message(action, directive, with_dialog_id);
voice_unlock();
if (text == NULL) {
return ESP_ERR_NO_MEM;
}
esp_err_t err = voice_send_text(text, (int)strlen(text));
ESP_LOGI(TAG,
"[stage] directive: action=%s directive=%s with_dialog_id=%d result=%s",
action,
directive,
with_dialog_id,
esp_err_to_name(err));
cJSON_free(text);
return err;
}
static void voice_handle_downstream_packet(const uint8_t *data, size_t len) {
if (data == NULL || len == 0) {
return;
}
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
return;
}
void *opus_dec = s_voice.opus_dec;
uint8_t *pcm_buf = s_voice.pcm_rx_buf;
size_t pcm_buf_size = s_voice.pcm_rx_buf_size;
voice_unlock();
if (opus_dec == NULL || pcm_buf == NULL || pcm_buf_size == 0) {
return;
}
esp_audio_dec_in_raw_t raw = {
.buffer = (uint8_t *)data,
.len = (uint32_t)len,
.consumed = 0,
};
while (raw.len > 0) {
esp_audio_dec_out_frame_t out = {
.buffer = pcm_buf,
.len = (uint32_t)pcm_buf_size,
.needed_size = 0,
.decoded_size = 0,
};
esp_audio_dec_info_t dec_info = {0};
esp_audio_err_t ret = esp_opus_dec_decode(opus_dec, &raw, &out, &dec_info);
if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) {
if (out.needed_size > pcm_buf_size) {
uint8_t *new_buf = (uint8_t *)voice_realloc_prefer_psram(pcm_buf, out.needed_size);
if (new_buf == NULL) {
ESP_LOGE(TAG, "No memory to extend pcm rx buffer to %u", out.needed_size);
break;
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.pcm_rx_buf = new_buf;
s_voice.pcm_rx_buf_size = out.needed_size;
voice_unlock();
}
pcm_buf = new_buf;
pcm_buf_size = out.needed_size;
continue;
}
break;
}
if (ret != ESP_AUDIO_ERR_OK) {
ESP_LOGW(TAG, "opus decode failed: %d", ret);
break;
}
if (out.decoded_size > 0) {
char audio_err[64] = {0};
size_t samples = (size_t)out.decoded_size / VOICE_SAMPLE_BYTES;
esp_err_t write_err = voice_audio_write_pcm((const int16_t *)out.buffer,
samples,
VOICE_AUDIO_IO_TIMEOUT_MS,
audio_err,
sizeof(audio_err));
if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t pcm_ms = ((int64_t)samples * 1000 +
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) /
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE;
int64_t base_ms = s_voice.playback_deadline_ms > now_ms ?
s_voice.playback_deadline_ms : now_ms;
s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS;
s_voice.last_downstream_ms = now_ms;
voice_unlock();
}
}
if (raw.consumed == 0 || raw.consumed > raw.len) {
break;
}
raw.buffer += raw.consumed;
raw.len -= raw.consumed;
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.downstream_packets++;
s_voice.last_event_ms = voice_now_ms();
voice_unlock();
}
}
static const char *voice_get_json_str(cJSON *obj, const char *name) {
if (obj == NULL || name == NULL) {
return NULL;
}
cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name);
if (cJSON_IsString(item) && item->valuestring != NULL) {
return item->valuestring;
}
return NULL;
}
static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) {
if (obj == NULL || name == NULL || out_value == NULL) {
return false;
}
cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name);
if (!cJSON_IsBool(item)) {
return false;
}
*out_value = cJSON_IsTrue(item);
return true;
}
static void voice_log_final_text(const char *stage, const char *text) {
if (stage == NULL || text == NULL || text[0] == '\0') {
return;
}
size_t len = strlen(text);
const int preview = 240;
ESP_LOGI(TAG,
"[stage] %s: len=%u text=%.*s%s",
stage,
(unsigned)len,
preview,
text,
(len > (size_t)preview) ? "..." : "");
}
static void voice_handle_output_event(cJSON *output) {
const char *event_name = voice_get_json_str(output, "event");
if (event_name == NULL) {
return;
}
const char *dialog_id = voice_get_json_str(output, "dialog_id");
const char *state = NULL;
if (strcmp(event_name, "DialogStateChanged") == 0) {
state = voice_get_json_str(output, "state");
}
ESP_LOGI(TAG,
"[stage] ws_event_output: event=%s dialog_id=%s state=%s",
event_name,
dialog_id != NULL ? dialog_id : "-",
state != NULL ? state : "-");
bool finished = false;
bool has_finished = voice_get_json_bool(output, "finished", &finished);
if (has_finished && finished) {
if (strcmp(event_name, "SpeechContent") == 0) {
voice_log_final_text("asr_final_text", voice_get_json_str(output, "text"));
} else if (strcmp(event_name, "RespondingContent") == 0) {
const char *final_text = voice_get_json_str(output, "text");
if (final_text == NULL || final_text[0] == '\0') {
final_text = voice_get_json_str(output, "spoken");
}
voice_log_final_text("response_final_text", final_text);
}
}
bool should_close_audio = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
if (dialog_id != NULL) {
strlcpy(s_voice.dialog_id, dialog_id, sizeof(s_voice.dialog_id));
}
s_voice.last_event_ms = voice_now_ms();
if (strcmp(event_name, "Started") == 0) {
s_voice.started = true;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id);
} else if (strcmp(event_name, "DialogStateChanged") == 0) {
if (state != NULL) {
if (strcmp(state, "Listening") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING;
} else if (strcmp(state, "Thinking") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING;
} else if (strcmp(state, "Responding") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING;
}
}
ESP_LOGI(TAG,
"[stage] dialog_state_changed: state=%s current=%s",
state != NULL ? state : "-",
voice_interaction_dialog_state_str(s_voice.dialog_state));
} else if (strcmp(event_name, "SpeechEnded") == 0) {
ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active);
} else if (strcmp(event_name, "Stopped") == 0) {
s_voice.started = false;
s_voice.tap_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
should_close_audio = true;
ESP_LOGI(TAG, "[stage] session_stopped_by_server");
} else if (strcmp(event_name, "Error") == 0) {
const char *error_msg = voice_get_json_str(output, "error_message");
if (error_msg != NULL) {
voice_set_last_error_locked(error_msg);
ESP_LOGW(TAG, "[stage] server_error: %s", error_msg);
}
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.tap_active = false;
should_close_audio = true;
}
voice_unlock();
}
if (should_close_audio) {
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close");
}
if (strcmp(event_name, "RespondingStarted") == 0) {
(void)voice_send_directive("continue-task", "LocalRespondingStarted", true);
} else if (strcmp(event_name, "RespondingEnded") == 0) {
(void)voice_send_directive("continue-task", "LocalRespondingEnded", true);
}
}
static void voice_handle_text_message(const char *text, size_t len) {
cJSON *root = cJSON_ParseWithLength(text, len);
if (root == NULL) {
ESP_LOGW(TAG, "Invalid ws text payload");
return;
}
cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header");
const char *header_event = voice_get_json_str(header, "event");
cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload");
cJSON *output = NULL;
if (payload != NULL) {
output = cJSON_GetObjectItemCaseSensitive(payload, "output");
}
if (header != NULL) {
cJSON *status_code = cJSON_GetObjectItemCaseSensitive(header, "status_code");
cJSON *status_msg = cJSON_GetObjectItemCaseSensitive(header, "status_message");
if (cJSON_IsNumber(status_code) && status_code->valueint >= 400) {
ESP_LOGW(TAG,
"[stage] ws_header_error: status_code=%d status_message=%s",
status_code->valueint,
(cJSON_IsString(status_msg) && status_msg->valuestring != NULL) ? status_msg->valuestring : "-");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
if (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) {
voice_set_last_error_locked(status_msg->valuestring);
} else {
voice_set_last_error_locked("websocket status error");
}
voice_unlock();
}
}
}
if (output != NULL) {
voice_handle_output_event(output);
} else if (header_event != NULL && strcmp(header_event, "task-failed") == 0) {
const char *error_msg = voice_get_json_str(header, "error_message");
ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed");
voice_unlock();
}
}
cJSON_Delete(root);
}
static void voice_handle_text_chunk(esp_websocket_event_data_t *data) {
if (data->payload_offset == 0) {
free(s_voice.text_agg);
s_voice.text_agg = NULL;
s_voice.text_agg_size = 0;
if (data->payload_len <= 0 || data->payload_len > 4096) {
return;
}
s_voice.text_agg = (uint8_t *)voice_calloc_prefer_psram(1, (size_t)data->payload_len + 1);
if (s_voice.text_agg == NULL) {
ESP_LOGW(TAG, "[stage] text_agg_alloc_failed: len=%d", data->payload_len);
return;
}
s_voice.text_agg_size = (size_t)data->payload_len;
}
if (s_voice.text_agg == NULL || s_voice.text_agg_size == 0) {
return;
}
if ((size_t)data->payload_offset + (size_t)data->data_len > s_voice.text_agg_size) {
free(s_voice.text_agg);
s_voice.text_agg = NULL;
s_voice.text_agg_size = 0;
return;
}
memcpy(s_voice.text_agg + data->payload_offset, data->data_ptr, (size_t)data->data_len);
bool complete = data->fin &&
((size_t)data->payload_offset + (size_t)data->data_len == s_voice.text_agg_size);
if (!complete) {
return;
}
voice_handle_text_message((const char *)s_voice.text_agg, s_voice.text_agg_size);
free(s_voice.text_agg);
s_voice.text_agg = NULL;
s_voice.text_agg_size = 0;
}
static void voice_handle_binary_chunk(esp_websocket_event_data_t *data) {
if (data->payload_offset == 0) {
free(s_voice.bin_agg);
s_voice.bin_agg = NULL;
s_voice.bin_agg_size = 0;
if (data->payload_len <= 0 || data->payload_len > 16384) {
return;
}
s_voice.bin_agg = (uint8_t *)voice_malloc_prefer_psram((size_t)data->payload_len);
if (s_voice.bin_agg == NULL) {
ESP_LOGW(TAG, "[stage] bin_agg_alloc_failed: len=%d", data->payload_len);
return;
}
s_voice.bin_agg_size = (size_t)data->payload_len;
}
if (s_voice.bin_agg == NULL || s_voice.bin_agg_size == 0) {
return;
}
if ((size_t)data->payload_offset + (size_t)data->data_len > s_voice.bin_agg_size) {
free(s_voice.bin_agg);
s_voice.bin_agg = NULL;
s_voice.bin_agg_size = 0;
return;
}
memcpy(s_voice.bin_agg + data->payload_offset, data->data_ptr, (size_t)data->data_len);
bool complete = data->fin &&
((size_t)data->payload_offset + (size_t)data->data_len == s_voice.bin_agg_size);
if (!complete) {
return;
}
voice_handle_downstream_packet(s_voice.bin_agg, s_voice.bin_agg_size);
free(s_voice.bin_agg);
s_voice.bin_agg = NULL;
s_voice.bin_agg_size = 0;
}
void voice_websocket_event_handler(void *handler_args,
esp_event_base_t base,
int32_t event_id,
void *event_data) {
(void)handler_args;
(void)base;
UBaseType_t ws_hwm = uxTaskGetStackHighWaterMark(NULL);
if (!s_voice.ws_low_stack_warned && ws_hwm < 256) {
s_voice.ws_low_stack_warned = true;
ESP_LOGW(TAG, "[stage] ws_task_low_stack: hwm_words=%u", (unsigned)ws_hwm);
}
esp_websocket_event_data_t *data = (esp_websocket_event_data_t *)event_data;
switch ((esp_websocket_event_id_t)event_id) {
case WEBSOCKET_EVENT_CONNECTED: {
ESP_LOGI(TAG, "[stage] ws_connected");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_connected = true;
s_voice.last_event_ms = voice_now_ms();
voice_unlock();
}
char *start_msg = NULL;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
start_msg = voice_build_start_message();
voice_unlock();
}
if (start_msg != NULL) {
esp_err_t send_err = voice_send_text(start_msg, (int)strlen(start_msg));
ESP_LOGI(TAG, "[stage] start_message_sent: result=%s", esp_err_to_name(send_err));
cJSON_free(start_msg);
} else {
ESP_LOGE(TAG, "[stage] start_message_build_failed");
}
break;
}
case WEBSOCKET_EVENT_DISCONNECTED:
ESP_LOGW(TAG, "[stage] ws_disconnected");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
voice_unlock();
}
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "ws_disconnected_close");
break;
case WEBSOCKET_EVENT_DATA:
if (data == NULL || data->data_ptr == NULL || data->data_len <= 0) {
break;
}
if (data->op_code == WS_TRANSPORT_OPCODES_TEXT) {
voice_handle_text_chunk(data);
} else if (data->op_code == WS_TRANSPORT_OPCODES_BINARY) {
voice_handle_binary_chunk(data);
}
break;
case WEBSOCKET_EVENT_ERROR:
ESP_LOGW(TAG, "[stage] ws_error");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_set_last_error_locked("websocket error");
s_voice.last_event_ms = voice_now_ms();
voice_unlock();
}
break;
default:
break;
}
}