Bairan.Zhang/AI_Printer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): migrate to push2talk and serialize websocket text sends

Browse Source
This commit is contained in:
admin
2026-03-01 01:21:18 +08:00
parent 9deeab738e
commit dd108e2658
19 changed files with 891 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
components/domain/CMakeLists.txt
View File

@@ -17,6 +17,7 @@ idf_component_register(
"src/voice_interaction_ws_audio.c"
"src/voice_interaction_marker.c"
"src/voice_interaction_tasks.c"
"src/voice_interaction_mic_key.c"
"third_party/qrcodegen.c"
INCLUDE_DIRS
"include"

20
components/domain/internal/voice_interaction_internal.h
View File

@@ -8,6 +8,7 @@
#include "esp_opus_enc.h"
#include "esp_websocket_client.h"
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#include "domain.h"
@@ -21,7 +22,7 @@
#define VOICE_SAMPLE_BYTES 2
#define VOICE_STATUS_LOCK_TIMEOUT_MS 1000
#define VOICE_WS_SEND_TIMEOUT_MS 1000
#define VOICE_WS_SEND_TIMEOUT_MS 3000
#define VOICE_WS_TASK_STACK 24576
#define VOICE_UPLINK_TASK_STACK 32768
#define VOICE_HEARTBEAT_TASK_STACK 4096
@@ -31,15 +32,25 @@
#define VOICE_AUDIO_DRAIN_QUIET_MS 180
#define VOICE_AUDIO_DRAIN_WAIT_MS 2200
#define VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS 3500
#define VOICE_WS_TEXT_QUEUE_LEN 16
#define VOICE_WS_TEXT_QUEUE_RX_TIMEOUT_MS 100
typedef struct {
char *text;
int len;
} voice_ws_text_msg_t;
typedef struct {
SemaphoreHandle_t lock;
esp_websocket_client_handle_t ws;
QueueHandle_t ws_text_queue;
TaskHandle_t uplink_task;
TaskHandle_t heartbeat_task;
TaskHandle_t ws_text_task;
bool uplink_task_with_caps;
bool heartbeat_task_with_caps;
bool ws_text_task_with_caps;
void *opus_enc;
void *opus_dec;
@@ -63,7 +74,10 @@ typedef struct {
bool ws_connected;
bool started;
bool tap_active;
bool speech_active;
bool stop_requested;
bool ws_text_shutdown;
bool ws_text_busy;
bool ws_low_stack_warned;
bool response_first_sentence_checked;
@@ -129,6 +143,10 @@ void voice_websocket_event_handler(void *handler_args,
void *event_data);
void voice_interaction_ws_init_cjson_hooks(void);
esp_err_t voice_interaction_try_send_speech(char *err, size_t err_len);
void voice_uplink_task(void *arg);
void voice_heartbeat_task(void *arg);
void voice_ws_text_task(void *arg);
esp_err_t voice_interaction_mic_key_init(void);

217
components/domain/src/voice_interaction.c
View File

@@ -20,18 +20,44 @@ static uint32_t clamp_timeout_ms(uint32_t timeout_ms, uint32_t default_ms, uint3
return value;
}
static bool voice_wait_ws_text_idle(uint32_t timeout_ms) {
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
bool busy = false;
QueueHandle_t queue = NULL;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
busy = s_voice.ws_text_busy;
queue = s_voice.ws_text_queue;
voice_unlock();
}
UBaseType_t pending = 0;
if (queue != NULL) {
pending = uxQueueMessagesWaiting(queue);
}
if (!busy && pending == 0) {
return true;
}
vTaskDelay(pdMS_TO_TICKS(20));
}
return false;
}
static bool voice_wait_worker_tasks_exit(uint32_t timeout_ms) {
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
bool uplink_alive = false;
bool heartbeat_alive = false;
bool ws_text_alive = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
uplink_alive = (s_voice.uplink_task != NULL);
heartbeat_alive = (s_voice.heartbeat_task != NULL);
ws_text_alive = (s_voice.ws_text_task != NULL);
voice_unlock();
}
if (!uplink_alive && !heartbeat_alive) {
if (!uplink_alive && !heartbeat_alive && !ws_text_alive) {
return true;
}
vTaskDelay(pdMS_TO_TICKS(20));
@@ -39,9 +65,10 @@ static bool voice_wait_worker_tasks_exit(uint32_t timeout_ms) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
ESP_LOGW(TAG,
"[stage] worker_task_exit_timeout: uplink_alive=%d heartbeat_alive=%d",
"[stage] worker_task_exit_timeout: uplink_alive=%d heartbeat_alive=%d ws_text_alive=%d",
s_voice.uplink_task != NULL,
s_voice.heartbeat_task != NULL);
s_voice.heartbeat_task != NULL,
s_voice.ws_text_task != NULL);
voice_unlock();
}
return false;
@@ -76,6 +103,14 @@ esp_err_t voice_interaction_init(void) {
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
esp_err_t mic_key_err = voice_interaction_mic_key_init();
if (mic_key_err != ESP_OK) {
vSemaphoreDelete(s_voice.lock);
memset(&s_voice, 0, sizeof(s_voice));
return mic_key_err;
}
s_voice.ready = true;
return ESP_OK;
}
@@ -152,10 +187,14 @@ esp_err_t voice_interaction_start_with_timeout(uint32_t timeout_ms, char *err, s
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.stop_requested = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.ws_low_stack_warned = false;
s_voice.response_first_sentence_checked = false;
s_voice.uplink_task_with_caps = false;
s_voice.heartbeat_task_with_caps = false;
s_voice.ws_text_task_with_caps = false;
s_voice.ws_text_shutdown = false;
s_voice.ws_text_busy = false;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGI(TAG,
"[stage] session_context_prepared: task_id=%s device_uuid=%s",
@@ -172,9 +211,12 @@ esp_err_t voice_interaction_start_with_timeout(uint32_t timeout_ms, char *err, s
.buffer_size = 2048,
.network_timeout_ms = 10000,
.reconnect_timeout_ms = 3000,
.ping_interval_sec = 20,
.ping_interval_sec = 55,
.disable_auto_reconnect = false,
.pingpong_timeout_sec = 10,
// DashScope protocol already requires application-level HeartBeat keepalive.
// Some networks may drop/ignore ws-level PONG intermittently, causing false
// disconnects if we enforce pingpong timeout aggressively.
.disable_pingpong_discon = true,
#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
.crt_bundle_attach = esp_crt_bundle_attach,
#endif
@@ -236,17 +278,53 @@ esp_err_t voice_interaction_start_with_timeout(uint32_t timeout_ms, char *err, s
s_voice.session_active = true;
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.speech_active = false;
s_voice.ws_text_shutdown = false;
s_voice.ws_text_busy = false;
voice_unlock();
voice_log_status_snapshot("session_marked_active");
voice_log_heap_snapshot("before_create_uplink");
ESP_LOGI(TAG, "[stage] uplink_task_create_begin");
BaseType_t ok = voice_create_task_prefer_psram(voice_uplink_task,
"voice_uplink",
VOICE_UPLINK_TASK_STACK,
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_fill_err(err, err_len, "voice lock timeout");
ESP_LOGE(TAG, "[stage] session_start_abort: lock timeout before ws_text_task create");
(void)voice_interaction_stop(NULL, 0);
return ESP_ERR_TIMEOUT;
}
if (s_voice.ws_text_queue == NULL) {
s_voice.ws_text_queue = xQueueCreate(VOICE_WS_TEXT_QUEUE_LEN, sizeof(voice_ws_text_msg_t));
}
QueueHandle_t ws_text_queue = s_voice.ws_text_queue;
voice_unlock();
if (ws_text_queue == NULL) {
voice_fill_err(err, err_len, "create ws text queue failed");
ESP_LOGE(TAG, "[stage] ws_text_queue_create_failed");
(void)voice_interaction_stop(NULL, 0);
return ESP_ERR_NO_MEM;
}
ESP_LOGI(TAG, "[stage] ws_text_task_create_begin");
BaseType_t ok = voice_create_task_prefer_psram(voice_ws_text_task,
"voice_ws_text",
VOICE_HEARTBEAT_TASK_STACK,
5,
&s_voice.uplink_task,
&s_voice.uplink_task_with_caps);
&s_voice.ws_text_task,
&s_voice.ws_text_task_with_caps);
if (ok != pdPASS) {
voice_fill_err(err, err_len, "create ws text task failed");
ESP_LOGE(TAG, "[stage] ws_text_task_create_failed");
(void)voice_interaction_stop(NULL, 0);
return ESP_ERR_NO_MEM;
}
ESP_LOGI(TAG, "[stage] ws_text_task_create_ok");
ESP_LOGI(TAG, "[stage] uplink_task_create_begin");
ok = voice_create_task_prefer_psram(voice_uplink_task,
"voice_uplink",
VOICE_UPLINK_TASK_STACK,
5,
&s_voice.uplink_task,
&s_voice.uplink_task_with_caps);
if (ok != pdPASS) {
voice_fill_err(err, err_len, "create uplink task failed");
ESP_LOGE(TAG, "[stage] uplink_task_create_failed");
@@ -346,22 +424,35 @@ esp_err_t voice_interaction_stop_with_timeout(uint32_t timeout_ms, char *err, si
bool ws_connected = s_voice.ws_connected;
TaskHandle_t uplink_task = s_voice.uplink_task;
TaskHandle_t heartbeat_task = s_voice.heartbeat_task;
TaskHandle_t ws_text_task = s_voice.ws_text_task;
QueueHandle_t ws_text_queue = s_voice.ws_text_queue;
esp_websocket_client_handle_t ws = s_voice.ws;
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
voice_log_status_snapshot("session_stop_requested");
if (ws_connected) {
(void)voice_send_directive("finish-task", "Stop", true);
if (!voice_wait_ws_text_idle(VOICE_WS_SEND_TIMEOUT_MS + 1200)) {
ESP_LOGW(TAG, "[stage] ws_text_idle_timeout_before_shutdown");
}
}
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS, "session_stop_close");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_text_shutdown = true;
voice_unlock();
}
if (uplink_task != NULL) {
(void)xTaskAbortDelay(uplink_task);
}
if (heartbeat_task != NULL) {
(void)xTaskAbortDelay(heartbeat_task);
}
if (ws_text_task != NULL) {
(void)xTaskAbortDelay(ws_text_task);
}
bool workers_exited = voice_wait_worker_tasks_exit(worker_exit_timeout_ms);
if (!workers_exited) {
voice_fill_err(err, err_len, "voice worker exit timeout");
@@ -373,6 +464,11 @@ esp_err_t voice_interaction_stop_with_timeout(uint32_t timeout_ms, char *err, si
return ESP_ERR_TIMEOUT;
}
if (ws_text_queue != NULL) {
vQueueDelete(ws_text_queue);
ws_text_queue = NULL;
}
if (ws != NULL) {
ESP_LOGI(TAG, "[stage] ws_stop_destroy_begin");
esp_websocket_unregister_events(ws,
@@ -385,14 +481,20 @@ esp_err_t voice_interaction_stop_with_timeout(uint32_t timeout_ms, char *err, si
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws = NULL;
s_voice.ws_text_queue = NULL;
s_voice.uplink_task = NULL;
s_voice.heartbeat_task = NULL;
s_voice.ws_text_task = NULL;
s_voice.uplink_task_with_caps = false;
s_voice.heartbeat_task_with_caps = false;
s_voice.ws_text_task_with_caps = false;
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.session_active = false;
s_voice.stop_requested = false;
s_voice.speech_active = false;
s_voice.ws_text_shutdown = false;
s_voice.ws_text_busy = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.dialog_id[0] = '\0';
s_voice.response_first_sentence_checked = false;
@@ -412,6 +514,60 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) {
return voice_interaction_stop_with_timeout(1500, err, err_len);
}
esp_err_t voice_interaction_try_send_speech(char *err, size_t err_len) {
bool should_send = false;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_fill_err(err, err_len, "voice lock timeout");
return ESP_ERR_TIMEOUT;
}
if (!s_voice.session_active || !s_voice.ws_connected || !s_voice.started) {
voice_unlock();
voice_fill_err(err, err_len, "voice session not ready");
return ESP_ERR_INVALID_STATE;
}
should_send = s_voice.tap_active &&
!s_voice.speech_active &&
s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING;
voice_unlock();
if (!should_send) {
return ESP_OK;
}
esp_err_t send_err = voice_send_directive("continue-task", "SendSpeech", true);
if (send_err != ESP_OK) {
voice_fill_err(err, err_len, "send speech failed");
ESP_LOGW(TAG, "[stage] send_speech_failed: rc=0x%x", (unsigned)send_err);
return send_err;
}
bool should_send_stop = false;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
(void)voice_send_directive("continue-task", "StopSpeech", true);
voice_fill_err(err, err_len, "voice lock timeout");
return ESP_ERR_TIMEOUT;
}
if (s_voice.session_active &&
s_voice.ws_connected &&
s_voice.started &&
s_voice.tap_active &&
s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING) {
s_voice.speech_active = true;
s_voice.last_event_ms = voice_now_ms();
} else if (s_voice.session_active && s_voice.ws_connected && s_voice.started) {
should_send_stop = true;
}
voice_unlock();
if (should_send_stop) {
(void)voice_send_directive("continue-task", "StopSpeech", true);
}
return ESP_OK;
}
esp_err_t voice_interaction_tap_start(char *err, size_t err_len) {
ESP_LOGI(TAG, "[stage] tap_start_enter");
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
@@ -461,11 +617,33 @@ esp_err_t voice_interaction_tap_start(char *err, size_t err_len) {
bool listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING);
s_voice.tap_active = true;
s_voice.speech_active = false;
voice_unlock();
ESP_LOGI(TAG, "[stage] tap_start_marked_active: listening=%d", listening);
if (!listening) {
(void)voice_send_directive("continue-task", "RequestToSpeak", true);
esp_err_t req_rc = voice_send_directive("continue-task", "RequestToSpeak", true);
if (req_rc != ESP_OK) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
}
voice_fill_err(err, err_len, "request to speak failed");
ESP_LOGW(TAG, "[stage] tap_start_abort: request_to_speak failed rc=0x%x", (unsigned)req_rc);
return req_rc;
}
} else {
esp_err_t send_speech_err = voice_interaction_try_send_speech(err, err_len);
if (send_speech_err != ESP_OK) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
}
voice_audio_close();
return send_speech_err;
}
}
voice_log_status_snapshot("tap_start_exit_ok");
@@ -491,12 +669,21 @@ esp_err_t voice_interaction_tap_cancel(char *err, size_t err_len) {
return ESP_ERR_INVALID_STATE;
}
bool should_send_stop = s_voice.speech_active;
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
voice_log_status_snapshot("tap_cancel_marked_inactive");
esp_err_t cancel_err = voice_send_directive("continue-task", "CancelSpeech", true);
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS, "tap_cancel_close");
return cancel_err;
esp_err_t stop_err = ESP_OK;
if (should_send_stop) {
stop_err = voice_send_directive("continue-task", "StopSpeech", true);
if (stop_err != ESP_OK) {
// Fail-safe close to avoid leaving audio device open in abnormal flow.
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "tap_cancel_stop_failed_close");
}
}
return stop_err;
}
void voice_interaction_get_status(voice_interaction_status_t *out_status) {

4
components/domain/src/voice_interaction_common.c
View File

@@ -54,12 +54,14 @@ void voice_log_status_snapshot(const char *stage) {
}
ESP_LOGI(TAG,
"[stage] %s: active=%d ws=%d started=%d tap=%d state=%s up=%" PRIu32 " down=%" PRIu32 " err=%s",
"[stage] %s: active=%d ws=%d started=%d tap=%d speech=%d state=%s up=%" PRIu32
" down=%" PRIu32 " err=%s",
stage,
s_voice.session_active,
s_voice.ws_connected,
s_voice.started,
s_voice.tap_active,
s_voice.speech_active,
voice_interaction_dialog_state_str(s_voice.dialog_state),
s_voice.upstream_packets,
s_voice.downstream_packets,

227
components/domain/src/voice_interaction_mic_key.c Normal file
View File

@@ -0,0 +1,227 @@
#include "voice_interaction_internal.h"
#include <inttypes.h>
#include "esp_log.h"
#include "freertos/queue.h"
#define VOICE_MIC_KEY_TASK_STACK 6144
#define VOICE_MIC_KEY_QUEUE_LEN 8
#define VOICE_MIC_KEY_SESSION_READY_WAIT_MS 2000
#define VOICE_MIC_KEY_SESSION_READY_POLL_MS 30
typedef enum {
VOICE_MIC_KEY_CMD_PRESS_DOWN = 0,
VOICE_MIC_KEY_CMD_PRESS_UP,
} voice_mic_key_cmd_t;
static const char *TAG = "voice_mic_key";
static QueueHandle_t s_mic_key_queue;
static TaskHandle_t s_mic_key_task;
static bool s_mic_key_task_with_caps;
static bool s_mic_key_ready;
static bool voice_mic_key_round_busy(const voice_interaction_status_t *voice,
const printer_runtime_status_t *printer) {
if (voice == NULL || printer == NULL) {
return false;
}
bool marker_busy = voice->marker_image_gen_running;
bool print_busy = printer->busy || printer->queue_depth > 0;
return marker_busy || print_busy;
}
static bool voice_mic_key_session_ready(const voice_interaction_status_t *voice) {
return voice != NULL && voice->session_active && voice->ws_connected && voice->started;
}
static bool voice_mic_key_wait_session_ready(uint32_t timeout_ms) {
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
voice_interaction_status_t voice = {0};
voice_interaction_get_status(&voice);
if (voice_mic_key_session_ready(&voice)) {
return true;
}
vTaskDelay(pdMS_TO_TICKS(VOICE_MIC_KEY_SESSION_READY_POLL_MS));
}
return false;
}
static void voice_mic_key_handle_press_down(void) {
voice_interaction_status_t voice = {0};
voice_interaction_get_status(&voice);
if (voice.tap_active) {
ESP_LOGI(TAG, "MIC_KEY press ignored: tap already active");
return;
}
printer_runtime_status_t printer = {0};
printer_protocol_get_runtime_status(&printer);
if (voice_mic_key_round_busy(&voice, &printer)) {
ESP_LOGI(TAG,
"MIC_KEY press ignored: state=%s marker_busy=%d print_busy=%d queue=%" PRIu32,
voice_interaction_dialog_state_str(voice.dialog_state),
voice.marker_image_gen_running,
printer.busy,
printer.queue_depth);
return;
}
if (!voice_mic_key_session_ready(&voice)) {
char start_err[128] = {0};
esp_err_t start_rc = voice_interaction_start_with_timeout(0, start_err, sizeof(start_err));
if (start_rc != ESP_OK) {
ESP_LOGW(TAG,
"MIC_KEY auto session start failed: %s (%s)",
esp_err_to_name(start_rc),
start_err[0] != '\0' ? start_err : "-");
return;
}
}
if (!voice_mic_key_wait_session_ready(VOICE_MIC_KEY_SESSION_READY_WAIT_MS)) {
voice_interaction_get_status(&voice);
ESP_LOGW(TAG,
"MIC_KEY wait session ready timeout: active=%d ws=%d started=%d",
voice.session_active,
voice.ws_connected,
voice.started);
return;
}
char tap_err[128] = {0};
esp_err_t tap_rc = voice_interaction_tap_start(tap_err, sizeof(tap_err));
if (tap_rc != ESP_OK) {
ESP_LOGW(TAG,
"MIC_KEY tap start failed: %s (%s)",
esp_err_to_name(tap_rc),
tap_err[0] != '\0' ? tap_err : "-");
return;
}
ESP_LOGI(TAG, "MIC_KEY tap started");
}
static void voice_mic_key_handle_press_up(void) {
voice_interaction_status_t voice = {0};
voice_interaction_get_status(&voice);
if (!voice.tap_active) {
return;
}
char tap_err[128] = {0};
esp_err_t tap_rc = voice_interaction_tap_cancel(tap_err, sizeof(tap_err));
if (tap_rc != ESP_OK) {
ESP_LOGW(TAG,
"MIC_KEY tap stop failed: %s (%s)",
esp_err_to_name(tap_rc),
tap_err[0] != '\0' ? tap_err : "-");
return;
}
ESP_LOGI(TAG, "MIC_KEY tap stopped");
}
static void voice_mic_key_task_main(void *arg) {
(void)arg;
while (true) {
voice_mic_key_cmd_t cmd = VOICE_MIC_KEY_CMD_PRESS_UP;
if (xQueueReceive(s_mic_key_queue, &cmd, portMAX_DELAY) != pdTRUE) {
continue;
}
switch (cmd) {
case VOICE_MIC_KEY_CMD_PRESS_DOWN:
voice_mic_key_handle_press_down();
break;
case VOICE_MIC_KEY_CMD_PRESS_UP:
voice_mic_key_handle_press_up();
break;
default:
break;
}
}
}
static void voice_mic_key_platform_event_cb(platform_mic_key_event_t event, void *user_data) {
(void)user_data;
if (s_mic_key_queue == NULL) {
return;
}
voice_mic_key_cmd_t cmd = VOICE_MIC_KEY_CMD_PRESS_UP;
switch (event) {
case PLATFORM_MIC_KEY_EVENT_PRESS_DOWN:
cmd = VOICE_MIC_KEY_CMD_PRESS_DOWN;
break;
case PLATFORM_MIC_KEY_EVENT_PRESS_UP:
cmd = VOICE_MIC_KEY_CMD_PRESS_UP;
break;
default:
return;
}
BaseType_t queued = xQueueSend(s_mic_key_queue, &cmd, 0);
if (queued == pdTRUE) {
return;
}
voice_mic_key_cmd_t dropped = VOICE_MIC_KEY_CMD_PRESS_UP;
(void)xQueueReceive(s_mic_key_queue, &dropped, 0);
(void)xQueueSend(s_mic_key_queue, &cmd, 0);
}
esp_err_t voice_interaction_mic_key_init(void) {
if (s_mic_key_ready) {
return ESP_OK;
}
esp_err_t err = platform_mic_key_init();
if (err == ESP_ERR_NOT_SUPPORTED) {
ESP_LOGW(TAG, "MIC_KEY trigger disabled by board config");
s_mic_key_ready = true;
return ESP_OK;
}
if (err != ESP_OK) {
return err;
}
s_mic_key_queue = xQueueCreate(VOICE_MIC_KEY_QUEUE_LEN, sizeof(voice_mic_key_cmd_t));
if (s_mic_key_queue == NULL) {
platform_mic_key_deinit();
return ESP_ERR_NO_MEM;
}
BaseType_t ok = voice_create_task_prefer_psram(voice_mic_key_task_main,
"voice_mic_key",
VOICE_MIC_KEY_TASK_STACK,
4,
&s_mic_key_task,
&s_mic_key_task_with_caps);
if (ok != pdPASS) {
vQueueDelete(s_mic_key_queue);
s_mic_key_queue = NULL;
platform_mic_key_deinit();
return ESP_ERR_NO_MEM;
}
err = platform_mic_key_set_event_callback(voice_mic_key_platform_event_cb, NULL);
if (err != ESP_OK) {
voice_delete_task(s_mic_key_task, s_mic_key_task_with_caps);
s_mic_key_task = NULL;
s_mic_key_task_with_caps = false;
vQueueDelete(s_mic_key_queue);
s_mic_key_queue = NULL;
platform_mic_key_deinit();
return err;
}
s_mic_key_ready = true;
ESP_LOGI(TAG, "MIC_KEY trigger ready: press to speak, release to process");
return ESP_OK;
}

5
components/domain/src/voice_interaction_tasks.c
View File

@@ -27,6 +27,7 @@ void voice_uplink_task(void *arg) {
bool connected = false;
bool started = false;
bool tap_active = false;
bool speech_active = false;
bool listening = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
@@ -34,6 +35,7 @@ void voice_uplink_task(void *arg) {
connected = s_voice.ws_connected;
started = s_voice.started;
tap_active = s_voice.tap_active;
speech_active = s_voice.speech_active;
listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING);
voice_unlock();
}
@@ -42,7 +44,7 @@ void voice_uplink_task(void *arg) {
break;
}
if (!(connected && started && listening && tap_active)) {
if (!(connected && started && listening && tap_active && speech_active)) {
vTaskDelay(pdMS_TO_TICKS(20));
continue;
}
@@ -83,6 +85,7 @@ void voice_uplink_task(void *arg) {
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGW(TAG, "[stage] uplink_detected_ws_disconnected");

105
components/domain/src/voice_interaction_ws.c
View File

@@ -88,7 +88,7 @@ static char *voice_build_start_message(void) {
cJSON *parameters = cJSON_AddObjectToObject(payload, "parameters");
cJSON *upstream = cJSON_AddObjectToObject(parameters, "upstream");
cJSON_AddStringToObject(upstream, "type", "AudioOnly");
cJSON_AddStringToObject(upstream, "mode", "tap2talk");
cJSON_AddStringToObject(upstream, "mode", "push2talk");
cJSON_AddStringToObject(upstream, "audio_format", "raw-opus");
cJSON_AddNumberToObject(upstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE);
@@ -113,7 +113,7 @@ static char *voice_build_start_message(void) {
return text;
}
static esp_err_t voice_send_text(const char *text, int len) {
static esp_err_t voice_send_text_immediate(const char *text, int len) {
if (text == NULL || len <= 0) {
return ESP_ERR_INVALID_ARG;
}
@@ -132,6 +132,7 @@ static esp_err_t voice_send_text(const char *text, int len) {
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGW(TAG, "[stage] ws_state_reconciled: transport disconnected");
@@ -156,6 +157,43 @@ static esp_err_t voice_send_text(const char *text, int len) {
return ESP_OK;
}
static esp_err_t voice_send_text(const char *text, int len) {
if (text == NULL || len <= 0) {
return ESP_ERR_INVALID_ARG;
}
QueueHandle_t queue = NULL;
TaskHandle_t ws_text_task = NULL;
bool session_active = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
queue = s_voice.ws_text_queue;
ws_text_task = s_voice.ws_text_task;
session_active = s_voice.session_active;
voice_unlock();
}
if (queue == NULL || ws_text_task == NULL || !session_active) {
return voice_send_text_immediate(text, len);
}
char *copy = (char *)voice_malloc_prefer_psram((size_t)len + 1);
if (copy == NULL) {
return ESP_ERR_NO_MEM;
}
memcpy(copy, text, (size_t)len);
copy[len] = '\0';
voice_ws_text_msg_t msg = {
.text = copy,
.len = len,
};
if (xQueueSend(queue, &msg, pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS)) != pdTRUE) {
free(copy);
return ESP_ERR_TIMEOUT;
}
return ESP_OK;
}
esp_err_t voice_send_directive(const char *action,
const char *directive,
bool with_dialog_id) {
@@ -181,6 +219,68 @@ esp_err_t voice_send_directive(const char *action,
return err;
}
void voice_ws_text_task(void *arg) {
(void)arg;
while (true) {
bool shutdown = false;
QueueHandle_t queue = NULL;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
shutdown = s_voice.ws_text_shutdown;
queue = s_voice.ws_text_queue;
voice_unlock();
}
if (shutdown || queue == NULL) {
break;
}
voice_ws_text_msg_t msg = {0};
if (xQueueReceive(queue, &msg, pdMS_TO_TICKS(VOICE_WS_TEXT_QUEUE_RX_TIMEOUT_MS)) != pdTRUE) {
continue;
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_text_busy = true;
voice_unlock();
}
esp_err_t err = voice_send_text_immediate(msg.text, msg.len);
if (err != ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_set_last_error_locked("ws text send failed");
voice_unlock();
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_text_busy = false;
voice_unlock();
}
free(msg.text);
}
QueueHandle_t queue = NULL;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
queue = s_voice.ws_text_queue;
s_voice.ws_text_busy = false;
voice_unlock();
}
if (queue != NULL) {
voice_ws_text_msg_t msg = {0};
while (xQueueReceive(queue, &msg, 0) == pdTRUE) {
free(msg.text);
}
}
bool delete_with_caps = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
delete_with_caps = s_voice.ws_text_task_with_caps;
s_voice.ws_text_task = NULL;
s_voice.ws_text_task_with_caps = false;
voice_unlock();
}
voice_delete_self_task(delete_with_caps);
}
static void voice_handle_text_chunk(esp_websocket_event_data_t *data) {
if (data->payload_offset == 0) {
free(s_voice.text_agg);
@@ -310,6 +410,7 @@ void voice_websocket_event_handler(void *handler_args,
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
voice_unlock();

28
components/domain/src/voice_interaction_ws_audio.c
View File

@@ -71,16 +71,24 @@ void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len) {
VOICE_AUDIO_IO_TIMEOUT_MS,
audio_err,
sizeof(audio_err));
if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t pcm_ms = ((int64_t)samples * 1000 +
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) /
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE;
int64_t base_ms = s_voice.playback_deadline_ms > now_ms ?
s_voice.playback_deadline_ms : now_ms;
s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS;
s_voice.last_downstream_ms = now_ms;
voice_unlock();
if (write_err == ESP_OK) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t pcm_ms = ((int64_t)samples * 1000 +
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) /
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE;
int64_t base_ms = s_voice.playback_deadline_ms > now_ms ?
s_voice.playback_deadline_ms : now_ms;
s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS;
s_voice.last_downstream_ms = now_ms;
voice_unlock();
}
} else {
ESP_LOGW(TAG,
"[stage] audio_downstream_write_failed: rc=0x%x err=%s samples=%u",
(unsigned)write_err,
audio_err[0] != '\0' ? audio_err : "-",
(unsigned)samples);
}
}

153
components/domain/src/voice_interaction_ws_protocol.c
View File

@@ -11,6 +11,111 @@
static const char *TAG = "voice_interaction";
#define VOICE_RESPONSE_FINALIZE_TASK_STACK 6144
#define VOICE_RESPONSE_FINALIZE_TASK_RETRY_COUNT 3
#define VOICE_RESPONSE_FINALIZE_TASK_RETRY_DELAY_MS 120
static portMUX_TYPE s_response_finalize_lock = portMUX_INITIALIZER_UNLOCKED;
static bool s_response_finalize_running;
static esp_err_t voice_send_directive_with_retry(const char *action,
const char *directive,
bool with_dialog_id) {
esp_err_t rc = ESP_FAIL;
for (uint32_t i = 0; i < VOICE_RESPONSE_FINALIZE_TASK_RETRY_COUNT; ++i) {
rc = voice_send_directive(action, directive, with_dialog_id);
if (rc == ESP_OK || rc == ESP_ERR_INVALID_STATE) {
return rc;
}
vTaskDelay(pdMS_TO_TICKS(VOICE_RESPONSE_FINALIZE_TASK_RETRY_DELAY_MS));
}
return rc;
}
static uint32_t voice_calc_response_finalize_wait_ms(void) {
uint32_t playback_wait_ms = VOICE_AUDIO_DRAIN_WAIT_MS;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
return playback_wait_ms;
}
int64_t now_ms = voice_now_ms();
int64_t deadline_ms = s_voice.playback_deadline_ms;
voice_unlock();
if (deadline_ms <= now_ms) {
return playback_wait_ms;
}
int64_t remain_ms = (deadline_ms - now_ms) + VOICE_AUDIO_DRAIN_QUIET_MS + 120;
if (remain_ms <= (int64_t)playback_wait_ms) {
return playback_wait_ms;
}
return (remain_ms > 12000) ? 12000 : (uint32_t)remain_ms;
}
static void voice_response_finalize_task(void *arg) {
(void)arg;
uint32_t playback_wait_ms = voice_calc_response_finalize_wait_ms();
ESP_LOGI(TAG,
"[stage] responding_ended_local_playback_wait: timeout_ms=%u",
(unsigned)playback_wait_ms);
voice_wait_audio_playback_done(playback_wait_ms, "responding_ended_local_playback");
esp_err_t local_end_rc =
voice_send_directive_with_retry("continue-task", "LocalRespondingEnded", true);
if (local_end_rc != ESP_OK) {
ESP_LOGW(TAG,
"[stage] responding_ended_local_end_failed: rc=0x%x",
(unsigned)local_end_rc);
}
esp_err_t stop_rc =
voice_send_directive_with_retry("finish-task", "Stop", true);
if (stop_rc != ESP_OK) {
ESP_LOGW(TAG,
"[stage] responding_ended_stop_failed: rc=0x%x",
(unsigned)stop_rc);
}
portENTER_CRITICAL(&s_response_finalize_lock);
s_response_finalize_running = false;
portEXIT_CRITICAL(&s_response_finalize_lock);
vTaskDelete(NULL);
}
static void voice_schedule_response_finalize(void) {
bool should_start = false;
portENTER_CRITICAL(&s_response_finalize_lock);
if (!s_response_finalize_running) {
s_response_finalize_running = true;
should_start = true;
}
portEXIT_CRITICAL(&s_response_finalize_lock);
if (!should_start) {
ESP_LOGI(TAG, "[stage] responding_ended_finalize_skip: task already running");
return;
}
BaseType_t ok = xTaskCreate(voice_response_finalize_task,
"voice_rsp_fin",
VOICE_RESPONSE_FINALIZE_TASK_STACK,
NULL,
4,
NULL);
if (ok != pdPASS) {
portENTER_CRITICAL(&s_response_finalize_lock);
s_response_finalize_running = false;
portEXIT_CRITICAL(&s_response_finalize_lock);
ESP_LOGW(TAG, "[stage] responding_ended_finalize_start_failed");
} else {
ESP_LOGI(TAG, "[stage] responding_ended_finalize_task_started");
}
}
static const char *voice_get_json_str(cJSON *obj, const char *name) {
if (obj == NULL || name == NULL) {
return NULL;
@@ -73,6 +178,7 @@ static void voice_handle_output_event(cJSON *output) {
bool has_finished = voice_get_json_bool(output, "finished", &finished);
const char *responding_final_text = NULL;
bool should_check_marker_in_first_sentence = false;
bool should_try_send_speech = false;
if (has_finished && finished) {
if (strcmp(event_name, "SpeechContent") == 0) {
voice_log_final_text("asr_final_text", voice_get_json_str(output, "text"));
@@ -114,13 +220,16 @@ static void voice_handle_output_event(cJSON *output) {
if (prev_state != VOICE_DIALOG_STATE_LISTENING) {
s_voice.response_first_sentence_checked = false;
}
should_try_send_speech = s_voice.tap_active && !s_voice.speech_active;
} else if (strcmp(state, "Thinking") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING;
s_voice.speech_active = false;
if (prev_state != VOICE_DIALOG_STATE_THINKING) {
s_voice.response_first_sentence_checked = false;
}
} else if (strcmp(state, "Responding") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING;
s_voice.speech_active = false;
if (prev_state != VOICE_DIALOG_STATE_RESPONDING) {
s_voice.response_first_sentence_checked = false;
}
@@ -131,14 +240,14 @@ static void voice_handle_output_event(cJSON *output) {
state != NULL ? state : "-",
voice_interaction_dialog_state_str(s_voice.dialog_state));
} else if (strcmp(event_name, "SpeechEnded") == 0) {
s_voice.tap_active = false;
ESP_LOGI(TAG, "[stage] speech_ended: mark single_round_input_done");
s_voice.speech_active = false;
ESP_LOGI(TAG, "[stage] speech_ended: stop uplink");
} else if (strcmp(event_name, "RespondingEnded") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
ESP_LOGI(TAG, "[stage] responding_ended: wait_local_playback_before_ack");
ESP_LOGI(TAG, "[stage] responding_ended: schedule_local_playback_finalize");
} else if (strcmp(event_name, "Stopped") == 0) {
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.response_first_sentence_checked = false;
should_close_audio = true;
@@ -151,6 +260,7 @@ static void voice_handle_output_event(cJSON *output) {
}
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.response_first_sentence_checked = false;
should_close_audio = true;
}
@@ -165,31 +275,21 @@ static void voice_handle_output_event(cJSON *output) {
voice_marker_try_trigger_image_generation(dialog_id, responding_final_text);
}
if (should_try_send_speech) {
char speech_err[96] = {0};
esp_err_t rc = voice_interaction_try_send_speech(speech_err, sizeof(speech_err));
if (rc != ESP_OK && rc != ESP_ERR_INVALID_STATE) {
ESP_LOGW(TAG,
"[stage] auto_send_speech_failed: rc=0x%x err=%s",
(unsigned)rc,
speech_err[0] != '\0' ? speech_err : "-");
}
}
if (strcmp(event_name, "RespondingStarted") == 0) {
(void)voice_send_directive("continue-task", "LocalRespondingStarted", true);
} else if (strcmp(event_name, "RespondingEnded") == 0) {
uint32_t playback_wait_ms = VOICE_AUDIO_DRAIN_WAIT_MS;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t deadline_ms = s_voice.playback_deadline_ms;
voice_unlock();
if (deadline_ms > now_ms) {
int64_t remain_ms = (deadline_ms - now_ms) + VOICE_AUDIO_DRAIN_QUIET_MS + 120;
if (remain_ms > (int64_t)playback_wait_ms) {
if (remain_ms > 12000) {
playback_wait_ms = 12000;
} else {
playback_wait_ms = (uint32_t)remain_ms;
}
}
}
}
ESP_LOGI(TAG, "[stage] responding_ended_local_playback_wait: timeout_ms=%u", (unsigned)playback_wait_ms);
voice_wait_audio_playback_done(playback_wait_ms,
"responding_ended_local_playback");
(void)voice_send_directive("continue-task", "LocalRespondingEnded", true);
(void)voice_send_directive("finish-task", "Stop", true);
voice_schedule_response_finalize();
}
}
@@ -243,6 +343,7 @@ void voice_ws_protocol_handle_text_message(const char *text, size_t len) {
voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed");
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
voice_unlock();
}