Bairan.Zhang/AI_Printer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(voice): migrate to push2talk and serialize websocket text sends

Browse Source
This commit is contained in:
admin
2026-03-01 01:21:18 +08:00
parent 9deeab738e
commit dd108e2658
19 changed files with 891 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
components/control_plane/src/rest_server.c
View File

@@ -21,29 +21,22 @@ static const char *s_web_index =
" .box { max-width: 760px; margin: 0 auto; background:#fff; border:1px solid #dfe6ee; border-radius:12px; padding:16px; }\n"
" h1 { margin: 0 0 8px 0; font-size: 22px; }\n"
" p { margin: 6px 0 12px 0; color:#4b5d73; }\n"
" .row { display:flex; gap:8px; flex-wrap:wrap; margin: 8px 0; }\n"
" button { border:0; border-radius:8px; padding:10px 14px; cursor:pointer; background:#198754; color:#fff; font: inherit; min-width:220px; }\n"
" button.active { background:#d68910; }\n"
" button:disabled { background:#5f6f82; cursor:default; }\n"
" .hint { font-size: 13px; color:#29583f; background:#e7f4ec; border:1px solid #cbe4d5; border-radius:8px; padding:10px; margin: 8px 0 12px 0; }\n"
" .small { font-size: 13px; color:#44566c; }\n"
" </style>\n"
"</head>\n"
"<body>\n"
" <div class=\"box\">\n"
" <h1>TQ Voice Console</h1>\n"
" <p>ESP32-S3 本地语音控制页。用于单轮语音对话（Tap2Talk）。</p>\n"
" <p>ESP32-S3 本地语音控制页。已切换为 MIC_KEY（IO14）触发单轮语音对话。</p>\n"
"\n"
" <div class=\"row\">\n"
" <button id=\"btnVoiceTalk\">点击开始单轮对话</button>\n"
" </div>\n"
" <div class=\"hint\">按住机身 MIC_KEY 说话，松开后自动进入“语音交互→生图→显示→处理→打印”。</div>\n"
" <div id=\"voiceState\" class=\"small\">语音状态：初始化中</div>\n"
" </div>\n"
"\n"
" <script>\n"
" const btnVoiceTalkEl = document.getElementById(\"btnVoiceTalk\");\n"
" const voiceStateEl = document.getElementById(\"voiceState\");\n"
"\n"
" let voiceTalkBusy = false;\n"
" let voiceStatusSyncing = false;\n"
"\n"
" function headers(withJson) {\n"
@@ -52,10 +45,6 @@ static const char *s_web_index =
" return h;\n"
" }\n"
"\n"
" function sleep(ms) {\n"
" return new Promise((resolve) => setTimeout(resolve, ms));\n"
" }\n"
"\n"
" async function callApi(path, method, body) {\n"
" const init = { method, headers: headers(body !== undefined) };\n"
" if (body !== undefined) init.body = JSON.stringify(body);\n"
@@ -69,27 +58,17 @@ static const char *s_web_index =
" return data;\n"
" }\n"
"\n"
" function voiceRoundBusy(st) {\n"
" if (!st) return false;\n"
" const state = (st.voice_dialog_state || \"\").toLowerCase();\n"
" const voiceBusy = state === \"thinking\" || state === \"responding\";\n"
" const markerBusy = !!st.voice_marker_image_gen_running;\n"
" const queueDepth = Number(st.queue_depth || 0);\n"
" const printBusy = !!st.printer_busy || queueDepth > 0;\n"
" return voiceBusy || markerBusy || printBusy;\n"
" }\n"
"\n"
" function voiceStatusText(st) {\n"
" if (!st) return \"语音状态：未知\";\n"
" const state = (st.voice_dialog_state || \"\").toLowerCase();\n"
" const queueDepth = Number(st.queue_depth || 0);\n"
" if (!st.voice_session_active) return \"语音状态：会话未启动\";\n"
" if (!st.voice_ws_connected) return \"语音状态：会话断开，点击开始将自动重建\";\n"
" if (!st.voice_ws_connected) return \"语音状态：会话断开，按 MIC_KEY 将自动重建\";\n"
" if (st.voice_tap_active) return \"语音状态：录音中\";\n"
" if (state === \"thinking\" || state === \"responding\") return \"语音状态：语音处理中\";\n"
" if (!!st.voice_marker_image_gen_running) return \"语音状态：图片生成中\";\n"
" if (!!st.printer_busy || queueDepth > 0) return \"语音状态：打印处理中\";\n"
" return \"语音状态：空闲，可开始下一轮\";\n"
" return \"语音状态：空闲，等待 MIC_KEY 触发\";\n"
" }\n"
"\n"
" function voiceErrorText(st) {\n"
@@ -99,138 +78,21 @@ static const char *s_web_index =
" return \"最近错误：\" + msg;\n"
" }\n"
"\n"
" function voiceRoundBusyText(st) {\n"
" if (!st) return \"处理中，请等待\";\n"
" const state = (st.voice_dialog_state || \"\").toLowerCase();\n"
" if (state === \"thinking\" || state === \"responding\") return \"语音处理中，请等待\";\n"
" if (!!st.voice_marker_image_gen_running) return \"图片生成中，请等待\";\n"
" const queueDepth = Number(st.queue_depth || 0);\n"
" if (!!st.printer_busy || queueDepth > 0) return \"打印处理中，请等待\";\n"
" return \"处理中，请等待\";\n"
" }\n"
"\n"
" function setVoiceTalkButtonState(active) {\n"
" if (active) {\n"
" btnVoiceTalkEl.classList.add(\"active\");\n"
" btnVoiceTalkEl.textContent = \"点击取消本轮录音\";\n"
" } else {\n"
" btnVoiceTalkEl.classList.remove(\"active\");\n"
" btnVoiceTalkEl.textContent = \"点击开始单轮对话\";\n"
" }\n"
" }\n"
"\n"
" function syncVoiceTalkUi(st) {\n"
" const active = !!(st && st.voice_tap_active && st.voice_ws_connected && st.voice_started);\n"
" const busy = voiceRoundBusy(st);\n"
" let status = voiceStatusText(st);\n"
" const err = voiceErrorText(st);\n"
" if (err) status += \"；\" + err;\n"
" if (voiceStateEl) voiceStateEl.textContent = status;\n"
"\n"
" if (busy) {\n"
" btnVoiceTalkEl.classList.remove(\"active\");\n"
" btnVoiceTalkEl.disabled = true;\n"
" btnVoiceTalkEl.textContent = voiceRoundBusyText(st);\n"
" return;\n"
" }\n"
" btnVoiceTalkEl.disabled = false;\n"
" setVoiceTalkButtonState(active);\n"
" }\n"
"\n"
" function voiceReady(st) {\n"
" return !!(st && st.voice_session_active && st.voice_ws_connected && st.voice_started);\n"
" }\n"
"\n"
" async function ensureVoiceSessionReady() {\n"
" let st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" if (voiceReady(st)) return st;\n"
"\n"
" await callApi(\"/v1/voice/session/start\", \"POST\", {});\n"
" const deadline = Date.now() + 10000;\n"
" while (Date.now() < deadline) {\n"
" await sleep(250);\n"
" st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" if (voiceReady(st)) return st;\n"
" }\n"
" throw new Error(\"voice session is not ready, please retry\");\n"
" }\n"
"\n"
" async function refreshVoiceStatus() {\n"
" if (voiceStatusSyncing) return null;\n"
" voiceStatusSyncing = true;\n"
" try {\n"
" const st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" let status = voiceStatusText(st);\n"
" const err = voiceErrorText(st);\n"
" if (err) status += \"；\" + err;\n"
" if (voiceStateEl) voiceStateEl.textContent = status;\n"
" return st;\n"
" } finally {\n"
" voiceStatusSyncing = false;\n"
" }\n"
" }\n"
"\n"
" async function beginVoiceTalk() {\n"
" if (voiceTalkBusy) {\n"
" const st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" return st;\n"
" }\n"
" voiceTalkBusy = true;\n"
" try {\n"
" let st = await ensureVoiceSessionReady();\n"
" if (st && st.voice_tap_active) {\n"
" syncVoiceTalkUi(st);\n"
" return st;\n"
" }\n"
" await callApi(\"/v1/voice/tap/start\", \"POST\", {});\n"
" st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" return st;\n"
" } finally {\n"
" voiceTalkBusy = false;\n"
" }\n"
" }\n"
"\n"
" async function endVoiceTalk() {\n"
" if (voiceTalkBusy) {\n"
" const st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" return st;\n"
" }\n"
" voiceTalkBusy = true;\n"
" try {\n"
" let st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" if (!(st && st.voice_tap_active)) return st;\n"
" await callApi(\"/v1/voice/tap/cancel\", \"POST\", {});\n"
" st = await callApi(\"/v1/voice/status\", \"GET\");\n"
" syncVoiceTalkUi(st);\n"
" return st;\n"
" } finally {\n"
" voiceTalkBusy = false;\n"
" }\n"
" }\n"
"\n"
" btnVoiceTalkEl.addEventListener(\"click\", async () => {\n"
" try {\n"
" const st = await refreshVoiceStatus();\n"
" if (!st) return;\n"
" if (voiceRoundBusy(st)) return;\n"
" if (st && st.voice_tap_active) {\n"
" await endVoiceTalk();\n"
" } else {\n"
" await beginVoiceTalk();\n"
" }\n"
" } catch (e) {\n"
" let msg = \"语音状态：请求失败\";\n"
" if (e && e.data && typeof e.data === \"object\" && e.data.error) {\n"
" msg += \"；\" + String(e.data.error);\n"
" }\n"
" voiceStateEl.textContent = msg;\n"
" }\n"
" });\n"
"\n"
" setVoiceTalkButtonState(false);\n"
" (async () => {\n"
" try { await refreshVoiceStatus(); } catch (_) {}\n"
" })();\n"

2
components/control_plane/src/rest_server_voice.c
View File

@@ -204,7 +204,7 @@ esp_err_t rest_server_voice_tap_cancel_post(httpd_req_t *req) {
cJSON *root = cJSON_CreateObject();
cJSON_AddBoolToObject(root, "ok", true);
cJSON_AddStringToObject(root, "message", "tap canceled");
cJSON_AddStringToObject(root, "message", "tap stopped");
rest_server_add_voice_status(root);
esp_err_t err = rest_server_send_json(req, "200 OK", root);

1
components/domain/CMakeLists.txt
View File

@@ -17,6 +17,7 @@ idf_component_register(
"src/voice_interaction_ws_audio.c"
"src/voice_interaction_marker.c"
"src/voice_interaction_tasks.c"
"src/voice_interaction_mic_key.c"
"third_party/qrcodegen.c"
INCLUDE_DIRS
"include"

20
components/domain/internal/voice_interaction_internal.h
View File

@@ -8,6 +8,7 @@
#include "esp_opus_enc.h"
#include "esp_websocket_client.h"
#include "freertos/FreeRTOS.h"
#include "freertos/queue.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#include "domain.h"
@@ -21,7 +22,7 @@
#define VOICE_SAMPLE_BYTES 2
#define VOICE_STATUS_LOCK_TIMEOUT_MS 1000
#define VOICE_WS_SEND_TIMEOUT_MS 1000
#define VOICE_WS_SEND_TIMEOUT_MS 3000
#define VOICE_WS_TASK_STACK 24576
#define VOICE_UPLINK_TASK_STACK 32768
#define VOICE_HEARTBEAT_TASK_STACK 4096
@@ -31,15 +32,25 @@
#define VOICE_AUDIO_DRAIN_QUIET_MS 180
#define VOICE_AUDIO_DRAIN_WAIT_MS 2200
#define VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS 3500
#define VOICE_WS_TEXT_QUEUE_LEN 16
#define VOICE_WS_TEXT_QUEUE_RX_TIMEOUT_MS 100
typedef struct {
char *text;
int len;
} voice_ws_text_msg_t;
typedef struct {
SemaphoreHandle_t lock;
esp_websocket_client_handle_t ws;
QueueHandle_t ws_text_queue;
TaskHandle_t uplink_task;
TaskHandle_t heartbeat_task;
TaskHandle_t ws_text_task;
bool uplink_task_with_caps;
bool heartbeat_task_with_caps;
bool ws_text_task_with_caps;
void *opus_enc;
void *opus_dec;
@@ -63,7 +74,10 @@ typedef struct {
bool ws_connected;
bool started;
bool tap_active;
bool speech_active;
bool stop_requested;
bool ws_text_shutdown;
bool ws_text_busy;
bool ws_low_stack_warned;
bool response_first_sentence_checked;
@@ -129,6 +143,10 @@ void voice_websocket_event_handler(void *handler_args,
void *event_data);
void voice_interaction_ws_init_cjson_hooks(void);
esp_err_t voice_interaction_try_send_speech(char *err, size_t err_len);
void voice_uplink_task(void *arg);
void voice_heartbeat_task(void *arg);
void voice_ws_text_task(void *arg);
esp_err_t voice_interaction_mic_key_init(void);

217
components/domain/src/voice_interaction.c
View File

@@ -20,18 +20,44 @@ static uint32_t clamp_timeout_ms(uint32_t timeout_ms, uint32_t default_ms, uint3
return value;
}
static bool voice_wait_ws_text_idle(uint32_t timeout_ms) {
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
bool busy = false;
QueueHandle_t queue = NULL;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
busy = s_voice.ws_text_busy;
queue = s_voice.ws_text_queue;
voice_unlock();
}
UBaseType_t pending = 0;
if (queue != NULL) {
pending = uxQueueMessagesWaiting(queue);
}
if (!busy && pending == 0) {
return true;
}
vTaskDelay(pdMS_TO_TICKS(20));
}
return false;
}
static bool voice_wait_worker_tasks_exit(uint32_t timeout_ms) {
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
bool uplink_alive = false;
bool heartbeat_alive = false;
bool ws_text_alive = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
uplink_alive = (s_voice.uplink_task != NULL);
heartbeat_alive = (s_voice.heartbeat_task != NULL);
ws_text_alive = (s_voice.ws_text_task != NULL);
voice_unlock();
}
if (!uplink_alive && !heartbeat_alive) {
if (!uplink_alive && !heartbeat_alive && !ws_text_alive) {
return true;
}
vTaskDelay(pdMS_TO_TICKS(20));
@@ -39,9 +65,10 @@ static bool voice_wait_worker_tasks_exit(uint32_t timeout_ms) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
ESP_LOGW(TAG,
"[stage] worker_task_exit_timeout: uplink_alive=%d heartbeat_alive=%d",
"[stage] worker_task_exit_timeout: uplink_alive=%d heartbeat_alive=%d ws_text_alive=%d",
s_voice.uplink_task != NULL,
s_voice.heartbeat_task != NULL);
s_voice.heartbeat_task != NULL,
s_voice.ws_text_task != NULL);
voice_unlock();
}
return false;
@@ -76,6 +103,14 @@ esp_err_t voice_interaction_init(void) {
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
esp_err_t mic_key_err = voice_interaction_mic_key_init();
if (mic_key_err != ESP_OK) {
vSemaphoreDelete(s_voice.lock);
memset(&s_voice, 0, sizeof(s_voice));
return mic_key_err;
}
s_voice.ready = true;
return ESP_OK;
}
@@ -152,10 +187,14 @@ esp_err_t voice_interaction_start_with_timeout(uint32_t timeout_ms, char *err, s
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.stop_requested = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.ws_low_stack_warned = false;
s_voice.response_first_sentence_checked = false;
s_voice.uplink_task_with_caps = false;
s_voice.heartbeat_task_with_caps = false;
s_voice.ws_text_task_with_caps = false;
s_voice.ws_text_shutdown = false;
s_voice.ws_text_busy = false;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGI(TAG,
"[stage] session_context_prepared: task_id=%s device_uuid=%s",
@@ -172,9 +211,12 @@ esp_err_t voice_interaction_start_with_timeout(uint32_t timeout_ms, char *err, s
.buffer_size = 2048,
.network_timeout_ms = 10000,
.reconnect_timeout_ms = 3000,
.ping_interval_sec = 20,
.ping_interval_sec = 55,
.disable_auto_reconnect = false,
.pingpong_timeout_sec = 10,
// DashScope protocol already requires application-level HeartBeat keepalive.
// Some networks may drop/ignore ws-level PONG intermittently, causing false
// disconnects if we enforce pingpong timeout aggressively.
.disable_pingpong_discon = true,
#if CONFIG_MBEDTLS_CERTIFICATE_BUNDLE
.crt_bundle_attach = esp_crt_bundle_attach,
#endif
@@ -236,17 +278,53 @@ esp_err_t voice_interaction_start_with_timeout(uint32_t timeout_ms, char *err, s
s_voice.session_active = true;
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.speech_active = false;
s_voice.ws_text_shutdown = false;
s_voice.ws_text_busy = false;
voice_unlock();
voice_log_status_snapshot("session_marked_active");
voice_log_heap_snapshot("before_create_uplink");
ESP_LOGI(TAG, "[stage] uplink_task_create_begin");
BaseType_t ok = voice_create_task_prefer_psram(voice_uplink_task,
"voice_uplink",
VOICE_UPLINK_TASK_STACK,
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_fill_err(err, err_len, "voice lock timeout");
ESP_LOGE(TAG, "[stage] session_start_abort: lock timeout before ws_text_task create");
(void)voice_interaction_stop(NULL, 0);
return ESP_ERR_TIMEOUT;
}
if (s_voice.ws_text_queue == NULL) {
s_voice.ws_text_queue = xQueueCreate(VOICE_WS_TEXT_QUEUE_LEN, sizeof(voice_ws_text_msg_t));
}
QueueHandle_t ws_text_queue = s_voice.ws_text_queue;
voice_unlock();
if (ws_text_queue == NULL) {
voice_fill_err(err, err_len, "create ws text queue failed");
ESP_LOGE(TAG, "[stage] ws_text_queue_create_failed");
(void)voice_interaction_stop(NULL, 0);
return ESP_ERR_NO_MEM;
}
ESP_LOGI(TAG, "[stage] ws_text_task_create_begin");
BaseType_t ok = voice_create_task_prefer_psram(voice_ws_text_task,
"voice_ws_text",
VOICE_HEARTBEAT_TASK_STACK,
5,
&s_voice.uplink_task,
&s_voice.uplink_task_with_caps);
&s_voice.ws_text_task,
&s_voice.ws_text_task_with_caps);
if (ok != pdPASS) {
voice_fill_err(err, err_len, "create ws text task failed");
ESP_LOGE(TAG, "[stage] ws_text_task_create_failed");
(void)voice_interaction_stop(NULL, 0);
return ESP_ERR_NO_MEM;
}
ESP_LOGI(TAG, "[stage] ws_text_task_create_ok");
ESP_LOGI(TAG, "[stage] uplink_task_create_begin");
ok = voice_create_task_prefer_psram(voice_uplink_task,
"voice_uplink",
VOICE_UPLINK_TASK_STACK,
5,
&s_voice.uplink_task,
&s_voice.uplink_task_with_caps);
if (ok != pdPASS) {
voice_fill_err(err, err_len, "create uplink task failed");
ESP_LOGE(TAG, "[stage] uplink_task_create_failed");
@@ -346,22 +424,35 @@ esp_err_t voice_interaction_stop_with_timeout(uint32_t timeout_ms, char *err, si
bool ws_connected = s_voice.ws_connected;
TaskHandle_t uplink_task = s_voice.uplink_task;
TaskHandle_t heartbeat_task = s_voice.heartbeat_task;
TaskHandle_t ws_text_task = s_voice.ws_text_task;
QueueHandle_t ws_text_queue = s_voice.ws_text_queue;
esp_websocket_client_handle_t ws = s_voice.ws;
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
voice_log_status_snapshot("session_stop_requested");
if (ws_connected) {
(void)voice_send_directive("finish-task", "Stop", true);
if (!voice_wait_ws_text_idle(VOICE_WS_SEND_TIMEOUT_MS + 1200)) {
ESP_LOGW(TAG, "[stage] ws_text_idle_timeout_before_shutdown");
}
}
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS, "session_stop_close");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_text_shutdown = true;
voice_unlock();
}
if (uplink_task != NULL) {
(void)xTaskAbortDelay(uplink_task);
}
if (heartbeat_task != NULL) {
(void)xTaskAbortDelay(heartbeat_task);
}
if (ws_text_task != NULL) {
(void)xTaskAbortDelay(ws_text_task);
}
bool workers_exited = voice_wait_worker_tasks_exit(worker_exit_timeout_ms);
if (!workers_exited) {
voice_fill_err(err, err_len, "voice worker exit timeout");
@@ -373,6 +464,11 @@ esp_err_t voice_interaction_stop_with_timeout(uint32_t timeout_ms, char *err, si
return ESP_ERR_TIMEOUT;
}
if (ws_text_queue != NULL) {
vQueueDelete(ws_text_queue);
ws_text_queue = NULL;
}
if (ws != NULL) {
ESP_LOGI(TAG, "[stage] ws_stop_destroy_begin");
esp_websocket_unregister_events(ws,
@@ -385,14 +481,20 @@ esp_err_t voice_interaction_stop_with_timeout(uint32_t timeout_ms, char *err, si
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws = NULL;
s_voice.ws_text_queue = NULL;
s_voice.uplink_task = NULL;
s_voice.heartbeat_task = NULL;
s_voice.ws_text_task = NULL;
s_voice.uplink_task_with_caps = false;
s_voice.heartbeat_task_with_caps = false;
s_voice.ws_text_task_with_caps = false;
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.session_active = false;
s_voice.stop_requested = false;
s_voice.speech_active = false;
s_voice.ws_text_shutdown = false;
s_voice.ws_text_busy = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.dialog_id[0] = '\0';
s_voice.response_first_sentence_checked = false;
@@ -412,6 +514,60 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) {
return voice_interaction_stop_with_timeout(1500, err, err_len);
}
esp_err_t voice_interaction_try_send_speech(char *err, size_t err_len) {
bool should_send = false;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_fill_err(err, err_len, "voice lock timeout");
return ESP_ERR_TIMEOUT;
}
if (!s_voice.session_active || !s_voice.ws_connected || !s_voice.started) {
voice_unlock();
voice_fill_err(err, err_len, "voice session not ready");
return ESP_ERR_INVALID_STATE;
}
should_send = s_voice.tap_active &&
!s_voice.speech_active &&
s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING;
voice_unlock();
if (!should_send) {
return ESP_OK;
}
esp_err_t send_err = voice_send_directive("continue-task", "SendSpeech", true);
if (send_err != ESP_OK) {
voice_fill_err(err, err_len, "send speech failed");
ESP_LOGW(TAG, "[stage] send_speech_failed: rc=0x%x", (unsigned)send_err);
return send_err;
}
bool should_send_stop = false;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
(void)voice_send_directive("continue-task", "StopSpeech", true);
voice_fill_err(err, err_len, "voice lock timeout");
return ESP_ERR_TIMEOUT;
}
if (s_voice.session_active &&
s_voice.ws_connected &&
s_voice.started &&
s_voice.tap_active &&
s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING) {
s_voice.speech_active = true;
s_voice.last_event_ms = voice_now_ms();
} else if (s_voice.session_active && s_voice.ws_connected && s_voice.started) {
should_send_stop = true;
}
voice_unlock();
if (should_send_stop) {
(void)voice_send_directive("continue-task", "StopSpeech", true);
}
return ESP_OK;
}
esp_err_t voice_interaction_tap_start(char *err, size_t err_len) {
ESP_LOGI(TAG, "[stage] tap_start_enter");
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
@@ -461,11 +617,33 @@ esp_err_t voice_interaction_tap_start(char *err, size_t err_len) {
bool listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING);
s_voice.tap_active = true;
s_voice.speech_active = false;
voice_unlock();
ESP_LOGI(TAG, "[stage] tap_start_marked_active: listening=%d", listening);
if (!listening) {
(void)voice_send_directive("continue-task", "RequestToSpeak", true);
esp_err_t req_rc = voice_send_directive("continue-task", "RequestToSpeak", true);
if (req_rc != ESP_OK) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
}
voice_fill_err(err, err_len, "request to speak failed");
ESP_LOGW(TAG, "[stage] tap_start_abort: request_to_speak failed rc=0x%x", (unsigned)req_rc);
return req_rc;
}
} else {
esp_err_t send_speech_err = voice_interaction_try_send_speech(err, err_len);
if (send_speech_err != ESP_OK) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
}
voice_audio_close();
return send_speech_err;
}
}
voice_log_status_snapshot("tap_start_exit_ok");
@@ -491,12 +669,21 @@ esp_err_t voice_interaction_tap_cancel(char *err, size_t err_len) {
return ESP_ERR_INVALID_STATE;
}
bool should_send_stop = s_voice.speech_active;
s_voice.tap_active = false;
s_voice.speech_active = false;
voice_unlock();
voice_log_status_snapshot("tap_cancel_marked_inactive");
esp_err_t cancel_err = voice_send_directive("continue-task", "CancelSpeech", true);
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS, "tap_cancel_close");
return cancel_err;
esp_err_t stop_err = ESP_OK;
if (should_send_stop) {
stop_err = voice_send_directive("continue-task", "StopSpeech", true);
if (stop_err != ESP_OK) {
// Fail-safe close to avoid leaving audio device open in abnormal flow.
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "tap_cancel_stop_failed_close");
}
}
return stop_err;
}
void voice_interaction_get_status(voice_interaction_status_t *out_status) {

4
components/domain/src/voice_interaction_common.c
View File

@@ -54,12 +54,14 @@ void voice_log_status_snapshot(const char *stage) {
}
ESP_LOGI(TAG,
"[stage] %s: active=%d ws=%d started=%d tap=%d state=%s up=%" PRIu32 " down=%" PRIu32 " err=%s",
"[stage] %s: active=%d ws=%d started=%d tap=%d speech=%d state=%s up=%" PRIu32
" down=%" PRIu32 " err=%s",
stage,
s_voice.session_active,
s_voice.ws_connected,
s_voice.started,
s_voice.tap_active,
s_voice.speech_active,
voice_interaction_dialog_state_str(s_voice.dialog_state),
s_voice.upstream_packets,
s_voice.downstream_packets,

227
components/domain/src/voice_interaction_mic_key.c Normal file
View File

@@ -0,0 +1,227 @@
#include "voice_interaction_internal.h"
#include <inttypes.h>
#include "esp_log.h"
#include "freertos/queue.h"
#define VOICE_MIC_KEY_TASK_STACK 6144
#define VOICE_MIC_KEY_QUEUE_LEN 8
#define VOICE_MIC_KEY_SESSION_READY_WAIT_MS 2000
#define VOICE_MIC_KEY_SESSION_READY_POLL_MS 30
typedef enum {
VOICE_MIC_KEY_CMD_PRESS_DOWN = 0,
VOICE_MIC_KEY_CMD_PRESS_UP,
} voice_mic_key_cmd_t;
static const char *TAG = "voice_mic_key";
static QueueHandle_t s_mic_key_queue;
static TaskHandle_t s_mic_key_task;
static bool s_mic_key_task_with_caps;
static bool s_mic_key_ready;
static bool voice_mic_key_round_busy(const voice_interaction_status_t *voice,
const printer_runtime_status_t *printer) {
if (voice == NULL || printer == NULL) {
return false;
}
bool marker_busy = voice->marker_image_gen_running;
bool print_busy = printer->busy || printer->queue_depth > 0;
return marker_busy || print_busy;
}
static bool voice_mic_key_session_ready(const voice_interaction_status_t *voice) {
return voice != NULL && voice->session_active && voice->ws_connected && voice->started;
}
static bool voice_mic_key_wait_session_ready(uint32_t timeout_ms) {
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
voice_interaction_status_t voice = {0};
voice_interaction_get_status(&voice);
if (voice_mic_key_session_ready(&voice)) {
return true;
}
vTaskDelay(pdMS_TO_TICKS(VOICE_MIC_KEY_SESSION_READY_POLL_MS));
}
return false;
}
static void voice_mic_key_handle_press_down(void) {
voice_interaction_status_t voice = {0};
voice_interaction_get_status(&voice);
if (voice.tap_active) {
ESP_LOGI(TAG, "MIC_KEY press ignored: tap already active");
return;
}
printer_runtime_status_t printer = {0};
printer_protocol_get_runtime_status(&printer);
if (voice_mic_key_round_busy(&voice, &printer)) {
ESP_LOGI(TAG,
"MIC_KEY press ignored: state=%s marker_busy=%d print_busy=%d queue=%" PRIu32,
voice_interaction_dialog_state_str(voice.dialog_state),
voice.marker_image_gen_running,
printer.busy,
printer.queue_depth);
return;
}
if (!voice_mic_key_session_ready(&voice)) {
char start_err[128] = {0};
esp_err_t start_rc = voice_interaction_start_with_timeout(0, start_err, sizeof(start_err));
if (start_rc != ESP_OK) {
ESP_LOGW(TAG,
"MIC_KEY auto session start failed: %s (%s)",
esp_err_to_name(start_rc),
start_err[0] != '\0' ? start_err : "-");
return;
}
}
if (!voice_mic_key_wait_session_ready(VOICE_MIC_KEY_SESSION_READY_WAIT_MS)) {
voice_interaction_get_status(&voice);
ESP_LOGW(TAG,
"MIC_KEY wait session ready timeout: active=%d ws=%d started=%d",
voice.session_active,
voice.ws_connected,
voice.started);
return;
}
char tap_err[128] = {0};
esp_err_t tap_rc = voice_interaction_tap_start(tap_err, sizeof(tap_err));
if (tap_rc != ESP_OK) {
ESP_LOGW(TAG,
"MIC_KEY tap start failed: %s (%s)",
esp_err_to_name(tap_rc),
tap_err[0] != '\0' ? tap_err : "-");
return;
}
ESP_LOGI(TAG, "MIC_KEY tap started");
}
static void voice_mic_key_handle_press_up(void) {
voice_interaction_status_t voice = {0};
voice_interaction_get_status(&voice);
if (!voice.tap_active) {
return;
}
char tap_err[128] = {0};
esp_err_t tap_rc = voice_interaction_tap_cancel(tap_err, sizeof(tap_err));
if (tap_rc != ESP_OK) {
ESP_LOGW(TAG,
"MIC_KEY tap stop failed: %s (%s)",
esp_err_to_name(tap_rc),
tap_err[0] != '\0' ? tap_err : "-");
return;
}
ESP_LOGI(TAG, "MIC_KEY tap stopped");
}
static void voice_mic_key_task_main(void *arg) {
(void)arg;
while (true) {
voice_mic_key_cmd_t cmd = VOICE_MIC_KEY_CMD_PRESS_UP;
if (xQueueReceive(s_mic_key_queue, &cmd, portMAX_DELAY) != pdTRUE) {
continue;
}
switch (cmd) {
case VOICE_MIC_KEY_CMD_PRESS_DOWN:
voice_mic_key_handle_press_down();
break;
case VOICE_MIC_KEY_CMD_PRESS_UP:
voice_mic_key_handle_press_up();
break;
default:
break;
}
}
}
static void voice_mic_key_platform_event_cb(platform_mic_key_event_t event, void *user_data) {
(void)user_data;
if (s_mic_key_queue == NULL) {
return;
}
voice_mic_key_cmd_t cmd = VOICE_MIC_KEY_CMD_PRESS_UP;
switch (event) {
case PLATFORM_MIC_KEY_EVENT_PRESS_DOWN:
cmd = VOICE_MIC_KEY_CMD_PRESS_DOWN;
break;
case PLATFORM_MIC_KEY_EVENT_PRESS_UP:
cmd = VOICE_MIC_KEY_CMD_PRESS_UP;
break;
default:
return;
}
BaseType_t queued = xQueueSend(s_mic_key_queue, &cmd, 0);
if (queued == pdTRUE) {
return;
}
voice_mic_key_cmd_t dropped = VOICE_MIC_KEY_CMD_PRESS_UP;
(void)xQueueReceive(s_mic_key_queue, &dropped, 0);
(void)xQueueSend(s_mic_key_queue, &cmd, 0);
}
esp_err_t voice_interaction_mic_key_init(void) {
if (s_mic_key_ready) {
return ESP_OK;
}
esp_err_t err = platform_mic_key_init();
if (err == ESP_ERR_NOT_SUPPORTED) {
ESP_LOGW(TAG, "MIC_KEY trigger disabled by board config");
s_mic_key_ready = true;
return ESP_OK;
}
if (err != ESP_OK) {
return err;
}
s_mic_key_queue = xQueueCreate(VOICE_MIC_KEY_QUEUE_LEN, sizeof(voice_mic_key_cmd_t));
if (s_mic_key_queue == NULL) {
platform_mic_key_deinit();
return ESP_ERR_NO_MEM;
}
BaseType_t ok = voice_create_task_prefer_psram(voice_mic_key_task_main,
"voice_mic_key",
VOICE_MIC_KEY_TASK_STACK,
4,
&s_mic_key_task,
&s_mic_key_task_with_caps);
if (ok != pdPASS) {
vQueueDelete(s_mic_key_queue);
s_mic_key_queue = NULL;
platform_mic_key_deinit();
return ESP_ERR_NO_MEM;
}
err = platform_mic_key_set_event_callback(voice_mic_key_platform_event_cb, NULL);
if (err != ESP_OK) {
voice_delete_task(s_mic_key_task, s_mic_key_task_with_caps);
s_mic_key_task = NULL;
s_mic_key_task_with_caps = false;
vQueueDelete(s_mic_key_queue);
s_mic_key_queue = NULL;
platform_mic_key_deinit();
return err;
}
s_mic_key_ready = true;
ESP_LOGI(TAG, "MIC_KEY trigger ready: press to speak, release to process");
return ESP_OK;
}

5
components/domain/src/voice_interaction_tasks.c
View File

@@ -27,6 +27,7 @@ void voice_uplink_task(void *arg) {
bool connected = false;
bool started = false;
bool tap_active = false;
bool speech_active = false;
bool listening = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
@@ -34,6 +35,7 @@ void voice_uplink_task(void *arg) {
connected = s_voice.ws_connected;
started = s_voice.started;
tap_active = s_voice.tap_active;
speech_active = s_voice.speech_active;
listening = (s_voice.dialog_state == VOICE_DIALOG_STATE_LISTENING);
voice_unlock();
}
@@ -42,7 +44,7 @@ void voice_uplink_task(void *arg) {
break;
}
if (!(connected && started && listening && tap_active)) {
if (!(connected && started && listening && tap_active && speech_active)) {
vTaskDelay(pdMS_TO_TICKS(20));
continue;
}
@@ -83,6 +85,7 @@ void voice_uplink_task(void *arg) {
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGW(TAG, "[stage] uplink_detected_ws_disconnected");

105
components/domain/src/voice_interaction_ws.c
View File

@@ -88,7 +88,7 @@ static char *voice_build_start_message(void) {
cJSON *parameters = cJSON_AddObjectToObject(payload, "parameters");
cJSON *upstream = cJSON_AddObjectToObject(parameters, "upstream");
cJSON_AddStringToObject(upstream, "type", "AudioOnly");
cJSON_AddStringToObject(upstream, "mode", "tap2talk");
cJSON_AddStringToObject(upstream, "mode", "push2talk");
cJSON_AddStringToObject(upstream, "audio_format", "raw-opus");
cJSON_AddNumberToObject(upstream, "sample_rate", CONFIG_TQ_VOICE_SAMPLE_RATE);
@@ -113,7 +113,7 @@ static char *voice_build_start_message(void) {
return text;
}
static esp_err_t voice_send_text(const char *text, int len) {
static esp_err_t voice_send_text_immediate(const char *text, int len) {
if (text == NULL || len <= 0) {
return ESP_ERR_INVALID_ARG;
}
@@ -132,6 +132,7 @@ static esp_err_t voice_send_text(const char *text, int len) {
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGW(TAG, "[stage] ws_state_reconciled: transport disconnected");
@@ -156,6 +157,43 @@ static esp_err_t voice_send_text(const char *text, int len) {
return ESP_OK;
}
static esp_err_t voice_send_text(const char *text, int len) {
if (text == NULL || len <= 0) {
return ESP_ERR_INVALID_ARG;
}
QueueHandle_t queue = NULL;
TaskHandle_t ws_text_task = NULL;
bool session_active = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
queue = s_voice.ws_text_queue;
ws_text_task = s_voice.ws_text_task;
session_active = s_voice.session_active;
voice_unlock();
}
if (queue == NULL || ws_text_task == NULL || !session_active) {
return voice_send_text_immediate(text, len);
}
char *copy = (char *)voice_malloc_prefer_psram((size_t)len + 1);
if (copy == NULL) {
return ESP_ERR_NO_MEM;
}
memcpy(copy, text, (size_t)len);
copy[len] = '\0';
voice_ws_text_msg_t msg = {
.text = copy,
.len = len,
};
if (xQueueSend(queue, &msg, pdMS_TO_TICKS(VOICE_WS_SEND_TIMEOUT_MS)) != pdTRUE) {
free(copy);
return ESP_ERR_TIMEOUT;
}
return ESP_OK;
}
esp_err_t voice_send_directive(const char *action,
const char *directive,
bool with_dialog_id) {
@@ -181,6 +219,68 @@ esp_err_t voice_send_directive(const char *action,
return err;
}
void voice_ws_text_task(void *arg) {
(void)arg;
while (true) {
bool shutdown = false;
QueueHandle_t queue = NULL;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
shutdown = s_voice.ws_text_shutdown;
queue = s_voice.ws_text_queue;
voice_unlock();
}
if (shutdown || queue == NULL) {
break;
}
voice_ws_text_msg_t msg = {0};
if (xQueueReceive(queue, &msg, pdMS_TO_TICKS(VOICE_WS_TEXT_QUEUE_RX_TIMEOUT_MS)) != pdTRUE) {
continue;
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_text_busy = true;
voice_unlock();
}
esp_err_t err = voice_send_text_immediate(msg.text, msg.len);
if (err != ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_set_last_error_locked("ws text send failed");
voice_unlock();
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
s_voice.ws_text_busy = false;
voice_unlock();
}
free(msg.text);
}
QueueHandle_t queue = NULL;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
queue = s_voice.ws_text_queue;
s_voice.ws_text_busy = false;
voice_unlock();
}
if (queue != NULL) {
voice_ws_text_msg_t msg = {0};
while (xQueueReceive(queue, &msg, 0) == pdTRUE) {
free(msg.text);
}
}
bool delete_with_caps = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
delete_with_caps = s_voice.ws_text_task_with_caps;
s_voice.ws_text_task = NULL;
s_voice.ws_text_task_with_caps = false;
voice_unlock();
}
voice_delete_self_task(delete_with_caps);
}
static void voice_handle_text_chunk(esp_websocket_event_data_t *data) {
if (data->payload_offset == 0) {
free(s_voice.text_agg);
@@ -310,6 +410,7 @@ void voice_websocket_event_handler(void *handler_args,
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
voice_unlock();

28
components/domain/src/voice_interaction_ws_audio.c
View File

@@ -71,16 +71,24 @@ void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len) {
VOICE_AUDIO_IO_TIMEOUT_MS,
audio_err,
sizeof(audio_err));
if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t pcm_ms = ((int64_t)samples * 1000 +
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) /
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE;
int64_t base_ms = s_voice.playback_deadline_ms > now_ms ?
s_voice.playback_deadline_ms : now_ms;
s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS;
s_voice.last_downstream_ms = now_ms;
voice_unlock();
if (write_err == ESP_OK) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t pcm_ms = ((int64_t)samples * 1000 +
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) /
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE;
int64_t base_ms = s_voice.playback_deadline_ms > now_ms ?
s_voice.playback_deadline_ms : now_ms;
s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS;
s_voice.last_downstream_ms = now_ms;
voice_unlock();
}
} else {
ESP_LOGW(TAG,
"[stage] audio_downstream_write_failed: rc=0x%x err=%s samples=%u",
(unsigned)write_err,
audio_err[0] != '\0' ? audio_err : "-",
(unsigned)samples);
}
}

153
components/domain/src/voice_interaction_ws_protocol.c
View File

@@ -11,6 +11,111 @@
static const char *TAG = "voice_interaction";
#define VOICE_RESPONSE_FINALIZE_TASK_STACK 6144
#define VOICE_RESPONSE_FINALIZE_TASK_RETRY_COUNT 3
#define VOICE_RESPONSE_FINALIZE_TASK_RETRY_DELAY_MS 120
static portMUX_TYPE s_response_finalize_lock = portMUX_INITIALIZER_UNLOCKED;
static bool s_response_finalize_running;
static esp_err_t voice_send_directive_with_retry(const char *action,
const char *directive,
bool with_dialog_id) {
esp_err_t rc = ESP_FAIL;
for (uint32_t i = 0; i < VOICE_RESPONSE_FINALIZE_TASK_RETRY_COUNT; ++i) {
rc = voice_send_directive(action, directive, with_dialog_id);
if (rc == ESP_OK || rc == ESP_ERR_INVALID_STATE) {
return rc;
}
vTaskDelay(pdMS_TO_TICKS(VOICE_RESPONSE_FINALIZE_TASK_RETRY_DELAY_MS));
}
return rc;
}
static uint32_t voice_calc_response_finalize_wait_ms(void) {
uint32_t playback_wait_ms = VOICE_AUDIO_DRAIN_WAIT_MS;
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
return playback_wait_ms;
}
int64_t now_ms = voice_now_ms();
int64_t deadline_ms = s_voice.playback_deadline_ms;
voice_unlock();
if (deadline_ms <= now_ms) {
return playback_wait_ms;
}
int64_t remain_ms = (deadline_ms - now_ms) + VOICE_AUDIO_DRAIN_QUIET_MS + 120;
if (remain_ms <= (int64_t)playback_wait_ms) {
return playback_wait_ms;
}
return (remain_ms > 12000) ? 12000 : (uint32_t)remain_ms;
}
static void voice_response_finalize_task(void *arg) {
(void)arg;
uint32_t playback_wait_ms = voice_calc_response_finalize_wait_ms();
ESP_LOGI(TAG,
"[stage] responding_ended_local_playback_wait: timeout_ms=%u",
(unsigned)playback_wait_ms);
voice_wait_audio_playback_done(playback_wait_ms, "responding_ended_local_playback");
esp_err_t local_end_rc =
voice_send_directive_with_retry("continue-task", "LocalRespondingEnded", true);
if (local_end_rc != ESP_OK) {
ESP_LOGW(TAG,
"[stage] responding_ended_local_end_failed: rc=0x%x",
(unsigned)local_end_rc);
}
esp_err_t stop_rc =
voice_send_directive_with_retry("finish-task", "Stop", true);
if (stop_rc != ESP_OK) {
ESP_LOGW(TAG,
"[stage] responding_ended_stop_failed: rc=0x%x",
(unsigned)stop_rc);
}
portENTER_CRITICAL(&s_response_finalize_lock);
s_response_finalize_running = false;
portEXIT_CRITICAL(&s_response_finalize_lock);
vTaskDelete(NULL);
}
static void voice_schedule_response_finalize(void) {
bool should_start = false;
portENTER_CRITICAL(&s_response_finalize_lock);
if (!s_response_finalize_running) {
s_response_finalize_running = true;
should_start = true;
}
portEXIT_CRITICAL(&s_response_finalize_lock);
if (!should_start) {
ESP_LOGI(TAG, "[stage] responding_ended_finalize_skip: task already running");
return;
}
BaseType_t ok = xTaskCreate(voice_response_finalize_task,
"voice_rsp_fin",
VOICE_RESPONSE_FINALIZE_TASK_STACK,
NULL,
4,
NULL);
if (ok != pdPASS) {
portENTER_CRITICAL(&s_response_finalize_lock);
s_response_finalize_running = false;
portEXIT_CRITICAL(&s_response_finalize_lock);
ESP_LOGW(TAG, "[stage] responding_ended_finalize_start_failed");
} else {
ESP_LOGI(TAG, "[stage] responding_ended_finalize_task_started");
}
}
static const char *voice_get_json_str(cJSON *obj, const char *name) {
if (obj == NULL || name == NULL) {
return NULL;
@@ -73,6 +178,7 @@ static void voice_handle_output_event(cJSON *output) {
bool has_finished = voice_get_json_bool(output, "finished", &finished);
const char *responding_final_text = NULL;
bool should_check_marker_in_first_sentence = false;
bool should_try_send_speech = false;
if (has_finished && finished) {
if (strcmp(event_name, "SpeechContent") == 0) {
voice_log_final_text("asr_final_text", voice_get_json_str(output, "text"));
@@ -114,13 +220,16 @@ static void voice_handle_output_event(cJSON *output) {
if (prev_state != VOICE_DIALOG_STATE_LISTENING) {
s_voice.response_first_sentence_checked = false;
}
should_try_send_speech = s_voice.tap_active && !s_voice.speech_active;
} else if (strcmp(state, "Thinking") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING;
s_voice.speech_active = false;
if (prev_state != VOICE_DIALOG_STATE_THINKING) {
s_voice.response_first_sentence_checked = false;
}
} else if (strcmp(state, "Responding") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING;
s_voice.speech_active = false;
if (prev_state != VOICE_DIALOG_STATE_RESPONDING) {
s_voice.response_first_sentence_checked = false;
}
@@ -131,14 +240,14 @@ static void voice_handle_output_event(cJSON *output) {
state != NULL ? state : "-",
voice_interaction_dialog_state_str(s_voice.dialog_state));
} else if (strcmp(event_name, "SpeechEnded") == 0) {
s_voice.tap_active = false;
ESP_LOGI(TAG, "[stage] speech_ended: mark single_round_input_done");
s_voice.speech_active = false;
ESP_LOGI(TAG, "[stage] speech_ended: stop uplink");
} else if (strcmp(event_name, "RespondingEnded") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
ESP_LOGI(TAG, "[stage] responding_ended: wait_local_playback_before_ack");
ESP_LOGI(TAG, "[stage] responding_ended: schedule_local_playback_finalize");
} else if (strcmp(event_name, "Stopped") == 0) {
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.response_first_sentence_checked = false;
should_close_audio = true;
@@ -151,6 +260,7 @@ static void voice_handle_output_event(cJSON *output) {
}
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.response_first_sentence_checked = false;
should_close_audio = true;
}
@@ -165,31 +275,21 @@ static void voice_handle_output_event(cJSON *output) {
voice_marker_try_trigger_image_generation(dialog_id, responding_final_text);
}
if (should_try_send_speech) {
char speech_err[96] = {0};
esp_err_t rc = voice_interaction_try_send_speech(speech_err, sizeof(speech_err));
if (rc != ESP_OK && rc != ESP_ERR_INVALID_STATE) {
ESP_LOGW(TAG,
"[stage] auto_send_speech_failed: rc=0x%x err=%s",
(unsigned)rc,
speech_err[0] != '\0' ? speech_err : "-");
}
}
if (strcmp(event_name, "RespondingStarted") == 0) {
(void)voice_send_directive("continue-task", "LocalRespondingStarted", true);
} else if (strcmp(event_name, "RespondingEnded") == 0) {
uint32_t playback_wait_ms = VOICE_AUDIO_DRAIN_WAIT_MS;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t deadline_ms = s_voice.playback_deadline_ms;
voice_unlock();
if (deadline_ms > now_ms) {
int64_t remain_ms = (deadline_ms - now_ms) + VOICE_AUDIO_DRAIN_QUIET_MS + 120;
if (remain_ms > (int64_t)playback_wait_ms) {
if (remain_ms > 12000) {
playback_wait_ms = 12000;
} else {
playback_wait_ms = (uint32_t)remain_ms;
}
}
}
}
ESP_LOGI(TAG, "[stage] responding_ended_local_playback_wait: timeout_ms=%u", (unsigned)playback_wait_ms);
voice_wait_audio_playback_done(playback_wait_ms,
"responding_ended_local_playback");
(void)voice_send_directive("continue-task", "LocalRespondingEnded", true);
(void)voice_send_directive("finish-task", "Stop", true);
voice_schedule_response_finalize();
}
}
@@ -243,6 +343,7 @@ void voice_ws_protocol_handle_text_message(const char *text, size_t len) {
voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed");
s_voice.started = false;
s_voice.tap_active = false;
s_voice.speech_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
voice_unlock();
}

2
components/platform/CMakeLists.txt
View File

@@ -7,6 +7,7 @@ idf_component_register(
"src/display_lcd_module.c"
"src/display_lcd_decode_image.c"
"src/display_st7789p3.c"
"src/platform_mic_key.c"
"src/voice_audio.c"
"src/runtime_policy.c"
"src/runtime_diagnostics.c"
@@ -26,6 +27,7 @@ idf_component_register(
esp_driver_spi
lvgl
espressif__libpng
espressif__button
esp_codec_dev
esp_audio_codec
driver

12
components/platform/include/platform.h
View File

@@ -95,6 +95,18 @@ esp_err_t platform_display_show_png(const uint8_t *png,
void platform_display_mark_shared_spi_dirty(void);
void platform_display_deinit(void);
// ---------- mic_key ----------
typedef enum {
PLATFORM_MIC_KEY_EVENT_PRESS_DOWN = 0,
PLATFORM_MIC_KEY_EVENT_PRESS_UP,
} platform_mic_key_event_t;
typedef void (*platform_mic_key_event_cb_t)(platform_mic_key_event_t event, void *user_data);
esp_err_t platform_mic_key_init(void);
void platform_mic_key_deinit(void);
esp_err_t platform_mic_key_set_event_callback(platform_mic_key_event_cb_t cb, void *user_data);
// ---------- voice_audio ----------
typedef struct {
uint32_t sample_rate;

126
components/platform/src/platform_mic_key.c Normal file
View File

@@ -0,0 +1,126 @@
#include "platform.h"
#include <stddef.h>
#include "button_gpio.h"
#include "esp_log.h"
#include "freertos/FreeRTOS.h"
#include "iot_button.h"
#include "sdkconfig.h"
static const char *TAG = "platform_mic_key";
static const uint16_t MIC_KEY_SHORT_PRESS_MS = 40;
static const uint16_t MIC_KEY_HOLD_START_MS = 80;
static button_handle_t s_mic_key_button;
static platform_mic_key_event_cb_t s_event_cb;
static void *s_event_cb_user_data;
static portMUX_TYPE s_event_cb_lock = portMUX_INITIALIZER_UNLOCKED;
static void mic_key_dispatch_event(platform_mic_key_event_t event) {
platform_mic_key_event_cb_t cb = NULL;
void *user_data = NULL;
portENTER_CRITICAL(&s_event_cb_lock);
cb = s_event_cb;
user_data = s_event_cb_user_data;
portEXIT_CRITICAL(&s_event_cb_lock);
if (cb != NULL) {
cb(event, user_data);
}
}
static void mic_key_hold_start_cb(void *button_handle, void *usr_data) {
(void)button_handle;
(void)usr_data;
mic_key_dispatch_event(PLATFORM_MIC_KEY_EVENT_PRESS_DOWN);
}
static void mic_key_press_up_cb(void *button_handle, void *usr_data) {
(void)button_handle;
(void)usr_data;
mic_key_dispatch_event(PLATFORM_MIC_KEY_EVENT_PRESS_UP);
}
esp_err_t platform_mic_key_init(void) {
if (CONFIG_TQ_MIC_KEY_PIN < 0) {
ESP_LOGW(TAG, "MIC_KEY disabled by config: pin=%d", CONFIG_TQ_MIC_KEY_PIN);
return ESP_ERR_NOT_SUPPORTED;
}
if (s_mic_key_button != NULL) {
return ESP_OK;
}
button_config_t button_cfg = {
.long_press_time = MIC_KEY_HOLD_START_MS,
.short_press_time = MIC_KEY_SHORT_PRESS_MS,
};
button_gpio_config_t gpio_cfg = {
.gpio_num = CONFIG_TQ_MIC_KEY_PIN,
.active_level = 0,
.enable_power_save = false,
.disable_pull = false,
};
esp_err_t err = iot_button_new_gpio_device(&button_cfg, &gpio_cfg, &s_mic_key_button);
if (err != ESP_OK) {
ESP_LOGE(TAG,
"iot_button_new_gpio_device failed: pin=%d err=%s",
CONFIG_TQ_MIC_KEY_PIN,
esp_err_to_name(err));
return err;
}
err = iot_button_register_cb(s_mic_key_button,
BUTTON_LONG_PRESS_START,
NULL,
mic_key_hold_start_cb,
NULL);
if (err != ESP_OK) {
ESP_LOGE(TAG, "register BUTTON_LONG_PRESS_START failed: %s", esp_err_to_name(err));
(void)iot_button_delete(s_mic_key_button);
s_mic_key_button = NULL;
return err;
}
err = iot_button_register_cb(s_mic_key_button, BUTTON_PRESS_UP, NULL, mic_key_press_up_cb, NULL);
if (err != ESP_OK) {
ESP_LOGE(TAG, "register BUTTON_PRESS_UP failed: %s", esp_err_to_name(err));
(void)iot_button_delete(s_mic_key_button);
s_mic_key_button = NULL;
return err;
}
ESP_LOGI(TAG,
"MIC_KEY initialized: gpio=%d active_level=low short_press_ms=%u hold_start_ms=%u",
CONFIG_TQ_MIC_KEY_PIN,
(unsigned)MIC_KEY_SHORT_PRESS_MS,
(unsigned)MIC_KEY_HOLD_START_MS);
return ESP_OK;
}
void platform_mic_key_deinit(void) {
if (s_mic_key_button != NULL) {
(void)iot_button_delete(s_mic_key_button);
s_mic_key_button = NULL;
}
portENTER_CRITICAL(&s_event_cb_lock);
s_event_cb = NULL;
s_event_cb_user_data = NULL;
portEXIT_CRITICAL(&s_event_cb_lock);
}
esp_err_t platform_mic_key_set_event_callback(platform_mic_key_event_cb_t cb, void *user_data) {
if (s_mic_key_button == NULL) {
return ESP_ERR_INVALID_STATE;
}
portENTER_CRITICAL(&s_event_cb_lock);
s_event_cb = cb;
s_event_cb_user_data = user_data;
portEXIT_CRITICAL(&s_event_cb_lock);
return ESP_OK;
}