Bairan.Zhang/AI_Printer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(voice): stabilize single-round flow and sync web UI state

Browse Source
This commit is contained in:
admin
2026-02-26 14:33:02 +08:00
parent e6a6b69ced
commit dde759d0a4
10 changed files with 251 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@@ -336,7 +336,8 @@ curl -X POST http://<esp-ip>/v1/voice/session/stop
- `/v1/print/text` and `/v1/print/receipt` now support UTF-8 Chinese via embedded 16x16 GB2312 glyphs.
- Characters outside embedded glyph set are rendered as square fallback boxes.
- QR encoding uses embedded `qrcodegen` (Project Nayuki C implementation).
- Tap2talk uses server-side VAD and keeps uploading 100ms raw-opus frames in `Listening` state (silence when user is not tapping), matching `docs/websocket.md` requirements.
- Voice flow is now single-round: audio uplink stops after `SpeechEnded`, then `LocalRespondingEnded` and `Stop` are sent after local playback drain.
- Web UI blocks starting the next voice round while voice is Thinking/Responding, marker image generation is running, or printer queue is still busy.
## Font Assets

132
components/control_plane/src/rest_server.c
View File

@@ -105,11 +105,13 @@ static const char *s_web_index =
" </div>\n"
" <div class='small'>Submit prompt only: ESP32-S3 calls DashScope Z-Image, downloads PNG, rasterizes and prints.</div>\n"
"\n"
" <label>Voice Tap2Talk</label>\n"
" <label>Voice 单轮对话</label>\n"
" <div class='row'>\n"
" <button id='btnVoiceTalk' class='talk'>点击开始对话</button>\n"
" <button id='btnVoiceTalk' class='talk'>点击开始单轮对话</button>\n"
" </div>\n"
" <div class='small'>点击一次开始语音输入，再点击一次结束（自动建立语音会话）。</div>\n"
" <div id='voiceState' class='small'>语音状态：初始化中</div>\n"
" <div id='voiceErr' class='small' style='color:#b00020;'></div>\n"
" <div class='small'>单击开始本轮语音输入；识别结束后自动停止上传，等待语音播报和打印完成。</div>\n"
"\n"
" <div class='small'>Tip: connect printer first, then submit print jobs.</div>\n"
" <pre id='out'>Ready.</pre>\n"
@@ -131,9 +133,12 @@ static const char *s_web_index =
" const imgInvertEl = document.getElementById('imgInvert');\n"
" const imgPromptExtendEl = document.getElementById('imgPromptExtend');\n"
" const btnVoiceTalkEl = document.getElementById('btnVoiceTalk');\n"
" const voiceStateEl = document.getElementById('voiceState');\n"
" const voiceErrEl = document.getElementById('voiceErr');\n"
" const thermalPromptSuffix = 'black and white line art, pure white background, high contrast, clean edges, minimal texture, no dark background, no heavy shadows, centered subject, thermal printer friendly';\n"
" let voiceTalkHolding = false;\n"
" let voiceTalkBusy = false;\n"
" let voiceStatusSyncing = false;\n"
"\n"
" function log(obj) {\n"
" if (typeof obj === 'string') {\n"
@@ -163,19 +168,95 @@ static const char *s_web_index =
" return new Promise((resolve) => setTimeout(resolve, ms));\n"
" }\n"
"\n"
" function voiceRoundBusy(st) {\n"
" if (!st) return false;\n"
" const state = (st.voice_dialog_state || '').toLowerCase();\n"
" const voiceBusy = state === 'thinking' || state === 'responding';\n"
" const markerBusy = !!st.voice_marker_image_gen_running;\n"
" const queueDepth = Number(st.queue_depth || 0);\n"
" const printBusy = !!st.printer_busy || queueDepth > 0;\n"
" return voiceBusy || markerBusy || printBusy;\n"
" }\n"
"\n"
" function voiceStatusText(st) {\n"
" if (!st) return '语音状态：未知';\n"
" const state = (st.voice_dialog_state || '').toLowerCase();\n"
" const queueDepth = Number(st.queue_depth || 0);\n"
" if (!st.voice_session_active) {\n"
" return '语音状态：会话未启动';\n"
" }\n"
" if (!st.voice_ws_connected) {\n"
" return '语音状态：会话断开，点击开始将自动重建';\n"
" }\n"
" if (st.voice_tap_active) {\n"
" return '语音状态：录音中';\n"
" }\n"
" if (state === 'thinking' || state === 'responding') {\n"
" return '语音状态：语音处理中';\n"
" }\n"
" if (!!st.voice_marker_image_gen_running) {\n"
" return '语音状态：图片生成中';\n"
" }\n"
" if (!!st.printer_busy || queueDepth > 0) {\n"
" return '语音状态：打印处理中';\n"
" }\n"
" return '语音状态：空闲，可开始下一轮';\n"
" }\n"
"\n"
" function voiceErrorText(st) {\n"
" if (!st || st.voice_last_error === undefined || st.voice_last_error === null) {\n"
" return '';\n"
" }\n"
" const msg = String(st.voice_last_error).trim();\n"
" if (!msg || msg === '-') {\n"
" return '';\n"
" }\n"
" return '最近错误：' + msg;\n"
" }\n"
"\n"
" function voiceRoundBusyText(st) {\n"
" if (!st) return '处理中，请等待';\n"
" const state = (st.voice_dialog_state || '').toLowerCase();\n"
" if (state === 'thinking' || state === 'responding') {\n"
" return '语音处理中，请等待';\n"
" }\n"
" if (!!st.voice_marker_image_gen_running) {\n"
" return '图片生成中，请等待';\n"
" }\n"
" const queueDepth = Number(st.queue_depth || 0);\n"
" if (!!st.printer_busy || queueDepth > 0) {\n"
" return '打印处理中，请等待';\n"
" }\n"
" return '处理中，请等待';\n"
" }\n"
"\n"
" function setVoiceTalkButtonState(active) {\n"
" if (active) {\n"
" btnVoiceTalkEl.classList.add('active');\n"
" btnVoiceTalkEl.textContent = '点击结束对话';\n"
" btnVoiceTalkEl.textContent = '点击取消本轮录音';\n"
" } else {\n"
" btnVoiceTalkEl.classList.remove('active');\n"
" btnVoiceTalkEl.textContent = '点击开始对话';\n"
" btnVoiceTalkEl.textContent = '点击开始单轮对话';\n"
" }\n"
" }\n"
"\n"
" function syncVoiceTalkUi(st) {\n"
" const active = !!(st && st.voice_tap_active);\n"
" const active = !!(st && st.voice_tap_active && st.voice_ws_connected && st.voice_started);\n"
" const busy = voiceRoundBusy(st);\n"
" voiceTalkHolding = active;\n"
" if (voiceStateEl) {\n"
" voiceStateEl.textContent = voiceStatusText(st);\n"
" }\n"
" if (voiceErrEl) {\n"
" voiceErrEl.textContent = voiceErrorText(st);\n"
" }\n"
" if (busy) {\n"
" btnVoiceTalkEl.classList.remove('active');\n"
" btnVoiceTalkEl.disabled = true;\n"
" btnVoiceTalkEl.textContent = voiceRoundBusyText(st);\n"
" return;\n"
" }\n"
" btnVoiceTalkEl.disabled = false;\n"
" setVoiceTalkButtonState(active);\n"
" }\n"
"\n"
@@ -204,6 +285,20 @@ static const char *s_web_index =
" throw new Error('voice session is not ready, please retry');\n"
" }\n"
"\n"
" async function refreshVoiceStatus() {\n"
" if (voiceStatusSyncing) {\n"
" return null;\n"
" }\n"
" voiceStatusSyncing = true;\n"
" try {\n"
" const st = await callApi('/v1/voice/status', 'GET');\n"
" syncVoiceTalkUi(st);\n"
" return st;\n"
" } finally {\n"
" voiceStatusSyncing = false;\n"
" }\n"
" }\n"
"\n"
" async function beginVoiceTalk() {\n"
" if (voiceTalkBusy) {\n"
" const st = await callApi('/v1/voice/status', 'GET');\n"
@@ -327,16 +422,23 @@ static const char *s_web_index =
" });\n"
"\n"
" btnVoiceTalkEl.addEventListener('click', () => run(async () => {\n"
" const st = await callApi('/v1/voice/status', 'GET');\n"
" syncVoiceTalkUi(st);\n"
" const st = await refreshVoiceStatus();\n"
" if (!st) {\n"
" return null;\n"
" }\n"
" if (voiceRoundBusy(st)) {\n"
" return st;\n"
" }\n"
" if (st && st.voice_tap_active) {\n"
" return endVoiceTalk();\n"
" }\n"
" return beginVoiceTalk();\n"
" }));\n"
" window.addEventListener('blur', () => run(async () => {\n"
" const st = await callApi('/v1/voice/status', 'GET');\n"
" syncVoiceTalkUi(st);\n"
" const st = await refreshVoiceStatus();\n"
" if (!st) {\n"
" return null;\n"
" }\n"
" if (st && st.voice_tap_active) {\n"
" return endVoiceTalk();\n"
" }\n"
@@ -345,10 +447,16 @@ static const char *s_web_index =
" setVoiceTalkButtonState(false);\n"
" (async () => {\n"
" try {\n"
" const st = await callApi('/v1/voice/status', 'GET');\n"
" syncVoiceTalkUi(st);\n"
" await refreshVoiceStatus();\n"
" } catch (_) {}\n"
" })();\n"
" setInterval(() => {\n"
" refreshVoiceStatus().catch(() => {\n"
" if (voiceStateEl) {\n"
" voiceStateEl.textContent = '语音状态：状态获取失败';\n"
" }\n"
" });\n"
" }, 1500);\n"
" </script>\n"
"</body>\n"
"</html>\n";

5
components/control_plane/src/rest_server_voice.c
View File

@@ -7,12 +7,15 @@
static void rest_server_add_voice_status(cJSON *root) {
voice_interaction_status_t st = {0};
voice_interaction_get_status(&st);
printer_runtime_status_t printer = {0};
printer_protocol_get_runtime_status(&printer);
cJSON_AddBoolToObject(root, "voice_ready", st.ready);
cJSON_AddBoolToObject(root, "voice_session_active", st.session_active);
cJSON_AddBoolToObject(root, "voice_ws_connected", st.ws_connected);
cJSON_AddBoolToObject(root, "voice_started", st.started);
cJSON_AddBoolToObject(root, "voice_tap_active", st.tap_active);
cJSON_AddBoolToObject(root, "voice_marker_image_gen_running", st.marker_image_gen_running);
cJSON_AddStringToObject(root,
"voice_dialog_state",
voice_interaction_dialog_state_str(st.dialog_state));
@@ -22,6 +25,8 @@ static void rest_server_add_voice_status(cJSON *root) {
cJSON_AddNumberToObject(root, "voice_last_event_ms", (double)st.last_event_ms);
cJSON_AddNumberToObject(root, "voice_upstream_packets", (double)st.upstream_packets);
cJSON_AddNumberToObject(root, "voice_downstream_packets", (double)st.downstream_packets);
cJSON_AddBoolToObject(root, "printer_busy", printer.busy);
cJSON_AddNumberToObject(root, "queue_depth", (double)printer.queue_depth);
}
esp_err_t rest_server_voice_status_get(httpd_req_t *req) {

1
components/domain/include/domain.h
View File

@@ -203,6 +203,7 @@ typedef struct {
bool ws_connected;
bool started;
bool tap_active;
bool marker_image_gen_running;
voice_dialog_state_t dialog_state;
char task_id[40];
char dialog_id[40];

1
components/domain/internal/voice_interaction_internal.h
View File

@@ -120,6 +120,7 @@ esp_err_t voice_send_directive(const char *action,
void voice_ws_protocol_handle_text_message(const char *text, size_t len);
void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len);
void voice_marker_try_trigger_image_generation(const char *dialog_id, const char *final_text);
bool voice_marker_image_generation_is_running(void);
void voice_websocket_event_handler(void *handler_args,
esp_event_base_t base,

56
components/domain/src/voice_interaction.c
View File

@@ -9,6 +9,32 @@
static const char *TAG = "voice_interaction";
static void voice_wait_worker_tasks_exit(uint32_t timeout_ms) {
int64_t deadline_ms = voice_now_ms() + (int64_t)timeout_ms;
while (voice_now_ms() < deadline_ms) {
bool uplink_alive = false;
bool heartbeat_alive = false;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
uplink_alive = (s_voice.uplink_task != NULL);
heartbeat_alive = (s_voice.heartbeat_task != NULL);
voice_unlock();
}
if (!uplink_alive && !heartbeat_alive) {
return;
}
vTaskDelay(pdMS_TO_TICKS(20));
}
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
ESP_LOGW(TAG,
"[stage] worker_task_exit_timeout: uplink_alive=%d heartbeat_alive=%d",
s_voice.uplink_task != NULL,
s_voice.heartbeat_task != NULL);
voice_unlock();
}
}
const char *voice_interaction_dialog_state_str(voice_dialog_state_t state) {
switch (state) {
case VOICE_DIALOG_STATE_IDLE:
@@ -71,12 +97,23 @@ esp_err_t voice_interaction_start(char *err, size_t err_len) {
return ESP_ERR_TIMEOUT;
}
bool already_active = s_voice.session_active;
bool ws_connected = s_voice.ws_connected;
bool started = s_voice.started;
voice_unlock();
if (already_active) {
ESP_LOGI(TAG, "[stage] session_start_skip: already active");
voice_log_status_snapshot("already_active");
return ESP_OK;
if (ws_connected && started) {
ESP_LOGI(TAG, "[stage] session_start_skip: already active");
voice_log_status_snapshot("already_active");
return ESP_OK;
}
ESP_LOGW(TAG,
"[stage] session_start_recover: active=%d ws_connected=%d started=%d",
already_active,
ws_connected,
started);
(void)voice_interaction_stop(NULL, 0);
}
voice_interaction_ws_init_cjson_hooks();
@@ -290,8 +327,6 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) {
bool ws_connected = s_voice.ws_connected;
TaskHandle_t uplink_task = s_voice.uplink_task;
TaskHandle_t heartbeat_task = s_voice.heartbeat_task;
bool uplink_task_with_caps = s_voice.uplink_task_with_caps;
bool heartbeat_task_with_caps = s_voice.heartbeat_task_with_caps;
esp_websocket_client_handle_t ws = s_voice.ws;
s_voice.tap_active = false;
@@ -302,9 +337,13 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) {
(void)voice_send_directive("finish-task", "Stop", true);
}
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS, "session_stop_close");
voice_delete_task(uplink_task, uplink_task_with_caps);
voice_delete_task(heartbeat_task, heartbeat_task_with_caps);
if (uplink_task != NULL) {
(void)xTaskAbortDelay(uplink_task);
}
if (heartbeat_task != NULL) {
(void)xTaskAbortDelay(heartbeat_task);
}
voice_wait_worker_tasks_exit(1500);
if (ws != NULL) {
ESP_LOGI(TAG, "[stage] ws_stop_destroy_begin");
@@ -444,6 +483,7 @@ void voice_interaction_get_status(voice_interaction_status_t *out_status) {
out_status->ws_connected = s_voice.ws_connected;
out_status->started = s_voice.started;
out_status->tap_active = s_voice.tap_active;
out_status->marker_image_gen_running = voice_marker_image_generation_is_running();
out_status->dialog_state = s_voice.dialog_state;
out_status->last_event_ms = s_voice.last_event_ms;
out_status->upstream_packets = s_voice.upstream_packets;

8
components/domain/src/voice_interaction_marker.c
View File

@@ -472,3 +472,11 @@ void voice_marker_try_trigger_image_generation(const char *dialog_id, const char
voice_log_text_preview("marker_image_gen_prompt_templated", templated_prompt);
voice_schedule_marker_image_generation(templated_prompt, dialog_id);
}
bool voice_marker_image_generation_is_running(void) {
bool running = false;
portENTER_CRITICAL(&s_marker_image_gen_lock);
running = s_marker_image_gen_running;
portEXIT_CRITICAL(&s_marker_image_gen_lock);
return running;
}

39
components/domain/src/voice_interaction_tasks.c
View File

@@ -10,7 +10,6 @@ static const char *TAG = "voice_interaction";
void voice_uplink_task(void *arg) {
(void)arg;
const int frame_ms = CONFIG_TQ_VOICE_OPUS_FRAME_MS;
bool low_stack_warned = false;
UBaseType_t start_hwm = uxTaskGetStackHighWaterMark(NULL);
ESP_LOGI(TAG, "[stage] uplink_task_started: stack_hwm_words=%u", (unsigned)start_hwm);
@@ -43,24 +42,19 @@ void voice_uplink_task(void *arg) {
break;
}
if (!(connected && started && listening)) {
if (!(connected && started && listening && tap_active)) {
vTaskDelay(pdMS_TO_TICKS(20));
continue;
}
if (tap_active) {
char read_err[64] = {0};
esp_err_t rc = voice_audio_read_pcm(s_voice.pcm_tx_buf,
(size_t)s_voice.opus_enc_in_size / VOICE_SAMPLE_BYTES,
VOICE_AUDIO_IO_TIMEOUT_MS,
read_err,
sizeof(read_err));
if (rc != ESP_OK) {
memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size);
}
} else {
char read_err[64] = {0};
esp_err_t rc = voice_audio_read_pcm(s_voice.pcm_tx_buf,
(size_t)s_voice.opus_enc_in_size / VOICE_SAMPLE_BYTES,
VOICE_AUDIO_IO_TIMEOUT_MS,
read_err,
sizeof(read_err));
if (rc != ESP_OK) {
memset(s_voice.pcm_tx_buf, 0, (size_t)s_voice.opus_enc_in_size);
vTaskDelay(pdMS_TO_TICKS(frame_ms));
}
esp_audio_enc_in_frame_t in_frame = {
@@ -80,10 +74,23 @@ void voice_uplink_task(void *arg) {
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
esp_websocket_client_handle_t ws = s_voice.ws;
bool can_send = s_voice.ws_connected;
bool ws_connected = s_voice.ws_connected;
voice_unlock();
if (ws != NULL && can_send) {
bool transport_connected = (ws != NULL) && esp_websocket_client_is_connected(ws);
if (ws_connected && !transport_connected && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
if (s_voice.ws_connected) {
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGW(TAG, "[stage] uplink_detected_ws_disconnected");
}
voice_unlock();
}
if (ws != NULL && ws_connected && transport_connected) {
int ret = esp_websocket_client_send_bin(ws,
(const char *)s_voice.opus_tx_buf,
(int)out_frame.encoded_bytes,

15
components/domain/src/voice_interaction_ws.c
View File

@@ -126,7 +126,20 @@ static esp_err_t voice_send_text(const char *text, int len) {
bool connected = s_voice.ws_connected;
voice_unlock();
if (ws == NULL || !connected) {
bool transport_connected = (ws != NULL) && esp_websocket_client_is_connected(ws);
if (connected && !transport_connected && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
if (s_voice.ws_connected) {
s_voice.ws_connected = false;
s_voice.started = false;
s_voice.tap_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
s_voice.last_event_ms = voice_now_ms();
ESP_LOGW(TAG, "[stage] ws_state_reconciled: transport disconnected");
}
voice_unlock();
}
if (ws == NULL || !connected || !transport_connected) {
ESP_LOGW(TAG, "[stage] ws_send_text skipped: ws=%p connected=%d len=%d", (void *)ws, connected, len);
return ESP_ERR_INVALID_STATE;
}

30
components/domain/src/voice_interaction_ws_protocol.c
View File

@@ -131,7 +131,11 @@ static void voice_handle_output_event(cJSON *output) {
state != NULL ? state : "-",
voice_interaction_dialog_state_str(s_voice.dialog_state));
} else if (strcmp(event_name, "SpeechEnded") == 0) {
ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active);
s_voice.tap_active = false;
ESP_LOGI(TAG, "[stage] speech_ended: mark single_round_input_done");
} else if (strcmp(event_name, "RespondingEnded") == 0) {
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
ESP_LOGI(TAG, "[stage] responding_ended: wait_local_playback_before_ack");
} else if (strcmp(event_name, "Stopped") == 0) {
s_voice.started = false;
s_voice.tap_active = false;
@@ -164,7 +168,28 @@ static void voice_handle_output_event(cJSON *output) {
if (strcmp(event_name, "RespondingStarted") == 0) {
(void)voice_send_directive("continue-task", "LocalRespondingStarted", true);
} else if (strcmp(event_name, "RespondingEnded") == 0) {
uint32_t playback_wait_ms = VOICE_AUDIO_DRAIN_WAIT_MS;
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
int64_t now_ms = voice_now_ms();
int64_t deadline_ms = s_voice.playback_deadline_ms;
voice_unlock();
if (deadline_ms > now_ms) {
int64_t remain_ms = (deadline_ms - now_ms) + VOICE_AUDIO_DRAIN_QUIET_MS + 120;
if (remain_ms > (int64_t)playback_wait_ms) {
if (remain_ms > 12000) {
playback_wait_ms = 12000;
} else {
playback_wait_ms = (uint32_t)remain_ms;
}
}
}
}
ESP_LOGI(TAG, "[stage] responding_ended_local_playback_wait: timeout_ms=%u", (unsigned)playback_wait_ms);
voice_wait_audio_playback_done(playback_wait_ms,
"responding_ended_local_playback");
(void)voice_send_directive("continue-task", "LocalRespondingEnded", true);
(void)voice_send_directive("finish-task", "Stop", true);
}
}
@@ -216,6 +241,9 @@ void voice_ws_protocol_handle_text_message(const char *text, size_t len) {
ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed");
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed");
s_voice.started = false;
s_voice.tap_active = false;
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
voice_unlock();
}
}