From dd108e2658abd9fa9bc6e6be85bae9ac6a443c41 Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 1 Mar 2026 01:21:18 +0800 Subject: [PATCH] feat(voice): migrate to push2talk and serialize websocket text sends --- .gitignore | 3 +- .vscode/settings.json | 11 - README.md | 15 +- components/control_plane/src/rest_server.c | 156 +----------- .../control_plane/src/rest_server_voice.c | 2 +- components/domain/CMakeLists.txt | 1 + .../internal/voice_interaction_internal.h | 20 +- components/domain/src/voice_interaction.c | 217 +++++++++++++++-- .../domain/src/voice_interaction_common.c | 4 +- .../domain/src/voice_interaction_mic_key.c | 227 ++++++++++++++++++ .../domain/src/voice_interaction_tasks.c | 5 +- components/domain/src/voice_interaction_ws.c | 105 +++++++- .../domain/src/voice_interaction_ws_audio.c | 28 ++- .../src/voice_interaction_ws_protocol.c | 153 ++++++++++-- components/platform/CMakeLists.txt | 2 + components/platform/include/platform.h | 12 + components/platform/src/platform_mic_key.c | 126 ++++++++++ dependencies.lock | 27 ++- main/idf_component.yml | 1 + 19 files changed, 891 insertions(+), 224 deletions(-) delete mode 100644 .vscode/settings.json create mode 100644 components/domain/src/voice_interaction_mic_key.c create mode 100644 components/platform/src/platform_mic_key.c diff --git a/.gitignore b/.gitignore index eef899c..74109bd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,9 @@ sdkconfig* # Local editor/OS/cache noise .DS_Store +.vscode/ __pycache__/ .pytest_cache/ managed_components -JX_2R_01测试程序_STM32F103RF \ No newline at end of file +JX_2R_01测试程序_STM32F103RF diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 235fa80..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "idf.currentSetup": "/Users/moyyang/esp/v5.5.2/esp-idf", - "idf.flashType": "UART", - "idf.port": "/dev/tty.usbserial-110", - "idf.openOcdConfigs": [ - "board/esp32s3-builtin.cfg" - ], - "idf.customExtraVars": { - "IDF_TARGET": "esp32s3" - } -} \ No newline at end of file diff --git a/README.md b/README.md index 93f31d7..e45cf39 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,12 @@ ESP32-S3 works as a Wi-Fi REST controller and replaces Android App logic: ## Features - Wi-Fi STA-only mode (configured SSID/password; no SoftAP fallback) -- Built-in minimal web UI at `/` for voice tap2talk +- Built-in minimal web UI at `/` for voice status and MIC_KEY guidance - Direct thermal printer control with sensor precheck - Async print queue with jobs (`queued/running/success/failed/canceled`) - Printer precheck (paper / battery / temperature) - Built-in UTF-8 text rendering with Chinese support (GB2312 character set, 16x16 bitmap) -- Tap2talk multimodal voice interaction over DashScope WebSocket: +- Push2talk multimodal voice interaction over DashScope WebSocket: - ES8311 microphone/speaker via `esp_codec_dev` - raw-opus uplink/downlink via `esp_audio_codec` - WebSocket transport via `esp_websocket_client` @@ -125,7 +125,7 @@ http:/// ``` The page provides a minimal voice console: -- single-round tap2talk button +- MIC_KEY(IO14) trigger guide (press to talk, release to process) - voice status display ## REST Examples @@ -205,20 +205,20 @@ curl http:///v1/jobs curl http:///v1/jobs/1 ``` -### Voice tap2talk +### Voice push2talk Start session: ```bash curl -X POST http:///v1/voice/session/start ``` -Start one tap2talk round: +Press-to-talk start (sends `SendSpeech` when entering `Listening`): ```bash curl -X POST http:///v1/voice/tap/start ``` -Cancel current tap2talk round: +Release-to-stop (sends `StopSpeech`): ```bash curl -X POST http:///v1/voice/tap/cancel @@ -244,7 +244,8 @@ curl -X POST http:///v1/voice/session/stop - Characters outside embedded glyph set are rendered as square fallback boxes. - Large buffers for image upload/decode prefer PSRAM first, then fallback to internal RAM. - Partition table uses `partitions.csv` with a 4MB `factory` app partition on 16MB flash modules. -- Voice flow is now single-round: audio uplink stops after `SpeechEnded`, then `LocalRespondingEnded` and `Stop` are sent after local playback drain. +- Voice flow uses `push2talk`: client sends `SendSpeech` to start uplink and `StopSpeech` on release to end input. +- Voice flow is now single-round: after recognition/response completes, `LocalRespondingEnded` and `Stop` are sent after local playback drain. - Voice flow blocks starting the next round while voice is Thinking/Responding, marker image generation is running, or printer queue is still busy. ## Font Assets diff --git a/components/control_plane/src/rest_server.c b/components/control_plane/src/rest_server.c index 08a31cc..ec271a7 100644 --- a/components/control_plane/src/rest_server.c +++ b/components/control_plane/src/rest_server.c @@ -21,29 +21,22 @@ static const char *s_web_index = " .box { max-width: 760px; margin: 0 auto; background:#fff; border:1px solid #dfe6ee; border-radius:12px; padding:16px; }\n" " h1 { margin: 0 0 8px 0; font-size: 22px; }\n" " p { margin: 6px 0 12px 0; color:#4b5d73; }\n" - " .row { display:flex; gap:8px; flex-wrap:wrap; margin: 8px 0; }\n" - " button { border:0; border-radius:8px; padding:10px 14px; cursor:pointer; background:#198754; color:#fff; font: inherit; min-width:220px; }\n" - " button.active { background:#d68910; }\n" - " button:disabled { background:#5f6f82; cursor:default; }\n" + " .hint { font-size: 13px; color:#29583f; background:#e7f4ec; border:1px solid #cbe4d5; border-radius:8px; padding:10px; margin: 8px 0 12px 0; }\n" " .small { font-size: 13px; color:#44566c; }\n" " \n" "\n" "\n" "
\n" "

TQ Voice Console

\n" - "

ESP32-S3 本地语音控制页。用于单轮语音对话（Tap2Talk）。

\n" + "

ESP32-S3 本地语音控制页。已切换为 MIC_KEY（IO14）触发单轮语音对话。

\n" "\n" - "
\n" - " \n" - "
\n" + "
按住机身 MIC_KEY 说话，松开后自动进入“语音交互→生图→显示→处理→打印”。
\n" "
语音状态：初始化中
\n" "
\n" "\n" "