From 6780bb278908d50f59af60fa80c583b44d92c716 Mon Sep 17 00:00:00 2001 From: admin Date: Mon, 2 Mar 2026 03:52:10 +0800 Subject: [PATCH] feat(voice): improve on-screen dialog text and wifi provisioning hints --- components/domain/include/domain.h | 4 + components/domain/src/raster_tools.c | 13 +- components/domain/src/screen_preview.c | 57 ++++++++ components/domain/src/system_runtime.c | 58 ++++++++ .../src/voice_interaction_ws_protocol.c | 125 +++++++++++++++++- components/platform/include/platform.h | 8 ++ components/platform/src/wifi_manager.c | 22 +++ 7 files changed, 283 insertions(+), 4 deletions(-) diff --git a/components/domain/include/domain.h b/components/domain/include/domain.h index bbcaa13..e47b0b7 100644 --- a/components/domain/include/domain.h +++ b/components/domain/include/domain.h @@ -115,6 +115,10 @@ esp_err_t screen_preview_show_png(const uint8_t *png, uint32_t timeout_ms, char *err, size_t err_len); +esp_err_t screen_preview_show_text(const char *text, + uint32_t timeout_ms, + char *err, + size_t err_len); // ---------- printer_protocol ---------- typedef enum { diff --git a/components/domain/src/raster_tools.c b/components/domain/src/raster_tools.c index f2a7092..df62528 100644 --- a/components/domain/src/raster_tools.c +++ b/components/domain/src/raster_tools.c @@ -125,6 +125,13 @@ static void set_black(uint8_t *buf, uint16_t width, uint16_t x, uint16_t y) { buf[idx] |= (uint8_t)(1u << bit); } +static void set_black_bold(uint8_t *buf, uint16_t width, uint16_t x, uint16_t y) { + set_black(buf, width, x, y); + if ((uint16_t)(x + 1) < width) { + set_black(buf, width, (uint16_t)(x + 1), y); + } +} + static size_t cn16_index_count(void) { size_t bytes = (size_t)(_binary_cn16_index_bin_end - _binary_cn16_index_bin_start); return bytes / CN16_INDEX_ENTRY_BYTES; @@ -211,7 +218,7 @@ static void draw_char(uint8_t *buf, uint16_t px = (uint16_t)(x + col * scale + sx); uint16_t py = (uint16_t)(y + row * scale + sy); if (py < height) { - set_black(buf, width, px, py); + set_black_bold(buf, width, px, py); } } } @@ -241,7 +248,7 @@ static void draw_cn16_glyph(uint8_t *buf, uint16_t px = (uint16_t)(x + col * scale + sx); uint16_t py = (uint16_t)(y + row * scale + sy); if (py < height && px < width) { - set_black(buf, width, px, py); + set_black_bold(buf, width, px, py); } } } @@ -265,7 +272,7 @@ static void draw_missing_box(uint8_t *buf, uint16_t px = (uint16_t)(x + c); uint16_t py = (uint16_t)(y + r); if (px < width && py < height) { - set_black(buf, width, px, py); + set_black_bold(buf, width, px, py); } } } diff --git a/components/domain/src/screen_preview.c b/components/domain/src/screen_preview.c index 961b3aa..6740bef 100644 --- a/components/domain/src/screen_preview.c +++ b/components/domain/src/screen_preview.c @@ -1,6 +1,13 @@ #include "domain.h" #include "platform.h" +#include +#include + +#define SCREEN_PREVIEW_TEXT_SCALE 2 +#define SCREEN_PREVIEW_TEXT_LINE_SPACING 1 +#define SCREEN_PREVIEW_TEXT_MAX_HEIGHT 3000 + esp_err_t screen_preview_show_raster(const uint8_t *raster, size_t raster_len, uint16_t width, @@ -20,3 +27,53 @@ esp_err_t screen_preview_show_png(const uint8_t *png, { return platform_display_show_png(png, png_len, timeout_ms, err, err_len); } + +esp_err_t screen_preview_show_text(const char *text, + uint32_t timeout_ms, + char *err, + size_t err_len) +{ + if (text == NULL || text[0] == '\0') { + if (err != NULL && err_len > 0) { + strlcpy(err, "invalid text", err_len); + } + return ESP_ERR_INVALID_ARG; + } + + uint16_t width = 0; + uint16_t height = 0; + uint8_t *raster = NULL; + size_t raster_len = 0; + char render_err[96] = {0}; + esp_err_t render_rc = raster_tools_render_text_384(text, + SCREEN_PREVIEW_TEXT_SCALE, + SCREEN_PREVIEW_TEXT_LINE_SPACING, + SCREEN_PREVIEW_TEXT_MAX_HEIGHT, + &width, + &height, + &raster, + &raster_len, + render_err, + sizeof(render_err)); + if (render_rc != ESP_OK) { + if (err != NULL && err_len > 0 && render_err[0] != '\0') { + strlcpy(err, render_err, err_len); + } + return render_rc; + } + + char preview_err[96] = {0}; + esp_err_t preview_rc = screen_preview_show_raster(raster, + raster_len, + width, + height, + timeout_ms, + preview_err, + sizeof(preview_err)); + free(raster); + + if (preview_rc != ESP_OK && err != NULL && err_len > 0 && preview_err[0] != '\0') { + strlcpy(err, preview_err, err_len); + } + return preview_rc; +} diff --git a/components/domain/src/system_runtime.c b/components/domain/src/system_runtime.c index ad3fd64..dff2e99 100644 --- a/components/domain/src/system_runtime.c +++ b/components/domain/src/system_runtime.c @@ -1,12 +1,69 @@ #include "domain.h" #include "platform.h" +#include + +#include "esp_log.h" + +static const char *TAG = "system_runtime"; + +#define SYSTEM_RUNTIME_WIFI_HINT_TIMEOUT_MS 220 + +static bool s_wifi_hint_shown; + +static void system_runtime_show_wifi_hint(void) { + if (s_wifi_hint_shown) { + return; + } + s_wifi_hint_shown = true; + + const char *ap_ssid = CONFIG_TQ_WIFI_PROV_SOFTAP_SSID; + if (ap_ssid == NULL || ap_ssid[0] == '\0') { + ap_ssid = "TalkingQ-Setup"; + } + + char hint[256] = {0}; + snprintf(hint, + sizeof(hint), + "未检测到Wi-Fi配置\n连接热点: %s\n打开: http://192.168.4.1", + ap_ssid); + + char preview_err[96] = {0}; + esp_err_t preview_rc = screen_preview_show_text(hint, + SYSTEM_RUNTIME_WIFI_HINT_TIMEOUT_MS, + preview_err, + sizeof(preview_err)); + if (preview_rc != ESP_OK && preview_rc != ESP_ERR_NOT_SUPPORTED) { + ESP_LOGW(TAG, + "wifi_hint_preview_failed: rc=0x%x msg=%s", + (unsigned)preview_rc, + preview_err[0] != '\0' ? preview_err : "show text failed"); + } +} + +static void system_runtime_try_show_wifi_hint_for_unprovisioned(void) { + if (!wifi_manager_has_saved_credentials()) { + system_runtime_show_wifi_hint(); + } +} + +static void system_runtime_wifi_event_handler(wifi_manager_event_t event, void *user_data) { + (void)user_data; + if (event == WIFI_MANAGER_EVENT_PROVISIONING_STARTED) { + system_runtime_show_wifi_hint(); + } +} + esp_err_t system_runtime_start(void) { esp_err_t err = platform_bootstrap_init(); if (err != ESP_OK) { return err; } + s_wifi_hint_shown = false; + (void)wifi_manager_set_event_callback(system_runtime_wifi_event_handler, NULL); + system_runtime_try_show_wifi_hint_for_unprovisioned(); + err = wifi_manager_start(); if (err != ESP_OK) { // Reset platform state so lifecycle retry runs from a clean baseline. @@ -18,6 +75,7 @@ esp_err_t system_runtime_start(void) { } esp_err_t system_runtime_shutdown(void) { + (void)wifi_manager_set_event_callback(NULL, NULL); return wifi_manager_stop(); } diff --git a/components/domain/src/voice_interaction_ws_protocol.c b/components/domain/src/voice_interaction_ws_protocol.c index 572c0dc..3d0f72a 100644 --- a/components/domain/src/voice_interaction_ws_protocol.c +++ b/components/domain/src/voice_interaction_ws_protocol.c @@ -1,5 +1,7 @@ #include "voice_interaction_internal.h" +#include +#include #include #define cJSON_malloc voice_malloc_prefer_psram @@ -14,10 +16,99 @@ static const char *TAG = "voice_interaction"; #define VOICE_RESPONSE_FINALIZE_TASK_STACK 6144 #define VOICE_RESPONSE_FINALIZE_TASK_RETRY_COUNT 3 #define VOICE_RESPONSE_FINALIZE_TASK_RETRY_DELAY_MS 120 +#define VOICE_SCREEN_PREVIEW_TIMEOUT_MS 180 static portMUX_TYPE s_response_finalize_lock = portMUX_INITIALIZER_UNLOCKED; static bool s_response_finalize_running; +static bool voice_is_space_char(char c) { + return isspace((unsigned char)c) != 0; +} + +static size_t voice_find_first_sentence_end(const char *text, size_t len) { + if (text == NULL || len == 0) { + return 0; + } + + for (size_t i = 0; i < len; ++i) { + unsigned char c = (unsigned char)text[i]; + if (c == '.' || c == '!' || c == '?' || c == ':' || c == ';' || + c == '\n' || c == '\r') { + return i + 1; + } + + if (i + 2 < len) { + unsigned char b1 = (unsigned char)text[i]; + unsigned char b2 = (unsigned char)text[i + 1]; + unsigned char b3 = (unsigned char)text[i + 2]; + + // UTF-8: 。 (E3 80 82) + if (b1 == 0xE3 && b2 == 0x80 && b3 == 0x82) { + return i + 3; + } + // UTF-8: ！(EF BC 81) ？(EF BC 9F) ；(EF BC 9B) ：(EF BC 9A) + if (b1 == 0xEF && b2 == 0xBC && + (b3 == 0x81 || b3 == 0x9F || b3 == 0x9B || b3 == 0x9A)) { + return i + 3; + } + } + } + + return len; +} + +static const char *voice_get_text_without_first_sentence(const char *text) { + if (text == NULL || text[0] == '\0') { + return NULL; + } + + size_t len = strlen(text); + size_t begin = 0; + while (begin < len && voice_is_space_char(text[begin])) { + begin++; + } + if (begin >= len) { + return NULL; + } + + size_t first_end = begin + voice_find_first_sentence_end(text + begin, len - begin); + while (first_end < len && voice_is_space_char(text[first_end])) { + first_end++; + } + + if (first_end < len) { + return text + first_end; + } + + // Fallback: keep answer body after marker prefix when punctuation split + // does not leave a second sentence. + const char *marker_begin = strstr(text + begin, "/*"); + if (marker_begin == NULL) { + return NULL; + } + + const char *marker_end = strstr(marker_begin + 2, "*/"); + if (marker_end == NULL) { + return NULL; + } + + const char *cursor = marker_end + 2; + while (*cursor != '\0') { + unsigned char ch = (unsigned char)*cursor; + if (!voice_is_space_char((char)ch) && + ch != '.' && ch != '!' && ch != '?' && + ch != ':' && ch != ';' && ch != ',') { + break; + } + cursor++; + } + + if (*cursor == '\0') { + return NULL; + } + return cursor; +} + static esp_err_t voice_send_directive_with_retry(const char *action, const char *directive, bool with_dialog_id) { @@ -136,6 +227,24 @@ static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) { return true; } +static void voice_try_preview_final_text(const char *text) { + if (text == NULL || text[0] == '\0') { + return; + } + + char preview_err[96] = {0}; + esp_err_t preview_rc = screen_preview_show_text(text, + VOICE_SCREEN_PREVIEW_TIMEOUT_MS, + preview_err, + sizeof(preview_err)); + if (preview_rc != ESP_OK && preview_rc != ESP_ERR_NOT_SUPPORTED) { + ESP_LOGW(TAG, + "dialog_text_preview_failed: rc=0x%x msg=%s", + (unsigned)preview_rc, + preview_err[0] != '\0' ? preview_err : "show text failed"); + } +} + static void voice_handle_output_event(cJSON *output) { const char *event_name = voice_get_json_str(output, "event"); if (event_name == NULL) { @@ -150,11 +259,14 @@ static void voice_handle_output_event(cJSON *output) { bool finished = false; bool has_finished = voice_get_json_bool(output, "finished", &finished); + const char *speech_final_text = NULL; const char *responding_final_text = NULL; bool should_check_marker_in_first_sentence = false; bool should_try_send_speech = false; if (has_finished && finished) { - if (strcmp(event_name, "RespondingContent") == 0) { + if (strcmp(event_name, "SpeechContent") == 0) { + speech_final_text = voice_get_json_str(output, "text"); + } else if (strcmp(event_name, "RespondingContent") == 0) { const char *final_text = voice_get_json_str(output, "text"); if (final_text == NULL || final_text[0] == '\0') { final_text = voice_get_json_str(output, "spoken"); @@ -233,6 +345,17 @@ static void voice_handle_output_event(cJSON *output) { voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close"); } + if (speech_final_text != NULL && speech_final_text[0] != '\0') { + voice_try_preview_final_text(speech_final_text); + } + + if (responding_final_text != NULL && responding_final_text[0] != '\0') { + const char *preview_text = voice_get_text_without_first_sentence(responding_final_text); + if (preview_text != NULL && preview_text[0] != '\0') { + voice_try_preview_final_text(preview_text); + } + } + if (should_check_marker_in_first_sentence) { voice_marker_try_trigger_image_generation(dialog_id, responding_final_text); } diff --git a/components/platform/include/platform.h b/components/platform/include/platform.h index 1d17d67..8022f48 100644 --- a/components/platform/include/platform.h +++ b/components/platform/include/platform.h @@ -14,10 +14,18 @@ extern "C" { esp_err_t platform_bootstrap_init(void); // ---------- wifi_manager ---------- +typedef enum { + WIFI_MANAGER_EVENT_PROVISIONING_STARTED = 0, +} wifi_manager_event_t; + +typedef void (*wifi_manager_event_cb_t)(wifi_manager_event_t event, void *user_data); + esp_err_t wifi_manager_start(void); esp_err_t wifi_manager_stop(void); +esp_err_t wifi_manager_set_event_callback(wifi_manager_event_cb_t cb, void *user_data); bool wifi_manager_is_ready(void); +bool wifi_manager_has_saved_credentials(void); void wifi_manager_get_ip(char *buf, size_t buf_len); // ---------- thermal_printer ---------- diff --git a/components/platform/src/wifi_manager.c b/components/platform/src/wifi_manager.c index cbadc1e..7780e57 100644 --- a/components/platform/src/wifi_manager.c +++ b/components/platform/src/wifi_manager.c @@ -59,12 +59,22 @@ static bool s_ready; static bool s_started; static bool s_sta_reconnect_enabled; static bool s_provisioning_active; +static wifi_manager_event_cb_t s_event_cb; +static void *s_event_cb_user_data; static esp_netif_t *s_sta_netif; static esp_netif_t *s_ap_netif; static httpd_handle_t s_prov_httpd; static esp_event_handler_instance_t s_wifi_event_inst; static esp_event_handler_instance_t s_ip_event_inst; +static void wifi_manager_notify_event(wifi_manager_event_t event) { + wifi_manager_event_cb_t cb = s_event_cb; + if (cb == NULL) { + return; + } + cb(event, s_event_cb_user_data); +} + static void wifi_manager_apply_sta_throughput_profile(void) { const uint8_t protocol = WIFI_PROTOCOL_11B | WIFI_PROTOCOL_11G | WIFI_PROTOCOL_11N; esp_err_t proto_rc = esp_wifi_set_protocol(WIFI_IF_STA, protocol); @@ -716,6 +726,7 @@ static esp_err_t wifi_manager_start_softap_provisioning(void) { } s_provisioning_active = true; + wifi_manager_notify_event(WIFI_MANAGER_EVENT_PROVISIONING_STARTED); ESP_LOGI(TAG, "SoftAP provisioning started: ssid=%s password=%s ip=192.168.4.1", @@ -726,6 +737,12 @@ static esp_err_t wifi_manager_start_softap_provisioning(void) { return ESP_OK; } +esp_err_t wifi_manager_set_event_callback(wifi_manager_event_cb_t cb, void *user_data) { + s_event_cb = cb; + s_event_cb_user_data = user_data; + return ESP_OK; +} + static void wifi_manager_stop_softap_provisioning(void) { if (!s_provisioning_active) { return; @@ -942,6 +959,11 @@ bool wifi_manager_is_ready(void) { return s_ready; } +bool wifi_manager_has_saved_credentials(void) { + wifi_sta_credentials_t creds = {0}; + return wifi_manager_load_credentials(&creds) == ESP_OK; +} + void wifi_manager_get_ip(char *buf, size_t buf_len) { if (buf == NULL || buf_len == 0) { return;