意图识别

main/Kconfig.projbuild

@@ -84,7 +84,7 @@ config TQ_VOICE_WORKSPACE_ID
 
 config TQ_VOICE_APP_ID
     string "Voice app_id"
-    default "fe83b756f6634dcc842c6e7f356c0cf8"
+    default "106a67b690db4686a5d6b40d119653fa"
 
 config TQ_VOICE_USER_ID
     string "Voice client user_id"

main/domain/internal/voice_interaction_internal.h

@@ -64,6 +64,7 @@ typedef struct {
     bool tap_active;
     bool stop_requested;
     bool ws_low_stack_warned;
+    bool response_first_sentence_checked;
 
     voice_dialog_state_t dialog_state;
 

main/domain/src/voice_interaction.c

@@ -101,6 +101,7 @@ esp_err_t voice_interaction_start(char *err, size_t err_len) {
         s_voice.stop_requested = false;
         s_voice.tap_active = false;
         s_voice.ws_low_stack_warned = false;
+        s_voice.response_first_sentence_checked = false;
         s_voice.uplink_task_with_caps = false;
         s_voice.heartbeat_task_with_caps = false;
         s_voice.last_event_ms = voice_now_ms();
@@ -326,6 +327,7 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) {
         s_voice.stop_requested = false;
         s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
         s_voice.dialog_id[0] = '\0';
+        s_voice.response_first_sentence_checked = false;
         s_voice.last_downstream_ms = 0;
         s_voice.playback_deadline_ms = 0;
         voice_unlock();

main/domain/src/voice_interaction_ws.c

@@ -1,5 +1,6 @@
 #include "voice_interaction_internal.h"
 
+#include <ctype.h>
 #include <inttypes.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -9,10 +10,21 @@
 #include "esp_audio_dec.h"
 #include "esp_audio_enc.h"
 #include "esp_log.h"
+#include "image_generation.h"
 #include "voice_audio.h"
 
 static const char *TAG = "voice_interaction";
 
+#define VOICE_IMAGE_MARKER_PROMPT_MAX_LEN 800
+#define VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN 1024
+#define VOICE_IMAGE_GEN_TASK_STACK 16384
+#define VOICE_IMAGE_GEN_TASK_PRIO 3
+
+typedef struct {
+    char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1];
+    char dialog_id[40];
+} voice_image_generation_task_arg_t;
+
 static char *voice_build_directive_message(const char *action,
                                            const char *directive,
                                            bool with_dialog_id) {
@@ -274,6 +286,220 @@ static void voice_log_final_text(const char *stage, const char *text) {
              (len > (size_t)preview) ? "..." : "");
 }
 
+static bool voice_is_space_char(char c) {
+    return isspace((unsigned char)c) != 0;
+}
+
+static size_t voice_find_first_sentence_end(const char *text, size_t len) {
+    if (text == NULL || len == 0) {
+        return 0;
+    }
+
+    for (size_t i = 0; i < len; ++i) {
+        unsigned char c = (unsigned char)text[i];
+        if (c == '.' || c == '!' || c == '?' || c == '\n' || c == '\r') {
+            return i + 1;
+        }
+
+        if (i + 2 < len) {
+            unsigned char b1 = (unsigned char)text[i];
+            unsigned char b2 = (unsigned char)text[i + 1];
+            unsigned char b3 = (unsigned char)text[i + 2];
+
+            if (b1 == 0xE3 && b2 == 0x80 && b3 == 0x82) {
+                return i + 3;
+            }
+            if (b1 == 0xEF && b2 == 0xBC &&
+                (b3 == 0x81 || b3 == 0x9F || b3 == 0x9B)) {
+                return i + 3;
+            }
+        }
+    }
+
+    return len;
+}
+
+static bool voice_extract_first_sentence_text(const char *text,
+                                              char *out_sentence,
+                                              size_t out_len) {
+    if (text == NULL || out_sentence == NULL || out_len == 0) {
+        return false;
+    }
+
+    size_t len = strlen(text);
+    size_t begin = 0;
+    while (begin < len && voice_is_space_char(text[begin])) {
+        begin++;
+    }
+    if (begin >= len) {
+        return false;
+    }
+
+    size_t end = begin + voice_find_first_sentence_end(text + begin, len - begin);
+    while (end > begin && voice_is_space_char(text[end - 1])) {
+        end--;
+    }
+    if (end <= begin) {
+        return false;
+    }
+
+    size_t copy_len = end - begin;
+    if (copy_len >= out_len) {
+        copy_len = out_len - 1;
+    }
+    memcpy(out_sentence, text + begin, copy_len);
+    out_sentence[copy_len] = '\0';
+    return copy_len > 0;
+}
+
+static bool voice_extract_marker_prompt(const char *text, char *out_prompt, size_t out_len) {
+    if (text == NULL || out_prompt == NULL || out_len == 0) {
+        return false;
+    }
+
+    const char *open = strstr(text, "/*");
+    if (open == NULL) {
+        return false;
+    }
+
+    const char *close = strstr(open + 2, "*/");
+    if (close == NULL) {
+        return false;
+    }
+
+    const char *begin = open + 2;
+    const char *end = close;
+    while (begin < end && voice_is_space_char(*begin)) {
+        begin++;
+    }
+    while (end > begin && voice_is_space_char(*(end - 1))) {
+        end--;
+    }
+    if (end <= begin) {
+        return false;
+    }
+
+    size_t prompt_len = (size_t)(end - begin);
+    if (prompt_len >= out_len) {
+        prompt_len = out_len - 1;
+    }
+    memcpy(out_prompt, begin, prompt_len);
+    out_prompt[prompt_len] = '\0';
+    return prompt_len > 0;
+}
+
+static void voice_image_generation_task(void *arg) {
+    voice_image_generation_task_arg_t *task_arg = (voice_image_generation_task_arg_t *)arg;
+    if (task_arg == NULL) {
+        vTaskDelete(NULL);
+        return;
+    }
+
+    image_generation_request_t req = {
+        .prompt = task_arg->prompt,
+        .size = NULL,
+        .prompt_extend = false,
+        .has_seed = false,
+        .seed = 0,
+        .generation_timeout_ms = 0,
+        .download_timeout_ms = 0,
+    };
+
+    image_generation_result_t result = {0};
+    image_generation_result_reset(&result);
+
+    char gen_err[160] = {0};
+    ESP_LOGI(TAG,
+             "[stage] marker_image_gen_start: dialog_id=%s prompt_len=%u",
+             task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
+             (unsigned)strlen(task_arg->prompt));
+    esp_err_t rc = image_generation_generate_png(&req, &result, gen_err, sizeof(gen_err));
+    if (rc == ESP_OK) {
+        ESP_LOGI(TAG,
+                 "[stage] marker_image_gen_done: dialog_id=%s request_id=%s size=%ux%u png_bytes=%u",
+                 task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
+                 result.request_id[0] != '\0' ? result.request_id : "-",
+                 (unsigned)result.width,
+                 (unsigned)result.height,
+                 (unsigned)result.png_len);
+    } else {
+        ESP_LOGW(TAG,
+                 "[stage] marker_image_gen_failed: dialog_id=%s rc=0x%x msg=%s",
+                 task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
+                 (unsigned)rc,
+                 gen_err[0] != '\0' ? gen_err : "image generation failed");
+    }
+
+    image_generation_result_free(&result);
+    image_generation_schedule_prewarm();
+    free(task_arg);
+    vTaskDelete(NULL);
+}
+
+static void voice_schedule_marker_image_generation(const char *prompt, const char *dialog_id) {
+    if (prompt == NULL || prompt[0] == '\0') {
+        return;
+    }
+
+    size_t prompt_len = strlen(prompt);
+    if (prompt_len > VOICE_IMAGE_MARKER_PROMPT_MAX_LEN) {
+        ESP_LOGW(TAG,
+                 "[stage] marker_image_gen_skip: prompt too long, len=%u",
+                 (unsigned)prompt_len);
+        return;
+    }
+
+    voice_image_generation_task_arg_t *task_arg =
+        (voice_image_generation_task_arg_t *)calloc(1, sizeof(*task_arg));
+    if (task_arg == NULL) {
+        ESP_LOGW(TAG, "[stage] marker_image_gen_skip: no memory for task arg");
+        return;
+    }
+
+    strlcpy(task_arg->prompt, prompt, sizeof(task_arg->prompt));
+    if (dialog_id != NULL) {
+        strlcpy(task_arg->dialog_id, dialog_id, sizeof(task_arg->dialog_id));
+    }
+
+    BaseType_t ok = xTaskCreate(voice_image_generation_task,
+                                "voice_img_gen",
+                                VOICE_IMAGE_GEN_TASK_STACK,
+                                task_arg,
+                                VOICE_IMAGE_GEN_TASK_PRIO,
+                                NULL);
+    if (ok != pdPASS) {
+        ESP_LOGW(TAG, "[stage] marker_image_gen_skip: create task failed");
+        free(task_arg);
+        return;
+    }
+
+    ESP_LOGI(TAG,
+             "[stage] marker_image_gen_queued: dialog_id=%s prompt=%s",
+             task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
+             task_arg->prompt);
+}
+
+static void voice_try_trigger_marker_image_generation(const char *dialog_id, const char *final_text) {
+    if (final_text == NULL || final_text[0] == '\0') {
+        return;
+    }
+
+    char first_sentence[VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN] = {0};
+    if (!voice_extract_first_sentence_text(final_text, first_sentence, sizeof(first_sentence))) {
+        return;
+    }
+
+    char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0};
+    if (!voice_extract_marker_prompt(first_sentence, prompt, sizeof(prompt))) {
+        return;
+    }
+
+    ESP_LOGI(TAG,
+             "[stage] marker_image_gen_detected: first_sentence=%s",
+             first_sentence);
+    voice_schedule_marker_image_generation(prompt, dialog_id);
+}
+
 static void voice_handle_output_event(cJSON *output) {
     const char *event_name = voice_get_json_str(output, "event");
     if (event_name == NULL) {
@@ -293,6 +519,8 @@ static void voice_handle_output_event(cJSON *output) {
 
     bool finished = false;
     bool has_finished = voice_get_json_bool(output, "finished", &finished);
+    const char *responding_final_text = NULL;
+    bool should_check_marker_in_first_sentence = false;
     if (has_finished && finished) {
         if (strcmp(event_name, "SpeechContent") == 0) {
             voice_log_final_text("asr_final_text", voice_get_json_str(output, "text"));
@@ -302,6 +530,15 @@ static void voice_handle_output_event(cJSON *output) {
                 final_text = voice_get_json_str(output, "spoken");
             }
             voice_log_final_text("response_final_text", final_text);
+            responding_final_text = final_text;
+            if (responding_final_text != NULL && responding_final_text[0] != '\0' &&
+                voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
+                if (!s_voice.response_first_sentence_checked) {
+                    s_voice.response_first_sentence_checked = true;
+                    should_check_marker_in_first_sentence = true;
+                }
+                voice_unlock();
+            }
         }
     }
 
@@ -315,15 +552,26 @@ static void voice_handle_output_event(cJSON *output) {
         if (strcmp(event_name, "Started") == 0) {
             s_voice.started = true;
             s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
+            s_voice.response_first_sentence_checked = false;
             ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id);
         } else if (strcmp(event_name, "DialogStateChanged") == 0) {
             if (state != NULL) {
+                voice_dialog_state_t prev_state = s_voice.dialog_state;
                 if (strcmp(state, "Listening") == 0) {
                     s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING;
+                    if (prev_state != VOICE_DIALOG_STATE_LISTENING) {
+                        s_voice.response_first_sentence_checked = false;
+                    }
                 } else if (strcmp(state, "Thinking") == 0) {
                     s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING;
+                    if (prev_state != VOICE_DIALOG_STATE_THINKING) {
+                        s_voice.response_first_sentence_checked = false;
+                    }
                 } else if (strcmp(state, "Responding") == 0) {
                     s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING;
+                    if (prev_state != VOICE_DIALOG_STATE_RESPONDING) {
+                        s_voice.response_first_sentence_checked = false;
+                    }
                 }
             }
             ESP_LOGI(TAG,
@@ -336,6 +584,7 @@ static void voice_handle_output_event(cJSON *output) {
             s_voice.started = false;
             s_voice.tap_active = false;
             s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
+            s_voice.response_first_sentence_checked = false;
             should_close_audio = true;
             ESP_LOGI(TAG, "[stage] session_stopped_by_server");
         } else if (strcmp(event_name, "Error") == 0) {
@@ -346,6 +595,7 @@ static void voice_handle_output_event(cJSON *output) {
             }
             s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
             s_voice.tap_active = false;
+            s_voice.response_first_sentence_checked = false;
             should_close_audio = true;
         }
         voice_unlock();
@@ -355,6 +605,10 @@ static void voice_handle_output_event(cJSON *output) {
         voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close");
     }
 
+    if (should_check_marker_in_first_sentence) {
+        voice_try_trigger_marker_image_generation(dialog_id, responding_final_text);
+    }
+
     if (strcmp(event_name, "RespondingStarted") == 0) {
         (void)voice_send_directive("continue-task", "LocalRespondingStarted", true);
     } else if (strcmp(event_name, "RespondingEnded") == 0) {

sdkconfig.defaults

@@ -43,7 +43,7 @@ CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM=y
 CONFIG_TQ_VOICE_API_KEY="sk-7a50eca6856d4afb968ac3bf512f6d1b"
 CONFIG_TQ_VOICE_WS_URI="wss://dashscope.aliyuncs.com/api-ws/v1/inference"
 CONFIG_TQ_VOICE_WORKSPACE_ID="llm-igyrxzdvbwja5gyt"
-CONFIG_TQ_VOICE_APP_ID="fe83b756f6634dcc842c6e7f356c0cf8"
+CONFIG_TQ_VOICE_APP_ID="106a67b690db4686a5d6b40d119653fa"
 CONFIG_TQ_VOICE_USER_ID="esp32-user"
 CONFIG_TQ_VOICE_DEVICE_UUID=""
 CONFIG_TQ_VOICE_TTS_VOICE="longling_v3"