diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index c6866ab..742fc7e 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -84,7 +84,7 @@ config TQ_VOICE_WORKSPACE_ID config TQ_VOICE_APP_ID string "Voice app_id" - default "fe83b756f6634dcc842c6e7f356c0cf8" + default "106a67b690db4686a5d6b40d119653fa" config TQ_VOICE_USER_ID string "Voice client user_id" diff --git a/main/domain/internal/voice_interaction_internal.h b/main/domain/internal/voice_interaction_internal.h index 4bee06e..aef072b 100644 --- a/main/domain/internal/voice_interaction_internal.h +++ b/main/domain/internal/voice_interaction_internal.h @@ -64,6 +64,7 @@ typedef struct { bool tap_active; bool stop_requested; bool ws_low_stack_warned; + bool response_first_sentence_checked; voice_dialog_state_t dialog_state; diff --git a/main/domain/src/voice_interaction.c b/main/domain/src/voice_interaction.c index eec83cc..0e6443c 100644 --- a/main/domain/src/voice_interaction.c +++ b/main/domain/src/voice_interaction.c @@ -101,6 +101,7 @@ esp_err_t voice_interaction_start(char *err, size_t err_len) { s_voice.stop_requested = false; s_voice.tap_active = false; s_voice.ws_low_stack_warned = false; + s_voice.response_first_sentence_checked = false; s_voice.uplink_task_with_caps = false; s_voice.heartbeat_task_with_caps = false; s_voice.last_event_ms = voice_now_ms(); @@ -326,6 +327,7 @@ esp_err_t voice_interaction_stop(char *err, size_t err_len) { s_voice.stop_requested = false; s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; s_voice.dialog_id[0] = '\0'; + s_voice.response_first_sentence_checked = false; s_voice.last_downstream_ms = 0; s_voice.playback_deadline_ms = 0; voice_unlock(); diff --git a/main/domain/src/voice_interaction_ws.c b/main/domain/src/voice_interaction_ws.c index a261a89..979f918 100644 --- a/main/domain/src/voice_interaction_ws.c +++ b/main/domain/src/voice_interaction_ws.c @@ -1,5 +1,6 @@ #include "voice_interaction_internal.h" +#include #include #include #include @@ -9,10 +10,21 @@ #include "esp_audio_dec.h" #include "esp_audio_enc.h" #include "esp_log.h" +#include "image_generation.h" #include "voice_audio.h" static const char *TAG = "voice_interaction"; +#define VOICE_IMAGE_MARKER_PROMPT_MAX_LEN 800 +#define VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN 1024 +#define VOICE_IMAGE_GEN_TASK_STACK 16384 +#define VOICE_IMAGE_GEN_TASK_PRIO 3 + +typedef struct { + char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1]; + char dialog_id[40]; +} voice_image_generation_task_arg_t; + static char *voice_build_directive_message(const char *action, const char *directive, bool with_dialog_id) { @@ -274,6 +286,220 @@ static void voice_log_final_text(const char *stage, const char *text) { (len > (size_t)preview) ? "..." : ""); } +static bool voice_is_space_char(char c) { + return isspace((unsigned char)c) != 0; +} + +static size_t voice_find_first_sentence_end(const char *text, size_t len) { + if (text == NULL || len == 0) { + return 0; + } + + for (size_t i = 0; i < len; ++i) { + unsigned char c = (unsigned char)text[i]; + if (c == '.' || c == '!' || c == '?' || c == '\n' || c == '\r') { + return i + 1; + } + + if (i + 2 < len) { + unsigned char b1 = (unsigned char)text[i]; + unsigned char b2 = (unsigned char)text[i + 1]; + unsigned char b3 = (unsigned char)text[i + 2]; + + if (b1 == 0xE3 && b2 == 0x80 && b3 == 0x82) { + return i + 3; + } + if (b1 == 0xEF && b2 == 0xBC && + (b3 == 0x81 || b3 == 0x9F || b3 == 0x9B)) { + return i + 3; + } + } + } + + return len; +} + +static bool voice_extract_first_sentence_text(const char *text, + char *out_sentence, + size_t out_len) { + if (text == NULL || out_sentence == NULL || out_len == 0) { + return false; + } + + size_t len = strlen(text); + size_t begin = 0; + while (begin < len && voice_is_space_char(text[begin])) { + begin++; + } + if (begin >= len) { + return false; + } + + size_t end = begin + voice_find_first_sentence_end(text + begin, len - begin); + while (end > begin && voice_is_space_char(text[end - 1])) { + end--; + } + if (end <= begin) { + return false; + } + + size_t copy_len = end - begin; + if (copy_len >= out_len) { + copy_len = out_len - 1; + } + memcpy(out_sentence, text + begin, copy_len); + out_sentence[copy_len] = '\0'; + return copy_len > 0; +} + +static bool voice_extract_marker_prompt(const char *text, char *out_prompt, size_t out_len) { + if (text == NULL || out_prompt == NULL || out_len == 0) { + return false; + } + + const char *open = strstr(text, "/*"); + if (open == NULL) { + return false; + } + + const char *close = strstr(open + 2, "*/"); + if (close == NULL) { + return false; + } + + const char *begin = open + 2; + const char *end = close; + while (begin < end && voice_is_space_char(*begin)) { + begin++; + } + while (end > begin && voice_is_space_char(*(end - 1))) { + end--; + } + if (end <= begin) { + return false; + } + + size_t prompt_len = (size_t)(end - begin); + if (prompt_len >= out_len) { + prompt_len = out_len - 1; + } + memcpy(out_prompt, begin, prompt_len); + out_prompt[prompt_len] = '\0'; + return prompt_len > 0; +} + +static void voice_image_generation_task(void *arg) { + voice_image_generation_task_arg_t *task_arg = (voice_image_generation_task_arg_t *)arg; + if (task_arg == NULL) { + vTaskDelete(NULL); + return; + } + + image_generation_request_t req = { + .prompt = task_arg->prompt, + .size = NULL, + .prompt_extend = false, + .has_seed = false, + .seed = 0, + .generation_timeout_ms = 0, + .download_timeout_ms = 0, + }; + + image_generation_result_t result = {0}; + image_generation_result_reset(&result); + + char gen_err[160] = {0}; + ESP_LOGI(TAG, + "[stage] marker_image_gen_start: dialog_id=%s prompt_len=%u", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)strlen(task_arg->prompt)); + esp_err_t rc = image_generation_generate_png(&req, &result, gen_err, sizeof(gen_err)); + if (rc == ESP_OK) { + ESP_LOGI(TAG, + "[stage] marker_image_gen_done: dialog_id=%s request_id=%s size=%ux%u png_bytes=%u", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + result.request_id[0] != '\0' ? result.request_id : "-", + (unsigned)result.width, + (unsigned)result.height, + (unsigned)result.png_len); + } else { + ESP_LOGW(TAG, + "[stage] marker_image_gen_failed: dialog_id=%s rc=0x%x msg=%s", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)rc, + gen_err[0] != '\0' ? gen_err : "image generation failed"); + } + + image_generation_result_free(&result); + image_generation_schedule_prewarm(); + free(task_arg); + vTaskDelete(NULL); +} + +static void voice_schedule_marker_image_generation(const char *prompt, const char *dialog_id) { + if (prompt == NULL || prompt[0] == '\0') { + return; + } + + size_t prompt_len = strlen(prompt); + if (prompt_len > VOICE_IMAGE_MARKER_PROMPT_MAX_LEN) { + ESP_LOGW(TAG, + "[stage] marker_image_gen_skip: prompt too long, len=%u", + (unsigned)prompt_len); + return; + } + + voice_image_generation_task_arg_t *task_arg = + (voice_image_generation_task_arg_t *)calloc(1, sizeof(*task_arg)); + if (task_arg == NULL) { + ESP_LOGW(TAG, "[stage] marker_image_gen_skip: no memory for task arg"); + return; + } + + strlcpy(task_arg->prompt, prompt, sizeof(task_arg->prompt)); + if (dialog_id != NULL) { + strlcpy(task_arg->dialog_id, dialog_id, sizeof(task_arg->dialog_id)); + } + + BaseType_t ok = xTaskCreate(voice_image_generation_task, + "voice_img_gen", + VOICE_IMAGE_GEN_TASK_STACK, + task_arg, + VOICE_IMAGE_GEN_TASK_PRIO, + NULL); + if (ok != pdPASS) { + ESP_LOGW(TAG, "[stage] marker_image_gen_skip: create task failed"); + free(task_arg); + return; + } + + ESP_LOGI(TAG, + "[stage] marker_image_gen_queued: dialog_id=%s prompt=%s", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + task_arg->prompt); +} + +static void voice_try_trigger_marker_image_generation(const char *dialog_id, const char *final_text) { + if (final_text == NULL || final_text[0] == '\0') { + return; + } + + char first_sentence[VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN] = {0}; + if (!voice_extract_first_sentence_text(final_text, first_sentence, sizeof(first_sentence))) { + return; + } + + char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0}; + if (!voice_extract_marker_prompt(first_sentence, prompt, sizeof(prompt))) { + return; + } + + ESP_LOGI(TAG, + "[stage] marker_image_gen_detected: first_sentence=%s", + first_sentence); + voice_schedule_marker_image_generation(prompt, dialog_id); +} + static void voice_handle_output_event(cJSON *output) { const char *event_name = voice_get_json_str(output, "event"); if (event_name == NULL) { @@ -293,6 +519,8 @@ static void voice_handle_output_event(cJSON *output) { bool finished = false; bool has_finished = voice_get_json_bool(output, "finished", &finished); + const char *responding_final_text = NULL; + bool should_check_marker_in_first_sentence = false; if (has_finished && finished) { if (strcmp(event_name, "SpeechContent") == 0) { voice_log_final_text("asr_final_text", voice_get_json_str(output, "text")); @@ -302,6 +530,15 @@ static void voice_handle_output_event(cJSON *output) { final_text = voice_get_json_str(output, "spoken"); } voice_log_final_text("response_final_text", final_text); + responding_final_text = final_text; + if (responding_final_text != NULL && responding_final_text[0] != '\0' && + voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + if (!s_voice.response_first_sentence_checked) { + s_voice.response_first_sentence_checked = true; + should_check_marker_in_first_sentence = true; + } + voice_unlock(); + } } } @@ -315,15 +552,26 @@ static void voice_handle_output_event(cJSON *output) { if (strcmp(event_name, "Started") == 0) { s_voice.started = true; s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + s_voice.response_first_sentence_checked = false; ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id); } else if (strcmp(event_name, "DialogStateChanged") == 0) { if (state != NULL) { + voice_dialog_state_t prev_state = s_voice.dialog_state; if (strcmp(state, "Listening") == 0) { s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING; + if (prev_state != VOICE_DIALOG_STATE_LISTENING) { + s_voice.response_first_sentence_checked = false; + } } else if (strcmp(state, "Thinking") == 0) { s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING; + if (prev_state != VOICE_DIALOG_STATE_THINKING) { + s_voice.response_first_sentence_checked = false; + } } else if (strcmp(state, "Responding") == 0) { s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING; + if (prev_state != VOICE_DIALOG_STATE_RESPONDING) { + s_voice.response_first_sentence_checked = false; + } } } ESP_LOGI(TAG, @@ -336,6 +584,7 @@ static void voice_handle_output_event(cJSON *output) { s_voice.started = false; s_voice.tap_active = false; s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + s_voice.response_first_sentence_checked = false; should_close_audio = true; ESP_LOGI(TAG, "[stage] session_stopped_by_server"); } else if (strcmp(event_name, "Error") == 0) { @@ -346,6 +595,7 @@ static void voice_handle_output_event(cJSON *output) { } s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; s_voice.tap_active = false; + s_voice.response_first_sentence_checked = false; should_close_audio = true; } voice_unlock(); @@ -355,6 +605,10 @@ static void voice_handle_output_event(cJSON *output) { voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close"); } + if (should_check_marker_in_first_sentence) { + voice_try_trigger_marker_image_generation(dialog_id, responding_final_text); + } + if (strcmp(event_name, "RespondingStarted") == 0) { (void)voice_send_directive("continue-task", "LocalRespondingStarted", true); } else if (strcmp(event_name, "RespondingEnded") == 0) { diff --git a/sdkconfig.defaults b/sdkconfig.defaults index f3c6702..9bb93d3 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -43,7 +43,7 @@ CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM=y CONFIG_TQ_VOICE_API_KEY="sk-7a50eca6856d4afb968ac3bf512f6d1b" CONFIG_TQ_VOICE_WS_URI="wss://dashscope.aliyuncs.com/api-ws/v1/inference" CONFIG_TQ_VOICE_WORKSPACE_ID="llm-igyrxzdvbwja5gyt" -CONFIG_TQ_VOICE_APP_ID="fe83b756f6634dcc842c6e7f356c0cf8" +CONFIG_TQ_VOICE_APP_ID="106a67b690db4686a5d6b40d119653fa" CONFIG_TQ_VOICE_USER_ID="esp32-user" CONFIG_TQ_VOICE_DEVICE_UUID="" CONFIG_TQ_VOICE_TTS_VOICE="longling_v3"