diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 9ff3db3..2491619 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -21,6 +21,9 @@ idf_component_register( "domain/src/voice_interaction.c" "domain/src/voice_interaction_common.c" "domain/src/voice_interaction_ws.c" + "domain/src/voice_interaction_ws_protocol.c" + "domain/src/voice_interaction_ws_audio.c" + "domain/src/voice_interaction_marker.c" "domain/src/voice_interaction_tasks.c" "platform/src/platform_bootstrap.c" "platform/src/wifi_manager.c" diff --git a/main/domain/internal/voice_interaction_internal.h b/main/domain/internal/voice_interaction_internal.h index 959a162..731457e 100644 --- a/main/domain/internal/voice_interaction_internal.h +++ b/main/domain/internal/voice_interaction_internal.h @@ -117,6 +117,10 @@ esp_err_t voice_send_directive(const char *action, const char *directive, bool with_dialog_id); +void voice_ws_protocol_handle_text_message(const char *text, size_t len); +void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len); +void voice_marker_try_trigger_image_generation(const char *dialog_id, const char *final_text); + void voice_websocket_event_handler(void *handler_args, esp_event_base_t base, int32_t event_id, diff --git a/main/domain/src/voice_interaction_marker.c b/main/domain/src/voice_interaction_marker.c new file mode 100644 index 0000000..c16dcc9 --- /dev/null +++ b/main/domain/src/voice_interaction_marker.c @@ -0,0 +1,480 @@ +#include "voice_interaction_internal.h" + +#include +#include +#include +#include + +#include "esp_log.h" +#include "image_generation.h" +#include "printer_protocol.h" +#include "raster_tools.h" + +static const char *TAG = "voice_interaction"; + +#define VOICE_IMAGE_MARKER_PROMPT_MAX_LEN 800 +#define VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN 1024 +#define VOICE_IMAGE_GEN_TASK_STACK 16384 +#define VOICE_IMAGE_GEN_TASK_PRIO 3 +#define VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS 45000 +#define VOICE_IMAGE_PRINT_THRESHOLD 160 +#define VOICE_IMAGE_PRINT_MAX_HEIGHT 2200 +#define VOICE_IMAGE_MARKER_TEMPLATE \ + "%s，黑白线稿风格，白色背景，主体居中，边缘清晰，细节简洁，高对比，避免大面积阴影和灰度渐变，适配热敏打印机打印。" +#define VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT "简洁卡通图案" + +typedef struct { + char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1]; + char dialog_id[40]; +} voice_image_generation_task_arg_t; + +static portMUX_TYPE s_marker_image_gen_lock = portMUX_INITIALIZER_UNLOCKED; +static bool s_marker_image_gen_running; +static int64_t s_marker_last_success_ms; +static char s_marker_last_success_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1]; +static char s_marker_last_success_dialog_id[40]; + +static bool voice_marker_image_generation_try_acquire(const char *prompt, + const char *dialog_id, + int64_t now_ms, + uint32_t *out_remain_ms) { + bool acquired = false; + uint32_t remain_ms = 0; + const char *safe_prompt = prompt != NULL ? prompt : ""; + const char *safe_dialog = dialog_id != NULL ? dialog_id : ""; + + portENTER_CRITICAL(&s_marker_image_gen_lock); + if (s_marker_image_gen_running) { + acquired = false; + } else if (s_marker_last_success_ms > 0 && + strcmp(safe_prompt, s_marker_last_success_prompt) == 0 && + strcmp(safe_dialog, s_marker_last_success_dialog_id) == 0) { + int64_t elapsed_ms = now_ms - s_marker_last_success_ms; + if (elapsed_ms >= 0 && elapsed_ms < VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS) { + remain_ms = (uint32_t)(VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS - elapsed_ms); + acquired = false; + } else { + s_marker_image_gen_running = true; + acquired = true; + } + } else { + s_marker_image_gen_running = true; + acquired = true; + } + portEXIT_CRITICAL(&s_marker_image_gen_lock); + + if (out_remain_ms != NULL) { + *out_remain_ms = remain_ms; + } + return acquired; +} + +static void voice_marker_image_generation_release(void) { + portENTER_CRITICAL(&s_marker_image_gen_lock); + s_marker_image_gen_running = false; + portEXIT_CRITICAL(&s_marker_image_gen_lock); +} + +static void voice_marker_image_generation_record_success(const char *prompt, + const char *dialog_id, + int64_t now_ms) { + const char *safe_prompt = prompt != NULL ? prompt : ""; + const char *safe_dialog = dialog_id != NULL ? dialog_id : ""; + + portENTER_CRITICAL(&s_marker_image_gen_lock); + strlcpy(s_marker_last_success_prompt, safe_prompt, sizeof(s_marker_last_success_prompt)); + strlcpy(s_marker_last_success_dialog_id, safe_dialog, sizeof(s_marker_last_success_dialog_id)); + s_marker_last_success_ms = now_ms; + portEXIT_CRITICAL(&s_marker_image_gen_lock); +} + +static void voice_log_text_preview(const char *stage, const char *text) { + if (stage == NULL || text == NULL || text[0] == '\0') { + return; + } + + size_t len = strlen(text); + const int preview = 240; + ESP_LOGI(TAG, + "[stage] %s: len=%u text=%.*s%s", + stage, + (unsigned)len, + preview, + text, + (len > (size_t)preview) ? "..." : ""); +} + +static bool voice_is_space_char(char c) { + return isspace((unsigned char)c) != 0; +} + +static size_t voice_find_first_sentence_end(const char *text, size_t len) { + if (text == NULL || len == 0) { + return 0; + } + + for (size_t i = 0; i < len; ++i) { + unsigned char c = (unsigned char)text[i]; + if (c == '.' || c == '!' || c == '?' || c == '\n' || c == '\r') { + return i + 1; + } + + if (i + 2 < len) { + unsigned char b1 = (unsigned char)text[i]; + unsigned char b2 = (unsigned char)text[i + 1]; + unsigned char b3 = (unsigned char)text[i + 2]; + + if (b1 == 0xE3 && b2 == 0x80 && b3 == 0x82) { + return i + 3; + } + if (b1 == 0xEF && b2 == 0xBC && + (b3 == 0x81 || b3 == 0x9F || b3 == 0x9B)) { + return i + 3; + } + } + } + + return len; +} + +static bool voice_extract_first_sentence_text(const char *text, + char *out_sentence, + size_t out_len) { + if (text == NULL || out_sentence == NULL || out_len == 0) { + return false; + } + + size_t len = strlen(text); + size_t begin = 0; + while (begin < len && voice_is_space_char(text[begin])) { + begin++; + } + if (begin >= len) { + return false; + } + + size_t end = begin + voice_find_first_sentence_end(text + begin, len - begin); + while (end > begin && voice_is_space_char(text[end - 1])) { + end--; + } + if (end <= begin) { + return false; + } + + size_t copy_len = end - begin; + if (copy_len >= out_len) { + copy_len = out_len - 1; + } + memcpy(out_sentence, text + begin, copy_len); + out_sentence[copy_len] = '\0'; + return copy_len > 0; +} + +static bool voice_extract_marker_prompt(const char *text, char *out_prompt, size_t out_len) { + if (text == NULL || out_prompt == NULL || out_len == 0) { + return false; + } + + const char *open = strstr(text, "/*"); + if (open == NULL) { + return false; + } + + const char *close = strstr(open + 2, "*/"); + if (close == NULL) { + return false; + } + + const char *begin = open + 2; + const char *end = close; + while (begin < end && voice_is_space_char(*begin)) { + begin++; + } + while (end > begin && voice_is_space_char(*(end - 1))) { + end--; + } + if (end <= begin) { + return false; + } + + size_t prompt_len = (size_t)(end - begin); + if (prompt_len >= out_len) { + prompt_len = out_len - 1; + } + memcpy(out_prompt, begin, prompt_len); + out_prompt[prompt_len] = '\0'; + return prompt_len > 0; +} + +static void voice_build_marker_image_prompt(const char *subject, + char *out_prompt, + size_t out_len) { + if (out_prompt == NULL || out_len == 0) { + return; + } + + const char *safe_subject = (subject != NULL && subject[0] != '\0') ? + subject : + VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT; + int written = snprintf(out_prompt, + out_len, + VOICE_IMAGE_MARKER_TEMPLATE, + safe_subject); + if (written < 0) { + out_prompt[0] = '\0'; + return; + } + + if ((size_t)written >= out_len) { + out_prompt[out_len - 1] = '\0'; + } +} + +static esp_err_t voice_submit_marker_generated_image(const voice_image_generation_task_arg_t *task_arg, + const image_generation_result_t *result, + char *err, + size_t err_len) { + if (task_arg == NULL || result == NULL || result->png == NULL || result->png_len == 0) { + voice_fill_err(err, err_len, "invalid image result"); + return ESP_ERR_INVALID_ARG; + } + + uint16_t raster_width = 0; + uint16_t raster_height = 0; + uint8_t *raster = NULL; + size_t raster_len = 0; + char decode_err[128] = {0}; + ESP_LOGI(TAG, + "[stage] marker_image_print_decode_start: dialog_id=%s request_id=%s png_bytes=%u", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + result->request_id[0] != '\0' ? result->request_id : "-", + (unsigned)result->png_len); + esp_err_t decode_rc = raster_tools_convert_png_to_raster_384(result->png, + result->png_len, + true, + VOICE_IMAGE_PRINT_THRESHOLD, + false, + VOICE_IMAGE_PRINT_MAX_HEIGHT, + &raster_width, + &raster_height, + &raster, + &raster_len, + decode_err, + sizeof(decode_err)); + if (decode_rc != ESP_OK) { + voice_fill_err(err, + err_len, + decode_err[0] != '\0' ? decode_err : "decode generated image failed"); + ESP_LOGW(TAG, + "[stage] marker_image_print_decode_failed: dialog_id=%s rc=0x%x msg=%s", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)decode_rc, + err != NULL && err[0] != '\0' ? err : "decode generated image failed"); + return decode_rc; + } + + ESP_LOGI(TAG, + "[stage] marker_image_print_decode_done: dialog_id=%s raster_size=%ux%u raster_bytes=%u", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)raster_width, + (unsigned)raster_height, + (unsigned)raster_len); + + uint32_t job_id = 0; + char submit_err[128] = {0}; + esp_err_t submit_rc = printer_protocol_submit_raster_job(raster, + raster_len, + raster_width, + raster_height, + NULL, + &job_id, + submit_err, + sizeof(submit_err)); + if (submit_rc != ESP_OK) { + voice_fill_err(err, + err_len, + submit_err[0] != '\0' ? submit_err : "submit print job failed"); + ESP_LOGW(TAG, + "[stage] marker_image_print_submit_failed: dialog_id=%s rc=0x%x msg=%s", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)submit_rc, + err != NULL && err[0] != '\0' ? err : "submit print job failed"); + } else { + ESP_LOGI(TAG, + "[stage] marker_image_print_submit_ok: dialog_id=%s request_id=%s job_id=%u", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + result->request_id[0] != '\0' ? result->request_id : "-", + (unsigned)job_id); + } + + free(raster); + return submit_rc; +} + +static void voice_image_generation_task(void *arg) { + voice_image_generation_task_arg_t *task_arg = (voice_image_generation_task_arg_t *)arg; + if (task_arg == NULL) { + vTaskDelete(NULL); + return; + } + + image_generation_request_t req = { + .prompt = task_arg->prompt, + .size = NULL, + .prompt_extend = false, + .has_seed = false, + .seed = 0, + .generation_timeout_ms = 0, + .download_timeout_ms = 0, + }; + + image_generation_result_t result = {0}; + image_generation_result_reset(&result); + + bool status_poll_paused = false; + char gen_err[160] = {0}; + ESP_LOGI(TAG, + "[stage] marker_image_gen_start: dialog_id=%s prompt_len=%u", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)strlen(task_arg->prompt)); + printer_protocol_set_status_poll_paused(true); + status_poll_paused = true; + esp_err_t rc = image_generation_generate_png(&req, &result, gen_err, sizeof(gen_err)); + printer_protocol_set_status_poll_paused(false); + status_poll_paused = false; + if (rc == ESP_OK) { + ESP_LOGI(TAG, + "[stage] marker_image_gen_done: dialog_id=%s request_id=%s size=%ux%u png_bytes=%u", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + result.request_id[0] != '\0' ? result.request_id : "-", + (unsigned)result.width, + (unsigned)result.height, + (unsigned)result.png_len); + + char submit_err[160] = {0}; + esp_err_t submit_rc = voice_submit_marker_generated_image(task_arg, + &result, + submit_err, + sizeof(submit_err)); + if (submit_rc == ESP_OK) { + voice_marker_image_generation_record_success(task_arg->prompt, + task_arg->dialog_id, + voice_now_ms()); + } else { + ESP_LOGW(TAG, + "[stage] marker_image_print_failed: dialog_id=%s rc=0x%x msg=%s", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)submit_rc, + submit_err[0] != '\0' ? submit_err : "marker image print failed"); + } + } else { + ESP_LOGW(TAG, + "[stage] marker_image_gen_failed: dialog_id=%s rc=0x%x msg=%s", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + (unsigned)rc, + gen_err[0] != '\0' ? gen_err : "image generation failed"); + } + + if (status_poll_paused) { + printer_protocol_set_status_poll_paused(false); + } + image_generation_result_free(&result); + voice_marker_image_generation_release(); + image_generation_schedule_prewarm(); + free(task_arg); + vTaskDelete(NULL); +} + +static void voice_schedule_marker_image_generation(const char *prompt, const char *dialog_id) { + if (prompt == NULL || prompt[0] == '\0') { + return; + } + + size_t prompt_len = strlen(prompt); + if (prompt_len > VOICE_IMAGE_MARKER_PROMPT_MAX_LEN) { + ESP_LOGW(TAG, + "[stage] marker_image_gen_skip: prompt too long, len=%u", + (unsigned)prompt_len); + return; + } + + uint32_t repeat_remain_ms = 0; + if (!voice_marker_image_generation_try_acquire(prompt, + dialog_id, + voice_now_ms(), + &repeat_remain_ms)) { + if (repeat_remain_ms > 0) { + ESP_LOGI(TAG, + "[stage] marker_image_gen_skip: duplicate prompt suppressed, remain_ms=%u", + (unsigned)repeat_remain_ms); + } else { + ESP_LOGI(TAG, "[stage] marker_image_gen_skip: previous marker task running"); + } + return; + } + + voice_image_generation_task_arg_t *task_arg = + (voice_image_generation_task_arg_t *)calloc(1, sizeof(*task_arg)); + if (task_arg == NULL) { + ESP_LOGW(TAG, "[stage] marker_image_gen_skip: no memory for task arg"); + voice_marker_image_generation_release(); + return; + } + + strlcpy(task_arg->prompt, prompt, sizeof(task_arg->prompt)); + if (dialog_id != NULL) { + strlcpy(task_arg->dialog_id, dialog_id, sizeof(task_arg->dialog_id)); + } + + BaseType_t ok = xTaskCreate(voice_image_generation_task, + "voice_img_gen", + VOICE_IMAGE_GEN_TASK_STACK, + task_arg, + VOICE_IMAGE_GEN_TASK_PRIO, + NULL); + if (ok != pdPASS) { + ESP_LOGW(TAG, "[stage] marker_image_gen_skip: create task failed"); + voice_marker_image_generation_release(); + free(task_arg); + return; + } + + ESP_LOGI(TAG, + "[stage] marker_image_gen_queued: dialog_id=%s prompt=%s", + task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", + task_arg->prompt); +} + +void voice_marker_try_trigger_image_generation(const char *dialog_id, const char *final_text) { + if (final_text == NULL || final_text[0] == '\0') { + return; + } + + char first_sentence[VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN] = {0}; + if (!voice_extract_first_sentence_text(final_text, first_sentence, sizeof(first_sentence))) { + return; + } + + char marker_subject[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0}; + if (!voice_extract_marker_prompt(first_sentence, marker_subject, sizeof(marker_subject))) { + return; + } + + char templated_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0}; + voice_build_marker_image_prompt(marker_subject, + templated_prompt, + sizeof(templated_prompt)); + if (templated_prompt[0] == '\0') { + ESP_LOGW(TAG, "[stage] marker_image_gen_skip: build prompt failed"); + return; + } + + ESP_LOGI(TAG, + "[stage] marker_image_gen_detected: first_sentence=%s", + first_sentence); + ESP_LOGI(TAG, + "[stage] marker_image_gen_subject: keyword=%s", + marker_subject); + voice_log_text_preview("marker_image_gen_prompt_templated", templated_prompt); + voice_schedule_marker_image_generation(templated_prompt, dialog_id); +} diff --git a/main/domain/src/voice_interaction_ws.c b/main/domain/src/voice_interaction_ws.c index 0f7c9e8..04afa7e 100644 --- a/main/domain/src/voice_interaction_ws.c +++ b/main/domain/src/voice_interaction_ws.c @@ -1,8 +1,5 @@ #include "voice_interaction_internal.h" -#include -#include -#include #include #include @@ -11,92 +8,10 @@ #include "cJSON.h" #undef cJSON_realloc #undef cJSON_malloc -#include "esp_audio_dec.h" -#include "esp_audio_enc.h" #include "esp_log.h" -#include "image_generation.h" -#include "printer_protocol.h" -#include "raster_tools.h" -#include "voice_audio.h" static const char *TAG = "voice_interaction"; -#define VOICE_IMAGE_MARKER_PROMPT_MAX_LEN 800 -#define VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN 1024 -#define VOICE_IMAGE_GEN_TASK_STACK 16384 -#define VOICE_IMAGE_GEN_TASK_PRIO 3 -#define VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS 45000 -#define VOICE_IMAGE_PRINT_THRESHOLD 160 -#define VOICE_IMAGE_PRINT_MAX_HEIGHT 2200 -#define VOICE_IMAGE_MARKER_TEMPLATE \ - "%s，黑白线稿风格，白色背景，主体居中，边缘清晰，细节简洁，高对比，避免大面积阴影和灰度渐变，适配热敏打印机打印。" -#define VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT "简洁卡通图案" - -typedef struct { - char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1]; - char dialog_id[40]; -} voice_image_generation_task_arg_t; - -static portMUX_TYPE s_marker_image_gen_lock = portMUX_INITIALIZER_UNLOCKED; -static bool s_marker_image_gen_running; -static int64_t s_marker_last_success_ms; -static char s_marker_last_success_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1]; -static char s_marker_last_success_dialog_id[40]; - -static bool voice_marker_image_generation_try_acquire(const char *prompt, - const char *dialog_id, - int64_t now_ms, - uint32_t *out_remain_ms) { - bool acquired = false; - uint32_t remain_ms = 0; - const char *safe_prompt = prompt != NULL ? prompt : ""; - const char *safe_dialog = dialog_id != NULL ? dialog_id : ""; - - portENTER_CRITICAL(&s_marker_image_gen_lock); - if (s_marker_image_gen_running) { - acquired = false; - } else if (s_marker_last_success_ms > 0 && - strcmp(safe_prompt, s_marker_last_success_prompt) == 0 && - strcmp(safe_dialog, s_marker_last_success_dialog_id) == 0) { - int64_t elapsed_ms = now_ms - s_marker_last_success_ms; - if (elapsed_ms >= 0 && elapsed_ms < VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS) { - remain_ms = (uint32_t)(VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS - elapsed_ms); - acquired = false; - } else { - s_marker_image_gen_running = true; - acquired = true; - } - } else { - s_marker_image_gen_running = true; - acquired = true; - } - portEXIT_CRITICAL(&s_marker_image_gen_lock); - - if (out_remain_ms != NULL) { - *out_remain_ms = remain_ms; - } - return acquired; -} - -static void voice_marker_image_generation_release(void) { - portENTER_CRITICAL(&s_marker_image_gen_lock); - s_marker_image_gen_running = false; - portEXIT_CRITICAL(&s_marker_image_gen_lock); -} - -static void voice_marker_image_generation_record_success(const char *prompt, - const char *dialog_id, - int64_t now_ms) { - const char *safe_prompt = prompt != NULL ? prompt : ""; - const char *safe_dialog = dialog_id != NULL ? dialog_id : ""; - - portENTER_CRITICAL(&s_marker_image_gen_lock); - strlcpy(s_marker_last_success_prompt, safe_prompt, sizeof(s_marker_last_success_prompt)); - strlcpy(s_marker_last_success_dialog_id, safe_dialog, sizeof(s_marker_last_success_dialog_id)); - s_marker_last_success_ms = now_ms; - portEXIT_CRITICAL(&s_marker_image_gen_lock); -} - static void *voice_cjson_malloc_hook(size_t size) { return voice_malloc_prefer_psram(size); } @@ -253,679 +168,6 @@ esp_err_t voice_send_directive(const char *action, return err; } -static void voice_handle_downstream_packet(const uint8_t *data, size_t len) { - if (data == NULL || len == 0) { - return; - } - - if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - return; - } - - void *opus_dec = s_voice.opus_dec; - uint8_t *pcm_buf = s_voice.pcm_rx_buf; - size_t pcm_buf_size = s_voice.pcm_rx_buf_size; - voice_unlock(); - - if (opus_dec == NULL || pcm_buf == NULL || pcm_buf_size == 0) { - return; - } - - esp_audio_dec_in_raw_t raw = { - .buffer = (uint8_t *)data, - .len = (uint32_t)len, - .consumed = 0, - }; - - while (raw.len > 0) { - esp_audio_dec_out_frame_t out = { - .buffer = pcm_buf, - .len = (uint32_t)pcm_buf_size, - .needed_size = 0, - .decoded_size = 0, - }; - esp_audio_dec_info_t dec_info = {0}; - - esp_audio_err_t ret = esp_opus_dec_decode(opus_dec, &raw, &out, &dec_info); - if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) { - if (out.needed_size > pcm_buf_size) { - uint8_t *new_buf = (uint8_t *)voice_realloc_prefer_psram(pcm_buf, out.needed_size); - if (new_buf == NULL) { - ESP_LOGE(TAG, "No memory to extend pcm rx buffer to %u", out.needed_size); - break; - } - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - s_voice.pcm_rx_buf = new_buf; - s_voice.pcm_rx_buf_size = out.needed_size; - voice_unlock(); - } - pcm_buf = new_buf; - pcm_buf_size = out.needed_size; - continue; - } - break; - } - - if (ret != ESP_AUDIO_ERR_OK) { - ESP_LOGW(TAG, "opus decode failed: %d", ret); - break; - } - - if (out.decoded_size > 0) { - char audio_err[64] = {0}; - size_t samples = (size_t)out.decoded_size / VOICE_SAMPLE_BYTES; - esp_err_t write_err = voice_audio_write_pcm((const int16_t *)out.buffer, - samples, - VOICE_AUDIO_IO_TIMEOUT_MS, - audio_err, - sizeof(audio_err)); - if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - int64_t now_ms = voice_now_ms(); - int64_t pcm_ms = ((int64_t)samples * 1000 + - (int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) / - (int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE; - int64_t base_ms = s_voice.playback_deadline_ms > now_ms ? - s_voice.playback_deadline_ms : now_ms; - s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS; - s_voice.last_downstream_ms = now_ms; - voice_unlock(); - } - } - - if (raw.consumed == 0 || raw.consumed > raw.len) { - break; - } - raw.buffer += raw.consumed; - raw.len -= raw.consumed; - } - - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - s_voice.downstream_packets++; - s_voice.last_event_ms = voice_now_ms(); - voice_unlock(); - } -} - -static const char *voice_get_json_str(cJSON *obj, const char *name) { - if (obj == NULL || name == NULL) { - return NULL; - } - cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); - if (cJSON_IsString(item) && item->valuestring != NULL) { - return item->valuestring; - } - return NULL; -} - -static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) { - if (obj == NULL || name == NULL || out_value == NULL) { - return false; - } - cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); - if (!cJSON_IsBool(item)) { - return false; - } - *out_value = cJSON_IsTrue(item); - return true; -} - -static void voice_log_final_text(const char *stage, const char *text) { - if (stage == NULL || text == NULL || text[0] == '\0') { - return; - } - size_t len = strlen(text); - const int preview = 240; - ESP_LOGI(TAG, - "[stage] %s: len=%u text=%.*s%s", - stage, - (unsigned)len, - preview, - text, - (len > (size_t)preview) ? "..." : ""); -} - -static bool voice_is_space_char(char c) { - return isspace((unsigned char)c) != 0; -} - -static size_t voice_find_first_sentence_end(const char *text, size_t len) { - if (text == NULL || len == 0) { - return 0; - } - - for (size_t i = 0; i < len; ++i) { - unsigned char c = (unsigned char)text[i]; - if (c == '.' || c == '!' || c == '?' || c == '\n' || c == '\r') { - return i + 1; - } - - if (i + 2 < len) { - unsigned char b1 = (unsigned char)text[i]; - unsigned char b2 = (unsigned char)text[i + 1]; - unsigned char b3 = (unsigned char)text[i + 2]; - - if (b1 == 0xE3 && b2 == 0x80 && b3 == 0x82) { - return i + 3; - } - if (b1 == 0xEF && b2 == 0xBC && - (b3 == 0x81 || b3 == 0x9F || b3 == 0x9B)) { - return i + 3; - } - } - } - - return len; -} - -static bool voice_extract_first_sentence_text(const char *text, - char *out_sentence, - size_t out_len) { - if (text == NULL || out_sentence == NULL || out_len == 0) { - return false; - } - - size_t len = strlen(text); - size_t begin = 0; - while (begin < len && voice_is_space_char(text[begin])) { - begin++; - } - if (begin >= len) { - return false; - } - - size_t end = begin + voice_find_first_sentence_end(text + begin, len - begin); - while (end > begin && voice_is_space_char(text[end - 1])) { - end--; - } - if (end <= begin) { - return false; - } - - size_t copy_len = end - begin; - if (copy_len >= out_len) { - copy_len = out_len - 1; - } - memcpy(out_sentence, text + begin, copy_len); - out_sentence[copy_len] = '\0'; - return copy_len > 0; -} - -static bool voice_extract_marker_prompt(const char *text, char *out_prompt, size_t out_len) { - if (text == NULL || out_prompt == NULL || out_len == 0) { - return false; - } - - const char *open = strstr(text, "/*"); - if (open == NULL) { - return false; - } - - const char *close = strstr(open + 2, "*/"); - if (close == NULL) { - return false; - } - - const char *begin = open + 2; - const char *end = close; - while (begin < end && voice_is_space_char(*begin)) { - begin++; - } - while (end > begin && voice_is_space_char(*(end - 1))) { - end--; - } - if (end <= begin) { - return false; - } - - size_t prompt_len = (size_t)(end - begin); - if (prompt_len >= out_len) { - prompt_len = out_len - 1; - } - memcpy(out_prompt, begin, prompt_len); - out_prompt[prompt_len] = '\0'; - return prompt_len > 0; -} - -static void voice_build_marker_image_prompt(const char *subject, - char *out_prompt, - size_t out_len) { - if (out_prompt == NULL || out_len == 0) { - return; - } - - const char *safe_subject = (subject != NULL && subject[0] != '\0') ? - subject : - VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT; - int written = snprintf(out_prompt, - out_len, - VOICE_IMAGE_MARKER_TEMPLATE, - safe_subject); - if (written < 0) { - out_prompt[0] = '\0'; - return; - } - - if ((size_t)written >= out_len) { - out_prompt[out_len - 1] = '\0'; - } -} - -static esp_err_t voice_submit_marker_generated_image(const voice_image_generation_task_arg_t *task_arg, - const image_generation_result_t *result, - char *err, - size_t err_len) { - if (task_arg == NULL || result == NULL || result->png == NULL || result->png_len == 0) { - voice_fill_err(err, err_len, "invalid image result"); - return ESP_ERR_INVALID_ARG; - } - - uint16_t raster_width = 0; - uint16_t raster_height = 0; - uint8_t *raster = NULL; - size_t raster_len = 0; - char decode_err[128] = {0}; - ESP_LOGI(TAG, - "[stage] marker_image_print_decode_start: dialog_id=%s request_id=%s png_bytes=%u", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - result->request_id[0] != '\0' ? result->request_id : "-", - (unsigned)result->png_len); - esp_err_t decode_rc = raster_tools_convert_png_to_raster_384(result->png, - result->png_len, - true, - VOICE_IMAGE_PRINT_THRESHOLD, - false, - VOICE_IMAGE_PRINT_MAX_HEIGHT, - &raster_width, - &raster_height, - &raster, - &raster_len, - decode_err, - sizeof(decode_err)); - if (decode_rc != ESP_OK) { - voice_fill_err(err, - err_len, - decode_err[0] != '\0' ? decode_err : "decode generated image failed"); - ESP_LOGW(TAG, - "[stage] marker_image_print_decode_failed: dialog_id=%s rc=0x%x msg=%s", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - (unsigned)decode_rc, - err != NULL && err[0] != '\0' ? err : "decode generated image failed"); - return decode_rc; - } - - ESP_LOGI(TAG, - "[stage] marker_image_print_decode_done: dialog_id=%s raster_size=%ux%u raster_bytes=%u", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - (unsigned)raster_width, - (unsigned)raster_height, - (unsigned)raster_len); - - uint32_t job_id = 0; - char submit_err[128] = {0}; - esp_err_t submit_rc = printer_protocol_submit_raster_job(raster, - raster_len, - raster_width, - raster_height, - NULL, - &job_id, - submit_err, - sizeof(submit_err)); - if (submit_rc != ESP_OK) { - voice_fill_err(err, - err_len, - submit_err[0] != '\0' ? submit_err : "submit print job failed"); - ESP_LOGW(TAG, - "[stage] marker_image_print_submit_failed: dialog_id=%s rc=0x%x msg=%s", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - (unsigned)submit_rc, - err != NULL && err[0] != '\0' ? err : "submit print job failed"); - } else { - ESP_LOGI(TAG, - "[stage] marker_image_print_submit_ok: dialog_id=%s request_id=%s job_id=%u", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - result->request_id[0] != '\0' ? result->request_id : "-", - (unsigned)job_id); - } - - free(raster); - return submit_rc; -} - -static void voice_image_generation_task(void *arg) { - voice_image_generation_task_arg_t *task_arg = (voice_image_generation_task_arg_t *)arg; - if (task_arg == NULL) { - vTaskDelete(NULL); - return; - } - - image_generation_request_t req = { - .prompt = task_arg->prompt, - .size = NULL, - .prompt_extend = false, - .has_seed = false, - .seed = 0, - .generation_timeout_ms = 0, - .download_timeout_ms = 0, - }; - - image_generation_result_t result = {0}; - image_generation_result_reset(&result); - - bool status_poll_paused = false; - char gen_err[160] = {0}; - ESP_LOGI(TAG, - "[stage] marker_image_gen_start: dialog_id=%s prompt_len=%u", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - (unsigned)strlen(task_arg->prompt)); - printer_protocol_set_status_poll_paused(true); - status_poll_paused = true; - esp_err_t rc = image_generation_generate_png(&req, &result, gen_err, sizeof(gen_err)); - printer_protocol_set_status_poll_paused(false); - status_poll_paused = false; - if (rc == ESP_OK) { - ESP_LOGI(TAG, - "[stage] marker_image_gen_done: dialog_id=%s request_id=%s size=%ux%u png_bytes=%u", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - result.request_id[0] != '\0' ? result.request_id : "-", - (unsigned)result.width, - (unsigned)result.height, - (unsigned)result.png_len); - - char submit_err[160] = {0}; - esp_err_t submit_rc = voice_submit_marker_generated_image(task_arg, - &result, - submit_err, - sizeof(submit_err)); - if (submit_rc == ESP_OK) { - voice_marker_image_generation_record_success(task_arg->prompt, - task_arg->dialog_id, - voice_now_ms()); - } else { - ESP_LOGW(TAG, - "[stage] marker_image_print_failed: dialog_id=%s rc=0x%x msg=%s", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - (unsigned)submit_rc, - submit_err[0] != '\0' ? submit_err : "marker image print failed"); - } - } else { - ESP_LOGW(TAG, - "[stage] marker_image_gen_failed: dialog_id=%s rc=0x%x msg=%s", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - (unsigned)rc, - gen_err[0] != '\0' ? gen_err : "image generation failed"); - } - - if (status_poll_paused) { - printer_protocol_set_status_poll_paused(false); - } - image_generation_result_free(&result); - voice_marker_image_generation_release(); - image_generation_schedule_prewarm(); - free(task_arg); - vTaskDelete(NULL); -} - -static void voice_schedule_marker_image_generation(const char *prompt, const char *dialog_id) { - if (prompt == NULL || prompt[0] == '\0') { - return; - } - - size_t prompt_len = strlen(prompt); - if (prompt_len > VOICE_IMAGE_MARKER_PROMPT_MAX_LEN) { - ESP_LOGW(TAG, - "[stage] marker_image_gen_skip: prompt too long, len=%u", - (unsigned)prompt_len); - return; - } - - uint32_t repeat_remain_ms = 0; - if (!voice_marker_image_generation_try_acquire(prompt, - dialog_id, - voice_now_ms(), - &repeat_remain_ms)) { - if (repeat_remain_ms > 0) { - ESP_LOGI(TAG, - "[stage] marker_image_gen_skip: duplicate prompt suppressed, remain_ms=%u", - (unsigned)repeat_remain_ms); - } else { - ESP_LOGI(TAG, "[stage] marker_image_gen_skip: previous marker task running"); - } - return; - } - - voice_image_generation_task_arg_t *task_arg = - (voice_image_generation_task_arg_t *)calloc(1, sizeof(*task_arg)); - if (task_arg == NULL) { - ESP_LOGW(TAG, "[stage] marker_image_gen_skip: no memory for task arg"); - voice_marker_image_generation_release(); - return; - } - - strlcpy(task_arg->prompt, prompt, sizeof(task_arg->prompt)); - if (dialog_id != NULL) { - strlcpy(task_arg->dialog_id, dialog_id, sizeof(task_arg->dialog_id)); - } - - BaseType_t ok = xTaskCreate(voice_image_generation_task, - "voice_img_gen", - VOICE_IMAGE_GEN_TASK_STACK, - task_arg, - VOICE_IMAGE_GEN_TASK_PRIO, - NULL); - if (ok != pdPASS) { - ESP_LOGW(TAG, "[stage] marker_image_gen_skip: create task failed"); - voice_marker_image_generation_release(); - free(task_arg); - return; - } - - ESP_LOGI(TAG, - "[stage] marker_image_gen_queued: dialog_id=%s prompt=%s", - task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-", - task_arg->prompt); -} - -static void voice_try_trigger_marker_image_generation(const char *dialog_id, const char *final_text) { - if (final_text == NULL || final_text[0] == '\0') { - return; - } - - char first_sentence[VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN] = {0}; - if (!voice_extract_first_sentence_text(final_text, first_sentence, sizeof(first_sentence))) { - return; - } - - char marker_subject[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0}; - if (!voice_extract_marker_prompt(first_sentence, marker_subject, sizeof(marker_subject))) { - return; - } - - char templated_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0}; - voice_build_marker_image_prompt(marker_subject, - templated_prompt, - sizeof(templated_prompt)); - if (templated_prompt[0] == '\0') { - ESP_LOGW(TAG, "[stage] marker_image_gen_skip: build prompt failed"); - return; - } - - ESP_LOGI(TAG, - "[stage] marker_image_gen_detected: first_sentence=%s", - first_sentence); - ESP_LOGI(TAG, - "[stage] marker_image_gen_subject: keyword=%s", - marker_subject); - voice_log_final_text("marker_image_gen_prompt_templated", templated_prompt); - voice_schedule_marker_image_generation(templated_prompt, dialog_id); -} - -static void voice_handle_output_event(cJSON *output) { - const char *event_name = voice_get_json_str(output, "event"); - if (event_name == NULL) { - return; - } - - const char *dialog_id = voice_get_json_str(output, "dialog_id"); - const char *state = NULL; - if (strcmp(event_name, "DialogStateChanged") == 0) { - state = voice_get_json_str(output, "state"); - } - ESP_LOGI(TAG, - "[stage] ws_event_output: event=%s dialog_id=%s state=%s", - event_name, - dialog_id != NULL ? dialog_id : "-", - state != NULL ? state : "-"); - - bool finished = false; - bool has_finished = voice_get_json_bool(output, "finished", &finished); - const char *responding_final_text = NULL; - bool should_check_marker_in_first_sentence = false; - if (has_finished && finished) { - if (strcmp(event_name, "SpeechContent") == 0) { - voice_log_final_text("asr_final_text", voice_get_json_str(output, "text")); - } else if (strcmp(event_name, "RespondingContent") == 0) { - const char *final_text = voice_get_json_str(output, "text"); - if (final_text == NULL || final_text[0] == '\0') { - final_text = voice_get_json_str(output, "spoken"); - } - voice_log_final_text("response_final_text", final_text); - responding_final_text = final_text; - if (responding_final_text != NULL && responding_final_text[0] != '\0' && - voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - if (!s_voice.response_first_sentence_checked) { - s_voice.response_first_sentence_checked = true; - should_check_marker_in_first_sentence = true; - } - voice_unlock(); - } - } - } - - bool should_close_audio = false; - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - if (dialog_id != NULL) { - strlcpy(s_voice.dialog_id, dialog_id, sizeof(s_voice.dialog_id)); - } - s_voice.last_event_ms = voice_now_ms(); - - if (strcmp(event_name, "Started") == 0) { - s_voice.started = true; - s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; - s_voice.response_first_sentence_checked = false; - ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id); - } else if (strcmp(event_name, "DialogStateChanged") == 0) { - if (state != NULL) { - voice_dialog_state_t prev_state = s_voice.dialog_state; - if (strcmp(state, "Listening") == 0) { - s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING; - if (prev_state != VOICE_DIALOG_STATE_LISTENING) { - s_voice.response_first_sentence_checked = false; - } - } else if (strcmp(state, "Thinking") == 0) { - s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING; - if (prev_state != VOICE_DIALOG_STATE_THINKING) { - s_voice.response_first_sentence_checked = false; - } - } else if (strcmp(state, "Responding") == 0) { - s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING; - if (prev_state != VOICE_DIALOG_STATE_RESPONDING) { - s_voice.response_first_sentence_checked = false; - } - } - } - ESP_LOGI(TAG, - "[stage] dialog_state_changed: state=%s current=%s", - state != NULL ? state : "-", - voice_interaction_dialog_state_str(s_voice.dialog_state)); - } else if (strcmp(event_name, "SpeechEnded") == 0) { - ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active); - } else if (strcmp(event_name, "Stopped") == 0) { - s_voice.started = false; - s_voice.tap_active = false; - s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; - s_voice.response_first_sentence_checked = false; - should_close_audio = true; - ESP_LOGI(TAG, "[stage] session_stopped_by_server"); - } else if (strcmp(event_name, "Error") == 0) { - const char *error_msg = voice_get_json_str(output, "error_message"); - if (error_msg != NULL) { - voice_set_last_error_locked(error_msg); - ESP_LOGW(TAG, "[stage] server_error: %s", error_msg); - } - s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; - s_voice.tap_active = false; - s_voice.response_first_sentence_checked = false; - should_close_audio = true; - } - voice_unlock(); - } - - if (should_close_audio) { - voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close"); - } - - if (should_check_marker_in_first_sentence) { - voice_try_trigger_marker_image_generation(dialog_id, responding_final_text); - } - - if (strcmp(event_name, "RespondingStarted") == 0) { - (void)voice_send_directive("continue-task", "LocalRespondingStarted", true); - } else if (strcmp(event_name, "RespondingEnded") == 0) { - (void)voice_send_directive("continue-task", "LocalRespondingEnded", true); - } -} - -static void voice_handle_text_message(const char *text, size_t len) { - voice_cjson_init_hooks_once(); - - cJSON *root = cJSON_ParseWithLength(text, len); - if (root == NULL) { - ESP_LOGW(TAG, "Invalid ws text payload"); - return; - } - - cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header"); - const char *header_event = voice_get_json_str(header, "event"); - - cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload"); - cJSON *output = NULL; - if (payload != NULL) { - output = cJSON_GetObjectItemCaseSensitive(payload, "output"); - } - - if (header != NULL) { - cJSON *status_code = cJSON_GetObjectItemCaseSensitive(header, "status_code"); - cJSON *status_msg = cJSON_GetObjectItemCaseSensitive(header, "status_message"); - if (cJSON_IsNumber(status_code) && status_code->valueint >= 400) { - ESP_LOGW(TAG, - "[stage] ws_header_error: status_code=%d status_message=%s", - status_code->valueint, - (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) ? status_msg->valuestring : "-"); - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - if (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) { - voice_set_last_error_locked(status_msg->valuestring); - } else { - voice_set_last_error_locked("websocket status error"); - } - voice_unlock(); - } - } - } - - if (output != NULL) { - voice_handle_output_event(output); - } else if (header_event != NULL && strcmp(header_event, "task-failed") == 0) { - const char *error_msg = voice_get_json_str(header, "error_message"); - ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed"); - if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { - voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed"); - voice_unlock(); - } - } - - cJSON_Delete(root); -} - static void voice_handle_text_chunk(esp_websocket_event_data_t *data) { if (data->payload_offset == 0) { free(s_voice.text_agg); @@ -963,7 +205,7 @@ static void voice_handle_text_chunk(esp_websocket_event_data_t *data) { return; } - voice_handle_text_message((const char *)s_voice.text_agg, s_voice.text_agg_size); + voice_ws_protocol_handle_text_message((const char *)s_voice.text_agg, s_voice.text_agg_size); free(s_voice.text_agg); s_voice.text_agg = NULL; s_voice.text_agg_size = 0; @@ -1006,7 +248,7 @@ static void voice_handle_binary_chunk(esp_websocket_event_data_t *data) { return; } - voice_handle_downstream_packet(s_voice.bin_agg, s_voice.bin_agg_size); + voice_ws_audio_handle_downstream_packet(s_voice.bin_agg, s_voice.bin_agg_size); free(s_voice.bin_agg); s_voice.bin_agg = NULL; s_voice.bin_agg_size = 0; diff --git a/main/domain/src/voice_interaction_ws_audio.c b/main/domain/src/voice_interaction_ws_audio.c new file mode 100644 index 0000000..200e7ec --- /dev/null +++ b/main/domain/src/voice_interaction_ws_audio.c @@ -0,0 +1,100 @@ +#include "voice_interaction_internal.h" + +#include "esp_audio_dec.h" +#include "esp_log.h" +#include "voice_audio.h" + +static const char *TAG = "voice_interaction"; + +void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len) { + if (data == NULL || len == 0) { + return; + } + + if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + return; + } + + void *opus_dec = s_voice.opus_dec; + uint8_t *pcm_buf = s_voice.pcm_rx_buf; + size_t pcm_buf_size = s_voice.pcm_rx_buf_size; + voice_unlock(); + + if (opus_dec == NULL || pcm_buf == NULL || pcm_buf_size == 0) { + return; + } + + esp_audio_dec_in_raw_t raw = { + .buffer = (uint8_t *)data, + .len = (uint32_t)len, + .consumed = 0, + }; + + while (raw.len > 0) { + esp_audio_dec_out_frame_t out = { + .buffer = pcm_buf, + .len = (uint32_t)pcm_buf_size, + .needed_size = 0, + .decoded_size = 0, + }; + esp_audio_dec_info_t dec_info = {0}; + + esp_audio_err_t ret = esp_opus_dec_decode(opus_dec, &raw, &out, &dec_info); + if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) { + if (out.needed_size > pcm_buf_size) { + uint8_t *new_buf = (uint8_t *)voice_realloc_prefer_psram(pcm_buf, out.needed_size); + if (new_buf == NULL) { + ESP_LOGE(TAG, "No memory to extend pcm rx buffer to %u", out.needed_size); + break; + } + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.pcm_rx_buf = new_buf; + s_voice.pcm_rx_buf_size = out.needed_size; + voice_unlock(); + } + pcm_buf = new_buf; + pcm_buf_size = out.needed_size; + continue; + } + break; + } + + if (ret != ESP_AUDIO_ERR_OK) { + ESP_LOGW(TAG, "opus decode failed: %d", ret); + break; + } + + if (out.decoded_size > 0) { + char audio_err[64] = {0}; + size_t samples = (size_t)out.decoded_size / VOICE_SAMPLE_BYTES; + esp_err_t write_err = voice_audio_write_pcm((const int16_t *)out.buffer, + samples, + VOICE_AUDIO_IO_TIMEOUT_MS, + audio_err, + sizeof(audio_err)); + if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + int64_t now_ms = voice_now_ms(); + int64_t pcm_ms = ((int64_t)samples * 1000 + + (int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) / + (int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE; + int64_t base_ms = s_voice.playback_deadline_ms > now_ms ? + s_voice.playback_deadline_ms : now_ms; + s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS; + s_voice.last_downstream_ms = now_ms; + voice_unlock(); + } + } + + if (raw.consumed == 0 || raw.consumed > raw.len) { + break; + } + raw.buffer += raw.consumed; + raw.len -= raw.consumed; + } + + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + s_voice.downstream_packets++; + s_voice.last_event_ms = voice_now_ms(); + voice_unlock(); + } +} diff --git a/main/domain/src/voice_interaction_ws_protocol.c b/main/domain/src/voice_interaction_ws_protocol.c new file mode 100644 index 0000000..e747a57 --- /dev/null +++ b/main/domain/src/voice_interaction_ws_protocol.c @@ -0,0 +1,224 @@ +#include "voice_interaction_internal.h" + +#include + +#define cJSON_malloc voice_malloc_prefer_psram +#define cJSON_realloc voice_realloc_prefer_psram +#include "cJSON.h" +#undef cJSON_realloc +#undef cJSON_malloc +#include "esp_log.h" + +static const char *TAG = "voice_interaction"; + +static const char *voice_get_json_str(cJSON *obj, const char *name) { + if (obj == NULL || name == NULL) { + return NULL; + } + + cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); + if (cJSON_IsString(item) && item->valuestring != NULL) { + return item->valuestring; + } + return NULL; +} + +static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) { + if (obj == NULL || name == NULL || out_value == NULL) { + return false; + } + + cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); + if (!cJSON_IsBool(item)) { + return false; + } + *out_value = cJSON_IsTrue(item); + return true; +} + +static void voice_log_final_text(const char *stage, const char *text) { + if (stage == NULL || text == NULL || text[0] == '\0') { + return; + } + + size_t len = strlen(text); + const int preview = 240; + ESP_LOGI(TAG, + "[stage] %s: len=%u text=%.*s%s", + stage, + (unsigned)len, + preview, + text, + (len > (size_t)preview) ? "..." : ""); +} + +static void voice_handle_output_event(cJSON *output) { + const char *event_name = voice_get_json_str(output, "event"); + if (event_name == NULL) { + return; + } + + const char *dialog_id = voice_get_json_str(output, "dialog_id"); + const char *state = NULL; + if (strcmp(event_name, "DialogStateChanged") == 0) { + state = voice_get_json_str(output, "state"); + } + ESP_LOGI(TAG, + "[stage] ws_event_output: event=%s dialog_id=%s state=%s", + event_name, + dialog_id != NULL ? dialog_id : "-", + state != NULL ? state : "-"); + + bool finished = false; + bool has_finished = voice_get_json_bool(output, "finished", &finished); + const char *responding_final_text = NULL; + bool should_check_marker_in_first_sentence = false; + if (has_finished && finished) { + if (strcmp(event_name, "SpeechContent") == 0) { + voice_log_final_text("asr_final_text", voice_get_json_str(output, "text")); + } else if (strcmp(event_name, "RespondingContent") == 0) { + const char *final_text = voice_get_json_str(output, "text"); + if (final_text == NULL || final_text[0] == '\0') { + final_text = voice_get_json_str(output, "spoken"); + } + voice_log_final_text("response_final_text", final_text); + responding_final_text = final_text; + if (responding_final_text != NULL && responding_final_text[0] != '\0' && + voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + if (!s_voice.response_first_sentence_checked) { + s_voice.response_first_sentence_checked = true; + should_check_marker_in_first_sentence = true; + } + voice_unlock(); + } + } + } + + bool should_close_audio = false; + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + if (dialog_id != NULL) { + strlcpy(s_voice.dialog_id, dialog_id, sizeof(s_voice.dialog_id)); + } + s_voice.last_event_ms = voice_now_ms(); + + if (strcmp(event_name, "Started") == 0) { + s_voice.started = true; + s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + s_voice.response_first_sentence_checked = false; + ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id); + } else if (strcmp(event_name, "DialogStateChanged") == 0) { + if (state != NULL) { + voice_dialog_state_t prev_state = s_voice.dialog_state; + if (strcmp(state, "Listening") == 0) { + s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING; + if (prev_state != VOICE_DIALOG_STATE_LISTENING) { + s_voice.response_first_sentence_checked = false; + } + } else if (strcmp(state, "Thinking") == 0) { + s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING; + if (prev_state != VOICE_DIALOG_STATE_THINKING) { + s_voice.response_first_sentence_checked = false; + } + } else if (strcmp(state, "Responding") == 0) { + s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING; + if (prev_state != VOICE_DIALOG_STATE_RESPONDING) { + s_voice.response_first_sentence_checked = false; + } + } + } + ESP_LOGI(TAG, + "[stage] dialog_state_changed: state=%s current=%s", + state != NULL ? state : "-", + voice_interaction_dialog_state_str(s_voice.dialog_state)); + } else if (strcmp(event_name, "SpeechEnded") == 0) { + ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active); + } else if (strcmp(event_name, "Stopped") == 0) { + s_voice.started = false; + s_voice.tap_active = false; + s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + s_voice.response_first_sentence_checked = false; + should_close_audio = true; + ESP_LOGI(TAG, "[stage] session_stopped_by_server"); + } else if (strcmp(event_name, "Error") == 0) { + const char *error_msg = voice_get_json_str(output, "error_message"); + if (error_msg != NULL) { + voice_set_last_error_locked(error_msg); + ESP_LOGW(TAG, "[stage] server_error: %s", error_msg); + } + s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE; + s_voice.tap_active = false; + s_voice.response_first_sentence_checked = false; + should_close_audio = true; + } + voice_unlock(); + } + + if (should_close_audio) { + voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close"); + } + + if (should_check_marker_in_first_sentence) { + voice_marker_try_trigger_image_generation(dialog_id, responding_final_text); + } + + if (strcmp(event_name, "RespondingStarted") == 0) { + (void)voice_send_directive("continue-task", "LocalRespondingStarted", true); + } else if (strcmp(event_name, "RespondingEnded") == 0) { + (void)voice_send_directive("continue-task", "LocalRespondingEnded", true); + } +} + +void voice_ws_protocol_handle_text_message(const char *text, size_t len) { + if (text == NULL || len == 0) { + return; + } + + voice_interaction_ws_init_cjson_hooks(); + + cJSON *root = cJSON_ParseWithLength(text, len); + if (root == NULL) { + ESP_LOGW(TAG, "Invalid ws text payload"); + return; + } + + cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header"); + const char *header_event = voice_get_json_str(header, "event"); + + cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload"); + cJSON *output = NULL; + if (payload != NULL) { + output = cJSON_GetObjectItemCaseSensitive(payload, "output"); + } + + if (header != NULL) { + cJSON *status_code = cJSON_GetObjectItemCaseSensitive(header, "status_code"); + cJSON *status_msg = cJSON_GetObjectItemCaseSensitive(header, "status_message"); + if (cJSON_IsNumber(status_code) && status_code->valueint >= 400) { + ESP_LOGW(TAG, + "[stage] ws_header_error: status_code=%d status_message=%s", + status_code->valueint, + (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) ? status_msg->valuestring : "-"); + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + if (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) { + voice_set_last_error_locked(status_msg->valuestring); + } else { + voice_set_last_error_locked("websocket status error"); + } + voice_unlock(); + } + } + } + + if (output != NULL) { + voice_handle_output_event(output); + } else if (header_event != NULL && strcmp(header_event, "task-failed") == 0) { + const char *error_msg = voice_get_json_str(header, "error_message"); + ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed"); + if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) { + voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed"); + voice_unlock(); + } + } + + cJSON_Delete(root); +}