refactor(voice): split ws transport protocol audio marker modules
This commit is contained in:
@@ -21,6 +21,9 @@ idf_component_register(
|
|||||||
"domain/src/voice_interaction.c"
|
"domain/src/voice_interaction.c"
|
||||||
"domain/src/voice_interaction_common.c"
|
"domain/src/voice_interaction_common.c"
|
||||||
"domain/src/voice_interaction_ws.c"
|
"domain/src/voice_interaction_ws.c"
|
||||||
|
"domain/src/voice_interaction_ws_protocol.c"
|
||||||
|
"domain/src/voice_interaction_ws_audio.c"
|
||||||
|
"domain/src/voice_interaction_marker.c"
|
||||||
"domain/src/voice_interaction_tasks.c"
|
"domain/src/voice_interaction_tasks.c"
|
||||||
"platform/src/platform_bootstrap.c"
|
"platform/src/platform_bootstrap.c"
|
||||||
"platform/src/wifi_manager.c"
|
"platform/src/wifi_manager.c"
|
||||||
|
|||||||
@@ -117,6 +117,10 @@ esp_err_t voice_send_directive(const char *action,
|
|||||||
const char *directive,
|
const char *directive,
|
||||||
bool with_dialog_id);
|
bool with_dialog_id);
|
||||||
|
|
||||||
|
void voice_ws_protocol_handle_text_message(const char *text, size_t len);
|
||||||
|
void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len);
|
||||||
|
void voice_marker_try_trigger_image_generation(const char *dialog_id, const char *final_text);
|
||||||
|
|
||||||
void voice_websocket_event_handler(void *handler_args,
|
void voice_websocket_event_handler(void *handler_args,
|
||||||
esp_event_base_t base,
|
esp_event_base_t base,
|
||||||
int32_t event_id,
|
int32_t event_id,
|
||||||
|
|||||||
480
main/domain/src/voice_interaction_marker.c
Normal file
480
main/domain/src/voice_interaction_marker.c
Normal file
@@ -0,0 +1,480 @@
|
|||||||
|
#include "voice_interaction_internal.h"
|
||||||
|
|
||||||
|
#include <ctype.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "esp_log.h"
|
||||||
|
#include "image_generation.h"
|
||||||
|
#include "printer_protocol.h"
|
||||||
|
#include "raster_tools.h"
|
||||||
|
|
||||||
|
static const char *TAG = "voice_interaction";
|
||||||
|
|
||||||
|
#define VOICE_IMAGE_MARKER_PROMPT_MAX_LEN 800
|
||||||
|
#define VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN 1024
|
||||||
|
#define VOICE_IMAGE_GEN_TASK_STACK 16384
|
||||||
|
#define VOICE_IMAGE_GEN_TASK_PRIO 3
|
||||||
|
#define VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS 45000
|
||||||
|
#define VOICE_IMAGE_PRINT_THRESHOLD 160
|
||||||
|
#define VOICE_IMAGE_PRINT_MAX_HEIGHT 2200
|
||||||
|
#define VOICE_IMAGE_MARKER_TEMPLATE \
|
||||||
|
"%s,黑白线稿风格,白色背景,主体居中,边缘清晰,细节简洁,高对比,避免大面积阴影和灰度渐变,适配热敏打印机打印。"
|
||||||
|
#define VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT "简洁卡通图案"
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1];
|
||||||
|
char dialog_id[40];
|
||||||
|
} voice_image_generation_task_arg_t;
|
||||||
|
|
||||||
|
static portMUX_TYPE s_marker_image_gen_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||||
|
static bool s_marker_image_gen_running;
|
||||||
|
static int64_t s_marker_last_success_ms;
|
||||||
|
static char s_marker_last_success_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1];
|
||||||
|
static char s_marker_last_success_dialog_id[40];
|
||||||
|
|
||||||
|
static bool voice_marker_image_generation_try_acquire(const char *prompt,
|
||||||
|
const char *dialog_id,
|
||||||
|
int64_t now_ms,
|
||||||
|
uint32_t *out_remain_ms) {
|
||||||
|
bool acquired = false;
|
||||||
|
uint32_t remain_ms = 0;
|
||||||
|
const char *safe_prompt = prompt != NULL ? prompt : "";
|
||||||
|
const char *safe_dialog = dialog_id != NULL ? dialog_id : "";
|
||||||
|
|
||||||
|
portENTER_CRITICAL(&s_marker_image_gen_lock);
|
||||||
|
if (s_marker_image_gen_running) {
|
||||||
|
acquired = false;
|
||||||
|
} else if (s_marker_last_success_ms > 0 &&
|
||||||
|
strcmp(safe_prompt, s_marker_last_success_prompt) == 0 &&
|
||||||
|
strcmp(safe_dialog, s_marker_last_success_dialog_id) == 0) {
|
||||||
|
int64_t elapsed_ms = now_ms - s_marker_last_success_ms;
|
||||||
|
if (elapsed_ms >= 0 && elapsed_ms < VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS) {
|
||||||
|
remain_ms = (uint32_t)(VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS - elapsed_ms);
|
||||||
|
acquired = false;
|
||||||
|
} else {
|
||||||
|
s_marker_image_gen_running = true;
|
||||||
|
acquired = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
s_marker_image_gen_running = true;
|
||||||
|
acquired = true;
|
||||||
|
}
|
||||||
|
portEXIT_CRITICAL(&s_marker_image_gen_lock);
|
||||||
|
|
||||||
|
if (out_remain_ms != NULL) {
|
||||||
|
*out_remain_ms = remain_ms;
|
||||||
|
}
|
||||||
|
return acquired;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_marker_image_generation_release(void) {
|
||||||
|
portENTER_CRITICAL(&s_marker_image_gen_lock);
|
||||||
|
s_marker_image_gen_running = false;
|
||||||
|
portEXIT_CRITICAL(&s_marker_image_gen_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_marker_image_generation_record_success(const char *prompt,
|
||||||
|
const char *dialog_id,
|
||||||
|
int64_t now_ms) {
|
||||||
|
const char *safe_prompt = prompt != NULL ? prompt : "";
|
||||||
|
const char *safe_dialog = dialog_id != NULL ? dialog_id : "";
|
||||||
|
|
||||||
|
portENTER_CRITICAL(&s_marker_image_gen_lock);
|
||||||
|
strlcpy(s_marker_last_success_prompt, safe_prompt, sizeof(s_marker_last_success_prompt));
|
||||||
|
strlcpy(s_marker_last_success_dialog_id, safe_dialog, sizeof(s_marker_last_success_dialog_id));
|
||||||
|
s_marker_last_success_ms = now_ms;
|
||||||
|
portEXIT_CRITICAL(&s_marker_image_gen_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_log_text_preview(const char *stage, const char *text) {
|
||||||
|
if (stage == NULL || text == NULL || text[0] == '\0') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t len = strlen(text);
|
||||||
|
const int preview = 240;
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] %s: len=%u text=%.*s%s",
|
||||||
|
stage,
|
||||||
|
(unsigned)len,
|
||||||
|
preview,
|
||||||
|
text,
|
||||||
|
(len > (size_t)preview) ? "..." : "");
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool voice_is_space_char(char c) {
|
||||||
|
return isspace((unsigned char)c) != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static size_t voice_find_first_sentence_end(const char *text, size_t len) {
|
||||||
|
if (text == NULL || len == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (size_t i = 0; i < len; ++i) {
|
||||||
|
unsigned char c = (unsigned char)text[i];
|
||||||
|
if (c == '.' || c == '!' || c == '?' || c == '\n' || c == '\r') {
|
||||||
|
return i + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (i + 2 < len) {
|
||||||
|
unsigned char b1 = (unsigned char)text[i];
|
||||||
|
unsigned char b2 = (unsigned char)text[i + 1];
|
||||||
|
unsigned char b3 = (unsigned char)text[i + 2];
|
||||||
|
|
||||||
|
if (b1 == 0xE3 && b2 == 0x80 && b3 == 0x82) {
|
||||||
|
return i + 3;
|
||||||
|
}
|
||||||
|
if (b1 == 0xEF && b2 == 0xBC &&
|
||||||
|
(b3 == 0x81 || b3 == 0x9F || b3 == 0x9B)) {
|
||||||
|
return i + 3;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return len;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool voice_extract_first_sentence_text(const char *text,
|
||||||
|
char *out_sentence,
|
||||||
|
size_t out_len) {
|
||||||
|
if (text == NULL || out_sentence == NULL || out_len == 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t len = strlen(text);
|
||||||
|
size_t begin = 0;
|
||||||
|
while (begin < len && voice_is_space_char(text[begin])) {
|
||||||
|
begin++;
|
||||||
|
}
|
||||||
|
if (begin >= len) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t end = begin + voice_find_first_sentence_end(text + begin, len - begin);
|
||||||
|
while (end > begin && voice_is_space_char(text[end - 1])) {
|
||||||
|
end--;
|
||||||
|
}
|
||||||
|
if (end <= begin) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t copy_len = end - begin;
|
||||||
|
if (copy_len >= out_len) {
|
||||||
|
copy_len = out_len - 1;
|
||||||
|
}
|
||||||
|
memcpy(out_sentence, text + begin, copy_len);
|
||||||
|
out_sentence[copy_len] = '\0';
|
||||||
|
return copy_len > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool voice_extract_marker_prompt(const char *text, char *out_prompt, size_t out_len) {
|
||||||
|
if (text == NULL || out_prompt == NULL || out_len == 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *open = strstr(text, "/*");
|
||||||
|
if (open == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *close = strstr(open + 2, "*/");
|
||||||
|
if (close == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *begin = open + 2;
|
||||||
|
const char *end = close;
|
||||||
|
while (begin < end && voice_is_space_char(*begin)) {
|
||||||
|
begin++;
|
||||||
|
}
|
||||||
|
while (end > begin && voice_is_space_char(*(end - 1))) {
|
||||||
|
end--;
|
||||||
|
}
|
||||||
|
if (end <= begin) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t prompt_len = (size_t)(end - begin);
|
||||||
|
if (prompt_len >= out_len) {
|
||||||
|
prompt_len = out_len - 1;
|
||||||
|
}
|
||||||
|
memcpy(out_prompt, begin, prompt_len);
|
||||||
|
out_prompt[prompt_len] = '\0';
|
||||||
|
return prompt_len > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_build_marker_image_prompt(const char *subject,
|
||||||
|
char *out_prompt,
|
||||||
|
size_t out_len) {
|
||||||
|
if (out_prompt == NULL || out_len == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *safe_subject = (subject != NULL && subject[0] != '\0') ?
|
||||||
|
subject :
|
||||||
|
VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT;
|
||||||
|
int written = snprintf(out_prompt,
|
||||||
|
out_len,
|
||||||
|
VOICE_IMAGE_MARKER_TEMPLATE,
|
||||||
|
safe_subject);
|
||||||
|
if (written < 0) {
|
||||||
|
out_prompt[0] = '\0';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((size_t)written >= out_len) {
|
||||||
|
out_prompt[out_len - 1] = '\0';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static esp_err_t voice_submit_marker_generated_image(const voice_image_generation_task_arg_t *task_arg,
|
||||||
|
const image_generation_result_t *result,
|
||||||
|
char *err,
|
||||||
|
size_t err_len) {
|
||||||
|
if (task_arg == NULL || result == NULL || result->png == NULL || result->png_len == 0) {
|
||||||
|
voice_fill_err(err, err_len, "invalid image result");
|
||||||
|
return ESP_ERR_INVALID_ARG;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint16_t raster_width = 0;
|
||||||
|
uint16_t raster_height = 0;
|
||||||
|
uint8_t *raster = NULL;
|
||||||
|
size_t raster_len = 0;
|
||||||
|
char decode_err[128] = {0};
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_print_decode_start: dialog_id=%s request_id=%s png_bytes=%u",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
result->request_id[0] != '\0' ? result->request_id : "-",
|
||||||
|
(unsigned)result->png_len);
|
||||||
|
esp_err_t decode_rc = raster_tools_convert_png_to_raster_384(result->png,
|
||||||
|
result->png_len,
|
||||||
|
true,
|
||||||
|
VOICE_IMAGE_PRINT_THRESHOLD,
|
||||||
|
false,
|
||||||
|
VOICE_IMAGE_PRINT_MAX_HEIGHT,
|
||||||
|
&raster_width,
|
||||||
|
&raster_height,
|
||||||
|
&raster,
|
||||||
|
&raster_len,
|
||||||
|
decode_err,
|
||||||
|
sizeof(decode_err));
|
||||||
|
if (decode_rc != ESP_OK) {
|
||||||
|
voice_fill_err(err,
|
||||||
|
err_len,
|
||||||
|
decode_err[0] != '\0' ? decode_err : "decode generated image failed");
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"[stage] marker_image_print_decode_failed: dialog_id=%s rc=0x%x msg=%s",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
(unsigned)decode_rc,
|
||||||
|
err != NULL && err[0] != '\0' ? err : "decode generated image failed");
|
||||||
|
return decode_rc;
|
||||||
|
}
|
||||||
|
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_print_decode_done: dialog_id=%s raster_size=%ux%u raster_bytes=%u",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
(unsigned)raster_width,
|
||||||
|
(unsigned)raster_height,
|
||||||
|
(unsigned)raster_len);
|
||||||
|
|
||||||
|
uint32_t job_id = 0;
|
||||||
|
char submit_err[128] = {0};
|
||||||
|
esp_err_t submit_rc = printer_protocol_submit_raster_job(raster,
|
||||||
|
raster_len,
|
||||||
|
raster_width,
|
||||||
|
raster_height,
|
||||||
|
NULL,
|
||||||
|
&job_id,
|
||||||
|
submit_err,
|
||||||
|
sizeof(submit_err));
|
||||||
|
if (submit_rc != ESP_OK) {
|
||||||
|
voice_fill_err(err,
|
||||||
|
err_len,
|
||||||
|
submit_err[0] != '\0' ? submit_err : "submit print job failed");
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"[stage] marker_image_print_submit_failed: dialog_id=%s rc=0x%x msg=%s",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
(unsigned)submit_rc,
|
||||||
|
err != NULL && err[0] != '\0' ? err : "submit print job failed");
|
||||||
|
} else {
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_print_submit_ok: dialog_id=%s request_id=%s job_id=%u",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
result->request_id[0] != '\0' ? result->request_id : "-",
|
||||||
|
(unsigned)job_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
free(raster);
|
||||||
|
return submit_rc;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_image_generation_task(void *arg) {
|
||||||
|
voice_image_generation_task_arg_t *task_arg = (voice_image_generation_task_arg_t *)arg;
|
||||||
|
if (task_arg == NULL) {
|
||||||
|
vTaskDelete(NULL);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
image_generation_request_t req = {
|
||||||
|
.prompt = task_arg->prompt,
|
||||||
|
.size = NULL,
|
||||||
|
.prompt_extend = false,
|
||||||
|
.has_seed = false,
|
||||||
|
.seed = 0,
|
||||||
|
.generation_timeout_ms = 0,
|
||||||
|
.download_timeout_ms = 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
image_generation_result_t result = {0};
|
||||||
|
image_generation_result_reset(&result);
|
||||||
|
|
||||||
|
bool status_poll_paused = false;
|
||||||
|
char gen_err[160] = {0};
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_gen_start: dialog_id=%s prompt_len=%u",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
(unsigned)strlen(task_arg->prompt));
|
||||||
|
printer_protocol_set_status_poll_paused(true);
|
||||||
|
status_poll_paused = true;
|
||||||
|
esp_err_t rc = image_generation_generate_png(&req, &result, gen_err, sizeof(gen_err));
|
||||||
|
printer_protocol_set_status_poll_paused(false);
|
||||||
|
status_poll_paused = false;
|
||||||
|
if (rc == ESP_OK) {
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_gen_done: dialog_id=%s request_id=%s size=%ux%u png_bytes=%u",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
result.request_id[0] != '\0' ? result.request_id : "-",
|
||||||
|
(unsigned)result.width,
|
||||||
|
(unsigned)result.height,
|
||||||
|
(unsigned)result.png_len);
|
||||||
|
|
||||||
|
char submit_err[160] = {0};
|
||||||
|
esp_err_t submit_rc = voice_submit_marker_generated_image(task_arg,
|
||||||
|
&result,
|
||||||
|
submit_err,
|
||||||
|
sizeof(submit_err));
|
||||||
|
if (submit_rc == ESP_OK) {
|
||||||
|
voice_marker_image_generation_record_success(task_arg->prompt,
|
||||||
|
task_arg->dialog_id,
|
||||||
|
voice_now_ms());
|
||||||
|
} else {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"[stage] marker_image_print_failed: dialog_id=%s rc=0x%x msg=%s",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
(unsigned)submit_rc,
|
||||||
|
submit_err[0] != '\0' ? submit_err : "marker image print failed");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"[stage] marker_image_gen_failed: dialog_id=%s rc=0x%x msg=%s",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
(unsigned)rc,
|
||||||
|
gen_err[0] != '\0' ? gen_err : "image generation failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status_poll_paused) {
|
||||||
|
printer_protocol_set_status_poll_paused(false);
|
||||||
|
}
|
||||||
|
image_generation_result_free(&result);
|
||||||
|
voice_marker_image_generation_release();
|
||||||
|
image_generation_schedule_prewarm();
|
||||||
|
free(task_arg);
|
||||||
|
vTaskDelete(NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_schedule_marker_image_generation(const char *prompt, const char *dialog_id) {
|
||||||
|
if (prompt == NULL || prompt[0] == '\0') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t prompt_len = strlen(prompt);
|
||||||
|
if (prompt_len > VOICE_IMAGE_MARKER_PROMPT_MAX_LEN) {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"[stage] marker_image_gen_skip: prompt too long, len=%u",
|
||||||
|
(unsigned)prompt_len);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t repeat_remain_ms = 0;
|
||||||
|
if (!voice_marker_image_generation_try_acquire(prompt,
|
||||||
|
dialog_id,
|
||||||
|
voice_now_ms(),
|
||||||
|
&repeat_remain_ms)) {
|
||||||
|
if (repeat_remain_ms > 0) {
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_gen_skip: duplicate prompt suppressed, remain_ms=%u",
|
||||||
|
(unsigned)repeat_remain_ms);
|
||||||
|
} else {
|
||||||
|
ESP_LOGI(TAG, "[stage] marker_image_gen_skip: previous marker task running");
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
voice_image_generation_task_arg_t *task_arg =
|
||||||
|
(voice_image_generation_task_arg_t *)calloc(1, sizeof(*task_arg));
|
||||||
|
if (task_arg == NULL) {
|
||||||
|
ESP_LOGW(TAG, "[stage] marker_image_gen_skip: no memory for task arg");
|
||||||
|
voice_marker_image_generation_release();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
strlcpy(task_arg->prompt, prompt, sizeof(task_arg->prompt));
|
||||||
|
if (dialog_id != NULL) {
|
||||||
|
strlcpy(task_arg->dialog_id, dialog_id, sizeof(task_arg->dialog_id));
|
||||||
|
}
|
||||||
|
|
||||||
|
BaseType_t ok = xTaskCreate(voice_image_generation_task,
|
||||||
|
"voice_img_gen",
|
||||||
|
VOICE_IMAGE_GEN_TASK_STACK,
|
||||||
|
task_arg,
|
||||||
|
VOICE_IMAGE_GEN_TASK_PRIO,
|
||||||
|
NULL);
|
||||||
|
if (ok != pdPASS) {
|
||||||
|
ESP_LOGW(TAG, "[stage] marker_image_gen_skip: create task failed");
|
||||||
|
voice_marker_image_generation_release();
|
||||||
|
free(task_arg);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_gen_queued: dialog_id=%s prompt=%s",
|
||||||
|
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
||||||
|
task_arg->prompt);
|
||||||
|
}
|
||||||
|
|
||||||
|
void voice_marker_try_trigger_image_generation(const char *dialog_id, const char *final_text) {
|
||||||
|
if (final_text == NULL || final_text[0] == '\0') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
char first_sentence[VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN] = {0};
|
||||||
|
if (!voice_extract_first_sentence_text(final_text, first_sentence, sizeof(first_sentence))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
char marker_subject[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0};
|
||||||
|
if (!voice_extract_marker_prompt(first_sentence, marker_subject, sizeof(marker_subject))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
char templated_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0};
|
||||||
|
voice_build_marker_image_prompt(marker_subject,
|
||||||
|
templated_prompt,
|
||||||
|
sizeof(templated_prompt));
|
||||||
|
if (templated_prompt[0] == '\0') {
|
||||||
|
ESP_LOGW(TAG, "[stage] marker_image_gen_skip: build prompt failed");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_gen_detected: first_sentence=%s",
|
||||||
|
first_sentence);
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] marker_image_gen_subject: keyword=%s",
|
||||||
|
marker_subject);
|
||||||
|
voice_log_text_preview("marker_image_gen_prompt_templated", templated_prompt);
|
||||||
|
voice_schedule_marker_image_generation(templated_prompt, dialog_id);
|
||||||
|
}
|
||||||
@@ -1,8 +1,5 @@
|
|||||||
#include "voice_interaction_internal.h"
|
#include "voice_interaction_internal.h"
|
||||||
|
|
||||||
#include <ctype.h>
|
|
||||||
#include <inttypes.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
@@ -11,92 +8,10 @@
|
|||||||
#include "cJSON.h"
|
#include "cJSON.h"
|
||||||
#undef cJSON_realloc
|
#undef cJSON_realloc
|
||||||
#undef cJSON_malloc
|
#undef cJSON_malloc
|
||||||
#include "esp_audio_dec.h"
|
|
||||||
#include "esp_audio_enc.h"
|
|
||||||
#include "esp_log.h"
|
#include "esp_log.h"
|
||||||
#include "image_generation.h"
|
|
||||||
#include "printer_protocol.h"
|
|
||||||
#include "raster_tools.h"
|
|
||||||
#include "voice_audio.h"
|
|
||||||
|
|
||||||
static const char *TAG = "voice_interaction";
|
static const char *TAG = "voice_interaction";
|
||||||
|
|
||||||
#define VOICE_IMAGE_MARKER_PROMPT_MAX_LEN 800
|
|
||||||
#define VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN 1024
|
|
||||||
#define VOICE_IMAGE_GEN_TASK_STACK 16384
|
|
||||||
#define VOICE_IMAGE_GEN_TASK_PRIO 3
|
|
||||||
#define VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS 45000
|
|
||||||
#define VOICE_IMAGE_PRINT_THRESHOLD 160
|
|
||||||
#define VOICE_IMAGE_PRINT_MAX_HEIGHT 2200
|
|
||||||
#define VOICE_IMAGE_MARKER_TEMPLATE \
|
|
||||||
"%s,黑白线稿风格,白色背景,主体居中,边缘清晰,细节简洁,高对比,避免大面积阴影和灰度渐变,适配热敏打印机打印。"
|
|
||||||
#define VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT "简洁卡通图案"
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
char prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1];
|
|
||||||
char dialog_id[40];
|
|
||||||
} voice_image_generation_task_arg_t;
|
|
||||||
|
|
||||||
static portMUX_TYPE s_marker_image_gen_lock = portMUX_INITIALIZER_UNLOCKED;
|
|
||||||
static bool s_marker_image_gen_running;
|
|
||||||
static int64_t s_marker_last_success_ms;
|
|
||||||
static char s_marker_last_success_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1];
|
|
||||||
static char s_marker_last_success_dialog_id[40];
|
|
||||||
|
|
||||||
static bool voice_marker_image_generation_try_acquire(const char *prompt,
|
|
||||||
const char *dialog_id,
|
|
||||||
int64_t now_ms,
|
|
||||||
uint32_t *out_remain_ms) {
|
|
||||||
bool acquired = false;
|
|
||||||
uint32_t remain_ms = 0;
|
|
||||||
const char *safe_prompt = prompt != NULL ? prompt : "";
|
|
||||||
const char *safe_dialog = dialog_id != NULL ? dialog_id : "";
|
|
||||||
|
|
||||||
portENTER_CRITICAL(&s_marker_image_gen_lock);
|
|
||||||
if (s_marker_image_gen_running) {
|
|
||||||
acquired = false;
|
|
||||||
} else if (s_marker_last_success_ms > 0 &&
|
|
||||||
strcmp(safe_prompt, s_marker_last_success_prompt) == 0 &&
|
|
||||||
strcmp(safe_dialog, s_marker_last_success_dialog_id) == 0) {
|
|
||||||
int64_t elapsed_ms = now_ms - s_marker_last_success_ms;
|
|
||||||
if (elapsed_ms >= 0 && elapsed_ms < VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS) {
|
|
||||||
remain_ms = (uint32_t)(VOICE_IMAGE_MARKER_REPEAT_SUPPRESS_MS - elapsed_ms);
|
|
||||||
acquired = false;
|
|
||||||
} else {
|
|
||||||
s_marker_image_gen_running = true;
|
|
||||||
acquired = true;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
s_marker_image_gen_running = true;
|
|
||||||
acquired = true;
|
|
||||||
}
|
|
||||||
portEXIT_CRITICAL(&s_marker_image_gen_lock);
|
|
||||||
|
|
||||||
if (out_remain_ms != NULL) {
|
|
||||||
*out_remain_ms = remain_ms;
|
|
||||||
}
|
|
||||||
return acquired;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_marker_image_generation_release(void) {
|
|
||||||
portENTER_CRITICAL(&s_marker_image_gen_lock);
|
|
||||||
s_marker_image_gen_running = false;
|
|
||||||
portEXIT_CRITICAL(&s_marker_image_gen_lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_marker_image_generation_record_success(const char *prompt,
|
|
||||||
const char *dialog_id,
|
|
||||||
int64_t now_ms) {
|
|
||||||
const char *safe_prompt = prompt != NULL ? prompt : "";
|
|
||||||
const char *safe_dialog = dialog_id != NULL ? dialog_id : "";
|
|
||||||
|
|
||||||
portENTER_CRITICAL(&s_marker_image_gen_lock);
|
|
||||||
strlcpy(s_marker_last_success_prompt, safe_prompt, sizeof(s_marker_last_success_prompt));
|
|
||||||
strlcpy(s_marker_last_success_dialog_id, safe_dialog, sizeof(s_marker_last_success_dialog_id));
|
|
||||||
s_marker_last_success_ms = now_ms;
|
|
||||||
portEXIT_CRITICAL(&s_marker_image_gen_lock);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void *voice_cjson_malloc_hook(size_t size) {
|
static void *voice_cjson_malloc_hook(size_t size) {
|
||||||
return voice_malloc_prefer_psram(size);
|
return voice_malloc_prefer_psram(size);
|
||||||
}
|
}
|
||||||
@@ -253,679 +168,6 @@ esp_err_t voice_send_directive(const char *action,
|
|||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void voice_handle_downstream_packet(const uint8_t *data, size_t len) {
|
|
||||||
if (data == NULL || len == 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
void *opus_dec = s_voice.opus_dec;
|
|
||||||
uint8_t *pcm_buf = s_voice.pcm_rx_buf;
|
|
||||||
size_t pcm_buf_size = s_voice.pcm_rx_buf_size;
|
|
||||||
voice_unlock();
|
|
||||||
|
|
||||||
if (opus_dec == NULL || pcm_buf == NULL || pcm_buf_size == 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
esp_audio_dec_in_raw_t raw = {
|
|
||||||
.buffer = (uint8_t *)data,
|
|
||||||
.len = (uint32_t)len,
|
|
||||||
.consumed = 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
while (raw.len > 0) {
|
|
||||||
esp_audio_dec_out_frame_t out = {
|
|
||||||
.buffer = pcm_buf,
|
|
||||||
.len = (uint32_t)pcm_buf_size,
|
|
||||||
.needed_size = 0,
|
|
||||||
.decoded_size = 0,
|
|
||||||
};
|
|
||||||
esp_audio_dec_info_t dec_info = {0};
|
|
||||||
|
|
||||||
esp_audio_err_t ret = esp_opus_dec_decode(opus_dec, &raw, &out, &dec_info);
|
|
||||||
if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) {
|
|
||||||
if (out.needed_size > pcm_buf_size) {
|
|
||||||
uint8_t *new_buf = (uint8_t *)voice_realloc_prefer_psram(pcm_buf, out.needed_size);
|
|
||||||
if (new_buf == NULL) {
|
|
||||||
ESP_LOGE(TAG, "No memory to extend pcm rx buffer to %u", out.needed_size);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
s_voice.pcm_rx_buf = new_buf;
|
|
||||||
s_voice.pcm_rx_buf_size = out.needed_size;
|
|
||||||
voice_unlock();
|
|
||||||
}
|
|
||||||
pcm_buf = new_buf;
|
|
||||||
pcm_buf_size = out.needed_size;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ret != ESP_AUDIO_ERR_OK) {
|
|
||||||
ESP_LOGW(TAG, "opus decode failed: %d", ret);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (out.decoded_size > 0) {
|
|
||||||
char audio_err[64] = {0};
|
|
||||||
size_t samples = (size_t)out.decoded_size / VOICE_SAMPLE_BYTES;
|
|
||||||
esp_err_t write_err = voice_audio_write_pcm((const int16_t *)out.buffer,
|
|
||||||
samples,
|
|
||||||
VOICE_AUDIO_IO_TIMEOUT_MS,
|
|
||||||
audio_err,
|
|
||||||
sizeof(audio_err));
|
|
||||||
if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
int64_t now_ms = voice_now_ms();
|
|
||||||
int64_t pcm_ms = ((int64_t)samples * 1000 +
|
|
||||||
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) /
|
|
||||||
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE;
|
|
||||||
int64_t base_ms = s_voice.playback_deadline_ms > now_ms ?
|
|
||||||
s_voice.playback_deadline_ms : now_ms;
|
|
||||||
s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS;
|
|
||||||
s_voice.last_downstream_ms = now_ms;
|
|
||||||
voice_unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (raw.consumed == 0 || raw.consumed > raw.len) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
raw.buffer += raw.consumed;
|
|
||||||
raw.len -= raw.consumed;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
s_voice.downstream_packets++;
|
|
||||||
s_voice.last_event_ms = voice_now_ms();
|
|
||||||
voice_unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static const char *voice_get_json_str(cJSON *obj, const char *name) {
|
|
||||||
if (obj == NULL || name == NULL) {
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name);
|
|
||||||
if (cJSON_IsString(item) && item->valuestring != NULL) {
|
|
||||||
return item->valuestring;
|
|
||||||
}
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) {
|
|
||||||
if (obj == NULL || name == NULL || out_value == NULL) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name);
|
|
||||||
if (!cJSON_IsBool(item)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
*out_value = cJSON_IsTrue(item);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_log_final_text(const char *stage, const char *text) {
|
|
||||||
if (stage == NULL || text == NULL || text[0] == '\0') {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
size_t len = strlen(text);
|
|
||||||
const int preview = 240;
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] %s: len=%u text=%.*s%s",
|
|
||||||
stage,
|
|
||||||
(unsigned)len,
|
|
||||||
preview,
|
|
||||||
text,
|
|
||||||
(len > (size_t)preview) ? "..." : "");
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool voice_is_space_char(char c) {
|
|
||||||
return isspace((unsigned char)c) != 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t voice_find_first_sentence_end(const char *text, size_t len) {
|
|
||||||
if (text == NULL || len == 0) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (size_t i = 0; i < len; ++i) {
|
|
||||||
unsigned char c = (unsigned char)text[i];
|
|
||||||
if (c == '.' || c == '!' || c == '?' || c == '\n' || c == '\r') {
|
|
||||||
return i + 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (i + 2 < len) {
|
|
||||||
unsigned char b1 = (unsigned char)text[i];
|
|
||||||
unsigned char b2 = (unsigned char)text[i + 1];
|
|
||||||
unsigned char b3 = (unsigned char)text[i + 2];
|
|
||||||
|
|
||||||
if (b1 == 0xE3 && b2 == 0x80 && b3 == 0x82) {
|
|
||||||
return i + 3;
|
|
||||||
}
|
|
||||||
if (b1 == 0xEF && b2 == 0xBC &&
|
|
||||||
(b3 == 0x81 || b3 == 0x9F || b3 == 0x9B)) {
|
|
||||||
return i + 3;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return len;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool voice_extract_first_sentence_text(const char *text,
|
|
||||||
char *out_sentence,
|
|
||||||
size_t out_len) {
|
|
||||||
if (text == NULL || out_sentence == NULL || out_len == 0) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t len = strlen(text);
|
|
||||||
size_t begin = 0;
|
|
||||||
while (begin < len && voice_is_space_char(text[begin])) {
|
|
||||||
begin++;
|
|
||||||
}
|
|
||||||
if (begin >= len) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t end = begin + voice_find_first_sentence_end(text + begin, len - begin);
|
|
||||||
while (end > begin && voice_is_space_char(text[end - 1])) {
|
|
||||||
end--;
|
|
||||||
}
|
|
||||||
if (end <= begin) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t copy_len = end - begin;
|
|
||||||
if (copy_len >= out_len) {
|
|
||||||
copy_len = out_len - 1;
|
|
||||||
}
|
|
||||||
memcpy(out_sentence, text + begin, copy_len);
|
|
||||||
out_sentence[copy_len] = '\0';
|
|
||||||
return copy_len > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool voice_extract_marker_prompt(const char *text, char *out_prompt, size_t out_len) {
|
|
||||||
if (text == NULL || out_prompt == NULL || out_len == 0) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char *open = strstr(text, "/*");
|
|
||||||
if (open == NULL) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char *close = strstr(open + 2, "*/");
|
|
||||||
if (close == NULL) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char *begin = open + 2;
|
|
||||||
const char *end = close;
|
|
||||||
while (begin < end && voice_is_space_char(*begin)) {
|
|
||||||
begin++;
|
|
||||||
}
|
|
||||||
while (end > begin && voice_is_space_char(*(end - 1))) {
|
|
||||||
end--;
|
|
||||||
}
|
|
||||||
if (end <= begin) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t prompt_len = (size_t)(end - begin);
|
|
||||||
if (prompt_len >= out_len) {
|
|
||||||
prompt_len = out_len - 1;
|
|
||||||
}
|
|
||||||
memcpy(out_prompt, begin, prompt_len);
|
|
||||||
out_prompt[prompt_len] = '\0';
|
|
||||||
return prompt_len > 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_build_marker_image_prompt(const char *subject,
|
|
||||||
char *out_prompt,
|
|
||||||
size_t out_len) {
|
|
||||||
if (out_prompt == NULL || out_len == 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char *safe_subject = (subject != NULL && subject[0] != '\0') ?
|
|
||||||
subject :
|
|
||||||
VOICE_IMAGE_MARKER_TEMPLATE_FALLBACK_SUBJECT;
|
|
||||||
int written = snprintf(out_prompt,
|
|
||||||
out_len,
|
|
||||||
VOICE_IMAGE_MARKER_TEMPLATE,
|
|
||||||
safe_subject);
|
|
||||||
if (written < 0) {
|
|
||||||
out_prompt[0] = '\0';
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((size_t)written >= out_len) {
|
|
||||||
out_prompt[out_len - 1] = '\0';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static esp_err_t voice_submit_marker_generated_image(const voice_image_generation_task_arg_t *task_arg,
|
|
||||||
const image_generation_result_t *result,
|
|
||||||
char *err,
|
|
||||||
size_t err_len) {
|
|
||||||
if (task_arg == NULL || result == NULL || result->png == NULL || result->png_len == 0) {
|
|
||||||
voice_fill_err(err, err_len, "invalid image result");
|
|
||||||
return ESP_ERR_INVALID_ARG;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint16_t raster_width = 0;
|
|
||||||
uint16_t raster_height = 0;
|
|
||||||
uint8_t *raster = NULL;
|
|
||||||
size_t raster_len = 0;
|
|
||||||
char decode_err[128] = {0};
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_print_decode_start: dialog_id=%s request_id=%s png_bytes=%u",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
result->request_id[0] != '\0' ? result->request_id : "-",
|
|
||||||
(unsigned)result->png_len);
|
|
||||||
esp_err_t decode_rc = raster_tools_convert_png_to_raster_384(result->png,
|
|
||||||
result->png_len,
|
|
||||||
true,
|
|
||||||
VOICE_IMAGE_PRINT_THRESHOLD,
|
|
||||||
false,
|
|
||||||
VOICE_IMAGE_PRINT_MAX_HEIGHT,
|
|
||||||
&raster_width,
|
|
||||||
&raster_height,
|
|
||||||
&raster,
|
|
||||||
&raster_len,
|
|
||||||
decode_err,
|
|
||||||
sizeof(decode_err));
|
|
||||||
if (decode_rc != ESP_OK) {
|
|
||||||
voice_fill_err(err,
|
|
||||||
err_len,
|
|
||||||
decode_err[0] != '\0' ? decode_err : "decode generated image failed");
|
|
||||||
ESP_LOGW(TAG,
|
|
||||||
"[stage] marker_image_print_decode_failed: dialog_id=%s rc=0x%x msg=%s",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
(unsigned)decode_rc,
|
|
||||||
err != NULL && err[0] != '\0' ? err : "decode generated image failed");
|
|
||||||
return decode_rc;
|
|
||||||
}
|
|
||||||
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_print_decode_done: dialog_id=%s raster_size=%ux%u raster_bytes=%u",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
(unsigned)raster_width,
|
|
||||||
(unsigned)raster_height,
|
|
||||||
(unsigned)raster_len);
|
|
||||||
|
|
||||||
uint32_t job_id = 0;
|
|
||||||
char submit_err[128] = {0};
|
|
||||||
esp_err_t submit_rc = printer_protocol_submit_raster_job(raster,
|
|
||||||
raster_len,
|
|
||||||
raster_width,
|
|
||||||
raster_height,
|
|
||||||
NULL,
|
|
||||||
&job_id,
|
|
||||||
submit_err,
|
|
||||||
sizeof(submit_err));
|
|
||||||
if (submit_rc != ESP_OK) {
|
|
||||||
voice_fill_err(err,
|
|
||||||
err_len,
|
|
||||||
submit_err[0] != '\0' ? submit_err : "submit print job failed");
|
|
||||||
ESP_LOGW(TAG,
|
|
||||||
"[stage] marker_image_print_submit_failed: dialog_id=%s rc=0x%x msg=%s",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
(unsigned)submit_rc,
|
|
||||||
err != NULL && err[0] != '\0' ? err : "submit print job failed");
|
|
||||||
} else {
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_print_submit_ok: dialog_id=%s request_id=%s job_id=%u",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
result->request_id[0] != '\0' ? result->request_id : "-",
|
|
||||||
(unsigned)job_id);
|
|
||||||
}
|
|
||||||
|
|
||||||
free(raster);
|
|
||||||
return submit_rc;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_image_generation_task(void *arg) {
|
|
||||||
voice_image_generation_task_arg_t *task_arg = (voice_image_generation_task_arg_t *)arg;
|
|
||||||
if (task_arg == NULL) {
|
|
||||||
vTaskDelete(NULL);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
image_generation_request_t req = {
|
|
||||||
.prompt = task_arg->prompt,
|
|
||||||
.size = NULL,
|
|
||||||
.prompt_extend = false,
|
|
||||||
.has_seed = false,
|
|
||||||
.seed = 0,
|
|
||||||
.generation_timeout_ms = 0,
|
|
||||||
.download_timeout_ms = 0,
|
|
||||||
};
|
|
||||||
|
|
||||||
image_generation_result_t result = {0};
|
|
||||||
image_generation_result_reset(&result);
|
|
||||||
|
|
||||||
bool status_poll_paused = false;
|
|
||||||
char gen_err[160] = {0};
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_gen_start: dialog_id=%s prompt_len=%u",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
(unsigned)strlen(task_arg->prompt));
|
|
||||||
printer_protocol_set_status_poll_paused(true);
|
|
||||||
status_poll_paused = true;
|
|
||||||
esp_err_t rc = image_generation_generate_png(&req, &result, gen_err, sizeof(gen_err));
|
|
||||||
printer_protocol_set_status_poll_paused(false);
|
|
||||||
status_poll_paused = false;
|
|
||||||
if (rc == ESP_OK) {
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_gen_done: dialog_id=%s request_id=%s size=%ux%u png_bytes=%u",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
result.request_id[0] != '\0' ? result.request_id : "-",
|
|
||||||
(unsigned)result.width,
|
|
||||||
(unsigned)result.height,
|
|
||||||
(unsigned)result.png_len);
|
|
||||||
|
|
||||||
char submit_err[160] = {0};
|
|
||||||
esp_err_t submit_rc = voice_submit_marker_generated_image(task_arg,
|
|
||||||
&result,
|
|
||||||
submit_err,
|
|
||||||
sizeof(submit_err));
|
|
||||||
if (submit_rc == ESP_OK) {
|
|
||||||
voice_marker_image_generation_record_success(task_arg->prompt,
|
|
||||||
task_arg->dialog_id,
|
|
||||||
voice_now_ms());
|
|
||||||
} else {
|
|
||||||
ESP_LOGW(TAG,
|
|
||||||
"[stage] marker_image_print_failed: dialog_id=%s rc=0x%x msg=%s",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
(unsigned)submit_rc,
|
|
||||||
submit_err[0] != '\0' ? submit_err : "marker image print failed");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
ESP_LOGW(TAG,
|
|
||||||
"[stage] marker_image_gen_failed: dialog_id=%s rc=0x%x msg=%s",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
(unsigned)rc,
|
|
||||||
gen_err[0] != '\0' ? gen_err : "image generation failed");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status_poll_paused) {
|
|
||||||
printer_protocol_set_status_poll_paused(false);
|
|
||||||
}
|
|
||||||
image_generation_result_free(&result);
|
|
||||||
voice_marker_image_generation_release();
|
|
||||||
image_generation_schedule_prewarm();
|
|
||||||
free(task_arg);
|
|
||||||
vTaskDelete(NULL);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_schedule_marker_image_generation(const char *prompt, const char *dialog_id) {
|
|
||||||
if (prompt == NULL || prompt[0] == '\0') {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t prompt_len = strlen(prompt);
|
|
||||||
if (prompt_len > VOICE_IMAGE_MARKER_PROMPT_MAX_LEN) {
|
|
||||||
ESP_LOGW(TAG,
|
|
||||||
"[stage] marker_image_gen_skip: prompt too long, len=%u",
|
|
||||||
(unsigned)prompt_len);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint32_t repeat_remain_ms = 0;
|
|
||||||
if (!voice_marker_image_generation_try_acquire(prompt,
|
|
||||||
dialog_id,
|
|
||||||
voice_now_ms(),
|
|
||||||
&repeat_remain_ms)) {
|
|
||||||
if (repeat_remain_ms > 0) {
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_gen_skip: duplicate prompt suppressed, remain_ms=%u",
|
|
||||||
(unsigned)repeat_remain_ms);
|
|
||||||
} else {
|
|
||||||
ESP_LOGI(TAG, "[stage] marker_image_gen_skip: previous marker task running");
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
voice_image_generation_task_arg_t *task_arg =
|
|
||||||
(voice_image_generation_task_arg_t *)calloc(1, sizeof(*task_arg));
|
|
||||||
if (task_arg == NULL) {
|
|
||||||
ESP_LOGW(TAG, "[stage] marker_image_gen_skip: no memory for task arg");
|
|
||||||
voice_marker_image_generation_release();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
strlcpy(task_arg->prompt, prompt, sizeof(task_arg->prompt));
|
|
||||||
if (dialog_id != NULL) {
|
|
||||||
strlcpy(task_arg->dialog_id, dialog_id, sizeof(task_arg->dialog_id));
|
|
||||||
}
|
|
||||||
|
|
||||||
BaseType_t ok = xTaskCreate(voice_image_generation_task,
|
|
||||||
"voice_img_gen",
|
|
||||||
VOICE_IMAGE_GEN_TASK_STACK,
|
|
||||||
task_arg,
|
|
||||||
VOICE_IMAGE_GEN_TASK_PRIO,
|
|
||||||
NULL);
|
|
||||||
if (ok != pdPASS) {
|
|
||||||
ESP_LOGW(TAG, "[stage] marker_image_gen_skip: create task failed");
|
|
||||||
voice_marker_image_generation_release();
|
|
||||||
free(task_arg);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_gen_queued: dialog_id=%s prompt=%s",
|
|
||||||
task_arg->dialog_id[0] != '\0' ? task_arg->dialog_id : "-",
|
|
||||||
task_arg->prompt);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_try_trigger_marker_image_generation(const char *dialog_id, const char *final_text) {
|
|
||||||
if (final_text == NULL || final_text[0] == '\0') {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
char first_sentence[VOICE_IMAGE_MARKER_FIRST_SENTENCE_MAX_LEN] = {0};
|
|
||||||
if (!voice_extract_first_sentence_text(final_text, first_sentence, sizeof(first_sentence))) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
char marker_subject[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0};
|
|
||||||
if (!voice_extract_marker_prompt(first_sentence, marker_subject, sizeof(marker_subject))) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
char templated_prompt[VOICE_IMAGE_MARKER_PROMPT_MAX_LEN + 1] = {0};
|
|
||||||
voice_build_marker_image_prompt(marker_subject,
|
|
||||||
templated_prompt,
|
|
||||||
sizeof(templated_prompt));
|
|
||||||
if (templated_prompt[0] == '\0') {
|
|
||||||
ESP_LOGW(TAG, "[stage] marker_image_gen_skip: build prompt failed");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_gen_detected: first_sentence=%s",
|
|
||||||
first_sentence);
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] marker_image_gen_subject: keyword=%s",
|
|
||||||
marker_subject);
|
|
||||||
voice_log_final_text("marker_image_gen_prompt_templated", templated_prompt);
|
|
||||||
voice_schedule_marker_image_generation(templated_prompt, dialog_id);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_handle_output_event(cJSON *output) {
|
|
||||||
const char *event_name = voice_get_json_str(output, "event");
|
|
||||||
if (event_name == NULL) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const char *dialog_id = voice_get_json_str(output, "dialog_id");
|
|
||||||
const char *state = NULL;
|
|
||||||
if (strcmp(event_name, "DialogStateChanged") == 0) {
|
|
||||||
state = voice_get_json_str(output, "state");
|
|
||||||
}
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] ws_event_output: event=%s dialog_id=%s state=%s",
|
|
||||||
event_name,
|
|
||||||
dialog_id != NULL ? dialog_id : "-",
|
|
||||||
state != NULL ? state : "-");
|
|
||||||
|
|
||||||
bool finished = false;
|
|
||||||
bool has_finished = voice_get_json_bool(output, "finished", &finished);
|
|
||||||
const char *responding_final_text = NULL;
|
|
||||||
bool should_check_marker_in_first_sentence = false;
|
|
||||||
if (has_finished && finished) {
|
|
||||||
if (strcmp(event_name, "SpeechContent") == 0) {
|
|
||||||
voice_log_final_text("asr_final_text", voice_get_json_str(output, "text"));
|
|
||||||
} else if (strcmp(event_name, "RespondingContent") == 0) {
|
|
||||||
const char *final_text = voice_get_json_str(output, "text");
|
|
||||||
if (final_text == NULL || final_text[0] == '\0') {
|
|
||||||
final_text = voice_get_json_str(output, "spoken");
|
|
||||||
}
|
|
||||||
voice_log_final_text("response_final_text", final_text);
|
|
||||||
responding_final_text = final_text;
|
|
||||||
if (responding_final_text != NULL && responding_final_text[0] != '\0' &&
|
|
||||||
voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
if (!s_voice.response_first_sentence_checked) {
|
|
||||||
s_voice.response_first_sentence_checked = true;
|
|
||||||
should_check_marker_in_first_sentence = true;
|
|
||||||
}
|
|
||||||
voice_unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bool should_close_audio = false;
|
|
||||||
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
if (dialog_id != NULL) {
|
|
||||||
strlcpy(s_voice.dialog_id, dialog_id, sizeof(s_voice.dialog_id));
|
|
||||||
}
|
|
||||||
s_voice.last_event_ms = voice_now_ms();
|
|
||||||
|
|
||||||
if (strcmp(event_name, "Started") == 0) {
|
|
||||||
s_voice.started = true;
|
|
||||||
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
|
|
||||||
s_voice.response_first_sentence_checked = false;
|
|
||||||
ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id);
|
|
||||||
} else if (strcmp(event_name, "DialogStateChanged") == 0) {
|
|
||||||
if (state != NULL) {
|
|
||||||
voice_dialog_state_t prev_state = s_voice.dialog_state;
|
|
||||||
if (strcmp(state, "Listening") == 0) {
|
|
||||||
s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING;
|
|
||||||
if (prev_state != VOICE_DIALOG_STATE_LISTENING) {
|
|
||||||
s_voice.response_first_sentence_checked = false;
|
|
||||||
}
|
|
||||||
} else if (strcmp(state, "Thinking") == 0) {
|
|
||||||
s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING;
|
|
||||||
if (prev_state != VOICE_DIALOG_STATE_THINKING) {
|
|
||||||
s_voice.response_first_sentence_checked = false;
|
|
||||||
}
|
|
||||||
} else if (strcmp(state, "Responding") == 0) {
|
|
||||||
s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING;
|
|
||||||
if (prev_state != VOICE_DIALOG_STATE_RESPONDING) {
|
|
||||||
s_voice.response_first_sentence_checked = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ESP_LOGI(TAG,
|
|
||||||
"[stage] dialog_state_changed: state=%s current=%s",
|
|
||||||
state != NULL ? state : "-",
|
|
||||||
voice_interaction_dialog_state_str(s_voice.dialog_state));
|
|
||||||
} else if (strcmp(event_name, "SpeechEnded") == 0) {
|
|
||||||
ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active);
|
|
||||||
} else if (strcmp(event_name, "Stopped") == 0) {
|
|
||||||
s_voice.started = false;
|
|
||||||
s_voice.tap_active = false;
|
|
||||||
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
|
|
||||||
s_voice.response_first_sentence_checked = false;
|
|
||||||
should_close_audio = true;
|
|
||||||
ESP_LOGI(TAG, "[stage] session_stopped_by_server");
|
|
||||||
} else if (strcmp(event_name, "Error") == 0) {
|
|
||||||
const char *error_msg = voice_get_json_str(output, "error_message");
|
|
||||||
if (error_msg != NULL) {
|
|
||||||
voice_set_last_error_locked(error_msg);
|
|
||||||
ESP_LOGW(TAG, "[stage] server_error: %s", error_msg);
|
|
||||||
}
|
|
||||||
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
|
|
||||||
s_voice.tap_active = false;
|
|
||||||
s_voice.response_first_sentence_checked = false;
|
|
||||||
should_close_audio = true;
|
|
||||||
}
|
|
||||||
voice_unlock();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (should_close_audio) {
|
|
||||||
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (should_check_marker_in_first_sentence) {
|
|
||||||
voice_try_trigger_marker_image_generation(dialog_id, responding_final_text);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (strcmp(event_name, "RespondingStarted") == 0) {
|
|
||||||
(void)voice_send_directive("continue-task", "LocalRespondingStarted", true);
|
|
||||||
} else if (strcmp(event_name, "RespondingEnded") == 0) {
|
|
||||||
(void)voice_send_directive("continue-task", "LocalRespondingEnded", true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_handle_text_message(const char *text, size_t len) {
|
|
||||||
voice_cjson_init_hooks_once();
|
|
||||||
|
|
||||||
cJSON *root = cJSON_ParseWithLength(text, len);
|
|
||||||
if (root == NULL) {
|
|
||||||
ESP_LOGW(TAG, "Invalid ws text payload");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header");
|
|
||||||
const char *header_event = voice_get_json_str(header, "event");
|
|
||||||
|
|
||||||
cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload");
|
|
||||||
cJSON *output = NULL;
|
|
||||||
if (payload != NULL) {
|
|
||||||
output = cJSON_GetObjectItemCaseSensitive(payload, "output");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (header != NULL) {
|
|
||||||
cJSON *status_code = cJSON_GetObjectItemCaseSensitive(header, "status_code");
|
|
||||||
cJSON *status_msg = cJSON_GetObjectItemCaseSensitive(header, "status_message");
|
|
||||||
if (cJSON_IsNumber(status_code) && status_code->valueint >= 400) {
|
|
||||||
ESP_LOGW(TAG,
|
|
||||||
"[stage] ws_header_error: status_code=%d status_message=%s",
|
|
||||||
status_code->valueint,
|
|
||||||
(cJSON_IsString(status_msg) && status_msg->valuestring != NULL) ? status_msg->valuestring : "-");
|
|
||||||
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
if (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) {
|
|
||||||
voice_set_last_error_locked(status_msg->valuestring);
|
|
||||||
} else {
|
|
||||||
voice_set_last_error_locked("websocket status error");
|
|
||||||
}
|
|
||||||
voice_unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (output != NULL) {
|
|
||||||
voice_handle_output_event(output);
|
|
||||||
} else if (header_event != NULL && strcmp(header_event, "task-failed") == 0) {
|
|
||||||
const char *error_msg = voice_get_json_str(header, "error_message");
|
|
||||||
ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed");
|
|
||||||
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
|
||||||
voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed");
|
|
||||||
voice_unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cJSON_Delete(root);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void voice_handle_text_chunk(esp_websocket_event_data_t *data) {
|
static void voice_handle_text_chunk(esp_websocket_event_data_t *data) {
|
||||||
if (data->payload_offset == 0) {
|
if (data->payload_offset == 0) {
|
||||||
free(s_voice.text_agg);
|
free(s_voice.text_agg);
|
||||||
@@ -963,7 +205,7 @@ static void voice_handle_text_chunk(esp_websocket_event_data_t *data) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
voice_handle_text_message((const char *)s_voice.text_agg, s_voice.text_agg_size);
|
voice_ws_protocol_handle_text_message((const char *)s_voice.text_agg, s_voice.text_agg_size);
|
||||||
free(s_voice.text_agg);
|
free(s_voice.text_agg);
|
||||||
s_voice.text_agg = NULL;
|
s_voice.text_agg = NULL;
|
||||||
s_voice.text_agg_size = 0;
|
s_voice.text_agg_size = 0;
|
||||||
@@ -1006,7 +248,7 @@ static void voice_handle_binary_chunk(esp_websocket_event_data_t *data) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
voice_handle_downstream_packet(s_voice.bin_agg, s_voice.bin_agg_size);
|
voice_ws_audio_handle_downstream_packet(s_voice.bin_agg, s_voice.bin_agg_size);
|
||||||
free(s_voice.bin_agg);
|
free(s_voice.bin_agg);
|
||||||
s_voice.bin_agg = NULL;
|
s_voice.bin_agg = NULL;
|
||||||
s_voice.bin_agg_size = 0;
|
s_voice.bin_agg_size = 0;
|
||||||
|
|||||||
100
main/domain/src/voice_interaction_ws_audio.c
Normal file
100
main/domain/src/voice_interaction_ws_audio.c
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
#include "voice_interaction_internal.h"
|
||||||
|
|
||||||
|
#include "esp_audio_dec.h"
|
||||||
|
#include "esp_log.h"
|
||||||
|
#include "voice_audio.h"
|
||||||
|
|
||||||
|
static const char *TAG = "voice_interaction";
|
||||||
|
|
||||||
|
void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len) {
|
||||||
|
if (data == NULL || len == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
void *opus_dec = s_voice.opus_dec;
|
||||||
|
uint8_t *pcm_buf = s_voice.pcm_rx_buf;
|
||||||
|
size_t pcm_buf_size = s_voice.pcm_rx_buf_size;
|
||||||
|
voice_unlock();
|
||||||
|
|
||||||
|
if (opus_dec == NULL || pcm_buf == NULL || pcm_buf_size == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
esp_audio_dec_in_raw_t raw = {
|
||||||
|
.buffer = (uint8_t *)data,
|
||||||
|
.len = (uint32_t)len,
|
||||||
|
.consumed = 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
while (raw.len > 0) {
|
||||||
|
esp_audio_dec_out_frame_t out = {
|
||||||
|
.buffer = pcm_buf,
|
||||||
|
.len = (uint32_t)pcm_buf_size,
|
||||||
|
.needed_size = 0,
|
||||||
|
.decoded_size = 0,
|
||||||
|
};
|
||||||
|
esp_audio_dec_info_t dec_info = {0};
|
||||||
|
|
||||||
|
esp_audio_err_t ret = esp_opus_dec_decode(opus_dec, &raw, &out, &dec_info);
|
||||||
|
if (ret == ESP_AUDIO_ERR_BUFF_NOT_ENOUGH) {
|
||||||
|
if (out.needed_size > pcm_buf_size) {
|
||||||
|
uint8_t *new_buf = (uint8_t *)voice_realloc_prefer_psram(pcm_buf, out.needed_size);
|
||||||
|
if (new_buf == NULL) {
|
||||||
|
ESP_LOGE(TAG, "No memory to extend pcm rx buffer to %u", out.needed_size);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
s_voice.pcm_rx_buf = new_buf;
|
||||||
|
s_voice.pcm_rx_buf_size = out.needed_size;
|
||||||
|
voice_unlock();
|
||||||
|
}
|
||||||
|
pcm_buf = new_buf;
|
||||||
|
pcm_buf_size = out.needed_size;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ret != ESP_AUDIO_ERR_OK) {
|
||||||
|
ESP_LOGW(TAG, "opus decode failed: %d", ret);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (out.decoded_size > 0) {
|
||||||
|
char audio_err[64] = {0};
|
||||||
|
size_t samples = (size_t)out.decoded_size / VOICE_SAMPLE_BYTES;
|
||||||
|
esp_err_t write_err = voice_audio_write_pcm((const int16_t *)out.buffer,
|
||||||
|
samples,
|
||||||
|
VOICE_AUDIO_IO_TIMEOUT_MS,
|
||||||
|
audio_err,
|
||||||
|
sizeof(audio_err));
|
||||||
|
if (write_err == ESP_OK && voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
int64_t now_ms = voice_now_ms();
|
||||||
|
int64_t pcm_ms = ((int64_t)samples * 1000 +
|
||||||
|
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE - 1) /
|
||||||
|
(int64_t)CONFIG_TQ_VOICE_SAMPLE_RATE;
|
||||||
|
int64_t base_ms = s_voice.playback_deadline_ms > now_ms ?
|
||||||
|
s_voice.playback_deadline_ms : now_ms;
|
||||||
|
s_voice.playback_deadline_ms = base_ms + pcm_ms + VOICE_AUDIO_DRAIN_MARGIN_MS;
|
||||||
|
s_voice.last_downstream_ms = now_ms;
|
||||||
|
voice_unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (raw.consumed == 0 || raw.consumed > raw.len) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
raw.buffer += raw.consumed;
|
||||||
|
raw.len -= raw.consumed;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
s_voice.downstream_packets++;
|
||||||
|
s_voice.last_event_ms = voice_now_ms();
|
||||||
|
voice_unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
224
main/domain/src/voice_interaction_ws_protocol.c
Normal file
224
main/domain/src/voice_interaction_ws_protocol.c
Normal file
@@ -0,0 +1,224 @@
|
|||||||
|
#include "voice_interaction_internal.h"
|
||||||
|
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#define cJSON_malloc voice_malloc_prefer_psram
|
||||||
|
#define cJSON_realloc voice_realloc_prefer_psram
|
||||||
|
#include "cJSON.h"
|
||||||
|
#undef cJSON_realloc
|
||||||
|
#undef cJSON_malloc
|
||||||
|
#include "esp_log.h"
|
||||||
|
|
||||||
|
static const char *TAG = "voice_interaction";
|
||||||
|
|
||||||
|
static const char *voice_get_json_str(cJSON *obj, const char *name) {
|
||||||
|
if (obj == NULL || name == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name);
|
||||||
|
if (cJSON_IsString(item) && item->valuestring != NULL) {
|
||||||
|
return item->valuestring;
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool voice_get_json_bool(cJSON *obj, const char *name, bool *out_value) {
|
||||||
|
if (obj == NULL || name == NULL || out_value == NULL) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name);
|
||||||
|
if (!cJSON_IsBool(item)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*out_value = cJSON_IsTrue(item);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_log_final_text(const char *stage, const char *text) {
|
||||||
|
if (stage == NULL || text == NULL || text[0] == '\0') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t len = strlen(text);
|
||||||
|
const int preview = 240;
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] %s: len=%u text=%.*s%s",
|
||||||
|
stage,
|
||||||
|
(unsigned)len,
|
||||||
|
preview,
|
||||||
|
text,
|
||||||
|
(len > (size_t)preview) ? "..." : "");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void voice_handle_output_event(cJSON *output) {
|
||||||
|
const char *event_name = voice_get_json_str(output, "event");
|
||||||
|
if (event_name == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *dialog_id = voice_get_json_str(output, "dialog_id");
|
||||||
|
const char *state = NULL;
|
||||||
|
if (strcmp(event_name, "DialogStateChanged") == 0) {
|
||||||
|
state = voice_get_json_str(output, "state");
|
||||||
|
}
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] ws_event_output: event=%s dialog_id=%s state=%s",
|
||||||
|
event_name,
|
||||||
|
dialog_id != NULL ? dialog_id : "-",
|
||||||
|
state != NULL ? state : "-");
|
||||||
|
|
||||||
|
bool finished = false;
|
||||||
|
bool has_finished = voice_get_json_bool(output, "finished", &finished);
|
||||||
|
const char *responding_final_text = NULL;
|
||||||
|
bool should_check_marker_in_first_sentence = false;
|
||||||
|
if (has_finished && finished) {
|
||||||
|
if (strcmp(event_name, "SpeechContent") == 0) {
|
||||||
|
voice_log_final_text("asr_final_text", voice_get_json_str(output, "text"));
|
||||||
|
} else if (strcmp(event_name, "RespondingContent") == 0) {
|
||||||
|
const char *final_text = voice_get_json_str(output, "text");
|
||||||
|
if (final_text == NULL || final_text[0] == '\0') {
|
||||||
|
final_text = voice_get_json_str(output, "spoken");
|
||||||
|
}
|
||||||
|
voice_log_final_text("response_final_text", final_text);
|
||||||
|
responding_final_text = final_text;
|
||||||
|
if (responding_final_text != NULL && responding_final_text[0] != '\0' &&
|
||||||
|
voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
if (!s_voice.response_first_sentence_checked) {
|
||||||
|
s_voice.response_first_sentence_checked = true;
|
||||||
|
should_check_marker_in_first_sentence = true;
|
||||||
|
}
|
||||||
|
voice_unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool should_close_audio = false;
|
||||||
|
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
if (dialog_id != NULL) {
|
||||||
|
strlcpy(s_voice.dialog_id, dialog_id, sizeof(s_voice.dialog_id));
|
||||||
|
}
|
||||||
|
s_voice.last_event_ms = voice_now_ms();
|
||||||
|
|
||||||
|
if (strcmp(event_name, "Started") == 0) {
|
||||||
|
s_voice.started = true;
|
||||||
|
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
|
||||||
|
s_voice.response_first_sentence_checked = false;
|
||||||
|
ESP_LOGI(TAG, "[stage] session_started: dialog_id=%s", s_voice.dialog_id);
|
||||||
|
} else if (strcmp(event_name, "DialogStateChanged") == 0) {
|
||||||
|
if (state != NULL) {
|
||||||
|
voice_dialog_state_t prev_state = s_voice.dialog_state;
|
||||||
|
if (strcmp(state, "Listening") == 0) {
|
||||||
|
s_voice.dialog_state = VOICE_DIALOG_STATE_LISTENING;
|
||||||
|
if (prev_state != VOICE_DIALOG_STATE_LISTENING) {
|
||||||
|
s_voice.response_first_sentence_checked = false;
|
||||||
|
}
|
||||||
|
} else if (strcmp(state, "Thinking") == 0) {
|
||||||
|
s_voice.dialog_state = VOICE_DIALOG_STATE_THINKING;
|
||||||
|
if (prev_state != VOICE_DIALOG_STATE_THINKING) {
|
||||||
|
s_voice.response_first_sentence_checked = false;
|
||||||
|
}
|
||||||
|
} else if (strcmp(state, "Responding") == 0) {
|
||||||
|
s_voice.dialog_state = VOICE_DIALOG_STATE_RESPONDING;
|
||||||
|
if (prev_state != VOICE_DIALOG_STATE_RESPONDING) {
|
||||||
|
s_voice.response_first_sentence_checked = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"[stage] dialog_state_changed: state=%s current=%s",
|
||||||
|
state != NULL ? state : "-",
|
||||||
|
voice_interaction_dialog_state_str(s_voice.dialog_state));
|
||||||
|
} else if (strcmp(event_name, "SpeechEnded") == 0) {
|
||||||
|
ESP_LOGI(TAG, "[stage] speech_ended: keep tap_active=%d for continuous rounds", s_voice.tap_active);
|
||||||
|
} else if (strcmp(event_name, "Stopped") == 0) {
|
||||||
|
s_voice.started = false;
|
||||||
|
s_voice.tap_active = false;
|
||||||
|
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
|
||||||
|
s_voice.response_first_sentence_checked = false;
|
||||||
|
should_close_audio = true;
|
||||||
|
ESP_LOGI(TAG, "[stage] session_stopped_by_server");
|
||||||
|
} else if (strcmp(event_name, "Error") == 0) {
|
||||||
|
const char *error_msg = voice_get_json_str(output, "error_message");
|
||||||
|
if (error_msg != NULL) {
|
||||||
|
voice_set_last_error_locked(error_msg);
|
||||||
|
ESP_LOGW(TAG, "[stage] server_error: %s", error_msg);
|
||||||
|
}
|
||||||
|
s_voice.dialog_state = VOICE_DIALOG_STATE_IDLE;
|
||||||
|
s_voice.tap_active = false;
|
||||||
|
s_voice.response_first_sentence_checked = false;
|
||||||
|
should_close_audio = true;
|
||||||
|
}
|
||||||
|
voice_unlock();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (should_close_audio) {
|
||||||
|
voice_close_audio_with_drain(VOICE_AUDIO_DRAIN_WAIT_MS, "server_event_close");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (should_check_marker_in_first_sentence) {
|
||||||
|
voice_marker_try_trigger_image_generation(dialog_id, responding_final_text);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (strcmp(event_name, "RespondingStarted") == 0) {
|
||||||
|
(void)voice_send_directive("continue-task", "LocalRespondingStarted", true);
|
||||||
|
} else if (strcmp(event_name, "RespondingEnded") == 0) {
|
||||||
|
(void)voice_send_directive("continue-task", "LocalRespondingEnded", true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void voice_ws_protocol_handle_text_message(const char *text, size_t len) {
|
||||||
|
if (text == NULL || len == 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
voice_interaction_ws_init_cjson_hooks();
|
||||||
|
|
||||||
|
cJSON *root = cJSON_ParseWithLength(text, len);
|
||||||
|
if (root == NULL) {
|
||||||
|
ESP_LOGW(TAG, "Invalid ws text payload");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
cJSON *header = cJSON_GetObjectItemCaseSensitive(root, "header");
|
||||||
|
const char *header_event = voice_get_json_str(header, "event");
|
||||||
|
|
||||||
|
cJSON *payload = cJSON_GetObjectItemCaseSensitive(root, "payload");
|
||||||
|
cJSON *output = NULL;
|
||||||
|
if (payload != NULL) {
|
||||||
|
output = cJSON_GetObjectItemCaseSensitive(payload, "output");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header != NULL) {
|
||||||
|
cJSON *status_code = cJSON_GetObjectItemCaseSensitive(header, "status_code");
|
||||||
|
cJSON *status_msg = cJSON_GetObjectItemCaseSensitive(header, "status_message");
|
||||||
|
if (cJSON_IsNumber(status_code) && status_code->valueint >= 400) {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"[stage] ws_header_error: status_code=%d status_message=%s",
|
||||||
|
status_code->valueint,
|
||||||
|
(cJSON_IsString(status_msg) && status_msg->valuestring != NULL) ? status_msg->valuestring : "-");
|
||||||
|
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
if (cJSON_IsString(status_msg) && status_msg->valuestring != NULL) {
|
||||||
|
voice_set_last_error_locked(status_msg->valuestring);
|
||||||
|
} else {
|
||||||
|
voice_set_last_error_locked("websocket status error");
|
||||||
|
}
|
||||||
|
voice_unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (output != NULL) {
|
||||||
|
voice_handle_output_event(output);
|
||||||
|
} else if (header_event != NULL && strcmp(header_event, "task-failed") == 0) {
|
||||||
|
const char *error_msg = voice_get_json_str(header, "error_message");
|
||||||
|
ESP_LOGW(TAG, "[stage] ws_task_failed: %s", error_msg != NULL ? error_msg : "task failed");
|
||||||
|
if (voice_lock(VOICE_STATUS_LOCK_TIMEOUT_MS)) {
|
||||||
|
voice_set_last_error_locked(error_msg != NULL ? error_msg : "task failed");
|
||||||
|
voice_unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cJSON_Delete(root);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user