From e5c1564130f8c5d18dbf66ecca6fdff982aa6460 Mon Sep 17 00:00:00 2001 From: chenjintao Date: Mon, 10 Aug 2026 16:46:55 +0800 Subject: [PATCH] feat(voice): add on-device Miao Zhang Gui wake word Add an optional local wake response prompt configuration. The prompt asset is intentionally not included yet. --- components/control_plane/CMakeLists.txt | 1 + .../internal/control_plane_internal.h | 5 + .../control_plane_multimodal_local_wake.c | 384 +++++++ .../control_plane_multimodal_runtime.c | 10 + ...control_plane_multimodal_session_service.c | 8 +- .../runtime/control_plane_runtime_workers.c | 31 + components/domain/CMakeLists.txt | 2 + components/domain/include/domain.h | 42 + .../domain/src/runtime/domain_local_command.c | 976 ++++++++++++++++++ components/platform/idf_component.yml | 1 + components/platform/include/platform.h | 24 + .../internal/multimodal/audio_pipeline.h | 11 + .../platform/src/multimodal/audio_pipeline.c | 33 +- .../audio_pipeline_codec_impl.c | 86 +- .../audio_pipeline_helpers_impl.c | 9 +- .../audio_pipeline_runtime_impl.c | 194 +++- .../src/runtime/platform_multimodal_runtime.c | 18 + dependencies.lock | 55 +- main/Kconfig.projbuild | 116 ++- sdkconfig.defaults | 20 +- tools/check_architecture_guards.sh | 40 +- 21 files changed, 2010 insertions(+), 56 deletions(-) create mode 100644 components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c create mode 100644 components/domain/src/runtime/domain_local_command.c diff --git a/components/control_plane/CMakeLists.txt b/components/control_plane/CMakeLists.txt index 5ab7e78..d8e6779 100644 --- a/components/control_plane/CMakeLists.txt +++ b/components/control_plane/CMakeLists.txt @@ -6,6 +6,7 @@ idf_component_register( "src/multimodal/control_plane_model_command_extractor.c" "src/multimodal/control_plane_multimodal_protocol.c" "src/multimodal/control_plane_multimodal_runtime.c" + "src/multimodal/control_plane_multimodal_local_wake.c" "src/multimodal/control_plane_multimodal_device_auth.c" "src/multimodal/control_plane_multimodal_auth_service.c" "src/multimodal/control_plane_multimodal_session_service.c" diff --git a/components/control_plane/internal/control_plane_internal.h b/components/control_plane/internal/control_plane_internal.h index 59c8005..3b3f9b9 100644 --- a/components/control_plane/internal/control_plane_internal.h +++ b/components/control_plane/internal/control_plane_internal.h @@ -133,6 +133,11 @@ esp_err_t control_plane_multimodal_play_downstream_audio(const uint8_t *data, si esp_err_t control_plane_multimodal_handle_ptt_press_step(void); esp_err_t control_plane_multimodal_handle_ptt_release(void); void control_plane_multimodal_reset_uplink_failure_window(void); +esp_err_t control_plane_local_wake_start(void); +esp_err_t control_plane_local_wake_stop(int timeout_ms); +void control_plane_local_wake_pause_for_foreground(void); +void control_plane_local_wake_resume_after_foreground(void); +bool control_plane_voice_turn_is_idle(void); esp_err_t control_plane_start_system_event_task(void); void control_plane_stop_system_event_task(void); diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c new file mode 100644 index 0000000..5b930c4 --- /dev/null +++ b/components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c @@ -0,0 +1,384 @@ +#include "control_plane_internal.h" + +#include + +#include "domain.h" +#include "esp_log.h" +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "freertos/task.h" +#include "sdkconfig.h" + +#ifndef CONFIG_TQ_LOCAL_WAKE_ENABLE +#define CONFIG_TQ_LOCAL_WAKE_ENABLE 0 +#endif + +#ifndef CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE +#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE 0 +#endif + +#ifndef CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE +#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE \ + "/littlefs/audio_prompts/local_wake_response.opus" +#endif + +#ifndef CONFIG_TQ_LOCAL_WAKE_RETRY_MS +#define CONFIG_TQ_LOCAL_WAKE_RETRY_MS 5000 +#endif + +#ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS +#define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000 +#endif + +#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES +#define CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES 6144 +#endif + +#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY +#define CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY 4 +#endif + +#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_CORE +#define CONFIG_TQ_LOCAL_WAKE_TASK_CORE 0 +#endif + +#define CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN 4U +#define CONTROL_PLANE_LOCAL_WAKE_POLL_MS 100U + +#if CONFIG_TQ_LOCAL_WAKE_TASK_CORE < 0 +#define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE tskNO_AFFINITY +#else +#define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE CONFIG_TQ_LOCAL_WAKE_TASK_CORE +#endif + +typedef struct { + TaskHandle_t task; + QueueHandle_t detection_queue; + bool stop_requested; + bool started; + bool paused_for_foreground; + bool capture_owned; +} local_wake_state_t; + +static const char *TAG = "cp_local_wake"; +static local_wake_state_t s_local_wake; +static portMUX_TYPE s_local_wake_lock = portMUX_INITIALIZER_UNLOCKED; + +static local_wake_state_t local_wake_snapshot(void) +{ + local_wake_state_t snapshot; + portENTER_CRITICAL(&s_local_wake_lock); + snapshot = s_local_wake; + portEXIT_CRITICAL(&s_local_wake_lock); + return snapshot; +} + +static void local_wake_set_capture_owned(bool owned) +{ + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.capture_owned = owned; + portEXIT_CRITICAL(&s_local_wake_lock); +} + +static void local_wake_domain_event( + const domain_local_command_event_t *event, + void *user_data) +{ + (void)user_data; + local_wake_state_t state = local_wake_snapshot(); + if (event == NULL || state.stop_requested || state.detection_queue == NULL) { + return; + } + if (xQueueSend(state.detection_queue, event, 0U) != pdTRUE) { + ESP_EARLY_LOGW(TAG, "Drop local wake detection: queue full"); + } +} + +static void local_wake_stop_owned_capture(void) +{ + local_wake_state_t state = local_wake_snapshot(); + if (!state.capture_owned) { + return; + } + local_wake_set_capture_owned(false); + esp_err_t err = domain_runtime_multimodal_audio_stop_uplink( + CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); + if (err != ESP_OK && err != ESP_ERR_TIMEOUT) { + ESP_LOGW(TAG, "Stop local wake capture failed: %s", esp_err_to_name(err)); + } +} + +static esp_err_t local_wake_start_listening(void) +{ + local_wake_state_t state = local_wake_snapshot(); + if (state.stop_requested || state.paused_for_foreground || + !control_plane_voice_turn_is_idle()) { + return ESP_ERR_INVALID_STATE; + } + + if (!domain_runtime_multimodal_audio_is_initialized()) { + esp_err_t init_err = domain_runtime_multimodal_audio_init(); + if (init_err != ESP_OK) { + return init_err; + } + } + + domain_local_command_snapshot_t command_state = {0}; + esp_err_t snapshot_err = + domain_runtime_local_command_get_snapshot(&command_state); + if (snapshot_err != ESP_OK || !command_state.started) { + esp_err_t start_err = domain_runtime_local_command_start( + local_wake_domain_event, + NULL); + if (start_err != ESP_OK) { + return start_err; + } + } else if (!command_state.detection_running) { + esp_err_t resume_err = domain_runtime_local_command_resume( + CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); + if (resume_err != ESP_OK) { + return resume_err; + } + } + + if (!domain_runtime_multimodal_audio_is_uplink_running()) { + esp_err_t capture_err = + domain_runtime_multimodal_audio_start_uplink(NULL, NULL); + if (capture_err != ESP_OK) { + return capture_err; + } + } + local_wake_set_capture_owned(true); + return ESP_OK; +} + +static void local_wake_handle_detection( + const domain_local_command_event_t *event) +{ + if (event == NULL || + event->type != DOMAIN_LOCAL_COMMAND_EVENT_DETECTED || + event->command_id != DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI) { + return; + } + + portENTER_CRITICAL(&s_local_wake_lock); + if (s_local_wake.stop_requested || s_local_wake.paused_for_foreground) { + portEXIT_CRITICAL(&s_local_wake_lock); + return; + } + s_local_wake.paused_for_foreground = true; + portEXIT_CRITICAL(&s_local_wake_lock); + + local_wake_stop_owned_capture(); + (void)domain_runtime_local_command_suspend( + CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); + + ESP_LOGI(TAG, + "Local wake accepted: phrase=喵掌柜 confidence_x10000=%u generation=%u", + (unsigned)event->confidence_x10000, + (unsigned)event->detection_generation); + +#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE + if (CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE[0] != '\0') { + esp_err_t prompt_err = domain_runtime_behavior_play_prompt_file( + CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE); + if (prompt_err != ESP_OK) { + ESP_LOGW(TAG, + "Local wake response prompt failed: %s", + esp_err_to_name(prompt_err)); + } + } +#endif + + control_plane_set_ptt_active(true); +} + +static void local_wake_task(void *arg) +{ + (void)arg; + TickType_t retry_not_before = 0U; + bool listening_logged = false; + + ESP_LOGI(TAG, "Local wake worker started: phrase=喵掌柜"); + while (!local_wake_snapshot().stop_requested) { + local_wake_state_t state = local_wake_snapshot(); + TickType_t now = xTaskGetTickCount(); + if (!state.paused_for_foreground && + control_plane_voice_turn_is_idle() && + (retry_not_before == 0U || now >= retry_not_before)) { + esp_err_t listen_err = local_wake_start_listening(); + if (listen_err == ESP_OK) { + retry_not_before = 0U; + if (!listening_logged) { + ESP_LOGI(TAG, "Local wake listening ready: phrase=喵掌柜"); + listening_logged = true; + } + } else if (listen_err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, + "Local wake listener unavailable: %s; retry in %ums", + esp_err_to_name(listen_err), + (unsigned)CONFIG_TQ_LOCAL_WAKE_RETRY_MS); + retry_not_before = + now + pdMS_TO_TICKS(CONFIG_TQ_LOCAL_WAKE_RETRY_MS); + listening_logged = false; + } + } + + domain_local_command_event_t event = {0}; + if (state.detection_queue != NULL && + xQueueReceive(state.detection_queue, + &event, + pdMS_TO_TICKS(CONTROL_PLANE_LOCAL_WAKE_POLL_MS)) == + pdTRUE) { + local_wake_handle_detection(&event); + } else { + ulTaskNotifyTake(pdTRUE, 0U); + } + } + + local_wake_stop_owned_capture(); + (void)domain_runtime_local_command_stop( + CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.task = NULL; + s_local_wake.started = false; + portEXIT_CRITICAL(&s_local_wake_lock); + ESP_LOGI(TAG, "Local wake worker stopped"); + vTaskDelete(NULL); +} + +esp_err_t control_plane_local_wake_start(void) +{ +#if !CONFIG_TQ_LOCAL_WAKE_ENABLE + return ESP_OK; +#else + portENTER_CRITICAL(&s_local_wake_lock); + if (s_local_wake.task != NULL || s_local_wake.started) { + portEXIT_CRITICAL(&s_local_wake_lock); + return ESP_OK; + } + portEXIT_CRITICAL(&s_local_wake_lock); + + if (s_local_wake.detection_queue == NULL) { + QueueHandle_t queue = xQueueCreate( + CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN, + sizeof(domain_local_command_event_t)); + if (queue == NULL) { + return ESP_ERR_NO_MEM; + } + portENTER_CRITICAL(&s_local_wake_lock); + if (s_local_wake.detection_queue == NULL) { + s_local_wake.detection_queue = queue; + queue = NULL; + } + portEXIT_CRITICAL(&s_local_wake_lock); + if (queue != NULL) { + vQueueDelete(queue); + } + } + + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.stop_requested = false; + s_local_wake.paused_for_foreground = false; + s_local_wake.capture_owned = false; + s_local_wake.started = true; + portEXIT_CRITICAL(&s_local_wake_lock); + + TaskHandle_t task = NULL; + BaseType_t created = xTaskCreatePinnedToCore( + local_wake_task, + "cp_local_wake", + CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES, + NULL, + CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY, + &task, + CONTROL_PLANE_LOCAL_WAKE_TASK_CORE); + if (created != pdPASS) { + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.started = false; + portEXIT_CRITICAL(&s_local_wake_lock); + return ESP_ERR_NO_MEM; + } + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.task = task; + portEXIT_CRITICAL(&s_local_wake_lock); + return ESP_OK; +#endif +} + +esp_err_t control_plane_local_wake_stop(int timeout_ms) +{ +#if !CONFIG_TQ_LOCAL_WAKE_ENABLE + (void)timeout_ms; + return ESP_OK; +#else + if (timeout_ms < 0) { + return ESP_ERR_INVALID_ARG; + } + local_wake_state_t state = local_wake_snapshot(); + if (!state.started && state.task == NULL) { + return ESP_OK; + } + + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.stop_requested = true; + TaskHandle_t task = s_local_wake.task; + portEXIT_CRITICAL(&s_local_wake_lock); + if (task != NULL) { + xTaskNotifyGive(task); + } + + TickType_t started = xTaskGetTickCount(); + TickType_t timeout_ticks = pdMS_TO_TICKS((uint32_t)timeout_ms); + if (timeout_ms > 0 && timeout_ticks == 0U) { + timeout_ticks = 1U; + } + while (local_wake_snapshot().task != NULL && + timeout_ms > 0 && + (xTaskGetTickCount() - started) < timeout_ticks) { + vTaskDelay(pdMS_TO_TICKS(20)); + } + state = local_wake_snapshot(); + if (state.task != NULL) { + vTaskDelete(state.task); + local_wake_stop_owned_capture(); + (void)domain_runtime_local_command_stop( + CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.task = NULL; + s_local_wake.started = false; + portEXIT_CRITICAL(&s_local_wake_lock); + return ESP_ERR_TIMEOUT; + } + return ESP_OK; +#endif +} + +void control_plane_local_wake_pause_for_foreground(void) +{ +#if CONFIG_TQ_LOCAL_WAKE_ENABLE + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.paused_for_foreground = true; + TaskHandle_t task = s_local_wake.task; + portEXIT_CRITICAL(&s_local_wake_lock); + (void)domain_runtime_local_command_suspend( + CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); + local_wake_stop_owned_capture(); + if (task != NULL) { + xTaskNotifyGive(task); + } +#endif +} + +void control_plane_local_wake_resume_after_foreground(void) +{ +#if CONFIG_TQ_LOCAL_WAKE_ENABLE + portENTER_CRITICAL(&s_local_wake_lock); + s_local_wake.paused_for_foreground = false; + TaskHandle_t task = s_local_wake.task; + portEXIT_CRITICAL(&s_local_wake_lock); + if (task != NULL) { + xTaskNotifyGive(task); + } +#endif +} diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c index 72602ad..724a9fc 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c @@ -575,6 +575,16 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event ESP_LOGI(TAG, "Pause uplink during TTS responding"); control_plane_multimodal_stop_uplink(); } +#if CONFIG_TQ_PTT_MODE_CLICK_TO_TALK + if (!control_plane_voice_turn_is_idle()) { + /* + * The server response closes this one-shot voice turn. Once + * teardown reaches IDLE, the local listener can resume with + * AEC-clean PCM and accept 喵掌柜 while TTS is playing. + */ + control_plane_set_ptt_active(false); + } +#endif break; } diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c index ac04616..c2eff79 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c @@ -67,12 +67,14 @@ #define CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS 24 #endif +#define CONTROL_PLANE_MULTIMODAL_OPUS_CHANNELS 1 + #if CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE != 16000 #error "Multimodal voice requires 16 kHz audio" #endif -#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1 -#error "Multimodal voice requires mono audio" +#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2 +#error "Multimodal AEC requires stereo codec capture" #endif #if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60 @@ -1943,7 +1945,7 @@ esp_err_t control_plane_multimodal_send_start(void) (unsigned)sample_rate, (unsigned)opus_frame_duration_ms, (unsigned)CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS, - CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS, + CONTROL_PLANE_MULTIMODAL_OPUS_CHANNELS, CONFIG_MULTIMODAL_AUDIO_PA_GPIO); return ESP_OK; diff --git a/components/control_plane/src/runtime/control_plane_runtime_workers.c b/components/control_plane/src/runtime/control_plane_runtime_workers.c index 447f4bc..a3765f6 100644 --- a/components/control_plane/src/runtime/control_plane_runtime_workers.c +++ b/components/control_plane/src/runtime/control_plane_runtime_workers.c @@ -88,6 +88,10 @@ #define CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS 8000 #endif +#ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS +#define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000 +#endif + #ifndef CONFIG_TQ_NETWORK_TASK_CORE #define CONFIG_TQ_NETWORK_TASK_CORE 0 #endif @@ -303,6 +307,9 @@ static void control_plane_notify_wifi_state_task(void) void control_plane_set_ptt_active(bool active) { + if (active) { + control_plane_local_wake_pause_for_foreground(); + } bool notify_worker = false; ptt_state_t old_state = PTT_STATE_IDLE; ptt_state_t new_state = PTT_STATE_IDLE; @@ -356,6 +363,20 @@ void control_plane_set_ptt_active(bool active) if (notify_worker) { control_plane_notify_wifi_state_task(); } + if (!active) { + control_plane_local_wake_resume_after_foreground(); + } +} + +bool control_plane_voice_turn_is_idle(void) +{ + bool idle = false; + portENTER_CRITICAL(&s_runtime_lock); + idle = s_ptt_runtime.state == PTT_STATE_IDLE && + !s_ptt_runtime.active && + !s_ptt_runtime.pending_press_after_teardown; + portEXIT_CRITICAL(&s_runtime_lock); + return idle; } static bool control_plane_get_ptt_active(void) @@ -1880,6 +1901,15 @@ esp_err_t control_plane_start_runtime_workers(void) return key_input_err; } + esp_err_t local_wake_err = control_plane_local_wake_start(); + if (local_wake_err != ESP_OK) { + ESP_LOGE(TAG, + "Start local wake worker failed: %s", + esp_err_to_name(local_wake_err)); + control_plane_stop_runtime_workers(); + return local_wake_err; + } + esp_err_t remote_log_err = control_plane_remote_log_start(); if (remote_log_err != ESP_OK && remote_log_err != ESP_ERR_NOT_SUPPORTED) { ESP_LOGW(TAG, "Remote log start failed: %s", esp_err_to_name(remote_log_err)); @@ -1946,6 +1976,7 @@ void control_plane_stop_runtime_workers(void) { control_plane_set_runtime_stop_requested(true); control_plane_stop_user_config_server(); + (void)control_plane_local_wake_stop(CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS); #if CONFIG_TQ_FACTORY_CLI_ENABLE control_plane_factory_cli_stop(); diff --git a/components/domain/CMakeLists.txt b/components/domain/CMakeLists.txt index 2ff319c..d27048f 100644 --- a/components/domain/CMakeLists.txt +++ b/components/domain/CMakeLists.txt @@ -8,12 +8,14 @@ idf_component_register( "src/fortune/domain_fortune_store.c" "src/fortune/domain_user_config_server.c" "src/runtime/domain_runtime_adapters.c" + "src/runtime/domain_local_command.c" INCLUDE_DIRS "include" PRIV_INCLUDE_DIRS "internal" PRIV_REQUIRES platform + esp-sr log json EMBED_TXTFILES diff --git a/components/domain/include/domain.h b/components/domain/include/domain.h index 73ebedc..33d066e 100644 --- a/components/domain/include/domain.h +++ b/components/domain/include/domain.h @@ -66,6 +66,34 @@ typedef esp_err_t (*domain_multimodal_audio_uplink_cb_t)( size_t len, void *user_data); +typedef enum { + DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI = 1, +} domain_local_command_id_t; + +typedef enum { + DOMAIN_LOCAL_COMMAND_EVENT_DETECTED = 0, +} domain_local_command_event_type_t; + +typedef struct { + domain_local_command_event_type_t type; + domain_local_command_id_t command_id; + uint32_t confidence_x10000; + uint32_t detection_generation; +} domain_local_command_event_t; + +typedef void (*domain_local_command_event_callback_t)( + const domain_local_command_event_t *event, + void *user_data); + +typedef struct { + bool started; + bool detection_running; + uint32_t detection_generation; + uint32_t detection_count; + uint32_t dropped_pcm_frames; + uint32_t task_stack_high_water_bytes; +} domain_local_command_snapshot_t; + typedef enum { DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED = 0, DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED, @@ -180,6 +208,20 @@ esp_err_t domain_runtime_multimodal_audio_start_uplink( esp_err_t domain_runtime_multimodal_audio_stop_uplink(int timeout_ms); bool domain_runtime_multimodal_audio_is_uplink_running(void); bool domain_runtime_multimodal_audio_is_initialized(void); +/** + * @brief Start local MultiNet detection for the Chinese phrase "喵掌柜". + * + * The callback runs in the domain inference task and must be bounded and + * non-blocking. Repeated start with the same callback is idempotent. + */ +esp_err_t domain_runtime_local_command_start( + domain_local_command_event_callback_t callback, + void *user_data); +esp_err_t domain_runtime_local_command_suspend(int timeout_ms); +esp_err_t domain_runtime_local_command_resume(int timeout_ms); +esp_err_t domain_runtime_local_command_stop(int timeout_ms); +esp_err_t domain_runtime_local_command_get_snapshot( + domain_local_command_snapshot_t *out_snapshot); esp_err_t domain_runtime_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len); esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(void); esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void); diff --git a/components/domain/src/runtime/domain_local_command.c b/components/domain/src/runtime/domain_local_command.c new file mode 100644 index 0000000..d62892a --- /dev/null +++ b/components/domain/src/runtime/domain_local_command.c @@ -0,0 +1,976 @@ +#include "domain.h" + +#include +#include +#include + +#include "esp_heap_caps.h" +#include "esp_log.h" +#include "esp_mn_iface.h" +#include "esp_mn_models.h" +#include "esp_mn_speech_commands.h" +#include "freertos/FreeRTOS.h" +#include "freertos/event_groups.h" +#include "freertos/idf_additions.h" +#include "freertos/queue.h" +#include "freertos/semphr.h" +#include "freertos/task.h" +#include "model_path.h" +#include "platform.h" +#include "sdkconfig.h" + +#ifndef CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES +#define CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES 8192 +#endif + +#ifndef CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY +#define CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY 7 +#endif + +#ifndef CONFIG_TQ_LOCAL_COMMAND_TASK_CORE +#define CONFIG_TQ_LOCAL_COMMAND_TASK_CORE 0 +#endif + +#ifndef CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS +#define CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS 1000 +#endif + +#ifndef CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS +#define CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS 6000 +#endif + +#ifndef CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000 +#define CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000 1200 +#endif + +#define DOMAIN_LOCAL_COMMAND_MODEL_PARTITION "model" +#define DOMAIN_LOCAL_COMMAND_MODEL_NAME "mn7_cn" +#define DOMAIN_LOCAL_COMMAND_TEXT "喵掌柜" +#define DOMAIN_LOCAL_COMMAND_GRAPH_TEXT "miao zhang gui" +#define DOMAIN_LOCAL_COMMAND_PCM_QUEUE_LEN 8U +#define DOMAIN_LOCAL_COMMAND_PCM_MAX_SAMPLES 512U +#define DOMAIN_LOCAL_COMMAND_QUEUE_WAIT_MS 100U +#define DOMAIN_LOCAL_COMMAND_LOCK_TIMEOUT_MS 100U +#define DOMAIN_LOCAL_COMMAND_STACK_MIN_BYTES 2048U +#define DOMAIN_LOCAL_COMMAND_STATS_INTERVAL_MS 2000U + +#define DOMAIN_LOCAL_COMMAND_STOP_BIT BIT0 +#define DOMAIN_LOCAL_COMMAND_STOPPED_BIT BIT1 +#define DOMAIN_LOCAL_COMMAND_RUNNING_BIT BIT2 +#define DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT BIT3 + +#if CONFIG_TQ_LOCAL_COMMAND_TASK_CORE < 0 +#define DOMAIN_LOCAL_COMMAND_TASK_CORE tskNO_AFFINITY +#else +#define DOMAIN_LOCAL_COMMAND_TASK_CORE CONFIG_TQ_LOCAL_COMMAND_TASK_CORE +#endif + +typedef struct { + uint16_t sample_count; + int16_t samples[DOMAIN_LOCAL_COMMAND_PCM_MAX_SAMPLES]; +} domain_local_command_pcm_frame_t; + +typedef struct { + uint32_t received_frames; + uint32_t processed_chunks; + uint32_t dropped_frames; + uint32_t timeout_count; + uint32_t queue_high_water; + uint32_t max_inference_us; +} domain_local_command_pipeline_stats_t; + +typedef struct { + SemaphoreHandle_t lock; + EventGroupHandle_t events; + QueueHandle_t pcm_queue; + TaskHandle_t task; + srmodel_list_t *models; + const esp_mn_iface_t *multinet_iface; + model_iface_data_t *multinet_data; + bool commands_allocated; + int multinet_chunk_samples; + int16_t *inference_pcm; + domain_local_command_event_callback_t callback; + void *callback_user_data; + bool started; + bool stop_requested; + bool suspended; + bool detection_running; + bool reset_pending; + uint32_t control_generation; + uint32_t applied_generation; + uint32_t detection_generation; + uint32_t detection_count; + uint32_t task_stack_high_water_bytes; +} domain_local_command_state_t; + +static const char *TAG = "domain_local_cmd"; +static domain_local_command_state_t s_local_command; +static StaticTask_t s_local_command_task_buffer; +static StackType_t *s_local_command_task_stack; +static portMUX_TYPE s_pcm_ingress_lock = portMUX_INITIALIZER_UNLOCKED; +static bool s_pcm_ingress_enabled; +static domain_local_command_pipeline_stats_t s_pipeline_stats; + +static bool local_command_lock(void) +{ + if (s_local_command.lock == NULL) { + return false; + } + TickType_t ticks = pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_LOCK_TIMEOUT_MS); + return xSemaphoreTake(s_local_command.lock, ticks > 0U ? ticks : 1U) == pdTRUE; +} +static void local_command_unlock(void) +{ + xSemaphoreGive(s_local_command.lock); +} + +static uint32_t local_command_default_timeout(int timeout_ms) +{ + return timeout_ms == 0 ? CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS : (uint32_t)timeout_ms; +} + +static uint32_t local_command_remaining_ms(TickType_t started, uint32_t timeout_ms) +{ + const uint32_t elapsed_ms = (uint32_t)((xTaskGetTickCount() - started) * portTICK_PERIOD_MS); + return elapsed_ms >= timeout_ms ? 0U : timeout_ms - elapsed_ms; +} + +static void local_command_set_pcm_admission(bool enabled) +{ + portENTER_CRITICAL(&s_pcm_ingress_lock); + s_pcm_ingress_enabled = enabled; + portEXIT_CRITICAL(&s_pcm_ingress_lock); +} + +static uint32_t local_command_dropped_pcm_frames(void) +{ + uint32_t dropped = 0U; + portENTER_CRITICAL(&s_pcm_ingress_lock); + dropped = s_pipeline_stats.dropped_frames; + portEXIT_CRITICAL(&s_pcm_ingress_lock); + return dropped; +} + +static void local_command_note_dropped_pcm(void) +{ + portENTER_CRITICAL(&s_pcm_ingress_lock); + ++s_pipeline_stats.dropped_frames; + portEXIT_CRITICAL(&s_pcm_ingress_lock); +} + +static void local_command_note_queued_pcm(uint32_t queue_depth) +{ + portENTER_CRITICAL(&s_pcm_ingress_lock); + ++s_pipeline_stats.received_frames; + if (queue_depth > s_pipeline_stats.queue_high_water) { + s_pipeline_stats.queue_high_water = queue_depth; + } + portEXIT_CRITICAL(&s_pcm_ingress_lock); +} + +static void local_command_note_processed_chunk(uint32_t inference_us, bool timed_out) +{ + portENTER_CRITICAL(&s_pcm_ingress_lock); + ++s_pipeline_stats.processed_chunks; + if (timed_out) { + ++s_pipeline_stats.timeout_count; + } + if (inference_us > s_pipeline_stats.max_inference_us) { + s_pipeline_stats.max_inference_us = inference_us; + } + portEXIT_CRITICAL(&s_pcm_ingress_lock); +} + +static domain_local_command_pipeline_stats_t local_command_pipeline_stats(void) +{ + domain_local_command_pipeline_stats_t stats; + portENTER_CRITICAL(&s_pcm_ingress_lock); + stats = s_pipeline_stats; + portEXIT_CRITICAL(&s_pcm_ingress_lock); + return stats; +} + +static void local_command_reset_pipeline_stats(void) +{ + portENTER_CRITICAL(&s_pcm_ingress_lock); + memset(&s_pipeline_stats, 0, sizeof(s_pipeline_stats)); + portEXIT_CRITICAL(&s_pcm_ingress_lock); +} + +static void local_command_pcm_observer(const int16_t *pcm, + size_t samples, + uint32_t sample_rate_hz, + void *user_data) +{ + (void)user_data; + bool admitted = false; + portENTER_CRITICAL(&s_pcm_ingress_lock); + admitted = s_pcm_ingress_enabled; + portEXIT_CRITICAL(&s_pcm_ingress_lock); + + if (!admitted || pcm == NULL || samples == 0U) { + return; + } + if (sample_rate_hz != 16000U || samples > DOMAIN_LOCAL_COMMAND_PCM_MAX_SAMPLES || + s_local_command.pcm_queue == NULL) { + local_command_note_dropped_pcm(); + return; + } + + domain_local_command_pcm_frame_t frame = { + .sample_count = (uint16_t)samples, + }; + memcpy(frame.samples, pcm, samples * sizeof(int16_t)); + if (xQueueSend(s_local_command.pcm_queue, &frame, 0U) != pdTRUE) { + local_command_note_dropped_pcm(); + } else { + local_command_note_queued_pcm( + (uint32_t)uxQueueMessagesWaiting(s_local_command.pcm_queue)); + } +} + +static void local_command_drain_pcm_queue(void) +{ + domain_local_command_pcm_frame_t frame; + if (s_local_command.pcm_queue == NULL) { + return; + } + while (xQueueReceive(s_local_command.pcm_queue, &frame, 0U) == pdTRUE) { + } +} + +static esp_err_t local_command_configure_model(void) +{ + s_local_command.models = esp_srmodel_init(DOMAIN_LOCAL_COMMAND_MODEL_PARTITION); + if (s_local_command.models == NULL || s_local_command.models->num <= 0) { + ESP_LOGE(TAG, + "ESP-SR model partition unavailable: partition=%s action=full_flash_required", + DOMAIN_LOCAL_COMMAND_MODEL_PARTITION); + return ESP_ERR_NOT_FOUND; + } + if (esp_srmodel_exists(s_local_command.models, + (char *)DOMAIN_LOCAL_COMMAND_MODEL_NAME) < 0) { + ESP_LOGE(TAG, "Chinese MultiNet model unavailable: model=%s", + DOMAIN_LOCAL_COMMAND_MODEL_NAME); + return ESP_ERR_NOT_FOUND; + } + + s_local_command.multinet_iface = esp_mn_handle_from_name( + (char *)DOMAIN_LOCAL_COMMAND_MODEL_NAME); + const esp_mn_iface_t *iface = s_local_command.multinet_iface; + if (iface == NULL || iface->create == NULL || iface->destroy == NULL || + iface->get_samp_rate == NULL || iface->get_samp_chunksize == NULL || + iface->detect == NULL || iface->get_results == NULL || iface->clean == NULL || + iface->check_speech_command == NULL || iface->set_det_threshold == NULL) { + return ESP_ERR_NOT_SUPPORTED; + } + + s_local_command.multinet_data = iface->create( + DOMAIN_LOCAL_COMMAND_MODEL_NAME, + CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS); + if (s_local_command.multinet_data == NULL) { + return ESP_ERR_NO_MEM; + } + if (iface->get_samp_rate(s_local_command.multinet_data) != 16000) { + return ESP_ERR_NOT_SUPPORTED; + } + s_local_command.multinet_chunk_samples = + iface->get_samp_chunksize(s_local_command.multinet_data); + if (s_local_command.multinet_chunk_samples <= 0 || + s_local_command.multinet_chunk_samples > INT16_MAX) { + return ESP_ERR_INVALID_SIZE; + } + if (iface->check_speech_command(s_local_command.multinet_data, + DOMAIN_LOCAL_COMMAND_GRAPH_TEXT) == 0) { + ESP_LOGE(TAG, "MultiNet command pronunciation is not tokenizable: %s", + DOMAIN_LOCAL_COMMAND_GRAPH_TEXT); + return ESP_ERR_INVALID_ARG; + } + + esp_err_t err = esp_mn_commands_alloc(iface, s_local_command.multinet_data); + if (err != ESP_OK) { + return err; + } + s_local_command.commands_allocated = true; + err = esp_mn_commands_clear(); + if (err == ESP_OK) { + err = esp_mn_commands_phoneme_add( + DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI, + DOMAIN_LOCAL_COMMAND_GRAPH_TEXT, + DOMAIN_LOCAL_COMMAND_GRAPH_TEXT); + } + if (err != ESP_OK) { + return err; + } + esp_mn_error_t *command_errors = esp_mn_commands_update(); + if (command_errors != NULL) { + ESP_LOGE(TAG, "MultiNet rejected local command graph: invalid_count=%d", + command_errors->num); + return ESP_ERR_INVALID_ARG; + } + const float detection_threshold = + (float)CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000 / 10000.0f; + (void)iface->set_det_threshold(s_local_command.multinet_data, + detection_threshold); + const esp_mn_phrase_t *command = esp_mn_commands_get_from_index(0); + if (command == NULL || command->command_id != DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI || + command->string == NULL || command->phonemes == NULL || + strcmp(command->string, DOMAIN_LOCAL_COMMAND_GRAPH_TEXT) != 0 || + strcmp(command->phonemes, DOMAIN_LOCAL_COMMAND_GRAPH_TEXT) != 0 || + esp_mn_commands_get_from_index(1) != NULL) { + return ESP_ERR_INVALID_RESPONSE; + } + + s_local_command.inference_pcm = calloc( + (size_t)s_local_command.multinet_chunk_samples, + sizeof(int16_t)); + if (s_local_command.inference_pcm == NULL) { + return ESP_ERR_NO_MEM; + } + ESP_LOGI(TAG, + "Local command model ready: model=%s phrase=%s command=%s samples=%d " + "rate=16000 threshold_x10000=%u", + DOMAIN_LOCAL_COMMAND_MODEL_NAME, + DOMAIN_LOCAL_COMMAND_TEXT, + DOMAIN_LOCAL_COMMAND_GRAPH_TEXT, + s_local_command.multinet_chunk_samples, + (unsigned)CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000); + return ESP_OK; +} + +static void local_command_release_model(void) +{ + if (s_local_command.commands_allocated) { + (void)esp_mn_commands_free(); + s_local_command.commands_allocated = false; + } + if (s_local_command.multinet_iface != NULL && + s_local_command.multinet_data != NULL) { + s_local_command.multinet_iface->destroy(s_local_command.multinet_data); + } + s_local_command.multinet_data = NULL; + s_local_command.multinet_iface = NULL; + if (s_local_command.models != NULL) { + esp_srmodel_deinit(s_local_command.models); + } + s_local_command.models = NULL; + free(s_local_command.inference_pcm); + s_local_command.inference_pcm = NULL; + s_local_command.multinet_chunk_samples = 0; +} + +static bool local_command_apply_control(size_t *fill_samples) +{ + bool stop_requested = true; + bool suspended = true; + bool changed = false; + bool reset_pending = false; + bool applied = false; + uint32_t generation = 0U; + + if (local_command_lock()) { + stop_requested = s_local_command.stop_requested; + suspended = s_local_command.suspended; + generation = s_local_command.control_generation; + changed = s_local_command.applied_generation != generation; + reset_pending = s_local_command.reset_pending; + local_command_unlock(); + } + if (stop_requested) { + return false; + } + if (!changed) { + return !suspended; + } + + if (reset_pending) { + s_local_command.multinet_iface->clean(s_local_command.multinet_data); + } + local_command_drain_pcm_queue(); + *fill_samples = 0U; + + if (!local_command_lock()) { + return false; + } + if (s_local_command.stop_requested) { + local_command_unlock(); + return false; + } + if (generation == s_local_command.control_generation) { + applied = true; + s_local_command.applied_generation = generation; + s_local_command.detection_running = !s_local_command.suspended; + s_local_command.reset_pending = false; + suspended = s_local_command.suspended; + xEventGroupClearBits(s_local_command.events, + DOMAIN_LOCAL_COMMAND_RUNNING_BIT | + DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT); + xEventGroupSetBits(s_local_command.events, + suspended ? DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT : + DOMAIN_LOCAL_COMMAND_RUNNING_BIT); + } + local_command_unlock(); + if (applied) { + ESP_LOGI(TAG, + "Local command detection %s: control_generation=%u queue_depth=%u dropped_total=%u", + suspended ? "suspended" : "running", + (unsigned)generation, + s_local_command.pcm_queue != NULL ? + (unsigned)uxQueueMessagesWaiting(s_local_command.pcm_queue) : 0U, + (unsigned)local_command_dropped_pcm_frames()); + } + return !suspended; +} + +static void local_command_publish_detection(float probability) +{ + domain_local_command_event_callback_t callback = NULL; + void *callback_user_data = NULL; + domain_local_command_event_t event = { + .type = DOMAIN_LOCAL_COMMAND_EVENT_DETECTED, + .command_id = DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI, + }; + if (probability > 0.0f) { + event.confidence_x10000 = probability >= 0.9999f ? + 9999U : (uint32_t)(probability * 10000.0f + 0.5f); + } + + if (!local_command_lock()) { + return; + } + if (!s_local_command.started || s_local_command.stop_requested || + s_local_command.suspended) { + local_command_unlock(); + return; + } + s_local_command.suspended = true; + s_local_command.detection_running = false; + s_local_command.reset_pending = false; + local_command_set_pcm_admission(false); + ++s_local_command.control_generation; + s_local_command.applied_generation = s_local_command.control_generation; + event.detection_generation = ++s_local_command.detection_generation; + ++s_local_command.detection_count; + callback = s_local_command.callback; + callback_user_data = s_local_command.callback_user_data; + xEventGroupClearBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_RUNNING_BIT); + xEventGroupSetBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT); + local_command_unlock(); + + const domain_local_command_pipeline_stats_t stats = local_command_pipeline_stats(); + ESP_LOGI(TAG, + "Local command detected: phrase=%s confidence_x10000=%u generation=%u " + "received=%u processed=%u drops=%u queue_high=%u inference_max_us=%u", + DOMAIN_LOCAL_COMMAND_TEXT, + (unsigned)event.confidence_x10000, + (unsigned)event.detection_generation, + (unsigned)stats.received_frames, + (unsigned)stats.processed_chunks, + (unsigned)stats.dropped_frames, + (unsigned)stats.queue_high_water, + (unsigned)stats.max_inference_us); + if (callback != NULL) { + callback(&event, callback_user_data); + } +} + +static void local_command_task(void *arg) +{ + (void)arg; + size_t fill_samples = 0U; + uint32_t frame_count = 0U; + bool stack_reported = false; + bool stack_warned = false; + TickType_t stats_last_log_tick = xTaskGetTickCount(); + uint32_t previous_received_frames = 0U; + uint32_t previous_processed_chunks = 0U; + uint32_t previous_dropped_frames = 0U; + uint32_t previous_timeout_count = 0U; + + for (;;) { + const bool detection_running = local_command_apply_control(&fill_samples); + bool stop_requested = true; + if (local_command_lock()) { + stop_requested = s_local_command.stop_requested; + local_command_unlock(); + } + if (stop_requested) { + break; + } + if (!detection_running) { + (void)ulTaskNotifyTake(pdTRUE, + pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_QUEUE_WAIT_MS)); + continue; + } + + domain_local_command_pcm_frame_t frame; + if (xQueueReceive(s_local_command.pcm_queue, + &frame, + pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_QUEUE_WAIT_MS)) != pdTRUE) { + continue; + } + if (!local_command_apply_control(&fill_samples)) { + continue; + } + + size_t offset = 0U; + while (offset < frame.sample_count) { + const size_t needed = (size_t)s_local_command.multinet_chunk_samples - fill_samples; + const size_t available = (size_t)frame.sample_count - offset; + const size_t copy_samples = available < needed ? available : needed; + memcpy(s_local_command.inference_pcm + fill_samples, + frame.samples + offset, + copy_samples * sizeof(int16_t)); + fill_samples += copy_samples; + offset += copy_samples; + if (fill_samples != (size_t)s_local_command.multinet_chunk_samples) { + continue; + } + + const TickType_t inference_started_tick = xTaskGetTickCount(); + const esp_mn_state_t state = s_local_command.multinet_iface->detect( + s_local_command.multinet_data, + s_local_command.inference_pcm); + const uint64_t inference_elapsed_us = + (uint64_t)(xTaskGetTickCount() - inference_started_tick) * + (uint64_t)portTICK_PERIOD_MS * 1000U; + const uint32_t inference_us = inference_elapsed_us > UINT32_MAX ? + UINT32_MAX : (uint32_t)inference_elapsed_us; + fill_samples = 0U; + ++frame_count; + local_command_note_processed_chunk( + inference_us, state == ESP_MN_STATE_TIMEOUT); + /* MultiNet can consume a full core while the PCM queue is nonempty. + * Give the pinned core's idle task a bounded scheduling point so + * continuous playback-time inference cannot trip the task WDT. */ + vTaskDelay(1); + if (state == ESP_MN_STATE_TIMEOUT) { + s_local_command.multinet_iface->clean(s_local_command.multinet_data); + } else if (state == ESP_MN_STATE_DETECTED) { + esp_mn_results_t *results = s_local_command.multinet_iface->get_results( + s_local_command.multinet_data); + const bool accepted = results != NULL && results->num > 0 && + results->command_id[0] == DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI; + const float probability = accepted ? results->prob[0] : 0.0f; + s_local_command.multinet_iface->clean(s_local_command.multinet_data); + if (accepted) { + local_command_publish_detection(probability); + fill_samples = 0U; + break; + } + ESP_LOGW(TAG, "Ignore unexpected MultiNet result"); + } else if (state != ESP_MN_STATE_DETECTING) { + ESP_LOGE(TAG, "Invalid MultiNet state: %d", (int)state); + if (local_command_lock()) { + s_local_command.stop_requested = true; + local_command_unlock(); + } + break; + } + + const TickType_t now = xTaskGetTickCount(); + if ((now - stats_last_log_tick) >= + pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_STATS_INTERVAL_MS)) { + const domain_local_command_pipeline_stats_t stats = + local_command_pipeline_stats(); + const uint32_t queue_depth = s_local_command.pcm_queue != NULL ? + (uint32_t)uxQueueMessagesWaiting(s_local_command.pcm_queue) : 0U; + ESP_LOGI(TAG, + "Local command pipeline: received=%u processed=%u drops=%u timeouts=%u " + "queue=%u high=%u inference_max_us=%u", + (unsigned)(stats.received_frames - previous_received_frames), + (unsigned)(stats.processed_chunks - previous_processed_chunks), + (unsigned)(stats.dropped_frames - previous_dropped_frames), + (unsigned)(stats.timeout_count - previous_timeout_count), + (unsigned)queue_depth, + (unsigned)stats.queue_high_water, + (unsigned)stats.max_inference_us); + previous_received_frames = stats.received_frames; + previous_processed_chunks = stats.processed_chunks; + previous_dropped_frames = stats.dropped_frames; + previous_timeout_count = stats.timeout_count; + stats_last_log_tick = now; + } + + if ((frame_count % 100U) == 0U) { + const uint32_t high_water = (uint32_t)uxTaskGetStackHighWaterMark(NULL); + if (local_command_lock()) { + s_local_command.task_stack_high_water_bytes = high_water; + local_command_unlock(); + } + const uint32_t required = + CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES / 4U > + DOMAIN_LOCAL_COMMAND_STACK_MIN_BYTES ? + CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES / 4U : + DOMAIN_LOCAL_COMMAND_STACK_MIN_BYTES; + if (!stack_reported) { + stack_reported = true; + ESP_LOGI(TAG, + "Task stack headroom=%u required=%u configured=%u bytes", + (unsigned)high_water, + (unsigned)required, + (unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES); + } + if (!stack_warned && high_water < required) { + stack_warned = true; + ESP_LOGW(TAG, + "Local command task stack margin low: high_water=%u required=%u configured=%u", + (unsigned)high_water, + (unsigned)required, + (unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES); + } + } + } + } + + if (local_command_lock()) { + s_local_command.task_stack_high_water_bytes = + (uint32_t)uxTaskGetStackHighWaterMark(NULL); + s_local_command.detection_running = false; + s_local_command.reset_pending = false; + local_command_unlock(); + } + xEventGroupClearBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_RUNNING_BIT); + xEventGroupSetBits(s_local_command.events, + DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT | + DOMAIN_LOCAL_COMMAND_STOPPED_BIT); + vTaskSuspend(NULL); + abort(); +} + +static esp_err_t local_command_reserve_task_stack(void) +{ + if (s_local_command_task_stack != NULL) { + return ESP_OK; + } +#if (configSUPPORT_STATIC_ALLOCATION == 1) && \ + ((defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) || \ + (defined(CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY) && CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY)) + s_local_command_task_stack = (StackType_t *)heap_caps_malloc( + CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES, + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (s_local_command_task_stack != NULL) { + ESP_LOGI(TAG, + "Task stack reserved in PSRAM (%uB)", + (unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES); + return ESP_OK; + } +#endif + ESP_LOGE(TAG, + "PSRAM task stack reservation failed bytes=%u internal_free=%u " + "internal_largest=%u psram_free=%u psram_largest=%u", + (unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES, + (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), + (unsigned)heap_caps_get_largest_free_block( + MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), + (unsigned)heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT), + (unsigned)heap_caps_get_largest_free_block( + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + return ESP_ERR_NO_MEM; +} + +static void local_command_cleanup_after_stop(void) +{ + local_command_set_pcm_admission(false); + local_command_release_model(); + if (local_command_lock()) { + s_local_command.callback = NULL; + s_local_command.callback_user_data = NULL; + s_local_command.started = false; + s_local_command.stop_requested = false; + s_local_command.suspended = true; + s_local_command.detection_running = false; + s_local_command.reset_pending = false; + s_local_command.control_generation = 0U; + s_local_command.applied_generation = 0U; + local_command_unlock(); + } +} + +esp_err_t domain_runtime_local_command_start( + domain_local_command_event_callback_t callback, + void *user_data) +{ + if (callback == NULL) { + return ESP_ERR_INVALID_ARG; + } + if (s_local_command.lock == NULL) { + s_local_command.lock = xSemaphoreCreateMutex(); + } + if (s_local_command.events == NULL) { + s_local_command.events = xEventGroupCreate(); + } + if (s_local_command.lock == NULL || s_local_command.events == NULL) { + return ESP_ERR_NO_MEM; + } + + if (!local_command_lock()) { + return ESP_ERR_TIMEOUT; + } + if (s_local_command.started) { + const bool same = s_local_command.callback == callback && + s_local_command.callback_user_data == user_data; + local_command_unlock(); + return same ? ESP_OK : ESP_ERR_INVALID_STATE; + } + s_local_command.callback = callback; + s_local_command.callback_user_data = user_data; + s_local_command.stop_requested = false; + s_local_command.suspended = false; + s_local_command.detection_running = false; + s_local_command.reset_pending = false; + s_local_command.control_generation = 1U; + s_local_command.applied_generation = 0U; + local_command_unlock(); + local_command_reset_pipeline_stats(); + + esp_err_t err = local_command_reserve_task_stack(); + if (err != ESP_OK) { + local_command_cleanup_after_stop(); + return err; + } + if (s_local_command.pcm_queue == NULL) { + s_local_command.pcm_queue = xQueueCreateWithCaps( + DOMAIN_LOCAL_COMMAND_PCM_QUEUE_LEN, + sizeof(domain_local_command_pcm_frame_t), + MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + } + if (s_local_command.pcm_queue == NULL) { + local_command_cleanup_after_stop(); + return ESP_ERR_NO_MEM; + } + (void)xQueueReset(s_local_command.pcm_queue); + err = local_command_configure_model(); + if (err != ESP_OK) { + local_command_cleanup_after_stop(); + return err; + } + err = platform_multimodal_audio_register_pcm_observer( + local_command_pcm_observer, + NULL); + if (err != ESP_OK) { + local_command_cleanup_after_stop(); + return err; + } + local_command_set_pcm_admission(true); + + xEventGroupClearBits(s_local_command.events, + DOMAIN_LOCAL_COMMAND_STOP_BIT | + DOMAIN_LOCAL_COMMAND_STOPPED_BIT | + DOMAIN_LOCAL_COMMAND_RUNNING_BIT | + DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT); + if (!local_command_lock()) { + (void)platform_multimodal_audio_unregister_pcm_observer( + local_command_pcm_observer, NULL, + CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS); + local_command_cleanup_after_stop(); + return ESP_ERR_TIMEOUT; + } + s_local_command.started = true; + local_command_unlock(); + + /* PSRAM-backed static stack. This inference-only task never initiates flash + * writes; OTA stops local wake before image installation. Its worst case is + * a 1026-byte PCM frame, MultiNet inference, and one bounded typed callback. */ + TaskHandle_t created = xTaskCreateStaticPinnedToCore( + local_command_task, + "domain_cmd", + CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES, + NULL, + CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY, + s_local_command_task_stack, + &s_local_command_task_buffer, + DOMAIN_LOCAL_COMMAND_TASK_CORE); + if (created == NULL) { + (void)platform_multimodal_audio_unregister_pcm_observer( + local_command_pcm_observer, NULL, + CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS); + local_command_cleanup_after_stop(); + return ESP_ERR_NO_MEM; + } + if (local_command_lock()) { + s_local_command.task = created; + local_command_unlock(); + } else { + vTaskDelete(created); + (void)platform_multimodal_audio_unregister_pcm_observer( + local_command_pcm_observer, NULL, + CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS); + local_command_cleanup_after_stop(); + return ESP_ERR_TIMEOUT; + } + + EventBits_t bits = xEventGroupWaitBits( + s_local_command.events, + DOMAIN_LOCAL_COMMAND_RUNNING_BIT | DOMAIN_LOCAL_COMMAND_STOPPED_BIT, + pdFALSE, + pdFALSE, + pdMS_TO_TICKS(CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS)); + if ((bits & DOMAIN_LOCAL_COMMAND_RUNNING_BIT) == 0U) { + (void)domain_runtime_local_command_stop(CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS); + return ESP_ERR_TIMEOUT; + } + return ESP_OK; +} + +static esp_err_t local_command_request_detection_state(bool suspended, int timeout_ms) +{ + if (timeout_ms < 0) { + return ESP_ERR_INVALID_ARG; + } + const uint32_t bounded_ms = local_command_default_timeout(timeout_ms); + if (suspended) { + local_command_set_pcm_admission(false); + } + if (!local_command_lock()) { + return ESP_ERR_TIMEOUT; + } + if (!s_local_command.started || s_local_command.stop_requested || + s_local_command.task == NULL) { + local_command_unlock(); + return ESP_ERR_INVALID_STATE; + } + if (s_local_command.suspended == suspended && + s_local_command.applied_generation == s_local_command.control_generation) { + local_command_unlock(); + return ESP_OK; + } + const bool resume_from_clean_stop = !suspended && + s_local_command.suspended && + !s_local_command.reset_pending && + !s_local_command.detection_running && + s_local_command.applied_generation == s_local_command.control_generation; + s_local_command.suspended = suspended; + if (suspended || !resume_from_clean_stop) { + s_local_command.reset_pending = true; + } + const uint32_t generation = ++s_local_command.control_generation; + const EventBits_t expected = suspended ? + DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT : DOMAIN_LOCAL_COMMAND_RUNNING_BIT; + xEventGroupClearBits(s_local_command.events, expected); + TaskHandle_t task = s_local_command.task; + local_command_unlock(); + xTaskNotifyGive(task); + + EventBits_t bits = xEventGroupWaitBits( + s_local_command.events, + expected | DOMAIN_LOCAL_COMMAND_STOPPED_BIT, + pdFALSE, + pdFALSE, + pdMS_TO_TICKS(bounded_ms)); + if ((bits & expected) == 0U) { + return (bits & DOMAIN_LOCAL_COMMAND_STOPPED_BIT) != 0U ? + ESP_ERR_INVALID_STATE : ESP_ERR_TIMEOUT; + } + if (!local_command_lock()) { + return ESP_ERR_TIMEOUT; + } + const bool applied = s_local_command.started && + !s_local_command.stop_requested && + s_local_command.suspended == suspended && + s_local_command.applied_generation == generation; + local_command_unlock(); + if (applied && !suspended) { + local_command_set_pcm_admission(true); + } + return applied ? ESP_OK : ESP_ERR_INVALID_STATE; +} + +esp_err_t domain_runtime_local_command_suspend(int timeout_ms) +{ + return local_command_request_detection_state(true, timeout_ms); +} + +esp_err_t domain_runtime_local_command_resume(int timeout_ms) +{ + return local_command_request_detection_state(false, timeout_ms); +} + +esp_err_t domain_runtime_local_command_stop(int timeout_ms) +{ + if (timeout_ms < 0) { + return ESP_ERR_INVALID_ARG; + } + const uint32_t bounded_ms = local_command_default_timeout(timeout_ms); + const TickType_t started = xTaskGetTickCount(); + if (s_local_command.lock == NULL) { + return ESP_OK; + } + local_command_set_pcm_admission(false); + + esp_err_t result = ESP_OK; + esp_err_t observer_err = platform_multimodal_audio_unregister_pcm_observer( + local_command_pcm_observer, + NULL, + (int)local_command_remaining_ms(started, bounded_ms)); + if (observer_err != ESP_OK && observer_err != ESP_ERR_INVALID_STATE) { + result = observer_err; + } + if (!local_command_lock()) { + return result != ESP_OK ? result : ESP_ERR_TIMEOUT; + } + if (!s_local_command.started) { + local_command_unlock(); + return result; + } + s_local_command.stop_requested = true; + s_local_command.detection_running = false; + TaskHandle_t task = s_local_command.task; + local_command_unlock(); + xEventGroupSetBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_STOP_BIT); + if (task != NULL) { + xTaskNotifyGive(task); + } + + const uint32_t remaining_ms = local_command_remaining_ms(started, bounded_ms); + EventBits_t bits = xEventGroupWaitBits( + s_local_command.events, + DOMAIN_LOCAL_COMMAND_STOPPED_BIT, + pdFALSE, + pdTRUE, + pdMS_TO_TICKS(remaining_ms)); + if ((bits & DOMAIN_LOCAL_COMMAND_STOPPED_BIT) == 0U && result == ESP_OK) { + result = ESP_ERR_TIMEOUT; + } + TaskHandle_t stopped_task = task; + if (local_command_lock()) { + if (s_local_command.task == stopped_task) { + s_local_command.task = NULL; + } + local_command_unlock(); + } + if (stopped_task != NULL) { + vTaskDelete(stopped_task); + } + if ((bits & DOMAIN_LOCAL_COMMAND_STOPPED_BIT) == 0U) { + ESP_LOGE(TAG, "Local command worker forced down after stop timeout"); + } + local_command_cleanup_after_stop(); + return result; +} + +esp_err_t domain_runtime_local_command_get_snapshot( + domain_local_command_snapshot_t *out_snapshot) +{ + if (out_snapshot == NULL) { + return ESP_ERR_INVALID_ARG; + } + if (s_local_command.lock == NULL) { + memset(out_snapshot, 0, sizeof(*out_snapshot)); + return ESP_OK; + } + if (!local_command_lock()) { + return ESP_ERR_TIMEOUT; + } + *out_snapshot = (domain_local_command_snapshot_t) { + .started = s_local_command.started, + .detection_running = s_local_command.detection_running, + .detection_generation = s_local_command.detection_generation, + .detection_count = s_local_command.detection_count, + .dropped_pcm_frames = local_command_dropped_pcm_frames(), + .task_stack_high_water_bytes = s_local_command.task_stack_high_water_bytes, + }; + local_command_unlock(); + return ESP_OK; +} diff --git a/components/platform/idf_component.yml b/components/platform/idf_component.yml index bc3ea83..69507a2 100644 --- a/components/platform/idf_component.yml +++ b/components/platform/idf_component.yml @@ -4,5 +4,6 @@ dependencies: espressif/esp_websocket_client: ^1.6.1 espressif/esp_codec_dev: ^1.5.4 espressif/esp_audio_codec: ^2.4.1 + espressif/esp-sr: ^2.4.7 espressif/mdns: ^1.11.0 78/esp-wifi-connect: ^3.1.4 diff --git a/components/platform/include/platform.h b/components/platform/include/platform.h index 69e5ef4..1e67ebb 100644 --- a/components/platform/include/platform.h +++ b/components/platform/include/platform.h @@ -49,6 +49,12 @@ typedef struct { void *user_data; } platform_multimodal_downstream_playback_callbacks_t; +typedef void (*platform_multimodal_audio_pcm_observer_t)( + const int16_t *pcm, + size_t samples, + uint32_t sample_rate_hz, + void *user_data); + #define PLATFORM_LAN_HTTP_PATH_MAX_LEN 96 #define PLATFORM_LAN_HTTP_HOSTNAME_MAX_LEN 32 #define PLATFORM_LAN_HTTP_REQUEST_BODY_MAX_LEN 1024 @@ -321,6 +327,24 @@ esp_err_t platform_multimodal_audio_start_uplink( esp_err_t platform_multimodal_audio_stop_uplink(int timeout_ms); bool platform_multimodal_audio_is_uplink_running(void); bool platform_multimodal_audio_is_initialized(void); +/** + * @brief Register the single mono PCM observer for the active capture path. + * + * The callback runs synchronously in the platform capture task, outside the + * registration lock, and must remain bounded and non-blocking. + */ +esp_err_t platform_multimodal_audio_register_pcm_observer( + platform_multimodal_audio_pcm_observer_t observer, + void *user_data); +/** + * @brief Remove a matching PCM observer and wait for in-flight callbacks. + * + * Negative timeout is invalid. Zero performs a non-blocking drain check. + */ +esp_err_t platform_multimodal_audio_unregister_pcm_observer( + platform_multimodal_audio_pcm_observer_t observer, + void *user_data, + int timeout_ms); esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len); esp_err_t platform_multimodal_audio_begin_downstream_response(void); esp_err_t platform_multimodal_audio_finish_downstream_response(void); diff --git a/components/platform/internal/multimodal/audio_pipeline.h b/components/platform/internal/multimodal/audio_pipeline.h index 6c5d81d..ad076be 100644 --- a/components/platform/internal/multimodal/audio_pipeline.h +++ b/components/platform/internal/multimodal/audio_pipeline.h @@ -11,6 +11,10 @@ extern "C" { #endif typedef esp_err_t (*multimodal_audio_uplink_cb_t)(const uint8_t *data, size_t len, void *ctx); +typedef void (*multimodal_audio_pcm_observer_t)(const int16_t *pcm, + size_t samples, + uint32_t sample_rate_hz, + void *ctx); typedef enum { MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED = 0, @@ -33,6 +37,13 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c esp_err_t multimodal_audio_stop_uplink(int timeout_ms); bool multimodal_audio_is_uplink_running(void); bool multimodal_audio_is_initialized(void); +esp_err_t multimodal_audio_register_pcm_observer( + multimodal_audio_pcm_observer_t observer, + void *observer_ctx); +esp_err_t multimodal_audio_unregister_pcm_observer( + multimodal_audio_pcm_observer_t observer, + void *observer_ctx, + int timeout_ms); esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t len); esp_err_t multimodal_audio_begin_downstream_response(void); diff --git a/components/platform/src/multimodal/audio_pipeline.c b/components/platform/src/multimodal/audio_pipeline.c index 11d7681..83a7e3c 100644 --- a/components/platform/src/multimodal/audio_pipeline.c +++ b/components/platform/src/multimodal/audio_pipeline.c @@ -15,6 +15,7 @@ #include "esp_codec_dev_defaults.h" #include "esp_audio_dec.h" #include "esp_audio_enc.h" +#include "esp_afe_aec.h" #include "esp_err.h" #include "esp_heap_caps.h" #include "esp_log.h" @@ -114,6 +115,14 @@ #define CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL 0 #endif +#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL +#define CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL 1 +#endif + +#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH +#define CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH 4 +#endif + #ifndef CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS #define CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS 1 #endif @@ -208,8 +217,16 @@ #error "Multimodal voice requires 16 kHz audio" #endif -#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1 -#error "Multimodal voice requires mono audio" +#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2 +#error "ES8311 local wake AEC requires two physical codec slots" +#endif + +#if CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE != 16 +#error "ESP AEC requires signed 16-bit PCM" +#endif + +#if CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL == CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL +#error "Microphone and AEC reference must use distinct codec slots" #endif #if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60 @@ -235,6 +252,9 @@ #define MULTIMODAL_DECODER_BUFFER_GUARD_BYTES 512U #define MULTIMODAL_AUDIO_DOWNSTREAM_TX_BOUNCE_BYTES 3200U #define MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL ((uint8_t)CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL) +#define MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL \ + ((uint8_t)CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL) +#define MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT 16U static const char *TAG = "multimodal_audio"; static portMUX_TYPE s_audio_runtime_lock = portMUX_INITIALIZER_UNLOCKED; @@ -253,6 +273,7 @@ typedef struct { uint8_t i2c_ctrl_addr_8bit; esp_audio_enc_handle_t opus_enc; esp_audio_dec_handle_t opus_dec; + afe_aec_handle_t *afe_aec_handle; int opus_enc_in_bytes; int opus_enc_out_bytes; uint32_t opus_sample_rate; @@ -260,8 +281,11 @@ typedef struct { uint32_t opus_bitrate_bps; uint8_t *uplink_codec_buf; int16_t *uplink_mono_buf; + int16_t *uplink_aec_clean_buf; uint8_t *uplink_opus_buf; size_t uplink_codec_read_bytes; + size_t uplink_aec_frame_bytes; + size_t uplink_aec_frame_samples; size_t uplink_mono_bytes; size_t uplink_mono_buf_capacity_bytes; size_t uplink_mono_fill_bytes; @@ -290,6 +314,7 @@ typedef struct { bool downstream_playback_keep_stopping_on_exit; bool audio_deinit_in_progress; uint8_t mic_capture_channel; + uint8_t aec_reference_channel; QueueHandle_t uplink_send_queue; bool uplink_send_queue_with_caps; TaskHandle_t uplink_task; @@ -306,6 +331,10 @@ typedef struct { static multimodal_audio_ctx_t s_audio; static multimodal_audio_downstream_playback_callback_t s_downstream_playback_cb; static void *s_downstream_playback_cb_ctx; +static multimodal_audio_pcm_observer_t s_pcm_observer; +static void *s_pcm_observer_ctx; +static uint32_t s_pcm_observer_in_flight; +static portMUX_TYPE s_pcm_observer_lock = portMUX_INITIALIZER_UNLOCKED; static bool s_uplink_stack_warned; static bool s_uplink_send_stack_warned; static bool s_uplink_send_queue_warned; diff --git a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c index c4ce0ef..430f1f8 100644 --- a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c +++ b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c @@ -50,10 +50,17 @@ static void cleanup_opus_path(multimodal_audio_ctx_t *ctx) ctx->opus_dec = NULL; } + if (ctx->afe_aec_handle != NULL) { + afe_aec_destroy(ctx->afe_aec_handle); + ctx->afe_aec_handle = NULL; + } + free(ctx->uplink_codec_buf); ctx->uplink_codec_buf = NULL; free(ctx->uplink_mono_buf); ctx->uplink_mono_buf = NULL; + free(ctx->uplink_aec_clean_buf); + ctx->uplink_aec_clean_buf = NULL; free(ctx->uplink_opus_buf); ctx->uplink_opus_buf = NULL; free(ctx->downstream_pcm_buf); @@ -64,6 +71,8 @@ static void cleanup_opus_path(multimodal_audio_ctx_t *ctx) ctx->downstream_tx_bounce_buf = NULL; ctx->uplink_codec_read_bytes = 0; + ctx->uplink_aec_frame_bytes = 0; + ctx->uplink_aec_frame_samples = 0; ctx->uplink_mono_bytes = 0; ctx->uplink_mono_buf_capacity_bytes = 0; ctx->uplink_mono_fill_bytes = 0; @@ -566,15 +575,54 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx) return audio_err_to_esp_err(audio_err); } + const char *aec_input_format = NULL; + if (ctx->mic_capture_channel == 0U && + ctx->aec_reference_channel == 1U) { + aec_input_format = "MR"; + } else if (ctx->mic_capture_channel == 1U && + ctx->aec_reference_channel == 0U) { + aec_input_format = "RM"; + } else { + ESP_LOGE(TAG, + "Unsupported AEC slot map: mic=%u ref=%u", + (unsigned)ctx->mic_capture_channel, + (unsigned)ctx->aec_reference_channel); + return ESP_ERR_NOT_SUPPORTED; + } + ctx->afe_aec_handle = afe_aec_create( + aec_input_format, + CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH, + AFE_TYPE_FD, + AFE_MODE_LOW_COST); + if (ctx->afe_aec_handle == NULL) { + ESP_LOGE(TAG, "Create ESP-SR AEC failed"); + return ESP_ERR_NO_MEM; + } + if (ctx->afe_aec_handle->handle != NULL) { + (void)aec_set_nlp_level( + ctx->afe_aec_handle->handle, + AEC_NLP_LEVEL_AGGR); + } + int aec_frame_samples = afe_aec_get_chunksize(ctx->afe_aec_handle); + if (aec_frame_samples <= 0) { + ESP_LOGE(TAG, "Invalid ESP-SR AEC frame size: %d", aec_frame_samples); + return ESP_ERR_INVALID_SIZE; + } + ctx->uplink_aec_frame_samples = (size_t)aec_frame_samples; + ctx->uplink_aec_frame_bytes = + ctx->uplink_aec_frame_samples * sizeof(int16_t); + ctx->uplink_mono_bytes = (size_t)ctx->opus_enc_in_bytes; if (ctx->uplink_mono_bytes == 0 || ctx->fs.channel == 0U) { return ESP_ERR_INVALID_SIZE; } - if (ctx->uplink_mono_bytes > (SIZE_MAX / (size_t)ctx->fs.channel)) { + if (ctx->uplink_aec_frame_bytes > + (SIZE_MAX / (size_t)ctx->fs.channel)) { return ESP_ERR_INVALID_SIZE; } - size_t codec_read_bytes = ctx->uplink_mono_bytes * (size_t)ctx->fs.channel; + size_t codec_read_bytes = + ctx->uplink_aec_frame_bytes * (size_t)ctx->fs.channel; ctx->uplink_codec_read_bytes = codec_read_bytes; ctx->uplink_opus_buf_bytes = (size_t)ctx->opus_enc_out_bytes; if (ctx->uplink_opus_buf_bytes < 256U) { @@ -586,8 +634,14 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx) ctx->uplink_codec_buf = (uint8_t *)audio_psram_first_calloc(1, ctx->uplink_codec_read_bytes); ctx->uplink_mono_buf = (int16_t *)audio_psram_first_calloc(1, ctx->uplink_mono_buf_capacity_bytes); + ctx->uplink_aec_clean_buf = (int16_t *)heap_caps_aligned_calloc( + MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT, + 1, + ctx->uplink_aec_frame_bytes, + MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); ctx->uplink_opus_buf = (uint8_t *)audio_psram_first_calloc(1, ctx->uplink_opus_buf_bytes); - if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf) { + if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || + !ctx->uplink_aec_clean_buf || !ctx->uplink_opus_buf) { return ESP_ERR_NO_MEM; } @@ -601,38 +655,20 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx) } ESP_LOGI(TAG, - "Opus ready: %uHz frame=%ums bitrate=%ukbps enc(in=%d out=%d)", + "Opus/AEC ready: %uHz opus_frame=%ums bitrate=%ukbps enc(in=%d out=%d) aec_frame=%u input=%s", (unsigned)ctx->opus_sample_rate, (unsigned)ctx->opus_frame_duration_ms, (unsigned)bitrate_kbps, ctx->opus_enc_in_bytes, - ctx->opus_enc_out_bytes); + ctx->opus_enc_out_bytes, + (unsigned)ctx->uplink_aec_frame_samples, + aec_input_format); ctx->opus_ready = true; log_heap_snapshot("after_opus_init"); return ESP_OK; } -static void extract_mic_channel_16(const int16_t *input, - uint8_t channels, - uint8_t mic_channel, - int16_t *output, - size_t frames) -{ - if (!input || !output || channels == 0 || mic_channel >= channels) { - return; - } - - if (channels == 1) { - memcpy(output, input, frames * sizeof(int16_t)); - return; - } - - for (size_t frame = 0; frame < frames; ++frame) { - output[frame] = input[frame * channels + mic_channel]; - } -} - static void expand_mono_to_multichannel_16(const int16_t *input, uint8_t channels, int16_t *output, diff --git a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_helpers_impl.c b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_helpers_impl.c index dc80302..ab197f7 100644 --- a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_helpers_impl.c +++ b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_helpers_impl.c @@ -207,6 +207,8 @@ static void reset_capture_channel_map(multimodal_audio_ctx_t *ctx) uint8_t mic_channel = MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL > 0U ? 1U : 0U; ctx->mic_capture_channel = mic_channel; + ctx->aec_reference_channel = + MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL > 0U ? 1U : 0U; } static void fill_codec_sample_info(multimodal_audio_ctx_t *ctx) @@ -220,9 +222,14 @@ static void fill_codec_sample_info(multimodal_audio_ctx_t *ctx) ctx->fs.channel_mask = 0; ctx->fs.mclk_multiple = 0; reset_capture_channel_map(ctx); - if (ctx->fs.channel == 1U || ctx->mic_capture_channel >= ctx->fs.channel) { + if (ctx->mic_capture_channel >= ctx->fs.channel) { ctx->mic_capture_channel = 0U; } + if (ctx->aec_reference_channel >= ctx->fs.channel || + ctx->aec_reference_channel == ctx->mic_capture_channel) { + ctx->aec_reference_channel = + ctx->mic_capture_channel == 0U ? 1U : 0U; + } } static bool is_supported_opus_sample_rate(uint32_t sample_rate) diff --git a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c index 8982d2c..9282a44 100644 --- a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c +++ b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c @@ -157,6 +157,34 @@ static void audio_set_uplink_callback(multimodal_audio_ctx_t *ctx, portEXIT_CRITICAL(&s_audio_runtime_lock); } +static void notify_pcm_observer(const int16_t *pcm, size_t samples) +{ + multimodal_audio_pcm_observer_t observer = NULL; + void *observer_ctx = NULL; + + portENTER_CRITICAL(&s_pcm_observer_lock); + observer = s_pcm_observer; + observer_ctx = s_pcm_observer_ctx; + if (observer != NULL) { + ++s_pcm_observer_in_flight; + } + portEXIT_CRITICAL(&s_pcm_observer_lock); + + if (observer == NULL) { + return; + } + observer(pcm, + samples, + (uint32_t)CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE, + observer_ctx); + + portENTER_CRITICAL(&s_pcm_observer_lock); + if (s_pcm_observer_in_flight > 0U) { + --s_pcm_observer_in_flight; + } + portEXIT_CRITICAL(&s_pcm_observer_lock); +} + static void audio_snapshot_uplink_tasks(const multimodal_audio_ctx_t *ctx, TaskHandle_t *out_capture_task, TaskHandle_t *out_send_task) @@ -610,6 +638,47 @@ static esp_err_t encode_and_queue_mono_pcm(multimodal_audio_ctx_t *ctx, return ESP_OK; } +static esp_err_t append_and_queue_mono_pcm( + multimodal_audio_ctx_t *ctx, + const int16_t *pcm, + size_t pcm_bytes) +{ + if (ctx == NULL || pcm == NULL || pcm_bytes == 0U || + ctx->uplink_mono_buf == NULL || + ctx->uplink_mono_buf_capacity_bytes != ctx->uplink_mono_bytes) { + return ESP_ERR_INVALID_ARG; + } + + const uint8_t *input = (const uint8_t *)pcm; + size_t remaining = pcm_bytes; + while (remaining > 0U) { + size_t available = + ctx->uplink_mono_buf_capacity_bytes - + ctx->uplink_mono_fill_bytes; + size_t copy_bytes = remaining < available ? remaining : available; + memcpy((uint8_t *)ctx->uplink_mono_buf + + ctx->uplink_mono_fill_bytes, + input, + copy_bytes); + ctx->uplink_mono_fill_bytes += copy_bytes; + input += copy_bytes; + remaining -= copy_bytes; + + if (ctx->uplink_mono_fill_bytes == + ctx->uplink_mono_buf_capacity_bytes) { + esp_err_t err = encode_and_queue_mono_pcm( + ctx, + ctx->uplink_mono_buf, + ctx->uplink_mono_bytes); + ctx->uplink_mono_fill_bytes = 0U; + if (err != ESP_OK) { + return err; + } + } + } + return ESP_OK; +} + static void uplink_send_task(void *arg) { (void)arg; @@ -666,9 +735,13 @@ static void uplink_capture_task(void *arg) (void)arg; multimodal_audio_ctx_t *ctx = &s_audio; - if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf || - !ctx->opus_enc || ctx->uplink_mono_bytes == 0U || ctx->uplink_codec_read_bytes == 0U) { - ESP_LOGE(TAG, "Uplink Opus buffers not ready"); + if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || + !ctx->uplink_aec_clean_buf || !ctx->uplink_opus_buf || + !ctx->opus_enc || !ctx->afe_aec_handle || + ctx->uplink_mono_bytes == 0U || + ctx->uplink_codec_read_bytes == 0U || + ctx->uplink_aec_frame_samples == 0U) { + ESP_LOGE(TAG, "Uplink Opus/AEC buffers not ready"); audio_set_uplink_capture_task(ctx, NULL); delete_audio_task_with_caps(NULL); return; @@ -681,13 +754,14 @@ static void uplink_capture_task(void *arg) } ESP_LOGI(TAG, - "Mic uplink capture task started (stack=%uB opus in=%uB out=%uB read=%uB channels=%u mic_ch=%u)", + "Mic uplink capture task started (stack=%uB opus in=%uB out=%uB read=%uB channels=%u mic_ch=%u ref_ch=%u)", (unsigned)get_uplink_task_stack_bytes(), (unsigned)ctx->uplink_mono_bytes, (unsigned)ctx->uplink_opus_buf_bytes, (unsigned)ctx->uplink_codec_read_bytes, (unsigned)ctx->fs.channel, - (unsigned)ctx->mic_capture_channel); + (unsigned)ctx->mic_capture_channel, + (unsigned)ctx->aec_reference_channel); while (!audio_get_uplink_task_exit_requested(ctx)) { if (!audio_get_uplink_recording_active(ctx)) { @@ -707,19 +781,37 @@ static void uplink_capture_task(void *arg) continue; } - size_t uplink_frames = ctx->uplink_mono_bytes / sizeof(int16_t); - extract_mic_channel_16((const int16_t *)ctx->uplink_codec_buf, - ctx->fs.channel, - ctx->mic_capture_channel, - ctx->uplink_mono_buf, - uplink_frames); - audio_debug_dump_pcm(ctx->uplink_mono_buf, ctx->uplink_mono_bytes); - esp_err_t err = encode_and_queue_mono_pcm(ctx, ctx->uplink_mono_buf, ctx->uplink_mono_bytes); + size_t clean_bytes = afe_aec_process( + ctx->afe_aec_handle, + (const int16_t *)ctx->uplink_codec_buf, + ctx->uplink_aec_clean_buf); + if (clean_bytes != ctx->uplink_aec_frame_bytes) { + ESP_LOGW(TAG, + "AEC output size=%u expected=%u", + (unsigned)clean_bytes, + (unsigned)ctx->uplink_aec_frame_bytes); + ctx->uplink_mono_fill_bytes = 0U; + continue; + } + size_t uplink_frames = ctx->uplink_aec_frame_samples; + notify_pcm_observer(ctx->uplink_aec_clean_buf, uplink_frames); + audio_debug_dump_pcm(ctx->uplink_aec_clean_buf, clean_bytes); + multimodal_audio_uplink_cb_t uplink_cb = NULL; + audio_snapshot_uplink_callback(ctx, &uplink_cb, NULL); + esp_err_t err = ESP_OK; + if (uplink_cb != NULL) { + err = append_and_queue_mono_pcm( + ctx, + ctx->uplink_aec_clean_buf, + clean_bytes); + } else { + ctx->uplink_mono_fill_bytes = 0U; + } maybe_log_uplink_levels(ctx, (const int16_t *)ctx->uplink_codec_buf, uplink_frames, - ctx->uplink_mono_buf, - ctx->uplink_mono_bytes); + ctx->uplink_aec_clean_buf, + clean_bytes); if (err != ESP_OK && err != ESP_ERR_INVALID_SIZE && err != ESP_ERR_TIMEOUT) { ESP_LOGW(TAG, "Encode uplink PCM failed: %s", esp_err_to_name(err)); vTaskDelay(pdMS_TO_TICKS(10)); @@ -739,7 +831,9 @@ static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx) return ESP_ERR_INVALID_ARG; } if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_enc || - !ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf) { + !ctx->afe_aec_handle || !ctx->uplink_codec_buf || + !ctx->uplink_mono_buf || !ctx->uplink_aec_clean_buf || + !ctx->uplink_opus_buf) { return ESP_ERR_INVALID_STATE; } @@ -970,18 +1064,82 @@ esp_err_t multimodal_audio_register_downstream_playback_callback( return ESP_OK; } -esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *cb_ctx) +esp_err_t multimodal_audio_register_pcm_observer( + multimodal_audio_pcm_observer_t observer, + void *observer_ctx) { - if (!cb) { + if (observer == NULL) { return ESP_ERR_INVALID_ARG; } + esp_err_t result = ESP_OK; + portENTER_CRITICAL(&s_pcm_observer_lock); + if (s_pcm_observer != NULL && + (s_pcm_observer != observer || s_pcm_observer_ctx != observer_ctx)) { + result = ESP_ERR_INVALID_STATE; + } else { + s_pcm_observer = observer; + s_pcm_observer_ctx = observer_ctx; + } + portEXIT_CRITICAL(&s_pcm_observer_lock); + return result; +} + +esp_err_t multimodal_audio_unregister_pcm_observer( + multimodal_audio_pcm_observer_t observer, + void *observer_ctx, + int timeout_ms) +{ + if (observer == NULL || timeout_ms < 0) { + return ESP_ERR_INVALID_ARG; + } + + portENTER_CRITICAL(&s_pcm_observer_lock); + if (s_pcm_observer == NULL) { + portEXIT_CRITICAL(&s_pcm_observer_lock); + return ESP_OK; + } + if (s_pcm_observer != observer || s_pcm_observer_ctx != observer_ctx) { + portEXIT_CRITICAL(&s_pcm_observer_lock); + return ESP_ERR_INVALID_STATE; + } + s_pcm_observer = NULL; + s_pcm_observer_ctx = NULL; + portEXIT_CRITICAL(&s_pcm_observer_lock); + + TickType_t started = xTaskGetTickCount(); + TickType_t timeout_ticks = pdMS_TO_TICKS((uint32_t)timeout_ms); + if (timeout_ms > 0 && timeout_ticks == 0U) { + timeout_ticks = 1U; + } + for (;;) { + uint32_t in_flight = 0U; + portENTER_CRITICAL(&s_pcm_observer_lock); + in_flight = s_pcm_observer_in_flight; + portEXIT_CRITICAL(&s_pcm_observer_lock); + if (in_flight == 0U) { + return ESP_OK; + } + if (timeout_ms == 0 || + (xTaskGetTickCount() - started) >= timeout_ticks) { + return ESP_ERR_TIMEOUT; + } + vTaskDelay(1U); + } +} + +esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *cb_ctx) +{ if (!s_audio.initialized || !s_audio.codec_dev || !s_audio.opus_enc || !s_audio.uplink_codec_buf || !s_audio.uplink_mono_buf || !s_audio.uplink_opus_buf) { return ESP_ERR_INVALID_STATE; } + bool already_active = audio_get_uplink_recording_active(&s_audio); audio_set_uplink_callback(&s_audio, cb, cb_ctx); + if (already_active) { + return ESP_OK; + } s_audio.uplink_mono_fill_bytes = 0U; memset(s_audio.uplink_mono_buf, 0, s_audio.uplink_mono_buf_capacity_bytes); reset_uplink_debug_stats(); diff --git a/components/platform/src/runtime/platform_multimodal_runtime.c b/components/platform/src/runtime/platform_multimodal_runtime.c index 45d259f..a0921b2 100644 --- a/components/platform/src/runtime/platform_multimodal_runtime.c +++ b/components/platform/src/runtime/platform_multimodal_runtime.c @@ -177,6 +177,24 @@ bool platform_multimodal_audio_is_initialized(void) return multimodal_audio_is_initialized(); } +esp_err_t platform_multimodal_audio_register_pcm_observer( + platform_multimodal_audio_pcm_observer_t observer, + void *user_data) +{ + return multimodal_audio_register_pcm_observer(observer, user_data); +} + +esp_err_t platform_multimodal_audio_unregister_pcm_observer( + platform_multimodal_audio_pcm_observer_t observer, + void *user_data, + int timeout_ms) +{ + return multimodal_audio_unregister_pcm_observer( + observer, + user_data, + timeout_ms); +} + esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len) { return multimodal_audio_handle_downstream_opus(data, len); diff --git a/dependencies.lock b/dependencies.lock index 63f0be0..b7f689a 100644 --- a/dependencies.lock +++ b/dependencies.lock @@ -15,6 +15,58 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 3.1.4 + espressif/cjson: + component_hash: e788323270d90738662d66fffa910bfe1fba019bba087f01557e70c40485b469 + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 1.7.19~2 + espressif/dl_fft: + component_hash: a955ca29581de2177ddf25759273bd83f082b3c805e735fe6ee105858643989b + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 0.6.0 + espressif/esp-dsp: + component_hash: 12e44db246517a627bc0fe2b255b8335410c0215c98bfba2df5a8e3edea839ef + dependencies: + - name: idf + require: private + version: '>=4.2' + source: + registry_url: https://components.espressif.com + type: service + version: 1.8.0 + espressif/esp-sr: + component_hash: 809d0041cdddd98a278f0d5afef7bb60a451290577b98cf718dfffc91bdcbd9b + dependencies: + - name: espressif/cjson + registry_url: https://components.espressif.com + require: private + version: ^1.7.19 + - name: espressif/dl_fft + registry_url: https://components.espressif.com + require: private + version: '>=0.6.0' + - name: espressif/esp-dsp + registry_url: https://components.espressif.com + require: private + version: 1.8.0 + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com/ + type: service + version: 2.4.7 espressif/esp_audio_codec: component_hash: af4663830bc73eca58febcc34a976d003465e78ed57458aa00a14b231e392ca1 dependencies: @@ -71,12 +123,13 @@ dependencies: version: 1.20.4 direct_dependencies: - 78/esp-wifi-connect +- espressif/esp-sr - espressif/esp_audio_codec - espressif/esp_codec_dev - espressif/esp_websocket_client - espressif/mdns - idf - joltwallet/littlefs -manifest_hash: b7c0e21c22598b04ad72173217dbd4f7c984674b1b7df2f688b54eb19cff8a12 +manifest_hash: c05cc39447d3758499f6a9a2b947c3fe21cf4fd47e0e9905cde8d79dc68d0472 target: esp32s3 version: 2.0.0 diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index cce5281..61fd282 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -879,6 +879,99 @@ menu "Example Configuration" help Leave disabled for the default active-low Button 2 wiring. + config TQ_LOCAL_WAKE_ENABLE + bool "Enable local wake-word listener" + default y + help + Use the on-device Chinese MultiNet model to detect 喵掌柜. + The physical PTT button remains enabled as a parallel entry. + + config TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE + bool "Enable local wake response prompt" + depends on TQ_LOCAL_WAKE_ENABLE + default n + help + Play a short acknowledgement after 喵掌柜 is detected and + before the normal single-turn dialog starts. Keep disabled + until the placeholder Opus asset has been supplied. + + config TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE + string "Local wake response prompt file" + depends on TQ_LOCAL_WAKE_ENABLE + default "/littlefs/audio_prompts/local_wake_response.opus" + help + Reserved LittleFS path for the future wake acknowledgement. + Add local_wake_response.opus under + components/platform/assets/audio_prompts before enabling it. + + config TQ_LOCAL_WAKE_RETRY_MS + int "Local wake startup retry delay (ms)" + depends on TQ_LOCAL_WAKE_ENABLE + default 5000 + range 500 60000 + + config TQ_LOCAL_WAKE_STOP_WAIT_MS + int "Local wake lifecycle wait (ms)" + depends on TQ_LOCAL_WAKE_ENABLE + default 1000 + range 100 5000 + + config TQ_LOCAL_WAKE_TASK_STACK_BYTES + int "Local wake control task stack (bytes)" + depends on TQ_LOCAL_WAKE_ENABLE + default 6144 + range 4096 32768 + + config TQ_LOCAL_WAKE_TASK_PRIORITY + int "Local wake control task priority" + depends on TQ_LOCAL_WAKE_ENABLE + default 4 + range 1 10 + + config TQ_LOCAL_WAKE_TASK_CORE + int "Local wake control task core (-1 for no affinity)" + depends on TQ_LOCAL_WAKE_ENABLE + default 0 + range -1 1 + + config TQ_LOCAL_COMMAND_TASK_STACK_BYTES + int "MultiNet inference task stack (bytes)" + depends on TQ_LOCAL_WAKE_ENABLE + default 8192 + range 6144 32768 + + config TQ_LOCAL_COMMAND_TASK_PRIORITY + int "MultiNet inference task priority" + depends on TQ_LOCAL_WAKE_ENABLE + default 7 + range 1 10 + + config TQ_LOCAL_COMMAND_TASK_CORE + int "MultiNet inference task core (-1 for no affinity)" + depends on TQ_LOCAL_WAKE_ENABLE + default 0 + range -1 1 + + config TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS + int "MultiNet control acknowledgement timeout (ms)" + depends on TQ_LOCAL_WAKE_ENABLE + default 1000 + range 100 5000 + + config TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS + int "MultiNet utterance detection timeout (ms)" + depends on TQ_LOCAL_WAKE_ENABLE + default 6000 + range 1000 15000 + + config TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000 + int "MultiNet detection threshold (x10000)" + depends on TQ_LOCAL_WAKE_ENABLE + default 1200 + range 1 9999 + help + Initial threshold is 0.12 and must be tuned with device audio. + config MULTIMODAL_HEARTBEAT_INTERVAL_MS int "Heartbeat interval (ms)" default 50000 @@ -966,10 +1059,11 @@ menu "Example Configuration" config MULTIMODAL_AUDIO_CODEC_CHANNELS int "Codec channels" - default 1 - range 1 1 + default 2 + range 2 2 help - Fixed at 1 for the raw-opus 16kHz mono voice contract. + Two physical ES8311 slots are captured: microphone and DAC + playback reference. The external Opus contract remains mono. config MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE int "Codec bits per sample" @@ -1051,6 +1145,22 @@ menu "Example Configuration" speech. Use 0 for the default left/first channel, or 1 if level logs show speech energy on raw_ch1 instead of raw_ch0. + config MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL + int "AEC playback reference channel" + default 1 + range 0 1 + help + Interleaved ES8311 slot carrying the DAC playback reference. + It must differ from the microphone channel. + + config MULTIMODAL_AUDIO_AEC_FILTER_LENGTH + int "AEC filter length" + default 4 + range 1 8 + help + ESP-SR synchronous AEC filter length. Four is the recommended + starting point for ESP32-S3. + config MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS bool "Log microphone uplink audio levels" default n diff --git a/sdkconfig.defaults b/sdkconfig.defaults index b45c537..1f021c4 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -59,6 +59,22 @@ CONFIG_TQ_KEY_POST_DOUBLE_CLICK_SUPPRESS_MS=450 CONFIG_TQ_PTT_MODE_CLICK_TO_TALK=y # CONFIG_TQ_PTT_MODE_HOLD_TO_TALK is not set CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS=8000 +CONFIG_TQ_LOCAL_WAKE_ENABLE=y +# CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE is not set +CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE="/littlefs/audio_prompts/local_wake_response.opus" +CONFIG_TQ_LOCAL_WAKE_RETRY_MS=5000 +CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS=1000 +CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES=6144 +CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY=4 +CONFIG_TQ_LOCAL_WAKE_TASK_CORE=0 +CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES=8192 +CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY=7 +CONFIG_TQ_LOCAL_COMMAND_TASK_CORE=0 +CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS=1000 +CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS=6000 +CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000=1200 +CONFIG_SR_MN_CN_MULTINET7_QUANT=y +CONFIG_SR_MN_EN_NONE=y # Internal DMA-capable SRAM budget. TLS/auth/fortune flows, hardware AES DMA # bounce buffers, LCD DMA, and worker task stacks all need internal-only memory. @@ -150,7 +166,9 @@ CONFIG_MULTIMODAL_AUDIO_UPLINK_TASK_STACK=32768 CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK=12288 CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_LEN=24 CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TIMEOUT_MS=5 -CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS=1 +CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS=2 +CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL=1 +CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH=4 CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES=1024 CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK=12288 CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_LEN=1024 diff --git a/tools/check_architecture_guards.sh b/tools/check_architecture_guards.sh index 847029b..fc79092 100644 --- a/tools/check_architecture_guards.sh +++ b/tools/check_architecture_guards.sh @@ -31,8 +31,44 @@ fi if rg -n "platform_multimodal_downstream_playback_event|PLATFORM_MULTIMODAL_DOWNSTREAM_PLAYBACK" components/platform/include/platform.h >/dev/null 2>&1; then fail "platform facade must not duplicate domain-owned multimodal semantic events" fi -if rg -n "esp-sr|esp_sr|esp_afe|afe_config|afe_sr|model_path\\.h|srmodels|CONFIG_SR_|SR_WN|MULTIMODAL_WAKE_WORD|wake_word|Wake word|Wake-word|wn9_" components main sdkconfig.defaults dependencies.lock >/dev/null 2>&1; then - fail "ESP-SR and wake-word code/dependencies must stay removed" +if rg -n "esp_afe_sr|afe_config_init|afe_config_check|esp_afe_handle_from_config|SR_WN|MULTIMODAL_WAKE_WORD|wake_word|Wake word|Wake-word|wn9_" \ + components/platform/{include,internal,src} \ + components/control_plane/{include,internal,src} \ + components/app_composition/{include,internal,src} >/dev/null 2>&1 || + rg -n 'model_path\\.h|esp_srmodel_|esp_mn_' components/domain/{include,internal,src} \ + --glob '!domain_local_command.c' >/dev/null 2>&1; then + fail "Local speech models must stay domain-owned and WakeNet/full AFE SR must stay absent" +fi +if ! rg -n 'espressif/esp-sr: \^2\.4\.7' components/platform/idf_component.yml >/dev/null 2>&1 || + ! rg -n 'DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI' \ + components/domain/include/domain.h \ + components/domain/src/runtime/domain_local_command.c >/dev/null 2>&1 || + ! rg -n 'miao zhang gui' \ + components/domain/src/runtime/domain_local_command.c >/dev/null 2>&1; then + fail "Local command detection must keep the ESP-SR dependency and 喵掌柜 domain contract" +fi +if ! rg -n 'TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE' \ + main/Kconfig.projbuild \ + components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1 || + ! rg -n '/littlefs/audio_prompts/local_wake_response\.opus' \ + main/Kconfig.projbuild \ + sdkconfig.defaults \ + components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1; then + fail "Local wake response prompt must keep its disabled asset placeholder contract" +fi +if ! rg -n '#include "esp_afe_aec.h"' components/platform/src/multimodal/audio_pipeline.c >/dev/null 2>&1 || + ! rg -n 'afe_aec_create\(' components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c >/dev/null 2>&1 || + ! rg -n 'afe_aec_process\(' components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c >/dev/null 2>&1; then + fail "Platform must own the synchronous ESP-SR AEC implementation" +fi +if rg -n 'esp_afe_aec\.h|afe_aec_process\(' \ + components/domain/{include,internal,src} \ + components/control_plane/{include,internal,src} \ + components/app_composition/{include,internal,src} >/dev/null 2>&1; then + fail "AEC execution must remain a platform primitive" +fi +if ! rg -n '^CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS=2\r?$' sdkconfig.defaults >/dev/null 2>&1; then + fail "ES8311 microphone and DAC reference capture requires two physical codec slots" fi if rg -n "platform_lcd_trigger_slot_animation|lcd_trigger_slot_animation" components/platform/include components/platform/internal components/platform/src/runtime >/dev/null 2>&1; then fail "platform must expose LCD presentation primitives, not one-off expression APIs"