feat(voice): add on-device Miao Zhang Gui wake word

Add an optional local wake response prompt configuration. The prompt asset is intentionally not included yet.

components/control_plane/CMakeLists.txt

@@ -6,6 +6,7 @@ idf_component_register(
         "src/multimodal/control_plane_model_command_extractor.c"
         "src/multimodal/control_plane_multimodal_protocol.c"
         "src/multimodal/control_plane_multimodal_runtime.c"
+        "src/multimodal/control_plane_multimodal_local_wake.c"
         "src/multimodal/control_plane_multimodal_device_auth.c"
         "src/multimodal/control_plane_multimodal_auth_service.c"
         "src/multimodal/control_plane_multimodal_session_service.c"

components/control_plane/internal/control_plane_internal.h

@@ -133,6 +133,11 @@ esp_err_t control_plane_multimodal_play_downstream_audio(const uint8_t *data, si
 esp_err_t control_plane_multimodal_handle_ptt_press_step(void);
 esp_err_t control_plane_multimodal_handle_ptt_release(void);
 void control_plane_multimodal_reset_uplink_failure_window(void);
+esp_err_t control_plane_local_wake_start(void);
+esp_err_t control_plane_local_wake_stop(int timeout_ms);
+void control_plane_local_wake_pause_for_foreground(void);
+void control_plane_local_wake_resume_after_foreground(void);
+bool control_plane_voice_turn_is_idle(void);
 
 esp_err_t control_plane_start_system_event_task(void);
 void control_plane_stop_system_event_task(void);

components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c

@@ -0,0 +1,384 @@
+#include "control_plane_internal.h"
+
+#include <stdint.h>
+
+#include "domain.h"
+#include "esp_log.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+#include "sdkconfig.h"
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_ENABLE
+#define CONFIG_TQ_LOCAL_WAKE_ENABLE 0
+#endif
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
+#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE 0
+#endif
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE
+#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE \
+    "/littlefs/audio_prompts/local_wake_response.opus"
+#endif
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_RETRY_MS
+#define CONFIG_TQ_LOCAL_WAKE_RETRY_MS 5000
+#endif
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS
+#define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000
+#endif
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES
+#define CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES 6144
+#endif
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY
+#define CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY 4
+#endif
+
+#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_CORE
+#define CONFIG_TQ_LOCAL_WAKE_TASK_CORE 0
+#endif
+
+#define CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN 4U
+#define CONTROL_PLANE_LOCAL_WAKE_POLL_MS 100U
+
+#if CONFIG_TQ_LOCAL_WAKE_TASK_CORE < 0
+#define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE tskNO_AFFINITY
+#else
+#define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE CONFIG_TQ_LOCAL_WAKE_TASK_CORE
+#endif
+
+typedef struct {
+    TaskHandle_t task;
+    QueueHandle_t detection_queue;
+    bool stop_requested;
+    bool started;
+    bool paused_for_foreground;
+    bool capture_owned;
+} local_wake_state_t;
+
+static const char *TAG = "cp_local_wake";
+static local_wake_state_t s_local_wake;
+static portMUX_TYPE s_local_wake_lock = portMUX_INITIALIZER_UNLOCKED;
+
+static local_wake_state_t local_wake_snapshot(void)
+{
+    local_wake_state_t snapshot;
+    portENTER_CRITICAL(&s_local_wake_lock);
+    snapshot = s_local_wake;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+    return snapshot;
+}
+
+static void local_wake_set_capture_owned(bool owned)
+{
+    portENTER_CRITICAL(&s_local_wake_lock);
+    s_local_wake.capture_owned = owned;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+}
+
+static void local_wake_domain_event(
+        const domain_local_command_event_t *event,
+        void *user_data)
+{
+    (void)user_data;
+    local_wake_state_t state = local_wake_snapshot();
+    if (event == NULL || state.stop_requested || state.detection_queue == NULL) {
+        return;
+    }
+    if (xQueueSend(state.detection_queue, event, 0U) != pdTRUE) {
+        ESP_EARLY_LOGW(TAG, "Drop local wake detection: queue full");
+    }
+}
+
+static void local_wake_stop_owned_capture(void)
+{
+    local_wake_state_t state = local_wake_snapshot();
+    if (!state.capture_owned) {
+        return;
+    }
+    local_wake_set_capture_owned(false);
+    esp_err_t err = domain_runtime_multimodal_audio_stop_uplink(
+            CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
+    if (err != ESP_OK && err != ESP_ERR_TIMEOUT) {
+        ESP_LOGW(TAG, "Stop local wake capture failed: %s", esp_err_to_name(err));
+    }
+}
+
+static esp_err_t local_wake_start_listening(void)
+{
+    local_wake_state_t state = local_wake_snapshot();
+    if (state.stop_requested || state.paused_for_foreground ||
+        !control_plane_voice_turn_is_idle()) {
+        return ESP_ERR_INVALID_STATE;
+    }
+
+    if (!domain_runtime_multimodal_audio_is_initialized()) {
+        esp_err_t init_err = domain_runtime_multimodal_audio_init();
+        if (init_err != ESP_OK) {
+            return init_err;
+        }
+    }
+
+    domain_local_command_snapshot_t command_state = {0};
+    esp_err_t snapshot_err =
+            domain_runtime_local_command_get_snapshot(&command_state);
+    if (snapshot_err != ESP_OK || !command_state.started) {
+        esp_err_t start_err = domain_runtime_local_command_start(
+                local_wake_domain_event,
+                NULL);
+        if (start_err != ESP_OK) {
+            return start_err;
+        }
+    } else if (!command_state.detection_running) {
+        esp_err_t resume_err = domain_runtime_local_command_resume(
+                CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
+        if (resume_err != ESP_OK) {
+            return resume_err;
+        }
+    }
+
+    if (!domain_runtime_multimodal_audio_is_uplink_running()) {
+        esp_err_t capture_err =
+                domain_runtime_multimodal_audio_start_uplink(NULL, NULL);
+        if (capture_err != ESP_OK) {
+            return capture_err;
+        }
+    }
+    local_wake_set_capture_owned(true);
+    return ESP_OK;
+}
+
+static void local_wake_handle_detection(
+        const domain_local_command_event_t *event)
+{
+    if (event == NULL ||
+        event->type != DOMAIN_LOCAL_COMMAND_EVENT_DETECTED ||
+        event->command_id != DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI) {
+        return;
+    }
+
+    portENTER_CRITICAL(&s_local_wake_lock);
+    if (s_local_wake.stop_requested || s_local_wake.paused_for_foreground) {
+        portEXIT_CRITICAL(&s_local_wake_lock);
+        return;
+    }
+    s_local_wake.paused_for_foreground = true;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+
+    local_wake_stop_owned_capture();
+    (void)domain_runtime_local_command_suspend(
+            CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
+
+    ESP_LOGI(TAG,
+             "Local wake accepted: phrase=喵掌柜 confidence_x10000=%u generation=%u",
+             (unsigned)event->confidence_x10000,
+             (unsigned)event->detection_generation);
+
+#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
+    if (CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE[0] != '\0') {
+        esp_err_t prompt_err = domain_runtime_behavior_play_prompt_file(
+                CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE);
+        if (prompt_err != ESP_OK) {
+            ESP_LOGW(TAG,
+                     "Local wake response prompt failed: %s",
+                     esp_err_to_name(prompt_err));
+        }
+    }
+#endif
+
+    control_plane_set_ptt_active(true);
+}
+
+static void local_wake_task(void *arg)
+{
+    (void)arg;
+    TickType_t retry_not_before = 0U;
+    bool listening_logged = false;
+
+    ESP_LOGI(TAG, "Local wake worker started: phrase=喵掌柜");
+    while (!local_wake_snapshot().stop_requested) {
+        local_wake_state_t state = local_wake_snapshot();
+        TickType_t now = xTaskGetTickCount();
+        if (!state.paused_for_foreground &&
+            control_plane_voice_turn_is_idle() &&
+            (retry_not_before == 0U || now >= retry_not_before)) {
+            esp_err_t listen_err = local_wake_start_listening();
+            if (listen_err == ESP_OK) {
+                retry_not_before = 0U;
+                if (!listening_logged) {
+                    ESP_LOGI(TAG, "Local wake listening ready: phrase=喵掌柜");
+                    listening_logged = true;
+                }
+            } else if (listen_err != ESP_ERR_INVALID_STATE) {
+                ESP_LOGW(TAG,
+                         "Local wake listener unavailable: %s; retry in %ums",
+                         esp_err_to_name(listen_err),
+                         (unsigned)CONFIG_TQ_LOCAL_WAKE_RETRY_MS);
+                retry_not_before =
+                        now + pdMS_TO_TICKS(CONFIG_TQ_LOCAL_WAKE_RETRY_MS);
+                listening_logged = false;
+            }
+        }
+
+        domain_local_command_event_t event = {0};
+        if (state.detection_queue != NULL &&
+            xQueueReceive(state.detection_queue,
+                          &event,
+                          pdMS_TO_TICKS(CONTROL_PLANE_LOCAL_WAKE_POLL_MS)) ==
+                    pdTRUE) {
+            local_wake_handle_detection(&event);
+        } else {
+            ulTaskNotifyTake(pdTRUE, 0U);
+        }
+    }
+
+    local_wake_stop_owned_capture();
+    (void)domain_runtime_local_command_stop(
+            CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
+    portENTER_CRITICAL(&s_local_wake_lock);
+    s_local_wake.task = NULL;
+    s_local_wake.started = false;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+    ESP_LOGI(TAG, "Local wake worker stopped");
+    vTaskDelete(NULL);
+}
+
+esp_err_t control_plane_local_wake_start(void)
+{
+#if !CONFIG_TQ_LOCAL_WAKE_ENABLE
+    return ESP_OK;
+#else
+    portENTER_CRITICAL(&s_local_wake_lock);
+    if (s_local_wake.task != NULL || s_local_wake.started) {
+        portEXIT_CRITICAL(&s_local_wake_lock);
+        return ESP_OK;
+    }
+    portEXIT_CRITICAL(&s_local_wake_lock);
+
+    if (s_local_wake.detection_queue == NULL) {
+        QueueHandle_t queue = xQueueCreate(
+                CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN,
+                sizeof(domain_local_command_event_t));
+        if (queue == NULL) {
+            return ESP_ERR_NO_MEM;
+        }
+        portENTER_CRITICAL(&s_local_wake_lock);
+        if (s_local_wake.detection_queue == NULL) {
+            s_local_wake.detection_queue = queue;
+            queue = NULL;
+        }
+        portEXIT_CRITICAL(&s_local_wake_lock);
+        if (queue != NULL) {
+            vQueueDelete(queue);
+        }
+    }
+
+    portENTER_CRITICAL(&s_local_wake_lock);
+    s_local_wake.stop_requested = false;
+    s_local_wake.paused_for_foreground = false;
+    s_local_wake.capture_owned = false;
+    s_local_wake.started = true;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+
+    TaskHandle_t task = NULL;
+    BaseType_t created = xTaskCreatePinnedToCore(
+            local_wake_task,
+            "cp_local_wake",
+            CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES,
+            NULL,
+            CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY,
+            &task,
+            CONTROL_PLANE_LOCAL_WAKE_TASK_CORE);
+    if (created != pdPASS) {
+        portENTER_CRITICAL(&s_local_wake_lock);
+        s_local_wake.started = false;
+        portEXIT_CRITICAL(&s_local_wake_lock);
+        return ESP_ERR_NO_MEM;
+    }
+    portENTER_CRITICAL(&s_local_wake_lock);
+    s_local_wake.task = task;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+    return ESP_OK;
+#endif
+}
+
+esp_err_t control_plane_local_wake_stop(int timeout_ms)
+{
+#if !CONFIG_TQ_LOCAL_WAKE_ENABLE
+    (void)timeout_ms;
+    return ESP_OK;
+#else
+    if (timeout_ms < 0) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    local_wake_state_t state = local_wake_snapshot();
+    if (!state.started && state.task == NULL) {
+        return ESP_OK;
+    }
+
+    portENTER_CRITICAL(&s_local_wake_lock);
+    s_local_wake.stop_requested = true;
+    TaskHandle_t task = s_local_wake.task;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+    if (task != NULL) {
+        xTaskNotifyGive(task);
+    }
+
+    TickType_t started = xTaskGetTickCount();
+    TickType_t timeout_ticks = pdMS_TO_TICKS((uint32_t)timeout_ms);
+    if (timeout_ms > 0 && timeout_ticks == 0U) {
+        timeout_ticks = 1U;
+    }
+    while (local_wake_snapshot().task != NULL &&
+           timeout_ms > 0 &&
+           (xTaskGetTickCount() - started) < timeout_ticks) {
+        vTaskDelay(pdMS_TO_TICKS(20));
+    }
+    state = local_wake_snapshot();
+    if (state.task != NULL) {
+        vTaskDelete(state.task);
+        local_wake_stop_owned_capture();
+        (void)domain_runtime_local_command_stop(
+                CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
+        portENTER_CRITICAL(&s_local_wake_lock);
+        s_local_wake.task = NULL;
+        s_local_wake.started = false;
+        portEXIT_CRITICAL(&s_local_wake_lock);
+        return ESP_ERR_TIMEOUT;
+    }
+    return ESP_OK;
+#endif
+}
+
+void control_plane_local_wake_pause_for_foreground(void)
+{
+#if CONFIG_TQ_LOCAL_WAKE_ENABLE
+    portENTER_CRITICAL(&s_local_wake_lock);
+    s_local_wake.paused_for_foreground = true;
+    TaskHandle_t task = s_local_wake.task;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+    (void)domain_runtime_local_command_suspend(
+            CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
+    local_wake_stop_owned_capture();
+    if (task != NULL) {
+        xTaskNotifyGive(task);
+    }
+#endif
+}
+
+void control_plane_local_wake_resume_after_foreground(void)
+{
+#if CONFIG_TQ_LOCAL_WAKE_ENABLE
+    portENTER_CRITICAL(&s_local_wake_lock);
+    s_local_wake.paused_for_foreground = false;
+    TaskHandle_t task = s_local_wake.task;
+    portEXIT_CRITICAL(&s_local_wake_lock);
+    if (task != NULL) {
+        xTaskNotifyGive(task);
+    }
+#endif
+}

components/control_plane/src/multimodal/control_plane_multimodal_runtime.c

@@ -575,6 +575,16 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
                 ESP_LOGI(TAG, "Pause uplink during TTS responding");
                 control_plane_multimodal_stop_uplink();
             }
+#if CONFIG_TQ_PTT_MODE_CLICK_TO_TALK
+            if (!control_plane_voice_turn_is_idle()) {
+                /*
+                 * The server response closes this one-shot voice turn.  Once
+                 * teardown reaches IDLE, the local listener can resume with
+                 * AEC-clean PCM and accept 喵掌柜 while TTS is playing.
+                 */
+                control_plane_set_ptt_active(false);
+            }
+#endif
             break;
         }
 

components/control_plane/src/multimodal/control_plane_multimodal_session_service.c

@@ -67,12 +67,14 @@
 #define CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS 24
 #endif
 
+#define CONTROL_PLANE_MULTIMODAL_OPUS_CHANNELS 1
+
 #if CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE != 16000
 #error "Multimodal voice requires 16 kHz audio"
 #endif
 
-#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1
-#error "Multimodal voice requires mono audio"
+#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2
+#error "Multimodal AEC requires stereo codec capture"
 #endif
 
 #if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60
@@ -1943,7 +1945,7 @@ esp_err_t control_plane_multimodal_send_start(void)
              (unsigned)sample_rate,
              (unsigned)opus_frame_duration_ms,
              (unsigned)CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS,
-             CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS,
+             CONTROL_PLANE_MULTIMODAL_OPUS_CHANNELS,
              CONFIG_MULTIMODAL_AUDIO_PA_GPIO);
 
     return ESP_OK;

components/control_plane/src/runtime/control_plane_runtime_workers.c

@@ -88,6 +88,10 @@
 #define CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS 8000
 #endif
 
+#ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS
+#define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000
+#endif
+
 #ifndef CONFIG_TQ_NETWORK_TASK_CORE
 #define CONFIG_TQ_NETWORK_TASK_CORE 0
 #endif
@@ -303,6 +307,9 @@ static void control_plane_notify_wifi_state_task(void)
 
 void control_plane_set_ptt_active(bool active)
 {
+    if (active) {
+        control_plane_local_wake_pause_for_foreground();
+    }
     bool notify_worker = false;
     ptt_state_t old_state = PTT_STATE_IDLE;
     ptt_state_t new_state = PTT_STATE_IDLE;
@@ -356,6 +363,20 @@ void control_plane_set_ptt_active(bool active)
     if (notify_worker) {
         control_plane_notify_wifi_state_task();
     }
+    if (!active) {
+        control_plane_local_wake_resume_after_foreground();
+    }
+}
+
+bool control_plane_voice_turn_is_idle(void)
+{
+    bool idle = false;
+    portENTER_CRITICAL(&s_runtime_lock);
+    idle = s_ptt_runtime.state == PTT_STATE_IDLE &&
+           !s_ptt_runtime.active &&
+           !s_ptt_runtime.pending_press_after_teardown;
+    portEXIT_CRITICAL(&s_runtime_lock);
+    return idle;
 }
 
 static bool control_plane_get_ptt_active(void)
@@ -1880,6 +1901,15 @@ esp_err_t control_plane_start_runtime_workers(void)
         return key_input_err;
     }
 
+    esp_err_t local_wake_err = control_plane_local_wake_start();
+    if (local_wake_err != ESP_OK) {
+        ESP_LOGE(TAG,
+                 "Start local wake worker failed: %s",
+                 esp_err_to_name(local_wake_err));
+        control_plane_stop_runtime_workers();
+        return local_wake_err;
+    }
+
     esp_err_t remote_log_err = control_plane_remote_log_start();
     if (remote_log_err != ESP_OK && remote_log_err != ESP_ERR_NOT_SUPPORTED) {
         ESP_LOGW(TAG, "Remote log start failed: %s", esp_err_to_name(remote_log_err));
@@ -1946,6 +1976,7 @@ void control_plane_stop_runtime_workers(void)
 {
     control_plane_set_runtime_stop_requested(true);
     control_plane_stop_user_config_server();
+    (void)control_plane_local_wake_stop(CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
 
 #if CONFIG_TQ_FACTORY_CLI_ENABLE
     control_plane_factory_cli_stop();