feat(voice): add local VAD speech endpoint detection

2026-10-23 10:59:26 +08:00
parent ee32b4d63f
commit f17ce857ff
17 changed files with 1183 additions and 141 deletions
@@ -11,6 +11,7 @@ idf_component_register(
        "src/runtime/domain_runtime_adapters.c"
        "src/runtime/domain_ota.c"
        "src/runtime/domain_local_command.c"
+        "src/runtime/domain_speech_endpoint.c"
        "src/runtime/domain_mini_program_provisioning.c"
    INCLUDE_DIRS
        "include"
@@ -297,16 +297,44 @@ esp_err_t domain_runtime_multimodal_audio_start_uplink(
        void *user_data);

 /**
- * @brief 有界停止多模态上行采集和流发送。 / Stop uplink capture/stream with bounded teardown.
+ * @brief 暂停采集、补齐尾帧并限时排空发送；常驻任务由 deinit 回收。
 *
- * `timeout_ms < 0` is invalid, zero requests asynchronous stop, and positive
- * values wait up to the bound before lower layers apply deterministic fallback.
+ * 负超时无效；零尝试立即完成，否则 ESP_ERR_TIMEOUT；正数为完整调用预算。
+ * 成功后没有在途采集/发送；超时保留资源供重试，不后台清理，不应发送 StopSpeech。
 */
 esp_err_t domain_runtime_multimodal_audio_stop_uplink(int timeout_ms);
 bool domain_runtime_multimodal_audio_is_uplink_running(void);
 bool domain_runtime_multimodal_audio_is_initialized(void);
 /** 保留麦克风采集，仅控制扬声器输出。 / Gate speaker output while preserving capture. */
 esp_err_t domain_runtime_multimodal_audio_set_speaker_output_enabled(bool enabled);
+
+typedef struct {
+    uint32_t generation;
+    uint32_t min_speech_ms;
+    uint32_t end_silence_ms;
+} domain_speech_endpoint_config_t;
+
+typedef struct {
+    uint32_t generation;
+    bool active;
+    bool speech_confirmed;
+    bool ended;
+    uint32_t audio_ms;
+    uint32_t silence_ms;
+    uint32_t max_process_us;
+    uint32_t discontinuities;
+} domain_speech_endpoint_snapshot_t;
+
+/**
+ * 本地端点检测，无新 worker。start 对相同配置幂等，不同配置须先 stop。
+ * start/stop/get_snapshot 最多等锁 100 ms；stop 另以同一总预算等待 PCM 回调。
+ * stop 超时保持资源可重试，绝不后台释放；成功移除订阅并销毁 VAD。
+ * PCM 回调只用零等待锁及固定 20 ms 分帧，不执行协议或阻塞式收尾。
+ * 配置关闭时 start 返回 ESP_ERR_NOT_SUPPORTED，stop 幂等，快照为 inactive。
+ */
+esp_err_t domain_speech_endpoint_start(const domain_speech_endpoint_config_t *config);
+esp_err_t domain_speech_endpoint_stop(void);
+esp_err_t domain_speech_endpoint_get_snapshot(domain_speech_endpoint_snapshot_t *out);
 /**
 * @brief 启动“喵掌柜”本地 MultiNet 检测。 / Start local MultiNet detection for "喵掌柜".
 *
@@ -0,0 +1,241 @@
+#include "domain.h"
+
+#include <string.h>
+
+#include "sdkconfig.h"
+
+#if CONFIG_TQ_SPEECH_ENDPOINT_ENABLE
+#include "esp_heap_caps.h"
+#include "esp_log.h"
+#include "esp_timer.h"
+#include "esp_vad.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/semphr.h"
+#include "freertos/task.h"
+#include "platform.h"
+
+#define ENDPOINT_LOCK_MS 100U
+#define ENDPOINT_RATE_HZ 16000U
+#define ENDPOINT_FRAME_MS 20U
+#define ENDPOINT_FRAME_SAMPLES 320U
+#define ENDPOINT_MAX_INPUT_SAMPLES 512U
+#define ENDPOINT_GAP_US 100000LL
+
+static const char *TAG = "domain_endpoint";
+static portMUX_TYPE s_init_lock = portMUX_INITIALIZER_UNLOCKED;
+static StaticSemaphore_t s_mutex_storage;
+static SemaphoreHandle_t s_mutex;
+static bool s_initializing;
+static bool s_gap_pending;
+static vad_handle_t s_vad;
+static int16_t s_pcm[ENDPOINT_FRAME_SAMPLES];
+static size_t s_fill;
+static uint32_t s_speech_run_ms;
+static int64_t s_last_pcm_us;
+static domain_speech_endpoint_config_t s_config;
+static domain_speech_endpoint_snapshot_t s_snapshot;
+
+static bool endpoint_lock(void)
+{
+    portENTER_CRITICAL(&s_init_lock);
+    SemaphoreHandle_t mutex = s_mutex;
+    bool create = mutex == NULL && !s_initializing;
+    if (create) {
+        s_initializing = true;
+    }
+    portEXIT_CRITICAL(&s_init_lock);
+    if (create) {
+        mutex = xSemaphoreCreateMutexStatic(&s_mutex_storage);
+        portENTER_CRITICAL(&s_init_lock);
+        s_mutex = mutex;
+        s_initializing = false;
+        portEXIT_CRITICAL(&s_init_lock);
+    }
+    return mutex != NULL &&
+           xSemaphoreTake(mutex, pdMS_TO_TICKS(ENDPOINT_LOCK_MS)) == pdTRUE;
+}
+
+static void endpoint_reset_continuity(void)
+{
+    s_fill = 0U;
+    s_speech_run_ms = 0U;
+    s_snapshot.silence_ms = 0U;
+    ++s_snapshot.discontinuities;
+}
+
+static void endpoint_pcm_observer(const int16_t *pcm, size_t samples,
+                                  uint32_t sample_rate_hz, void *user_data)
+{
+    (void)user_data;
+    /* 注册发生在锁创建后。回调绝不等待控制线程，也不调用停止/发送接口。 */
+    if (xSemaphoreTake(s_mutex, 0U) != pdTRUE) {
+        portENTER_CRITICAL(&s_init_lock);
+        s_gap_pending = true;
+        portEXIT_CRITICAL(&s_init_lock);
+        return;
+    }
+    if (!s_snapshot.active || s_snapshot.ended) {
+        xSemaphoreGive(s_mutex);
+        return;
+    }
+    const int64_t started_us = esp_timer_get_time();
+    portENTER_CRITICAL(&s_init_lock);
+    bool gap = s_gap_pending;
+    s_gap_pending = false;
+    portEXIT_CRITICAL(&s_init_lock);
+    if (gap || (s_last_pcm_us != 0 && started_us - s_last_pcm_us > ENDPOINT_GAP_US)) {
+        endpoint_reset_continuity();
+    }
+    s_last_pcm_us = started_us;
+    if (pcm == NULL || samples == 0U || samples > ENDPOINT_MAX_INPUT_SAMPLES ||
+        sample_rate_hz != ENDPOINT_RATE_HZ) {
+        endpoint_reset_continuity();
+        xSemaphoreGive(s_mutex);
+        return;
+    }
+
+    while (samples > 0U && !s_snapshot.ended) {
+        size_t count = ENDPOINT_FRAME_SAMPLES - s_fill;
+        if (count > samples) {
+            count = samples;
+        }
+        memcpy(s_pcm + s_fill, pcm, count * sizeof(int16_t));
+        s_fill += count;
+        pcm += count;
+        samples -= count;
+        if (s_fill != ENDPOINT_FRAME_SAMPLES) {
+            continue;
+        }
+        s_fill = 0U;
+        vad_state_t activity = vad_process(s_vad, s_pcm,
+                                           ENDPOINT_RATE_HZ, ENDPOINT_FRAME_MS);
+        s_snapshot.audio_ms += ENDPOINT_FRAME_MS;
+        if (activity == VAD_SPEECH) {
+            s_snapshot.silence_ms = 0U;
+            if (!s_snapshot.speech_confirmed) {
+                s_speech_run_ms += ENDPOINT_FRAME_MS;
+                s_snapshot.speech_confirmed = s_speech_run_ms >= s_config.min_speech_ms;
+            }
+        } else {
+            s_speech_run_ms = 0U;
+            if (s_snapshot.speech_confirmed) {
+                s_snapshot.silence_ms += ENDPOINT_FRAME_MS;
+                s_snapshot.ended = s_snapshot.silence_ms >= s_config.end_silence_ms;
+            }
+        }
+    }
+    uint32_t elapsed_us = (uint32_t)(esp_timer_get_time() - started_us);
+    if (elapsed_us > s_snapshot.max_process_us) {
+        s_snapshot.max_process_us = elapsed_us;
+    }
+    xSemaphoreGive(s_mutex);
+}
+
+esp_err_t domain_speech_endpoint_start(const domain_speech_endpoint_config_t *config)
+{
+    if (config == NULL || config->generation == 0U ||
+        config->min_speech_ms < 100U || config->min_speech_ms > 1000U ||
+        config->end_silence_ms < 400U || config->end_silence_ms > 2000U) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    if (!endpoint_lock()) {
+        return ESP_ERR_TIMEOUT;
+    }
+    if (s_vad != NULL) {
+        bool same = s_snapshot.active && s_config.generation == config->generation &&
+                    s_config.min_speech_ms == config->min_speech_ms &&
+                    s_config.end_silence_ms == config->end_silence_ms;
+        xSemaphoreGive(s_mutex);
+        return same ? ESP_OK : ESP_ERR_INVALID_STATE;
+    }
+    s_vad = vad_create(VAD_MODE_1);
+    if (s_vad == NULL) {
+        xSemaphoreGive(s_mutex);
+        return ESP_ERR_NO_MEM;
+    }
+    s_config = *config;
+    s_snapshot = (domain_speech_endpoint_snapshot_t) {
+        .generation = config->generation,
+        .active = true,
+    };
+    s_fill = 0U;
+    s_speech_run_ms = 0U;
+    s_last_pcm_us = 0;
+    portENTER_CRITICAL(&s_init_lock);
+    s_gap_pending = false;
+    portEXIT_CRITICAL(&s_init_lock);
+    esp_err_t err = platform_multimodal_audio_register_pcm_observer(endpoint_pcm_observer, NULL);
+    if (err != ESP_OK) {
+        s_snapshot.active = false;
+        vad_destroy(s_vad);
+        s_vad = NULL;
+    }
+    xSemaphoreGive(s_mutex);
+    if (err == ESP_OK) {
+        ESP_LOGI(TAG, "VAD armed: generation=%u speech_ms=%u silence_ms=%u internal_free=%u psram_free=%u",
+                 (unsigned)config->generation, (unsigned)config->min_speech_ms,
+                 (unsigned)config->end_silence_ms,
+                 (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),
+                 (unsigned)heap_caps_get_free_size(MALLOC_CAP_SPIRAM));
+    }
+    return err;
+}
+
+esp_err_t domain_speech_endpoint_stop(void)
+{
+    const TickType_t started = xTaskGetTickCount();
+    if (!endpoint_lock()) {
+        return ESP_ERR_TIMEOUT;
+    }
+    s_snapshot.active = false;
+    if (s_vad == NULL) {
+        xSemaphoreGive(s_mutex);
+        return ESP_OK;
+    }
+    uint32_t elapsed_ms = (uint32_t)pdTICKS_TO_MS(xTaskGetTickCount() - started);
+    int remaining_ms = elapsed_ms < ENDPOINT_LOCK_MS ? (int)(ENDPOINT_LOCK_MS - elapsed_ms) : 0;
+    /* 回调仅 try-lock，所以持有状态锁等待注销不会形成互等。 */
+    esp_err_t err = platform_multimodal_audio_unregister_pcm_observer(
+            endpoint_pcm_observer, NULL, remaining_ms);
+    if (err == ESP_OK) {
+        vad_destroy(s_vad);
+        s_vad = NULL;
+        s_fill = 0U;
+    }
+    xSemaphoreGive(s_mutex);
+    return err;
+}
+
+esp_err_t domain_speech_endpoint_get_snapshot(domain_speech_endpoint_snapshot_t *out)
+{
+    if (out == NULL) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    if (!endpoint_lock()) {
+        return ESP_ERR_TIMEOUT;
+    }
+    *out = s_snapshot;
+    xSemaphoreGive(s_mutex);
+    return ESP_OK;
+}
+#else
+esp_err_t domain_speech_endpoint_start(const domain_speech_endpoint_config_t *config)
+{
+    (void)config;
+    return ESP_ERR_NOT_SUPPORTED;
+}
+
+esp_err_t domain_speech_endpoint_stop(void)
+{
+    return ESP_OK;
+}
+
+esp_err_t domain_speech_endpoint_get_snapshot(domain_speech_endpoint_snapshot_t *out)
+{
+    if (out == NULL) {
+        return ESP_ERR_INVALID_ARG;
+    }
+    memset(out, 0, sizeof(*out));
+    return ESP_OK;
+}
+#endif