feat(voice): add on-device Miao Zhang Gui wake word
Add an optional local wake response prompt configuration. The prompt asset is intentionally not included yet.
This commit is contained in:
chenjintao
@@ -6,6 +6,7 @@ idf_component_register(
|
||||
"src/multimodal/control_plane_model_command_extractor.c"
|
||||
"src/multimodal/control_plane_multimodal_protocol.c"
|
||||
"src/multimodal/control_plane_multimodal_runtime.c"
|
||||
"src/multimodal/control_plane_multimodal_local_wake.c"
|
||||
"src/multimodal/control_plane_multimodal_device_auth.c"
|
||||
"src/multimodal/control_plane_multimodal_auth_service.c"
|
||||
"src/multimodal/control_plane_multimodal_session_service.c"
|
||||
|
||||
@@ -133,6 +133,11 @@ esp_err_t control_plane_multimodal_play_downstream_audio(const uint8_t *data, si
|
||||
esp_err_t control_plane_multimodal_handle_ptt_press_step(void);
|
||||
esp_err_t control_plane_multimodal_handle_ptt_release(void);
|
||||
void control_plane_multimodal_reset_uplink_failure_window(void);
|
||||
esp_err_t control_plane_local_wake_start(void);
|
||||
esp_err_t control_plane_local_wake_stop(int timeout_ms);
|
||||
void control_plane_local_wake_pause_for_foreground(void);
|
||||
void control_plane_local_wake_resume_after_foreground(void);
|
||||
bool control_plane_voice_turn_is_idle(void);
|
||||
|
||||
esp_err_t control_plane_start_system_event_task(void);
|
||||
void control_plane_stop_system_event_task(void);
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
#include "control_plane_internal.h"
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "domain.h"
|
||||
#include "esp_log.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/queue.h"
|
||||
#include "freertos/task.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_ENABLE
|
||||
#define CONFIG_TQ_LOCAL_WAKE_ENABLE 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
|
||||
#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE
|
||||
#define CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE \
|
||||
"/littlefs/audio_prompts/local_wake_response.opus"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_RETRY_MS
|
||||
#define CONFIG_TQ_LOCAL_WAKE_RETRY_MS 5000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS
|
||||
#define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES
|
||||
#define CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES 6144
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY
|
||||
#define CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY 4
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_TASK_CORE
|
||||
#define CONFIG_TQ_LOCAL_WAKE_TASK_CORE 0
|
||||
#endif
|
||||
|
||||
#define CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN 4U
|
||||
#define CONTROL_PLANE_LOCAL_WAKE_POLL_MS 100U
|
||||
|
||||
#if CONFIG_TQ_LOCAL_WAKE_TASK_CORE < 0
|
||||
#define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE tskNO_AFFINITY
|
||||
#else
|
||||
#define CONTROL_PLANE_LOCAL_WAKE_TASK_CORE CONFIG_TQ_LOCAL_WAKE_TASK_CORE
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
TaskHandle_t task;
|
||||
QueueHandle_t detection_queue;
|
||||
bool stop_requested;
|
||||
bool started;
|
||||
bool paused_for_foreground;
|
||||
bool capture_owned;
|
||||
} local_wake_state_t;
|
||||
|
||||
static const char *TAG = "cp_local_wake";
|
||||
static local_wake_state_t s_local_wake;
|
||||
static portMUX_TYPE s_local_wake_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||
|
||||
static local_wake_state_t local_wake_snapshot(void)
|
||||
{
|
||||
local_wake_state_t snapshot;
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
snapshot = s_local_wake;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
static void local_wake_set_capture_owned(bool owned)
|
||||
{
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.capture_owned = owned;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
}
|
||||
|
||||
static void local_wake_domain_event(
|
||||
const domain_local_command_event_t *event,
|
||||
void *user_data)
|
||||
{
|
||||
(void)user_data;
|
||||
local_wake_state_t state = local_wake_snapshot();
|
||||
if (event == NULL || state.stop_requested || state.detection_queue == NULL) {
|
||||
return;
|
||||
}
|
||||
if (xQueueSend(state.detection_queue, event, 0U) != pdTRUE) {
|
||||
ESP_EARLY_LOGW(TAG, "Drop local wake detection: queue full");
|
||||
}
|
||||
}
|
||||
|
||||
static void local_wake_stop_owned_capture(void)
|
||||
{
|
||||
local_wake_state_t state = local_wake_snapshot();
|
||||
if (!state.capture_owned) {
|
||||
return;
|
||||
}
|
||||
local_wake_set_capture_owned(false);
|
||||
esp_err_t err = domain_runtime_multimodal_audio_stop_uplink(
|
||||
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
|
||||
if (err != ESP_OK && err != ESP_ERR_TIMEOUT) {
|
||||
ESP_LOGW(TAG, "Stop local wake capture failed: %s", esp_err_to_name(err));
|
||||
}
|
||||
}
|
||||
|
||||
static esp_err_t local_wake_start_listening(void)
|
||||
{
|
||||
local_wake_state_t state = local_wake_snapshot();
|
||||
if (state.stop_requested || state.paused_for_foreground ||
|
||||
!control_plane_voice_turn_is_idle()) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
if (!domain_runtime_multimodal_audio_is_initialized()) {
|
||||
esp_err_t init_err = domain_runtime_multimodal_audio_init();
|
||||
if (init_err != ESP_OK) {
|
||||
return init_err;
|
||||
}
|
||||
}
|
||||
|
||||
domain_local_command_snapshot_t command_state = {0};
|
||||
esp_err_t snapshot_err =
|
||||
domain_runtime_local_command_get_snapshot(&command_state);
|
||||
if (snapshot_err != ESP_OK || !command_state.started) {
|
||||
esp_err_t start_err = domain_runtime_local_command_start(
|
||||
local_wake_domain_event,
|
||||
NULL);
|
||||
if (start_err != ESP_OK) {
|
||||
return start_err;
|
||||
}
|
||||
} else if (!command_state.detection_running) {
|
||||
esp_err_t resume_err = domain_runtime_local_command_resume(
|
||||
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
|
||||
if (resume_err != ESP_OK) {
|
||||
return resume_err;
|
||||
}
|
||||
}
|
||||
|
||||
if (!domain_runtime_multimodal_audio_is_uplink_running()) {
|
||||
esp_err_t capture_err =
|
||||
domain_runtime_multimodal_audio_start_uplink(NULL, NULL);
|
||||
if (capture_err != ESP_OK) {
|
||||
return capture_err;
|
||||
}
|
||||
}
|
||||
local_wake_set_capture_owned(true);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static void local_wake_handle_detection(
|
||||
const domain_local_command_event_t *event)
|
||||
{
|
||||
if (event == NULL ||
|
||||
event->type != DOMAIN_LOCAL_COMMAND_EVENT_DETECTED ||
|
||||
event->command_id != DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI) {
|
||||
return;
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
if (s_local_wake.stop_requested || s_local_wake.paused_for_foreground) {
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
return;
|
||||
}
|
||||
s_local_wake.paused_for_foreground = true;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
|
||||
local_wake_stop_owned_capture();
|
||||
(void)domain_runtime_local_command_suspend(
|
||||
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
|
||||
|
||||
ESP_LOGI(TAG,
|
||||
"Local wake accepted: phrase=喵掌柜 confidence_x10000=%u generation=%u",
|
||||
(unsigned)event->confidence_x10000,
|
||||
(unsigned)event->detection_generation);
|
||||
|
||||
#if CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
|
||||
if (CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE[0] != '\0') {
|
||||
esp_err_t prompt_err = domain_runtime_behavior_play_prompt_file(
|
||||
CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE);
|
||||
if (prompt_err != ESP_OK) {
|
||||
ESP_LOGW(TAG,
|
||||
"Local wake response prompt failed: %s",
|
||||
esp_err_to_name(prompt_err));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
control_plane_set_ptt_active(true);
|
||||
}
|
||||
|
||||
static void local_wake_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
TickType_t retry_not_before = 0U;
|
||||
bool listening_logged = false;
|
||||
|
||||
ESP_LOGI(TAG, "Local wake worker started: phrase=喵掌柜");
|
||||
while (!local_wake_snapshot().stop_requested) {
|
||||
local_wake_state_t state = local_wake_snapshot();
|
||||
TickType_t now = xTaskGetTickCount();
|
||||
if (!state.paused_for_foreground &&
|
||||
control_plane_voice_turn_is_idle() &&
|
||||
(retry_not_before == 0U || now >= retry_not_before)) {
|
||||
esp_err_t listen_err = local_wake_start_listening();
|
||||
if (listen_err == ESP_OK) {
|
||||
retry_not_before = 0U;
|
||||
if (!listening_logged) {
|
||||
ESP_LOGI(TAG, "Local wake listening ready: phrase=喵掌柜");
|
||||
listening_logged = true;
|
||||
}
|
||||
} else if (listen_err != ESP_ERR_INVALID_STATE) {
|
||||
ESP_LOGW(TAG,
|
||||
"Local wake listener unavailable: %s; retry in %ums",
|
||||
esp_err_to_name(listen_err),
|
||||
(unsigned)CONFIG_TQ_LOCAL_WAKE_RETRY_MS);
|
||||
retry_not_before =
|
||||
now + pdMS_TO_TICKS(CONFIG_TQ_LOCAL_WAKE_RETRY_MS);
|
||||
listening_logged = false;
|
||||
}
|
||||
}
|
||||
|
||||
domain_local_command_event_t event = {0};
|
||||
if (state.detection_queue != NULL &&
|
||||
xQueueReceive(state.detection_queue,
|
||||
&event,
|
||||
pdMS_TO_TICKS(CONTROL_PLANE_LOCAL_WAKE_POLL_MS)) ==
|
||||
pdTRUE) {
|
||||
local_wake_handle_detection(&event);
|
||||
} else {
|
||||
ulTaskNotifyTake(pdTRUE, 0U);
|
||||
}
|
||||
}
|
||||
|
||||
local_wake_stop_owned_capture();
|
||||
(void)domain_runtime_local_command_stop(
|
||||
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.task = NULL;
|
||||
s_local_wake.started = false;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
ESP_LOGI(TAG, "Local wake worker stopped");
|
||||
vTaskDelete(NULL);
|
||||
}
|
||||
|
||||
esp_err_t control_plane_local_wake_start(void)
|
||||
{
|
||||
#if !CONFIG_TQ_LOCAL_WAKE_ENABLE
|
||||
return ESP_OK;
|
||||
#else
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
if (s_local_wake.task != NULL || s_local_wake.started) {
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
return ESP_OK;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
|
||||
if (s_local_wake.detection_queue == NULL) {
|
||||
QueueHandle_t queue = xQueueCreate(
|
||||
CONTROL_PLANE_LOCAL_WAKE_QUEUE_LEN,
|
||||
sizeof(domain_local_command_event_t));
|
||||
if (queue == NULL) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
if (s_local_wake.detection_queue == NULL) {
|
||||
s_local_wake.detection_queue = queue;
|
||||
queue = NULL;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
if (queue != NULL) {
|
||||
vQueueDelete(queue);
|
||||
}
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.stop_requested = false;
|
||||
s_local_wake.paused_for_foreground = false;
|
||||
s_local_wake.capture_owned = false;
|
||||
s_local_wake.started = true;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
|
||||
TaskHandle_t task = NULL;
|
||||
BaseType_t created = xTaskCreatePinnedToCore(
|
||||
local_wake_task,
|
||||
"cp_local_wake",
|
||||
CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES,
|
||||
NULL,
|
||||
CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY,
|
||||
&task,
|
||||
CONTROL_PLANE_LOCAL_WAKE_TASK_CORE);
|
||||
if (created != pdPASS) {
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.started = false;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.task = task;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
return ESP_OK;
|
||||
#endif
|
||||
}
|
||||
|
||||
esp_err_t control_plane_local_wake_stop(int timeout_ms)
|
||||
{
|
||||
#if !CONFIG_TQ_LOCAL_WAKE_ENABLE
|
||||
(void)timeout_ms;
|
||||
return ESP_OK;
|
||||
#else
|
||||
if (timeout_ms < 0) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
local_wake_state_t state = local_wake_snapshot();
|
||||
if (!state.started && state.task == NULL) {
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.stop_requested = true;
|
||||
TaskHandle_t task = s_local_wake.task;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
if (task != NULL) {
|
||||
xTaskNotifyGive(task);
|
||||
}
|
||||
|
||||
TickType_t started = xTaskGetTickCount();
|
||||
TickType_t timeout_ticks = pdMS_TO_TICKS((uint32_t)timeout_ms);
|
||||
if (timeout_ms > 0 && timeout_ticks == 0U) {
|
||||
timeout_ticks = 1U;
|
||||
}
|
||||
while (local_wake_snapshot().task != NULL &&
|
||||
timeout_ms > 0 &&
|
||||
(xTaskGetTickCount() - started) < timeout_ticks) {
|
||||
vTaskDelay(pdMS_TO_TICKS(20));
|
||||
}
|
||||
state = local_wake_snapshot();
|
||||
if (state.task != NULL) {
|
||||
vTaskDelete(state.task);
|
||||
local_wake_stop_owned_capture();
|
||||
(void)domain_runtime_local_command_stop(
|
||||
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.task = NULL;
|
||||
s_local_wake.started = false;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
return ESP_OK;
|
||||
#endif
|
||||
}
|
||||
|
||||
void control_plane_local_wake_pause_for_foreground(void)
|
||||
{
|
||||
#if CONFIG_TQ_LOCAL_WAKE_ENABLE
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.paused_for_foreground = true;
|
||||
TaskHandle_t task = s_local_wake.task;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
(void)domain_runtime_local_command_suspend(
|
||||
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
|
||||
local_wake_stop_owned_capture();
|
||||
if (task != NULL) {
|
||||
xTaskNotifyGive(task);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
void control_plane_local_wake_resume_after_foreground(void)
|
||||
{
|
||||
#if CONFIG_TQ_LOCAL_WAKE_ENABLE
|
||||
portENTER_CRITICAL(&s_local_wake_lock);
|
||||
s_local_wake.paused_for_foreground = false;
|
||||
TaskHandle_t task = s_local_wake.task;
|
||||
portEXIT_CRITICAL(&s_local_wake_lock);
|
||||
if (task != NULL) {
|
||||
xTaskNotifyGive(task);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -575,6 +575,16 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event
|
||||
ESP_LOGI(TAG, "Pause uplink during TTS responding");
|
||||
control_plane_multimodal_stop_uplink();
|
||||
}
|
||||
#if CONFIG_TQ_PTT_MODE_CLICK_TO_TALK
|
||||
if (!control_plane_voice_turn_is_idle()) {
|
||||
/*
|
||||
* The server response closes this one-shot voice turn. Once
|
||||
* teardown reaches IDLE, the local listener can resume with
|
||||
* AEC-clean PCM and accept 喵掌柜 while TTS is playing.
|
||||
*/
|
||||
control_plane_set_ptt_active(false);
|
||||
}
|
||||
#endif
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
@@ -67,12 +67,14 @@
|
||||
#define CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS 24
|
||||
#endif
|
||||
|
||||
#define CONTROL_PLANE_MULTIMODAL_OPUS_CHANNELS 1
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE != 16000
|
||||
#error "Multimodal voice requires 16 kHz audio"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1
|
||||
#error "Multimodal voice requires mono audio"
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2
|
||||
#error "Multimodal AEC requires stereo codec capture"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60
|
||||
@@ -1943,7 +1945,7 @@ esp_err_t control_plane_multimodal_send_start(void)
|
||||
(unsigned)sample_rate,
|
||||
(unsigned)opus_frame_duration_ms,
|
||||
(unsigned)CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS,
|
||||
CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS,
|
||||
CONTROL_PLANE_MULTIMODAL_OPUS_CHANNELS,
|
||||
CONFIG_MULTIMODAL_AUDIO_PA_GPIO);
|
||||
|
||||
return ESP_OK;
|
||||
|
||||
@@ -88,6 +88,10 @@
|
||||
#define CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS 8000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS
|
||||
#define CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS 1000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_NETWORK_TASK_CORE
|
||||
#define CONFIG_TQ_NETWORK_TASK_CORE 0
|
||||
#endif
|
||||
@@ -303,6 +307,9 @@ static void control_plane_notify_wifi_state_task(void)
|
||||
|
||||
void control_plane_set_ptt_active(bool active)
|
||||
{
|
||||
if (active) {
|
||||
control_plane_local_wake_pause_for_foreground();
|
||||
}
|
||||
bool notify_worker = false;
|
||||
ptt_state_t old_state = PTT_STATE_IDLE;
|
||||
ptt_state_t new_state = PTT_STATE_IDLE;
|
||||
@@ -356,6 +363,20 @@ void control_plane_set_ptt_active(bool active)
|
||||
if (notify_worker) {
|
||||
control_plane_notify_wifi_state_task();
|
||||
}
|
||||
if (!active) {
|
||||
control_plane_local_wake_resume_after_foreground();
|
||||
}
|
||||
}
|
||||
|
||||
bool control_plane_voice_turn_is_idle(void)
|
||||
{
|
||||
bool idle = false;
|
||||
portENTER_CRITICAL(&s_runtime_lock);
|
||||
idle = s_ptt_runtime.state == PTT_STATE_IDLE &&
|
||||
!s_ptt_runtime.active &&
|
||||
!s_ptt_runtime.pending_press_after_teardown;
|
||||
portEXIT_CRITICAL(&s_runtime_lock);
|
||||
return idle;
|
||||
}
|
||||
|
||||
static bool control_plane_get_ptt_active(void)
|
||||
@@ -1880,6 +1901,15 @@ esp_err_t control_plane_start_runtime_workers(void)
|
||||
return key_input_err;
|
||||
}
|
||||
|
||||
esp_err_t local_wake_err = control_plane_local_wake_start();
|
||||
if (local_wake_err != ESP_OK) {
|
||||
ESP_LOGE(TAG,
|
||||
"Start local wake worker failed: %s",
|
||||
esp_err_to_name(local_wake_err));
|
||||
control_plane_stop_runtime_workers();
|
||||
return local_wake_err;
|
||||
}
|
||||
|
||||
esp_err_t remote_log_err = control_plane_remote_log_start();
|
||||
if (remote_log_err != ESP_OK && remote_log_err != ESP_ERR_NOT_SUPPORTED) {
|
||||
ESP_LOGW(TAG, "Remote log start failed: %s", esp_err_to_name(remote_log_err));
|
||||
@@ -1946,6 +1976,7 @@ void control_plane_stop_runtime_workers(void)
|
||||
{
|
||||
control_plane_set_runtime_stop_requested(true);
|
||||
control_plane_stop_user_config_server();
|
||||
(void)control_plane_local_wake_stop(CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS);
|
||||
|
||||
#if CONFIG_TQ_FACTORY_CLI_ENABLE
|
||||
control_plane_factory_cli_stop();
|
||||
|
||||
@@ -8,12 +8,14 @@ idf_component_register(
|
||||
"src/fortune/domain_fortune_store.c"
|
||||
"src/fortune/domain_user_config_server.c"
|
||||
"src/runtime/domain_runtime_adapters.c"
|
||||
"src/runtime/domain_local_command.c"
|
||||
INCLUDE_DIRS
|
||||
"include"
|
||||
PRIV_INCLUDE_DIRS
|
||||
"internal"
|
||||
PRIV_REQUIRES
|
||||
platform
|
||||
esp-sr
|
||||
log
|
||||
json
|
||||
EMBED_TXTFILES
|
||||
|
||||
@@ -66,6 +66,34 @@ typedef esp_err_t (*domain_multimodal_audio_uplink_cb_t)(
|
||||
size_t len,
|
||||
void *user_data);
|
||||
|
||||
typedef enum {
|
||||
DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI = 1,
|
||||
} domain_local_command_id_t;
|
||||
|
||||
typedef enum {
|
||||
DOMAIN_LOCAL_COMMAND_EVENT_DETECTED = 0,
|
||||
} domain_local_command_event_type_t;
|
||||
|
||||
typedef struct {
|
||||
domain_local_command_event_type_t type;
|
||||
domain_local_command_id_t command_id;
|
||||
uint32_t confidence_x10000;
|
||||
uint32_t detection_generation;
|
||||
} domain_local_command_event_t;
|
||||
|
||||
typedef void (*domain_local_command_event_callback_t)(
|
||||
const domain_local_command_event_t *event,
|
||||
void *user_data);
|
||||
|
||||
typedef struct {
|
||||
bool started;
|
||||
bool detection_running;
|
||||
uint32_t detection_generation;
|
||||
uint32_t detection_count;
|
||||
uint32_t dropped_pcm_frames;
|
||||
uint32_t task_stack_high_water_bytes;
|
||||
} domain_local_command_snapshot_t;
|
||||
|
||||
typedef enum {
|
||||
DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_STARTED = 0,
|
||||
DOMAIN_MULTIMODAL_DOWNSTREAM_PLAYBACK_COMPLETED,
|
||||
@@ -180,6 +208,20 @@ esp_err_t domain_runtime_multimodal_audio_start_uplink(
|
||||
esp_err_t domain_runtime_multimodal_audio_stop_uplink(int timeout_ms);
|
||||
bool domain_runtime_multimodal_audio_is_uplink_running(void);
|
||||
bool domain_runtime_multimodal_audio_is_initialized(void);
|
||||
/**
|
||||
* @brief Start local MultiNet detection for the Chinese phrase "喵掌柜".
|
||||
*
|
||||
* The callback runs in the domain inference task and must be bounded and
|
||||
* non-blocking. Repeated start with the same callback is idempotent.
|
||||
*/
|
||||
esp_err_t domain_runtime_local_command_start(
|
||||
domain_local_command_event_callback_t callback,
|
||||
void *user_data);
|
||||
esp_err_t domain_runtime_local_command_suspend(int timeout_ms);
|
||||
esp_err_t domain_runtime_local_command_resume(int timeout_ms);
|
||||
esp_err_t domain_runtime_local_command_stop(int timeout_ms);
|
||||
esp_err_t domain_runtime_local_command_get_snapshot(
|
||||
domain_local_command_snapshot_t *out_snapshot);
|
||||
esp_err_t domain_runtime_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
|
||||
esp_err_t domain_runtime_multimodal_audio_begin_downstream_response(void);
|
||||
esp_err_t domain_runtime_multimodal_audio_finish_downstream_response(void);
|
||||
|
||||
@@ -0,0 +1,976 @@
|
||||
#include "domain.h"
|
||||
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "esp_heap_caps.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_mn_iface.h"
|
||||
#include "esp_mn_models.h"
|
||||
#include "esp_mn_speech_commands.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/event_groups.h"
|
||||
#include "freertos/idf_additions.h"
|
||||
#include "freertos/queue.h"
|
||||
#include "freertos/semphr.h"
|
||||
#include "freertos/task.h"
|
||||
#include "model_path.h"
|
||||
#include "platform.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES
|
||||
#define CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES 8192
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY
|
||||
#define CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY 7
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_COMMAND_TASK_CORE
|
||||
#define CONFIG_TQ_LOCAL_COMMAND_TASK_CORE 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS
|
||||
#define CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS 1000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS
|
||||
#define CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS 6000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000
|
||||
#define CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000 1200
|
||||
#endif
|
||||
|
||||
#define DOMAIN_LOCAL_COMMAND_MODEL_PARTITION "model"
|
||||
#define DOMAIN_LOCAL_COMMAND_MODEL_NAME "mn7_cn"
|
||||
#define DOMAIN_LOCAL_COMMAND_TEXT "喵掌柜"
|
||||
#define DOMAIN_LOCAL_COMMAND_GRAPH_TEXT "miao zhang gui"
|
||||
#define DOMAIN_LOCAL_COMMAND_PCM_QUEUE_LEN 8U
|
||||
#define DOMAIN_LOCAL_COMMAND_PCM_MAX_SAMPLES 512U
|
||||
#define DOMAIN_LOCAL_COMMAND_QUEUE_WAIT_MS 100U
|
||||
#define DOMAIN_LOCAL_COMMAND_LOCK_TIMEOUT_MS 100U
|
||||
#define DOMAIN_LOCAL_COMMAND_STACK_MIN_BYTES 2048U
|
||||
#define DOMAIN_LOCAL_COMMAND_STATS_INTERVAL_MS 2000U
|
||||
|
||||
#define DOMAIN_LOCAL_COMMAND_STOP_BIT BIT0
|
||||
#define DOMAIN_LOCAL_COMMAND_STOPPED_BIT BIT1
|
||||
#define DOMAIN_LOCAL_COMMAND_RUNNING_BIT BIT2
|
||||
#define DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT BIT3
|
||||
|
||||
#if CONFIG_TQ_LOCAL_COMMAND_TASK_CORE < 0
|
||||
#define DOMAIN_LOCAL_COMMAND_TASK_CORE tskNO_AFFINITY
|
||||
#else
|
||||
#define DOMAIN_LOCAL_COMMAND_TASK_CORE CONFIG_TQ_LOCAL_COMMAND_TASK_CORE
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
uint16_t sample_count;
|
||||
int16_t samples[DOMAIN_LOCAL_COMMAND_PCM_MAX_SAMPLES];
|
||||
} domain_local_command_pcm_frame_t;
|
||||
|
||||
typedef struct {
|
||||
uint32_t received_frames;
|
||||
uint32_t processed_chunks;
|
||||
uint32_t dropped_frames;
|
||||
uint32_t timeout_count;
|
||||
uint32_t queue_high_water;
|
||||
uint32_t max_inference_us;
|
||||
} domain_local_command_pipeline_stats_t;
|
||||
|
||||
typedef struct {
|
||||
SemaphoreHandle_t lock;
|
||||
EventGroupHandle_t events;
|
||||
QueueHandle_t pcm_queue;
|
||||
TaskHandle_t task;
|
||||
srmodel_list_t *models;
|
||||
const esp_mn_iface_t *multinet_iface;
|
||||
model_iface_data_t *multinet_data;
|
||||
bool commands_allocated;
|
||||
int multinet_chunk_samples;
|
||||
int16_t *inference_pcm;
|
||||
domain_local_command_event_callback_t callback;
|
||||
void *callback_user_data;
|
||||
bool started;
|
||||
bool stop_requested;
|
||||
bool suspended;
|
||||
bool detection_running;
|
||||
bool reset_pending;
|
||||
uint32_t control_generation;
|
||||
uint32_t applied_generation;
|
||||
uint32_t detection_generation;
|
||||
uint32_t detection_count;
|
||||
uint32_t task_stack_high_water_bytes;
|
||||
} domain_local_command_state_t;
|
||||
|
||||
static const char *TAG = "domain_local_cmd";
|
||||
static domain_local_command_state_t s_local_command;
|
||||
static StaticTask_t s_local_command_task_buffer;
|
||||
static StackType_t *s_local_command_task_stack;
|
||||
static portMUX_TYPE s_pcm_ingress_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||
static bool s_pcm_ingress_enabled;
|
||||
static domain_local_command_pipeline_stats_t s_pipeline_stats;
|
||||
|
||||
static bool local_command_lock(void)
|
||||
{
|
||||
if (s_local_command.lock == NULL) {
|
||||
return false;
|
||||
}
|
||||
TickType_t ticks = pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_LOCK_TIMEOUT_MS);
|
||||
return xSemaphoreTake(s_local_command.lock, ticks > 0U ? ticks : 1U) == pdTRUE;
|
||||
}
|
||||
static void local_command_unlock(void)
|
||||
{
|
||||
xSemaphoreGive(s_local_command.lock);
|
||||
}
|
||||
|
||||
static uint32_t local_command_default_timeout(int timeout_ms)
|
||||
{
|
||||
return timeout_ms == 0 ? CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS : (uint32_t)timeout_ms;
|
||||
}
|
||||
|
||||
static uint32_t local_command_remaining_ms(TickType_t started, uint32_t timeout_ms)
|
||||
{
|
||||
const uint32_t elapsed_ms = (uint32_t)((xTaskGetTickCount() - started) * portTICK_PERIOD_MS);
|
||||
return elapsed_ms >= timeout_ms ? 0U : timeout_ms - elapsed_ms;
|
||||
}
|
||||
|
||||
static void local_command_set_pcm_admission(bool enabled)
|
||||
{
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
s_pcm_ingress_enabled = enabled;
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
}
|
||||
|
||||
static uint32_t local_command_dropped_pcm_frames(void)
|
||||
{
|
||||
uint32_t dropped = 0U;
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
dropped = s_pipeline_stats.dropped_frames;
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
return dropped;
|
||||
}
|
||||
|
||||
static void local_command_note_dropped_pcm(void)
|
||||
{
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
++s_pipeline_stats.dropped_frames;
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
}
|
||||
|
||||
static void local_command_note_queued_pcm(uint32_t queue_depth)
|
||||
{
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
++s_pipeline_stats.received_frames;
|
||||
if (queue_depth > s_pipeline_stats.queue_high_water) {
|
||||
s_pipeline_stats.queue_high_water = queue_depth;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
}
|
||||
|
||||
static void local_command_note_processed_chunk(uint32_t inference_us, bool timed_out)
|
||||
{
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
++s_pipeline_stats.processed_chunks;
|
||||
if (timed_out) {
|
||||
++s_pipeline_stats.timeout_count;
|
||||
}
|
||||
if (inference_us > s_pipeline_stats.max_inference_us) {
|
||||
s_pipeline_stats.max_inference_us = inference_us;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
}
|
||||
|
||||
static domain_local_command_pipeline_stats_t local_command_pipeline_stats(void)
|
||||
{
|
||||
domain_local_command_pipeline_stats_t stats;
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
stats = s_pipeline_stats;
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
return stats;
|
||||
}
|
||||
|
||||
static void local_command_reset_pipeline_stats(void)
|
||||
{
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
memset(&s_pipeline_stats, 0, sizeof(s_pipeline_stats));
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
}
|
||||
|
||||
static void local_command_pcm_observer(const int16_t *pcm,
|
||||
size_t samples,
|
||||
uint32_t sample_rate_hz,
|
||||
void *user_data)
|
||||
{
|
||||
(void)user_data;
|
||||
bool admitted = false;
|
||||
portENTER_CRITICAL(&s_pcm_ingress_lock);
|
||||
admitted = s_pcm_ingress_enabled;
|
||||
portEXIT_CRITICAL(&s_pcm_ingress_lock);
|
||||
|
||||
if (!admitted || pcm == NULL || samples == 0U) {
|
||||
return;
|
||||
}
|
||||
if (sample_rate_hz != 16000U || samples > DOMAIN_LOCAL_COMMAND_PCM_MAX_SAMPLES ||
|
||||
s_local_command.pcm_queue == NULL) {
|
||||
local_command_note_dropped_pcm();
|
||||
return;
|
||||
}
|
||||
|
||||
domain_local_command_pcm_frame_t frame = {
|
||||
.sample_count = (uint16_t)samples,
|
||||
};
|
||||
memcpy(frame.samples, pcm, samples * sizeof(int16_t));
|
||||
if (xQueueSend(s_local_command.pcm_queue, &frame, 0U) != pdTRUE) {
|
||||
local_command_note_dropped_pcm();
|
||||
} else {
|
||||
local_command_note_queued_pcm(
|
||||
(uint32_t)uxQueueMessagesWaiting(s_local_command.pcm_queue));
|
||||
}
|
||||
}
|
||||
|
||||
static void local_command_drain_pcm_queue(void)
|
||||
{
|
||||
domain_local_command_pcm_frame_t frame;
|
||||
if (s_local_command.pcm_queue == NULL) {
|
||||
return;
|
||||
}
|
||||
while (xQueueReceive(s_local_command.pcm_queue, &frame, 0U) == pdTRUE) {
|
||||
}
|
||||
}
|
||||
|
||||
static esp_err_t local_command_configure_model(void)
|
||||
{
|
||||
s_local_command.models = esp_srmodel_init(DOMAIN_LOCAL_COMMAND_MODEL_PARTITION);
|
||||
if (s_local_command.models == NULL || s_local_command.models->num <= 0) {
|
||||
ESP_LOGE(TAG,
|
||||
"ESP-SR model partition unavailable: partition=%s action=full_flash_required",
|
||||
DOMAIN_LOCAL_COMMAND_MODEL_PARTITION);
|
||||
return ESP_ERR_NOT_FOUND;
|
||||
}
|
||||
if (esp_srmodel_exists(s_local_command.models,
|
||||
(char *)DOMAIN_LOCAL_COMMAND_MODEL_NAME) < 0) {
|
||||
ESP_LOGE(TAG, "Chinese MultiNet model unavailable: model=%s",
|
||||
DOMAIN_LOCAL_COMMAND_MODEL_NAME);
|
||||
return ESP_ERR_NOT_FOUND;
|
||||
}
|
||||
|
||||
s_local_command.multinet_iface = esp_mn_handle_from_name(
|
||||
(char *)DOMAIN_LOCAL_COMMAND_MODEL_NAME);
|
||||
const esp_mn_iface_t *iface = s_local_command.multinet_iface;
|
||||
if (iface == NULL || iface->create == NULL || iface->destroy == NULL ||
|
||||
iface->get_samp_rate == NULL || iface->get_samp_chunksize == NULL ||
|
||||
iface->detect == NULL || iface->get_results == NULL || iface->clean == NULL ||
|
||||
iface->check_speech_command == NULL || iface->set_det_threshold == NULL) {
|
||||
return ESP_ERR_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
s_local_command.multinet_data = iface->create(
|
||||
DOMAIN_LOCAL_COMMAND_MODEL_NAME,
|
||||
CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS);
|
||||
if (s_local_command.multinet_data == NULL) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
if (iface->get_samp_rate(s_local_command.multinet_data) != 16000) {
|
||||
return ESP_ERR_NOT_SUPPORTED;
|
||||
}
|
||||
s_local_command.multinet_chunk_samples =
|
||||
iface->get_samp_chunksize(s_local_command.multinet_data);
|
||||
if (s_local_command.multinet_chunk_samples <= 0 ||
|
||||
s_local_command.multinet_chunk_samples > INT16_MAX) {
|
||||
return ESP_ERR_INVALID_SIZE;
|
||||
}
|
||||
if (iface->check_speech_command(s_local_command.multinet_data,
|
||||
DOMAIN_LOCAL_COMMAND_GRAPH_TEXT) == 0) {
|
||||
ESP_LOGE(TAG, "MultiNet command pronunciation is not tokenizable: %s",
|
||||
DOMAIN_LOCAL_COMMAND_GRAPH_TEXT);
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
esp_err_t err = esp_mn_commands_alloc(iface, s_local_command.multinet_data);
|
||||
if (err != ESP_OK) {
|
||||
return err;
|
||||
}
|
||||
s_local_command.commands_allocated = true;
|
||||
err = esp_mn_commands_clear();
|
||||
if (err == ESP_OK) {
|
||||
err = esp_mn_commands_phoneme_add(
|
||||
DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI,
|
||||
DOMAIN_LOCAL_COMMAND_GRAPH_TEXT,
|
||||
DOMAIN_LOCAL_COMMAND_GRAPH_TEXT);
|
||||
}
|
||||
if (err != ESP_OK) {
|
||||
return err;
|
||||
}
|
||||
esp_mn_error_t *command_errors = esp_mn_commands_update();
|
||||
if (command_errors != NULL) {
|
||||
ESP_LOGE(TAG, "MultiNet rejected local command graph: invalid_count=%d",
|
||||
command_errors->num);
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
const float detection_threshold =
|
||||
(float)CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000 / 10000.0f;
|
||||
(void)iface->set_det_threshold(s_local_command.multinet_data,
|
||||
detection_threshold);
|
||||
const esp_mn_phrase_t *command = esp_mn_commands_get_from_index(0);
|
||||
if (command == NULL || command->command_id != DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI ||
|
||||
command->string == NULL || command->phonemes == NULL ||
|
||||
strcmp(command->string, DOMAIN_LOCAL_COMMAND_GRAPH_TEXT) != 0 ||
|
||||
strcmp(command->phonemes, DOMAIN_LOCAL_COMMAND_GRAPH_TEXT) != 0 ||
|
||||
esp_mn_commands_get_from_index(1) != NULL) {
|
||||
return ESP_ERR_INVALID_RESPONSE;
|
||||
}
|
||||
|
||||
s_local_command.inference_pcm = calloc(
|
||||
(size_t)s_local_command.multinet_chunk_samples,
|
||||
sizeof(int16_t));
|
||||
if (s_local_command.inference_pcm == NULL) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
ESP_LOGI(TAG,
|
||||
"Local command model ready: model=%s phrase=%s command=%s samples=%d "
|
||||
"rate=16000 threshold_x10000=%u",
|
||||
DOMAIN_LOCAL_COMMAND_MODEL_NAME,
|
||||
DOMAIN_LOCAL_COMMAND_TEXT,
|
||||
DOMAIN_LOCAL_COMMAND_GRAPH_TEXT,
|
||||
s_local_command.multinet_chunk_samples,
|
||||
(unsigned)CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000);
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static void local_command_release_model(void)
|
||||
{
|
||||
if (s_local_command.commands_allocated) {
|
||||
(void)esp_mn_commands_free();
|
||||
s_local_command.commands_allocated = false;
|
||||
}
|
||||
if (s_local_command.multinet_iface != NULL &&
|
||||
s_local_command.multinet_data != NULL) {
|
||||
s_local_command.multinet_iface->destroy(s_local_command.multinet_data);
|
||||
}
|
||||
s_local_command.multinet_data = NULL;
|
||||
s_local_command.multinet_iface = NULL;
|
||||
if (s_local_command.models != NULL) {
|
||||
esp_srmodel_deinit(s_local_command.models);
|
||||
}
|
||||
s_local_command.models = NULL;
|
||||
free(s_local_command.inference_pcm);
|
||||
s_local_command.inference_pcm = NULL;
|
||||
s_local_command.multinet_chunk_samples = 0;
|
||||
}
|
||||
|
||||
static bool local_command_apply_control(size_t *fill_samples)
|
||||
{
|
||||
bool stop_requested = true;
|
||||
bool suspended = true;
|
||||
bool changed = false;
|
||||
bool reset_pending = false;
|
||||
bool applied = false;
|
||||
uint32_t generation = 0U;
|
||||
|
||||
if (local_command_lock()) {
|
||||
stop_requested = s_local_command.stop_requested;
|
||||
suspended = s_local_command.suspended;
|
||||
generation = s_local_command.control_generation;
|
||||
changed = s_local_command.applied_generation != generation;
|
||||
reset_pending = s_local_command.reset_pending;
|
||||
local_command_unlock();
|
||||
}
|
||||
if (stop_requested) {
|
||||
return false;
|
||||
}
|
||||
if (!changed) {
|
||||
return !suspended;
|
||||
}
|
||||
|
||||
if (reset_pending) {
|
||||
s_local_command.multinet_iface->clean(s_local_command.multinet_data);
|
||||
}
|
||||
local_command_drain_pcm_queue();
|
||||
*fill_samples = 0U;
|
||||
|
||||
if (!local_command_lock()) {
|
||||
return false;
|
||||
}
|
||||
if (s_local_command.stop_requested) {
|
||||
local_command_unlock();
|
||||
return false;
|
||||
}
|
||||
if (generation == s_local_command.control_generation) {
|
||||
applied = true;
|
||||
s_local_command.applied_generation = generation;
|
||||
s_local_command.detection_running = !s_local_command.suspended;
|
||||
s_local_command.reset_pending = false;
|
||||
suspended = s_local_command.suspended;
|
||||
xEventGroupClearBits(s_local_command.events,
|
||||
DOMAIN_LOCAL_COMMAND_RUNNING_BIT |
|
||||
DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT);
|
||||
xEventGroupSetBits(s_local_command.events,
|
||||
suspended ? DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT :
|
||||
DOMAIN_LOCAL_COMMAND_RUNNING_BIT);
|
||||
}
|
||||
local_command_unlock();
|
||||
if (applied) {
|
||||
ESP_LOGI(TAG,
|
||||
"Local command detection %s: control_generation=%u queue_depth=%u dropped_total=%u",
|
||||
suspended ? "suspended" : "running",
|
||||
(unsigned)generation,
|
||||
s_local_command.pcm_queue != NULL ?
|
||||
(unsigned)uxQueueMessagesWaiting(s_local_command.pcm_queue) : 0U,
|
||||
(unsigned)local_command_dropped_pcm_frames());
|
||||
}
|
||||
return !suspended;
|
||||
}
|
||||
|
||||
static void local_command_publish_detection(float probability)
|
||||
{
|
||||
domain_local_command_event_callback_t callback = NULL;
|
||||
void *callback_user_data = NULL;
|
||||
domain_local_command_event_t event = {
|
||||
.type = DOMAIN_LOCAL_COMMAND_EVENT_DETECTED,
|
||||
.command_id = DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI,
|
||||
};
|
||||
if (probability > 0.0f) {
|
||||
event.confidence_x10000 = probability >= 0.9999f ?
|
||||
9999U : (uint32_t)(probability * 10000.0f + 0.5f);
|
||||
}
|
||||
|
||||
if (!local_command_lock()) {
|
||||
return;
|
||||
}
|
||||
if (!s_local_command.started || s_local_command.stop_requested ||
|
||||
s_local_command.suspended) {
|
||||
local_command_unlock();
|
||||
return;
|
||||
}
|
||||
s_local_command.suspended = true;
|
||||
s_local_command.detection_running = false;
|
||||
s_local_command.reset_pending = false;
|
||||
local_command_set_pcm_admission(false);
|
||||
++s_local_command.control_generation;
|
||||
s_local_command.applied_generation = s_local_command.control_generation;
|
||||
event.detection_generation = ++s_local_command.detection_generation;
|
||||
++s_local_command.detection_count;
|
||||
callback = s_local_command.callback;
|
||||
callback_user_data = s_local_command.callback_user_data;
|
||||
xEventGroupClearBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_RUNNING_BIT);
|
||||
xEventGroupSetBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT);
|
||||
local_command_unlock();
|
||||
|
||||
const domain_local_command_pipeline_stats_t stats = local_command_pipeline_stats();
|
||||
ESP_LOGI(TAG,
|
||||
"Local command detected: phrase=%s confidence_x10000=%u generation=%u "
|
||||
"received=%u processed=%u drops=%u queue_high=%u inference_max_us=%u",
|
||||
DOMAIN_LOCAL_COMMAND_TEXT,
|
||||
(unsigned)event.confidence_x10000,
|
||||
(unsigned)event.detection_generation,
|
||||
(unsigned)stats.received_frames,
|
||||
(unsigned)stats.processed_chunks,
|
||||
(unsigned)stats.dropped_frames,
|
||||
(unsigned)stats.queue_high_water,
|
||||
(unsigned)stats.max_inference_us);
|
||||
if (callback != NULL) {
|
||||
callback(&event, callback_user_data);
|
||||
}
|
||||
}
|
||||
|
||||
static void local_command_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
size_t fill_samples = 0U;
|
||||
uint32_t frame_count = 0U;
|
||||
bool stack_reported = false;
|
||||
bool stack_warned = false;
|
||||
TickType_t stats_last_log_tick = xTaskGetTickCount();
|
||||
uint32_t previous_received_frames = 0U;
|
||||
uint32_t previous_processed_chunks = 0U;
|
||||
uint32_t previous_dropped_frames = 0U;
|
||||
uint32_t previous_timeout_count = 0U;
|
||||
|
||||
for (;;) {
|
||||
const bool detection_running = local_command_apply_control(&fill_samples);
|
||||
bool stop_requested = true;
|
||||
if (local_command_lock()) {
|
||||
stop_requested = s_local_command.stop_requested;
|
||||
local_command_unlock();
|
||||
}
|
||||
if (stop_requested) {
|
||||
break;
|
||||
}
|
||||
if (!detection_running) {
|
||||
(void)ulTaskNotifyTake(pdTRUE,
|
||||
pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_QUEUE_WAIT_MS));
|
||||
continue;
|
||||
}
|
||||
|
||||
domain_local_command_pcm_frame_t frame;
|
||||
if (xQueueReceive(s_local_command.pcm_queue,
|
||||
&frame,
|
||||
pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_QUEUE_WAIT_MS)) != pdTRUE) {
|
||||
continue;
|
||||
}
|
||||
if (!local_command_apply_control(&fill_samples)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
size_t offset = 0U;
|
||||
while (offset < frame.sample_count) {
|
||||
const size_t needed = (size_t)s_local_command.multinet_chunk_samples - fill_samples;
|
||||
const size_t available = (size_t)frame.sample_count - offset;
|
||||
const size_t copy_samples = available < needed ? available : needed;
|
||||
memcpy(s_local_command.inference_pcm + fill_samples,
|
||||
frame.samples + offset,
|
||||
copy_samples * sizeof(int16_t));
|
||||
fill_samples += copy_samples;
|
||||
offset += copy_samples;
|
||||
if (fill_samples != (size_t)s_local_command.multinet_chunk_samples) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const TickType_t inference_started_tick = xTaskGetTickCount();
|
||||
const esp_mn_state_t state = s_local_command.multinet_iface->detect(
|
||||
s_local_command.multinet_data,
|
||||
s_local_command.inference_pcm);
|
||||
const uint64_t inference_elapsed_us =
|
||||
(uint64_t)(xTaskGetTickCount() - inference_started_tick) *
|
||||
(uint64_t)portTICK_PERIOD_MS * 1000U;
|
||||
const uint32_t inference_us = inference_elapsed_us > UINT32_MAX ?
|
||||
UINT32_MAX : (uint32_t)inference_elapsed_us;
|
||||
fill_samples = 0U;
|
||||
++frame_count;
|
||||
local_command_note_processed_chunk(
|
||||
inference_us, state == ESP_MN_STATE_TIMEOUT);
|
||||
/* MultiNet can consume a full core while the PCM queue is nonempty.
|
||||
* Give the pinned core's idle task a bounded scheduling point so
|
||||
* continuous playback-time inference cannot trip the task WDT. */
|
||||
vTaskDelay(1);
|
||||
if (state == ESP_MN_STATE_TIMEOUT) {
|
||||
s_local_command.multinet_iface->clean(s_local_command.multinet_data);
|
||||
} else if (state == ESP_MN_STATE_DETECTED) {
|
||||
esp_mn_results_t *results = s_local_command.multinet_iface->get_results(
|
||||
s_local_command.multinet_data);
|
||||
const bool accepted = results != NULL && results->num > 0 &&
|
||||
results->command_id[0] == DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI;
|
||||
const float probability = accepted ? results->prob[0] : 0.0f;
|
||||
s_local_command.multinet_iface->clean(s_local_command.multinet_data);
|
||||
if (accepted) {
|
||||
local_command_publish_detection(probability);
|
||||
fill_samples = 0U;
|
||||
break;
|
||||
}
|
||||
ESP_LOGW(TAG, "Ignore unexpected MultiNet result");
|
||||
} else if (state != ESP_MN_STATE_DETECTING) {
|
||||
ESP_LOGE(TAG, "Invalid MultiNet state: %d", (int)state);
|
||||
if (local_command_lock()) {
|
||||
s_local_command.stop_requested = true;
|
||||
local_command_unlock();
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
const TickType_t now = xTaskGetTickCount();
|
||||
if ((now - stats_last_log_tick) >=
|
||||
pdMS_TO_TICKS(DOMAIN_LOCAL_COMMAND_STATS_INTERVAL_MS)) {
|
||||
const domain_local_command_pipeline_stats_t stats =
|
||||
local_command_pipeline_stats();
|
||||
const uint32_t queue_depth = s_local_command.pcm_queue != NULL ?
|
||||
(uint32_t)uxQueueMessagesWaiting(s_local_command.pcm_queue) : 0U;
|
||||
ESP_LOGI(TAG,
|
||||
"Local command pipeline: received=%u processed=%u drops=%u timeouts=%u "
|
||||
"queue=%u high=%u inference_max_us=%u",
|
||||
(unsigned)(stats.received_frames - previous_received_frames),
|
||||
(unsigned)(stats.processed_chunks - previous_processed_chunks),
|
||||
(unsigned)(stats.dropped_frames - previous_dropped_frames),
|
||||
(unsigned)(stats.timeout_count - previous_timeout_count),
|
||||
(unsigned)queue_depth,
|
||||
(unsigned)stats.queue_high_water,
|
||||
(unsigned)stats.max_inference_us);
|
||||
previous_received_frames = stats.received_frames;
|
||||
previous_processed_chunks = stats.processed_chunks;
|
||||
previous_dropped_frames = stats.dropped_frames;
|
||||
previous_timeout_count = stats.timeout_count;
|
||||
stats_last_log_tick = now;
|
||||
}
|
||||
|
||||
if ((frame_count % 100U) == 0U) {
|
||||
const uint32_t high_water = (uint32_t)uxTaskGetStackHighWaterMark(NULL);
|
||||
if (local_command_lock()) {
|
||||
s_local_command.task_stack_high_water_bytes = high_water;
|
||||
local_command_unlock();
|
||||
}
|
||||
const uint32_t required =
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES / 4U >
|
||||
DOMAIN_LOCAL_COMMAND_STACK_MIN_BYTES ?
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES / 4U :
|
||||
DOMAIN_LOCAL_COMMAND_STACK_MIN_BYTES;
|
||||
if (!stack_reported) {
|
||||
stack_reported = true;
|
||||
ESP_LOGI(TAG,
|
||||
"Task stack headroom=%u required=%u configured=%u bytes",
|
||||
(unsigned)high_water,
|
||||
(unsigned)required,
|
||||
(unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES);
|
||||
}
|
||||
if (!stack_warned && high_water < required) {
|
||||
stack_warned = true;
|
||||
ESP_LOGW(TAG,
|
||||
"Local command task stack margin low: high_water=%u required=%u configured=%u",
|
||||
(unsigned)high_water,
|
||||
(unsigned)required,
|
||||
(unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (local_command_lock()) {
|
||||
s_local_command.task_stack_high_water_bytes =
|
||||
(uint32_t)uxTaskGetStackHighWaterMark(NULL);
|
||||
s_local_command.detection_running = false;
|
||||
s_local_command.reset_pending = false;
|
||||
local_command_unlock();
|
||||
}
|
||||
xEventGroupClearBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_RUNNING_BIT);
|
||||
xEventGroupSetBits(s_local_command.events,
|
||||
DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT |
|
||||
DOMAIN_LOCAL_COMMAND_STOPPED_BIT);
|
||||
vTaskSuspend(NULL);
|
||||
abort();
|
||||
}
|
||||
|
||||
static esp_err_t local_command_reserve_task_stack(void)
|
||||
{
|
||||
if (s_local_command_task_stack != NULL) {
|
||||
return ESP_OK;
|
||||
}
|
||||
#if (configSUPPORT_STATIC_ALLOCATION == 1) && \
|
||||
((defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) || \
|
||||
(defined(CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY) && CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY))
|
||||
s_local_command_task_stack = (StackType_t *)heap_caps_malloc(
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES,
|
||||
MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
||||
if (s_local_command_task_stack != NULL) {
|
||||
ESP_LOGI(TAG,
|
||||
"Task stack reserved in PSRAM (%uB)",
|
||||
(unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES);
|
||||
return ESP_OK;
|
||||
}
|
||||
#endif
|
||||
ESP_LOGE(TAG,
|
||||
"PSRAM task stack reservation failed bytes=%u internal_free=%u "
|
||||
"internal_largest=%u psram_free=%u psram_largest=%u",
|
||||
(unsigned)CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES,
|
||||
(unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),
|
||||
(unsigned)heap_caps_get_largest_free_block(
|
||||
MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),
|
||||
(unsigned)heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT),
|
||||
(unsigned)heap_caps_get_largest_free_block(
|
||||
MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
|
||||
static void local_command_cleanup_after_stop(void)
|
||||
{
|
||||
local_command_set_pcm_admission(false);
|
||||
local_command_release_model();
|
||||
if (local_command_lock()) {
|
||||
s_local_command.callback = NULL;
|
||||
s_local_command.callback_user_data = NULL;
|
||||
s_local_command.started = false;
|
||||
s_local_command.stop_requested = false;
|
||||
s_local_command.suspended = true;
|
||||
s_local_command.detection_running = false;
|
||||
s_local_command.reset_pending = false;
|
||||
s_local_command.control_generation = 0U;
|
||||
s_local_command.applied_generation = 0U;
|
||||
local_command_unlock();
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t domain_runtime_local_command_start(
|
||||
domain_local_command_event_callback_t callback,
|
||||
void *user_data)
|
||||
{
|
||||
if (callback == NULL) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
if (s_local_command.lock == NULL) {
|
||||
s_local_command.lock = xSemaphoreCreateMutex();
|
||||
}
|
||||
if (s_local_command.events == NULL) {
|
||||
s_local_command.events = xEventGroupCreate();
|
||||
}
|
||||
if (s_local_command.lock == NULL || s_local_command.events == NULL) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
|
||||
if (!local_command_lock()) {
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
if (s_local_command.started) {
|
||||
const bool same = s_local_command.callback == callback &&
|
||||
s_local_command.callback_user_data == user_data;
|
||||
local_command_unlock();
|
||||
return same ? ESP_OK : ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
s_local_command.callback = callback;
|
||||
s_local_command.callback_user_data = user_data;
|
||||
s_local_command.stop_requested = false;
|
||||
s_local_command.suspended = false;
|
||||
s_local_command.detection_running = false;
|
||||
s_local_command.reset_pending = false;
|
||||
s_local_command.control_generation = 1U;
|
||||
s_local_command.applied_generation = 0U;
|
||||
local_command_unlock();
|
||||
local_command_reset_pipeline_stats();
|
||||
|
||||
esp_err_t err = local_command_reserve_task_stack();
|
||||
if (err != ESP_OK) {
|
||||
local_command_cleanup_after_stop();
|
||||
return err;
|
||||
}
|
||||
if (s_local_command.pcm_queue == NULL) {
|
||||
s_local_command.pcm_queue = xQueueCreateWithCaps(
|
||||
DOMAIN_LOCAL_COMMAND_PCM_QUEUE_LEN,
|
||||
sizeof(domain_local_command_pcm_frame_t),
|
||||
MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
||||
}
|
||||
if (s_local_command.pcm_queue == NULL) {
|
||||
local_command_cleanup_after_stop();
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
(void)xQueueReset(s_local_command.pcm_queue);
|
||||
err = local_command_configure_model();
|
||||
if (err != ESP_OK) {
|
||||
local_command_cleanup_after_stop();
|
||||
return err;
|
||||
}
|
||||
err = platform_multimodal_audio_register_pcm_observer(
|
||||
local_command_pcm_observer,
|
||||
NULL);
|
||||
if (err != ESP_OK) {
|
||||
local_command_cleanup_after_stop();
|
||||
return err;
|
||||
}
|
||||
local_command_set_pcm_admission(true);
|
||||
|
||||
xEventGroupClearBits(s_local_command.events,
|
||||
DOMAIN_LOCAL_COMMAND_STOP_BIT |
|
||||
DOMAIN_LOCAL_COMMAND_STOPPED_BIT |
|
||||
DOMAIN_LOCAL_COMMAND_RUNNING_BIT |
|
||||
DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT);
|
||||
if (!local_command_lock()) {
|
||||
(void)platform_multimodal_audio_unregister_pcm_observer(
|
||||
local_command_pcm_observer, NULL,
|
||||
CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS);
|
||||
local_command_cleanup_after_stop();
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
s_local_command.started = true;
|
||||
local_command_unlock();
|
||||
|
||||
/* PSRAM-backed static stack. This inference-only task never initiates flash
|
||||
* writes; OTA stops local wake before image installation. Its worst case is
|
||||
* a 1026-byte PCM frame, MultiNet inference, and one bounded typed callback. */
|
||||
TaskHandle_t created = xTaskCreateStaticPinnedToCore(
|
||||
local_command_task,
|
||||
"domain_cmd",
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES,
|
||||
NULL,
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY,
|
||||
s_local_command_task_stack,
|
||||
&s_local_command_task_buffer,
|
||||
DOMAIN_LOCAL_COMMAND_TASK_CORE);
|
||||
if (created == NULL) {
|
||||
(void)platform_multimodal_audio_unregister_pcm_observer(
|
||||
local_command_pcm_observer, NULL,
|
||||
CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS);
|
||||
local_command_cleanup_after_stop();
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
if (local_command_lock()) {
|
||||
s_local_command.task = created;
|
||||
local_command_unlock();
|
||||
} else {
|
||||
vTaskDelete(created);
|
||||
(void)platform_multimodal_audio_unregister_pcm_observer(
|
||||
local_command_pcm_observer, NULL,
|
||||
CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS);
|
||||
local_command_cleanup_after_stop();
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
|
||||
EventBits_t bits = xEventGroupWaitBits(
|
||||
s_local_command.events,
|
||||
DOMAIN_LOCAL_COMMAND_RUNNING_BIT | DOMAIN_LOCAL_COMMAND_STOPPED_BIT,
|
||||
pdFALSE,
|
||||
pdFALSE,
|
||||
pdMS_TO_TICKS(CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS));
|
||||
if ((bits & DOMAIN_LOCAL_COMMAND_RUNNING_BIT) == 0U) {
|
||||
(void)domain_runtime_local_command_stop(CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS);
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static esp_err_t local_command_request_detection_state(bool suspended, int timeout_ms)
|
||||
{
|
||||
if (timeout_ms < 0) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
const uint32_t bounded_ms = local_command_default_timeout(timeout_ms);
|
||||
if (suspended) {
|
||||
local_command_set_pcm_admission(false);
|
||||
}
|
||||
if (!local_command_lock()) {
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
if (!s_local_command.started || s_local_command.stop_requested ||
|
||||
s_local_command.task == NULL) {
|
||||
local_command_unlock();
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
if (s_local_command.suspended == suspended &&
|
||||
s_local_command.applied_generation == s_local_command.control_generation) {
|
||||
local_command_unlock();
|
||||
return ESP_OK;
|
||||
}
|
||||
const bool resume_from_clean_stop = !suspended &&
|
||||
s_local_command.suspended &&
|
||||
!s_local_command.reset_pending &&
|
||||
!s_local_command.detection_running &&
|
||||
s_local_command.applied_generation == s_local_command.control_generation;
|
||||
s_local_command.suspended = suspended;
|
||||
if (suspended || !resume_from_clean_stop) {
|
||||
s_local_command.reset_pending = true;
|
||||
}
|
||||
const uint32_t generation = ++s_local_command.control_generation;
|
||||
const EventBits_t expected = suspended ?
|
||||
DOMAIN_LOCAL_COMMAND_SUSPENDED_BIT : DOMAIN_LOCAL_COMMAND_RUNNING_BIT;
|
||||
xEventGroupClearBits(s_local_command.events, expected);
|
||||
TaskHandle_t task = s_local_command.task;
|
||||
local_command_unlock();
|
||||
xTaskNotifyGive(task);
|
||||
|
||||
EventBits_t bits = xEventGroupWaitBits(
|
||||
s_local_command.events,
|
||||
expected | DOMAIN_LOCAL_COMMAND_STOPPED_BIT,
|
||||
pdFALSE,
|
||||
pdFALSE,
|
||||
pdMS_TO_TICKS(bounded_ms));
|
||||
if ((bits & expected) == 0U) {
|
||||
return (bits & DOMAIN_LOCAL_COMMAND_STOPPED_BIT) != 0U ?
|
||||
ESP_ERR_INVALID_STATE : ESP_ERR_TIMEOUT;
|
||||
}
|
||||
if (!local_command_lock()) {
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
const bool applied = s_local_command.started &&
|
||||
!s_local_command.stop_requested &&
|
||||
s_local_command.suspended == suspended &&
|
||||
s_local_command.applied_generation == generation;
|
||||
local_command_unlock();
|
||||
if (applied && !suspended) {
|
||||
local_command_set_pcm_admission(true);
|
||||
}
|
||||
return applied ? ESP_OK : ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
esp_err_t domain_runtime_local_command_suspend(int timeout_ms)
|
||||
{
|
||||
return local_command_request_detection_state(true, timeout_ms);
|
||||
}
|
||||
|
||||
esp_err_t domain_runtime_local_command_resume(int timeout_ms)
|
||||
{
|
||||
return local_command_request_detection_state(false, timeout_ms);
|
||||
}
|
||||
|
||||
esp_err_t domain_runtime_local_command_stop(int timeout_ms)
|
||||
{
|
||||
if (timeout_ms < 0) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
const uint32_t bounded_ms = local_command_default_timeout(timeout_ms);
|
||||
const TickType_t started = xTaskGetTickCount();
|
||||
if (s_local_command.lock == NULL) {
|
||||
return ESP_OK;
|
||||
}
|
||||
local_command_set_pcm_admission(false);
|
||||
|
||||
esp_err_t result = ESP_OK;
|
||||
esp_err_t observer_err = platform_multimodal_audio_unregister_pcm_observer(
|
||||
local_command_pcm_observer,
|
||||
NULL,
|
||||
(int)local_command_remaining_ms(started, bounded_ms));
|
||||
if (observer_err != ESP_OK && observer_err != ESP_ERR_INVALID_STATE) {
|
||||
result = observer_err;
|
||||
}
|
||||
if (!local_command_lock()) {
|
||||
return result != ESP_OK ? result : ESP_ERR_TIMEOUT;
|
||||
}
|
||||
if (!s_local_command.started) {
|
||||
local_command_unlock();
|
||||
return result;
|
||||
}
|
||||
s_local_command.stop_requested = true;
|
||||
s_local_command.detection_running = false;
|
||||
TaskHandle_t task = s_local_command.task;
|
||||
local_command_unlock();
|
||||
xEventGroupSetBits(s_local_command.events, DOMAIN_LOCAL_COMMAND_STOP_BIT);
|
||||
if (task != NULL) {
|
||||
xTaskNotifyGive(task);
|
||||
}
|
||||
|
||||
const uint32_t remaining_ms = local_command_remaining_ms(started, bounded_ms);
|
||||
EventBits_t bits = xEventGroupWaitBits(
|
||||
s_local_command.events,
|
||||
DOMAIN_LOCAL_COMMAND_STOPPED_BIT,
|
||||
pdFALSE,
|
||||
pdTRUE,
|
||||
pdMS_TO_TICKS(remaining_ms));
|
||||
if ((bits & DOMAIN_LOCAL_COMMAND_STOPPED_BIT) == 0U && result == ESP_OK) {
|
||||
result = ESP_ERR_TIMEOUT;
|
||||
}
|
||||
TaskHandle_t stopped_task = task;
|
||||
if (local_command_lock()) {
|
||||
if (s_local_command.task == stopped_task) {
|
||||
s_local_command.task = NULL;
|
||||
}
|
||||
local_command_unlock();
|
||||
}
|
||||
if (stopped_task != NULL) {
|
||||
vTaskDelete(stopped_task);
|
||||
}
|
||||
if ((bits & DOMAIN_LOCAL_COMMAND_STOPPED_BIT) == 0U) {
|
||||
ESP_LOGE(TAG, "Local command worker forced down after stop timeout");
|
||||
}
|
||||
local_command_cleanup_after_stop();
|
||||
return result;
|
||||
}
|
||||
|
||||
esp_err_t domain_runtime_local_command_get_snapshot(
|
||||
domain_local_command_snapshot_t *out_snapshot)
|
||||
{
|
||||
if (out_snapshot == NULL) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
if (s_local_command.lock == NULL) {
|
||||
memset(out_snapshot, 0, sizeof(*out_snapshot));
|
||||
return ESP_OK;
|
||||
}
|
||||
if (!local_command_lock()) {
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
*out_snapshot = (domain_local_command_snapshot_t) {
|
||||
.started = s_local_command.started,
|
||||
.detection_running = s_local_command.detection_running,
|
||||
.detection_generation = s_local_command.detection_generation,
|
||||
.detection_count = s_local_command.detection_count,
|
||||
.dropped_pcm_frames = local_command_dropped_pcm_frames(),
|
||||
.task_stack_high_water_bytes = s_local_command.task_stack_high_water_bytes,
|
||||
};
|
||||
local_command_unlock();
|
||||
return ESP_OK;
|
||||
}
|
||||
@@ -4,5 +4,6 @@ dependencies:
|
||||
espressif/esp_websocket_client: ^1.6.1
|
||||
espressif/esp_codec_dev: ^1.5.4
|
||||
espressif/esp_audio_codec: ^2.4.1
|
||||
espressif/esp-sr: ^2.4.7
|
||||
espressif/mdns: ^1.11.0
|
||||
78/esp-wifi-connect: ^3.1.4
|
||||
|
||||
@@ -49,6 +49,12 @@ typedef struct {
|
||||
void *user_data;
|
||||
} platform_multimodal_downstream_playback_callbacks_t;
|
||||
|
||||
typedef void (*platform_multimodal_audio_pcm_observer_t)(
|
||||
const int16_t *pcm,
|
||||
size_t samples,
|
||||
uint32_t sample_rate_hz,
|
||||
void *user_data);
|
||||
|
||||
#define PLATFORM_LAN_HTTP_PATH_MAX_LEN 96
|
||||
#define PLATFORM_LAN_HTTP_HOSTNAME_MAX_LEN 32
|
||||
#define PLATFORM_LAN_HTTP_REQUEST_BODY_MAX_LEN 1024
|
||||
@@ -321,6 +327,24 @@ esp_err_t platform_multimodal_audio_start_uplink(
|
||||
esp_err_t platform_multimodal_audio_stop_uplink(int timeout_ms);
|
||||
bool platform_multimodal_audio_is_uplink_running(void);
|
||||
bool platform_multimodal_audio_is_initialized(void);
|
||||
/**
|
||||
* @brief Register the single mono PCM observer for the active capture path.
|
||||
*
|
||||
* The callback runs synchronously in the platform capture task, outside the
|
||||
* registration lock, and must remain bounded and non-blocking.
|
||||
*/
|
||||
esp_err_t platform_multimodal_audio_register_pcm_observer(
|
||||
platform_multimodal_audio_pcm_observer_t observer,
|
||||
void *user_data);
|
||||
/**
|
||||
* @brief Remove a matching PCM observer and wait for in-flight callbacks.
|
||||
*
|
||||
* Negative timeout is invalid. Zero performs a non-blocking drain check.
|
||||
*/
|
||||
esp_err_t platform_multimodal_audio_unregister_pcm_observer(
|
||||
platform_multimodal_audio_pcm_observer_t observer,
|
||||
void *user_data,
|
||||
int timeout_ms);
|
||||
esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
|
||||
esp_err_t platform_multimodal_audio_begin_downstream_response(void);
|
||||
esp_err_t platform_multimodal_audio_finish_downstream_response(void);
|
||||
|
||||
@@ -11,6 +11,10 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
typedef esp_err_t (*multimodal_audio_uplink_cb_t)(const uint8_t *data, size_t len, void *ctx);
|
||||
typedef void (*multimodal_audio_pcm_observer_t)(const int16_t *pcm,
|
||||
size_t samples,
|
||||
uint32_t sample_rate_hz,
|
||||
void *ctx);
|
||||
|
||||
typedef enum {
|
||||
MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED = 0,
|
||||
@@ -33,6 +37,13 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
|
||||
esp_err_t multimodal_audio_stop_uplink(int timeout_ms);
|
||||
bool multimodal_audio_is_uplink_running(void);
|
||||
bool multimodal_audio_is_initialized(void);
|
||||
esp_err_t multimodal_audio_register_pcm_observer(
|
||||
multimodal_audio_pcm_observer_t observer,
|
||||
void *observer_ctx);
|
||||
esp_err_t multimodal_audio_unregister_pcm_observer(
|
||||
multimodal_audio_pcm_observer_t observer,
|
||||
void *observer_ctx,
|
||||
int timeout_ms);
|
||||
|
||||
esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t len);
|
||||
esp_err_t multimodal_audio_begin_downstream_response(void);
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "esp_codec_dev_defaults.h"
|
||||
#include "esp_audio_dec.h"
|
||||
#include "esp_audio_enc.h"
|
||||
#include "esp_afe_aec.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "esp_log.h"
|
||||
@@ -114,6 +115,14 @@
|
||||
#define CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL
|
||||
#define CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL 1
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH
|
||||
#define CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH 4
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
|
||||
#define CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS 1
|
||||
#endif
|
||||
@@ -208,8 +217,16 @@
|
||||
#error "Multimodal voice requires 16 kHz audio"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1
|
||||
#error "Multimodal voice requires mono audio"
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2
|
||||
#error "ES8311 local wake AEC requires two physical codec slots"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE != 16
|
||||
#error "ESP AEC requires signed 16-bit PCM"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL == CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL
|
||||
#error "Microphone and AEC reference must use distinct codec slots"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60
|
||||
@@ -235,6 +252,9 @@
|
||||
#define MULTIMODAL_DECODER_BUFFER_GUARD_BYTES 512U
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_TX_BOUNCE_BYTES 3200U
|
||||
#define MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL ((uint8_t)CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL)
|
||||
#define MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL \
|
||||
((uint8_t)CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL)
|
||||
#define MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT 16U
|
||||
|
||||
static const char *TAG = "multimodal_audio";
|
||||
static portMUX_TYPE s_audio_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||
@@ -253,6 +273,7 @@ typedef struct {
|
||||
uint8_t i2c_ctrl_addr_8bit;
|
||||
esp_audio_enc_handle_t opus_enc;
|
||||
esp_audio_dec_handle_t opus_dec;
|
||||
afe_aec_handle_t *afe_aec_handle;
|
||||
int opus_enc_in_bytes;
|
||||
int opus_enc_out_bytes;
|
||||
uint32_t opus_sample_rate;
|
||||
@@ -260,8 +281,11 @@ typedef struct {
|
||||
uint32_t opus_bitrate_bps;
|
||||
uint8_t *uplink_codec_buf;
|
||||
int16_t *uplink_mono_buf;
|
||||
int16_t *uplink_aec_clean_buf;
|
||||
uint8_t *uplink_opus_buf;
|
||||
size_t uplink_codec_read_bytes;
|
||||
size_t uplink_aec_frame_bytes;
|
||||
size_t uplink_aec_frame_samples;
|
||||
size_t uplink_mono_bytes;
|
||||
size_t uplink_mono_buf_capacity_bytes;
|
||||
size_t uplink_mono_fill_bytes;
|
||||
@@ -290,6 +314,7 @@ typedef struct {
|
||||
bool downstream_playback_keep_stopping_on_exit;
|
||||
bool audio_deinit_in_progress;
|
||||
uint8_t mic_capture_channel;
|
||||
uint8_t aec_reference_channel;
|
||||
QueueHandle_t uplink_send_queue;
|
||||
bool uplink_send_queue_with_caps;
|
||||
TaskHandle_t uplink_task;
|
||||
@@ -306,6 +331,10 @@ typedef struct {
|
||||
static multimodal_audio_ctx_t s_audio;
|
||||
static multimodal_audio_downstream_playback_callback_t s_downstream_playback_cb;
|
||||
static void *s_downstream_playback_cb_ctx;
|
||||
static multimodal_audio_pcm_observer_t s_pcm_observer;
|
||||
static void *s_pcm_observer_ctx;
|
||||
static uint32_t s_pcm_observer_in_flight;
|
||||
static portMUX_TYPE s_pcm_observer_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||
static bool s_uplink_stack_warned;
|
||||
static bool s_uplink_send_stack_warned;
|
||||
static bool s_uplink_send_queue_warned;
|
||||
|
||||
@@ -50,10 +50,17 @@ static void cleanup_opus_path(multimodal_audio_ctx_t *ctx)
|
||||
ctx->opus_dec = NULL;
|
||||
}
|
||||
|
||||
if (ctx->afe_aec_handle != NULL) {
|
||||
afe_aec_destroy(ctx->afe_aec_handle);
|
||||
ctx->afe_aec_handle = NULL;
|
||||
}
|
||||
|
||||
free(ctx->uplink_codec_buf);
|
||||
ctx->uplink_codec_buf = NULL;
|
||||
free(ctx->uplink_mono_buf);
|
||||
ctx->uplink_mono_buf = NULL;
|
||||
free(ctx->uplink_aec_clean_buf);
|
||||
ctx->uplink_aec_clean_buf = NULL;
|
||||
free(ctx->uplink_opus_buf);
|
||||
ctx->uplink_opus_buf = NULL;
|
||||
free(ctx->downstream_pcm_buf);
|
||||
@@ -64,6 +71,8 @@ static void cleanup_opus_path(multimodal_audio_ctx_t *ctx)
|
||||
ctx->downstream_tx_bounce_buf = NULL;
|
||||
|
||||
ctx->uplink_codec_read_bytes = 0;
|
||||
ctx->uplink_aec_frame_bytes = 0;
|
||||
ctx->uplink_aec_frame_samples = 0;
|
||||
ctx->uplink_mono_bytes = 0;
|
||||
ctx->uplink_mono_buf_capacity_bytes = 0;
|
||||
ctx->uplink_mono_fill_bytes = 0;
|
||||
@@ -566,15 +575,54 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx)
|
||||
return audio_err_to_esp_err(audio_err);
|
||||
}
|
||||
|
||||
const char *aec_input_format = NULL;
|
||||
if (ctx->mic_capture_channel == 0U &&
|
||||
ctx->aec_reference_channel == 1U) {
|
||||
aec_input_format = "MR";
|
||||
} else if (ctx->mic_capture_channel == 1U &&
|
||||
ctx->aec_reference_channel == 0U) {
|
||||
aec_input_format = "RM";
|
||||
} else {
|
||||
ESP_LOGE(TAG,
|
||||
"Unsupported AEC slot map: mic=%u ref=%u",
|
||||
(unsigned)ctx->mic_capture_channel,
|
||||
(unsigned)ctx->aec_reference_channel);
|
||||
return ESP_ERR_NOT_SUPPORTED;
|
||||
}
|
||||
ctx->afe_aec_handle = afe_aec_create(
|
||||
aec_input_format,
|
||||
CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH,
|
||||
AFE_TYPE_FD,
|
||||
AFE_MODE_LOW_COST);
|
||||
if (ctx->afe_aec_handle == NULL) {
|
||||
ESP_LOGE(TAG, "Create ESP-SR AEC failed");
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
if (ctx->afe_aec_handle->handle != NULL) {
|
||||
(void)aec_set_nlp_level(
|
||||
ctx->afe_aec_handle->handle,
|
||||
AEC_NLP_LEVEL_AGGR);
|
||||
}
|
||||
int aec_frame_samples = afe_aec_get_chunksize(ctx->afe_aec_handle);
|
||||
if (aec_frame_samples <= 0) {
|
||||
ESP_LOGE(TAG, "Invalid ESP-SR AEC frame size: %d", aec_frame_samples);
|
||||
return ESP_ERR_INVALID_SIZE;
|
||||
}
|
||||
ctx->uplink_aec_frame_samples = (size_t)aec_frame_samples;
|
||||
ctx->uplink_aec_frame_bytes =
|
||||
ctx->uplink_aec_frame_samples * sizeof(int16_t);
|
||||
|
||||
ctx->uplink_mono_bytes = (size_t)ctx->opus_enc_in_bytes;
|
||||
if (ctx->uplink_mono_bytes == 0 || ctx->fs.channel == 0U) {
|
||||
return ESP_ERR_INVALID_SIZE;
|
||||
}
|
||||
|
||||
if (ctx->uplink_mono_bytes > (SIZE_MAX / (size_t)ctx->fs.channel)) {
|
||||
if (ctx->uplink_aec_frame_bytes >
|
||||
(SIZE_MAX / (size_t)ctx->fs.channel)) {
|
||||
return ESP_ERR_INVALID_SIZE;
|
||||
}
|
||||
size_t codec_read_bytes = ctx->uplink_mono_bytes * (size_t)ctx->fs.channel;
|
||||
size_t codec_read_bytes =
|
||||
ctx->uplink_aec_frame_bytes * (size_t)ctx->fs.channel;
|
||||
ctx->uplink_codec_read_bytes = codec_read_bytes;
|
||||
ctx->uplink_opus_buf_bytes = (size_t)ctx->opus_enc_out_bytes;
|
||||
if (ctx->uplink_opus_buf_bytes < 256U) {
|
||||
@@ -586,8 +634,14 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx)
|
||||
|
||||
ctx->uplink_codec_buf = (uint8_t *)audio_psram_first_calloc(1, ctx->uplink_codec_read_bytes);
|
||||
ctx->uplink_mono_buf = (int16_t *)audio_psram_first_calloc(1, ctx->uplink_mono_buf_capacity_bytes);
|
||||
ctx->uplink_aec_clean_buf = (int16_t *)heap_caps_aligned_calloc(
|
||||
MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT,
|
||||
1,
|
||||
ctx->uplink_aec_frame_bytes,
|
||||
MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||
ctx->uplink_opus_buf = (uint8_t *)audio_psram_first_calloc(1, ctx->uplink_opus_buf_bytes);
|
||||
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf) {
|
||||
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf ||
|
||||
!ctx->uplink_aec_clean_buf || !ctx->uplink_opus_buf) {
|
||||
return ESP_ERR_NO_MEM;
|
||||
}
|
||||
|
||||
@@ -601,38 +655,20 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx)
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG,
|
||||
"Opus ready: %uHz frame=%ums bitrate=%ukbps enc(in=%d out=%d)",
|
||||
"Opus/AEC ready: %uHz opus_frame=%ums bitrate=%ukbps enc(in=%d out=%d) aec_frame=%u input=%s",
|
||||
(unsigned)ctx->opus_sample_rate,
|
||||
(unsigned)ctx->opus_frame_duration_ms,
|
||||
(unsigned)bitrate_kbps,
|
||||
ctx->opus_enc_in_bytes,
|
||||
ctx->opus_enc_out_bytes);
|
||||
ctx->opus_enc_out_bytes,
|
||||
(unsigned)ctx->uplink_aec_frame_samples,
|
||||
aec_input_format);
|
||||
ctx->opus_ready = true;
|
||||
log_heap_snapshot("after_opus_init");
|
||||
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static void extract_mic_channel_16(const int16_t *input,
|
||||
uint8_t channels,
|
||||
uint8_t mic_channel,
|
||||
int16_t *output,
|
||||
size_t frames)
|
||||
{
|
||||
if (!input || !output || channels == 0 || mic_channel >= channels) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (channels == 1) {
|
||||
memcpy(output, input, frames * sizeof(int16_t));
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t frame = 0; frame < frames; ++frame) {
|
||||
output[frame] = input[frame * channels + mic_channel];
|
||||
}
|
||||
}
|
||||
|
||||
static void expand_mono_to_multichannel_16(const int16_t *input,
|
||||
uint8_t channels,
|
||||
int16_t *output,
|
||||
|
||||
@@ -207,6 +207,8 @@ static void reset_capture_channel_map(multimodal_audio_ctx_t *ctx)
|
||||
|
||||
uint8_t mic_channel = MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL > 0U ? 1U : 0U;
|
||||
ctx->mic_capture_channel = mic_channel;
|
||||
ctx->aec_reference_channel =
|
||||
MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL > 0U ? 1U : 0U;
|
||||
}
|
||||
|
||||
static void fill_codec_sample_info(multimodal_audio_ctx_t *ctx)
|
||||
@@ -220,9 +222,14 @@ static void fill_codec_sample_info(multimodal_audio_ctx_t *ctx)
|
||||
ctx->fs.channel_mask = 0;
|
||||
ctx->fs.mclk_multiple = 0;
|
||||
reset_capture_channel_map(ctx);
|
||||
if (ctx->fs.channel == 1U || ctx->mic_capture_channel >= ctx->fs.channel) {
|
||||
if (ctx->mic_capture_channel >= ctx->fs.channel) {
|
||||
ctx->mic_capture_channel = 0U;
|
||||
}
|
||||
if (ctx->aec_reference_channel >= ctx->fs.channel ||
|
||||
ctx->aec_reference_channel == ctx->mic_capture_channel) {
|
||||
ctx->aec_reference_channel =
|
||||
ctx->mic_capture_channel == 0U ? 1U : 0U;
|
||||
}
|
||||
}
|
||||
|
||||
static bool is_supported_opus_sample_rate(uint32_t sample_rate)
|
||||
|
||||
@@ -157,6 +157,34 @@ static void audio_set_uplink_callback(multimodal_audio_ctx_t *ctx,
|
||||
portEXIT_CRITICAL(&s_audio_runtime_lock);
|
||||
}
|
||||
|
||||
static void notify_pcm_observer(const int16_t *pcm, size_t samples)
|
||||
{
|
||||
multimodal_audio_pcm_observer_t observer = NULL;
|
||||
void *observer_ctx = NULL;
|
||||
|
||||
portENTER_CRITICAL(&s_pcm_observer_lock);
|
||||
observer = s_pcm_observer;
|
||||
observer_ctx = s_pcm_observer_ctx;
|
||||
if (observer != NULL) {
|
||||
++s_pcm_observer_in_flight;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_pcm_observer_lock);
|
||||
|
||||
if (observer == NULL) {
|
||||
return;
|
||||
}
|
||||
observer(pcm,
|
||||
samples,
|
||||
(uint32_t)CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE,
|
||||
observer_ctx);
|
||||
|
||||
portENTER_CRITICAL(&s_pcm_observer_lock);
|
||||
if (s_pcm_observer_in_flight > 0U) {
|
||||
--s_pcm_observer_in_flight;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_pcm_observer_lock);
|
||||
}
|
||||
|
||||
static void audio_snapshot_uplink_tasks(const multimodal_audio_ctx_t *ctx,
|
||||
TaskHandle_t *out_capture_task,
|
||||
TaskHandle_t *out_send_task)
|
||||
@@ -610,6 +638,47 @@ static esp_err_t encode_and_queue_mono_pcm(multimodal_audio_ctx_t *ctx,
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static esp_err_t append_and_queue_mono_pcm(
|
||||
multimodal_audio_ctx_t *ctx,
|
||||
const int16_t *pcm,
|
||||
size_t pcm_bytes)
|
||||
{
|
||||
if (ctx == NULL || pcm == NULL || pcm_bytes == 0U ||
|
||||
ctx->uplink_mono_buf == NULL ||
|
||||
ctx->uplink_mono_buf_capacity_bytes != ctx->uplink_mono_bytes) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
const uint8_t *input = (const uint8_t *)pcm;
|
||||
size_t remaining = pcm_bytes;
|
||||
while (remaining > 0U) {
|
||||
size_t available =
|
||||
ctx->uplink_mono_buf_capacity_bytes -
|
||||
ctx->uplink_mono_fill_bytes;
|
||||
size_t copy_bytes = remaining < available ? remaining : available;
|
||||
memcpy((uint8_t *)ctx->uplink_mono_buf +
|
||||
ctx->uplink_mono_fill_bytes,
|
||||
input,
|
||||
copy_bytes);
|
||||
ctx->uplink_mono_fill_bytes += copy_bytes;
|
||||
input += copy_bytes;
|
||||
remaining -= copy_bytes;
|
||||
|
||||
if (ctx->uplink_mono_fill_bytes ==
|
||||
ctx->uplink_mono_buf_capacity_bytes) {
|
||||
esp_err_t err = encode_and_queue_mono_pcm(
|
||||
ctx,
|
||||
ctx->uplink_mono_buf,
|
||||
ctx->uplink_mono_bytes);
|
||||
ctx->uplink_mono_fill_bytes = 0U;
|
||||
if (err != ESP_OK) {
|
||||
return err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static void uplink_send_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
@@ -666,9 +735,13 @@ static void uplink_capture_task(void *arg)
|
||||
(void)arg;
|
||||
|
||||
multimodal_audio_ctx_t *ctx = &s_audio;
|
||||
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf ||
|
||||
!ctx->opus_enc || ctx->uplink_mono_bytes == 0U || ctx->uplink_codec_read_bytes == 0U) {
|
||||
ESP_LOGE(TAG, "Uplink Opus buffers not ready");
|
||||
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf ||
|
||||
!ctx->uplink_aec_clean_buf || !ctx->uplink_opus_buf ||
|
||||
!ctx->opus_enc || !ctx->afe_aec_handle ||
|
||||
ctx->uplink_mono_bytes == 0U ||
|
||||
ctx->uplink_codec_read_bytes == 0U ||
|
||||
ctx->uplink_aec_frame_samples == 0U) {
|
||||
ESP_LOGE(TAG, "Uplink Opus/AEC buffers not ready");
|
||||
audio_set_uplink_capture_task(ctx, NULL);
|
||||
delete_audio_task_with_caps(NULL);
|
||||
return;
|
||||
@@ -681,13 +754,14 @@ static void uplink_capture_task(void *arg)
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG,
|
||||
"Mic uplink capture task started (stack=%uB opus in=%uB out=%uB read=%uB channels=%u mic_ch=%u)",
|
||||
"Mic uplink capture task started (stack=%uB opus in=%uB out=%uB read=%uB channels=%u mic_ch=%u ref_ch=%u)",
|
||||
(unsigned)get_uplink_task_stack_bytes(),
|
||||
(unsigned)ctx->uplink_mono_bytes,
|
||||
(unsigned)ctx->uplink_opus_buf_bytes,
|
||||
(unsigned)ctx->uplink_codec_read_bytes,
|
||||
(unsigned)ctx->fs.channel,
|
||||
(unsigned)ctx->mic_capture_channel);
|
||||
(unsigned)ctx->mic_capture_channel,
|
||||
(unsigned)ctx->aec_reference_channel);
|
||||
|
||||
while (!audio_get_uplink_task_exit_requested(ctx)) {
|
||||
if (!audio_get_uplink_recording_active(ctx)) {
|
||||
@@ -707,19 +781,37 @@ static void uplink_capture_task(void *arg)
|
||||
continue;
|
||||
}
|
||||
|
||||
size_t uplink_frames = ctx->uplink_mono_bytes / sizeof(int16_t);
|
||||
extract_mic_channel_16((const int16_t *)ctx->uplink_codec_buf,
|
||||
ctx->fs.channel,
|
||||
ctx->mic_capture_channel,
|
||||
ctx->uplink_mono_buf,
|
||||
uplink_frames);
|
||||
audio_debug_dump_pcm(ctx->uplink_mono_buf, ctx->uplink_mono_bytes);
|
||||
esp_err_t err = encode_and_queue_mono_pcm(ctx, ctx->uplink_mono_buf, ctx->uplink_mono_bytes);
|
||||
size_t clean_bytes = afe_aec_process(
|
||||
ctx->afe_aec_handle,
|
||||
(const int16_t *)ctx->uplink_codec_buf,
|
||||
ctx->uplink_aec_clean_buf);
|
||||
if (clean_bytes != ctx->uplink_aec_frame_bytes) {
|
||||
ESP_LOGW(TAG,
|
||||
"AEC output size=%u expected=%u",
|
||||
(unsigned)clean_bytes,
|
||||
(unsigned)ctx->uplink_aec_frame_bytes);
|
||||
ctx->uplink_mono_fill_bytes = 0U;
|
||||
continue;
|
||||
}
|
||||
size_t uplink_frames = ctx->uplink_aec_frame_samples;
|
||||
notify_pcm_observer(ctx->uplink_aec_clean_buf, uplink_frames);
|
||||
audio_debug_dump_pcm(ctx->uplink_aec_clean_buf, clean_bytes);
|
||||
multimodal_audio_uplink_cb_t uplink_cb = NULL;
|
||||
audio_snapshot_uplink_callback(ctx, &uplink_cb, NULL);
|
||||
esp_err_t err = ESP_OK;
|
||||
if (uplink_cb != NULL) {
|
||||
err = append_and_queue_mono_pcm(
|
||||
ctx,
|
||||
ctx->uplink_aec_clean_buf,
|
||||
clean_bytes);
|
||||
} else {
|
||||
ctx->uplink_mono_fill_bytes = 0U;
|
||||
}
|
||||
maybe_log_uplink_levels(ctx,
|
||||
(const int16_t *)ctx->uplink_codec_buf,
|
||||
uplink_frames,
|
||||
ctx->uplink_mono_buf,
|
||||
ctx->uplink_mono_bytes);
|
||||
ctx->uplink_aec_clean_buf,
|
||||
clean_bytes);
|
||||
if (err != ESP_OK && err != ESP_ERR_INVALID_SIZE && err != ESP_ERR_TIMEOUT) {
|
||||
ESP_LOGW(TAG, "Encode uplink PCM failed: %s", esp_err_to_name(err));
|
||||
vTaskDelay(pdMS_TO_TICKS(10));
|
||||
@@ -739,7 +831,9 @@ static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx)
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_enc ||
|
||||
!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf) {
|
||||
!ctx->afe_aec_handle || !ctx->uplink_codec_buf ||
|
||||
!ctx->uplink_mono_buf || !ctx->uplink_aec_clean_buf ||
|
||||
!ctx->uplink_opus_buf) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
@@ -970,18 +1064,82 @@ esp_err_t multimodal_audio_register_downstream_playback_callback(
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *cb_ctx)
|
||||
esp_err_t multimodal_audio_register_pcm_observer(
|
||||
multimodal_audio_pcm_observer_t observer,
|
||||
void *observer_ctx)
|
||||
{
|
||||
if (!cb) {
|
||||
if (observer == NULL) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
esp_err_t result = ESP_OK;
|
||||
portENTER_CRITICAL(&s_pcm_observer_lock);
|
||||
if (s_pcm_observer != NULL &&
|
||||
(s_pcm_observer != observer || s_pcm_observer_ctx != observer_ctx)) {
|
||||
result = ESP_ERR_INVALID_STATE;
|
||||
} else {
|
||||
s_pcm_observer = observer;
|
||||
s_pcm_observer_ctx = observer_ctx;
|
||||
}
|
||||
portEXIT_CRITICAL(&s_pcm_observer_lock);
|
||||
return result;
|
||||
}
|
||||
|
||||
esp_err_t multimodal_audio_unregister_pcm_observer(
|
||||
multimodal_audio_pcm_observer_t observer,
|
||||
void *observer_ctx,
|
||||
int timeout_ms)
|
||||
{
|
||||
if (observer == NULL || timeout_ms < 0) {
|
||||
return ESP_ERR_INVALID_ARG;
|
||||
}
|
||||
|
||||
portENTER_CRITICAL(&s_pcm_observer_lock);
|
||||
if (s_pcm_observer == NULL) {
|
||||
portEXIT_CRITICAL(&s_pcm_observer_lock);
|
||||
return ESP_OK;
|
||||
}
|
||||
if (s_pcm_observer != observer || s_pcm_observer_ctx != observer_ctx) {
|
||||
portEXIT_CRITICAL(&s_pcm_observer_lock);
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
s_pcm_observer = NULL;
|
||||
s_pcm_observer_ctx = NULL;
|
||||
portEXIT_CRITICAL(&s_pcm_observer_lock);
|
||||
|
||||
TickType_t started = xTaskGetTickCount();
|
||||
TickType_t timeout_ticks = pdMS_TO_TICKS((uint32_t)timeout_ms);
|
||||
if (timeout_ms > 0 && timeout_ticks == 0U) {
|
||||
timeout_ticks = 1U;
|
||||
}
|
||||
for (;;) {
|
||||
uint32_t in_flight = 0U;
|
||||
portENTER_CRITICAL(&s_pcm_observer_lock);
|
||||
in_flight = s_pcm_observer_in_flight;
|
||||
portEXIT_CRITICAL(&s_pcm_observer_lock);
|
||||
if (in_flight == 0U) {
|
||||
return ESP_OK;
|
||||
}
|
||||
if (timeout_ms == 0 ||
|
||||
(xTaskGetTickCount() - started) >= timeout_ticks) {
|
||||
return ESP_ERR_TIMEOUT;
|
||||
}
|
||||
vTaskDelay(1U);
|
||||
}
|
||||
}
|
||||
|
||||
esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *cb_ctx)
|
||||
{
|
||||
if (!s_audio.initialized || !s_audio.codec_dev || !s_audio.opus_enc ||
|
||||
!s_audio.uplink_codec_buf || !s_audio.uplink_mono_buf || !s_audio.uplink_opus_buf) {
|
||||
return ESP_ERR_INVALID_STATE;
|
||||
}
|
||||
|
||||
bool already_active = audio_get_uplink_recording_active(&s_audio);
|
||||
audio_set_uplink_callback(&s_audio, cb, cb_ctx);
|
||||
if (already_active) {
|
||||
return ESP_OK;
|
||||
}
|
||||
s_audio.uplink_mono_fill_bytes = 0U;
|
||||
memset(s_audio.uplink_mono_buf, 0, s_audio.uplink_mono_buf_capacity_bytes);
|
||||
reset_uplink_debug_stats();
|
||||
|
||||
@@ -177,6 +177,24 @@ bool platform_multimodal_audio_is_initialized(void)
|
||||
return multimodal_audio_is_initialized();
|
||||
}
|
||||
|
||||
esp_err_t platform_multimodal_audio_register_pcm_observer(
|
||||
platform_multimodal_audio_pcm_observer_t observer,
|
||||
void *user_data)
|
||||
{
|
||||
return multimodal_audio_register_pcm_observer(observer, user_data);
|
||||
}
|
||||
|
||||
esp_err_t platform_multimodal_audio_unregister_pcm_observer(
|
||||
platform_multimodal_audio_pcm_observer_t observer,
|
||||
void *user_data,
|
||||
int timeout_ms)
|
||||
{
|
||||
return multimodal_audio_unregister_pcm_observer(
|
||||
observer,
|
||||
user_data,
|
||||
timeout_ms);
|
||||
}
|
||||
|
||||
esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len)
|
||||
{
|
||||
return multimodal_audio_handle_downstream_opus(data, len);
|
||||
|
||||
+54
-1
@@ -15,6 +15,58 @@ dependencies:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
version: 3.1.4
|
||||
espressif/cjson:
|
||||
component_hash: e788323270d90738662d66fffa910bfe1fba019bba087f01557e70c40485b469
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com
|
||||
type: service
|
||||
version: 1.7.19~2
|
||||
espressif/dl_fft:
|
||||
component_hash: a955ca29581de2177ddf25759273bd83f082b3c805e735fe6ee105858643989b
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com
|
||||
type: service
|
||||
version: 0.6.0
|
||||
espressif/esp-dsp:
|
||||
component_hash: 12e44db246517a627bc0fe2b255b8335410c0215c98bfba2df5a8e3edea839ef
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=4.2'
|
||||
source:
|
||||
registry_url: https://components.espressif.com
|
||||
type: service
|
||||
version: 1.8.0
|
||||
espressif/esp-sr:
|
||||
component_hash: 809d0041cdddd98a278f0d5afef7bb60a451290577b98cf718dfffc91bdcbd9b
|
||||
dependencies:
|
||||
- name: espressif/cjson
|
||||
registry_url: https://components.espressif.com
|
||||
require: private
|
||||
version: ^1.7.19
|
||||
- name: espressif/dl_fft
|
||||
registry_url: https://components.espressif.com
|
||||
require: private
|
||||
version: '>=0.6.0'
|
||||
- name: espressif/esp-dsp
|
||||
registry_url: https://components.espressif.com
|
||||
require: private
|
||||
version: 1.8.0
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
version: 2.4.7
|
||||
espressif/esp_audio_codec:
|
||||
component_hash: af4663830bc73eca58febcc34a976d003465e78ed57458aa00a14b231e392ca1
|
||||
dependencies:
|
||||
@@ -71,12 +123,13 @@ dependencies:
|
||||
version: 1.20.4
|
||||
direct_dependencies:
|
||||
- 78/esp-wifi-connect
|
||||
- espressif/esp-sr
|
||||
- espressif/esp_audio_codec
|
||||
- espressif/esp_codec_dev
|
||||
- espressif/esp_websocket_client
|
||||
- espressif/mdns
|
||||
- idf
|
||||
- joltwallet/littlefs
|
||||
manifest_hash: b7c0e21c22598b04ad72173217dbd4f7c984674b1b7df2f688b54eb19cff8a12
|
||||
manifest_hash: c05cc39447d3758499f6a9a2b947c3fe21cf4fd47e0e9905cde8d79dc68d0472
|
||||
target: esp32s3
|
||||
version: 2.0.0
|
||||
|
||||
+113
-3
@@ -879,6 +879,99 @@ menu "Example Configuration"
|
||||
help
|
||||
Leave disabled for the default active-low Button 2 wiring.
|
||||
|
||||
config TQ_LOCAL_WAKE_ENABLE
|
||||
bool "Enable local wake-word listener"
|
||||
default y
|
||||
help
|
||||
Use the on-device Chinese MultiNet model to detect 喵掌柜.
|
||||
The physical PTT button remains enabled as a parallel entry.
|
||||
|
||||
config TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE
|
||||
bool "Enable local wake response prompt"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default n
|
||||
help
|
||||
Play a short acknowledgement after 喵掌柜 is detected and
|
||||
before the normal single-turn dialog starts. Keep disabled
|
||||
until the placeholder Opus asset has been supplied.
|
||||
|
||||
config TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE
|
||||
string "Local wake response prompt file"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default "/littlefs/audio_prompts/local_wake_response.opus"
|
||||
help
|
||||
Reserved LittleFS path for the future wake acknowledgement.
|
||||
Add local_wake_response.opus under
|
||||
components/platform/assets/audio_prompts before enabling it.
|
||||
|
||||
config TQ_LOCAL_WAKE_RETRY_MS
|
||||
int "Local wake startup retry delay (ms)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 5000
|
||||
range 500 60000
|
||||
|
||||
config TQ_LOCAL_WAKE_STOP_WAIT_MS
|
||||
int "Local wake lifecycle wait (ms)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 1000
|
||||
range 100 5000
|
||||
|
||||
config TQ_LOCAL_WAKE_TASK_STACK_BYTES
|
||||
int "Local wake control task stack (bytes)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 6144
|
||||
range 4096 32768
|
||||
|
||||
config TQ_LOCAL_WAKE_TASK_PRIORITY
|
||||
int "Local wake control task priority"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 4
|
||||
range 1 10
|
||||
|
||||
config TQ_LOCAL_WAKE_TASK_CORE
|
||||
int "Local wake control task core (-1 for no affinity)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 0
|
||||
range -1 1
|
||||
|
||||
config TQ_LOCAL_COMMAND_TASK_STACK_BYTES
|
||||
int "MultiNet inference task stack (bytes)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 8192
|
||||
range 6144 32768
|
||||
|
||||
config TQ_LOCAL_COMMAND_TASK_PRIORITY
|
||||
int "MultiNet inference task priority"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 7
|
||||
range 1 10
|
||||
|
||||
config TQ_LOCAL_COMMAND_TASK_CORE
|
||||
int "MultiNet inference task core (-1 for no affinity)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 0
|
||||
range -1 1
|
||||
|
||||
config TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS
|
||||
int "MultiNet control acknowledgement timeout (ms)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 1000
|
||||
range 100 5000
|
||||
|
||||
config TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS
|
||||
int "MultiNet utterance detection timeout (ms)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 6000
|
||||
range 1000 15000
|
||||
|
||||
config TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000
|
||||
int "MultiNet detection threshold (x10000)"
|
||||
depends on TQ_LOCAL_WAKE_ENABLE
|
||||
default 1200
|
||||
range 1 9999
|
||||
help
|
||||
Initial threshold is 0.12 and must be tuned with device audio.
|
||||
|
||||
config MULTIMODAL_HEARTBEAT_INTERVAL_MS
|
||||
int "Heartbeat interval (ms)"
|
||||
default 50000
|
||||
@@ -966,10 +1059,11 @@ menu "Example Configuration"
|
||||
|
||||
config MULTIMODAL_AUDIO_CODEC_CHANNELS
|
||||
int "Codec channels"
|
||||
default 1
|
||||
range 1 1
|
||||
default 2
|
||||
range 2 2
|
||||
help
|
||||
Fixed at 1 for the raw-opus 16kHz mono voice contract.
|
||||
Two physical ES8311 slots are captured: microphone and DAC
|
||||
playback reference. The external Opus contract remains mono.
|
||||
|
||||
config MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE
|
||||
int "Codec bits per sample"
|
||||
@@ -1051,6 +1145,22 @@ menu "Example Configuration"
|
||||
speech. Use 0 for the default left/first channel, or 1 if
|
||||
level logs show speech energy on raw_ch1 instead of raw_ch0.
|
||||
|
||||
config MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL
|
||||
int "AEC playback reference channel"
|
||||
default 1
|
||||
range 0 1
|
||||
help
|
||||
Interleaved ES8311 slot carrying the DAC playback reference.
|
||||
It must differ from the microphone channel.
|
||||
|
||||
config MULTIMODAL_AUDIO_AEC_FILTER_LENGTH
|
||||
int "AEC filter length"
|
||||
default 4
|
||||
range 1 8
|
||||
help
|
||||
ESP-SR synchronous AEC filter length. Four is the recommended
|
||||
starting point for ESP32-S3.
|
||||
|
||||
config MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
|
||||
bool "Log microphone uplink audio levels"
|
||||
default n
|
||||
|
||||
+19
-1
@@ -59,6 +59,22 @@ CONFIG_TQ_KEY_POST_DOUBLE_CLICK_SUPPRESS_MS=450
|
||||
CONFIG_TQ_PTT_MODE_CLICK_TO_TALK=y
|
||||
# CONFIG_TQ_PTT_MODE_HOLD_TO_TALK is not set
|
||||
CONFIG_TQ_PTT_CLICK_CAPTURE_TIMEOUT_MS=8000
|
||||
CONFIG_TQ_LOCAL_WAKE_ENABLE=y
|
||||
# CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE is not set
|
||||
CONFIG_TQ_LOCAL_WAKE_RESPONSE_PROMPT_FILE="/littlefs/audio_prompts/local_wake_response.opus"
|
||||
CONFIG_TQ_LOCAL_WAKE_RETRY_MS=5000
|
||||
CONFIG_TQ_LOCAL_WAKE_STOP_WAIT_MS=1000
|
||||
CONFIG_TQ_LOCAL_WAKE_TASK_STACK_BYTES=6144
|
||||
CONFIG_TQ_LOCAL_WAKE_TASK_PRIORITY=4
|
||||
CONFIG_TQ_LOCAL_WAKE_TASK_CORE=0
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_STACK_BYTES=8192
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_PRIORITY=7
|
||||
CONFIG_TQ_LOCAL_COMMAND_TASK_CORE=0
|
||||
CONFIG_TQ_LOCAL_COMMAND_CONTROL_TIMEOUT_MS=1000
|
||||
CONFIG_TQ_LOCAL_COMMAND_DETECTION_TIMEOUT_MS=6000
|
||||
CONFIG_TQ_LOCAL_COMMAND_DETECTION_THRESHOLD_X10000=1200
|
||||
CONFIG_SR_MN_CN_MULTINET7_QUANT=y
|
||||
CONFIG_SR_MN_EN_NONE=y
|
||||
|
||||
# Internal DMA-capable SRAM budget. TLS/auth/fortune flows, hardware AES DMA
|
||||
# bounce buffers, LCD DMA, and worker task stacks all need internal-only memory.
|
||||
@@ -150,7 +166,9 @@ CONFIG_MULTIMODAL_AUDIO_UPLINK_TASK_STACK=32768
|
||||
CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK=12288
|
||||
CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_LEN=24
|
||||
CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TIMEOUT_MS=5
|
||||
CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS=1
|
||||
CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS=2
|
||||
CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL=1
|
||||
CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH=4
|
||||
CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES=1024
|
||||
CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK=12288
|
||||
CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_LEN=1024
|
||||
|
||||
@@ -31,8 +31,44 @@ fi
|
||||
if rg -n "platform_multimodal_downstream_playback_event|PLATFORM_MULTIMODAL_DOWNSTREAM_PLAYBACK" components/platform/include/platform.h >/dev/null 2>&1; then
|
||||
fail "platform facade must not duplicate domain-owned multimodal semantic events"
|
||||
fi
|
||||
if rg -n "esp-sr|esp_sr|esp_afe|afe_config|afe_sr|model_path\\.h|srmodels|CONFIG_SR_|SR_WN|MULTIMODAL_WAKE_WORD|wake_word|Wake word|Wake-word|wn9_" components main sdkconfig.defaults dependencies.lock >/dev/null 2>&1; then
|
||||
fail "ESP-SR and wake-word code/dependencies must stay removed"
|
||||
if rg -n "esp_afe_sr|afe_config_init|afe_config_check|esp_afe_handle_from_config|SR_WN|MULTIMODAL_WAKE_WORD|wake_word|Wake word|Wake-word|wn9_" \
|
||||
components/platform/{include,internal,src} \
|
||||
components/control_plane/{include,internal,src} \
|
||||
components/app_composition/{include,internal,src} >/dev/null 2>&1 ||
|
||||
rg -n 'model_path\\.h|esp_srmodel_|esp_mn_' components/domain/{include,internal,src} \
|
||||
--glob '!domain_local_command.c' >/dev/null 2>&1; then
|
||||
fail "Local speech models must stay domain-owned and WakeNet/full AFE SR must stay absent"
|
||||
fi
|
||||
if ! rg -n 'espressif/esp-sr: \^2\.4\.7' components/platform/idf_component.yml >/dev/null 2>&1 ||
|
||||
! rg -n 'DOMAIN_LOCAL_COMMAND_MIAO_ZHANG_GUI' \
|
||||
components/domain/include/domain.h \
|
||||
components/domain/src/runtime/domain_local_command.c >/dev/null 2>&1 ||
|
||||
! rg -n 'miao zhang gui' \
|
||||
components/domain/src/runtime/domain_local_command.c >/dev/null 2>&1; then
|
||||
fail "Local command detection must keep the ESP-SR dependency and 喵掌柜 domain contract"
|
||||
fi
|
||||
if ! rg -n 'TQ_LOCAL_WAKE_RESPONSE_PROMPT_ENABLE' \
|
||||
main/Kconfig.projbuild \
|
||||
components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1 ||
|
||||
! rg -n '/littlefs/audio_prompts/local_wake_response\.opus' \
|
||||
main/Kconfig.projbuild \
|
||||
sdkconfig.defaults \
|
||||
components/control_plane/src/multimodal/control_plane_multimodal_local_wake.c >/dev/null 2>&1; then
|
||||
fail "Local wake response prompt must keep its disabled asset placeholder contract"
|
||||
fi
|
||||
if ! rg -n '#include "esp_afe_aec.h"' components/platform/src/multimodal/audio_pipeline.c >/dev/null 2>&1 ||
|
||||
! rg -n 'afe_aec_create\(' components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c >/dev/null 2>&1 ||
|
||||
! rg -n 'afe_aec_process\(' components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c >/dev/null 2>&1; then
|
||||
fail "Platform must own the synchronous ESP-SR AEC implementation"
|
||||
fi
|
||||
if rg -n 'esp_afe_aec\.h|afe_aec_process\(' \
|
||||
components/domain/{include,internal,src} \
|
||||
components/control_plane/{include,internal,src} \
|
||||
components/app_composition/{include,internal,src} >/dev/null 2>&1; then
|
||||
fail "AEC execution must remain a platform primitive"
|
||||
fi
|
||||
if ! rg -n '^CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS=2\r?$' sdkconfig.defaults >/dev/null 2>&1; then
|
||||
fail "ES8311 microphone and DAC reference capture requires two physical codec slots"
|
||||
fi
|
||||
if rg -n "platform_lcd_trigger_slot_animation|lcd_trigger_slot_animation" components/platform/include components/platform/internal components/platform/src/runtime >/dev/null 2>&1; then
|
||||
fail "platform must expose LCD presentation primitives, not one-off expression APIs"
|
||||
|
||||
Reference in New Issue
Block a user