chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(voice): add on-device Miao Zhang Gui wake word

Browse Source 
Add an optional local wake response prompt configuration. The prompt asset is intentionally not included yet.
This commit is contained in:
chenjintao
2026-08-10 16:54:42 +08:00
parent d9660164a6
commit e5c1564130
21 changed files with 2010 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/platform/idf_component.yml
+1
View File
@@ -4,5 +4,6 @@ dependencies:
espressif/esp_websocket_client: ^1.6.1
espressif/esp_codec_dev: ^1.5.4
espressif/esp_audio_codec: ^2.4.1
espressif/esp-sr: ^2.4.7
espressif/mdns: ^1.11.0
78/esp-wifi-connect: ^3.1.4
components/platform/include/platform.h
+24
View File
@@ -49,6 +49,12 @@ typedef struct {
void *user_data;
} platform_multimodal_downstream_playback_callbacks_t;
typedef void (*platform_multimodal_audio_pcm_observer_t)(
const int16_t *pcm,
size_t samples,
uint32_t sample_rate_hz,
void *user_data);
#define PLATFORM_LAN_HTTP_PATH_MAX_LEN 96
#define PLATFORM_LAN_HTTP_HOSTNAME_MAX_LEN 32
#define PLATFORM_LAN_HTTP_REQUEST_BODY_MAX_LEN 1024
@@ -321,6 +327,24 @@ esp_err_t platform_multimodal_audio_start_uplink(
esp_err_t platform_multimodal_audio_stop_uplink(int timeout_ms);
bool platform_multimodal_audio_is_uplink_running(void);
bool platform_multimodal_audio_is_initialized(void);
/**
* @brief Register the single mono PCM observer for the active capture path.
*
* The callback runs synchronously in the platform capture task, outside the
* registration lock, and must remain bounded and non-blocking.
*/
esp_err_t platform_multimodal_audio_register_pcm_observer(
platform_multimodal_audio_pcm_observer_t observer,
void *user_data);
/**
* @brief Remove a matching PCM observer and wait for in-flight callbacks.
*
* Negative timeout is invalid. Zero performs a non-blocking drain check.
*/
esp_err_t platform_multimodal_audio_unregister_pcm_observer(
platform_multimodal_audio_pcm_observer_t observer,
void *user_data,
int timeout_ms);
esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len);
esp_err_t platform_multimodal_audio_begin_downstream_response(void);
esp_err_t platform_multimodal_audio_finish_downstream_response(void);
components/platform/internal/multimodal/audio_pipeline.h
+11
View File
@@ -11,6 +11,10 @@ extern "C" {
#endif
typedef esp_err_t (*multimodal_audio_uplink_cb_t)(const uint8_t *data, size_t len, void *ctx);
typedef void (*multimodal_audio_pcm_observer_t)(const int16_t *pcm,
size_t samples,
uint32_t sample_rate_hz,
void *ctx);
typedef enum {
MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_STARTED = 0,
@@ -33,6 +37,13 @@ esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *c
esp_err_t multimodal_audio_stop_uplink(int timeout_ms);
bool multimodal_audio_is_uplink_running(void);
bool multimodal_audio_is_initialized(void);
esp_err_t multimodal_audio_register_pcm_observer(
multimodal_audio_pcm_observer_t observer,
void *observer_ctx);
esp_err_t multimodal_audio_unregister_pcm_observer(
multimodal_audio_pcm_observer_t observer,
void *observer_ctx,
int timeout_ms);
esp_err_t multimodal_audio_handle_downstream_opus(const uint8_t *data, size_t len);
esp_err_t multimodal_audio_begin_downstream_response(void);
components/platform/src/multimodal/audio_pipeline.c
+31 -2
View File
@@ -15,6 +15,7 @@
#include "esp_codec_dev_defaults.h"
#include "esp_audio_dec.h"
#include "esp_audio_enc.h"
#include "esp_afe_aec.h"
#include "esp_err.h"
#include "esp_heap_caps.h"
#include "esp_log.h"
@@ -114,6 +115,14 @@
#define CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL 0
#endif
#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL
#define CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL 1
#endif
#ifndef CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH
#define CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH 4
#endif
#ifndef CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
#define CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS 1
#endif
@@ -208,8 +217,16 @@
#error "Multimodal voice requires 16 kHz audio"
#endif
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1
#error "Multimodal voice requires mono audio"
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 2
#error "ES8311 local wake AEC requires two physical codec slots"
#endif
#if CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE != 16
#error "ESP AEC requires signed 16-bit PCM"
#endif
#if CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL == CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL
#error "Microphone and AEC reference must use distinct codec slots"
#endif
#if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60
@@ -235,6 +252,9 @@
#define MULTIMODAL_DECODER_BUFFER_GUARD_BYTES 512U
#define MULTIMODAL_AUDIO_DOWNSTREAM_TX_BOUNCE_BYTES 3200U
#define MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL ((uint8_t)CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL)
#define MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL \
((uint8_t)CONFIG_MULTIMODAL_AUDIO_AEC_REFERENCE_CHANNEL)
#define MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT 16U
static const char *TAG = "multimodal_audio";
static portMUX_TYPE s_audio_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
@@ -253,6 +273,7 @@ typedef struct {
uint8_t i2c_ctrl_addr_8bit;
esp_audio_enc_handle_t opus_enc;
esp_audio_dec_handle_t opus_dec;
afe_aec_handle_t *afe_aec_handle;
int opus_enc_in_bytes;
int opus_enc_out_bytes;
uint32_t opus_sample_rate;
@@ -260,8 +281,11 @@ typedef struct {
uint32_t opus_bitrate_bps;
uint8_t *uplink_codec_buf;
int16_t *uplink_mono_buf;
int16_t *uplink_aec_clean_buf;
uint8_t *uplink_opus_buf;
size_t uplink_codec_read_bytes;
size_t uplink_aec_frame_bytes;
size_t uplink_aec_frame_samples;
size_t uplink_mono_bytes;
size_t uplink_mono_buf_capacity_bytes;
size_t uplink_mono_fill_bytes;
@@ -290,6 +314,7 @@ typedef struct {
bool downstream_playback_keep_stopping_on_exit;
bool audio_deinit_in_progress;
uint8_t mic_capture_channel;
uint8_t aec_reference_channel;
QueueHandle_t uplink_send_queue;
bool uplink_send_queue_with_caps;
TaskHandle_t uplink_task;
@@ -306,6 +331,10 @@ typedef struct {
static multimodal_audio_ctx_t s_audio;
static multimodal_audio_downstream_playback_callback_t s_downstream_playback_cb;
static void *s_downstream_playback_cb_ctx;
static multimodal_audio_pcm_observer_t s_pcm_observer;
static void *s_pcm_observer_ctx;
static uint32_t s_pcm_observer_in_flight;
static portMUX_TYPE s_pcm_observer_lock = portMUX_INITIALIZER_UNLOCKED;
static bool s_uplink_stack_warned;
static bool s_uplink_send_stack_warned;
static bool s_uplink_send_queue_warned;
components/platform/src/multimodal/audio_pipeline/audio_pipeline_codec_impl.c
+61 -25
View File
@@ -50,10 +50,17 @@ static void cleanup_opus_path(multimodal_audio_ctx_t *ctx)
ctx->opus_dec = NULL;
}
if (ctx->afe_aec_handle != NULL) {
afe_aec_destroy(ctx->afe_aec_handle);
ctx->afe_aec_handle = NULL;
}
free(ctx->uplink_codec_buf);
ctx->uplink_codec_buf = NULL;
free(ctx->uplink_mono_buf);
ctx->uplink_mono_buf = NULL;
free(ctx->uplink_aec_clean_buf);
ctx->uplink_aec_clean_buf = NULL;
free(ctx->uplink_opus_buf);
ctx->uplink_opus_buf = NULL;
free(ctx->downstream_pcm_buf);
@@ -64,6 +71,8 @@ static void cleanup_opus_path(multimodal_audio_ctx_t *ctx)
ctx->downstream_tx_bounce_buf = NULL;
ctx->uplink_codec_read_bytes = 0;
ctx->uplink_aec_frame_bytes = 0;
ctx->uplink_aec_frame_samples = 0;
ctx->uplink_mono_bytes = 0;
ctx->uplink_mono_buf_capacity_bytes = 0;
ctx->uplink_mono_fill_bytes = 0;
@@ -566,15 +575,54 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx)
return audio_err_to_esp_err(audio_err);
}
const char *aec_input_format = NULL;
if (ctx->mic_capture_channel == 0U &&
ctx->aec_reference_channel == 1U) {
aec_input_format = "MR";
} else if (ctx->mic_capture_channel == 1U &&
ctx->aec_reference_channel == 0U) {
aec_input_format = "RM";
} else {
ESP_LOGE(TAG,
"Unsupported AEC slot map: mic=%u ref=%u",
(unsigned)ctx->mic_capture_channel,
(unsigned)ctx->aec_reference_channel);
return ESP_ERR_NOT_SUPPORTED;
}
ctx->afe_aec_handle = afe_aec_create(
aec_input_format,
CONFIG_MULTIMODAL_AUDIO_AEC_FILTER_LENGTH,
AFE_TYPE_FD,
AFE_MODE_LOW_COST);
if (ctx->afe_aec_handle == NULL) {
ESP_LOGE(TAG, "Create ESP-SR AEC failed");
return ESP_ERR_NO_MEM;
}
if (ctx->afe_aec_handle->handle != NULL) {
(void)aec_set_nlp_level(
ctx->afe_aec_handle->handle,
AEC_NLP_LEVEL_AGGR);
}
int aec_frame_samples = afe_aec_get_chunksize(ctx->afe_aec_handle);
if (aec_frame_samples <= 0) {
ESP_LOGE(TAG, "Invalid ESP-SR AEC frame size: %d", aec_frame_samples);
return ESP_ERR_INVALID_SIZE;
}
ctx->uplink_aec_frame_samples = (size_t)aec_frame_samples;
ctx->uplink_aec_frame_bytes =
ctx->uplink_aec_frame_samples * sizeof(int16_t);
ctx->uplink_mono_bytes = (size_t)ctx->opus_enc_in_bytes;
if (ctx->uplink_mono_bytes == 0 || ctx->fs.channel == 0U) {
return ESP_ERR_INVALID_SIZE;
}
if (ctx->uplink_mono_bytes > (SIZE_MAX / (size_t)ctx->fs.channel)) {
if (ctx->uplink_aec_frame_bytes >
(SIZE_MAX / (size_t)ctx->fs.channel)) {
return ESP_ERR_INVALID_SIZE;
}
size_t codec_read_bytes = ctx->uplink_mono_bytes * (size_t)ctx->fs.channel;
size_t codec_read_bytes =
ctx->uplink_aec_frame_bytes * (size_t)ctx->fs.channel;
ctx->uplink_codec_read_bytes = codec_read_bytes;
ctx->uplink_opus_buf_bytes = (size_t)ctx->opus_enc_out_bytes;
if (ctx->uplink_opus_buf_bytes < 256U) {
@@ -586,8 +634,14 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx)
ctx->uplink_codec_buf = (uint8_t *)audio_psram_first_calloc(1, ctx->uplink_codec_read_bytes);
ctx->uplink_mono_buf = (int16_t *)audio_psram_first_calloc(1, ctx->uplink_mono_buf_capacity_bytes);
ctx->uplink_aec_clean_buf = (int16_t *)heap_caps_aligned_calloc(
MULTIMODAL_AUDIO_AEC_BUFFER_ALIGNMENT,
1,
ctx->uplink_aec_frame_bytes,
MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
ctx->uplink_opus_buf = (uint8_t *)audio_psram_first_calloc(1, ctx->uplink_opus_buf_bytes);
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf) {
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf ||
!ctx->uplink_aec_clean_buf || !ctx->uplink_opus_buf) {
return ESP_ERR_NO_MEM;
}
@@ -601,38 +655,20 @@ static esp_err_t init_opus_codec(multimodal_audio_ctx_t *ctx)
}
ESP_LOGI(TAG,
"Opus ready: %uHz frame=%ums bitrate=%ukbps enc(in=%d out=%d)",
"Opus/AEC ready: %uHz opus_frame=%ums bitrate=%ukbps enc(in=%d out=%d) aec_frame=%u input=%s",
(unsigned)ctx->opus_sample_rate,
(unsigned)ctx->opus_frame_duration_ms,
(unsigned)bitrate_kbps,
ctx->opus_enc_in_bytes,
ctx->opus_enc_out_bytes);
ctx->opus_enc_out_bytes,
(unsigned)ctx->uplink_aec_frame_samples,
aec_input_format);
ctx->opus_ready = true;
log_heap_snapshot("after_opus_init");
return ESP_OK;
}
static void extract_mic_channel_16(const int16_t *input,
uint8_t channels,
uint8_t mic_channel,
int16_t *output,
size_t frames)
{
if (!input || !output || channels == 0 || mic_channel >= channels) {
return;
}
if (channels == 1) {
memcpy(output, input, frames * sizeof(int16_t));
return;
}
for (size_t frame = 0; frame < frames; ++frame) {
output[frame] = input[frame * channels + mic_channel];
}
}
static void expand_mono_to_multichannel_16(const int16_t *input,
uint8_t channels,
int16_t *output,
components/platform/src/multimodal/audio_pipeline/audio_pipeline_helpers_impl.c
+8 -1
View File
@@ -207,6 +207,8 @@ static void reset_capture_channel_map(multimodal_audio_ctx_t *ctx)
uint8_t mic_channel = MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL > 0U ? 1U : 0U;
ctx->mic_capture_channel = mic_channel;
ctx->aec_reference_channel =
MULTIMODAL_AUDIO_DEFAULT_AEC_REFERENCE_CHANNEL > 0U ? 1U : 0U;
}
static void fill_codec_sample_info(multimodal_audio_ctx_t *ctx)
@@ -220,9 +222,14 @@ static void fill_codec_sample_info(multimodal_audio_ctx_t *ctx)
ctx->fs.channel_mask = 0;
ctx->fs.mclk_multiple = 0;
reset_capture_channel_map(ctx);
if (ctx->fs.channel == 1U || ctx->mic_capture_channel >= ctx->fs.channel) {
if (ctx->mic_capture_channel >= ctx->fs.channel) {
ctx->mic_capture_channel = 0U;
}
if (ctx->aec_reference_channel >= ctx->fs.channel ||
ctx->aec_reference_channel == ctx->mic_capture_channel) {
ctx->aec_reference_channel =
ctx->mic_capture_channel == 0U ? 1U : 0U;
}
}
static bool is_supported_opus_sample_rate(uint32_t sample_rate)
components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c
+176 -18
View File
@@ -157,6 +157,34 @@ static void audio_set_uplink_callback(multimodal_audio_ctx_t *ctx,
portEXIT_CRITICAL(&s_audio_runtime_lock);
}
static void notify_pcm_observer(const int16_t *pcm, size_t samples)
{
multimodal_audio_pcm_observer_t observer = NULL;
void *observer_ctx = NULL;
portENTER_CRITICAL(&s_pcm_observer_lock);
observer = s_pcm_observer;
observer_ctx = s_pcm_observer_ctx;
if (observer != NULL) {
++s_pcm_observer_in_flight;
}
portEXIT_CRITICAL(&s_pcm_observer_lock);
if (observer == NULL) {
return;
}
observer(pcm,
samples,
(uint32_t)CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE,
observer_ctx);
portENTER_CRITICAL(&s_pcm_observer_lock);
if (s_pcm_observer_in_flight > 0U) {
--s_pcm_observer_in_flight;
}
portEXIT_CRITICAL(&s_pcm_observer_lock);
}
static void audio_snapshot_uplink_tasks(const multimodal_audio_ctx_t *ctx,
TaskHandle_t *out_capture_task,
TaskHandle_t *out_send_task)
@@ -610,6 +638,47 @@ static esp_err_t encode_and_queue_mono_pcm(multimodal_audio_ctx_t *ctx,
return ESP_OK;
}
static esp_err_t append_and_queue_mono_pcm(
multimodal_audio_ctx_t *ctx,
const int16_t *pcm,
size_t pcm_bytes)
{
if (ctx == NULL || pcm == NULL || pcm_bytes == 0U ||
ctx->uplink_mono_buf == NULL ||
ctx->uplink_mono_buf_capacity_bytes != ctx->uplink_mono_bytes) {
return ESP_ERR_INVALID_ARG;
}
const uint8_t *input = (const uint8_t *)pcm;
size_t remaining = pcm_bytes;
while (remaining > 0U) {
size_t available =
ctx->uplink_mono_buf_capacity_bytes -
ctx->uplink_mono_fill_bytes;
size_t copy_bytes = remaining < available ? remaining : available;
memcpy((uint8_t *)ctx->uplink_mono_buf +
ctx->uplink_mono_fill_bytes,
input,
copy_bytes);
ctx->uplink_mono_fill_bytes += copy_bytes;
input += copy_bytes;
remaining -= copy_bytes;
if (ctx->uplink_mono_fill_bytes ==
ctx->uplink_mono_buf_capacity_bytes) {
esp_err_t err = encode_and_queue_mono_pcm(
ctx,
ctx->uplink_mono_buf,
ctx->uplink_mono_bytes);
ctx->uplink_mono_fill_bytes = 0U;
if (err != ESP_OK) {
return err;
}
}
}
return ESP_OK;
}
static void uplink_send_task(void *arg)
{
(void)arg;
@@ -666,9 +735,13 @@ static void uplink_capture_task(void *arg)
(void)arg;
multimodal_audio_ctx_t *ctx = &s_audio;
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf ||
!ctx->opus_enc || ctx->uplink_mono_bytes == 0U || ctx->uplink_codec_read_bytes == 0U) {
ESP_LOGE(TAG, "Uplink Opus buffers not ready");
if (!ctx->uplink_codec_buf || !ctx->uplink_mono_buf ||
!ctx->uplink_aec_clean_buf || !ctx->uplink_opus_buf ||
!ctx->opus_enc || !ctx->afe_aec_handle ||
ctx->uplink_mono_bytes == 0U ||
ctx->uplink_codec_read_bytes == 0U ||
ctx->uplink_aec_frame_samples == 0U) {
ESP_LOGE(TAG, "Uplink Opus/AEC buffers not ready");
audio_set_uplink_capture_task(ctx, NULL);
delete_audio_task_with_caps(NULL);
return;
@@ -681,13 +754,14 @@ static void uplink_capture_task(void *arg)
}
ESP_LOGI(TAG,
"Mic uplink capture task started (stack=%uB opus in=%uB out=%uB read=%uB channels=%u mic_ch=%u)",
"Mic uplink capture task started (stack=%uB opus in=%uB out=%uB read=%uB channels=%u mic_ch=%u ref_ch=%u)",
(unsigned)get_uplink_task_stack_bytes(),
(unsigned)ctx->uplink_mono_bytes,
(unsigned)ctx->uplink_opus_buf_bytes,
(unsigned)ctx->uplink_codec_read_bytes,
(unsigned)ctx->fs.channel,
(unsigned)ctx->mic_capture_channel);
(unsigned)ctx->mic_capture_channel,
(unsigned)ctx->aec_reference_channel);
while (!audio_get_uplink_task_exit_requested(ctx)) {
if (!audio_get_uplink_recording_active(ctx)) {
@@ -707,19 +781,37 @@ static void uplink_capture_task(void *arg)
continue;
}
size_t uplink_frames = ctx->uplink_mono_bytes / sizeof(int16_t);
extract_mic_channel_16((const int16_t *)ctx->uplink_codec_buf,
ctx->fs.channel,
ctx->mic_capture_channel,
ctx->uplink_mono_buf,
uplink_frames);
audio_debug_dump_pcm(ctx->uplink_mono_buf, ctx->uplink_mono_bytes);
esp_err_t err = encode_and_queue_mono_pcm(ctx, ctx->uplink_mono_buf, ctx->uplink_mono_bytes);
size_t clean_bytes = afe_aec_process(
ctx->afe_aec_handle,
(const int16_t *)ctx->uplink_codec_buf,
ctx->uplink_aec_clean_buf);
if (clean_bytes != ctx->uplink_aec_frame_bytes) {
ESP_LOGW(TAG,
"AEC output size=%u expected=%u",
(unsigned)clean_bytes,
(unsigned)ctx->uplink_aec_frame_bytes);
ctx->uplink_mono_fill_bytes = 0U;
continue;
}
size_t uplink_frames = ctx->uplink_aec_frame_samples;
notify_pcm_observer(ctx->uplink_aec_clean_buf, uplink_frames);
audio_debug_dump_pcm(ctx->uplink_aec_clean_buf, clean_bytes);
multimodal_audio_uplink_cb_t uplink_cb = NULL;
audio_snapshot_uplink_callback(ctx, &uplink_cb, NULL);
esp_err_t err = ESP_OK;
if (uplink_cb != NULL) {
err = append_and_queue_mono_pcm(
ctx,
ctx->uplink_aec_clean_buf,
clean_bytes);
} else {
ctx->uplink_mono_fill_bytes = 0U;
}
maybe_log_uplink_levels(ctx,
(const int16_t *)ctx->uplink_codec_buf,
uplink_frames,
ctx->uplink_mono_buf,
ctx->uplink_mono_bytes);
ctx->uplink_aec_clean_buf,
clean_bytes);
if (err != ESP_OK && err != ESP_ERR_INVALID_SIZE && err != ESP_ERR_TIMEOUT) {
ESP_LOGW(TAG, "Encode uplink PCM failed: %s", esp_err_to_name(err));
vTaskDelay(pdMS_TO_TICKS(10));
@@ -739,7 +831,9 @@ static esp_err_t ensure_uplink_worker_tasks(multimodal_audio_ctx_t *ctx)
return ESP_ERR_INVALID_ARG;
}
if (!ctx->initialized || !ctx->codec_dev || !ctx->opus_enc ||
!ctx->uplink_codec_buf || !ctx->uplink_mono_buf || !ctx->uplink_opus_buf) {
!ctx->afe_aec_handle || !ctx->uplink_codec_buf ||
!ctx->uplink_mono_buf || !ctx->uplink_aec_clean_buf ||
!ctx->uplink_opus_buf) {
return ESP_ERR_INVALID_STATE;
}
@@ -970,18 +1064,82 @@ esp_err_t multimodal_audio_register_downstream_playback_callback(
return ESP_OK;
}
esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *cb_ctx)
esp_err_t multimodal_audio_register_pcm_observer(
multimodal_audio_pcm_observer_t observer,
void *observer_ctx)
{
if (!cb) {
if (observer == NULL) {
return ESP_ERR_INVALID_ARG;
}
esp_err_t result = ESP_OK;
portENTER_CRITICAL(&s_pcm_observer_lock);
if (s_pcm_observer != NULL &&
(s_pcm_observer != observer || s_pcm_observer_ctx != observer_ctx)) {
result = ESP_ERR_INVALID_STATE;
} else {
s_pcm_observer = observer;
s_pcm_observer_ctx = observer_ctx;
}
portEXIT_CRITICAL(&s_pcm_observer_lock);
return result;
}
esp_err_t multimodal_audio_unregister_pcm_observer(
multimodal_audio_pcm_observer_t observer,
void *observer_ctx,
int timeout_ms)
{
if (observer == NULL || timeout_ms < 0) {
return ESP_ERR_INVALID_ARG;
}
portENTER_CRITICAL(&s_pcm_observer_lock);
if (s_pcm_observer == NULL) {
portEXIT_CRITICAL(&s_pcm_observer_lock);
return ESP_OK;
}
if (s_pcm_observer != observer || s_pcm_observer_ctx != observer_ctx) {
portEXIT_CRITICAL(&s_pcm_observer_lock);
return ESP_ERR_INVALID_STATE;
}
s_pcm_observer = NULL;
s_pcm_observer_ctx = NULL;
portEXIT_CRITICAL(&s_pcm_observer_lock);
TickType_t started = xTaskGetTickCount();
TickType_t timeout_ticks = pdMS_TO_TICKS((uint32_t)timeout_ms);
if (timeout_ms > 0 && timeout_ticks == 0U) {
timeout_ticks = 1U;
}
for (;;) {
uint32_t in_flight = 0U;
portENTER_CRITICAL(&s_pcm_observer_lock);
in_flight = s_pcm_observer_in_flight;
portEXIT_CRITICAL(&s_pcm_observer_lock);
if (in_flight == 0U) {
return ESP_OK;
}
if (timeout_ms == 0 ||
(xTaskGetTickCount() - started) >= timeout_ticks) {
return ESP_ERR_TIMEOUT;
}
vTaskDelay(1U);
}
}
esp_err_t multimodal_audio_start_uplink(multimodal_audio_uplink_cb_t cb, void *cb_ctx)
{
if (!s_audio.initialized || !s_audio.codec_dev || !s_audio.opus_enc ||
!s_audio.uplink_codec_buf || !s_audio.uplink_mono_buf || !s_audio.uplink_opus_buf) {
return ESP_ERR_INVALID_STATE;
}
bool already_active = audio_get_uplink_recording_active(&s_audio);
audio_set_uplink_callback(&s_audio, cb, cb_ctx);
if (already_active) {
return ESP_OK;
}
s_audio.uplink_mono_fill_bytes = 0U;
memset(s_audio.uplink_mono_buf, 0, s_audio.uplink_mono_buf_capacity_bytes);
reset_uplink_debug_stats();
components/platform/src/runtime/platform_multimodal_runtime.c
+18
View File
@@ -177,6 +177,24 @@ bool platform_multimodal_audio_is_initialized(void)
return multimodal_audio_is_initialized();
}
esp_err_t platform_multimodal_audio_register_pcm_observer(
platform_multimodal_audio_pcm_observer_t observer,
void *user_data)
{
return multimodal_audio_register_pcm_observer(observer, user_data);
}
esp_err_t platform_multimodal_audio_unregister_pcm_observer(
platform_multimodal_audio_pcm_observer_t observer,
void *user_data,
int timeout_ms)
{
return multimodal_audio_unregister_pcm_observer(
observer,
user_data,
timeout_ms);
}
esp_err_t platform_multimodal_audio_play_downstream_opus(const uint8_t *data, size_t len)
{
return multimodal_audio_handle_downstream_opus(data, len);