chore: initial commit
This commit is contained in:
CSBOOK\44744
@@ -0,0 +1,346 @@
|
||||
#include "multimodal/audio_pipeline.h"
|
||||
|
||||
#include <limits.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "audio_codec_ctrl_if.h"
|
||||
#include "audio_codec_data_if.h"
|
||||
#include "audio_codec_gpio_if.h"
|
||||
#include "audio_codec_if.h"
|
||||
#include "es8311_codec.h"
|
||||
#include "esp_codec_dev.h"
|
||||
#include "esp_codec_dev_defaults.h"
|
||||
#include "esp_audio_dec.h"
|
||||
#include "esp_audio_enc.h"
|
||||
#include "esp_err.h"
|
||||
#include "esp_heap_caps.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_opus_dec.h"
|
||||
#include "esp_opus_enc.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/idf_additions.h"
|
||||
#include "freertos/queue.h"
|
||||
#include "freertos/task.h"
|
||||
#include "soc/soc_caps.h"
|
||||
#include "sdkconfig.h"
|
||||
#include "driver/i2s_std.h"
|
||||
#include "driver/i2c_master.h"
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_ENABLE_CODEC_PATH
|
||||
#define CONFIG_MULTIMODAL_AUDIO_ENABLE_CODEC_PATH 1
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE
|
||||
#define CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE 16000
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS
|
||||
#define CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS 1
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE
|
||||
#define CONFIG_MULTIMODAL_AUDIO_CODEC_BITS_PER_SAMPLE 16
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2C_PORT
|
||||
#define CONFIG_MULTIMODAL_AUDIO_I2C_PORT 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2C_SCL_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_I2C_SCL_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2C_SDA_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_I2C_SDA_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2C_ADDR
|
||||
#define CONFIG_MULTIMODAL_AUDIO_I2C_ADDR 0x18
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_PORT
|
||||
#define CONFIG_MULTIMODAL_AUDIO_I2S_PORT 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_MCLK_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_I2S_MCLK_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_BCLK_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_I2S_BCLK_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_WS_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_I2S_WS_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DOUT_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_I2S_DOUT_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DIN_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_I2S_DIN_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_USE_MCLK
|
||||
#define CONFIG_MULTIMODAL_AUDIO_USE_MCLK 1
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_PA_GPIO
|
||||
#error "CONFIG_MULTIMODAL_AUDIO_PA_GPIO must be defined in Kconfig"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DMA_DESC_NUM
|
||||
#define CONFIG_MULTIMODAL_AUDIO_I2S_DMA_DESC_NUM 4
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_I2S_DMA_FRAME_NUM
|
||||
#define CONFIG_MULTIMODAL_AUDIO_I2S_DMA_FRAME_NUM 160
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_OUT_VOLUME
|
||||
#define CONFIG_MULTIMODAL_AUDIO_OUT_VOLUME 60
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_IN_GAIN_DB
|
||||
#define CONFIG_MULTIMODAL_AUDIO_IN_GAIN_DB 10
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL
|
||||
#define CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS
|
||||
#define CONFIG_MULTIMODAL_AUDIO_LOG_UPLINK_LEVELS 1
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_IDLE_MUTE_MS
|
||||
#define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_IDLE_MUTE_MS 250
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK
|
||||
#define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK 12288
|
||||
#endif
|
||||
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_TASK_STACK_MIN_BYTES 8192U
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_LEN
|
||||
#define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_LEN 1024
|
||||
#endif
|
||||
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_MIN_LEN 16U
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_QUEUE_CAP_LEN 1024U
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_MAX_PACKET_BYTES
|
||||
#define CONFIG_MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_MAX_PACKET_BYTES 2048
|
||||
#endif
|
||||
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_PACKET_MIN_BYTES 256U
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_PLAYBACK_PACKET_CAP_BYTES 8192U
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_TASK_STACK
|
||||
#define CONFIG_MULTIMODAL_AUDIO_UPLINK_TASK_STACK 32768
|
||||
#endif
|
||||
|
||||
#define MULTIMODAL_AUDIO_UPLINK_TASK_STACK_MIN_BYTES 32768U
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK
|
||||
#define CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK 12288
|
||||
#endif
|
||||
|
||||
#define MULTIMODAL_AUDIO_UPLINK_SEND_TASK_STACK_MIN_BYTES 8192U
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_LEN
|
||||
#define CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_LEN 24
|
||||
#endif
|
||||
|
||||
#define MULTIMODAL_AUDIO_UPLINK_SEND_QUEUE_MIN_LEN 24U
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TIMEOUT_MS
|
||||
#define CONFIG_MULTIMODAL_AUDIO_UPLINK_SEND_TIMEOUT_MS 5
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES
|
||||
#define CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES 1024
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_INIT_RETRY_COUNT
|
||||
#define CONFIG_MULTIMODAL_AUDIO_INIT_RETRY_COUNT 1
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_DMA
|
||||
#define CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_DMA 4096
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_INTERNAL
|
||||
#define CONFIG_MULTIMODAL_AUDIO_INIT_MIN_LARGEST_INTERNAL 6144
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS
|
||||
#define CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS 60
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS
|
||||
#define CONFIG_MULTIMODAL_OPUS_BITRATE_KBPS 24
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_OPUS_COMPLEXITY
|
||||
#define CONFIG_MULTIMODAL_OPUS_COMPLEXITY 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_OPUS_ENABLE_VBR
|
||||
#define CONFIG_MULTIMODAL_OPUS_ENABLE_VBR 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_OPUS_ENABLE_DTX
|
||||
#define CONFIG_MULTIMODAL_OPUS_ENABLE_DTX 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_OPUS_ENABLE_FEC
|
||||
#define CONFIG_MULTIMODAL_OPUS_ENABLE_FEC 0
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_SAMPLE_RATE != 16000
|
||||
#error "Multimodal voice requires 16 kHz audio"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_CODEC_CHANNELS != 1
|
||||
#error "Multimodal voice requires mono audio"
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_OPUS_FRAME_DURATION_MS != 60
|
||||
#error "Multimodal voice requires 60 ms raw-opus frames"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_OPUS_DEC_SELF_DELIMITED
|
||||
#define CONFIG_MULTIMODAL_OPUS_DEC_SELF_DELIMITED 0
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_MULTIMODAL_AUDIO_TASK_CORE
|
||||
#define CONFIG_MULTIMODAL_AUDIO_TASK_CORE 1
|
||||
#endif
|
||||
|
||||
#if CONFIG_MULTIMODAL_AUDIO_TASK_CORE < 0
|
||||
#define MULTIMODAL_AUDIO_CORE_ID tskNO_AFFINITY
|
||||
#else
|
||||
#define MULTIMODAL_AUDIO_CORE_ID ((BaseType_t)CONFIG_MULTIMODAL_AUDIO_TASK_CORE)
|
||||
#endif
|
||||
|
||||
#define MULTIMODAL_PCM_SAMPLE_BYTES 2U
|
||||
#define MULTIMODAL_MIN_DECODER_PCM_BUFFER_BYTES 2048U
|
||||
#define MULTIMODAL_DECODER_BUFFER_GUARD_BYTES 512U
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_IDLE_MUTE_TASK_STACK_BYTES 4096U
|
||||
#define MULTIMODAL_AUDIO_DOWNSTREAM_TX_BOUNCE_BYTES 3200U
|
||||
#define MULTIMODAL_AUDIO_DEFAULT_MIC_CHANNEL ((uint8_t)CONFIG_MULTIMODAL_AUDIO_MIC_CHANNEL)
|
||||
|
||||
static const char *TAG = "multimodal_audio";
|
||||
static portMUX_TYPE s_audio_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||
|
||||
typedef struct {
|
||||
const audio_codec_data_if_t *data_if;
|
||||
const audio_codec_ctrl_if_t *ctrl_if;
|
||||
const audio_codec_gpio_if_t *gpio_if;
|
||||
const audio_codec_if_t *codec_if;
|
||||
esp_codec_dev_handle_t codec_dev;
|
||||
esp_codec_dev_sample_info_t fs;
|
||||
i2s_chan_handle_t tx_handle;
|
||||
i2s_chan_handle_t rx_handle;
|
||||
i2c_master_bus_handle_t i2c_bus;
|
||||
uint8_t i2c_probe_addr_7bit;
|
||||
uint8_t i2c_ctrl_addr_8bit;
|
||||
esp_audio_enc_handle_t opus_enc;
|
||||
esp_audio_dec_handle_t opus_dec;
|
||||
int opus_enc_in_bytes;
|
||||
int opus_enc_out_bytes;
|
||||
uint32_t opus_sample_rate;
|
||||
uint32_t opus_frame_duration_ms;
|
||||
uint32_t opus_bitrate_bps;
|
||||
uint8_t *uplink_codec_buf;
|
||||
int16_t *uplink_mono_buf;
|
||||
uint8_t *uplink_opus_buf;
|
||||
size_t uplink_codec_read_bytes;
|
||||
size_t uplink_mono_bytes;
|
||||
size_t uplink_mono_buf_capacity_bytes;
|
||||
size_t uplink_mono_fill_bytes;
|
||||
size_t uplink_opus_buf_bytes;
|
||||
uint8_t *downstream_pcm_buf;
|
||||
size_t downstream_pcm_buf_bytes;
|
||||
int16_t *downstream_codec_out_buf;
|
||||
size_t downstream_codec_out_buf_bytes;
|
||||
uint8_t *downstream_tx_bounce_buf;
|
||||
size_t downstream_tx_bounce_buf_bytes;
|
||||
uint32_t downstream_playback_generation;
|
||||
uint32_t downstream_audio_activity_generation;
|
||||
volatile bool downstream_decoder_reset_pending;
|
||||
bool downstream_playback_in_progress;
|
||||
bool downstream_output_muted;
|
||||
bool downstream_output_mute_state_known;
|
||||
bool downstream_output_primed;
|
||||
bool downstream_response_active;
|
||||
bool downstream_response_end_received;
|
||||
TaskHandle_t downstream_idle_mute_task;
|
||||
bool downstream_idle_mute_stop_requested;
|
||||
QueueHandle_t downstream_playback_queue;
|
||||
bool downstream_playback_queue_with_caps;
|
||||
TaskHandle_t downstream_playback_task;
|
||||
bool downstream_playback_starting;
|
||||
volatile bool downstream_playback_stop_requested;
|
||||
bool downstream_playback_stopping;
|
||||
bool downstream_playback_keep_stopping_on_exit;
|
||||
bool audio_deinit_in_progress;
|
||||
uint8_t mic_capture_channel;
|
||||
QueueHandle_t uplink_send_queue;
|
||||
bool uplink_send_queue_with_caps;
|
||||
TaskHandle_t uplink_task;
|
||||
TaskHandle_t uplink_send_task;
|
||||
volatile bool uplink_stop_requested;
|
||||
multimodal_audio_uplink_cb_t uplink_cb;
|
||||
void *uplink_cb_ctx;
|
||||
bool opus_ready;
|
||||
bool initialized;
|
||||
} multimodal_audio_ctx_t;
|
||||
|
||||
static multimodal_audio_ctx_t s_audio;
|
||||
static multimodal_audio_downstream_playback_callback_t s_downstream_playback_cb;
|
||||
static void *s_downstream_playback_cb_ctx;
|
||||
static bool s_uplink_stack_warned;
|
||||
static bool s_uplink_send_stack_warned;
|
||||
static bool s_uplink_send_queue_warned;
|
||||
static bool s_downstream_playback_stack_warned;
|
||||
static bool s_downstream_playback_queue_warned;
|
||||
static bool s_downstream_playback_packet_warned;
|
||||
static uint32_t s_uplink_overflow_count;
|
||||
static TickType_t s_uplink_overflow_last_log_tick;
|
||||
static uint32_t s_downstream_overflow_count;
|
||||
static TickType_t s_downstream_overflow_last_log_tick;
|
||||
static uint32_t s_downstream_queued_packet_total;
|
||||
static uint32_t s_downstream_played_packet_total;
|
||||
static SemaphoreHandle_t s_downstream_playback_queue_mutex;
|
||||
static bool s_downstream_playback_queue_mutex_initializing;
|
||||
#if ( configSUPPORT_STATIC_ALLOCATION == 1 )
|
||||
static StaticSemaphore_t s_downstream_playback_queue_mutex_buf;
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
uint16_t len;
|
||||
uint8_t data[CONFIG_MULTIMODAL_AUDIO_UPLINK_MAX_PACKET_BYTES];
|
||||
} uplink_packet_t;
|
||||
|
||||
typedef struct {
|
||||
size_t len;
|
||||
uint8_t *data;
|
||||
} downstream_packet_t;
|
||||
|
||||
/* Split implementation by concern while keeping a single translation unit:
|
||||
* - helpers: memory, config normalization, utility helpers
|
||||
* - codec: codec/i2c/i2s/opus setup and teardown
|
||||
* - runtime: uplink/downlink processing and public APIs
|
||||
*/
|
||||
|
||||
#include "audio_pipeline/audio_pipeline_helpers_impl.c"
|
||||
#include "audio_pipeline/audio_pipeline_codec_impl.c"
|
||||
#include "audio_pipeline/audio_debug_dump_impl.c"
|
||||
#include "audio_pipeline/audio_pipeline_runtime_impl.c"
|
||||
Reference in New Issue
Block a user