150 lines
4.5 KiB
C
150 lines
4.5 KiB
C
#pragma once
|
|
|
|
#include <stdbool.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "esp_opus_dec.h"
|
|
#include "esp_opus_enc.h"
|
|
#include "esp_websocket_client.h"
|
|
#include "freertos/FreeRTOS.h"
|
|
#include "freertos/queue.h"
|
|
#include "freertos/semphr.h"
|
|
#include "freertos/task.h"
|
|
#include "domain.h"
|
|
#include "platform.h"
|
|
|
|
#define VOICE_STREAMING_MODE "duplex"
|
|
#define VOICE_TASK_GROUP "aigc"
|
|
#define VOICE_TASK_NAME "multimodal-generation"
|
|
#define VOICE_FUNCTION_NAME "generation"
|
|
#define VOICE_MODEL_NAME "multimodal-dialog"
|
|
|
|
#define VOICE_SAMPLE_BYTES 2
|
|
#define VOICE_STATUS_LOCK_TIMEOUT_MS 1000
|
|
#define VOICE_WS_SEND_TIMEOUT_MS 3000
|
|
#define VOICE_WS_TASK_STACK 24576
|
|
#define VOICE_UPLINK_TASK_STACK 32768
|
|
#define VOICE_HEARTBEAT_TASK_STACK 4096
|
|
#define VOICE_START_CONNECT_TIMEOUT_MS 8000
|
|
#define VOICE_AUDIO_IO_TIMEOUT_MS 1200
|
|
#define VOICE_AUDIO_DRAIN_MARGIN_MS 120
|
|
#define VOICE_AUDIO_DRAIN_QUIET_MS 180
|
|
#define VOICE_AUDIO_DRAIN_WAIT_MS 2200
|
|
#define VOICE_AUDIO_DRAIN_WAIT_ON_STOP_MS 3500
|
|
#define VOICE_WS_TEXT_QUEUE_LEN 16
|
|
#define VOICE_WS_TEXT_QUEUE_RX_TIMEOUT_MS 100
|
|
|
|
typedef struct {
|
|
char *text;
|
|
int len;
|
|
} voice_ws_text_msg_t;
|
|
|
|
typedef struct {
|
|
SemaphoreHandle_t lock;
|
|
esp_websocket_client_handle_t ws;
|
|
QueueHandle_t ws_text_queue;
|
|
|
|
TaskHandle_t uplink_task;
|
|
TaskHandle_t heartbeat_task;
|
|
TaskHandle_t ws_text_task;
|
|
bool uplink_task_with_caps;
|
|
bool heartbeat_task_with_caps;
|
|
bool ws_text_task_with_caps;
|
|
|
|
void *opus_enc;
|
|
void *opus_dec;
|
|
int opus_enc_in_size;
|
|
int opus_enc_out_size;
|
|
|
|
uint8_t *opus_tx_buf;
|
|
int16_t *pcm_tx_buf;
|
|
|
|
uint8_t *pcm_rx_buf;
|
|
size_t pcm_rx_buf_size;
|
|
|
|
uint8_t *text_agg;
|
|
size_t text_agg_size;
|
|
|
|
uint8_t *bin_agg;
|
|
size_t bin_agg_size;
|
|
|
|
bool ready;
|
|
bool session_active;
|
|
bool ws_connected;
|
|
bool started;
|
|
bool tap_active;
|
|
bool speech_active;
|
|
bool stop_requested;
|
|
bool ws_text_shutdown;
|
|
bool ws_text_busy;
|
|
bool response_first_sentence_checked;
|
|
|
|
voice_dialog_state_t dialog_state;
|
|
|
|
char task_id[40];
|
|
char dialog_id[40];
|
|
char device_uuid[40];
|
|
char last_error[128];
|
|
|
|
int64_t last_event_ms;
|
|
int64_t last_downstream_ms;
|
|
int64_t playback_deadline_ms;
|
|
uint32_t upstream_packets;
|
|
uint32_t downstream_packets;
|
|
} voice_context_t;
|
|
|
|
extern voice_context_t s_voice;
|
|
|
|
int64_t voice_now_ms(void);
|
|
void voice_fill_err(char *err, size_t err_len, const char *msg);
|
|
void voice_set_last_error_locked(const char *msg);
|
|
bool voice_lock(uint32_t timeout_ms);
|
|
void voice_unlock(void);
|
|
void voice_wait_audio_playback_done(uint32_t timeout_ms, const char *stage);
|
|
void voice_close_audio_with_drain(uint32_t timeout_ms, const char *stage);
|
|
|
|
void *voice_malloc_prefer_psram(size_t size);
|
|
void *voice_calloc_prefer_psram(size_t n, size_t size);
|
|
void *voice_realloc_prefer_psram(void *ptr, size_t size);
|
|
|
|
BaseType_t voice_create_task_prefer_psram(TaskFunction_t task_fn,
|
|
const char *name,
|
|
uint32_t stack_size,
|
|
UBaseType_t priority,
|
|
TaskHandle_t *out_task,
|
|
bool *out_with_caps);
|
|
void voice_delete_task(TaskHandle_t task, bool with_caps);
|
|
void voice_delete_self_task(bool with_caps);
|
|
|
|
void voice_make_uuid(char out[40]);
|
|
void voice_make_device_uuid(char out[40]);
|
|
const char *voice_safe_user_id(void);
|
|
|
|
esp_err_t voice_setup_opus(char *err, size_t err_len);
|
|
esp_err_t voice_open_audio_for_dialog(char *err, size_t err_len);
|
|
void voice_release_runtime_resources(void);
|
|
|
|
esp_err_t voice_send_directive(const char *action,
|
|
const char *directive,
|
|
bool with_dialog_id);
|
|
|
|
void voice_ws_protocol_handle_text_message(const char *text, size_t len);
|
|
void voice_ws_audio_handle_downstream_packet(const uint8_t *data, size_t len);
|
|
void voice_marker_try_trigger_image_generation(const char *dialog_id, const char *final_text);
|
|
bool voice_marker_image_generation_is_running(void);
|
|
|
|
void voice_websocket_event_handler(void *handler_args,
|
|
esp_event_base_t base,
|
|
int32_t event_id,
|
|
void *event_data);
|
|
|
|
void voice_interaction_ws_init_cjson_hooks(void);
|
|
esp_err_t voice_interaction_try_send_speech(char *err, size_t err_len);
|
|
|
|
void voice_uplink_task(void *arg);
|
|
void voice_heartbeat_task(void *arg);
|
|
void voice_ws_text_task(void *arg);
|
|
|
|
esp_err_t voice_interaction_mic_key_init(void);
|