444 lines
13 KiB
C
444 lines
13 KiB
C
#include <stdlib.h>
|
||
#include <string.h>
|
||
#include <librws.h>
|
||
#include "cJSON.h"
|
||
#include "app_inc.h"
|
||
#include "ql_rtos.h"
|
||
#include "ql_application.h"
|
||
#include "ql_audio.h"
|
||
#include "ql_gpio.h"
|
||
#include "sockets.h"
|
||
#include "ql_uart.h"
|
||
#include "ql_func.h"
|
||
#include "netdb.h"
|
||
#include "ql_sim.h"
|
||
#include "ql_dev.h"
|
||
#include "ql_wtd.h"
|
||
#include "ql_fs.h"
|
||
#include "ql_fota.h"
|
||
#include "ql_sim.h"
|
||
#include "ql_ping.h"
|
||
#include "ql_nw.h"
|
||
#include "ql_atcmd.h"
|
||
#include "layer3.h"
|
||
#include "ql_uart.h"
|
||
#include "hal_api.h"
|
||
#include "fvad.h"
|
||
|
||
|
||
// 在文件顶部添加状态变量
|
||
static uint8_t speech_detect_count = 0; // 连续语音帧计数器
|
||
#define SPEECH_DETECT_THRESHOLD 20 // 连续检测到语音的阈值
|
||
uint8_t no_speech_timeout = 0;
|
||
extern int g_mic_working;
|
||
|
||
extern int key_start_leave;
|
||
|
||
|
||
static uint8_t vad_leave_message_flag = 0;//留言模式vad只做单次
|
||
// FVAD 实例
|
||
static Fvad *fvad_inst = NULL;
|
||
extern struct ringbuffer *ring_sender;
|
||
extern ql_sem_t sender_sem;
|
||
extern unsigned char wss_connect_start; //表示vad拾音结束 开始等待后台返回 如果超过5s没有返回 则报超时语音
|
||
|
||
// 配置
|
||
static int sample_rate = 16000; // FVAD 支持的采样率：8k/16k/32k/48k
|
||
static int frame_ms = 20; // 帧长：10ms/20ms/30ms
|
||
static size_t frame_samples; // 每帧采样点数
|
||
|
||
// 状态
|
||
static volatile system_state_t g_state = STATE_IDLE;
|
||
static volatile bool g_triggered = false;
|
||
|
||
// 计时器（微秒）
|
||
static int64_t silence_start_time = 0;
|
||
static int64_t no_sound_start_time = 0;
|
||
static int64_t cmd_wait_start_time = 0;
|
||
static bool first_sound_detected = false;
|
||
static int64_t recording_start_time = 0; // 记录录音开始时间（首次检测到声音时）
|
||
|
||
static int vad_tts_playback_finished_status = 0;
|
||
// 统计
|
||
static uint32 frame_count = 0;
|
||
static uint32 speech_frame_count = 0;
|
||
extern int audio_packet_type;
|
||
|
||
|
||
// 内部函数声明
|
||
void set_state(system_state_t new_state);
|
||
static vad_state_t process_vad_frame(const int16 *data, size_t len);
|
||
static void fvad_init_config(void);
|
||
system_state_t get_state(void);
|
||
|
||
static audio_record_handle record_hd;
|
||
|
||
static void ql_audio_record_callback(acm_audio_track_handle handle, acm_audio_record_event_t event)
|
||
{
|
||
//printf("audio_record_callback event:%d\r\n", event);
|
||
}
|
||
|
||
void vad_tts_playback_finished()
|
||
{
|
||
vad_tts_playback_finished_status = 1;
|
||
}
|
||
|
||
void vad_set_speech_timeout_flag(uint8_t value)
|
||
{
|
||
no_speech_timeout = value;
|
||
}
|
||
|
||
// 状态转换
|
||
void set_state(system_state_t new_state) {
|
||
if (g_state != new_state) {
|
||
printf("State: %d -> %d", g_state, new_state);
|
||
g_state = new_state;
|
||
|
||
// 状态初始化
|
||
int64_t now = hal_os_up_time_ms();
|
||
switch (new_state) {
|
||
case STATE_RECORDING_WAIT_SOUND:
|
||
no_sound_start_time = now;
|
||
first_sound_detected = false;
|
||
speech_frame_count = 0;
|
||
g_mic_working = 0;
|
||
break;
|
||
case STATE_RECORDING_ACTIVE:
|
||
silence_start_time = now;
|
||
recording_start_time = now; // 记录进入活跃录音状态的时间
|
||
g_mic_working = 0;
|
||
break;
|
||
case STATE_WAITING_COMMAND:
|
||
cmd_wait_start_time = now;
|
||
break;
|
||
default:
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
// FVAD 初始化配置
|
||
static void fvad_init_config(void) {
|
||
if (fvad_inst == NULL) {
|
||
fvad_inst = fvad_new();
|
||
if (fvad_inst == NULL) {
|
||
printf("Failed to create FVAD instance");
|
||
return;
|
||
}
|
||
}
|
||
|
||
// 设置采样率（16kHz 是推荐值，平衡性能和效果）
|
||
int ret = fvad_set_sample_rate(fvad_inst, sample_rate);
|
||
if (ret != 0) {
|
||
printf("Failed to set sample rate %d", sample_rate);
|
||
return;
|
||
}
|
||
|
||
// 设置 VAD 模式（0-3，越高级越严格，3 最严格）
|
||
// 0: 宽松（噪声环境） 1: 正常 2: 严格 3: 最严格
|
||
fvad_set_mode(fvad_inst, 3);
|
||
|
||
// 计算每帧采样点数
|
||
frame_samples = (sample_rate / 1000) * frame_ms;
|
||
|
||
printf("FVAD initialized: rate=%dHz, frame=%dms, samples=%d",
|
||
sample_rate, frame_ms, frame_samples);
|
||
}
|
||
|
||
// VAD处理单帧 - 使用 FVAD 替换自定义算法
|
||
static vad_state_t process_vad_frame(const int16_t *data, size_t len) {
|
||
// 确保长度符合 FVAD 要求
|
||
#if 1
|
||
if (len != frame_samples) {
|
||
printf("Frame length mismatch: %d vs expected %d", len, frame_samples);
|
||
// 如果长度不匹配，返回静音状态
|
||
return VAD_STATE_SILENCE;
|
||
}
|
||
#endif
|
||
// 调用 FVAD 处理
|
||
int result = fvad_process(fvad_inst, data, len);
|
||
|
||
frame_count++;
|
||
|
||
// result: 0 = 静音, 1 = 语音, -1 = 错误
|
||
if (result < 0) {
|
||
printf("FVAD process error");
|
||
return VAD_STATE_SILENCE;
|
||
}
|
||
|
||
return (result == 1) ? VAD_STATE_ACTIVE : VAD_STATE_SILENCE;
|
||
}
|
||
|
||
// 初始化
|
||
int vad_audio_init(void) {
|
||
printf("Initializing VAD Audio with FVAD...");
|
||
|
||
// 初始化 FVAD
|
||
fvad_init_config();
|
||
if (fvad_inst == NULL) {
|
||
return -1;
|
||
}
|
||
|
||
app_codec_pa_close();
|
||
|
||
record_hd = ql_audio_record_open(16000, 1, 0x00, ql_audio_record_callback);
|
||
|
||
return 0;
|
||
}
|
||
|
||
// 启动录音
|
||
int vad_audio_start(void) {
|
||
vad_audio_init();
|
||
|
||
frame_count = 0;
|
||
speech_frame_count = 0;
|
||
|
||
printf("Audio recording started with FVAD");
|
||
return 0;
|
||
}
|
||
|
||
// 停止录音
|
||
int vad_audio_stop(void) {
|
||
|
||
printf("Audio recording stopped, frames: %d, speech: %d",
|
||
frame_count, speech_frame_count);
|
||
|
||
vad_audio_deinit();
|
||
|
||
ql_audio_record_close(record_hd, 1);
|
||
|
||
wss_connect_start = 1;//用于判断后台响应超时
|
||
|
||
key_start_leave = 0;
|
||
//ql_rtos_semaphore_release(sender_sem);
|
||
|
||
return 0;
|
||
}
|
||
|
||
// 停止录音
|
||
int vad_audio_stop_no_wait_resp(void) {
|
||
|
||
printf("Audio recording stopped no_wait_resp, frames: %d, speech: %d",
|
||
frame_count, speech_frame_count);
|
||
|
||
vad_audio_deinit();
|
||
|
||
ql_audio_record_close(record_hd, 1);
|
||
|
||
wss_connect_start = 0;
|
||
|
||
return 0;
|
||
}
|
||
|
||
|
||
// 触发录音（按键触发）
|
||
void vad_audio_trigger_record(void) {
|
||
g_triggered = true;
|
||
printf("Record triggered");
|
||
}
|
||
|
||
// 强制停止
|
||
void vad_audio_stop_record(void) {
|
||
set_state(STATE_IDLE);
|
||
vad_audio_stop_no_wait_resp();
|
||
}
|
||
|
||
// 获取状态
|
||
system_state_t vad_audio_get_state(void) {
|
||
return g_state;
|
||
}
|
||
|
||
bool vad_audio_is_recording(void) {
|
||
return (g_state == STATE_RECORDING_WAIT_SOUND ||
|
||
g_state == STATE_RECORDING_ACTIVE);
|
||
}
|
||
|
||
// 设置 FVAD 模式（0-3）
|
||
void vad_set_mode(int mode) {
|
||
if (fvad_inst && mode >= 0 && mode <= 3) {
|
||
fvad_set_mode(fvad_inst, mode);
|
||
printf("FVAD mode set to %d", mode);
|
||
}
|
||
}
|
||
|
||
// 设置采样率（必须在初始化前调用）
|
||
void vad_set_sample_rate(int rate) {
|
||
// FVAD 支持：8000, 16000, 32000, 48000
|
||
if (rate == 8000 || rate == 16000 || rate == 32000 || rate == 48000) {
|
||
sample_rate = rate;
|
||
printf("Sample rate set to %d", rate);
|
||
} else {
|
||
printf("Invalid sample rate %d, use 8000/16000/32000/48000", rate);
|
||
}
|
||
}
|
||
|
||
// 设置帧长（必须在初始化前调用）
|
||
void vad_set_frame_ms(int ms) {
|
||
// FVAD 支持：10ms, 20ms, 30ms
|
||
if (ms == 10 || ms == 20 || ms == 30) {
|
||
frame_ms = ms;
|
||
printf("Frame time set to %dms", ms);
|
||
} else {
|
||
printf("Invalid frame time %d, use 10/20/30", ms);
|
||
}
|
||
}
|
||
|
||
void vad_leave_message(void)
|
||
{
|
||
if(vad_audio_is_recording()) return;
|
||
|
||
audio_packet_type = 4;
|
||
|
||
vad_leave_message_flag = 1;
|
||
vad_audio_trigger_record();
|
||
}
|
||
|
||
// 主处理函数 - 需要在循环中定期调用
|
||
void vad_audio_process(void) {
|
||
size_t bytes_read = 0;
|
||
int64_t now = hal_os_up_time_ms();
|
||
|
||
static size_t buffer_offset = 0;
|
||
uint8_t mic_buf[640]={0};
|
||
int record_len = 640;
|
||
uint16 frame[320];
|
||
int index = 0, i = 0;
|
||
|
||
// 状态机处理
|
||
switch (g_state) {
|
||
case STATE_IDLE:
|
||
no_speech_timeout = 0;
|
||
vad_leave_message_flag = 0;
|
||
speech_detect_count = 0; // 空闲时清零计数器
|
||
vad_tts_playback_finished_status = 0;
|
||
|
||
if (g_triggered) {
|
||
g_triggered = false;
|
||
vad_audio_start();
|
||
set_state(STATE_RECORDING_WAIT_SOUND);
|
||
buffer_offset = 0;
|
||
}
|
||
return; // 空闲时不读取数据
|
||
|
||
case STATE_RECORDING_WAIT_SOUND:
|
||
case STATE_RECORDING_ACTIVE:
|
||
memset(mic_buf, 0, sizeof(mic_buf));
|
||
//每次读取640字节音频数据
|
||
ql_audio_record_read(record_hd, mic_buf, &record_len);
|
||
|
||
//如果检测到声音再去传输
|
||
// if(first_sound_detected){
|
||
//实际上每次编码需要输入1152字节的PCM数据
|
||
if(hal_os_rb_write(ring_sender, (uint8*)mic_buf, record_len, 25)>0){
|
||
|
||
//quec_usbcdc_write((uint8*)mic_buf, record_len);
|
||
ql_rtos_semaphore_release(sender_sem); //告诉线程去获取录音数据
|
||
}
|
||
// }
|
||
#if 1
|
||
for(index=0; index<320; index++)
|
||
{
|
||
frame[index] = mic_buf[i] + (mic_buf[i+1]<<8);
|
||
i=i+2;
|
||
}
|
||
#endif
|
||
// 缓冲区已满，处理整帧
|
||
vad_state_t current_vad = process_vad_frame(frame, 320);
|
||
|
||
// 首次声音检测
|
||
if (!first_sound_detected) {
|
||
if (current_vad == VAD_STATE_ACTIVE) {
|
||
speech_detect_count++;
|
||
printf("Speech frame count: %d\r\n", speech_detect_count);
|
||
|
||
// 连续达到阈值才确认检测到声音
|
||
if (speech_detect_count >= SPEECH_DETECT_THRESHOLD) {
|
||
|
||
#if 0
|
||
//实际上每次编码需要输入1152字节的PCM数据
|
||
if(hal_os_rb_write(ring_sender, (uint8*)mic_buf, record_len, 25)>0){
|
||
|
||
//quec_usbcdc_write((uint8*)mic_buf, record_len);
|
||
ql_rtos_semaphore_release(sender_sem); //告诉线程去获取录音数据
|
||
}
|
||
#endif
|
||
|
||
first_sound_detected = true;
|
||
set_state(STATE_RECORDING_ACTIVE);
|
||
printf("First speech confirmed by FVAD (continuous %d frames)", SPEECH_DETECT_THRESHOLD);
|
||
}
|
||
} else {
|
||
// 出现静音帧，计数器清零
|
||
if (speech_detect_count > 0) {
|
||
printf("Speech interrupted, reset count\r\n");
|
||
speech_detect_count = 0;
|
||
}
|
||
|
||
// 检查无声音超时
|
||
if ((now - no_sound_start_time) > VAD_NO_SOUND_TIMEOUT_MS) {
|
||
printf("No speech for timeout, stopping");
|
||
no_speech_timeout = 1;
|
||
set_state(STATE_WAITING_COMMAND);
|
||
vad_audio_stop();
|
||
buffer_offset = 0;
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
// ==========================================
|
||
|
||
// 活跃状态处理
|
||
if (g_state == STATE_RECORDING_ACTIVE) {
|
||
|
||
// 10秒录音时长限制
|
||
if ((now - recording_start_time) > RECORDING_MAX_TIME) {
|
||
printf("Recording time limit reached (10s), stopping");
|
||
set_state(STATE_WAITING_COMMAND);
|
||
vad_audio_stop();
|
||
buffer_offset = 0;
|
||
return;
|
||
}
|
||
|
||
// 更新静音计时
|
||
if (current_vad == VAD_STATE_ACTIVE) {
|
||
silence_start_time = now;
|
||
speech_frame_count++;
|
||
} else {
|
||
if ((now - silence_start_time) > VAD_SILENCE_TIMEOUT_MS) {
|
||
printf("Silence timeout, stopping output, waiting command");
|
||
set_state(STATE_WAITING_COMMAND);
|
||
vad_audio_stop();
|
||
buffer_offset = 0;
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
// 重置缓冲区偏移
|
||
buffer_offset = 0;
|
||
break;
|
||
|
||
case STATE_WAITING_COMMAND:
|
||
if(vad_tts_playback_finished_status == 1) {
|
||
printf("finished_status:%d, no_speech_timeout:%d\n", vad_tts_playback_finished_status, no_speech_timeout);
|
||
set_state(STATE_IDLE);
|
||
ql_rtos_task_sleep_ms(50);
|
||
if(no_speech_timeout == 0&&vad_leave_message_flag==0) {
|
||
vad_audio_trigger_record();
|
||
}
|
||
}
|
||
|
||
key_start_leave = 0;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 清理资源（添加清理函数）
|
||
void vad_audio_deinit(void) {
|
||
if (fvad_inst) {
|
||
fvad_free(fvad_inst);
|
||
fvad_inst = NULL;
|
||
}
|
||
printf("VAD audio deinitialized");
|
||
}
|