#include #include #include #include "cJSON.h" #include "app_inc.h" #include "ql_rtos.h" #include "ql_application.h" #include "ql_audio.h" #include "ql_gpio.h" #include "sockets.h" #include "ql_uart.h" #include "ql_func.h" #include "netdb.h" #include "ql_sim.h" #include "ql_dev.h" #include "ql_wtd.h" #include "ql_fs.h" #include "ql_fota.h" #include "ql_sim.h" #include "ql_ping.h" #include "ql_nw.h" #include "ql_atcmd.h" #include "layer3.h" #include "ql_uart.h" #include "hal_api.h" #include "fvad.h" // 在文件顶部添加状态变量 static uint8_t speech_detect_count = 0; // 连续语音帧计数器 #define SPEECH_DETECT_THRESHOLD 20 // 连续检测到语音的阈值 uint8_t no_speech_timeout = 0; extern int g_mic_working; extern int key_start_leave; static uint8_t vad_leave_message_flag = 0;//留言模式vad只做单次 // FVAD 实例 static Fvad *fvad_inst = NULL; extern struct ringbuffer *ring_sender; extern ql_sem_t sender_sem; extern unsigned char wss_connect_start; //表示vad拾音结束 开始等待后台返回 如果超过5s没有返回 则报超时语音 // 配置 static int sample_rate = 16000; // FVAD 支持的采样率:8k/16k/32k/48k static int frame_ms = 20; // 帧长:10ms/20ms/30ms static size_t frame_samples; // 每帧采样点数 // 状态 static volatile system_state_t g_state = STATE_IDLE; static volatile bool g_triggered = false; // 计时器(微秒) static int64_t silence_start_time = 0; static int64_t no_sound_start_time = 0; static int64_t cmd_wait_start_time = 0; static bool first_sound_detected = false; static int64_t recording_start_time = 0; // 记录录音开始时间(首次检测到声音时) static int vad_tts_playback_finished_status = 0; // 统计 static uint32 frame_count = 0; static uint32 speech_frame_count = 0; extern int audio_packet_type; // 内部函数声明 void set_state(system_state_t new_state); static vad_state_t process_vad_frame(const int16 *data, size_t len); static void fvad_init_config(void); system_state_t get_state(void); static audio_record_handle record_hd; static void ql_audio_record_callback(acm_audio_track_handle handle, acm_audio_record_event_t event) { //printf("audio_record_callback event:%d\r\n", event); } void vad_tts_playback_finished() { vad_tts_playback_finished_status = 1; } void vad_set_speech_timeout_flag(uint8_t value) { no_speech_timeout = value; } // 状态转换 void set_state(system_state_t new_state) { if (g_state != new_state) { printf("State: %d -> %d", g_state, new_state); g_state = new_state; // 状态初始化 int64_t now = hal_os_up_time_ms(); switch (new_state) { case STATE_RECORDING_WAIT_SOUND: no_sound_start_time = now; first_sound_detected = false; speech_frame_count = 0; g_mic_working = 0; break; case STATE_RECORDING_ACTIVE: silence_start_time = now; recording_start_time = now; // 记录进入活跃录音状态的时间 g_mic_working = 0; break; case STATE_WAITING_COMMAND: cmd_wait_start_time = now; break; default: break; } } } // FVAD 初始化配置 static void fvad_init_config(void) { if (fvad_inst == NULL) { fvad_inst = fvad_new(); if (fvad_inst == NULL) { printf("Failed to create FVAD instance"); return; } } // 设置采样率(16kHz 是推荐值,平衡性能和效果) int ret = fvad_set_sample_rate(fvad_inst, sample_rate); if (ret != 0) { printf("Failed to set sample rate %d", sample_rate); return; } // 设置 VAD 模式(0-3,越高级越严格,3 最严格) // 0: 宽松(噪声环境) 1: 正常 2: 严格 3: 最严格 fvad_set_mode(fvad_inst, 3); // 计算每帧采样点数 frame_samples = (sample_rate / 1000) * frame_ms; printf("FVAD initialized: rate=%dHz, frame=%dms, samples=%d", sample_rate, frame_ms, frame_samples); } // VAD处理单帧 - 使用 FVAD 替换自定义算法 static vad_state_t process_vad_frame(const int16_t *data, size_t len) { // 确保长度符合 FVAD 要求 #if 1 if (len != frame_samples) { printf("Frame length mismatch: %d vs expected %d", len, frame_samples); // 如果长度不匹配,返回静音状态 return VAD_STATE_SILENCE; } #endif // 调用 FVAD 处理 int result = fvad_process(fvad_inst, data, len); frame_count++; // result: 0 = 静音, 1 = 语音, -1 = 错误 if (result < 0) { printf("FVAD process error"); return VAD_STATE_SILENCE; } return (result == 1) ? VAD_STATE_ACTIVE : VAD_STATE_SILENCE; } // 初始化 int vad_audio_init(void) { printf("Initializing VAD Audio with FVAD..."); // 初始化 FVAD fvad_init_config(); if (fvad_inst == NULL) { return -1; } app_codec_pa_close(); record_hd = ql_audio_record_open(16000, 1, 0x00, ql_audio_record_callback); return 0; } // 启动录音 int vad_audio_start(void) { vad_audio_init(); frame_count = 0; speech_frame_count = 0; printf("Audio recording started with FVAD"); return 0; } // 停止录音 int vad_audio_stop(void) { printf("Audio recording stopped, frames: %d, speech: %d", frame_count, speech_frame_count); vad_audio_deinit(); ql_audio_record_close(record_hd, 1); wss_connect_start = 1;//用于判断后台响应超时 key_start_leave = 0; //ql_rtos_semaphore_release(sender_sem); return 0; } // 停止录音 int vad_audio_stop_no_wait_resp(void) { printf("Audio recording stopped no_wait_resp, frames: %d, speech: %d", frame_count, speech_frame_count); vad_audio_deinit(); ql_audio_record_close(record_hd, 1); wss_connect_start = 0; return 0; } // 触发录音(按键触发) void vad_audio_trigger_record(void) { g_triggered = true; printf("Record triggered"); } // 强制停止 void vad_audio_stop_record(void) { set_state(STATE_IDLE); vad_audio_stop_no_wait_resp(); } // 获取状态 system_state_t vad_audio_get_state(void) { return g_state; } bool vad_audio_is_recording(void) { return (g_state == STATE_RECORDING_WAIT_SOUND || g_state == STATE_RECORDING_ACTIVE); } // 设置 FVAD 模式(0-3) void vad_set_mode(int mode) { if (fvad_inst && mode >= 0 && mode <= 3) { fvad_set_mode(fvad_inst, mode); printf("FVAD mode set to %d", mode); } } // 设置采样率(必须在初始化前调用) void vad_set_sample_rate(int rate) { // FVAD 支持:8000, 16000, 32000, 48000 if (rate == 8000 || rate == 16000 || rate == 32000 || rate == 48000) { sample_rate = rate; printf("Sample rate set to %d", rate); } else { printf("Invalid sample rate %d, use 8000/16000/32000/48000", rate); } } // 设置帧长(必须在初始化前调用) void vad_set_frame_ms(int ms) { // FVAD 支持:10ms, 20ms, 30ms if (ms == 10 || ms == 20 || ms == 30) { frame_ms = ms; printf("Frame time set to %dms", ms); } else { printf("Invalid frame time %d, use 10/20/30", ms); } } void vad_leave_message(void) { if(vad_audio_is_recording()) return; audio_packet_type = 4; vad_leave_message_flag = 1; vad_audio_trigger_record(); } // 主处理函数 - 需要在循环中定期调用 void vad_audio_process(void) { size_t bytes_read = 0; int64_t now = hal_os_up_time_ms(); static size_t buffer_offset = 0; uint8_t mic_buf[640]={0}; int record_len = 640; uint16 frame[320]; int index = 0, i = 0; // 状态机处理 switch (g_state) { case STATE_IDLE: no_speech_timeout = 0; vad_leave_message_flag = 0; speech_detect_count = 0; // 空闲时清零计数器 vad_tts_playback_finished_status = 0; if (g_triggered) { g_triggered = false; vad_audio_start(); set_state(STATE_RECORDING_WAIT_SOUND); buffer_offset = 0; } return; // 空闲时不读取数据 case STATE_RECORDING_WAIT_SOUND: case STATE_RECORDING_ACTIVE: memset(mic_buf, 0, sizeof(mic_buf)); //每次读取640字节音频数据 ql_audio_record_read(record_hd, mic_buf, &record_len); //如果检测到声音再去传输 if(first_sound_detected){ //实际上每次编码需要输入1152字节的PCM数据 if(hal_os_rb_write(ring_sender, (uint8*)mic_buf, record_len, 25)>0){ //quec_usbcdc_write((uint8*)mic_buf, record_len); ql_rtos_semaphore_release(sender_sem); //告诉线程去获取录音数据 } } #if 1 for(index=0; index<320; index++) { frame[index] = mic_buf[i] + (mic_buf[i+1]<<8); i=i+2; } #endif // 缓冲区已满,处理整帧 vad_state_t current_vad = process_vad_frame(frame, 320); // 首次声音检测 if (!first_sound_detected) { if (current_vad == VAD_STATE_ACTIVE) { speech_detect_count++; printf("Speech frame count: %d\r\n", speech_detect_count); // 连续达到阈值才确认检测到声音 if (speech_detect_count >= SPEECH_DETECT_THRESHOLD) { //实际上每次编码需要输入1152字节的PCM数据 if(hal_os_rb_write(ring_sender, (uint8*)mic_buf, record_len, 25)>0){ //quec_usbcdc_write((uint8*)mic_buf, record_len); ql_rtos_semaphore_release(sender_sem); //告诉线程去获取录音数据 } first_sound_detected = true; set_state(STATE_RECORDING_ACTIVE); printf("First speech confirmed by FVAD (continuous %d frames)", SPEECH_DETECT_THRESHOLD); } } else { // 出现静音帧,计数器清零 if (speech_detect_count > 0) { printf("Speech interrupted, reset count\r\n"); speech_detect_count = 0; } // 检查无声音超时 if ((now - no_sound_start_time) > VAD_NO_SOUND_TIMEOUT_MS) { printf("No speech for timeout, stopping"); no_speech_timeout = 1; set_state(STATE_WAITING_COMMAND); vad_audio_stop(); buffer_offset = 0; return; } } } // ========================================== // 活跃状态处理 if (g_state == STATE_RECORDING_ACTIVE) { // 10秒录音时长限制 if ((now - recording_start_time) > RECORDING_MAX_TIME) { printf("Recording time limit reached (10s), stopping"); set_state(STATE_WAITING_COMMAND); vad_audio_stop(); buffer_offset = 0; return; } // 更新静音计时 if (current_vad == VAD_STATE_ACTIVE) { silence_start_time = now; speech_frame_count++; } else { if ((now - silence_start_time) > VAD_SILENCE_TIMEOUT_MS) { printf("Silence timeout, stopping output, waiting command"); set_state(STATE_WAITING_COMMAND); vad_audio_stop(); buffer_offset = 0; return; } } } // 重置缓冲区偏移 buffer_offset = 0; break; case STATE_WAITING_COMMAND: if(vad_tts_playback_finished_status == 1) { printf("finished_status:%d, no_speech_timeout:%d\n", vad_tts_playback_finished_status, no_speech_timeout); set_state(STATE_IDLE); ql_rtos_task_sleep_ms(50); if(no_speech_timeout == 0&&vad_leave_message_flag==0) { vad_audio_trigger_record(); } } key_start_leave = 0; break; } } // 清理资源(添加清理函数) void vad_audio_deinit(void) { if (fvad_inst) { fvad_free(fvad_inst); fvad_inst = NULL; } printf("VAD audio deinitialized"); }