fix(audio): prevent empty session replay

ql-application/threadx/interface/custom_app/mp3_player.c

@@ -25,6 +25,7 @@ static unsigned char mp3_player_status = 0;
 QFILE *local_mp3_file_fd=NULL;
 
 static int mp3_player_end_frame = 0;
+static int mp3_player_last_result = 0;
 
 int current_sample_rate = 0;
 static	audio_track_handle track_hd;
@@ -278,9 +279,10 @@ u32 local_mp3_data_fetch(const char *uri, uint8_t *buffer, u32 length)
 	return read_bytes;
 }
 
-size_t url_mp3_data_fetch(const char *uri, uint8_t *buffer, size_t length)
+int url_mp3_data_fetch(const char *uri, uint8_t *buffer, size_t length)
 {
 	int resp_status = -1;
+	int result = -1;
 	uint32_t nw = length;
 	uint32_t page_pos = 0;
 	uint32_t content_length = 0;
@@ -354,6 +356,9 @@ size_t url_mp3_data_fetch(const char *uri, uint8_t *buffer, size_t length)
 	}
 
 	printf("end read:%d, %d\n", total_length, session->content_length);
+	if(total_length == session->content_length){
+		result = 0;
+	}
 
 __exit:
 
@@ -363,12 +368,12 @@ __exit:
 		session = NULL;
 	}
 
-	if(total_length<content_length){
+	if(result != 0 || total_length < content_length){
 		total_length = 0;
 		return -1;
 	}
 
-	return 0;
+	return result;
 }
 
 
@@ -390,11 +395,24 @@ int aofei_mp3_player_init(void)
 void aofei_mp3_player(char *file_name)
 {
 	int ret = 0;
-	uint8_t *mp3_buff = malloc(100*1024);
+	uint8_t *mp3_buff = NULL;
+
+	mp3_player_last_result = -1;
+	if(file_name == NULL){
+		printf("mp3 player file name is null\r\n");
+		return;
+	}
+
+	mp3_buff = malloc(100*1024);
+	if(mp3_buff == NULL){
+		printf("mp3 player buffer malloc failed\r\n");
+		return;
+	}
 
 	memset(mp3_buff, 0, 100*1024);
 	//直接下载到一个100k的buffer
-	if(strstr(file_name, "http:")){//播放url
+	if(strncmp(file_name, "http://", strlen("http://")) == 0 ||
+	   strncmp(file_name, "https://", strlen("https://")) == 0){//播放url
 		ret = url_mp3_data_fetch(file_name, mp3_buff, 100*1024);
 	}else if(strstr(file_name, "test.mp3")){//播放本地音频
 		total_length = test_data_size;
@@ -403,10 +421,18 @@ void aofei_mp3_player(char *file_name)
 		total_length = local_mp3_data_fetch(file_name, mp3_buff, 100*1024);
 	}
 
+	if(ret < 0 || total_length == 0){
+		ret = -1;
+		goto __exit;
+	}
+  
 	mp3_player_end_frame = 0;
 	
 
-	if(aofei_mp3_player_init()==-1)return;
+	if(aofei_mp3_player_init()==-1){
+		ret = -1;
+		goto __exit;
+	}
 
 	if(ret>=0){
 		aofei_mp3_decoder->read_ptr = mp3_buff;
@@ -457,6 +483,7 @@ void aofei_mp3_player(char *file_name)
 		app_codec_pa_close();
 	}
 
+__exit:
 	if(aofei_mp3_decoder!=NULL){
 		mp3_decoder_delete(aofei_mp3_decoder);
 		aofei_mp3_decoder = NULL;
@@ -480,6 +507,13 @@ void aofei_mp3_player(char *file_name)
 	total_length = 0;
 	mp3_player_end_frame = 0;
 	mp3_player_status = 0;
+	mp3_player_last_result = ret;
+}
+
+/* 播放调用是同步的，队列线程可在返回后读取本次结果决定是否使用本地兜底。 */
+int aofei_mp3_player_get_result(void)
+{
+	return mp3_player_last_result;
 }
 
 

ql-application/threadx/interface/custom_app/vad_det.c

@@ -29,6 +29,9 @@
 // 在文件顶部添加状态变量
 static uint8_t speech_detect_count = 0;  // 连续语音帧计数器
 #define SPEECH_DETECT_THRESHOLD  20        // 连续检测到语音的阈值
+#define PLAYBACK_VAD_GUARD_MS     800       // 屏蔽播放结束后的尾音和音频通路切换噪声
+#define VAD_FRAME_BYTES           640       // 20ms @ 16kHz, 16-bit mono
+#define VAD_PREROLL_FRAMES        25        // 保留500ms预录音，避免吞掉用户首字
 uint8_t no_speech_timeout = 0;
 extern int g_mic_working;
 
@@ -57,6 +60,9 @@ static int64_t no_sound_start_time = 0;
 static int64_t cmd_wait_start_time = 0;
 static bool first_sound_detected = false;
 static int64_t recording_start_time = 0;  // 记录录音开始时间（首次检测到声音时）
+static uint8_t vad_preroll[VAD_PREROLL_FRAMES][VAD_FRAME_BYTES];
+static size_t vad_preroll_next = 0;
+static size_t vad_preroll_count = 0;
 
 static int vad_tts_playback_finished_status = 0;
 // 统计
@@ -101,6 +107,9 @@ void set_state(system_state_t new_state) {
                 no_sound_start_time = now;
                 first_sound_detected = false;
                 speech_frame_count = 0;
+				speech_detect_count = 0;
+				vad_preroll_next = 0;
+				vad_preroll_count = 0;
 				g_mic_working = 0;
                 break;
             case STATE_RECORDING_ACTIVE:
@@ -290,17 +299,55 @@ void vad_leave_message(void)
 	audio_packet_type = 4;
 	
 	vad_leave_message_flag = 1;
+	/* A previous conversation may still be parked in WAITING_COMMAND.  NFC
+	 * leave-message recording must enter IDLE first, otherwise g_triggered is
+	 * left pending forever and its 10-second no-speech timer never starts. */
+	set_state(STATE_IDLE);
 	vad_audio_trigger_record();
 }
 
+int vad_leave_message_is_active(void)
+{
+	return vad_leave_message_flag != 0 || audio_packet_type == 4;
+}
+
+/* The conversation key has priority over an unfinished NFC leave-message
+ * transaction.  Clear every local latch before the normal conversation prompt
+ * is queued so that its recording can never inherit packetType=4. */
+int vad_cancel_leave_message(void)
+{
+	int was_active = vad_leave_message_is_active();
+
+	if (!was_active) return 0;
+
+	printf("cancel pending leave message\r\n");
+	vad_leave_message_flag = 0;
+	g_triggered = false;
+	audio_packet_type = 0;
+	no_speech_timeout = 0;
+	speech_detect_count = 0;
+	vad_preroll_next = 0;
+	vad_preroll_count = 0;
+	key_start_leave = 0;
+
+	if (vad_audio_is_recording()) {
+		set_state(STATE_IDLE);
+		vad_audio_stop_no_wait_resp();
+	} else {
+		set_state(STATE_IDLE);
+	}
+
+	return 1;
+}
+
 // 主处理函数 - 需要在循环中定期调用
 void vad_audio_process(void) {
     size_t bytes_read = 0;
     int64_t now = hal_os_up_time_ms();
     
     static size_t buffer_offset = 0;
-	uint8_t mic_buf[640]={0};
+	uint8_t mic_buf[VAD_FRAME_BYTES]={0};
-	int record_len = 640;
+	int record_len = VAD_FRAME_BYTES;
 	uint16 frame[320];
 	int index = 0, i = 0;
     
@@ -308,7 +355,6 @@ void vad_audio_process(void) {
     switch (g_state) {
         case STATE_IDLE:
 			no_speech_timeout = 0;
-			vad_leave_message_flag = 0;
             speech_detect_count = 0;  // 空闲时清零计数器
             vad_tts_playback_finished_status = 0;
 			
@@ -326,15 +372,19 @@ void vad_audio_process(void) {
 			//每次读取640字节音频数据
 			ql_audio_record_read(record_hd, mic_buf, &record_len); 
 
-			//如果检测到声音再去传输
+			if (g_state == STATE_RECORDING_WAIT_SOUND) {
-		//	if(first_sound_detected){
+				/* 首次开口确认前不向服务端发送静音。只在本地保留最近500ms，
-			//实际上每次编码需要输入1152字节的PCM数据
+				 * 确认开口时再回填，防止截掉用户首字。 */
+				memcpy(vad_preroll[vad_preroll_next], mic_buf, VAD_FRAME_BYTES);
+				vad_preroll_next = (vad_preroll_next + 1) % VAD_PREROLL_FRAMES;
+				if (vad_preroll_count < VAD_PREROLL_FRAMES) {
+					vad_preroll_count++;
+				}
+			} else {
 				if(hal_os_rb_write(ring_sender, (uint8*)mic_buf, record_len, 25)>0){
-
+					ql_rtos_semaphore_release(sender_sem);
-				//quec_usbcdc_write((uint8*)mic_buf, record_len);
+				}
-				ql_rtos_semaphore_release(sender_sem); //告诉线程去获取录音数据
 			}
-		//	}
 		#if 1
 			for(index=0; index<320; index++)
 			{
@@ -347,6 +397,12 @@ void vad_audio_process(void) {
             
             // 首次声音检测
             if (!first_sound_detected) {
+				/* 播放刚结束时扬声器尾音和PA切换噪声容易被FVAD认为用户开口。 */
+				if ((now - no_sound_start_time) < PLAYBACK_VAD_GUARD_MS) {
+					speech_detect_count = 0;
+					buffer_offset = 0;
+					break;
+				}
                 if (current_vad == VAD_STATE_ACTIVE) {
                     speech_detect_count++;
                     printf("Speech frame count: %d\r\n", speech_detect_count);
@@ -364,6 +420,21 @@ void vad_audio_process(void) {
 						#endif
 						
 	                        first_sound_detected = true;
+						{
+							size_t first = (vad_preroll_next + VAD_PREROLL_FRAMES - vad_preroll_count) % VAD_PREROLL_FRAMES;
+							size_t n;
+							int preroll_queued = 0;
+							for (n = 0; n < vad_preroll_count; n++) {
+								size_t frame_index = (first + n) % VAD_PREROLL_FRAMES;
+								if (hal_os_rb_write(ring_sender, vad_preroll[frame_index], VAD_FRAME_BYTES, 25) > 0) {
+									preroll_queued = 1;
+								}
+							}
+							if (preroll_queued) {
+								ql_rtos_semaphore_release(sender_sem);
+							}
+							vad_preroll_count = 0;
+						}
 	                        set_state(STATE_RECORDING_ACTIVE);
                         printf("First speech confirmed by FVAD (continuous %d frames)", SPEECH_DETECT_THRESHOLD);
                     }
@@ -420,12 +491,17 @@ void vad_audio_process(void) {
             
         case STATE_WAITING_COMMAND:
 			if(vad_tts_playback_finished_status == 1) {
+				int leave_message_finished = vad_leave_message_flag != 0;
                 printf("finished_status:%d, no_speech_timeout:%d\n", vad_tts_playback_finished_status, no_speech_timeout);
                 set_state(STATE_IDLE);
                 ql_rtos_task_sleep_ms(50);
                 if(no_speech_timeout == 0&&vad_leave_message_flag==0) {
                     vad_audio_trigger_record();
                 }
+				if(leave_message_finished) {
+					vad_leave_message_flag = 0;
+					audio_packet_type = 0;
+				}
             }
 
 			key_start_leave = 0;

ql-application/threadx/interface/custom_app/ws_talkingq.c

@@ -32,6 +32,7 @@
 #define PCM_FRAME_LENGTH    1152
 #define CLOUD_TTS_ERROR_URL_FORMAT \
     "http://101.35.224.118:8080/assets/roles/banban/tts_error/tts_error%02u.mp3"
+#define LOCAL_TTS_ERROR_PATH "U:/tts_error.mp3"
 
 static int g_sim_ready = 0;
 
@@ -56,7 +57,7 @@ static volatile int player_mq_couter = 0;
 /* Set while a barge-in prompt bridges the old and new sessions.  The old
  * server response can still deliver URLs after its player queue was cleared. */
 static volatile int barge_in_transition = 0;
-static unsigned char audio_in_task_cancel = 0;
+static volatile unsigned char audio_in_task_cancel = 0;
 
 static unsigned char xiaoice_wss_connect_status = 0; 
 static unsigned char wss_connect_counter = 0;
@@ -69,13 +70,20 @@ extern int mqtt_task_run_flag;
 static unsigned char shine_init = 0;
 extern int fisrt_net_start;
 extern u8 power_down_status;
+extern int aofei_mp3_player_get_result(void);
+extern int vad_leave_message_is_active(void);
+extern int vad_cancel_leave_message(void);
+
+void sendAuthentication(void);
+void vad_start_func(void);
+void xiaoice_close_all_audio(void);
 
 int audio_packet_type = 0;
 
 
 typedef struct ice_work_item_s
 {
-    int type;  //0-tts_out 1-session_continue 2-session_end 3-error
+    int type;  //0-tts_out 1-session_continue 2-session_end 3-云端超时提示（失败时播放本地兜底）
     int error;
 	int del_url;
     void *data; 
@@ -198,15 +206,11 @@ static void remove_underscores(char *str) {
     *dst = '\0'; // 添加字符串结束符
 }
 
-/*
+/* 生成 banban 角色的随机云端对话超时提示音 URL。 */
- * 将后台下发的任意角色对话超时音频 URL 改为 banban 角色的 HTTP 地址，
+static int build_random_tts_error_url(char *url, size_t url_size)
- * 并在 tts_error00~tts_error09 中随机选择一条。
- */
-static int randomize_tts_error_url(char *url, size_t url_size)
 {
     static uint32_t random_sequence = 0;
     static unsigned int last_random_index = 10;
-    char *file_name;
     uint32_t random_value;
     unsigned int random_index;
     int written;
@@ -215,14 +219,6 @@ static int randomize_tts_error_url(char *url, size_t url_size)
         return 0;
     }
 
-    file_name = strrchr(url, '/');
-    file_name = (file_name == NULL) ? url : file_name + 1;
-
-    if(strncmp(file_name, "tts_error", strlen("tts_error")) != 0 &&
-       strncmp(file_name, "tts_err", strlen("tts_err")) != 0){
-        return 0;
-    }
-
     /*
      * 某些设备上硬件随机接口可能持续返回 0，因此混合伪随机数、系统 tick
      * 和递增序列；若仍与上一次相同，则向后偏移，避免连续重复播放。
@@ -248,6 +244,29 @@ static int randomize_tts_error_url(char *url, size_t url_size)
     return 1;
 }
 
+/*
+ * 后台下发任意角色的对话超时 URL 时，统一替换成 banban 的随机云端资源。
+ * 非对话超时 URL 保持不变。
+ */
+static int randomize_tts_error_url(char *url, size_t url_size)
+{
+    char *file_name;
+
+    if(url == NULL || url_size == 0){
+        return 0;
+    }
+
+    file_name = strrchr(url, '/');
+    file_name = (file_name == NULL) ? url : file_name + 1;
+
+    if(strncmp(file_name, "tts_error", strlen("tts_error")) != 0 &&
+       strncmp(file_name, "tts_err", strlen("tts_err")) != 0){
+        return 0;
+    }
+
+    return build_random_tts_error_url(url, url_size);
+}
+
 void add_file_to_play_player_queue(char *path, int type)
 {
 	ice_work_item *item = NULL;
@@ -334,6 +353,7 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
 			char *ret_str2 = NULL;
 			size_t url_len = 0;
 			int is_tts_error_url = 0;
+			int is_message_ok_url = 0;
 			if(ret_str1 == NULL){
 				ret_str1 = strstr(text, "http://");
 			}
@@ -350,6 +370,14 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
 				}
 				memcpy(url_buff, ret_str1, url_len);
 				url_buff[url_len] = '\0';
+				is_message_ok_url = strstr(url_buff, "message_ok") != NULL;
+
+				/* A cancelled NFC transaction may still have a response in flight.
+				 * Never let that acknowledgement enter a normal conversation. */
+				if(is_message_ok_url && !vad_leave_message_is_active()){
+					printf("drop stale leave message ack\r\n");
+					return;
+				}
 
 				/* 两个角色统一使用 banban 的 HTTP 随机超时提示音 */
 				is_tts_error_url = randomize_tts_error_url(url_buff, sizeof(url_buff));
@@ -374,7 +402,7 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
 					tts_play_status = 2;
 				}
 
-				if(is_tts_error_url == 1 || strstr(url_buff, "message_ok")!=NULL){
+				if(is_tts_error_url == 1 || is_message_ok_url){
 					printf("tts_error or message_ok\n");
 					vad_set_speech_timeout_flag(1);
 				}
@@ -383,7 +411,8 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
 				if(tts_playing==2) return;
 				
 		
-				add_file_to_play_player_queue(url_buff, 0);
+				/* 对话超时提示使用 type=3，云端播放失败时由播放线程切换到本地兜底。 */
+				add_file_to_play_player_queue(url_buff, is_tts_error_url == 1 ? 3 : 0);
 			}
 		}
 
@@ -970,6 +999,7 @@ int app_play_ota_tips_file(void)
 void app_player_thread(void *arvg)
 {
 	ice_work_item *player_item=NULL;
+	int play_ret;
 	
     while(1){
       ql_rtos_queue_wait(player_queue, (u8*)(&player_item), sizeof(ice_work_item *), 0xFFFFFFFF);
@@ -981,9 +1011,26 @@ void app_player_thread(void *arvg)
 
 				  app_set_app_operation(1);
 				  aofei_mp3_player(player_item->data);
+				  play_ret = aofei_mp3_player_get_result();
 				  app_set_app_operation(1);
 
 				  if(player_mq_couter>0) player_mq_couter--;
+
+				  /*
+				   * type=3 只用于对话超时提示。云端连接、HTTP、读取或播放器初始化
+				   * 失败时才排入本地兜底；如果用户已经开始下一轮录音，则丢弃旧提示。
+				   */
+				  if(play_ret != 0 && player_item->type == 3 &&
+				     !vad_audio_is_recording() && tts_playing_ret() != 2){
+					  printf("cloud tts error play failed:%d, fallback:%s\r\n",
+					         play_ret, LOCAL_TTS_ERROR_PATH);
+					  if(ql_access(LOCAL_TTS_ERROR_PATH, 0) == 0){
+						  add_file_to_play_player_queue(LOCAL_TTS_ERROR_PATH, 0);
+					  }else{
+						  printf("local tts error fallback missing:%s\r\n", LOCAL_TTS_ERROR_PATH);
+					  }
+				  }
+
 				  if(player_mq_couter==0){
 					  printf("player_mq_couter==0\r\n");
 
@@ -1025,6 +1072,29 @@ void app_player_thread(void *arvg)
 int tts_play_interrupt_func(void)
 {
 	int playback_interrupted = 0;
+	int leave_message_cancelled = 0;
+
+	/* A short press explicitly starts a normal conversation.  Cancel an NFC
+	 * leave-message before the generic "already recording" guard, otherwise a
+	 * pending/recording leave-message can leak packetType=4 into this session. */
+	leave_message_cancelled = vad_cancel_leave_message();
+	if(leave_message_cancelled){
+		uint64_t cancel_start;
+
+		/* Let the sender task discard buffered PCM and its encoder without
+		 * emitting END.  Only then close the old leave-message socket. */
+		audio_in_task_cancel = 1;
+		ql_rtos_semaphore_release(sender_sem);
+		cancel_start = hal_os_up_time_ms();
+		while(audio_in_task_cancel &&
+		      hal_os_up_time_ms() - cancel_start < OTA_AUDIO_STOP_TIMEOUT_MS){
+			ql_rtos_task_sleep_ms(20);
+		}
+		if(audio_in_task_cancel){
+			printf("wait leave message sender cancel timeout\r\n");
+		}
+		talkingq_ws_disconnect();
+	}
 
 	//如果在vad状态，则直接返回
 	if(vad_audio_is_recording()||g_mic_working) return -1;
@@ -1191,9 +1261,19 @@ static void check_connect_thread_entry(void *parameter)
 			printf("tts_error_set_speech1\n");
 			vad_set_speech_timeout_flag(1);
 			
+			/*
+			 * 后台没有返回任何音频 URL 时，也先由设备生成随机云端提示音。
+			 * type=3 会在 HTTP 下载或播放初始化失败时自动切换到本地音频。
+			 */
+			{
+				char cloud_tts_error_url[128] = {0};
+				if(build_random_tts_error_url(cloud_tts_error_url,
+				                              sizeof(cloud_tts_error_url)) == 1){
+					add_file_to_play_player_queue(cloud_tts_error_url, 3);
+				}else{
 					app_play_tips_file(9);
-			//talkingq_ws_send_text("REQUEST_PROMPT_SOUND:tts_error");
+				}
-			//改为播放本地音频 听不清 请再说一遍
+			}
 		}
 	#endif
 	
@@ -1267,9 +1347,34 @@ void app_sender_thread(void *arvg)
     		ql_rtos_semaphore_wait(sender_sem, 0xFFFFFFFF);
 
 sender_loop:
+			if(audio_in_task_cancel){
+				left = rb_get_space_used(ring_sender);
+				while(left > 0){
+					int discard_len = left > PCM_FRAME_LENGTH ? PCM_FRAME_LENGTH : left;
+					hal_os_rb_read(ring_sender, sender_buf, discard_len, 0);
+					left = rb_get_space_used(ring_sender);
+				}
+				if(shine_init != 0){
+					shine_close(sh_shine);
+					sh_shine = NULL;
+					shine_init = 0;
+				}
+				audio_packet_type = 0;
+				audio_in_task_cancel = 0;
+				printf("leave message sender cancelled\r\n");
+				continue;
+			}
 
 			if(!app_get_data_call_net_status()) continue;
 
+			/* One recording can release the semaphore many times while the sender
+			 * drains the ring buffer.  A stale token must never create an empty
+			 * session after the previous encoder has already been closed. */
+			left = rb_get_space_used(ring_sender);
+			if(shine_init == 0 && left <= 0){
+				continue;
+			}
+
 			if(shine_init==0){
 				shine_set_config_mpeg_defaults(&(sh_config.mpeg)); //--init sh_config struct
 				sh_config.wave.channels=PCM_MONO; //PCM_STEREO;//PCM_MONO;
@@ -1323,6 +1428,9 @@ sender_loop:
 				shine_init = 0;
 
 				audio_packet_type = 0;
+				/* Discard semaphore tokens already represented by the PCM that this
+				 * session drained.  Recording has stopped, so no new frames race here. */
+				while(ql_rtos_semaphore_wait(sender_sem, 0) == 0) {}
 			}
 	}
 }