fix(audio): prefer cloud timeout prompts with local fallback

ql-application/threadx/interface/custom_app/hal_key.c

@@ -187,6 +187,9 @@ static void key_thread_entry(void *parameter)
 					if(ql_rtos_get_systicks() - key_press_mic_time > KEY_LONG_PRESS_3S && power_down_status==0){
 						nfc_tx_click_count = 0;
 						power_down_status = 1;
+						/* Shutdown audio must own the codec.  Stop VAD and any
+						 * interrupted TTS before queueing the power-off prompt. */
+						stop_player_func();
 						app_play_tips_file(10);
 						power_down_status = 2;
 						//ql_rtos_task_sleep_s(3);

ql-application/threadx/interface/custom_app/http_downfile.c

@@ -34,6 +34,7 @@ const char *mp3_file[] ={
 	"volume_up_zh.mp3",//16  
 	"volume_down_zh.mp3",//17  
 	"ai_active_zh.mp3",//18
+	"barge_in_cute.mp3",//19  对话打断提示音
 };
 
 //http://banban.api.talkingq.com:8080/assets/roles/banban/tts_error.mp3
@@ -183,11 +184,13 @@ void app_file_check_download(void)
 	char filename_buff[64]={0x00};
 	int ret = 0, cnt = 0;
 	
-	for(index=1;index<19;index++){	
+	for(index=1;index<20;index++){
-		
+
-		if(index==4||index==5||index==6||index==7||index==8){
+		/* barge_in_cute.mp3 is shipped in customer_fs and is not hosted by the
-			continue;
+		 * legacy prompt download service. */
-		}
+		if(index==4||index==5||index==6||index==7||index==8||index==19){
+			continue;
+		}
 		
 		if(strlen(mp3_file[index])>0){
 			sprintf(filename_buff, "U:/%s", mp3_file[index]);
@@ -203,14 +206,16 @@ down_loop:
 	}
 }
 
-void app_file_check_del(void)
+void app_file_check_del(void)
 {
 	int index = 1;
 	char filename_buff[64]={0x00};
 	
-	for(index=1;index<19;index++){	
+	for(index=1;index<20;index++){
 		
-		if(index==4||index==5||index==6||index==7||index==8){
+		/* Keep the bundled barge-in prompt across app-only FOTA.  Unlike the
+		 * server-managed prompts, it cannot be restored by the downloader. */
+		if(index==4||index==5||index==6||index==7||index==8||index==19){
 			continue;
 		}
 		

ql-application/threadx/interface/custom_app/mp3_player.c

@@ -24,7 +24,8 @@ static Mp3PlayConfigInfo  mp3_configInfo;
 static unsigned char mp3_player_status = 0;
 QFILE *local_mp3_file_fd=NULL;
 
-static int mp3_player_end_frame = 0;
+static int mp3_player_end_frame = 0;
+static int mp3_player_last_result = 0;
 
 int current_sample_rate = 0;
 static	audio_track_handle track_hd;
@@ -278,9 +279,10 @@ u32 local_mp3_data_fetch(const char *uri, uint8_t *buffer, u32 length)
 	return read_bytes;
 }
 
-size_t url_mp3_data_fetch(const char *uri, uint8_t *buffer, size_t length)
+int url_mp3_data_fetch(const char *uri, uint8_t *buffer, size_t length)
-{
+{
 	int resp_status = -1;
+	int result = -1;
 	uint32_t nw = length;
 	uint32_t page_pos = 0;
 	uint32_t content_length = 0;
@@ -353,7 +355,10 @@ size_t url_mp3_data_fetch(const char *uri, uint8_t *buffer, size_t length)
 		total_length += page_pos;
 	}
 
-	 printf("end read:%d, %d\n", total_length, session->content_length);
+	printf("end read:%d, %d\n", total_length, session->content_length);
+	if(total_length == session->content_length){
+		result = 0;
+	}
 
 __exit:
 
@@ -363,13 +368,13 @@ __exit:
 		session = NULL;
 	}
 
-	if(total_length<content_length){
+	if(result != 0 || total_length < content_length){
-		total_length = 0;
+		total_length = 0;
-		return -1;
+		return -1;
-	}
+	}
-
+
-	return 0;
+	return result;
-}
+}
 
 
 struct mp3_decoder* aofei_mp3_decoder = NULL;
@@ -389,24 +394,45 @@ int aofei_mp3_player_init(void)
 
 void aofei_mp3_player(char *file_name)
 {
-	int ret = 0;
+	int ret = 0;
-	uint8_t *mp3_buff = malloc(100*1024);
+	uint8_t *mp3_buff = NULL;
-
+
-	memset(mp3_buff, 0, 100*1024);
+	mp3_player_last_result = -1;
-	//直接下载到一个100k的buffer
+	if(file_name == NULL){
-	if(strstr(file_name, "http:")){//播放url
+		printf("mp3 player file name is null\r\n");
-		ret = url_mp3_data_fetch(file_name, mp3_buff, 100*1024);
+		return;
+	}
+
+	mp3_buff = malloc(100*1024);
+	if(mp3_buff == NULL){
+		printf("mp3 player buffer malloc failed\r\n");
+		return;
+	}
+
+	memset(mp3_buff, 0, 100*1024);
+	//直接下载到一个100k的buffer
+	if(strncmp(file_name, "http://", strlen("http://")) == 0 ||
+	   strncmp(file_name, "https://", strlen("https://")) == 0){//播放url
+		ret = url_mp3_data_fetch(file_name, mp3_buff, 100*1024);
 	}else if(strstr(file_name, "test.mp3")){//播放本地音频
 		total_length = test_data_size;
 		memcpy(mp3_buff, test_data, total_length);
-	}else{//播放本地音频
+	}else{//播放本地音频
-		total_length = local_mp3_data_fetch(file_name, mp3_buff, 100*1024);
+		total_length = local_mp3_data_fetch(file_name, mp3_buff, 100*1024);
-	}
+	}
+
+	if(ret < 0 || total_length == 0){
+		ret = -1;
+		goto __exit;
+	}
   
 	mp3_player_end_frame = 0;
 	
 
-	if(aofei_mp3_player_init()==-1)return;
+	if(aofei_mp3_player_init()==-1){
+		ret = -1;
+		goto __exit;
+	}
 
 	if(ret>=0){
 		aofei_mp3_decoder->read_ptr = mp3_buff;
@@ -457,6 +483,7 @@ void aofei_mp3_player(char *file_name)
 		app_codec_pa_close();
 	}
 
+__exit:
 	if(aofei_mp3_decoder!=NULL){
 		mp3_decoder_delete(aofei_mp3_decoder);
 		aofei_mp3_decoder = NULL;
@@ -477,10 +504,17 @@ void aofei_mp3_player(char *file_name)
 		mp3_buff = NULL;
 	}
 
-	total_length = 0;
+	total_length = 0;
-	mp3_player_end_frame = 0;
+	mp3_player_end_frame = 0;
-	mp3_player_status = 0;
+	mp3_player_status = 0;
-}
+	mp3_player_last_result = ret;
+}
+
+/* 播放调用是同步的，队列线程可在返回后读取本次结果决定是否使用本地兜底。 */
+int aofei_mp3_player_get_result(void)
+{
+	return mp3_player_last_result;
+}
 
 
 

ql-application/threadx/interface/custom_app/ws_talkingq.c

@@ -30,8 +30,9 @@
 
 
 #define PCM_FRAME_LENGTH    1152
-#define CLOUD_TTS_ERROR_URL_FORMAT \
+#define CLOUD_TTS_ERROR_URL_FORMAT \
-    "http://101.35.224.118:8080/assets/roles/banban/tts_error/tts_error%02u.mp3"
+    "http://101.35.224.118:8080/assets/roles/banban/tts_error/tts_error%02u.mp3"
+#define LOCAL_TTS_ERROR_PATH "U:/tts_error.mp3"
 
 static int g_sim_ready = 0;
 
@@ -49,11 +50,14 @@ static uint8_t sender_buf[PCM_FRAME_LENGTH]={0};
 static int16 mp3_buff[2*PCM_FRAME_LENGTH];
 static int16 *ppbuff=mp3_buff;
 
-static int onmessage_status = -1;
+static int onmessage_status = -1;
-volatile int tts_playing = 0;
+volatile int tts_playing = 0;
-unsigned char wss_connect_start = 0;
+unsigned char wss_connect_start = 0;
 static volatile int player_mq_couter = 0;
-static unsigned char audio_in_task_cancel = 0;
+/* Set while a barge-in prompt bridges the old and new sessions.  The old
+ * server response can still deliver URLs after its player queue was cleared. */
+static volatile int barge_in_transition = 0;
+static unsigned char audio_in_task_cancel = 0;
 
 static unsigned char xiaoice_wss_connect_status = 0; 
 static unsigned char wss_connect_counter = 0;
@@ -62,17 +66,22 @@ int fota_start = 0;
 int mqtt_task_run_flag = 0;
 
 extern int sub_topic_status;
-extern int mqtt_task_run_flag;
+extern int mqtt_task_run_flag;
-static unsigned char shine_init = 0;
+static unsigned char shine_init = 0;
-extern int fisrt_net_start;
+extern int fisrt_net_start;
-extern u8 power_down_status;
+extern u8 power_down_status;
+extern int aofei_mp3_player_get_result(void);
+
+void sendAuthentication(void);
+void vad_start_func(void);
+void xiaoice_close_all_audio(void);
 
 int audio_packet_type = 0;
 
 
-typedef struct ice_work_item_s
+typedef struct ice_work_item_s
-{
+{
-    int type;  //0-tts_out 1-session_continue 2-session_end 3-error
+    int type;  //0-tts_out 1-session_continue 2-session_end 3-云端超时提示（失败时播放本地兜底）
     int error;
 	int del_url;
     void *data; 
@@ -195,16 +204,12 @@ static void remove_underscores(char *str) {
     *dst = '\0'; // 添加字符串结束符
 }
 
-/*
+/* 生成 banban 角色的随机云端对话超时提示音 URL。 */
- * 将后台下发的任意角色对话超时音频 URL 改为 banban 角色的 HTTP 地址，
+static int build_random_tts_error_url(char *url, size_t url_size)
- * 并在 tts_error00~tts_error09 中随机选择一条。
+{
- */
+    static uint32_t random_sequence = 0;
-static int randomize_tts_error_url(char *url, size_t url_size)
+    static unsigned int last_random_index = 10;
-{
+    uint32_t random_value;
-    static uint32_t random_sequence = 0;
-    static unsigned int last_random_index = 10;
-    char *file_name;
-    uint32_t random_value;
     unsigned int random_index;
     int written;
 
@@ -212,14 +217,6 @@ static int randomize_tts_error_url(char *url, size_t url_size)
         return 0;
     }
 
-    file_name = strrchr(url, '/');
-    file_name = (file_name == NULL) ? url : file_name + 1;
-
-    if(strncmp(file_name, "tts_error", strlen("tts_error")) != 0 &&
-       strncmp(file_name, "tts_err", strlen("tts_err")) != 0){
-        return 0;
-    }
-
     /*
      * 某些设备上硬件随机接口可能持续返回 0，因此混合伪随机数、系统 tick
      * 和递增序列；若仍与上一次相同，则向后偏移，避免连续重复播放。
@@ -241,9 +238,32 @@ static int randomize_tts_error_url(char *url, size_t url_size)
         return -1;
     }
 
-    printf("random cloud tts error index:%02u, url:%s\n", random_index, url);
+    printf("random cloud tts error index:%02u, url:%s\n", random_index, url);
-    return 1;
+    return 1;
-}
+}
+
+/*
+ * 后台下发任意角色的对话超时 URL 时，统一替换成 banban 的随机云端资源。
+ * 非对话超时 URL 保持不变。
+ */
+static int randomize_tts_error_url(char *url, size_t url_size)
+{
+    char *file_name;
+
+    if(url == NULL || url_size == 0){
+        return 0;
+    }
+
+    file_name = strrchr(url, '/');
+    file_name = (file_name == NULL) ? url : file_name + 1;
+
+    if(strncmp(file_name, "tts_error", strlen("tts_error")) != 0 &&
+       strncmp(file_name, "tts_err", strlen("tts_err")) != 0){
+        return 0;
+    }
+
+    return build_random_tts_error_url(url, url_size);
+}
 
 void add_file_to_play_player_queue(char *path, int type)
 {
@@ -299,6 +319,14 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
 			return;
 		}
 
+		/* The new session id is generated only when recording starts.  Until
+		 * then, every TTS message on this socket belongs to the interrupted
+		 * session and must not be allowed to refill the cleared player queue. */
+		if (barge_in_transition) {
+			printf("drop stale websocket message during barge-in\r\n");
+			return;
+		}
+
 	//	printf("message(txt), %d(byte): %s ", len, text);
 		
 		char url_buff[128]={0x00};
@@ -372,7 +400,8 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
 				if(tts_playing==2) return;
 				
 		
-				add_file_to_play_player_queue(url_buff, 0);
+				/* 对话超时提示使用 type=3，云端播放失败时由播放线程切换到本地兜底。 */
+				add_file_to_play_player_queue(url_buff, is_tts_error_url == 1 ? 3 : 0);
 			}
 		}
 
@@ -743,9 +772,12 @@ void generateSessionId(void)
 
 
 #if 1
-void sendStartPacket(void) {
+void sendStartPacket(void) {
-    generateSessionId();
+    generateSessionId();
-	generateDeviceId();
+	/* sessionId now identifies the new conversation.  Late URLs from the old
+	 * conversation will fail the existing session-id check below. */
+	barge_in_transition = 0;
+	generateDeviceId();
     gSequenceNumber = 0;
 
     AudioPacket startPacket = {0};
@@ -862,17 +894,23 @@ void sendRecordingAudioPacket(AudioPacket *packet) {
 
 extern const char *mp3_file[];
 
-int app_play_tips_file(int index)
+int app_play_tips_file(int index)
-{
+{
-	int file_valid_check = 0;
+	int file_valid_check = 0;
-	
+	int access_ret = 0;
-	printf("xiaoice_play_index:%d\n", index);
+
-	
+	printf("xiaoice_play_index:%d\n", index);
-		//正在录音过程中不允许播放
+
-//	if(xiaoice_get_tts_play_status() || get_start_mic_flag()==1) return -1;
+		//正在录音过程中不允许播放
-    if(power_down_status!=1){
+//	if(xiaoice_get_tts_play_status() || get_start_mic_flag()==1) return -1;
-		if(vad_audio_is_recording() || tts_playing!=0 || fota_start==1 || power_down_status==2 || get_sleep_mode()==1) return -1;
+    if(power_down_status!=1){
-    }
+		if(vad_audio_is_recording() || tts_playing!=0 || fota_start==1 || power_down_status==2 || get_sleep_mode()==1) {
+			printf("tips blocked:index=%d vad=%d tts=%d fota=%d power=%d sleep=%d\r\n",
+			       index, vad_audio_is_recording(), tts_playing, fota_start,
+			       power_down_status, get_sleep_mode());
+			return -1;
+		}
+    }
 	
 	char filename_buff[64]={0x00};
 	
@@ -881,15 +919,18 @@ int app_play_tips_file(int index)
 	if(index==0) {
 		file_valid_check = 0;
 	}else{ 
-		if(ql_access(filename_buff, 0)==0){	
+		access_ret = ql_access(filename_buff, 0);
-			file_valid_check = 1;
+		if(access_ret==0){
-		}else{
+			file_valid_check = 1;
-			file_valid_check = 2;
+		}else{
-		}
+			printf("tips file missing:index=%d path=%s access=%d\r\n",
+			       index, filename_buff, access_ret);
+			file_valid_check = 2;
+		}
 	}
 
 	if(file_valid_check!=2){
-		if(index==18){
+		if(index==18 || index==19){
 			add_file_to_play_player_queue(filename_buff, 2);
 		}else{
 			add_file_to_play_player_queue(filename_buff, 0);
@@ -944,9 +985,10 @@ int app_play_ota_tips_file(void)
 	return player_mq_couter > 0 ? 0 : -1;
 }
 
-void app_player_thread(void *arvg)
+void app_player_thread(void *arvg)
-{
+{
- 	ice_work_item *player_item=NULL; 	
+	ice_work_item *player_item=NULL;
+	int play_ret;
 	
     while(1){
       ql_rtos_queue_wait(player_queue, (u8*)(&player_item), sizeof(ice_work_item *), 0xFFFFFFFF);
@@ -956,12 +998,29 @@ void app_player_thread(void *arvg)
 			  if(player_item->data){
 			  	 // printf("xiaoice player! %s\n",player_item->data);		  
 
-			  	  app_set_app_operation(1);
+				  app_set_app_operation(1);
-				  aofei_mp3_player(player_item->data);
+				  aofei_mp3_player(player_item->data);
-			  	  app_set_app_operation(1);
+				  play_ret = aofei_mp3_player_get_result();
-
+				  app_set_app_operation(1);
-				  if(player_mq_couter>0) player_mq_couter--;  
+
-				  if(player_mq_couter==0){
+				  if(player_mq_couter>0) player_mq_couter--;
+
+				  /*
+				   * type=3 只用于对话超时提示。云端连接、HTTP、读取或播放器初始化
+				   * 失败时才排入本地兜底；如果用户已经开始下一轮录音，则丢弃旧提示。
+				   */
+				  if(play_ret != 0 && player_item->type == 3 &&
+				     !vad_audio_is_recording() && tts_playing_ret() != 2){
+					  printf("cloud tts error play failed:%d, fallback:%s\r\n",
+					         play_ret, LOCAL_TTS_ERROR_PATH);
+					  if(ql_access(LOCAL_TTS_ERROR_PATH, 0) == 0){
+						  add_file_to_play_player_queue(LOCAL_TTS_ERROR_PATH, 0);
+					  }else{
+						  printf("local tts error fallback missing:%s\r\n", LOCAL_TTS_ERROR_PATH);
+					  }
+				  }
+
+				  if(player_mq_couter==0){
 					  printf("player_mq_couter==0\r\n");
 
 					   if(tts_play_status==2){
@@ -1001,26 +1060,46 @@ void app_player_thread(void *arvg)
 
 int tts_play_interrupt_func(void)
 {
+	int playback_interrupted = 0;
+
 	//如果在vad状态，则直接返回
 	if(vad_audio_is_recording()||g_mic_working) return -1;
 
 	g_mic_working = 1;//等待转为vad状态就清0 或者拾音超时 或者ws没有正常
-	
+
-	xiaoice_close_all_audio();
+	/* Remember whether this key press is interrupting TTS.  tts_playing == 2
+	 * is only a cancellation signal for the old player; it must not leak into
+	 * the next conversation, otherwise prompts and error audio are discarded. */
+	playback_interrupted = (tts_playing != 0 || player_mq_couter > 0);
+	barge_in_transition = playback_interrupted;
+	xiaoice_close_all_audio();
 
 	talkingq_ws_connect();
 
-	if(get_xiaoice_connected_status()!=2) {
+	if(get_xiaoice_connected_status()!=2) {
-		g_mic_working = 0;
+		/* Let the caller play its local network-error prompt. */
-		return -2;
+		tts_play_status = 0;
-	}
+		tts_playing = 0;
-	
+		barge_in_transition = 0;
-	if(tts_playing==2){
+		g_mic_working = 0;
-		printf("vad_start_func0\n");
+		return -2;
-		vad_start_func();//如果是打断就不播报
+	}
+
+	if(playback_interrupted){
+		printf("barge_in_prompt\n");
+		/* The old player has finished consuming the cancellation state. */
+		tts_play_status = 0;
+		tts_playing = 0;
+		/* Use the same local-prompt flow as ai_active_zh.mp3. Queue type 2
+		 * starts VAD after playback; fall back to VAD if the file is absent. */
+		if(app_play_tips_file(19) != 0){
+			vad_start_func();
+		}
 	}else{
 		app_play_tips_file(18);
 	}
+
+	return 0;
 }
 
 void stop_player_func(void)
@@ -1028,6 +1107,7 @@ void stop_player_func(void)
 	vad_audio_stop_record();
 	xiaoice_close_all_audio();
 
+	tts_play_status = 0;
 	tts_playing = 0;
 }
 
@@ -1147,9 +1227,19 @@ static void check_connect_thread_entry(void *parameter)
 			printf("tts_error_set_speech1\n");
 			vad_set_speech_timeout_flag(1);
 			
-			app_play_tips_file(9);
+			/*
-			//talkingq_ws_send_text("REQUEST_PROMPT_SOUND:tts_error");
+			 * 后台没有返回任何音频 URL 时，也先由设备生成随机云端提示音。
-			//改为播放本地音频 听不清 请再说一遍
+			 * type=3 会在 HTTP 下载或播放初始化失败时自动切换到本地音频。
+			 */
+			{
+				char cloud_tts_error_url[128] = {0};
+				if(build_random_tts_error_url(cloud_tts_error_url,
+				                              sizeof(cloud_tts_error_url)) == 1){
+					add_file_to_play_player_queue(cloud_tts_error_url, 3);
+				}else{
+					app_play_tips_file(9);
+				}
+			}
 		}
 	#endif
 	
@@ -1194,9 +1284,16 @@ void xiaoice_close_all_audio(void)
 
 		timeout = hal_os_up_time_ms();
 
-		while(tts_play_status&&tts_playing!=2){
+		/* The queue counter also includes the item currently owned by the
+		 * player task.  Wait for that item to observe tts_playing == 2 and
+		 * finish cleanup before the cancellation state is reused/reset. */
+		while(player_mq_couter>0){
 			ql_rtos_task_sleep_ms(20);
-			if(hal_os_up_time_ms()-timeout>10000) break;
+			if(hal_os_up_time_ms()-timeout>OTA_AUDIO_STOP_TIMEOUT_MS){
+				printf("wait player stop timeout, counter:%d\r\n",
+				       player_mq_couter);
+				break;
+			}
 		}
 	}
 #endif