From 8b01debe4d4442e3dc19bd5afcfa2f501d8bb43a Mon Sep 17 00:00:00 2001 From: chenjintao Date: Tue, 22 Sep 2026 17:50:42 +0800 Subject: [PATCH 1/6] fix(audio): recover cleanly after barge-in --- .../threadx/interface/custom_app/hal_key.c | 3 ++ .../interface/custom_app/ws_talkingq.c | 40 ++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/ql-application/threadx/interface/custom_app/hal_key.c b/ql-application/threadx/interface/custom_app/hal_key.c index 0441fda..8eb650d 100644 --- a/ql-application/threadx/interface/custom_app/hal_key.c +++ b/ql-application/threadx/interface/custom_app/hal_key.c @@ -187,6 +187,9 @@ static void key_thread_entry(void *parameter) if(ql_rtos_get_systicks() - key_press_mic_time > KEY_LONG_PRESS_3S && power_down_status==0){ nfc_tx_click_count = 0; power_down_status = 1; + /* Shutdown audio must own the codec. Stop VAD and any + * interrupted TTS before queueing the power-off prompt. */ + stop_player_func(); app_play_tips_file(10); power_down_status = 2; //ql_rtos_task_sleep_s(3); diff --git a/ql-application/threadx/interface/custom_app/ws_talkingq.c b/ql-application/threadx/interface/custom_app/ws_talkingq.c index f0bc864..5046bab 100644 --- a/ql-application/threadx/interface/custom_app/ws_talkingq.c +++ b/ql-application/threadx/interface/custom_app/ws_talkingq.c @@ -1001,26 +1001,40 @@ void app_player_thread(void *arvg) int tts_play_interrupt_func(void) { + int playback_interrupted = 0; + //如果在vad状态，则直接返回 if(vad_audio_is_recording()||g_mic_working) return -1; g_mic_working = 1;//等待转为vad状态就清0 或者拾音超时 或者ws没有正常 - + + /* Remember whether this key press is interrupting TTS. tts_playing == 2 + * is only a cancellation signal for the old player; it must not leak into + * the next conversation, otherwise prompts and error audio are discarded. */ + playback_interrupted = (tts_playing != 0 || player_mq_couter > 0); xiaoice_close_all_audio(); talkingq_ws_connect(); - if(get_xiaoice_connected_status()!=2) { - g_mic_working = 0; - return -2; - } - - if(tts_playing==2){ + if(get_xiaoice_connected_status()!=2) { + /* Let the caller play its local network-error prompt. */ + tts_play_status = 0; + tts_playing = 0; + g_mic_working = 0; + return -2; + } + + if(playback_interrupted){ printf("vad_start_func0\n"); + /* The old player has finished consuming the cancellation state. */ + tts_play_status = 0; + tts_playing = 0; vad_start_func();//如果是打断就不播报 }else{ app_play_tips_file(18); } + + return 0; } void stop_player_func(void) @@ -1028,6 +1042,7 @@ void stop_player_func(void) vad_audio_stop_record(); xiaoice_close_all_audio(); + tts_play_status = 0; tts_playing = 0; } @@ -1194,9 +1209,16 @@ void xiaoice_close_all_audio(void) timeout = hal_os_up_time_ms(); - while(tts_play_status&&tts_playing!=2){ + /* The queue counter also includes the item currently owned by the + * player task. Wait for that item to observe tts_playing == 2 and + * finish cleanup before the cancellation state is reused/reset. */ + while(player_mq_couter>0){ ql_rtos_task_sleep_ms(20); - if(hal_os_up_time_ms()-timeout>10000) break; + if(hal_os_up_time_ms()-timeout>OTA_AUDIO_STOP_TIMEOUT_MS){ + printf("wait player stop timeout, counter:%d\r\n", + player_mq_couter); + break; + } } } #endif From cfdb193e75f643b8ac79195a46ae23119d33229b Mon Sep 17 00:00:00 2001 From: chenjintao Date: Tue, 22 Sep 2026 18:06:41 +0800 Subject: [PATCH 2/6] feat(audio): add local barge-in prompt --- .../threadx/interface/custom_app/http_downfile.c | 5 +++-- .../threadx/interface/custom_app/ws_talkingq.c | 10 +++++++--- ql-config/config/EG800AKCN_11LG/customer_fs.bin | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ql-application/threadx/interface/custom_app/http_downfile.c b/ql-application/threadx/interface/custom_app/http_downfile.c index 2be48a8..d3b05c0 100644 --- a/ql-application/threadx/interface/custom_app/http_downfile.c +++ b/ql-application/threadx/interface/custom_app/http_downfile.c @@ -34,6 +34,7 @@ const char *mp3_file[] ={ "volume_up_zh.mp3",//16 "volume_down_zh.mp3",//17 "ai_active_zh.mp3",//18 + "barge_in_cute.mp3",//19 对话打断提示音 }; //http://banban.api.talkingq.com:8080/assets/roles/banban/tts_error.mp3 @@ -183,7 +184,7 @@ void app_file_check_download(void) char filename_buff[64]={0x00}; int ret = 0, cnt = 0; - for(index=1;index<19;index++){ + for(index=1;index<20;index++){ if(index==4||index==5||index==6||index==7||index==8){ continue; @@ -208,7 +209,7 @@ void app_file_check_del(void) int index = 1; char filename_buff[64]={0x00}; - for(index=1;index<19;index++){ + for(index=1;index<20;index++){ if(index==4||index==5||index==6||index==7||index==8){ continue; diff --git a/ql-application/threadx/interface/custom_app/ws_talkingq.c b/ql-application/threadx/interface/custom_app/ws_talkingq.c index 5046bab..23a948e 100644 --- a/ql-application/threadx/interface/custom_app/ws_talkingq.c +++ b/ql-application/threadx/interface/custom_app/ws_talkingq.c @@ -889,7 +889,7 @@ int app_play_tips_file(int index) } if(file_valid_check!=2){ - if(index==18){ + if(index==18 || index==19){ add_file_to_play_player_queue(filename_buff, 2); }else{ add_file_to_play_player_queue(filename_buff, 0); @@ -1025,11 +1025,15 @@ int tts_play_interrupt_func(void) } if(playback_interrupted){ - printf("vad_start_func0\n"); + printf("barge_in_prompt\n"); /* The old player has finished consuming the cancellation state. */ tts_play_status = 0; tts_playing = 0; - vad_start_func();//如果是打断就不播报 + /* Use the same local-prompt flow as ai_active_zh.mp3. Queue type 2 + * starts VAD after playback; fall back to VAD if the file is absent. */ + if(app_play_tips_file(19) != 0){ + vad_start_func(); + } }else{ app_play_tips_file(18); } diff --git a/ql-config/config/EG800AKCN_11LG/customer_fs.bin b/ql-config/config/EG800AKCN_11LG/customer_fs.bin index 78c7470..d0e6854 100644 --- a/ql-config/config/EG800AKCN_11LG/customer_fs.bin +++ b/ql-config/config/EG800AKCN_11LG/customer_fs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:08eba5e80158dc312326f81f1ffef9fc8af648f084e2d8d1bc623b58d275c67a +oid sha256:827033779b1fddea9a360e560f379655fd9378d373dc5b278e92fc7bf94a769c size 262144 From d07b3f6771a6992636ef2c48045949620ab01577 Mon Sep 17 00:00:00 2001 From: chenjintao Date: Wed, 23 Sep 2026 10:57:12 +0800 Subject: [PATCH 3/6] fix(audio): preserve barge-in prompt across OTA --- .../interface/custom_app/http_downfile.c | 16 +++++--- .../interface/custom_app/ws_talkingq.c | 41 +++++++++++-------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/ql-application/threadx/interface/custom_app/http_downfile.c b/ql-application/threadx/interface/custom_app/http_downfile.c index d3b05c0..9c1819a 100644 --- a/ql-application/threadx/interface/custom_app/http_downfile.c +++ b/ql-application/threadx/interface/custom_app/http_downfile.c @@ -185,10 +185,12 @@ void app_file_check_download(void) int ret = 0, cnt = 0; for(index=1;index<20;index++){ - - if(index==4||index==5||index==6||index==7||index==8){ - continue; - } + + /* barge_in_cute.mp3 is shipped in customer_fs and is not hosted by the + * legacy prompt download service. */ + if(index==4||index==5||index==6||index==7||index==8||index==19){ + continue; + } if(strlen(mp3_file[index])>0){ sprintf(filename_buff, "U:/%s", mp3_file[index]); @@ -204,14 +206,16 @@ down_loop: } } -void app_file_check_del(void) +void app_file_check_del(void) { int index = 1; char filename_buff[64]={0x00}; for(index=1;index<20;index++){ - if(index==4||index==5||index==6||index==7||index==8){ + /* Keep the bundled barge-in prompt across app-only FOTA. Unlike the + * server-managed prompts, it cannot be restored by the downloader. */ + if(index==4||index==5||index==6||index==7||index==8||index==19){ continue; } diff --git a/ql-application/threadx/interface/custom_app/ws_talkingq.c b/ql-application/threadx/interface/custom_app/ws_talkingq.c index 23a948e..6eb3f15 100644 --- a/ql-application/threadx/interface/custom_app/ws_talkingq.c +++ b/ql-application/threadx/interface/custom_app/ws_talkingq.c @@ -862,17 +862,23 @@ void sendRecordingAudioPacket(AudioPacket *packet) { extern const char *mp3_file[]; -int app_play_tips_file(int index) -{ - int file_valid_check = 0; - - printf("xiaoice_play_index:%d\n", index); - - //正在录音过程中不允许播放 -// if(xiaoice_get_tts_play_status() || get_start_mic_flag()==1) return -1; - if(power_down_status!=1){ - if(vad_audio_is_recording() || tts_playing!=0 || fota_start==1 || power_down_status==2 || get_sleep_mode()==1) return -1; - } +int app_play_tips_file(int index) +{ + int file_valid_check = 0; + int access_ret = 0; + + printf("xiaoice_play_index:%d\n", index); + + //正在录音过程中不允许播放 +// if(xiaoice_get_tts_play_status() || get_start_mic_flag()==1) return -1; + if(power_down_status!=1){ + if(vad_audio_is_recording() || tts_playing!=0 || fota_start==1 || power_down_status==2 || get_sleep_mode()==1) { + printf("tips blocked:index=%d vad=%d tts=%d fota=%d power=%d sleep=%d\r\n", + index, vad_audio_is_recording(), tts_playing, fota_start, + power_down_status, get_sleep_mode()); + return -1; + } + } char filename_buff[64]={0x00}; @@ -881,11 +887,14 @@ int app_play_tips_file(int index) if(index==0) { file_valid_check = 0; }else{ - if(ql_access(filename_buff, 0)==0){ - file_valid_check = 1; - }else{ - file_valid_check = 2; - } + access_ret = ql_access(filename_buff, 0); + if(access_ret==0){ + file_valid_check = 1; + }else{ + printf("tips file missing:index=%d path=%s access=%d\r\n", + index, filename_buff, access_ret); + file_valid_check = 2; + } } if(file_valid_check!=2){ From d3d1b6e7fbaae8e615067d19e301f5bf1a302a6e Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 23 Sep 2026 11:49:45 +0800 Subject: [PATCH 4/6] fix(audio): rebuild valid customer fs image --- ql-config/config/EG800AKCN_11LG/customer_fs.bin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ql-config/config/EG800AKCN_11LG/customer_fs.bin b/ql-config/config/EG800AKCN_11LG/customer_fs.bin index d0e6854..5eb3fbf 100644 --- a/ql-config/config/EG800AKCN_11LG/customer_fs.bin +++ b/ql-config/config/EG800AKCN_11LG/customer_fs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:827033779b1fddea9a360e560f379655fd9378d373dc5b278e92fc7bf94a769c +oid sha256:b0451bcad51575a2050c152af84ce1fc13acc1f226cc1793b0043b7357197920 size 262144 From 1cd3c638901c3c0af520e8a9736c476813e690ac Mon Sep 17 00:00:00 2001 From: chenjintao Date: Wed, 23 Sep 2026 12:05:30 +0800 Subject: [PATCH 5/6] fix:drop ws message during barge in --- .../interface/custom_app/ws_talkingq.c | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/ql-application/threadx/interface/custom_app/ws_talkingq.c b/ql-application/threadx/interface/custom_app/ws_talkingq.c index 6eb3f15..a1f0e4f 100644 --- a/ql-application/threadx/interface/custom_app/ws_talkingq.c +++ b/ql-application/threadx/interface/custom_app/ws_talkingq.c @@ -49,11 +49,14 @@ static uint8_t sender_buf[PCM_FRAME_LENGTH]={0}; static int16 mp3_buff[2*PCM_FRAME_LENGTH]; static int16 *ppbuff=mp3_buff; -static int onmessage_status = -1; -volatile int tts_playing = 0; -unsigned char wss_connect_start = 0; +static int onmessage_status = -1; +volatile int tts_playing = 0; +unsigned char wss_connect_start = 0; static volatile int player_mq_couter = 0; -static unsigned char audio_in_task_cancel = 0; +/* Set while a barge-in prompt bridges the old and new sessions. The old + * server response can still deliver URLs after its player queue was cleared. */ +static volatile int barge_in_transition = 0; +static unsigned char audio_in_task_cancel = 0; static unsigned char xiaoice_wss_connect_status = 0; static unsigned char wss_connect_counter = 0; @@ -299,6 +302,14 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i return; } + /* The new session id is generated only when recording starts. Until + * then, every TTS message on this socket belongs to the interrupted + * session and must not be allowed to refill the cleared player queue. */ + if (barge_in_transition) { + printf("drop stale websocket message during barge-in\r\n"); + return; + } + // printf("message(txt), %d(byte): %s ", len, text); char url_buff[128]={0x00}; @@ -743,9 +754,12 @@ void generateSessionId(void) #if 1 -void sendStartPacket(void) { - generateSessionId(); - generateDeviceId(); +void sendStartPacket(void) { + generateSessionId(); + /* sessionId now identifies the new conversation. Late URLs from the old + * conversation will fail the existing session-id check below. */ + barge_in_transition = 0; + generateDeviceId(); gSequenceNumber = 0; AudioPacket startPacket = {0}; @@ -1021,7 +1035,8 @@ int tts_play_interrupt_func(void) * is only a cancellation signal for the old player; it must not leak into * the next conversation, otherwise prompts and error audio are discarded. */ playback_interrupted = (tts_playing != 0 || player_mq_couter > 0); - xiaoice_close_all_audio(); + barge_in_transition = playback_interrupted; + xiaoice_close_all_audio(); talkingq_ws_connect(); @@ -1029,6 +1044,7 @@ int tts_play_interrupt_func(void) /* Let the caller play its local network-error prompt. */ tts_play_status = 0; tts_playing = 0; + barge_in_transition = 0; g_mic_working = 0; return -2; } From 5f5604e45bf31d8f590b55b75bc512bb43728972 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 23 Sep 2026 12:14:11 +0800 Subject: [PATCH 6/6] fix(audio): increase barge-in prompt volume --- ql-config/config/EG800AKCN_11LG/customer_fs.bin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ql-config/config/EG800AKCN_11LG/customer_fs.bin b/ql-config/config/EG800AKCN_11LG/customer_fs.bin index 5eb3fbf..5066bf7 100644 --- a/ql-config/config/EG800AKCN_11LG/customer_fs.bin +++ b/ql-config/config/EG800AKCN_11LG/customer_fs.bin @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b0451bcad51575a2050c152af84ce1fc13acc1f226cc1793b0043b7357197920 +oid sha256:1c498e97811cc8e4f4732936cc26cd42fb37c9c9f4795d2cab9135dca2636c16 size 262144