chenjintao/banban
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(audio): reject stale replies during voice startup

Browse Source
This commit is contained in:
chenjintao
2026-10-13 10:04:31 +08:00
parent 00e9509dd0
commit 109a95497a
2 changed files with 170 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ql-application/threadx/interface/custom_app/ws_talkingq.c
+134 -16
View File
@@ -62,6 +62,9 @@ enum { WAIT_NONE, WAIT_NO_SPEECH, WAIT_REPLY, WAIT_SEND_FAILURE };
static int conversation_wait = WAIT_NONE;
static int timeout_action_busy = 0;
static int round_response_received = 0;
static unsigned int active_session_round = 0;
static int active_session_valid = 0;
static uint64_t startup_start_ms = 0;
static void conversation_lock(void)
{
@@ -79,6 +82,8 @@ static void conversation_begin_round(void)
conversation_round++;
conversation_wait = WAIT_NONE;
round_response_received = 0;
active_session_valid = 0;
startup_start_ms = hal_os_up_time_ms();
}
void conversation_record_begin(void)
@@ -319,27 +324,27 @@ static int randomize_tts_error_url(char *url, size_t url_size)
return build_random_tts_error_url(url, url_size);
}
void add_file_to_play_player_queue(char *path, int type)
int add_file_to_play_player_queue(char *path, int type)
{
ice_work_item *item = NULL;
// 安全修复：检查path参数有效性
if(path == NULL || strlen(path) == 0){
printf("add_file_to_play_player_queue: invalid path\n");
return;
return -1;
}
item = malloc(sizeof(ice_work_item));
if(item == NULL){
printf("add_file_to_play_player_queue: malloc item failed\n");
return;
return -1;
}
item->data = malloc(strlen(path)+1);
if(item->data == NULL){
printf("add_file_to_play_player_queue: malloc data failed\n");
free(item);
return;
return -1;
}
item->type = type;
@@ -350,7 +355,7 @@ void add_file_to_play_player_queue(char *path, int type)
printf("play file:%s\r\n", (char *)item->data);
if (ql_rtos_queue_release(player_queue, sizeof(item), &item, (type == 3 || type == 4) ? 0 : 0xFFFFFFFF) != 0)
if (ql_rtos_queue_release(player_queue, sizeof(item), &item, 0) != 0)
{
printf("player_mq rt_mq_send ERR\n");
free(item->data);
@@ -358,16 +363,18 @@ void add_file_to_play_player_queue(char *path, int type)
free(item);
item = NULL;
return -1;
}else{
player_mq_couter++;
printf("player_mq_couter:%d\r\n", player_mq_couter);
return 0;
}
}
//如果录音后 超过10s 还没有收到信息 提示本轮对话结束
static void onmessage_text(rws_socket socket, const char *text, const unsigned int len)
static void onmessage_text_locked(rws_socket socket, const char *text, const unsigned int len)
{
if (fota_start == 1) {
printf("ignore websocket audio during OTA\r\n");
@@ -454,10 +461,9 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
tts_play_status = 2;
}
conversation_lock();
conversation_wait = WAIT_NONE;
round_response_received = 1;
conversation_unlock();
if(is_tts_error_url == 1 || is_message_ok_url){
printf("tts_error or message_ok\n");
@@ -475,6 +481,81 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i
}
/* Untagged error prompts cannot identify an old server request. At least
* reject them throughout local startup and before this round sends START. */
static int conversation_accept_voice_message(const char *text)
{
int voice = strstr(text, "TTS_") != NULL ||
strstr(text, "END_ACK") != NULL ||
strstr(text, "START_ACK") != NULL ||
strstr(text, "/tts_error") != NULL ||
strstr(text, "/tts_err") != NULL;
return !voice || (active_session_valid &&
active_session_round == conversation_round && !g_mic_working);
}
/* Recognize the device's 32-character session-id format, including URL
* underscores. This is a candidate check, not a generic protocol parser. */
static int conversation_find_session_id(const char *text, unsigned int len, char *id)
{
unsigned int i, j, n;
for (i = 0; i < len; i++) {
if (text[i] < '0' || text[i] > '9') continue;
n = 0;
for (j = i; j < len && n < 32; j++) {
char ch = text[j];
if (ch == '_') continue;
if (n < 14 ? (ch < '0' || ch > '9') :
!((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z'))) break;
id[n++] = ch;
}
if (n == 32 && (j == len ||
!((text[j] >= '0' && text[j] <= '9') ||
(text[j] >= 'A' && text[j] <= 'Z') ||
(text[j] >= 'a' && text[j] <= 'z')))) {
id[32] = '\0';
return 1;
}
}
id[0] = '\0';
return 0;
}
static void conversation_log_error_message(const char *text, unsigned int len)
{
char candidate[33];
unsigned int offset;
int found = conversation_find_session_id(text, len, candidate);
int field = strstr(text, "sessionId") != NULL ||
strstr(text, "session_id") != NULL || strstr(text, "sessionID") != NULL;
printf("[WSS-ERR-ID] round=%u session_valid=%d last_session=%s field=%d candidate=%s match_last=%d len=%u\n",
conversation_round, active_session_valid, sessionId, field,
found ? candidate : "none", found && strcmp(candidate, sessionId) == 0, len);
/* Log before URL randomization and before rejecting stale replies. */
for (offset = 0; offset < len; offset += 128) {
unsigned int chunk = len - offset;
if (chunk > 128) chunk = 128;
printf("[WSS-ERR-RAW] offset=%u/%u %.*s\n", offset, len,
(int)chunk, text + offset);
}
}
static void onmessage_text(rws_socket socket, const char *text, const unsigned int len)
{
conversation_lock();
if (strstr(text, "tts_error") != NULL || strstr(text, "tts_err") != NULL ||
strstr(text, "PROMPT_SOUND_URL") != NULL) {
conversation_log_error_message(text, len);
}
if (!conversation_accept_voice_message(text)) {
printf("drop old voice reply:round=%u startup=%d\n",
conversation_round, g_mic_working);
} else {
onmessage_text_locked(socket, text, len);
}
conversation_unlock();
}
static void onmessage_bin(rws_socket socket, const void * data, const unsigned int len)
{
LOG_D("message(bin): %d ", len);
@@ -839,7 +920,9 @@ void generateSessionId(void)
#if 1
void sendStartPacket(void) {
conversation_lock();
generateSessionId();
active_session_round = conversation_round;
/* sessionId now identifies the new conversation. Late URLs from the old
* conversation will fail the existing session-id check below. */
barge_in_transition = 0;
@@ -851,7 +934,8 @@ void sendStartPacket(void) {
startPacket.packetType = 1;
startPacket.dataSize = 0;
sendRecordingAudioPacket(&startPacket);
active_session_valid = sendRecordingAudioPacket(&startPacket) == 0;
conversation_unlock();
// rt_kprintf("sendStartPacket, seq=%u\r\n", startPacket.sequenceNumber);
}
@@ -1007,9 +1091,9 @@ int app_play_tips_file(int index)
if(file_valid_check!=2){
if(index==18 || index==19){
add_file_to_play_player_queue(filename_buff, 2);
return add_file_to_play_player_queue(filename_buff, 2);
}else{
add_file_to_play_player_queue(filename_buff, 0);
return add_file_to_play_player_queue(filename_buff, 0);
}
}else{
return -2;
@@ -1061,6 +1145,20 @@ int app_play_ota_tips_file(void)
return player_mq_couter > 0 ? 0 : -1;
}
static void conversation_finish_startup_prompt(unsigned int round, int result)
{
int start;
conversation_lock();
start = round == conversation_round && g_mic_working && tts_playing != 2;
if (start) startup_start_ms = hal_os_up_time_ms();
conversation_unlock();
if (start) {
printf("voice startup prompt finished:round=%u result=%d\n", round, result);
/* Queue fullness must not suppress this prompt's startup action. */
vad_start_func();
}
}
void app_player_thread(void *arvg)
{
ice_work_item *player_item=NULL;
@@ -1072,7 +1170,7 @@ void app_player_thread(void *arvg)
if(player_item!=NULL){
int stale_timeout;
conversation_lock();
stale_timeout = (player_item->type == 3 || player_item->type == 4) &&
stale_timeout = (player_item->type == 2 || player_item->type == 3 || player_item->type == 4) &&
player_item->timeout_round != conversation_round;
conversation_unlock();
if (stale_timeout) {
@@ -1133,11 +1231,11 @@ void app_player_thread(void *arvg)
if(player_item->type==1){
printf("vad_leave_message\r\n");
vad_leave_message();
}else if(player_item->type==2){
// vad_tts_playback_finished();
vad_start_func();
}
}
if (player_item->type == 2) {
conversation_finish_startup_prompt(player_item->timeout_round, play_ret);
}
// }
}
if(player_item->data){
@@ -1154,6 +1252,9 @@ static int conversation_try_begin(void)
{
conversation_lock();
if(vad_audio_is_recording() || g_mic_working || shine_init || timeout_action_busy) {
printf("voice key ignored:round=%u vad=%d startup=%d sender=%d timeout=%d\n",
conversation_round, vad_audio_is_recording(), g_mic_working,
shine_init, timeout_action_busy);
conversation_unlock();
return -1;
}
@@ -1226,7 +1327,13 @@ int tts_play_interrupt_func(void)
vad_start_func();
}
}else{
app_play_tips_file(18);
if (app_play_tips_file(18) != 0) {
conversation_lock();
g_mic_working = 0;
conversation_wait = WAIT_NONE;
conversation_unlock();
return -2;
}
}
return 0;
@@ -1254,6 +1361,17 @@ static void conversation_check_timeout(void)
int expired = WAIT_NONE;
unsigned int expired_round = 0;
conversation_lock();
/* A lost prompt/trigger must not leave all future keys permanently blocked.
* Never recover during an active player, upload or recording. */
if (g_mic_working && !vad_audio_is_recording() && !shine_init &&
player_mq_couter == 0 && !timeout_action_busy &&
hal_os_up_time_ms() - startup_start_ms > 30000) {
printf("recover stalled voice startup:round=%u\n", conversation_round);
g_mic_working = 0;
conversation_wait = WAIT_NONE;
active_session_valid = 0;
vad_reset_conversation_flags();
}
if (conversation_wait != WAIT_NONE &&
wait_round == conversation_round &&
!vad_audio_is_recording() && !g_mic_working &&
tools/test_conversation_timeout.py
+36 -4
View File
@@ -24,10 +24,11 @@ harness = r'''
#include <stdint.h>
#include <stdio.h>
#include <stddef.h>
#include <string.h>
typedef void *ql_mutex_t;
static int g_mic_working, recording, shine_init, fota_start;
static int tts_playing, tts_play_status, audio_packet_type, no_speech_timeout;
static int queued, disconnected, flags_reset, mutex_locked;
static int queued, disconnected, flags_reset, mutex_locked, player_mq_couter, vad_starts;
static uint64_t now;
static unsigned int gSequenceNumber;
static int end_result;
@@ -42,6 +43,7 @@ static void vad_set_speech_timeout_flag(int value) { no_speech_timeout = value;
static void vad_reset_conversation_flags(void) { no_speech_timeout = 0; flags_reset++; }
static int build_random_tts_error_url(char *p, size_t n) { (void)p; (void)n; return 1; }
static void add_file_to_play_player_queue(char *p, int type) { (void)p; assert(type == 3); queued++; }
static void vad_start_func(void) { vad_starts++; recording = 1; g_mic_working = 0; }
static void app_play_tips_file(int n) { (void)n; assert(0); }
static int conversation_try_begin(void);
static void talkingq_ws_disconnect(void) {
@@ -52,7 +54,7 @@ static void talkingq_ws_disconnect(void) {
static int sendRecordingAudioPacket(AudioPacket *p) { assert(p->packetType == 2); return end_result; }
'''
harness += source[state_start:state_end]
for name in ['conversation_try_begin', 'sendEndPacket', 'conversation_check_timeout']:
for name in ['conversation_try_begin', 'sendEndPacket', 'conversation_check_timeout', 'conversation_accept_voice_message', 'conversation_finish_startup_prompt', 'conversation_find_session_id']:
harness += function(name)
harness += r'''
static void reset(void) {
@@ -60,9 +62,16 @@ static void reset(void) {
conversation_round = wait_round = sender_round = 0;
conversation_wait = WAIT_NONE; timeout_action_busy = round_response_received = 0;
queued = disconnected = flags_reset = no_speech_timeout = mutex_locked = 0;
now = 1000; end_result = 0;
now = 1000; end_result = 0; player_mq_couter = vad_starts = 0; active_session_valid = 0; startup_start_ms = 0; tts_playing = 0;
}
int main(void) {
char id[33];
const char *tagged = "PROMPT_SOUND_URL:http://host/26101309423304_EFGHIJKLMNOPQRSTUV.mp3";
const char *plain = "PROMPT_SOUND_URL:http://host/tts_error.mp3";
assert(conversation_find_session_id(tagged, (unsigned int)strlen(tagged), id));
assert(strcmp(id, "26101309423304EFGHIJKLMNOPQRSTUV") == 0);
assert(!conversation_find_session_id(plain, (unsigned int)strlen(plain), id));
assert(!conversation_find_session_id(tagged, 20, id));
/* No confirmed speech: prompt remains, without disconnect or reply wait. */
reset(); assert(conversation_try_begin() == 0); g_mic_working = 0;
conversation_no_speech(); conversation_check_timeout();
@@ -94,7 +103,30 @@ int main(void) {
reset(); conversation_no_speech(); conversation_cancel_wait(); conversation_check_timeout(); assert(!queued);
reset(); conversation_wait = WAIT_REPLY; wait_round = 9; now += 20000;
conversation_check_timeout(); assert(!queued && !disconnected);
puts("PASS: no-speech prompt, new-round cancellation, ignored recording keys, END deadline, stale rounds, early reply, send failure, timeout/key race, auto-listening");
/* Reproduce ai_active followed by a late untagged backend error. */
reset(); active_session_valid = 1; active_session_round = 0;
assert(conversation_try_begin() == 0);
assert(!conversation_accept_voice_message("PROMPT_SOUND_URL:http://host/tts_error.mp3"));
assert(!conversation_accept_voice_message("TTS_START"));
assert(conversation_accept_voice_message("NFC_SOUND_URL:http://host/card.mp3"));
/* A following queue entry cannot consume the startup action. */
player_mq_couter = 1;
conversation_finish_startup_prompt(conversation_round, 0);
assert(vad_starts == 1 && recording && !g_mic_working);
assert(!conversation_accept_voice_message("PROMPT_SOUND_URL:http://host/tts_error.mp3"));
active_session_valid = 1; active_session_round = conversation_round;
assert(conversation_accept_voice_message("PROMPT_SOUND_URL:http://host/tts_error.mp3"));
/* Stale prompt completion must not start another round. */
reset(); assert(conversation_try_begin() == 0);
conversation_finish_startup_prompt(conversation_round - 1, 0); assert(!vad_starts);
conversation_finish_startup_prompt(conversation_round, -1); assert(vad_starts == 1);
/* Lost startup releases reservation; active work is never recovered. */
reset(); assert(conversation_try_begin() == 0); now += 30001;
conversation_check_timeout(); assert(!g_mic_working);
assert(conversation_try_begin() == 0);
now += 30001; player_mq_couter = 1; conversation_check_timeout(); assert(g_mic_working);
player_mq_couter = 0; recording = 1; conversation_check_timeout(); assert(g_mic_working);
puts("PASS: late backend error admission, prompt completion with queued audio, stale prompt, failed playback, stalled-startup recovery; no-speech prompt, new-round cancellation, ignored recording keys, END deadline, stale rounds, early reply, send failure, timeout/key race, auto-listening");
}
'''
with tempfile.TemporaryDirectory(prefix='banban-timeout-') as directory: