diff --git a/ql-application/threadx/interface/custom_app/ws_talkingq.c b/ql-application/threadx/interface/custom_app/ws_talkingq.c index bbbaad8..b2f41db 100644 --- a/ql-application/threadx/interface/custom_app/ws_talkingq.c +++ b/ql-application/threadx/interface/custom_app/ws_talkingq.c @@ -62,6 +62,9 @@ enum { WAIT_NONE, WAIT_NO_SPEECH, WAIT_REPLY, WAIT_SEND_FAILURE }; static int conversation_wait = WAIT_NONE; static int timeout_action_busy = 0; static int round_response_received = 0; +static unsigned int active_session_round = 0; +static int active_session_valid = 0; +static uint64_t startup_start_ms = 0; static void conversation_lock(void) { @@ -79,6 +82,8 @@ static void conversation_begin_round(void) conversation_round++; conversation_wait = WAIT_NONE; round_response_received = 0; + active_session_valid = 0; + startup_start_ms = hal_os_up_time_ms(); } void conversation_record_begin(void) @@ -319,27 +324,27 @@ static int randomize_tts_error_url(char *url, size_t url_size) return build_random_tts_error_url(url, url_size); } -void add_file_to_play_player_queue(char *path, int type) +int add_file_to_play_player_queue(char *path, int type) { ice_work_item *item = NULL; // 安全修复：检查path参数有效性 if(path == NULL || strlen(path) == 0){ printf("add_file_to_play_player_queue: invalid path\n"); - return; + return -1; } item = malloc(sizeof(ice_work_item)); if(item == NULL){ printf("add_file_to_play_player_queue: malloc item failed\n"); - return; + return -1; } item->data = malloc(strlen(path)+1); if(item->data == NULL){ printf("add_file_to_play_player_queue: malloc data failed\n"); free(item); - return; + return -1; } item->type = type; @@ -350,7 +355,7 @@ void add_file_to_play_player_queue(char *path, int type) printf("play file:%s\r\n", (char *)item->data); - if (ql_rtos_queue_release(player_queue, sizeof(item), &item, (type == 3 || type == 4) ? 0 : 0xFFFFFFFF) != 0) + if (ql_rtos_queue_release(player_queue, sizeof(item), &item, 0) != 0) { printf("player_mq rt_mq_send ERR\n"); free(item->data); @@ -358,16 +363,18 @@ void add_file_to_play_player_queue(char *path, int type) free(item); item = NULL; + return -1; }else{ player_mq_couter++; printf("player_mq_couter:%d\r\n", player_mq_couter); + return 0; } } //如果录音后 超过10s 还没有收到信息 提示本轮对话结束 -static void onmessage_text(rws_socket socket, const char *text, const unsigned int len) +static void onmessage_text_locked(rws_socket socket, const char *text, const unsigned int len) { if (fota_start == 1) { printf("ignore websocket audio during OTA\r\n"); @@ -454,10 +461,9 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i tts_play_status = 2; } - conversation_lock(); + conversation_wait = WAIT_NONE; round_response_received = 1; - conversation_unlock(); if(is_tts_error_url == 1 || is_message_ok_url){ printf("tts_error or message_ok\n"); @@ -475,6 +481,81 @@ static void onmessage_text(rws_socket socket, const char *text, const unsigned i } +/* Untagged error prompts cannot identify an old server request. At least + * reject them throughout local startup and before this round sends START. */ +static int conversation_accept_voice_message(const char *text) +{ + int voice = strstr(text, "TTS_") != NULL || + strstr(text, "END_ACK") != NULL || + strstr(text, "START_ACK") != NULL || + strstr(text, "/tts_error") != NULL || + strstr(text, "/tts_err") != NULL; + return !voice || (active_session_valid && + active_session_round == conversation_round && !g_mic_working); +} + +/* Recognize the device's 32-character session-id format, including URL + * underscores. This is a candidate check, not a generic protocol parser. */ +static int conversation_find_session_id(const char *text, unsigned int len, char *id) +{ + unsigned int i, j, n; + for (i = 0; i < len; i++) { + if (text[i] < '0' || text[i] > '9') continue; + n = 0; + for (j = i; j < len && n < 32; j++) { + char ch = text[j]; + if (ch == '_') continue; + if (n < 14 ? (ch < '0' || ch > '9') : + !((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z'))) break; + id[n++] = ch; + } + if (n == 32 && (j == len || + !((text[j] >= '0' && text[j] <= '9') || + (text[j] >= 'A' && text[j] <= 'Z') || + (text[j] >= 'a' && text[j] <= 'z')))) { + id[32] = '\0'; + return 1; + } + } + id[0] = '\0'; + return 0; +} + +static void conversation_log_error_message(const char *text, unsigned int len) +{ + char candidate[33]; + unsigned int offset; + int found = conversation_find_session_id(text, len, candidate); + int field = strstr(text, "sessionId") != NULL || + strstr(text, "session_id") != NULL || strstr(text, "sessionID") != NULL; + printf("[WSS-ERR-ID] round=%u session_valid=%d last_session=%s field=%d candidate=%s match_last=%d len=%u\n", + conversation_round, active_session_valid, sessionId, field, + found ? candidate : "none", found && strcmp(candidate, sessionId) == 0, len); + /* Log before URL randomization and before rejecting stale replies. */ + for (offset = 0; offset < len; offset += 128) { + unsigned int chunk = len - offset; + if (chunk > 128) chunk = 128; + printf("[WSS-ERR-RAW] offset=%u/%u %.*s\n", offset, len, + (int)chunk, text + offset); + } +} + +static void onmessage_text(rws_socket socket, const char *text, const unsigned int len) +{ + conversation_lock(); + if (strstr(text, "tts_error") != NULL || strstr(text, "tts_err") != NULL || + strstr(text, "PROMPT_SOUND_URL") != NULL) { + conversation_log_error_message(text, len); + } + if (!conversation_accept_voice_message(text)) { + printf("drop old voice reply:round=%u startup=%d\n", + conversation_round, g_mic_working); + } else { + onmessage_text_locked(socket, text, len); + } + conversation_unlock(); +} + static void onmessage_bin(rws_socket socket, const void * data, const unsigned int len) { LOG_D("message(bin): %d ", len); @@ -839,7 +920,9 @@ void generateSessionId(void) #if 1 void sendStartPacket(void) { + conversation_lock(); generateSessionId(); + active_session_round = conversation_round; /* sessionId now identifies the new conversation. Late URLs from the old * conversation will fail the existing session-id check below. */ barge_in_transition = 0; @@ -851,7 +934,8 @@ void sendStartPacket(void) { startPacket.packetType = 1; startPacket.dataSize = 0; - sendRecordingAudioPacket(&startPacket); + active_session_valid = sendRecordingAudioPacket(&startPacket) == 0; + conversation_unlock(); // rt_kprintf("sendStartPacket, seq=%u\r\n", startPacket.sequenceNumber); } @@ -1007,9 +1091,9 @@ int app_play_tips_file(int index) if(file_valid_check!=2){ if(index==18 || index==19){ - add_file_to_play_player_queue(filename_buff, 2); + return add_file_to_play_player_queue(filename_buff, 2); }else{ - add_file_to_play_player_queue(filename_buff, 0); + return add_file_to_play_player_queue(filename_buff, 0); } }else{ return -2; @@ -1061,6 +1145,20 @@ int app_play_ota_tips_file(void) return player_mq_couter > 0 ? 0 : -1; } +static void conversation_finish_startup_prompt(unsigned int round, int result) +{ + int start; + conversation_lock(); + start = round == conversation_round && g_mic_working && tts_playing != 2; + if (start) startup_start_ms = hal_os_up_time_ms(); + conversation_unlock(); + if (start) { + printf("voice startup prompt finished:round=%u result=%d\n", round, result); + /* Queue fullness must not suppress this prompt's startup action. */ + vad_start_func(); + } +} + void app_player_thread(void *arvg) { ice_work_item *player_item=NULL; @@ -1072,7 +1170,7 @@ void app_player_thread(void *arvg) if(player_item!=NULL){ int stale_timeout; conversation_lock(); - stale_timeout = (player_item->type == 3 || player_item->type == 4) && + stale_timeout = (player_item->type == 2 || player_item->type == 3 || player_item->type == 4) && player_item->timeout_round != conversation_round; conversation_unlock(); if (stale_timeout) { @@ -1133,11 +1231,11 @@ void app_player_thread(void *arvg) if(player_item->type==1){ printf("vad_leave_message\r\n"); vad_leave_message(); - }else if(player_item->type==2){ - // vad_tts_playback_finished(); - vad_start_func(); } } + if (player_item->type == 2) { + conversation_finish_startup_prompt(player_item->timeout_round, play_ret); + } // } } if(player_item->data){ @@ -1154,6 +1252,9 @@ static int conversation_try_begin(void) { conversation_lock(); if(vad_audio_is_recording() || g_mic_working || shine_init || timeout_action_busy) { + printf("voice key ignored:round=%u vad=%d startup=%d sender=%d timeout=%d\n", + conversation_round, vad_audio_is_recording(), g_mic_working, + shine_init, timeout_action_busy); conversation_unlock(); return -1; } @@ -1226,7 +1327,13 @@ int tts_play_interrupt_func(void) vad_start_func(); } }else{ - app_play_tips_file(18); + if (app_play_tips_file(18) != 0) { + conversation_lock(); + g_mic_working = 0; + conversation_wait = WAIT_NONE; + conversation_unlock(); + return -2; + } } return 0; @@ -1254,6 +1361,17 @@ static void conversation_check_timeout(void) int expired = WAIT_NONE; unsigned int expired_round = 0; conversation_lock(); + /* A lost prompt/trigger must not leave all future keys permanently blocked. + * Never recover during an active player, upload or recording. */ + if (g_mic_working && !vad_audio_is_recording() && !shine_init && + player_mq_couter == 0 && !timeout_action_busy && + hal_os_up_time_ms() - startup_start_ms > 30000) { + printf("recover stalled voice startup:round=%u\n", conversation_round); + g_mic_working = 0; + conversation_wait = WAIT_NONE; + active_session_valid = 0; + vad_reset_conversation_flags(); + } if (conversation_wait != WAIT_NONE && wait_round == conversation_round && !vad_audio_is_recording() && !g_mic_working && diff --git a/tools/test_conversation_timeout.py b/tools/test_conversation_timeout.py index 5a4f61e..9f45809 100644 --- a/tools/test_conversation_timeout.py +++ b/tools/test_conversation_timeout.py @@ -24,10 +24,11 @@ harness = r''' #include #include #include +#include typedef void *ql_mutex_t; static int g_mic_working, recording, shine_init, fota_start; static int tts_playing, tts_play_status, audio_packet_type, no_speech_timeout; -static int queued, disconnected, flags_reset, mutex_locked; +static int queued, disconnected, flags_reset, mutex_locked, player_mq_couter, vad_starts; static uint64_t now; static unsigned int gSequenceNumber; static int end_result; @@ -42,6 +43,7 @@ static void vad_set_speech_timeout_flag(int value) { no_speech_timeout = value; static void vad_reset_conversation_flags(void) { no_speech_timeout = 0; flags_reset++; } static int build_random_tts_error_url(char *p, size_t n) { (void)p; (void)n; return 1; } static void add_file_to_play_player_queue(char *p, int type) { (void)p; assert(type == 3); queued++; } +static void vad_start_func(void) { vad_starts++; recording = 1; g_mic_working = 0; } static void app_play_tips_file(int n) { (void)n; assert(0); } static int conversation_try_begin(void); static void talkingq_ws_disconnect(void) { @@ -52,7 +54,7 @@ static void talkingq_ws_disconnect(void) { static int sendRecordingAudioPacket(AudioPacket *p) { assert(p->packetType == 2); return end_result; } ''' harness += source[state_start:state_end] -for name in ['conversation_try_begin', 'sendEndPacket', 'conversation_check_timeout']: +for name in ['conversation_try_begin', 'sendEndPacket', 'conversation_check_timeout', 'conversation_accept_voice_message', 'conversation_finish_startup_prompt', 'conversation_find_session_id']: harness += function(name) harness += r''' static void reset(void) { @@ -60,9 +62,16 @@ static void reset(void) { conversation_round = wait_round = sender_round = 0; conversation_wait = WAIT_NONE; timeout_action_busy = round_response_received = 0; queued = disconnected = flags_reset = no_speech_timeout = mutex_locked = 0; - now = 1000; end_result = 0; + now = 1000; end_result = 0; player_mq_couter = vad_starts = 0; active_session_valid = 0; startup_start_ms = 0; tts_playing = 0; } int main(void) { + char id[33]; + const char *tagged = "PROMPT_SOUND_URL:http://host/26101309423304_EFGHIJKLMNOPQRSTUV.mp3"; + const char *plain = "PROMPT_SOUND_URL:http://host/tts_error.mp3"; + assert(conversation_find_session_id(tagged, (unsigned int)strlen(tagged), id)); + assert(strcmp(id, "26101309423304EFGHIJKLMNOPQRSTUV") == 0); + assert(!conversation_find_session_id(plain, (unsigned int)strlen(plain), id)); + assert(!conversation_find_session_id(tagged, 20, id)); /* No confirmed speech: prompt remains, without disconnect or reply wait. */ reset(); assert(conversation_try_begin() == 0); g_mic_working = 0; conversation_no_speech(); conversation_check_timeout(); @@ -94,7 +103,30 @@ int main(void) { reset(); conversation_no_speech(); conversation_cancel_wait(); conversation_check_timeout(); assert(!queued); reset(); conversation_wait = WAIT_REPLY; wait_round = 9; now += 20000; conversation_check_timeout(); assert(!queued && !disconnected); - puts("PASS: no-speech prompt, new-round cancellation, ignored recording keys, END deadline, stale rounds, early reply, send failure, timeout/key race, auto-listening"); + /* Reproduce ai_active followed by a late untagged backend error. */ + reset(); active_session_valid = 1; active_session_round = 0; + assert(conversation_try_begin() == 0); + assert(!conversation_accept_voice_message("PROMPT_SOUND_URL:http://host/tts_error.mp3")); + assert(!conversation_accept_voice_message("TTS_START")); + assert(conversation_accept_voice_message("NFC_SOUND_URL:http://host/card.mp3")); + /* A following queue entry cannot consume the startup action. */ + player_mq_couter = 1; + conversation_finish_startup_prompt(conversation_round, 0); + assert(vad_starts == 1 && recording && !g_mic_working); + assert(!conversation_accept_voice_message("PROMPT_SOUND_URL:http://host/tts_error.mp3")); + active_session_valid = 1; active_session_round = conversation_round; + assert(conversation_accept_voice_message("PROMPT_SOUND_URL:http://host/tts_error.mp3")); + /* Stale prompt completion must not start another round. */ + reset(); assert(conversation_try_begin() == 0); + conversation_finish_startup_prompt(conversation_round - 1, 0); assert(!vad_starts); + conversation_finish_startup_prompt(conversation_round, -1); assert(vad_starts == 1); + /* Lost startup releases reservation; active work is never recovered. */ + reset(); assert(conversation_try_begin() == 0); now += 30001; + conversation_check_timeout(); assert(!g_mic_working); + assert(conversation_try_begin() == 0); + now += 30001; player_mq_couter = 1; conversation_check_timeout(); assert(g_mic_working); + player_mq_couter = 0; recording = 1; conversation_check_timeout(); assert(g_mic_working); + puts("PASS: late backend error admission, prompt completion with queued audio, stale prompt, failed playback, stalled-startup recovery; no-speech prompt, new-round cancellation, ignored recording keys, END deadline, stale rounds, early reply, send failure, timeout/key race, auto-listening"); } ''' with tempfile.TemporaryDirectory(prefix='banban-timeout-') as directory: