From 23983ccdc88726167e264cf95ebee14b34ec7615 Mon Sep 17 00:00:00 2001 From: chenjintao Date: Tue, 13 Oct 2026 15:56:00 +0800 Subject: [PATCH] fix(audio): defer and coalesce role switch notices --- .../threadx/interface/custom_app/mqtt.c | 5 +- .../threadx/interface/custom_app/vad_det.c | 2 + .../interface/custom_app/ws_talkingq.c | 127 +++++++++++++++++- tools/test_conversation_timeout.py | 91 ++++++++++++- 4 files changed, 214 insertions(+), 11 deletions(-) diff --git a/ql-application/threadx/interface/custom_app/mqtt.c b/ql-application/threadx/interface/custom_app/mqtt.c index fa565c2..534c1b8 100644 --- a/ql-application/threadx/interface/custom_app/mqtt.c +++ b/ql-application/threadx/interface/custom_app/mqtt.c @@ -108,6 +108,7 @@ static char payload[MQTT_PAYLOAD_SIZE] = {0}; extern int fota_start; int app_play_ota_tips_file(void); +int app_role_notice_handle_url(const char *url); int xiaoice_get_play_counter(void); typedef struct { @@ -1510,7 +1511,9 @@ void handle_command(command_t *cmd) app_mqtt_publish(resp, strlen(resp), 0); free(resp); - add_file_to_play_player_queue(cmd->params.nfc.url, 0); + if (!app_role_notice_handle_url(cmd->params.nfc.url)) { + add_file_to_play_player_queue(cmd->params.nfc.url, 0); + } break; } diff --git a/ql-application/threadx/interface/custom_app/vad_det.c b/ql-application/threadx/interface/custom_app/vad_det.c index fad7bf7..973df56 100644 --- a/ql-application/threadx/interface/custom_app/vad_det.c +++ b/ql-application/threadx/interface/custom_app/vad_det.c @@ -46,6 +46,7 @@ extern ql_sem_t sender_sem; extern void conversation_no_speech(void); extern void conversation_record_begin(void); extern void conversation_cancel_wait(void); +extern int app_role_notice_before_finish(void); // 配置 static int sample_rate = 16000; // FVAD 支持的采样率：8k/16k/32k/48k @@ -502,6 +503,7 @@ void vad_audio_process(void) { case STATE_WAITING_COMMAND: if(vad_tts_playback_finished_status == 1) { + if (app_role_notice_before_finish()) break; int leave_message_finished = vad_leave_message_flag != 0; printf("finished_status:%d, no_speech_timeout:%d\n", vad_tts_playback_finished_status, no_speech_timeout); set_state(STATE_IDLE); diff --git a/ql-application/threadx/interface/custom_app/ws_talkingq.c b/ql-application/threadx/interface/custom_app/ws_talkingq.c index b2f41db..7b13919 100644 --- a/ql-application/threadx/interface/custom_app/ws_talkingq.c +++ b/ql-application/threadx/interface/custom_app/ws_talkingq.c @@ -65,6 +65,13 @@ static int round_response_received = 0; static unsigned int active_session_round = 0; static int active_session_valid = 0; static uint64_t startup_start_ms = 0; +#define ROLE_NOTICE_TYPE 5 +#define ROLE_NOTICE_URL_SIZE 256 +static char role_notice_url[ROLE_NOTICE_URL_SIZE]; +static int role_notice_pending = 0; +static int role_notice_playing = 0; +static int role_notice_consumed = 0; +static unsigned int role_notice_round = 0; static void conversation_lock(void) { @@ -84,6 +91,9 @@ static void conversation_begin_round(void) round_response_received = 0; active_session_valid = 0; startup_start_ms = hal_os_up_time_ms(); + /* Accepted restart cancels the preceding round's deferred notice. */ + role_notice_pending = role_notice_playing = role_notice_consumed = 0; + role_notice_url[0] = '\0'; } void conversation_record_begin(void) @@ -98,6 +108,8 @@ void conversation_cancel_wait(void) { conversation_lock(); conversation_wait = WAIT_NONE; + role_notice_pending = role_notice_playing = role_notice_consumed = 0; + role_notice_url[0] = '\0'; conversation_unlock(); } @@ -129,6 +141,7 @@ extern int aofei_mp3_player_get_result(void); extern int vad_leave_message_is_active(void); extern int vad_cancel_leave_message(void); extern void vad_reset_conversation_flags(void); +extern system_state_t vad_audio_get_state(void); void sendAuthentication(void); void vad_start_func(void); @@ -139,8 +152,8 @@ int audio_packet_type = 0; typedef struct ice_work_item_s { - int type; //0-tts_out 1-session_continue 2-session_end 3-云端超时提示（失败时播放本地兜底） 4-本地超时兜底 - unsigned int timeout_round; /* Timeout items (type 3/4) use this cancellation token. */ + int type; //0-tts_out 1-session_continue 2-session_end 3-云端超时提示（失败时播放本地兜底） 4-本地超时兜底 5-角色切换提示 + unsigned int timeout_round; /* Startup, timeout and role notices use this round token. */ int error; int del_url; void *data; @@ -373,6 +386,99 @@ int add_file_to_play_player_queue(char *path, int type) } +/* Only this resource is a role-change notice. Other NFC notices keep their + * existing immediate queue path, including binding and card audio. */ +static int role_notice_is_url(const char *url) +{ + const char *host, *path; + const char resource[] = "/assets/audio/switch_role.mp3"; + size_t n = sizeof(resource) - 1; + if (!url) return 0; + host = strstr(url, "://"); + if (!host) return 0; + path = strchr(host + 3, '/'); + return path && strncmp(path, resource, n) == 0 && + (path[n] == '\0' || path[n] == '?' || path[n] == '#'); +} + +/* Caller holds conversation_mutex. A full queue/failed download must not + * prevent the original conversation completion from proceeding. */ +static int role_notice_start_locked(void) +{ + role_notice_pending = 0; + role_notice_consumed = 1; + role_notice_playing = 1; + role_notice_round = conversation_round; + if (add_file_to_play_player_queue(role_notice_url, ROLE_NOTICE_TYPE) != 0) { + role_notice_playing = 0; + printf("role notice queue failed:round=%u\n", conversation_round); + return 0; + } + printf("role notice queued once:round=%u\n", conversation_round); + return 1; +} + +/* Called only after the normal VAD playback-finished signal, or while idle. + * The original finished flag/no-speech flag stays untouched during the notice. */ +static int role_notice_poll_locked(int finishing) +{ + if (fota_start || power_down_status || get_sleep_mode()) { + role_notice_pending = role_notice_playing = 0; + return 0; + } + if (role_notice_playing) return 1; + if (!role_notice_pending) return 0; + if (role_notice_round != conversation_round) { + role_notice_pending = 0; + return 0; + } + if (vad_audio_is_recording() || g_mic_working || shine_init || + timeout_action_busy || player_mq_couter > 0 || tts_playing != 0 || + conversation_wait != WAIT_NONE) return 1; + if (!finishing && vad_audio_get_state() != STATE_IDLE) return 1; + return role_notice_start_locked(); +} + +int app_role_notice_handle_url(const char *url) +{ + if (!role_notice_is_url(url)) return 0; + conversation_lock(); + if (strlen(url) >= sizeof(role_notice_url)) { + printf("role notice URL too long\n"); + } else if (role_notice_playing || + (role_notice_consumed && vad_audio_get_state() != STATE_IDLE)) { + printf("role notice coalesced:round=%u\n", conversation_round); + } else { + /* Repeated commands before playback replace one pending slot. */ + snprintf(role_notice_url, sizeof(role_notice_url), "%s", url); + role_notice_pending = 1; + role_notice_round = conversation_round; + printf("role notice pending:round=%u\n", conversation_round); + role_notice_poll_locked(0); + } + conversation_unlock(); + return 1; +} + +int app_role_notice_before_finish(void) +{ + int wait; + conversation_lock(); + wait = role_notice_poll_locked(1); + conversation_unlock(); + return wait; +} + +static void role_notice_finished(unsigned int round, int result) +{ + conversation_lock(); + if (role_notice_round == round && round == conversation_round) { + role_notice_playing = 0; + printf("role notice finished:round=%u result=%d\n", round, result); + } + conversation_unlock(); +} + //如果录音后 超过10s 还没有收到信息 提示本轮对话结束 static void onmessage_text_locked(rws_socket socket, const char *text, const unsigned int len) { @@ -1104,6 +1210,9 @@ int app_play_tips_file(int index) int app_play_ota_tips_file(void) { + conversation_lock(); + role_notice_pending = role_notice_playing = role_notice_consumed = 0; + conversation_unlock(); char filename_buff[64] = {0}; uint32_t timeout; @@ -1170,8 +1279,10 @@ void app_player_thread(void *arvg) if(player_item!=NULL){ int stale_timeout; conversation_lock(); - stale_timeout = (player_item->type == 2 || player_item->type == 3 || player_item->type == 4) && - player_item->timeout_round != conversation_round; + stale_timeout = ((player_item->type == 2 || player_item->type == 3 || player_item->type == 4 || + player_item->type == ROLE_NOTICE_TYPE) && + player_item->timeout_round != conversation_round) || + (player_item->type == ROLE_NOTICE_TYPE && !role_notice_playing); conversation_unlock(); if (stale_timeout) { if (player_mq_couter > 0) player_mq_couter--; @@ -1210,7 +1321,7 @@ void app_player_thread(void *arvg) conversation_unlock(); - if(player_mq_couter==0){ + if(player_mq_couter==0 && player_item->type != ROLE_NOTICE_TYPE){ printf("player_mq_couter==0\r\n"); if(tts_play_status==2){ @@ -1233,6 +1344,9 @@ void app_player_thread(void *arvg) vad_leave_message(); } } + if (player_item->type == ROLE_NOTICE_TYPE) { + role_notice_finished(player_item->timeout_round, play_ret); + } if (player_item->type == 2) { conversation_finish_startup_prompt(player_item->timeout_round, play_ret); } @@ -1507,6 +1621,9 @@ static void check_connect_thread_entry(void *parameter) #endif #if 1 conversation_check_timeout(); + conversation_lock(); + role_notice_poll_locked(0); + conversation_unlock(); #endif ql_rtos_task_sleep_ms(3000); diff --git a/tools/test_conversation_timeout.py b/tools/test_conversation_timeout.py index 9f45809..7aeb794 100644 --- a/tools/test_conversation_timeout.py +++ b/tools/test_conversation_timeout.py @@ -1,4 +1,4 @@ -"""Run firmware timeout/round functions on host with scripted RTOS and transport.""" +"""Run timeout, round and role-notice scheduling with scripted RTOS and transport.""" from pathlib import Path import subprocess import tempfile @@ -27,7 +27,13 @@ harness = r''' #include typedef void *ql_mutex_t; static int g_mic_working, recording, shine_init, fota_start; +enum { STATE_IDLE, STATE_RECORDING_WAIT_SOUND, STATE_RECORDING_ACTIVE, STATE_WAITING_COMMAND }; static int tts_playing, tts_play_status, audio_packet_type, no_speech_timeout; +static int power_down_status, sleeping, vad_state; +static int queue_failure, last_queue_type; +static char last_queue_url[256]; +static int get_sleep_mode(void) { return sleeping; } +static int vad_audio_get_state(void) { return vad_state; } static int queued, disconnected, flags_reset, mutex_locked, player_mq_couter, vad_starts; static uint64_t now; static unsigned int gSequenceNumber; @@ -42,8 +48,14 @@ static uint64_t hal_os_up_time_ms(void) { return now; } static void vad_set_speech_timeout_flag(int value) { no_speech_timeout = value; } static void vad_reset_conversation_flags(void) { no_speech_timeout = 0; flags_reset++; } static int build_random_tts_error_url(char *p, size_t n) { (void)p; (void)n; return 1; } -static void add_file_to_play_player_queue(char *p, int type) { (void)p; assert(type == 3); queued++; } -static void vad_start_func(void) { vad_starts++; recording = 1; g_mic_working = 0; } +static int add_file_to_play_player_queue(char *p, int type) { + assert(type == 3 || type == 5); + if (queue_failure) return -1; + queued++; player_mq_couter++; last_queue_type = type; + snprintf(last_queue_url, sizeof(last_queue_url), "%s", p); + return 0; +} +static void vad_start_func(void) { vad_starts++; recording = 1; vad_state = STATE_RECORDING_WAIT_SOUND; g_mic_working = 0; } static void app_play_tips_file(int n) { (void)n; assert(0); } static int conversation_try_begin(void); static void talkingq_ws_disconnect(void) { @@ -54,7 +66,7 @@ static void talkingq_ws_disconnect(void) { static int sendRecordingAudioPacket(AudioPacket *p) { assert(p->packetType == 2); return end_result; } ''' harness += source[state_start:state_end] -for name in ['conversation_try_begin', 'sendEndPacket', 'conversation_check_timeout', 'conversation_accept_voice_message', 'conversation_finish_startup_prompt', 'conversation_find_session_id']: +for name in ['conversation_try_begin', 'sendEndPacket', 'conversation_check_timeout', 'conversation_accept_voice_message', 'conversation_finish_startup_prompt', 'conversation_find_session_id', 'role_notice_is_url', 'role_notice_start_locked', 'role_notice_poll_locked', 'app_role_notice_handle_url', 'app_role_notice_before_finish', 'role_notice_finished']: harness += function(name) harness += r''' static void reset(void) { @@ -62,6 +74,9 @@ static void reset(void) { conversation_round = wait_round = sender_round = 0; conversation_wait = WAIT_NONE; timeout_action_busy = round_response_received = 0; queued = disconnected = flags_reset = no_speech_timeout = mutex_locked = 0; + power_down_status = sleeping = queue_failure = 0; vad_state = STATE_IDLE; + role_notice_pending = role_notice_playing = role_notice_consumed = 0; + role_notice_round = 0; last_queue_type = 0; last_queue_url[0] = 0; now = 1000; end_result = 0; player_mq_couter = vad_starts = 0; active_session_valid = 0; startup_start_ms = 0; tts_playing = 0; } int main(void) { @@ -126,7 +141,73 @@ int main(void) { assert(conversation_try_begin() == 0); now += 30001; player_mq_couter = 1; conversation_check_timeout(); assert(g_mic_working); player_mq_couter = 0; recording = 1; conversation_check_timeout(); assert(g_mic_working); - puts("PASS: late backend error admission, prompt completion with queued audio, stale prompt, failed playback, stalled-startup recovery; no-speech prompt, new-round cancellation, ignored recording keys, END deadline, stale rounds, early reply, send failure, timeout/key race, auto-listening"); + /* The exact resource alone is routed; unrelated notices retain original path. */ + reset(); + assert(!app_role_notice_handle_url("http://host/assets/audio/card.mp3")); + assert(!app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3.extra")); + assert(!role_notice_is_url("http://host/other/switch_role.mp3")); + assert(!queued); + /* No speech: retain one notice until tts_error + optional local fallback ends. */ + reset(); recording = 1; vad_state = STATE_RECORDING_WAIT_SOUND; + assert(app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3")); + assert(!queued && role_notice_pending); + recording = 0; vad_state = STATE_WAITING_COMMAND; conversation_wait = WAIT_NO_SPEECH; + assert(app_role_notice_before_finish() && !queued); + conversation_wait = WAIT_NONE; player_mq_couter = 1; no_speech_timeout = 1; + assert(app_role_notice_before_finish() && !queued); + player_mq_couter = 0; + assert(app_role_notice_before_finish() && queued == 1 && last_queue_type == 5); + assert(no_speech_timeout == 1); /* Error flag never becomes auto-listening. */ + assert(app_role_notice_before_finish() && queued == 1); + player_mq_couter = 0; role_notice_finished(conversation_round, 0); + assert(!app_role_notice_before_finish() && no_speech_timeout == 1); + /* Speech: zero insertion while uploading/waiting or between reply segments. */ + reset(); recording = 1; vad_state = STATE_RECORDING_ACTIVE; + app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + recording = 0; vad_state = STATE_WAITING_COMMAND; conversation_wait = WAIT_REPLY; + assert(app_role_notice_before_finish() && !queued); + conversation_wait = WAIT_NONE; tts_playing = 1; + assert(app_role_notice_before_finish() && !queued); + tts_playing = 0; player_mq_couter = 2; + assert(app_role_notice_before_finish() && !queued); + player_mq_couter = 0; + assert(app_role_notice_before_finish() && queued == 1); + role_notice_finished(conversation_round, 0); player_mq_couter = 0; + assert(!app_role_notice_before_finish() && no_speech_timeout == 0); + /* Repeated switches produce one latest URL at the end, never six entries. */ + reset(); vad_state = STATE_WAITING_COMMAND; tts_playing = 1; + for (int n = 0; n < 6; n++) { + char url[128]; snprintf(url, sizeof(url), "http://host/assets/audio/switch_role.mp3?v=%d", n); + assert(app_role_notice_handle_url(url)); assert(!queued); + } + tts_playing = 0; + assert(app_role_notice_before_finish() && queued == 1); + assert(strstr(last_queue_url, "v=5")); + app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + assert(queued == 1 && !role_notice_pending); + role_notice_finished(conversation_round, 0); player_mq_couter = 0; + app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + assert(queued == 1 && !app_role_notice_before_finish()); + /* Idle plays promptly; playback/queue failure releases completion too. */ + reset(); app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + assert(queued == 1); player_mq_couter = 0; + role_notice_finished(conversation_round, -1); assert(!app_role_notice_before_finish()); + reset(); queue_failure = 1; + app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + assert(!queued && !app_role_notice_before_finish()); + /* Accepted restart/force-stop cancels notice; no replay into next round. */ + reset(); vad_state = STATE_WAITING_COMMAND; tts_playing = 1; + app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + assert(conversation_try_begin() == 0 && !role_notice_pending); + reset(); recording = 1; app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + conversation_cancel_wait(); assert(!role_notice_pending && !role_notice_playing); + for (int mode = 0; mode < 3; mode++) { + reset(); vad_state = STATE_WAITING_COMMAND; + app_role_notice_handle_url("http://host/assets/audio/switch_role.mp3"); + fota_start = mode == 0; power_down_status = mode == 1; sleeping = mode == 2; + assert(!app_role_notice_before_finish() && !role_notice_pending && !queued); + } + puts("PASS: role notices: silent/speech end, latest-only coalescing, idle, failure, restart/OTA/shutdown/sleep isolation; late backend error admission, prompt completion with queued audio, stale prompt, failed playback, stalled-startup recovery; no-speech prompt, new-round cancellation, ignored recording keys, END deadline, stale rounds, early reply, send failure, timeout/key race, auto-listening"); } ''' with tempfile.TemporaryDirectory(prefix='banban-timeout-') as directory: