From c61d1a5043314afd232373aeae5d8e8340cd32cf Mon Sep 17 00:00:00 2001 From: "CSBOOK\\44744" <447449914@qq.com> Date: Fri, 26 Jun 2026 15:56:26 +0800 Subject: [PATCH] =?UTF-8?q?WS=E5=88=9A=E8=BF=9E=E6=8E=A5=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E5=8F=AF=E4=BB=A5=E8=BF=9B=E8=A1=8C=20=E5=AF=B9=E8=AF=9D?= =?UTF-8?q?=EF=BC=8C=E8=B6=85=E8=BF=8730sWS=E6=96=AD=E5=BC=80=E5=90=8E?= =?UTF-8?q?=EF=BC=8C=E8=AF=AD=E9=9F=B3=E4=B8=8A=E8=A1=8C=E5=81=9C=E6=AD=A2?= =?UTF-8?q?=EF=BC=8C=E9=9C=80=E8=A6=81=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal/control_plane_internal.h | 2 + .../control_plane_multimodal_runtime.c | 9 ++ ...control_plane_multimodal_session_service.c | 53 ++++++++- .../control_plane_connectivity_runtime.c | 9 +- .../src/runtime/control_plane_key_input.c | 19 +++- .../runtime/control_plane_runtime_workers.c | 2 +- .../src/runtime/control_plane_system_events.c | 107 ++++++++++++++++++ .../platform/src/multimodal/audio_pipeline.c | 1 + .../audio_pipeline_runtime_impl.c | 24 +++- sdkconfig.defaults | 3 +- 10 files changed, 218 insertions(+), 11 deletions(-) diff --git a/components/control_plane/internal/control_plane_internal.h b/components/control_plane/internal/control_plane_internal.h index ce0c3cb..83ba86b 100644 --- a/components/control_plane/internal/control_plane_internal.h +++ b/components/control_plane/internal/control_plane_internal.h @@ -107,6 +107,7 @@ esp_err_t control_plane_multimodal_register_downstream_audio_callback( esp_err_t control_plane_multimodal_register_responding_text_callback( control_plane_multimodal_responding_text_callback_t cb, void *user_data); +esp_err_t control_plane_multimodal_start_capture(void); esp_err_t control_plane_multimodal_start_uplink(void); void control_plane_multimodal_stop_uplink(void); bool control_plane_multimodal_is_uplink_running(void); @@ -123,3 +124,4 @@ const char *control_plane_key_event_type_name(control_plane_key_event_type_t typ esp_err_t control_plane_start_key_input_task(void); void control_plane_stop_key_input_task(void); +esp_err_t control_plane_schedule_websocket_wait_task(void); diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c index 0562579..af07327 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c @@ -46,6 +46,7 @@ #define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220 #define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000 #define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 6144 +#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus" #ifndef CONFIG_TQ_NETWORK_TASK_CORE #define CONFIG_TQ_NETWORK_TASK_CORE 0 @@ -295,6 +296,13 @@ static esp_err_t control_plane_start_uplink_if_ready(void) esp_err_t control_plane_multimodal_handle_ptt_press_step(void) { + esp_err_t capture_err = control_plane_multimodal_start_capture(); + if (capture_err != ESP_OK && + capture_err != ESP_ERR_INVALID_STATE && + capture_err != ESP_ERR_NOT_SUPPORTED) { + ESP_LOGW(TAG, "PTT capture start failed: %s", esp_err_to_name(capture_err)); + } + control_plane_multimodal_runtime_state_t state = {0}; esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state); if (state_err != ESP_OK) { @@ -580,6 +588,7 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event switch (event) { case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_CONNECTED: + (void)domain_runtime_behavior_play_prompt_file(CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE); if (CONTROL_PLANE_MULTIMODAL_AUTO_START_FLAG) { schedule_autostart_if_enabled(); } diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c index 84c1651..5207687 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c @@ -83,6 +83,8 @@ #define CONTROL_PLANE_LOCAL_RESPONDING_EVENT_TASK_PRIORITY 5U #define CONTROL_PLANE_LOCAL_RESPONDING_EVENT_QUEUE_LEN 4U #define CONTROL_PLANE_MULTIMODAL_BINARY_SUMMARY_INTERVAL 200U +#define CONTROL_PLANE_UPLINK_READY_WAIT_MS 6000U +#define CONTROL_PLANE_UPLINK_READY_POLL_MS 20U #ifndef CONFIG_TQ_LOCAL_TASK_CORE #define CONFIG_TQ_LOCAL_TASK_CORE 1 @@ -1322,7 +1324,12 @@ static void transport_event_bridge(const domain_multimodal_transport_event_t *ev break; case DOMAIN_MULTIMODAL_TRANSPORT_EVENT_DISCONNECTED: - reset_session_state(); + if (lock_state()) { + s_started = false; + memset(&s_auth, 0, sizeof(s_auth)); + reset_session_state_locked(); + unlock_state(); + } notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED); break; @@ -1426,6 +1433,30 @@ static void downstream_playback_event_bridge( static esp_err_t control_plane_audio_uplink_send_cb(const uint8_t *data, size_t len, void *ctx) { (void)ctx; + uint32_t waited_ms = 0U; + while (waited_ms < CONTROL_PLANE_UPLINK_READY_WAIT_MS) { + esp_err_t err = control_plane_multimodal_send_audio_realtime(data, len); + if (err != ESP_ERR_INVALID_STATE) { + return err; + } + + control_plane_multimodal_runtime_state_t state = {0}; + if (control_plane_multimodal_get_runtime_state(&state) == ESP_OK && + state.connected) { + if (!state.has_active_task) { + (void)control_plane_multimodal_send_start(); + } else if (state.server_listening && + !state.client_speech_active && + !state.ptt_release_required && + !state.remote_responding) { + (void)control_plane_multimodal_send_speech_start(); + } + } + + vTaskDelay(pdMS_TO_TICKS(CONTROL_PLANE_UPLINK_READY_POLL_MS)); + waited_ms += CONTROL_PLANE_UPLINK_READY_POLL_MS; + } + return control_plane_multimodal_send_audio_realtime(data, len); } @@ -1439,7 +1470,7 @@ esp_err_t control_plane_multimodal_start(void) if (lock_state()) { bool already_started = s_started; unlock_state(); - if (already_started && domain_runtime_multimodal_transport_is_connected()) { + if (already_started) { return ESP_OK; } } @@ -2056,6 +2087,22 @@ static esp_err_t control_plane_multimodal_prepare_audio_runtime(void) return domain_runtime_multimodal_audio_init(); } +esp_err_t control_plane_multimodal_start_capture(void) +{ + if (!domain_runtime_multimodal_audio_is_initialized()) { + esp_err_t init_err = domain_runtime_multimodal_audio_init(); + if (init_err != ESP_OK) { + return init_err; + } + } + + if (domain_runtime_multimodal_audio_is_uplink_running()) { + return ESP_OK; + } + + return domain_runtime_multimodal_audio_start_uplink(control_plane_audio_uplink_send_cb, NULL); +} + esp_err_t control_plane_multimodal_start_uplink(void) { esp_err_t ready_err = control_plane_multimodal_prepare_audio_runtime(); @@ -2072,7 +2119,7 @@ esp_err_t control_plane_multimodal_start_uplink(void) void control_plane_multimodal_stop_uplink(void) { - esp_err_t err = domain_runtime_multimodal_audio_stop_uplink(2000); + esp_err_t err = domain_runtime_multimodal_audio_stop_uplink(CONTROL_PLANE_UPLINK_READY_WAIT_MS); if (err != ESP_OK) { ESP_LOGW(TAG, "Mic uplink stop ended with %s", esp_err_to_name(err)); } diff --git a/components/control_plane/src/runtime/control_plane_connectivity_runtime.c b/components/control_plane/src/runtime/control_plane_connectivity_runtime.c index 5342cba..6e35229 100644 --- a/components/control_plane/src/runtime/control_plane_connectivity_runtime.c +++ b/components/control_plane/src/runtime/control_plane_connectivity_runtime.c @@ -24,6 +24,8 @@ #define CONFIG_TQ_BOOT_SOUND_FILE "" #endif +#define CONTROL_PLANE_WIFI_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/network_connect_success.opus" + #define CONTROL_PLANE_STA_CONNECT_MAX_ATTEMPTS 3 #define CONTROL_PLANE_STA_RETRY_BASE_DELAY_MS 300 @@ -79,8 +81,13 @@ static int control_plane_normalize_connect_timeout_ms(int timeout_ms, int min_ti static void control_plane_play_wifi_success_prompt(void) { bool played = false; + esp_err_t audio_err = domain_runtime_multimodal_audio_init(); + if (audio_err != ESP_OK && audio_err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, "Prepare audio for Wi-Fi success prompt failed: %s", esp_err_to_name(audio_err)); + } + played = (control_plane_wifi_play_prompt_file(CONTROL_PLANE_WIFI_SUCCESS_PROMPT_FILE) == ESP_OK); #ifdef CONFIG_VOICE_PROMPT_WIFI_SUCCESS_FILE - if (CONFIG_VOICE_PROMPT_WIFI_SUCCESS_FILE[0] != '\0') { + if (!played && CONFIG_VOICE_PROMPT_WIFI_SUCCESS_FILE[0] != '\0') { played = (control_plane_wifi_play_prompt_file(CONFIG_VOICE_PROMPT_WIFI_SUCCESS_FILE) == ESP_OK); } #endif diff --git a/components/control_plane/src/runtime/control_plane_key_input.c b/components/control_plane/src/runtime/control_plane_key_input.c index e45c8c5..b87397a 100644 --- a/components/control_plane/src/runtime/control_plane_key_input.c +++ b/components/control_plane/src/runtime/control_plane_key_input.c @@ -94,6 +94,7 @@ typedef struct { int raw_level; bool debounced_pressed; bool debounce_pending; + bool wait_inactive_before_press; bool long_press_sent; bool click_pending; int64_t debounce_deadline_ms; @@ -252,6 +253,14 @@ static void control_plane_handle_debounced_transition(const control_plane_key_co int level, int64_t now_ms) { + if (state->wait_inactive_before_press) { + if (pressed) { + state->debounced_pressed = false; + return; + } + state->wait_inactive_before_press = false; + } + if (pressed == state->debounced_pressed) { return; } @@ -374,8 +383,14 @@ static void control_plane_key_input_task(void *arg) const control_plane_key_config_t *config = &s_key_configs[i]; s_key_states[i].initialized = true; s_key_states[i].raw_level = gpio_get_level(config->gpio_num); - s_key_states[i].debounced_pressed = - control_plane_key_level_is_pressed(config, s_key_states[i].raw_level); + bool initial_pressed = control_plane_key_level_is_pressed(config, s_key_states[i].raw_level); + s_key_states[i].debounced_pressed = initial_pressed; + if (config->key_id == CONTROL_PLANE_KEY_ID_PTT && initial_pressed) { + ESP_LOGW(TAG, + "PTT initial level is active; wait for next stable press before emitting press event"); + s_key_states[i].debounced_pressed = false; + s_key_states[i].wait_inactive_before_press = true; + } setup_err = control_plane_configure_key_gpio(config); if (setup_err != ESP_OK) { diff --git a/components/control_plane/src/runtime/control_plane_runtime_workers.c b/components/control_plane/src/runtime/control_plane_runtime_workers.c index 0cb630c..5090cf8 100644 --- a/components/control_plane/src/runtime/control_plane_runtime_workers.c +++ b/components/control_plane/src/runtime/control_plane_runtime_workers.c @@ -726,7 +726,7 @@ static void control_plane_websocket_wait_task(void *arg) return; } -static esp_err_t control_plane_schedule_websocket_wait_task(void) +esp_err_t control_plane_schedule_websocket_wait_task(void) { bool should_create = false; diff --git a/components/control_plane/src/runtime/control_plane_system_events.c b/components/control_plane/src/runtime/control_plane_system_events.c index a72036d..77bc5fd 100644 --- a/components/control_plane/src/runtime/control_plane_system_events.c +++ b/components/control_plane/src/runtime/control_plane_system_events.c @@ -30,6 +30,10 @@ #define CONFIG_TQ_SYSTEM_EVENT_TASK_STACK 4096 #endif +#define CONTROL_PLANE_PTT_HOLD_STEP_MS 80 +#define CONTROL_PLANE_PTT_HOLD_TASK_STACK 4096 +#define CONTROL_PLANE_PTT_HOLD_TASK_PRIORITY 4 + #ifndef CONFIG_TQ_LOCAL_TASK_CORE #define CONFIG_TQ_LOCAL_TASK_CORE 1 #endif @@ -44,7 +48,9 @@ static const char *TAG = "cp_system_event"; static QueueHandle_t s_system_event_queue; static TaskHandle_t s_system_event_task; +static TaskHandle_t s_ptt_hold_task; static volatile bool s_stop_requested; +static volatile bool s_ptt_hold_active; static const char *control_plane_event_family_name(control_plane_event_family_t family) { @@ -106,6 +112,80 @@ const char *control_plane_key_event_type_name(control_plane_key_event_type_t typ } } +static void control_plane_ptt_hold_task(void *arg) +{ + (void)arg; + ESP_LOGI(TAG, "PTT hold task started"); + bool start_requested = false; + + while (s_ptt_hold_active && !s_stop_requested) { + if (!control_plane_is_sta_connected()) { + vTaskDelay(pdMS_TO_TICKS(CONTROL_PLANE_PTT_HOLD_STEP_MS)); + continue; + } + + if (!control_plane_multimodal_is_connected() && !start_requested) { + esp_err_t schedule_err = control_plane_schedule_websocket_wait_task(); + if (schedule_err != ESP_OK && schedule_err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, + "PTT multimodal start schedule failed: %s", + esp_err_to_name(schedule_err)); + } else { + start_requested = true; + } + } + + esp_err_t press_err = control_plane_multimodal_handle_ptt_press_step(); + if (press_err != ESP_OK && + press_err != ESP_ERR_INVALID_STATE && + press_err != ESP_ERR_NOT_SUPPORTED) { + ESP_LOGW(TAG, "PTT press step failed: %s", esp_err_to_name(press_err)); + } + + vTaskDelay(pdMS_TO_TICKS(CONTROL_PLANE_PTT_HOLD_STEP_MS)); + } + + ESP_LOGI(TAG, "PTT hold task stopped"); + s_ptt_hold_task = NULL; + vTaskDelete(NULL); +} + +static esp_err_t control_plane_start_ptt_hold_task(void) +{ + s_ptt_hold_active = true; + if (s_ptt_hold_task != NULL) { + return ESP_OK; + } + + BaseType_t created = xTaskCreatePinnedToCore(control_plane_ptt_hold_task, + "cp_ptt_hold", + CONTROL_PLANE_PTT_HOLD_TASK_STACK, + NULL, + CONTROL_PLANE_PTT_HOLD_TASK_PRIORITY, + &s_ptt_hold_task, + CONTROL_PLANE_SYSTEM_EVENT_TASK_CORE); + if (created != pdPASS) { + s_ptt_hold_task = NULL; + s_ptt_hold_active = false; + return ESP_ERR_NO_MEM; + } + + return ESP_OK; +} + +static void control_plane_stop_ptt_hold_task(void) +{ + s_ptt_hold_active = false; + + for (int i = 0; i < 10 && s_ptt_hold_task != NULL; ++i) { + vTaskDelay(pdMS_TO_TICKS(20)); + } + if (s_ptt_hold_task != NULL) { + vTaskDelete(s_ptt_hold_task); + s_ptt_hold_task = NULL; + } +} + static void control_plane_dispatch_key_event(const control_plane_key_event_t *event) { if (event == NULL) { @@ -120,6 +200,32 @@ static void control_plane_dispatch_key_event(const control_plane_key_event_t *ev event->level, event->press_duration_ms, event->click_count); + + if (event->key_id != CONTROL_PLANE_KEY_ID_PTT) { + return; + } + + switch (event->type) { + case CONTROL_PLANE_KEY_EVENT_PRESSED: { + esp_err_t start_err = control_plane_start_ptt_hold_task(); + if (start_err != ESP_OK) { + ESP_LOGW(TAG, "PTT hold task start failed: %s", esp_err_to_name(start_err)); + } + break; + } + + case CONTROL_PLANE_KEY_EVENT_RELEASED: { + control_plane_stop_ptt_hold_task(); + esp_err_t release_err = control_plane_multimodal_handle_ptt_release(); + if (release_err != ESP_OK && release_err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, "PTT release failed: %s", esp_err_to_name(release_err)); + } + break; + } + + default: + break; + } } static void control_plane_dispatch_system_event(const control_plane_system_event_t *event) @@ -225,6 +331,7 @@ void control_plane_stop_system_event_task(void) { #if CONFIG_TQ_SYSTEM_EVENT_ENABLE s_stop_requested = true; + control_plane_stop_ptt_hold_task(); for (int i = 0; i < 20 && s_system_event_task != NULL; ++i) { vTaskDelay(pdMS_TO_TICKS(50)); diff --git a/components/platform/src/multimodal/audio_pipeline.c b/components/platform/src/multimodal/audio_pipeline.c index 3d23ab1..cc06bae 100644 --- a/components/platform/src/multimodal/audio_pipeline.c +++ b/components/platform/src/multimodal/audio_pipeline.c @@ -298,6 +298,7 @@ typedef struct { TaskHandle_t uplink_send_task; volatile bool uplink_recording_active; volatile bool uplink_task_exit_requested; + volatile bool uplink_send_in_flight; multimodal_audio_uplink_cb_t uplink_cb; void *uplink_cb_ctx; bool opus_ready; diff --git a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c index 7a40774..fdb3c67 100644 --- a/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c +++ b/components/platform/src/multimodal/audio_pipeline/audio_pipeline_runtime_impl.c @@ -664,7 +664,9 @@ static void uplink_send_task(void *arg) continue; } + ctx->uplink_send_in_flight = true; esp_err_t send_err = cb(packet.data, packet.len, cb_ctx); + ctx->uplink_send_in_flight = false; track_uplink_send_result(send_err, packet.len); if (send_err != ESP_OK && send_err != ESP_ERR_INVALID_STATE) { if (send_err != ESP_ERR_TIMEOUT) { @@ -1030,9 +1032,27 @@ esp_err_t multimodal_audio_stop_uplink(int timeout_ms) return ESP_ERR_INVALID_ARG; } - (void)timeout_ms; audio_set_uplink_recording_active(&s_audio, false); - reset_uplink_send_queue(&s_audio); + + TickType_t deadline = xTaskGetTickCount() + pdMS_TO_TICKS((uint32_t)timeout_ms); + while (((s_audio.uplink_send_queue != NULL && + uxQueueMessagesWaiting(s_audio.uplink_send_queue) > 0U) || + s_audio.uplink_send_in_flight) && + xTaskGetTickCount() < deadline) { + vTaskDelay(pdMS_TO_TICKS(20)); + } + + if ((s_audio.uplink_send_queue != NULL && + uxQueueMessagesWaiting(s_audio.uplink_send_queue) > 0U) || + s_audio.uplink_send_in_flight) { + ESP_LOGW(TAG, + "Mic uplink stop timed out with %u queued packet(s), in_flight=%d", + (unsigned)((s_audio.uplink_send_queue != NULL) ? + uxQueueMessagesWaiting(s_audio.uplink_send_queue) : 0U), + s_audio.uplink_send_in_flight ? 1 : 0); + reset_uplink_send_queue(&s_audio); + } + audio_set_uplink_callback(&s_audio, NULL, NULL); s_audio.uplink_mono_fill_bytes = 0U; log_heap_snapshot("after_uplink_stop"); diff --git a/sdkconfig.defaults b/sdkconfig.defaults index c2105cb..edd27f0 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -13,7 +13,7 @@ CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y CONFIG_SPIRAM_ALLOW_NOINIT_SEG_EXTERNAL_MEMORY=y CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM=y CONFIG_FREERTOS_USE_TRACE_FACILITY=y -CONFIG_TQ_TASK_MONITOR_ENABLE=y +# CONFIG_TQ_TASK_MONITOR_ENABLE is not set # CONFIG_TQ_BOOT_SEQUENCE_DRY_RUN is not set CONFIG_TQ_BOOT_SOUND_TIMEOUT_MS=3000 CONFIG_TQ_BOOT_SOUND_FILE="boot.opus" @@ -78,7 +78,6 @@ CONFIG_MULTIMODAL_DEVICE_ID="" CONFIG_MULTIMODAL_SERIAL_NUMBER="" CONFIG_MULTIMODAL_WIFI_SSID="" CONFIG_MULTIMODAL_WIFI_PASSWORD="" -CONFIG_ESPTOOLPY_BAUD_921600B=y CONFIG_ESPTOOLPY_MONITOR_BAUD=921600 CONFIG_ESP_CONSOLE_UART_BAUDRATE=921600 CONFIG_MULTIMODAL_DEVICE_TOKEN_URL="https://gz.api.talkingq.com/token/api/device/token"