diff --git a/components/control_plane/internal/control_plane_internal.h b/components/control_plane/internal/control_plane_internal.h index 1a99b23..1758cd3 100644 --- a/components/control_plane/internal/control_plane_internal.h +++ b/components/control_plane/internal/control_plane_internal.h @@ -79,6 +79,15 @@ typedef struct { uint32_t last_downstream_audio_ms; } control_plane_multimodal_runtime_state_t; +typedef enum { + CONTROL_PLANE_TRANSPORT_LOSS_WS_ERROR = 0, + CONTROL_PLANE_TRANSPORT_LOSS_WS_DISCONNECTED, + CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE, + CONTROL_PLANE_TRANSPORT_LOSS_SEND_BINARY_FAILURE, + CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT, + CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT, +} control_plane_transport_loss_reason_t; + esp_err_t control_plane_multimodal_start(void); void control_plane_multimodal_stop(void); bool control_plane_multimodal_is_connected(void); @@ -96,6 +105,9 @@ esp_err_t control_plane_multimodal_send_local_responding_started(void); esp_err_t control_plane_multimodal_send_local_responding_ended(void); esp_err_t control_plane_multimodal_send_heartbeat(void); esp_err_t control_plane_multimodal_send_audio_realtime(const uint8_t *data, size_t len); +void control_plane_multimodal_mark_transport_unavailable( + control_plane_transport_loss_reason_t reason, + const char *caller); esp_err_t control_plane_multimodal_get_runtime_state( control_plane_multimodal_runtime_state_t *out_state); esp_err_t control_plane_multimodal_register_event_callback( @@ -114,6 +126,7 @@ bool control_plane_multimodal_is_uplink_running(void); esp_err_t control_plane_multimodal_play_downstream_audio(const uint8_t *data, size_t len); esp_err_t control_plane_multimodal_handle_ptt_press_step(void); esp_err_t control_plane_multimodal_handle_ptt_release(void); +void control_plane_multimodal_reset_uplink_failure_window(void); esp_err_t control_plane_start_system_event_task(void); void control_plane_stop_system_event_task(void); diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c index 8afa6d2..d72bda2 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_runtime.c @@ -286,13 +286,6 @@ static esp_err_t control_plane_start_uplink_if_ready(void) esp_err_t control_plane_multimodal_handle_ptt_press_step(void) { - esp_err_t capture_err = control_plane_multimodal_start_capture(); - if (capture_err != ESP_OK && - capture_err != ESP_ERR_INVALID_STATE && - capture_err != ESP_ERR_NOT_SUPPORTED) { - ESP_LOGW(TAG, "PTT capture start failed: %s", esp_err_to_name(capture_err)); - } - control_plane_multimodal_runtime_state_t state = {0}; esp_err_t state_err = control_plane_multimodal_get_runtime_state(&state); if (state_err != ESP_OK) { @@ -327,6 +320,7 @@ esp_err_t control_plane_multimodal_handle_ptt_press_step(void) if (speech_err != ESP_OK) { return speech_err; } + return ESP_ERR_INVALID_STATE; } if (control_plane_multimodal_is_uplink_running()) { @@ -496,6 +490,10 @@ static void control_plane_multimodal_event_bridge(control_plane_multimodal_event control_plane_multimodal_stop_uplink(); break; + case CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_ERROR: + control_plane_multimodal_stop_uplink(); + break; + case CONTROL_PLANE_MULTIMODAL_EVENT_TASK_FAILED: control_plane_request_server_error_expression(DOMAIN_EXPRESSION_TRIGGER_SERVER_TASK_FAILED); control_plane_multimodal_stop_uplink(); diff --git a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c index b2f1ec7..3f536c6 100644 --- a/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c +++ b/components/control_plane/src/multimodal/control_plane_multimodal_session_service.c @@ -85,7 +85,8 @@ #define CONTROL_PLANE_LOCAL_RESPONDING_EVENT_IDLE_EXIT_MS 5000U #define CONTROL_PLANE_MULTIMODAL_BINARY_SUMMARY_INTERVAL 200U #define CONTROL_PLANE_UPLINK_READY_WAIT_MS 6000U -#define CONTROL_PLANE_UPLINK_READY_POLL_MS 20U +#define CONTROL_PLANE_AUDIO_SEND_FAILURE_WINDOW_MS 100U +#define CONTROL_PLANE_AUDIO_SEND_FAILURE_THRESHOLD 2U #ifndef CONFIG_TQ_LOCAL_TASK_CORE #define CONFIG_TQ_LOCAL_TASK_CORE 1 @@ -124,6 +125,7 @@ static char s_task_id[40]; static char s_pending_start_task_id[40]; static char s_dialog_id[CONTROL_PLANE_MULTIMODAL_AUTH_ID_MAX_LEN]; static bool s_start_in_progress; +static bool s_transport_available; static TickType_t s_last_heartbeat_tick; static TickType_t s_last_downstream_tick; @@ -141,6 +143,8 @@ static bool s_downstream_playback_completed; static bool s_drop_downstream_audio; static bool s_received_downstream_audio; static bool s_current_response_received_audio; +static uint32_t s_audio_send_failure_count; +static TickType_t s_audio_send_failure_window_start_tick; static size_t s_last_asr_text_len; static size_t s_last_tts_text_len; static uint32_t s_downstream_binary_packet_total; @@ -376,12 +380,79 @@ static void reset_session_state_locked(void) s_drop_downstream_audio = false; s_received_downstream_audio = false; s_current_response_received_audio = false; + s_audio_send_failure_count = 0; + s_audio_send_failure_window_start_tick = 0; s_local_responding_started_send_pending = false; s_local_responding_ended_send_pending = false; s_last_asr_text_len = 0; s_last_tts_text_len = 0; } +static bool transport_available_locked(void) +{ + return s_transport_available && domain_runtime_multimodal_transport_is_connected(); +} + +static bool transport_available(void) +{ + bool available = false; + if (lock_state()) { + available = transport_available_locked(); + unlock_state(); + } + return available; +} + +static const char *transport_loss_reason_name(control_plane_transport_loss_reason_t reason) +{ + switch (reason) { + case CONTROL_PLANE_TRANSPORT_LOSS_WS_ERROR: + return "ws_error"; + case CONTROL_PLANE_TRANSPORT_LOSS_WS_DISCONNECTED: + return "ws_disconnected"; + case CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE: + return "send_text_failure"; + case CONTROL_PLANE_TRANSPORT_LOSS_SEND_BINARY_FAILURE: + return "send_binary_failure"; + case CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT: + return "listening_timeout"; + case CONTROL_PLANE_TRANSPORT_LOSS_NO_RESPONSE_TIMEOUT: + return "no_response_timeout"; + default: + return "unknown"; + } +} + +void control_plane_multimodal_reset_uplink_failure_window(void) +{ + if (!lock_state()) { + return; + } + s_audio_send_failure_count = 0; + s_audio_send_failure_window_start_tick = 0; + unlock_state(); +} + +void control_plane_multimodal_mark_transport_unavailable( + control_plane_transport_loss_reason_t reason, + const char *caller) +{ + ESP_LOGW(TAG, + "Transport unavailable: reason=%s caller=%s", + transport_loss_reason_name(reason), + caller != NULL ? caller : "-"); + + if (lock_state()) { + s_started = false; + s_transport_available = false; + memset(&s_auth, 0, sizeof(s_auth)); + reset_session_state_locked(); + unlock_state(); + } + + control_plane_multimodal_stop_uplink(); +} + static void reset_session_state(void) { if (!lock_state()) { @@ -915,6 +986,11 @@ static esp_err_t send_directive(const char *action, domain_runtime_multimodal_transport_send_text_realtime(directive, json) : domain_runtime_multimodal_transport_send_text("directive", json); free(json); + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE, + __func__); + } return err; } @@ -972,6 +1048,11 @@ static esp_err_t send_request_to_respond_directive( err = domain_runtime_multimodal_transport_send_text("request_to_respond", json); free(json); + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE, + __func__); + } return err; } @@ -1026,6 +1107,11 @@ static esp_err_t send_update_info_directive( err = domain_runtime_multimodal_transport_send_text("update_info", json); free(json); + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE, + __func__); + } return err; } @@ -1323,6 +1409,7 @@ static void transport_event_bridge(const domain_multimodal_transport_event_t *ev switch (event->type) { case DOMAIN_MULTIMODAL_TRANSPORT_EVENT_CONNECTED: if (lock_state()) { + s_transport_available = true; s_last_heartbeat_tick = xTaskGetTickCount(); s_last_downstream_tick = s_last_heartbeat_tick; unlock_state(); @@ -1331,16 +1418,16 @@ static void transport_event_bridge(const domain_multimodal_transport_event_t *ev break; case DOMAIN_MULTIMODAL_TRANSPORT_EVENT_DISCONNECTED: - if (lock_state()) { - s_started = false; - memset(&s_auth, 0, sizeof(s_auth)); - reset_session_state_locked(); - unlock_state(); - } + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_WS_DISCONNECTED, + __func__); notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_DISCONNECTED); break; case DOMAIN_MULTIMODAL_TRANSPORT_EVENT_ERROR: + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_WS_ERROR, + __func__); notify_event(CONTROL_PLANE_MULTIMODAL_EVENT_TRANSPORT_ERROR); break; @@ -1440,30 +1527,6 @@ static void downstream_playback_event_bridge( static esp_err_t control_plane_audio_uplink_send_cb(const uint8_t *data, size_t len, void *ctx) { (void)ctx; - uint32_t waited_ms = 0U; - while (waited_ms < CONTROL_PLANE_UPLINK_READY_WAIT_MS) { - esp_err_t err = control_plane_multimodal_send_audio_realtime(data, len); - if (err != ESP_ERR_INVALID_STATE) { - return err; - } - - control_plane_multimodal_runtime_state_t state = {0}; - if (control_plane_multimodal_get_runtime_state(&state) == ESP_OK && - state.connected) { - if (!state.has_active_task) { - (void)control_plane_multimodal_send_start(); - } else if (state.server_listening && - !state.client_speech_active && - !state.ptt_release_required && - !state.remote_responding) { - (void)control_plane_multimodal_send_speech_start(); - } - } - - vTaskDelay(pdMS_TO_TICKS(CONTROL_PLANE_UPLINK_READY_POLL_MS)); - waited_ms += CONTROL_PLANE_UPLINK_READY_POLL_MS; - } - return control_plane_multimodal_send_audio_realtime(data, len); } @@ -1534,14 +1597,14 @@ void control_plane_multimodal_stop(void) bool control_plane_multimodal_is_connected(void) { - return domain_runtime_multimodal_transport_is_connected(); + return transport_available(); } esp_err_t control_plane_multimodal_send_start(void) { control_plane_multimodal_auth_info_t auth = {0}; - if (!domain_runtime_multimodal_transport_is_connected()) { + if (!transport_available()) { return ESP_ERR_INVALID_STATE; } @@ -1701,6 +1764,11 @@ esp_err_t control_plane_multimodal_send_start(void) free(json); if (err != ESP_OK) { clear_start_in_progress(); + if (err != ESP_ERR_INVALID_STATE) { + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_SEND_TEXT_FAILURE, + __func__); + } return err; } @@ -1761,7 +1829,7 @@ esp_err_t control_plane_multimodal_send_stop(const char *reason) esp_err_t control_plane_multimodal_send_request_to_speak(void) { - if (!domain_runtime_multimodal_transport_is_connected()) { + if (!transport_available()) { return ESP_ERR_INVALID_STATE; } @@ -1794,7 +1862,7 @@ esp_err_t control_plane_multimodal_send_request_to_speak(void) esp_err_t control_plane_multimodal_send_speech_start(void) { - if (!domain_runtime_multimodal_transport_is_connected()) { + if (!transport_available()) { return ESP_ERR_INVALID_STATE; } @@ -1841,7 +1909,7 @@ esp_err_t control_plane_multimodal_send_speech_stop(void) unlock_state(); esp_err_t err = ESP_OK; - if (domain_runtime_multimodal_transport_is_connected() && has_active_task) { + if (transport_available() && has_active_task) { err = send_directive("continue-task", "StopSpeech", true, false); } @@ -1860,7 +1928,7 @@ esp_err_t control_plane_multimodal_send_speech_stop(void) esp_err_t control_plane_multimodal_send_cancel_speech(void) { - if (!domain_runtime_multimodal_transport_is_connected()) { + if (!transport_available()) { return ESP_ERR_INVALID_STATE; } @@ -1896,7 +1964,7 @@ esp_err_t control_plane_multimodal_send_request_to_respond( return ESP_ERR_INVALID_ARG; } - if (!domain_runtime_multimodal_transport_is_connected()) { + if (!transport_available()) { return ESP_ERR_INVALID_STATE; } @@ -1928,7 +1996,7 @@ esp_err_t control_plane_multimodal_send_update_info( return ESP_ERR_INVALID_ARG; } - if (!domain_runtime_multimodal_transport_is_connected()) { + if (!transport_available()) { return ESP_ERR_INVALID_STATE; } @@ -1988,7 +2056,7 @@ esp_err_t control_plane_multimodal_send_audio_realtime(const uint8_t *data, size return ESP_ERR_INVALID_ARG; } - if (!domain_runtime_multimodal_transport_is_connected()) { + if (!transport_available()) { return ESP_ERR_INVALID_STATE; } @@ -2008,7 +2076,41 @@ esp_err_t control_plane_multimodal_send_audio_realtime(const uint8_t *data, size return ESP_ERR_INVALID_STATE; } - return domain_runtime_multimodal_transport_send_binary_realtime(data, len); + esp_err_t err = domain_runtime_multimodal_transport_send_binary_realtime(data, len); + if (err == ESP_OK) { + control_plane_multimodal_reset_uplink_failure_window(); + return ESP_OK; + } + + if (err == ESP_ERR_INVALID_STATE) { + return err; + } + + bool threshold_reached = false; + if (lock_state()) { + TickType_t now = xTaskGetTickCount(); + TickType_t window_ticks = pdMS_TO_TICKS(CONTROL_PLANE_AUDIO_SEND_FAILURE_WINDOW_MS); + if (window_ticks == 0) { + window_ticks = 1; + } + if (s_audio_send_failure_window_start_tick == 0 || + (now - s_audio_send_failure_window_start_tick) > window_ticks) { + s_audio_send_failure_window_start_tick = now; + s_audio_send_failure_count = 0; + } + s_audio_send_failure_count++; + threshold_reached = + s_audio_send_failure_count >= CONTROL_PLANE_AUDIO_SEND_FAILURE_THRESHOLD; + unlock_state(); + } + + if (threshold_reached) { + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_SEND_BINARY_FAILURE, + __func__); + } + + return err; } esp_err_t control_plane_multimodal_get_runtime_state(control_plane_multimodal_runtime_state_t *out_state) @@ -2020,7 +2122,7 @@ esp_err_t control_plane_multimodal_get_runtime_state(control_plane_multimodal_ru if (!lock_state()) { return ESP_ERR_TIMEOUT; } - out_state->connected = domain_runtime_multimodal_transport_is_connected(); + out_state->connected = transport_available_locked(); out_state->has_active_task = s_task_id[0] != '\0'; out_state->server_listening = s_server_listening; out_state->remote_responding = s_remote_responding; @@ -2082,7 +2184,7 @@ static esp_err_t control_plane_multimodal_prepare_audio_runtime(void) unlock_state(); } - if (!domain_runtime_multimodal_transport_is_connected() || !has_active_task) { + if (!transport_available() || !has_active_task) { return ESP_ERR_INVALID_STATE; } diff --git a/components/control_plane/src/runtime/control_plane_runtime_workers.c b/components/control_plane/src/runtime/control_plane_runtime_workers.c index e2c2f46..1a36db6 100644 --- a/components/control_plane/src/runtime/control_plane_runtime_workers.c +++ b/components/control_plane/src/runtime/control_plane_runtime_workers.c @@ -40,6 +40,11 @@ #define CONTROL_PLANE_USER_WEB_RETRY_MS 5000 #define CONTROL_PLANE_USER_WEB_STOP_TIMEOUT_MS 1500 #define CONTROL_PLANE_WS_START_RETRY_DELAY_MS 2000 +#define CONTROL_PLANE_PTT_LISTENING_TIMEOUT_MS 3000 +#define CONTROL_PLANE_PTT_SPEECH_STARTED_TIMEOUT_MS 1500 +#define CONTROL_PLANE_PTT_TEARDOWN_TIMEOUT_MS 2000 +#define CONTROL_PLANE_PTT_DENIED_TIMEOUT_MS 10000 +#define CONTROL_PLANE_PTT_FOREGROUND_RECONNECT_MAX_ATTEMPTS 2 /* Liveness safety bound for the fixed two-cycle wealth spin intro (not the * animation length). On timeout the worker proceeds to reveal + rotate anyway. */ #define CONTROL_PLANE_SPIN_INTRO_TIMEOUT_MS 6000 @@ -130,6 +135,30 @@ typedef struct { char value[32]; } control_plane_model_command_t; +typedef enum { + PTT_STATE_IDLE = 0, + PTT_STATE_WAIT_WIFI, + PTT_STATE_WAIT_WS, + PTT_STATE_WAIT_START_SENT, + PTT_STATE_WAIT_LISTENING, + PTT_STATE_WAIT_SPEECH_STARTED, + PTT_STATE_STREAMING, + PTT_STATE_TEARDOWN_LOCAL, + PTT_STATE_TEARDOWN_REMOTE, + PTT_STATE_DENIED, +} ptt_state_t; + +typedef struct { + ptt_state_t state; + uint32_t generation; + TickType_t state_enter_tick; + TickType_t press_tick; + bool active; + bool foreground_connect; + uint8_t foreground_attempts; + bool pending_press_after_teardown; +} ptt_runtime_t; + typedef struct { TaskFunction_t task; bool task_with_caps; @@ -140,6 +169,7 @@ static bool s_model_cmd_queue_with_caps; static TaskHandle_t s_model_cmd_task; static bool s_ws_start_requested; static bool s_ptt_active; +static ptt_runtime_t s_ptt_runtime; #if CONFIG_TQ_DIRECTION_CONSOLE_ENABLE static TaskHandle_t s_direction_console_task; #endif @@ -156,6 +186,8 @@ static uint32_t s_fortune_refresh_generation; static volatile bool s_runtime_stop_requested; static portMUX_TYPE s_runtime_lock = portMUX_INITIALIZER_UNLOCKED; +static TaskHandle_t control_plane_get_task_handle(TaskHandle_t *slot); + static bool control_plane_runtime_stop_requested(void) { bool stop_requested = false; @@ -172,14 +204,125 @@ static void control_plane_set_runtime_stop_requested(bool stop_requested) portEXIT_CRITICAL(&s_runtime_lock); } +static const char *ptt_state_name(ptt_state_t state) +{ + switch (state) { + case PTT_STATE_IDLE: + return "IDLE"; + case PTT_STATE_WAIT_WIFI: + return "WAIT_WIFI"; + case PTT_STATE_WAIT_WS: + return "WAIT_WS"; + case PTT_STATE_WAIT_START_SENT: + return "WAIT_START_SENT"; + case PTT_STATE_WAIT_LISTENING: + return "WAIT_LISTENING"; + case PTT_STATE_WAIT_SPEECH_STARTED: + return "WAIT_SPEECH_STARTED"; + case PTT_STATE_STREAMING: + return "STREAMING"; + case PTT_STATE_TEARDOWN_LOCAL: + return "TEARDOWN_LOCAL"; + case PTT_STATE_TEARDOWN_REMOTE: + return "TEARDOWN_REMOTE"; + case PTT_STATE_DENIED: + return "DENIED"; + default: + return "UNKNOWN"; + } +} + +static void control_plane_ptt_transition_locked(ptt_state_t next_state, const char *reason) +{ + if (s_ptt_runtime.state == next_state) { + return; + } + (void)reason; + s_ptt_runtime.state = next_state; + s_ptt_runtime.state_enter_tick = xTaskGetTickCount(); + if (next_state == PTT_STATE_IDLE || + next_state == PTT_STATE_STREAMING || + next_state == PTT_STATE_DENIED) { + s_ptt_runtime.foreground_connect = false; + } +} + +static void control_plane_ptt_log_transition(ptt_state_t old_state, + ptt_state_t next_state, + uint32_t generation, + const char *reason) +{ + if (old_state == next_state) { + return; + } + ESP_LOGI(TAG, + "PTT state %s -> %s generation=%u reason=%s", + ptt_state_name(old_state), + ptt_state_name(next_state), + (unsigned)generation, + reason != NULL ? reason : "-"); +} + +static void control_plane_notify_wifi_state_task(void) +{ + TaskHandle_t task = control_plane_get_task_handle(&s_wifi_state_task); + if (task != NULL) { + xTaskNotifyGive(task); + } +} + void control_plane_set_ptt_active(bool active) { + bool notify_worker = false; + ptt_state_t old_state = PTT_STATE_IDLE; + ptt_state_t new_state = PTT_STATE_IDLE; + uint32_t generation = 0; + const char *transition_reason = active ? "press" : "release"; portENTER_CRITICAL(&s_runtime_lock); + old_state = s_ptt_runtime.state; s_ptt_active = active; if (active) { s_ws_start_requested = true; + s_ptt_runtime.generation++; + s_ptt_runtime.active = true; + s_ptt_runtime.press_tick = xTaskGetTickCount(); + s_ptt_runtime.foreground_connect = true; + s_ptt_runtime.foreground_attempts = 0; + if (s_ptt_runtime.state == PTT_STATE_TEARDOWN_LOCAL || + s_ptt_runtime.state == PTT_STATE_TEARDOWN_REMOTE) { + s_ptt_runtime.pending_press_after_teardown = true; + } else { + s_ptt_runtime.pending_press_after_teardown = false; + control_plane_ptt_transition_locked(PTT_STATE_WAIT_WIFI, "press"); + } + notify_worker = true; + } else { + s_ptt_runtime.active = false; + s_ptt_runtime.foreground_connect = false; + s_ptt_runtime.foreground_attempts = 0; + s_ptt_runtime.pending_press_after_teardown = false; + s_ptt_runtime.generation++; + if (s_ptt_runtime.state == PTT_STATE_WAIT_WIFI || + s_ptt_runtime.state == PTT_STATE_WAIT_WS || + s_ptt_runtime.state == PTT_STATE_WAIT_START_SENT || + s_ptt_runtime.state == PTT_STATE_WAIT_LISTENING || + s_ptt_runtime.state == PTT_STATE_WAIT_SPEECH_STARTED || + s_ptt_runtime.state == PTT_STATE_DENIED) { + control_plane_ptt_transition_locked(PTT_STATE_IDLE, "release_before_streaming"); + transition_reason = "release_before_streaming"; + } else if (s_ptt_runtime.state == PTT_STATE_STREAMING) { + control_plane_ptt_transition_locked(PTT_STATE_TEARDOWN_LOCAL, "release_streaming"); + transition_reason = "release_streaming"; + } + notify_worker = true; } + new_state = s_ptt_runtime.state; + generation = s_ptt_runtime.generation; portEXIT_CRITICAL(&s_runtime_lock); + control_plane_ptt_log_transition(old_state, new_state, generation, transition_reason); + if (notify_worker) { + control_plane_notify_wifi_state_task(); + } } static bool control_plane_get_ptt_active(void) @@ -984,17 +1127,19 @@ static esp_err_t control_plane_ensure_user_config_server(void) #endif } -static void control_plane_try_start_multimodal_from_state_machine(TickType_t *last_attempt_tick) +static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *last_attempt_tick, + bool foreground) { TickType_t now = xTaskGetTickCount(); TickType_t retry_ticks = pdMS_TO_TICKS(CONTROL_PLANE_WS_START_RETRY_DELAY_MS); if (retry_ticks == 0) { retry_ticks = 1; } - if (last_attempt_tick != NULL && + if (!foreground && + last_attempt_tick != NULL && *last_attempt_tick != 0 && (now - *last_attempt_tick) < retry_ticks) { - return; + return false; } if (last_attempt_tick != NULL) { *last_attempt_tick = now; @@ -1003,30 +1148,196 @@ static void control_plane_try_start_multimodal_from_state_machine(TickType_t *la esp_err_t err = control_plane_start_multimodal_client(); if (err == ESP_OK) { ESP_LOGI(TAG, "Multimodal client started"); - return; + return true; } ESP_LOGW(TAG, "Multimodal client start failed: %s", esp_err_to_name(err)); + return false; } -static void control_plane_drive_ptt_state_machine(void) +static void control_plane_ptt_set_state(ptt_state_t next_state, const char *reason) { - if (!control_plane_get_ptt_active()) { + ptt_state_t old_state = PTT_STATE_IDLE; + uint32_t generation = 0; + portENTER_CRITICAL(&s_runtime_lock); + old_state = s_ptt_runtime.state; + control_plane_ptt_transition_locked(next_state, reason); + generation = s_ptt_runtime.generation; + portEXIT_CRITICAL(&s_runtime_lock); + control_plane_ptt_log_transition(old_state, next_state, generation, reason); +} + +static ptt_runtime_t control_plane_ptt_snapshot(void) +{ + ptt_runtime_t snapshot = {0}; + portENTER_CRITICAL(&s_runtime_lock); + snapshot = s_ptt_runtime; + portEXIT_CRITICAL(&s_runtime_lock); + return snapshot; +} + +static void control_plane_handle_ptt_teardown(void) +{ + ptt_runtime_t ptt = control_plane_ptt_snapshot(); + if (ptt.state == PTT_STATE_TEARDOWN_LOCAL) { + control_plane_multimodal_stop_uplink(); + control_plane_ptt_set_state(PTT_STATE_TEARDOWN_REMOTE, "uplink_stopped"); + ptt = control_plane_ptt_snapshot(); + } + + if (ptt.state != PTT_STATE_TEARDOWN_REMOTE) { + return; + } + + esp_err_t release_err = control_plane_multimodal_send_speech_stop(); + if (release_err != ESP_OK && release_err != ESP_ERR_INVALID_STATE) { + ESP_LOGW(TAG, "PTT release failed: %s", esp_err_to_name(release_err)); + } + + TickType_t now = xTaskGetTickCount(); + TickType_t timeout_ticks = pdMS_TO_TICKS(CONTROL_PLANE_PTT_TEARDOWN_TIMEOUT_MS); + if (timeout_ticks == 0) { + timeout_ticks = 1; + } + if (release_err == ESP_OK || + (now - ptt.state_enter_tick) >= timeout_ticks || + release_err == ESP_ERR_INVALID_STATE) { + bool pending_press = false; + bool active = false; + ptt_state_t old_state = PTT_STATE_IDLE; + uint32_t generation = 0; + portENTER_CRITICAL(&s_runtime_lock); + old_state = s_ptt_runtime.state; + pending_press = s_ptt_runtime.pending_press_after_teardown; + active = s_ptt_runtime.active; + s_ptt_runtime.pending_press_after_teardown = false; + control_plane_ptt_transition_locked(PTT_STATE_IDLE, "teardown_complete"); + if (pending_press && active) { + s_ws_start_requested = true; + s_ptt_runtime.generation++; + s_ptt_runtime.foreground_connect = true; + s_ptt_runtime.foreground_attempts = 0; + s_ptt_runtime.press_tick = xTaskGetTickCount(); + control_plane_ptt_transition_locked(PTT_STATE_WAIT_WIFI, "pending_press"); + } + generation = s_ptt_runtime.generation; + ptt_state_t next_state = s_ptt_runtime.state; + portEXIT_CRITICAL(&s_runtime_lock); + control_plane_ptt_log_transition(old_state, next_state, generation, "teardown_complete"); + } +} + +static void control_plane_drive_ptt_state_machine(TickType_t *last_ws_start_attempt_tick) +{ + ptt_runtime_t ptt = control_plane_ptt_snapshot(); + if (ptt.state == PTT_STATE_IDLE) { + return; + } + + if (ptt.state == PTT_STATE_TEARDOWN_LOCAL || + ptt.state == PTT_STATE_TEARDOWN_REMOTE) { + control_plane_handle_ptt_teardown(); + return; + } + + if (!ptt.active) { + if (ptt.state != PTT_STATE_IDLE) { + control_plane_ptt_set_state(PTT_STATE_IDLE, "inactive"); + } + return; + } + + if (ptt.state == PTT_STATE_DENIED) { + return; + } + + TickType_t now = xTaskGetTickCount(); + TickType_t denied_ticks = pdMS_TO_TICKS(CONTROL_PLANE_PTT_DENIED_TIMEOUT_MS); + if (denied_ticks == 0) { + denied_ticks = 1; + } + if (ptt.press_tick != 0 && + (now - ptt.press_tick) >= denied_ticks && + ptt.state != PTT_STATE_STREAMING && + ptt.state != PTT_STATE_DENIED) { + control_plane_ptt_set_state(PTT_STATE_DENIED, "wait_timeout"); return; } if (!control_plane_is_sta_connected()) { + control_plane_ptt_set_state(PTT_STATE_WAIT_WIFI, "wifi_unavailable"); return; } if (!control_plane_multimodal_is_connected()) { + control_plane_ptt_set_state(PTT_STATE_WAIT_WS, "ws_unavailable"); + bool try_foreground = ptt.foreground_connect && + ptt.foreground_attempts < + CONTROL_PLANE_PTT_FOREGROUND_RECONNECT_MAX_ATTEMPTS; + if (try_foreground) { + bool started = control_plane_try_start_multimodal_from_state_machine( + last_ws_start_attempt_tick, + true); + portENTER_CRITICAL(&s_runtime_lock); + s_ptt_runtime.foreground_attempts++; + portEXIT_CRITICAL(&s_runtime_lock); + if (!started && + ptt.foreground_attempts + 1U >= + CONTROL_PLANE_PTT_FOREGROUND_RECONNECT_MAX_ATTEMPTS) { + control_plane_ptt_set_state(PTT_STATE_DENIED, "ws_foreground_attempts_exhausted"); + } + } else { + (void)control_plane_schedule_websocket_wait_task(); + } + return; + } + + control_plane_multimodal_runtime_state_t state = {0}; + if (control_plane_multimodal_get_runtime_state(&state) != ESP_OK) { + return; + } + + if (!state.has_active_task) { + control_plane_ptt_set_state(PTT_STATE_WAIT_START_SENT, "send_start"); + } else if (!state.server_listening) { + control_plane_ptt_set_state(PTT_STATE_WAIT_LISTENING, "wait_listening"); + } else if (!state.client_speech_active) { + control_plane_ptt_set_state(PTT_STATE_WAIT_SPEECH_STARTED, "send_speech"); + } else if (control_plane_multimodal_is_uplink_running()) { + control_plane_ptt_set_state(PTT_STATE_STREAMING, "uplink_running"); + } else { + control_plane_ptt_set_state(PTT_STATE_STREAMING, "start_uplink"); + } + + ptt = control_plane_ptt_snapshot(); + if ((ptt.state == PTT_STATE_WAIT_START_SENT || + ptt.state == PTT_STATE_WAIT_LISTENING) && + (now - ptt.state_enter_tick) >= + pdMS_TO_TICKS(CONTROL_PLANE_PTT_LISTENING_TIMEOUT_MS)) { + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT, + __func__); + control_plane_ptt_set_state(PTT_STATE_WAIT_WS, "listening_timeout"); (void)control_plane_schedule_websocket_wait_task(); + return; + } + + if (ptt.state == PTT_STATE_WAIT_SPEECH_STARTED && + (now - ptt.state_enter_tick) >= + pdMS_TO_TICKS(CONTROL_PLANE_PTT_SPEECH_STARTED_TIMEOUT_MS)) { + control_plane_multimodal_mark_transport_unavailable( + CONTROL_PLANE_TRANSPORT_LOSS_LISTENING_TIMEOUT, + __func__); + control_plane_ptt_set_state(PTT_STATE_WAIT_WS, "speech_started_timeout"); + (void)control_plane_schedule_websocket_wait_task(); + return; } esp_err_t press_err = control_plane_multimodal_handle_ptt_press_step(); - if (press_err != ESP_OK && - press_err != ESP_ERR_INVALID_STATE && - press_err != ESP_ERR_NOT_SUPPORTED) { + if (press_err == ESP_OK) { + control_plane_multimodal_reset_uplink_failure_window(); + } else if (press_err != ESP_ERR_INVALID_STATE && + press_err != ESP_ERR_NOT_SUPPORTED) { ESP_LOGW(TAG, "PTT press step failed: %s", esp_err_to_name(press_err)); } } @@ -1066,6 +1377,7 @@ static void control_plane_wifi_state_task(void *arg) TickType_t last_ws_start_attempt_tick = 0; uint32_t reconnect_delay_ms = CONTROL_PLANE_WIFI_RECONNECT_BASE_DELAY_MS; bool reconnect_attempted = false; + uint8_t ptt_wifi_foreground_attempts = 0; if (!was_connected) { ESP_LOGI(TAG, "Wi-Fi STA unavailable at runtime start; requesting disconnected expression"); @@ -1088,7 +1400,7 @@ static void control_plane_wifi_state_task(void *arg) TickType_t next_delay_ticks = control_plane_get_ptt_active() ? pdMS_TO_TICKS(CONTROL_PLANE_PTT_ACTIVE_POLL_INTERVAL_MS) : poll_ticks; - vTaskDelay(next_delay_ticks > 0 ? next_delay_ticks : 1); + (void)ulTaskNotifyTake(pdTRUE, next_delay_ticks > 0 ? next_delay_ticks : 1); bool connected = control_plane_is_sta_connected(); TickType_t now_tick = xTaskGetTickCount(); @@ -1101,7 +1413,9 @@ static void control_plane_wifi_state_task(void *arg) (unsigned)CONTROL_PLANE_WIFI_DISCONNECT_GRACE_MS); } + bool ptt_foreground_disconnect = control_plane_get_ptt_active(); bool disconnect_grace_elapsed = + ptt_foreground_disconnect || disconnected_expression_active || (now_tick - disconnect_started_tick) >= disconnect_grace_ticks; if (!disconnect_grace_elapsed) { @@ -1132,20 +1446,31 @@ static void control_plane_wifi_state_task(void *arg) if (reconnect_delay_ticks == 0) { reconnect_delay_ticks = 1; } - bool reconnect_due = !reconnect_attempted || + bool ptt_foreground = control_plane_get_ptt_active() && + ptt_wifi_foreground_attempts < + CONTROL_PLANE_PTT_FOREGROUND_RECONNECT_MAX_ATTEMPTS; + bool reconnect_due = ptt_foreground || + !reconnect_attempted || (now_tick - last_reconnect_attempt_tick) >= reconnect_delay_ticks; if (reconnect_due && domain_runtime_wifi_portal_has_saved_credentials()) { esp_err_t reconnect_err = control_plane_attempt_wifi_reconnect(); now_tick = xTaskGetTickCount(); reconnect_attempted = true; last_reconnect_attempt_tick = now_tick; + if (ptt_foreground) { + ptt_wifi_foreground_attempts++; + } if (reconnect_err == ESP_OK) { connected = true; reconnect_delay_ms = CONTROL_PLANE_WIFI_RECONNECT_BASE_DELAY_MS; + ptt_wifi_foreground_attempts = 0; } else { reconnect_delay_ms = control_plane_next_wifi_reconnect_delay_ms(reconnect_delay_ms); } } + if (!control_plane_get_ptt_active()) { + ptt_wifi_foreground_attempts = 0; + } } if (connected && disconnect_pending && !disconnected_expression_active) { @@ -1178,10 +1503,10 @@ static void control_plane_wifi_state_task(void *arg) if (connected) { bool ws_start_due = control_plane_consume_ws_start_requested(); if (ws_start_due && !control_plane_multimodal_is_connected()) { - control_plane_try_start_multimodal_from_state_machine(&last_ws_start_attempt_tick); + control_plane_try_start_multimodal_from_state_machine(&last_ws_start_attempt_tick, false); now_tick = xTaskGetTickCount(); } - control_plane_drive_ptt_state_machine(); + control_plane_drive_ptt_state_machine(&last_ws_start_attempt_tick); TickType_t web_retry_ticks = pdMS_TO_TICKS(CONTROL_PLANE_USER_WEB_RETRY_MS); if (web_retry_ticks == 0) { diff --git a/components/control_plane/src/runtime/control_plane_system_events.c b/components/control_plane/src/runtime/control_plane_system_events.c index 34aaf34..adc14f0 100644 --- a/components/control_plane/src/runtime/control_plane_system_events.c +++ b/components/control_plane/src/runtime/control_plane_system_events.c @@ -135,10 +135,6 @@ static void control_plane_dispatch_key_event(const control_plane_key_event_t *ev case CONTROL_PLANE_KEY_EVENT_RELEASED: { control_plane_set_ptt_active(false); - esp_err_t release_err = control_plane_multimodal_handle_ptt_release(); - if (release_err != ESP_OK && release_err != ESP_ERR_INVALID_STATE) { - ESP_LOGW(TAG, "PTT release failed: %s", esp_err_to_name(release_err)); - } break; } diff --git a/components/platform/src/lcd_direct.c b/components/platform/src/lcd_direct.c index 572d9a0..109bf1a 100644 --- a/components/platform/src/lcd_direct.c +++ b/components/platform/src/lcd_direct.c @@ -419,6 +419,17 @@ static esp_err_t lcd_direct_draw_asset_sequence_once(const lcd_direct_request_t vTaskDelay(pdMS_TO_TICKS(frame_ms)); } + static TickType_t s_last_draw_success_tick; + TickType_t now = xTaskGetTickCount(); + if (s_last_draw_success_tick == 0 || + (now - s_last_draw_success_tick) >= pdMS_TO_TICKS(5000)) { + s_last_draw_success_tick = now; + ESP_LOGI(TAG, + "LCD asset sequence drawn asset=%s frames=%u", + request->asset_dir, + (unsigned)frame_count); + } + return ESP_OK; }