diff --git a/components/control_plane/src/runtime/control_plane_runtime_workers.c b/components/control_plane/src/runtime/control_plane_runtime_workers.c index 81a7eb5..c60ed57 100644 --- a/components/control_plane/src/runtime/control_plane_runtime_workers.c +++ b/components/control_plane/src/runtime/control_plane_runtime_workers.c @@ -49,6 +49,8 @@ #define CONTROL_PLANE_SPIN_INTRO_TIMEOUT_MS 6000 #define CONTROL_PLANE_COMMAND_ROTATION_DEG 100.0f #define CONTROL_PLANE_COMMAND_RETURN_DELAY_MS 5000 +#define CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES 512U +#define CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES 256U #if (configSUPPORT_STATIC_ALLOCATION == 1) && \ ((defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) || \ @@ -80,6 +82,38 @@ #define CONFIG_TQ_TASK_MONITOR_MAX_TASKS 64 #endif +#ifndef CONFIG_TQ_MEMORY_MONITOR_ENABLE +#define CONFIG_TQ_MEMORY_MONITOR_ENABLE 1 +#endif + +#ifndef CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS +#define CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS 1000 +#endif + +#ifndef CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS +#define CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS 10000 +#endif + +#ifndef CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES +#define CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES 16384 +#endif + +#ifndef CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES +#define CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES 4096 +#endif + +#ifndef CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE +#define CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE 0 +#endif + +#ifndef CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE +#define CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE 0 +#endif + +#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE +#define CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD 2U +#endif + #if !defined(CONFIG_TQ_PTT_MODE_HOLD_TO_TALK) && !defined(CONFIG_TQ_PTT_MODE_CLICK_TO_TALK) #define CONFIG_TQ_PTT_MODE_CLICK_TO_TALK 1 #endif @@ -182,6 +216,10 @@ static QueueHandle_t s_model_cmd_queue; static bool s_model_cmd_queue_with_caps; static TaskHandle_t s_model_cmd_task; static bool s_ws_start_requested; +#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE +static uint32_t s_multimodal_network_failure_count; +static bool s_wifi_reassociation_requested; +#endif static bool s_ptt_active; static ptt_runtime_t s_ptt_runtime; static TaskHandle_t s_lcd_render_loop_task; @@ -197,7 +235,200 @@ static uint32_t s_fortune_refresh_generation; static volatile bool s_runtime_stop_requested; static portMUX_TYPE s_runtime_lock = portMUX_INITIALIZER_UNLOCKED; +#if CONFIG_TQ_MEMORY_MONITOR_ENABLE +typedef struct { + TickType_t last_sample_tick; + TickType_t last_report_tick; + uint32_t reported_alloc_fail_count; + size_t reported_internal_free; + size_t reported_dma_largest; + bool initialized; + bool pressure_active; +} control_plane_memory_monitor_t; + +static control_plane_memory_monitor_t s_memory_monitor; +static volatile uint32_t s_alloc_fail_count; +static volatile size_t s_last_alloc_fail_size; +static volatile uint32_t s_last_alloc_fail_caps; +static const char *volatile s_last_alloc_fail_function; +static bool s_alloc_failure_hook_registered; +#endif + static TaskHandle_t control_plane_get_task_handle(TaskHandle_t *slot); +static const char *ptt_state_name(ptt_state_t state); + +#if CONFIG_TQ_MEMORY_MONITOR_ENABLE +static void IRAM_ATTR control_plane_alloc_failed_hook(size_t size, + uint32_t caps, + const char *function_name) +{ + __atomic_store_n(&s_last_alloc_fail_size, size, __ATOMIC_RELAXED); + __atomic_store_n(&s_last_alloc_fail_caps, caps, __ATOMIC_RELAXED); + __atomic_store_n(&s_last_alloc_fail_function, function_name, __ATOMIC_RELAXED); + (void)__atomic_add_fetch(&s_alloc_fail_count, 1U, __ATOMIC_RELEASE); +} + +static void control_plane_memory_monitor_register_hook(void) +{ + if (s_alloc_failure_hook_registered) { + return; + } + + esp_err_t err = heap_caps_register_failed_alloc_callback(control_plane_alloc_failed_hook); + if (err == ESP_OK) { + s_alloc_failure_hook_registered = true; + ESP_LOGI(TAG, "Heap allocation failure hook registered"); + } else { + ESP_LOGW(TAG, "Heap allocation failure hook registration failed: %s", + esp_err_to_name(err)); + } +} + +static unsigned control_plane_heap_contiguous_percent(const multi_heap_info_t *info) +{ + if (info == NULL || info->total_free_bytes == 0U) { + return 0U; + } + return (unsigned)((info->largest_free_block * 100U) / info->total_free_bytes); +} + +static void control_plane_memory_monitor_sample(TickType_t now_tick, bool force) +{ + TickType_t sample_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS); + TickType_t report_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS); + if (sample_ticks == 0) { + sample_ticks = 1; + } + if (report_ticks == 0) { + report_ticks = 1; + } + + if (!force && s_memory_monitor.initialized && + (now_tick - s_memory_monitor.last_sample_tick) < sample_ticks) { + return; + } + s_memory_monitor.last_sample_tick = now_tick; + + multi_heap_info_t internal = {0}; + multi_heap_info_t dma = {0}; + multi_heap_info_t psram = {0}; + heap_caps_get_info(&internal, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + heap_caps_get_info(&dma, MALLOC_CAP_DMA | MALLOC_CAP_8BIT); + heap_caps_get_info(&psram, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + + uint32_t alloc_fail_count = __atomic_load_n(&s_alloc_fail_count, __ATOMIC_ACQUIRE); + bool alloc_failure_pending = alloc_fail_count != s_memory_monitor.reported_alloc_fail_count; + bool pressure = internal.total_free_bytes < CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES || + dma.largest_free_block < CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES; + bool pressure_changed = s_memory_monitor.initialized && + pressure != s_memory_monitor.pressure_active; + bool watermark_drop = s_memory_monitor.initialized && + (internal.total_free_bytes + CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES <= + s_memory_monitor.reported_internal_free || + dma.largest_free_block + CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES <= + s_memory_monitor.reported_dma_largest); + bool periodic_due = !s_memory_monitor.initialized || + (now_tick - s_memory_monitor.last_report_tick) >= report_ticks; + if (!force && !alloc_failure_pending && !pressure_changed && !watermark_drop && !periodic_due) { + return; + } + + const char *reason = force || !s_memory_monitor.initialized ? "startup" : + alloc_failure_pending ? "alloc_failure" : + pressure_changed ? (pressure ? "pressure_enter" : "pressure_exit") : + watermark_drop ? "watermark_drop" : + pressure ? "pressure_periodic" : "periodic"; + control_plane_multimodal_runtime_state_t mm = {0}; + bool have_mm = control_plane_multimodal_get_runtime_state(&mm) == ESP_OK; + domain_local_command_snapshot_t local_command = {0}; + bool have_local_command = + domain_runtime_local_command_get_snapshot(&local_command) == ESP_OK; + ptt_state_t ptt_state; + bool ptt_active; + portENTER_CRITICAL(&s_runtime_lock); + ptt_state = s_ptt_runtime.state; + ptt_active = s_ptt_runtime.active; + portEXIT_CRITICAL(&s_runtime_lock); + + if (pressure || alloc_failure_pending) { + ESP_LOGW(TAG, + "Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)", + reason, + (unsigned)internal.total_free_bytes, + (unsigned)internal.minimum_free_bytes, + (unsigned)internal.largest_free_block, + (unsigned)internal.allocated_blocks, + (unsigned)internal.free_blocks, + control_plane_heap_contiguous_percent(&internal), + (unsigned)dma.total_free_bytes, + (unsigned)dma.minimum_free_bytes, + (unsigned)dma.largest_free_block, + (unsigned)dma.allocated_blocks, + (unsigned)dma.free_blocks, + control_plane_heap_contiguous_percent(&dma), + (unsigned)psram.total_free_bytes, + (unsigned)psram.minimum_free_bytes, + (unsigned)psram.largest_free_block); + } else { + ESP_LOGI(TAG, + "Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)", + reason, + (unsigned)internal.total_free_bytes, + (unsigned)internal.minimum_free_bytes, + (unsigned)internal.largest_free_block, + (unsigned)internal.allocated_blocks, + (unsigned)internal.free_blocks, + control_plane_heap_contiguous_percent(&internal), + (unsigned)dma.total_free_bytes, + (unsigned)dma.minimum_free_bytes, + (unsigned)dma.largest_free_block, + (unsigned)dma.allocated_blocks, + (unsigned)dma.free_blocks, + control_plane_heap_contiguous_percent(&dma), + (unsigned)psram.total_free_bytes, + (unsigned)psram.minimum_free_bytes, + (unsigned)psram.largest_free_block); + } + + ESP_LOGI(TAG, + "Memory context ptt=%s active=%d wifi=%d ws(conn=%d starting=%d stopping=%d) task=%d listening=%d responding=%d uplink=%d local_kws(started=%d detecting=%d detections=%u stack_hwm=%u) alloc_failures=%u", + ptt_state_name(ptt_state), + ptt_active ? 1 : 0, + control_plane_is_sta_connected() ? 1 : 0, + (have_mm && mm.connected) ? 1 : 0, + (have_mm && mm.starting) ? 1 : 0, + (have_mm && mm.stopping) ? 1 : 0, + (have_mm && mm.has_active_task) ? 1 : 0, + (have_mm && mm.server_listening) ? 1 : 0, + (have_mm && mm.remote_responding) ? 1 : 0, + control_plane_multimodal_is_uplink_running() ? 1 : 0, + (have_local_command && local_command.started) ? 1 : 0, + (have_local_command && local_command.detection_running) ? 1 : 0, + have_local_command ? (unsigned)local_command.detection_count : 0U, + have_local_command ? (unsigned)local_command.task_stack_high_water_bytes : 0U, + (unsigned)alloc_fail_count); + + if (alloc_failure_pending) { + size_t failed_size = __atomic_load_n(&s_last_alloc_fail_size, __ATOMIC_RELAXED); + uint32_t failed_caps = __atomic_load_n(&s_last_alloc_fail_caps, __ATOMIC_RELAXED); + const char *failed_function = __atomic_load_n(&s_last_alloc_fail_function, + __ATOMIC_RELAXED); + ESP_LOGE(TAG, + "Heap allocation failure count=%u newest(size=%u caps=0x%08x function=%s)", + (unsigned)alloc_fail_count, + (unsigned)failed_size, + (unsigned)failed_caps, + failed_function != NULL ? failed_function : "unknown"); + s_memory_monitor.reported_alloc_fail_count = alloc_fail_count; + } + + s_memory_monitor.initialized = true; + s_memory_monitor.pressure_active = pressure; + s_memory_monitor.reported_internal_free = internal.total_free_bytes; + s_memory_monitor.reported_dma_largest = dma.largest_free_block; + s_memory_monitor.last_report_tick = now_tick; +} +#endif static bool control_plane_runtime_stop_requested(void) { @@ -880,6 +1111,58 @@ esp_err_t control_plane_schedule_websocket_wait_task(void) return ESP_OK; } +#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE +static void control_plane_record_multimodal_network_result(esp_err_t result) +{ + uint32_t failures = 0U; + bool reassociation_requested = false; + portENTER_CRITICAL(&s_runtime_lock); + if (result == ESP_OK) { + s_multimodal_network_failure_count = 0U; + } else if (domain_runtime_is_network_connect_error(result)) { + s_multimodal_network_failure_count++; + failures = s_multimodal_network_failure_count; + if (failures >= CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD && + !s_wifi_reassociation_requested) { + s_wifi_reassociation_requested = true; + reassociation_requested = true; + } + } + portEXIT_CRITICAL(&s_runtime_lock); + +#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE + if (result == ESP_OK) { + ESP_LOGI(TAG, "Multimodal network failure streak cleared"); + } else if (domain_runtime_is_network_connect_error(result)) { + ESP_LOGW(TAG, + "Multimodal network failure streak=%u reassociation_requested=%d", + (unsigned)failures, + reassociation_requested ? 1 : 0); + } +#else + (void)failures; + (void)reassociation_requested; +#endif +} + +static bool control_plane_consume_wifi_reassociation_request(void) +{ + bool requested = false; + portENTER_CRITICAL(&s_runtime_lock); + requested = s_wifi_reassociation_requested; + s_wifi_reassociation_requested = false; + portEXIT_CRITICAL(&s_runtime_lock); + return requested; +} + +static void control_plane_reset_multimodal_network_failure_streak(void) +{ + portENTER_CRITICAL(&s_runtime_lock); + s_multimodal_network_failure_count = 0U; + portEXIT_CRITICAL(&s_runtime_lock); +} +#endif + #if CONFIG_TQ_TASK_MONITOR_ENABLE static const char *control_plane_task_state_name(eTaskState state) { @@ -1130,6 +1413,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la esp_err_t err = control_plane_start_multimodal_client(); if (err == ESP_OK) { +#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE + control_plane_record_multimodal_network_result(ESP_OK); +#endif ESP_LOGI(TAG, "Multimodal client started"); return true; } @@ -1137,6 +1423,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la return false; } +#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE + control_plane_record_multimodal_network_result(err); +#endif ESP_LOGW(TAG, "Multimodal client start failed: %s", esp_err_to_name(err)); return false; } @@ -1421,6 +1710,10 @@ static void control_plane_wifi_state_task(void *arg) bool reconnect_attempted = false; uint8_t ptt_wifi_foreground_attempts = 0; +#if CONFIG_TQ_MEMORY_MONITOR_ENABLE + control_plane_memory_monitor_sample(xTaskGetTickCount(), true); +#endif + if (!was_connected) { ESP_LOGI(TAG, "Wi-Fi STA unavailable at runtime start; requesting disconnected expression"); control_plane_request_wifi_state_expression(DOMAIN_EXPRESSION_WIFI_DISCONNECTED, @@ -1447,6 +1740,10 @@ static void control_plane_wifi_state_task(void *arg) bool connected = control_plane_is_sta_connected(); TickType_t now_tick = xTaskGetTickCount(); +#if CONFIG_TQ_MEMORY_MONITOR_ENABLE + control_plane_memory_monitor_sample(now_tick, false); +#endif + if (!connected) { if (!disconnect_pending) { disconnect_pending = true; @@ -1515,6 +1812,28 @@ static void control_plane_wifi_state_task(void *arg) } } +#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE + if (connected && control_plane_consume_wifi_reassociation_request()) { + ESP_LOGW(TAG, + "Network recovery experiment: controlled STA reassociation started after %u HTTPS connection failures", + (unsigned)CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD); + domain_runtime_behavior_set_keep_awake(true); + esp_err_t reassoc_err = domain_runtime_wifi_portal_reconnect_sta( + CONTROL_PLANE_WIFI_RECONNECT_TIMEOUT_MS); + domain_runtime_behavior_set_keep_awake(false); + connected = control_plane_is_sta_connected(); + ESP_LOGW(TAG, + "Network recovery experiment: controlled STA reassociation result=%s connected=%d", + esp_err_to_name(reassoc_err), + connected ? 1 : 0); + if (reassoc_err == ESP_OK && connected) { + control_plane_reset_multimodal_network_failure_streak(); + last_ws_start_attempt_tick = 0; + (void)control_plane_schedule_websocket_wait_task(); + } + } +#endif + if (connected && disconnect_pending && !disconnected_expression_active) { ESP_LOGI(TAG, "Wi-Fi STA reconnected within disconnect grace"); disconnect_pending = false; @@ -1873,6 +2192,11 @@ esp_err_t control_plane_start_runtime_workers(void) { control_plane_set_runtime_stop_requested(false); +#if CONFIG_TQ_MEMORY_MONITOR_ENABLE + memset(&s_memory_monitor, 0, sizeof(s_memory_monitor)); + control_plane_memory_monitor_register_hook(); +#endif + esp_err_t queue_err = control_plane_ensure_model_cmd_queue(); if (queue_err != ESP_OK) { return queue_err; diff --git a/components/domain/include/domain.h b/components/domain/include/domain.h index 5772071..9e8da07 100644 --- a/components/domain/include/domain.h +++ b/components/domain/include/domain.h @@ -430,6 +430,8 @@ bool domain_runtime_wifi_connect_error_keeps_credentials(esp_err_t err); * - Returns explicit failure on timeout/connection error. */ esp_err_t domain_runtime_wifi_portal_connect_sta(int timeout_ms); +esp_err_t domain_runtime_wifi_portal_reconnect_sta(int timeout_ms); +bool domain_runtime_is_network_connect_error(esp_err_t err); esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password); esp_err_t domain_runtime_wifi_portal_clear_saved_credentials(void); diff --git a/components/domain/src/runtime/domain_runtime_adapters.c b/components/domain/src/runtime/domain_runtime_adapters.c index 719c931..f2ea6d6 100644 --- a/components/domain/src/runtime/domain_runtime_adapters.c +++ b/components/domain/src/runtime/domain_runtime_adapters.c @@ -1237,6 +1237,16 @@ esp_err_t domain_runtime_wifi_portal_connect_sta(int timeout_ms) return platform_wifi_portal_connect_sta(timeout_ms); } +esp_err_t domain_runtime_wifi_portal_reconnect_sta(int timeout_ms) +{ + return platform_wifi_portal_reconnect_sta(timeout_ms); +} + +bool domain_runtime_is_network_connect_error(esp_err_t err) +{ + return platform_https_is_connect_error(err); +} + esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password) { return platform_wifi_portal_save_credentials(ssid, password); diff --git a/components/platform/CMakeLists.txt b/components/platform/CMakeLists.txt index 4b1a626..cb42efe 100644 --- a/components/platform/CMakeLists.txt +++ b/components/platform/CMakeLists.txt @@ -52,6 +52,10 @@ set(platform_priv_requires nvs_flash ) +if(CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE) + list(APPEND platform_priv_requires lwip) +endif() + idf_component_register( SRCS ${platform_srcs} diff --git a/components/platform/include/platform.h b/components/platform/include/platform.h index 56d953b..9e6f373 100644 --- a/components/platform/include/platform.h +++ b/components/platform/include/platform.h @@ -223,6 +223,7 @@ bool platform_time_is_synced(void); */ esp_err_t platform_https_post_json(const platform_https_post_json_request_t *request, platform_https_response_t *response); +bool platform_https_is_connect_error(esp_err_t err); /** * @brief Rotate stepper synchronously with a bounded timeout. @@ -258,6 +259,7 @@ esp_err_t platform_wifi_portal_start_config_ap(const char *ssid_prefix); * - On timeout/failure, no hidden destructive continuation is started after return. */ esp_err_t platform_wifi_portal_connect_sta(int timeout_ms); +esp_err_t platform_wifi_portal_reconnect_sta(int timeout_ms); esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password); esp_err_t platform_wifi_portal_clear_saved_credentials(void); bool platform_wifi_connect_error_keeps_credentials(esp_err_t err); diff --git a/components/platform/internal/wifi_portal.h b/components/platform/internal/wifi_portal.h index 796b6a1..cc525b3 100644 --- a/components/platform/internal/wifi_portal.h +++ b/components/platform/internal/wifi_portal.h @@ -47,6 +47,7 @@ esp_err_t wifi_portal_start_config_ap(const char *ssid_prefix); * for the connection to succeed within the provided timeout (milliseconds). */ esp_err_t wifi_portal_connect_sta(int timeout_ms); +esp_err_t wifi_portal_reconnect_sta(int timeout_ms); /** * @brief Save Wi-Fi credentials into NVS. diff --git a/components/platform/src/runtime/platform_behavior_runtime.c b/components/platform/src/runtime/platform_behavior_runtime.c index 79f62d7..a07cc54 100644 --- a/components/platform/src/runtime/platform_behavior_runtime.c +++ b/components/platform/src/runtime/platform_behavior_runtime.c @@ -138,6 +138,11 @@ esp_err_t platform_wifi_portal_connect_sta(int timeout_ms) return wifi_portal_connect_sta(timeout_ms); } +esp_err_t platform_wifi_portal_reconnect_sta(int timeout_ms) +{ + return wifi_portal_reconnect_sta(timeout_ms); +} + esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password) { return wifi_portal_save_credentials(ssid, password); diff --git a/components/platform/src/runtime/platform_https_client.c b/components/platform/src/runtime/platform_https_client.c index ef1f84b..2f19ecf 100644 --- a/components/platform/src/runtime/platform_https_client.c +++ b/components/platform/src/runtime/platform_https_client.c @@ -18,12 +18,355 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" +#ifndef CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE +#define CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE 0 +#endif + +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE +#include +#include +#include + +#include "esp_heap_caps.h" +#include "esp_netif.h" +#include "esp_timer.h" +#include "esp_wifi.h" +#include "freertos/semphr.h" +#include "ping/ping_sock.h" +#endif + #define PLATFORM_HTTPS_DEFAULT_TIMEOUT_MS 10000 #define PLATFORM_HTTPS_DNS_RETRY_COUNT 2 #define PLATFORM_HTTPS_DNS_RETRY_DELAY_MS 500 #define PLATFORM_HTTPS_HOST_MAX_LEN 256 +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE +#define PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS 400 +#define PLATFORM_HTTPS_PROBE_TCP_TIMEOUT_MS 800 +#endif static const char *TAG = "platform_https"; +static const char *dns_error_name(int err); + +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE +static volatile uint32_t s_dns_lookup_count; +static volatile uint32_t s_dns_lookup_failure_count; +static volatile uint32_t s_dns_lookup_active; +static struct sockaddr_storage s_last_resolved_addr; +static socklen_t s_last_resolved_addr_len; +static char s_last_resolved_host[PLATFORM_HTTPS_HOST_MAX_LEN]; +static portMUX_TYPE s_probe_cache_lock = portMUX_INITIALIZER_UNLOCKED; + +typedef struct { + SemaphoreHandle_t done; + volatile uint32_t replies; +} https_ping_probe_context_t; + +static void https_ping_success(esp_ping_handle_t handle, void *args) +{ + (void)handle; + https_ping_probe_context_t *context = (https_ping_probe_context_t *)args; + if (context != NULL) { + context->replies++; + } +} + +static void https_ping_end(esp_ping_handle_t handle, void *args) +{ + (void)handle; + https_ping_probe_context_t *context = (https_ping_probe_context_t *)args; + if (context != NULL && context->done != NULL) { + (void)xSemaphoreGive(context->done); + } +} + +static void run_ipv4_ping_probe(const char *label, const esp_ip4_addr_t *target) +{ + if (target == NULL || target->addr == 0U) { + ESP_LOGW(TAG, "Network probe ping label=%s skipped: no IPv4 target", label); + return; + } + + https_ping_probe_context_t context = { + .done = xSemaphoreCreateBinary(), + }; + if (context.done == NULL) { + ESP_LOGW(TAG, "Network probe ping label=%s failed: no semaphore memory", label); + return; + } + + esp_ping_config_t config = ESP_PING_DEFAULT_CONFIG(); + config.count = 1; + config.interval_ms = 0; + config.timeout_ms = PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS; + ip_addr_set_ip4_u32(&config.target_addr, target->addr); + + esp_ping_callbacks_t callbacks = { + .cb_args = &context, + .on_ping_success = https_ping_success, + .on_ping_timeout = NULL, + .on_ping_end = https_ping_end, + }; + esp_ping_handle_t session = NULL; + int64_t started_us = esp_timer_get_time(); + esp_err_t err = esp_ping_new_session(&config, &callbacks, &session); + if (err == ESP_OK) { + err = esp_ping_start(session); + } + if (err == ESP_OK) { + TickType_t wait_ticks = pdMS_TO_TICKS(PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS + 300); + if (wait_ticks == 0) { + wait_ticks = 1; + } + if (xSemaphoreTake(context.done, wait_ticks) != pdTRUE) { + err = ESP_ERR_TIMEOUT; + } + } + if (session != NULL) { + (void)esp_ping_stop(session); + (void)esp_ping_delete_session(session); + } + + ESP_LOGW(TAG, + "Network probe ping label=%s target=" IPSTR " result=%s replies=%u elapsed_ms=%lld", + label, + IP2STR(target), + err == ESP_OK && context.replies > 0U ? "reachable" : + (err == ESP_OK ? "no_reply" : esp_err_to_name(err)), + (unsigned)context.replies, + (long long)((esp_timer_get_time() - started_us) / 1000)); + vSemaphoreDelete(context.done); +} + +static void cache_resolved_endpoint(const char *host, const struct addrinfo *addr) +{ + const struct addrinfo *selected = addr; + while (selected != NULL && + selected->ai_family != AF_INET && + selected->ai_family != AF_INET6) { + selected = selected->ai_next; + } + if (host == NULL || selected == NULL || selected->ai_addr == NULL || + selected->ai_addrlen > sizeof(s_last_resolved_addr)) { + return; + } + + struct sockaddr_storage endpoint = {0}; + memcpy(&endpoint, selected->ai_addr, selected->ai_addrlen); + if (selected->ai_family == AF_INET) { + ((struct sockaddr_in *)&endpoint)->sin_port = htons(443); + } else { + ((struct sockaddr_in6 *)&endpoint)->sin6_port = htons(443); + } + + portENTER_CRITICAL(&s_probe_cache_lock); + s_last_resolved_addr = endpoint; + s_last_resolved_addr_len = selected->ai_addrlen; + strlcpy(s_last_resolved_host, host, sizeof(s_last_resolved_host)); + portEXIT_CRITICAL(&s_probe_cache_lock); +} + +static void run_cached_endpoint_probe(void) +{ + struct sockaddr_storage endpoint = {0}; + socklen_t endpoint_len = 0; + char host[PLATFORM_HTTPS_HOST_MAX_LEN] = {0}; + portENTER_CRITICAL(&s_probe_cache_lock); + endpoint = s_last_resolved_addr; + endpoint_len = s_last_resolved_addr_len; + strlcpy(host, s_last_resolved_host, sizeof(host)); + portEXIT_CRITICAL(&s_probe_cache_lock); + + if (endpoint_len == 0 || host[0] == '\0') { + ESP_LOGW(TAG, "Network probe cached TCP skipped: no successful DNS result cached"); + return; + } + + char address[64] = {0}; + void *address_ptr = endpoint.ss_family == AF_INET ? + (void *)&((struct sockaddr_in *)&endpoint)->sin_addr : + (void *)&((struct sockaddr_in6 *)&endpoint)->sin6_addr; + const char *formatted = inet_ntop(endpoint.ss_family, + address_ptr, + address, + sizeof(address)); + int64_t started_us = esp_timer_get_time(); + int sock = socket(endpoint.ss_family, SOCK_STREAM, IPPROTO_TCP); + int socket_error = 0; + bool connected = false; + if (sock >= 0) { + int flags = fcntl(sock, F_GETFL, 0); + if (flags >= 0 && fcntl(sock, F_SETFL, flags | O_NONBLOCK) == 0) { + int connect_result = connect(sock, + (const struct sockaddr *)&endpoint, + endpoint_len); + if (connect_result == 0) { + connected = true; + } else if (errno == EINPROGRESS) { + fd_set write_set; + FD_ZERO(&write_set); + FD_SET(sock, &write_set); + struct timeval timeout = { + .tv_sec = 0, + .tv_usec = PLATFORM_HTTPS_PROBE_TCP_TIMEOUT_MS * 1000, + }; + int selected = select(sock + 1, NULL, &write_set, NULL, &timeout); + socklen_t error_len = sizeof(socket_error); + if (selected > 0 && + getsockopt(sock, + SOL_SOCKET, + SO_ERROR, + &socket_error, + &error_len) == 0 && + socket_error == 0) { + connected = true; + } else if (selected == 0) { + socket_error = ETIMEDOUT; + } else if (socket_error == 0) { + socket_error = errno; + } + } else { + socket_error = errno; + } + } else { + socket_error = errno; + } + close(sock); + } else { + socket_error = errno; + } + + ESP_LOGW(TAG, + "Network probe cached TCP host=%s address=%s:443 result=%s socket_error=%d elapsed_ms=%lld", + host, + formatted != NULL ? address : "unknown", + connected ? "reachable" : "failed", + socket_error, + (long long)((esp_timer_get_time() - started_us) / 1000)); +} + +static void run_dns_failure_network_probes(const esp_netif_ip_info_t *ip, + const esp_netif_dns_info_t *fallback_dns) +{ + ESP_LOGW(TAG, "Network probe bundle started after DNS failure"); + run_ipv4_ping_probe("gateway", ip != NULL ? &ip->gw : NULL); + if (fallback_dns != NULL && fallback_dns->ip.type == ESP_IPADDR_TYPE_V4) { + run_ipv4_ping_probe("fallback_dns", &fallback_dns->ip.u_addr.ip4); + } else { + ESP_LOGW(TAG, "Network probe fallback DNS ping skipped: no IPv4 address"); + } + run_cached_endpoint_probe(); + ESP_LOGW(TAG, "Network probe bundle completed"); +} + +static const char *dhcp_status_name(esp_netif_dhcp_status_t status) +{ + switch (status) { + case ESP_NETIF_DHCP_INIT: + return "init"; + case ESP_NETIF_DHCP_STARTED: + return "started"; + case ESP_NETIF_DHCP_STOPPED: + return "stopped"; + default: + return "unknown"; + } +} + +static const char *format_dns_address(const esp_netif_dns_info_t *dns, + char *buffer, + size_t buffer_size) +{ + if (dns == NULL || buffer == NULL || buffer_size == 0U) { + return "invalid"; + } + if (dns->ip.type == ESP_IPADDR_TYPE_V6) { + int written = snprintf(buffer, + buffer_size, + IPV6STR, + IPV62STR(dns->ip.u_addr.ip6)); + return written >= 0 && (size_t)written < buffer_size ? buffer : "unknown"; + } + + int written = snprintf(buffer, + buffer_size, + IPSTR, + IP2STR(&dns->ip.u_addr.ip4)); + return written >= 0 && (size_t)written < buffer_size ? buffer : "unknown"; +} + +static void log_dns_failure_diagnostics(const char *host, + int attempt, + int dns_err, + uint32_t lookup_id, + uint32_t elapsed_ms) +{ + esp_netif_t *sta = esp_netif_get_handle_from_ifkey("WIFI_STA_DEF"); + esp_netif_ip_info_t ip = {0}; + esp_netif_dns_info_t dns_main = {0}; + esp_netif_dns_info_t dns_backup = {0}; + esp_netif_dns_info_t dns_fallback = {0}; + esp_netif_dhcp_status_t dhcp_status = ESP_NETIF_DHCP_INIT; + esp_err_t ip_err = sta != NULL ? esp_netif_get_ip_info(sta, &ip) : ESP_ERR_NOT_FOUND; + esp_err_t main_err = esp_netif_get_dns_info(sta, ESP_NETIF_DNS_MAIN, &dns_main); + esp_err_t backup_err = esp_netif_get_dns_info(sta, ESP_NETIF_DNS_BACKUP, &dns_backup); + esp_err_t fallback_err = esp_netif_get_dns_info(sta, + ESP_NETIF_DNS_FALLBACK, + &dns_fallback); + esp_err_t dhcp_err = sta != NULL ? esp_netif_dhcpc_get_status(sta, &dhcp_status) : + ESP_ERR_NOT_FOUND; + + wifi_ap_record_t ap = {0}; + esp_err_t ap_err = esp_wifi_sta_get_ap_info(&ap); + char main_buffer[48] = {0}; + char backup_buffer[48] = {0}; + char fallback_buffer[48] = {0}; + const char *main_address = main_err == ESP_OK ? + format_dns_address(&dns_main, main_buffer, sizeof(main_buffer)) : "unavailable"; + const char *backup_address = backup_err == ESP_OK ? + format_dns_address(&dns_backup, backup_buffer, sizeof(backup_buffer)) : "unavailable"; + const char *fallback_address = fallback_err == ESP_OK ? + format_dns_address(&dns_fallback, fallback_buffer, sizeof(fallback_buffer)) : + "unavailable"; + + ESP_LOGW(TAG, + "DNS diagnostics lookup=%u attempt=%d elapsed_ms=%u host=%s err=%s(%d) active=%u failures=%u sta=%d ip=" IPSTR " netmask=" IPSTR " gw=" IPSTR " dhcp=%d/%s dns(main=%s backup=%s fallback=%s status=%s/%s/%s) rssi=%d ap=%s heap(internal=%u largest=%u dma_largest=%u)", + (unsigned)lookup_id, + attempt, + (unsigned)elapsed_ms, + host != NULL ? host : "-", + dns_error_name(dns_err), + dns_err, + (unsigned)__atomic_load_n(&s_dns_lookup_active, __ATOMIC_RELAXED), + (unsigned)__atomic_load_n(&s_dns_lookup_failure_count, __ATOMIC_RELAXED), + sta != NULL ? 1 : 0, + IP2STR(&ip.ip), + IP2STR(&ip.netmask), + IP2STR(&ip.gw), + dhcp_err == ESP_OK ? (int)dhcp_status : -1, + dhcp_err == ESP_OK ? dhcp_status_name(dhcp_status) : esp_err_to_name(dhcp_err), + main_address, + backup_address, + fallback_address, + esp_err_to_name(main_err), + esp_err_to_name(backup_err), + esp_err_to_name(fallback_err), + ap_err == ESP_OK ? (int)ap.rssi : 0, + esp_err_to_name(ap_err), + (unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT), + (unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_DMA | MALLOC_CAP_8BIT)); + + if (ip_err != ESP_OK) { + ESP_LOGW(TAG, + "DNS diagnostics STA IP query failed: %s", + esp_err_to_name(ip_err)); + } + if (attempt == 1) { + run_dns_failure_network_probes(ip_err == ESP_OK ? &ip : NULL, + fallback_err == ESP_OK ? &dns_fallback : NULL); + } +} +#endif static int effective_positive_or_zero(int value) { @@ -106,23 +449,54 @@ static esp_err_t resolve_host_before_open(const char *url) return err; } +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE + uint32_t lookup_id = __atomic_add_fetch(&s_dns_lookup_count, 1U, __ATOMIC_RELAXED); + (void)__atomic_add_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED); +#endif for (int attempt = 1; attempt <= PLATFORM_HTTPS_DNS_RETRY_COUNT; ++attempt) { struct addrinfo hints = { .ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM, }; struct addrinfo *addr = NULL; +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE + int64_t attempt_started_us = esp_timer_get_time(); +#endif int dns_err = getaddrinfo(host, NULL, &hints, &addr); if (dns_err == 0 && addr != NULL) { +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE + uint32_t elapsed_ms = (uint32_t)((esp_timer_get_time() - attempt_started_us) / 1000); + cache_resolved_endpoint(host, addr); + if (__atomic_load_n(&s_dns_lookup_failure_count, __ATOMIC_RELAXED) > 0U || + elapsed_ms >= 1000U) { + ESP_LOGI(TAG, + "DNS resolved lookup=%u attempt=%d elapsed_ms=%u host=%s", + (unsigned)lookup_id, + attempt, + (unsigned)elapsed_ms, + host); + } +#endif freeaddrinfo(addr); +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE + (void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED); +#endif return ESP_OK; } if (addr != NULL) { freeaddrinfo(addr); } +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE + (void)__atomic_add_fetch(&s_dns_lookup_failure_count, 1U, __ATOMIC_RELAXED); + uint32_t elapsed_ms = (uint32_t)((esp_timer_get_time() - attempt_started_us) / 1000); + log_dns_failure_diagnostics(host, attempt, dns_err, lookup_id, elapsed_ms); +#endif if (dns_err == EAI_MEMORY) { ESP_LOGW(TAG, "DNS resolve failed host=%s err=%s(%d)", host, dns_error_name(dns_err), dns_err); +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE + (void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED); +#endif return ESP_ERR_NO_MEM; } @@ -143,6 +517,9 @@ static esp_err_t resolve_host_before_open(const char *url) } } +#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE + (void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED); +#endif return ESP_ERR_HTTP_CONNECT; } @@ -291,3 +668,8 @@ esp_err_t platform_https_post_json(const platform_https_post_json_request_t *req response->body_len = total_read; return ESP_OK; } + +bool platform_https_is_connect_error(esp_err_t err) +{ + return err == ESP_ERR_HTTP_CONNECT; +} diff --git a/components/platform/src/wifi_portal_bridge.cpp b/components/platform/src/wifi_portal_bridge.cpp index 766a231..108b328 100644 --- a/components/platform/src/wifi_portal_bridge.cpp +++ b/components/platform/src/wifi_portal_bridge.cpp @@ -343,6 +343,65 @@ esp_err_t wifi_portal_connect_sta(int timeout_ms) return ESP_ERR_TIMEOUT; } +esp_err_t wifi_portal_reconnect_sta(int timeout_ms) +{ + int effective_timeout_ms = normalize_connect_timeout_ms(timeout_ms); + int64_t deadline_us = esp_timer_get_time() + (int64_t)effective_timeout_ms * 1000; + + { + ESP_RETURN_ON_ERROR(wifi_portal_lock(), TAG, "Failed to lock Wi-Fi portal"); + PortalUnlockGuard guard; + + if (!wifi_portal_has_saved_credentials()) { + return ESP_ERR_NOT_FOUND; + } + ESP_RETURN_ON_ERROR(ensure_wifi_manager_initialized(), + TAG, + "Failed to initialize Wi-Fi manager"); + + ESP_LOGW(TAG, + "Controlled STA reassociation started timeout_ms=%d connected=%d", + effective_timeout_ms, + WifiManager::GetInstance().IsConnected() ? 1 : 0); + esp_err_t disconnect_err = esp_wifi_disconnect(); + if (disconnect_err != ESP_OK && disconnect_err != ESP_ERR_WIFI_NOT_CONNECT) { + ESP_LOGW(TAG, + "Controlled STA disconnect returned %s", + esp_err_to_name(disconnect_err)); + } + } + + int64_t disconnect_deadline_us = esp_timer_get_time() + 2000LL * 1000LL; + if (disconnect_deadline_us > deadline_us) { + disconnect_deadline_us = deadline_us; + } + while (esp_timer_get_time() < disconnect_deadline_us && + WifiManager::GetInstance().IsConnected()) { + vTaskDelay(pdMS_TO_TICKS(WIFI_PORTAL_CONNECT_POLL_MS)); + } + + esp_err_t connect_err = esp_wifi_connect(); + if (connect_err != ESP_OK) { + ESP_LOGI(TAG, + "Controlled STA reconnect command returned %s; wait for manager auto-reconnect", + esp_err_to_name(connect_err)); + } + + while (esp_timer_get_time() < deadline_us) { + if (WifiManager::GetInstance().IsConnected()) { + (void)apply_sta_stability_policy(); + ESP_LOGI(TAG, "Controlled STA reassociation completed"); + return ESP_OK; + } + vTaskDelay(pdMS_TO_TICKS(WIFI_PORTAL_CONNECT_POLL_MS)); + } + + ESP_LOGW(TAG, + "Controlled STA reassociation timed out after %d ms", + effective_timeout_ms); + return ESP_ERR_TIMEOUT; +} + esp_err_t wifi_portal_save_credentials(const char *ssid, const char *password) { install_wifi_storage_hooks(); diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 042a592..5f201ab 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -110,6 +110,77 @@ menu "Example Configuration" help Maximum FreeRTOS task entries captured per report. + config TQ_MEMORY_MONITOR_ENABLE + bool "Enable runtime heap watermark monitor" + default y + help + Track internal RAM, DMA-capable RAM, and PSRAM watermarks. + Allocation failures and low contiguous DMA memory are logged + together with the current voice-session state. + + config TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS + int "Memory monitor sample interval (ms)" + default 1000 + range 100 60000 + depends on TQ_MEMORY_MONITOR_ENABLE + help + Sampling interval for heap watermarks. Periodic reports are + emitted less frequently to avoid flooding the log. + + config TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS + int "Memory monitor periodic report interval (ms)" + default 30000 + range 5000 600000 + depends on TQ_MEMORY_MONITOR_ENABLE + help + Interval for normal heap watermark reports. Pressure changes + and allocation failures are reported immediately. + + config TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES + int "Internal RAM warning threshold (bytes)" + default 16384 + range 4096 262144 + depends on TQ_MEMORY_MONITOR_ENABLE + help + Emit a pressure warning when free 8-bit internal RAM falls + below this value. + + config TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES + int "Largest DMA block warning threshold (bytes)" + default 4096 + range 1024 65536 + depends on TQ_MEMORY_MONITOR_ENABLE + help + Emit a pressure warning when the largest contiguous + DMA-capable allocation falls below this value. + + config TQ_NETWORK_DIAGNOSTICS_ENABLE + bool "Enable detailed network diagnostics" + default y + help + Collect detailed DNS, Wi-Fi, heap, ping, and cached-endpoint + TCP probe data when HTTPS name resolution fails. The probes + can add up to roughly two seconds to a failed DNS attempt, so + disable this option for production builds after diagnosis. + + config TQ_MULTIMODAL_DIAGNOSTICS_ENABLE + bool "Enable detailed multimodal session diagnostics" + default y + help + Log multimodal session generations, transport cleanup stages, + and network failure streaks. + Functional recovery warnings remain visible when disabled. + + config TQ_NETWORK_AUTO_REASSOCIATION_ENABLE + bool "Enable automatic Wi-Fi reassociation after HTTPS failures" + default n + help + After two consecutive multimodal HTTPS connection failures, + disconnect and reconnect the Wi-Fi station before retrying. + This interrupts all device network traffic while reassociation + is in progress. Keep disabled unless field testing shows that + reconnecting only the multimodal transport is insufficient. + config TQ_SYSTEM_EVENT_ENABLE bool "Enable control-plane system event task" default y diff --git a/sdkconfig.defaults b/sdkconfig.defaults index db63ba1..0ef90f6 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -21,6 +21,14 @@ CONFIG_TQ_NETWORK_TASK_CORE=0 CONFIG_TQ_LOCAL_TASK_CORE=1 CONFIG_TQ_TASK_MONITOR_INTERVAL_MS=10000 CONFIG_TQ_TASK_MONITOR_MAX_TASKS=64 +CONFIG_TQ_MEMORY_MONITOR_ENABLE=y +CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS=1000 +CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS=30000 +CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES=16384 +CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES=4096 +CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE=y +CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE=y +# CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE is not set CONFIG_TQ_SYSTEM_EVENT_ENABLE=y CONFIG_TQ_SYSTEM_EVENT_QUEUE_LEN=32 CONFIG_TQ_SYSTEM_EVENT_TASK_STACK=4096