chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(runtime): add diagnostic and network recovery controls

Browse Source
This commit is contained in:
chenjintao
2026-08-12 16:32:40 +08:00
parent a94df9545d
commit 1c32d9f0e7
11 changed files with 868 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/control_plane/src/runtime/control_plane_runtime_workers.c
+324
View File
@@ -49,6 +49,8 @@
#define CONTROL_PLANE_SPIN_INTRO_TIMEOUT_MS 6000
#define CONTROL_PLANE_COMMAND_ROTATION_DEG 100.0f
#define CONTROL_PLANE_COMMAND_RETURN_DELAY_MS 5000
#define CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES 512U
#define CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES 256U
#if (configSUPPORT_STATIC_ALLOCATION == 1) && \
((defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) || \
@@ -80,6 +82,38 @@
#define CONFIG_TQ_TASK_MONITOR_MAX_TASKS 64
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_ENABLE
#define CONFIG_TQ_MEMORY_MONITOR_ENABLE 1
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS
#define CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS 1000
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS
#define CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS 10000
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES
#define CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES 16384
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES
#define CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES 4096
#endif
#ifndef CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
#define CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE 0
#endif
#ifndef CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
#define CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE 0
#endif
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
#define CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD 2U
#endif
#if !defined(CONFIG_TQ_PTT_MODE_HOLD_TO_TALK) && !defined(CONFIG_TQ_PTT_MODE_CLICK_TO_TALK)
#define CONFIG_TQ_PTT_MODE_CLICK_TO_TALK 1
#endif
@@ -182,6 +216,10 @@ static QueueHandle_t s_model_cmd_queue;
static bool s_model_cmd_queue_with_caps;
static TaskHandle_t s_model_cmd_task;
static bool s_ws_start_requested;
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
static uint32_t s_multimodal_network_failure_count;
static bool s_wifi_reassociation_requested;
#endif
static bool s_ptt_active;
static ptt_runtime_t s_ptt_runtime;
static TaskHandle_t s_lcd_render_loop_task;
@@ -197,7 +235,200 @@ static uint32_t s_fortune_refresh_generation;
static volatile bool s_runtime_stop_requested;
static portMUX_TYPE s_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
typedef struct {
TickType_t last_sample_tick;
TickType_t last_report_tick;
uint32_t reported_alloc_fail_count;
size_t reported_internal_free;
size_t reported_dma_largest;
bool initialized;
bool pressure_active;
} control_plane_memory_monitor_t;
static control_plane_memory_monitor_t s_memory_monitor;
static volatile uint32_t s_alloc_fail_count;
static volatile size_t s_last_alloc_fail_size;
static volatile uint32_t s_last_alloc_fail_caps;
static const char *volatile s_last_alloc_fail_function;
static bool s_alloc_failure_hook_registered;
#endif
static TaskHandle_t control_plane_get_task_handle(TaskHandle_t *slot);
static const char *ptt_state_name(ptt_state_t state);
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
static void IRAM_ATTR control_plane_alloc_failed_hook(size_t size,
uint32_t caps,
const char *function_name)
{
__atomic_store_n(&s_last_alloc_fail_size, size, __ATOMIC_RELAXED);
__atomic_store_n(&s_last_alloc_fail_caps, caps, __ATOMIC_RELAXED);
__atomic_store_n(&s_last_alloc_fail_function, function_name, __ATOMIC_RELAXED);
(void)__atomic_add_fetch(&s_alloc_fail_count, 1U, __ATOMIC_RELEASE);
}
static void control_plane_memory_monitor_register_hook(void)
{
if (s_alloc_failure_hook_registered) {
return;
}
esp_err_t err = heap_caps_register_failed_alloc_callback(control_plane_alloc_failed_hook);
if (err == ESP_OK) {
s_alloc_failure_hook_registered = true;
ESP_LOGI(TAG, "Heap allocation failure hook registered");
} else {
ESP_LOGW(TAG, "Heap allocation failure hook registration failed: %s",
esp_err_to_name(err));
}
}
static unsigned control_plane_heap_contiguous_percent(const multi_heap_info_t *info)
{
if (info == NULL || info->total_free_bytes == 0U) {
return 0U;
}
return (unsigned)((info->largest_free_block * 100U) / info->total_free_bytes);
}
static void control_plane_memory_monitor_sample(TickType_t now_tick, bool force)
{
TickType_t sample_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS);
TickType_t report_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS);
if (sample_ticks == 0) {
sample_ticks = 1;
}
if (report_ticks == 0) {
report_ticks = 1;
}
if (!force && s_memory_monitor.initialized &&
(now_tick - s_memory_monitor.last_sample_tick) < sample_ticks) {
return;
}
s_memory_monitor.last_sample_tick = now_tick;
multi_heap_info_t internal = {0};
multi_heap_info_t dma = {0};
multi_heap_info_t psram = {0};
heap_caps_get_info(&internal, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
heap_caps_get_info(&dma, MALLOC_CAP_DMA | MALLOC_CAP_8BIT);
heap_caps_get_info(&psram, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
uint32_t alloc_fail_count = __atomic_load_n(&s_alloc_fail_count, __ATOMIC_ACQUIRE);
bool alloc_failure_pending = alloc_fail_count != s_memory_monitor.reported_alloc_fail_count;
bool pressure = internal.total_free_bytes < CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES ||
dma.largest_free_block < CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES;
bool pressure_changed = s_memory_monitor.initialized &&
pressure != s_memory_monitor.pressure_active;
bool watermark_drop = s_memory_monitor.initialized &&
(internal.total_free_bytes + CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES <=
s_memory_monitor.reported_internal_free ||
dma.largest_free_block + CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES <=
s_memory_monitor.reported_dma_largest);
bool periodic_due = !s_memory_monitor.initialized ||
(now_tick - s_memory_monitor.last_report_tick) >= report_ticks;
if (!force && !alloc_failure_pending && !pressure_changed && !watermark_drop && !periodic_due) {
return;
}
const char *reason = force || !s_memory_monitor.initialized ? "startup" :
alloc_failure_pending ? "alloc_failure" :
pressure_changed ? (pressure ? "pressure_enter" : "pressure_exit") :
watermark_drop ? "watermark_drop" :
pressure ? "pressure_periodic" : "periodic";
control_plane_multimodal_runtime_state_t mm = {0};
bool have_mm = control_plane_multimodal_get_runtime_state(&mm) == ESP_OK;
domain_local_command_snapshot_t local_command = {0};
bool have_local_command =
domain_runtime_local_command_get_snapshot(&local_command) == ESP_OK;
ptt_state_t ptt_state;
bool ptt_active;
portENTER_CRITICAL(&s_runtime_lock);
ptt_state = s_ptt_runtime.state;
ptt_active = s_ptt_runtime.active;
portEXIT_CRITICAL(&s_runtime_lock);
if (pressure || alloc_failure_pending) {
ESP_LOGW(TAG,
"Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)",
reason,
(unsigned)internal.total_free_bytes,
(unsigned)internal.minimum_free_bytes,
(unsigned)internal.largest_free_block,
(unsigned)internal.allocated_blocks,
(unsigned)internal.free_blocks,
control_plane_heap_contiguous_percent(&internal),
(unsigned)dma.total_free_bytes,
(unsigned)dma.minimum_free_bytes,
(unsigned)dma.largest_free_block,
(unsigned)dma.allocated_blocks,
(unsigned)dma.free_blocks,
control_plane_heap_contiguous_percent(&dma),
(unsigned)psram.total_free_bytes,
(unsigned)psram.minimum_free_bytes,
(unsigned)psram.largest_free_block);
} else {
ESP_LOGI(TAG,
"Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)",
reason,
(unsigned)internal.total_free_bytes,
(unsigned)internal.minimum_free_bytes,
(unsigned)internal.largest_free_block,
(unsigned)internal.allocated_blocks,
(unsigned)internal.free_blocks,
control_plane_heap_contiguous_percent(&internal),
(unsigned)dma.total_free_bytes,
(unsigned)dma.minimum_free_bytes,
(unsigned)dma.largest_free_block,
(unsigned)dma.allocated_blocks,
(unsigned)dma.free_blocks,
control_plane_heap_contiguous_percent(&dma),
(unsigned)psram.total_free_bytes,
(unsigned)psram.minimum_free_bytes,
(unsigned)psram.largest_free_block);
}
ESP_LOGI(TAG,
"Memory context ptt=%s active=%d wifi=%d ws(conn=%d starting=%d stopping=%d) task=%d listening=%d responding=%d uplink=%d local_kws(started=%d detecting=%d detections=%u stack_hwm=%u) alloc_failures=%u",
ptt_state_name(ptt_state),
ptt_active ? 1 : 0,
control_plane_is_sta_connected() ? 1 : 0,
(have_mm && mm.connected) ? 1 : 0,
(have_mm && mm.starting) ? 1 : 0,
(have_mm && mm.stopping) ? 1 : 0,
(have_mm && mm.has_active_task) ? 1 : 0,
(have_mm && mm.server_listening) ? 1 : 0,
(have_mm && mm.remote_responding) ? 1 : 0,
control_plane_multimodal_is_uplink_running() ? 1 : 0,
(have_local_command && local_command.started) ? 1 : 0,
(have_local_command && local_command.detection_running) ? 1 : 0,
have_local_command ? (unsigned)local_command.detection_count : 0U,
have_local_command ? (unsigned)local_command.task_stack_high_water_bytes : 0U,
(unsigned)alloc_fail_count);
if (alloc_failure_pending) {
size_t failed_size = __atomic_load_n(&s_last_alloc_fail_size, __ATOMIC_RELAXED);
uint32_t failed_caps = __atomic_load_n(&s_last_alloc_fail_caps, __ATOMIC_RELAXED);
const char *failed_function = __atomic_load_n(&s_last_alloc_fail_function,
__ATOMIC_RELAXED);
ESP_LOGE(TAG,
"Heap allocation failure count=%u newest(size=%u caps=0x%08x function=%s)",
(unsigned)alloc_fail_count,
(unsigned)failed_size,
(unsigned)failed_caps,
failed_function != NULL ? failed_function : "unknown");
s_memory_monitor.reported_alloc_fail_count = alloc_fail_count;
}
s_memory_monitor.initialized = true;
s_memory_monitor.pressure_active = pressure;
s_memory_monitor.reported_internal_free = internal.total_free_bytes;
s_memory_monitor.reported_dma_largest = dma.largest_free_block;
s_memory_monitor.last_report_tick = now_tick;
}
#endif
static bool control_plane_runtime_stop_requested(void)
{
@@ -880,6 +1111,58 @@ esp_err_t control_plane_schedule_websocket_wait_task(void)
return ESP_OK;
}
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
static void control_plane_record_multimodal_network_result(esp_err_t result)
{
uint32_t failures = 0U;
bool reassociation_requested = false;
portENTER_CRITICAL(&s_runtime_lock);
if (result == ESP_OK) {
s_multimodal_network_failure_count = 0U;
} else if (domain_runtime_is_network_connect_error(result)) {
s_multimodal_network_failure_count++;
failures = s_multimodal_network_failure_count;
if (failures >= CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD &&
!s_wifi_reassociation_requested) {
s_wifi_reassociation_requested = true;
reassociation_requested = true;
}
}
portEXIT_CRITICAL(&s_runtime_lock);
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
if (result == ESP_OK) {
ESP_LOGI(TAG, "Multimodal network failure streak cleared");
} else if (domain_runtime_is_network_connect_error(result)) {
ESP_LOGW(TAG,
"Multimodal network failure streak=%u reassociation_requested=%d",
(unsigned)failures,
reassociation_requested ? 1 : 0);
}
#else
(void)failures;
(void)reassociation_requested;
#endif
}
static bool control_plane_consume_wifi_reassociation_request(void)
{
bool requested = false;
portENTER_CRITICAL(&s_runtime_lock);
requested = s_wifi_reassociation_requested;
s_wifi_reassociation_requested = false;
portEXIT_CRITICAL(&s_runtime_lock);
return requested;
}
static void control_plane_reset_multimodal_network_failure_streak(void)
{
portENTER_CRITICAL(&s_runtime_lock);
s_multimodal_network_failure_count = 0U;
portEXIT_CRITICAL(&s_runtime_lock);
}
#endif
#if CONFIG_TQ_TASK_MONITOR_ENABLE
static const char *control_plane_task_state_name(eTaskState state)
{
@@ -1130,6 +1413,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la
esp_err_t err = control_plane_start_multimodal_client();
if (err == ESP_OK) {
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
control_plane_record_multimodal_network_result(ESP_OK);
#endif
ESP_LOGI(TAG, "Multimodal client started");
return true;
}
@@ -1137,6 +1423,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la
return false;
}
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
control_plane_record_multimodal_network_result(err);
#endif
ESP_LOGW(TAG, "Multimodal client start failed: %s", esp_err_to_name(err));
return false;
}
@@ -1421,6 +1710,10 @@ static void control_plane_wifi_state_task(void *arg)
bool reconnect_attempted = false;
uint8_t ptt_wifi_foreground_attempts = 0;
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
control_plane_memory_monitor_sample(xTaskGetTickCount(), true);
#endif
if (!was_connected) {
ESP_LOGI(TAG, "Wi-Fi STA unavailable at runtime start; requesting disconnected expression");
control_plane_request_wifi_state_expression(DOMAIN_EXPRESSION_WIFI_DISCONNECTED,
@@ -1447,6 +1740,10 @@ static void control_plane_wifi_state_task(void *arg)
bool connected = control_plane_is_sta_connected();
TickType_t now_tick = xTaskGetTickCount();
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
control_plane_memory_monitor_sample(now_tick, false);
#endif
if (!connected) {
if (!disconnect_pending) {
disconnect_pending = true;
@@ -1515,6 +1812,28 @@ static void control_plane_wifi_state_task(void *arg)
}
}
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
if (connected && control_plane_consume_wifi_reassociation_request()) {
ESP_LOGW(TAG,
"Network recovery experiment: controlled STA reassociation started after %u HTTPS connection failures",
(unsigned)CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD);
domain_runtime_behavior_set_keep_awake(true);
esp_err_t reassoc_err = domain_runtime_wifi_portal_reconnect_sta(
CONTROL_PLANE_WIFI_RECONNECT_TIMEOUT_MS);
domain_runtime_behavior_set_keep_awake(false);
connected = control_plane_is_sta_connected();
ESP_LOGW(TAG,
"Network recovery experiment: controlled STA reassociation result=%s connected=%d",
esp_err_to_name(reassoc_err),
connected ? 1 : 0);
if (reassoc_err == ESP_OK && connected) {
control_plane_reset_multimodal_network_failure_streak();
last_ws_start_attempt_tick = 0;
(void)control_plane_schedule_websocket_wait_task();
}
}
#endif
if (connected && disconnect_pending && !disconnected_expression_active) {
ESP_LOGI(TAG, "Wi-Fi STA reconnected within disconnect grace");
disconnect_pending = false;
@@ -1873,6 +2192,11 @@ esp_err_t control_plane_start_runtime_workers(void)
{
control_plane_set_runtime_stop_requested(false);
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
memset(&s_memory_monitor, 0, sizeof(s_memory_monitor));
control_plane_memory_monitor_register_hook();
#endif
esp_err_t queue_err = control_plane_ensure_model_cmd_queue();
if (queue_err != ESP_OK) {
return queue_err;
components/domain/include/domain.h
+2
View File
@@ -430,6 +430,8 @@ bool domain_runtime_wifi_connect_error_keeps_credentials(esp_err_t err);
* - Returns explicit failure on timeout/connection error.
*/
esp_err_t domain_runtime_wifi_portal_connect_sta(int timeout_ms);
esp_err_t domain_runtime_wifi_portal_reconnect_sta(int timeout_ms);
bool domain_runtime_is_network_connect_error(esp_err_t err);
esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password);
esp_err_t domain_runtime_wifi_portal_clear_saved_credentials(void);
components/domain/src/runtime/domain_runtime_adapters.c
+10
View File
@@ -1237,6 +1237,16 @@ esp_err_t domain_runtime_wifi_portal_connect_sta(int timeout_ms)
return platform_wifi_portal_connect_sta(timeout_ms);
}
esp_err_t domain_runtime_wifi_portal_reconnect_sta(int timeout_ms)
{
return platform_wifi_portal_reconnect_sta(timeout_ms);
}
bool domain_runtime_is_network_connect_error(esp_err_t err)
{
return platform_https_is_connect_error(err);
}
esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password)
{
return platform_wifi_portal_save_credentials(ssid, password);
components/platform/CMakeLists.txt
+4
View File
@@ -52,6 +52,10 @@ set(platform_priv_requires
nvs_flash
)
if(CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE)
list(APPEND platform_priv_requires lwip)
endif()
idf_component_register(
SRCS
${platform_srcs}
components/platform/include/platform.h
+2
View File
@@ -223,6 +223,7 @@ bool platform_time_is_synced(void);
*/
esp_err_t platform_https_post_json(const platform_https_post_json_request_t *request,
platform_https_response_t *response);
bool platform_https_is_connect_error(esp_err_t err);
/**
* @brief Rotate stepper synchronously with a bounded timeout.
@@ -258,6 +259,7 @@ esp_err_t platform_wifi_portal_start_config_ap(const char *ssid_prefix);
* - On timeout/failure, no hidden destructive continuation is started after return.
*/
esp_err_t platform_wifi_portal_connect_sta(int timeout_ms);
esp_err_t platform_wifi_portal_reconnect_sta(int timeout_ms);
esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password);
esp_err_t platform_wifi_portal_clear_saved_credentials(void);
bool platform_wifi_connect_error_keeps_credentials(esp_err_t err);
components/platform/internal/wifi_portal.h
+1
View File
@@ -47,6 +47,7 @@ esp_err_t wifi_portal_start_config_ap(const char *ssid_prefix);
* for the connection to succeed within the provided timeout (milliseconds).
*/
esp_err_t wifi_portal_connect_sta(int timeout_ms);
esp_err_t wifi_portal_reconnect_sta(int timeout_ms);
/**
* @brief Save Wi-Fi credentials into NVS.
components/platform/src/runtime/platform_behavior_runtime.c
+5
View File
@@ -138,6 +138,11 @@ esp_err_t platform_wifi_portal_connect_sta(int timeout_ms)
return wifi_portal_connect_sta(timeout_ms);
}
esp_err_t platform_wifi_portal_reconnect_sta(int timeout_ms)
{
return wifi_portal_reconnect_sta(timeout_ms);
}
esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password)
{
return wifi_portal_save_credentials(ssid, password);
components/platform/src/runtime/platform_https_client.c
+382
View File
@@ -18,12 +18,355 @@
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#ifndef CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
#define CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE 0
#endif
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include "esp_heap_caps.h"
#include "esp_netif.h"
#include "esp_timer.h"
#include "esp_wifi.h"
#include "freertos/semphr.h"
#include "ping/ping_sock.h"
#endif
#define PLATFORM_HTTPS_DEFAULT_TIMEOUT_MS 10000
#define PLATFORM_HTTPS_DNS_RETRY_COUNT 2
#define PLATFORM_HTTPS_DNS_RETRY_DELAY_MS 500
#define PLATFORM_HTTPS_HOST_MAX_LEN 256
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
#define PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS 400
#define PLATFORM_HTTPS_PROBE_TCP_TIMEOUT_MS 800
#endif
static const char *TAG = "platform_https";
static const char *dns_error_name(int err);
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
static volatile uint32_t s_dns_lookup_count;
static volatile uint32_t s_dns_lookup_failure_count;
static volatile uint32_t s_dns_lookup_active;
static struct sockaddr_storage s_last_resolved_addr;
static socklen_t s_last_resolved_addr_len;
static char s_last_resolved_host[PLATFORM_HTTPS_HOST_MAX_LEN];
static portMUX_TYPE s_probe_cache_lock = portMUX_INITIALIZER_UNLOCKED;
typedef struct {
SemaphoreHandle_t done;
volatile uint32_t replies;
} https_ping_probe_context_t;
static void https_ping_success(esp_ping_handle_t handle, void *args)
{
(void)handle;
https_ping_probe_context_t *context = (https_ping_probe_context_t *)args;
if (context != NULL) {
context->replies++;
}
}
static void https_ping_end(esp_ping_handle_t handle, void *args)
{
(void)handle;
https_ping_probe_context_t *context = (https_ping_probe_context_t *)args;
if (context != NULL && context->done != NULL) {
(void)xSemaphoreGive(context->done);
}
}
static void run_ipv4_ping_probe(const char *label, const esp_ip4_addr_t *target)
{
if (target == NULL || target->addr == 0U) {
ESP_LOGW(TAG, "Network probe ping label=%s skipped: no IPv4 target", label);
return;
}
https_ping_probe_context_t context = {
.done = xSemaphoreCreateBinary(),
};
if (context.done == NULL) {
ESP_LOGW(TAG, "Network probe ping label=%s failed: no semaphore memory", label);
return;
}
esp_ping_config_t config = ESP_PING_DEFAULT_CONFIG();
config.count = 1;
config.interval_ms = 0;
config.timeout_ms = PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS;
ip_addr_set_ip4_u32(&config.target_addr, target->addr);
esp_ping_callbacks_t callbacks = {
.cb_args = &context,
.on_ping_success = https_ping_success,
.on_ping_timeout = NULL,
.on_ping_end = https_ping_end,
};
esp_ping_handle_t session = NULL;
int64_t started_us = esp_timer_get_time();
esp_err_t err = esp_ping_new_session(&config, &callbacks, &session);
if (err == ESP_OK) {
err = esp_ping_start(session);
}
if (err == ESP_OK) {
TickType_t wait_ticks = pdMS_TO_TICKS(PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS + 300);
if (wait_ticks == 0) {
wait_ticks = 1;
}
if (xSemaphoreTake(context.done, wait_ticks) != pdTRUE) {
err = ESP_ERR_TIMEOUT;
}
}
if (session != NULL) {
(void)esp_ping_stop(session);
(void)esp_ping_delete_session(session);
}
ESP_LOGW(TAG,
"Network probe ping label=%s target=" IPSTR " result=%s replies=%u elapsed_ms=%lld",
label,
IP2STR(target),
err == ESP_OK && context.replies > 0U ? "reachable" :
(err == ESP_OK ? "no_reply" : esp_err_to_name(err)),
(unsigned)context.replies,
(long long)((esp_timer_get_time() - started_us) / 1000));
vSemaphoreDelete(context.done);
}
static void cache_resolved_endpoint(const char *host, const struct addrinfo *addr)
{
const struct addrinfo *selected = addr;
while (selected != NULL &&
selected->ai_family != AF_INET &&
selected->ai_family != AF_INET6) {
selected = selected->ai_next;
}
if (host == NULL || selected == NULL || selected->ai_addr == NULL ||
selected->ai_addrlen > sizeof(s_last_resolved_addr)) {
return;
}
struct sockaddr_storage endpoint = {0};
memcpy(&endpoint, selected->ai_addr, selected->ai_addrlen);
if (selected->ai_family == AF_INET) {
((struct sockaddr_in *)&endpoint)->sin_port = htons(443);
} else {
((struct sockaddr_in6 *)&endpoint)->sin6_port = htons(443);
}
portENTER_CRITICAL(&s_probe_cache_lock);
s_last_resolved_addr = endpoint;
s_last_resolved_addr_len = selected->ai_addrlen;
strlcpy(s_last_resolved_host, host, sizeof(s_last_resolved_host));
portEXIT_CRITICAL(&s_probe_cache_lock);
}
static void run_cached_endpoint_probe(void)
{
struct sockaddr_storage endpoint = {0};
socklen_t endpoint_len = 0;
char host[PLATFORM_HTTPS_HOST_MAX_LEN] = {0};
portENTER_CRITICAL(&s_probe_cache_lock);
endpoint = s_last_resolved_addr;
endpoint_len = s_last_resolved_addr_len;
strlcpy(host, s_last_resolved_host, sizeof(host));
portEXIT_CRITICAL(&s_probe_cache_lock);
if (endpoint_len == 0 || host[0] == '\0') {
ESP_LOGW(TAG, "Network probe cached TCP skipped: no successful DNS result cached");
return;
}
char address[64] = {0};
void *address_ptr = endpoint.ss_family == AF_INET ?
(void *)&((struct sockaddr_in *)&endpoint)->sin_addr :
(void *)&((struct sockaddr_in6 *)&endpoint)->sin6_addr;
const char *formatted = inet_ntop(endpoint.ss_family,
address_ptr,
address,
sizeof(address));
int64_t started_us = esp_timer_get_time();
int sock = socket(endpoint.ss_family, SOCK_STREAM, IPPROTO_TCP);
int socket_error = 0;
bool connected = false;
if (sock >= 0) {
int flags = fcntl(sock, F_GETFL, 0);
if (flags >= 0 && fcntl(sock, F_SETFL, flags | O_NONBLOCK) == 0) {
int connect_result = connect(sock,
(const struct sockaddr *)&endpoint,
endpoint_len);
if (connect_result == 0) {
connected = true;
} else if (errno == EINPROGRESS) {
fd_set write_set;
FD_ZERO(&write_set);
FD_SET(sock, &write_set);
struct timeval timeout = {
.tv_sec = 0,
.tv_usec = PLATFORM_HTTPS_PROBE_TCP_TIMEOUT_MS * 1000,
};
int selected = select(sock + 1, NULL, &write_set, NULL, &timeout);
socklen_t error_len = sizeof(socket_error);
if (selected > 0 &&
getsockopt(sock,
SOL_SOCKET,
SO_ERROR,
&socket_error,
&error_len) == 0 &&
socket_error == 0) {
connected = true;
} else if (selected == 0) {
socket_error = ETIMEDOUT;
} else if (socket_error == 0) {
socket_error = errno;
}
} else {
socket_error = errno;
}
} else {
socket_error = errno;
}
close(sock);
} else {
socket_error = errno;
}
ESP_LOGW(TAG,
"Network probe cached TCP host=%s address=%s:443 result=%s socket_error=%d elapsed_ms=%lld",
host,
formatted != NULL ? address : "unknown",
connected ? "reachable" : "failed",
socket_error,
(long long)((esp_timer_get_time() - started_us) / 1000));
}
static void run_dns_failure_network_probes(const esp_netif_ip_info_t *ip,
const esp_netif_dns_info_t *fallback_dns)
{
ESP_LOGW(TAG, "Network probe bundle started after DNS failure");
run_ipv4_ping_probe("gateway", ip != NULL ? &ip->gw : NULL);
if (fallback_dns != NULL && fallback_dns->ip.type == ESP_IPADDR_TYPE_V4) {
run_ipv4_ping_probe("fallback_dns", &fallback_dns->ip.u_addr.ip4);
} else {
ESP_LOGW(TAG, "Network probe fallback DNS ping skipped: no IPv4 address");
}
run_cached_endpoint_probe();
ESP_LOGW(TAG, "Network probe bundle completed");
}
static const char *dhcp_status_name(esp_netif_dhcp_status_t status)
{
switch (status) {
case ESP_NETIF_DHCP_INIT:
return "init";
case ESP_NETIF_DHCP_STARTED:
return "started";
case ESP_NETIF_DHCP_STOPPED:
return "stopped";
default:
return "unknown";
}
}
static const char *format_dns_address(const esp_netif_dns_info_t *dns,
char *buffer,
size_t buffer_size)
{
if (dns == NULL || buffer == NULL || buffer_size == 0U) {
return "invalid";
}
if (dns->ip.type == ESP_IPADDR_TYPE_V6) {
int written = snprintf(buffer,
buffer_size,
IPV6STR,
IPV62STR(dns->ip.u_addr.ip6));
return written >= 0 && (size_t)written < buffer_size ? buffer : "unknown";
}
int written = snprintf(buffer,
buffer_size,
IPSTR,
IP2STR(&dns->ip.u_addr.ip4));
return written >= 0 && (size_t)written < buffer_size ? buffer : "unknown";
}
static void log_dns_failure_diagnostics(const char *host,
int attempt,
int dns_err,
uint32_t lookup_id,
uint32_t elapsed_ms)
{
esp_netif_t *sta = esp_netif_get_handle_from_ifkey("WIFI_STA_DEF");
esp_netif_ip_info_t ip = {0};
esp_netif_dns_info_t dns_main = {0};
esp_netif_dns_info_t dns_backup = {0};
esp_netif_dns_info_t dns_fallback = {0};
esp_netif_dhcp_status_t dhcp_status = ESP_NETIF_DHCP_INIT;
esp_err_t ip_err = sta != NULL ? esp_netif_get_ip_info(sta, &ip) : ESP_ERR_NOT_FOUND;
esp_err_t main_err = esp_netif_get_dns_info(sta, ESP_NETIF_DNS_MAIN, &dns_main);
esp_err_t backup_err = esp_netif_get_dns_info(sta, ESP_NETIF_DNS_BACKUP, &dns_backup);
esp_err_t fallback_err = esp_netif_get_dns_info(sta,
ESP_NETIF_DNS_FALLBACK,
&dns_fallback);
esp_err_t dhcp_err = sta != NULL ? esp_netif_dhcpc_get_status(sta, &dhcp_status) :
ESP_ERR_NOT_FOUND;
wifi_ap_record_t ap = {0};
esp_err_t ap_err = esp_wifi_sta_get_ap_info(&ap);
char main_buffer[48] = {0};
char backup_buffer[48] = {0};
char fallback_buffer[48] = {0};
const char *main_address = main_err == ESP_OK ?
format_dns_address(&dns_main, main_buffer, sizeof(main_buffer)) : "unavailable";
const char *backup_address = backup_err == ESP_OK ?
format_dns_address(&dns_backup, backup_buffer, sizeof(backup_buffer)) : "unavailable";
const char *fallback_address = fallback_err == ESP_OK ?
format_dns_address(&dns_fallback, fallback_buffer, sizeof(fallback_buffer)) :
"unavailable";
ESP_LOGW(TAG,
"DNS diagnostics lookup=%u attempt=%d elapsed_ms=%u host=%s err=%s(%d) active=%u failures=%u sta=%d ip=" IPSTR " netmask=" IPSTR " gw=" IPSTR " dhcp=%d/%s dns(main=%s backup=%s fallback=%s status=%s/%s/%s) rssi=%d ap=%s heap(internal=%u largest=%u dma_largest=%u)",
(unsigned)lookup_id,
attempt,
(unsigned)elapsed_ms,
host != NULL ? host : "-",
dns_error_name(dns_err),
dns_err,
(unsigned)__atomic_load_n(&s_dns_lookup_active, __ATOMIC_RELAXED),
(unsigned)__atomic_load_n(&s_dns_lookup_failure_count, __ATOMIC_RELAXED),
sta != NULL ? 1 : 0,
IP2STR(&ip.ip),
IP2STR(&ip.netmask),
IP2STR(&ip.gw),
dhcp_err == ESP_OK ? (int)dhcp_status : -1,
dhcp_err == ESP_OK ? dhcp_status_name(dhcp_status) : esp_err_to_name(dhcp_err),
main_address,
backup_address,
fallback_address,
esp_err_to_name(main_err),
esp_err_to_name(backup_err),
esp_err_to_name(fallback_err),
ap_err == ESP_OK ? (int)ap.rssi : 0,
esp_err_to_name(ap_err),
(unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),
(unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),
(unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_DMA | MALLOC_CAP_8BIT));
if (ip_err != ESP_OK) {
ESP_LOGW(TAG,
"DNS diagnostics STA IP query failed: %s",
esp_err_to_name(ip_err));
}
if (attempt == 1) {
run_dns_failure_network_probes(ip_err == ESP_OK ? &ip : NULL,
fallback_err == ESP_OK ? &dns_fallback : NULL);
}
}
#endif
static int effective_positive_or_zero(int value)
{
@@ -106,23 +449,54 @@ static esp_err_t resolve_host_before_open(const char *url)
return err;
}
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
uint32_t lookup_id = __atomic_add_fetch(&s_dns_lookup_count, 1U, __ATOMIC_RELAXED);
(void)__atomic_add_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
#endif
for (int attempt = 1; attempt <= PLATFORM_HTTPS_DNS_RETRY_COUNT; ++attempt) {
struct addrinfo hints = {
.ai_family = AF_UNSPEC,
.ai_socktype = SOCK_STREAM,
};
struct addrinfo *addr = NULL;
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
int64_t attempt_started_us = esp_timer_get_time();
#endif
int dns_err = getaddrinfo(host, NULL, &hints, &addr);
if (dns_err == 0 && addr != NULL) {
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
uint32_t elapsed_ms = (uint32_t)((esp_timer_get_time() - attempt_started_us) / 1000);
cache_resolved_endpoint(host, addr);
if (__atomic_load_n(&s_dns_lookup_failure_count, __ATOMIC_RELAXED) > 0U ||
elapsed_ms >= 1000U) {
ESP_LOGI(TAG,
"DNS resolved lookup=%u attempt=%d elapsed_ms=%u host=%s",
(unsigned)lookup_id,
attempt,
(unsigned)elapsed_ms,
host);
}
#endif
freeaddrinfo(addr);
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
(void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
#endif
return ESP_OK;
}
if (addr != NULL) {
freeaddrinfo(addr);
}
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
(void)__atomic_add_fetch(&s_dns_lookup_failure_count, 1U, __ATOMIC_RELAXED);
uint32_t elapsed_ms = (uint32_t)((esp_timer_get_time() - attempt_started_us) / 1000);
log_dns_failure_diagnostics(host, attempt, dns_err, lookup_id, elapsed_ms);
#endif
if (dns_err == EAI_MEMORY) {
ESP_LOGW(TAG, "DNS resolve failed host=%s err=%s(%d)", host, dns_error_name(dns_err), dns_err);
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
(void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
#endif
return ESP_ERR_NO_MEM;
}
@@ -143,6 +517,9 @@ static esp_err_t resolve_host_before_open(const char *url)
}
}
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
(void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
#endif
return ESP_ERR_HTTP_CONNECT;
}
@@ -291,3 +668,8 @@ esp_err_t platform_https_post_json(const platform_https_post_json_request_t *req
response->body_len = total_read;
return ESP_OK;
}
bool platform_https_is_connect_error(esp_err_t err)
{
return err == ESP_ERR_HTTP_CONNECT;
}
components/platform/src/wifi_portal_bridge.cpp
+59
View File
@@ -343,6 +343,65 @@ esp_err_t wifi_portal_connect_sta(int timeout_ms)
return ESP_ERR_TIMEOUT;
}
esp_err_t wifi_portal_reconnect_sta(int timeout_ms)
{
int effective_timeout_ms = normalize_connect_timeout_ms(timeout_ms);
int64_t deadline_us = esp_timer_get_time() + (int64_t)effective_timeout_ms * 1000;
{
ESP_RETURN_ON_ERROR(wifi_portal_lock(), TAG, "Failed to lock Wi-Fi portal");
PortalUnlockGuard guard;
if (!wifi_portal_has_saved_credentials()) {
return ESP_ERR_NOT_FOUND;
}
ESP_RETURN_ON_ERROR(ensure_wifi_manager_initialized(),
TAG,
"Failed to initialize Wi-Fi manager");
ESP_LOGW(TAG,
"Controlled STA reassociation started timeout_ms=%d connected=%d",
effective_timeout_ms,
WifiManager::GetInstance().IsConnected() ? 1 : 0);
esp_err_t disconnect_err = esp_wifi_disconnect();
if (disconnect_err != ESP_OK && disconnect_err != ESP_ERR_WIFI_NOT_CONNECT) {
ESP_LOGW(TAG,
"Controlled STA disconnect returned %s",
esp_err_to_name(disconnect_err));
}
}
int64_t disconnect_deadline_us = esp_timer_get_time() + 2000LL * 1000LL;
if (disconnect_deadline_us > deadline_us) {
disconnect_deadline_us = deadline_us;
}
while (esp_timer_get_time() < disconnect_deadline_us &&
WifiManager::GetInstance().IsConnected()) {
vTaskDelay(pdMS_TO_TICKS(WIFI_PORTAL_CONNECT_POLL_MS));
}
esp_err_t connect_err = esp_wifi_connect();
if (connect_err != ESP_OK) {
ESP_LOGI(TAG,
"Controlled STA reconnect command returned %s; wait for manager auto-reconnect",
esp_err_to_name(connect_err));
}
while (esp_timer_get_time() < deadline_us) {
if (WifiManager::GetInstance().IsConnected()) {
(void)apply_sta_stability_policy();
ESP_LOGI(TAG, "Controlled STA reassociation completed");
return ESP_OK;
}
vTaskDelay(pdMS_TO_TICKS(WIFI_PORTAL_CONNECT_POLL_MS));
}
ESP_LOGW(TAG,
"Controlled STA reassociation timed out after %d ms",
effective_timeout_ms);
return ESP_ERR_TIMEOUT;
}
esp_err_t wifi_portal_save_credentials(const char *ssid, const char *password)
{
install_wifi_storage_hooks();
main/Kconfig.projbuild
+71
View File
@@ -110,6 +110,77 @@ menu "Example Configuration"
help
Maximum FreeRTOS task entries captured per report.
config TQ_MEMORY_MONITOR_ENABLE
bool "Enable runtime heap watermark monitor"
default y
help
Track internal RAM, DMA-capable RAM, and PSRAM watermarks.
Allocation failures and low contiguous DMA memory are logged
together with the current voice-session state.
config TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS
int "Memory monitor sample interval (ms)"
default 1000
range 100 60000
depends on TQ_MEMORY_MONITOR_ENABLE
help
Sampling interval for heap watermarks. Periodic reports are
emitted less frequently to avoid flooding the log.
config TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS
int "Memory monitor periodic report interval (ms)"
default 30000
range 5000 600000
depends on TQ_MEMORY_MONITOR_ENABLE
help
Interval for normal heap watermark reports. Pressure changes
and allocation failures are reported immediately.
config TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES
int "Internal RAM warning threshold (bytes)"
default 16384
range 4096 262144
depends on TQ_MEMORY_MONITOR_ENABLE
help
Emit a pressure warning when free 8-bit internal RAM falls
below this value.
config TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES
int "Largest DMA block warning threshold (bytes)"
default 4096
range 1024 65536
depends on TQ_MEMORY_MONITOR_ENABLE
help
Emit a pressure warning when the largest contiguous
DMA-capable allocation falls below this value.
config TQ_NETWORK_DIAGNOSTICS_ENABLE
bool "Enable detailed network diagnostics"
default y
help
Collect detailed DNS, Wi-Fi, heap, ping, and cached-endpoint
TCP probe data when HTTPS name resolution fails. The probes
can add up to roughly two seconds to a failed DNS attempt, so
disable this option for production builds after diagnosis.
config TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
bool "Enable detailed multimodal session diagnostics"
default y
help
Log multimodal session generations, transport cleanup stages,
and network failure streaks.
Functional recovery warnings remain visible when disabled.
config TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
bool "Enable automatic Wi-Fi reassociation after HTTPS failures"
default n
help
After two consecutive multimodal HTTPS connection failures,
disconnect and reconnect the Wi-Fi station before retrying.
This interrupts all device network traffic while reassociation
is in progress. Keep disabled unless field testing shows that
reconnecting only the multimodal transport is insufficient.
config TQ_SYSTEM_EVENT_ENABLE
bool "Enable control-plane system event task"
default y
sdkconfig.defaults
+8
View File
@@ -21,6 +21,14 @@ CONFIG_TQ_NETWORK_TASK_CORE=0
CONFIG_TQ_LOCAL_TASK_CORE=1
CONFIG_TQ_TASK_MONITOR_INTERVAL_MS=10000
CONFIG_TQ_TASK_MONITOR_MAX_TASKS=64
CONFIG_TQ_MEMORY_MONITOR_ENABLE=y
CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS=1000
CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS=30000
CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES=16384
CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES=4096
CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE=y
CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE=y
# CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE is not set
CONFIG_TQ_SYSTEM_EVENT_ENABLE=y
CONFIG_TQ_SYSTEM_EVENT_QUEUE_LEN=32
CONFIG_TQ_SYSTEM_EVENT_TASK_STACK=4096