feat(runtime): add diagnostic and network recovery controls
This commit is contained in:
@@ -49,6 +49,8 @@
|
|||||||
#define CONTROL_PLANE_SPIN_INTRO_TIMEOUT_MS 6000
|
#define CONTROL_PLANE_SPIN_INTRO_TIMEOUT_MS 6000
|
||||||
#define CONTROL_PLANE_COMMAND_ROTATION_DEG 100.0f
|
#define CONTROL_PLANE_COMMAND_ROTATION_DEG 100.0f
|
||||||
#define CONTROL_PLANE_COMMAND_RETURN_DELAY_MS 5000
|
#define CONTROL_PLANE_COMMAND_RETURN_DELAY_MS 5000
|
||||||
|
#define CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES 512U
|
||||||
|
#define CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES 256U
|
||||||
|
|
||||||
#if (configSUPPORT_STATIC_ALLOCATION == 1) && \
|
#if (configSUPPORT_STATIC_ALLOCATION == 1) && \
|
||||||
((defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) || \
|
((defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) || \
|
||||||
@@ -80,6 +82,38 @@
|
|||||||
#define CONFIG_TQ_TASK_MONITOR_MAX_TASKS 64
|
#define CONFIG_TQ_TASK_MONITOR_MAX_TASKS 64
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
#define CONFIG_TQ_MEMORY_MONITOR_ENABLE 1
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS
|
||||||
|
#define CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS 1000
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS
|
||||||
|
#define CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS 10000
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES
|
||||||
|
#define CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES 16384
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES
|
||||||
|
#define CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES 4096
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
||||||
|
#define CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE 0
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
#define CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE 0
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
#define CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD 2U
|
||||||
|
#endif
|
||||||
|
|
||||||
#if !defined(CONFIG_TQ_PTT_MODE_HOLD_TO_TALK) && !defined(CONFIG_TQ_PTT_MODE_CLICK_TO_TALK)
|
#if !defined(CONFIG_TQ_PTT_MODE_HOLD_TO_TALK) && !defined(CONFIG_TQ_PTT_MODE_CLICK_TO_TALK)
|
||||||
#define CONFIG_TQ_PTT_MODE_CLICK_TO_TALK 1
|
#define CONFIG_TQ_PTT_MODE_CLICK_TO_TALK 1
|
||||||
#endif
|
#endif
|
||||||
@@ -182,6 +216,10 @@ static QueueHandle_t s_model_cmd_queue;
|
|||||||
static bool s_model_cmd_queue_with_caps;
|
static bool s_model_cmd_queue_with_caps;
|
||||||
static TaskHandle_t s_model_cmd_task;
|
static TaskHandle_t s_model_cmd_task;
|
||||||
static bool s_ws_start_requested;
|
static bool s_ws_start_requested;
|
||||||
|
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
static uint32_t s_multimodal_network_failure_count;
|
||||||
|
static bool s_wifi_reassociation_requested;
|
||||||
|
#endif
|
||||||
static bool s_ptt_active;
|
static bool s_ptt_active;
|
||||||
static ptt_runtime_t s_ptt_runtime;
|
static ptt_runtime_t s_ptt_runtime;
|
||||||
static TaskHandle_t s_lcd_render_loop_task;
|
static TaskHandle_t s_lcd_render_loop_task;
|
||||||
@@ -197,7 +235,200 @@ static uint32_t s_fortune_refresh_generation;
|
|||||||
static volatile bool s_runtime_stop_requested;
|
static volatile bool s_runtime_stop_requested;
|
||||||
static portMUX_TYPE s_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
static portMUX_TYPE s_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||||
|
|
||||||
|
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
typedef struct {
|
||||||
|
TickType_t last_sample_tick;
|
||||||
|
TickType_t last_report_tick;
|
||||||
|
uint32_t reported_alloc_fail_count;
|
||||||
|
size_t reported_internal_free;
|
||||||
|
size_t reported_dma_largest;
|
||||||
|
bool initialized;
|
||||||
|
bool pressure_active;
|
||||||
|
} control_plane_memory_monitor_t;
|
||||||
|
|
||||||
|
static control_plane_memory_monitor_t s_memory_monitor;
|
||||||
|
static volatile uint32_t s_alloc_fail_count;
|
||||||
|
static volatile size_t s_last_alloc_fail_size;
|
||||||
|
static volatile uint32_t s_last_alloc_fail_caps;
|
||||||
|
static const char *volatile s_last_alloc_fail_function;
|
||||||
|
static bool s_alloc_failure_hook_registered;
|
||||||
|
#endif
|
||||||
|
|
||||||
static TaskHandle_t control_plane_get_task_handle(TaskHandle_t *slot);
|
static TaskHandle_t control_plane_get_task_handle(TaskHandle_t *slot);
|
||||||
|
static const char *ptt_state_name(ptt_state_t state);
|
||||||
|
|
||||||
|
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
static void IRAM_ATTR control_plane_alloc_failed_hook(size_t size,
|
||||||
|
uint32_t caps,
|
||||||
|
const char *function_name)
|
||||||
|
{
|
||||||
|
__atomic_store_n(&s_last_alloc_fail_size, size, __ATOMIC_RELAXED);
|
||||||
|
__atomic_store_n(&s_last_alloc_fail_caps, caps, __ATOMIC_RELAXED);
|
||||||
|
__atomic_store_n(&s_last_alloc_fail_function, function_name, __ATOMIC_RELAXED);
|
||||||
|
(void)__atomic_add_fetch(&s_alloc_fail_count, 1U, __ATOMIC_RELEASE);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void control_plane_memory_monitor_register_hook(void)
|
||||||
|
{
|
||||||
|
if (s_alloc_failure_hook_registered) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
esp_err_t err = heap_caps_register_failed_alloc_callback(control_plane_alloc_failed_hook);
|
||||||
|
if (err == ESP_OK) {
|
||||||
|
s_alloc_failure_hook_registered = true;
|
||||||
|
ESP_LOGI(TAG, "Heap allocation failure hook registered");
|
||||||
|
} else {
|
||||||
|
ESP_LOGW(TAG, "Heap allocation failure hook registration failed: %s",
|
||||||
|
esp_err_to_name(err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static unsigned control_plane_heap_contiguous_percent(const multi_heap_info_t *info)
|
||||||
|
{
|
||||||
|
if (info == NULL || info->total_free_bytes == 0U) {
|
||||||
|
return 0U;
|
||||||
|
}
|
||||||
|
return (unsigned)((info->largest_free_block * 100U) / info->total_free_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void control_plane_memory_monitor_sample(TickType_t now_tick, bool force)
|
||||||
|
{
|
||||||
|
TickType_t sample_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS);
|
||||||
|
TickType_t report_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS);
|
||||||
|
if (sample_ticks == 0) {
|
||||||
|
sample_ticks = 1;
|
||||||
|
}
|
||||||
|
if (report_ticks == 0) {
|
||||||
|
report_ticks = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!force && s_memory_monitor.initialized &&
|
||||||
|
(now_tick - s_memory_monitor.last_sample_tick) < sample_ticks) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
s_memory_monitor.last_sample_tick = now_tick;
|
||||||
|
|
||||||
|
multi_heap_info_t internal = {0};
|
||||||
|
multi_heap_info_t dma = {0};
|
||||||
|
multi_heap_info_t psram = {0};
|
||||||
|
heap_caps_get_info(&internal, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
|
||||||
|
heap_caps_get_info(&dma, MALLOC_CAP_DMA | MALLOC_CAP_8BIT);
|
||||||
|
heap_caps_get_info(&psram, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
||||||
|
|
||||||
|
uint32_t alloc_fail_count = __atomic_load_n(&s_alloc_fail_count, __ATOMIC_ACQUIRE);
|
||||||
|
bool alloc_failure_pending = alloc_fail_count != s_memory_monitor.reported_alloc_fail_count;
|
||||||
|
bool pressure = internal.total_free_bytes < CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES ||
|
||||||
|
dma.largest_free_block < CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES;
|
||||||
|
bool pressure_changed = s_memory_monitor.initialized &&
|
||||||
|
pressure != s_memory_monitor.pressure_active;
|
||||||
|
bool watermark_drop = s_memory_monitor.initialized &&
|
||||||
|
(internal.total_free_bytes + CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES <=
|
||||||
|
s_memory_monitor.reported_internal_free ||
|
||||||
|
dma.largest_free_block + CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES <=
|
||||||
|
s_memory_monitor.reported_dma_largest);
|
||||||
|
bool periodic_due = !s_memory_monitor.initialized ||
|
||||||
|
(now_tick - s_memory_monitor.last_report_tick) >= report_ticks;
|
||||||
|
if (!force && !alloc_failure_pending && !pressure_changed && !watermark_drop && !periodic_due) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *reason = force || !s_memory_monitor.initialized ? "startup" :
|
||||||
|
alloc_failure_pending ? "alloc_failure" :
|
||||||
|
pressure_changed ? (pressure ? "pressure_enter" : "pressure_exit") :
|
||||||
|
watermark_drop ? "watermark_drop" :
|
||||||
|
pressure ? "pressure_periodic" : "periodic";
|
||||||
|
control_plane_multimodal_runtime_state_t mm = {0};
|
||||||
|
bool have_mm = control_plane_multimodal_get_runtime_state(&mm) == ESP_OK;
|
||||||
|
domain_local_command_snapshot_t local_command = {0};
|
||||||
|
bool have_local_command =
|
||||||
|
domain_runtime_local_command_get_snapshot(&local_command) == ESP_OK;
|
||||||
|
ptt_state_t ptt_state;
|
||||||
|
bool ptt_active;
|
||||||
|
portENTER_CRITICAL(&s_runtime_lock);
|
||||||
|
ptt_state = s_ptt_runtime.state;
|
||||||
|
ptt_active = s_ptt_runtime.active;
|
||||||
|
portEXIT_CRITICAL(&s_runtime_lock);
|
||||||
|
|
||||||
|
if (pressure || alloc_failure_pending) {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)",
|
||||||
|
reason,
|
||||||
|
(unsigned)internal.total_free_bytes,
|
||||||
|
(unsigned)internal.minimum_free_bytes,
|
||||||
|
(unsigned)internal.largest_free_block,
|
||||||
|
(unsigned)internal.allocated_blocks,
|
||||||
|
(unsigned)internal.free_blocks,
|
||||||
|
control_plane_heap_contiguous_percent(&internal),
|
||||||
|
(unsigned)dma.total_free_bytes,
|
||||||
|
(unsigned)dma.minimum_free_bytes,
|
||||||
|
(unsigned)dma.largest_free_block,
|
||||||
|
(unsigned)dma.allocated_blocks,
|
||||||
|
(unsigned)dma.free_blocks,
|
||||||
|
control_plane_heap_contiguous_percent(&dma),
|
||||||
|
(unsigned)psram.total_free_bytes,
|
||||||
|
(unsigned)psram.minimum_free_bytes,
|
||||||
|
(unsigned)psram.largest_free_block);
|
||||||
|
} else {
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)",
|
||||||
|
reason,
|
||||||
|
(unsigned)internal.total_free_bytes,
|
||||||
|
(unsigned)internal.minimum_free_bytes,
|
||||||
|
(unsigned)internal.largest_free_block,
|
||||||
|
(unsigned)internal.allocated_blocks,
|
||||||
|
(unsigned)internal.free_blocks,
|
||||||
|
control_plane_heap_contiguous_percent(&internal),
|
||||||
|
(unsigned)dma.total_free_bytes,
|
||||||
|
(unsigned)dma.minimum_free_bytes,
|
||||||
|
(unsigned)dma.largest_free_block,
|
||||||
|
(unsigned)dma.allocated_blocks,
|
||||||
|
(unsigned)dma.free_blocks,
|
||||||
|
control_plane_heap_contiguous_percent(&dma),
|
||||||
|
(unsigned)psram.total_free_bytes,
|
||||||
|
(unsigned)psram.minimum_free_bytes,
|
||||||
|
(unsigned)psram.largest_free_block);
|
||||||
|
}
|
||||||
|
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"Memory context ptt=%s active=%d wifi=%d ws(conn=%d starting=%d stopping=%d) task=%d listening=%d responding=%d uplink=%d local_kws(started=%d detecting=%d detections=%u stack_hwm=%u) alloc_failures=%u",
|
||||||
|
ptt_state_name(ptt_state),
|
||||||
|
ptt_active ? 1 : 0,
|
||||||
|
control_plane_is_sta_connected() ? 1 : 0,
|
||||||
|
(have_mm && mm.connected) ? 1 : 0,
|
||||||
|
(have_mm && mm.starting) ? 1 : 0,
|
||||||
|
(have_mm && mm.stopping) ? 1 : 0,
|
||||||
|
(have_mm && mm.has_active_task) ? 1 : 0,
|
||||||
|
(have_mm && mm.server_listening) ? 1 : 0,
|
||||||
|
(have_mm && mm.remote_responding) ? 1 : 0,
|
||||||
|
control_plane_multimodal_is_uplink_running() ? 1 : 0,
|
||||||
|
(have_local_command && local_command.started) ? 1 : 0,
|
||||||
|
(have_local_command && local_command.detection_running) ? 1 : 0,
|
||||||
|
have_local_command ? (unsigned)local_command.detection_count : 0U,
|
||||||
|
have_local_command ? (unsigned)local_command.task_stack_high_water_bytes : 0U,
|
||||||
|
(unsigned)alloc_fail_count);
|
||||||
|
|
||||||
|
if (alloc_failure_pending) {
|
||||||
|
size_t failed_size = __atomic_load_n(&s_last_alloc_fail_size, __ATOMIC_RELAXED);
|
||||||
|
uint32_t failed_caps = __atomic_load_n(&s_last_alloc_fail_caps, __ATOMIC_RELAXED);
|
||||||
|
const char *failed_function = __atomic_load_n(&s_last_alloc_fail_function,
|
||||||
|
__ATOMIC_RELAXED);
|
||||||
|
ESP_LOGE(TAG,
|
||||||
|
"Heap allocation failure count=%u newest(size=%u caps=0x%08x function=%s)",
|
||||||
|
(unsigned)alloc_fail_count,
|
||||||
|
(unsigned)failed_size,
|
||||||
|
(unsigned)failed_caps,
|
||||||
|
failed_function != NULL ? failed_function : "unknown");
|
||||||
|
s_memory_monitor.reported_alloc_fail_count = alloc_fail_count;
|
||||||
|
}
|
||||||
|
|
||||||
|
s_memory_monitor.initialized = true;
|
||||||
|
s_memory_monitor.pressure_active = pressure;
|
||||||
|
s_memory_monitor.reported_internal_free = internal.total_free_bytes;
|
||||||
|
s_memory_monitor.reported_dma_largest = dma.largest_free_block;
|
||||||
|
s_memory_monitor.last_report_tick = now_tick;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
static bool control_plane_runtime_stop_requested(void)
|
static bool control_plane_runtime_stop_requested(void)
|
||||||
{
|
{
|
||||||
@@ -880,6 +1111,58 @@ esp_err_t control_plane_schedule_websocket_wait_task(void)
|
|||||||
return ESP_OK;
|
return ESP_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
static void control_plane_record_multimodal_network_result(esp_err_t result)
|
||||||
|
{
|
||||||
|
uint32_t failures = 0U;
|
||||||
|
bool reassociation_requested = false;
|
||||||
|
portENTER_CRITICAL(&s_runtime_lock);
|
||||||
|
if (result == ESP_OK) {
|
||||||
|
s_multimodal_network_failure_count = 0U;
|
||||||
|
} else if (domain_runtime_is_network_connect_error(result)) {
|
||||||
|
s_multimodal_network_failure_count++;
|
||||||
|
failures = s_multimodal_network_failure_count;
|
||||||
|
if (failures >= CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD &&
|
||||||
|
!s_wifi_reassociation_requested) {
|
||||||
|
s_wifi_reassociation_requested = true;
|
||||||
|
reassociation_requested = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
portEXIT_CRITICAL(&s_runtime_lock);
|
||||||
|
|
||||||
|
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
||||||
|
if (result == ESP_OK) {
|
||||||
|
ESP_LOGI(TAG, "Multimodal network failure streak cleared");
|
||||||
|
} else if (domain_runtime_is_network_connect_error(result)) {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Multimodal network failure streak=%u reassociation_requested=%d",
|
||||||
|
(unsigned)failures,
|
||||||
|
reassociation_requested ? 1 : 0);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
(void)failures;
|
||||||
|
(void)reassociation_requested;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool control_plane_consume_wifi_reassociation_request(void)
|
||||||
|
{
|
||||||
|
bool requested = false;
|
||||||
|
portENTER_CRITICAL(&s_runtime_lock);
|
||||||
|
requested = s_wifi_reassociation_requested;
|
||||||
|
s_wifi_reassociation_requested = false;
|
||||||
|
portEXIT_CRITICAL(&s_runtime_lock);
|
||||||
|
return requested;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void control_plane_reset_multimodal_network_failure_streak(void)
|
||||||
|
{
|
||||||
|
portENTER_CRITICAL(&s_runtime_lock);
|
||||||
|
s_multimodal_network_failure_count = 0U;
|
||||||
|
portEXIT_CRITICAL(&s_runtime_lock);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
#if CONFIG_TQ_TASK_MONITOR_ENABLE
|
#if CONFIG_TQ_TASK_MONITOR_ENABLE
|
||||||
static const char *control_plane_task_state_name(eTaskState state)
|
static const char *control_plane_task_state_name(eTaskState state)
|
||||||
{
|
{
|
||||||
@@ -1130,6 +1413,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la
|
|||||||
|
|
||||||
esp_err_t err = control_plane_start_multimodal_client();
|
esp_err_t err = control_plane_start_multimodal_client();
|
||||||
if (err == ESP_OK) {
|
if (err == ESP_OK) {
|
||||||
|
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
control_plane_record_multimodal_network_result(ESP_OK);
|
||||||
|
#endif
|
||||||
ESP_LOGI(TAG, "Multimodal client started");
|
ESP_LOGI(TAG, "Multimodal client started");
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -1137,6 +1423,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
control_plane_record_multimodal_network_result(err);
|
||||||
|
#endif
|
||||||
ESP_LOGW(TAG, "Multimodal client start failed: %s", esp_err_to_name(err));
|
ESP_LOGW(TAG, "Multimodal client start failed: %s", esp_err_to_name(err));
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -1421,6 +1710,10 @@ static void control_plane_wifi_state_task(void *arg)
|
|||||||
bool reconnect_attempted = false;
|
bool reconnect_attempted = false;
|
||||||
uint8_t ptt_wifi_foreground_attempts = 0;
|
uint8_t ptt_wifi_foreground_attempts = 0;
|
||||||
|
|
||||||
|
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
control_plane_memory_monitor_sample(xTaskGetTickCount(), true);
|
||||||
|
#endif
|
||||||
|
|
||||||
if (!was_connected) {
|
if (!was_connected) {
|
||||||
ESP_LOGI(TAG, "Wi-Fi STA unavailable at runtime start; requesting disconnected expression");
|
ESP_LOGI(TAG, "Wi-Fi STA unavailable at runtime start; requesting disconnected expression");
|
||||||
control_plane_request_wifi_state_expression(DOMAIN_EXPRESSION_WIFI_DISCONNECTED,
|
control_plane_request_wifi_state_expression(DOMAIN_EXPRESSION_WIFI_DISCONNECTED,
|
||||||
@@ -1447,6 +1740,10 @@ static void control_plane_wifi_state_task(void *arg)
|
|||||||
bool connected = control_plane_is_sta_connected();
|
bool connected = control_plane_is_sta_connected();
|
||||||
TickType_t now_tick = xTaskGetTickCount();
|
TickType_t now_tick = xTaskGetTickCount();
|
||||||
|
|
||||||
|
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
control_plane_memory_monitor_sample(now_tick, false);
|
||||||
|
#endif
|
||||||
|
|
||||||
if (!connected) {
|
if (!connected) {
|
||||||
if (!disconnect_pending) {
|
if (!disconnect_pending) {
|
||||||
disconnect_pending = true;
|
disconnect_pending = true;
|
||||||
@@ -1515,6 +1812,28 @@ static void control_plane_wifi_state_task(void *arg)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
if (connected && control_plane_consume_wifi_reassociation_request()) {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Network recovery experiment: controlled STA reassociation started after %u HTTPS connection failures",
|
||||||
|
(unsigned)CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD);
|
||||||
|
domain_runtime_behavior_set_keep_awake(true);
|
||||||
|
esp_err_t reassoc_err = domain_runtime_wifi_portal_reconnect_sta(
|
||||||
|
CONTROL_PLANE_WIFI_RECONNECT_TIMEOUT_MS);
|
||||||
|
domain_runtime_behavior_set_keep_awake(false);
|
||||||
|
connected = control_plane_is_sta_connected();
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Network recovery experiment: controlled STA reassociation result=%s connected=%d",
|
||||||
|
esp_err_to_name(reassoc_err),
|
||||||
|
connected ? 1 : 0);
|
||||||
|
if (reassoc_err == ESP_OK && connected) {
|
||||||
|
control_plane_reset_multimodal_network_failure_streak();
|
||||||
|
last_ws_start_attempt_tick = 0;
|
||||||
|
(void)control_plane_schedule_websocket_wait_task();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
if (connected && disconnect_pending && !disconnected_expression_active) {
|
if (connected && disconnect_pending && !disconnected_expression_active) {
|
||||||
ESP_LOGI(TAG, "Wi-Fi STA reconnected within disconnect grace");
|
ESP_LOGI(TAG, "Wi-Fi STA reconnected within disconnect grace");
|
||||||
disconnect_pending = false;
|
disconnect_pending = false;
|
||||||
@@ -1873,6 +2192,11 @@ esp_err_t control_plane_start_runtime_workers(void)
|
|||||||
{
|
{
|
||||||
control_plane_set_runtime_stop_requested(false);
|
control_plane_set_runtime_stop_requested(false);
|
||||||
|
|
||||||
|
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
memset(&s_memory_monitor, 0, sizeof(s_memory_monitor));
|
||||||
|
control_plane_memory_monitor_register_hook();
|
||||||
|
#endif
|
||||||
|
|
||||||
esp_err_t queue_err = control_plane_ensure_model_cmd_queue();
|
esp_err_t queue_err = control_plane_ensure_model_cmd_queue();
|
||||||
if (queue_err != ESP_OK) {
|
if (queue_err != ESP_OK) {
|
||||||
return queue_err;
|
return queue_err;
|
||||||
|
|||||||
@@ -430,6 +430,8 @@ bool domain_runtime_wifi_connect_error_keeps_credentials(esp_err_t err);
|
|||||||
* - Returns explicit failure on timeout/connection error.
|
* - Returns explicit failure on timeout/connection error.
|
||||||
*/
|
*/
|
||||||
esp_err_t domain_runtime_wifi_portal_connect_sta(int timeout_ms);
|
esp_err_t domain_runtime_wifi_portal_connect_sta(int timeout_ms);
|
||||||
|
esp_err_t domain_runtime_wifi_portal_reconnect_sta(int timeout_ms);
|
||||||
|
bool domain_runtime_is_network_connect_error(esp_err_t err);
|
||||||
esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password);
|
esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password);
|
||||||
esp_err_t domain_runtime_wifi_portal_clear_saved_credentials(void);
|
esp_err_t domain_runtime_wifi_portal_clear_saved_credentials(void);
|
||||||
|
|
||||||
|
|||||||
@@ -1237,6 +1237,16 @@ esp_err_t domain_runtime_wifi_portal_connect_sta(int timeout_ms)
|
|||||||
return platform_wifi_portal_connect_sta(timeout_ms);
|
return platform_wifi_portal_connect_sta(timeout_ms);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
esp_err_t domain_runtime_wifi_portal_reconnect_sta(int timeout_ms)
|
||||||
|
{
|
||||||
|
return platform_wifi_portal_reconnect_sta(timeout_ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool domain_runtime_is_network_connect_error(esp_err_t err)
|
||||||
|
{
|
||||||
|
return platform_https_is_connect_error(err);
|
||||||
|
}
|
||||||
|
|
||||||
esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password)
|
esp_err_t domain_runtime_wifi_portal_save_credentials(const char *ssid, const char *password)
|
||||||
{
|
{
|
||||||
return platform_wifi_portal_save_credentials(ssid, password);
|
return platform_wifi_portal_save_credentials(ssid, password);
|
||||||
|
|||||||
@@ -52,6 +52,10 @@ set(platform_priv_requires
|
|||||||
nvs_flash
|
nvs_flash
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if(CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE)
|
||||||
|
list(APPEND platform_priv_requires lwip)
|
||||||
|
endif()
|
||||||
|
|
||||||
idf_component_register(
|
idf_component_register(
|
||||||
SRCS
|
SRCS
|
||||||
${platform_srcs}
|
${platform_srcs}
|
||||||
|
|||||||
@@ -223,6 +223,7 @@ bool platform_time_is_synced(void);
|
|||||||
*/
|
*/
|
||||||
esp_err_t platform_https_post_json(const platform_https_post_json_request_t *request,
|
esp_err_t platform_https_post_json(const platform_https_post_json_request_t *request,
|
||||||
platform_https_response_t *response);
|
platform_https_response_t *response);
|
||||||
|
bool platform_https_is_connect_error(esp_err_t err);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Rotate stepper synchronously with a bounded timeout.
|
* @brief Rotate stepper synchronously with a bounded timeout.
|
||||||
@@ -258,6 +259,7 @@ esp_err_t platform_wifi_portal_start_config_ap(const char *ssid_prefix);
|
|||||||
* - On timeout/failure, no hidden destructive continuation is started after return.
|
* - On timeout/failure, no hidden destructive continuation is started after return.
|
||||||
*/
|
*/
|
||||||
esp_err_t platform_wifi_portal_connect_sta(int timeout_ms);
|
esp_err_t platform_wifi_portal_connect_sta(int timeout_ms);
|
||||||
|
esp_err_t platform_wifi_portal_reconnect_sta(int timeout_ms);
|
||||||
esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password);
|
esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password);
|
||||||
esp_err_t platform_wifi_portal_clear_saved_credentials(void);
|
esp_err_t platform_wifi_portal_clear_saved_credentials(void);
|
||||||
bool platform_wifi_connect_error_keeps_credentials(esp_err_t err);
|
bool platform_wifi_connect_error_keeps_credentials(esp_err_t err);
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ esp_err_t wifi_portal_start_config_ap(const char *ssid_prefix);
|
|||||||
* for the connection to succeed within the provided timeout (milliseconds).
|
* for the connection to succeed within the provided timeout (milliseconds).
|
||||||
*/
|
*/
|
||||||
esp_err_t wifi_portal_connect_sta(int timeout_ms);
|
esp_err_t wifi_portal_connect_sta(int timeout_ms);
|
||||||
|
esp_err_t wifi_portal_reconnect_sta(int timeout_ms);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief Save Wi-Fi credentials into NVS.
|
* @brief Save Wi-Fi credentials into NVS.
|
||||||
|
|||||||
@@ -138,6 +138,11 @@ esp_err_t platform_wifi_portal_connect_sta(int timeout_ms)
|
|||||||
return wifi_portal_connect_sta(timeout_ms);
|
return wifi_portal_connect_sta(timeout_ms);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
esp_err_t platform_wifi_portal_reconnect_sta(int timeout_ms)
|
||||||
|
{
|
||||||
|
return wifi_portal_reconnect_sta(timeout_ms);
|
||||||
|
}
|
||||||
|
|
||||||
esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password)
|
esp_err_t platform_wifi_portal_save_credentials(const char *ssid, const char *password)
|
||||||
{
|
{
|
||||||
return wifi_portal_save_credentials(ssid, password);
|
return wifi_portal_save_credentials(ssid, password);
|
||||||
|
|||||||
@@ -18,12 +18,355 @@
|
|||||||
#include "freertos/FreeRTOS.h"
|
#include "freertos/FreeRTOS.h"
|
||||||
#include "freertos/task.h"
|
#include "freertos/task.h"
|
||||||
|
|
||||||
|
#ifndef CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
#define CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE 0
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
#include <errno.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
|
#include "esp_heap_caps.h"
|
||||||
|
#include "esp_netif.h"
|
||||||
|
#include "esp_timer.h"
|
||||||
|
#include "esp_wifi.h"
|
||||||
|
#include "freertos/semphr.h"
|
||||||
|
#include "ping/ping_sock.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
#define PLATFORM_HTTPS_DEFAULT_TIMEOUT_MS 10000
|
#define PLATFORM_HTTPS_DEFAULT_TIMEOUT_MS 10000
|
||||||
#define PLATFORM_HTTPS_DNS_RETRY_COUNT 2
|
#define PLATFORM_HTTPS_DNS_RETRY_COUNT 2
|
||||||
#define PLATFORM_HTTPS_DNS_RETRY_DELAY_MS 500
|
#define PLATFORM_HTTPS_DNS_RETRY_DELAY_MS 500
|
||||||
#define PLATFORM_HTTPS_HOST_MAX_LEN 256
|
#define PLATFORM_HTTPS_HOST_MAX_LEN 256
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
#define PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS 400
|
||||||
|
#define PLATFORM_HTTPS_PROBE_TCP_TIMEOUT_MS 800
|
||||||
|
#endif
|
||||||
|
|
||||||
static const char *TAG = "platform_https";
|
static const char *TAG = "platform_https";
|
||||||
|
static const char *dns_error_name(int err);
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
static volatile uint32_t s_dns_lookup_count;
|
||||||
|
static volatile uint32_t s_dns_lookup_failure_count;
|
||||||
|
static volatile uint32_t s_dns_lookup_active;
|
||||||
|
static struct sockaddr_storage s_last_resolved_addr;
|
||||||
|
static socklen_t s_last_resolved_addr_len;
|
||||||
|
static char s_last_resolved_host[PLATFORM_HTTPS_HOST_MAX_LEN];
|
||||||
|
static portMUX_TYPE s_probe_cache_lock = portMUX_INITIALIZER_UNLOCKED;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
SemaphoreHandle_t done;
|
||||||
|
volatile uint32_t replies;
|
||||||
|
} https_ping_probe_context_t;
|
||||||
|
|
||||||
|
static void https_ping_success(esp_ping_handle_t handle, void *args)
|
||||||
|
{
|
||||||
|
(void)handle;
|
||||||
|
https_ping_probe_context_t *context = (https_ping_probe_context_t *)args;
|
||||||
|
if (context != NULL) {
|
||||||
|
context->replies++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void https_ping_end(esp_ping_handle_t handle, void *args)
|
||||||
|
{
|
||||||
|
(void)handle;
|
||||||
|
https_ping_probe_context_t *context = (https_ping_probe_context_t *)args;
|
||||||
|
if (context != NULL && context->done != NULL) {
|
||||||
|
(void)xSemaphoreGive(context->done);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void run_ipv4_ping_probe(const char *label, const esp_ip4_addr_t *target)
|
||||||
|
{
|
||||||
|
if (target == NULL || target->addr == 0U) {
|
||||||
|
ESP_LOGW(TAG, "Network probe ping label=%s skipped: no IPv4 target", label);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
https_ping_probe_context_t context = {
|
||||||
|
.done = xSemaphoreCreateBinary(),
|
||||||
|
};
|
||||||
|
if (context.done == NULL) {
|
||||||
|
ESP_LOGW(TAG, "Network probe ping label=%s failed: no semaphore memory", label);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
esp_ping_config_t config = ESP_PING_DEFAULT_CONFIG();
|
||||||
|
config.count = 1;
|
||||||
|
config.interval_ms = 0;
|
||||||
|
config.timeout_ms = PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS;
|
||||||
|
ip_addr_set_ip4_u32(&config.target_addr, target->addr);
|
||||||
|
|
||||||
|
esp_ping_callbacks_t callbacks = {
|
||||||
|
.cb_args = &context,
|
||||||
|
.on_ping_success = https_ping_success,
|
||||||
|
.on_ping_timeout = NULL,
|
||||||
|
.on_ping_end = https_ping_end,
|
||||||
|
};
|
||||||
|
esp_ping_handle_t session = NULL;
|
||||||
|
int64_t started_us = esp_timer_get_time();
|
||||||
|
esp_err_t err = esp_ping_new_session(&config, &callbacks, &session);
|
||||||
|
if (err == ESP_OK) {
|
||||||
|
err = esp_ping_start(session);
|
||||||
|
}
|
||||||
|
if (err == ESP_OK) {
|
||||||
|
TickType_t wait_ticks = pdMS_TO_TICKS(PLATFORM_HTTPS_PROBE_PING_TIMEOUT_MS + 300);
|
||||||
|
if (wait_ticks == 0) {
|
||||||
|
wait_ticks = 1;
|
||||||
|
}
|
||||||
|
if (xSemaphoreTake(context.done, wait_ticks) != pdTRUE) {
|
||||||
|
err = ESP_ERR_TIMEOUT;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (session != NULL) {
|
||||||
|
(void)esp_ping_stop(session);
|
||||||
|
(void)esp_ping_delete_session(session);
|
||||||
|
}
|
||||||
|
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Network probe ping label=%s target=" IPSTR " result=%s replies=%u elapsed_ms=%lld",
|
||||||
|
label,
|
||||||
|
IP2STR(target),
|
||||||
|
err == ESP_OK && context.replies > 0U ? "reachable" :
|
||||||
|
(err == ESP_OK ? "no_reply" : esp_err_to_name(err)),
|
||||||
|
(unsigned)context.replies,
|
||||||
|
(long long)((esp_timer_get_time() - started_us) / 1000));
|
||||||
|
vSemaphoreDelete(context.done);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void cache_resolved_endpoint(const char *host, const struct addrinfo *addr)
|
||||||
|
{
|
||||||
|
const struct addrinfo *selected = addr;
|
||||||
|
while (selected != NULL &&
|
||||||
|
selected->ai_family != AF_INET &&
|
||||||
|
selected->ai_family != AF_INET6) {
|
||||||
|
selected = selected->ai_next;
|
||||||
|
}
|
||||||
|
if (host == NULL || selected == NULL || selected->ai_addr == NULL ||
|
||||||
|
selected->ai_addrlen > sizeof(s_last_resolved_addr)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct sockaddr_storage endpoint = {0};
|
||||||
|
memcpy(&endpoint, selected->ai_addr, selected->ai_addrlen);
|
||||||
|
if (selected->ai_family == AF_INET) {
|
||||||
|
((struct sockaddr_in *)&endpoint)->sin_port = htons(443);
|
||||||
|
} else {
|
||||||
|
((struct sockaddr_in6 *)&endpoint)->sin6_port = htons(443);
|
||||||
|
}
|
||||||
|
|
||||||
|
portENTER_CRITICAL(&s_probe_cache_lock);
|
||||||
|
s_last_resolved_addr = endpoint;
|
||||||
|
s_last_resolved_addr_len = selected->ai_addrlen;
|
||||||
|
strlcpy(s_last_resolved_host, host, sizeof(s_last_resolved_host));
|
||||||
|
portEXIT_CRITICAL(&s_probe_cache_lock);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void run_cached_endpoint_probe(void)
|
||||||
|
{
|
||||||
|
struct sockaddr_storage endpoint = {0};
|
||||||
|
socklen_t endpoint_len = 0;
|
||||||
|
char host[PLATFORM_HTTPS_HOST_MAX_LEN] = {0};
|
||||||
|
portENTER_CRITICAL(&s_probe_cache_lock);
|
||||||
|
endpoint = s_last_resolved_addr;
|
||||||
|
endpoint_len = s_last_resolved_addr_len;
|
||||||
|
strlcpy(host, s_last_resolved_host, sizeof(host));
|
||||||
|
portEXIT_CRITICAL(&s_probe_cache_lock);
|
||||||
|
|
||||||
|
if (endpoint_len == 0 || host[0] == '\0') {
|
||||||
|
ESP_LOGW(TAG, "Network probe cached TCP skipped: no successful DNS result cached");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
char address[64] = {0};
|
||||||
|
void *address_ptr = endpoint.ss_family == AF_INET ?
|
||||||
|
(void *)&((struct sockaddr_in *)&endpoint)->sin_addr :
|
||||||
|
(void *)&((struct sockaddr_in6 *)&endpoint)->sin6_addr;
|
||||||
|
const char *formatted = inet_ntop(endpoint.ss_family,
|
||||||
|
address_ptr,
|
||||||
|
address,
|
||||||
|
sizeof(address));
|
||||||
|
int64_t started_us = esp_timer_get_time();
|
||||||
|
int sock = socket(endpoint.ss_family, SOCK_STREAM, IPPROTO_TCP);
|
||||||
|
int socket_error = 0;
|
||||||
|
bool connected = false;
|
||||||
|
if (sock >= 0) {
|
||||||
|
int flags = fcntl(sock, F_GETFL, 0);
|
||||||
|
if (flags >= 0 && fcntl(sock, F_SETFL, flags | O_NONBLOCK) == 0) {
|
||||||
|
int connect_result = connect(sock,
|
||||||
|
(const struct sockaddr *)&endpoint,
|
||||||
|
endpoint_len);
|
||||||
|
if (connect_result == 0) {
|
||||||
|
connected = true;
|
||||||
|
} else if (errno == EINPROGRESS) {
|
||||||
|
fd_set write_set;
|
||||||
|
FD_ZERO(&write_set);
|
||||||
|
FD_SET(sock, &write_set);
|
||||||
|
struct timeval timeout = {
|
||||||
|
.tv_sec = 0,
|
||||||
|
.tv_usec = PLATFORM_HTTPS_PROBE_TCP_TIMEOUT_MS * 1000,
|
||||||
|
};
|
||||||
|
int selected = select(sock + 1, NULL, &write_set, NULL, &timeout);
|
||||||
|
socklen_t error_len = sizeof(socket_error);
|
||||||
|
if (selected > 0 &&
|
||||||
|
getsockopt(sock,
|
||||||
|
SOL_SOCKET,
|
||||||
|
SO_ERROR,
|
||||||
|
&socket_error,
|
||||||
|
&error_len) == 0 &&
|
||||||
|
socket_error == 0) {
|
||||||
|
connected = true;
|
||||||
|
} else if (selected == 0) {
|
||||||
|
socket_error = ETIMEDOUT;
|
||||||
|
} else if (socket_error == 0) {
|
||||||
|
socket_error = errno;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
socket_error = errno;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
socket_error = errno;
|
||||||
|
}
|
||||||
|
close(sock);
|
||||||
|
} else {
|
||||||
|
socket_error = errno;
|
||||||
|
}
|
||||||
|
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Network probe cached TCP host=%s address=%s:443 result=%s socket_error=%d elapsed_ms=%lld",
|
||||||
|
host,
|
||||||
|
formatted != NULL ? address : "unknown",
|
||||||
|
connected ? "reachable" : "failed",
|
||||||
|
socket_error,
|
||||||
|
(long long)((esp_timer_get_time() - started_us) / 1000));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void run_dns_failure_network_probes(const esp_netif_ip_info_t *ip,
|
||||||
|
const esp_netif_dns_info_t *fallback_dns)
|
||||||
|
{
|
||||||
|
ESP_LOGW(TAG, "Network probe bundle started after DNS failure");
|
||||||
|
run_ipv4_ping_probe("gateway", ip != NULL ? &ip->gw : NULL);
|
||||||
|
if (fallback_dns != NULL && fallback_dns->ip.type == ESP_IPADDR_TYPE_V4) {
|
||||||
|
run_ipv4_ping_probe("fallback_dns", &fallback_dns->ip.u_addr.ip4);
|
||||||
|
} else {
|
||||||
|
ESP_LOGW(TAG, "Network probe fallback DNS ping skipped: no IPv4 address");
|
||||||
|
}
|
||||||
|
run_cached_endpoint_probe();
|
||||||
|
ESP_LOGW(TAG, "Network probe bundle completed");
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char *dhcp_status_name(esp_netif_dhcp_status_t status)
|
||||||
|
{
|
||||||
|
switch (status) {
|
||||||
|
case ESP_NETIF_DHCP_INIT:
|
||||||
|
return "init";
|
||||||
|
case ESP_NETIF_DHCP_STARTED:
|
||||||
|
return "started";
|
||||||
|
case ESP_NETIF_DHCP_STOPPED:
|
||||||
|
return "stopped";
|
||||||
|
default:
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char *format_dns_address(const esp_netif_dns_info_t *dns,
|
||||||
|
char *buffer,
|
||||||
|
size_t buffer_size)
|
||||||
|
{
|
||||||
|
if (dns == NULL || buffer == NULL || buffer_size == 0U) {
|
||||||
|
return "invalid";
|
||||||
|
}
|
||||||
|
if (dns->ip.type == ESP_IPADDR_TYPE_V6) {
|
||||||
|
int written = snprintf(buffer,
|
||||||
|
buffer_size,
|
||||||
|
IPV6STR,
|
||||||
|
IPV62STR(dns->ip.u_addr.ip6));
|
||||||
|
return written >= 0 && (size_t)written < buffer_size ? buffer : "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
int written = snprintf(buffer,
|
||||||
|
buffer_size,
|
||||||
|
IPSTR,
|
||||||
|
IP2STR(&dns->ip.u_addr.ip4));
|
||||||
|
return written >= 0 && (size_t)written < buffer_size ? buffer : "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
static void log_dns_failure_diagnostics(const char *host,
|
||||||
|
int attempt,
|
||||||
|
int dns_err,
|
||||||
|
uint32_t lookup_id,
|
||||||
|
uint32_t elapsed_ms)
|
||||||
|
{
|
||||||
|
esp_netif_t *sta = esp_netif_get_handle_from_ifkey("WIFI_STA_DEF");
|
||||||
|
esp_netif_ip_info_t ip = {0};
|
||||||
|
esp_netif_dns_info_t dns_main = {0};
|
||||||
|
esp_netif_dns_info_t dns_backup = {0};
|
||||||
|
esp_netif_dns_info_t dns_fallback = {0};
|
||||||
|
esp_netif_dhcp_status_t dhcp_status = ESP_NETIF_DHCP_INIT;
|
||||||
|
esp_err_t ip_err = sta != NULL ? esp_netif_get_ip_info(sta, &ip) : ESP_ERR_NOT_FOUND;
|
||||||
|
esp_err_t main_err = esp_netif_get_dns_info(sta, ESP_NETIF_DNS_MAIN, &dns_main);
|
||||||
|
esp_err_t backup_err = esp_netif_get_dns_info(sta, ESP_NETIF_DNS_BACKUP, &dns_backup);
|
||||||
|
esp_err_t fallback_err = esp_netif_get_dns_info(sta,
|
||||||
|
ESP_NETIF_DNS_FALLBACK,
|
||||||
|
&dns_fallback);
|
||||||
|
esp_err_t dhcp_err = sta != NULL ? esp_netif_dhcpc_get_status(sta, &dhcp_status) :
|
||||||
|
ESP_ERR_NOT_FOUND;
|
||||||
|
|
||||||
|
wifi_ap_record_t ap = {0};
|
||||||
|
esp_err_t ap_err = esp_wifi_sta_get_ap_info(&ap);
|
||||||
|
char main_buffer[48] = {0};
|
||||||
|
char backup_buffer[48] = {0};
|
||||||
|
char fallback_buffer[48] = {0};
|
||||||
|
const char *main_address = main_err == ESP_OK ?
|
||||||
|
format_dns_address(&dns_main, main_buffer, sizeof(main_buffer)) : "unavailable";
|
||||||
|
const char *backup_address = backup_err == ESP_OK ?
|
||||||
|
format_dns_address(&dns_backup, backup_buffer, sizeof(backup_buffer)) : "unavailable";
|
||||||
|
const char *fallback_address = fallback_err == ESP_OK ?
|
||||||
|
format_dns_address(&dns_fallback, fallback_buffer, sizeof(fallback_buffer)) :
|
||||||
|
"unavailable";
|
||||||
|
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"DNS diagnostics lookup=%u attempt=%d elapsed_ms=%u host=%s err=%s(%d) active=%u failures=%u sta=%d ip=" IPSTR " netmask=" IPSTR " gw=" IPSTR " dhcp=%d/%s dns(main=%s backup=%s fallback=%s status=%s/%s/%s) rssi=%d ap=%s heap(internal=%u largest=%u dma_largest=%u)",
|
||||||
|
(unsigned)lookup_id,
|
||||||
|
attempt,
|
||||||
|
(unsigned)elapsed_ms,
|
||||||
|
host != NULL ? host : "-",
|
||||||
|
dns_error_name(dns_err),
|
||||||
|
dns_err,
|
||||||
|
(unsigned)__atomic_load_n(&s_dns_lookup_active, __ATOMIC_RELAXED),
|
||||||
|
(unsigned)__atomic_load_n(&s_dns_lookup_failure_count, __ATOMIC_RELAXED),
|
||||||
|
sta != NULL ? 1 : 0,
|
||||||
|
IP2STR(&ip.ip),
|
||||||
|
IP2STR(&ip.netmask),
|
||||||
|
IP2STR(&ip.gw),
|
||||||
|
dhcp_err == ESP_OK ? (int)dhcp_status : -1,
|
||||||
|
dhcp_err == ESP_OK ? dhcp_status_name(dhcp_status) : esp_err_to_name(dhcp_err),
|
||||||
|
main_address,
|
||||||
|
backup_address,
|
||||||
|
fallback_address,
|
||||||
|
esp_err_to_name(main_err),
|
||||||
|
esp_err_to_name(backup_err),
|
||||||
|
esp_err_to_name(fallback_err),
|
||||||
|
ap_err == ESP_OK ? (int)ap.rssi : 0,
|
||||||
|
esp_err_to_name(ap_err),
|
||||||
|
(unsigned)heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),
|
||||||
|
(unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT),
|
||||||
|
(unsigned)heap_caps_get_largest_free_block(MALLOC_CAP_DMA | MALLOC_CAP_8BIT));
|
||||||
|
|
||||||
|
if (ip_err != ESP_OK) {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"DNS diagnostics STA IP query failed: %s",
|
||||||
|
esp_err_to_name(ip_err));
|
||||||
|
}
|
||||||
|
if (attempt == 1) {
|
||||||
|
run_dns_failure_network_probes(ip_err == ESP_OK ? &ip : NULL,
|
||||||
|
fallback_err == ESP_OK ? &dns_fallback : NULL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
static int effective_positive_or_zero(int value)
|
static int effective_positive_or_zero(int value)
|
||||||
{
|
{
|
||||||
@@ -106,23 +449,54 @@ static esp_err_t resolve_host_before_open(const char *url)
|
|||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
uint32_t lookup_id = __atomic_add_fetch(&s_dns_lookup_count, 1U, __ATOMIC_RELAXED);
|
||||||
|
(void)__atomic_add_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
|
||||||
|
#endif
|
||||||
for (int attempt = 1; attempt <= PLATFORM_HTTPS_DNS_RETRY_COUNT; ++attempt) {
|
for (int attempt = 1; attempt <= PLATFORM_HTTPS_DNS_RETRY_COUNT; ++attempt) {
|
||||||
struct addrinfo hints = {
|
struct addrinfo hints = {
|
||||||
.ai_family = AF_UNSPEC,
|
.ai_family = AF_UNSPEC,
|
||||||
.ai_socktype = SOCK_STREAM,
|
.ai_socktype = SOCK_STREAM,
|
||||||
};
|
};
|
||||||
struct addrinfo *addr = NULL;
|
struct addrinfo *addr = NULL;
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
int64_t attempt_started_us = esp_timer_get_time();
|
||||||
|
#endif
|
||||||
int dns_err = getaddrinfo(host, NULL, &hints, &addr);
|
int dns_err = getaddrinfo(host, NULL, &hints, &addr);
|
||||||
if (dns_err == 0 && addr != NULL) {
|
if (dns_err == 0 && addr != NULL) {
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
uint32_t elapsed_ms = (uint32_t)((esp_timer_get_time() - attempt_started_us) / 1000);
|
||||||
|
cache_resolved_endpoint(host, addr);
|
||||||
|
if (__atomic_load_n(&s_dns_lookup_failure_count, __ATOMIC_RELAXED) > 0U ||
|
||||||
|
elapsed_ms >= 1000U) {
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"DNS resolved lookup=%u attempt=%d elapsed_ms=%u host=%s",
|
||||||
|
(unsigned)lookup_id,
|
||||||
|
attempt,
|
||||||
|
(unsigned)elapsed_ms,
|
||||||
|
host);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
freeaddrinfo(addr);
|
freeaddrinfo(addr);
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
(void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
|
||||||
|
#endif
|
||||||
return ESP_OK;
|
return ESP_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (addr != NULL) {
|
if (addr != NULL) {
|
||||||
freeaddrinfo(addr);
|
freeaddrinfo(addr);
|
||||||
}
|
}
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
(void)__atomic_add_fetch(&s_dns_lookup_failure_count, 1U, __ATOMIC_RELAXED);
|
||||||
|
uint32_t elapsed_ms = (uint32_t)((esp_timer_get_time() - attempt_started_us) / 1000);
|
||||||
|
log_dns_failure_diagnostics(host, attempt, dns_err, lookup_id, elapsed_ms);
|
||||||
|
#endif
|
||||||
if (dns_err == EAI_MEMORY) {
|
if (dns_err == EAI_MEMORY) {
|
||||||
ESP_LOGW(TAG, "DNS resolve failed host=%s err=%s(%d)", host, dns_error_name(dns_err), dns_err);
|
ESP_LOGW(TAG, "DNS resolve failed host=%s err=%s(%d)", host, dns_error_name(dns_err), dns_err);
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
(void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
|
||||||
|
#endif
|
||||||
return ESP_ERR_NO_MEM;
|
return ESP_ERR_NO_MEM;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -143,6 +517,9 @@ static esp_err_t resolve_host_before_open(const char *url)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
(void)__atomic_sub_fetch(&s_dns_lookup_active, 1U, __ATOMIC_RELAXED);
|
||||||
|
#endif
|
||||||
return ESP_ERR_HTTP_CONNECT;
|
return ESP_ERR_HTTP_CONNECT;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -291,3 +668,8 @@ esp_err_t platform_https_post_json(const platform_https_post_json_request_t *req
|
|||||||
response->body_len = total_read;
|
response->body_len = total_read;
|
||||||
return ESP_OK;
|
return ESP_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool platform_https_is_connect_error(esp_err_t err)
|
||||||
|
{
|
||||||
|
return err == ESP_ERR_HTTP_CONNECT;
|
||||||
|
}
|
||||||
|
|||||||
@@ -343,6 +343,65 @@ esp_err_t wifi_portal_connect_sta(int timeout_ms)
|
|||||||
return ESP_ERR_TIMEOUT;
|
return ESP_ERR_TIMEOUT;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
esp_err_t wifi_portal_reconnect_sta(int timeout_ms)
|
||||||
|
{
|
||||||
|
int effective_timeout_ms = normalize_connect_timeout_ms(timeout_ms);
|
||||||
|
int64_t deadline_us = esp_timer_get_time() + (int64_t)effective_timeout_ms * 1000;
|
||||||
|
|
||||||
|
{
|
||||||
|
ESP_RETURN_ON_ERROR(wifi_portal_lock(), TAG, "Failed to lock Wi-Fi portal");
|
||||||
|
PortalUnlockGuard guard;
|
||||||
|
|
||||||
|
if (!wifi_portal_has_saved_credentials()) {
|
||||||
|
return ESP_ERR_NOT_FOUND;
|
||||||
|
}
|
||||||
|
ESP_RETURN_ON_ERROR(ensure_wifi_manager_initialized(),
|
||||||
|
TAG,
|
||||||
|
"Failed to initialize Wi-Fi manager");
|
||||||
|
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Controlled STA reassociation started timeout_ms=%d connected=%d",
|
||||||
|
effective_timeout_ms,
|
||||||
|
WifiManager::GetInstance().IsConnected() ? 1 : 0);
|
||||||
|
esp_err_t disconnect_err = esp_wifi_disconnect();
|
||||||
|
if (disconnect_err != ESP_OK && disconnect_err != ESP_ERR_WIFI_NOT_CONNECT) {
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Controlled STA disconnect returned %s",
|
||||||
|
esp_err_to_name(disconnect_err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int64_t disconnect_deadline_us = esp_timer_get_time() + 2000LL * 1000LL;
|
||||||
|
if (disconnect_deadline_us > deadline_us) {
|
||||||
|
disconnect_deadline_us = deadline_us;
|
||||||
|
}
|
||||||
|
while (esp_timer_get_time() < disconnect_deadline_us &&
|
||||||
|
WifiManager::GetInstance().IsConnected()) {
|
||||||
|
vTaskDelay(pdMS_TO_TICKS(WIFI_PORTAL_CONNECT_POLL_MS));
|
||||||
|
}
|
||||||
|
|
||||||
|
esp_err_t connect_err = esp_wifi_connect();
|
||||||
|
if (connect_err != ESP_OK) {
|
||||||
|
ESP_LOGI(TAG,
|
||||||
|
"Controlled STA reconnect command returned %s; wait for manager auto-reconnect",
|
||||||
|
esp_err_to_name(connect_err));
|
||||||
|
}
|
||||||
|
|
||||||
|
while (esp_timer_get_time() < deadline_us) {
|
||||||
|
if (WifiManager::GetInstance().IsConnected()) {
|
||||||
|
(void)apply_sta_stability_policy();
|
||||||
|
ESP_LOGI(TAG, "Controlled STA reassociation completed");
|
||||||
|
return ESP_OK;
|
||||||
|
}
|
||||||
|
vTaskDelay(pdMS_TO_TICKS(WIFI_PORTAL_CONNECT_POLL_MS));
|
||||||
|
}
|
||||||
|
|
||||||
|
ESP_LOGW(TAG,
|
||||||
|
"Controlled STA reassociation timed out after %d ms",
|
||||||
|
effective_timeout_ms);
|
||||||
|
return ESP_ERR_TIMEOUT;
|
||||||
|
}
|
||||||
|
|
||||||
esp_err_t wifi_portal_save_credentials(const char *ssid, const char *password)
|
esp_err_t wifi_portal_save_credentials(const char *ssid, const char *password)
|
||||||
{
|
{
|
||||||
install_wifi_storage_hooks();
|
install_wifi_storage_hooks();
|
||||||
|
|||||||
@@ -110,6 +110,77 @@ menu "Example Configuration"
|
|||||||
help
|
help
|
||||||
Maximum FreeRTOS task entries captured per report.
|
Maximum FreeRTOS task entries captured per report.
|
||||||
|
|
||||||
|
config TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
bool "Enable runtime heap watermark monitor"
|
||||||
|
default y
|
||||||
|
help
|
||||||
|
Track internal RAM, DMA-capable RAM, and PSRAM watermarks.
|
||||||
|
Allocation failures and low contiguous DMA memory are logged
|
||||||
|
together with the current voice-session state.
|
||||||
|
|
||||||
|
config TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS
|
||||||
|
int "Memory monitor sample interval (ms)"
|
||||||
|
default 1000
|
||||||
|
range 100 60000
|
||||||
|
depends on TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
help
|
||||||
|
Sampling interval for heap watermarks. Periodic reports are
|
||||||
|
emitted less frequently to avoid flooding the log.
|
||||||
|
|
||||||
|
config TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS
|
||||||
|
int "Memory monitor periodic report interval (ms)"
|
||||||
|
default 30000
|
||||||
|
range 5000 600000
|
||||||
|
depends on TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
help
|
||||||
|
Interval for normal heap watermark reports. Pressure changes
|
||||||
|
and allocation failures are reported immediately.
|
||||||
|
|
||||||
|
config TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES
|
||||||
|
int "Internal RAM warning threshold (bytes)"
|
||||||
|
default 16384
|
||||||
|
range 4096 262144
|
||||||
|
depends on TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
help
|
||||||
|
Emit a pressure warning when free 8-bit internal RAM falls
|
||||||
|
below this value.
|
||||||
|
|
||||||
|
config TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES
|
||||||
|
int "Largest DMA block warning threshold (bytes)"
|
||||||
|
default 4096
|
||||||
|
range 1024 65536
|
||||||
|
depends on TQ_MEMORY_MONITOR_ENABLE
|
||||||
|
help
|
||||||
|
Emit a pressure warning when the largest contiguous
|
||||||
|
DMA-capable allocation falls below this value.
|
||||||
|
|
||||||
|
config TQ_NETWORK_DIAGNOSTICS_ENABLE
|
||||||
|
bool "Enable detailed network diagnostics"
|
||||||
|
default y
|
||||||
|
help
|
||||||
|
Collect detailed DNS, Wi-Fi, heap, ping, and cached-endpoint
|
||||||
|
TCP probe data when HTTPS name resolution fails. The probes
|
||||||
|
can add up to roughly two seconds to a failed DNS attempt, so
|
||||||
|
disable this option for production builds after diagnosis.
|
||||||
|
|
||||||
|
config TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
|
||||||
|
bool "Enable detailed multimodal session diagnostics"
|
||||||
|
default y
|
||||||
|
help
|
||||||
|
Log multimodal session generations, transport cleanup stages,
|
||||||
|
and network failure streaks.
|
||||||
|
Functional recovery warnings remain visible when disabled.
|
||||||
|
|
||||||
|
config TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
|
||||||
|
bool "Enable automatic Wi-Fi reassociation after HTTPS failures"
|
||||||
|
default n
|
||||||
|
help
|
||||||
|
After two consecutive multimodal HTTPS connection failures,
|
||||||
|
disconnect and reconnect the Wi-Fi station before retrying.
|
||||||
|
This interrupts all device network traffic while reassociation
|
||||||
|
is in progress. Keep disabled unless field testing shows that
|
||||||
|
reconnecting only the multimodal transport is insufficient.
|
||||||
|
|
||||||
config TQ_SYSTEM_EVENT_ENABLE
|
config TQ_SYSTEM_EVENT_ENABLE
|
||||||
bool "Enable control-plane system event task"
|
bool "Enable control-plane system event task"
|
||||||
default y
|
default y
|
||||||
|
|||||||
@@ -21,6 +21,14 @@ CONFIG_TQ_NETWORK_TASK_CORE=0
|
|||||||
CONFIG_TQ_LOCAL_TASK_CORE=1
|
CONFIG_TQ_LOCAL_TASK_CORE=1
|
||||||
CONFIG_TQ_TASK_MONITOR_INTERVAL_MS=10000
|
CONFIG_TQ_TASK_MONITOR_INTERVAL_MS=10000
|
||||||
CONFIG_TQ_TASK_MONITOR_MAX_TASKS=64
|
CONFIG_TQ_TASK_MONITOR_MAX_TASKS=64
|
||||||
|
CONFIG_TQ_MEMORY_MONITOR_ENABLE=y
|
||||||
|
CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS=1000
|
||||||
|
CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS=30000
|
||||||
|
CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES=16384
|
||||||
|
CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES=4096
|
||||||
|
CONFIG_TQ_NETWORK_DIAGNOSTICS_ENABLE=y
|
||||||
|
CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE=y
|
||||||
|
# CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE is not set
|
||||||
CONFIG_TQ_SYSTEM_EVENT_ENABLE=y
|
CONFIG_TQ_SYSTEM_EVENT_ENABLE=y
|
||||||
CONFIG_TQ_SYSTEM_EVENT_QUEUE_LEN=32
|
CONFIG_TQ_SYSTEM_EVENT_QUEUE_LEN=32
|
||||||
CONFIG_TQ_SYSTEM_EVENT_TASK_STACK=4096
|
CONFIG_TQ_SYSTEM_EVENT_TASK_STACK=4096
|
||||||
|
|||||||
Reference in New Issue
Block a user