chenjintao/lucky-cat
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(runtime): add diagnostic and network recovery controls

Browse Source
This commit is contained in:
chenjintao
2026-08-12 16:32:40 +08:00
parent a94df9545d
commit 1c32d9f0e7
11 changed files with 868 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/control_plane/src/runtime/control_plane_runtime_workers.c
+324
View File
@@ -49,6 +49,8 @@
#define CONTROL_PLANE_SPIN_INTRO_TIMEOUT_MS 6000
#define CONTROL_PLANE_COMMAND_ROTATION_DEG 100.0f
#define CONTROL_PLANE_COMMAND_RETURN_DELAY_MS 5000
#define CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES 512U
#define CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES 256U
#if (configSUPPORT_STATIC_ALLOCATION == 1) && \
((defined(CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) && CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM) || \
@@ -80,6 +82,38 @@
#define CONFIG_TQ_TASK_MONITOR_MAX_TASKS 64
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_ENABLE
#define CONFIG_TQ_MEMORY_MONITOR_ENABLE 1
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS
#define CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS 1000
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS
#define CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS 10000
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES
#define CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES 16384
#endif
#ifndef CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES
#define CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES 4096
#endif
#ifndef CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
#define CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE 0
#endif
#ifndef CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
#define CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE 0
#endif
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
#define CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD 2U
#endif
#if !defined(CONFIG_TQ_PTT_MODE_HOLD_TO_TALK) && !defined(CONFIG_TQ_PTT_MODE_CLICK_TO_TALK)
#define CONFIG_TQ_PTT_MODE_CLICK_TO_TALK 1
#endif
@@ -182,6 +216,10 @@ static QueueHandle_t s_model_cmd_queue;
static bool s_model_cmd_queue_with_caps;
static TaskHandle_t s_model_cmd_task;
static bool s_ws_start_requested;
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
static uint32_t s_multimodal_network_failure_count;
static bool s_wifi_reassociation_requested;
#endif
static bool s_ptt_active;
static ptt_runtime_t s_ptt_runtime;
static TaskHandle_t s_lcd_render_loop_task;
@@ -197,7 +235,200 @@ static uint32_t s_fortune_refresh_generation;
static volatile bool s_runtime_stop_requested;
static portMUX_TYPE s_runtime_lock = portMUX_INITIALIZER_UNLOCKED;
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
typedef struct {
TickType_t last_sample_tick;
TickType_t last_report_tick;
uint32_t reported_alloc_fail_count;
size_t reported_internal_free;
size_t reported_dma_largest;
bool initialized;
bool pressure_active;
} control_plane_memory_monitor_t;
static control_plane_memory_monitor_t s_memory_monitor;
static volatile uint32_t s_alloc_fail_count;
static volatile size_t s_last_alloc_fail_size;
static volatile uint32_t s_last_alloc_fail_caps;
static const char *volatile s_last_alloc_fail_function;
static bool s_alloc_failure_hook_registered;
#endif
static TaskHandle_t control_plane_get_task_handle(TaskHandle_t *slot);
static const char *ptt_state_name(ptt_state_t state);
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
static void IRAM_ATTR control_plane_alloc_failed_hook(size_t size,
uint32_t caps,
const char *function_name)
{
__atomic_store_n(&s_last_alloc_fail_size, size, __ATOMIC_RELAXED);
__atomic_store_n(&s_last_alloc_fail_caps, caps, __ATOMIC_RELAXED);
__atomic_store_n(&s_last_alloc_fail_function, function_name, __ATOMIC_RELAXED);
(void)__atomic_add_fetch(&s_alloc_fail_count, 1U, __ATOMIC_RELEASE);
}
static void control_plane_memory_monitor_register_hook(void)
{
if (s_alloc_failure_hook_registered) {
return;
}
esp_err_t err = heap_caps_register_failed_alloc_callback(control_plane_alloc_failed_hook);
if (err == ESP_OK) {
s_alloc_failure_hook_registered = true;
ESP_LOGI(TAG, "Heap allocation failure hook registered");
} else {
ESP_LOGW(TAG, "Heap allocation failure hook registration failed: %s",
esp_err_to_name(err));
}
}
static unsigned control_plane_heap_contiguous_percent(const multi_heap_info_t *info)
{
if (info == NULL || info->total_free_bytes == 0U) {
return 0U;
}
return (unsigned)((info->largest_free_block * 100U) / info->total_free_bytes);
}
static void control_plane_memory_monitor_sample(TickType_t now_tick, bool force)
{
TickType_t sample_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS);
TickType_t report_ticks = pdMS_TO_TICKS(CONFIG_TQ_MEMORY_MONITOR_REPORT_INTERVAL_MS);
if (sample_ticks == 0) {
sample_ticks = 1;
}
if (report_ticks == 0) {
report_ticks = 1;
}
if (!force && s_memory_monitor.initialized &&
(now_tick - s_memory_monitor.last_sample_tick) < sample_ticks) {
return;
}
s_memory_monitor.last_sample_tick = now_tick;
multi_heap_info_t internal = {0};
multi_heap_info_t dma = {0};
multi_heap_info_t psram = {0};
heap_caps_get_info(&internal, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
heap_caps_get_info(&dma, MALLOC_CAP_DMA | MALLOC_CAP_8BIT);
heap_caps_get_info(&psram, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
uint32_t alloc_fail_count = __atomic_load_n(&s_alloc_fail_count, __ATOMIC_ACQUIRE);
bool alloc_failure_pending = alloc_fail_count != s_memory_monitor.reported_alloc_fail_count;
bool pressure = internal.total_free_bytes < CONFIG_TQ_MEMORY_MONITOR_INTERNAL_WARN_BYTES ||
dma.largest_free_block < CONFIG_TQ_MEMORY_MONITOR_DMA_LARGEST_WARN_BYTES;
bool pressure_changed = s_memory_monitor.initialized &&
pressure != s_memory_monitor.pressure_active;
bool watermark_drop = s_memory_monitor.initialized &&
(internal.total_free_bytes + CONTROL_PLANE_MEMORY_INTERNAL_DROP_BYTES <=
s_memory_monitor.reported_internal_free ||
dma.largest_free_block + CONTROL_PLANE_MEMORY_DMA_LARGEST_DROP_BYTES <=
s_memory_monitor.reported_dma_largest);
bool periodic_due = !s_memory_monitor.initialized ||
(now_tick - s_memory_monitor.last_report_tick) >= report_ticks;
if (!force && !alloc_failure_pending && !pressure_changed && !watermark_drop && !periodic_due) {
return;
}
const char *reason = force || !s_memory_monitor.initialized ? "startup" :
alloc_failure_pending ? "alloc_failure" :
pressure_changed ? (pressure ? "pressure_enter" : "pressure_exit") :
watermark_drop ? "watermark_drop" :
pressure ? "pressure_periodic" : "periodic";
control_plane_multimodal_runtime_state_t mm = {0};
bool have_mm = control_plane_multimodal_get_runtime_state(&mm) == ESP_OK;
domain_local_command_snapshot_t local_command = {0};
bool have_local_command =
domain_runtime_local_command_get_snapshot(&local_command) == ESP_OK;
ptt_state_t ptt_state;
bool ptt_active;
portENTER_CRITICAL(&s_runtime_lock);
ptt_state = s_ptt_runtime.state;
ptt_active = s_ptt_runtime.active;
portEXIT_CRITICAL(&s_runtime_lock);
if (pressure || alloc_failure_pending) {
ESP_LOGW(TAG,
"Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)",
reason,
(unsigned)internal.total_free_bytes,
(unsigned)internal.minimum_free_bytes,
(unsigned)internal.largest_free_block,
(unsigned)internal.allocated_blocks,
(unsigned)internal.free_blocks,
control_plane_heap_contiguous_percent(&internal),
(unsigned)dma.total_free_bytes,
(unsigned)dma.minimum_free_bytes,
(unsigned)dma.largest_free_block,
(unsigned)dma.allocated_blocks,
(unsigned)dma.free_blocks,
control_plane_heap_contiguous_percent(&dma),
(unsigned)psram.total_free_bytes,
(unsigned)psram.minimum_free_bytes,
(unsigned)psram.largest_free_block);
} else {
ESP_LOGI(TAG,
"Memory watermark reason=%s internal(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) dma(free=%u min=%u largest=%u blocks=%u/%u contiguous=%u%%) psram(free=%u min=%u largest=%u)",
reason,
(unsigned)internal.total_free_bytes,
(unsigned)internal.minimum_free_bytes,
(unsigned)internal.largest_free_block,
(unsigned)internal.allocated_blocks,
(unsigned)internal.free_blocks,
control_plane_heap_contiguous_percent(&internal),
(unsigned)dma.total_free_bytes,
(unsigned)dma.minimum_free_bytes,
(unsigned)dma.largest_free_block,
(unsigned)dma.allocated_blocks,
(unsigned)dma.free_blocks,
control_plane_heap_contiguous_percent(&dma),
(unsigned)psram.total_free_bytes,
(unsigned)psram.minimum_free_bytes,
(unsigned)psram.largest_free_block);
}
ESP_LOGI(TAG,
"Memory context ptt=%s active=%d wifi=%d ws(conn=%d starting=%d stopping=%d) task=%d listening=%d responding=%d uplink=%d local_kws(started=%d detecting=%d detections=%u stack_hwm=%u) alloc_failures=%u",
ptt_state_name(ptt_state),
ptt_active ? 1 : 0,
control_plane_is_sta_connected() ? 1 : 0,
(have_mm && mm.connected) ? 1 : 0,
(have_mm && mm.starting) ? 1 : 0,
(have_mm && mm.stopping) ? 1 : 0,
(have_mm && mm.has_active_task) ? 1 : 0,
(have_mm && mm.server_listening) ? 1 : 0,
(have_mm && mm.remote_responding) ? 1 : 0,
control_plane_multimodal_is_uplink_running() ? 1 : 0,
(have_local_command && local_command.started) ? 1 : 0,
(have_local_command && local_command.detection_running) ? 1 : 0,
have_local_command ? (unsigned)local_command.detection_count : 0U,
have_local_command ? (unsigned)local_command.task_stack_high_water_bytes : 0U,
(unsigned)alloc_fail_count);
if (alloc_failure_pending) {
size_t failed_size = __atomic_load_n(&s_last_alloc_fail_size, __ATOMIC_RELAXED);
uint32_t failed_caps = __atomic_load_n(&s_last_alloc_fail_caps, __ATOMIC_RELAXED);
const char *failed_function = __atomic_load_n(&s_last_alloc_fail_function,
__ATOMIC_RELAXED);
ESP_LOGE(TAG,
"Heap allocation failure count=%u newest(size=%u caps=0x%08x function=%s)",
(unsigned)alloc_fail_count,
(unsigned)failed_size,
(unsigned)failed_caps,
failed_function != NULL ? failed_function : "unknown");
s_memory_monitor.reported_alloc_fail_count = alloc_fail_count;
}
s_memory_monitor.initialized = true;
s_memory_monitor.pressure_active = pressure;
s_memory_monitor.reported_internal_free = internal.total_free_bytes;
s_memory_monitor.reported_dma_largest = dma.largest_free_block;
s_memory_monitor.last_report_tick = now_tick;
}
#endif
static bool control_plane_runtime_stop_requested(void)
{
@@ -880,6 +1111,58 @@ esp_err_t control_plane_schedule_websocket_wait_task(void)
return ESP_OK;
}
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
static void control_plane_record_multimodal_network_result(esp_err_t result)
{
uint32_t failures = 0U;
bool reassociation_requested = false;
portENTER_CRITICAL(&s_runtime_lock);
if (result == ESP_OK) {
s_multimodal_network_failure_count = 0U;
} else if (domain_runtime_is_network_connect_error(result)) {
s_multimodal_network_failure_count++;
failures = s_multimodal_network_failure_count;
if (failures >= CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD &&
!s_wifi_reassociation_requested) {
s_wifi_reassociation_requested = true;
reassociation_requested = true;
}
}
portEXIT_CRITICAL(&s_runtime_lock);
#if CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE
if (result == ESP_OK) {
ESP_LOGI(TAG, "Multimodal network failure streak cleared");
} else if (domain_runtime_is_network_connect_error(result)) {
ESP_LOGW(TAG,
"Multimodal network failure streak=%u reassociation_requested=%d",
(unsigned)failures,
reassociation_requested ? 1 : 0);
}
#else
(void)failures;
(void)reassociation_requested;
#endif
}
static bool control_plane_consume_wifi_reassociation_request(void)
{
bool requested = false;
portENTER_CRITICAL(&s_runtime_lock);
requested = s_wifi_reassociation_requested;
s_wifi_reassociation_requested = false;
portEXIT_CRITICAL(&s_runtime_lock);
return requested;
}
static void control_plane_reset_multimodal_network_failure_streak(void)
{
portENTER_CRITICAL(&s_runtime_lock);
s_multimodal_network_failure_count = 0U;
portEXIT_CRITICAL(&s_runtime_lock);
}
#endif
#if CONFIG_TQ_TASK_MONITOR_ENABLE
static const char *control_plane_task_state_name(eTaskState state)
{
@@ -1130,6 +1413,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la
esp_err_t err = control_plane_start_multimodal_client();
if (err == ESP_OK) {
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
control_plane_record_multimodal_network_result(ESP_OK);
#endif
ESP_LOGI(TAG, "Multimodal client started");
return true;
}
@@ -1137,6 +1423,9 @@ static bool control_plane_try_start_multimodal_from_state_machine(TickType_t *la
return false;
}
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
control_plane_record_multimodal_network_result(err);
#endif
ESP_LOGW(TAG, "Multimodal client start failed: %s", esp_err_to_name(err));
return false;
}
@@ -1421,6 +1710,10 @@ static void control_plane_wifi_state_task(void *arg)
bool reconnect_attempted = false;
uint8_t ptt_wifi_foreground_attempts = 0;
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
control_plane_memory_monitor_sample(xTaskGetTickCount(), true);
#endif
if (!was_connected) {
ESP_LOGI(TAG, "Wi-Fi STA unavailable at runtime start; requesting disconnected expression");
control_plane_request_wifi_state_expression(DOMAIN_EXPRESSION_WIFI_DISCONNECTED,
@@ -1447,6 +1740,10 @@ static void control_plane_wifi_state_task(void *arg)
bool connected = control_plane_is_sta_connected();
TickType_t now_tick = xTaskGetTickCount();
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
control_plane_memory_monitor_sample(now_tick, false);
#endif
if (!connected) {
if (!disconnect_pending) {
disconnect_pending = true;
@@ -1515,6 +1812,28 @@ static void control_plane_wifi_state_task(void *arg)
}
}
#if CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE
if (connected && control_plane_consume_wifi_reassociation_request()) {
ESP_LOGW(TAG,
"Network recovery experiment: controlled STA reassociation started after %u HTTPS connection failures",
(unsigned)CONTROL_PLANE_NETWORK_FAILURE_REASSOC_THRESHOLD);
domain_runtime_behavior_set_keep_awake(true);
esp_err_t reassoc_err = domain_runtime_wifi_portal_reconnect_sta(
CONTROL_PLANE_WIFI_RECONNECT_TIMEOUT_MS);
domain_runtime_behavior_set_keep_awake(false);
connected = control_plane_is_sta_connected();
ESP_LOGW(TAG,
"Network recovery experiment: controlled STA reassociation result=%s connected=%d",
esp_err_to_name(reassoc_err),
connected ? 1 : 0);
if (reassoc_err == ESP_OK && connected) {
control_plane_reset_multimodal_network_failure_streak();
last_ws_start_attempt_tick = 0;
(void)control_plane_schedule_websocket_wait_task();
}
}
#endif
if (connected && disconnect_pending && !disconnected_expression_active) {
ESP_LOGI(TAG, "Wi-Fi STA reconnected within disconnect grace");
disconnect_pending = false;
@@ -1873,6 +2192,11 @@ esp_err_t control_plane_start_runtime_workers(void)
{
control_plane_set_runtime_stop_requested(false);
#if CONFIG_TQ_MEMORY_MONITOR_ENABLE
memset(&s_memory_monitor, 0, sizeof(s_memory_monitor));
control_plane_memory_monitor_register_hook();
#endif
esp_err_t queue_err = control_plane_ensure_model_cmd_queue();
if (queue_err != ESP_OK) {
return queue_err;