fix(control-plane): optimize task monitoring and worker stacks

This commit is contained in:
chenjintao
2026-08-21 17:52:10 +08:00
parent cb203a74f4
commit a60f98a4c6
6 changed files with 139 additions and 37 deletions
@@ -36,10 +36,14 @@
#define CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS 1 #define CONFIG_MULTIMODAL_AUDIO_PREWARM_OPUS_BEFORE_WS 1
#endif #endif
#ifndef CONFIG_TQ_MULTIMODAL_WATCHDOG_TASK_STACK_BYTES
#define CONFIG_TQ_MULTIMODAL_WATCHDOG_TASK_STACK_BYTES (6U * 1024U)
#endif
#define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220 #define CONTROL_PLANE_DOWNSTREAM_PLAYBACK_GUARD_MS 220
#define CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS 2000 #define CONTROL_PLANE_DOWNSTREAM_STALL_STOP_GRACE_MS 2000
#define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000 #define CONTROL_PLANE_IDLE_TRANSPORT_SHUTDOWN_MS 30000
#define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE 8192 #define CONTROL_PLANE_WATCHDOG_TASK_STACK_SIZE CONFIG_TQ_MULTIMODAL_WATCHDOG_TASK_STACK_BYTES
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus" #define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_FILE "/littlefs/audio_prompts/server_connect_success.opus"
#define CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS 30000U #define CONTROL_PLANE_CLIENT_LIFECYCLE_WAIT_MS 30000U
#define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS 1000U #define CONTROL_PLANE_SERVER_SUCCESS_PROMPT_RETRY_MS 1000U
@@ -30,7 +30,11 @@
#define FACTORY_CLI_SERIAL_MAX_LEN 64 #define FACTORY_CLI_SERIAL_MAX_LEN 64
#define FACTORY_CLI_JSON_MAX_LEN 640 #define FACTORY_CLI_JSON_MAX_LEN 640
#define FACTORY_CLI_READ_TIMEOUT_MS 100 #define FACTORY_CLI_READ_TIMEOUT_MS 100
#define FACTORY_CLI_WORKER_STACK_BYTES (12U * 1024U) #ifndef CONFIG_TQ_FACTORY_CLI_TASK_STACK_BYTES
#define CONFIG_TQ_FACTORY_CLI_TASK_STACK_BYTES (10U * 1024U)
#endif
#define FACTORY_CLI_WORKER_STACK_BYTES CONFIG_TQ_FACTORY_CLI_TASK_STACK_BYTES
#define FACTORY_CLI_WORKER_PRIORITY 4 #define FACTORY_CLI_WORKER_PRIORITY 4
#define FACTORY_CLI_STOP_WAIT_STEPS 100 #define FACTORY_CLI_STOP_WAIT_STEPS 100
#define FACTORY_CLI_STOP_WAIT_STEP_MS 20 #define FACTORY_CLI_STOP_WAIT_STEP_MS 20
@@ -20,10 +20,16 @@
#include "freertos/task.h" #include "freertos/task.h"
#include "sdkconfig.h" #include "sdkconfig.h"
#ifndef CONFIG_TQ_LCD_RENDER_TASK_STACK_BYTES
#define CONFIG_TQ_LCD_RENDER_TASK_STACK_BYTES (10U * 1024U)
#endif
#define CONTROL_PLANE_MODEL_CMD_TASK_STACK_SIZE (1024 * 4) #define CONTROL_PLANE_MODEL_CMD_TASK_STACK_SIZE (1024 * 4)
#define CONTROL_PLANE_LCD_RENDER_LOOP_TASK_STACK_SIZE (1024 * 12) #define CONTROL_PLANE_LCD_RENDER_LOOP_TASK_STACK_SIZE CONFIG_TQ_LCD_RENDER_TASK_STACK_BYTES
#define CONTROL_PLANE_WIFI_STATE_TASK_STACK_SIZE (1024 * 10) #define CONTROL_PLANE_WIFI_STATE_TASK_STACK_SIZE (1024 * 10)
#define CONTROL_PLANE_TASK_MONITOR_TASK_STACK_SIZE (1024 * 4) #define CONTROL_PLANE_TASK_MONITOR_TASK_STACK_SIZE (1024 * 4)
#define CONTROL_PLANE_TASK_MONITOR_STACK_WARN_BYTES 1024U
#define CONTROL_PLANE_TASK_MONITOR_FULL_REPORT_PERIOD 5U
#define CONTROL_PLANE_REMOTE_LOG_STOP_WAIT_MS 1500 #define CONTROL_PLANE_REMOTE_LOG_STOP_WAIT_MS 1500
#define CONTROL_PLANE_MODEL_CMD_QUEUE_LEN 4 #define CONTROL_PLANE_MODEL_CMD_QUEUE_LEN 4
#define CONTROL_PLANE_MODEL_CMD_RECV_TIMEOUT_MS 200 #define CONTROL_PLANE_MODEL_CMD_RECV_TIMEOUT_MS 200
@@ -1404,7 +1410,39 @@ static const char *control_plane_task_core_name(BaseType_t core_id,
return buffer; return buffer;
} }
static void control_plane_log_task_table(void) static void control_plane_log_task_status(const TaskStatus_t *task, bool warn)
{
if (task == NULL) {
return;
}
char core_buf[12] = {0};
const char *name = task->pcTaskName != NULL ? task->pcTaskName : "?";
const char *state = control_plane_task_state_name(task->eCurrentState);
const char *core = control_plane_task_core_name(
xTaskGetCoreID(task->xHandle), core_buf, sizeof(core_buf));
if (warn) {
ESP_LOGW(TAG,
"Task stack headroom low: name=%s state=%s priority=%u headroom=%uB core=%s",
name,
state,
(unsigned int)task->uxCurrentPriority,
(unsigned int)task->usStackHighWaterMark,
core);
return;
}
ESP_LOGI(TAG,
"%-16s %-5s %-4u %-10u %-4s %-10lu",
name,
state,
(unsigned int)task->uxCurrentPriority,
(unsigned int)task->usStackHighWaterMark,
core,
(unsigned long)task->ulRunTimeCounter);
}
static void control_plane_log_task_table(bool full_report)
{ {
#if !configUSE_TRACE_FACILITY #if !configUSE_TRACE_FACILITY
ESP_LOGW(TAG, "Task monitor requires CONFIG_FREERTOS_USE_TRACE_FACILITY"); ESP_LOGW(TAG, "Task monitor requires CONFIG_FREERTOS_USE_TRACE_FACILITY");
@@ -1422,42 +1460,61 @@ static void control_plane_log_task_table(void)
return; return;
} }
TaskStatus_t *tasks = (TaskStatus_t *)calloc(capture_count, sizeof(TaskStatus_t)); TaskStatus_t *tasks = (TaskStatus_t *)heap_caps_calloc(
capture_count,
sizeof(TaskStatus_t),
MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (tasks == NULL) { if (tasks == NULL) {
ESP_LOGW(TAG, "Task monitor allocation failed: entries=%u", ESP_LOGW(TAG, "Task monitor PSRAM snapshot allocation failed: entries=%u",
(unsigned int)capture_count); (unsigned int)capture_count);
return; return;
} }
uint32_t total_runtime = 0; uint32_t total_runtime = 0;
UBaseType_t captured = uxTaskGetSystemState(tasks, capture_count, &total_runtime); UBaseType_t captured = uxTaskGetSystemState(tasks, capture_count, &total_runtime);
const TaskStatus_t *lowest = NULL;
UBaseType_t low_count = 0U;
for (UBaseType_t i = 0; i < captured; ++i) {
if (lowest == NULL ||
tasks[i].usStackHighWaterMark < lowest->usStackHighWaterMark) {
lowest = &tasks[i];
}
if (tasks[i].usStackHighWaterMark <
CONTROL_PLANE_TASK_MONITOR_STACK_WARN_BYTES) {
low_count++;
}
}
ESP_LOGI(TAG, ESP_LOGI(TAG,
"Task monitor: total=%u captured=%u runtime=%lu", "Task monitor summary: total=%u captured=%u runtime=%lu lowest=%s/%uB low=%u snapshot=PSRAM full=%d",
(unsigned int)task_count, (unsigned int)task_count,
(unsigned int)captured, (unsigned int)captured,
(unsigned long)total_runtime); (unsigned long)total_runtime,
ESP_LOGI(TAG, "%-16s %-5s %-4s %-7s %-4s %-10s", lowest != NULL && lowest->pcTaskName != NULL ? lowest->pcTaskName : "?",
lowest != NULL ? (unsigned int)lowest->usStackHighWaterMark : 0U,
(unsigned int)low_count,
full_report ? 1 : 0);
if (!full_report) {
heap_caps_free(tasks);
return;
}
ESP_LOGI(TAG, "%-16s %-5s %-4s %-10s %-4s %-10s",
"name", "name",
"state", "state",
"prio", "prio",
"stack", "headroom",
"core", "core",
"runtime"); "runtime");
for (UBaseType_t i = 0; i < captured; ++i) { for (UBaseType_t i = 0; i < captured; ++i) {
char core_buf[12] = {0}; control_plane_log_task_status(
ESP_LOGI(TAG, &tasks[i],
"%-16s %-5s %-4u %-7u %-4s %-10lu", tasks[i].usStackHighWaterMark <
tasks[i].pcTaskName != NULL ? tasks[i].pcTaskName : "?", CONTROL_PLANE_TASK_MONITOR_STACK_WARN_BYTES);
control_plane_task_state_name(tasks[i].eCurrentState),
(unsigned int)tasks[i].uxCurrentPriority,
(unsigned int)tasks[i].usStackHighWaterMark,
control_plane_task_core_name(xTaskGetCoreID(tasks[i].xHandle), core_buf, sizeof(core_buf)),
(unsigned long)tasks[i].ulRunTimeCounter);
} }
free(tasks); heap_caps_free(tasks);
#endif #endif
} }
@@ -1466,9 +1523,11 @@ static void control_plane_task_monitor_task(void *arg)
(void)arg; (void)arg;
ESP_LOGI(TAG, ESP_LOGI(TAG,
"Task monitor started interval=%dms max_tasks=%d", "Task monitor started interval=%dms max_tasks=%d stack=PSRAM-preferred snapshot=PSRAM full_period=%u warn=%uB",
CONFIG_TQ_TASK_MONITOR_INTERVAL_MS, CONFIG_TQ_TASK_MONITOR_INTERVAL_MS,
CONFIG_TQ_TASK_MONITOR_MAX_TASKS); CONFIG_TQ_TASK_MONITOR_MAX_TASKS,
(unsigned)CONTROL_PLANE_TASK_MONITOR_FULL_REPORT_PERIOD,
(unsigned)CONTROL_PLANE_TASK_MONITOR_STACK_WARN_BYTES);
TickType_t interval_ticks = pdMS_TO_TICKS(CONFIG_TQ_TASK_MONITOR_INTERVAL_MS); TickType_t interval_ticks = pdMS_TO_TICKS(CONFIG_TQ_TASK_MONITOR_INTERVAL_MS);
if (interval_ticks == 0) { if (interval_ticks == 0) {
@@ -1478,8 +1537,12 @@ static void control_plane_task_monitor_task(void *arg)
interval_ticks = 1; interval_ticks = 1;
} }
uint32_t report_count = 0U;
while (!control_plane_runtime_stop_requested()) { while (!control_plane_runtime_stop_requested()) {
control_plane_log_task_table(); bool full_report =
(report_count % CONTROL_PLANE_TASK_MONITOR_FULL_REPORT_PERIOD) == 0U;
control_plane_log_task_table(full_report);
report_count++;
vTaskDelay(interval_ticks); vTaskDelay(interval_ticks);
} }
@@ -2549,12 +2612,13 @@ esp_err_t control_plane_start_runtime_workers(void)
#if CONFIG_TQ_TASK_MONITOR_ENABLE #if CONFIG_TQ_TASK_MONITOR_ENABLE
if (control_plane_get_task_handle(&s_task_monitor_task) == NULL) { if (control_plane_get_task_handle(&s_task_monitor_task) == NULL) {
esp_err_t task_err = control_plane_create_task(control_plane_task_monitor_task, esp_err_t task_err = control_plane_create_task_psram_first(
"cp_task_monitor", control_plane_task_monitor_task,
CONTROL_PLANE_TASK_MONITOR_TASK_STACK_SIZE, "cp_task_monitor",
3, CONTROL_PLANE_TASK_MONITOR_TASK_STACK_SIZE,
CONTROL_PLANE_LOCAL_TASK_CORE, 3,
&s_task_monitor_task); CONTROL_PLANE_LOCAL_TASK_CORE,
&s_task_monitor_task);
if (task_err != ESP_OK) { if (task_err != ESP_OK) {
control_plane_stop_runtime_workers(); control_plane_stop_runtime_workers();
return task_err; return task_err;
@@ -2703,7 +2767,7 @@ void control_plane_stop_runtime_workers(void)
} }
#if CONFIG_TQ_TASK_MONITOR_ENABLE #if CONFIG_TQ_TASK_MONITOR_ENABLE
if (!control_plane_wait_task_exit(&s_task_monitor_task, CONTROL_PLANE_TASK_STOP_WAIT_MS)) { if (!control_plane_wait_task_exit(&s_task_monitor_task, CONTROL_PLANE_TASK_STOP_WAIT_MS)) {
control_plane_force_delete_task(&s_task_monitor_task, "cp_task_monitor"); control_plane_force_delete_task_with_caps(&s_task_monitor_task, "cp_task_monitor");
} }
#endif #endif
#if CONFIG_FORTUNE_ENABLE #if CONFIG_FORTUNE_ENABLE
@@ -27,7 +27,7 @@
#endif #endif
#ifndef CONFIG_TQ_SYSTEM_EVENT_TASK_STACK #ifndef CONFIG_TQ_SYSTEM_EVENT_TASK_STACK
#define CONFIG_TQ_SYSTEM_EVENT_TASK_STACK 4096 #define CONFIG_TQ_SYSTEM_EVENT_TASK_STACK 6144
#endif #endif
#if !defined(CONFIG_TQ_PTT_MODE_HOLD_TO_TALK) && !defined(CONFIG_TQ_PTT_MODE_CLICK_TO_TALK) #if !defined(CONFIG_TQ_PTT_MODE_HOLD_TO_TALK) && !defined(CONFIG_TQ_PTT_MODE_CLICK_TO_TALK)
+31 -4
View File
@@ -211,15 +211,18 @@ menu "Example Configuration"
bool "Enable periodic FreeRTOS task monitor log" bool "Enable periodic FreeRTOS task monitor log"
default y default y
help help
Periodically print a compact FreeRTOS task table to the log. Run a low-impact task monitor with a PSRAM-preferred worker
stack and PSRAM-only snapshots. Periodic summaries report the
lowest stack headroom; a full task table is emitted less often.
config TQ_TASK_MONITOR_INTERVAL_MS config TQ_TASK_MONITOR_INTERVAL_MS
int "Task monitor log interval (ms)" int "Task monitor log interval (ms)"
default 10000 default 30000
range 1000 600000 range 1000 600000
depends on TQ_TASK_MONITOR_ENABLE depends on TQ_TASK_MONITOR_ENABLE
help help
Interval for periodic task table logging. Interval for task summaries. The complete table is emitted on
the first sample and every five samples after that.
config TQ_TASK_MONITOR_MAX_TASKS config TQ_TASK_MONITOR_MAX_TASKS
int "Task monitor max task entries" int "Task monitor max task entries"
@@ -317,7 +320,7 @@ menu "Example Configuration"
config TQ_SYSTEM_EVENT_TASK_STACK config TQ_SYSTEM_EVENT_TASK_STACK
int "System event task stack size" int "System event task stack size"
default 4096 default 6144
range 2048 8192 range 2048 8192
depends on TQ_SYSTEM_EVENT_ENABLE depends on TQ_SYSTEM_EVENT_ENABLE
help help
@@ -680,6 +683,14 @@ menu "Example Configuration"
Disable when the connected display hardware does not match this Disable when the connected display hardware does not match this
firmware's LCD driver. firmware's LCD driver.
config TQ_LCD_RENDER_TASK_STACK_BYTES
int "LCD render task stack (bytes)"
default 10240
range 8192 16384
depends on TQ_SCREEN_ENABLE
help
Stack size in bytes for the control-plane LCD render worker.
config TQ_SCREEN_COLOR_ORDER_BGR config TQ_SCREEN_COLOR_ORDER_BGR
bool "Use BGR color order" bool "Use BGR color order"
default n default n
@@ -1154,6 +1165,14 @@ menu "Example Configuration"
help help
Master switch for multimodal WebSocket interaction features. Master switch for multimodal WebSocket interaction features.
config TQ_MULTIMODAL_WATCHDOG_TASK_STACK_BYTES
int "Multimodal watchdog task stack (bytes)"
default 6144
range 4096 12288
depends on ENABLE_MULTIMODAL_INTERACTION
help
Stack size in bytes for the multimodal session watchdog task.
config MULTIMODAL_USE_STATIC_CREDENTIALS config MULTIMODAL_USE_STATIC_CREDENTIALS
bool "Use static WebSocket credentials (skip token request)" bool "Use static WebSocket credentials (skip token request)"
default n default n
@@ -1230,6 +1249,14 @@ menu "Example Configuration"
device identity provisioning, public-key retrieval, and device identity provisioning, public-key retrieval, and
on-device challenge signing. on-device challenge signing.
config TQ_FACTORY_CLI_TASK_STACK_BYTES
int "Factory CLI task stack (bytes)"
default 10240
range 8192 16384
depends on TQ_FACTORY_CLI_ENABLE
help
Stack size in bytes for the serial factory provisioning worker.
config AUTH_REGISTER_BASE_URL config AUTH_REGISTER_BASE_URL
string "Auth V2 registration base URL" string "Auth V2 registration base URL"
depends on AUTH_V2_ENABLE depends on AUTH_V2_ENABLE
+5 -2
View File
@@ -37,7 +37,8 @@ CONFIG_TQ_BOOT_SOUND_TIMEOUT_MS=3000
CONFIG_TQ_BOOT_SOUND_FILE="boot.opus" CONFIG_TQ_BOOT_SOUND_FILE="boot.opus"
CONFIG_TQ_NETWORK_TASK_CORE=0 CONFIG_TQ_NETWORK_TASK_CORE=0
CONFIG_TQ_LOCAL_TASK_CORE=1 CONFIG_TQ_LOCAL_TASK_CORE=1
CONFIG_TQ_TASK_MONITOR_INTERVAL_MS=10000 CONFIG_TQ_LCD_RENDER_TASK_STACK_BYTES=10240
CONFIG_TQ_TASK_MONITOR_INTERVAL_MS=30000
CONFIG_TQ_TASK_MONITOR_MAX_TASKS=64 CONFIG_TQ_TASK_MONITOR_MAX_TASKS=64
CONFIG_TQ_MEMORY_MONITOR_ENABLE=y CONFIG_TQ_MEMORY_MONITOR_ENABLE=y
CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS=1000 CONFIG_TQ_MEMORY_MONITOR_SAMPLE_INTERVAL_MS=1000
@@ -49,7 +50,7 @@ CONFIG_TQ_MULTIMODAL_DIAGNOSTICS_ENABLE=y
CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE=y CONFIG_TQ_NETWORK_AUTO_REASSOCIATION_ENABLE=y
CONFIG_TQ_SYSTEM_EVENT_ENABLE=y CONFIG_TQ_SYSTEM_EVENT_ENABLE=y
CONFIG_TQ_SYSTEM_EVENT_QUEUE_LEN=32 CONFIG_TQ_SYSTEM_EVENT_QUEUE_LEN=32
CONFIG_TQ_SYSTEM_EVENT_TASK_STACK=4096 CONFIG_TQ_SYSTEM_EVENT_TASK_STACK=6144
CONFIG_TQ_LIGHT_STANDBY_ENABLE=y CONFIG_TQ_LIGHT_STANDBY_ENABLE=y
CONFIG_TQ_LIGHT_STANDBY_IDLE_TIMEOUT_MS=300000 CONFIG_TQ_LIGHT_STANDBY_IDLE_TIMEOUT_MS=300000
CONFIG_TQ_LIGHT_STANDBY_DIM_BACKLIGHT=y CONFIG_TQ_LIGHT_STANDBY_DIM_BACKLIGHT=y
@@ -190,10 +191,12 @@ CONFIG_FORTUNE_USER_CONFIG_HTTP_PORT=80
CONFIG_FORTUNE_USER_CONFIG_MDNS_HOSTNAME="tq-cat" CONFIG_FORTUNE_USER_CONFIG_MDNS_HOSTNAME="tq-cat"
# CONFIG_FORTUNE_USE_DEVICE_JWT is not set # CONFIG_FORTUNE_USE_DEVICE_JWT is not set
CONFIG_ENABLE_MULTIMODAL_INTERACTION=y CONFIG_ENABLE_MULTIMODAL_INTERACTION=y
CONFIG_TQ_MULTIMODAL_WATCHDOG_TASK_STACK_BYTES=6144
# CONFIG_MULTIMODAL_USE_STATIC_CREDENTIALS is not set # CONFIG_MULTIMODAL_USE_STATIC_CREDENTIALS is not set
# CONFIG_MULTIMODAL_AUTH_BYPASS_TOKEN_SERVICE is not set # CONFIG_MULTIMODAL_AUTH_BYPASS_TOKEN_SERVICE is not set
CONFIG_AUTH_V2_ENABLE=y CONFIG_AUTH_V2_ENABLE=y
CONFIG_TQ_FACTORY_CLI_ENABLE=y CONFIG_TQ_FACTORY_CLI_ENABLE=y
CONFIG_TQ_FACTORY_CLI_TASK_STACK_BYTES=10240
CONFIG_MULTIMODAL_UPSTREAM_MODE="push2talk" CONFIG_MULTIMODAL_UPSTREAM_MODE="push2talk"
CONFIG_MULTIMODAL_PTT_BUTTON_GPIO=13 CONFIG_MULTIMODAL_PTT_BUTTON_GPIO=13
CONFIG_MULTIMODAL_PTT_BUTTON_ACTIVE_LEVEL=0 CONFIG_MULTIMODAL_PTT_BUTTON_ACTIVE_LEVEL=0