fix(control_plane): bound provisioning recovery and prevent reboot loops

This commit is contained in:
chenjintao
2026-10-23 15:37:59 +08:00
parent aede687dc9
commit 415c75b0e6
12 changed files with 947 additions and 57 deletions
+378
View File
@@ -0,0 +1,378 @@
#!/usr/bin/env python3
"""Fault-inject the production provisioning coordinator and domain start functions.
Uses host stubs for RTOS/BLE/storage only; policy, startup, error handling,
recovery-boot consumption and factory save are compiled from production code.
"""
from pathlib import Path
import re
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
CP = (ROOT / 'components/control_plane/src/runtime/control_plane_connectivity_runtime.c').read_text()
DOMAIN = (ROOT / 'components/domain/src/runtime/domain_mini_program_provisioning.c').read_text()
CLI = (ROOT / 'components/control_plane/src/runtime/control_plane_factory_cli.c').read_text()
FACADE = (ROOT / 'components/domain/include/domain.h').read_text()
def extract(source, name):
match = re.search(r'^(?:static )?(?:esp_err_t|bool|void|uint32_t|const char \*|domain_wifi_provisioning_start_status_t)\s*' + name + r'\([^;]*?\)\s*\{', source, re.M)
if not match:
raise ValueError(name)
pos, depth = match.end(), 1
while depth:
depth += (source[pos] == '{') - (source[pos] == '}')
pos += 1
return source[match.start():pos]
EVENTS = re.search(r'typedef enum \{\s*DOMAIN_WIFI_PROVISIONING_EVENT_CREDENTIALS.*?\} domain_wifi_provisioning_event_type_t;', FACADE, re.S).group()
NOTIFY = re.search(r'typedef enum \{\s*DOMAIN_WIFI_PROVISIONING_STATUS_WIFI_CONNECTING.*?\} domain_wifi_provisioning_status_t;', FACADE, re.S).group()
STATUS = re.search(r'typedef enum \{\s*DOMAIN_WIFI_PROVISIONING_START_READY.*?\} domain_wifi_provisioning_start_status_t;', FACADE, re.S).group()
CONSTANTS = '\n'.join(re.findall(r'^#define CONTROL_PLANE_PROVISIONING_.*$', CP, re.M))
STUBS = r'''
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include "control_plane_provisioning_policy.h"
typedef int esp_err_t;
typedef void *TaskHandle_t;
typedef void *QueueHandle_t;
typedef void *EventGroupHandle_t;
typedef int BaseType_t;
typedef struct {int type; struct {char ssid[33], password[65];} credentials;} domain_wifi_provisioning_event_t;
typedef uint32_t TickType_t;
typedef uint32_t EventBits_t;
typedef void (*domain_wifi_provisioning_event_callback_t)(const void *, void *);
typedef struct {uint32_t free_internal, largest_internal, largest_dma;} domain_ota_memory_headroom_t;
#define ESP_OK 0
#define ESP_ERR_INVALID_ARG 1
#define ESP_ERR_NO_MEM 2
#define ESP_ERR_TIMEOUT 3
#define ESP_ERR_NOT_FOUND 4
#define ESP_ERR_NOT_SUPPORTED 5
#define ESP_ERR_INVALID_STATE 6
#define ESP_FAIL 7
#define BIT0 1
#define BIT1 2
#define BIT2 4
#define CONFIG_TQ_MINI_PROGRAM_PROVISIONING_ENABLE 1
#define CONFIG_TQ_MINI_PROGRAM_PROVISIONING_MIN_FREE_INTERNAL_BYTES 49152
#define CONFIG_TQ_MINI_PROGRAM_PROVISIONING_MIN_LARGEST_DMA_BYTES 12288
#define DOMAIN_PROVISIONING_ID_MAX_LEN 64
#define DOMAIN_PROVISIONING_SERIAL_MAX_LEN 64
#define DOMAIN_PROVISIONING_INFO_MAX_LEN 512
#define PLATFORM_BLE_GATT_NAME_MAX_LEN 32
#define FACTORY_CLI_DEVICE_ID_MAX_LEN 64
#define FACTORY_CLI_SERIAL_MAX_LEN 64
#define pdMS_TO_TICKS(ms) (ms)
#define pdPASS 1
#define pdTRUE 1
#define pdFALSE 0
#define CONFIG_TQ_MINI_PROGRAM_PROVISIONING_CONNECT_TIMEOUT_MS 30000
static int lock_depth;
#define portENTER_CRITICAL(lock) do {assert(lock_depth++ == 0);} while(0)
#define portEXIT_CRITICAL(lock) do {assert(--lock_depth == 0);} while(0)
static char logs[32768];
static void record(const char *fmt, ...) {
va_list ap; va_start(ap, fmt);
vsnprintf(logs+strlen(logs), sizeof(logs)-strlen(logs), fmt, ap);
va_end(ap);
}
#define ESP_LOGI(tag, ...) record(__VA_ARGS__)
#define ESP_LOGW(tag, ...) record(__VA_ARGS__)
#define ESP_LOGE(tag, ...) record(__VA_ARGS__)
static const char *esp_err_to_name(int err) {return err==ESP_ERR_NOT_FOUND ? "NOT_FOUND" : "error";}
static uint64_t now_ms;
static int restarts, memory_calls, quiesce_calls, audio_deinits, ble_calls, ble_stops, notifications;
static int ble_error, stop_error, memory_error_after, create_error, queue_error, local_wake_error, sta_error, save_error;
static int cli_ok, cli_errors, config_notifications;
static bool identity_ready, keep_awake;
static unsigned event_bits;
static int runtime_starts, task_deletes, connect_calls, connect_failures, wifi_failed_notifications;
static domain_ota_memory_headroom_t memory;
static cp_provisioning_policy_t s_provisioning_policy;
static bool s_provisioning_recovery_boot;
static uint32_t s_provisioning_recovery_marker, s_provisioning_recovery_marker_inverse;
static TaskHandle_t s_provisioning_task;
static QueueHandle_t s_provisioning_queue;
static EventGroupHandle_t s_provisioning_events;
static domain_wifi_provisioning_event_callback_t s_event_callback;
static void *s_event_user_data;
static char s_pending_device_id[64], s_pending_serial_number[64];
static uint64_t esp_timer_get_time(void) {return now_ms*1000;}
static void vTaskDelay(unsigned ms) {now_ms+=ms;}
static void domain_runtime_restart(void) {++restarts;}
static esp_err_t domain_runtime_ota_get_memory_headroom(domain_ota_memory_headroom_t *out) {
assert(!lock_depth); ++memory_calls;
if (memory_error_after && memory_calls >= memory_error_after) return ESP_FAIL;
*out=memory; return ESP_OK;
}
static void domain_runtime_read_device_identity(char *id, size_t id_len, char *sn, size_t sn_len) {
snprintf(id,id_len,"%s",identity_ready?"device":"");
snprintf(sn,sn_len,"%s",identity_ready?"serial":"");
}
static void control_plane_request_wifi_reconnect_now(void) {++config_notifications;}
static void control_plane_wifi_set_keep_awake(bool value) {keep_awake=value;}
static void control_plane_set_ptt_active(bool active) {assert(!active);}
static int wake_restarts;
static bool control_plane_ota_voice_services_allowed(void) {return !s_provisioning_policy.runtime_quiesced;}
static esp_err_t control_plane_local_wake_start(void) {++wake_restarts; return ESP_OK;}
static esp_err_t control_plane_local_wake_stop(int ms) {assert(ms==2000); ++quiesce_calls; return local_wake_error;}
static esp_err_t control_plane_stop_multimodal_client(void) {return ESP_OK;}
static esp_err_t domain_runtime_multimodal_audio_stop_downstream_playback(void) {return ESP_OK;}
static void domain_runtime_multimodal_audio_deinit(void) {++audio_deinits;}
static void control_plane_show_provisioning_ui(void) {}
static esp_err_t domain_runtime_wifi_portal_stop_sta(void) {return sta_error;}
static void *xQueueCreate(unsigned len, unsigned size) {return queue_error ? NULL : (void*)1;}
static void *xEventGroupCreate(void) {return (void*)2;}
static void xQueueReset(void *queue) {assert(queue);}
static void xEventGroupClearBits(void *events, unsigned bits) {assert(events);}
static void control_plane_provisioning_event_callback(const void *event, void *user) {}
static uint32_t xTaskGetTickCount(void) {return (uint32_t)now_ms;}
static unsigned xEventGroupGetBits(void *events) {return event_bits;}
static unsigned xEventGroupWaitBits(void *events, unsigned bits, int clear, int all, unsigned timeout) {return event_bits;}
static int xQueueReceive(void *queue, domain_wifi_provisioning_event_t *event, unsigned timeout) {
event->type=DOMAIN_WIFI_PROVISIONING_EVENT_CREDENTIALS; return pdTRUE;
}
static void vTaskDelete(void *task) {++task_deletes;}
static void control_plane_provisioning_notify(domain_wifi_provisioning_status_t status) {
if (status==DOMAIN_WIFI_PROVISIONING_STATUS_FAIL_CONNECT || status==DOMAIN_WIFI_PROVISIONING_STATUS_FAIL_TIMEOUT)
++wifi_failed_notifications;
}
static esp_err_t domain_runtime_wifi_portal_connect_sta_with_credentials(const char *ssid, const char *password, int timeout) {
++connect_calls; return connect_calls<=connect_failures ? ESP_FAIL : ESP_OK;
}
static void control_plane_play_wifi_success_prompt(void) {}
static void control_plane_finish_provisioning_ui(void) {}
static esp_err_t control_plane_start_runtime_workers(void) {
assert(!s_provisioning_policy.runtime_quiesced); ++runtime_starts; return ESP_OK;
}
static int xTaskCreate(void (*fn)(void*), const char *name, unsigned stack, void *arg, unsigned priority, void **task) {
if (create_error) return 0;
*task=(void*)3; return pdPASS;
}
static void xEventGroupSetBits(void *events, unsigned bits) {
assert(events==s_provisioning_events);
event_bits |= bits;
if (bits==BIT2) {
assert(s_provisioning_task && s_provisioning_policy.phase==CP_PROVISIONING_ACTIVE);
++notifications;
}
}
static void domain_provisioning_reset_write_buffer(void) {}
static void domain_provisioning_handle_write(void) {}
static const char *s_service_uuid="service", *s_wifi_write_uuid="write", *s_status_uuid="status", *s_device_info_uuid="info";
typedef struct {
const char *advertised_name, *service_uuid, *write_uuid, *notify_uuid, *read_notify_uuid;
const uint8_t *read_value; size_t read_value_len;
void (*on_write)(void); void *user_data;
} platform_ble_gatt_server_config_t;
static esp_err_t platform_ble_gatt_server_start(const platform_ble_gatt_server_config_t *cfg) {++ble_calls; return ble_error;}
static esp_err_t domain_runtime_mini_program_provisioning_stop(int timeout) {assert(timeout==2000); ++ble_stops; return stop_error;}
static esp_err_t domain_provisioning_build_device_info(char *name, size_t n, char *info, size_t i) {
name[0]=0; snprintf(info,i,"{}"); return identity_ready?ESP_OK:ESP_ERR_NOT_FOUND;
}
static void control_plane_register_wifi_behavior_hooks(void) {}
static esp_err_t domain_runtime_wifi_portal_init_defaults(const char *ssid, const char *password) {return ESP_OK;}
static bool domain_runtime_wifi_portal_has_saved_credentials(void) {return false;}
static int control_plane_normalize_connect_timeout_ms(int ms, int unused) {return ms;}
static esp_err_t control_plane_connect_sta_with_feedback(int ms, bool feedback) {return ESP_FAIL;}
static esp_err_t control_plane_start_lcd_render_worker(void) {return ESP_OK;}
static bool is_text_empty(const char *s) {return !s || !s[0];}
#define strlcpy test_strlcpy
static size_t test_strlcpy(char *dst, const char *src, size_t len) {size_t n=strlen(src); snprintf(dst,len,"%s",src); return n;}
static void trim_text(char *s) {}
static esp_err_t domain_runtime_write_device_identity(const char *id, const char *sn) {
if (save_error) return save_error;
identity_ready=id[0] && sn[0]; return ESP_OK;
}
static void control_plane_multimodal_device_auth_invalidate(unsigned unused) {}
static void cli_write_error(const char *error) {++cli_errors;}
static void cli_write_ok(void) {++cli_ok;}
'''
TESTS = r'''
static void reset(void) {
memset(&s_provisioning_policy,0,sizeof(s_provisioning_policy));
s_provisioning_recovery_boot=false;
s_provisioning_recovery_marker=s_provisioning_recovery_marker_inverse=0;
s_provisioning_task=s_provisioning_queue=s_provisioning_events=NULL;
now_ms=0; restarts=memory_calls=quiesce_calls=audio_deinits=ble_calls=ble_stops=notifications=0;
ble_error=stop_error=memory_error_after=create_error=queue_error=local_wake_error=sta_error=save_error=0;
cli_ok=cli_errors=config_notifications=0;
wake_restarts=0; event_bits=0; runtime_starts=task_deletes=connect_calls=connect_failures=wifi_failed_notifications=0;
identity_ready=false; keep_awake=false;
s_pending_device_id[0]=s_pending_serial_number[0]=logs[0]=0;
memory=(domain_ota_memory_headroom_t){135167,73728,73728};
}
int main(void) {
reset();
assert(control_plane_bootstrap_connectivity("","",20000)==ESP_ERR_NOT_FOUND);
assert(s_provisioning_policy.phase==CP_PROVISIONING_WAIT_IDENTITY);
for (unsigned i=0;i<10000;i++) {now_ms+=1000; assert(!control_plane_provisioning_retry_due());}
assert(!restarts && !ble_calls && !quiesce_calls && !audio_deinits);
assert(control_plane_provisioning_allows_voice());
assert(strstr(logs,"identity_required"));
/* Failed save cannot wake policy; successful save only notifies, BLE runs later. */
snprintf(s_pending_device_id,64,"new-id"); snprintf(s_pending_serial_number,64,"new-sn");
save_error=ESP_FAIL; handle_save_config();
assert(cli_errors==1 && !config_notifications && !control_plane_provisioning_retry_due());
save_error=0; handle_save_config();
assert(cli_ok==1 && config_notifications==1 && !ble_calls);
assert(control_plane_provisioning_retry_due());
assert(control_plane_enter_mini_program_reprovisioning("identity_saved")==ESP_OK);
assert(ble_calls==1 && quiesce_calls==1 && notifications==1 && !restarts);
assert(!control_plane_provisioning_allows_voice() && !control_plane_provisioning_retry_due());
/* Actual timeout classification and exact bounded backoff, including uint32 wrap. */
reset(); identity_ready=true; ble_error=ESP_ERR_TIMEOUT; now_ms=UINT32_MAX-500;
const unsigned delays[]={1000,3000,10000};
for (unsigned i=0;i<3;i++) {
assert(control_plane_enter_mini_program_reprovisioning("timeout")==ESP_ERR_TIMEOUT);
assert(s_provisioning_policy.phase==CP_PROVISIONING_BACKOFF);
assert(s_provisioning_policy.retry_delay_ms==delays[i]);
assert(!control_plane_provisioning_retry_due()); now_ms+=delays[i]-1;
assert(!control_plane_provisioning_retry_due()); ++now_ms;
assert(control_plane_provisioning_retry_due());
}
assert(control_plane_enter_mini_program_reprovisioning("timeout")==ESP_ERR_TIMEOUT);
assert(s_provisioning_policy.phase==CP_PROVISIONING_BLOCKED && !restarts && ble_calls==4);
now_ms+=100000; assert(!control_plane_provisioning_retry_due());
control_plane_provisioning_configuration_changed(); ble_error=ESP_OK;
assert(control_plane_enter_mini_program_reprovisioning("manual")==ESP_OK);
const int permanent[]={ESP_ERR_INVALID_ARG,ESP_ERR_NOT_SUPPORTED,ESP_ERR_NOT_FOUND,ESP_FAIL};
for (unsigned i=0;i<4;i++) {
reset(); identity_ready=true; ble_error=permanent[i];
assert(control_plane_enter_mini_program_reprovisioning("permanent")==permanent[i]);
assert(s_provisioning_policy.phase==CP_PROVISIONING_BLOCKED);
assert(!restarts && !control_plane_provisioning_retry_due());
assert(!s_event_callback && !s_event_user_data);
}
/* Memory threshold and actual allocator failure both permit only ONE recovery. */
for (unsigned scenario=0;scenario<5;scenario++) {
reset(); identity_ready=true;
if (scenario==0) memory.free_internal=49151;
if (scenario==1) memory.largest_dma=12287;
if (scenario==2) ble_error=ESP_ERR_NO_MEM;
if (scenario==3) create_error=1;
if (scenario==4) queue_error=1;
assert(control_plane_bootstrap_connectivity("","",20000)==ESP_ERR_NO_MEM);
assert(restarts==1 && s_provisioning_policy.recovery_attempted);
assert(control_plane_take_mini_program_recovery_boot_request());
assert(!control_plane_take_mini_program_recovery_boot_request());
assert(s_provisioning_policy.recovery_attempted);
/* Simulate fresh globals after RTC consumption, retaining the attempt latch. */
s_provisioning_policy.phase=CP_PROVISIONING_IDLE;
assert(control_plane_bootstrap_connectivity("","",20000)==ESP_ERR_NO_MEM);
assert(!s_provisioning_recovery_boot && s_provisioning_policy.recovery_attempted);
assert(restarts==1 && !control_plane_provisioning_retry_due());
for (unsigned retry=0;retry<20;retry++) {
control_plane_provisioning_configuration_changed();
assert(control_plane_enter_mini_program_reprovisioning("explicit")==ESP_ERR_NO_MEM);
assert(restarts==1);
}
}
reset(); identity_ready=true; memory.free_internal=49152; memory.largest_dma=12288;
assert(control_plane_enter_mini_program_reprovisioning("boundary")==ESP_OK && !restarts);
reset(); identity_ready=true; memory_error_after=1;
assert(control_plane_enter_mini_program_reprovisioning("measurement_error")==ESP_FAIL);
assert(!restarts && !ble_calls && !control_plane_provisioning_retry_due());
/* Snapshot failure after a confirmed BLE allocation failure prints unknown, never fake zeros. */
reset(); identity_ready=true; ble_error=ESP_ERR_NO_MEM; memory_error_after=2;
assert(control_plane_enter_mini_program_reprovisioning("allocation")==ESP_ERR_NO_MEM);
assert(restarts==1 && strstr(logs,"memory=unknown") && !strstr(logs,"free_internal=0"));
reset(); identity_ready=true; local_wake_error=ESP_ERR_TIMEOUT;
assert(control_plane_enter_mini_program_reprovisioning("stop_timeout")==ESP_ERR_TIMEOUT);
assert(!audio_deinits && !ble_calls && !restarts);
reset(); identity_ready=true; create_error=1; stop_error=ESP_ERR_TIMEOUT;
assert(control_plane_enter_mini_program_reprovisioning("cleanup_timeout")==ESP_ERR_TIMEOUT);
assert(ble_stops==1 && !restarts && s_provisioning_policy.phase==CP_PROVISIONING_BLOCKED);
/* Run the real worker: bad Wi-Fi credentials retain BLE; success restores runtime. */
reset(); identity_ready=true; connect_failures=1;
assert(control_plane_enter_mini_program_reprovisioning("normal")==ESP_OK);
control_plane_provisioning_worker(NULL);
assert(connect_calls==2 && wifi_failed_notifications==1 && ble_calls==1 && ble_stops==1);
assert(runtime_starts==1 && !restarts && !s_provisioning_task && task_deletes==1);
assert(control_plane_provisioning_allows_voice() && !keep_awake);
reset(); identity_ready=true; stop_error=ESP_ERR_TIMEOUT;
assert(control_plane_enter_mini_program_reprovisioning("teardown")==ESP_OK);
control_plane_provisioning_worker(NULL);
assert(!runtime_starts && !restarts && s_provisioning_policy.phase==CP_PROVISIONING_BLOCKED);
assert(!control_plane_provisioning_allows_voice() && s_provisioning_policy.cleanup_required);
control_plane_provisioning_sta_connected();
assert(!control_plane_provisioning_allows_voice() && !wake_restarts);
control_plane_provisioning_configuration_changed();
assert(control_plane_enter_mini_program_reprovisioning("cleanup_still_fails")==ESP_ERR_TIMEOUT);
assert(ble_calls==1 && !restarts && !control_plane_provisioning_retry_due());
stop_error=ESP_OK;
control_plane_provisioning_configuration_changed();
assert(control_plane_enter_mini_program_reprovisioning("cleanup_then_retry")==ESP_OK);
assert(ble_stops==3 && ble_calls==2);
reset(); identity_ready=true;
assert(control_plane_enter_mini_program_reprovisioning("cancel")==ESP_OK);
event_bits |= BIT0;
control_plane_provisioning_worker(NULL);
assert(!runtime_starts && !restarts && !control_plane_provisioning_retry_due());
reset(); identity_ready=true;
assert(control_plane_enter_mini_program_reprovisioning("gate_timeout")==ESP_OK);
event_bits=0;
control_plane_provisioning_worker(NULL);
assert(!s_provisioning_task && !restarts && ble_stops==1 && !control_plane_provisioning_retry_due());
reset(); identity_ready=true; ble_error=ESP_ERR_TIMEOUT;
assert(control_plane_enter_mini_program_reprovisioning("direct_sta_recovery")==ESP_ERR_TIMEOUT);
control_plane_provisioning_sta_connected();
assert(control_plane_provisioning_allows_voice() && wake_restarts==1);
/* One failure-recovery reboot, then the existing success-mode-switch reboot. */
reset(); identity_ready=true; memory.free_internal=1;
assert(control_plane_bootstrap_connectivity("","",20000)==ESP_ERR_NO_MEM && restarts==1);
assert(control_plane_take_mini_program_recovery_boot_request());
s_provisioning_policy.phase=CP_PROVISIONING_IDLE; memory.free_internal=135167;
assert(control_plane_bootstrap_connectivity("","",20000)==ESP_OK);
control_plane_provisioning_worker(NULL);
assert(restarts==2 && !runtime_starts && !s_provisioning_task);
assert(control_plane_provisioning_allows_voice() && !s_provisioning_recovery_marker);
puts("PASS: worker Wi-Fi failure/success, teardown failure, cancellation and bounded start gate");
puts("PASS: identity wait/save resume, production BLE error classification, bounded retry/wrap, memory thresholds/allocation failures, recovery-boot loop prevention, cleanup and truthful diagnostics");
}
'''
def main():
domain_functions = ['domain_runtime_mini_program_provisioning_preflight',
'domain_provisioning_classify_start_error',
'domain_runtime_mini_program_provisioning_start']
cp_functions = ['control_plane_provisioning_now_ms', 'control_plane_provisioning_is_recovery_boot',
'control_plane_provisioning_retry_due', 'control_plane_provisioning_configuration_changed',
'control_plane_provisioning_allows_voice', 'control_plane_provisioning_sta_connected',
'control_plane_take_mini_program_recovery_boot_request',
'control_plane_request_provisioning_recovery_restart', 'control_plane_provisioning_failure_name',
'control_plane_provisioning_start_failed', 'control_plane_check_provisioning_memory_headroom',
'control_plane_provisioning_stop_requested', 'control_plane_provisioning_worker',
'control_plane_start_provisioning_transport', 'control_plane_is_mini_program_provisioning_running',
'control_plane_try_provisioning', 'control_plane_enter_mini_program_reprovisioning',
'control_plane_bootstrap_connectivity']
with tempfile.TemporaryDirectory(prefix='provisioning-recovery-') as tmp:
tmp = Path(tmp)
(tmp / 'domain.h').write_text('#pragma once\n'+STATUS+'\n'+EVENTS+'\n'+NOTIFY+'\n')
source = STUBS+'\n'+CONSTANTS+'\n'
source += '\n'.join(extract(DOMAIN, name) for name in domain_functions)+'\n'
source += 'bool control_plane_is_mini_program_provisioning_running(void);\n'
source += '\n'.join(extract(CP, name) for name in cp_functions)+'\n'
source += extract(CLI, 'handle_save_config')+'\n'+TESTS
(tmp / 'test.c').write_text(source)
subprocess.run(['clang', '-std=c11', '-Wall', '-Wextra', '-Werror',
'-Wno-unused-parameter', '-Wno-unused-function',
'-fsanitize=address,undefined', '-I',str(tmp),
'-I',str(ROOT/'components/control_plane/internal'),
str(tmp/'test.c'), '-o',str(tmp/'test')],check=True)
subprocess.run([str(tmp/'test')],check=True)
if __name__ == '__main__':
main()