286 lines
8.7 KiB
C
286 lines
8.7 KiB
C
#include "control_plane.h"
|
|
|
|
#include <string.h>
|
|
|
|
#include "esp_log.h"
|
|
#include "esp_timer.h"
|
|
#include "domain.h"
|
|
#include "freertos/FreeRTOS.h"
|
|
#include "freertos/semphr.h"
|
|
#include "freertos/task.h"
|
|
|
|
static const char *TAG = "controller_lifecycle";
|
|
static SemaphoreHandle_t s_lock;
|
|
static controller_lifecycle_status_t s_status = {
|
|
.state = CONTROLLER_LIFECYCLE_STATE_STOPPED,
|
|
.last_error = ESP_OK,
|
|
.start_attempt = 0,
|
|
.last_transition_ms = 0,
|
|
.last_stage = "stopped",
|
|
};
|
|
|
|
static void lifecycle_lock_init_if_needed(void) {
|
|
if (s_lock == NULL) {
|
|
s_lock = xSemaphoreCreateMutex();
|
|
}
|
|
}
|
|
|
|
static bool lifecycle_lock_take(uint32_t timeout_ms) {
|
|
lifecycle_lock_init_if_needed();
|
|
return s_lock != NULL && xSemaphoreTake(s_lock, pdMS_TO_TICKS(timeout_ms)) == pdTRUE;
|
|
}
|
|
|
|
static void lifecycle_set_state_locked(controller_lifecycle_state_t state,
|
|
esp_err_t last_error,
|
|
const char *stage) {
|
|
s_status.state = state;
|
|
s_status.last_error = last_error;
|
|
s_status.last_transition_ms = esp_timer_get_time() / 1000;
|
|
strlcpy(s_status.last_stage, stage != NULL ? stage : "unknown", sizeof(s_status.last_stage));
|
|
domain_diag_set_gauge(DOMAIN_DIAG_GAUGE_LIFECYCLE_STATE, (int32_t)state);
|
|
}
|
|
|
|
const char *controller_lifecycle_state_str(controller_lifecycle_state_t state) {
|
|
switch (state) {
|
|
case CONTROLLER_LIFECYCLE_STATE_STOPPED:
|
|
return "stopped";
|
|
case CONTROLLER_LIFECYCLE_STATE_STARTING:
|
|
return "starting";
|
|
case CONTROLLER_LIFECYCLE_STATE_RUNNING:
|
|
return "running";
|
|
case CONTROLLER_LIFECYCLE_STATE_DEGRADED:
|
|
return "degraded";
|
|
case CONTROLLER_LIFECYCLE_STATE_STOPPING:
|
|
return "stopping";
|
|
default:
|
|
return "unknown";
|
|
}
|
|
}
|
|
|
|
static esp_err_t lifecycle_start_system_runtime(void) {
|
|
return system_runtime_start();
|
|
}
|
|
|
|
static esp_err_t lifecycle_start_printer_protocol(void) {
|
|
return printer_protocol_init();
|
|
}
|
|
|
|
static esp_err_t lifecycle_start_voice(void) {
|
|
return voice_interaction_init();
|
|
}
|
|
|
|
static esp_err_t lifecycle_stop_printer_protocol(void) {
|
|
esp_err_t rc = printer_protocol_stop(domain_policy_printer_stop_timeout_ms());
|
|
if (rc == ESP_OK || rc == ESP_ERR_INVALID_STATE) {
|
|
return ESP_OK;
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
static esp_err_t lifecycle_stop_voice(void) {
|
|
esp_err_t rc = voice_interaction_stop(NULL, 0);
|
|
if (rc == ESP_OK || rc == ESP_ERR_INVALID_STATE) {
|
|
return ESP_OK;
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
static esp_err_t lifecycle_stop_system_runtime(void) {
|
|
return system_runtime_shutdown();
|
|
}
|
|
|
|
typedef esp_err_t (*lifecycle_step_fn_t)(void);
|
|
|
|
static esp_err_t lifecycle_run_step_with_retry(const char *stage, lifecycle_step_fn_t fn) {
|
|
uint32_t retry_count = domain_policy_lifecycle_start_retry_count();
|
|
uint32_t max_attempts = retry_count + 1;
|
|
|
|
for (uint32_t attempt = 0; attempt < max_attempts; ++attempt) {
|
|
esp_err_t rc = fn();
|
|
if (rc == ESP_OK) {
|
|
if (attempt > 0) {
|
|
ESP_LOGW(TAG, "stage %s recovered after retries, attempt=%u", stage, (unsigned)(attempt + 1));
|
|
}
|
|
return ESP_OK;
|
|
}
|
|
|
|
bool can_retry = (attempt + 1 < max_attempts) && domain_policy_is_retryable_error(rc);
|
|
ESP_LOGW(TAG,
|
|
"stage %s failed, rc=0x%x, attempt=%u/%u, retry=%d",
|
|
stage,
|
|
(unsigned)rc,
|
|
(unsigned)(attempt + 1),
|
|
(unsigned)max_attempts,
|
|
can_retry ? 1 : 0);
|
|
|
|
if (!can_retry) {
|
|
return rc;
|
|
}
|
|
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_START_RETRY, 1);
|
|
vTaskDelay(pdMS_TO_TICKS(domain_policy_lifecycle_retry_backoff_ms(attempt)));
|
|
}
|
|
|
|
return ESP_FAIL;
|
|
}
|
|
|
|
esp_err_t controller_lifecycle_start(void) {
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_START_ATTEMPT, 1);
|
|
|
|
if (!lifecycle_lock_take(1000)) {
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_START_FAILED, 1);
|
|
domain_diag_record_error("lifecycle_start", ESP_ERR_TIMEOUT, "lifecycle lock timeout");
|
|
return ESP_ERR_TIMEOUT;
|
|
}
|
|
|
|
if (s_status.state == CONTROLLER_LIFECYCLE_STATE_RUNNING) {
|
|
xSemaphoreGive(s_lock);
|
|
return ESP_OK;
|
|
}
|
|
if (s_status.state == CONTROLLER_LIFECYCLE_STATE_STARTING ||
|
|
s_status.state == CONTROLLER_LIFECYCLE_STATE_STOPPING) {
|
|
xSemaphoreGive(s_lock);
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
s_status.start_attempt++;
|
|
lifecycle_set_state_locked(CONTROLLER_LIFECYCLE_STATE_STARTING, ESP_OK, "bootstrap");
|
|
xSemaphoreGive(s_lock);
|
|
|
|
bool system_started = false;
|
|
bool printer_started = false;
|
|
bool voice_started = false;
|
|
const char *failed_stage = "unknown";
|
|
|
|
failed_stage = "system_runtime";
|
|
esp_err_t rc = lifecycle_run_step_with_retry("system_runtime", lifecycle_start_system_runtime);
|
|
if (rc != ESP_OK) {
|
|
goto start_failed;
|
|
}
|
|
system_started = true;
|
|
|
|
failed_stage = "printer_protocol";
|
|
rc = lifecycle_run_step_with_retry("printer_protocol", lifecycle_start_printer_protocol);
|
|
if (rc != ESP_OK) {
|
|
goto start_failed;
|
|
}
|
|
printer_started = true;
|
|
|
|
failed_stage = "voice_interaction";
|
|
rc = lifecycle_run_step_with_retry("voice_interaction", lifecycle_start_voice);
|
|
if (rc != ESP_OK) {
|
|
goto start_failed;
|
|
}
|
|
voice_started = true;
|
|
|
|
image_generation_schedule_prewarm();
|
|
|
|
if (lifecycle_lock_take(1000)) {
|
|
lifecycle_set_state_locked(CONTROLLER_LIFECYCLE_STATE_RUNNING, ESP_OK, "running");
|
|
xSemaphoreGive(s_lock);
|
|
}
|
|
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_START_SUCCESS, 1);
|
|
ESP_LOGI(TAG, "controller lifecycle started");
|
|
return ESP_OK;
|
|
|
|
start_failed:
|
|
if (voice_started) {
|
|
(void)lifecycle_stop_voice();
|
|
}
|
|
if (printer_started) {
|
|
(void)lifecycle_stop_printer_protocol();
|
|
}
|
|
if (system_started) {
|
|
(void)lifecycle_stop_system_runtime();
|
|
}
|
|
|
|
if (lifecycle_lock_take(1000)) {
|
|
lifecycle_set_state_locked(CONTROLLER_LIFECYCLE_STATE_DEGRADED, rc, failed_stage);
|
|
xSemaphoreGive(s_lock);
|
|
}
|
|
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_START_FAILED, 1);
|
|
domain_diag_record_error("lifecycle_start", rc, failed_stage);
|
|
ESP_LOGE(TAG,
|
|
"controller lifecycle start failed at stage=%s, rc=0x%x",
|
|
failed_stage,
|
|
(unsigned)rc);
|
|
return rc;
|
|
}
|
|
|
|
esp_err_t controller_lifecycle_stop(void) {
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_STOP_ATTEMPT, 1);
|
|
|
|
if (!lifecycle_lock_take(1000)) {
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_STOP_FAILED, 1);
|
|
domain_diag_record_error("lifecycle_stop", ESP_ERR_TIMEOUT, "lifecycle lock timeout");
|
|
return ESP_ERR_TIMEOUT;
|
|
}
|
|
|
|
if (s_status.state == CONTROLLER_LIFECYCLE_STATE_STOPPED) {
|
|
xSemaphoreGive(s_lock);
|
|
return ESP_OK;
|
|
}
|
|
if (s_status.state == CONTROLLER_LIFECYCLE_STATE_STARTING ||
|
|
s_status.state == CONTROLLER_LIFECYCLE_STATE_STOPPING) {
|
|
xSemaphoreGive(s_lock);
|
|
return ESP_ERR_INVALID_STATE;
|
|
}
|
|
|
|
lifecycle_set_state_locked(CONTROLLER_LIFECYCLE_STATE_STOPPING, ESP_OK, "stopping");
|
|
xSemaphoreGive(s_lock);
|
|
|
|
esp_err_t first_err = ESP_OK;
|
|
|
|
esp_err_t rc = lifecycle_stop_voice();
|
|
if (rc != ESP_OK && first_err == ESP_OK) {
|
|
first_err = rc;
|
|
}
|
|
|
|
rc = lifecycle_stop_printer_protocol();
|
|
if (rc != ESP_OK && first_err == ESP_OK) {
|
|
first_err = rc;
|
|
}
|
|
|
|
rc = lifecycle_stop_system_runtime();
|
|
if (rc != ESP_OK && first_err == ESP_OK) {
|
|
first_err = rc;
|
|
}
|
|
|
|
if (lifecycle_lock_take(1000)) {
|
|
if (first_err == ESP_OK) {
|
|
lifecycle_set_state_locked(CONTROLLER_LIFECYCLE_STATE_STOPPED, ESP_OK, "stopped");
|
|
} else {
|
|
lifecycle_set_state_locked(CONTROLLER_LIFECYCLE_STATE_DEGRADED, first_err, "stop_failed");
|
|
}
|
|
xSemaphoreGive(s_lock);
|
|
}
|
|
|
|
if (first_err == ESP_OK) {
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_STOP_SUCCESS, 1);
|
|
ESP_LOGI(TAG, "controller lifecycle stopped");
|
|
} else {
|
|
domain_diag_counter_add(DOMAIN_DIAG_COUNTER_LIFECYCLE_STOP_FAILED, 1);
|
|
domain_diag_record_error("lifecycle_stop", first_err, "controller stop failed");
|
|
ESP_LOGE(TAG, "controller lifecycle stop failed, rc=0x%x", (unsigned)first_err);
|
|
}
|
|
|
|
return first_err;
|
|
}
|
|
|
|
void controller_lifecycle_get_status(controller_lifecycle_status_t *out_status) {
|
|
if (out_status == NULL) {
|
|
return;
|
|
}
|
|
|
|
memset(out_status, 0, sizeof(*out_status));
|
|
|
|
if (!lifecycle_lock_take(200)) {
|
|
return;
|
|
}
|
|
|
|
*out_status = s_status;
|
|
xSemaphoreGive(s_lock);
|
|
}
|