#include "image_generation.h" #include #include #include #include "cJSON.h" #include "esp_crt_bundle.h" #include "esp_heap_caps.h" #include "esp_http_client.h" #include "esp_log.h" #ifndef CONFIG_TQ_Z_IMAGE_API_ENDPOINT #define CONFIG_TQ_Z_IMAGE_API_ENDPOINT "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" #endif #ifndef CONFIG_TQ_Z_IMAGE_MODEL #define CONFIG_TQ_Z_IMAGE_MODEL "z-image-turbo" #endif #ifndef CONFIG_TQ_Z_IMAGE_DEFAULT_SIZE #define CONFIG_TQ_Z_IMAGE_DEFAULT_SIZE "1024*1536" #endif #ifndef CONFIG_TQ_Z_IMAGE_TIMEOUT_MS #define CONFIG_TQ_Z_IMAGE_TIMEOUT_MS 45000 #endif #ifndef CONFIG_TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS #define CONFIG_TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS 15000 #endif #ifndef CONFIG_TQ_Z_IMAGE_MAX_RESPONSE_BYTES #define CONFIG_TQ_Z_IMAGE_MAX_RESPONSE_BYTES 65536 #endif #ifndef CONFIG_TQ_Z_IMAGE_MAX_PNG_BYTES #define CONFIG_TQ_Z_IMAGE_MAX_PNG_BYTES (4 * 1024 * 1024) #endif static const char *TAG = "image_generation"; typedef struct { uint8_t *data; size_t len; size_t cap; size_t max_len; } bytes_buffer_t; static void image_generation_fill_err(char *err, size_t err_len, const char *msg) { if (err != NULL && err_len > 0) { snprintf(err, err_len, "%s", msg != NULL ? msg : "unknown error"); } } static void *malloc_prefer_psram(size_t size) { void *ptr = heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); if (ptr == NULL) { ptr = malloc(size); } return ptr; } static void *realloc_prefer_psram(void *old, size_t size) { if (old == NULL) { return malloc_prefer_psram(size); } return realloc(old, size); } static esp_err_t bytes_buffer_reserve(bytes_buffer_t *buf, size_t need_extra) { if (buf == NULL) { return ESP_ERR_INVALID_ARG; } if (need_extra > SIZE_MAX - buf->len) { return ESP_ERR_INVALID_SIZE; } size_t required = buf->len + need_extra; if (required > buf->max_len) { return ESP_ERR_INVALID_SIZE; } if (required <= buf->cap) { return ESP_OK; } size_t new_cap = buf->cap == 0 ? 1024 : buf->cap; while (new_cap < required) { if (new_cap > (SIZE_MAX / 2)) { new_cap = required; break; } new_cap *= 2; } uint8_t *new_data = (uint8_t *)realloc_prefer_psram(buf->data, new_cap); if (new_data == NULL) { return ESP_ERR_NO_MEM; } buf->data = new_data; buf->cap = new_cap; return ESP_OK; } static esp_err_t bytes_buffer_append(bytes_buffer_t *buf, const void *data, size_t len) { if (buf == NULL || data == NULL) { return ESP_ERR_INVALID_ARG; } if (len == 0) { return ESP_OK; } esp_err_t rc = bytes_buffer_reserve(buf, len); if (rc != ESP_OK) { return rc; } memcpy(buf->data + buf->len, data, len); buf->len += len; return ESP_OK; } static void bytes_buffer_free(bytes_buffer_t *buf) { if (buf == NULL) { return; } free(buf->data); buf->data = NULL; buf->len = 0; buf->cap = 0; } static esp_err_t bytes_buffer_terminate(bytes_buffer_t *buf) { if (buf == NULL) { return ESP_ERR_INVALID_ARG; } esp_err_t rc = bytes_buffer_reserve(buf, 1); if (rc != ESP_OK) { return rc; } buf->data[buf->len] = '\0'; return ESP_OK; } static const char *image_generation_api_key(void) { if (strlen(CONFIG_TQ_Z_IMAGE_API_KEY) > 0) { return CONFIG_TQ_Z_IMAGE_API_KEY; } if (strlen(CONFIG_TQ_VOICE_API_KEY) > 0) { return CONFIG_TQ_VOICE_API_KEY; } return NULL; } static esp_err_t http_read_full_response(esp_http_client_handle_t client, bytes_buffer_t *out_body, char *err, size_t err_len) { if (client == NULL || out_body == NULL) { image_generation_fill_err(err, err_len, "invalid http read args"); return ESP_ERR_INVALID_ARG; } char tmp[1024]; while (true) { int n = esp_http_client_read(client, tmp, sizeof(tmp)); if (n < 0) { image_generation_fill_err(err, err_len, "http read failed"); return ESP_FAIL; } if (n == 0) { if (esp_http_client_is_complete_data_received(client)) { return ESP_OK; } image_generation_fill_err(err, err_len, "http response incomplete"); return ESP_FAIL; } esp_err_t append_rc = bytes_buffer_append(out_body, tmp, (size_t)n); if (append_rc != ESP_OK) { image_generation_fill_err(err, err_len, "http response too large"); return append_rc; } } } static esp_err_t http_request_collect(const esp_http_client_config_t *config, const char *auth_header, const char *body, int *out_status_code, bytes_buffer_t *out_body, char *err, size_t err_len) { if (config == NULL || out_body == NULL) { image_generation_fill_err(err, err_len, "invalid http request args"); return ESP_ERR_INVALID_ARG; } esp_http_client_handle_t client = esp_http_client_init(config); if (client == NULL) { image_generation_fill_err(err, err_len, "http client init failed"); return ESP_FAIL; } if (auth_header != NULL) { esp_http_client_set_header(client, "Authorization", auth_header); } if (body != NULL) { esp_http_client_set_header(client, "Content-Type", "application/json"); } size_t body_len = body != NULL ? strlen(body) : 0; esp_err_t open_rc = esp_http_client_open(client, (int)body_len); if (open_rc != ESP_OK) { esp_http_client_cleanup(client); image_generation_fill_err(err, err_len, "http open failed"); return open_rc; } if (body != NULL && body_len > 0) { int written = esp_http_client_write(client, body, (int)body_len); if (written < 0 || (size_t)written != body_len) { esp_http_client_close(client); esp_http_client_cleanup(client); image_generation_fill_err(err, err_len, "http request write failed"); return ESP_FAIL; } } int64_t content_len = esp_http_client_fetch_headers(client); if (content_len > 0 && (size_t)content_len > out_body->max_len) { esp_http_client_close(client); esp_http_client_cleanup(client); image_generation_fill_err(err, err_len, "http response too large"); return ESP_ERR_INVALID_SIZE; } if (out_status_code != NULL) { *out_status_code = esp_http_client_get_status_code(client); } esp_err_t read_rc = http_read_full_response(client, out_body, err, err_len); esp_http_client_close(client); esp_http_client_cleanup(client); return read_rc; } static const char *json_get_string(cJSON *obj, const char *name) { if (obj == NULL || name == NULL) { return NULL; } cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); if (cJSON_IsString(item) && item->valuestring != NULL) { return item->valuestring; } return NULL; } static bool json_extract_image_url(cJSON *root, char *out_url, size_t out_url_len, char *out_prompt, size_t out_prompt_len) { if (root == NULL || out_url == NULL || out_url_len == 0) { return false; } cJSON *output = cJSON_GetObjectItemCaseSensitive(root, "output"); cJSON *choices = cJSON_GetObjectItemCaseSensitive(output, "choices"); if (!cJSON_IsArray(choices)) { return false; } cJSON *choice = cJSON_GetArrayItem(choices, 0); if (!cJSON_IsObject(choice)) { return false; } cJSON *message = cJSON_GetObjectItemCaseSensitive(choice, "message"); cJSON *content = cJSON_GetObjectItemCaseSensitive(message, "content"); if (!cJSON_IsArray(content)) { return false; } out_url[0] = '\0'; if (out_prompt != NULL && out_prompt_len > 0) { out_prompt[0] = '\0'; } int count = cJSON_GetArraySize(content); for (int i = 0; i < count; ++i) { cJSON *item = cJSON_GetArrayItem(content, i); if (!cJSON_IsObject(item)) { continue; } const char *image = json_get_string(item, "image"); if (image != NULL && out_url[0] == '\0') { strlcpy(out_url, image, out_url_len); } if (out_prompt != NULL && out_prompt_len > 0 && out_prompt[0] == '\0') { const char *text = json_get_string(item, "text"); if (text != NULL) { strlcpy(out_prompt, text, out_prompt_len); } } } return out_url[0] != '\0'; } static void extract_remote_error_message(const char *json_text, int status_code, char *err, size_t err_len) { if (json_text == NULL) { image_generation_fill_err(err, err_len, "upstream request failed"); return; } cJSON *root = cJSON_Parse(json_text); if (root == NULL) { if (status_code > 0) { snprintf(err, err_len, "upstream request failed, http_status=%d", status_code); } else { image_generation_fill_err(err, err_len, "upstream request failed"); } return; } const char *code = json_get_string(root, "code"); const char *message = json_get_string(root, "message"); if (code != NULL && message != NULL) { snprintf(err, err_len, "%s: %s", code, message); } else if (message != NULL) { image_generation_fill_err(err, err_len, message); } else if (status_code > 0) { snprintf(err, err_len, "upstream request failed, http_status=%d", status_code); } else { image_generation_fill_err(err, err_len, "upstream request failed"); } cJSON_Delete(root); } static char *build_generation_request_json(const image_generation_request_t *req) { cJSON *root = cJSON_CreateObject(); if (root == NULL) { return NULL; } cJSON_AddStringToObject(root, "model", CONFIG_TQ_Z_IMAGE_MODEL); cJSON *input = cJSON_AddObjectToObject(root, "input"); cJSON *messages = cJSON_AddArrayToObject(input, "messages"); cJSON *message = cJSON_CreateObject(); cJSON_AddItemToArray(messages, message); cJSON_AddStringToObject(message, "role", "user"); cJSON *content = cJSON_AddArrayToObject(message, "content"); cJSON *content_item = cJSON_CreateObject(); cJSON_AddItemToArray(content, content_item); cJSON_AddStringToObject(content_item, "text", req->prompt); cJSON *parameters = cJSON_AddObjectToObject(root, "parameters"); cJSON_AddBoolToObject(parameters, "prompt_extend", req->prompt_extend); const char *size = (req->size != NULL && req->size[0] != '\0') ? req->size : CONFIG_TQ_Z_IMAGE_DEFAULT_SIZE; cJSON_AddStringToObject(parameters, "size", size); if (req->has_seed) { cJSON_AddNumberToObject(parameters, "seed", (double)req->seed); } char *body = cJSON_PrintUnformatted(root); cJSON_Delete(root); return body; } static esp_err_t image_generation_invoke_model(const image_generation_request_t *req, image_generation_result_t *out_result, char *err, size_t err_len) { char *req_body = build_generation_request_json(req); if (req_body == NULL) { image_generation_fill_err(err, err_len, "build request json failed"); return ESP_ERR_NO_MEM; } const char *api_key = image_generation_api_key(); if (api_key == NULL) { cJSON_free(req_body); image_generation_fill_err(err, err_len, "z-image api key not configured"); return ESP_ERR_INVALID_STATE; } char auth_header[192] = {0}; snprintf(auth_header, sizeof(auth_header), "Bearer %s", api_key); esp_http_client_config_t config = { .url = CONFIG_TQ_Z_IMAGE_API_ENDPOINT, .method = HTTP_METHOD_POST, .timeout_ms = (int)req->generation_timeout_ms, .keep_alive_enable = false, .buffer_size = 2048, .buffer_size_tx = 2048, .crt_bundle_attach = esp_crt_bundle_attach, }; bytes_buffer_t resp = { .data = NULL, .len = 0, .cap = 0, .max_len = CONFIG_TQ_Z_IMAGE_MAX_RESPONSE_BYTES, }; int http_status = 0; esp_err_t rc = http_request_collect(&config, auth_header, req_body, &http_status, &resp, err, err_len); cJSON_free(req_body); if (rc != ESP_OK) { bytes_buffer_free(&resp); return rc; } rc = bytes_buffer_terminate(&resp); if (rc != ESP_OK) { bytes_buffer_free(&resp); image_generation_fill_err(err, err_len, "generation response too large"); return rc; } if (http_status < 200 || http_status >= 300) { extract_remote_error_message((const char *)resp.data, http_status, err, err_len); bytes_buffer_free(&resp); return ESP_FAIL; } cJSON *root = cJSON_Parse((const char *)resp.data); bytes_buffer_free(&resp); if (root == NULL) { image_generation_fill_err(err, err_len, "invalid generation response"); return ESP_FAIL; } const char *request_id = json_get_string(root, "request_id"); if (request_id != NULL) { strlcpy(out_result->request_id, request_id, sizeof(out_result->request_id)); } cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage"); cJSON *usage_w = cJSON_GetObjectItemCaseSensitive(usage, "width"); cJSON *usage_h = cJSON_GetObjectItemCaseSensitive(usage, "height"); if (cJSON_IsNumber(usage_w) && usage_w->valuedouble >= 1 && usage_w->valuedouble <= UINT16_MAX) { out_result->width = (uint16_t)usage_w->valuedouble; } if (cJSON_IsNumber(usage_h) && usage_h->valuedouble >= 1 && usage_h->valuedouble <= UINT16_MAX) { out_result->height = (uint16_t)usage_h->valuedouble; } bool has_url = json_extract_image_url(root, out_result->image_url, sizeof(out_result->image_url), out_result->output_prompt, sizeof(out_result->output_prompt)); if (!has_url) { const char *code = json_get_string(root, "code"); const char *message = json_get_string(root, "message"); if (code != NULL && message != NULL) { snprintf(err, err_len, "%s: %s", code, message); } else { image_generation_fill_err(err, err_len, "generation response has no image url"); } cJSON_Delete(root); return ESP_FAIL; } cJSON_Delete(root); return ESP_OK; } static esp_err_t image_generation_download_png(image_generation_result_t *out_result, uint32_t timeout_ms, char *err, size_t err_len) { esp_http_client_config_t config = { .url = out_result->image_url, .method = HTTP_METHOD_GET, .timeout_ms = (int)timeout_ms, .keep_alive_enable = false, .buffer_size = 2048, .crt_bundle_attach = esp_crt_bundle_attach, }; bytes_buffer_t resp = { .data = NULL, .len = 0, .cap = 0, .max_len = CONFIG_TQ_Z_IMAGE_MAX_PNG_BYTES, }; int http_status = 0; esp_err_t rc = http_request_collect(&config, NULL, NULL, &http_status, &resp, err, err_len); if (rc != ESP_OK) { bytes_buffer_free(&resp); return rc; } if (http_status < 200 || http_status >= 300) { rc = bytes_buffer_terminate(&resp); if (rc != ESP_OK) { bytes_buffer_free(&resp); image_generation_fill_err(err, err_len, "image download failed"); return ESP_FAIL; } extract_remote_error_message((const char *)resp.data, http_status, err, err_len); bytes_buffer_free(&resp); return ESP_FAIL; } static const uint8_t png_header[8] = { 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, }; if (resp.len < sizeof(png_header) || memcmp(resp.data, png_header, sizeof(png_header)) != 0) { bytes_buffer_free(&resp); image_generation_fill_err(err, err_len, "downloaded image is not png"); return ESP_FAIL; } out_result->png = resp.data; out_result->png_len = resp.len; return ESP_OK; } void image_generation_result_reset(image_generation_result_t *result) { if (result == NULL) { return; } memset(result, 0, sizeof(*result)); } void image_generation_result_free(image_generation_result_t *result) { if (result == NULL) { return; } free(result->png); result->png = NULL; result->png_len = 0; } esp_err_t image_generation_generate_png(const image_generation_request_t *req, image_generation_result_t *out_result, char *err, size_t err_len) { if (req == NULL || out_result == NULL || req->prompt == NULL || req->prompt[0] == '\0') { image_generation_fill_err(err, err_len, "prompt is required"); return ESP_ERR_INVALID_ARG; } image_generation_result_reset(out_result); image_generation_request_t actual = *req; if (actual.generation_timeout_ms == 0) { actual.generation_timeout_ms = CONFIG_TQ_Z_IMAGE_TIMEOUT_MS; } if (actual.download_timeout_ms == 0) { actual.download_timeout_ms = CONFIG_TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS; } ESP_LOGI(TAG, "invoke z-image, prompt_len=%u", (unsigned)strlen(actual.prompt)); esp_err_t gen_rc = image_generation_invoke_model(&actual, out_result, err, err_len); if (gen_rc != ESP_OK) { image_generation_result_free(out_result); return gen_rc; } ESP_LOGI(TAG, "download image from model output"); esp_err_t dl_rc = image_generation_download_png(out_result, actual.download_timeout_ms, err, err_len); if (dl_rc != ESP_OK) { image_generation_result_free(out_result); return dl_rc; } return ESP_OK; }