From 92ac92f514a7e95cb3ee699cb7a17b1e0183db0f Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 24 Feb 2026 23:16:33 +0800 Subject: [PATCH] z-image --- README.md | 38 +- main/CMakeLists.txt | 2 + main/Kconfig.projbuild | 44 +- main/control_plane/src/rest_server.c | 95 ++-- main/control_plane/src/rest_server_print.c | 355 ++++++++++++- main/domain/include/image_generation.h | 35 ++ main/domain/src/image_generation.c | 591 +++++++++++++++++++++ partitions.csv | 4 + sdkconfig.defaults | 4 +- 9 files changed, 1090 insertions(+), 78 deletions(-) create mode 100644 main/domain/include/image_generation.h create mode 100644 main/domain/src/image_generation.c create mode 100644 partitions.csv diff --git a/README.md b/README.md index 06c91c0..c9f9060 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ESP32-S3 works as a Wi-Fi REST controller and replaces Android App logic: ## Features - Wi-Fi STA-only mode (configured SSID/password; no SoftAP fallback) -- Built-in web UI at `/` for connect/status, text print, and image upload print +- Built-in web UI at `/` for connect/status, text print, and Z-Image prompt print - BLE central client auto-scan/connect to printer name - Async print queue with jobs (`queued/running/success/failed/canceled`) - Printer precheck (paper / battery / temperature) @@ -33,7 +33,7 @@ ESP32-S3 works as a Wi-Fi REST controller and replaces Android App logic: - `GET /v1/printer/status` - Print: - `POST /v1/print/raster` - - `POST /v1/print/image` (direct PNG binary upload, no base64) + - `POST /v1/print/image` (JSON prompt -> generate image -> print, also compatible with direct PNG binary upload) - `POST /v1/print/qr` - `POST /v1/print/text` - `POST /v1/print/receipt` @@ -81,6 +81,13 @@ In `menuconfig -> TQ Controller Config`: - `TQ_WIFI_PASSWORD` - `TQ_HTTP_PORT` - `TQ_API_KEY` (optional) +- Z-Image HTTP auth and model fields: + - `TQ_Z_IMAGE_API_KEY` (optional, empty means fallback to `TQ_VOICE_API_KEY`) + - `TQ_Z_IMAGE_API_ENDPOINT` + - `TQ_Z_IMAGE_MODEL` + - `TQ_Z_IMAGE_DEFAULT_SIZE` + - `TQ_Z_IMAGE_TIMEOUT_MS` + - `TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS` - Voice WebSocket auth and app fields: - `TQ_VOICE_API_KEY` - `TQ_VOICE_WORKSPACE_ID` @@ -121,7 +128,7 @@ The page provides: - health/status check - connect/disconnect printer - submit a simple text print job -- upload a PNG image (binary payload) and let ESP32-S3 decode/threshold/scale/raster conversion +- input an image prompt, then let ESP32-S3 call DashScope Z-Image, download PNG, decode/threshold/scale/raster conversion, and print - view jobs list If API key is enabled, input it in the page before invoking actions. @@ -163,7 +170,25 @@ curl -X POST http:///v1/print/raster \ }' ``` -### Image print (direct PNG upload, no base64) +### Image print (generate with prompt, then print) +```bash +curl -X POST http:///v1/print/image \ + -H 'Content-Type: application/json' \ + -d '{ + "prompt":"一只坐在窗边的橘猫，午后阳光，胶片质感，写实风格。", + "size":"1120*1440", + "prompt_extend":false, + "threshold":160, + "max_height":2200, + "scale_to_width":true, + "invert":false, + "density":"中等", + "timeout_ms":45000, + "fetch_timeout_ms":15000 + }' +``` + +### Image print (direct PNG upload, backward-compatible) ```bash curl -X POST 'http:///v1/print/image?scale_to_width=1&threshold=160&invert=0&max_height=2200&density=medium' \ -H 'Content-Type: image/png' \ @@ -172,7 +197,7 @@ curl -X POST 'http:///v1/print/image?scale_to_width=1&threshold=160&inve Request constraints: - Request body limit: 4 MB -- Suggested image size: <= 3 MB when using built-in Web UI uploader +- Suggested image size: <= 3 MB when using direct PNG upload compatibility mode ### QR print ```bash @@ -305,8 +330,9 @@ curl -X POST http:///v1/voice/session/stop - BLE side is central/client role, not printer peripheral role. - `base64_msb_1bpp` uses Android-compatible bit order (MSB first). -- `/v1/print/image` accepts raw `image/png` bytes directly (no base64 wrapper). +- `/v1/print/image` supports two modes: JSON prompt generation (DashScope Z-Image) and raw `image/png` upload (no base64 wrapper). - Large buffers for image upload/decode prefer PSRAM first, then fallback to internal RAM. +- Partition table uses `partitions.csv` with a 4MB `factory` app partition on 16MB flash modules. - `/v1/print/text` and `/v1/print/receipt` now support UTF-8 Chinese via embedded 16x16 GB2312 glyphs. - Characters outside embedded glyph set are rendered as square fallback boxes. - QR encoding uses embedded `qrcodegen` (Project Nayuki C implementation). diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 9aa2231..a044e17 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -12,6 +12,7 @@ idf_component_register( "domain/src/printer_protocol_commands.c" "domain/src/raster_tools.c" "domain/src/raster_tools_image_qr.c" + "domain/src/image_generation.c" "domain/src/system_runtime.c" "domain/src/voice_interaction.c" "platform/src/platform_bootstrap.c" @@ -36,6 +37,7 @@ idf_component_register( esp_netif esp_event esp_http_server + esp_http_client nvs_flash json mbedtls diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 3b50922..a91d39b 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -20,9 +20,45 @@ config TQ_API_KEY string "REST API Key (optional, empty means disabled)" default "" +config TQ_Z_IMAGE_API_KEY + string "Z-Image API Key for DashScope HTTP (optional, fallback to Voice API key)" + default "sk-7a50eca6856d4afb968ac3bf512f6d1b" + +config TQ_Z_IMAGE_API_ENDPOINT + string "Z-Image HTTP endpoint" + default "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" + +config TQ_Z_IMAGE_MODEL + string "Z-Image model" + default "z-image-turbo" + +config TQ_Z_IMAGE_DEFAULT_SIZE + string "Z-Image default output size" + default "1024*1536" + +config TQ_Z_IMAGE_TIMEOUT_MS + int "Z-Image generation timeout (ms)" + range 5000 180000 + default 45000 + +config TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS + int "Z-Image image download timeout (ms)" + range 2000 120000 + default 15000 + +config TQ_Z_IMAGE_MAX_RESPONSE_BYTES + int "Z-Image max response body bytes" + range 1024 262144 + default 65536 + +config TQ_Z_IMAGE_MAX_PNG_BYTES + int "Z-Image max downloaded PNG bytes" + range 65536 8388608 + default 4194304 + config TQ_VOICE_API_KEY string "Voice API Key for DashScope WebSocket (optional)" - default "" + default "sk-7a50eca6856d4afb968ac3bf512f6d1b" config TQ_VOICE_WS_URI string "Voice WebSocket URI" @@ -30,11 +66,11 @@ config TQ_VOICE_WS_URI config TQ_VOICE_WORKSPACE_ID string "Voice workspace_id" - default "" + default "llm-igyrxzdvbwja5gyt" config TQ_VOICE_APP_ID string "Voice app_id" - default "" + default "fe83b756f6634dcc842c6e7f356c0cf8" config TQ_VOICE_USER_ID string "Voice client user_id" @@ -46,7 +82,7 @@ config TQ_VOICE_DEVICE_UUID config TQ_VOICE_TTS_VOICE string "Voice TTS voice (optional)" - default "" + default "longling_v3" config TQ_VOICE_SAMPLE_RATE int "Voice sample rate" diff --git a/main/control_plane/src/rest_server.c b/main/control_plane/src/rest_server.c index c2219de..a6d4021 100644 --- a/main/control_plane/src/rest_server.c +++ b/main/control_plane/src/rest_server.c @@ -63,8 +63,18 @@ static const char *s_web_index = " \n" " \n" "\n" - " \n" - " \n" + " \n" + " \n" + "
\n" + " \n" + " \n" + " \n" + "
\n" "
\n" " \n" " \n" @@ -84,9 +94,13 @@ static const char *s_web_index = " \n" " Invert\n" " \n" - " \n" + " \n" + " \n" "
\n" - "
Direct PNG upload: browser sends binary PNG; ESP32-S3 decodes/scales/thresholds then prints.
\n" + "
Submit prompt only: ESP32-S3 calls DashScope Z-Image, downloads PNG, rasterizes and prints.
\n" "\n" "
Tip: connect printer first, then submit print jobs.
\n" " 
Ready.
\n" @@ -97,12 +111,16 @@ static const char *s_web_index = " const apiKeyEl = document.getElementById('apiKey');\n" " const printerNameEl = document.getElementById('printerName');\n" " const printTextEl = document.getElementById('printText');\n" - " const imageFileEl = document.getElementById('imageFile');\n" + " const imagePromptEl = document.getElementById('imagePrompt');\n" + " const imgSizeEl = document.getElementById('imgSize');\n" + " const imgTimeoutEl = document.getElementById('imgTimeout');\n" + " const imgFetchTimeoutEl = document.getElementById('imgFetchTimeout');\n" " const imgThresholdEl = document.getElementById('imgThreshold');\n" " const imgMaxHeightEl = document.getElementById('imgMaxHeight');\n" " const imgDensityEl = document.getElementById('imgDensity');\n" " const imgScaleToWidthEl = document.getElementById('imgScaleToWidth');\n" " const imgInvertEl = document.getElementById('imgInvert');\n" + " const imgPromptExtendEl = document.getElementById('imgPromptExtend');\n" "\n" " function log(obj) {\n" " if (typeof obj === 'string') {\n" @@ -128,41 +146,23 @@ static const char *s_web_index = " return n;\n" " }\n" "\n" - " function buildImageUploadRequest(file) {\n" - " if (!file) {\n" - " throw new Error('please select a png image first');\n" + " function buildImagePromptRequest() {\n" + " const prompt = imagePromptEl.value.trim();\n" + " if (!prompt) {\n" + " throw new Error('please input image prompt');\n" " }\n" - " const name = (file.name || '').toLowerCase();\n" - " const type = (file.type || '').toLowerCase();\n" - " const isPng = type.includes('png') || name.endsWith('.png');\n" - " if (!isPng) {\n" - " throw new Error('only .png is supported in direct upload mode');\n" - " }\n" - " if (file.size <= 0) {\n" - " throw new Error('empty image file');\n" - " }\n" - " if (file.size > 3 * 1024 * 1024) {\n" - " throw new Error('png too large (>3MB), backend limit is 4MB');\n" - " }\n" - "\n" - " const densityMap = {\n" - " '较淡': 'light',\n" - " '中等': 'medium',\n" - " '较浓': 'dark',\n" - " '最深': 'max'\n" - " };\n" - "\n" - " const q = new URLSearchParams();\n" - " q.set('scale_to_width', imgScaleToWidthEl.checked ? '1' : '0');\n" - " q.set('threshold', String(toInt(imgThresholdEl.value, 160, 0, 255)));\n" - " q.set('invert', imgInvertEl.checked ? '1' : '0');\n" - " q.set('max_height', String(toInt(imgMaxHeightEl.value, 2200, 64, 3000)));\n" - " q.set('density', densityMap[imgDensityEl.value] || 'medium');\n" "\n" " return {\n" - " path: '/v1/print/image?' + q.toString(),\n" - " body: file,\n" - " contentType: file.type || 'image/png'\n" + " prompt,\n" + " size: imgSizeEl.value || '1120*1440',\n" + " prompt_extend: imgPromptExtendEl.checked,\n" + " threshold: toInt(imgThresholdEl.value, 160, 0, 255),\n" + " max_height: toInt(imgMaxHeightEl.value, 2200, 64, 3000),\n" + " density: imgDensityEl.value || '中等',\n" + " scale_to_width: imgScaleToWidthEl.checked,\n" + " invert: imgInvertEl.checked,\n" + " timeout_ms: toInt(imgTimeoutEl.value, 45000, 5000, 180000),\n" + " fetch_timeout_ms: toInt(imgFetchTimeoutEl.value, 15000, 2000, 120000)\n" " };\n" " }\n" "\n" @@ -179,19 +179,6 @@ static const char *s_web_index = " return data;\n" " }\n" "\n" - " async function callBinaryApi(path, body, contentType) {\n" - " const h = headers(false);\n" - " h['Content-Type'] = contentType || 'application/octet-stream';\n" - " const r = await fetch(path, { method: 'POST', headers: h, body });\n" - " const text = await r.text();\n" - " let data = text;\n" - " try { data = JSON.parse(text); } catch (_) {}\n" - " if (!r.ok) {\n" - " throw { status: r.status, data };\n" - " }\n" - " return data;\n" - " }\n" - "\n" " async function run(fn) {\n" " try {\n" " const data = await fn();\n" @@ -226,11 +213,8 @@ static const char *s_web_index = " }));\n" "\n" " document.getElementById('btnPrintImage').onclick = () => run(async () => {\n" - " const file = imageFileEl.files && imageFileEl.files[0] ? imageFileEl.files[0] : null;\n" - " const req = buildImageUploadRequest(file);\n" - " const result = await callBinaryApi(req.path, req.body, req.contentType);\n" - " result.client_note = 'uploaded png bytes: ' + file.size;\n" - " return result;\n" + " const req = buildImagePromptRequest();\n" + " return callApi('/v1/print/image', 'POST', req);\n" " });\n" " \n" "\n" @@ -252,6 +236,7 @@ esp_err_t rest_server_start(void) { config.server_port = CONFIG_TQ_HTTP_PORT; config.uri_match_fn = httpd_uri_match_wildcard; config.max_uri_handlers = 32; + config.stack_size = 10240; esp_err_t err = httpd_start(&s_server, &config); if (err != ESP_OK) { diff --git a/main/control_plane/src/rest_server_print.c b/main/control_plane/src/rest_server_print.c index 0a2838f..83e6678 100644 --- a/main/control_plane/src/rest_server_print.c +++ b/main/control_plane/src/rest_server_print.c @@ -5,6 +5,7 @@ #include #include +#include "image_generation.h" #include "printer_protocol.h" #include "raster_tools.h" static esp_err_t submit_raster_job_and_reply(httpd_req_t *req, @@ -154,6 +155,194 @@ static void parse_image_upload_options(httpd_req_t *req, free(query); } +typedef struct { + bool scale_to_width; + uint8_t threshold; + bool invert; + uint16_t max_height; + const char *density; +} image_print_options_t; + +typedef struct { + char prompt[801]; + char size[24]; + bool has_size; + bool prompt_extend; + bool has_seed; + uint32_t seed; + uint32_t timeout_ms; + uint32_t fetch_timeout_ms; +} image_generate_options_t; + +static void image_print_options_set_defaults(image_print_options_t *out) { + if (out == NULL) { + return; + } + out->scale_to_width = true; + out->threshold = 160; + out->invert = false; + out->max_height = 2200; + out->density = "中等"; +} + +static void parse_image_print_options_from_json(cJSON *json, image_print_options_t *out) { + if (json == NULL || out == NULL) { + return; + } + + cJSON *jscale = cJSON_GetObjectItemCaseSensitive(json, "scale_to_width"); + cJSON *jthreshold = cJSON_GetObjectItemCaseSensitive(json, "threshold"); + cJSON *jinvert = cJSON_GetObjectItemCaseSensitive(json, "invert"); + cJSON *jmaxh = cJSON_GetObjectItemCaseSensitive(json, "max_height"); + cJSON *jdensity = cJSON_GetObjectItemCaseSensitive(json, "density"); + + out->scale_to_width = rest_server_json_bool_with_default(jscale, out->scale_to_width); + out->invert = rest_server_json_bool_with_default(jinvert, out->invert); + if (cJSON_IsNumber(jthreshold) && jthreshold->valuedouble >= 0 && jthreshold->valuedouble <= 255) { + out->threshold = (uint8_t)jthreshold->valuedouble; + } + if (cJSON_IsNumber(jmaxh) && jmaxh->valuedouble >= 64 && jmaxh->valuedouble <= 3000) { + out->max_height = (uint16_t)jmaxh->valuedouble; + } + if (cJSON_IsString(jdensity) && jdensity->valuestring != NULL) { + out->density = map_density_text(jdensity->valuestring); + } +} + +static bool parse_image_size_text(const char *text, uint32_t *out_w, uint32_t *out_h) { + if (text == NULL || out_w == NULL || out_h == NULL) { + return false; + } + + const char *sep = strchr(text, '*'); + if (sep == NULL || sep == text || *(sep + 1) == '\0') { + return false; + } + + char wbuf[12] = {0}; + char hbuf[12] = {0}; + size_t wlen = (size_t)(sep - text); + size_t hlen = strlen(sep + 1); + if (wlen == 0 || hlen == 0 || wlen >= sizeof(wbuf) || hlen >= sizeof(hbuf)) { + return false; + } + + memcpy(wbuf, text, wlen); + memcpy(hbuf, sep + 1, hlen); + wbuf[wlen] = '\0'; + hbuf[hlen] = '\0'; + + char *wend = NULL; + char *hend = NULL; + long w = strtol(wbuf, &wend, 10); + long h = strtol(hbuf, &hend, 10); + if (wend == wbuf || hend == hbuf || *wend != '\0' || *hend != '\0') { + return false; + } + if (w <= 0 || h <= 0) { + return false; + } + + *out_w = (uint32_t)w; + *out_h = (uint32_t)h; + return true; +} + +static bool parse_image_generation_options(cJSON *json, + image_generate_options_t *out, + char *err, + size_t err_len) { + if (json == NULL || out == NULL) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "invalid args"); + } + return false; + } + + memset(out, 0, sizeof(*out)); + out->timeout_ms = CONFIG_TQ_Z_IMAGE_TIMEOUT_MS; + out->fetch_timeout_ms = CONFIG_TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS; + + cJSON *jprompt = cJSON_GetObjectItemCaseSensitive(json, "prompt"); + if (!cJSON_IsString(jprompt) || jprompt->valuestring == NULL || jprompt->valuestring[0] == '\0') { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "prompt is required"); + } + return false; + } + + size_t prompt_len = strlen(jprompt->valuestring); + if (prompt_len > 800) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "prompt too long, max 800 chars"); + } + return false; + } + strlcpy(out->prompt, jprompt->valuestring, sizeof(out->prompt)); + + cJSON *jsize = cJSON_GetObjectItemCaseSensitive(json, "size"); + if (cJSON_IsString(jsize) && jsize->valuestring != NULL && jsize->valuestring[0] != '\0') { + uint32_t w = 0; + uint32_t h = 0; + if (!parse_image_size_text(jsize->valuestring, &w, &h)) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "size must be like 1120*1440"); + } + return false; + } + + uint64_t pixels = (uint64_t)w * (uint64_t)h; + if (pixels < (512ULL * 512ULL) || pixels > (2048ULL * 2048ULL)) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "size pixels must be in [512*512, 2048*2048]"); + } + return false; + } + + out->has_size = true; + strlcpy(out->size, jsize->valuestring, sizeof(out->size)); + } + + cJSON *jextend = cJSON_GetObjectItemCaseSensitive(json, "prompt_extend"); + out->prompt_extend = rest_server_json_bool_with_default(jextend, false); + + cJSON *jseed = cJSON_GetObjectItemCaseSensitive(json, "seed"); + if (cJSON_IsNumber(jseed)) { + if (jseed->valuedouble < 0 || jseed->valuedouble > 2147483647.0) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "seed must be in [0, 2147483647]"); + } + return false; + } + out->has_seed = true; + out->seed = (uint32_t)jseed->valuedouble; + } + + cJSON *jtimeout = cJSON_GetObjectItemCaseSensitive(json, "timeout_ms"); + if (cJSON_IsNumber(jtimeout)) { + if (jtimeout->valuedouble < 5000 || jtimeout->valuedouble > 180000) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "timeout_ms must be 5000..180000"); + } + return false; + } + out->timeout_ms = (uint32_t)jtimeout->valuedouble; + } + + cJSON *jfetch_timeout = cJSON_GetObjectItemCaseSensitive(json, "fetch_timeout_ms"); + if (cJSON_IsNumber(jfetch_timeout)) { + if (jfetch_timeout->valuedouble < 2000 || jfetch_timeout->valuedouble > 120000) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "fetch_timeout_ms must be 2000..120000"); + } + return false; + } + out->fetch_timeout_ms = (uint32_t)jfetch_timeout->valuedouble; + } + + return true; +} + static esp_err_t decode_image_json_to_raster(cJSON *json, uint16_t *out_width, uint16_t *out_height, @@ -341,21 +530,26 @@ esp_err_t rest_server_print_raster_post(httpd_req_t *req) { return submit_rc; } -esp_err_t rest_server_print_image_post(httpd_req_t *req) { - if (!rest_server_auth_ok(req)) { - return rest_server_send_error(req, "401 Unauthorized", "unauthorized"); +static esp_err_t rest_server_send_image_generation_error(httpd_req_t *req, + esp_err_t rc, + const char *msg) { + const char *message = (msg != NULL && msg[0] != '\0') ? msg : "image generation failed"; + if (rc == ESP_ERR_INVALID_ARG) { + return rest_server_send_error(req, "400 Bad Request", message); } - - char content_type[96] = {0}; - if (httpd_req_get_hdr_value_str(req, "Content-Type", content_type, sizeof(content_type)) == ESP_OK) { - if (strstr(content_type, "application/json") != NULL) { - return rest_server_send_error(req, "400 Bad Request", "use binary png body, not json"); - } - if (strstr(content_type, "image/png") == NULL) { - return rest_server_send_error(req, "415 Unsupported Media Type", "Content-Type must be image/png"); - } + if (rc == ESP_ERR_INVALID_STATE) { + return rest_server_send_error(req, "503 Service Unavailable", message); } + if (rc == ESP_ERR_TIMEOUT) { + return rest_server_send_error(req, "504 Gateway Timeout", message); + } + if (rc == ESP_ERR_NO_MEM) { + return rest_server_send_error(req, "500 Internal Server Error", message); + } + return rest_server_send_error(req, "502 Bad Gateway", message); +} +static esp_err_t rest_server_print_image_binary(httpd_req_t *req) { bool scale_to_width = true; uint8_t threshold = 160; bool invert = false; @@ -409,6 +603,143 @@ esp_err_t rest_server_print_image_post(httpd_req_t *req) { return submit_rc; } +static esp_err_t rest_server_print_image_generate(httpd_req_t *req) { + char *body = NULL; + cJSON *json = NULL; + image_generation_result_t *gen_result = NULL; + uint8_t *raster = NULL; + char request_id[80] = {0}; + esp_err_t ret = ESP_FAIL; + + esp_err_t body_err = rest_server_read_body(req, &body); + if (body_err != ESP_OK) { + return rest_server_send_error(req, "400 Bad Request", "invalid request body"); + } + + json = cJSON_Parse(body); + free(body); + body = NULL; + if (json == NULL) { + return rest_server_send_error(req, "400 Bad Request", "invalid json"); + } + + image_generate_options_t gen_opt = {0}; + char parse_err[128] = {0}; + if (!parse_image_generation_options(json, &gen_opt, parse_err, sizeof(parse_err))) { + ret = rest_server_send_error(req, + "400 Bad Request", + parse_err[0] != '\0' ? parse_err : "invalid image options"); + goto cleanup; + } + + image_print_options_t print_opt = {0}; + image_print_options_set_defaults(&print_opt); + parse_image_print_options_from_json(json, &print_opt); + cJSON_Delete(json); + json = NULL; + + image_generation_request_t gen_req = { + .prompt = gen_opt.prompt, + .size = gen_opt.has_size ? gen_opt.size : NULL, + .prompt_extend = gen_opt.prompt_extend, + .has_seed = gen_opt.has_seed, + .seed = gen_opt.seed, + .generation_timeout_ms = gen_opt.timeout_ms, + .download_timeout_ms = gen_opt.fetch_timeout_ms, + }; + + gen_result = (image_generation_result_t *)calloc(1, sizeof(*gen_result)); + if (gen_result == NULL) { + ret = rest_server_send_error(req, "500 Internal Server Error", "no memory"); + goto cleanup; + } + image_generation_result_reset(gen_result); + + char model_err[160] = {0}; + esp_err_t gen_rc = image_generation_generate_png(&gen_req, + gen_result, + model_err, + sizeof(model_err)); + if (gen_rc != ESP_OK) { + ret = rest_server_send_image_generation_error(req, gen_rc, model_err); + goto cleanup; + } + + if (gen_result->request_id[0] != '\0') { + strlcpy(request_id, gen_result->request_id, sizeof(request_id)); + } + + uint16_t width = 0; + uint16_t height = 0; + size_t raster_len = 0; + char decode_err[128] = {0}; + esp_err_t decode_rc = raster_tools_convert_png_to_raster_384(gen_result->png, + gen_result->png_len, + print_opt.scale_to_width, + print_opt.threshold, + print_opt.invert, + print_opt.max_height, + &width, + &height, + &raster, + &raster_len, + decode_err, + sizeof(decode_err)); + if (decode_rc != ESP_OK) { + ret = rest_server_send_error(req, + "400 Bad Request", + decode_err[0] != '\0' ? decode_err : "decode generated image failed"); + goto cleanup; + } + + char warning[160] = {0}; + if (request_id[0] != '\0') { + snprintf(warning, sizeof(warning), "z-image request_id=%s", request_id); + } + + ret = submit_raster_job_and_reply(req, + raster, + raster_len, + width, + height, + print_opt.density, + warning); + +cleanup: + if (body != NULL) { + free(body); + } + if (json != NULL) { + cJSON_Delete(json); + } + if (raster != NULL) { + free(raster); + } + if (gen_result != NULL) { + image_generation_result_free(gen_result); + free(gen_result); + } + return ret; +} + +esp_err_t rest_server_print_image_post(httpd_req_t *req) { + if (!rest_server_auth_ok(req)) { + return rest_server_send_error(req, "401 Unauthorized", "unauthorized"); + } + + char content_type[96] = {0}; + if (httpd_req_get_hdr_value_str(req, "Content-Type", content_type, sizeof(content_type)) == ESP_OK) { + if (strstr(content_type, "application/json") != NULL) { + return rest_server_print_image_generate(req); + } + if (strstr(content_type, "image/png") == NULL) { + return rest_server_send_error(req, "415 Unsupported Media Type", "Content-Type must be application/json or image/png"); + } + } + + return rest_server_print_image_binary(req); +} + esp_err_t rest_server_print_qr_post(httpd_req_t *req) { if (!rest_server_auth_ok(req)) { return rest_server_send_error(req, "401 Unauthorized", "unauthorized"); diff --git a/main/domain/include/image_generation.h b/main/domain/include/image_generation.h new file mode 100644 index 0000000..6a58c11 --- /dev/null +++ b/main/domain/include/image_generation.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include "esp_err.h" + +typedef struct { + const char *prompt; + const char *size; + bool prompt_extend; + bool has_seed; + uint32_t seed; + uint32_t generation_timeout_ms; + uint32_t download_timeout_ms; +} image_generation_request_t; + +typedef struct { + uint8_t *png; + size_t png_len; + char image_url[1024]; + char output_prompt[256]; + char request_id[80]; + uint16_t width; + uint16_t height; +} image_generation_result_t; + +void image_generation_result_reset(image_generation_result_t *result); +void image_generation_result_free(image_generation_result_t *result); + +esp_err_t image_generation_generate_png(const image_generation_request_t *req, + image_generation_result_t *out_result, + char *err, + size_t err_len); diff --git a/main/domain/src/image_generation.c b/main/domain/src/image_generation.c new file mode 100644 index 0000000..09ee7f9 --- /dev/null +++ b/main/domain/src/image_generation.c @@ -0,0 +1,591 @@ +#include "image_generation.h" + +#include +#include +#include + +#include "cJSON.h" +#include "esp_crt_bundle.h" +#include "esp_heap_caps.h" +#include "esp_http_client.h" +#include "esp_log.h" + +#ifndef CONFIG_TQ_Z_IMAGE_API_ENDPOINT +#define CONFIG_TQ_Z_IMAGE_API_ENDPOINT "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +#endif + +#ifndef CONFIG_TQ_Z_IMAGE_MODEL +#define CONFIG_TQ_Z_IMAGE_MODEL "z-image-turbo" +#endif + +#ifndef CONFIG_TQ_Z_IMAGE_DEFAULT_SIZE +#define CONFIG_TQ_Z_IMAGE_DEFAULT_SIZE "1024*1536" +#endif + +#ifndef CONFIG_TQ_Z_IMAGE_TIMEOUT_MS +#define CONFIG_TQ_Z_IMAGE_TIMEOUT_MS 45000 +#endif + +#ifndef CONFIG_TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS +#define CONFIG_TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS 15000 +#endif + +#ifndef CONFIG_TQ_Z_IMAGE_MAX_RESPONSE_BYTES +#define CONFIG_TQ_Z_IMAGE_MAX_RESPONSE_BYTES 65536 +#endif + +#ifndef CONFIG_TQ_Z_IMAGE_MAX_PNG_BYTES +#define CONFIG_TQ_Z_IMAGE_MAX_PNG_BYTES (4 * 1024 * 1024) +#endif + +static const char *TAG = "image_generation"; + +typedef struct { + uint8_t *data; + size_t len; + size_t cap; + size_t max_len; +} bytes_buffer_t; + +static void image_generation_fill_err(char *err, size_t err_len, const char *msg) { + if (err != NULL && err_len > 0) { + snprintf(err, err_len, "%s", msg != NULL ? msg : "unknown error"); + } +} + +static void *malloc_prefer_psram(size_t size) { + void *ptr = heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (ptr == NULL) { + ptr = malloc(size); + } + return ptr; +} + +static void *realloc_prefer_psram(void *old, size_t size) { + if (old == NULL) { + return malloc_prefer_psram(size); + } + return realloc(old, size); +} + +static esp_err_t bytes_buffer_reserve(bytes_buffer_t *buf, size_t need_extra) { + if (buf == NULL) { + return ESP_ERR_INVALID_ARG; + } + if (need_extra > SIZE_MAX - buf->len) { + return ESP_ERR_INVALID_SIZE; + } + size_t required = buf->len + need_extra; + if (required > buf->max_len) { + return ESP_ERR_INVALID_SIZE; + } + if (required <= buf->cap) { + return ESP_OK; + } + + size_t new_cap = buf->cap == 0 ? 1024 : buf->cap; + while (new_cap < required) { + if (new_cap > (SIZE_MAX / 2)) { + new_cap = required; + break; + } + new_cap *= 2; + } + + uint8_t *new_data = (uint8_t *)realloc_prefer_psram(buf->data, new_cap); + if (new_data == NULL) { + return ESP_ERR_NO_MEM; + } + buf->data = new_data; + buf->cap = new_cap; + return ESP_OK; +} + +static esp_err_t bytes_buffer_append(bytes_buffer_t *buf, const void *data, size_t len) { + if (buf == NULL || data == NULL) { + return ESP_ERR_INVALID_ARG; + } + if (len == 0) { + return ESP_OK; + } + esp_err_t rc = bytes_buffer_reserve(buf, len); + if (rc != ESP_OK) { + return rc; + } + memcpy(buf->data + buf->len, data, len); + buf->len += len; + return ESP_OK; +} + +static void bytes_buffer_free(bytes_buffer_t *buf) { + if (buf == NULL) { + return; + } + free(buf->data); + buf->data = NULL; + buf->len = 0; + buf->cap = 0; +} + +static esp_err_t bytes_buffer_terminate(bytes_buffer_t *buf) { + if (buf == NULL) { + return ESP_ERR_INVALID_ARG; + } + esp_err_t rc = bytes_buffer_reserve(buf, 1); + if (rc != ESP_OK) { + return rc; + } + buf->data[buf->len] = '\0'; + return ESP_OK; +} + +static const char *image_generation_api_key(void) { + if (strlen(CONFIG_TQ_Z_IMAGE_API_KEY) > 0) { + return CONFIG_TQ_Z_IMAGE_API_KEY; + } + if (strlen(CONFIG_TQ_VOICE_API_KEY) > 0) { + return CONFIG_TQ_VOICE_API_KEY; + } + return NULL; +} + +static esp_err_t http_read_full_response(esp_http_client_handle_t client, + bytes_buffer_t *out_body, + char *err, + size_t err_len) { + if (client == NULL || out_body == NULL) { + image_generation_fill_err(err, err_len, "invalid http read args"); + return ESP_ERR_INVALID_ARG; + } + + char tmp[1024]; + while (true) { + int n = esp_http_client_read(client, tmp, sizeof(tmp)); + if (n < 0) { + image_generation_fill_err(err, err_len, "http read failed"); + return ESP_FAIL; + } + if (n == 0) { + if (esp_http_client_is_complete_data_received(client)) { + return ESP_OK; + } + image_generation_fill_err(err, err_len, "http response incomplete"); + return ESP_FAIL; + } + + esp_err_t append_rc = bytes_buffer_append(out_body, tmp, (size_t)n); + if (append_rc != ESP_OK) { + image_generation_fill_err(err, err_len, "http response too large"); + return append_rc; + } + } +} + +static esp_err_t http_request_collect(const esp_http_client_config_t *config, + const char *auth_header, + const char *body, + int *out_status_code, + bytes_buffer_t *out_body, + char *err, + size_t err_len) { + if (config == NULL || out_body == NULL) { + image_generation_fill_err(err, err_len, "invalid http request args"); + return ESP_ERR_INVALID_ARG; + } + + esp_http_client_handle_t client = esp_http_client_init(config); + if (client == NULL) { + image_generation_fill_err(err, err_len, "http client init failed"); + return ESP_FAIL; + } + + if (auth_header != NULL) { + esp_http_client_set_header(client, "Authorization", auth_header); + } + if (body != NULL) { + esp_http_client_set_header(client, "Content-Type", "application/json"); + } + + size_t body_len = body != NULL ? strlen(body) : 0; + esp_err_t open_rc = esp_http_client_open(client, (int)body_len); + if (open_rc != ESP_OK) { + esp_http_client_cleanup(client); + image_generation_fill_err(err, err_len, "http open failed"); + return open_rc; + } + + if (body != NULL && body_len > 0) { + int written = esp_http_client_write(client, body, (int)body_len); + if (written < 0 || (size_t)written != body_len) { + esp_http_client_close(client); + esp_http_client_cleanup(client); + image_generation_fill_err(err, err_len, "http request write failed"); + return ESP_FAIL; + } + } + + int64_t content_len = esp_http_client_fetch_headers(client); + if (content_len > 0 && (size_t)content_len > out_body->max_len) { + esp_http_client_close(client); + esp_http_client_cleanup(client); + image_generation_fill_err(err, err_len, "http response too large"); + return ESP_ERR_INVALID_SIZE; + } + if (out_status_code != NULL) { + *out_status_code = esp_http_client_get_status_code(client); + } + + esp_err_t read_rc = http_read_full_response(client, out_body, err, err_len); + esp_http_client_close(client); + esp_http_client_cleanup(client); + return read_rc; +} + +static const char *json_get_string(cJSON *obj, const char *name) { + if (obj == NULL || name == NULL) { + return NULL; + } + cJSON *item = cJSON_GetObjectItemCaseSensitive(obj, name); + if (cJSON_IsString(item) && item->valuestring != NULL) { + return item->valuestring; + } + return NULL; +} + +static bool json_extract_image_url(cJSON *root, + char *out_url, + size_t out_url_len, + char *out_prompt, + size_t out_prompt_len) { + if (root == NULL || out_url == NULL || out_url_len == 0) { + return false; + } + + cJSON *output = cJSON_GetObjectItemCaseSensitive(root, "output"); + cJSON *choices = cJSON_GetObjectItemCaseSensitive(output, "choices"); + if (!cJSON_IsArray(choices)) { + return false; + } + + cJSON *choice = cJSON_GetArrayItem(choices, 0); + if (!cJSON_IsObject(choice)) { + return false; + } + + cJSON *message = cJSON_GetObjectItemCaseSensitive(choice, "message"); + cJSON *content = cJSON_GetObjectItemCaseSensitive(message, "content"); + if (!cJSON_IsArray(content)) { + return false; + } + + out_url[0] = '\0'; + if (out_prompt != NULL && out_prompt_len > 0) { + out_prompt[0] = '\0'; + } + + int count = cJSON_GetArraySize(content); + for (int i = 0; i < count; ++i) { + cJSON *item = cJSON_GetArrayItem(content, i); + if (!cJSON_IsObject(item)) { + continue; + } + const char *image = json_get_string(item, "image"); + if (image != NULL && out_url[0] == '\0') { + strlcpy(out_url, image, out_url_len); + } + + if (out_prompt != NULL && out_prompt_len > 0 && out_prompt[0] == '\0') { + const char *text = json_get_string(item, "text"); + if (text != NULL) { + strlcpy(out_prompt, text, out_prompt_len); + } + } + } + + return out_url[0] != '\0'; +} + +static void extract_remote_error_message(const char *json_text, + int status_code, + char *err, + size_t err_len) { + if (json_text == NULL) { + image_generation_fill_err(err, err_len, "upstream request failed"); + return; + } + + cJSON *root = cJSON_Parse(json_text); + if (root == NULL) { + if (status_code > 0) { + snprintf(err, err_len, "upstream request failed, http_status=%d", status_code); + } else { + image_generation_fill_err(err, err_len, "upstream request failed"); + } + return; + } + + const char *code = json_get_string(root, "code"); + const char *message = json_get_string(root, "message"); + + if (code != NULL && message != NULL) { + snprintf(err, err_len, "%s: %s", code, message); + } else if (message != NULL) { + image_generation_fill_err(err, err_len, message); + } else if (status_code > 0) { + snprintf(err, err_len, "upstream request failed, http_status=%d", status_code); + } else { + image_generation_fill_err(err, err_len, "upstream request failed"); + } + + cJSON_Delete(root); +} + +static char *build_generation_request_json(const image_generation_request_t *req) { + cJSON *root = cJSON_CreateObject(); + if (root == NULL) { + return NULL; + } + + cJSON_AddStringToObject(root, "model", CONFIG_TQ_Z_IMAGE_MODEL); + + cJSON *input = cJSON_AddObjectToObject(root, "input"); + cJSON *messages = cJSON_AddArrayToObject(input, "messages"); + cJSON *message = cJSON_CreateObject(); + cJSON_AddItemToArray(messages, message); + cJSON_AddStringToObject(message, "role", "user"); + cJSON *content = cJSON_AddArrayToObject(message, "content"); + cJSON *content_item = cJSON_CreateObject(); + cJSON_AddItemToArray(content, content_item); + cJSON_AddStringToObject(content_item, "text", req->prompt); + + cJSON *parameters = cJSON_AddObjectToObject(root, "parameters"); + cJSON_AddBoolToObject(parameters, "prompt_extend", req->prompt_extend); + + const char *size = (req->size != NULL && req->size[0] != '\0') + ? req->size + : CONFIG_TQ_Z_IMAGE_DEFAULT_SIZE; + cJSON_AddStringToObject(parameters, "size", size); + if (req->has_seed) { + cJSON_AddNumberToObject(parameters, "seed", (double)req->seed); + } + + char *body = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + return body; +} + +static esp_err_t image_generation_invoke_model(const image_generation_request_t *req, + image_generation_result_t *out_result, + char *err, + size_t err_len) { + char *req_body = build_generation_request_json(req); + if (req_body == NULL) { + image_generation_fill_err(err, err_len, "build request json failed"); + return ESP_ERR_NO_MEM; + } + + const char *api_key = image_generation_api_key(); + if (api_key == NULL) { + cJSON_free(req_body); + image_generation_fill_err(err, err_len, "z-image api key not configured"); + return ESP_ERR_INVALID_STATE; + } + + char auth_header[192] = {0}; + snprintf(auth_header, sizeof(auth_header), "Bearer %s", api_key); + + esp_http_client_config_t config = { + .url = CONFIG_TQ_Z_IMAGE_API_ENDPOINT, + .method = HTTP_METHOD_POST, + .timeout_ms = (int)req->generation_timeout_ms, + .keep_alive_enable = false, + .buffer_size = 2048, + .buffer_size_tx = 2048, + .crt_bundle_attach = esp_crt_bundle_attach, + }; + + bytes_buffer_t resp = { + .data = NULL, + .len = 0, + .cap = 0, + .max_len = CONFIG_TQ_Z_IMAGE_MAX_RESPONSE_BYTES, + }; + int http_status = 0; + esp_err_t rc = http_request_collect(&config, + auth_header, + req_body, + &http_status, + &resp, + err, + err_len); + cJSON_free(req_body); + if (rc != ESP_OK) { + bytes_buffer_free(&resp); + return rc; + } + + rc = bytes_buffer_terminate(&resp); + if (rc != ESP_OK) { + bytes_buffer_free(&resp); + image_generation_fill_err(err, err_len, "generation response too large"); + return rc; + } + + if (http_status < 200 || http_status >= 300) { + extract_remote_error_message((const char *)resp.data, http_status, err, err_len); + bytes_buffer_free(&resp); + return ESP_FAIL; + } + + cJSON *root = cJSON_Parse((const char *)resp.data); + bytes_buffer_free(&resp); + if (root == NULL) { + image_generation_fill_err(err, err_len, "invalid generation response"); + return ESP_FAIL; + } + + const char *request_id = json_get_string(root, "request_id"); + if (request_id != NULL) { + strlcpy(out_result->request_id, request_id, sizeof(out_result->request_id)); + } + + cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage"); + cJSON *usage_w = cJSON_GetObjectItemCaseSensitive(usage, "width"); + cJSON *usage_h = cJSON_GetObjectItemCaseSensitive(usage, "height"); + if (cJSON_IsNumber(usage_w) && usage_w->valuedouble >= 1 && usage_w->valuedouble <= UINT16_MAX) { + out_result->width = (uint16_t)usage_w->valuedouble; + } + if (cJSON_IsNumber(usage_h) && usage_h->valuedouble >= 1 && usage_h->valuedouble <= UINT16_MAX) { + out_result->height = (uint16_t)usage_h->valuedouble; + } + + bool has_url = json_extract_image_url(root, + out_result->image_url, + sizeof(out_result->image_url), + out_result->output_prompt, + sizeof(out_result->output_prompt)); + if (!has_url) { + const char *code = json_get_string(root, "code"); + const char *message = json_get_string(root, "message"); + if (code != NULL && message != NULL) { + snprintf(err, err_len, "%s: %s", code, message); + } else { + image_generation_fill_err(err, err_len, "generation response has no image url"); + } + cJSON_Delete(root); + return ESP_FAIL; + } + + cJSON_Delete(root); + return ESP_OK; +} + +static esp_err_t image_generation_download_png(image_generation_result_t *out_result, + uint32_t timeout_ms, + char *err, + size_t err_len) { + esp_http_client_config_t config = { + .url = out_result->image_url, + .method = HTTP_METHOD_GET, + .timeout_ms = (int)timeout_ms, + .keep_alive_enable = false, + .buffer_size = 2048, + .crt_bundle_attach = esp_crt_bundle_attach, + }; + + bytes_buffer_t resp = { + .data = NULL, + .len = 0, + .cap = 0, + .max_len = CONFIG_TQ_Z_IMAGE_MAX_PNG_BYTES, + }; + int http_status = 0; + esp_err_t rc = http_request_collect(&config, + NULL, + NULL, + &http_status, + &resp, + err, + err_len); + if (rc != ESP_OK) { + bytes_buffer_free(&resp); + return rc; + } + + if (http_status < 200 || http_status >= 300) { + rc = bytes_buffer_terminate(&resp); + if (rc != ESP_OK) { + bytes_buffer_free(&resp); + image_generation_fill_err(err, err_len, "image download failed"); + return ESP_FAIL; + } + extract_remote_error_message((const char *)resp.data, http_status, err, err_len); + bytes_buffer_free(&resp); + return ESP_FAIL; + } + + static const uint8_t png_header[8] = { + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, + }; + if (resp.len < sizeof(png_header) || memcmp(resp.data, png_header, sizeof(png_header)) != 0) { + bytes_buffer_free(&resp); + image_generation_fill_err(err, err_len, "downloaded image is not png"); + return ESP_FAIL; + } + + out_result->png = resp.data; + out_result->png_len = resp.len; + return ESP_OK; +} + +void image_generation_result_reset(image_generation_result_t *result) { + if (result == NULL) { + return; + } + memset(result, 0, sizeof(*result)); +} + +void image_generation_result_free(image_generation_result_t *result) { + if (result == NULL) { + return; + } + free(result->png); + result->png = NULL; + result->png_len = 0; +} + +esp_err_t image_generation_generate_png(const image_generation_request_t *req, + image_generation_result_t *out_result, + char *err, + size_t err_len) { + if (req == NULL || out_result == NULL || req->prompt == NULL || req->prompt[0] == '\0') { + image_generation_fill_err(err, err_len, "prompt is required"); + return ESP_ERR_INVALID_ARG; + } + + image_generation_result_reset(out_result); + + image_generation_request_t actual = *req; + if (actual.generation_timeout_ms == 0) { + actual.generation_timeout_ms = CONFIG_TQ_Z_IMAGE_TIMEOUT_MS; + } + if (actual.download_timeout_ms == 0) { + actual.download_timeout_ms = CONFIG_TQ_Z_IMAGE_DOWNLOAD_TIMEOUT_MS; + } + + ESP_LOGI(TAG, "invoke z-image, prompt_len=%u", (unsigned)strlen(actual.prompt)); + esp_err_t gen_rc = image_generation_invoke_model(&actual, out_result, err, err_len); + if (gen_rc != ESP_OK) { + image_generation_result_free(out_result); + return gen_rc; + } + + ESP_LOGI(TAG, "download image from model output"); + esp_err_t dl_rc = image_generation_download_png(out_result, actual.download_timeout_ms, err, err_len); + if (dl_rc != ESP_OK) { + image_generation_result_free(out_result); + return dl_rc; + } + + return ESP_OK; +} diff --git a/partitions.csv b/partitions.csv new file mode 100644 index 0000000..c721d7e --- /dev/null +++ b/partitions.csv @@ -0,0 +1,4 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, , 0x6000, +phy_init, data, phy, , 0x1000, +factory, app, factory, , 4M, diff --git a/sdkconfig.defaults b/sdkconfig.defaults index ec2a233..9ef8d5e 100644 --- a/sdkconfig.defaults +++ b/sdkconfig.defaults @@ -8,7 +8,9 @@ CONFIG_BT_NIMBLE_ATT_PREFERRED_MTU=247 CONFIG_ESP_WIFI_STA_DISCONNECTED_PM_ENABLE=n CONFIG_TQ_WIFI_SSID="TalkingQ" CONFIG_TQ_WIFI_PASSWORD="TalkingQ123" -CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE=y +# CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE is not set +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y CONFIG_ESPTOOLPY_FLASHSIZE="16MB"