From b48d9179307560d112678c41bb01e92fb8d297de Mon Sep 17 00:00:00 2001 From: xyzroe Date: Fri, 8 Nov 2024 14:13:22 +0100 Subject: [PATCH] 407 - Added support for OTA updates using zlib compression. This reduces the size of the update files and consequently halves the update time. Special thanks to [ginkage/CeilingCW-C6](https://github.com/ginkage/CeilingCW-C6) for the inspiration and implementation details. --- .vscode/settings.json | 13 +- README.md | 3 +- main/const.h | 4 +- main/idf_component.yml | 2 +- main/ota.c | 264 +++++++++++++++++++++++++++++++++++++++++ main/ota.h | 13 ++ main/zigbee.c | 6 +- main/zigbee.h | 2 +- tools/create-ota.py | 8 +- 9 files changed, 304 insertions(+), 11 deletions(-) create mode 100644 main/ota.c create mode 100644 main/ota.h diff --git a/.vscode/settings.json b/.vscode/settings.json index 06ea9de..23f9169 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -63,5 +63,16 @@ "freertos.h": "c", "zigbee.h": "c" }, - "editor.formatOnSave": true + "editor.formatOnSave": false, + "idf.projectDirPath": "${workspaceFolder}/firmware", + "idf.port": "/dev/tty.usbmodemflip_Elevlox1", + "idf.espIdfPath": "/Users/lost/esp/v5.3.1/esp-idf", + "idf.pythonBinPath": "/Users/lost/.espressif/python_env/idf5.3_py3.9_env/bin/python", + "idf.toolsPath": "/Users/lost/.espressif", + "idf.customExtraPaths": "/Users/lost/.espressif/tools/xtensa-esp-elf-gdb/14.2_20240403/xtensa-esp-elf-gdb/bin:/Users/lost/.espressif/tools/riscv32-esp-elf-gdb/14.2_20240403/riscv32-esp-elf-gdb/bin:/Users/lost/.espressif/tools/xtensa-esp-elf/esp-13.2.0_20240530/xtensa-esp-elf/bin:/Users/lost/.espressif/tools/riscv32-esp-elf/esp-13.2.0_20240530/riscv32-esp-elf/bin:/Users/lost/.espressif/tools/esp32ulp-elf/2.38_20240113/esp32ulp-elf/bin:/Users/lost/.espressif/tools/cmake/3.24.0/CMake.app/Contents/bin:/Users/lost/.espressif/tools/openocd-esp32/v0.12.0-esp32-20240318/openocd-esp32/bin:/Users/lost/.espressif/tools/ninja/1.11.1:/Users/lost/.espressif/tools/esp-rom-elfs/20240305", + "idf.customExtraVars": { + "OPENOCD_SCRIPTS": "/Users/lost/.espressif/tools/openocd-esp32/v0.12.0-esp32-20240318/openocd-esp32/share/openocd/scripts", + "ESP_ROM_ELF_DIR": "/Users/lost/.espressif/tools/esp-rom-elfs/20240305/" + }, + "idf.gitPath": "git" } diff --git a/README.md b/README.md index ccd2028..3ab4455 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,10 @@ ota: ### Verified Supported Zigbee Systems -- [zigbee2mqtt](https://www.zigbee2mqtt.io/) - Full support, still requires [external converter](https://github.com/xyzroe/ZigUSB_C6/tree/main/external_converter/ZigUSB_C6.js) ⭐⭐⭐⭐⭐ +- [zigbee2mqtt](https://www.zigbee2mqtt.io/) - Full support, no longer requires an [external converter](https://github.com/xyzroe/ZigUSB_C6/tree/main/external_converter/ZigUSB_C6.js) ⭐⭐⭐⭐⭐ - [HOMEd](https://wiki.homed.dev/page/HOMEd) - Partial support ⭐⭐⭐⭐ - [ZHA](https://www.home-assistant.io/integrations/zha/) - Partial support ⭐⭐⭐⭐ +- Other systems must be tested. The device uses standard clusters and attributes, so most coordinators can support it out of the box. ### Where to buy? diff --git a/main/const.h b/main/const.h index 19d671f..828ae2e 100644 --- a/main/const.h +++ b/main/const.h @@ -27,8 +27,8 @@ #define EXT_LED_ENDPOINT 3 /* the endpoint number for the external LED */ #define INV_USB_ENDPOINT 4 /* the endpoint number for the USB switch (inverted logic) */ -#define OTA_FW_VERSION 0x00000143 /* The attribute indicates the version of the firmware */ -#define FW_BUILD_DATE "20241024" /* The parameter indicates the build date of the firmware */ +#define OTA_FW_VERSION 0x00000133 /* The attribute indicates the version of the firmware */ +#define FW_BUILD_DATE "20241108" /* The parameter indicates the build date of the firmware */ /* GPIO configuration */ #define BTN_GPIO_1 5 /* Button from v0.3 */ diff --git a/main/idf_component.yml b/main/idf_component.yml index d874bc1..4884142 100644 --- a/main/idf_component.yml +++ b/main/idf_component.yml @@ -3,7 +3,7 @@ dependencies: espressif/button: "^3.2.0" espressif/esp-zboss-lib: "1.5.0" espressif/esp-zigbee-lib: "1.5.0" - ## espressif/zlib: "1.3.0" + espressif/zlib: "1.3.0" ## Required IDF version idf: version: "5.3.1" diff --git a/main/ota.c b/main/ota.c new file mode 100644 index 0000000..be4fb06 --- /dev/null +++ b/main/ota.c @@ -0,0 +1,264 @@ +#include "ota.h" + +#include +#include +#include +#include +#include +#include + +bool zlib_init = false; +static const esp_partition_t *s_ota_partition = NULL; +static esp_ota_handle_t s_ota_handle = 0; +z_stream zlib_stream; + +uint8_t ota_header[6]; +size_t ota_header_len = 0; +bool ota_upgrade_subelement = false; +size_t ota_data_len = 0; +uint64_t ota_last_receive_us = 0; +size_t ota_receive_not_logged = 0; + +void ota_reset() +{ + if (s_ota_partition) { + esp_ota_abort(s_ota_handle); + s_ota_partition = NULL; + } + + if (zlib_init) { + inflateEnd(&zlib_stream); + zlib_init = false; + } +} + +bool ota_start() +{ + zlib_init = false; + s_ota_partition = NULL; + s_ota_handle = 0; + + memset(&zlib_stream, 0, sizeof(zlib_stream)); + int ret = inflateInit(&zlib_stream); + if (ret == Z_OK) { + zlib_init = true; + } else { + ESP_LOGE(__func__, "zlib init failed: %d", ret); + return false; + } + + s_ota_partition = esp_ota_get_next_update_partition(NULL); + if (!s_ota_partition) { + ESP_LOGE(__func__, "No next OTA partition"); + return false; + } + + esp_err_t err = esp_ota_begin(s_ota_partition, OTA_WITH_SEQUENTIAL_WRITES, &s_ota_handle); + if (err != ESP_OK) { + ESP_LOGE(__func__, "Error starting OTA: %d", err); + s_ota_partition = NULL; + return false; + } + + return true; +} + +bool ota_write(const uint8_t *data, size_t size, bool flush) +{ + uint8_t buf[256]; + + if (!s_ota_partition) { + return false; + } + + zlib_stream.avail_in = size; + zlib_stream.next_in = data; + + do { + zlib_stream.avail_out = sizeof(buf); + zlib_stream.next_out = buf; + + int ret = inflate(&zlib_stream, flush ? Z_FINISH : Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR) { + ESP_LOGE(__func__, "zlib error: %d", ret); + esp_ota_abort(s_ota_handle); + s_ota_partition = NULL; + return false; + } + + size_t available = sizeof(buf) - zlib_stream.avail_out; + if (available > 0) { + esp_err_t err = esp_ota_write(s_ota_handle, buf, available); + if (err != ESP_OK) { + ESP_LOGE(__func__, "Error writing OTA: %d", err); + esp_ota_abort(s_ota_handle); + s_ota_partition = NULL; + return false; + } + } + } while (zlib_stream.avail_in > 0 || zlib_stream.avail_out == 0); + + return true; +} + +bool ota_finish() +{ + if (!s_ota_partition) { + ESP_LOGE(__func__, "OTA not running"); + return false; + } + + if (!ota_write(NULL, 0, true)) { + return false; + } + + esp_err_t err = esp_ota_end(s_ota_handle); + if (err != ESP_OK) { + ESP_LOGE(__func__, "Error ending OTA: %d", err); + s_ota_partition = NULL; + return false; + } + + inflateEnd(&zlib_stream); + zlib_init = false; + + err = esp_ota_set_boot_partition(s_ota_partition); + if (err != ESP_OK) { + ESP_LOGE(__func__, "Error setting boot partition: %d", err); + s_ota_partition = NULL; + return false; + } + + s_ota_partition = NULL; + return true; +} + +esp_err_t zb_ota_upgrade_status_handler(esp_zb_zcl_ota_upgrade_value_message_t message) +{ + esp_err_t ret = ESP_OK; + + if (message.info.status == ESP_ZB_ZCL_STATUS_SUCCESS) { + if (message.upgrade_status != ESP_ZB_ZCL_OTA_UPGRADE_STATUS_RECEIVE) { + if (ota_receive_not_logged) { + ESP_LOGD(__func__, "OTA (%zu receive data messages suppressed)", + ota_receive_not_logged); + ota_receive_not_logged = 0; + } + ota_last_receive_us = 0; + } + + switch (message.upgrade_status) { + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_START: + ESP_LOGI(__func__, "OTA start"); + ota_reset(); + ota_header_len = 0; + ota_upgrade_subelement = false; + ota_data_len = 0; + if (!ota_start()) { + ota_reset(); + ret = ESP_FAIL; + } + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_RECEIVE: + const uint8_t *payload = message.payload; + size_t payload_size = message.payload_size; + + // Read and process the first sub-element, ignoring everything else + while (ota_header_len < 6 && payload_size > 0) { + ota_header[ota_header_len++] = payload[0]; + payload++; + payload_size--; + } + + if (!ota_upgrade_subelement && ota_header_len == 6) { + if (ota_header[0] == 0 && ota_header[1] == 0) { + ota_upgrade_subelement = true; + ota_data_len = + (((int)ota_header[5] & 0xFF) << 24) + | (((int)ota_header[4] & 0xFF) << 16) + | (((int)ota_header[3] & 0xFF) << 8 ) + | ((int)ota_header[2] & 0xFF); + ESP_LOGD(__func__, "OTA sub-element size %zu", ota_data_len); + } else { + ESP_LOGE(__func__, "OTA sub-element type %02x%02x not supported", ota_header[0], ota_header[1]); + ota_reset(); + ret = ESP_FAIL; + } + } + + if (ota_data_len) { + if (payload_size > ota_data_len) + payload_size = ota_data_len; + ota_data_len -= payload_size; + + if (ota_write(payload, payload_size, false)) { + uint64_t now_us = esp_timer_get_time(); + if (!ota_last_receive_us + || now_us - ota_last_receive_us >= 30 * 1000 * 1000) { + ESP_LOGD(__func__, "OTA receive data (%zu messages suppressed)", + ota_receive_not_logged); + ota_last_receive_us = now_us; + ota_receive_not_logged = 0; + } else { + ota_receive_not_logged++; + } + } else { + ota_reset(); + ret = ESP_FAIL; + } + } + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_APPLY: + ESP_LOGI(__func__, "OTA apply"); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_CHECK: + ESP_LOGI(__func__, "OTA data complete"); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_FINISH: + ESP_LOGI(__func__, "OTA finished"); + bool ok = ota_finish(); + ota_reset(); + if (ok) + esp_restart(); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_ABORT: + ESP_LOGI(__func__, "OTA aborted"); + ota_reset(); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_OK: + ESP_LOGI(__func__, "OTA data ok"); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_ERROR: + ESP_LOGI(__func__, "OTA data error"); + ota_reset(); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_IMAGE_STATUS_NORMAL: + ESP_LOGI(__func__, "OTA image accepted"); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_BUSY: + ESP_LOGI(__func__, "OTA busy"); + ota_reset(); + break; + + case ESP_ZB_ZCL_OTA_UPGRADE_STATUS_SERVER_NOT_FOUND: + ESP_LOGI(__func__, "OTA server not found"); + ota_reset(); + break; + + default: + ESP_LOGI(__func__, "OTA status: %d", message.upgrade_status); + break; + } + } + + return ret; +} \ No newline at end of file diff --git a/main/ota.h b/main/ota.h new file mode 100644 index 0000000..43d4720 --- /dev/null +++ b/main/ota.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +esp_err_t zb_ota_upgrade_status_handler(esp_zb_zcl_ota_upgrade_value_message_t message); + +#ifdef __cplusplus +} // extern "C" +#endif \ No newline at end of file diff --git a/main/zigbee.c b/main/zigbee.c index 0eba741..51dc400 100644 --- a/main/zigbee.c +++ b/main/zigbee.c @@ -24,6 +24,7 @@ #include "perf.h" #include "tools.h" #include "zigbee.h" +#include "ota.h" /*------ Global definitions -----------*/ @@ -310,6 +311,7 @@ size_t ota_header_len_; bool ota_upgrade_subelement_; uint8_t ota_header_[6]; +/* static esp_err_t zb_ota_upgrade_status_handler(esp_zb_zcl_ota_upgrade_value_message_t message) { static uint32_t total_size = 0; @@ -338,7 +340,7 @@ static esp_err_t zb_ota_upgrade_status_handler(esp_zb_zcl_ota_upgrade_value_mess ESP_LOGI(__func__, "-- OTA Client receives data: progress [%ld/%ld]", offset, total_size); - /* Read and process the first sub-element, ignoring everything else */ + // Read and process the first sub-element, ignoring everything else while (ota_header_len_ < 6 && payload_size > 0) { ota_header_[ota_header_len_] = payload[0]; @@ -401,6 +403,7 @@ static esp_err_t zb_ota_upgrade_status_handler(esp_zb_zcl_ota_upgrade_value_mess } return ret; } +*/ static esp_err_t zb_ota_upgrade_query_image_resp_handler(esp_zb_zcl_ota_upgrade_query_image_resp_message_t message) { @@ -422,6 +425,7 @@ static esp_err_t zb_ota_upgrade_query_image_resp_handler(esp_zb_zcl_ota_upgrade_ return ret; } + static esp_err_t zb_action_handler(esp_zb_core_action_callback_id_t callback_id, const void *message) { esp_err_t ret = ESP_OK; diff --git a/main/zigbee.h b/main/zigbee.h index 851135e..6c853cf 100644 --- a/main/zigbee.h +++ b/main/zigbee.h @@ -57,7 +57,7 @@ extern "C" static esp_err_t zb_attribute_handler(const esp_zb_zcl_set_attr_value_message_t *message); static esp_err_t zb_read_attr_resp_handler(const esp_zb_zcl_cmd_read_attr_resp_message_t *message); - static esp_err_t zb_ota_upgrade_status_handler(esp_zb_zcl_ota_upgrade_value_message_t message); + //static esp_err_t zb_ota_upgrade_status_handler(esp_zb_zcl_ota_upgrade_value_message_t message); static esp_err_t zb_ota_upgrade_query_image_resp_handler(esp_zb_zcl_ota_upgrade_query_image_resp_message_t message); static esp_err_t zb_action_handler(esp_zb_core_action_callback_id_t callback_id, const void *message); diff --git a/tools/create-ota.py b/tools/create-ota.py index 00a8b73..8b3f4f5 100644 --- a/tools/create-ota.py +++ b/tools/create-ota.py @@ -26,10 +26,10 @@ def create(filename, manufacturer_id, image_type, file_version, header_string): with open(filename, "rb") as f: data = f.read() - #zobj = zlib.compressobj(level=zlib.Z_BEST_COMPRESSION) - #zdata = zobj.compress(data) - #zdata += zobj.flush() - zdata = data + zobj = zlib.compressobj(level=zlib.Z_BEST_COMPRESSION) + zdata = zobj.compress(data) + zdata += zobj.flush() + #zdata = data image = zigpy.ota.image.OTAImage( header=zigpy.ota.image.OTAImageHeader(