diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ebdbc9..fd7a7961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Input validation for unsigned int Configs ([#344](https://github.com/matth-x/MicroOcpp/pull/344)) - Support for TransactionMessageAttempts/-RetryInterval ([#345](https://github.com/matth-x/MicroOcpp/pull/345)) - Heap profiler and custom allocator support ([#350](https://github.com/matth-x/MicroOcpp/pull/350)) +- Migration of persistent storage ([#355](https://github.com/matth-x/MicroOcpp/pull/355)) ### Removed diff --git a/CMakeLists.txt b/CMakeLists.txt index ca91ef7d..9d8fffc8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,7 +180,7 @@ target_compile_definitions(mo_unit_tests PUBLIC MO_PLATFORM=MO_PLATFORM_UNIX MO_NUMCONNECTORS=3 MO_CUSTOM_TIMER - MO_DBG_LEVEL=MO_DL_DEBUG + MO_DBG_LEVEL=MO_DL_INFO MO_TRAFFIC_OUT MO_FILENAME_PREFIX="./mo_store/" MO_LocalAuthListMaxLength=8 diff --git a/src/MicroOcpp.cpp b/src/MicroOcpp.cpp index 2fc39ba3..3b2d070c 100644 --- a/src/MicroOcpp.cpp +++ b/src/MicroOcpp.cpp @@ -259,21 +259,12 @@ void mocpp_initialize(Connection& connection, const char *bootNotificationCreden BootService::loadBootStats(filesystem, bootstats); if (autoRecover && bootstats.getBootFailureCount() > 3) { - MO_DBG_ERR("multiple initialization failures detected"); - if (filesystem) { - bool success = FilesystemUtils::remove_if(filesystem, [] (const char *fname) -> bool { - return !strncmp(fname, "sd", strlen("sd")) || - !strncmp(fname, "tx", strlen("tx")) || - !strncmp(fname, "op", strlen("op")) || - !strncmp(fname, "sc-", strlen("sc-")) || - !strncmp(fname, "reservation", strlen("reservation")); - }); - MO_DBG_ERR("clear local state files (recovery): %s", success ? "success" : "not completed"); - - bootstats = BootStats(); - } + BootService::recover(filesystem, bootstats); + bootstats = BootStats(); } + BootService::migrate(filesystem, bootstats); + bootstats.bootNr++; //assign new boot number to this run BootService::storeBootStats(filesystem, bootstats); @@ -373,13 +364,8 @@ void mocpp_initialize(Connection& connection, const char *bootNotificationCreden } credsJson.reset(); - auto mocppVersion = declareConfiguration("MicroOcppVersion", MO_VERSION, MO_KEYVALUE_FN, false, false, false); - configuration_load(); - if (mocppVersion) { - mocppVersion->setString(MO_VERSION); - } MO_DBG_INFO("initialized MicroOcpp v" MO_VERSION); } diff --git a/src/MicroOcpp/Core/Configuration.cpp b/src/MicroOcpp/Core/Configuration.cpp index ec1b9841..9a00fb12 100644 --- a/src/MicroOcpp/Core/Configuration.cpp +++ b/src/MicroOcpp/Core/Configuration.cpp @@ -239,4 +239,11 @@ bool configuration_save() { return success; } +bool configuration_clean_unused() { + for (auto& container : configurationContainers) { + container->removeUnused(); + } + return configuration_save(); +} + } //end namespace MicroOcpp diff --git a/src/MicroOcpp/Core/Configuration.h b/src/MicroOcpp/Core/Configuration.h index 6b33f43c..f445ffce 100644 --- a/src/MicroOcpp/Core/Configuration.h +++ b/src/MicroOcpp/Core/Configuration.h @@ -36,5 +36,7 @@ bool configuration_load(const char *filename = nullptr); bool configuration_save(); +bool configuration_clean_unused(); //remove configs which haven't been accessed + } //end namespace MicroOcpp #endif diff --git a/src/MicroOcpp/Core/ConfigurationContainer.h b/src/MicroOcpp/Core/ConfigurationContainer.h index f31c613c..9a3ff3ae 100644 --- a/src/MicroOcpp/Core/ConfigurationContainer.h +++ b/src/MicroOcpp/Core/ConfigurationContainer.h @@ -35,6 +35,8 @@ class ConfigurationContainer { virtual std::shared_ptr getConfiguration(const char *key) = 0; virtual void loadStaticKey(Configuration& config, const char *key) { } //possible optimization: can replace internal key with passed static key + + virtual void removeUnused() { } //remove configs which haven't been accessed (optional and only if known) }; class ConfigurationContainerVolatile : public ConfigurationContainer, public MemoryManaged { diff --git a/src/MicroOcpp/Core/ConfigurationContainerFlash.cpp b/src/MicroOcpp/Core/ConfigurationContainerFlash.cpp index 9f83631f..b32f1af8 100644 --- a/src/MicroOcpp/Core/ConfigurationContainerFlash.cpp +++ b/src/MicroOcpp/Core/ConfigurationContainerFlash.cpp @@ -334,6 +334,25 @@ class ConfigurationContainerFlash : public ConfigurationContainer, public Memory config.setKey(key); clearKeyPool(key); } + + void removeUnused() override { + //if a config's key is still in the keyPool, we know it's unused because it has never been declared in FW (originates from an older FW version) + + auto key = keyPool.begin(); + while (key != keyPool.end()) { + + for (auto config = configurations.begin(); config != configurations.end(); ++config) { + if ((*config)->getKey() == *key) { + MO_DBG_DEBUG("remove unused config %s", (*config)->getKey()); + configurations.erase(config); + break; + } + } + + MO_FREE(*key); + key = keyPool.erase(key); + } + } }; std::unique_ptr makeConfigurationContainerFlash(std::shared_ptr filesystem, const char *filename, bool accessible) { diff --git a/src/MicroOcpp/Model/Boot/BootService.cpp b/src/MicroOcpp/Model/Boot/BootService.cpp index cf569449..1aabb200 100644 --- a/src/MicroOcpp/Model/Boot/BootService.cpp +++ b/src/MicroOcpp/Model/Boot/BootService.cpp @@ -14,10 +14,6 @@ #include #include -#ifndef MO_BOOTSTATS_LONGTIME_MS -#define MO_BOOTSTATS_LONGTIME_MS 180 * 1000 -#endif - using namespace MicroOcpp; unsigned int PreBootQueue::getFrontRequestOpNr() { @@ -71,6 +67,9 @@ void BootService::loop() { if (!executedLongTime && mocpp_tick_ms() - firstExecutionTimestamp >= MO_BOOTSTATS_LONGTIME_MS) { executedLongTime = true; MO_DBG_DEBUG("boot success timer reached"); + + configuration_clean_unused(); + BootStats bootstats; loadBootStats(filesystem, bootstats); bootstats.lastBootSuccess = bootstats.bootNr; @@ -185,6 +184,14 @@ bool BootService::loadBootStats(std::shared_ptr filesystem, B } else { success = false; } + + const char *microOcppVersionIn = (*json)["MicroOcppVersion"] | (const char*)nullptr; + if (microOcppVersionIn) { + auto ret = snprintf(bstats.microOcppVersion, sizeof(bstats.microOcppVersion), "%s", microOcppVersionIn); + if (ret < 0 || (size_t)ret >= sizeof(bstats.microOcppVersion)) { + success = false; + } + } //else: version specifier can be missing after upgrade from pre 1.2.0 version } else { success = false; } @@ -201,15 +208,58 @@ bool BootService::loadBootStats(std::shared_ptr filesystem, B } } -bool BootService::storeBootStats(std::shared_ptr filesystem, BootStats bstats) { +bool BootService::storeBootStats(std::shared_ptr filesystem, BootStats& bstats) { if (!filesystem) { return false; } - auto json = initJsonDoc("v16.Boot.BootService", JSON_OBJECT_SIZE(2)); + auto json = initJsonDoc("v16.Boot.BootService", JSON_OBJECT_SIZE(3)); json["bootNr"] = bstats.bootNr; json["lastSuccess"] = bstats.lastBootSuccess; + json["MicroOcppVersion"] = (const char*)bstats.microOcppVersion; return FilesystemUtils::storeJson(filesystem, MO_FILENAME_PREFIX "bootstats.jsn", json); } + +bool BootService::recover(std::shared_ptr filesystem, BootStats& bstats) { + if (!filesystem) { + return false; + } + + bool success = FilesystemUtils::remove_if(filesystem, [] (const char *fname) -> bool { + return !strncmp(fname, "sd", strlen("sd")) || + !strncmp(fname, "tx", strlen("tx")) || + !strncmp(fname, "sc-", strlen("sc-")) || + !strncmp(fname, "reservation", strlen("reservation")) || + !strncmp(fname, "client-state", strlen("client-state")); + }); + MO_DBG_ERR("clear local state files (recovery): %s", success ? "success" : "not completed"); + + return success; +} + +bool BootService::migrate(std::shared_ptr filesystem, BootStats& bstats) { + if (!filesystem) { + return false; + } + + bool success = true; + + if (strcmp(bstats.microOcppVersion, MO_VERSION)) { + MO_DBG_INFO("migrate persistent storage to MO v" MO_VERSION); + success = FilesystemUtils::remove_if(filesystem, [] (const char *fname) -> bool { + return !strncmp(fname, "sd", strlen("sd")) || + !strncmp(fname, "tx", strlen("tx")) || + !strncmp(fname, "op", strlen("op")) || + !strncmp(fname, "sc-", strlen("sc-")) || + !strcmp(fname, "client-state.cnf") || + !strcmp(fname, "arduino-ocpp.cnf") || + !strcmp(fname, "ocpp-creds.jsn"); + }); + + snprintf(bstats.microOcppVersion, sizeof(bstats.microOcppVersion), "%s", MO_VERSION); + MO_DBG_DEBUG("clear local state files (migration): %s", success ? "success" : "not completed"); + } + return success; +} diff --git a/src/MicroOcpp/Model/Boot/BootService.h b/src/MicroOcpp/Model/Boot/BootService.h index dac4d91f..6091dc2c 100644 --- a/src/MicroOcpp/Model/Boot/BootService.h +++ b/src/MicroOcpp/Model/Boot/BootService.h @@ -13,8 +13,14 @@ #define MO_BOOT_INTERVAL_DEFAULT 60 +#ifndef MO_BOOTSTATS_LONGTIME_MS +#define MO_BOOTSTATS_LONGTIME_MS 180 * 1000 +#endif + namespace MicroOcpp { +#define MO_BOOTSTATS_VERSION_SIZE 10 + struct BootStats { uint16_t bootNr = 0; uint16_t lastBootSuccess = 0; @@ -22,6 +28,8 @@ struct BootStats { uint16_t getBootFailureCount() { return bootNr - lastBootSuccess; } + + char microOcppVersion [MO_BOOTSTATS_VERSION_SIZE] = {'\0'}; }; enum class RegistrationStatus { @@ -79,8 +87,12 @@ class BootService : public MemoryManaged { void notifyRegistrationStatus(RegistrationStatus status); void setRetryInterval(unsigned long interval); - static bool loadBootStats(std::shared_ptr filesystem, BootStats& out); - static bool storeBootStats(std::shared_ptr filesystem, BootStats bstats); + static bool loadBootStats(std::shared_ptr filesystem, BootStats& bstats); + static bool storeBootStats(std::shared_ptr filesystem, BootStats& bstats); + + static bool recover(std::shared_ptr filesystem, BootStats& bstats); //delete all persistent files which could lead to a crash + + static bool migrate(std::shared_ptr filesystem, BootStats& bstats); //migrate persistent storage if running on a new MO version }; } diff --git a/tests/Boot.cpp b/tests/Boot.cpp index 0e661b2a..4beadf31 100644 --- a/tests/Boot.cpp +++ b/tests/Boot.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -309,5 +310,121 @@ TEST_CASE( "Boot Behavior" ) { } + SECTION("Auto recovery") { + + //start transaction which will persist a few boot cycles, but then will be wiped by auto recovery + loop(); + beginTransaction("mIdTag"); + loop(); + REQUIRE( getChargePointStatus() == ChargePointStatus_Charging ); + + declareConfiguration("keepConfigOverRecovery", "originalVal"); + configuration_save(); + + mocpp_deinitialize(); + + //MO has 2 unexpected power cycles. Probably just back luck - keep the local state and configuration + + //Increase the power cycle counter manually because it's not possible to interrupt the MO lifecycle during unit tests + BootStats bootstats; + BootService::loadBootStats(filesystem, bootstats); + bootstats.bootNr += 2; + BootService::storeBootStats(filesystem, bootstats); + + mocpp_initialize(loopback, ChargerCredentials(), filesystem, /*enable auto recovery*/ true); + BootService::loadBootStats(filesystem, bootstats); + REQUIRE( bootstats.getBootFailureCount() == 2 + 1 ); //two boot failures have been measured, +1 because each power cycle is counted as potentially failing until reaching the long runtime barrier + + loop(); + + REQUIRE( getChargePointStatus() == ChargePointStatus_Charging ); + + REQUIRE( !strcmp(declareConfiguration("keepConfigOverRecovery", "otherVal")->getString(), "originalVal") ); + + //check that the power cycle counter has been updated properly after the controller has been running stable over a long time + mtime += MO_BOOTSTATS_LONGTIME_MS; + loop(); + BootService::loadBootStats(filesystem, bootstats); + REQUIRE( bootstats.getBootFailureCount() == 0 ); + + mocpp_deinitialize(); + + //MO has 10 power cycles without running for at least 3 minutes and wipes the local state, but keeps the configuration + + BootStats bootstats2; + BootService::loadBootStats(filesystem, bootstats2); + bootstats2.bootNr += 10; + BootService::storeBootStats(filesystem, bootstats2); + + mocpp_initialize(loopback, ChargerCredentials(), filesystem, /*enable auto recovery*/ true); + + REQUIRE( !strcmp(declareConfiguration("keepConfigOverRecovery", "otherVal")->getString(), "originalVal") ); + BootStats bootstats3; + BootService::loadBootStats(filesystem, bootstats3); + REQUIRE( bootstats3.getBootFailureCount() == 0 + 1 ); //failure count is reset, but +1 because each power cycle is counted as potentially failing until reaching the long runtime barrier + + loop(); + REQUIRE( getChargePointStatus() == ChargePointStatus_Available ); + + } + + SECTION("Migration") { + + //migration removes files from previous MO versions which were running on the controller. This includes the + //transaction cache, but configs are preserved + + auto old_opstore = filesystem->open(MO_FILENAME_PREFIX "opstore.jsn", "w"); //the opstore has been removed in MO v1.2.0 + old_opstore->write("example content", sizeof("example content") - 1); + old_opstore.reset(); //flushes the file + + loop(); + auto tx = beginTransaction("mIdTag"); //tx store will also be removed + auto txNr = tx->getTxNr(); //remember this for later usage + tx.reset(); //reset this smart pointer + loop(); + REQUIRE( getChargePointStatus() == ChargePointStatus_Charging ); + endTransaction(); + loop(); + + REQUIRE( getOcppContext()->getModel().getTransactionStore()->getTransaction(1, txNr) != nullptr ); //tx exists on flash + + declareConfiguration("keepConfigOverMigration", "originalVal"); //migration keeps configs + configuration_save(); + + mocpp_deinitialize(); + + //After a FW update, the tracked version number has changed + BootStats bootstats; + BootService::loadBootStats(filesystem, bootstats); + snprintf(bootstats.microOcppVersion, sizeof(bootstats.microOcppVersion), "oldFwVers"); + BootService::storeBootStats(filesystem, bootstats); + + mocpp_initialize(loopback, ChargerCredentials(), filesystem); //MO migrates here + + size_t msize = 0; + REQUIRE( filesystem->stat(MO_FILENAME_PREFIX "opstore.jsn", &msize) != 0 ); //opstore has been removed + + REQUIRE( getOcppContext()->getModel().getTransactionStore()->getTransaction(1, txNr) == nullptr ); //tx history entry has been removed + + REQUIRE( !strcmp(declareConfiguration("keepConfigOverMigration", "otherVal")->getString(), "originalVal") ); //config has been preserved + } + + SECTION("Clean unused configs") { + + declareConfiguration("neverDeclaredInsideMO", "originalVal"); //unused configs will be cleared automatically after the controller has been running for a long time + configuration_save(); + + mocpp_deinitialize(); + + mocpp_initialize(loopback, ChargerCredentials(), filesystem); //all configs are loaded here, including the test config of this section + loop(); + + //unused configs will be cleared automatically after long time + mtime += MO_BOOTSTATS_LONGTIME_MS; + loop(); + + REQUIRE( !strcmp(declareConfiguration("neverDeclaredInsideMO", "newVal")->getString(), "newVal") ); //config has been removed + } + mocpp_deinitialize(); }