From ecfe48755bda5f2978d62741fbc51fb8a6fccb51 Mon Sep 17 00:00:00 2001 From: Cristian Pop Date: Tue, 22 Feb 2022 16:20:26 -0800 Subject: [PATCH] Test fixes (#2240) * Removing test publishing for jobs not running tests. * Fixing Twin SFC test memory leak. * Permanently disabling Provisioning Service Client for Apple. * Removing the OpenSSL Valgrind workaround. * Adding E2E service client module creation/update retries. --- build/.vsts-ci.yml | 28 ++++-------------- build_all/linux/run_tests.sh | 23 +-------------- .../iothubclient_common_dt_e2e.c | 1 + .../iothubclient_amqp_dt_e2e_sfc.c | 2 -- .../iothubclient_mqtt_dt_e2e_sfc.c | 2 -- .../iothub_auth_client_ut.c | 2 +- .../provisioning_service_client_ut.c | 6 ++-- testtools/iothub_test/src/iothub_account.c | 29 ++++++++++++++++++- 8 files changed, 40 insertions(+), 53 deletions(-) diff --git a/build/.vsts-ci.yml b/build/.vsts-ci.yml index 491e569be7..8b637f144e 100644 --- a/build/.vsts-ci.yml +++ b/build/.vsts-ci.yml @@ -135,14 +135,6 @@ jobs: IOT_DPS_INDIVIDUAL_X509_CERTIFICATE: $(IOTHUB-E2E-X509-ECC-CERT-BASE64) IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64) IOT_DPS_INDIVIDUAL_REGISTRATION_ID: $(IOT-DPS-INDIVIDUAL-REGISTRATION-ID) - - task: PublishTestResults@2 - displayName: 'Publish Windows Dynamic Results' - inputs: - testRunner: CTest - testResultsFiles: '**/Test.xml' - mergeTestResults: true - testRunTitle: 'windowsdynamic' - condition: succeededOrFailed() - script: cd .. && rd /Q /S $(Agent.BuildDirectory)\s displayName: 'Cleanup' condition: always() @@ -172,7 +164,7 @@ jobs: sudo -E ./jenkins/ubuntu_clang.sh displayName: 'Build' - script: | - cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind + cd cmake && sudo -E ../build_all/linux/run_tests.sh displayName: "Run Tests" env: IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING) @@ -218,7 +210,7 @@ jobs: - script: | export OPENSSL_ia32cap=0x00000000 sudo chmod -R 755 . - cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind + cd cmake && sudo -E ../build_all/linux/run_tests.sh displayName: "Run Tests" env: IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING) @@ -264,7 +256,7 @@ jobs: - script: | export OPENSSL_ia32cap=0x00000000 sudo chmod -R 755 . - cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind + cd cmake && sudo -E ../build_all/linux/run_tests.sh displayName: "Run Tests" env: IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING) @@ -310,7 +302,7 @@ jobs: - script: | export OPENSSL_ia32cap=0x00000000 sudo chmod -R 755 . - cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind + cd cmake && sudo -E ../build_all/linux/run_tests.sh displayName: "Run Tests" env: IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING) @@ -354,7 +346,7 @@ jobs: sudo ./jenkins/debian_c.sh displayName: 'Build' - script: | - cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind + cd cmake && sudo -E ../build_all/linux/run_tests.sh displayName: "Run Tests" env: IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING) @@ -419,14 +411,6 @@ jobs: IOT_DPS_INDIVIDUAL_X509_CERTIFICATE: $(IOTHUB-E2E-X509-ECC-CERT-BASE64) IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64) IOT_DPS_INDIVIDUAL_REGISTRATION_ID: $(IOT-DPS-INDIVIDUAL-REGISTRATION-ID) - - task: PublishTestResults@2 - displayName: 'Publish Linux with Installed Deps Results' - inputs: - testRunner: CTest - testResultsFiles: '**/Test.xml' - mergeTestResults: true - testRunTitle: 'linux_installed_deps' - condition: succeededOrFailed() - script: sudo rm -rf $(Agent.BuildDirectory)/* displayName: 'Cleanup' condition: always() @@ -626,7 +610,7 @@ jobs: IOTHUB_E2E_X509_PRIVATE_KEY_BASE64: $(IOTHUB-E2E-X509-PRIVATE-KEY-BASE64) IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64) - script: | - cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind + cd cmake && sudo -E ../build_all/linux/run_tests.sh displayName: "Run Tests" env: IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING) diff --git a/build_all/linux/run_tests.sh b/build_all/linux/run_tests.sh index 8781fc439e..41835338ca 100755 --- a/build_all/linux/run_tests.sh +++ b/build_all/linux/run_tests.sh @@ -6,32 +6,11 @@ set -o errexit # Exit if command failed. set -o pipefail # Exit if pipe failed. -usage() { - echo "${0} OR ${0} [run_valgrind]" 1>&2 - exit 1 -} - -RUN_VALGRIND=${1:-""} - -if [[ "$RUN_VALGRIND" == "run_valgrind" ]]; then - echo "Using doctored openSSL to work with valgrind" -elif [[ "$RUN_VALGRIND" != "" ]]; then - usage -fi - # Only for testing E2E behaviour !!! TEST_CORES=16 # Refresh dynamic libs to link to sudo ldconfig -if [[ "$RUN_VALGRIND" == "run_valgrind" ]] ; -then - #use doctored openssl - export LD_LIBRARY_PATH=/usr/local/ssl/lib - ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random - export LD_LIBRARY_PATH= -else - ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random -fi +ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random diff --git a/iothub_client/tests/common_dt_e2e/iothubclient_common_dt_e2e.c b/iothub_client/tests/common_dt_e2e/iothubclient_common_dt_e2e.c index 884a78e1fb..c89bacb0cb 100644 --- a/iothub_client/tests/common_dt_e2e/iothubclient_common_dt_e2e.c +++ b/iothub_client/tests/common_dt_e2e/iothubclient_common_dt_e2e.c @@ -794,6 +794,7 @@ void dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(IOTHUB_CLIENT_TRANSPORT_P // Send the Event from the device client client_send_tcp_kill_via_d2c(device_to_use); + free(reported_payload); // Send a new reported payload. Register callback. Receive reported state message. reported_payload = calloc_and_fill_reported_payload(reported_twin_data->string_property, diff --git a/iothub_client/tests/iothubclient_amqp_dt_e2e_sfc/iothubclient_amqp_dt_e2e_sfc.c b/iothub_client/tests/iothubclient_amqp_dt_e2e_sfc/iothubclient_amqp_dt_e2e_sfc.c index e71edc8eb7..08ccaa9fc3 100644 --- a/iothub_client/tests/iothubclient_amqp_dt_e2e_sfc/iothubclient_amqp_dt_e2e_sfc.c +++ b/iothub_client/tests/iothubclient_amqp_dt_e2e_sfc/iothubclient_amqp_dt_e2e_sfc.c @@ -17,12 +17,10 @@ TEST_SUITE_CLEANUP(TestClassCleanup) dt_e2e_deinit(); } -#if 0 // TODO: #2238 - Fails Valgrind TEST_FUNCTION(IoTHub_AMQP_SendReported_e2e_svc_fault_ctrl_kill_Tcp) { dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(AMQP_Protocol, IOTHUB_ACCOUNT_AUTH_CONNSTRING); } -#endif TEST_FUNCTION(IoTHub_AMQP_GetFullDesired_e2e_svc_fault_ctrl_kill_Tcp) { diff --git a/iothub_client/tests/iothubclient_mqtt_dt_e2e_sfc/iothubclient_mqtt_dt_e2e_sfc.c b/iothub_client/tests/iothubclient_mqtt_dt_e2e_sfc/iothubclient_mqtt_dt_e2e_sfc.c index 3c06b37a6b..7664e05270 100644 --- a/iothub_client/tests/iothubclient_mqtt_dt_e2e_sfc/iothubclient_mqtt_dt_e2e_sfc.c +++ b/iothub_client/tests/iothubclient_mqtt_dt_e2e_sfc/iothubclient_mqtt_dt_e2e_sfc.c @@ -17,12 +17,10 @@ TEST_SUITE_CLEANUP(TestClassCleanup) dt_e2e_deinit(); } -#if 0 // TODO: #2238 - Fails Valgrind TEST_FUNCTION(IoTHub_MQTT_SendReported_e2e_svc_fault_ctrl_kill_Tcp) { dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(MQTT_Protocol, IOTHUB_ACCOUNT_AUTH_CONNSTRING); } -#endif TEST_FUNCTION(IoTHub_MQTT_GetFullDesired_e2e_svc_fault_ctrl_kill_Tcp) { diff --git a/provisioning_client/tests/iothub_auth_client_ut/iothub_auth_client_ut.c b/provisioning_client/tests/iothub_auth_client_ut/iothub_auth_client_ut.c index 57723f3628..1a8cd8016b 100644 --- a/provisioning_client/tests/iothub_auth_client_ut/iothub_auth_client_ut.c +++ b/provisioning_client/tests/iothub_auth_client_ut/iothub_auth_client_ut.c @@ -747,7 +747,7 @@ BEGIN_TEST_SUITE(iothub_auth_client_ut) iothub_device_auth_destroy(xda_handle); } -#ifndef __APPLE__ // TODO: #2235 +#ifndef __APPLE__ // Disabled for Apple builds. TEST_FUNCTION(iothub_device_auth_generate_credentials_key_succeed) { //arrange diff --git a/provisioning_service_client/tests/provisioning_service_client_ut/provisioning_service_client_ut.c b/provisioning_service_client/tests/provisioning_service_client_ut/provisioning_service_client_ut.c index d333080f3a..54a872d8b8 100644 --- a/provisioning_service_client/tests/provisioning_service_client_ut/provisioning_service_client_ut.c +++ b/provisioning_service_client/tests/provisioning_service_client_ut/provisioning_service_client_ut.c @@ -1620,7 +1620,7 @@ TEST_FUNCTION(prov_sc_create_or_update_individual_enrollment_FAIL_ALL_HTTP_OPTIO umock_c_negative_tests_deinit(); } -#ifndef __APPLE__ // TODO: #2235 +#ifndef __APPLE__ // Disabled for Apple builds. TEST_FUNCTION(prov_sc_delete_individual_enrollment_ERROR_INPUT_NULL_PROV_SC) { //arrange @@ -2304,7 +2304,7 @@ TEST_FUNCTION(prov_sc_create_or_update_enrollment_group_FAIL_w_etag) umock_c_negative_tests_deinit(); } -#ifndef __APPLE__ // TODO: #2235 +#ifndef __APPLE__ // Disabled for Apple builds. TEST_FUNCTION(prov_sc_delete_enrollment_group_ERROR_INPUT_NULL_PROV_SC) { //arrange @@ -2819,7 +2819,7 @@ TEST_FUNCTION(prov_sc_get_device_registration_state_GOLDEN) deviceRegistrationState_destroy(drs); } -#ifndef __APPLE__ // TODO: #2235 +#ifndef __APPLE__ // Disabled for Apple builds. TEST_FUNCTION(prov_sc_get_device_registration_state_FAIL) { //arrange diff --git a/testtools/iothub_test/src/iothub_account.c b/testtools/iothub_test/src/iothub_account.c index 095d3292b7..6eeb81bb6e 100644 --- a/testtools/iothub_test/src/iothub_account.c +++ b/testtools/iothub_test/src/iothub_account.c @@ -503,6 +503,33 @@ static int updateTestModule(IOTHUB_REGISTRYMANAGER_HANDLE iothub_registrymanager return result; } +static int updateTestModuleWithRetry(IOTHUB_REGISTRYMANAGER_HANDLE iothub_registrymanager_handle, IOTHUB_PROVISIONED_DEVICE* deviceToProvision) +{ + int result; + int attempts = 0; + + while (true) + { + LogInfo("Attempting to update test module on %s/%s", deviceToProvision->deviceId, deviceToProvision->moduleId); + if ((result = updateTestModule(iothub_registrymanager_handle, deviceToProvision)) == 0) + { + break; + } + + attempts++; + if (attempts == TEST_CREATE_MAX_RETRIES) + { + LogError("Updating device/module %s/%s failed with error %d. Exhausted retry attempts. Failing test", deviceToProvision->deviceId, deviceToProvision->moduleId, result); + break; + } + + LogError("Updating device/module %s/%s failed with error %d. Sleeping %d milliseconds", deviceToProvision->deviceId, deviceToProvision->moduleId, result, TEST_SLEEP_BETWEEN_CREATION_FAILURES_MSEC); + ThreadAPI_Sleep(TEST_SLEEP_BETWEEN_CREATION_FAILURES_MSEC); + } + + return result; +} + static int provisionModule(IOTHUB_ACCOUNT_INFO* accountInfo, IOTHUB_PROVISIONED_DEVICE* deviceToProvision) { IOTHUB_REGISTRY_MODULE_CREATE moduleCreate; @@ -563,7 +590,7 @@ static int provisionModule(IOTHUB_ACCOUNT_INFO* accountInfo, IOTHUB_PROVISIONED_ LogError("managedBy expected (%s) does not match what was returned from IoTHubRegistryManager_CreateModule (%s)", TEST_MANAGED_BY_1, moduleInfo.managedBy); result = MU_FAILURE; } - else if (updateTestModule(iothub_registrymanager_handle, deviceToProvision) != 0) + else if (updateTestModuleWithRetry(iothub_registrymanager_handle, deviceToProvision) != 0) { LogError("Unable to update test module"); result = MU_FAILURE;