Test fixes (#2240)

* Removing test publishing for jobs not running tests. * Fixing Twin SFC test memory leak. * Permanently disabling Provisioning Service Client for Apple. * Removing the OpenSSL Valgrind workaround. * Adding E2E service client module creation/update retries.
Azure · Feb 23, 2022 · ecfe487 · ecfe487
1 parent 1de399f
commit ecfe487
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 53 deletions.
diff --git a/build/.vsts-ci.yml b/build/.vsts-ci.yml
@@ -135,14 +135,6 @@ jobs:
       IOT_DPS_INDIVIDUAL_X509_CERTIFICATE: $(IOTHUB-E2E-X509-ECC-CERT-BASE64)
       IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64)
       IOT_DPS_INDIVIDUAL_REGISTRATION_ID: $(IOT-DPS-INDIVIDUAL-REGISTRATION-ID)
-  - task: PublishTestResults@2
-    displayName: 'Publish Windows Dynamic Results'
-    inputs:
-      testRunner: CTest
-      testResultsFiles: '**/Test.xml'
-      mergeTestResults: true
-      testRunTitle: 'windowsdynamic'
-    condition: succeededOrFailed()
   - script: cd .. && rd /Q /S $(Agent.BuildDirectory)\s
     displayName: 'Cleanup'
     condition: always()
@@ -172,7 +164,7 @@ jobs:
       sudo -E ./jenkins/ubuntu_clang.sh
     displayName: 'Build'
   - script: |
-      cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
+      cd cmake && sudo -E ../build_all/linux/run_tests.sh
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
@@ -218,7 +210,7 @@ jobs:
   - script: |
       export OPENSSL_ia32cap=0x00000000
       sudo chmod -R 755 .
-      cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
+      cd cmake && sudo -E ../build_all/linux/run_tests.sh
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
@@ -264,7 +256,7 @@ jobs:
   - script: |
       export OPENSSL_ia32cap=0x00000000
       sudo chmod -R 755 .
-      cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
+      cd cmake && sudo -E ../build_all/linux/run_tests.sh
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
@@ -310,7 +302,7 @@ jobs:
   - script: |
       export OPENSSL_ia32cap=0x00000000
       sudo chmod -R 755 .
-      cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
+      cd cmake && sudo -E ../build_all/linux/run_tests.sh
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
@@ -354,7 +346,7 @@ jobs:
       sudo ./jenkins/debian_c.sh
     displayName: 'Build'
   - script: |
-      cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
+      cd cmake && sudo -E ../build_all/linux/run_tests.sh
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
@@ -419,14 +411,6 @@ jobs:
       IOT_DPS_INDIVIDUAL_X509_CERTIFICATE: $(IOTHUB-E2E-X509-ECC-CERT-BASE64)
       IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64)
       IOT_DPS_INDIVIDUAL_REGISTRATION_ID: $(IOT-DPS-INDIVIDUAL-REGISTRATION-ID)
-  - task: PublishTestResults@2
-    displayName: 'Publish Linux with Installed Deps Results'
-    inputs:
-      testRunner: CTest
-      testResultsFiles: '**/Test.xml'
-      mergeTestResults: true
-      testRunTitle: 'linux_installed_deps'
-    condition: succeededOrFailed()
   - script: sudo rm -rf $(Agent.BuildDirectory)/*
     displayName: 'Cleanup'
     condition: always()
@@ -626,7 +610,7 @@ jobs:
       IOTHUB_E2E_X509_PRIVATE_KEY_BASE64: $(IOTHUB-E2E-X509-PRIVATE-KEY-BASE64)
       IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64)
   - script: |
-      cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
+      cd cmake && sudo -E ../build_all/linux/run_tests.sh
     displayName: "Run Tests"
     env:
       IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)

diff --git a/build_all/linux/run_tests.sh b/build_all/linux/run_tests.sh
@@ -6,32 +6,11 @@
 set -o errexit # Exit if command failed.
 set -o pipefail # Exit if pipe failed.
 
-usage() {
-    echo "${0} OR ${0} [run_valgrind]" 1>&2
-    exit 1
-}
-
-RUN_VALGRIND=${1:-""}
-
-if [[ "$RUN_VALGRIND" == "run_valgrind" ]]; then
-  echo "Using doctored openSSL to work with valgrind"
-elif [[ "$RUN_VALGRIND" != "" ]]; then
-  usage
-fi
-
 # Only for testing E2E behaviour !!! 
 TEST_CORES=16
 
 # Refresh dynamic libs to link to
 sudo ldconfig
 
-if [[ "$RUN_VALGRIND" == "run_valgrind" ]] ;
-then
-  #use doctored openssl
-  export LD_LIBRARY_PATH=/usr/local/ssl/lib
-  ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random
-  export LD_LIBRARY_PATH=
-else
-  ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random
-fi
+ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random
 
diff --git a/iothub_client/tests/common_dt_e2e/iothubclient_common_dt_e2e.c b/iothub_client/tests/common_dt_e2e/iothubclient_common_dt_e2e.c
@@ -794,6 +794,7 @@ void dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(IOTHUB_CLIENT_TRANSPORT_P
 
     // Send the Event from the device client
     client_send_tcp_kill_via_d2c(device_to_use);
+    free(reported_payload);
 
     // Send a new reported payload. Register callback. Receive reported state message.
     reported_payload = calloc_and_fill_reported_payload(reported_twin_data->string_property,

diff --git a/iothub_client/tests/iothubclient_amqp_dt_e2e_sfc/iothubclient_amqp_dt_e2e_sfc.c b/iothub_client/tests/iothubclient_amqp_dt_e2e_sfc/iothubclient_amqp_dt_e2e_sfc.c
@@ -17,12 +17,10 @@ TEST_SUITE_CLEANUP(TestClassCleanup)
     dt_e2e_deinit();
 }
 
-#if 0 // TODO: #2238 - Fails Valgrind
 TEST_FUNCTION(IoTHub_AMQP_SendReported_e2e_svc_fault_ctrl_kill_Tcp)
 {
     dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(AMQP_Protocol, IOTHUB_ACCOUNT_AUTH_CONNSTRING);
 }
-#endif
 
 TEST_FUNCTION(IoTHub_AMQP_GetFullDesired_e2e_svc_fault_ctrl_kill_Tcp)
 {

diff --git a/iothub_client/tests/iothubclient_mqtt_dt_e2e_sfc/iothubclient_mqtt_dt_e2e_sfc.c b/iothub_client/tests/iothubclient_mqtt_dt_e2e_sfc/iothubclient_mqtt_dt_e2e_sfc.c
@@ -17,12 +17,10 @@ TEST_SUITE_CLEANUP(TestClassCleanup)
     dt_e2e_deinit();
 }
 
-#if 0 // TODO: #2238 - Fails Valgrind
 TEST_FUNCTION(IoTHub_MQTT_SendReported_e2e_svc_fault_ctrl_kill_Tcp)
 {
     dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(MQTT_Protocol, IOTHUB_ACCOUNT_AUTH_CONNSTRING);
 }
-#endif
 
 TEST_FUNCTION(IoTHub_MQTT_GetFullDesired_e2e_svc_fault_ctrl_kill_Tcp)
 {

diff --git a/provisioning_client/tests/iothub_auth_client_ut/iothub_auth_client_ut.c b/provisioning_client/tests/iothub_auth_client_ut/iothub_auth_client_ut.c
@@ -747,7 +747,7 @@ BEGIN_TEST_SUITE(iothub_auth_client_ut)
         iothub_device_auth_destroy(xda_handle);
     }
 
-#ifndef __APPLE__   // TODO: #2235
+#ifndef __APPLE__   // Disabled for Apple builds.
     TEST_FUNCTION(iothub_device_auth_generate_credentials_key_succeed)
     {
         //arrange

diff --git a/...ning_service_client/tests/provisioning_service_client_ut/provisioning_service_client_ut.c b/...ning_service_client/tests/provisioning_service_client_ut/provisioning_service_client_ut.c
@@ -1620,7 +1620,7 @@ TEST_FUNCTION(prov_sc_create_or_update_individual_enrollment_FAIL_ALL_HTTP_OPTIO
     umock_c_negative_tests_deinit();
 }
 
-#ifndef __APPLE__   // TODO: #2235
+#ifndef __APPLE__   // Disabled for Apple builds.
 TEST_FUNCTION(prov_sc_delete_individual_enrollment_ERROR_INPUT_NULL_PROV_SC)
 {
     //arrange
@@ -2304,7 +2304,7 @@ TEST_FUNCTION(prov_sc_create_or_update_enrollment_group_FAIL_w_etag)
     umock_c_negative_tests_deinit();
 }
 
-#ifndef __APPLE__   // TODO: #2235
+#ifndef __APPLE__   // Disabled for Apple builds.
 TEST_FUNCTION(prov_sc_delete_enrollment_group_ERROR_INPUT_NULL_PROV_SC)
 {
     //arrange
@@ -2819,7 +2819,7 @@ TEST_FUNCTION(prov_sc_get_device_registration_state_GOLDEN)
     deviceRegistrationState_destroy(drs);
 }
 
-#ifndef __APPLE__   // TODO: #2235
+#ifndef __APPLE__   // Disabled for Apple builds.
 TEST_FUNCTION(prov_sc_get_device_registration_state_FAIL)
 {
     //arrange

diff --git a/testtools/iothub_test/src/iothub_account.c b/testtools/iothub_test/src/iothub_account.c
@@ -503,6 +503,33 @@ static int updateTestModule(IOTHUB_REGISTRYMANAGER_HANDLE iothub_registrymanager
     return result;
 }
 
+static int updateTestModuleWithRetry(IOTHUB_REGISTRYMANAGER_HANDLE iothub_registrymanager_handle, IOTHUB_PROVISIONED_DEVICE* deviceToProvision)
+{
+    int result;
+    int attempts = 0;
+
+    while (true)
+    {
+        LogInfo("Attempting to update test module on %s/%s", deviceToProvision->deviceId, deviceToProvision->moduleId);
+        if ((result = updateTestModule(iothub_registrymanager_handle, deviceToProvision)) == 0)
+        {
+            break;
+        }
+
+        attempts++;
+        if (attempts == TEST_CREATE_MAX_RETRIES)
+        {
+            LogError("Updating device/module %s/%s failed with error %d.  Exhausted retry attempts.  Failing test", deviceToProvision->deviceId, deviceToProvision->moduleId, result);
+            break;
+        }
+
+        LogError("Updating device/module %s/%s failed with error %d.  Sleeping %d milliseconds", deviceToProvision->deviceId, deviceToProvision->moduleId, result, TEST_SLEEP_BETWEEN_CREATION_FAILURES_MSEC);
+        ThreadAPI_Sleep(TEST_SLEEP_BETWEEN_CREATION_FAILURES_MSEC);
+    }
+
+    return result;
+}
+
 static int provisionModule(IOTHUB_ACCOUNT_INFO* accountInfo, IOTHUB_PROVISIONED_DEVICE* deviceToProvision)
 {
     IOTHUB_REGISTRY_MODULE_CREATE moduleCreate;
@@ -563,7 +590,7 @@ static int provisionModule(IOTHUB_ACCOUNT_INFO* accountInfo, IOTHUB_PROVISIONED_
         LogError("managedBy expected (%s) does not match what was returned from IoTHubRegistryManager_CreateModule (%s)", TEST_MANAGED_BY_1, moduleInfo.managedBy);
         result = MU_FAILURE;
     }
-    else if (updateTestModule(iothub_registrymanager_handle, deviceToProvision) != 0)
+    else if (updateTestModuleWithRetry(iothub_registrymanager_handle, deviceToProvision) != 0)
     {
         LogError("Unable to update test module");
         result = MU_FAILURE;