Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test fixes #2240

Merged
merged 5 commits into from
Feb 23, 2022
Merged

Test fixes #2240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5f2b23a
Removing test publishing for jobs not running tests.
CIPop Feb 22, 2022
1fa977d
Fixing Twin SFC test memory leak.
CIPop Feb 22, 2022
68d88d4
Permanently disabling Provisioning Service Client for Apple.
CIPop Feb 22, 2022
b95dc06
Removing the OpenSSL Valgrind workaround.
CIPop Feb 22, 2022
9fe9042
Adding E2E service client module creation/update retries.
CIPop Feb 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 6 additions & 22 deletions build/.vsts-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,14 +135,6 @@ jobs:
IOT_DPS_INDIVIDUAL_X509_CERTIFICATE: $(IOTHUB-E2E-X509-ECC-CERT-BASE64)
IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64)
IOT_DPS_INDIVIDUAL_REGISTRATION_ID: $(IOT-DPS-INDIVIDUAL-REGISTRATION-ID)
- task: PublishTestResults@2
displayName: 'Publish Windows Dynamic Results'
inputs:
testRunner: CTest
testResultsFiles: '**/Test.xml'
mergeTestResults: true
testRunTitle: 'windowsdynamic'
condition: succeededOrFailed()
- script: cd .. && rd /Q /S $(Agent.BuildDirectory)\s
displayName: 'Cleanup'
condition: always()
Expand Down Expand Up @@ -172,7 +164,7 @@ jobs:
sudo -E ./jenkins/ubuntu_clang.sh
displayName: 'Build'
- script: |
cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
cd cmake && sudo -E ../build_all/linux/run_tests.sh
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

does this run valgrind tests though?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh wait that's right I separated the build and test run steps. This is just for the openssl workaround

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. The previous run shows all SKUs ran valgrind correctly. The script was only adding the OpenSSL workaround. CMake generation is controlling Valgrind enable/disable.

image

displayName: "Run Tests"
env:
IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
Expand Down Expand Up @@ -218,7 +210,7 @@ jobs:
- script: |
export OPENSSL_ia32cap=0x00000000
sudo chmod -R 755 .
cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
cd cmake && sudo -E ../build_all/linux/run_tests.sh
displayName: "Run Tests"
env:
IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
Expand Down Expand Up @@ -264,7 +256,7 @@ jobs:
- script: |
export OPENSSL_ia32cap=0x00000000
sudo chmod -R 755 .
cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
cd cmake && sudo -E ../build_all/linux/run_tests.sh
displayName: "Run Tests"
env:
IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
Expand Down Expand Up @@ -310,7 +302,7 @@ jobs:
- script: |
export OPENSSL_ia32cap=0x00000000
sudo chmod -R 755 .
cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
cd cmake && sudo -E ../build_all/linux/run_tests.sh
displayName: "Run Tests"
env:
IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
Expand Down Expand Up @@ -354,7 +346,7 @@ jobs:
sudo ./jenkins/debian_c.sh
displayName: 'Build'
- script: |
cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
cd cmake && sudo -E ../build_all/linux/run_tests.sh
displayName: "Run Tests"
env:
IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
Expand Down Expand Up @@ -419,14 +411,6 @@ jobs:
IOT_DPS_INDIVIDUAL_X509_CERTIFICATE: $(IOTHUB-E2E-X509-ECC-CERT-BASE64)
IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64)
IOT_DPS_INDIVIDUAL_REGISTRATION_ID: $(IOT-DPS-INDIVIDUAL-REGISTRATION-ID)
- task: PublishTestResults@2
displayName: 'Publish Linux with Installed Deps Results'
inputs:
testRunner: CTest
testResultsFiles: '**/Test.xml'
mergeTestResults: true
testRunTitle: 'linux_installed_deps'
condition: succeededOrFailed()
- script: sudo rm -rf $(Agent.BuildDirectory)/*
displayName: 'Cleanup'
condition: always()
Expand Down Expand Up @@ -626,7 +610,7 @@ jobs:
IOTHUB_E2E_X509_PRIVATE_KEY_BASE64: $(IOTHUB-E2E-X509-PRIVATE-KEY-BASE64)
IOT_DPS_INDIVIDUAL_X509_KEY: $(IOTHUB-E2E-X509-ECC-PRIVATE-KEY-BASE64)
- script: |
cd cmake && sudo -E ../build_all/linux/run_tests.sh run_valgrind
cd cmake && sudo -E ../build_all/linux/run_tests.sh
displayName: "Run Tests"
env:
IOTHUB_CONNECTION_STRING: $(IOTHUB-CONNECTION-STRING)
Expand Down
23 changes: 1 addition & 22 deletions build_all/linux/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,32 +6,11 @@
set -o errexit # Exit if command failed.
set -o pipefail # Exit if pipe failed.

usage() {
echo "${0} OR ${0} [run_valgrind]" 1>&2
exit 1
}

RUN_VALGRIND=${1:-""}

if [[ "$RUN_VALGRIND" == "run_valgrind" ]]; then
echo "Using doctored openSSL to work with valgrind"
elif [[ "$RUN_VALGRIND" != "" ]]; then
usage
fi

# Only for testing E2E behaviour !!!
TEST_CORES=16

# Refresh dynamic libs to link to
sudo ldconfig

if [[ "$RUN_VALGRIND" == "run_valgrind" ]] ;
then
#use doctored openssl
export LD_LIBRARY_PATH=/usr/local/ssl/lib
ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random
export LD_LIBRARY_PATH=
else
ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random
fi
ctest -T test --no-compress-output -C "Debug" -V -j $TEST_CORES --schedule-random

1 change: 1 addition & 0 deletions iothub_client/tests/common_dt_e2e/iothubclient_common_dt_e2e.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,7 @@ void dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(IOTHUB_CLIENT_TRANSPORT_P

// Send the Event from the device client
client_send_tcp_kill_via_d2c(device_to_use);
free(reported_payload);

// Send a new reported payload. Register callback. Receive reported state message.
reported_payload = calloc_and_fill_reported_payload(reported_twin_data->string_property,
Expand Down
2 changes: 0 additions & 2 deletions iothub_client/tests/iothubclient_amqp_dt_e2e_sfc/iothubclient_amqp_dt_e2e_sfc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,10 @@ TEST_SUITE_CLEANUP(TestClassCleanup)
dt_e2e_deinit();
}

#if 0 // TODO: #2238 - Fails Valgrind
TEST_FUNCTION(IoTHub_AMQP_SendReported_e2e_svc_fault_ctrl_kill_Tcp)
{
dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(AMQP_Protocol, IOTHUB_ACCOUNT_AUTH_CONNSTRING);
}
#endif

TEST_FUNCTION(IoTHub_AMQP_GetFullDesired_e2e_svc_fault_ctrl_kill_Tcp)
{
Expand Down
2 changes: 0 additions & 2 deletions iothub_client/tests/iothubclient_mqtt_dt_e2e_sfc/iothubclient_mqtt_dt_e2e_sfc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,10 @@ TEST_SUITE_CLEANUP(TestClassCleanup)
dt_e2e_deinit();
}

#if 0 // TODO: #2238 - Fails Valgrind
TEST_FUNCTION(IoTHub_MQTT_SendReported_e2e_svc_fault_ctrl_kill_Tcp)
{
dt_e2e_send_reported_test_svc_fault_ctrl_kill_Tcp(MQTT_Protocol, IOTHUB_ACCOUNT_AUTH_CONNSTRING);
}
#endif

TEST_FUNCTION(IoTHub_MQTT_GetFullDesired_e2e_svc_fault_ctrl_kill_Tcp)
{
Expand Down
2 changes: 1 addition & 1 deletion provisioning_client/tests/iothub_auth_client_ut/iothub_auth_client_ut.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -747,7 +747,7 @@ BEGIN_TEST_SUITE(iothub_auth_client_ut)
iothub_device_auth_destroy(xda_handle);
}

#ifndef __APPLE__ // TODO: #2235
#ifndef __APPLE__ // Disabled for Apple builds.
TEST_FUNCTION(iothub_device_auth_generate_credentials_key_succeed)
{
//arrange
Expand Down
6 changes: 3 additions & 3 deletions ...ning_service_client/tests/provisioning_service_client_ut/provisioning_service_client_ut.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1620,7 +1620,7 @@ TEST_FUNCTION(prov_sc_create_or_update_individual_enrollment_FAIL_ALL_HTTP_OPTIO
umock_c_negative_tests_deinit();
}

#ifndef __APPLE__ // TODO: #2235
#ifndef __APPLE__ // Disabled for Apple builds.
TEST_FUNCTION(prov_sc_delete_individual_enrollment_ERROR_INPUT_NULL_PROV_SC)
{
//arrange
Expand Down Expand Up @@ -2304,7 +2304,7 @@ TEST_FUNCTION(prov_sc_create_or_update_enrollment_group_FAIL_w_etag)
umock_c_negative_tests_deinit();
}

#ifndef __APPLE__ // TODO: #2235
#ifndef __APPLE__ // Disabled for Apple builds.
TEST_FUNCTION(prov_sc_delete_enrollment_group_ERROR_INPUT_NULL_PROV_SC)
{
//arrange
Expand Down Expand Up @@ -2819,7 +2819,7 @@ TEST_FUNCTION(prov_sc_get_device_registration_state_GOLDEN)
deviceRegistrationState_destroy(drs);
}

#ifndef __APPLE__ // TODO: #2235
#ifndef __APPLE__ // Disabled for Apple builds.
TEST_FUNCTION(prov_sc_get_device_registration_state_FAIL)
{
//arrange
Expand Down
29 changes: 28 additions & 1 deletion testtools/iothub_test/src/iothub_account.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,33 @@ static int updateTestModule(IOTHUB_REGISTRYMANAGER_HANDLE iothub_registrymanager
return result;
}

static int updateTestModuleWithRetry(IOTHUB_REGISTRYMANAGER_HANDLE iothub_registrymanager_handle, IOTHUB_PROVISIONED_DEVICE* deviceToProvision)
{
int result;
int attempts = 0;

while (true)
{
LogInfo("Attempting to update test module on %s/%s", deviceToProvision->deviceId, deviceToProvision->moduleId);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

too verbose?

if ((result = updateTestModule(iothub_registrymanager_handle, deviceToProvision)) == 0)
{
break;
}

attempts++;
if (attempts == TEST_CREATE_MAX_RETRIES)
{
LogError("Updating device/module %s/%s failed with error %d. Exhausted retry attempts. Failing test", deviceToProvision->deviceId, deviceToProvision->moduleId, result);
break;
}

LogError("Updating device/module %s/%s failed with error %d. Sleeping %d milliseconds", deviceToProvision->deviceId, deviceToProvision->moduleId, result, TEST_SLEEP_BETWEEN_CREATION_FAILURES_MSEC);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LogInfo ?

ThreadAPI_Sleep(TEST_SLEEP_BETWEEN_CREATION_FAILURES_MSEC);
}

return result;
}

static int provisionModule(IOTHUB_ACCOUNT_INFO* accountInfo, IOTHUB_PROVISIONED_DEVICE* deviceToProvision)
{
IOTHUB_REGISTRY_MODULE_CREATE moduleCreate;
Expand Down Expand Up @@ -563,7 +590,7 @@ static int provisionModule(IOTHUB_ACCOUNT_INFO* accountInfo, IOTHUB_PROVISIONED_
LogError("managedBy expected (%s) does not match what was returned from IoTHubRegistryManager_CreateModule (%s)", TEST_MANAGED_BY_1, moduleInfo.managedBy);
result = MU_FAILURE;
}
else if (updateTestModule(iothub_registrymanager_handle, deviceToProvision) != 0)
else if (updateTestModuleWithRetry(iothub_registrymanager_handle, deviceToProvision) != 0)
{
LogError("Unable to update test module");
result = MU_FAILURE;
Expand Down