diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 00000000000..143c64680f3 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +*.config diff --git a/tests/end_to_end/etc/cntk.json b/tests/end_to_end/etc/cntk.json new file mode 100644 index 00000000000..2d25a92569f --- /dev/null +++ b/tests/end_to_end/etc/cntk.json @@ -0,0 +1,17 @@ +{ + "jobName": "CNTK_TEST", + "image": "aiplatform/pai.run.cntk", + "dataDir": "HDFS_URI/Test/cntk/Data", + "outputDir": "HDFS_URI/Test/cntk/CNTK_TEST", + "codeDir": "HDFS_URI/Test/cntk/BrainScript", + "taskRoles": [ + { + "name": "g2p-train", + "taskNumber": 1, + "cpuNumber": 4, + "memoryMB": 8196, + "gpuNumber": 1, + "command": "/bin/bash cntk.sh" + } + ] +} \ No newline at end of file diff --git a/tests/end_to_end/etc/cntk.sh b/tests/end_to_end/etc/cntk.sh new file mode 100644 index 00000000000..7135952b73d --- /dev/null +++ b/tests/end_to_end/etc/cntk.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright (c) Microsoft Corporation +# All rights reserved. +# +# MIT License +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +mnt_point=/mnt/hdfs +hdfs_addr=$(sed -e "s@hdfs://\(\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}:[0-9]\{1,5\}\).*@\1@" <<< $PAI_DATA_DIR) + +mkdir -p $mnt_point +hdfs-mount $hdfs_addr $mnt_point & +export DATA_DIR=$(sed -e "s@hdfs://\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}:[0-9]\{1,5\}@$mnt_point@g" <<< $PAI_DATA_DIR) +export OUTPUT_DIR=$(sed -e "s@hdfs://\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}:[0-9]\{1,5\}@$mnt_point@g" <<< $PAI_OUTPUT_DIR) + +sed -i "/maxEpochs/c\maxEpochs = 1" G2P.cntk +cntk configFile=G2P.cntk DataDir=$DATA_DIR OutDir=$OUTPUT_DIR \ No newline at end of file diff --git a/tests/end_to_end/etc/launcher.json b/tests/end_to_end/etc/launcher.json new file mode 100644 index 00000000000..b0c9c602da3 --- /dev/null +++ b/tests/end_to_end/etc/launcher.json @@ -0,0 +1,25 @@ +{ + "version": 10, + "user": { + "name": "test" + }, + "taskRoles": { + "Master": { + "taskNumber": 10, + "taskService": { + "version": 23, + "entryPoint": "echo 'TEST'", + "sourceLocations": [ + "/Test/launcher" + ], + "resource": { + "cpuNumber": 1, + "memoryMB": 512, + "portRanges": [], + "diskType": 0, + "diskMB": 0 + } + } + } + } +} \ No newline at end of file diff --git a/tests/end_to_end/install.sh b/tests/end_to_end/install.sh index 7f2db7c582a..77561392be5 100644 --- a/tests/end_to_end/install.sh +++ b/tests/end_to_end/install.sh @@ -18,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +account_file="./etc/account.config" + install_bats() { git clone https://github.com/sstephenson/bats.git cd bats @@ -25,5 +27,32 @@ install_bats() { cd - } +install_paifs() { + cp -r ../../../pai-fs ./ + cd pai-fs + pip install -r requirements.txt + cd - +} + +prepare_cntk_job() { + git clone https://github.com/Microsoft/CNTK.git +} + +get_test_account() { + printf "\nPlease provide a test account:\n" + read -p "Username: " username + read -p "Password: " -s password + printf "\n" + echo "$username:$password" > $account_file +} + + apt-get install -y dos2unix -install_bats \ No newline at end of file + +mkdir -p local +cd local +install_bats +install_paifs +prepare_cntk_job +cd .. +get_test_account \ No newline at end of file diff --git a/tests/end_to_end/start.sh b/tests/end_to_end/start.sh index 62ff3d055df..ef318454361 100644 --- a/tests/end_to_end/start.sh +++ b/tests/end_to_end/start.sh @@ -18,12 +18,30 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +. utils.sh + cluster_config=$1 +account_file="./etc/account.config" +token_file="./etc/token.config" +expiration="$((7*24*60*60))" dos2unix $cluster_config +eval $(parse_yaml $cluster_config "pai_") +rest_server_uri=$pai_clusterinfo_webportalinfo_rest_server_uri + +get_auth_token() { + account="$(cat $account_file)" + account=(${account//:/ }) + curl -X POST -d "username=${account[0]}" -d "password=${account[1]}" -d "expiration=$expiration" $rest_server_uri/api/v1/token | sed -e "s@{\"token\":\"\(.*\)\",.*}@\1@" > $token_file +} + printf "\nStarting end to end tests:\n" +if [ ! -f $token_file ] || [ $(( $(date +%s) - $(stat -c %Y $token_file) )) -gt $expiration ]; then + get_auth_token +fi + printf "\nTesting service ...\n" bats test_service.sh diff --git a/tests/end_to_end/test_hdfs.sh b/tests/end_to_end/test_hdfs.sh index 3414e61702b..9838bfba5eb 100644 --- a/tests/end_to_end/test_hdfs.sh +++ b/tests/end_to_end/test_hdfs.sh @@ -21,11 +21,37 @@ . utils.sh eval $(parse_yaml $cluster_config "pai_") -hdfs_uri=$pai_clusterinfo_restserverinfo_hdfs_uri -webhdfs_uri="$(echo $hdfs_uri | sed -e "s/^hdfs/http/g" | sed -e "s/9000/50070/g")" +hdfs_host=$pai_clusterinfo_hadoopinfo_hadoop_vip +paifs_arg="--host $hdfs_host --port 50070 --user root" -@test "check framework launcher health check" { - result="$(curl $webhdfs_uri/webhdfs/v1/?op=LISTSTATUS)" - [[ $result == *FileStatuses* ]] +@test "list hdfs root dir" { + result="$(python local/pai-fs/pai-fs.py $paifs_arg -ls hdfs://)" + [[ $result == *Launcher* ]] } + +@test "make hdfs test root dir" { + result="$(python local/pai-fs/pai-fs.py $paifs_arg -mkdir hdfs://Test)" + [[ ! $result == *Error* ]] + result="$(python local/pai-fs/pai-fs.py $paifs_arg -ls hdfs://)" + [[ $result == *Test* ]] +} + +@test "make hdfs test sub dir" { + result="$(python local/pai-fs/pai-fs.py $paifs_arg -mkdir hdfs://Test/launcher)" + [[ ! $result == *Error* ]] + result="$(python local/pai-fs/pai-fs.py $paifs_arg -mkdir hdfs://Test/cntk)" + [[ ! $result == *Error* ]] +} + +@test "upload cntk data to hdfs" { + result="$(python local/pai-fs/pai-fs.py $paifs_arg -cp -r local/CNTK/Examples/SequenceToSequence/CMUDict/Data hdfs://Test/cntk/)" + [[ ! $result == *Error* ]] + result="$(python local/pai-fs/pai-fs.py $paifs_arg -cp -r local/CNTK/Examples/SequenceToSequence/CMUDict/BrainScript hdfs://Test/cntk/)" + [[ ! $result == *Error* ]] +} + +@test "upload cntk start script to hdfs" { + result="$(python local/pai-fs/pai-fs.py $paifs_arg -cp etc/cntk.sh hdfs://Test/cntk/BrainScript/)" + [[ ! $result == *Error* ]] +} \ No newline at end of file diff --git a/tests/end_to_end/test_launcher.sh b/tests/end_to_end/test_launcher.sh index b4b63242caa..b974af4baa9 100644 --- a/tests/end_to_end/test_launcher.sh +++ b/tests/end_to_end/test_launcher.sh @@ -28,3 +28,9 @@ launcher_uri=$pai_clusterinfo_restserverinfo_webservice_uri result="$(curl $launcher_uri)" [[ $result == *Active* ]] } + +@test "submit framework launcher test job" { + job_name="launcher-test-$RANDOM-$RANDOM" + result="$(cat ./etc/launcher.json | curl -H "Content-Type: application/json" -X PUT -d @- $launcher_uri/v1/Frameworks/$job_name)" + [[ ! $result == *Error* ]] +} \ No newline at end of file diff --git a/tests/end_to_end/test_rest_server.sh b/tests/end_to_end/test_rest_server.sh index 0da8c16d527..e8568ff4db6 100644 --- a/tests/end_to_end/test_rest_server.sh +++ b/tests/end_to_end/test_rest_server.sh @@ -21,6 +21,7 @@ . utils.sh eval $(parse_yaml $cluster_config "pai_") +hdfs_uri=$pai_clusterinfo_restserverinfo_hdfs_uri rest_server_uri=$pai_clusterinfo_webportalinfo_rest_server_uri @@ -28,3 +29,10 @@ rest_server_uri=$pai_clusterinfo_webportalinfo_rest_server_uri result="$(curl $rest_server_uri)" [[ $result == *API* ]] } + +@test "submit cntk test job" { + job_name="cntk-test-$RANDOM-$RANDOM" + token="$(cat ./etc/token.config)" + result="$(cat ./etc/cntk.json | sed -e "s@CNTK_TEST@$job_name@g" -e "s@HDFS_URI@$hdfs_uri@g" | curl -H "Content-Type: application/json" -H "Authorization: Bearer $token" -X PUT -d @- $rest_server_uri/api/v1/jobs/$job_name)" + [[ ! $result == *Error* ]] +} \ No newline at end of file