Merge pull request apache#195 from Microsoft/xiongyf/e2e-test-dev

[Tests] Update end to end test

tests/.gitignore

@@ -0,0 +1 @@
+*.config

tests/end_to_end/etc/cntk.json

@@ -0,0 +1,17 @@
+{
+  "jobName": "CNTK_TEST",
+  "image": "aiplatform/pai.run.cntk",
+  "dataDir": "HDFS_URI/Test/cntk/Data",
+  "outputDir": "HDFS_URI/Test/cntk/CNTK_TEST",
+  "codeDir": "HDFS_URI/Test/cntk/BrainScript",
+  "taskRoles": [
+    {
+      "name": "g2p-train",
+      "taskNumber": 1,
+      "cpuNumber": 4,
+      "memoryMB": 8196,
+      "gpuNumber": 1,
+      "command": "/bin/bash cntk.sh"
+    }
+  ]
+}

tests/end_to_end/etc/cntk.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+mnt_point=/mnt/hdfs
+hdfs_addr=$(sed -e "s@hdfs://\(\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}:[0-9]\{1,5\}\).*@\1@" <<< $PAI_DATA_DIR)
+
+mkdir -p $mnt_point
+hdfs-mount $hdfs_addr $mnt_point &
+export DATA_DIR=$(sed -e "s@hdfs://\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}:[0-9]\{1,5\}@$mnt_point@g" <<< $PAI_DATA_DIR)
+export OUTPUT_DIR=$(sed -e "s@hdfs://\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}:[0-9]\{1,5\}@$mnt_point@g" <<< $PAI_OUTPUT_DIR)
+
+sed -i "/maxEpochs/c\maxEpochs = 1" G2P.cntk
+cntk configFile=G2P.cntk DataDir=$DATA_DIR OutDir=$OUTPUT_DIR

tests/end_to_end/etc/launcher.json

@@ -0,0 +1,25 @@
+{
+  "version": 10,
+  "user": {
+    "name": "test"
+  },
+  "taskRoles": {
+    "Master": {
+      "taskNumber": 10,
+      "taskService": {
+        "version": 23,
+        "entryPoint": "echo 'TEST'",
+        "sourceLocations": [
+          "/Test/launcher"
+        ],
+        "resource": {
+          "cpuNumber": 1,
+          "memoryMB": 512,
+          "portRanges": [],
+          "diskType": 0,
+          "diskMB": 0
+        }
+      }
+    }
+  }
+}

tests/end_to_end/install.sh

@@ -18,12 +18,41 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
+account_file="./etc/account.config"
+
 install_bats() {
   git clone https://github.com/sstephenson/bats.git
   cd bats
   ./install.sh /usr/local
   cd -
 }
 
+install_paifs() {
+  cp -r ../../../pai-fs ./
+  cd pai-fs
+  pip install -r requirements.txt
+  cd -
+}
+
+prepare_cntk_job() {
+  git clone https://github.com/Microsoft/CNTK.git
+}
+
+get_test_account() {
+  printf "\nPlease provide a test account:\n"
+  read -p "Username: " username
+  read -p "Password: " -s password
+  printf "\n"
+  echo "$username:$password" > $account_file
+}
+
+
 apt-get install -y dos2unix
-install_bats
+
+mkdir -p local
+cd local
+install_bats
+install_paifs
+prepare_cntk_job
+cd ..
+get_test_account

tests/end_to_end/start.sh

@@ -18,12 +18,30 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
+. utils.sh
+
 cluster_config=$1
+account_file="./etc/account.config"
+token_file="./etc/token.config"
+expiration="$((7*24*60*60))"
 dos2unix $cluster_config
 
+eval $(parse_yaml $cluster_config "pai_")
+rest_server_uri=$pai_clusterinfo_webportalinfo_rest_server_uri
+
+get_auth_token() {
+  account="$(cat $account_file)"
+  account=(${account//:/ })
+  curl -X POST -d "username=${account[0]}" -d "password=${account[1]}" -d "expiration=$expiration" $rest_server_uri/api/v1/token | sed -e "s@{\"token\":\"\(.*\)\",.*}@\1@" > $token_file
+}
+
 
 printf "\nStarting end to end tests:\n"
 
+if [ ! -f $token_file ] || [ $(( $(date +%s) - $(stat -c %Y $token_file) )) -gt $expiration ]; then
+  get_auth_token
+fi
+
 printf "\nTesting service ...\n"
 bats test_service.sh
 

tests/end_to_end/test_hdfs.sh

@@ -21,11 +21,37 @@
 . utils.sh
 
 eval $(parse_yaml $cluster_config "pai_")
-hdfs_uri=$pai_clusterinfo_restserverinfo_hdfs_uri
-webhdfs_uri="$(echo $hdfs_uri | sed -e "s/^hdfs/http/g" | sed -e "s/9000/50070/g")"
+hdfs_host=$pai_clusterinfo_hadoopinfo_hadoop_vip
+paifs_arg="--host $hdfs_host --port 50070 --user root"
 
 
-@test "check framework launcher health check" {
-  result="$(curl $webhdfs_uri/webhdfs/v1/?op=LISTSTATUS)"
-  [[ $result == *FileStatuses* ]]
+@test "list hdfs root dir" {
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -ls hdfs://)"
+  [[ $result == *Launcher* ]]
 }
+
+@test "make hdfs test root dir" {
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -mkdir hdfs://Test)"
+  [[ ! $result == *Error* ]]
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -ls hdfs://)"
+  [[ $result == *Test* ]]
+}
+
+@test "make hdfs test sub dir" {
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -mkdir hdfs://Test/launcher)"
+  [[ ! $result == *Error* ]]
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -mkdir hdfs://Test/cntk)"
+  [[ ! $result == *Error* ]]
+}
+
+@test "upload cntk data to hdfs" {
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -cp -r local/CNTK/Examples/SequenceToSequence/CMUDict/Data hdfs://Test/cntk/)"
+  [[ ! $result == *Error* ]]
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -cp -r local/CNTK/Examples/SequenceToSequence/CMUDict/BrainScript hdfs://Test/cntk/)"
+  [[ ! $result == *Error* ]]
+}
+
+@test "upload cntk start script to hdfs" {
+  result="$(python local/pai-fs/pai-fs.py $paifs_arg -cp etc/cntk.sh hdfs://Test/cntk/BrainScript/)"
+  [[ ! $result == *Error* ]]
+}

tests/end_to_end/test_launcher.sh

@@ -28,3 +28,9 @@ launcher_uri=$pai_clusterinfo_restserverinfo_webservice_uri
   result="$(curl $launcher_uri)"
   [[ $result == *Active* ]]
 }
+
+@test "submit framework launcher test job" {
+  job_name="launcher-test-$RANDOM-$RANDOM"
+  result="$(cat ./etc/launcher.json | curl -H "Content-Type: application/json" -X PUT -d @- $launcher_uri/v1/Frameworks/$job_name)"
+  [[ ! $result == *Error* ]]
+}

tests/end_to_end/test_rest_server.sh

@@ -21,10 +21,18 @@
 . utils.sh
 
 eval $(parse_yaml $cluster_config "pai_")
+hdfs_uri=$pai_clusterinfo_restserverinfo_hdfs_uri
 rest_server_uri=$pai_clusterinfo_webportalinfo_rest_server_uri
 
 
 @test "check rest server health check" {
   result="$(curl $rest_server_uri)"
   [[ $result == *API* ]]
 }
+
+@test "submit cntk test job" {
+  job_name="cntk-test-$RANDOM-$RANDOM"
+  token="$(cat ./etc/token.config)"
+  result="$(cat ./etc/cntk.json | sed -e "s@CNTK_TEST@$job_name@g" -e "s@HDFS_URI@$hdfs_uri@g" | curl -H "Content-Type: application/json" -H "Authorization: Bearer $token" -X PUT -d @- $rest_server_uri/api/v1/jobs/$job_name)"
+  [[ ! $result == *Error* ]]
+}