Add Sample for Release Test

.gitignore

@@ -2,12 +2,12 @@ node_modules/
 dist/
 .vscode/
 .vs/
+__pycache__/
+src/test/scenarios/testserver/input/
 package-lock.json
 tmpoutput
 .history
-__pycache__/
 test-results.xml
-
 *.njsproj
 *.user
 *.orig

.scripts/e2e/conftest.py

@@ -1,17 +1,22 @@
+from distutils.dir_util import copy_tree
 import pathlib
 
 def pytest_addoption(parser):
-    parser.addoption("--rp", action="store", default="defaultRP")
-    parser.addoption("--az", action="store", default="defaultAZ")
-    parser.addoption("--swagger", action="store", default="defaultSwagger")
-    parser.addoption("--cliext", action="store", default="defaultCliextf")
+    parser.addoption("--rp", action="store", default="")
+    parser.addoption("--az", action="store", default="")
+    parser.addoption("--swagger", action="store", default="")
+    parser.addoption("--cliext", action="store", default="")
+    parser.addoption("--azpack", action="store", default="")
+    parser.addoption("--testserver", action="store", default="")
     parser.addoption("--all", action="store_true", help="run all combinations")
 
 def pytest_generate_tests(metafunc):
     rp_name = metafunc.config.option.rp;
     az_path = metafunc.config.option.az;
     swagger_path = metafunc.config.option.swagger;
     cliext_path = metafunc.config.option.cliext;
+    azpack_path = metafunc.config.option.azpack;
+    testserver_path = metafunc.config.option.testserver;
     if 'rp' in metafunc.fixturenames:
         if metafunc.config.getoption("all"):
             path = pathlib.Path(__file__).parent.absolute().joinpath("rp_list");
@@ -27,6 +32,12 @@ def pytest_generate_tests(metafunc):
     if 'az' in metafunc.fixturenames:
         if az_path is not None:
             print("Az: " + az_path);
+            if testserver_path is not None:
+                print("TestServer: " + testserver_path)
+                from_path = testserver_path + '/swagger/'
+                to_path = az_path + '/src/test/scenarios/testserver/input/'
+                print("Copy from " + from_path + " to " + to_path);
+                copy_tree(from_path, to_path)
             metafunc.parametrize("az", [az_path])
     if 'swagger' in metafunc.fixturenames:
         if swagger_path is not None:
@@ -35,4 +46,8 @@ def pytest_generate_tests(metafunc):
     if 'cliext' in metafunc.fixturenames:
         if cliext_path is not None:
             print("Cliext: " + cliext_path);
-            metafunc.parametrize("cliext", [cliext_path])
+            metafunc.parametrize("cliext", [cliext_path])
+    if 'azpack' in metafunc.fixturenames:
+        if azpack_path is not None:
+            print("Azpack: " + azpack_path);
+            metafunc.parametrize("azpack", [azpack_path])

.scripts/e2e/rp_list

@@ -0,0 +1 @@
+testserver

.scripts/e2e/testcases.py

@@ -19,12 +19,21 @@ class bcolors:
     UNDERLINE = '\033[4m'
     ENDC = '\033[0m'
 
-def test_rp_e2e(rp, az, swagger, cliext):
+def test_rp_e2e(rp, az, swagger, cliext, azpack):
     print("");
     logStep ("Start test rp: %s" % rp);
     cliext_arg_str = '--azure-cli-extension-folder=' + cliext;
-    swagger_arg_str = swagger + '/specification/'+ rp + '/resource-manager/readme.md';
-    az_arg_str = '--use=' + az;
+
+    if azpack:
+        az_arg_str = '--use=' + azpack;
+    else:
+        az_arg_str = '--use=' + az;
+
+    if (rp == 'testserver'):
+        swagger_arg_str = az + '/src/test/scenarios/testserver/configuration/readme.md';
+    else:
+        swagger_arg_str = swagger + '/specification/'+ rp + '/resource-manager/readme.md';
+
     logInfo ("Az: " + az_arg_str);
     logInfo ("Swagger: " + swagger_arg_str);
     logInfo ("Cliext: " + cliext_arg_str);

azure-pipelines/release.yml

@@ -81,6 +81,8 @@ stages:
           git clone --single-branch -b azure-core-preview https://github.com/Azure/azure-cli.git ../azure-cli
           echo -e "\e[32m[$(date -u)] LOG: Clone azure-cli-extensions"
           git clone --single-branch -b master https://github.com/Azure/azure-cli-extensions.git ../azure-cli-extensions
+          echo -e "\e[32m[$(date -u)] LOG: Clone autorest.testserver"
+          git clone --single-branch -b master https://github.com/Azure/autorest.testserver.git ../autorest.testserver
         displayName: 'Clone Branch'
       - script: |
           echo -e "\e[32m[$(date -u)] LOG: Install and upgrade Pip"
@@ -97,20 +99,25 @@ stages:
           pip install azdev
           echo -e "\e[32m[$(date -u)] LOG: Install autorest"
           npm install -g autorest
-          echo -e "\e[32m[$(date -u)] LOG: Go to azure-cli folder"
-          cd ../azure-cli
           echo -e "\e[32m[$(date -u)] LOG: Azdev setup"
+          cd ../azure-cli
           azdev setup -r ../azure-cli-extensions
+          echo -e "\e[32m[$(date -u)] LOG: test server setup"
+          cd ../autorest.testserver
+          npm install
         displayName: 'Prepare env'
       - script: |
           CURRENT_VERSION=$(node -p "require('./package.json').version")
           echo -e "\e[32m[$(date -u)] LOG: CURRENT AZ VERSION: $CURRENT_VERSION"
           echo -e "\e[32m[$(date -u)] LOG: Active venv"
           . /opt/venv/bin/activate
-          echo -e "\e[32m[$(date -u)] LOG: Go to azure-cli folder"
-          cd ../azure-cli
+          echo -e "\e[32m[$(date -u)] LOG: Start test server"
+          cd ../autorest.testserver
+          npm start &
+          echo -e "\e[32m[$(date -u)] LOG: Az login"
+          az login --service-principal -u $(USER_SERVICE_PRINCIPAL_ID) -p $(USER_SERVICE_PRINCIPAL_SECRET) --tenant $(USER_SERVICE_PRINCIPAL_TENANT)
           echo -e "\e[32m[$(date -u)] LOG: Start test script"
-          pytest -s ../s/.scripts/e2e/testcases.py --all --az $(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)/$(VAR_ARIFACT_NAME)/autorest-az-$CURRENT_VERSION.tgz --swagger ../azure-rest-api-specs --cliext ../azure-cli-extensions --junitxml=../s/.scripts/e2e/test-results.xml
+          pytest -s ../s/.scripts/e2e/testcases.py --all --az ../s --azpack $(VAR_BUILD_ARTIFACT_STAGING_DIRECTORY)/$(VAR_ARIFACT_NAME)/autorest-az-$CURRENT_VERSION.tgz --swagger ../azure-rest-api-specs --cliext ../azure-cli-extensions --testserver ../autorest.testserver --junitxml=../s/.scripts/e2e/test-results.xml
         displayName: 'Pytest'
       - task: PublishTestResults@2
         condition: always()

src/test/scenario-test.ts

@@ -12,8 +12,8 @@ require('source-map-support').install();
     async runAz(directory: string, each: string) {
         let cmd = `${__dirname}/../../` + "node_modules/.bin/autorest --version=3.0.6271 --az --use=" + `${__dirname}/../../` + " " + directory + "/configuration/readme.md --azure-cli-extension-folder=" + directory + "/tmpoutput/";
         console.log(cmd);
-        return await new Promise<boolean>((resolve, reject) => { 
-            exec(cmd, function(error) {
+        return await new Promise<boolean>((resolve, reject) => {
+            exec(cmd, function (error) {
                 if (error !== null) {
                     console.log('exec error: ' + error);
                     // Reject if there is an error:
@@ -28,8 +28,8 @@ require('source-map-support').install();
     async compare(dir1: string, dir2: string) {
         let cmd = "diff -r --strip-trailing-cr " + dir1 + " " + dir2;
         console.log(cmd);
-        return await new Promise<boolean>((resolve, reject) => { 
-            exec(cmd, function(error, stdout) {
+        return await new Promise<boolean>((resolve, reject) => {
+            exec(cmd, function (error, stdout) {
                 if (error !== null) {
                     console.log('exec error: ' + error + ", " + stdout);
                     // Reject if there is an error:
@@ -38,7 +38,7 @@ require('source-map-support').install();
                 // Otherwise resolve the promise:
                 return resolve(true);
             });
-        });        
+        });
     }
 
 
@@ -49,38 +49,39 @@ require('source-map-support').install();
         let msg = "";
         let finalResult = true;
         for (const each of folders) {
-            console.log(`Processing: ${each}`);
-            try {
-                await this.runAz(dir + each, each).then(res => {
-                    if(res == false) {
-                        msg = "Run autorest not successfully!";
-                    }
-                    result = res;
-                }).catch(err => {
-                    msg = "Run autorest failed!";
-                    result = err;
-                });
-                if(result) {
-                    await this.compare(dir + each + "/output/src/" + each, dir + each + "/tmpoutput/src/" + each).then(res1 => {           
-                        if(res1 == false) {
-                            msg = "The generated files have changed!";
+            if (each != 'testserver') {
+                console.log(`Processing: ${each}`);
+                try {
+                    await this.runAz(dir + each, each).then(res => {
+                        if (res == false) {
+                            msg = "Run autorest not successfully!";
                         }
-                        result = res1;
-                    }).catch(e => {
-                        msg = "The diff has some error";
-                        result = e;
+                        result = res;
+                    }).catch(err => {
+                        msg = "Run autorest failed!";
+                        result = err;
                     });
-                }    
-            } catch (error) {
-                console.log(msg);
-                result = false;
-                break;
-            }           
-            if(!result) {
-                finalResult = false;
+                    if (result) {
+                        await this.compare(dir + each + "/output/src/" + each, dir + each + "/tmpoutput/src/" + each).then(res1 => {
+                            if (res1 == false) {
+                                msg = "The generated files have changed!";
+                            }
+                            result = res1;
+                        }).catch(e => {
+                            msg = "The diff has some error";
+                            result = e;
+                        });
+                    }
+                } catch (error) {
+                    console.log(msg);
+                    result = false;
+                    break;
+                }
+                if (!result) {
+                    finalResult = false;
+                }
+                assert.strictEqual(result, true, msg);
             }
-            assert.strictEqual(result, true, msg);
-
         }
         assert.strictEqual(finalResult, true, msg);
     }

src/test/scenarios/testserver/configuration/readme.md

@@ -26,21 +26,31 @@ These are the global settings for the Managed Network API.
 
 ``` yaml
 openapi-type: arm
-tag: testserver-2-7-8
+tag: testserver-local
 override-info:
   title: AutoRest Test Service
   description: Test Infrastructure for AutoRest.
 ```
 
-### Tag: testserver-2-7-8
+### Tag: testserver-2-10-45
 
-These settings apply only when `--tag=testserver-2-7-8` is specified on the command line.
+These settings apply only when `--tag=testserver-2-10-45` is specified on the command line.
 
-``` yaml $(tag) == 'testserver-2-7-8'
+``` yaml $(tag) == 'testserver-2-10-45'
 input-file:
 - ../../../../../node_modules/@microsoft.azure/autorest.testserver/swagger/body-array.json
 - ../../../../../node_modules/@microsoft.azure/autorest.testserver/swagger/body-boolean.json
 ```
+
+### Tag: testserver-local
+
+These settings apply only when `--tag=testserver-local` is specified on the command line.
+
+``` yaml $(tag) == 'testserver-local'
+input-file:
+- ../input/body-boolean.json
+```
+
 # Code Generation