Catch step exceptions (#345)

Azure · Apr 29, 2020 · 11c9cf5 · 11c9cf5
1 parent 9a58900
commit 11c9cf5
Show file tree

Hide file tree

Showing 9 changed files with 100 additions and 23 deletions.
diff --git a/src/plugins/azgenerator/Generator.ts b/src/plugins/azgenerator/Generator.ts
@@ -35,7 +35,7 @@ export async function GenerateAll(model: CodeModelAz,
         do
         {
             let pathTop = "";
-            let path = "azext_" + model.Extension_Name.split("-").join("_") + "/";
+            let path = "azext_" + model.Extension_NameUnderscored + "/";
 
             files[path + "generated/_params.py"] = GenerateAzureCliParams(model);
             files[path + "generated/commands.py"] = GenerateAzureCliCommands(model);
@@ -46,7 +46,7 @@ export async function GenerateAll(model: CodeModelAz,
             files[path + "generated/_help.py"] = GenerateAzureCliHelp(model);
             files[path + "generated/__init__.py"] = GenerateNamespaceInit(model);  
             files[path + "tests/__init__.py"] = GenerateAzureCliTestInit(model);
-            files[path + "tests/latest/test_" + model.Extension_Name + "_scenario.py"] = GenerateAzureCliTestScenario(model);   
+            files[path + "tests/latest/test_" + model.Extension_NameUnderscored + "_scenario.py"] = GenerateAzureCliTestScenario(model);   
             files[path + "tests/latest/preparers.py"] = GenerateAzureCliTestPrepare(model);
             files[path + "tests/latest/__init__.py"] = GenerateNamespaceInit(model);  
             files[path + "azext_metadata.json"] = GenerateAzureCliAzextMetadata(model);

diff --git a/src/plugins/azgenerator/TemplateAzureCliTestInit.ts b/src/plugins/azgenerator/TemplateAzureCliTestInit.ts
@@ -5,7 +5,7 @@
 
 import { CodeModelAz } from "./CodeModelAz"
 
-export function GenerateAzureCliTestInit(model: CodeModelAz) : string[] {
+export function GenerateAzureCliTestInit(model: CodeModelAz): string[] {
     var output: string[] = [];
 
     output.push("# coding=utf-8");
@@ -20,9 +20,12 @@ export function GenerateAzureCliTestInit(model: CodeModelAz) : string[] {
     output.push("# --------------------------------------------------------------------------");
     output.push("import inspect");
     output.push("import os");
+    output.push("import sys");
+    output.push("import traceback");
     output.push("");
     output.push("");
     output.push("__path__ = __import__('pkgutil').extend_path(__path__, __name__)");
+    output.push("exceptions = []");
     output.push('');
     output.push('');
     output.push('def try_manual(func):');
@@ -44,19 +47,38 @@ export function GenerateAzureCliTestInit(model: CodeModelAz) : string[] {
     output.push('        func_to_call = func');
     output.push('        try:');
     output.push('            func_to_call = import_manual_function(func)');
+    output.push('            print("Found manual override for {}(...)".format(func.__name__))')
     output.push('        except (ImportError, AttributeError):');
     output.push('            pass');
     output.push('        return func_to_call');
     output.push('');
     output.push('    def wrapper(*args, **kwargs):');
     output.push('        func_to_call = get_func_to_call()');
     output.push('        print("running {}()...".format(func.__name__))');
-    output.push('        return func_to_call(*args, **kwargs)');
+    output.push('        try:');
+    output.push('            return func_to_call(*args, **kwargs)');
+    output.push('        except BaseException as e:');
+    output.push('            print("--------------------------------------")');
+    output.push('            print("step exception: ", e)');
+    output.push('            print("--------------------------------------", file=sys.stderr)');
+    output.push('            print("step exception in {}: {}".format(func.__name__, e), file=sys.stderr)');
+    output.push('            traceback.print_exc()');
+    output.push('            exceptions.append((func.__name__, sys.exc_info()))');
     output.push('');
     output.push('    if inspect.isclass(func):');
     output.push('        return get_func_to_call()');
     output.push('    else:');
     output.push('        return wrapper');
     output.push('');
+    output.push('');
+    output.push('def raise_if():');
+    output.push('    if exceptions:');
+    output.push('        if len(exceptions) <= 1:');
+    output.push('            raise exceptions[0][1][1]');
+    output.push('        message = "{}\\nFollowed with exceptions in other steps:\\n".format(str(exceptions[0][1][1]))');
+    output.push('        message += "\\n".join(["{}: {}".format(h[0], h[1][1]) for h in exceptions[1:]])');
+    output.push('        raise exceptions[0][1][0](message).with_traceback(exceptions[0][1][2])');
+    output.push('');
+
     return output;
 }
diff --git a/src/plugins/azgenerator/TemplateAzureCliTestScenario.ts b/src/plugins/azgenerator/TemplateAzureCliTestScenario.ts
@@ -26,11 +26,8 @@ export function GenerateAzureCliTestScenario(model: CodeModelAz): string[] {
 
     head.push("");
     head.push("import os");
-    head.push("import unittest");
-    head.push("");
-    head.push("from azure_devtools.scenario_tests import AllowLargeResponse");
     head.push("from azure.cli.testsdk import ScenarioTest");
-    head.push("from .. import try_manual");
+    head.push("from .. import try_manual, raise_if");
     //head.push("from .preparers import (VirtualNetworkPreparer, VnetSubnetPreparer)");
     steps.push("");
     steps.push("");
@@ -131,6 +128,7 @@ export function GenerateAzureCliTestScenario(model: CodeModelAz): string[] {
     funcScenario.push("");
     funcScenario.push("");
     body.push(`        call_scenario(self${parameterLine()})`);
+    body.push(`        raise_if()`);
     body.push("");
 
     let output = head.concat(imports, steps, funcScenario, class_info, decorators, initiates, body);

diff --git a/src/test/scenarios/attestation/output/src/attestation/azext_attestation/tests/__init__.py b/src/test/scenarios/attestation/output/src/attestation/azext_attestation/tests/__init__.py
@@ -10,9 +10,12 @@
 # --------------------------------------------------------------------------
 import inspect
 import os
+import sys
+import traceback
 
 
 __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+exceptions = []
 
 
 def try_manual(func):
@@ -34,16 +37,34 @@ def get_func_to_call():
         func_to_call = func
         try:
             func_to_call = import_manual_function(func)
+            print("Found manual override for {}(...)".format(func.__name__))
         except (ImportError, AttributeError):
             pass
         return func_to_call
 
     def wrapper(*args, **kwargs):
         func_to_call = get_func_to_call()
         print("running {}()...".format(func.__name__))
-        return func_to_call(*args, **kwargs)
+        try:
+            return func_to_call(*args, **kwargs)
+        except BaseException as e:
+            print("--------------------------------------")
+            print("step exception: ", e)
+            print("--------------------------------------", file=sys.stderr)
+            print("step exception in {}: {}".format(func.__name__, e), file=sys.stderr)
+            traceback.print_exc()
+            exceptions.append((func.__name__, sys.exc_info()))
 
     if inspect.isclass(func):
         return get_func_to_call()
     else:
         return wrapper
+
+
+def raise_if():
+    if exceptions:
+        if len(exceptions) <= 1:
+            raise exceptions[0][1][1]
+        message = "{}\nFollowed with exceptions in other steps:\n".format(str(exceptions[0][1][1]))
+        message += "\n".join(["{}: {}".format(h[0], h[1][1]) for h in exceptions[1:]])
+        raise exceptions[0][1][0](message).with_traceback(exceptions[0][1][2])
diff --git a/...tation/output/src/attestation/azext_attestation/tests/latest/test_attestation_scenario.py b/...tation/output/src/attestation/azext_attestation/tests/latest/test_attestation_scenario.py
@@ -9,11 +9,8 @@
 # --------------------------------------------------------------------------
 
 import os
-import unittest
-
-from azure_devtools.scenario_tests import AllowLargeResponse
 from azure.cli.testsdk import ScenarioTest
-from .. import try_manual
+from .. import try_manual, raise_if
 from azure.cli.testsdk import ResourceGroupPreparer
 
 
@@ -107,3 +104,4 @@ class AttestationManagementClientScenarioTest(ScenarioTest):
     def test_attestation(self, rg, rg_2, rg_3):
 
         call_scenario(self, rg, rg_2, rg_3)
+        raise_if()
diff --git a/src/test/scenarios/datafactory/output/src/datafactory/azext_datafactory/tests/__init__.py b/src/test/scenarios/datafactory/output/src/datafactory/azext_datafactory/tests/__init__.py
@@ -10,9 +10,12 @@
 # --------------------------------------------------------------------------
 import inspect
 import os
+import sys
+import traceback
 
 
 __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+exceptions = []
 
 
 def try_manual(func):
@@ -34,16 +37,34 @@ def get_func_to_call():
         func_to_call = func
         try:
             func_to_call = import_manual_function(func)
+            print("Found manual override for {}(...)".format(func.__name__))
         except (ImportError, AttributeError):
             pass
         return func_to_call
 
     def wrapper(*args, **kwargs):
         func_to_call = get_func_to_call()
         print("running {}()...".format(func.__name__))
-        return func_to_call(*args, **kwargs)
+        try:
+            return func_to_call(*args, **kwargs)
+        except BaseException as e:
+            print("--------------------------------------")
+            print("step exception: ", e)
+            print("--------------------------------------", file=sys.stderr)
+            print("step exception in {}: {}".format(func.__name__, e), file=sys.stderr)
+            traceback.print_exc()
+            exceptions.append((func.__name__, sys.exc_info()))
 
     if inspect.isclass(func):
         return get_func_to_call()
     else:
         return wrapper
+
+
+def raise_if():
+    if exceptions:
+        if len(exceptions) <= 1:
+            raise exceptions[0][1][1]
+        message = "{}\nFollowed with exceptions in other steps:\n".format(str(exceptions[0][1][1]))
+        message += "\n".join(["{}: {}".format(h[0], h[1][1]) for h in exceptions[1:]])
+        raise exceptions[0][1][0](message).with_traceback(exceptions[0][1][2])
diff --git a/...actory/output/src/datafactory/azext_datafactory/tests/latest/test_datafactory_scenario.py b/...actory/output/src/datafactory/azext_datafactory/tests/latest/test_datafactory_scenario.py
@@ -9,11 +9,8 @@
 # --------------------------------------------------------------------------
 
 import os
-import unittest
-
-from azure_devtools.scenario_tests import AllowLargeResponse
 from azure.cli.testsdk import ScenarioTest
-from .. import try_manual
+from .. import try_manual, raise_if
 from azure.cli.testsdk import ResourceGroupPreparer
 
 
@@ -151,3 +148,4 @@ def test_datafactory(self, rg):
         })
 
         call_scenario(self, rg)
+        raise_if()
diff --git a/...narios/managed-network/output/src/managed-network/azext_managed_network/tests/__init__.py b/...narios/managed-network/output/src/managed-network/azext_managed_network/tests/__init__.py
@@ -10,9 +10,12 @@
 # --------------------------------------------------------------------------
 import inspect
 import os
+import sys
+import traceback
 
 
 __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+exceptions = []
 
 
 def try_manual(func):
@@ -34,16 +37,34 @@ def get_func_to_call():
         func_to_call = func
         try:
             func_to_call = import_manual_function(func)
+            print("Found manual override for {}(...)".format(func.__name__))
         except (ImportError, AttributeError):
             pass
         return func_to_call
 
     def wrapper(*args, **kwargs):
         func_to_call = get_func_to_call()
         print("running {}()...".format(func.__name__))
-        return func_to_call(*args, **kwargs)
+        try:
+            return func_to_call(*args, **kwargs)
+        except BaseException as e:
+            print("--------------------------------------")
+            print("step exception: ", e)
+            print("--------------------------------------", file=sys.stderr)
+            print("step exception in {}: {}".format(func.__name__, e), file=sys.stderr)
+            traceback.print_exc()
+            exceptions.append((func.__name__, sys.exc_info()))
 
     if inspect.isclass(func):
         return get_func_to_call()
     else:
         return wrapper
+
+
+def raise_if():
+    if exceptions:
+        if len(exceptions) <= 1:
+            raise exceptions[0][1][1]
+        message = "{}\nFollowed with exceptions in other steps:\n".format(str(exceptions[0][1][1]))
+        message += "\n".join(["{}: {}".format(h[0], h[1][1]) for h in exceptions[1:]])
+        raise exceptions[0][1][0](message).with_traceback(exceptions[0][1][2])
diff --git a/...s/latest/test_managed-network_scenario.py → ...s/latest/test_managed_network_scenario.py b/...s/latest/test_managed-network_scenario.py → ...s/latest/test_managed_network_scenario.py
@@ -9,11 +9,8 @@
 # --------------------------------------------------------------------------
 
 import os
-import unittest
-
-from azure_devtools.scenario_tests import AllowLargeResponse
 from azure.cli.testsdk import ScenarioTest
-from .. import try_manual
+from .. import try_manual, raise_if
 from azure.cli.testsdk import ResourceGroupPreparer
 from .preparers import VirtualNetworkPreparer
 
@@ -258,3 +255,4 @@ def test_managed_network(self, rg):
         })
 
         call_scenario(self, rg)
+        raise_if()