[Monitor Control Service] Update Data Collection Rules

src/monitor-control-service/HISTORY.rst

@@ -3,6 +3,13 @@
 Release History
 ===============
 
+0.3.0
+++++++
+* `az monitor data-collection rule create`: Add parameter --rule-file.
+* `az monitor data-collection rule create`: Remove parameters --data-flows, --log-analytics, --monitor-metrics, --performance-counters, --windows-event-logs, --syslog and --extensions.
+* `az monitor data-collection rule association list`: Add parameter --data-collection-endpoint-name.
+* Bump api version from 2021-04-01 to 2021_09_01_preview.
+
 0.2.0
 ++++++
 * GA release.

src/monitor-control-service/azext_amcs/generated/_help.py

@@ -63,12 +63,16 @@
 helps['monitor data-collection rule association list'] = """
     type: command
     short-summary: "Lists associations for the specified data collection rule. And Lists associations for the \
-specified resource."
+specified data collection endpoint. And Lists associations for the specified resource."
     examples:
       - name: List associations for specified data collection rule
         text: |-
                az monitor data-collection rule association list --rule-name "myCollectionRule" --resource-group \
 "myResourceGroup"
+      - name: List associations for specified data collection endpoint
+        text: |-
+               az monitor data-collection rule association list --data-collection-endpoint-name \
+"myDataCollectionEndpointName" --resource-group "myResourceGroup"
       - name: List associations for specified resource
         text: |-
                az monitor data-collection rule association list --resource "subscriptions/703362b3-f278-4e4b-9179-c76ea\

src/monitor-control-service/azext_amcs/generated/_params.py

@@ -32,6 +32,8 @@ def load_arguments(self, _):
         c.argument('resource_group_name', resource_group_name_type)
         c.argument('data_collection_rule_name', options_list=['--rule-name'], type=str, help='The name of the data '
                    'collection rule. The name is case insensitive.')
+        c.argument('data_collection_endpoint_name', options_list=['--endpoint-name'], type=str, help='The name of the data collection endpoint. The name '
+                   'is case insensitive.')
         c.argument('resource_uri', options_list=['--resource'], type=str, help='The identifier of the resource.')
 
     with self.argument_context('monitor data-collection rule association show') as c:

src/monitor-control-service/azext_amcs/generated/custom.py

@@ -34,10 +34,14 @@ def monitor_data_collection_endpoint_delete(client,
 def monitor_data_collection_rule_association_list(client,
                                                   resource_group_name=None,
                                                   data_collection_rule_name=None,
+                                                  data_collection_endpoint_name=None,
                                                   resource_uri=None):
     if resource_group_name and data_collection_rule_name is not None:
         return client.list_by_rule(resource_group_name=resource_group_name,
                                    data_collection_rule_name=data_collection_rule_name)
+    elif resource_group_name and data_collection_endpoint_name is not None:
+        return client.list_by_data_collection_endpoint(resource_group_name=resource_group_name,
+                                                       data_collection_endpoint_name=data_collection_endpoint_name)
     return client.list_by_resource(resource_uri=resource_uri)
 
 

src/monitor-control-service/azext_amcs/manual/_help.py

@@ -165,103 +165,13 @@
     type: command
     short-summary: "Create a data collection rule."
     parameters:
-      - name: --data-flows
-        short-summary: "The specification of data flows."
-        long-summary: |
-            Usage: --data-flows streams=XX1 streams=XX2 destinations=XX1 destinations=XX2
-
-            streams: Required. List of streams for this data flow.
-            destinations: Required. List of destinations for this data flow.
-
-            Multiple actions can be specified by using more than one --data-flows argument.
-      - name: --log-analytics
-        short-summary: "List of Log Analytics destinations."
-        long-summary: |
-            Usage: --log-analytics resource-id=XX name=XX
-
-            resource-id: Required. The resource ID of the Log Analytics workspace.
-            name: Required. A friendly name for the destination.  This name should be unique across all destinations \
-(regardless of type) within the data collection rule.
-
-            Multiple actions can be specified by using more than one --log-analytics argument.
-      - name: --monitor-metrics
-        short-summary: "Azure Monitor Metrics destination."
-        long-summary: |
-            Usage: --monitor-metrics name=XX
-
-            name: Required. A friendly name for the destination.  This name should be unique across all destinations \
-(regardless of type) within the data collection rule.
-      - name: --performance-counters
-        short-summary: "The list of performance counter data source configurations."
-        long-summary: |
-            Usage: --performance-counters streams=XX1 streams=XX2 \
-sampling-frequency=XX counter-specifiers=XX1 counter-specifiers=XX2 name=XX
-
-            streams: Required. List of streams that this data source will be sent to. A stream indicates what schema \
-will be used for this data and usually what table in Log Analytics the data will be sent to.
-            sampling-frequency: Required. The number of seconds between consecutive counter measurements \
-(samples).
-            counter-specifiers: Required. A list of specifier names of the performance counters you want to collect. \
-Use a wildcard (*) to collect a counter for all instances. To get a list of performance counters on Windows, run the \
-command 'typeperf'.
-            name: Required. A friendly name for the data source.  This name should be unique across all data sources \
-(regardless of type) within the data collection rule.
-
-            Multiple actions can be specified by using more than one --performance-counters argument.
-      - name: --windows-event-logs
-        short-summary: "The list of Windows Event Log data source configurations."
-        long-summary: |
-            Usage: --windows-event-logs streams=XX1 streams=XX2 x-path-queries=XX1 \
-x-path-queries=XX2 name=XX
-
-            streams: Required. List of streams that this data source will be sent to. A stream indicates what schema \
-will be used for this data and usually what table in Log Analytics the data will be sent to.
-            x-path-queries: Required. A list of Windows Event Log queries in XPATH format.
-            name: Required. A friendly name for the data source.  This name should be unique across all data sources \
-(regardless of type) within the data collection rule.
-
-            Multiple actions can be specified by using more than one --windows-event-logs argument.
-      - name: --syslog
-        short-summary: "The list of Syslog data source configurations."
-        long-summary: |
-            Usage: --syslog streams=XX1 streams=XX2 facility-names=XX1 facility-names=XX2 log-levels=XX1 log-levels=XX2 \
-name=XX
-
-            streams: Required. List of streams that this data source will be sent to. A stream indicates what schema \
-will be used for this data and usually what table in Log Analytics the data will be sent to.
-            facility-names: Required. The list of facility names.
-            log-levels: The log levels to collect.
-            name: Required. A friendly name for the data source.  This name should be unique across all data sources \
-(regardless of type) within the data collection rule.
-
-            Multiple actions can be specified by using more than one --syslog argument.
+      - name: --rule-file
+        short-summary: "The json file for rule parameters."
     examples:
       - name: Create data collection rule
         text: |-
                az monitor data-collection rule create --resource-group "myResourceGroup" --location "eastus" \
---name "myCollectionRule" \
---data-flows destinations="centralWorkspace" streams="Microsoft-Perf" streams="Microsoft-Syslog" \
-streams="Microsoft-WindowsEvent" \
---log-analytics name="centralWorkspace" \
-resource-id="/subscriptions/703362b3-f278-4e4b-9179-c76eaf41ffc2/resourceGroups/myResourceGroup/providers/Mic\
-rosoft.OperationalInsights/workspaces/centralTeamWorkspace" \
---performance-counters name="cloudTeamCoreCounters" counter-specifiers="\\\\Processor(_Total)\\\\% Processor Time" \
-counter-specifiers="\\\\Memory\\\\Committed Bytes" counter-specifiers="\\\\LogicalDisk(_Total)\\\\Free Megabytes" \
-counter-specifiers="\\\\PhysicalDisk(_Total)\\\\Avg. Disk Queue Length" sampling-frequency=15 \
-streams="Microsoft-Perf" \
---performance-counters name="appTeamExtraCounters" \
-counter-specifiers="\\\\Process(_Total)\\\\Thread Count" sampling-frequency=30 \
-streams="Microsoft-Perf" \
---syslog name="cronSyslog" facility-names="cron" log-levels="Debug" log-levels="Critical" log-levels="Emergency" \
-streams="Microsoft-Syslog" \
---syslog name="syslogBase" facility-names="syslog" log-levels="Alert" log-levels="Critical" log-levels="Emergency" \
-streams="Microsoft-Syslog" \
---windows-event-logs name="cloudSecurityTeamEvents" streams="Microsoft-WindowsEvent" \
-x-path-queries="Security!" \
---windows-event-logs name="appTeam1AppEvents" streams="Microsoft-WindowsEvent" \
-x-path-queries="System![System[(Level = 1 or Level = 2 or Level = 3)]]" \
-x-path-queries="Application!*[System[(Level = 1 or Level = 2 or Level = 3)]]"
-
+--name "myCollectionRule" --rule-file "C:\samples\dcrEx1.json"
 """
 
 helps['monitor data-collection rule update'] = """

src/monitor-control-service/azext_amcs/manual/_params.py

@@ -92,6 +92,7 @@ def load_arguments(self, _):
 
     with self.argument_context('monitor data-collection rule create') as c:
         c.argument('data_collection_rule_name', id_part=None)
+        c.argument('rule_file', type=str, help='The json file for rule parameters.', required=True)
         c.argument('location', arg_type=get_location_type(self.cli_ctx), required=False,
                    validator=get_default_location_from_resource_group)
 

src/monitor-control-service/azext_amcs/manual/custom.py

@@ -94,29 +94,52 @@ def _data_collection_rules_create(client,
 def data_collection_rules_create(client,
                                  resource_group_name,
                                  data_collection_rule_name,
+                                 rule_file,
                                  location=None,
                                  tags=None,
-                                 description=None,
-                                 data_flows=None,
-                                 destinations__log_analytics=None,
-                                 destinations__azure_monitor_metrics=None,
-                                 data_sources__performance_counters=None,
-                                 data_sources__windows_event_logs=None,
-                                 data_sources__syslog=None,
-                                 data_sources__extensions=None):
+                                 description=None):
+    from azure.cli.core.util import get_file_json
+    from azure.cli.core.azclierror import FileOperationError, UnclassifiedUserFault
     body = {}
     body['location'] = location
     body['tags'] = tags
     body['description'] = description
-    body['data_flows'] = data_flows
-    body['destinations'] = {}
-    body['destinations']['log_analytics'] = destinations__log_analytics
-    body['destinations']['azure_monitor_metrics'] = destinations__azure_monitor_metrics
-    body['data_sources'] = {}
-    body['data_sources']['performance_counters'] = data_sources__performance_counters
-    body['data_sources']['windows_event_logs'] = data_sources__windows_event_logs
-    body['data_sources']['syslog'] = data_sources__syslog
-    body['data_sources']['extensions'] = data_sources__extensions
+    try:
+        json_data = get_file_json(rule_file)
+    except FileNotFoundError:
+        raise FileOperationError("No such file: " + str(rule_file))
+    except IsADirectoryError:
+        raise FileOperationError("Is a directory: " + str(rule_file))
+    except PermissionError:
+        raise FileOperationError("Permission denied: " + str(rule_file))
+    except OSError as e:
+        raise UnclassifiedUserFault(e)
+    for key_prop in json_data:
+        if key_prop == 'properties':
+            data = json_data['properties']
+        else:
+            data = json_data
+    for key in data:
+        if key == 'dataSources':
+            body['data_sources'] = {}
+            for key_ds in data['dataSources']:
+                if key_ds == 'performanceCounters':
+                    body['data_sources']['performance_counters'] = data['dataSources']['performanceCounters']
+                if key_ds == 'windowsEventLogs':
+                    body['data_sources']['windows_event_logs'] = data['dataSources']['windowsEventLogs']
+                if key_ds == 'syslog':
+                    body['data_sources']['syslog'] = data['dataSources']['syslog']
+                if key_ds == 'extensions':
+                    body['data_sources']['extensions'] = data['dataSources']['extensions']
+        if key == 'destinations':
+            body['destinations'] = {}
+            for key_de in data['destinations']:
+                if key_de == 'logAnalytics':
+                    body['destinations']['log_analytics'] = data['destinations']['logAnalytics']
+                if key_de == 'azureMonitorMetrics':
+                    body['destinations']['azure_monitor_metrics'] = data['destinations']['azureMonitorMetrics']
+        if key == 'dataFlows':
+            body['data_flows'] = data['dataFlows']
     return _data_collection_rules_create(client,
                                          resource_group_name=resource_group_name,
                                          data_collection_rule_name=data_collection_rule_name,

src/monitor-control-service/azext_amcs/tests/__init__.py

@@ -0,0 +1,116 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+#
+# Code generated by Microsoft (R) AutoRest Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is
+# regenerated.
+# --------------------------------------------------------------------------
+import inspect
+import logging
+import os
+import sys
+import traceback
+import datetime as dt
+
+from azure.core.exceptions import AzureError
+from azure.cli.testsdk.exceptions import CliTestError, CliExecutionError, JMESPathCheckAssertionError
+
+
+logger = logging.getLogger('azure.cli.testsdk')
+logger.addHandler(logging.StreamHandler())
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+exceptions = []
+test_map = dict()
+SUCCESSED = "successed"
+FAILED = "failed"
+
+
+def try_manual(func):
+    def import_manual_function(origin_func):
+        from importlib import import_module
+        decorated_path = inspect.getfile(origin_func).lower()
+        module_path = __path__[0].lower()
+        if not decorated_path.startswith(module_path):
+            raise Exception("Decorator can only be used in submodules!")
+        manual_path = os.path.join(
+            decorated_path[module_path.rfind(os.path.sep) + 1:])
+        manual_file_path, manual_file_name = os.path.split(manual_path)
+        module_name, _ = os.path.splitext(manual_file_name)
+        manual_module = "..manual." + \
+            ".".join(manual_file_path.split(os.path.sep) + [module_name, ])
+        return getattr(import_module(manual_module, package=__name__), origin_func.__name__)
+
+    def get_func_to_call():
+        func_to_call = func
+        try:
+            func_to_call = import_manual_function(func)
+            logger.info("Found manual override for %s(...)", func.__name__)
+        except (ImportError, AttributeError):
+            pass
+        return func_to_call
+
+    def wrapper(*args, **kwargs):
+        func_to_call = get_func_to_call()
+        logger.info("running %s()...", func.__name__)
+        try:
+            test_map[func.__name__] = dict()
+            test_map[func.__name__]["result"] = SUCCESSED
+            test_map[func.__name__]["error_message"] = ""
+            test_map[func.__name__]["error_stack"] = ""
+            test_map[func.__name__]["error_normalized"] = ""
+            test_map[func.__name__]["start_dt"] = dt.datetime.utcnow()
+            ret = func_to_call(*args, **kwargs)
+        except (AssertionError, AzureError, CliTestError, CliExecutionError, SystemExit,
+                JMESPathCheckAssertionError) as e:
+            use_exception_cache = os.getenv("TEST_EXCEPTION_CACHE")
+            if use_exception_cache is None or use_exception_cache.lower() != "true":
+                raise
+            test_map[func.__name__]["end_dt"] = dt.datetime.utcnow()
+            test_map[func.__name__]["result"] = FAILED
+            test_map[func.__name__]["error_message"] = str(e).replace("\r\n", " ").replace("\n", " ")[:500]
+            test_map[func.__name__]["error_stack"] = traceback.format_exc().replace(
+                "\r\n", " ").replace("\n", " ")[:500]
+            logger.info("--------------------------------------")
+            logger.info("step exception: %s", e)
+            logger.error("--------------------------------------")
+            logger.error("step exception in %s: %s", func.__name__, e)
+            logger.info(traceback.format_exc())
+            exceptions.append((func.__name__, sys.exc_info()))
+        else:
+            test_map[func.__name__]["end_dt"] = dt.datetime.utcnow()
+            return ret
+
+    if inspect.isclass(func):
+        return get_func_to_call()
+    return wrapper
+
+
+def calc_coverage(filename):
+    filename = filename.split(".")[0]
+    coverage_name = filename + "_coverage.md"
+    with open(coverage_name, "w") as f:
+        f.write("|Scenario|Result|ErrorMessage|ErrorStack|ErrorNormalized|StartDt|EndDt|\n")
+        total = len(test_map)
+        covered = 0
+        for k, v in test_map.items():
+            if not k.startswith("step_"):
+                total -= 1
+                continue
+            if v["result"] == SUCCESSED:
+                covered += 1
+            f.write("|{step_name}|{result}|{error_message}|{error_stack}|{error_normalized}|{start_dt}|"
+                    "{end_dt}|\n".format(step_name=k, **v))
+        f.write("Coverage: {}/{}\n".format(covered, total))
+    print("Create coverage\n", file=sys.stderr)
+
+
+def raise_if():
+    if exceptions:
+        if len(exceptions) <= 1:
+            raise exceptions[0][1][1]
+        message = "{}\nFollowed with exceptions in other steps:\n".format(str(exceptions[0][1][1]))
+        message += "\n".join(["{}: {}".format(h[0], h[1][1]) for h in exceptions[1:]])
+        raise exceptions[0][1][0](message).with_traceback(exceptions[0][1][2])