Add support for organization level metrics

snowflake/README.md

@@ -31,7 +31,7 @@ datadog-agent integration install datadog-snowflake==2.0.1
 
 1. Create a Datadog specific role and user to monitor Snowflake. In Snowflake, run the following to create a custom role with access to the ACCOUNT_USAGE schema.
 
-    Note: By default, this integration monitors the `SNOWFLAKE` database and `ACCOUNT_USAGE` schema.
+    Note: By default, this integration monitors the `SNOWFLAKE` database and `ACCOUNT_USAGE` schema. See "Collecting Organization Data" for information on how to monitor the `ORGANIZATION_USAGE` schema.
     This database is available by default and only viewable by users in the `ACCOUNTADMIN` role or [any role granted by the ACCOUNTADMIN][4].
 
 
@@ -117,6 +117,39 @@ datadog-agent integration install datadog-snowflake==2.0.1
 
 3. [Restart the Agent][6].
 
+#### Collecting Organization Data
+
+By default, this integration monitors the `ACCOUNT_USAGE` schema, but it can be set to monitor organization-level metrics instead.
+
+To collect organization metrics, you can create configure the integration by changing the schema field to `ORGANIZATION_USAGE` and increasing the `min_collection_interval` to 43200 to reduce the number of queries to Snowflake, as most organization queries have a latency of up to 24 hours.
+
+Note: To monitor organization metrics, your `user` must have the `ORGADMIN` role.
+
+    ```yaml
+       - schema: ORGANIZATION_USAGE
+         min_collection_interval: 43200
+    ```
+
+
+Additionally, you can monitor both account and organization metrics at the same time:
+
+    ```yaml
+    instances:
+      - account: example-inc
+        username: DATADOG_ORG_ADMIN
+        password: '<PASSWORD>'
+        role: SYSADMIN
+        database: ORGANIZATION_USAGE
+        min_collection_interval: 3600
+
+      - account: example-inc
+        username: DATADOG_ACCOUNT_ADMIN
+        password: '<PASSWORD>'
+        role: DATADOG_ADMIN
+        database: ACCOUNT_USAGE
+        min_collection_interval: 43200
+    ```
+
 #### Collecting data for multiple environments
 
 If you want to collect data for multiple Snowflake environments, add each environment as an instance in your `snowflake.d/conf.yaml` file. For example, if you needed to collect data for two users named `DATADOG_SYSADMIN` and `DATADOG_USER`:

snowflake/assets/configuration/spec.yaml

@@ -64,7 +64,10 @@ files:
           type: string
           example: SNOWFLAKE
       - name: schema
-        description: Name of the default schema to use for the database.
+        description: |
+          Name of the default schema to use for the database.
+
+          To collect organization level metrics, set this to ORGANIZATION_USAGE
         value:
           type: string
           example: ACCOUNT_USAGE
@@ -144,7 +147,7 @@ files:
         description: |
           List Snowflake metric groups to collect. Metric groups are determined by the metric prefixes.
 
-          The available metric groups are:
+          When querying the ACCOUNT_USAGE schema, the available metric groups are:
 
             - snowflake.query (enabled by default)
             - snowflake.billing (enabled by default)
@@ -156,6 +159,16 @@ files:
             - snowflake.auto_recluster
             - snowflake.pipe
             - snowflake.replication
+
+          When querying the ORGANIZATION_USAGE schema, the available metric groups are:
+            - snowflake.organization.warehouse (enabled by default)
+            - snowflake.organization.currency (enabled by default)
+            - snowflake.organization.credit
+            - snowflake.organization.storage (enabled by default)
+            - snowflake.organization.contracts
+            - snowflake.organization.balance
+            - snowflake.organization.rate
+            - snowflake.organization.data_transfer
         value:
           type: array
           items:

snowflake/datadog_checks/snowflake/check.py

@@ -13,7 +13,7 @@
 from . import queries
 from .config import Config
 
-METRIC_GROUPS = {
+ACCOUNT_USAGE_METRIC_GROUPS = {
     'snowflake.query': [queries.WarehouseLoad, queries.QueryHistory],
     'snowflake.billing': [queries.CreditUsage, queries.WarehouseCreditUsage],
     'snowflake.storage': [queries.StorageUsageMetrics],
@@ -26,6 +26,17 @@
     'snowflake.replication': [queries.ReplicationUsage],
 }
 
+ORGANIZATION_USAGE_METRIC_GROUPS = {
+    'snowflake.organization.contracts': [queries.OrgContractItems],
+    'snowflake.organization.credit': [queries.OrgCreditUsage],
+    'snowflake.organization.currency': [queries.OrgCurrencyUsage],
+    'snowflake.organization.warehouse': [queries.OrgWarehouseCreditUsage],
+    'snowflake.organization.storage': [queries.OrgStorageDaily],
+    'snowflake.organization.balance': [queries.OrgBalance],
+    'snowflake.organization.rate': [queries.OrgRateSheet],
+    'snowflake.organization.data_transfer': [queries.OrgDataTransfer],
+}
+
 
 class SnowflakeCheck(AgentCheck):
     """
@@ -60,24 +71,32 @@ def __init__(self, *args, **kwargs):
                 'Snowflake `role` is set as `ACCOUNTADMIN` which should be used cautiously, '
                 'refer to docs about custom roles.'
             )
-
+        metric_groups = (
+            ORGANIZATION_USAGE_METRIC_GROUPS
+            if (self._config.schema == 'ORGANIZATION_USAGE')
+            else ACCOUNT_USAGE_METRIC_GROUPS
+        )
         self.metric_queries = []
         self.errors = []
         for mgroup in self._config.metric_groups:
             try:
                 if not self._config.aggregate_last_24_hours:
-                    for query in range(len(METRIC_GROUPS[mgroup])):
-                        METRIC_GROUPS[mgroup][query]['query'] = METRIC_GROUPS[mgroup][query]['query'].replace(
+                    for query in range(len(metric_groups[mgroup])):
+                        metric_groups[mgroup][query]['query'] = metric_groups[mgroup][query]['query'].replace(
                             'DATEADD(hour, -24, current_timestamp())', 'date_trunc(day, current_date)'
                         )
-                self.metric_queries.extend(METRIC_GROUPS[mgroup])
+                self.metric_queries.extend(metric_groups[mgroup])
             except KeyError:
                 self.errors.append(mgroup)
 
         if self.errors:
-            self.log.warning('Invalid metric_groups found in snowflake conf.yaml: %s', (', '.join(self.errors)))
+            self.log.warning('Invalid metric_groups forfound in snowflake conf.yaml: %s', (', '.join(self.errors)))
         if not self.metric_queries and not self._config.custom_queries_defined:
-            raise ConfigurationError('No valid metric_groups or custom query configured, please list at least one.')
+            raise ConfigurationError(
+                'No valid metric_groups for `{}` or custom query configured, please list at least one.'.format(
+                    self._config.schema
+                )
+            )
 
         self._query_manager = QueryManager(self, self.execute_query_raw, queries=self.metric_queries, tags=self._tags)
         self.check_initializations.append(self._query_manager.compile_queries)

snowflake/datadog_checks/snowflake/config.py

@@ -12,13 +12,19 @@ class Config(object):
     Encapsulates the validation of an `instance` dictionary and authentication options.
     """
 
-    DEFAULT_METRIC_GROUP = [
+    DEFAULT_METRIC_GROUPS = [
         'snowflake.query',
         'snowflake.billing',
         'snowflake.storage',
         'snowflake.logins',
     ]
 
+    DEFAULT_ORG_METRIC_GROUPS = [
+        'snowflake.organization.warehouse',
+        'snowflake.organization.currency',
+        'snowflake.organization.storage',
+    ]
+
     AUTHENTICATION_MODES = ['snowflake', 'oauth', 'snowflake_jwt']
 
     def __init__(self, instance=None):
@@ -47,7 +53,10 @@ def __init__(self, instance=None):
         aggregate_last_24_hours = instance.get('aggregate_last_24_hours', False)
         custom_queries_defined = len(instance.get('custom_queries', [])) > 0
 
-        metric_groups = instance.get('metric_groups', self.DEFAULT_METRIC_GROUP)
+        default_metric_groups = (
+            self.DEFAULT_ORG_METRIC_GROUPS if schema == 'ORGANIZATION_USAGE' else self.DEFAULT_METRIC_GROUPS
+        )
+        metric_groups = instance.get('metric_groups', default_metric_groups)
 
         if account is None:
             raise ConfigurationError('Must specify an account')

snowflake/datadog_checks/snowflake/data/conf.yaml.example

@@ -77,6 +77,8 @@ instances:
 
     ## @param schema - string - optional - default: ACCOUNT_USAGE
     ## Name of the default schema to use for the database.
+    ##
+    ## To collect organization level metrics, set this to ORGANIZATION_USAGE
     #
     # schema: ACCOUNT_USAGE
 
@@ -147,7 +149,7 @@ instances:
     ## @param metric_groups - list of strings - optional
     ## List Snowflake metric groups to collect. Metric groups are determined by the metric prefixes.
     ##
-    ## The available metric groups are:
+    ## When querying the ACCOUNT_USAGE schema, the available metric groups are:
     ##
     ##   - snowflake.query (enabled by default)
     ##   - snowflake.billing (enabled by default)
@@ -159,6 +161,16 @@ instances:
     ##   - snowflake.auto_recluster
     ##   - snowflake.pipe
     ##   - snowflake.replication
+    ##
+    ## When querying the ORGANIZATION_USAGE schema, the available metric groups are:
+    ##   - snowflake.organization.warehouse (enabled by default)
+    ##   - snowflake.organization.currency (enabled by default)
+    ##   - snowflake.organization.credit
+    ##   - snowflake.organization.storage (enabled by default)
+    ##   - snowflake.organization.contracts
+    ##   - snowflake.organization.balance
+    ##   - snowflake.organization.rate
+    ##   - snowflake.organization.data_transfer
     #
     # metric_groups:
     #   - snowflake.query

snowflake/datadog_checks/snowflake/queries.py

@@ -2,7 +2,7 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
-# https://docs.snowflak e.com/en/sql-reference/account-usage/storage_usage.html
+# https://docs.snowflake.com/en/sql-reference/account-usage/storage_usage.html
 StorageUsageMetrics = {
     'name': 'storage.metrics',
     'query': ('SELECT STORAGE_BYTES, STAGE_BYTES, FAILSAFE_BYTES from STORAGE_USAGE ORDER BY USAGE_DATE DESC LIMIT 1;'),
@@ -52,7 +52,7 @@
 
 # https://docs.snowflake.com/en/sql-reference/account-usage/warehouse_metering_history.html
 WarehouseCreditUsage = {
-    'name': 'billings.warehouse.metrics',
+    'name': 'billing.warehouse.metrics',
     'query': (
         'select WAREHOUSE_NAME, sum(CREDITS_USED_COMPUTE), avg(CREDITS_USED_COMPUTE), '
         'sum(CREDITS_USED_CLOUD_SERVICES), avg(CREDITS_USED_CLOUD_SERVICES), '
@@ -222,3 +222,146 @@
         {'name': 'replication.bytes_transferred.sum', 'type': 'gauge'},
     ],
 }
+
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/contract_items.html
+OrgContractItems = {
+    'name': 'organization.contract.metrics',
+    'query': ('select CONTRACT_NUMBER, CONTRACT_ITEM, CURRENCY, AMOUNT from CONTRACT_ITEMS group by 1, 2, 3;'),
+    'columns': [
+        {'name': 'contract_number', 'type': 'tag'},
+        {'name': 'contract_item', 'type': 'tag'},
+        {'name': 'currency', 'type': 'tag'},
+        {'name': 'organization.contract.amount', 'type': 'gauge'},
+    ],
+}
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/metering_daily_history.html
+OrgCreditUsage = {
+    'name': 'organization.credit.metrics',
+    'query': (
+        'select ACCOUNT_NAME, SERVICE_TYPE, '
+        'sum(CREDITS_USED_COMPUTE), avg(CREDITS_USED_COMPUTE), '
+        'sum(CREDITS_USED_CLOUD_SERVICES), avg(CREDITS_USED_CLOUD_SERVICES), '
+        'sum(CREDITS_ADJUSTMENT_CLOUD_SERVICES), avg(CREDITS_ADJUSTMENT_CLOUD_SERVICES), '
+        'sum(CREDITS_USED), avg(CREDITS_USED), sum(CREDITS_BILLED), avg(CREDITS_BILLED) from METERING_DAILY_HISTORY '
+        'where USAGE_DATE = DATEADD(day, -1, current_date) group by 1, 2;'
+    ),
+    'columns': [
+        {'name': 'billing_account', 'type': 'tag'},
+        {'name': 'service_type', 'type': 'tag'},
+        {'name': 'organization.virtual_warehouse.sum', 'type': 'gauge'},
+        {'name': 'organization.virtual_warehouse.avg', 'type': 'gauge'},
+        {'name': 'organization.cloud_service.sum', 'type': 'gauge'},
+        {'name': 'organization.cloud_service.avg', 'type': 'gauge'},
+        {'name': 'organization.cloud_service_adjustment.sum', 'type': 'gauge'},
+        {'name': 'organization.cloud_service_adjustment.avg', 'type': 'gauge'},
+        {'name': 'organization.total_credit.sum', 'type': 'gauge'},
+        {'name': 'organization.total_credit.avg', 'type': 'gauge'},
+        {'name': 'organization.total_credits_billed.sum', 'type': 'gauge'},
+        {'name': 'organization.total_credits_billed.avg', 'type': 'gauge'},
+    ],
+}
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/usage_in_currency_daily.html
+OrgCurrencyUsage = {
+    'name': 'organization.currency.metrics',
+    'query': (
+        'select ACCOUNT_NAME, SERVICE_LEVEL, USAGE_TYPE, CURRENCY, '
+        'sum(USAGE), sum(USAGE_IN_CURRENCY) from USAGE_IN_CURRENCY_DAILY '
+        'where USAGE_DATE = DATEADD(day, -1, current_date) group by 1, 2, 3, 4;'
+    ),
+    'columns': [
+        {'name': 'billing_account', 'type': 'tag'},
+        {'name': 'service_level', 'type': 'tag'},
+        {'name': 'usage_type', 'type': 'tag'},
+        {'name': 'currency', 'type': 'tag'},
+        {'name': 'organization.currency.usage', 'type': 'gauge'},
+        {'name': 'organization.currency.usage_in_currency', 'type': 'gauge'},
+    ],
+}
+
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/warehouse_metering_history.html
+OrgWarehouseCreditUsage = {
+    'name': 'organization.warehouse.metrics',
+    'query': (
+        'select WAREHOUSE_NAME, ACCOUNT_NAME, sum(CREDITS_USED_COMPUTE), avg(CREDITS_USED_COMPUTE), '
+        'sum(CREDITS_USED_CLOUD_SERVICES), avg(CREDITS_USED_CLOUD_SERVICES), '
+        'sum(CREDITS_USED), avg(CREDITS_USED) from WAREHOUSE_METERING_HISTORY '
+        'where start_time = DATEADD(day, -1, current_date) group by 1, 2;'
+    ),
+    'columns': [
+        {'name': 'warehouse', 'type': 'tag'},
+        {'name': 'billing_account', 'type': 'tag'},
+        {'name': 'organization.warehouse.virtual_warehouse.sum', 'type': 'gauge'},
+        {'name': 'organization.warehouse.virtual_warehouse.avg', 'type': 'gauge'},
+        {'name': 'organization.warehouse.cloud_service.sum', 'type': 'gauge'},
+        {'name': 'organization.warehouse.cloud_service.avg', 'type': 'gauge'},
+        {'name': 'organization.warehouse.total_credit.sum', 'type': 'gauge'},
+        {'name': 'organization.warehouse.total_credit.avg', 'type': 'gauge'},
+    ],
+}
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/storage_daily_history.html
+OrgStorageDaily = {
+    'name': 'organization.storage.metrics',
+    'query': (
+        'select ACCOUNT_NAME, sum(AVERAGE_BYTES), sum(CREDITS) from STORAGE_DAILY_HISTORY '
+        'where USAGE_DATE = DATEADD(day, -1, current_date) group by 1;'
+    ),
+    'columns': [
+        {'name': 'billing_account', 'type': 'tag'},
+        {'name': 'organization.storage.average_bytes', 'type': 'gauge'},
+        {'name': 'organization.storage.credits', 'type': 'gauge'},
+    ],
+}
+
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/remaining_balance_daily.html
+OrgBalance = {
+    'name': 'organization.balance.metrics',
+    'query': (
+        'select CONTRACT_NUMBER, CURRENCY, sum(FREE_USAGE_BALANCE), sum(CAPACITY_BALANCE), '
+        'sum(ON_DEMAND_CONSUMPTION_BALANCE), sum(ROLLOVER_BALANCE) from REMAINING_BALANCE_DAILY '
+        'where DATE = DATEADD(day, -1, current_date) group by 1, 2;'
+    ),
+    'columns': [
+        {'name': 'contract_number', 'type': 'tag'},
+        {'name': 'currency', 'type': 'tag'},
+        {'name': 'organization.balance.free_usage', 'type': 'gauge'},
+        {'name': 'organization.balance.capacity', 'type': 'gauge'},
+        {'name': 'organization.balance.on_demand_consumption', 'type': 'gauge'},
+        {'name': 'organization.balance.rollover', 'type': 'gauge'},
+    ],
+}
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/rate_sheet_daily.html
+OrgRateSheet = {
+    'name': 'organization.rate.metrics',
+    'query': (
+        'select CONTRACT_NUMBER, ACCOUNT_NAME, USAGE_TYPE, SERVICE_TYPE, CURRENCY, '
+        'sum(EFFECTIVE_RATE) from RATE_SHEET_DAILY '
+        'where DATE = DATEADD(day, -1, current_date) group by 1, 2, 3, 4, 5;'
+    ),
+    'columns': [
+        {'name': 'contract_number', 'type': 'tag'},
+        {'name': 'billing_account', 'type': 'tag'},
+        {'name': 'usage_type', 'type': 'tag'},
+        {'name': 'service_type', 'type': 'tag'},
+        {'name': 'currency', 'type': 'tag'},
+        {'name': 'organization.rate.effective_rate', 'type': 'gauge'},
+    ],
+}
+
+# https://docs.snowflake.com/en/sql-reference/organization-usage/data_transfer_history.html
+OrgDataTransfer = {
+    'name': 'organization.data_transfer.metrics',
+    'query': (
+        'select ACCOUNT_NAME, BYTES_TRANSFERRED from DATA_TRANSFER_DAILY_HISTORY ORDER BY USAGE_DATE DESC LIMIT 1;'
+    ),
+    'columns': [
+        {'name': 'billing_account', 'type': 'tag'},
+        {'name': 'organization.data_transfer.bytes_transferred', 'type': 'gauge'},
+    ],
+}