From 7edb623c0ab8a97e4a410f88e9c0f53a03c2da0e Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 21 Nov 2024 20:21:15 +0000 Subject: [PATCH 01/18] Deprecate usage of instance_config in favor of config in database_metrics --- sqlserver/datadog_checks/sqlserver/config.py | 29 ++++++++++++++----- .../sqlserver/config_models/defaults.py | 2 +- sqlserver/datadog_checks/sqlserver/const.py | 2 ++ .../sqlserver/database_metrics/ao_metrics.py | 2 +- .../availability_groups_metrics.py | 5 ++-- .../availability_replicas_metrics.py | 9 +++--- .../sqlserver/database_metrics/base.py | 2 -- .../database_agent_metrics.py | 4 +-- .../database_backup_metrics.py | 10 ++----- .../database_replication_stats_metrics.py | 6 ++-- .../db_fragmentation_metrics.py | 15 +++------- .../sqlserver/database_metrics/fci_metrics.py | 4 +-- .../database_metrics/index_usage_metrics.py | 13 ++------- .../database_metrics/master_files_metrics.py | 2 +- .../database_metrics/os_schedulers_metrics.py | 2 +- .../database_metrics/os_tasks_metrics.py | 2 +- .../primary_log_shipping_metrics.py | 4 +-- .../secondary_log_shipping_metrics.py | 4 +-- .../tempdb_file_space_usage_metrics.py | 5 ++-- .../database_metrics/xe_session_metrics.py | 4 +-- .../datadog_checks/sqlserver/sqlserver.py | 1 - sqlserver/tests/test_database_metrics.py | 24 ++------------- 22 files changed, 62 insertions(+), 89 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/config.py b/sqlserver/datadog_checks/sqlserver/config.py index 92508a44e5721..67a0fb426f24f 100644 --- a/sqlserver/datadog_checks/sqlserver/config.py +++ b/sqlserver/datadog_checks/sqlserver/config.py @@ -10,6 +10,7 @@ from datadog_checks.base.utils.db.utils import get_agent_host_tags from datadog_checks.sqlserver.const import ( DEFAULT_AUTODISCOVERY_INTERVAL, + DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, PROC_CHAR_LIMIT, ) @@ -28,17 +29,15 @@ def __init__(self, init_config, instance, log): self.autodiscovery_db_service_check: bool = is_affirmative(instance.get('autodiscovery_db_service_check', True)) self.min_collection_interval: int = instance.get('min_collection_interval', 15) self.autodiscovery_interval: int = instance.get('autodiscovery_interval', DEFAULT_AUTODISCOVERY_INTERVAL) + self.database_backup_metrics_interval: int = instance.get('database_backup_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) + self.index_usage_stats_interval: int = instance.get('index_usage_stats_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) + self.db_fragmentation_metrics_interval: int = instance.get('db_fragmentation_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) + self.database_instance_collection_interval: int = instance.get('database_instance_collection_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) self._include_patterns = self._compile_valid_patterns(self.autodiscovery_include) self._exclude_patterns = self._compile_valid_patterns(self.autodiscovery_exclude) self.proc: str = instance.get('stored_procedure') self.custom_metrics: list[dict] = init_config.get('custom_metrics', []) or [] - self.include_index_usage_metrics_tempdb: bool = is_affirmative( - instance.get('include_index_usage_metrics_tempdb', False) - ) - self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative( - instance.get('include_db_fragmentation_metrics_tempdb', False) - ) self.ignore_missing_database = is_affirmative(instance.get("ignore_missing_database", False)) if self.ignore_missing_database: self.log.warning( @@ -110,10 +109,26 @@ def __init__(self, init_config, instance, log): ) self.log_unobfuscated_queries: bool = is_affirmative(instance.get('log_unobfuscated_queries', False)) self.log_unobfuscated_plans: bool = is_affirmative(instance.get('log_unobfuscated_plans', False)) - self.database_instance_collection_interval: int = instance.get('database_instance_collection_interval', 300) self.stored_procedure_characters_limit: int = instance.get('stored_procedure_characters_limit', PROC_CHAR_LIMIT) self.connection_host: str = instance['host'] self.service = instance.get('service') or init_config.get('service') or '' + self.availability_group = instance.get('availability_group') # TODO type hint... string? + self.only_emit_local = is_affirmative(instance.get('only_emit_local', False)) + self.ao_database = instance.get('ao_database') + self.db_fragmentation_object_names = instance.get('db_fragmentation_object_names', []) or [] + # DBM Metrics Includes + self.include_ao_metrics: bool = is_affirmative(instance.get('include_ao_metrics', False)) + self.include_master_files_metrics: bool = is_affirmative(instance.get('include_master_files_metrics', False)) + self.include_fci_metrics: bool = is_affirmative(instance.get('include_fci_metrics', False)) + self.include_primary_log_shipping_metrics: bool = is_affirmative(instance.get('include_primary_log_shipping_metrics', False)) + self.include_secondary_log_shipping_metrics: bool = is_affirmative(instance.get('include_secondary_log_shipping_metrics', False)) + self.include_task_scheduler_metrics: bool = is_affirmative(instance.get('include_task_scheduler_metrics', False)) + self.include_db_fragmentation_metrics: bool = is_affirmative(instance.get('include_db_fragmentation_metrics', False)) + self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative(instance.get('include_db_fragmentation_metrics_tempdb', False)) + self.include_index_usage_metrics: bool = is_affirmative(instance.get('include_index_usage_metrics', True)) + self.include_index_usage_metrics_tempdb: bool = is_affirmative(instance.get('include_index_usage_metrics_tempdb', False)) + self.include_tempdb_file_space_usage_metrics: bool = is_affirmative(instance.get('include_tempdb_file_space_usage_metrics', True)) + self.include_xe_metrics: bool = is_affirmative(instance.get('include_xe_metrics', False)) def _compile_valid_patterns(self, patterns: list[str]) -> re.Pattern: valid_patterns = [] diff --git a/sqlserver/datadog_checks/sqlserver/config_models/defaults.py b/sqlserver/datadog_checks/sqlserver/config_models/defaults.py index 8dfc29b5ddfa7..e7c2d25da0041 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/defaults.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/defaults.py @@ -41,7 +41,7 @@ def instance_database_autodiscovery_interval(): def instance_database_instance_collection_interval(): - return False + return 300 def instance_dbm(): diff --git a/sqlserver/datadog_checks/sqlserver/const.py b/sqlserver/datadog_checks/sqlserver/const.py index 7fc1cf86a659e..53cc3bd0bc938 100644 --- a/sqlserver/datadog_checks/sqlserver/const.py +++ b/sqlserver/datadog_checks/sqlserver/const.py @@ -271,3 +271,5 @@ PROC_CHAR_LIMIT = 500 DEFAULT_SCHEMAS_COLLECTION_INTERVAL = 600 + +DEFAULT_LONG_METRICS_COLLECTION_INTERVAL = 300 \ No newline at end of file diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py index 31b3238ba8752..e6330c1d9577a 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py @@ -61,7 +61,7 @@ class SqlserverAoMetrics(SqlserverDatabaseMetricsBase): @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) + return self.config.include_ao_metrics @property def enabled(self) -> bool: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py index c3c7937f65c2b..290f708e4f27d 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py @@ -38,11 +38,10 @@ class SqlserverAvailabilityGroupsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-hadr-availability-group-states-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) - + return self.config.include_ao_metrics @property def availability_group(self): - return self.instance_config.get('availability_group') + return self.config.availability_group @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py index 95b0c9041108c..9cfd3c6fe026a 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py @@ -49,19 +49,18 @@ class SqlserverAvailabilityReplicasMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-availability-replicas-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) - + return self.config.include_ao_metrics @property def availability_group(self): - return self.instance_config.get('availability_group') + return self.config.availability_group @property def only_emit_local(self): - return is_affirmative(self.instance_config.get('only_emit_local', False)) + return self.config.only_emit_local @property def ao_database(self): - return self.instance_config.get('ao_database') + return self.config.ao_database @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/base.py b/sqlserver/datadog_checks/sqlserver/database_metrics/base.py index bc61f01c71491..013c68d9703e7 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/base.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/base.py @@ -15,7 +15,6 @@ class SqlserverDatabaseMetricsBase: def __init__( self, config, - instance_config, new_query_executor, server_static_info, execute_query_handler, @@ -23,7 +22,6 @@ def __init__( databases=None, ): self.config: SQLServerConfig = config - self.instance_config: dict = instance_config # TODO: Remove instance_config and use self.config self.server_static_info: dict = server_static_info self.new_query_executor: Callable[ [List[dict], Callable, Optional[List[str]], Optional[bool]], QueryExecutor diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py index 6d99655392eaf..0c2491b0b056f 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_agent_metrics.py @@ -110,7 +110,7 @@ class SqlserverAgentMetrics(SqlserverDatabaseMetricsBase): def include_agent_metrics(self) -> bool: if not self.config.dbm_enabled: return False - agent_jobs_config = self.instance_config.get('agent_jobs', {}) + agent_jobs_config = self.config.agent_jobs_config if agent_jobs_config: return is_affirmative(agent_jobs_config.get('enabled', False)) return False @@ -128,7 +128,7 @@ def collection_interval(self) -> int: Returns the interval in seconds at which to collect index usage metrics. Note: The index usage metrics query can be expensive, so it is recommended to set a higher interval. ''' - agent_jobs_config = self.instance_config.get('agent_jobs', {}) + agent_jobs_config = self.config.agent_jobs_config if agent_jobs_config: return int(agent_jobs_config.get('collection_interval', 15)) return 15 # 15 seconds diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py index e82cd65963204..bfc68397272da 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py @@ -32,24 +32,18 @@ class SqlserverDatabaseBackupMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-tables/backupset-transact-sql?view=sql-server-ver15 @property def enabled(self): + # Allen Change here if is_azure_sql_database(self.engine_edition): return False return True - @property - def _default_collection_interval(self) -> int: - ''' - Returns the default interval in seconds at which to collect database backup metrics. - ''' - return 5 * 60 # 5 minutes - @property def collection_interval(self) -> int: ''' Returns the interval in seconds at which to collect database backup metrics. Note: The database backup metrics query can be expensive, so it is recommended to set a higher interval. ''' - return int(self.instance_config.get('database_backup_metrics_interval', self._default_collection_interval)) + return self.config.database_backup_metrics_interval @property def queries(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py index e1233e353cf83..83653605629d1 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py @@ -34,15 +34,15 @@ class SqlserverDatabaseReplicationStatsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-hadr-database-replica-states-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_ao_metrics', False)) + return self.config.include_ao_metrics @property def availability_group(self): - return self.instance_config.get('availability_group') + return self.config.availability_group @property def only_emit_local(self): - return is_affirmative(self.instance_config.get('only_emit_local', False)) + return self.config.only_emit_local @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py index 2b05f9378d37c..47907272d3dd7 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py @@ -52,15 +52,15 @@ class SqlserverDBFragmentationMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-index-physical-stats-transact-sql?view=sql-server-ver15 @property def include_db_fragmentation_metrics(self): - return is_affirmative(self.instance_config.get('include_db_fragmentation_metrics', False)) + return self.config.include_db_fragmentation_metrics @property def include_db_fragmentation_metrics_tempdb(self): - return is_affirmative(self.instance_config.get('include_db_fragmentation_metrics_tempdb', False)) + return self.config.include_db_fragmentation_metrics_tempdb @property def db_fragmentation_object_names(self): - return self.instance_config.get('db_fragmentation_object_names', []) or [] + return self.config.db_fragmentation_object_names @property def enabled(self): @@ -68,20 +68,13 @@ def enabled(self): return False return True - @property - def _default_collection_interval(self) -> int: - ''' - Returns the default interval in seconds at which to collect database index fragmentation metrics. - ''' - return 5 * 60 # 5 minutes - @property def collection_interval(self) -> int: ''' Returns the interval in seconds at which to collect database index fragmentation metrics. Note: The index fragmentation metrics query can be expensive, so it is recommended to set a higher interval. ''' - return int(self.instance_config.get('db_fragmentation_metrics_interval', self._default_collection_interval)) + return self.config.db_fragmentation_metrics_interval @property def databases(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py index 81bbf485509d3..2045262436de6 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py @@ -37,8 +37,8 @@ class SqlserverFciMetrics(SqlserverDatabaseMetricsBase): @property - def include_fci_metrics(self): - return is_affirmative(self.instance_config.get('include_fci_metrics', False)) + def include_fci_metrics(self) -> bool: + return self.config.include_fci_metrics @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py index 34c9b24e8d668..cf9183c848ef7 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py @@ -45,18 +45,11 @@ class SqlserverIndexUsageMetrics(SqlserverDatabaseMetricsBase): @property def include_index_usage_metrics(self) -> bool: - return is_affirmative(self.instance_config.get('include_index_usage_metrics', True)) + return self.config.include_index_usage_metrics @property def include_index_usage_metrics_tempdb(self) -> bool: - return is_affirmative(self.instance_config.get('include_index_usage_metrics_tempdb', False)) - - @property - def _default_collection_interval(self) -> int: - ''' - Returns the default interval in seconds at which to collect index usage metrics. - ''' - return 5 * 60 # 5 minutes + return self.config.include_index_usage_metrics_tempdb @property def collection_interval(self) -> int: @@ -64,7 +57,7 @@ def collection_interval(self) -> int: Returns the interval in seconds at which to collect index usage metrics. Note: The index usage metrics query can be expensive, so it is recommended to set a higher interval. ''' - return int(self.instance_config.get('index_usage_stats_interval', self._default_collection_interval)) + return self.config.index_usage_stats_interval @property def databases(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py index a3ebf5edd89ef..cff9287519e76 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py @@ -48,7 +48,7 @@ class SqlserverMasterFilesMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-master-files-transact-sql @property def include_master_files_metrics(self): - return is_affirmative(self.instance_config.get('include_master_files_metrics', False)) + return self.config.include_master_files_metrics @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py index c1dcfd8092652..17b4715b88382 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py @@ -34,7 +34,7 @@ class SqlserverOsSchedulersMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-schedulers-transact-sql @property def include_task_scheduler_metrics(self): - return is_affirmative(self.instance_config.get('include_task_scheduler_metrics', False)) + return self.config.include_task_scheduler_metrics @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py index 036ef35e36bd8..d3da226cfb659 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py @@ -30,7 +30,7 @@ class SqlserverOsTasksMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-tasks-transact-sql @property def include_task_scheduler_metrics(self): - return is_affirmative(self.instance_config.get('include_task_scheduler_metrics', False)) + return self.config.include_task_scheduler_metrics @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py index 66724c1bacb09..bdd8973397357 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py @@ -29,8 +29,8 @@ class SqlserverPrimaryLogShippingMetrics(SqlserverDatabaseMetricsBase): @property - def include_primary_log_shipping_metrics(self): - return is_affirmative(self.instance_config.get('include_primary_log_shipping_metrics', False)) + def include_primary_log_shipping_metrics(self) -> bool: + return self.config.include_primary_log_shipping_metrics @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py index daa4476058b2d..72f59b4678aa9 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py @@ -37,8 +37,8 @@ class SqlserverSecondaryLogShippingMetrics(SqlserverDatabaseMetricsBase): @property - def include_secondary_log_shipping_metrics(self): - return is_affirmative(self.instance_config.get('include_secondary_log_shipping_metrics', False)) + def include_secondary_log_shipping_metrics(self) -> bool: + return self.config.include_secondary_log_shipping_metrics @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py index 6d926c0c2f74f..40bdb476348b4 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py @@ -32,9 +32,8 @@ class SqlserverTempDBFileSpaceUsageMetrics(SqlserverDatabaseMetricsBase): @property - def include_tempdb_file_space_usage_metrics(self): - return is_affirmative(self.instance_config.get('include_tempdb_file_space_usage_metrics', True)) - + def include_tempdb_file_space_usage_metrics(self) -> bool: + return self.config.include_tempdb_file_space_usage_metrics @property def enabled(self): if not self.include_tempdb_file_space_usage_metrics: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py index 5f5d57bbe938e..48a5c17eea795 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py @@ -44,8 +44,8 @@ class SQLServerXESessionMetrics(SqlserverDatabaseMetricsBase): @property def enabled(self): - self.deadlocks_config: dict = self.instance_config.get('deadlocks_collection', {}) or {} - return is_affirmative(self.instance_config.get("include_xe_metrics", False)) or is_affirmative( + self.deadlocks_config: dict = self.config.deadlocks_config + return self.config.include_xe_metrics or is_affirmative( self.deadlocks_config.get('enabled', False) ) diff --git a/sqlserver/datadog_checks/sqlserver/sqlserver.py b/sqlserver/datadog_checks/sqlserver/sqlserver.py index c9ba81885363d..134e09099115f 100644 --- a/sqlserver/datadog_checks/sqlserver/sqlserver.py +++ b/sqlserver/datadog_checks/sqlserver/sqlserver.py @@ -746,7 +746,6 @@ def check(self, _): def _new_database_metric_executor(self, database_metric_class, db_names=None): return database_metric_class( config=self._config, - instance_config=self.instance, new_query_executor=self._new_query_executor, server_static_info=self.static_info_cache, execute_query_handler=self.execute_query_raw, diff --git a/sqlserver/tests/test_database_metrics.py b/sqlserver/tests/test_database_metrics.py index 0330ebbe36a1e..2fd87781ba0d2 100644 --- a/sqlserver/tests/test_database_metrics.py +++ b/sqlserver/tests/test_database_metrics.py @@ -78,7 +78,6 @@ def execute_query_handler_mocked(query, db=None): file_stats_metrics = SqlserverFileStatsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -155,7 +154,6 @@ def test_sqlserver_ao_metrics( ao_metrics = SqlserverAoMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -265,7 +263,6 @@ def execute_query_handler_mocked(query, db=None): availability_groups_metrics = SqlserverAvailabilityGroupsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -355,7 +352,6 @@ def execute_query_handler_mocked(query, db=None): database_replication_stats_metrics = SqlserverDatabaseReplicationStatsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -490,7 +486,6 @@ def execute_query_handler_mocked(query, db=None): availability_replicas_metrics = SqlserverAvailabilityReplicasMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -561,7 +556,6 @@ def execute_query_handler_mocked(query, db=None): fci_metrics = SqlserverFciMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -609,7 +603,6 @@ def execute_query_handler_mocked(query, db=None): primary_log_shipping_metrics = SqlserverPrimaryLogShippingMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -669,7 +662,6 @@ def execute_query_handler_mocked(query, db=None): primary_log_shipping_metrics = SqlserverSecondaryLogShippingMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -716,7 +708,6 @@ def execute_query_handler_mocked(query, db=None): server_state_metrics = SqlserverServerStateMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -757,7 +748,6 @@ def execute_query_handler_mocked(query, db=None): tempdb_file_space_usage_metrics = SqlserverTempDBFileSpaceUsageMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -830,14 +820,13 @@ def test_sqlserver_index_usage_metrics( index_usage_metrics = SqlserverIndexUsageMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, databases=AUTODISCOVERY_DBS + ['tempdb'], ) - expected_collection_interval = index_usage_stats_interval or index_usage_metrics._default_collection_interval + expected_collection_interval = index_usage_stats_interval or index_usage_metrics.collection_interval assert index_usage_metrics.queries[0]['collection_interval'] == expected_collection_interval sqlserver_check._database_metrics = [index_usage_metrics] @@ -934,7 +923,6 @@ def test_sqlserver_db_fragmentation_metrics( db_fragmentation_metrics = SqlserverDBFragmentationMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -945,7 +933,7 @@ def test_sqlserver_db_fragmentation_metrics( assert db_fragmentation_metrics.db_fragmentation_object_names == db_fragmentation_object_names expected_collection_interval = ( - db_fragmentation_metrics_interval or db_fragmentation_metrics._default_collection_interval + db_fragmentation_metrics_interval or db_fragmentation_metrics.collection_interval ) assert db_fragmentation_metrics.queries[0]['collection_interval'] == expected_collection_interval @@ -1024,7 +1012,6 @@ def execute_query_handler_mocked(query, db=None): os_schedulers_metrics = SqlserverOsSchedulersMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1089,7 +1076,6 @@ def execute_query_handler_mocked(query, db=None): os_tasks_metrics = SqlserverOsTasksMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1155,7 +1141,6 @@ def execute_query_handler_mocked(query, db=None): master_files_metrics = SqlserverMasterFilesMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1226,7 +1211,6 @@ def test_sqlserver_database_files_metrics( database_files_metrics = SqlserverDatabaseFilesMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1282,7 +1266,6 @@ def execute_query_handler_mocked(query, db=None): database_stats_metrics = SqlserverDatabaseStatsMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, @@ -1335,14 +1318,13 @@ def execute_query_handler_mocked(query, db=None): database_backup_metrics = SqlserverDatabaseBackupMetrics( config=sqlserver_check._config, - instance_config=instance_docker_metrics, new_query_executor=sqlserver_check._new_query_executor, server_static_info=STATIC_SERVER_INFO, execute_query_handler=execute_query_handler_mocked, ) expected_collection_interval = ( - database_backup_metrics_interval or database_backup_metrics._default_collection_interval + database_backup_metrics_interval or database_backup_metrics.collection_interval ) assert database_backup_metrics.queries[0]['collection_interval'] == expected_collection_interval From 0905a0d57afddeb8f4450c1ba2e8377d9a49700d Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 21 Nov 2024 20:47:05 +0000 Subject: [PATCH 02/18] Remove reminder comments to myself --- .../sqlserver/database_metrics/database_backup_metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py index bfc68397272da..f82fac9e035a8 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py @@ -32,7 +32,6 @@ class SqlserverDatabaseBackupMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-tables/backupset-transact-sql?view=sql-server-ver15 @property def enabled(self): - # Allen Change here if is_azure_sql_database(self.engine_edition): return False return True From 64b654372a540fccef74add34f90c7066f703da7 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 21 Nov 2024 21:06:15 +0000 Subject: [PATCH 03/18] Applied linter suggertions and added changelog --- sqlserver/changelog.d/19111.deprecated | 1 + sqlserver/datadog_checks/sqlserver/config.py | 46 ++++++++++++++----- sqlserver/datadog_checks/sqlserver/const.py | 2 +- .../availability_groups_metrics.py | 1 + .../availability_replicas_metrics.py | 1 + .../tempdb_file_space_usage_metrics.py | 1 + .../database_metrics/xe_session_metrics.py | 4 +- sqlserver/tests/test_database_metrics.py | 8 +--- 8 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 sqlserver/changelog.d/19111.deprecated diff --git a/sqlserver/changelog.d/19111.deprecated b/sqlserver/changelog.d/19111.deprecated new file mode 100644 index 0000000000000..f60055f7e4266 --- /dev/null +++ b/sqlserver/changelog.d/19111.deprecated @@ -0,0 +1 @@ +Deprecate usage of instance_config in database_metrics ([#18586](https://github.com/DataDog/integrations-core/pull/19111)) diff --git a/sqlserver/datadog_checks/sqlserver/config.py b/sqlserver/datadog_checks/sqlserver/config.py index 67a0fb426f24f..e5f9bb33976d4 100644 --- a/sqlserver/datadog_checks/sqlserver/config.py +++ b/sqlserver/datadog_checks/sqlserver/config.py @@ -29,10 +29,18 @@ def __init__(self, init_config, instance, log): self.autodiscovery_db_service_check: bool = is_affirmative(instance.get('autodiscovery_db_service_check', True)) self.min_collection_interval: int = instance.get('min_collection_interval', 15) self.autodiscovery_interval: int = instance.get('autodiscovery_interval', DEFAULT_AUTODISCOVERY_INTERVAL) - self.database_backup_metrics_interval: int = instance.get('database_backup_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) - self.index_usage_stats_interval: int = instance.get('index_usage_stats_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) - self.db_fragmentation_metrics_interval: int = instance.get('db_fragmentation_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) - self.database_instance_collection_interval: int = instance.get('database_instance_collection_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL) + self.database_backup_metrics_interval: int = instance.get( + 'database_backup_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL + ) + self.index_usage_stats_interval: int = instance.get( + 'index_usage_stats_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL + ) + self.db_fragmentation_metrics_interval: int = instance.get( + 'db_fragmentation_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL + ) + self.database_instance_collection_interval: int = instance.get( + 'database_instance_collection_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL + ) self._include_patterns = self._compile_valid_patterns(self.autodiscovery_include) self._exclude_patterns = self._compile_valid_patterns(self.autodiscovery_exclude) @@ -112,7 +120,7 @@ def __init__(self, init_config, instance, log): self.stored_procedure_characters_limit: int = instance.get('stored_procedure_characters_limit', PROC_CHAR_LIMIT) self.connection_host: str = instance['host'] self.service = instance.get('service') or init_config.get('service') or '' - self.availability_group = instance.get('availability_group') # TODO type hint... string? + self.availability_group = instance.get('availability_group') self.only_emit_local = is_affirmative(instance.get('only_emit_local', False)) self.ao_database = instance.get('ao_database') self.db_fragmentation_object_names = instance.get('db_fragmentation_object_names', []) or [] @@ -120,14 +128,28 @@ def __init__(self, init_config, instance, log): self.include_ao_metrics: bool = is_affirmative(instance.get('include_ao_metrics', False)) self.include_master_files_metrics: bool = is_affirmative(instance.get('include_master_files_metrics', False)) self.include_fci_metrics: bool = is_affirmative(instance.get('include_fci_metrics', False)) - self.include_primary_log_shipping_metrics: bool = is_affirmative(instance.get('include_primary_log_shipping_metrics', False)) - self.include_secondary_log_shipping_metrics: bool = is_affirmative(instance.get('include_secondary_log_shipping_metrics', False)) - self.include_task_scheduler_metrics: bool = is_affirmative(instance.get('include_task_scheduler_metrics', False)) - self.include_db_fragmentation_metrics: bool = is_affirmative(instance.get('include_db_fragmentation_metrics', False)) - self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative(instance.get('include_db_fragmentation_metrics_tempdb', False)) + self.include_primary_log_shipping_metrics: bool = is_affirmative( + instance.get('include_primary_log_shipping_metrics', False) + ) + self.include_secondary_log_shipping_metrics: bool = is_affirmative( + instance.get('include_secondary_log_shipping_metrics', False) + ) + self.include_task_scheduler_metrics: bool = is_affirmative( + instance.get('include_task_scheduler_metrics', False) + ) + self.include_db_fragmentation_metrics: bool = is_affirmative( + instance.get('include_db_fragmentation_metrics', False) + ) + self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative( + instance.get('include_db_fragmentation_metrics_tempdb', False) + ) self.include_index_usage_metrics: bool = is_affirmative(instance.get('include_index_usage_metrics', True)) - self.include_index_usage_metrics_tempdb: bool = is_affirmative(instance.get('include_index_usage_metrics_tempdb', False)) - self.include_tempdb_file_space_usage_metrics: bool = is_affirmative(instance.get('include_tempdb_file_space_usage_metrics', True)) + self.include_index_usage_metrics_tempdb: bool = is_affirmative( + instance.get('include_index_usage_metrics_tempdb', False) + ) + self.include_tempdb_file_space_usage_metrics: bool = is_affirmative( + instance.get('include_tempdb_file_space_usage_metrics', True) + ) self.include_xe_metrics: bool = is_affirmative(instance.get('include_xe_metrics', False)) def _compile_valid_patterns(self, patterns: list[str]) -> re.Pattern: diff --git a/sqlserver/datadog_checks/sqlserver/const.py b/sqlserver/datadog_checks/sqlserver/const.py index 53cc3bd0bc938..5d29c54041040 100644 --- a/sqlserver/datadog_checks/sqlserver/const.py +++ b/sqlserver/datadog_checks/sqlserver/const.py @@ -272,4 +272,4 @@ DEFAULT_SCHEMAS_COLLECTION_INTERVAL = 600 -DEFAULT_LONG_METRICS_COLLECTION_INTERVAL = 300 \ No newline at end of file +DEFAULT_LONG_METRICS_COLLECTION_INTERVAL = 300 diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py index 290f708e4f27d..813efe9ee9e0a 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py @@ -39,6 +39,7 @@ class SqlserverAvailabilityGroupsMetrics(SqlserverDatabaseMetricsBase): @property def include_ao_metrics(self) -> bool: return self.config.include_ao_metrics + @property def availability_group(self): return self.config.availability_group diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py index 9cfd3c6fe026a..c34bd91dd6f9e 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py @@ -50,6 +50,7 @@ class SqlserverAvailabilityReplicasMetrics(SqlserverDatabaseMetricsBase): @property def include_ao_metrics(self) -> bool: return self.config.include_ao_metrics + @property def availability_group(self): return self.config.availability_group diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py index 40bdb476348b4..afab1c3da434c 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py @@ -34,6 +34,7 @@ class SqlserverTempDBFileSpaceUsageMetrics(SqlserverDatabaseMetricsBase): @property def include_tempdb_file_space_usage_metrics(self) -> bool: return self.config.include_tempdb_file_space_usage_metrics + @property def enabled(self): if not self.include_tempdb_file_space_usage_metrics: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py index 48a5c17eea795..738a36bbe61a9 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py @@ -45,9 +45,7 @@ class SQLServerXESessionMetrics(SqlserverDatabaseMetricsBase): @property def enabled(self): self.deadlocks_config: dict = self.config.deadlocks_config - return self.config.include_xe_metrics or is_affirmative( - self.deadlocks_config.get('enabled', False) - ) + return self.config.include_xe_metrics or is_affirmative(self.deadlocks_config.get('enabled', False)) @property def collection_interval(self) -> int: diff --git a/sqlserver/tests/test_database_metrics.py b/sqlserver/tests/test_database_metrics.py index 2fd87781ba0d2..d6af24be4fd24 100644 --- a/sqlserver/tests/test_database_metrics.py +++ b/sqlserver/tests/test_database_metrics.py @@ -932,9 +932,7 @@ def test_sqlserver_db_fragmentation_metrics( if db_fragmentation_object_names: assert db_fragmentation_metrics.db_fragmentation_object_names == db_fragmentation_object_names - expected_collection_interval = ( - db_fragmentation_metrics_interval or db_fragmentation_metrics.collection_interval - ) + expected_collection_interval = db_fragmentation_metrics_interval or db_fragmentation_metrics.collection_interval assert db_fragmentation_metrics.queries[0]['collection_interval'] == expected_collection_interval sqlserver_check._database_metrics = [db_fragmentation_metrics] @@ -1323,9 +1321,7 @@ def execute_query_handler_mocked(query, db=None): execute_query_handler=execute_query_handler_mocked, ) - expected_collection_interval = ( - database_backup_metrics_interval or database_backup_metrics.collection_interval - ) + expected_collection_interval = database_backup_metrics_interval or database_backup_metrics.collection_interval assert database_backup_metrics.queries[0]['collection_interval'] == expected_collection_interval sqlserver_check._database_metrics = [database_backup_metrics] From d700d4bca7a71e12d7de9ba3bb3b21202ad3a296 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 21 Nov 2024 21:14:15 +0000 Subject: [PATCH 04/18] default for database_instance_collection_interval should be a number --- sqlserver/assets/configuration/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 4fde7367ce475..153552df36830 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -764,7 +764,7 @@ files: value: type: number example: 1800 - display_default: false + display_default: 300 - name: schemas_collection description: | Available for Agent 7.56 and newer. From 9a962fda07175963c0f1573468822f49b085a29d Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 21 Nov 2024 16:17:39 -0500 Subject: [PATCH 05/18] Update 19111.deprecated --- sqlserver/changelog.d/19111.deprecated | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver/changelog.d/19111.deprecated b/sqlserver/changelog.d/19111.deprecated index f60055f7e4266..2b7ee7e596894 100644 --- a/sqlserver/changelog.d/19111.deprecated +++ b/sqlserver/changelog.d/19111.deprecated @@ -1 +1 @@ -Deprecate usage of instance_config in database_metrics ([#18586](https://github.com/DataDog/integrations-core/pull/19111)) +Deprecate usage of instance_config in database_metrics ([#19111](https://github.com/DataDog/integrations-core/pull/19111)) From 131a187b8f180373d40fee6d890d4c46c4c37145 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 21 Nov 2024 21:25:30 +0000 Subject: [PATCH 06/18] remove unused imports --- .../datadog_checks/sqlserver/database_metrics/ao_metrics.py | 1 - .../sqlserver/database_metrics/availability_groups_metrics.py | 2 -- .../database_metrics/availability_replicas_metrics.py | 2 -- .../database_metrics/database_replication_stats_metrics.py | 2 -- .../sqlserver/database_metrics/db_fragmentation_metrics.py | 1 - .../datadog_checks/sqlserver/database_metrics/fci_metrics.py | 1 - .../sqlserver/database_metrics/index_usage_metrics.py | 1 - .../sqlserver/database_metrics/master_files_metrics.py | 2 -- .../sqlserver/database_metrics/os_schedulers_metrics.py | 2 -- .../sqlserver/database_metrics/os_tasks_metrics.py | 2 -- .../sqlserver/database_metrics/primary_log_shipping_metrics.py | 3 --- .../database_metrics/secondary_log_shipping_metrics.py | 3 --- .../database_metrics/tempdb_file_space_usage_metrics.py | 1 - 13 files changed, 23 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py index e6330c1d9577a..4b7f1d359207a 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py @@ -4,7 +4,6 @@ from typing import List -from datadog_checks.base.config import is_affirmative from datadog_checks.sqlserver.utils import is_azure_database from .base import SqlserverDatabaseMetricsBase diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py index 813efe9ee9e0a..36850d07eb574 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase AVAILABILITY_GROUPS_METRICS_QUERY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py index c34bd91dd6f9e..f84a97590b77b 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase AVAILABILITY_REPLICAS_METRICS_QUERY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py index 83653605629d1..309f9c86045d4 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase DATABASE_REPLICATION_STATS_METRICS_QUERY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py index 47907272d3dd7..79fa052965580 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py @@ -5,7 +5,6 @@ import copy import functools -from datadog_checks.base.config import is_affirmative from datadog_checks.base.errors import ConfigurationError from .base import SqlserverDatabaseMetricsBase diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py index 2045262436de6..744bbc13d2ece 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py @@ -2,7 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative from datadog_checks.sqlserver.const import ( ENGINE_EDITION_AZURE_MANAGED_INSTANCE, ) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py index cf9183c848ef7..2f43f166ed252 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py @@ -4,7 +4,6 @@ import functools -from datadog_checks.base.config import is_affirmative from datadog_checks.base.errors import ConfigurationError from .base import SqlserverDatabaseMetricsBase diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py index cff9287519e76..7d54dbd3edfb8 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase MASTER_FILES_METRICS_QUERY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py index 17b4715b88382..92454bd859dfc 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase OS_SCHEDULERS_METRICS_QUERY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py index d3da226cfb659..fa542aafedfcd 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase OS_TASKS_METRICS_QUERY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py index bdd8973397357..cfe4436a8c69b 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py @@ -2,9 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) - -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase QUERY_LOG_SHIPPING_PRIMARY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py index 72f59b4678aa9..65704accee301 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py @@ -2,9 +2,6 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) - -from datadog_checks.base.config import is_affirmative - from .base import SqlserverDatabaseMetricsBase QUERY_LOG_SHIPPING_SECONDARY = { diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py index afab1c3da434c..9e29a0bb0994e 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py @@ -4,7 +4,6 @@ import functools -from datadog_checks.base.config import is_affirmative from datadog_checks.sqlserver.utils import is_azure_sql_database from .base import SqlserverDatabaseMetricsBase From 02e67406c22c70b3a214bf777baa2860654d45ee Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Fri, 22 Nov 2024 09:59:07 -0500 Subject: [PATCH 07/18] Update sqlserver/changelog.d/19111.deprecated Co-authored-by: Zhengda Lu --- sqlserver/changelog.d/19111.deprecated | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver/changelog.d/19111.deprecated b/sqlserver/changelog.d/19111.deprecated index 2b7ee7e596894..0519995f69655 100644 --- a/sqlserver/changelog.d/19111.deprecated +++ b/sqlserver/changelog.d/19111.deprecated @@ -1 +1 @@ -Deprecate usage of instance_config in database_metrics ([#19111](https://github.com/DataDog/integrations-core/pull/19111)) +Deprecate usage of instance_config in database_metrics From b0da6863737590ee4972c34673446f9bd327058f Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Tue, 26 Nov 2024 19:40:04 +0000 Subject: [PATCH 08/18] Make all database metrics configurable and update configuration structure --- sqlserver/assets/configuration/spec.yaml | 374 ++++++++++++------ sqlserver/datadog_checks/sqlserver/config.py | 103 +++-- .../sqlserver/database_metrics/ao_metrics.py | 2 +- .../availability_groups_metrics.py | 4 +- .../availability_replicas_metrics.py | 8 +- .../database_backup_metrics.py | 9 +- .../database_files_metrics.py | 12 +- .../database_replication_stats_metrics.py | 6 +- .../database_stats_metrics.py | 11 +- .../db_fragmentation_metrics.py | 6 +- .../sqlserver/database_metrics/fci_metrics.py | 2 +- .../database_metrics/file_stats_metrics.py | 5 + .../database_metrics/index_usage_metrics.py | 6 +- .../database_metrics/master_files_metrics.py | 2 +- .../database_metrics/os_schedulers_metrics.py | 2 +- .../database_metrics/os_tasks_metrics.py | 2 +- .../primary_log_shipping_metrics.py | 2 +- .../secondary_log_shipping_metrics.py | 2 +- .../database_metrics/server_state_metrics.py | 8 + .../tempdb_file_space_usage_metrics.py | 2 +- .../database_metrics/xe_session_metrics.py | 2 +- sqlserver/tests/conftest.py | 33 +- sqlserver/tests/test_database_metrics.py | 290 +++++++++----- 23 files changed, 596 insertions(+), 297 deletions(-) diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 153552df36830..e5d4d0e116095 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -129,12 +129,253 @@ files: type: boolean example: false display_default: true - - name: include_ao_metrics + - name: database_metrics description: | - Include AlwaysOn availability group metrics. - value: - type: boolean - example: false + Configure the collection of database metrics + options: + - name: ao_metrics + description: | + AlwaysOn availability group metrics. + options: + - name: enabled + description: | + Enable collection of AlwaysOn availability group metrics. + value: + type: boolean + example: false + - name: availability_group + description: | + When the `ao_metrics` is enabled, you can provide the resource + group id of a specific availability group that you would like to monitor. + If no availability group is specified, then we will collect AlwaysOn metrics + for all availability groups on the current replica. + value: + type: string + - name: only_emit_local + description: | + Primary replicas may emit metrics for remote secondary replicas + in the same availability group. If this option is set to true, + the primary replica will only emit information local to itself. + value: + type: boolean + example: false + - name: ao_database + description: | + AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. + value: + type: string + - name: db_fragmentation_metrics + description: | + Database fragmentation metrics. Note these queries can be resource intensive on large datasets. + Recommend to limit these via autodiscovery or specific database instances. + options: + - name: enabled + description: | + Enable collection of database fragmentation metrics. + value: + type: boolean + example: false + - name: enabled_tempdb + description: | + Enable collection of database index fragmentation statistics in tempdb database from the + `sys.dm_db_index_physical_stats` DMF. + + By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + might cause blocking. This configuration parameter allows enabling the collection of this metric. + This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + value: + type: boolean + example: false + - name: collection_interval + description: | + Configure the interval (in seconds) for the collection of database fragmentation metrics from the + `sys.dm_db_index_physical_stats` DMF. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. + value: + type: integer + example: 300 + display_default: 300 + - name: fci_metrics + description: | + Failover Cluster Instance metrics. Note that these metrics + requires a SQLServer set up with Failover Clustering enabled. + options: + - name: enabled + description: | + Enable collection of Failover Cluster Instance metrics. + value: + type: boolean + example: false + - name: index_usage_metrics + description: | + User table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. + + Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable + `database_autodiscovery` or set `database`. + options: + - name: enabled + description: | + Enable collection of index usage metrics. + value: + type: boolean + example: true + - name: enabled_tempdb + description: | + Enable the collection of user table index usage statistics in tempdb database from the + `sys.dm_db_index_usage_stats` DMV. + + By default, we do not collect index usage statistics in the tempdb database, as those queries + might cause blocking. This configuration parameter allows enabling the collection of this metric. + This parameter is ignored if the 'enabled' option for 'index_usage_metrics' is set to false. + value: + type: boolean + example: false + - name: collection_interval + description: | + Configure the interval (in seconds) for the collection of index usage statistics from the + `sys.dm_db_index_usage_stats` DMV. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. + value: + type: integer + example: 300 + display_default: 300 + - name: instance_metrics + description: | + Server-level instance metrics. When setting up multiple instances for + different databases on the same host these metrics will be duplicated unless this option is turned off. + options: + - name: enabled + description: | + Enable collection of instance metrics. + value: + type: boolean + example: true + - name: master_files_metrics + description: | + Database file size and state from `sys.master_files` + options: + - name: enabled + description: | + Enable collection of master files metrics. + value: + type: boolean + example: false + - name: primary_log_shipping_metrics + description: | + Metrics for a log shipping setup. Required to run against the + primary instance in a transaction log shipping configuration. Note that + the Datadog user needs to be present in msdb and must be added to the db_datareader role. + options: + - name: enabled + description: | + Enable collection of primary log shipping metrics. + value: + type: boolean + example: false + - name: secondary_log_shipping_metrics + description: | + Metrics for a log shipping setup. Required to run against the + secondary instance in a transaction log shipping configuration. Note that + the Datadog user needs to be present in msdb and must be added to the db_datareader role. + options: + - name: enabled + description: | + Enable collection of secondary log shipping metrics. + value: + type: boolean + example: false + - name: task_scheduler_metrics + description: | + Additional Task and Scheduler metrics. + options: + - name: enabled + description: | + Enable collection of Task and Scheduler metrics. + value: + type: boolean + example: false + - name: tempdb_file_space_usage_metrics + description: | + tempdb file space usage metrics for how space is used in tempdb data files. + options: + - name: enabled + description: | + Enable collection of tempdb file space usage metrics. + value: + type: boolean + example: true + - name: xe_metrics + description: | + Extended events (XE) metrics. + options: + - name: enabled + description: | + Enable collection of extended events (XE) metrics. The collection of XE metrics is automatically enabled + when `deadlocks_collection` is enabled. + value: + type: boolean + example: true + - name: db_backup_metrics + description: | + Database backup metrics. + options: + - name: enabled + description: | + Enable collection of database backup metrics + value: + type: boolean + example: true + - name: collection_interval + description: | + Set the interval (in seconds) for the collection of database backup metrics. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. + value: + type: integer + example: 300 + display_default: 300 + - name: db_files_metrics + description: | + Database files metrics. + options: + - name: enabled + description: | + Enable collection of database files metrics + value: + type: boolean + example: true + - name: db_stats_metrics + description: | + Database stats metrics + options: + - name: enabled + description: | + Enable collection of database stats metrics + value: + type: boolean + example: true + - name: file_stats_metrics + description: | + File stats metrics + options: + - name: enabled + description: | + Enable collection of file stats metrics + value: + type: boolean + example: true + - name: server_state_metrics + description: | + Server state metrics + options: + - name: enabled + description: | + Enable collection of server state metrics + value: + type: boolean + example: true - name: agent_jobs description: Configure collection of agent jobs events and metrics options: @@ -157,116 +398,6 @@ files: value: type: integer example: 10000 - - name: availability_group - description: | - When `include_ao_metrics` is enabled, you can provide the resource - group id of a specific availability group that you would like to monitor. - If no availability group is specified, then we will collect AlwaysOn metrics - for all availability groups on the current replica. - value: - type: string - - name: only_emit_local - description: | - Primary replicas may emit metrics for remote secondary replicas - in the same availability group. If this option is set to true, - the primary replica will only emit information local to itself. - value: - type: boolean - example: false - - name: ao_database - description: | - AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. - value: - type: string - - name: include_master_files_metrics - description: | - Include database file size and state from `sys.master_files`. - value: - type: boolean - example: false - - name: include_fci_metrics - description: | - Include Failover Cluster Instance metrics. Note that these metrics - requires a SQLServer set up with Failover Clustering enabled. - value: - type: boolean - example: false - - name: include_primary_log_shipping_metrics - description: | - Include log_shipping_primary metrics for a log shipping setup. Required to run - against the primary instance in a transaction log shipping configuration. Note that - the Datadog user needs to be present in msdb and must be added to the db_datareader role. - value: - type: boolean - example: false - - name: include_secondary_log_shipping_metrics - description: | - Include log_shipping_secondary metrics for a log shipping setup. Required to run - against a secondary instance in a transaction log shipping configuration. Note that - the Datadog user needs to be present in msdb and must be added to the db_datareader role. - value: - type: boolean - example: false - - name: include_instance_metrics - description: | - Include server-level instance metrics. When setting up multiple instances for - different databases on the same host these metrics will be duplicated unless this option is turned off. - value: - type: boolean - example: true - - name: include_task_scheduler_metrics - description: Include additional Task and Scheduler metrics. - value: - type: boolean - example: false - - name: include_db_fragmentation_metrics - description: | - Include database fragmentation metrics. Note these queries can be resource intensive on large datasets. - Recommend to limit these via autodiscovery or specific database instances. - value: - type: boolean - example: false - - name: include_db_fragmentation_metrics_tempdb - description: | - Configure the collection of database index fragmentation statistics in tempdb database from the - `sys.dm_db_index_physical_stats` DMF. - - By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - might cause blocking. This configuration parameter allows enabling the collection of this metric. - This parameter is ignored if 'include_db_fragmentation_metrics' is set to false. - value: - type: boolean - example: false - - name: include_index_usage_metrics - description: | - Configure the collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. - - Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable - `database_autodiscovery` or set `database`. - value: - type: boolean - example: true - - name: include_index_usage_metrics_tempdb - description: | - Configure the collection of user table index usage statistics in tempdb database from the - `sys.dm_db_index_usage_stats` DMV. - - By default, we do not collect index usage statistics in the tempdb database, as those queries - might cause blocking. This configuration parameter allows enabling the collection of this metric. - This parameter is ignored if 'include_index_usage_metrics' is set to false. - value: - type: boolean - example: false - - name: index_usage_metrics_interval - description: | - Configure the interval (in seconds) for the collection of index usage statistics from the - `sys.dm_db_index_usage_stats` DMV. - Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - to use a consistent value throughout all SQL Server agent deployments. - value: - type: integer - example: 300 - display_default: 300 - name: db_fragmentation_object_names description: | Fragmentation metrics normally emit metrics for all objects within a database. @@ -276,19 +407,6 @@ files: type: array items: type: string - - name: include_tempdb_file_space_usage_metrics - description: | - Include tempdb file space usage metrics for how space is used in tempdb data files. - value: - type: boolean - example: true - - name: include_xe_metrics - description: | - Include extended events (XE) metrics. The collection of XE metrics is automatically enabled - when `deadlocks_collection` is enabled. - value: - type: boolean - example: true - name: adoprovider description: | Choose the ADO provider. Note that the (default) provider diff --git a/sqlserver/datadog_checks/sqlserver/config.py b/sqlserver/datadog_checks/sqlserver/config.py index e5f9bb33976d4..e4205867e8572 100644 --- a/sqlserver/datadog_checks/sqlserver/config.py +++ b/sqlserver/datadog_checks/sqlserver/config.py @@ -29,15 +29,6 @@ def __init__(self, init_config, instance, log): self.autodiscovery_db_service_check: bool = is_affirmative(instance.get('autodiscovery_db_service_check', True)) self.min_collection_interval: int = instance.get('min_collection_interval', 15) self.autodiscovery_interval: int = instance.get('autodiscovery_interval', DEFAULT_AUTODISCOVERY_INTERVAL) - self.database_backup_metrics_interval: int = instance.get( - 'database_backup_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL - ) - self.index_usage_stats_interval: int = instance.get( - 'index_usage_stats_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL - ) - self.db_fragmentation_metrics_interval: int = instance.get( - 'db_fragmentation_metrics_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL - ) self.database_instance_collection_interval: int = instance.get( 'database_instance_collection_interval', DEFAULT_LONG_METRICS_COLLECTION_INTERVAL ) @@ -55,6 +46,7 @@ def __init__(self, init_config, instance, log): # DBM self.dbm_enabled: bool = is_affirmative(instance.get('dbm', False)) + self.database_metrics_config: dict = self._build_database_metrics_configs(instance) self.statement_metrics_config: dict = instance.get('query_metrics', {}) or {} self.agent_jobs_config: dict = instance.get('agent_jobs', {}) or {} self.procedure_metrics_config: dict = instance.get('procedure_metrics', {}) or {} @@ -120,37 +112,7 @@ def __init__(self, init_config, instance, log): self.stored_procedure_characters_limit: int = instance.get('stored_procedure_characters_limit', PROC_CHAR_LIMIT) self.connection_host: str = instance['host'] self.service = instance.get('service') or init_config.get('service') or '' - self.availability_group = instance.get('availability_group') - self.only_emit_local = is_affirmative(instance.get('only_emit_local', False)) - self.ao_database = instance.get('ao_database') self.db_fragmentation_object_names = instance.get('db_fragmentation_object_names', []) or [] - # DBM Metrics Includes - self.include_ao_metrics: bool = is_affirmative(instance.get('include_ao_metrics', False)) - self.include_master_files_metrics: bool = is_affirmative(instance.get('include_master_files_metrics', False)) - self.include_fci_metrics: bool = is_affirmative(instance.get('include_fci_metrics', False)) - self.include_primary_log_shipping_metrics: bool = is_affirmative( - instance.get('include_primary_log_shipping_metrics', False) - ) - self.include_secondary_log_shipping_metrics: bool = is_affirmative( - instance.get('include_secondary_log_shipping_metrics', False) - ) - self.include_task_scheduler_metrics: bool = is_affirmative( - instance.get('include_task_scheduler_metrics', False) - ) - self.include_db_fragmentation_metrics: bool = is_affirmative( - instance.get('include_db_fragmentation_metrics', False) - ) - self.include_db_fragmentation_metrics_tempdb: bool = is_affirmative( - instance.get('include_db_fragmentation_metrics_tempdb', False) - ) - self.include_index_usage_metrics: bool = is_affirmative(instance.get('include_index_usage_metrics', True)) - self.include_index_usage_metrics_tempdb: bool = is_affirmative( - instance.get('include_index_usage_metrics_tempdb', False) - ) - self.include_tempdb_file_space_usage_metrics: bool = is_affirmative( - instance.get('include_tempdb_file_space_usage_metrics', True) - ) - self.include_xe_metrics: bool = is_affirmative(instance.get('include_xe_metrics', False)) def _compile_valid_patterns(self, patterns: list[str]) -> re.Pattern: valid_patterns = [] @@ -207,3 +169,66 @@ def _should_propagate_agent_tags(instance, init_config) -> bool: return init_config_propagate_agent_tags # if neither the instance nor the init_config has set the value, return False return False + + def _build_database_metrics_configs(self, instance): + # Set defaults for database metrics + configurable_metrics = { + "ao_metrics" : {'enabled' : False, + 'availability_group' : None, + 'ao_database' : None, + 'only_emit_local' : False}, + "db_backup_metrics": {'enabled' : True, + 'collection_interval' : DEFAULT_LONG_METRICS_COLLECTION_INTERVAL}, + "db_files_metrics": {'enabled' : True}, + "db_stats_metrics": {'enabled' : True}, + "db_fragmentation_metrics" : {'enabled' : False, + 'enabled_tempdb' : False, + 'collection_interval' : DEFAULT_LONG_METRICS_COLLECTION_INTERVAL}, + "fci_metrics": {'enabled' : False}, + "file_stats_metrics": {'enabled' : True}, + "index_usage_metrics": {'enabled' : True, + 'collection_interval' : DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, + 'enabled_tempdb' : False}, + "instance_metrics": {'enabled' : True}, + "master_files_metrics": {'enabled' : False}, + "primary_log_shipping_metrics": {'enabled' : False}, + "secondary_log_shipping_metrics": {'enabled' : False}, + "server_state_metrics": {'enabled' : True}, + "task_scheduler_metrics": {'enabled' : False}, + "tempdb_file_space_usage_metrics": {'enabled' : False}, + "xe_metrics": {'enabled' : True}, + } + # Check if the instance has any configuration for the metrics in legacy structure + legacy_configuration_metrics = { + "include_ao_metrics" : "ao_metrics", + "include_master_files_metrics" : "master_files_metrics", + "include_fci_metrics" : "fci_metrics", + "include_primary_log_shipping_metrics" : "primary_log_shipping_metrics", + "include_secondary_log_shipping_metrics" : "secondary_log_shipping_metrics", + "include_instance_metrics" : "instance_metrics", + "include_task_scheduler_metrics" : "task_scheduler_metrics", + "include_db_fragmentation_metrics" : "db_fragmentation_metrics", + "include_index_usage_metrics" : "index_usage_metrics", + "include_tempdb_file_space_usage_metrics" : "tempdb_file_space_usage_metrics", + "include_xe_metrics" : "xe_metrics", + } + for metric, config_key in legacy_configuration_metrics.items(): + if instance.get(metric) is not None: + configurable_metrics[config_key]['enabled'] = instance[metric] + # Manual look ups for legacy configuration structure + configurable_metrics['ao_metrics']['availability_group'] = instance.get('availability_group', configurable_metrics['ao_metrics']['availability_group']) + configurable_metrics['ao_metrics']['ao_database'] = instance.get('ao_database', configurable_metrics['ao_metrics']['ao_database']) + configurable_metrics['ao_metrics']['only_emit_local'] = instance.get('only_emit_local', configurable_metrics['ao_metrics']['only_emit_local']) + configurable_metrics['db_backup_metrics']['collection_interval'] = instance.get('database_backup_metrics_interval', configurable_metrics['db_backup_metrics']['collection_interval']) + configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'] = instance.get('include_db_fragmentation_metrics_tempdb', configurable_metrics['db_fragmentation_metrics']['enabled_tempdb']) + configurable_metrics['db_fragmentation_metrics']['collection_interval'] = instance.get('db_fragmentation_metrics_interval', configurable_metrics['db_fragmentation_metrics']['collection_interval']) + configurable_metrics['index_usage_metrics']['enabled_tempdb'] = instance.get('include_index_usage_metrics_tempdb', configurable_metrics['index_usage_metrics']['enabled_tempdb']) + configurable_metrics['index_usage_metrics']['collection_interval'] = instance.get('index_usage_stats_interval', configurable_metrics['index_usage_metrics']['collection_interval']) + # Check if the instance has any configuration for the metrics + database_metrics = instance.get('database_metrics', {}) + for metric, config in configurable_metrics.items(): + metric_config = database_metrics.get(metric, {}) + for key, value in metric_config.items(): + if value is not None: + config[key] = value + return configurable_metrics diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py index 4b7f1d359207a..8e83034bda518 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/ao_metrics.py @@ -60,7 +60,7 @@ class SqlserverAoMetrics(SqlserverDatabaseMetricsBase): @property def include_ao_metrics(self) -> bool: - return self.config.include_ao_metrics + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def enabled(self) -> bool: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py index 36850d07eb574..ed3558c3d3a1c 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_groups_metrics.py @@ -36,11 +36,11 @@ class SqlserverAvailabilityGroupsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-hadr-availability-group-states-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return self.config.include_ao_metrics + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def availability_group(self): - return self.config.availability_group + return self.config.database_metrics_config["ao_metrics"]["availability_group"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py index f84a97590b77b..69bccb89af457 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/availability_replicas_metrics.py @@ -47,19 +47,19 @@ class SqlserverAvailabilityReplicasMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-availability-replicas-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return self.config.include_ao_metrics + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def availability_group(self): - return self.config.availability_group + return self.config.database_metrics_config["ao_metrics"]["availability_group"] @property def only_emit_local(self): - return self.config.only_emit_local + return self.config.database_metrics_config["ao_metrics"]["only_emit_local"] @property def ao_database(self): - return self.config.ao_database + return self.config.database_metrics_config["ao_metrics"]["ao_database"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py index f82fac9e035a8..170351230b2a0 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py @@ -30,9 +30,13 @@ class SqlserverDatabaseBackupMetrics(SqlserverDatabaseMetricsBase): # Contains a row for each backup set. A backup set # contains the backup from a single, successful backup operation. # https://docs.microsoft.com/en-us/sql/relational-databases/system-tables/backupset-transact-sql?view=sql-server-ver15 + @property + def include_database_backup_metrics(self) -> bool: + return self.config.database_metrics_config["db_backup_metrics"]["enabled"] + @property def enabled(self): - if is_azure_sql_database(self.engine_edition): + if not self.include_database_backup_metrics or is_azure_sql_database(self.engine_edition): return False return True @@ -42,7 +46,7 @@ def collection_interval(self) -> int: Returns the interval in seconds at which to collect database backup metrics. Note: The database backup metrics query can be expensive, so it is recommended to set a higher interval. ''' - return self.config.database_backup_metrics_interval + return self.config.database_metrics_config["db_backup_metrics"]["collection_interval"] @property def queries(self): @@ -56,6 +60,7 @@ def __repr__(self) -> str: return ( f"{self.__class__.__name__}(" f"enabled={self.enabled}, " + f"include_database_backup_metrics={self.include_database_backup_metrics}), " f"engine_edition={self.engine_edition}, " f"collection_interval={self.collection_interval})" ) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py index 0786ebd5e999c..17709dac548d3 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py @@ -47,7 +47,12 @@ class SqlserverDatabaseFilesMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-database-files-transact-sql @property + def include_database_files_metrics(self) -> bool: + return self.config.database_metrics_config["db_files_metrics"]["enabled"] + @property def enabled(self): + if not self.include_database_files_metrics: + return False return True @property @@ -55,8 +60,11 @@ def queries(self): return [DATABASE_FILES_METRICS_QUERY] def __repr__(self) -> str: - return f"{self.__class__.__name__}(" f"enabled={self.enabled})" - + return ( + f"{self.__class__.__name__}(" + f"enabled={self.enabled}, " + f"include_database_files_metrics={self.include_database_files_metrics})" + ) def _build_query_executors(self): executors = [] for database in self.databases: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py index 309f9c86045d4..cbc5503855c80 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_replication_stats_metrics.py @@ -32,15 +32,15 @@ class SqlserverDatabaseReplicationStatsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-hadr-database-replica-states-transact-sql?view=sql-server-ver15 @property def include_ao_metrics(self) -> bool: - return self.config.include_ao_metrics + return self.config.database_metrics_config["ao_metrics"]["enabled"] @property def availability_group(self): - return self.config.availability_group + return self.config.database_metrics_config["ao_metrics"]["availability_group"] @property def only_emit_local(self): - return self.config.only_emit_local + return self.config.database_metrics_config["ao_metrics"]["only_emit_local"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py index d8024f4deb929..6582a015a6909 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py @@ -34,7 +34,12 @@ class SqlserverDatabaseStatsMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-databases-transact-sql?view=sql-server-ver15 @property + def include_database_stats_metrics(self) -> bool: + return self.config.database_metrics_config["db_stats_metrics"]["enabled"] + @property def enabled(self): + if not self.include_database_stats_metrics: + return False return True @property @@ -42,4 +47,8 @@ def queries(self): return [DATABASE_STATS_METRICS_QUERY] def __repr__(self) -> str: - return f"{self.__class__.__name__}(" f"enabled={self.enabled}" + return ( + f"{self.__class__.__name__}(" + f"enabled={self.enabled}, " + f"include_database_stats_metrics={self.include_database_stats_metrics})" + ) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py index 79fa052965580..1411449ac5e21 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/db_fragmentation_metrics.py @@ -51,11 +51,11 @@ class SqlserverDBFragmentationMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-index-physical-stats-transact-sql?view=sql-server-ver15 @property def include_db_fragmentation_metrics(self): - return self.config.include_db_fragmentation_metrics + return self.config.database_metrics_config["db_fragmentation_metrics"]["enabled"] @property def include_db_fragmentation_metrics_tempdb(self): - return self.config.include_db_fragmentation_metrics_tempdb + return self.config.database_metrics_config["db_fragmentation_metrics"]["enabled_tempdb"] @property def db_fragmentation_object_names(self): @@ -73,7 +73,7 @@ def collection_interval(self) -> int: Returns the interval in seconds at which to collect database index fragmentation metrics. Note: The index fragmentation metrics query can be expensive, so it is recommended to set a higher interval. ''' - return self.config.db_fragmentation_metrics_interval + return self.config.database_metrics_config["db_fragmentation_metrics"]["collection_interval"] @property def databases(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py index 744bbc13d2ece..5bd821d07b8de 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/fci_metrics.py @@ -37,7 +37,7 @@ class SqlserverFciMetrics(SqlserverDatabaseMetricsBase): @property def include_fci_metrics(self) -> bool: - return self.config.include_fci_metrics + return self.config.database_metrics_config["fci_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py index 79e0877769d7a..c5def2d4dfdaf 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py @@ -10,7 +10,12 @@ class SqlserverFileStatsMetrics(SqlserverDatabaseMetricsBase): @property + def include_file_stats_metrics(self) -> bool: + return self.config.database_metrics_config["file_stats_metrics"]["enabled"] + @property def enabled(self): + if not self.include_file_stats_metrics: + return False if not self.major_version and not is_azure_database(self.engine_edition): return False return True diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py index 2f43f166ed252..d91a935c19133 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/index_usage_metrics.py @@ -44,11 +44,11 @@ class SqlserverIndexUsageMetrics(SqlserverDatabaseMetricsBase): @property def include_index_usage_metrics(self) -> bool: - return self.config.include_index_usage_metrics + return self.config.database_metrics_config["index_usage_metrics"]["enabled"] @property def include_index_usage_metrics_tempdb(self) -> bool: - return self.config.include_index_usage_metrics_tempdb + return self.config.database_metrics_config["index_usage_metrics"]["enabled_tempdb"] @property def collection_interval(self) -> int: @@ -56,7 +56,7 @@ def collection_interval(self) -> int: Returns the interval in seconds at which to collect index usage metrics. Note: The index usage metrics query can be expensive, so it is recommended to set a higher interval. ''' - return self.config.index_usage_stats_interval + return self.config.database_metrics_config["index_usage_metrics"]["collection_interval"] @property def databases(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py index 7d54dbd3edfb8..2dd429807c0dc 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/master_files_metrics.py @@ -46,7 +46,7 @@ class SqlserverMasterFilesMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-master-files-transact-sql @property def include_master_files_metrics(self): - return self.config.include_master_files_metrics + return self.config.database_metrics_config["master_files_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py index 92454bd859dfc..0363040d455f2 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_schedulers_metrics.py @@ -32,7 +32,7 @@ class SqlserverOsSchedulersMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-schedulers-transact-sql @property def include_task_scheduler_metrics(self): - return self.config.include_task_scheduler_metrics + return self.config.database_metrics_config["task_scheduler_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py index fa542aafedfcd..2fa631c28e7d7 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/os_tasks_metrics.py @@ -28,7 +28,7 @@ class SqlserverOsTasksMetrics(SqlserverDatabaseMetricsBase): # https://docs.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-os-tasks-transact-sql @property def include_task_scheduler_metrics(self): - return self.config.include_task_scheduler_metrics + return self.config.database_metrics_config["task_scheduler_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py index cfe4436a8c69b..709c8e48f2146 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/primary_log_shipping_metrics.py @@ -27,7 +27,7 @@ class SqlserverPrimaryLogShippingMetrics(SqlserverDatabaseMetricsBase): @property def include_primary_log_shipping_metrics(self) -> bool: - return self.config.include_primary_log_shipping_metrics + return self.config.database_metrics_config["primary_log_shipping_metrics"]["enabled"] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py index 65704accee301..4fe6fb3db106d 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/secondary_log_shipping_metrics.py @@ -35,7 +35,7 @@ class SqlserverSecondaryLogShippingMetrics(SqlserverDatabaseMetricsBase): @property def include_secondary_log_shipping_metrics(self) -> bool: - return self.config.include_secondary_log_shipping_metrics + return self.config.database_metrics_config['secondary_log_shipping_metrics']['enabled'] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py index 536b67f0be4ae..7d437637e3056 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/server_state_metrics.py @@ -30,10 +30,17 @@ class SqlserverServerStateMetrics(SqlserverDatabaseMetricsBase): + @property + def include_server_state_metrics(self) -> bool: + return self.config.database_metrics_config['server_state_metrics']['enabled'] + @property def enabled(self): # Server state queries require VIEW SERVER STATE permissions, which some managed database # versions do not support. + + if not self.include_server_state_metrics: + return False if self.engine_edition in [ENGINE_EDITION_SQL_DATABASE]: return False return True @@ -46,6 +53,7 @@ def __repr__(self) -> str: return ( f"{self.__class__.__name__}(" f"enabled={self.enabled}, " + f"include_server_state_metrics={self.include_server_state_metrics}, " f"major_version={self.major_version}, " f"engine_edition={self.engine_edition})" ) diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py index 9e29a0bb0994e..5e18c1838679c 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/tempdb_file_space_usage_metrics.py @@ -32,7 +32,7 @@ class SqlserverTempDBFileSpaceUsageMetrics(SqlserverDatabaseMetricsBase): @property def include_tempdb_file_space_usage_metrics(self) -> bool: - return self.config.include_tempdb_file_space_usage_metrics + return self.config.database_metrics_config['tempdb_file_space_usage_metrics']['enabled'] @property def enabled(self): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py index 738a36bbe61a9..993ea359a9e36 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py @@ -45,7 +45,7 @@ class SQLServerXESessionMetrics(SqlserverDatabaseMetricsBase): @property def enabled(self): self.deadlocks_config: dict = self.config.deadlocks_config - return self.config.include_xe_metrics or is_affirmative(self.deadlocks_config.get('enabled', False)) + return self.config.database_metrics_config["xe_metrics"]["enabled"] or is_affirmative(self.deadlocks_config.get('enabled', False)) @property def collection_interval(self) -> int: diff --git a/sqlserver/tests/conftest.py b/sqlserver/tests/conftest.py index b0985ea6a8414..dcad332a3cb6c 100644 --- a/sqlserver/tests/conftest.py +++ b/sqlserver/tests/conftest.py @@ -99,12 +99,24 @@ def instance_minimal_defaults(): def instance_docker(instance_docker_defaults): instance_docker_defaults.update( { - 'include_task_scheduler_metrics': True, - 'include_db_fragmentation_metrics': True, - 'include_fci_metrics': True, - 'include_ao_metrics': False, - 'include_master_files_metrics': True, + 'database_metrics': { + 'ao_metrics': { + 'enabled': False, + }, + 'task_scheduler_metrics': { + 'enabled': True, + }, + 'db_fragmentation_metrics': { + 'enabled': True, + }, + 'fci_metrics': { + 'enabled': True, + }, + 'master_files_metrics': { + 'enabled': True, + }, 'disable_generic_tags': True, + } } ) return instance_docker_defaults @@ -233,21 +245,26 @@ def instance_e2e(instance_docker): @pytest.fixture def instance_ao_docker_primary(instance_docker): - instance_docker['include_ao_metrics'] = True + # instance_docker['include_ao_metrics'] = True + instance_docker['database_metrics'] = { + 'ao_metrics': { + 'enabled': True, + } + } return instance_docker @pytest.fixture def instance_ao_docker_primary_local_only(instance_ao_docker_primary): instance = deepcopy(instance_ao_docker_primary) - instance['only_emit_local'] = True + instance['database_metrics']['ao_metrics']['only_emit_local'] = True return instance @pytest.fixture def instance_ao_docker_primary_non_existing_ag(instance_ao_docker_primary): instance = deepcopy(instance_ao_docker_primary) - instance['availability_group'] = 'AG2' + instance['database_metrics']['ao_metrics']['availability_group'] = 'AG2' return instance diff --git a/sqlserver/tests/test_database_metrics.py b/sqlserver/tests/test_database_metrics.py index d6af24be4fd24..129a067e8dd77 100644 --- a/sqlserver/tests/test_database_metrics.py +++ b/sqlserver/tests/test_database_metrics.py @@ -52,13 +52,18 @@ @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_file_stats_metrics', [True, False]) def test_sqlserver_file_stats_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, + include_file_stats_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'file_stats_metrics': {'enabled': include_file_stats_metrics}, + } mocked_results = [ ('master', 'ONLINE', 'master', '/xx/master.mdf', 89, 0, 0, 73, 16, 3153920, 933888, 59, 98, 4194304), @@ -87,18 +92,21 @@ def execute_query_handler_mocked(query, db=None): dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - db, state, logical_name, file_location, *metric_values = result - metrics = zip(file_stats_metrics.metric_names()[0], metric_values) - expected_tags = [ - f'db:{db}', - f'state:{state}', - f'logical_name:{logical_name}', - f'file_location:{file_location}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + if not include_file_stats_metrics: + assert file_stats_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + db, state, logical_name, file_location, *metric_values = result + metrics = zip(file_stats_metrics.metric_names()[0], metric_values) + expected_tags = [ + f'db:{db}', + f'state:{state}', + f'logical_name:{logical_name}', + f'file_location:{file_location}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) @pytest.mark.integration @@ -112,7 +120,9 @@ def test_sqlserver_ao_metrics( include_ao_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } # Mocked results mocked_ao_availability_groups = [ @@ -252,9 +262,11 @@ def test_sqlserver_availability_groups_metrics( mocked_results, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } if availability_group: - instance_docker_metrics['availability_group'] = availability_group + instance_docker_metrics['database_metrics']['ao_metrics']['availability_group'] = availability_group sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -339,11 +351,13 @@ def test_sqlserver_database_replication_stats_metrics( mocked_results, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } if availability_group: - instance_docker_metrics['availability_group'] = availability_group + instance_docker_metrics['database_metrics']['ao_metrics']['availability_group'] = availability_group if only_emit_local: - instance_docker_metrics['only_emit_local'] = only_emit_local + instance_docker_metrics['database_metrics']['ao_metrics']['only_emit_local'] = only_emit_local sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -471,13 +485,15 @@ def test_sqlserver_availability_replicas_metrics( mocked_results, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_ao_metrics'] = include_ao_metrics + instance_docker_metrics['database_metrics'] = { + 'ao_metrics': {'enabled': include_ao_metrics}, + } if availability_group: - instance_docker_metrics['availability_group'] = availability_group + instance_docker_metrics['database_metrics']['ao_metrics']['availability_group'] = availability_group if only_emit_local: - instance_docker_metrics['only_emit_local'] = only_emit_local + instance_docker_metrics['database_metrics']['ao_metrics']['only_emit_local'] = only_emit_local if ao_database: - instance_docker_metrics['ao_database'] = ao_database + instance_docker_metrics['database_metrics']['ao_metrics']['ao_database'] = ao_database sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -543,7 +559,9 @@ def test_sqlserver_fci_metrics( include_fci_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_fci_metrics'] = include_fci_metrics + instance_docker_metrics['database_metrics'] = { + 'fci_metrics': {'enabled': include_fci_metrics}, + } mocked_results = [ ('node1', 'up', 'cluster1', 0, 1), @@ -592,8 +610,9 @@ def test_sqlserver_primary_log_shipping_metrics( include_primary_log_shipping_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_primary_log_shipping_metrics'] = include_primary_log_shipping_metrics - + instance_docker_metrics['database_metrics'] = { + 'primary_log_shipping_metrics': {'enabled': include_primary_log_shipping_metrics}, + } mocked_results = [('97E29D89-2FA0-44FF-9EF7-65DA75FE0E3E', 'EC2AMAZ-Q0NCNV5', 'MyDummyDB', 500, 3600)] sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) @@ -639,8 +658,9 @@ def test_sqlserver_secondary_log_shipping_metrics( include_secondary_log_shipping_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_secondary_log_shipping_metrics'] = include_secondary_log_shipping_metrics - + instance_docker_metrics['database_metrics'] = { + 'secondary_log_shipping_metrics': {'enabled': include_secondary_log_shipping_metrics}, + } mocked_results = [ ( r'EC2AMAZ-Q0NCNV5\MYSECONDARY', @@ -691,13 +711,18 @@ def execute_query_handler_mocked(query, db=None): @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_server_state_metrics', [True, False]) def test_sqlserver_server_state_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, + include_server_state_metrics ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'server_state_metrics': {'enabled': include_server_state_metrics}, + } mocked_results = [(1000, 4, 8589934592, 17179869184, 4294967296, 8589934592)] @@ -717,11 +742,14 @@ def execute_query_handler_mocked(query, db=None): dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - metrics = zip(server_state_metrics.metric_names()[0], result) - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=tags) + if not include_server_state_metrics: + assert server_state_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + metrics = zip(server_state_metrics.metric_names()[0], result) + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=tags) @pytest.mark.integration @@ -735,8 +763,9 @@ def test_sqlserver_tempdb_file_space_usage_metrics( include_tempdb_file_space_usage_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_tempdb_file_space_usage_metrics'] = include_tempdb_file_space_usage_metrics - + instance_docker_metrics['tempdb_file_space_usage_metrics'] = { + 'tempdb_file_space_usage_metrics': {'enabled': include_tempdb_file_space_usage_metrics} + } mocked_results = [ [(2, Decimal('5.375000'), Decimal('0.000000'), Decimal('0.000000'), Decimal('1.312500'), Decimal('1.312500'))] ] @@ -788,10 +817,13 @@ def test_sqlserver_index_usage_metrics( index_usage_stats_interval, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_index_usage_metrics'] = include_index_usage_metrics - instance_docker_metrics['include_index_usage_metrics_tempdb'] = include_index_usage_metrics_tempdb + instance_docker_metrics['database_metrics'] = { + 'index_usage_metrics': {'enabled': include_index_usage_metrics, + 'enabled_tempdb': include_index_usage_metrics_tempdb + }, + } if index_usage_stats_interval: - instance_docker_metrics['index_usage_stats_interval'] = index_usage_stats_interval + instance_docker_metrics['database_metrics']['index_usage_metrics']['collection_interval'] = index_usage_stats_interval mocked_results_non_tempdb = [ [ @@ -877,11 +909,13 @@ def test_sqlserver_db_fragmentation_metrics( db_fragmentation_metrics_interval, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_db_fragmentation_metrics'] = include_db_fragmentation_metrics - instance_docker_metrics['include_db_fragmentation_metrics_tempdb'] = include_db_fragmentation_metrics_tempdb + instance_docker_metrics['database_metrics'] = { + 'db_fragmentation_metrics': {'enabled': include_db_fragmentation_metrics, + 'enabled_tempdb': include_db_fragmentation_metrics_tempdb}, + } if db_fragmentation_metrics_interval: - instance_docker_metrics['db_fragmentation_metrics_interval'] = db_fragmentation_metrics_interval - + instance_docker_metrics['database_metrics']['db_fragmentation_metrics']['collection_interval'] = db_fragmentation_metrics_interval + print(instance_docker_metrics) mocked_results = [ [ ('master', 'spt_fallback_db', 0, None, 0, 0.0, 0, 0.0), @@ -981,7 +1015,9 @@ def test_sqlserver_os_schedulers_metrics( include_task_scheduler_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_task_scheduler_metrics'] = include_task_scheduler_metrics + instance_docker_metrics['database_metrics'] = { + 'task_scheduler_metrics': {'enabled': include_task_scheduler_metrics}, + } mocked_results = [ (0, 0, 4, 6, 4, 0, 0), @@ -1045,8 +1081,9 @@ def test_sqlserver_os_tasks_metrics( include_task_scheduler_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_task_scheduler_metrics'] = include_task_scheduler_metrics - + instance_docker_metrics['database_metrics'] = { + 'task_scheduler_metrics': {'enabled': include_task_scheduler_metrics}, + } mocked_results = [ (0, 40, 0, 0, 0), (9, 46, 0, 0, 0), @@ -1108,8 +1145,9 @@ def test_sqlserver_master_files_metrics( include_master_files_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['include_master_files_metrics'] = include_master_files_metrics - + instance_docker_metrics['database_metrics'] = { + 'master_files_metrics': {'enabled': include_master_files_metrics}, + } mocked_results = [ ('master', 'master', 1, 'data', '/var/opt/mssql/data/master.mdf', 'ONLINE', 4096, 0), ('master', 'master', 2, 'transaction_log', '/var/opt/mssql/data/mastlog.ldf', 'ONLINE', 512, 0), @@ -1170,13 +1208,18 @@ def execute_query_handler_mocked(query, db=None): @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_database_files_metrics', [True, False]) def test_sqlserver_database_files_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, + include_database_files_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'db_files_metrics': {'enabled': include_database_files_metrics}, + } mocked_results = [ [ @@ -1219,35 +1262,43 @@ def test_sqlserver_database_files_metrics( dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for db, result in zip(AUTODISCOVERY_DBS, mocked_results): - for row in result: - file_id, file_type, file_location, file_name, database_files_state_desc, size, space_used, state = row - size *= 8 # size is in pages, 1 page = 8 KB - space_used *= 8 # space_used is in pages, 1 page = 8 KB - metrics = zip(database_files_metrics.metric_names()[0], [state, size, space_used]) - expected_tags = [ - f'db:{db}', - f'database:{db}', - f'file_id:{file_id}', - f'file_type:{file_type}', - f'file_location:{file_location}', - f'file_name:{file_name}', - f'database_files_state_desc:{database_files_state_desc}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + if not include_database_files_metrics: + assert database_files_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for db, result in zip(AUTODISCOVERY_DBS, mocked_results): + for row in result: + file_id, file_type, file_location, file_name, database_files_state_desc, size, space_used, state = row + size *= 8 # size is in pages, 1 page = 8 KB + space_used *= 8 # space_used is in pages, 1 page = 8 KB + metrics = zip(database_files_metrics.metric_names()[0], [state, size, space_used]) + expected_tags = [ + f'db:{db}', + f'database:{db}', + f'file_id:{file_id}', + f'file_type:{file_type}', + f'file_location:{file_location}', + f'file_name:{file_name}', + f'database_files_state_desc:{database_files_state_desc}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') +@pytest.mark.parametrize('include_database_files_metrics', [True, False]) def test_sqlserver_database_stats_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, + include_database_files_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'db_stats_metrics': {'enabled': include_database_files_metrics}, + } mocked_results = [ ('master', 'master', 'ONLINE', 'SIMPLE', 0, False, False, False), @@ -1273,33 +1324,41 @@ def execute_query_handler_mocked(query, db=None): dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - db, database, database_state_desc, database_recovery_model_desc, *metric_values = result - metrics = zip(database_stats_metrics.metric_names()[0], metric_values) - expected_tags = [ - f'db:{db}', - f'database:{database}', - f'database_state_desc:{database_state_desc}', - f'database_recovery_model_desc:{database_recovery_model_desc}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + if not include_database_files_metrics: + assert database_stats_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + db, database, database_state_desc, database_recovery_model_desc, *metric_values = result + metrics = zip(database_stats_metrics.metric_names()[0], metric_values) + expected_tags = [ + f'db:{db}', + f'database:{database}', + f'database_state_desc:{database_state_desc}', + f'database_recovery_model_desc:{database_recovery_model_desc}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') @pytest.mark.parametrize('database_backup_metrics_interval', [None, 600]) +@pytest.mark.parametrize('include_database_backup_metrics', [True, False]) def test_sqlserver_database_backup_metrics( aggregator, dd_run_check, init_config, instance_docker_metrics, database_backup_metrics_interval, + include_database_backup_metrics, ): instance_docker_metrics['database_autodiscovery'] = True + instance_docker_metrics['database_metrics'] = { + 'db_backup_metrics': {'enabled': include_database_backup_metrics}, + } if database_backup_metrics_interval: - instance_docker_metrics['database_backup_metrics_interval'] = database_backup_metrics_interval + instance_docker_metrics['database_metrics']['db_backup_metrics']['collection_interval'] = database_backup_metrics_interval mocked_results = [ ('master', 'master', 0), @@ -1327,22 +1386,25 @@ def execute_query_handler_mocked(query, db=None): sqlserver_check._database_metrics = [database_backup_metrics] dd_run_check(sqlserver_check) - tags = sqlserver_check._config.tags - for result in mocked_results: - db, database, *metric_values = result - metrics = zip(database_backup_metrics.metric_names()[0], metric_values) - expected_tags = [ - f'db:{db}', - f'database:{database}', - ] + tags - for metric_name, metric_value in metrics: - aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) - - # database_backup_metrics should not be collected because the collection interval is not reached - aggregator.reset() - dd_run_check(sqlserver_check) - for metric_name in database_backup_metrics.metric_names()[0]: - aggregator.assert_metric(metric_name, count=0) + if not include_database_backup_metrics: + assert database_backup_metrics.enabled is False + else: + tags = sqlserver_check._config.tags + for result in mocked_results: + db, database, *metric_values = result + metrics = zip(database_backup_metrics.metric_names()[0], metric_values) + expected_tags = [ + f'db:{db}', + f'database:{database}', + ] + tags + for metric_name, metric_value in metrics: + aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags) + + # database_backup_metrics should not be collected because the collection interval is not reached + aggregator.reset() + dd_run_check(sqlserver_check) + for metric_name in database_backup_metrics.metric_names()[0]: + aggregator.assert_metric(metric_name, count=0) @pytest.mark.integration @@ -1369,3 +1431,45 @@ def test_sqlserver_xe_session_metrics( expected_tags = sqlserver_check._config.tags expected_tags.append('session_name:datadog') aggregator.assert_metric("sqlserver.xe.session_status", value=1, tags=expected_tags) + +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +def test_sqlserver_database_metrics_defaults( + aggregator, + dd_run_check, + init_config, + instance_docker_metrics, +): + include_defaults = { + SqlserverAoMetrics : False, + SqlserverAvailabilityGroupsMetrics : False, + SqlserverAvailabilityReplicasMetrics : False, + SqlserverDatabaseBackupMetrics : True, + SqlserverDatabaseFilesMetrics : True, + SqlserverDatabaseReplicationStatsMetrics : False, + SqlserverDatabaseStatsMetrics : True, + SqlserverDBFragmentationMetrics : False, + SqlserverFciMetrics : False, + SqlserverFileStatsMetrics : True, + SqlserverIndexUsageMetrics : True, + SqlserverMasterFilesMetrics : False, + SqlserverOsSchedulersMetrics : False, + SqlserverOsTasksMetrics : False, + SqlserverPrimaryLogShippingMetrics : False, + SqlserverSecondaryLogShippingMetrics : False, + SqlserverServerStateMetrics : True, + SqlserverTempDBFileSpaceUsageMetrics : False, + } + instance_docker_metrics['database_autodiscovery'] = True + + sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics]) + + for metric, enabled in include_defaults.items(): + database_metrics = metric( + config=sqlserver_check._config, + new_query_executor=sqlserver_check._new_query_executor, + server_static_info=STATIC_SERVER_INFO, + execute_query_handler=None, + databases=AUTODISCOVERY_DBS, + ) + assert database_metrics.enabled == enabled \ No newline at end of file From 3b668d6b61c4892ae3362fcd2ea65ba3e3187609 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Tue, 26 Nov 2024 20:06:09 +0000 Subject: [PATCH 09/18] sync default config files and linter --- sqlserver/datadog_checks/sqlserver/config.py | 103 +++--- .../sqlserver/data/conf.yaml.example | 330 ++++++++++++------ .../database_backup_metrics.py | 2 +- .../database_files_metrics.py | 2 + .../database_stats_metrics.py | 1 + .../database_metrics/file_stats_metrics.py | 1 + .../database_metrics/xe_session_metrics.py | 4 +- sqlserver/tests/conftest.py | 2 +- sqlserver/tests/test_database_metrics.py | 70 ++-- 9 files changed, 326 insertions(+), 189 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/config.py b/sqlserver/datadog_checks/sqlserver/config.py index e4205867e8572..e93461aae828e 100644 --- a/sqlserver/datadog_checks/sqlserver/config.py +++ b/sqlserver/datadog_checks/sqlserver/config.py @@ -173,57 +173,74 @@ def _should_propagate_agent_tags(instance, init_config) -> bool: def _build_database_metrics_configs(self, instance): # Set defaults for database metrics configurable_metrics = { - "ao_metrics" : {'enabled' : False, - 'availability_group' : None, - 'ao_database' : None, - 'only_emit_local' : False}, - "db_backup_metrics": {'enabled' : True, - 'collection_interval' : DEFAULT_LONG_METRICS_COLLECTION_INTERVAL}, - "db_files_metrics": {'enabled' : True}, - "db_stats_metrics": {'enabled' : True}, - "db_fragmentation_metrics" : {'enabled' : False, - 'enabled_tempdb' : False, - 'collection_interval' : DEFAULT_LONG_METRICS_COLLECTION_INTERVAL}, - "fci_metrics": {'enabled' : False}, - "file_stats_metrics": {'enabled' : True}, - "index_usage_metrics": {'enabled' : True, - 'collection_interval' : DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, - 'enabled_tempdb' : False}, - "instance_metrics": {'enabled' : True}, - "master_files_metrics": {'enabled' : False}, - "primary_log_shipping_metrics": {'enabled' : False}, - "secondary_log_shipping_metrics": {'enabled' : False}, - "server_state_metrics": {'enabled' : True}, - "task_scheduler_metrics": {'enabled' : False}, - "tempdb_file_space_usage_metrics": {'enabled' : False}, - "xe_metrics": {'enabled' : True}, + "ao_metrics": {'enabled': False, 'availability_group': None, 'ao_database': None, 'only_emit_local': False}, + "db_backup_metrics": {'enabled': True, 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL}, + "db_files_metrics": {'enabled': True}, + "db_stats_metrics": {'enabled': True}, + "db_fragmentation_metrics": { + 'enabled': False, + 'enabled_tempdb': False, + 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, + }, + "fci_metrics": {'enabled': False}, + "file_stats_metrics": {'enabled': True}, + "index_usage_metrics": { + 'enabled': True, + 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL, + 'enabled_tempdb': False, + }, + "instance_metrics": {'enabled': True}, + "master_files_metrics": {'enabled': False}, + "primary_log_shipping_metrics": {'enabled': False}, + "secondary_log_shipping_metrics": {'enabled': False}, + "server_state_metrics": {'enabled': True}, + "task_scheduler_metrics": {'enabled': False}, + "tempdb_file_space_usage_metrics": {'enabled': False}, + "xe_metrics": {'enabled': True}, } # Check if the instance has any configuration for the metrics in legacy structure legacy_configuration_metrics = { - "include_ao_metrics" : "ao_metrics", - "include_master_files_metrics" : "master_files_metrics", - "include_fci_metrics" : "fci_metrics", - "include_primary_log_shipping_metrics" : "primary_log_shipping_metrics", - "include_secondary_log_shipping_metrics" : "secondary_log_shipping_metrics", - "include_instance_metrics" : "instance_metrics", - "include_task_scheduler_metrics" : "task_scheduler_metrics", - "include_db_fragmentation_metrics" : "db_fragmentation_metrics", - "include_index_usage_metrics" : "index_usage_metrics", - "include_tempdb_file_space_usage_metrics" : "tempdb_file_space_usage_metrics", - "include_xe_metrics" : "xe_metrics", + "include_ao_metrics": "ao_metrics", + "include_master_files_metrics": "master_files_metrics", + "include_fci_metrics": "fci_metrics", + "include_primary_log_shipping_metrics": "primary_log_shipping_metrics", + "include_secondary_log_shipping_metrics": "secondary_log_shipping_metrics", + "include_instance_metrics": "instance_metrics", + "include_task_scheduler_metrics": "task_scheduler_metrics", + "include_db_fragmentation_metrics": "db_fragmentation_metrics", + "include_index_usage_metrics": "index_usage_metrics", + "include_tempdb_file_space_usage_metrics": "tempdb_file_space_usage_metrics", + "include_xe_metrics": "xe_metrics", } for metric, config_key in legacy_configuration_metrics.items(): if instance.get(metric) is not None: configurable_metrics[config_key]['enabled'] = instance[metric] # Manual look ups for legacy configuration structure - configurable_metrics['ao_metrics']['availability_group'] = instance.get('availability_group', configurable_metrics['ao_metrics']['availability_group']) - configurable_metrics['ao_metrics']['ao_database'] = instance.get('ao_database', configurable_metrics['ao_metrics']['ao_database']) - configurable_metrics['ao_metrics']['only_emit_local'] = instance.get('only_emit_local', configurable_metrics['ao_metrics']['only_emit_local']) - configurable_metrics['db_backup_metrics']['collection_interval'] = instance.get('database_backup_metrics_interval', configurable_metrics['db_backup_metrics']['collection_interval']) - configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'] = instance.get('include_db_fragmentation_metrics_tempdb', configurable_metrics['db_fragmentation_metrics']['enabled_tempdb']) - configurable_metrics['db_fragmentation_metrics']['collection_interval'] = instance.get('db_fragmentation_metrics_interval', configurable_metrics['db_fragmentation_metrics']['collection_interval']) - configurable_metrics['index_usage_metrics']['enabled_tempdb'] = instance.get('include_index_usage_metrics_tempdb', configurable_metrics['index_usage_metrics']['enabled_tempdb']) - configurable_metrics['index_usage_metrics']['collection_interval'] = instance.get('index_usage_stats_interval', configurable_metrics['index_usage_metrics']['collection_interval']) + configurable_metrics['ao_metrics']['availability_group'] = instance.get( + 'availability_group', configurable_metrics['ao_metrics']['availability_group'] + ) + configurable_metrics['ao_metrics']['ao_database'] = instance.get( + 'ao_database', configurable_metrics['ao_metrics']['ao_database'] + ) + configurable_metrics['ao_metrics']['only_emit_local'] = instance.get( + 'only_emit_local', configurable_metrics['ao_metrics']['only_emit_local'] + ) + configurable_metrics['db_backup_metrics']['collection_interval'] = instance.get( + 'database_backup_metrics_interval', configurable_metrics['db_backup_metrics']['collection_interval'] + ) + configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'] = instance.get( + 'include_db_fragmentation_metrics_tempdb', + configurable_metrics['db_fragmentation_metrics']['enabled_tempdb'], + ) + configurable_metrics['db_fragmentation_metrics']['collection_interval'] = instance.get( + 'db_fragmentation_metrics_interval', configurable_metrics['db_fragmentation_metrics']['collection_interval'] + ) + configurable_metrics['index_usage_metrics']['enabled_tempdb'] = instance.get( + 'include_index_usage_metrics_tempdb', configurable_metrics['index_usage_metrics']['enabled_tempdb'] + ) + configurable_metrics['index_usage_metrics']['collection_interval'] = instance.get( + 'index_usage_stats_interval', configurable_metrics['index_usage_metrics']['collection_interval'] + ) # Check if the instance has any configuration for the metrics database_metrics = instance.get('database_metrics', {}) for metric, config in configurable_metrics.items(): diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index 5418c1b76b220..7ab38e0011d28 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -127,128 +127,247 @@ instances: # autodiscovery_db_service_check: false - ## @param include_ao_metrics - boolean - optional - default: false - ## Include AlwaysOn availability group metrics. - # - # include_ao_metrics: false + ## Configure the collection of database metrics + # + # database_metrics: + + ## AlwaysOn availability group metrics. + # + # ao_metrics: + + ## @param enabled - boolean - optional - default: false + ## Enable collection of AlwaysOn availability group metrics. + # + # enabled: false + + ## @param availability_group - string - optional + ## When the `ao_metrics` is enabled, you can provide the resource + ## group id of a specific availability group that you would like to monitor. + ## If no availability group is specified, then we will collect AlwaysOn metrics + ## for all availability groups on the current replica. + # + # availability_group: + + ## @param only_emit_local - boolean - optional - default: false + ## Primary replicas may emit metrics for remote secondary replicas + ## in the same availability group. If this option is set to true, + ## the primary replica will only emit information local to itself. + # + # only_emit_local: false + + ## @param ao_database - string - optional + ## AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. + # + # ao_database: + + ## Database fragmentation metrics. Note these queries can be resource intensive on large datasets. + ## Recommend to limit these via autodiscovery or specific database instances. + # + # db_fragmentation_metrics: + + ## @param enabled - boolean - optional - default: false + ## Enable collection of database fragmentation metrics. + # + # enabled: false + + ## @param enabled_tempdb - boolean - optional - default: false + ## Enable collection of database index fragmentation statistics in tempdb database from the + ## `sys.dm_db_index_physical_stats` DMF. + ## + ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + # + # enabled_tempdb: false + + ## @param collection_interval - integer - optional - default: 300 + ## Configure the interval (in seconds) for the collection of database fragmentation metrics from the + ## `sys.dm_db_index_physical_stats` DMF. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. + # + # collection_interval: 300 + + ## Failover Cluster Instance metrics. Note that these metrics + ## requires a SQLServer set up with Failover Clustering enabled. + # + # fci_metrics: + + ## @param enabled - boolean - optional - default: false + ## Enable collection of Failover Cluster Instance metrics. + # + # enabled: false + + ## User table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## + ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable + ## `database_autodiscovery` or set `database`. + # + # index_usage_metrics: - ## Configure collection of agent jobs events and metrics - # - # agent_jobs: + ## @param enabled - boolean - optional - default: true + ## Enable collection of index usage metrics. + # + # enabled: true - ## @param enabled - boolean - optional - default: false - ## Enable collection of agent jobs events and metrics. Requires `dbm: true`. + ## @param enabled_tempdb - boolean - optional - default: false + ## Enable the collection of user table index usage statistics in tempdb database from the + ## `sys.dm_db_index_usage_stats` DMV. + ## + ## By default, we do not collect index usage statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if the 'enabled' option for 'index_usage_metrics' is set to false. + # + # enabled_tempdb: false + + ## @param collection_interval - integer - optional - default: 300 + ## Configure the interval (in seconds) for the collection of index usage statistics from the + ## `sys.dm_db_index_usage_stats` DMV. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. + # + # collection_interval: 300 + + ## Server-level instance metrics. When setting up multiple instances for + ## different databases on the same host these metrics will be duplicated unless this option is turned off. # - # enabled: false + # instance_metrics: - ## @param collection_interval - number - optional - default: 15 - ## Configure the interval (in seconds) for the collection of agent jobs events and metrics. - ## Set the query metric collection interval (in seconds). + ## @param enabled - boolean - optional - default: true + ## Enable collection of instance metrics. + # + # enabled: true + + ## Database file size and state from `sys.master_files` # - # collection_interval: 15 + # master_files_metrics: - ## @param history_row_limit - integer - optional - default: 10000 - ## Set the maximum number of job step rows that can be retrieved in a single check run. + ## @param enabled - boolean - optional - default: false + ## Enable collection of master files metrics. + # + # enabled: false + + ## Metrics for a log shipping setup. Required to run against the + ## primary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. # - # history_row_limit: 10000 + # primary_log_shipping_metrics: - ## @param availability_group - string - optional - ## When `include_ao_metrics` is enabled, you can provide the resource - ## group id of a specific availability group that you would like to monitor. - ## If no availability group is specified, then we will collect AlwaysOn metrics - ## for all availability groups on the current replica. - # - # availability_group: + ## @param enabled - boolean - optional - default: false + ## Enable collection of primary log shipping metrics. + # + # enabled: false - ## @param only_emit_local - boolean - optional - default: false - ## Primary replicas may emit metrics for remote secondary replicas - ## in the same availability group. If this option is set to true, - ## the primary replica will only emit information local to itself. - # - # only_emit_local: false + ## Metrics for a log shipping setup. Required to run against the + ## secondary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + # + # secondary_log_shipping_metrics: - ## @param ao_database - string - optional - ## AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. - # - # ao_database: + ## @param enabled - boolean - optional - default: false + ## Enable collection of secondary log shipping metrics. + # + # enabled: false - ## @param include_master_files_metrics - boolean - optional - default: false - ## Include database file size and state from `sys.master_files`. - # - # include_master_files_metrics: false + ## Additional Task and Scheduler metrics. + # + # task_scheduler_metrics: - ## @param include_fci_metrics - boolean - optional - default: false - ## Include Failover Cluster Instance metrics. Note that these metrics - ## requires a SQLServer set up with Failover Clustering enabled. - # - # include_fci_metrics: false + ## @param enabled - boolean - optional - default: false + ## Enable collection of Task and Scheduler metrics. + # + # enabled: false - ## @param include_primary_log_shipping_metrics - boolean - optional - default: false - ## Include log_shipping_primary metrics for a log shipping setup. Required to run - ## against the primary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. - # - # include_primary_log_shipping_metrics: false + ## tempdb file space usage metrics for how space is used in tempdb data files. + # + # tempdb_file_space_usage_metrics: - ## @param include_secondary_log_shipping_metrics - boolean - optional - default: false - ## Include log_shipping_secondary metrics for a log shipping setup. Required to run - ## against a secondary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. - # - # include_secondary_log_shipping_metrics: false + ## @param enabled - boolean - optional - default: true + ## Enable collection of tempdb file space usage metrics. + # + # enabled: true - ## @param include_instance_metrics - boolean - optional - default: true - ## Include server-level instance metrics. When setting up multiple instances for - ## different databases on the same host these metrics will be duplicated unless this option is turned off. - # - # include_instance_metrics: true + ## Extended events (XE) metrics. + # + # xe_metrics: - ## @param include_task_scheduler_metrics - boolean - optional - default: false - ## Include additional Task and Scheduler metrics. - # - # include_task_scheduler_metrics: false + ## @param enabled - boolean - optional - default: true + ## Enable collection of extended events (XE) metrics. The collection of XE metrics is automatically enabled + ## when `deadlocks_collection` is enabled. + # + # enabled: true - ## @param include_db_fragmentation_metrics - boolean - optional - default: false - ## Include database fragmentation metrics. Note these queries can be resource intensive on large datasets. - ## Recommend to limit these via autodiscovery or specific database instances. - # - # include_db_fragmentation_metrics: false + ## Database backup metrics. + # + # db_backup_metrics: - ## @param include_db_fragmentation_metrics_tempdb - boolean - optional - default: false - ## Configure the collection of database index fragmentation statistics in tempdb database from the - ## `sys.dm_db_index_physical_stats` DMF. - ## - ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if 'include_db_fragmentation_metrics' is set to false. - # - # include_db_fragmentation_metrics_tempdb: false + ## @param enabled - boolean - optional - default: true + ## Enable collection of database backup metrics + # + # enabled: true - ## @param include_index_usage_metrics - boolean - optional - default: true - ## Configure the collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. - ## - ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable - ## `database_autodiscovery` or set `database`. - # - # include_index_usage_metrics: true + ## @param collection_interval - integer - optional - default: 300 + ## Set the interval (in seconds) for the collection of database backup metrics. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. + # + # collection_interval: 300 - ## @param include_index_usage_metrics_tempdb - boolean - optional - default: false - ## Configure the collection of user table index usage statistics in tempdb database from the - ## `sys.dm_db_index_usage_stats` DMV. - ## - ## By default, we do not collect index usage statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if 'include_index_usage_metrics' is set to false. - # - # include_index_usage_metrics_tempdb: false + ## Database files metrics. + # + # db_files_metrics: - ## @param index_usage_metrics_interval - integer - optional - default: 300 - ## Configure the interval (in seconds) for the collection of index usage statistics from the - ## `sys.dm_db_index_usage_stats` DMV. - ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - ## to use a consistent value throughout all SQL Server agent deployments. + ## @param enabled - boolean - optional - default: true + ## Enable collection of database files metrics + # + # enabled: true + + ## Database stats metrics + # + # db_stats_metrics: + + ## @param enabled - boolean - optional - default: true + ## Enable collection of database stats metrics + # + # enabled: true + + ## File stats metrics + # + # file_stats_metrics: + + ## @param enabled - boolean - optional - default: true + ## Enable collection of file stats metrics + # + # enabled: true + + ## Server state metrics + # + # server_state_metrics: + + ## @param enabled - boolean - optional - default: true + ## Enable collection of server state metrics + # + # enabled: true + + ## Configure collection of agent jobs events and metrics # - # index_usage_metrics_interval: 300 + # agent_jobs: + + ## @param enabled - boolean - optional - default: false + ## Enable collection of agent jobs events and metrics. Requires `dbm: true`. + # + # enabled: false + + ## @param collection_interval - number - optional - default: 15 + ## Configure the interval (in seconds) for the collection of agent jobs events and metrics. + ## Set the query metric collection interval (in seconds). + # + # collection_interval: 15 + + ## @param history_row_limit - integer - optional - default: 10000 + ## Set the maximum number of job step rows that can be retrieved in a single check run. + # + # history_row_limit: 10000 ## @param db_fragmentation_object_names - list of strings - optional ## Fragmentation metrics normally emit metrics for all objects within a database. @@ -257,17 +376,6 @@ instances: # # db_fragmentation_object_names: [] - ## @param include_tempdb_file_space_usage_metrics - boolean - optional - default: true - ## Include tempdb file space usage metrics for how space is used in tempdb data files. - # - # include_tempdb_file_space_usage_metrics: true - - ## @param include_xe_metrics - boolean - optional - default: true - ## Include extended events (XE) metrics. The collection of XE metrics is automatically enabled - ## when `deadlocks_collection` is enabled. - # - # include_xe_metrics: true - ## @param adoprovider - string - optional - default: SQLOLEDB ## Choose the ADO provider. Note that the (default) provider ## SQLOLEDB is being deprecated. To use the newer MSOLEDBSQL diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py index 170351230b2a0..21be687d112b1 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_backup_metrics.py @@ -33,7 +33,7 @@ class SqlserverDatabaseBackupMetrics(SqlserverDatabaseMetricsBase): @property def include_database_backup_metrics(self) -> bool: return self.config.database_metrics_config["db_backup_metrics"]["enabled"] - + @property def enabled(self): if not self.include_database_backup_metrics or is_azure_sql_database(self.engine_edition): diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py index 17709dac548d3..5dd458bfd6005 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_files_metrics.py @@ -49,6 +49,7 @@ class SqlserverDatabaseFilesMetrics(SqlserverDatabaseMetricsBase): @property def include_database_files_metrics(self) -> bool: return self.config.database_metrics_config["db_files_metrics"]["enabled"] + @property def enabled(self): if not self.include_database_files_metrics: @@ -65,6 +66,7 @@ def __repr__(self) -> str: f"enabled={self.enabled}, " f"include_database_files_metrics={self.include_database_files_metrics})" ) + def _build_query_executors(self): executors = [] for database in self.databases: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py index 6582a015a6909..3c34a16004f81 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/database_stats_metrics.py @@ -36,6 +36,7 @@ class SqlserverDatabaseStatsMetrics(SqlserverDatabaseMetricsBase): @property def include_database_stats_metrics(self) -> bool: return self.config.database_metrics_config["db_stats_metrics"]["enabled"] + @property def enabled(self): if not self.include_database_stats_metrics: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py index c5def2d4dfdaf..85b3a0d25c26d 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/file_stats_metrics.py @@ -12,6 +12,7 @@ class SqlserverFileStatsMetrics(SqlserverDatabaseMetricsBase): @property def include_file_stats_metrics(self) -> bool: return self.config.database_metrics_config["file_stats_metrics"]["enabled"] + @property def enabled(self): if not self.include_file_stats_metrics: diff --git a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py index 993ea359a9e36..99cacc6fcda5b 100644 --- a/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py +++ b/sqlserver/datadog_checks/sqlserver/database_metrics/xe_session_metrics.py @@ -45,7 +45,9 @@ class SQLServerXESessionMetrics(SqlserverDatabaseMetricsBase): @property def enabled(self): self.deadlocks_config: dict = self.config.deadlocks_config - return self.config.database_metrics_config["xe_metrics"]["enabled"] or is_affirmative(self.deadlocks_config.get('enabled', False)) + return self.config.database_metrics_config["xe_metrics"]["enabled"] or is_affirmative( + self.deadlocks_config.get('enabled', False) + ) @property def collection_interval(self) -> int: diff --git a/sqlserver/tests/conftest.py b/sqlserver/tests/conftest.py index dcad332a3cb6c..0d9bca2102661 100644 --- a/sqlserver/tests/conftest.py +++ b/sqlserver/tests/conftest.py @@ -115,8 +115,8 @@ def instance_docker(instance_docker_defaults): 'master_files_metrics': { 'enabled': True, }, + }, 'disable_generic_tags': True, - } } ) return instance_docker_defaults diff --git a/sqlserver/tests/test_database_metrics.py b/sqlserver/tests/test_database_metrics.py index 129a067e8dd77..46a96749128e8 100644 --- a/sqlserver/tests/test_database_metrics.py +++ b/sqlserver/tests/test_database_metrics.py @@ -713,11 +713,7 @@ def execute_query_handler_mocked(query, db=None): @pytest.mark.usefixtures('dd_environment') @pytest.mark.parametrize('include_server_state_metrics', [True, False]) def test_sqlserver_server_state_metrics( - aggregator, - dd_run_check, - init_config, - instance_docker_metrics, - include_server_state_metrics + aggregator, dd_run_check, init_config, instance_docker_metrics, include_server_state_metrics ): instance_docker_metrics['database_autodiscovery'] = True instance_docker_metrics['database_metrics'] = { @@ -818,12 +814,15 @@ def test_sqlserver_index_usage_metrics( ): instance_docker_metrics['database_autodiscovery'] = True instance_docker_metrics['database_metrics'] = { - 'index_usage_metrics': {'enabled': include_index_usage_metrics, - 'enabled_tempdb': include_index_usage_metrics_tempdb - }, + 'index_usage_metrics': { + 'enabled': include_index_usage_metrics, + 'enabled_tempdb': include_index_usage_metrics_tempdb, + }, } if index_usage_stats_interval: - instance_docker_metrics['database_metrics']['index_usage_metrics']['collection_interval'] = index_usage_stats_interval + instance_docker_metrics['database_metrics']['index_usage_metrics'][ + 'collection_interval' + ] = index_usage_stats_interval mocked_results_non_tempdb = [ [ @@ -910,11 +909,15 @@ def test_sqlserver_db_fragmentation_metrics( ): instance_docker_metrics['database_autodiscovery'] = True instance_docker_metrics['database_metrics'] = { - 'db_fragmentation_metrics': {'enabled': include_db_fragmentation_metrics, - 'enabled_tempdb': include_db_fragmentation_metrics_tempdb}, + 'db_fragmentation_metrics': { + 'enabled': include_db_fragmentation_metrics, + 'enabled_tempdb': include_db_fragmentation_metrics_tempdb, + }, } if db_fragmentation_metrics_interval: - instance_docker_metrics['database_metrics']['db_fragmentation_metrics']['collection_interval'] = db_fragmentation_metrics_interval + instance_docker_metrics['database_metrics']['db_fragmentation_metrics'][ + 'collection_interval' + ] = db_fragmentation_metrics_interval print(instance_docker_metrics) mocked_results = [ [ @@ -1358,7 +1361,9 @@ def test_sqlserver_database_backup_metrics( 'db_backup_metrics': {'enabled': include_database_backup_metrics}, } if database_backup_metrics_interval: - instance_docker_metrics['database_metrics']['db_backup_metrics']['collection_interval'] = database_backup_metrics_interval + instance_docker_metrics['database_metrics']['db_backup_metrics'][ + 'collection_interval' + ] = database_backup_metrics_interval mocked_results = [ ('master', 'master', 0), @@ -1432,6 +1437,7 @@ def test_sqlserver_xe_session_metrics( expected_tags.append('session_name:datadog') aggregator.assert_metric("sqlserver.xe.session_status", value=1, tags=expected_tags) + @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') def test_sqlserver_database_metrics_defaults( @@ -1441,24 +1447,24 @@ def test_sqlserver_database_metrics_defaults( instance_docker_metrics, ): include_defaults = { - SqlserverAoMetrics : False, - SqlserverAvailabilityGroupsMetrics : False, - SqlserverAvailabilityReplicasMetrics : False, - SqlserverDatabaseBackupMetrics : True, - SqlserverDatabaseFilesMetrics : True, - SqlserverDatabaseReplicationStatsMetrics : False, - SqlserverDatabaseStatsMetrics : True, - SqlserverDBFragmentationMetrics : False, - SqlserverFciMetrics : False, - SqlserverFileStatsMetrics : True, - SqlserverIndexUsageMetrics : True, - SqlserverMasterFilesMetrics : False, - SqlserverOsSchedulersMetrics : False, - SqlserverOsTasksMetrics : False, - SqlserverPrimaryLogShippingMetrics : False, - SqlserverSecondaryLogShippingMetrics : False, - SqlserverServerStateMetrics : True, - SqlserverTempDBFileSpaceUsageMetrics : False, + SqlserverAoMetrics: False, + SqlserverAvailabilityGroupsMetrics: False, + SqlserverAvailabilityReplicasMetrics: False, + SqlserverDatabaseBackupMetrics: True, + SqlserverDatabaseFilesMetrics: True, + SqlserverDatabaseReplicationStatsMetrics: False, + SqlserverDatabaseStatsMetrics: True, + SqlserverDBFragmentationMetrics: False, + SqlserverFciMetrics: False, + SqlserverFileStatsMetrics: True, + SqlserverIndexUsageMetrics: True, + SqlserverMasterFilesMetrics: False, + SqlserverOsSchedulersMetrics: False, + SqlserverOsTasksMetrics: False, + SqlserverPrimaryLogShippingMetrics: False, + SqlserverSecondaryLogShippingMetrics: False, + SqlserverServerStateMetrics: True, + SqlserverTempDBFileSpaceUsageMetrics: False, } instance_docker_metrics['database_autodiscovery'] = True @@ -1472,4 +1478,4 @@ def test_sqlserver_database_metrics_defaults( execute_query_handler=None, databases=AUTODISCOVERY_DBS, ) - assert database_metrics.enabled == enabled \ No newline at end of file + assert database_metrics.enabled == enabled From e8b3a0ec108d0cecaae01046a7f1e397ea615fb9 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Tue, 26 Nov 2024 21:12:40 +0000 Subject: [PATCH 10/18] fix failing tests --- sqlserver/assets/configuration/spec.yaml | 2 +- sqlserver/datadog_checks/sqlserver/config.py | 4 ++-- sqlserver/tests/conftest.py | 1 - sqlserver/tests/test_database_metrics.py | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index e5d4d0e116095..8b3f0310f74e0 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -316,7 +316,7 @@ files: when `deadlocks_collection` is enabled. value: type: boolean - example: true + example: false - name: db_backup_metrics description: | Database backup metrics. diff --git a/sqlserver/datadog_checks/sqlserver/config.py b/sqlserver/datadog_checks/sqlserver/config.py index e93461aae828e..bcbbb9be07b66 100644 --- a/sqlserver/datadog_checks/sqlserver/config.py +++ b/sqlserver/datadog_checks/sqlserver/config.py @@ -195,8 +195,8 @@ def _build_database_metrics_configs(self, instance): "secondary_log_shipping_metrics": {'enabled': False}, "server_state_metrics": {'enabled': True}, "task_scheduler_metrics": {'enabled': False}, - "tempdb_file_space_usage_metrics": {'enabled': False}, - "xe_metrics": {'enabled': True}, + "tempdb_file_space_usage_metrics": {'enabled': True}, + "xe_metrics": {'enabled': False}, } # Check if the instance has any configuration for the metrics in legacy structure legacy_configuration_metrics = { diff --git a/sqlserver/tests/conftest.py b/sqlserver/tests/conftest.py index 0d9bca2102661..bdcda43ccd91f 100644 --- a/sqlserver/tests/conftest.py +++ b/sqlserver/tests/conftest.py @@ -245,7 +245,6 @@ def instance_e2e(instance_docker): @pytest.fixture def instance_ao_docker_primary(instance_docker): - # instance_docker['include_ao_metrics'] = True instance_docker['database_metrics'] = { 'ao_metrics': { 'enabled': True, diff --git a/sqlserver/tests/test_database_metrics.py b/sqlserver/tests/test_database_metrics.py index 46a96749128e8..7203196159b4b 100644 --- a/sqlserver/tests/test_database_metrics.py +++ b/sqlserver/tests/test_database_metrics.py @@ -759,7 +759,7 @@ def test_sqlserver_tempdb_file_space_usage_metrics( include_tempdb_file_space_usage_metrics, ): instance_docker_metrics['database_autodiscovery'] = True - instance_docker_metrics['tempdb_file_space_usage_metrics'] = { + instance_docker_metrics['database_metrics'] = { 'tempdb_file_space_usage_metrics': {'enabled': include_tempdb_file_space_usage_metrics} } mocked_results = [ @@ -1464,7 +1464,7 @@ def test_sqlserver_database_metrics_defaults( SqlserverPrimaryLogShippingMetrics: False, SqlserverSecondaryLogShippingMetrics: False, SqlserverServerStateMetrics: True, - SqlserverTempDBFileSpaceUsageMetrics: False, + SqlserverTempDBFileSpaceUsageMetrics: True, } instance_docker_metrics['database_autodiscovery'] = True From e00176e17a9d38fc9c8f4d18fcbd10f8133dce13 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Tue, 26 Nov 2024 21:14:59 +0000 Subject: [PATCH 11/18] sync config files --- sqlserver/datadog_checks/sqlserver/data/conf.yaml.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index 7ab38e0011d28..4baab099f4c75 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -291,11 +291,11 @@ instances: # # xe_metrics: - ## @param enabled - boolean - optional - default: true + ## @param enabled - boolean - optional - default: false ## Enable collection of extended events (XE) metrics. The collection of XE metrics is automatically enabled ## when `deadlocks_collection` is enabled. # - # enabled: true + # enabled: false ## Database backup metrics. # From 76120cc295ba04a9d7c8af1c71632f1035667b03 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Tue, 26 Nov 2024 22:44:52 +0000 Subject: [PATCH 12/18] change nested options in config to objects --- sqlserver/assets/configuration/spec.yaml | 286 ++++++++---------- .../sqlserver/config_models/defaults.py | 60 ---- .../sqlserver/config_models/instance.py | 209 +++++++++++-- sqlserver/tests/conftest.py | 6 +- 4 files changed, 323 insertions(+), 238 deletions(-) diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 8b3f0310f74e0..f02d44c1b54cf 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -136,63 +136,92 @@ files: - name: ao_metrics description: | AlwaysOn availability group metrics. - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of AlwaysOn availability group metrics. - value: - type: boolean - example: false + type: boolean + example: false - name: availability_group description: | When the `ao_metrics` is enabled, you can provide the resource group id of a specific availability group that you would like to monitor. If no availability group is specified, then we will collect AlwaysOn metrics for all availability groups on the current replica. - value: - type: string + type: string + example: - name: only_emit_local description: | Primary replicas may emit metrics for remote secondary replicas in the same availability group. If this option is set to true, the primary replica will only emit information local to itself. - value: - type: boolean - example: false + type: boolean + example: false - name: ao_database description: | AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. - value: - type: string + type: string + - name: db_backup_metrics + description: | + Database backup metrics. + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: collection_interval + description: | + Set the interval (in seconds) for the collection of database backup metrics. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. + type: integer + example: 300 + display_default: 300 + - name: db_files_metrics + description: | + Database files metrics. + value: + type: object + properties: + - name: enabled + type: boolean + example: true + - name: db_stats_metrics + description: | + Database stats metrics + value: + type: object + properties: + - name: enabled + type: boolean + example: true - name: db_fragmentation_metrics description: | Database fragmentation metrics. Note these queries can be resource intensive on large datasets. Recommend to limit these via autodiscovery or specific database instances. - options: - - name: enabled - description: | - Enable collection of database fragmentation metrics. - value: + value: + type: object + properties: + - name: enabled type: boolean example: false - - name: enabled_tempdb - description: | - Enable collection of database index fragmentation statistics in tempdb database from the - `sys.dm_db_index_physical_stats` DMF. - - By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - might cause blocking. This configuration parameter allows enabling the collection of this metric. - This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. - value: + - name: enabled_tempdb + description: | + Enable collection of database index fragmentation statistics in tempdb database from the + `sys.dm_db_index_physical_stats` DMF. + + By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + might cause blocking. This configuration parameter allows enabling the collection of this metric. + This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. type: boolean example: false - - name: collection_interval - description: | - Configure the interval (in seconds) for the collection of database fragmentation metrics from the - `sys.dm_db_index_physical_stats` DMF. - Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - to use a consistent value throughout all SQL Server agent deployments. - value: + - name: collection_interval + description: | + Configure the interval (in seconds) for the collection of database fragmentation metrics from the + `sys.dm_db_index_physical_stats` DMF. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. type: integer example: 300 display_default: 300 @@ -200,26 +229,33 @@ files: description: | Failover Cluster Instance metrics. Note that these metrics requires a SQLServer set up with Failover Clustering enabled. - options: - - name: enabled - description: | - Enable collection of Failover Cluster Instance metrics. - value: + value: + type: object + properties: + - name: enabled type: boolean example: false + - name: file_stats_metrics + description: | + File stats metrics + value: + type: object + properties: + - name: enabled + type: boolean + example: true - name: index_usage_metrics description: | User table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable `database_autodiscovery` or set `database`. - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of index usage metrics. - value: - type: boolean - example: true + type: boolean + example: false - name: enabled_tempdb description: | Enable the collection of user table index usage statistics in tempdb database from the @@ -228,154 +264,94 @@ files: By default, we do not collect index usage statistics in the tempdb database, as those queries might cause blocking. This configuration parameter allows enabling the collection of this metric. This parameter is ignored if the 'enabled' option for 'index_usage_metrics' is set to false. - value: - type: boolean - example: false + type: boolean + example: false - name: collection_interval description: | Configure the interval (in seconds) for the collection of index usage statistics from the `sys.dm_db_index_usage_stats` DMV. Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. - value: - type: integer - example: 300 - display_default: 300 + type: integer + example: 300 + display_default: 300 - name: instance_metrics description: | Server-level instance metrics. When setting up multiple instances for different databases on the same host these metrics will be duplicated unless this option is turned off. - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of instance metrics. - value: - type: boolean - example: true + type: boolean + example: true - name: master_files_metrics description: | - Database file size and state from `sys.master_files` - options: + Database file size and state from `sys.master_files` + value: + type: object + properties: - name: enabled - description: | - Enable collection of master files metrics. - value: - type: boolean - example: false + type: boolean + example: false - name: primary_log_shipping_metrics description: | Metrics for a log shipping setup. Required to run against the primary instance in a transaction log shipping configuration. Note that the Datadog user needs to be present in msdb and must be added to the db_datareader role. - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of primary log shipping metrics. - value: - type: boolean - example: false + type: boolean + example: false - name: secondary_log_shipping_metrics description: | Metrics for a log shipping setup. Required to run against the secondary instance in a transaction log shipping configuration. Note that the Datadog user needs to be present in msdb and must be added to the db_datareader role. - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of secondary log shipping metrics. - value: - type: boolean - example: false + type: boolean + example: false + - name: server_state_metrics + description: | + Server state metrics + value: + type: object + properties: + - name: enabled + type: boolean + example: true - name: task_scheduler_metrics description: | Additional Task and Scheduler metrics. - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of Task and Scheduler metrics. - value: - type: boolean - example: false + type: boolean + example: false - name: tempdb_file_space_usage_metrics description: | tempdb file space usage metrics for how space is used in tempdb data files. - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of tempdb file space usage metrics. - value: - type: boolean - example: true + type: boolean + example: false - name: xe_metrics description: | Extended events (XE) metrics. - options: - - name: enabled - description: | - Enable collection of extended events (XE) metrics. The collection of XE metrics is automatically enabled - when `deadlocks_collection` is enabled. - value: - type: boolean - example: false - - name: db_backup_metrics - description: | - Database backup metrics. - options: - - name: enabled - description: | - Enable collection of database backup metrics - value: - type: boolean - example: true - - name: collection_interval - description: | - Set the interval (in seconds) for the collection of database backup metrics. - Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - to use a consistent value throughout all SQL Server agent deployments. - value: - type: integer - example: 300 - display_default: 300 - - name: db_files_metrics - description: | - Database files metrics. - options: - - name: enabled - description: | - Enable collection of database files metrics - value: - type: boolean - example: true - - name: db_stats_metrics - description: | - Database stats metrics - options: - - name: enabled - description: | - Enable collection of database stats metrics - value: - type: boolean - example: true - - name: file_stats_metrics - description: | - File stats metrics - options: - - name: enabled - description: | - Enable collection of file stats metrics - value: - type: boolean - example: true - - name: server_state_metrics - description: | - Server state metrics - options: + value: + type: object + properties: - name: enabled - description: | - Enable collection of server state metrics - value: - type: boolean - example: true + type: boolean + example: false - name: agent_jobs description: Configure collection of agent jobs events and metrics options: diff --git a/sqlserver/datadog_checks/sqlserver/config_models/defaults.py b/sqlserver/datadog_checks/sqlserver/config_models/defaults.py index e7c2d25da0041..1c6fcda461355 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/defaults.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/defaults.py @@ -64,62 +64,6 @@ def instance_ignore_missing_database(): return False -def instance_include_ao_metrics(): - return False - - -def instance_include_db_fragmentation_metrics(): - return False - - -def instance_include_db_fragmentation_metrics_tempdb(): - return False - - -def instance_include_fci_metrics(): - return False - - -def instance_include_index_usage_metrics(): - return True - - -def instance_include_index_usage_metrics_tempdb(): - return False - - -def instance_include_instance_metrics(): - return True - - -def instance_include_master_files_metrics(): - return False - - -def instance_include_primary_log_shipping_metrics(): - return False - - -def instance_include_secondary_log_shipping_metrics(): - return False - - -def instance_include_task_scheduler_metrics(): - return False - - -def instance_include_tempdb_file_space_usage_metrics(): - return True - - -def instance_include_xe_metrics(): - return True - - -def instance_index_usage_metrics_interval(): - return 300 - - def instance_log_unobfuscated_plans(): return False @@ -136,10 +80,6 @@ def instance_only_custom_queries(): return False -def instance_only_emit_local(): - return False - - def instance_proc_only_if_database(): return 'master' diff --git a/sqlserver/datadog_checks/sqlserver/config_models/instance.py b/sqlserver/datadog_checks/sqlserver/config_models/instance.py index ad93d2bdb6f48..9da5b0e64915b 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/instance.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/instance.py @@ -12,7 +12,7 @@ from types import MappingProxyType from typing import Any, Optional -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from datadog_checks.base.utils.functions import identity from datadog_checks.base.utils.models import validation @@ -69,6 +69,195 @@ class CustomQuery(BaseModel): tags: Optional[tuple[str, ...]] = None +class AoMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + ao_database: Optional[str] = Field( + None, description='AlwaysOn metrics are only emitted for the selected `ao_database` if not empty.\n' + ) + availability_group: Optional[str] = Field( + None, + description='When the `ao_metrics` is enabled, you can provide the resource \ngroup id of a specific availability group that you would like to monitor.\nIf no availability group is specified, then we will collect AlwaysOn metrics \nfor all availability groups on the current replica.\n', + examples=[''], + ) + enabled: Optional[bool] = Field(None, examples=[False]) + only_emit_local: Optional[bool] = Field( + None, + description='Primary replicas may emit metrics for remote secondary replicas\nin the same availability group. If this option is set to true,\nthe primary replica will only emit information local to itself.\n', + examples=[False], + ) + + +class DbBackupMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[int] = Field( + None, + description='Set the interval (in seconds) for the collection of database backup metrics.\nDefaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended\nto use a consistent value throughout all SQL Server agent deployments.\n', + examples=[300], + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class DbFilesMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class DbFragmentationMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[int] = Field( + None, + description='Configure the interval (in seconds) for the collection of database fragmentation metrics from the\n`sys.dm_db_index_physical_stats` DMF.\nDefaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended\nto use a consistent value throughout all SQL Server agent deployments.\n', + examples=[300], + ) + enabled: Optional[bool] = Field(None, examples=[False]) + enabled_tempdb: Optional[bool] = Field( + None, + description="Enable collection of database index fragmentation statistics in tempdb database from the\n`sys.dm_db_index_physical_stats` DMF.\n\nBy default, we do not collect index fragmentation statistics in the tempdb database, as those queries\nmight cause blocking. This configuration parameter allows enabling the collection of this metric.\nThis parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false.\n", + examples=[False], + ) + + +class DbStatsMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class FciMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class FileStatsMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class IndexUsageMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[int] = Field( + None, + description='Configure the interval (in seconds) for the collection of index usage statistics from the\n`sys.dm_db_index_usage_stats` DMV.\nDefaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended\nto use a consistent value throughout all SQL Server agent deployments.\n', + examples=[300], + ) + enabled: Optional[bool] = Field(None, examples=[False]) + enabled_tempdb: Optional[bool] = Field( + None, + description="Enable the collection of user table index usage statistics in tempdb database from the\n`sys.dm_db_index_usage_stats` DMV.\n\nBy default, we do not collect index usage statistics in the tempdb database, as those queries\nmight cause blocking. This configuration parameter allows enabling the collection of this metric.\nThis parameter is ignored if the 'enabled' option for 'index_usage_metrics' is set to false.\n", + examples=[False], + ) + + +class InstanceMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class MasterFilesMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class PrimaryLogShippingMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class SecondaryLogShippingMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class ServerStateMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[True]) + + +class TaskSchedulerMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class TempdbFileSpaceUsageMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class XeMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = Field(None, examples=[False]) + + +class DatabaseMetrics(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + ao_metrics: Optional[AoMetrics] = None + db_backup_metrics: Optional[DbBackupMetrics] = None + db_files_metrics: Optional[DbFilesMetrics] = None + db_fragmentation_metrics: Optional[DbFragmentationMetrics] = None + db_stats_metrics: Optional[DbStatsMetrics] = None + fci_metrics: Optional[FciMetrics] = None + file_stats_metrics: Optional[FileStatsMetrics] = None + index_usage_metrics: Optional[IndexUsageMetrics] = None + instance_metrics: Optional[InstanceMetrics] = None + master_files_metrics: Optional[MasterFilesMetrics] = None + primary_log_shipping_metrics: Optional[PrimaryLogShippingMetrics] = None + secondary_log_shipping_metrics: Optional[SecondaryLogShippingMetrics] = None + server_state_metrics: Optional[ServerStateMetrics] = None + task_scheduler_metrics: Optional[TaskSchedulerMetrics] = None + tempdb_file_space_usage_metrics: Optional[TempdbFileSpaceUsageMetrics] = None + xe_metrics: Optional[XeMetrics] = None + + class DeadlocksCollection(BaseModel): model_config = ConfigDict( arbitrary_types_allowed=True, @@ -179,11 +368,9 @@ class InstanceConfig(BaseModel): ) adoprovider: Optional[str] = None agent_jobs: Optional[AgentJobs] = None - ao_database: Optional[str] = None autodiscovery_db_service_check: Optional[bool] = None autodiscovery_exclude: Optional[tuple[str, ...]] = None autodiscovery_include: Optional[tuple[str, ...]] = None - availability_group: Optional[str] = None aws: Optional[Aws] = None azure: Optional[Azure] = None collect_settings: Optional[CollectSettings] = None @@ -195,6 +382,7 @@ class InstanceConfig(BaseModel): database_autodiscovery: Optional[bool] = None database_autodiscovery_interval: Optional[int] = None database_instance_collection_interval: Optional[float] = None + database_metrics: Optional[DatabaseMetrics] = None db_fragmentation_object_names: Optional[tuple[str, ...]] = None dbm: Optional[bool] = None deadlocks_collection: Optional[DeadlocksCollection] = None @@ -205,20 +393,6 @@ class InstanceConfig(BaseModel): gcp: Optional[Gcp] = None host: str ignore_missing_database: Optional[bool] = None - include_ao_metrics: Optional[bool] = None - include_db_fragmentation_metrics: Optional[bool] = None - include_db_fragmentation_metrics_tempdb: Optional[bool] = None - include_fci_metrics: Optional[bool] = None - include_index_usage_metrics: Optional[bool] = None - include_index_usage_metrics_tempdb: Optional[bool] = None - include_instance_metrics: Optional[bool] = None - include_master_files_metrics: Optional[bool] = None - include_primary_log_shipping_metrics: Optional[bool] = None - include_secondary_log_shipping_metrics: Optional[bool] = None - include_task_scheduler_metrics: Optional[bool] = None - include_tempdb_file_space_usage_metrics: Optional[bool] = None - include_xe_metrics: Optional[bool] = None - index_usage_metrics_interval: Optional[int] = None log_unobfuscated_plans: Optional[bool] = None log_unobfuscated_queries: Optional[bool] = None managed_identity: Optional[ManagedIdentity] = None @@ -226,7 +400,6 @@ class InstanceConfig(BaseModel): min_collection_interval: Optional[float] = None obfuscator_options: Optional[ObfuscatorOptions] = None only_custom_queries: Optional[bool] = None - only_emit_local: Optional[bool] = None password: Optional[str] = None proc_only_if: Optional[str] = None proc_only_if_database: Optional[str] = None diff --git a/sqlserver/tests/conftest.py b/sqlserver/tests/conftest.py index bdcda43ccd91f..bfe6dd7fdf71f 100644 --- a/sqlserver/tests/conftest.py +++ b/sqlserver/tests/conftest.py @@ -245,11 +245,7 @@ def instance_e2e(instance_docker): @pytest.fixture def instance_ao_docker_primary(instance_docker): - instance_docker['database_metrics'] = { - 'ao_metrics': { - 'enabled': True, - } - } + instance_docker['database_metrics']['ao_metrics']['enabled'] = True return instance_docker From 36d38926441234b58301ef0bb7ad2e703081fa60 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Tue, 26 Nov 2024 23:41:39 +0000 Subject: [PATCH 13/18] Update descriptions of database metrics configurations --- sqlserver/assets/configuration/spec.yaml | 114 +++++++++--------- .../sqlserver/config_models/instance.py | 46 ++----- 2 files changed, 63 insertions(+), 97 deletions(-) diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index f02d44c1b54cf..1968f5072ceef 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -135,7 +135,18 @@ files: options: - name: ao_metrics description: | - AlwaysOn availability group metrics. + Configure collection of AlwaysOn availability group metrics. + + When the `ao_metrics.enabled` is True, use `ao_metrics.availability_group` to specify the + resource group id of a specific availability group that you would like to monitor. + If no availability group is specified, then we will collect AlwaysOn metrics for all + availability groups on the current replica. + + Primary replicas may emit metrics for remote secondary replicas + in the same availability group. If `ao_metrics.only_emit_local` is set to true, + the primary replica will only emit information local to itself. + + If `ao_metrics.ao_database` is set, AlwaysOn metrics are only emitted for the selected `ao_database`. value: type: object properties: @@ -143,27 +154,18 @@ files: type: boolean example: false - name: availability_group - description: | - When the `ao_metrics` is enabled, you can provide the resource - group id of a specific availability group that you would like to monitor. - If no availability group is specified, then we will collect AlwaysOn metrics - for all availability groups on the current replica. type: string - example: - name: only_emit_local - description: | - Primary replicas may emit metrics for remote secondary replicas - in the same availability group. If this option is set to true, - the primary replica will only emit information local to itself. type: boolean example: false - name: ao_database - description: | - AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. type: string - name: db_backup_metrics description: | - Database backup metrics. + Configure collection of database backup metrics. + Use `db_backup_metrics.collection_interval` to set the interval (in seconds) for the collection of + database backup metrics. Defaults to 300 seconds (5 minutes). If you intend on updating this value, + it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. value: type: object properties: @@ -171,16 +173,12 @@ files: type: boolean example: true - name: collection_interval - description: | - Set the interval (in seconds) for the collection of database backup metrics. - Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - to use a consistent value throughout all SQL Server agent deployments. type: integer example: 300 display_default: 300 - name: db_files_metrics description: | - Database files metrics. + Configure collection of database files metrics. value: type: object properties: @@ -189,7 +187,7 @@ files: example: true - name: db_stats_metrics description: | - Database stats metrics + Configure collection of database stats metrics value: type: object properties: @@ -198,36 +196,36 @@ files: example: true - name: db_fragmentation_metrics description: | - Database fragmentation metrics. Note these queries can be resource intensive on large datasets. - Recommend to limit these via autodiscovery or specific database instances. + Configure collection of database fragmentation metrics. + Note these queries can be resource intensive on large datasets. Recommend to limit these via + autodiscovery or specific database instances. + + Use `db_fragmentation_metrics.enabled_tempdb` to enable collection of database index fragmentation statistics + in tempdb database from the `sys.dm_db_index_physical_stats` DMF. + By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + might cause blocking. This configuration parameter allows enabling the collection of this metric. + This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + + Use `db_fragmentation_metrics.collection_interval` to set the interval (in seconds) for the collection of + database fragmentation metrics from the `sys.dm_db_index_physical_stats` DMF. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. value: type: object properties: - name: enabled type: boolean example: false - - name: enabled_tempdb - description: | - Enable collection of database index fragmentation statistics in tempdb database from the - `sys.dm_db_index_physical_stats` DMF. - - By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - might cause blocking. This configuration parameter allows enabling the collection of this metric. - This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + - name: enabled_tempdb type: boolean example: false - name: collection_interval - description: | - Configure the interval (in seconds) for the collection of database fragmentation metrics from the - `sys.dm_db_index_physical_stats` DMF. - Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - to use a consistent value throughout all SQL Server agent deployments. type: integer example: 300 display_default: 300 - name: fci_metrics description: | - Failover Cluster Instance metrics. Note that these metrics + Configure collection of failover Cluster Instance metrics. Note that these metrics requires a SQLServer set up with Failover Clustering enabled. value: type: object @@ -237,7 +235,7 @@ files: example: false - name: file_stats_metrics description: | - File stats metrics + Configure collection of file stats metrics. value: type: object properties: @@ -246,10 +244,20 @@ files: example: true - name: index_usage_metrics description: | - User table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. - + Configure collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable `database_autodiscovery` or set `database`. + + Use `index_usage_metrics.enabled_tempdb` to enable collection of user table index usage statistics in tempdb + database from the `sys.dm_db_index_usage_stats` DMV. + By default, we do not collect index usage statistics in the tempdb database, as those queries + might cause blocking. This configuration parameter allows enabling the collection of this metric. + This parameter is ignored if 'index_usage_metrics.enabled' is set to false. + + Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index usage + statistics from the `sys.dm_db_index_usage_stats` DMV. + Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + to use a consistent value throughout all SQL Server agent deployments. value: type: object properties: @@ -257,27 +265,15 @@ files: type: boolean example: false - name: enabled_tempdb - description: | - Enable the collection of user table index usage statistics in tempdb database from the - `sys.dm_db_index_usage_stats` DMV. - - By default, we do not collect index usage statistics in the tempdb database, as those queries - might cause blocking. This configuration parameter allows enabling the collection of this metric. - This parameter is ignored if the 'enabled' option for 'index_usage_metrics' is set to false. type: boolean example: false - name: collection_interval - description: | - Configure the interval (in seconds) for the collection of index usage statistics from the - `sys.dm_db_index_usage_stats` DMV. - Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - to use a consistent value throughout all SQL Server agent deployments. type: integer example: 300 display_default: 300 - name: instance_metrics description: | - Server-level instance metrics. When setting up multiple instances for + Configure collection of server-level instance metrics. When setting up multiple instances for different databases on the same host these metrics will be duplicated unless this option is turned off. value: type: object @@ -287,7 +283,7 @@ files: example: true - name: master_files_metrics description: | - Database file size and state from `sys.master_files` + Configure collection of database file size and state from `sys.master_files` value: type: object properties: @@ -296,7 +292,7 @@ files: example: false - name: primary_log_shipping_metrics description: | - Metrics for a log shipping setup. Required to run against the + Configure collection of metrics for a log shipping setup. Required to run against the primary instance in a transaction log shipping configuration. Note that the Datadog user needs to be present in msdb and must be added to the db_datareader role. value: @@ -307,7 +303,7 @@ files: example: false - name: secondary_log_shipping_metrics description: | - Metrics for a log shipping setup. Required to run against the + Configure collection of metrics for a log shipping setup. Required to run against the secondary instance in a transaction log shipping configuration. Note that the Datadog user needs to be present in msdb and must be added to the db_datareader role. value: @@ -318,7 +314,7 @@ files: example: false - name: server_state_metrics description: | - Server state metrics + Configure collection of server state metrics value: type: object properties: @@ -327,7 +323,7 @@ files: example: true - name: task_scheduler_metrics description: | - Additional Task and Scheduler metrics. + Configure collection of additional Task and Scheduler metrics. value: type: object properties: @@ -336,7 +332,7 @@ files: example: false - name: tempdb_file_space_usage_metrics description: | - tempdb file space usage metrics for how space is used in tempdb data files. + Configure collection of tempdb file space usage metrics for how space is used in tempdb data files. value: type: object properties: @@ -345,7 +341,7 @@ files: example: false - name: xe_metrics description: | - Extended events (XE) metrics. + Configure collection of extended events (XE) metrics. value: type: object properties: diff --git a/sqlserver/datadog_checks/sqlserver/config_models/instance.py b/sqlserver/datadog_checks/sqlserver/config_models/instance.py index 9da5b0e64915b..1f83cbd0ccc1d 100644 --- a/sqlserver/datadog_checks/sqlserver/config_models/instance.py +++ b/sqlserver/datadog_checks/sqlserver/config_models/instance.py @@ -74,20 +74,10 @@ class AoMetrics(BaseModel): arbitrary_types_allowed=True, frozen=True, ) - ao_database: Optional[str] = Field( - None, description='AlwaysOn metrics are only emitted for the selected `ao_database` if not empty.\n' - ) - availability_group: Optional[str] = Field( - None, - description='When the `ao_metrics` is enabled, you can provide the resource \ngroup id of a specific availability group that you would like to monitor.\nIf no availability group is specified, then we will collect AlwaysOn metrics \nfor all availability groups on the current replica.\n', - examples=[''], - ) + ao_database: Optional[str] = None + availability_group: Optional[str] = None enabled: Optional[bool] = Field(None, examples=[False]) - only_emit_local: Optional[bool] = Field( - None, - description='Primary replicas may emit metrics for remote secondary replicas\nin the same availability group. If this option is set to true,\nthe primary replica will only emit information local to itself.\n', - examples=[False], - ) + only_emit_local: Optional[bool] = Field(None, examples=[False]) class DbBackupMetrics(BaseModel): @@ -95,11 +85,7 @@ class DbBackupMetrics(BaseModel): arbitrary_types_allowed=True, frozen=True, ) - collection_interval: Optional[int] = Field( - None, - description='Set the interval (in seconds) for the collection of database backup metrics.\nDefaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended\nto use a consistent value throughout all SQL Server agent deployments.\n', - examples=[300], - ) + collection_interval: Optional[int] = Field(None, examples=[300]) enabled: Optional[bool] = Field(None, examples=[True]) @@ -116,17 +102,9 @@ class DbFragmentationMetrics(BaseModel): arbitrary_types_allowed=True, frozen=True, ) - collection_interval: Optional[int] = Field( - None, - description='Configure the interval (in seconds) for the collection of database fragmentation metrics from the\n`sys.dm_db_index_physical_stats` DMF.\nDefaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended\nto use a consistent value throughout all SQL Server agent deployments.\n', - examples=[300], - ) + collection_interval: Optional[int] = Field(None, examples=[300]) enabled: Optional[bool] = Field(None, examples=[False]) - enabled_tempdb: Optional[bool] = Field( - None, - description="Enable collection of database index fragmentation statistics in tempdb database from the\n`sys.dm_db_index_physical_stats` DMF.\n\nBy default, we do not collect index fragmentation statistics in the tempdb database, as those queries\nmight cause blocking. This configuration parameter allows enabling the collection of this metric.\nThis parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false.\n", - examples=[False], - ) + enabled_tempdb: Optional[bool] = Field(None, examples=[False]) class DbStatsMetrics(BaseModel): @@ -158,17 +136,9 @@ class IndexUsageMetrics(BaseModel): arbitrary_types_allowed=True, frozen=True, ) - collection_interval: Optional[int] = Field( - None, - description='Configure the interval (in seconds) for the collection of index usage statistics from the\n`sys.dm_db_index_usage_stats` DMV.\nDefaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended\nto use a consistent value throughout all SQL Server agent deployments.\n', - examples=[300], - ) + collection_interval: Optional[int] = Field(None, examples=[300]) enabled: Optional[bool] = Field(None, examples=[False]) - enabled_tempdb: Optional[bool] = Field( - None, - description="Enable the collection of user table index usage statistics in tempdb database from the\n`sys.dm_db_index_usage_stats` DMV.\n\nBy default, we do not collect index usage statistics in the tempdb database, as those queries\nmight cause blocking. This configuration parameter allows enabling the collection of this metric.\nThis parameter is ignored if the 'enabled' option for 'index_usage_metrics' is set to false.\n", - examples=[False], - ) + enabled_tempdb: Optional[bool] = Field(None, examples=[False]) class InstanceMetrics(BaseModel): From 6ea4f4919628c52c790b8d83b4a712d3472bc8a2 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Tue, 26 Nov 2024 23:45:14 +0000 Subject: [PATCH 14/18] Linter fix --- sqlserver/assets/configuration/spec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 1968f5072ceef..a87490b6d05fa 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -254,8 +254,8 @@ files: might cause blocking. This configuration parameter allows enabling the collection of this metric. This parameter is ignored if 'index_usage_metrics.enabled' is set to false. - Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index usage - statistics from the `sys.dm_db_index_usage_stats` DMV. + Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index + usage statistics from the `sys.dm_db_index_usage_stats` DMV. Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. value: From 3ea9e0f5bf1e65b8cab55e088687d201556af751 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Wed, 27 Nov 2024 00:07:06 +0000 Subject: [PATCH 15/18] sync config files --- .../sqlserver/data/conf.yaml.example | 286 ++++++------------ 1 file changed, 97 insertions(+), 189 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index 4baab099f4c75..acd11f7c169c8 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -131,223 +131,131 @@ instances: # # database_metrics: - ## AlwaysOn availability group metrics. - # - # ao_metrics: - - ## @param enabled - boolean - optional - default: false - ## Enable collection of AlwaysOn availability group metrics. - # - # enabled: false - - ## @param availability_group - string - optional - ## When the `ao_metrics` is enabled, you can provide the resource - ## group id of a specific availability group that you would like to monitor. - ## If no availability group is specified, then we will collect AlwaysOn metrics - ## for all availability groups on the current replica. - # - # availability_group: - - ## @param only_emit_local - boolean - optional - default: false - ## Primary replicas may emit metrics for remote secondary replicas - ## in the same availability group. If this option is set to true, - ## the primary replica will only emit information local to itself. - # - # only_emit_local: false - - ## @param ao_database - string - optional - ## AlwaysOn metrics are only emitted for the selected `ao_database` if not empty. - # - # ao_database: - - ## Database fragmentation metrics. Note these queries can be resource intensive on large datasets. - ## Recommend to limit these via autodiscovery or specific database instances. - # - # db_fragmentation_metrics: - - ## @param enabled - boolean - optional - default: false - ## Enable collection of database fragmentation metrics. - # - # enabled: false - - ## @param enabled_tempdb - boolean - optional - default: false - ## Enable collection of database index fragmentation statistics in tempdb database from the - ## `sys.dm_db_index_physical_stats` DMF. - ## - ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. - # - # enabled_tempdb: false - - ## @param collection_interval - integer - optional - default: 300 - ## Configure the interval (in seconds) for the collection of database fragmentation metrics from the - ## `sys.dm_db_index_physical_stats` DMF. - ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - ## to use a consistent value throughout all SQL Server agent deployments. - # - # collection_interval: 300 - - ## Failover Cluster Instance metrics. Note that these metrics - ## requires a SQLServer set up with Failover Clustering enabled. - # - # fci_metrics: - - ## @param enabled - boolean - optional - default: false - ## Enable collection of Failover Cluster Instance metrics. - # - # enabled: false - - ## User table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## @param ao_metrics - mapping - optional + ## Configure collection of AlwaysOn availability group metrics. ## - ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable - ## `database_autodiscovery` or set `database`. - # - # index_usage_metrics: - - ## @param enabled - boolean - optional - default: true - ## Enable collection of index usage metrics. - # - # enabled: true - - ## @param enabled_tempdb - boolean - optional - default: false - ## Enable the collection of user table index usage statistics in tempdb database from the - ## `sys.dm_db_index_usage_stats` DMV. - ## - ## By default, we do not collect index usage statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if the 'enabled' option for 'index_usage_metrics' is set to false. - # - # enabled_tempdb: false - - ## @param collection_interval - integer - optional - default: 300 - ## Configure the interval (in seconds) for the collection of index usage statistics from the - ## `sys.dm_db_index_usage_stats` DMV. - ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - ## to use a consistent value throughout all SQL Server agent deployments. - # - # collection_interval: 300 - - ## Server-level instance metrics. When setting up multiple instances for - ## different databases on the same host these metrics will be duplicated unless this option is turned off. + ## When the `ao_metrics.enabled` is True, use `ao_metrics.availability_group` to specify the + ## resource group id of a specific availability group that you would like to monitor. + ## If no availability group is specified, then we will collect AlwaysOn metrics for all + ## availability groups on the current replica. + ## + ## Primary replicas may emit metrics for remote secondary replicas + ## in the same availability group. If `ao_metrics.only_emit_local` is set to true, + ## the primary replica will only emit information local to itself. + ## + ## If `ao_metrics.ao_database` is set, AlwaysOn metrics are only emitted for the selected `ao_database`. # - # instance_metrics: + # ao_metrics: {} - ## @param enabled - boolean - optional - default: true - ## Enable collection of instance metrics. - # - # enabled: true - - ## Database file size and state from `sys.master_files` + ## @param db_backup_metrics - mapping - optional + ## Configure collection of database backup metrics. + ## Use `db_backup_metrics.collection_interval` to set the interval (in seconds) for the collection of + ## database backup metrics. Defaults to 300 seconds (5 minutes). If you intend on updating this value, + ## it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. # - # master_files_metrics: - - ## @param enabled - boolean - optional - default: false - ## Enable collection of master files metrics. - # - # enabled: false + # db_backup_metrics: {} - ## Metrics for a log shipping setup. Required to run against the - ## primary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + ## @param db_files_metrics - mapping - optional + ## Configure collection of database files metrics. # - # primary_log_shipping_metrics: - - ## @param enabled - boolean - optional - default: false - ## Enable collection of primary log shipping metrics. - # - # enabled: false + # db_files_metrics: {} - ## Metrics for a log shipping setup. Required to run against the - ## secondary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + ## @param db_stats_metrics - mapping - optional + ## Configure collection of database stats metrics # - # secondary_log_shipping_metrics: - - ## @param enabled - boolean - optional - default: false - ## Enable collection of secondary log shipping metrics. - # - # enabled: false + # db_stats_metrics: {} - ## Additional Task and Scheduler metrics. + ## @param db_fragmentation_metrics - mapping - optional + ## Configure collection of database fragmentation metrics. + ## Note these queries can be resource intensive on large datasets. Recommend to limit these via + ## autodiscovery or specific database instances. + ## + ## Use `db_fragmentation_metrics.enabled_tempdb` to enable collection of database index fragmentation statistics + ## in tempdb database from the `sys.dm_db_index_physical_stats` DMF. + ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + ## + ## Use `db_fragmentation_metrics.collection_interval` to set the interval (in seconds) for the collection of + ## database fragmentation metrics from the `sys.dm_db_index_physical_stats` DMF. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. # - # task_scheduler_metrics: + # db_fragmentation_metrics: {} - ## @param enabled - boolean - optional - default: false - ## Enable collection of Task and Scheduler metrics. - # - # enabled: false - - ## tempdb file space usage metrics for how space is used in tempdb data files. + ## @param fci_metrics - mapping - optional + ## Configure collection of failover Cluster Instance metrics. Note that these metrics + ## requires a SQLServer set up with Failover Clustering enabled. # - # tempdb_file_space_usage_metrics: - - ## @param enabled - boolean - optional - default: true - ## Enable collection of tempdb file space usage metrics. - # - # enabled: true + # fci_metrics: {} - ## Extended events (XE) metrics. + ## @param file_stats_metrics - mapping - optional + ## Configure collection of file stats metrics. # - # xe_metrics: + # file_stats_metrics: {} - ## @param enabled - boolean - optional - default: false - ## Enable collection of extended events (XE) metrics. The collection of XE metrics is automatically enabled - ## when `deadlocks_collection` is enabled. - # - # enabled: false - - ## Database backup metrics. + ## @param index_usage_metrics - mapping - optional + ## Configure collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable + ## `database_autodiscovery` or set `database`. + ## + ## Use `index_usage_metrics.enabled_tempdb` to enable collection of user table index usage statistics in tempdb + ## database from the `sys.dm_db_index_usage_stats` DMV. + ## By default, we do not collect index usage statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if 'index_usage_metrics.enabled' is set to false. + ## + ## Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index + ## usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. # - # db_backup_metrics: - - ## @param enabled - boolean - optional - default: true - ## Enable collection of database backup metrics - # - # enabled: true - - ## @param collection_interval - integer - optional - default: 300 - ## Set the interval (in seconds) for the collection of database backup metrics. - ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - ## to use a consistent value throughout all SQL Server agent deployments. - # - # collection_interval: 300 + # index_usage_metrics: {} - ## Database files metrics. + ## @param instance_metrics - mapping - optional + ## Configure collection of server-level instance metrics. When setting up multiple instances for + ## different databases on the same host these metrics will be duplicated unless this option is turned off. # - # db_files_metrics: + # instance_metrics: {} - ## @param enabled - boolean - optional - default: true - ## Enable collection of database files metrics - # - # enabled: true + ## @param master_files_metrics - mapping - optional + ## Configure collection of database file size and state from `sys.master_files` + # + # master_files_metrics: {} - ## Database stats metrics + ## @param primary_log_shipping_metrics - mapping - optional + ## Configure collection of metrics for a log shipping setup. Required to run against the + ## primary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. # - # db_stats_metrics: + # primary_log_shipping_metrics: {} - ## @param enabled - boolean - optional - default: true - ## Enable collection of database stats metrics - # - # enabled: true + ## @param secondary_log_shipping_metrics - mapping - optional + ## Configure collection of metrics for a log shipping setup. Required to run against the + ## secondary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + # + # secondary_log_shipping_metrics: {} - ## File stats metrics + ## @param server_state_metrics - mapping - optional + ## Configure collection of server state metrics # - # file_stats_metrics: + # server_state_metrics: {} - ## @param enabled - boolean - optional - default: true - ## Enable collection of file stats metrics - # - # enabled: true + ## @param task_scheduler_metrics - mapping - optional + ## Configure collection of additional Task and Scheduler metrics. + # + # task_scheduler_metrics: {} - ## Server state metrics + ## @param tempdb_file_space_usage_metrics - mapping - optional + ## Configure collection of tempdb file space usage metrics for how space is used in tempdb data files. # - # server_state_metrics: + # tempdb_file_space_usage_metrics: {} - ## @param enabled - boolean - optional - default: true - ## Enable collection of server state metrics - # - # enabled: true + ## @param xe_metrics - mapping - optional + ## Configure collection of extended events (XE) metrics. + # + # xe_metrics: {} ## Configure collection of agent jobs events and metrics # From d2a5dc7d8f779ed4a57ff46874cc10cdb4b9195b Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Mon, 2 Dec 2024 13:10:04 +0000 Subject: [PATCH 16/18] Add hidden tag to database_metrics --- sqlserver/assets/configuration/spec.yaml | 1 + sqlserver/changelog.d/19111.added | 1 + sqlserver/changelog.d/19111.deprecated | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 sqlserver/changelog.d/19111.added delete mode 100644 sqlserver/changelog.d/19111.deprecated diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index a87490b6d05fa..5c71542637323 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -132,6 +132,7 @@ files: - name: database_metrics description: | Configure the collection of database metrics + hidden: true options: - name: ao_metrics description: | diff --git a/sqlserver/changelog.d/19111.added b/sqlserver/changelog.d/19111.added new file mode 100644 index 0000000000000..c3ae6706f8277 --- /dev/null +++ b/sqlserver/changelog.d/19111.added @@ -0,0 +1 @@ +Update configuration structure and allow configuration of all database metrics \ No newline at end of file diff --git a/sqlserver/changelog.d/19111.deprecated b/sqlserver/changelog.d/19111.deprecated deleted file mode 100644 index 0519995f69655..0000000000000 --- a/sqlserver/changelog.d/19111.deprecated +++ /dev/null @@ -1 +0,0 @@ -Deprecate usage of instance_config in database_metrics From 58ce0acaf7f8068438dfd315edc71bca65954ebd Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Mon, 2 Dec 2024 13:15:34 +0000 Subject: [PATCH 17/18] sync configs --- .../sqlserver/data/conf.yaml.example | 130 ------------------ 1 file changed, 130 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index acd11f7c169c8..060683dbf06fc 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -127,136 +127,6 @@ instances: # autodiscovery_db_service_check: false - ## Configure the collection of database metrics - # - # database_metrics: - - ## @param ao_metrics - mapping - optional - ## Configure collection of AlwaysOn availability group metrics. - ## - ## When the `ao_metrics.enabled` is True, use `ao_metrics.availability_group` to specify the - ## resource group id of a specific availability group that you would like to monitor. - ## If no availability group is specified, then we will collect AlwaysOn metrics for all - ## availability groups on the current replica. - ## - ## Primary replicas may emit metrics for remote secondary replicas - ## in the same availability group. If `ao_metrics.only_emit_local` is set to true, - ## the primary replica will only emit information local to itself. - ## - ## If `ao_metrics.ao_database` is set, AlwaysOn metrics are only emitted for the selected `ao_database`. - # - # ao_metrics: {} - - ## @param db_backup_metrics - mapping - optional - ## Configure collection of database backup metrics. - ## Use `db_backup_metrics.collection_interval` to set the interval (in seconds) for the collection of - ## database backup metrics. Defaults to 300 seconds (5 minutes). If you intend on updating this value, - ## it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. - # - # db_backup_metrics: {} - - ## @param db_files_metrics - mapping - optional - ## Configure collection of database files metrics. - # - # db_files_metrics: {} - - ## @param db_stats_metrics - mapping - optional - ## Configure collection of database stats metrics - # - # db_stats_metrics: {} - - ## @param db_fragmentation_metrics - mapping - optional - ## Configure collection of database fragmentation metrics. - ## Note these queries can be resource intensive on large datasets. Recommend to limit these via - ## autodiscovery or specific database instances. - ## - ## Use `db_fragmentation_metrics.enabled_tempdb` to enable collection of database index fragmentation statistics - ## in tempdb database from the `sys.dm_db_index_physical_stats` DMF. - ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. - ## - ## Use `db_fragmentation_metrics.collection_interval` to set the interval (in seconds) for the collection of - ## database fragmentation metrics from the `sys.dm_db_index_physical_stats` DMF. - ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - ## to use a consistent value throughout all SQL Server agent deployments. - # - # db_fragmentation_metrics: {} - - ## @param fci_metrics - mapping - optional - ## Configure collection of failover Cluster Instance metrics. Note that these metrics - ## requires a SQLServer set up with Failover Clustering enabled. - # - # fci_metrics: {} - - ## @param file_stats_metrics - mapping - optional - ## Configure collection of file stats metrics. - # - # file_stats_metrics: {} - - ## @param index_usage_metrics - mapping - optional - ## Configure collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. - ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable - ## `database_autodiscovery` or set `database`. - ## - ## Use `index_usage_metrics.enabled_tempdb` to enable collection of user table index usage statistics in tempdb - ## database from the `sys.dm_db_index_usage_stats` DMV. - ## By default, we do not collect index usage statistics in the tempdb database, as those queries - ## might cause blocking. This configuration parameter allows enabling the collection of this metric. - ## This parameter is ignored if 'index_usage_metrics.enabled' is set to false. - ## - ## Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index - ## usage statistics from the `sys.dm_db_index_usage_stats` DMV. - ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended - ## to use a consistent value throughout all SQL Server agent deployments. - # - # index_usage_metrics: {} - - ## @param instance_metrics - mapping - optional - ## Configure collection of server-level instance metrics. When setting up multiple instances for - ## different databases on the same host these metrics will be duplicated unless this option is turned off. - # - # instance_metrics: {} - - ## @param master_files_metrics - mapping - optional - ## Configure collection of database file size and state from `sys.master_files` - # - # master_files_metrics: {} - - ## @param primary_log_shipping_metrics - mapping - optional - ## Configure collection of metrics for a log shipping setup. Required to run against the - ## primary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. - # - # primary_log_shipping_metrics: {} - - ## @param secondary_log_shipping_metrics - mapping - optional - ## Configure collection of metrics for a log shipping setup. Required to run against the - ## secondary instance in a transaction log shipping configuration. Note that - ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. - # - # secondary_log_shipping_metrics: {} - - ## @param server_state_metrics - mapping - optional - ## Configure collection of server state metrics - # - # server_state_metrics: {} - - ## @param task_scheduler_metrics - mapping - optional - ## Configure collection of additional Task and Scheduler metrics. - # - # task_scheduler_metrics: {} - - ## @param tempdb_file_space_usage_metrics - mapping - optional - ## Configure collection of tempdb file space usage metrics for how space is used in tempdb data files. - # - # tempdb_file_space_usage_metrics: {} - - ## @param xe_metrics - mapping - optional - ## Configure collection of extended events (XE) metrics. - # - # xe_metrics: {} - ## Configure collection of agent jobs events and metrics # # agent_jobs: From f6c2a20e23b4afdfd3febd1c5b2bc5b0af586980 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Mon, 2 Dec 2024 16:15:57 +0000 Subject: [PATCH 18/18] Make only newly exposed configurations hidden --- sqlserver/assets/configuration/spec.yaml | 6 +- .../sqlserver/data/conf.yaml.example | 102 ++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 5c71542637323..6594c7f51011f 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -132,7 +132,6 @@ files: - name: database_metrics description: | Configure the collection of database metrics - hidden: true options: - name: ao_metrics description: | @@ -167,6 +166,7 @@ files: Use `db_backup_metrics.collection_interval` to set the interval (in seconds) for the collection of database backup metrics. Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended to use a consistent value throughout all SQL Server agent deployments. + hidden: true value: type: object properties: @@ -180,6 +180,7 @@ files: - name: db_files_metrics description: | Configure collection of database files metrics. + hidden: true value: type: object properties: @@ -189,6 +190,7 @@ files: - name: db_stats_metrics description: | Configure collection of database stats metrics + hidden: true value: type: object properties: @@ -237,6 +239,7 @@ files: - name: file_stats_metrics description: | Configure collection of file stats metrics. + hidden: true value: type: object properties: @@ -316,6 +319,7 @@ files: - name: server_state_metrics description: | Configure collection of server state metrics + hidden: true value: type: object properties: diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index 060683dbf06fc..bcd78b58b33c3 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -127,6 +127,108 @@ instances: # autodiscovery_db_service_check: false + ## Configure the collection of database metrics + # + # database_metrics: + + ## @param ao_metrics - mapping - optional + ## Configure collection of AlwaysOn availability group metrics. + ## + ## When the `ao_metrics.enabled` is True, use `ao_metrics.availability_group` to specify the + ## resource group id of a specific availability group that you would like to monitor. + ## If no availability group is specified, then we will collect AlwaysOn metrics for all + ## availability groups on the current replica. + ## + ## Primary replicas may emit metrics for remote secondary replicas + ## in the same availability group. If `ao_metrics.only_emit_local` is set to true, + ## the primary replica will only emit information local to itself. + ## + ## If `ao_metrics.ao_database` is set, AlwaysOn metrics are only emitted for the selected `ao_database`. + # + # ao_metrics: {} + + ## @param db_fragmentation_metrics - mapping - optional + ## Configure collection of database fragmentation metrics. + ## Note these queries can be resource intensive on large datasets. Recommend to limit these via + ## autodiscovery or specific database instances. + ## + ## Use `db_fragmentation_metrics.enabled_tempdb` to enable collection of database index fragmentation statistics + ## in tempdb database from the `sys.dm_db_index_physical_stats` DMF. + ## By default, we do not collect index fragmentation statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if the 'enabled' option for 'db_fragmentation_metrics' is set to false. + ## + ## Use `db_fragmentation_metrics.collection_interval` to set the interval (in seconds) for the collection of + ## database fragmentation metrics from the `sys.dm_db_index_physical_stats` DMF. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. + # + # db_fragmentation_metrics: {} + + ## @param fci_metrics - mapping - optional + ## Configure collection of failover Cluster Instance metrics. Note that these metrics + ## requires a SQLServer set up with Failover Clustering enabled. + # + # fci_metrics: {} + + ## @param index_usage_metrics - mapping - optional + ## Configure collection of user table index usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## Because the `sys.dm_db_index_usage_stats` view is scoped to the current database, enable + ## `database_autodiscovery` or set `database`. + ## + ## Use `index_usage_metrics.enabled_tempdb` to enable collection of user table index usage statistics in tempdb + ## database from the `sys.dm_db_index_usage_stats` DMV. + ## By default, we do not collect index usage statistics in the tempdb database, as those queries + ## might cause blocking. This configuration parameter allows enabling the collection of this metric. + ## This parameter is ignored if 'index_usage_metrics.enabled' is set to false. + ## + ## Use `index_usage_metrics.collection_interval` to set the interval (in seconds) for the collection of index + ## usage statistics from the `sys.dm_db_index_usage_stats` DMV. + ## Defaults to 300 seconds (5 minutes). If you intend on updating this value, it is strongly recommended + ## to use a consistent value throughout all SQL Server agent deployments. + # + # index_usage_metrics: {} + + ## @param instance_metrics - mapping - optional + ## Configure collection of server-level instance metrics. When setting up multiple instances for + ## different databases on the same host these metrics will be duplicated unless this option is turned off. + # + # instance_metrics: {} + + ## @param master_files_metrics - mapping - optional + ## Configure collection of database file size and state from `sys.master_files` + # + # master_files_metrics: {} + + ## @param primary_log_shipping_metrics - mapping - optional + ## Configure collection of metrics for a log shipping setup. Required to run against the + ## primary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + # + # primary_log_shipping_metrics: {} + + ## @param secondary_log_shipping_metrics - mapping - optional + ## Configure collection of metrics for a log shipping setup. Required to run against the + ## secondary instance in a transaction log shipping configuration. Note that + ## the Datadog user needs to be present in msdb and must be added to the db_datareader role. + # + # secondary_log_shipping_metrics: {} + + ## @param task_scheduler_metrics - mapping - optional + ## Configure collection of additional Task and Scheduler metrics. + # + # task_scheduler_metrics: {} + + ## @param tempdb_file_space_usage_metrics - mapping - optional + ## Configure collection of tempdb file space usage metrics for how space is used in tempdb data files. + # + # tempdb_file_space_usage_metrics: {} + + ## @param xe_metrics - mapping - optional + ## Configure collection of extended events (XE) metrics. + # + # xe_metrics: {} + ## Configure collection of agent jobs events and metrics # # agent_jobs: