From f46912f132ce80e73bb5b5acb70aa933bdad36d7 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Tue, 27 Sep 2022 13:54:34 -0400 Subject: [PATCH 01/13] Added 'AND' operator and parenthesis to properly build full query. --- products/sentinel_one.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index ab9ddd0..ff144a3 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -210,6 +210,8 @@ def build_query(self, filters: dict) -> Tuple[str, datetime, datetime]: # S1 requires the date range to be supplied in the query request, not the query text # therefore we return the from/to dates separately + if query_base: + query_base = f'({query_base}) AND' return query_base, from_date, to_date def _get_all_paginated_data(self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None, @@ -326,7 +328,7 @@ def _get_dv_events(self, query_id: str) -> list[dict]: def process_search(self, tag: Tag, base_query: dict, query: str) -> None: build_query, from_date, to_date = self.build_query(base_query) - query = query + build_query + query = f'{build_query} ({query})' self._echo(f'Built Query: {query}') if tag not in self._queries: From c538a0058d1cbf7ce47a8b71ec0e70383ec4aa3d Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Wed, 28 Sep 2022 10:29:26 -0400 Subject: [PATCH 02/13] fixed containscis parsing and existing operator,parameter pair check --- products/sentinel_one.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index ff144a3..2e8124b 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -391,9 +391,9 @@ def _process_queries(self): for tag, queries in self._queries.items(): for query in queries: - if query.operator == 'contains': + if query.operator == 'containscis': key = (query.operator, query.parameter) - if query.operator not in combined_queries: + if key not in combined_queries: combined_queries[key] = list() combined_queries[key].append((tag, query.search_value)) @@ -406,7 +406,7 @@ def _process_queries(self): # merge combined queries and add them to query_text data: list[Tuple[Tag, str]] for (operator, parameter), data in combined_queries.items(): - if operator == 'contains': + if operator == 'containscis': full_query = f'{parameter} in contains anycase ({", ".join(x[1] for x in data)})' tag = Tag(','.join(tag[0].tag for tag in data), ','.join(tag[0].data for tag in data)) From 7373070478808efe4a487f60851849f5c1508fc8 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Wed, 28 Sep 2022 11:26:23 -0400 Subject: [PATCH 03/13] added ability to use commandline filters with deffile --- products/sentinel_one.py | 19 +++++++++++-------- surveyor.py | 4 ++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index 2e8124b..b190d36 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -351,7 +351,6 @@ def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): parameter = PARAMETER_MAPPING[search_field] search_value = all_terms - if len(terms) > 1: search_value = f'({all_terms})' operator = 'in contains anycase' @@ -360,18 +359,19 @@ def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): if tag not in self._queries: self._queries[tag] = list() - self._queries[tag].append(Query(from_date, to_date, parameter, operator, search_value)) except KeyboardInterrupt: self._echo("Caught CTRL-C. Returning what we have...") - def _process_queries(self): + def _process_queries(self, base_query: dict): """ Process all cached queries. """ start_date = datetime.utcnow() end_date = start_date - + + query_base, from_date, to_date = self.build_query(base_query) + # determine earliest start date for tag, queries in self._queries.items(): for query in queries: @@ -408,7 +408,6 @@ def _process_queries(self): for (operator, parameter), data in combined_queries.items(): if operator == 'containscis': full_query = f'{parameter} in contains anycase ({", ".join(x[1] for x in data)})' - tag = Tag(','.join(tag[0].tag for tag in data), ','.join(tag[0].data for tag in data)) query_text.append((tag, full_query)) else: @@ -435,7 +434,11 @@ def _process_queries(self): # merge all query tags into a single string merged_tag = Tag(','.join(tag.tag for tag in merged_tags), ','.join(str(tag.data) for tag in merged_tags)) - + + if len(query_base): + # add base query filter if they exist + merged_query = f'{query_base} ({merged_query})' + if len(self._site_ids): # restrict query to specified sites # S1QL does not support restricting a query to a specified account ID @@ -494,13 +497,13 @@ def _process_queries(self): except KeyboardInterrupt: self._echo("Caught CTRL-C. Returning what we have . . .") - def get_results(self, final_call: bool = True) -> dict[Tag, list[Result]]: + def get_results(self, base_query: dict, final_call: bool = True) -> dict[Tag, list[Result]]: self.log.debug('Entered get_results') # process any unprocessed queries if final_call and len(self._queries) > 0: self.log.debug(f'Executing additional _process_queries') - self._process_queries() + self._process_queries(base_query) return self._results diff --git a/surveyor.py b/surveyor.py index dec267a..0a9e03b 100644 --- a/surveyor.py +++ b/surveyor.py @@ -344,7 +344,7 @@ def survey(ctx, product: str = 'cbr'): if product.has_results(): # write results as they become available - for tag, nested_results in product.get_results(final_call=False).items(): + for tag, nested_results in product.get_results(base_query, final_call=False).items(): _write_results(writer, nested_results, program, tag.data, tag, log, use_tqdm=True) @@ -352,7 +352,7 @@ def survey(ctx, product: str = 'cbr'): product.clear_results() # write any remaining results - for tag, nested_results in product.get_results().items(): + for tag, nested_results in product.get_results(base_query).items(): _write_results(writer, nested_results, tag.tag, tag.data, tag, log) if output_file: From a2b192be83207e91c721c916a048c2bb1c2afa8f Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Wed, 28 Sep 2022 11:53:33 -0400 Subject: [PATCH 04/13] added option for base_query to work for all commandline parameter options and resolved double merges of base_query. --- products/sentinel_one.py | 7 ++++--- surveyor.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index b190d36..db266a8 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -434,11 +434,12 @@ def _process_queries(self, base_query: dict): # merge all query tags into a single string merged_tag = Tag(','.join(tag.tag for tag in merged_tags), ','.join(str(tag.data) for tag in merged_tags)) - + if len(query_base): # add base query filter if they exist - merged_query = f'{query_base} ({merged_query})' - + if query_base not in merged_query: + merged_query = f'{query_base} ({merged_query})' + if len(self._site_ids): # restrict query to specified sites # S1QL does not support restricting a query to a specified account ID diff --git a/surveyor.py b/surveyor.py index 0a9e03b..2bd4fdc 100644 --- a/surveyor.py +++ b/surveyor.py @@ -288,7 +288,7 @@ def survey(ctx, product: str = 'cbr'): log_echo(f"Running Custom Query: {opt.query}", log) product.process_search(Tag('query'), base_query, opt.query) - for tag, results in product.get_results().items(): + for tag, results in product.get_results(base_query).items(): _write_results(writer, results, opt.query, "query", tag, log) # test if deffile exists @@ -328,7 +328,7 @@ def survey(ctx, product: str = 'cbr'): product.process_search(Tag(ioc), base_query, opt.query) del base_query[opt.ioc_type] - for tag, results in product.get_results().items(): + for tag, results in product.get_results(base_query).items(): _write_results(writer, results, ioc, 'ioc', tag, log) # run search against definition files and write to csv From 7d15be4c4c3238e8593be4b2460b909a58dab1a6 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Tue, 4 Oct 2022 16:13:30 -0400 Subject: [PATCH 05/13] Revert "added option for base_query to work for all commandline parameter options and resolved double merges of base_query." This reverts commit a2b192be83207e91c721c916a048c2bb1c2afa8f. --- products/sentinel_one.py | 7 +++---- surveyor.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index db266a8..b190d36 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -434,12 +434,11 @@ def _process_queries(self, base_query: dict): # merge all query tags into a single string merged_tag = Tag(','.join(tag.tag for tag in merged_tags), ','.join(str(tag.data) for tag in merged_tags)) - + if len(query_base): # add base query filter if they exist - if query_base not in merged_query: - merged_query = f'{query_base} ({merged_query})' - + merged_query = f'{query_base} ({merged_query})' + if len(self._site_ids): # restrict query to specified sites # S1QL does not support restricting a query to a specified account ID diff --git a/surveyor.py b/surveyor.py index 2bd4fdc..0a9e03b 100644 --- a/surveyor.py +++ b/surveyor.py @@ -288,7 +288,7 @@ def survey(ctx, product: str = 'cbr'): log_echo(f"Running Custom Query: {opt.query}", log) product.process_search(Tag('query'), base_query, opt.query) - for tag, results in product.get_results(base_query).items(): + for tag, results in product.get_results().items(): _write_results(writer, results, opt.query, "query", tag, log) # test if deffile exists @@ -328,7 +328,7 @@ def survey(ctx, product: str = 'cbr'): product.process_search(Tag(ioc), base_query, opt.query) del base_query[opt.ioc_type] - for tag, results in product.get_results(base_query).items(): + for tag, results in product.get_results().items(): _write_results(writer, results, ioc, 'ioc', tag, log) # run search against definition files and write to csv From 11c8426203c04bbbe6dd2c1e5fd855fd806ae4e5 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Tue, 4 Oct 2022 16:14:11 -0400 Subject: [PATCH 06/13] Revert "added ability to use commandline filters with deffile" This reverts commit 7373070478808efe4a487f60851849f5c1508fc8. --- products/sentinel_one.py | 19 ++++++++----------- surveyor.py | 4 ++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index b190d36..2e8124b 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -351,6 +351,7 @@ def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): parameter = PARAMETER_MAPPING[search_field] search_value = all_terms + if len(terms) > 1: search_value = f'({all_terms})' operator = 'in contains anycase' @@ -359,19 +360,18 @@ def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): if tag not in self._queries: self._queries[tag] = list() + self._queries[tag].append(Query(from_date, to_date, parameter, operator, search_value)) except KeyboardInterrupt: self._echo("Caught CTRL-C. Returning what we have...") - def _process_queries(self, base_query: dict): + def _process_queries(self): """ Process all cached queries. """ start_date = datetime.utcnow() end_date = start_date - - query_base, from_date, to_date = self.build_query(base_query) - + # determine earliest start date for tag, queries in self._queries.items(): for query in queries: @@ -408,6 +408,7 @@ def _process_queries(self, base_query: dict): for (operator, parameter), data in combined_queries.items(): if operator == 'containscis': full_query = f'{parameter} in contains anycase ({", ".join(x[1] for x in data)})' + tag = Tag(','.join(tag[0].tag for tag in data), ','.join(tag[0].data for tag in data)) query_text.append((tag, full_query)) else: @@ -434,11 +435,7 @@ def _process_queries(self, base_query: dict): # merge all query tags into a single string merged_tag = Tag(','.join(tag.tag for tag in merged_tags), ','.join(str(tag.data) for tag in merged_tags)) - - if len(query_base): - # add base query filter if they exist - merged_query = f'{query_base} ({merged_query})' - + if len(self._site_ids): # restrict query to specified sites # S1QL does not support restricting a query to a specified account ID @@ -497,13 +494,13 @@ def _process_queries(self, base_query: dict): except KeyboardInterrupt: self._echo("Caught CTRL-C. Returning what we have . . .") - def get_results(self, base_query: dict, final_call: bool = True) -> dict[Tag, list[Result]]: + def get_results(self, final_call: bool = True) -> dict[Tag, list[Result]]: self.log.debug('Entered get_results') # process any unprocessed queries if final_call and len(self._queries) > 0: self.log.debug(f'Executing additional _process_queries') - self._process_queries(base_query) + self._process_queries() return self._results diff --git a/surveyor.py b/surveyor.py index 0a9e03b..dec267a 100644 --- a/surveyor.py +++ b/surveyor.py @@ -344,7 +344,7 @@ def survey(ctx, product: str = 'cbr'): if product.has_results(): # write results as they become available - for tag, nested_results in product.get_results(base_query, final_call=False).items(): + for tag, nested_results in product.get_results(final_call=False).items(): _write_results(writer, nested_results, program, tag.data, tag, log, use_tqdm=True) @@ -352,7 +352,7 @@ def survey(ctx, product: str = 'cbr'): product.clear_results() # write any remaining results - for tag, nested_results in product.get_results(base_query).items(): + for tag, nested_results in product.get_results().items(): _write_results(writer, nested_results, tag.tag, tag.data, tag, log) if output_file: From 4689f0c7c070704575598b8b33798dc96a203c52 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Tue, 4 Oct 2022 16:19:26 -0400 Subject: [PATCH 07/13] fixed formatting for base_query --- products/sentinel_one.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index 2e8124b..c7841ac 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -210,8 +210,6 @@ def build_query(self, filters: dict) -> Tuple[str, datetime, datetime]: # S1 requires the date range to be supplied in the query request, not the query text # therefore we return the from/to dates separately - if query_base: - query_base = f'({query_base}) AND' return query_base, from_date, to_date def _get_all_paginated_data(self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None, @@ -328,7 +326,7 @@ def _get_dv_events(self, query_id: str) -> list[dict]: def process_search(self, tag: Tag, base_query: dict, query: str) -> None: build_query, from_date, to_date = self.build_query(base_query) - query = f'{build_query} ({query})' + query = f'({build_query}) AND ({query})' self._echo(f'Built Query: {query}') if tag not in self._queries: From cbfaf10314016a6ea3521898f95f4ffeb01e984b Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Tue, 4 Oct 2022 17:00:56 -0400 Subject: [PATCH 08/13] added abillity for variations of contains operator. --- products/sentinel_one.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index c7841ac..efc821b 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -389,7 +389,7 @@ def _process_queries(self): for tag, queries in self._queries.items(): for query in queries: - if query.operator == 'containscis': + if query.operator in ('contains', 'containscis', 'contains anycase'): key = (query.operator, query.parameter) if key not in combined_queries: combined_queries[key] = list() @@ -404,7 +404,7 @@ def _process_queries(self): # merge combined queries and add them to query_text data: list[Tuple[Tag, str]] for (operator, parameter), data in combined_queries.items(): - if operator == 'containscis': + if operator in ('contains', 'containscis', 'contains anycase'): full_query = f'{parameter} in contains anycase ({", ".join(x[1] for x in data)})' tag = Tag(','.join(tag[0].tag for tag in data), ','.join(tag[0].data for tag in data)) From abe1e04a9d7e7e280e2714e00034cdcfb2200ba0 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Wed, 5 Oct 2022 08:59:54 -0400 Subject: [PATCH 09/13] add base_query to self._queries if it exists --- products/sentinel_one.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index efc821b..4446f69 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -337,7 +337,10 @@ def process_search(self, tag: Tag, base_query: dict, query: str) -> None: def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): query_base, from_date, to_date = self.build_query(base_query) - + if query_base not in self._queries: + if tag not in self._queries: + self._queries[Tag("filter")] = list() + self._queries[Tag("filter")].append(Query(from_date, to_date, None, None, None, query_base)) try: for search_field, terms in criteria.items(): all_terms = ', '.join(f'"{term}"' for term in terms) From 97d358707c53b4246855908a6c8f49514de8cbc2 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Wed, 5 Oct 2022 09:26:28 -0400 Subject: [PATCH 10/13] add formatting for building query in order to not have double operators or missing parenthesis when mixing 'AND' and 'OR' operators --- products/sentinel_one.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index 4446f69..ed728b6 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -338,9 +338,10 @@ def process_search(self, tag: Tag, base_query: dict, query: str) -> None: def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): query_base, from_date, to_date = self.build_query(base_query) if query_base not in self._queries: + query = f'({query_base}) AND ' if tag not in self._queries: self._queries[Tag("filter")] = list() - self._queries[Tag("filter")].append(Query(from_date, to_date, None, None, None, query_base)) + self._queries[Tag("filter")].append(Query(from_date, to_date, None, None, None, query)) try: for search_field, terms in criteria.items(): all_terms = ', '.join(f'"{term}"' for term in terms) @@ -426,7 +427,7 @@ def _process_queries(self): merged_query = '' for tag, query in query_text[i:i + chunk_size]: # combine queries with ORs - if merged_query: + if merged_query and not merged_query.endswith("AND ") and not merged_query.endswith("OR "): merged_query += ' OR ' merged_query += query From 8c321db5a8244dcf9468d12eb5375c73f31783d7 Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Wed, 5 Oct 2022 14:05:10 -0400 Subject: [PATCH 11/13] prevent empty parenthesis and AND operator if build_query is empty --- products/sentinel_one.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index ed728b6..602debd 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -326,7 +326,8 @@ def _get_dv_events(self, query_id: str) -> list[dict]: def process_search(self, tag: Tag, base_query: dict, query: str) -> None: build_query, from_date, to_date = self.build_query(base_query) - query = f'({build_query}) AND ({query})' + if build_query: + query = f'({build_query}) AND ({query})' self._echo(f'Built Query: {query}') if tag not in self._queries: From aea382287e2de24ac563d07b251de3a98f5dab7d Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Fri, 7 Oct 2022 10:45:23 -0400 Subject: [PATCH 12/13] fix empty base_query for deffiles --- products/sentinel_one.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index 602debd..a61c2f3 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -338,7 +338,7 @@ def process_search(self, tag: Tag, base_query: dict, query: str) -> None: def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): query_base, from_date, to_date = self.build_query(base_query) - if query_base not in self._queries: + if query_base and query_base not in self._queries: query = f'({query_base}) AND ' if tag not in self._queries: self._queries[Tag("filter")] = list() From 60874436d5a6eb33d2f09b43ef3221df86ab137c Mon Sep 17 00:00:00 2001 From: Justin Lentz Date: Tue, 11 Oct 2022 15:33:13 -0400 Subject: [PATCH 13/13] add base_query to merged query instead of self._queries --- products/sentinel_one.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/products/sentinel_one.py b/products/sentinel_one.py index a61c2f3..ba3dc58 100644 --- a/products/sentinel_one.py +++ b/products/sentinel_one.py @@ -326,8 +326,7 @@ def _get_dv_events(self, query_id: str) -> list[dict]: def process_search(self, tag: Tag, base_query: dict, query: str) -> None: build_query, from_date, to_date = self.build_query(base_query) - if build_query: - query = f'({build_query}) AND ({query})' + self._query_base = build_query self._echo(f'Built Query: {query}') if tag not in self._queries: @@ -338,11 +337,7 @@ def process_search(self, tag: Tag, base_query: dict, query: str) -> None: def nested_process_search(self, tag: Tag, criteria: dict, base_query: dict): query_base, from_date, to_date = self.build_query(base_query) - if query_base and query_base not in self._queries: - query = f'({query_base}) AND ' - if tag not in self._queries: - self._queries[Tag("filter")] = list() - self._queries[Tag("filter")].append(Query(from_date, to_date, None, None, None, query)) + self._query_base = query_base try: for search_field, terms in criteria.items(): all_terms = ', '.join(f'"{term}"' for term in terms) @@ -428,7 +423,7 @@ def _process_queries(self): merged_query = '' for tag, query in query_text[i:i + chunk_size]: # combine queries with ORs - if merged_query and not merged_query.endswith("AND ") and not merged_query.endswith("OR "): + if merged_query: merged_query += ' OR ' merged_query += query @@ -439,11 +434,15 @@ def _process_queries(self): # merge all query tags into a single string merged_tag = Tag(','.join(tag.tag for tag in merged_tags), ','.join(str(tag.data) for tag in merged_tags)) + if len(self._query_base): + # add base_query filter to merged query string + merged_query = f'{self._query_base} AND ({merged_query})' + if len(self._site_ids): # restrict query to specified sites # S1QL does not support restricting a query to a specified account ID merged_query = f'SiteID in contains ("' + '", "'.join(self._site_ids) + f'") AND ({merged_query})' - + # build request body for DV API call params = self._get_default_body() params.update({