style: add pyupgrade and isort to pre-commit (#552)

* style: pre-commit update

* style: apply pyupgrade

* style: apply isort

.pre-commit-config.yaml

@@ -19,7 +19,17 @@ repos:
     hooks:
     -   id: check-merge-conflict
     -   id: debug-statements
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.31.0
+    hooks:
+    -   id: pyupgrade
+        args:
+        - --py37-plus
 -   repo: https://github.com/psf/black
-    rev: 21.12b0
+    rev: 22.1.0
     hooks:
     -   id: black
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Configuration file for the Sphinx documentation builder.
 #
@@ -14,21 +13,22 @@
 #
 import os
 import sys
+
 import sphinx_rtd_theme
 
 sys.path.insert(0, os.path.abspath("../pytest_splunk_addon"))
 
 
 # -- Project information -----------------------------------------------------
 
-project = u"pytest-splunk-addon"
-copyright = u"2021, Splunk, Inc."
-author = u"Splunk, Inc."
+project = "pytest-splunk-addon"
+copyright = "2021, Splunk, Inc."
+author = "Splunk, Inc."
 
 # The short X.Y version
-version = u""
+version = ""
 # The full version, including alpha/beta/rc tags
-release = u""
+release = ""
 
 
 # -- General configuration ---------------------------------------------------
@@ -134,8 +134,8 @@
     (
         master_doc,
         "pytest-splunk-addon.tex",
-        u"pytest-splunk-addon Documentation",
-        u"Splunk, Inc.",
+        "pytest-splunk-addon Documentation",
+        "Splunk, Inc.",
         "manual",
     ),
 ]
@@ -149,7 +149,7 @@
     (
         master_doc,
         "pytest-splunk-addon",
-        u"pytest-splunk-addon Documentation",
+        "pytest-splunk-addon Documentation",
         [author],
         1,
     )
@@ -165,7 +165,7 @@
     (
         master_doc,
         "pytest-splunk-addon",
-        u"pytest-splunk-addon Documentation",
+        "pytest-splunk-addon Documentation",
         author,
         "pytest-splunk-addon",
         "One line description of project.",

pyproject.toml

@@ -59,6 +59,9 @@ pytz = "^2021.1"
 sphinx-rtd-theme = "^1.0.0"
 sphinx-panels = "^0.6.0"
 
+[tool.isort]
+profile = "black"
+
 [tool.poetry.plugins]
 pytest11 = { plugin = "pytest_splunk_addon.plugin", "splunk" = "pytest_splunk_addon.splunk" }
 

pytest_splunk_addon/helmut/connector/sdk.py

@@ -16,8 +16,8 @@
 import logging
 import time
 
-from splunklib.binding import _NoAuthenticationToken, AuthenticationError
-from splunklib.client import Service, Endpoint
+from splunklib.binding import AuthenticationError, _NoAuthenticationToken
+from splunklib.client import Endpoint, Service
 
 LOGGER = logging.getLogger("helmut")
 
@@ -265,13 +265,11 @@ def is_logged_in(self):
             # FAST-8222
         except AuthenticationError as err:
             LOGGER.debug(
-                "SDKconnector %s:%s is NOT logged in" % (self.username, self.password)
+                f"SDKconnector {self.username}:{self.password} is NOT logged in"
             )
             return False
         else:
-            LOGGER.debug(
-                "SDKconnector %s:%s is logged in" % (self.username, self.password)
-            )
+            LOGGER.debug(f"SDKconnector {self.username}:{self.password} is logged in")
             return True
 
     def _was_logged_in(self):

pytest_splunk_addon/helmut/exceptions/job.py

@@ -16,8 +16,8 @@
 class JobNotFound(RuntimeError):
     def __init__(self, sid):
         self.sid = sid
-        super(JobNotFound, self).__init__(self._error_message)
+        super().__init__(self._error_message)
 
     @property
     def _error_message(self):
-        return "Could not find a job with SID {sid}".format(sid=self.sid)
+        return f"Could not find a job with SID {self.sid}"

pytest_splunk_addon/helmut/exceptions/search.py

@@ -20,7 +20,7 @@ class SearchFailure(RuntimeError):
 
     def __init__(self, search_message):
         self.search_message = search_message
-        super(SearchFailure, self).__init__(self._error_message)
+        super().__init__(self._error_message)
 
     @property
     def _error_message(self):

pytest_splunk_addon/helmut/exceptions/wait.py

@@ -20,7 +20,7 @@ class WaitTimedOut(RuntimeError):
 
     def __init__(self, seconds_waited):
         self.seconds_waited = seconds_waited
-        super(WaitTimedOut, self).__init__(self._error_message)
+        super().__init__(self._error_message)
 
     @property
     def _error_message(self):

pytest_splunk_addon/helmut/manager/jobs/results.py

@@ -16,7 +16,7 @@
 import copy
 
 
-class Results(object):
+class Results:
     """
     A class that represents a result set.
 
@@ -61,7 +61,7 @@ def __init__(self, results_):
         @param results_: The raw results as returned by ResultReader
         @type results_: list
         """
-        super(Results, self).__init__()
+        super().__init__()
 
         self._list = results_
         self._dict_cache = None
@@ -73,7 +73,7 @@ def __repr__(self):
         @return: The representation
         @rtype: str
         """
-        return "Results set with {count} result(s)".format(count=len(self))
+        return f"Results set with {len(self)} result(s)"
 
     def get_field(self, field):
         """

pytest_splunk_addon/helmut/manager/jobs/sdk/__init__.py

@@ -41,8 +41,7 @@ def __len__(self):
         return len(list(self.items()))
 
     def __iter__(self):
-        for item in list(self.items()):
-            yield item
+        yield from list(self.items())
 
     def __contains__(self, sid):
         for job in self:

pytest_splunk_addon/helmut/manager/jobs/sdk/job.py

@@ -18,9 +18,9 @@
 
 import splunklib.results as results
 
-from pytest_splunk_addon.helmut.manager.jobs.results import Results
 from pytest_splunk_addon.helmut.exceptions.search import SearchFailure
 from pytest_splunk_addon.helmut.exceptions.wait import WaitTimedOut
+from pytest_splunk_addon.helmut.manager.jobs.results import Results
 
 LOGGER = logging.getLogger("helmut")
 
@@ -49,7 +49,7 @@ def sid(self):
         return self.raw_sdk_job.sid
 
     def __str__(self):
-        return "SDK Job with SID {sid}".format(sid=self.sid)
+        return f"SDK Job with SID {self.sid}"
 
     def get_event_count(self):
         return int(self.raw_sdk_job.refresh().content.eventCount)

pytest_splunk_addon/helmut/splunk/cloud.py

@@ -19,8 +19,8 @@
 
 from splunklib.binding import HTTPError
 
-from pytest_splunk_addon.helmut.manager.jobs.sdk import SDKJobsWrapper
 from pytest_splunk_addon.helmut.connector.sdk import SDKConnector
+from pytest_splunk_addon.helmut.manager.jobs.sdk import SDKJobsWrapper
 
 LOGGER = logging.getLogger("helmut")
 
@@ -58,7 +58,7 @@ def __init__(
         self._start_listeners = set()
         self._connectors = {}
         self._name = name or id(self)
-        LOGGER.debug("Helmut Splunk created:{splunk}".format(splunk=self))
+        LOGGER.debug(f"Helmut Splunk created:{self}")
         self.set_credentials_to_use(username=username, password=password)
 
         server_web_scheme = server_web_host = server_web_port = None
@@ -77,7 +77,7 @@ def __init__(
         self._web_scheme = web_scheme or server_web_scheme or "http"
         self._web_host = web_host or server_web_host or self._splunkd_host
         self._web_port = web_port or server_web_port or ""
-        LOGGER.debug("Set web base to: {}".format(self.web_base()))
+        LOGGER.debug(f"Set web base to: {self.web_base()}")
 
     @property
     def _str_format(self):
@@ -210,7 +210,7 @@ def create_connector(self, **kwargs):
         connector_id = self._get_connector_id(user=conn.username)
 
         if connector_id in list(self._connectors.keys()):
-            LOGGER.warning("Connector {id} is being replaced".format(id=connector_id))
+            LOGGER.warning(f"Connector {connector_id} is being replaced")
             del self._connectors[connector_id]
         self._connectors[connector_id] = conn
 
@@ -264,7 +264,7 @@ def _get_connector_id(self, user):
         @param user: splunk username used by connector
         @type user: string
         """
-        return "{user}".format(user=user)
+        return f"{user}"
 
     @property
     def default_connector(self):
@@ -296,9 +296,7 @@ def connector(self):
 
         connector_id = self._get_connector_id(self.username)
         if connector_id not in list(self._connectors.keys()):
-            raise InvalidConnector(
-                "Connector {id} does not exist".format(id=connector_id)
-            )
+            raise InvalidConnector(f"Connector {connector_id} does not exist")
         connector = self._connectors[connector_id]
         self._attempt_login(connector)
         return connector
@@ -327,7 +325,7 @@ def get_event_count(self, search_string="*"):
         job = jobs.create("search %s" % search_string)
         job.wait()
         event_count = job.get_event_count()
-        LOGGER.debug("Event count: {ec}".format(ec=event_count))
+        LOGGER.debug(f"Event count: {event_count}")
         return event_count
 
     def get_final_event_count(
@@ -448,7 +446,7 @@ class InvalidStartListener(AttributeError):
 
     def __init__(self, message=None):
         message = message or "Start listeners must be callable"
-        super(InvalidStartListener, self).__init__(message)
+        super().__init__(message)
 
 
 class InvalidConnector(KeyError):

pytest_splunk_addon/helmut_lib/SearchUtil.py

@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import string
 import sys
 import time
-import string
 
 try:
     import py
@@ -34,7 +34,7 @@ def __str__(self):
         return repr(self.message)
 
 
-class SearchUtil(object):
+class SearchUtil:
     def __init__(self, jobs, logger):
         """
         Constructor of the SearchUtil object.

pytest_splunk_addon/plugin.py

@@ -14,13 +14,17 @@
 # limitations under the License.
 #
 import logging
-import pytest
-from .standard_lib.sample_generation.sample_xdist_generator import SampleXdistGenerator
 import traceback
-from .standard_lib import AppTestGenerator
-from .standard_lib.cim_compliance import CIMReportPlugin
+
+import pytest
 from filelock import FileLock
 
+from pytest_splunk_addon.standard_lib.app_test_generator import AppTestGenerator
+from pytest_splunk_addon.standard_lib.cim_compliance.plugin import CIMReportPlugin
+from pytest_splunk_addon.standard_lib.sample_generation.sample_xdist_generator import (
+    SampleXdistGenerator,
+)
+
 LOG_FILE = "pytest_splunk_addon.log"
 
 test_generator = None

pytest_splunk_addon/splunk.py

@@ -20,21 +20,25 @@
 - helmut_lib: Provides various Utility functions to search on Splunk. Source: splunk-sdk
 """
 
+import configparser
+import json
 import logging
 import os
 import shutil
 from time import sleep
-import json
+
 import pytest
 import requests
 import splunklib.client as client
-from .helmut.manager.jobs.sdk import SDKJobsWrapper
-from .helmut.splunk.cloud import CloudSplunk
-from .helmut_lib.SearchUtil import SearchUtil
-from .standard_lib.event_ingestors import IngestorHelper
-import configparser
 from filelock import FileLock
 
+from pytest_splunk_addon.helmut.manager.jobs.sdk import SDKJobsWrapper
+from pytest_splunk_addon.helmut.splunk.cloud import CloudSplunk
+from pytest_splunk_addon.helmut_lib.SearchUtil import SearchUtil
+from pytest_splunk_addon.standard_lib.event_ingestors.ingestor_helper import (
+    IngestorHelper,
+)
+
 RESPONSIVE_SPLUNK_TIMEOUT = 300  # seconds
 
 LOGGER = logging.getLogger("pytest-splunk-addon")
@@ -180,7 +184,7 @@ def pytest_addoption(parser):
             "Relative or absolute path can be provided."
             "Json files are expected in the directory."
             "Json files must follow the schema mentioned in DatamodelSchema.json"
-            "pytest-splunk-addon\pytest_splunk_addon\standard_lib\cim_tests\DatamodelSchema.json"
+            r"pytest-splunk-addon\pytest_splunk_addon\standard_lib\cim_tests\DatamodelSchema.json"
         ),
     )
     group.addoption(
@@ -402,14 +406,14 @@ def ignore_internal_errors(request):
     if request.config.getoption("ignore_addon_errors"):
         addon_error_file_path = request.config.getoption("ignore_addon_errors")
         if os.path.exists(addon_error_file_path):
-            with open(addon_error_file_path, "r") as addon_errors:
+            with open(addon_error_file_path) as addon_errors:
                 error_list.extend(
                     [each_error.strip() for each_error in addon_errors.readlines()]
                 )
     if request.config.getoption("ignore_errors_not_related_to_addon"):
         file_path = request.config.getoption("ignore_errors_not_related_to_addon")
         if os.path.exists(file_path):
-            with open(file_path, "r") as non_ta_errors:
+            with open(file_path) as non_ta_errors:
                 error_list.extend(
                     [each_error.strip() for each_error in non_ta_errors.readlines()]
                 )
@@ -886,7 +890,7 @@ def is_responsive_hec(request, splunk):
             f'{request.config.getoption("splunk_hec_scheme")}://{splunk["forwarder_host"]}:{splunk["port_hec"]}/services/collector/health/1.0',
             verify=False,
         )
-        LOGGER.debug("Status code: {}".format(response.status_code))
+        LOGGER.debug(f"Status code: {response.status_code}")
         if response.status_code in (200, 201):
             LOGGER.info("Splunk HEC is responsive.")
             return True

pytest_splunk_addon/standard_lib/__init__.py

@@ -31,6 +31,3 @@
 5. Other utility classes like Add-on parser & Data model handlers.
 
 """
-
-from .app_test_generator import AppTestGenerator
-from .addon_basic import Basic