diff --git a/.gitignore b/.gitignore index 228e620a..d0e9c0ba 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ stage/ local/ eventgen_wsgi.conf *.log -dist *.egg-info **/*.tgz .cache @@ -27,3 +26,4 @@ _book *.result venv/* *.log.* +splunk_eventgen-*/ diff --git a/Makefile b/Makefile index e49e4daf..81d1b419 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ image: setup_eventgen egg rm splunk_eventgen/default/eventgen_engine.conf || true docker build -f dockerfiles/Dockerfile . -t eventgen -test: egg image test_helper test_collection_cleanup +test: egg image test_helper run_tests test_collection_cleanup test_helper: docker run -d -t --net=host -v /var/run/docker.sock:/var/run/docker.sock --name ${EVENTGEN_TEST_IMAGE} eventgen:latest cat @@ -41,15 +41,10 @@ test_helper: @echo 'Installing test requirements' docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "pip install -r $(shell pwd)/tests/requirements.txt" || true +run_tests: @echo 'Running the super awesome tests' docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); python tests/run_tests.py ${SMALL} ${MEDIUM} ${LARGE} ${XLARGE}" || true - echo 'Collecting results' - #TODO: Should be paramaterized or generalized so that we don't need to add this here - docker cp ${EVENTGEN_TEST_IMAGE}:$(shell pwd)/tests_results.xml tests_results.xml || echo "no tests_results.xml" || true - - docker stop ${EVENTGEN_TEST_IMAGE} || true - test_collection_cleanup: @echo 'Collecting results' #TODO: Should be paramaterized or generalized so that we don't need to add this here diff --git a/setup.cfg b/setup.cfg index ab926b67..d8f0d78f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,12 +1,22 @@ [flake8] -exclude = .git,.tox,__pycache__,env,venv,build,dist,docs +exclude = .git,.tox,__pycache__,env,venv,splunk_eventgen/lib/concurrent,splunk_eventgen/lib/requests_futures max-line-length = 120 +ignore = E121,E123,E126,E226,E24,E704,W503,W504,E722,E731,W605 +# Includes default ignores, E722 (bare excepts), E731 (lambda usage), and W605 (escape sequences) [metadata] description-file = README.md version-from-file: __init__.py [yapf] -based_on_style = pep8 -spaces_before_comment = 4 -split_before_logical_operator = true +column_limit = 120 +split_all_comma_separated_values = false +split_before_named_assigns = false +split_before_first_argument = false +split_before_expression_after_opening_paren = false +split_before_closing_bracket = false +each_dict_entry_on_separate_line = false + +[isort] +# isort/yapf solutions to below files are not compatible +skip = splunk_eventgen/lib/concurrent,splunk_eventgen/lib/requests_futures diff --git a/setup.py b/setup.py index 3df22c4a..0cf07e05 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,16 @@ #!/usr/bin/env python # encoding: utf-8 -from setuptools import setup -from setuptools import find_packages -import splunk_eventgen +from setuptools import find_packages, setup +import splunk_eventgen VERSION = splunk_eventgen.__version__ try: import pypandoc long_description = pypandoc.convert('README.md', 'rst') -except(IOError, ImportError): +except (IOError, ImportError): long_description = open('README.md').read() @@ -27,12 +26,9 @@ def readme(): long_description=long_description, author='Splunk, Inc.', classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Software Development :: Build Tools', - 'Topic :: Software Development :: Testing', - 'Programming Language :: Python', + 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', + 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Software Development :: Build Tools', + 'Topic :: Software Development :: Testing', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7'], keywords='splunk eventgen container containers docker automation', entry_points={'console_scripts': ["splunk_eventgen = splunk_eventgen.__main__:main"]}, @@ -56,5 +52,4 @@ def readme(): 'flake8>=3.7.7', 'yapf>=0.26.0', 'isort>=4.3.15' - ] - ) + ]) diff --git a/splunk_eventgen/__init__.py b/splunk_eventgen/__init__.py index 6bbcdcc3..04a51e0b 100644 --- a/splunk_eventgen/__init__.py +++ b/splunk_eventgen/__init__.py @@ -3,6 +3,7 @@ import json import os + file_location = os.path.normpath(os.path.realpath(__file__)) VERSION_FILE = "version.json" VERSION_LOCATION = os.path.normpath(os.path.join(file_location, '..', VERSION_FILE)) @@ -10,7 +11,7 @@ def _get_version(versionfile): """ - @param versionfile: File to get the version info from + @param versionfile: File to get the version info from @return: Version Number """ with open(VERSION_LOCATION, 'r') as fp: @@ -19,6 +20,7 @@ def _get_version(versionfile): fp.close() return version + def _set_dev_version(): """ Write .dev at the end of version @@ -33,6 +35,7 @@ def _set_dev_version(): fp.write(json.dumps(json_data)) fp.close() + def _set_release_version(): """ Remove .dev at end of version if it exists diff --git a/splunk_eventgen/__main__.py b/splunk_eventgen/__main__.py index c4c5d453..d9e43d3d 100644 --- a/splunk_eventgen/__main__.py +++ b/splunk_eventgen/__main__.py @@ -6,6 +6,7 @@ import argparse import errno +import logging import os import shutil import sys @@ -13,64 +14,97 @@ import requests +import eventgen_core + FILE_LOCATION = os.path.dirname(os.path.abspath(__file__)) path_prepend = os.path.join(FILE_LOCATION, 'lib') sys.path.append(path_prepend) -import __init__ as splunk_eventgen_init -import logging -import eventgen_core + +import __init__ as splunk_eventgen_init # noqa isort:skip EVENTGEN_VERSION = splunk_eventgen_init.__version__ logger = logging.getLogger() + def parse_args(): """Parse command line arguments""" subparser_dict = {} - parser = argparse.ArgumentParser(prog='Eventgen', - description='Splunk Event Generation Tool') + parser = argparse.ArgumentParser(prog='Eventgen', description='Splunk Event Generation Tool') parser.add_argument("-v", "--verbosity", action="count", help="increase output verbosity") parser.add_argument("--version", action='version', default=False, version='%(prog)s ' + EVENTGEN_VERSION) parser.add_argument("--modinput-mode", default=False) subparsers = parser.add_subparsers(title='commands', help="valid subcommands", dest='subcommand') # Generate subparser generate_subparser = subparsers.add_parser('generate', help="Generate events using a supplied config file") - generate_subparser.add_argument("configfile", help="Location of eventgen.conf, app folder, or name of an app in $SPLUNK_HOME/etc/apps to run") + generate_subparser.add_argument( + "configfile", help="Location of eventgen.conf, app folder, or name of an app in $SPLUNK_HOME/etc/apps to run") generate_subparser.add_argument("-s", "--sample", help="Run specified sample only, disabling all other samples") generate_subparser.add_argument("--keepoutput", action="store_true", help="Keep original outputMode for the sample") generate_subparser.add_argument("--devnull", action="store_true", help="Set outputMode to devnull") - generate_subparser.add_argument("--modinput", action="store_true", help="Set outputMode to modinput, to see metadata") + generate_subparser.add_argument("--modinput", action="store_true", + help="Set outputMode to modinput, to see metadata") generate_subparser.add_argument("-c", "--count", type=int, help="Set sample count") generate_subparser.add_argument("-i", "--interval", type=int, help="Set sample interval") - generate_subparser.add_argument("-b", "--backfill", help="Set time to backfill from. Note: to use it, send the parameter with space in front like ' -60m'") - generate_subparser.add_argument("-e", "--end", help="Set time to end generation at or a number of intervals to run. Note: to use it with a time, send the parameter with space in front like ' -10m'") + generate_subparser.add_argument("-b", "--backfill", help="Set time to backfill from") + generate_subparser.add_argument("-e", "--end", help="Set time to end generation at or a number of intervals to run") generate_subparser.add_argument("--generators", type=int, help="Number of GeneratorWorkers (mappers)") generate_subparser.add_argument("--outputters", type=int, help="Number of OutputWorkers (reducers)") generate_subparser.add_argument("--disableOutputQueue", action="store_true", help="Disable reducer step") - generate_subparser.add_argument("--multiprocess", action="store_true", help="Use multiprocesing instead of threading") + generate_subparser.add_argument("--multiprocess", action="store_true", + help="Use multiprocesing instead of threading") generate_subparser.add_argument("--profiler", action="store_true", help="Turn on cProfiler") generate_subparser.add_argument("--log-path", type=str, default="{0}/logs".format(FILE_LOCATION)) # Build subparser build_subparser = subparsers.add_parser('build', help="Will build different forms of sa-eventgen") - build_subparser.add_argument("--mode", type=str, default="splunk-app", help="Specify what type of package to build, defaults to splunk-app mode.") + build_subparser.add_argument("--mode", type=str, default="splunk-app", + help="Specify what type of package to build, defaults to splunk-app mode.") build_subparser.add_argument("--destination", help="Specify where to store the output of the build command.") - build_subparser.add_argument("--remove", default=True, help="Remove the build directory after completion. Defaults to True") + build_subparser.add_argument("--remove", default=True, + help="Remove the build directory after completion. Defaults to True") # WSGI subparser wsgi_subparser = subparsers.add_parser('wsgi', help="start a wsgi server to interact with eventgen.") - wsgi_subparser.add_argument("--daemon", action="store_true", help="Daemon will tell the wsgi server to start in a daemon mode and will release the cli.") + wsgi_subparser.add_argument( + "--daemon", action="store_true", + help="Daemon will tell the wsgi server to start in a daemon mode and will release the cli.") # Service subparser - service_subparser = subparsers.add_parser('service', help="Run Eventgen as a Nameko service. Parameters for starting this service can be defined as either environment variables or CLI arguments, where environment variables takes precedence. See help for more info.") - service_subparser.add_argument("--role", "-r", type=str, default=None, required=True, choices=["controller", "server"], help="Define the role for this Eventgen node. Options: master, slave") - service_subparser.add_argument("--amqp-uri", type=str, default=None, help="Full URI to AMQP endpoint in the format pyamqp://:@:. This can also be set using the environment variable EVENTGEN_AMQP_URI") - service_subparser.add_argument("--amqp-host", type=str, default=None, help="Specify AMQP hostname. This can also be set using the environment variable EVENTGEN_AMQP_HOST. Default is localhost") - service_subparser.add_argument("--amqp-port", type=int, default=None, help="Specify AMQP port. This can also be set using the environment variable EVENTGEN_AMQP_PORT. Default is 5672") - service_subparser.add_argument("--amqp-webport", type=int, default=None, help="Specify AMQP web port. This can also be set using the environment variable EVENTGEN_AMQP_WEBPORT. Default is 15672") - service_subparser.add_argument("--amqp-user", type=str, default=None, help="Specify AMQP user. This can also be set using the environment variable EVENTGEN_AMQP_USER. Default is 'guest'") - service_subparser.add_argument("--amqp-pass", type=str, default=None, help="Specify AMQP password. This can also be set using the environment variable EVENTGEN_AMQP_PASS. Default is 'guest'") - service_subparser.add_argument("--web-server-address", type=str, default=None, help="Specify nameko webserver address. This can also be set using the environment variable EVENTGEN_WEB_SERVER_ADDR. Default is 0.0.0.0:9500") + service_subparser = subparsers.add_parser( + 'service', + help=("Run Eventgen as a Nameko service. Parameters for starting this service can be defined as either env" + "variables or CLI arguments, where env variables takes precedence. See help for more info.")) + service_subparser.add_argument("--role", "-r", type=str, default=None, required=True, choices=[ + "controller", "server"], help="Define the role for this Eventgen node. Options: master, slave") + service_subparser.add_argument( + "--amqp-uri", type=str, default=None, + help=("Full URI to AMQP endpoint in the format pyamqp://:@:." + "This can also be set using the environment variable EVENTGEN_AMQP_URI")) + service_subparser.add_argument( + "--amqp-host", type=str, default=None, + help=("Specify AMQP hostname. This can also be set using the environment variable EVENTGEN_AMQP_HOST." + + "Default is localhost")) + service_subparser.add_argument( + "--amqp-port", type=int, default=None, + help=("Specify AMQP port. This can also be set using the environment variable EVENTGEN_AMQP_PORT." + + "Default is 5672")) + service_subparser.add_argument( + "--amqp-webport", type=int, default=None, + help=("Specify AMQP web port. This can also be set using the environment variable EVENTGEN_AMQP_WEBPORT." + + "Default is 15672")) + service_subparser.add_argument( + "--amqp-user", type=str, default=None, + help=("Specify AMQP user. This can also be set using the environment variable EVENTGEN_AMQP_USER." + + "Default is 'guest'")) + service_subparser.add_argument( + "--amqp-pass", type=str, default=None, + help=("Specify AMQP password. This can also be set using the environment variable EVENTGEN_AMQP_PASS." + + "Default is 'guest'")) + service_subparser.add_argument( + "--web-server-address", type=str, default=None, + help=("Specify nameko webserver address. This can also be set using the environment variable" + + "EVENTGEN_WEB_SERVER_ADDR. Default is 0.0.0.0:9500")) # Help subparser # NOTE: Keep this at the end so we can use the subparser_dict.keys() to display valid commands help_subparser = subparsers.add_parser('help', help="Display usage on a subcommand") - helpstr = "Help on a specific command, valid commands are: " + ", ".join(subparser_dict.keys() + ["help"]) + helpstr = "Help on a specific command, valid commands are: " + ", ".join(subparser_dict.keys() + ["help"]) help_subparser.add_argument("command", nargs='?', default="default", help=helpstr) # add subparsers to the subparser dict, this will be used later for usage / help statements. subparser_dict['generate'] = generate_subparser @@ -91,7 +125,7 @@ def parse_args(): if 'subcommand' not in args: parser.print_help() sys.exit(2) - + if args.subcommand == "service": if not args.role: msg = "Role is undefined. Please specify a role for this Eventgen service using --role/-r." @@ -120,10 +154,11 @@ def parse_args(): args.configfile = None return args + def wait_for_response(address, webport, timeout=300): ''' - This function extracts the hostname off the given address in the form ://:@: - and builds a URL in the form http://:. Using this URL, it tries to verify the endpoint is reachable. + Extracts the hostname off the given address in the form ://:@: and + builds a URL in the form http://:. Using this URL, it tries to verify the endpoint is reachable. Retry will occur for ~300s ''' @@ -133,12 +168,12 @@ def wait_for_response(address, webport, timeout=300): userid, password = creds.split(":") start = time.time() end = start - while end-start < timeout: + while end - start < timeout: try: r = requests.get("http://{}:{}".format(host, webport)) r.raise_for_status() return - except requests.exceptions.ConnectionError as e: + except requests.exceptions.ConnectionError: time.sleep(1) finally: end = time.time() @@ -146,6 +181,7 @@ def wait_for_response(address, webport, timeout=300): logger.exception(msg) raise Exception(msg) + def parse_cli_vars(config, args): config["AMQP_URI"] = args.amqp_uri if args.amqp_uri else config["AMQP_URI"] config["AMQP_HOST"] = args.amqp_host if args.amqp_host else config["AMQP_HOST"] @@ -156,6 +192,7 @@ def parse_cli_vars(config, args): config["WEB_SERVER_ADDRESS"] = args.web_server_address if args.web_server_address else config["WEB_SERVER_ADDRESS"] return config + def parse_env_vars(): osvars, config = dict(os.environ), {} config["AMQP_URI"] = osvars.get("EVENTGEN_AMQP_URI", None) @@ -167,6 +204,7 @@ def parse_env_vars(): config["WEB_SERVER_ADDRESS"] = osvars.get("EVENTGEN_WEB_SERVER_ADDR", "0.0.0.0:9500") return config + def rectify_config(config): # For nameko purposes, all we need to pass into the config is AMQP_URI and WEB_SERVER_ADDRESS. new = {} @@ -176,22 +214,21 @@ def rectify_config(config): new["AMQP_URI"] = config["AMQP_URI"] else: if all([config["AMQP_HOST"], config["AMQP_PORT"], config["AMQP_USER"], config["AMQP_PASS"]]): - new["AMQP_URI"] = "pyamqp://{user}:{pw}@{host}:{port}".format(user=config["AMQP_USER"], - pw=config["AMQP_PASS"], - host=config["AMQP_HOST"], - port=config["AMQP_PORT"]) + new["AMQP_URI"] = "pyamqp://{user}:{pw}@{host}:{port}".format( + user=config["AMQP_USER"], pw=config["AMQP_PASS"], host=config["AMQP_HOST"], port=config["AMQP_PORT"]) else: msg = "AMQP_URI is not defined and cannot be constructed. Check environment variables/CLI arguments." logger.exception(msg) raise Exception(msg) return new + def run_nameko(args): # Running nameko imports here so that Eventgen as a module does not require nameko to run. import eventlet eventlet.monkey_patch() from nameko.runners import ServiceRunner - # In order to make this run locally as well as within a container-ized environment, we're to pull variables + # In order to make this run locally as well as within a container-ized environment, we're to pull variables # from both environment variables and CLI arguments, where CLI will take precendence. config = parse_env_vars() config = parse_cli_vars(config, args) @@ -229,6 +266,7 @@ def run_nameko(args): # runner.wait completed break + def exclude_function(filename): # removing any hidden . files. last_index = filename.rfind('/') @@ -240,13 +278,15 @@ def exclude_function(filename): else: return False + def make_tarfile(output_filename, source_dir): import tarfile with tarfile.open(output_filename, "w:gz") as tar: tar.add(source_dir, arcname=os.path.basename(source_dir), exclude=exclude_function) + def build_splunk_app(dest, source=os.getcwd(), remove=True): - import errno, imp + import imp cwd = os.getcwd() os.chdir(source) directory = os.path.join(dest, 'SA-Eventgen') @@ -275,6 +315,7 @@ def build_splunk_app(dest, source=os.getcwd(), remove=True): shutil.rmtree(directory) os.chdir(cwd) + def convert_verbosity_count_to_logging_level(verbosity): if verbosity == 0: return logging.ERROR @@ -285,6 +326,7 @@ def convert_verbosity_count_to_logging_level(verbosity): else: return logging.ERROR + def main(): cwd = os.getcwd() args = parse_args() diff --git a/splunk_eventgen/eventgen_core.py b/splunk_eventgen/eventgen_core.py index e9db47f6..71c80647 100644 --- a/splunk_eventgen/eventgen_core.py +++ b/splunk_eventgen/eventgen_core.py @@ -1,17 +1,19 @@ #!/usr/bin/env python # encoding: utf-8 -from lib.eventgenconfig import Config -from lib.eventgentimer import Timer -from lib.eventgenexceptions import PluginNotLoaded, FailedLoadingPlugin -from lib.outputcounter import OutputCounter +import imp +import json import logging import logging.config import os import sys -import imp -from Queue import Queue, Empty -from threading import Thread import time +from Queue import Empty, Queue +from threading import Thread + +from lib.eventgenconfig import Config +from lib.eventgenexceptions import PluginNotLoaded +from lib.eventgentimer import Timer +from lib.outputcounter import OutputCounter lib_path_prepend = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib') sys.path.insert(0, lib_path_prepend) @@ -26,23 +28,24 @@ import logutils import logutils.queue -file_path=os.path.dirname(os.path.realpath(__file__)) +file_path = os.path.dirname(os.path.realpath(__file__)) EVENTGEN_DIR = os.path.realpath(os.path.join(file_path, "..")) EVENTGEN_ENGINE_CONF_PATH = os.path.abspath(os.path.join(file_path, "default", "eventgen_engine.conf")) -import json + class JSONFormatter(logging.Formatter): """ Quick and dirty formatter that turns the log into a quick json format """ + def format(self, record): message = record.msg if not isinstance(message, dict): - #The record is probably a string + # The record is probably a string try: message = json.loads(message) except ValueError: - #Abort, just store the message as an attribute + # Abort, just store the message as an attribute message = {"message": message} if "timestamp" not in message: @@ -51,8 +54,9 @@ def format(self, record): message["level"] = logging.getLevelName(record.levelno) return json.dumps(message) + class EventGenerator(object): - def __init__(self, args=None ): + def __init__(self, args=None): ''' This object will allow you to generate and control eventgen. It should be handed the parse_args object from __main__ and will hand the argument object to the config parser of eventgen5. This will provide the @@ -81,12 +85,11 @@ def _load_config(self, configfile, **kwargs): :param configfile: :return: ''' - #TODO: The old eventgen had strange cli args, and usage of args. We should probably update the module args - #to match more of what this is doing... - new_args={} + # TODO: The old eventgen had strange cli args. We should probably update the module args to match this usage. + new_args = {} if "args" in kwargs: args = kwargs["args"] - outputer = [key for key in ["keepoutput","devnull","modinput"] if getattr(args, key)] + outputer = [key for key in ["keepoutput", "devnull", "modinput"] if getattr(args, key)] if len(outputer) > 0: new_args["override_outputter"] = outputer[0] if getattr(args, "count"): @@ -112,7 +115,7 @@ def _load_config(self, configfile, **kwargs): self.config = Config(configfile, **new_args) self.config.parse() self._reload_plugins() - #TODO: Probably should destroy pools better so processes are cleaned. + # TODO: Probably should destroy pools better so processes are cleaned. self._setup_pools() def _reload_plugins(self): @@ -120,11 +123,14 @@ def _reload_plugins(self): # Plugins must be loaded before objects that do work, otherwise threads and processes generated will not have # the modules loaded in active memory. try: - self.config.outputPlugins = { } - plugins = self._initializePlugins(os.path.join(file_path, 'lib', 'plugins', 'output'), self.config.outputPlugins, 'output') + self.config.outputPlugins = {} + plugins = self._initializePlugins( + os.path.join(file_path, 'lib', 'plugins', 'output'), self.config.outputPlugins, 'output') self.config.validOutputModes.extend(plugins) - self._initializePlugins(os.path.join(file_path, 'lib', 'plugins', 'generator'), self.config.plugins, 'generator') - plugins = self._initializePlugins(os.path.join(file_path, 'lib', 'plugins', 'rater'), self.config.plugins, 'rater') + self._initializePlugins( + os.path.join(file_path, 'lib', 'plugins', 'generator'), self.config.plugins, 'generator') + plugins = self._initializePlugins( + os.path.join(file_path, 'lib', 'plugins', 'rater'), self.config.plugins, 'rater') self.config._complexSettings['rater'] = plugins except Exception as e: self.logger.exception(e) @@ -133,13 +139,12 @@ def _load_custom_plugins(self, PluginNotLoadedException): plugintype = PluginNotLoadedException.type plugin = PluginNotLoadedException.name bindir = PluginNotLoadedException.bindir - libdir = PluginNotLoadedException.libdir plugindir = PluginNotLoadedException.plugindir pluginsdict = self.config.plugins if plugintype in ('generator', 'rater') else self.config.outputPlugins - #APPPERF-263: be picky when loading from an app bindir (only load name) + # APPPERF-263: be picky when loading from an app bindir (only load name) self._initializePlugins(bindir, pluginsdict, plugintype, name=plugin) - #APPPERF-263: be greedy when scanning plugin dir (eat all the pys) + # APPPERF-263: be greedy when scanning plugin dir (eat all the pys) self._initializePlugins(plugindir, pluginsdict, plugintype) def _setup_pools(self): @@ -165,31 +170,33 @@ def _create_timer_threadpool(self, threadcount=100): self.sampleQueue = Queue(maxsize=0) num_threads = threadcount for i in range(num_threads): - worker = Thread(target=self._worker_do_work, - args=(self.sampleQueue, self.loggingQueue, ), - name="TimeThread{0}".format(i)) + worker = Thread(target=self._worker_do_work, args=( + self.sampleQueue, + self.loggingQueue, + ), name="TimeThread{0}".format(i)) worker.setDaemon(True) worker.start() def _create_output_threadpool(self, threadcount=1): ''' the output thread pool is used for output plugins that need to control file locking, or only have 1 set thread - to send all the data out of. this FIFO queue just helps make sure there are file collisions or write collisions. + to send all the data out of. This FIFO queue just helps make sure there are file collisions or write collisions. There's only 1 active thread for this queue, if you're ever considering upping this, don't. Just shut off the outputQueue and let each generator directly output it's data. :param threadcount: is how many active output threads we want to allow inside of eventgen. Default 1 :return: ''' - #TODO: Make this take the config param and figure out what we want to do with this. + # TODO: Make this take the config param and figure out what we want to do with this. if getattr(self, "manager", None): self.outputQueue = self.manager.Queue(maxsize=500) else: self.outputQueue = Queue(maxsize=500) num_threads = threadcount for i in range(num_threads): - worker = Thread(target=self._worker_do_work, - args=(self.outputQueue, self.loggingQueue, ), - name="OutputThread{0}".format(i)) + worker = Thread(target=self._worker_do_work, args=( + self.outputQueue, + self.loggingQueue, + ), name="OutputThread{0}".format(i)) worker.setDaemon(True) worker.start() @@ -206,7 +213,7 @@ def _create_generator_pool(self, workercount=20): import multiprocessing self.manager = multiprocessing.Manager() self.loggingQueue = self.manager.Queue() - self.logging_pool = Thread(target=self.logger_thread, args=(self.loggingQueue,), name="LoggerThread") + self.logging_pool = Thread(target=self.logger_thread, args=(self.loggingQueue, ), name="LoggerThread") self.logging_pool.start() # since we're now in multiprocess, we need to use better queues. self.workerQueue = multiprocessing.JoinableQueue(maxsize=500) @@ -220,7 +227,8 @@ def _create_generator_pool(self, workercount=20): for i in range(workercount): self.output_counters.append(OutputCounter()) for i in range(worker_threads): - worker = Thread(target=self._generator_do_work, args=(self.workerQueue, self.loggingQueue, self.output_counters[i])) + worker = Thread(target=self._generator_do_work, args=(self.workerQueue, self.loggingQueue, + self.output_counters[i])) worker.setDaemon(True) worker.start() else: @@ -234,9 +242,12 @@ def _create_generator_workers(self, workercount=20): import multiprocessing self.workerPool = [] for worker in xrange(workercount): - #builds a list of tuples to use the map function - process = multiprocessing.Process(target=self._proc_worker_do_work, - args=(self.workerQueue, self.loggingQueue, self.genconfig, )) + # builds a list of tuples to use the map function + process = multiprocessing.Process(target=self._proc_worker_do_work, args=( + self.workerQueue, + self.loggingQueue, + self.genconfig, + )) self.workerPool.append(process) process.start() else: @@ -262,23 +273,28 @@ def _setup_loggers(self, args=None, config=None): console_handler.setFormatter(detailed_formatter) console_handler.setLevel(logging.DEBUG) - file_handler = logging.handlers.RotatingFileHandler(eventgen_main_logger_path, maxBytes=2500000, backupCount=20) + file_handler = logging.handlers.RotatingFileHandler(eventgen_main_logger_path, maxBytes=2500000, + backupCount=20) file_handler.setFormatter(detailed_formatter) file_handler.setLevel(logging.DEBUG) - eventgen_controller_file_handler = logging.handlers.RotatingFileHandler(eventgen_controller_logger_path, maxBytes=2500000, backupCount=20) + eventgen_controller_file_handler = logging.handlers.RotatingFileHandler(eventgen_controller_logger_path, + maxBytes=2500000, backupCount=20) eventgen_controller_file_handler.setFormatter(detailed_formatter) eventgen_controller_file_handler.setLevel(logging.DEBUG) - error_file_handler = logging.handlers.RotatingFileHandler(eventgen_error_logger_path, maxBytes=2500000, backupCount=20) + error_file_handler = logging.handlers.RotatingFileHandler(eventgen_error_logger_path, maxBytes=2500000, + backupCount=20) error_file_handler.setFormatter(detailed_formatter) error_file_handler.setLevel(logging.ERROR) - metrics_file_handler = logging.handlers.RotatingFileHandler(eventgen_metrics_logger_path, maxBytes=2500000, backupCount=20) + metrics_file_handler = logging.handlers.RotatingFileHandler(eventgen_metrics_logger_path, maxBytes=2500000, + backupCount=20) metrics_file_handler.setFormatter(json_formatter) metrics_file_handler.setLevel(logging.INFO) - server_file_handler = logging.handlers.RotatingFileHandler(eventgen_server_logger_path, maxBytes=2500000, backupCount=10) + server_file_handler = logging.handlers.RotatingFileHandler(eventgen_server_logger_path, maxBytes=2500000, + backupCount=10) server_file_handler.setFormatter(json_formatter) server_file_handler.setLevel(logging.INFO) @@ -321,10 +337,12 @@ def _setup_loggers(self, args=None, config=None): # We need to have debugv from the olderversions of eventgen. DEBUG_LEVELV_NUM = 9 logging.addLevelName(DEBUG_LEVELV_NUM, "DEBUGV") + def debugv(self, message, *args, **kws): # Yes, logger takes its '*args' as 'args'. if self.isEnabledFor(DEBUG_LEVELV_NUM): self._log(DEBUG_LEVELV_NUM, message, args, **kws) + logging.Logger.debugv = debugv self.logger = logging.getLogger('eventgen') self.loggingQueue = None @@ -344,6 +362,7 @@ def _worker_do_work(self, work_queue, logging_queue): except Exception as e: self.logger.exception(e) raise e + def _generator_do_work(self, work_queue, logging_queue, output_counter=None): while not self.stopping: try: @@ -408,13 +427,13 @@ def _initializePlugins(self, dirname, plugins, plugintype, name=None): syspathset = set(sys.path) dirname = os.path.abspath(dirname) - self.logger.debugv("looking for plugin(s) in {}".format(dirname)) + self.logger.debug("looking for plugin(s) in {}".format(dirname)) if not os.path.isdir(dirname): - self.logger.debugv("directory {} does not exist ... moving on".format(dirname)) + self.logger.debug("directory {} does not exist ... moving on".format(dirname)) return ret # Include all plugin directories in sys.path for includes - if not dirname in sys.path: + if dirname not in sys.path: syspathset.add(dirname) sys.path = list(syspathset) @@ -429,16 +448,16 @@ def _initializePlugins(self, dirname, plugins, plugintype, name=None): base, extension = os.path.splitext(basename) # If we're a python file and we don't start with _ - #if extension == ".py" and not basename.startswith("_"): + # if extension == ".py" and not basename.startswith("_"): # APPPERF-263: If name param is supplied, only attempt to load # {name}.py from {app}/bin directory if extension == ".py" and ((name is None and not basename.startswith("_")) or base == name): - self.logger.debugv("Searching for plugin in file '%s'" % filename) + self.logger.debug("Searching for plugin in file '%s'" % filename) try: # Import the module - #module = imp.load_source(base, filename) + # module = imp.load_source(base, filename) mod_name, mod_path, mod_desc = imp.find_module(base, [dirname]) - #TODO: Probably need to adjust module.load() to be added later so this can be pickled. + # TODO: Probably need to adjust module.load() to be added later so this can be pickled. module = imp.load_module(base, mod_name, mod_path, mod_desc) plugin = module.load() @@ -487,33 +506,36 @@ def start(self, join_after_start=True): self.logger.info("No samples found. Exiting.") for s in self.config.samples: if s.interval > 0 or s.mode == 'replay' or s.end != "0": - self.logger.info("Creating timer object for sample '%s' in app '%s'" % (s.name, s.app) ) + self.logger.info("Creating timer object for sample '%s' in app '%s'" % (s.name, s.app)) # This is where the timer is finally sent to a queue to be processed. Needs to move to this object. try: - t = Timer(1.0, sample=s, config=self.config, - genqueue=self.workerQueue, outputqueue=self.outputQueue, loggingqueue=self.loggingQueue) + t = Timer(1.0, sample=s, config=self.config, genqueue=self.workerQueue, + outputqueue=self.outputQueue, loggingqueue=self.loggingQueue) except PluginNotLoaded as pnl: self._load_custom_plugins(pnl) - t = Timer(1.0, sample=s, config=self.config, - genqueue=self.workerQueue, outputqueue=self.outputQueue, loggingqueue=self.loggingQueue) + t = Timer(1.0, sample=s, config=self.config, genqueue=self.workerQueue, + outputqueue=self.outputQueue, loggingqueue=self.loggingQueue) except Exception as e: raise e self.sampleQueue.put(t) if join_after_start: self.logger.info("All timers started, joining queue until it's empty.") self.join_process() - ## Only need to start timers once + # Only need to start timers once # Every 5 seconds, get values and output basic statistics about our operations - #TODO: Figure out how to do this better... - #generatorsPerSec = (generatorDecrements - generatorQueueCounter) / 5 - #outputtersPerSec = (outputDecrements - outputQueueCounter) / 5 - #outputQueueCounter = outputDecrements - #generatorQueueCounter = generatorDecrements - #self.logger.info('OutputQueueDepth=%d GeneratorQueueDepth=%d GeneratorsPerSec=%d OutputtersPerSec=%d' % (self.config.outputQueueSize.value(), self.config.generatorQueueSize.value(), generatorsPerSec, outputtersPerSec)) - #kiloBytesPerSec = self.config.bytesSent.valueAndClear() / 5 / 1024 - #gbPerDay = (kiloBytesPerSec / 1024 / 1024) * 60 * 60 * 24 - #eventsPerSec = self.config.eventsSent.valueAndClear() / 5 - #self.logger.info('GlobalEventsPerSec=%s KilobytesPerSec=%1f GigabytesPerDay=%1f' % (eventsPerSec, kiloBytesPerSec, gbPerDay)) + # TODO: Figure out how to do this better... + # generatorsPerSec = (generatorDecrements - generatorQueueCounter) / 5 + # outputtersPerSec = (outputDecrements - outputQueueCounter) / 5 + # outputQueueCounter = outputDecrements + # generatorQueueCounter = generatorDecrements + # self.logger.info('OutputQueueDepth=%d GeneratorQueueDepth=%d GeneratorsPerSec=%d OutputtersPerSec=%d' % + # (self.config.outputQueueSize.value(), self.config.generatorQueueSize.value(), + # generatorsPerSec, outputtersPerSec)) + # kiloBytesPerSec = self.config.bytesSent.valueAndClear() / 5 / 1024 + # gbPerDay = (kiloBytesPerSec / 1024 / 1024) * 60 * 60 * 24 + # eventsPerSec = self.config.eventsSent.valueAndClear() / 5 + # self.logger.info('GlobalEventsPerSec=%s KilobytesPerSec=%1f GigabytesPerDay=%1f' % + # (eventsPerSec, kiloBytesPerSec, gbPerDay)) def join_process(self): ''' @@ -522,7 +544,8 @@ def join_process(self): :return: ''' try: - while not self.sampleQueue.empty() or self.sampleQueue.unfinished_tasks > 0 or not self.workerQueue.empty() or self.workerQueue.unfinished_tasks > 0: + while not self.sampleQueue.empty() or self.sampleQueue.unfinished_tasks > 0 or not self.workerQueue.empty( + ) or self.workerQueue.unfinished_tasks > 0: time.sleep(5) self.logger.info("All timers have finished, signalling workers to exit.") self.stop() @@ -538,13 +561,13 @@ def stop(self): self.logger.info("All timers exited, joining generation queue until it's empty.") self.workerQueue.join() - # if we're in multiprocess, make sure that since all the timers stopped, we don't let any more generators get added. + # if we're in multiprocess, make sure we don't add more generators after the timers stopped. if self.args.multiprocess: self.genconfig["stopping"] = True for worker in self.workerPool: count = 0 # We wait for a minute until terminating the worker - while worker.exitcode == None and count != 20: + while worker.exitcode is None and count != 20: if count == 30: self.logger.info("Terminating worker {0}".format(worker._name)) worker.terminate() @@ -586,4 +609,3 @@ def check_running(self): else: return True return False - diff --git a/splunk_eventgen/eventgen_nameko_controller.py b/splunk_eventgen/eventgen_nameko_controller.py index 12cda2e7..52192859 100644 --- a/splunk_eventgen/eventgen_nameko_controller.py +++ b/splunk_eventgen/eventgen_nameko_controller.py @@ -1,22 +1,26 @@ -from nameko.rpc import rpc -from nameko.events import EventDispatcher, event_handler, BROADCAST -from nameko.web.handlers import http -from pyrabbit.api import Client import atexit +import json import logging import os import socket -from logger.logger_config import controller_logger_config import time -import json + +from pyrabbit.api import Client + +from logger.logger_config import controller_logger_config +from nameko.events import BROADCAST, EventDispatcher, event_handler +from nameko.rpc import rpc +from nameko.web.handlers import http FILE_PATH = os.path.dirname(os.path.realpath(__file__)) EVENTGEN_ENGINE_CONF_PATH = os.path.abspath(os.path.join(FILE_PATH, "default", "eventgen_engine.conf")) + def exit_handler(client, hostname, logger): client.delete_vhost(hostname) logger.info("Deleted vhost {}. Shutting down.".format(hostname)) + class EventgenController(object): name = "eventgen_controller" @@ -38,17 +42,14 @@ class EventgenController(object): config["AMQP_PASS"] = osvars.get("EVENTGEN_AMQP_PASS", "guest") pyrabbit_cl = Client('{0}:{1}'.format(config['AMQP_HOST'], config['AMQP_WEBPORT']), - '{0}'.format(config['AMQP_USER']), - '{0}'.format(config['AMQP_PASS'])) + '{0}'.format(config['AMQP_USER']), '{0}'.format(config['AMQP_PASS'])) pyrabbit_cl.create_vhost(host) log.info("Vhost set to {}".format(host)) log.info("Current Vhosts are {}".format(pyrabbit_cl.get_vhost_names())) atexit.register(exit_handler, client=pyrabbit_cl, hostname=host, logger=log) - ############################################## - ################ RPC Methods ################# - ############################################## + # RPC Methods @event_handler("eventgen_server", "server_status", handler_type=BROADCAST, reliable_delivery=False) def event_handler_server_status(self, payload): @@ -188,7 +189,7 @@ def bundle(self, target, data): except Exception as e: self.log.exception(e) return "500", "Exception: {}".format(e.message) - + @rpc def setup(self, target, data): try: @@ -210,7 +211,7 @@ def get_volume(self, target): except Exception as e: self.log.exception(e) return "500", "Exception: {}".format(e.message) - + @rpc def set_volume(self, target, data): try: @@ -238,10 +239,7 @@ def reset(self, target): self.log.exception(e) return '500', "Exception: {}".format(e.message) - - ############################################## - ################ HTTP Methods ################ - ############################################## + # HTTP Methods @http('GET', '/') def root_page(self, request): @@ -439,15 +437,13 @@ def http_set_volume_target(self, request, target="all"): def http_reset(self, request): return self.reset(target="all") - ############################################## - ############### Helper Methods ############### - ############################################## + # Helper Methods def receive_status(self, data): if data['server_name'] and data['server_status']: self.server_status[data['server_name']] = data['server_status'] rec_time = time.time() - self.log.info("receive {}'s status, update the status at time:{}".format(data['server_name'],rec_time)) + self.log.info("receive {}'s status, update the status at time:{}".format(data['server_name'], rec_time)) self.server_status['time'] = rec_time def receive_conf(self, data): @@ -461,14 +457,15 @@ def receive_volume(self, data): def process_server_status(self, current_time, num_retries=15, delay=0.3): current_server_vhosts = self.get_current_server_vhosts() server_time = self.server_status['time'] if 'time' in self.server_status else 0 - server_vhost_len = len(self.server_status) if 'time' not in self.server_status else len(self.server_status)-1 + server_vhost_len = len(self.server_status) if 'time' not in self.server_status else len(self.server_status) - 1 if current_server_vhosts: for i in range(num_retries): if server_vhost_len != len(current_server_vhosts) or server_time < current_time: time.sleep(delay) current_server_vhosts = self.get_current_server_vhosts() server_time = self.server_status['time'] if 'time' in self.server_status else 0 - server_vhost_len = len(self.server_status) if 'time' not in self.server_status else len(self.server_status)-1 + server_vhost_len = len( + self.server_status) if 'time' not in self.server_status else len(self.server_status) - 1 else: break dump_value = self.calculate_throughput(self.server_status) @@ -515,10 +512,7 @@ def check_vhost(self, vhost_name): return False def calculate_throughput(self, data): - throughput_summary = { 'TOTAL_VOLUME_MB': 0, - 'TOTAL_COUNT': 0, - 'THROUGHPUT_VOLUME_KB': 0, - 'THROUGHPUT_COUNT': 0} + throughput_summary = {'TOTAL_VOLUME_MB': 0, 'TOTAL_COUNT': 0, 'THROUGHPUT_VOLUME_KB': 0, 'THROUGHPUT_COUNT': 0} for server_name, server_status in data.items(): if server_name != 'time' and 'THROUGHPUT_STATUS' in server_status: server_throughput = server_status['THROUGHPUT_STATUS'] diff --git a/splunk_eventgen/eventgen_nameko_dependency.py b/splunk_eventgen/eventgen_nameko_dependency.py index a22577c9..6bbcd54e 100644 --- a/splunk_eventgen/eventgen_nameko_dependency.py +++ b/splunk_eventgen/eventgen_nameko_dependency.py @@ -1,9 +1,11 @@ -from nameko.extensions import DependencyProvider -import eventgen_core -import logging import argparse -import sys +import logging import os +import sys + +import eventgen_core +from nameko.extensions import DependencyProvider + FILE_PATH = os.path.dirname(os.path.realpath(__file__)) CUSTOM_CONFIG_PATH = os.path.realpath(os.path.join(FILE_PATH, "default", "eventgen_wsgi.conf")) @@ -12,6 +14,7 @@ # eventgen_nameko_dependency: error: unrecognized arguments: --role master --config server_conf.yml sys.argv = [sys.argv.pop(0)] + def create_args(): parser = argparse.ArgumentParser(prog="eventgen_nameko_dependency") args = parser.parse_args() @@ -38,6 +41,7 @@ def create_args(): args.modinput_mode = False return args + class EventgenDependency(DependencyProvider): arguments = create_args() diff --git a/splunk_eventgen/eventgen_nameko_server.py b/splunk_eventgen/eventgen_nameko_server.py index 4bb89e50..cdfe2f35 100644 --- a/splunk_eventgen/eventgen_nameko_server.py +++ b/splunk_eventgen/eventgen_nameko_server.py @@ -1,21 +1,23 @@ -from nameko.rpc import rpc -from nameko.web.handlers import http -from nameko.events import EventDispatcher, event_handler, BROADCAST -from pyrabbit.api import Client import atexit import ConfigParser -import yaml +import glob import json +import logging import os +import shutil import socket -import time -import requests -import glob import tarfile +import time import zipfile -import shutil + +import requests +import yaml +from pyrabbit.api import Client + import eventgen_nameko_dependency -import logging +from nameko.events import BROADCAST, EventDispatcher, event_handler +from nameko.rpc import rpc +from nameko.web.handlers import http FILE_PATH = os.path.dirname(os.path.realpath(__file__)) EVENTGEN_DIR = os.path.realpath(os.path.join(FILE_PATH, "..")) @@ -27,7 +29,6 @@ def get_eventgen_name_from_conf(): with open(os.path.abspath(os.path.join(FILE_PATH, "server_conf.yml"))) as config_yml: loaded_yml = yaml.load(config_yml) return loaded_yml['EVENTGEN_NAME'] if 'EVENTGEN_NAME' in loaded_yml else socket.gethostname() - return None def exit_handler(client, hostname, logger): @@ -53,8 +54,7 @@ class EventgenServer(object): config["AMQP_PASS"] = osvars.get("EVENTGEN_AMQP_PASS", "guest") pyrabbit_cl = Client('{0}:{1}'.format(config['AMQP_HOST'], config['AMQP_WEBPORT']), - '{0}'.format(config['AMQP_USER']), - '{0}'.format(config['AMQP_PASS'])) + '{0}'.format(config['AMQP_USER']), '{0}'.format(config['AMQP_PASS'])) pyrabbit_cl.create_vhost(host) log.info("Vhost set to {}".format(host)) @@ -85,7 +85,7 @@ def get_status(self): if self.eventgen_dependency.eventgen.check_running(): # running status = 1 - elif self.eventgen_dependency.eventgen.completed == True: + elif self.eventgen_dependency.eventgen.completed is True: # all samples completed and stop status = 2 else: @@ -96,9 +96,10 @@ def get_status(self): res["CONFIGURED"] = self.eventgen_dependency.configured res["CONFIG_FILE"] = self.eventgen_dependency.configfile res["TOTAL_VOLUME"] = self.total_volume - res["QUEUE_STATUS"] = {'SAMPLE_QUEUE': {'UNFINISHED_TASK': 'N/A', 'QUEUE_LENGTH': 'N/A'}, - 'OUTPUT_QUEUE': {'UNFINISHED_TASK': 'N/A', 'QUEUE_LENGTH': 'N/A'}, - 'WORKER_QUEUE': {'UNFINISHED_TASK': 'N/A', 'QUEUE_LENGTH': 'N/A'}} + res["QUEUE_STATUS"] = { + 'SAMPLE_QUEUE': {'UNFINISHED_TASK': 'N/A', 'QUEUE_LENGTH': 'N/A'}, 'OUTPUT_QUEUE': { + 'UNFINISHED_TASK': 'N/A', 'QUEUE_LENGTH': 'N/A'}, 'WORKER_QUEUE': { + 'UNFINISHED_TASK': 'N/A', 'QUEUE_LENGTH': 'N/A'}} res['THROUGHPUT_STATUS'] = self.get_throughput() if hasattr(self.eventgen_dependency.eventgen, "sampleQueue"): res["QUEUE_STATUS"]['SAMPLE_QUEUE'][ @@ -114,9 +115,7 @@ def get_status(self): res["QUEUE_STATUS"]['WORKER_QUEUE']['QUEUE_LENGTH'] = self.eventgen_dependency.eventgen.workerQueue.qsize() return res - ############################################## - ############### Real Methods ################# - ############################################## + # Real Methods def index(self): self.log.info("index method called") @@ -139,14 +138,8 @@ def index(self): sample_queue_status = status["QUEUE_STATUS"]["SAMPLE_QUEUE"] output_queue_status = status["QUEUE_STATUS"]["OUTPUT_QUEUE"] - return home_page.format(host, - eventgen_status, - configured, - config_file, - total_volume, - worker_queue_status, - sample_queue_status, - output_queue_status) + return home_page.format(host, eventgen_status, configured, config_file, total_volume, worker_queue_status, + sample_queue_status, output_queue_status) def status(self): self.log.info('Status method called.') @@ -344,28 +337,18 @@ def setup(self, data): new_key = bool(data.get("new_key", True)) def create_new_hec_key(hostname): - requests.post("https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http/http".format( - hostname, mgmt_port), - auth=("admin", password), - data={"disabled": "0"}, - verify=False) + requests.post( + "https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http/http".format( + hostname, mgmt_port), auth=("admin", password), data={"disabled": "0"}, verify=False) requests.delete( - "https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http/{2}".format( - hostname, mgmt_port,key_name), - verify=False, - auth=("admin", password) - ) + "https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http/{2}".format( + hostname, mgmt_port, key_name), verify=False, auth=("admin", password)) requests.post( - "https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http?output_mode=json".format( - hostname, mgmt_port), - verify=False, - auth=("admin", password), - data={"name": key_name}) + "https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http?output_mode=json".format( + hostname, mgmt_port), verify=False, auth=("admin", password), data={"name": key_name}) r = requests.post( - "https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http/{2}?output_mode=json".format( - hostname, mgmt_port, key_name), - verify=False, - auth=("admin", password)) + "https://{0}:{1}/servicesNS/admin/splunk_httpinput/data/inputs/http/{2}?output_mode=json".format( + hostname, mgmt_port, key_name), verify=False, auth=("admin", password)) return str(json.loads(r.text)["entry"][0]["content"]["token"]) self.discovered_servers = [] @@ -375,10 +358,9 @@ def create_new_hec_key(hostname): if new_key: key = create_new_hec_key(formatted_hostname) - self.discovered_servers.append({"protocol": str(protocol), - "address": str(formatted_hostname), - "port": str(hec_port), - "key": str(key)}) + self.discovered_servers.append({ + "protocol": str(protocol), "address": str(formatted_hostname), "port": str(hec_port), "key": + str(key)}) except socket.gaierror: continue @@ -389,10 +371,9 @@ def create_new_hec_key(hostname): if new_key: key = create_new_hec_key(formatted_hostname) - self.discovered_servers.append({"protocol": str(protocol), - "address": str(formatted_hostname), - "port": str(hec_port), - "key": str(key)}) + self.discovered_servers.append({ + "protocol": str(protocol), "address": str(formatted_hostname), "port": str(hec_port), "key": + str(key)}) counter += 1 except socket.gaierror: break @@ -477,9 +458,7 @@ def reset(self): self.log.exception(e) return '500', "Exception: {}".format(e.message) - ############################################## - ############ Event Handler Methods ########### - ############################################## + # Event Handler Methods @event_handler("eventgen_controller", "all_index", handler_type=BROADCAST, reliable_delivery=False) def event_handler_all_index(self, payload): @@ -602,9 +581,7 @@ def event_handler_set_volume(self, payload): def event_handler_reset(self, payload): return self.reset() - ############################################## - ################ HTTP Methods ################ - ############################################## + # HTTP Methods @http('GET', '/') def http_root(self, request): @@ -696,9 +673,7 @@ def http_set_volume(self, request): def http_reset(self, request): return json.dumps(self.reset()) - ############################################## - ################ Helper Methods ############## - ############################################## + # Helper Methods def parse_eventgen_conf(self, path): config = ConfigParser.ConfigParser() @@ -759,10 +734,7 @@ def get_data_volumes(self, config): def get_throughput(self): self.log.debug("Getting throughput ...") - empty_throughput = {'TOTAL_VOLUME_MB': 0, - 'TOTAL_COUNT': 0, - 'THROUGHPUT_VOLUME_KB': 0, - 'THROUGHPUT_COUNT': 0} + empty_throughput = {'TOTAL_VOLUME_MB': 0, 'TOTAL_COUNT': 0, 'THROUGHPUT_VOLUME_KB': 0, 'THROUGHPUT_COUNT': 0} if hasattr(self.eventgen_dependency.eventgen, 'output_counters'): total_volume = 0 total_count = 0 @@ -773,10 +745,9 @@ def get_throughput(self): total_count += output_counter.total_output_count throughput_volume += output_counter.throughput_volume throughput_count += output_counter.throughput_count - return {'TOTAL_VOLUME_MB': total_volume / (1024*1024), - 'TOTAL_COUNT': total_count, - 'THROUGHPUT_VOLUME_KB': throughput_volume / (1024), - 'THROUGHPUT_COUNT': throughput_count} + return { + 'TOTAL_VOLUME_MB': total_volume / (1024 * 1024), 'TOTAL_COUNT': total_count, 'THROUGHPUT_VOLUME_KB': + throughput_volume / (1024), 'THROUGHPUT_COUNT': throughput_count} else: self.log.debug("return empty throughput because of output_counters not found.") - return empty_throughput \ No newline at end of file + return empty_throughput diff --git a/splunk_eventgen/identitygen.py b/splunk_eventgen/identitygen.py index a0c29939..f95a1047 100644 --- a/splunk_eventgen/identitygen.py +++ b/splunk_eventgen/identitygen.py @@ -4,115 +4,135 @@ import time from string import ascii_uppercase -BASE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),"..")) +BASE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + class identityGenerator(object): - ''' - Generates csv file with the following values - ''' - CATEGORIES = ["cardholder","cardholder|pci","officer|pci","intern","default","default","default","default","sox","pci","officer","","","","","","","","","",""] - LOCATION = [["San Francisco","USA","americas","37.3382N","121.6663W"],["San Jose","USA","americas","37.78N","122.41W"]] - EMAIL_DOMAIN = "@splunk.com" - PRIORITIES = ["low","low","low","low","low","low","medium","medium","high","critical"] - def __init__(self): - try: - self.last = [i.split()[0] for i in open("%s/samples/dist.all.last"%BASE_PATH,"rb").readlines()] - except IOError as e: - self.last = [(''.join(random.choice(ascii_uppercase) for i in xrange(random.randint(4,12)))) for i in xrange(100)] - try: - self.female_first = [i.split()[0] for i in open("%s/samples/dist.female.first"%BASE_PATH,"rb").readlines()] - except IOError as e: - self.female_first = [(''.join(random.choice(ascii_uppercase) for i in xrange(random.randint(4,12)))) for i in xrange(100)] - try: - self.male_first = [i.split()[0] for i in open("%s/samples/dist.male.first"%BASE_PATH,"rb").readlines()] - except IOError as e: - self.male_first = [(''.join(random.choice(ascii_uppercase) for i in xrange(random.randint(4,12)))) for i in xrange(100)] + """ + Generates csv file with the following values + """ + CATEGORIES = [ + "cardholder", "cardholder|pci", "officer|pci", "intern", "default", "default", "default", "default", "sox", + "pci", "officer", "", "", "", "", "", "", "", "", "", ""] + LOCATION = [["San Francisco", "USA", "americas", "37.3382N", "121.6663W"], + ["San Jose", "USA", "americas", "37.78N", "122.41W"]] + EMAIL_DOMAIN = "@splunk.com" + PRIORITIES = ["low", "low", "low", "low", "low", "low", "medium", "medium", "high", "critical"] + + def __init__(self): + try: + self.last = [i.split()[0] for i in open("%s/samples/dist.all.last" % BASE_PATH, "rb").readlines()] + except IOError: + self.last = [ + (''.join(random.choice(ascii_uppercase) for i in xrange(random.randint(4, 12)))) for i in xrange(100)] + try: + self.female_first = [ + i.split()[0] for i in open("%s/samples/dist.female.first" % BASE_PATH, "rb").readlines()] + except IOError: + self.female_first = [ + (''.join(random.choice(ascii_uppercase) for i in xrange(random.randint(4, 12)))) for i in xrange(100)] + try: + self.male_first = [i.split()[0] for i in open("%s/samples/dist.male.first" % BASE_PATH, "rb").readlines()] + except IOError: + self.male_first = [ + (''.join(random.choice(ascii_uppercase) for i in xrange(random.randint(4, 12)))) for i in xrange(100)] + + def generate(self, count): + self.identities = [] + usernames = dict() + username = "" + len_last = len(self.last) + len_male_first = len(self.male_first) + len_female_first = len(self.female_first) + for i in xrange(count): + gender = random.choice(["m", "f"]) + last_name = self.last[int(random.triangular(0, len_last, 0))] + if gender == "m": + first_name = self.male_first[int(random.triangular(0, len_male_first, 0))] + else: + first_name = self.female_first[int(random.triangular(0, len_female_first, 0))] + category = random.choice(self.CATEGORIES) + priority = random.choice(self.PRIORITIES) + startDate = time.strftime("%m/%d/%Y", time.localtime(time.time() - random.randint(2592000, 77760000)) + ) # random start date between 30 days ago to 900 days ago + (work_city, work_country, bunit, work_lat, work_long) = random.choice(self.LOCATION) + identity = { + "first_name": first_name, "last_name": last_name, "work_city": work_city, "work_country": work_country, + "bunit": bunit, "work_lat": work_lat, "work_long": work_long, "priority": priority, "category": + category, "startDate": startDate} + base_username = identity["first_name"] + identity["last_name"] + if base_username in usernames: + tmp_val = 0 + while username + str(tmp_val) in usernames[base_username]: + tmp_val += 1 + username = base_username + str(tmp_val) + usernames[base_username].append(username) + else: + username = base_username + usernames[username] = list() + identity["username"] = username + identity["ip"] = self.int2InternalIP(i) + self.identities.append(identity) - def generate(self, count): - self.identities = [] - usernames = dict() - len_last = len(self.last) - len_male_first = len(self.male_first) - len_female_first = len(self.female_first) - prev_time = time.time() - for i in xrange(count): - gender = random.choice(["m","f"]) - last_name = self.last[int(random.triangular(0,len_last,0))] - if gender == "m": - first_name = self.male_first[int(random.triangular(0,len_male_first,0))] - else: - first_name = self.female_first[int(random.triangular(0,len_female_first,0))] - category = random.choice(self.CATEGORIES) - priority = random.choice(self.PRIORITIES) - startDate = time.strftime("%m/%d/%Y", time.localtime(time.time()-random.randint(2592000,77760000))) # random start date between 30 days ago to 900 days ago - (work_city, work_country, bunit, work_lat, work_long)= random.choice(self.LOCATION) - identity = {"first_name":first_name,"last_name":last_name,"work_city":work_city,"work_country":work_country,"bunit":bunit,"work_lat":work_lat,"work_long":work_long,"priority":priority,"category":category,"startDate":startDate} - base_username = identity["first_name"] + identity["last_name"] - t = time.time() - if base_username in usernames: - tmp_val = 0 - while username + str(tmp_val) in usernames[base_username]: - tmp_val += 1 - username = base_username + str(tmp_val) - usernames[base_username].append(username) - else: - username = base_username - usernames[username] = list() - identity["username"] = username - identity["ip"] = self.int2InternalIP(i) - self.identities.append(identity) - + def int2InternalIP(self, i): + return "10.%s.%s.%s" % (str(int(i / 65536)), str(int(i / 256) % 256), str(i % 256)) - def int2InternalIP(self,i): - return "10.%s.%s.%s" % (str(int(i/65536)), str(int(i/256)%256), str(i%256)) + def setLocations(self, new_locations): + for location in new_locations: + if len(location) != 5: + raise ValueError + self.CATEGORIES = new_locations - def setLocations(self,new_locations): - for location in new_locations: - if len(location)!=5: - raise ValueError - self.CATEGORIES = new_locations + def setCategories(self, new_categories): + self.CATEGORIES = new_categories - def setCategories(self,new_categories): - self.CATEGORIES = new_categories + def setEmail(self, new_email): + if "@" in new_email: + self.EMAIL_DOMAIN = new_email + else: + raise ValueError - def setEmail(self,new_email): - if "@" in new_email: - self.EMAIL_DOMAIN = new_email - else: - raise ValueError + def getFile(self, count=0, filename="../default", fields=["username", "first_name", "last_name"], fieldnames=[ + "username", "first_name", "last_name"]): + """ + Returns a rest endpoint to download a csv file + """ + if count == 0: + with open(filename, "wb") as lookupFile: + file = csv.writer(lookupFile) + file.writerow(fieldnames) + for identity in self.identities: + row = [] + for field in fields: + try: + row.append(identity[field]) + except KeyError: + row.append("") + file.writerow(row) + else: + with open(filename, "wb") as lookupFile: + file = csv.writer(lookupFile) + file.writerow(fieldnames) + for i in xrange(min(count + 1, len(self.identities))): # + 1 to account for the header + row = [] + identity = self.identities[i] + for field in fields: + try: + row.append(identity[field]) + except KeyError: + row.append("") + file.writerow(row) + return open(filename, "rb") - def getFile(self,count=0,filename="../default",fields=["username","first_name","last_name"],fieldnames=["username","first_name","last_name"]): - 'Returns a rest endpoint to download a csv file' - if count == 0: - with open(filename,"wb") as lookupFile: - file = csv.writer(lookupFile) - file.writerow(fieldnames) - for identity in self.identities: - row = [] - for field in fields: - try: - row.append(identity[field]) - except KeyError: - row.append("") - file.writerow(row) - else: - with open(filename,"wb") as lookupFile: - file = csv.writer(lookupFile) - file.writerow(fieldnames) - for i in xrange(min(count+1,len(identities))): # + 1 to account for the header - row = [] - identity = identities[i] - for field in fields: - try: - row.append(identity[field]) - except KeyError: - row.append("") - file.writerow(row) - return open(filename,"rb") if __name__ == "__main__": - identityGenerator = identityGenerator() - identityGenerator.generate(300000) - identityGenerator.getFile(filename="identities.csv", - fields=["username","prefix","username","first_name","last_name","suffix","email","phone","phone2","managedBy","priority","bunit","category","watchlist","startDate","endDate","work_city","work_country","work_lat","work_long"], - fieldnames = ["identity","prefix","nick","first","last","suffix","email","phone","phone2","managedBy","priority","bunit","category","watchlist","startDate","endDate","work_city","work_country","work_lat","work_long"]) \ No newline at end of file + identityGenerator = identityGenerator() + identityGenerator.generate(300000) + identityGenerator.getFile( + filename="identities.csv", fields=[ + "username", "prefix", "username", "first_name", "last_name", "suffix", "email", "phone", "phone2", + "managedBy", "priority", "bunit", "category", "watchlist", "startDate", "endDate", "work_city", + "work_country", "work_lat", "work_long"], fieldnames=[ + "identity", "prefix", "nick", "first", "last", "suffix", "email", "phone", "phone2", "managedBy", + "priority", "bunit", "category", "watchlist", "startDate", "endDate", "work_city", "work_country", + "work_lat", "work_long"]) diff --git a/splunk_eventgen/lib/eventgenconfig.py b/splunk_eventgen/lib/eventgenconfig.py index 161fca6b..b9f1db38 100644 --- a/splunk_eventgen/lib/eventgenconfig.py +++ b/splunk_eventgen/lib/eventgenconfig.py @@ -1,17 +1,20 @@ from __future__ import division -from ConfigParser import ConfigParser -import os + import datetime -import re -import logging, logging.handlers import json +import logging +import logging.handlers +import os import pprint +import random +import re +import types +import urllib +from ConfigParser import ConfigParser + +from eventgenexceptions import FailedLoadingPlugin, PluginNotLoaded from eventgensamples import Sample from eventgentoken import Token -from eventgenexceptions import PluginNotLoaded, FailedLoadingPlugin -import urllib -import types -import random # 4/21/14 CS Adding a defined constant whether we're running in standalone mode or not # Standalone mode is when we know we're Splunk embedded but we want to force @@ -27,6 +30,7 @@ STANDALONE = False + # 5/10/12 CS Some people consider Singleton to be lazy. Dunno, I like it for convenience. # My general thought on that sort of stuff is if you don't like it, reimplement it. I'll consider # your patch. @@ -50,11 +54,11 @@ class Config(object): sessionKey = None grandparentdir = None greatgrandparentdir = None - samples = [ ] + samples = [] sampleDir = None outputWorkers = None generatorWorkers = None - sampleTimers = [ ] + sampleTimers = [] # Config file options. We do not define defaults here, rather we pull them in # from eventgen.conf. @@ -65,47 +69,49 @@ class Config(object): disabled = None blacklist = ".*\.part" - __generatorworkers = [ ] - __outputworkers = [ ] - outputPlugins = { } - plugins = { } + __generatorworkers = [] + __outputworkers = [] + outputPlugins = {} + plugins = {} outputQueue = None generatorQueue = None args = None - ## Validations - _validSettings = ['disabled', 'blacklist', 'spoolDir', 'spoolFile', 'breaker', 'sampletype' , 'interval', - 'delay', 'count', 'bundlelines', 'earliest', 'latest', 'eai:acl', 'hourOfDayRate', - 'dayOfWeekRate', 'randomizeCount', 'randomizeEvents', 'outputMode', 'fileName', 'fileMaxBytes', - 'fileBackupFiles', 'index', 'source', 'sourcetype', 'host', 'hostRegex', 'projectID', 'accessToken', - 'mode', 'backfill', 'backfillSearch', 'eai:userName', 'eai:appName', 'timeMultiple', 'debug', - 'minuteOfHourRate', 'timezone', 'dayOfMonthRate', 'monthOfYearRate', 'perDayVolume', - 'outputWorkers', 'generator', 'rater', 'generatorWorkers', 'timeField', 'sampleDir', 'threading', - 'profiler', 'maxIntervalsBeforeFlush', 'maxQueueLength', 'splunkMethod', 'splunkPort', - 'verbosity', 'useOutputQueue', 'seed','end', 'autotimestamps', 'autotimestamp', 'httpeventWaitResponse', 'outputCounter', 'sequentialTimestamp'] + # Validations + _validSettings = [ + 'disabled', 'blacklist', 'spoolDir', 'spoolFile', 'breaker', 'sampletype', 'interval', 'delay', 'count', + 'bundlelines', 'earliest', 'latest', 'eai:acl', 'hourOfDayRate', 'dayOfWeekRate', 'randomizeCount', + 'randomizeEvents', 'outputMode', 'fileName', 'fileMaxBytes', 'fileBackupFiles', 'index', 'source', 'sourcetype', + 'host', 'hostRegex', 'projectID', 'accessToken', 'mode', 'backfill', 'backfillSearch', 'eai:userName', + 'eai:appName', 'timeMultiple', 'debug', 'minuteOfHourRate', 'timezone', 'dayOfMonthRate', 'monthOfYearRate', + 'perDayVolume', 'outputWorkers', 'generator', 'rater', 'generatorWorkers', 'timeField', 'sampleDir', + 'threading', 'profiler', 'maxIntervalsBeforeFlush', 'maxQueueLength', 'splunkMethod', 'splunkPort', 'verbosity', + 'useOutputQueue', 'seed', 'end', 'autotimestamps', 'autotimestamp', 'httpeventWaitResponse', 'outputCounter', + 'sequentialTimestamp'] _validTokenTypes = {'token': 0, 'replacementType': 1, 'replacement': 2} _validHostTokens = {'token': 0, 'replacement': 1} - _validReplacementTypes = ['static', 'timestamp', 'replaytimestamp', 'random', 'rated', 'file', 'mvfile', 'seqfile', 'integerid'] - validOutputModes = [ ] + _validReplacementTypes = [ + 'static', 'timestamp', 'replaytimestamp', 'random', 'rated', 'file', 'mvfile', 'seqfile', 'integerid'] + validOutputModes = [] _intSettings = ['interval', 'outputWorkers', 'generatorWorkers', 'maxIntervalsBeforeFlush', 'maxQueueLength'] _floatSettings = ['randomizeCount', 'delay', 'timeMultiple'] - _boolSettings = ['disabled', 'randomizeEvents', 'bundlelines', 'profiler', 'useOutputQueue', 'autotimestamp', 'httpeventWaitResponse', 'outputCounter', 'sequentialTimestamp'] - _jsonSettings = ['hourOfDayRate', 'dayOfWeekRate', 'minuteOfHourRate', 'dayOfMonthRate', 'monthOfYearRate', 'autotimestamps'] - _defaultableSettings = ['disabled', 'spoolDir', 'spoolFile', 'breaker', 'sampletype', 'interval', 'delay', - 'count', 'bundlelines', 'earliest', 'latest', 'hourOfDayRate', 'dayOfWeekRate', - 'randomizeCount', 'randomizeEvents', 'outputMode', 'fileMaxBytes', 'fileBackupFiles', - 'splunkHost', 'splunkPort', 'splunkMethod', 'index', 'source', 'sourcetype', 'host', 'hostRegex', - 'projectID', 'accessToken', 'mode', 'minuteOfHourRate', 'timeMultiple', 'dayOfMonthRate', - 'monthOfYearRate', 'perDayVolume', 'sessionKey', 'generator', 'rater', 'timeField', 'maxQueueLength', - 'maxIntervalsBeforeFlush', 'autotimestamp'] - _complexSettings = { 'sampletype': ['raw', 'csv'], - 'mode': ['sample', 'replay'], - 'threading': ['thread', 'process']} + _boolSettings = [ + 'disabled', 'randomizeEvents', 'bundlelines', 'profiler', 'useOutputQueue', 'autotimestamp', + 'httpeventWaitResponse', 'outputCounter', 'sequentialTimestamp'] + _jsonSettings = [ + 'hourOfDayRate', 'dayOfWeekRate', 'minuteOfHourRate', 'dayOfMonthRate', 'monthOfYearRate', 'autotimestamps'] + _defaultableSettings = [ + 'disabled', 'spoolDir', 'spoolFile', 'breaker', 'sampletype', 'interval', 'delay', 'count', 'bundlelines', + 'earliest', 'latest', 'hourOfDayRate', 'dayOfWeekRate', 'randomizeCount', 'randomizeEvents', 'outputMode', + 'fileMaxBytes', 'fileBackupFiles', 'splunkHost', 'splunkPort', 'splunkMethod', 'index', 'source', 'sourcetype', + 'host', 'hostRegex', 'projectID', 'accessToken', 'mode', 'minuteOfHourRate', 'timeMultiple', 'dayOfMonthRate', + 'monthOfYearRate', 'perDayVolume', 'sessionKey', 'generator', 'rater', 'timeField', 'maxQueueLength', + 'maxIntervalsBeforeFlush', 'autotimestamp'] + _complexSettings = {'sampletype': ['raw', 'csv'], 'mode': ['sample', 'replay'], 'threading': ['thread', 'process']} def __init__(self, configfile=None, sample=None, override_outputter=False, override_count=False, - override_interval=False, override_backfill=False, override_end=False, - threading="thread", override_generators=None, override_outputqueue=False, - profiler=False, verbosity=40): + override_interval=False, override_backfill=False, override_end=False, threading="thread", + override_generators=None, override_outputqueue=False, profiler=False, verbosity=40): """Setup Config object. Sets up Logging and path related variables.""" # Rebind the internal datastore of the class to an Instance variable self.__dict__ = self.__sharedState @@ -144,18 +150,18 @@ def __init__(self, configfile=None, sample=None, override_outputter=False, overr self.stopping = False - #self.copyLock = threading.Lock() if self.threading == 'thread' else multiprocessing.Lock() + # self.copyLock = threading.Lock() if self.threading == 'thread' else multiprocessing.Lock() self._firsttime = False def __str__(self): """Only used for debugging, outputs a pretty printed representation of our Config""" # Filter items from config we don't want to pretty print - filter_list = [ 'samples', 'sampleTimers', '__generatorworkers', '__outputworkers' ] + filter_list = ['samples', 'sampleTimers', '__generatorworkers', '__outputworkers'] # Eliminate recursive going back to parent - temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key not in filter_list ]) + temp = dict([(key, value) for (key, value) in self.__dict__.items() if key not in filter_list]) - return 'Config:'+pprint.pformat(temp)+'\nSamples:\n'+pprint.pformat(self.samples) + return 'Config:' + pprint.pformat(temp) + '\nSamples:\n' + pprint.pformat(self.samples) # loggers can't be pickled due to the lock object, remove them before we try to pickle anything. def __getstate__(self): @@ -173,14 +179,13 @@ def _setup_logging(self): def getPlugin(self, name, s=None): """Return a reference to a Python object (not an instance) referenced by passed name""" - ''' APPPERF-263: make sure we look in __outputPlugins as well. For some reason we keep 2 separate dicts of plugins. ''' - plugintype=name.split(".")[0] - if not name in self.plugins and not name in self.outputPlugins: + plugintype = name.split(".")[0] + if name not in self.plugins and name not in self.outputPlugins: # 2/1/15 CS If we haven't already seen the plugin, try to load it # Note, this will only work for plugins which do not specify config validation # parameters. If they do, configs may not validate for user provided plugins. @@ -189,14 +194,14 @@ def getPlugin(self, name, s=None): plugin = getattr(s, plugintype) else: plugin = getattr(s, 'outputMode') - if plugin != None: - self.logger.debug("Attempting to dynamically load plugintype '%s' named '%s' for sample '%s'" - % (plugintype, plugin, s.name)) + if plugin is not None: + self.logger.debug("Attempting to dynamically load plugintype '%s' named '%s' for sample '%s'" % + (plugintype, plugin, s.name)) bindir = os.path.join(s.sampleDir, os.pardir, 'bin') libdir = os.path.join(s.sampleDir, os.pardir, 'lib') plugindir = os.path.join(libdir, 'plugins', plugintype) - targetplugin = PluginNotLoaded(bindir=bindir, libdir=libdir, - plugindir=plugindir, name=plugin, type=plugintype) + targetplugin = PluginNotLoaded(bindir=bindir, libdir=libdir, plugindir=plugindir, name=plugin, + type=plugintype) if targetplugin.name not in self.extraplugins: self.extraplugins.append(targetplugin.name) raise targetplugin @@ -204,7 +209,7 @@ def getPlugin(self, name, s=None): raise FailedLoadingPlugin(name=plugin) # APPPERF-263: consult both __outputPlugins and __plugins - if not name in self.plugins and not name in self.outputPlugins: + if name not in self.plugins and name not in self.outputPlugins: raise KeyError('Plugin ' + name + ' not found') # return in order of precedence: __plugins, __outputPlugins, None @@ -218,7 +223,7 @@ def makeSplunkEmbedded(self, sessionKey): def getSplunkUrl(self, s): """ - Get Splunk URL. If we're embedded in Splunk, get it from Splunk's Python libraries, otherwise get it from config. + If we're embedded in Splunk, get it from Splunk's Python libraries, otherwise get it from config. Returns a tuple of ( splunkUrl, splunkMethod, splunkHost, splunkPort ) """ @@ -233,11 +238,14 @@ def getSplunkUrl(self, s): except: import traceback trace = traceback.format_exc() - self.logger.error('Error parsing host from splunk.auth.splunk.getLocalServerInfo() for sample %s. Stacktrace: %s' % (s.name, trace)) - raise ValueError('Error parsing host from splunk.auth.splunk.getLocalServerInfo() for sample %s' % s.name) + self.logger.error( + 'Error parsing host from splunk.auth.splunk.getLocalServerInfo() for sample %s. Stacktrace: %s' % + (s.name, trace)) + raise ValueError( + 'Error parsing host from splunk.auth.splunk.getLocalServerInfo() for sample %s' % s.name) else: # splunkMethod and splunkPort are defaulted so only check for splunkHost - if s.splunkHost == None: + if s.splunkHost is None: self.logger.error("Splunk URL Requested but splunkHost not set for sample '%s'" % s.name) raise ValueError("Splunk URL Requested but splunkHost not set for sample '%s'" % s.name) @@ -246,7 +254,8 @@ def getSplunkUrl(self, s): splunkHost = s.splunkHost splunkPort = s.splunkPort - self.logger.debug("Getting Splunk URL: %s Method: %s Host: %s Port: %s" % (splunkUrl, splunkMethod, splunkHost, splunkPort)) + self.logger.debug( + "Getting Splunk URL: %s Method: %s Host: %s Port: %s" % (splunkUrl, splunkMethod, splunkHost, splunkPort)) return (splunkUrl, splunkMethod, splunkHost, splunkPort) def parse(self): @@ -267,8 +276,8 @@ def parse(self): if 'default' in self._confDict: del self._confDict['default'] - tempsamples = [ ] - tempsamples2 = [ ] + tempsamples = [] + tempsamples2 = [] stanza_map = {} stanza_list = [] @@ -322,7 +331,8 @@ def parse(self): if 'token.{}.token'.format(i) in self._confDict[global_stanza]: token = self._confDict[global_stanza].get('token.{}.token'.format(i)) replacement = self._confDict[global_stanza].get('token.{}.replacement'.format(i)) - replacementType = self._confDict[global_stanza].get('token.{}.replacementType'.format(i)) + replacementType = self._confDict[global_stanza].get( + 'token.{}.replacementType'.format(i)) last_token_number += 1 if token: @@ -343,9 +353,9 @@ def parse(self): break keys = settings.keys() - for k,v in self._confDict[global_stanza].items(): + for k, v in self._confDict[global_stanza].items(): if 'token' not in k and k not in keys: - kv_pair_items.append((k,v)) + kv_pair_items.append((k, v)) for key, value in kv_pair_items: oldvalue = value @@ -359,7 +369,7 @@ def parse(self): # Token indices could be out of order, so we must check to # see whether we have enough items in the list to update the token # In general this will keep growing the list by whatever length we need - if(key.find("host.") > -1): + if (key.find("host.") > -1): # self.logger.info("hostToken.{} = {}".format(value[1],oldvalue)) if not isinstance(s.hostToken, Token): s.hostToken = Token(s) @@ -368,8 +378,8 @@ def parse(self): setattr(s.hostToken, value[0], oldvalue) else: if len(s.tokens) <= value[0]: - x = (value[0]+1) - len(s.tokens) - s.tokens.extend([None for i in xrange(0, x)]) + x = (value[0] + 1) - len(s.tokens) + s.tokens.extend([None for num in xrange(0, x)]) if not isinstance(s.tokens[value[0]], Token): s.tokens[value[0]] = Token(s) # logger.info("token[{}].{} = {}".format(value[0],value[1],oldvalue)) @@ -383,42 +393,39 @@ def parse(self): s._lockedSettings.append(key) # self.logger.debug("Appending '%s' to locked settings for sample '%s'" % (key, s.name)) - - # Validate all the tokens are fully setup, can't do this in _validateSettings # because they come over multiple lines # Don't error out at this point, just log it and remove the token and move on - deleteidx = [ ] + deleteidx = [] for i in xrange(0, len(s.tokens)): t = s.tokens[i] # If the index doesn't exist at all - if t == None: + if t is None: self.logger.error("Token at index %s invalid" % i) # Can't modify list in place while we're looping through it # so create a list to remove later deleteidx.append(i) - elif t.token == None or t.replacementType == None or t.replacement == None: + elif t.token is None or t.replacementType is None or t.replacement is None: self.logger.error("Token at index %s invalid" % i) deleteidx.append(i) - newtokens = [ ] + newtokens = [] for i in xrange(0, len(s.tokens)): if i not in deleteidx: newtokens.append(s.tokens[i]) s.tokens = newtokens - # Must have eai:acl key to determine app name which determines where actual files are - if s.app == None: + if s.app is None: self.logger.error("App not set for sample '%s' in stanza '%s'" % (s.name, stanza)) raise ValueError("App not set for sample '%s' in stanza '%s'" % (s.name, stanza)) # Set defaults for items not included in the config file for setting in self._defaultableSettings: - if not hasattr(s, setting) or getattr(s, setting) == None: + if not hasattr(s, setting) or getattr(s, setting) is None: setattr(s, setting, getattr(self, setting, None)) # Append to temporary holding list if not s.disabled: - s._priority = len(tempsamples)+1 + s._priority = len(tempsamples) + 1 tempsamples.append(s) # 6/22/12 CS Rewriting the config matching code yet again to handling flattening better. @@ -427,14 +434,15 @@ def parse(self): # every other match to that one. for s in tempsamples: # Now we need to match this up to real files. May generate multiple copies of the sample. - foundFiles = [ ] + foundFiles = [] # 1/5/14 Adding a config setting to override sample directory, primarily so I can put tests in their own # directories - if s.sampleDir == None: + if s.sampleDir is None: self.logger.debug("Sample directory not specified in config, setting based on standard") if self.splunkEmbedded and not STANDALONE: - s.sampleDir = os.path.normpath(os.path.join(self.grandparentdir, '..', '..', '..', s.app, 'samples')) + s.sampleDir = os.path.normpath( + os.path.join(self.grandparentdir, '..', '..', '..', s.app, 'samples')) else: # 2/1/15 CS Adding support for looking for samples based on the config file specified on # the command line. @@ -452,7 +460,8 @@ def parse(self): if not os.path.exists(s.sampleDir): newSampleDir = os.path.join(self.grandparentdir, 'samples') - self.logger.error("Path not found for samples '%s', trying '%s'" % (s.sampleDir, newSampleDir)) + self.logger.error( + "Path not found for samples '%s', trying '%s'" % (s.sampleDir, newSampleDir)) s.sampleDir = newSampleDir else: self.logger.debug("Sample directory specified in config, checking for relative") @@ -485,10 +494,12 @@ def parse(self): self.maxIntervalsBeforeFlush = 1 s.maxIntervalsBeforeFlush = 1 s.maxQueueLength = s.maxQueueLength or 1 - self.logger.debug("Sample '%s' setting maxQueueLength to '%s' from command line" % (s.name, s.maxQueueLength)) + self.logger.debug( + "Sample '%s' setting maxQueueLength to '%s' from command line" % (s.name, s.maxQueueLength)) if self.override_outputter: - self.logger.debug("Sample '%s' setting output to '%s' from command line" % (s.name, self.override_outputter)) + self.logger.debug( + "Sample '%s' setting output to '%s' from command line" % (s.name, self.override_outputter)) s.outputMode = self.override_outputter if self.override_count: @@ -498,11 +509,13 @@ def parse(self): s.backfill = None if self.override_interval: - self.logger.debug("Overriding interval to '%d' for sample '%s'" % (self.override_interval, s.name)) + self.logger.debug( + "Overriding interval to '%d' for sample '%s'" % (self.override_interval, s.name)) s.interval = self.override_interval if self.override_backfill: - self.logger.debug("Overriding backfill to '%s' for sample '%s'" % (self.override_backfill, s.name)) + self.logger.debug( + "Overriding backfill to '%s' for sample '%s'" % (self.override_backfill, s.name)) s.backfill = self.override_backfill.lstrip() if self.override_end: @@ -517,7 +530,7 @@ def parse(self): for token in s.tokens: if token.replacementType == 'integerid': try: - stateFile = open(os.path.join(s.sampleDir, 'state.'+urllib.pathname2url(token.token)), 'rU') + stateFile = open(os.path.join(s.sampleDir, 'state.' + urllib.pathname2url(token.token)), 'rU') token.replacement = stateFile.read() stateFile.close() # The file doesn't exist, use the default value in the config @@ -532,8 +545,9 @@ def parse(self): self.logger.debug("Matched file {0} with sample name {1}".format(results.group(0), s.name)) samplePath = os.path.join(s.sampleDir, sample) if os.path.isfile(samplePath): - self.logger.debug("Found sample file '%s' for app '%s' using config '%s' with priority '%s'; adding to list" \ - % (sample, s.app, s.name, s._priority) ) + self.logger.debug( + "Found sample file '%s' for app '%s' using config '%s' with priority '%s'" % + (sample, s.app, s.name, s._priority) + "; adding to list") foundFiles.append(samplePath) # If we didn't find any files, log about it @@ -554,9 +568,9 @@ def parse(self): # Override with real name if s.outputMode == 'spool' and s.spoolFile == self.spoolFile: news.spoolFile = f.split(os.sep)[-1] - if s.outputMode == 'file' and s.fileName == None and s.spoolFile == self.spoolFile: + if s.outputMode == 'file' and s.fileName is None and s.spoolFile == self.spoolFile: news.fileName = os.path.join(s.spoolDir, f.split(os.sep)[-1]) - elif s.outputMode == 'file' and s.fileName == None and s.spoolFile != None: + elif s.outputMode == 'file' and s.fileName is None and s.spoolFile is not None: news.fileName = os.path.join(s.spoolDir, s.spoolFile) # Override s.name with file name. Usually they'll match unless we've been a regex # 6/22/12 CS Save original name for later matching @@ -568,7 +582,7 @@ def parse(self): self.logger.info("Sample '%s' for app '%s' is marked disabled." % (news.name, news.app)) # Clear tempsamples, we're going to reuse it - tempsamples = [ ] + tempsamples = [] # We're now going go through the samples and attempt to apply any matches from other stanzas # This allows us to specify a wildcard at the beginning of the file and get more specific as we go on @@ -576,25 +590,26 @@ def parse(self): # Loop through all samples, create a list of the master samples for s in tempsamples2: foundHigherPriority = False - othermatches = [ ] + othermatches = [] # If we're an exact match, don't go looking for higher priorities if not s.name == s._origName: for matchs in tempsamples2: if matchs.filePath == s.filePath and s._origName != matchs._origName: # We have a match, now determine if we're higher priority or not - # If this is a longer pattern or our match is an exact match - # then we're a higher priority match + # If this is a longer pattern or our match is an exact match + # then we're a higher priority match if len(matchs._origName) > len(s._origName) or matchs.name == matchs._origName: # if s._priority < matchs._priority: - self.logger.debug("Found higher priority for sample '%s' with priority '%s' from sample '%s' with priority '%s'" \ - % (s._origName, s._priority, matchs._origName, matchs._priority)) + self.logger.debug("Found higher priority for sample '%s' with priority '%s' from sample " % + (s._origName, s._priority) + + "'%s' with priority '%s'" % (matchs._origName, matchs._priority)) foundHigherPriority = True break else: othermatches.append(matchs._origName) if not foundHigherPriority: - self.logger.debug("Chose sample '%s' from samples '%s' for file '%s'" \ - % (s._origName, othermatches, s.name)) + self.logger.debug( + "Chose sample '%s' from samples '%s' for file '%s'" % (s._origName, othermatches, s.name)) tempsamples.append(s) # Now we have two lists, tempsamples which contains only the highest priority matches, and @@ -625,13 +640,14 @@ def parse(self): # 6/22/12 CS Added support for non-overrideable (locked) settings # logger.debug("Locked settings: %s" % pprint.pformat(matchs._lockedSettings)) # if settingname in matchs._lockedSettings: - # logger.debug("Matched setting '%s' in sample '%s' lockedSettings" \ + # logger.debug("Matched setting '%s' in sample '%s' lockedSettings" # % (settingname, matchs.name)) - if (destsetting == None or destsetting == getattr(self, settingname)) \ - and sourcesetting != None and sourcesetting != getattr(self, settingname) \ - and not settingname in s._lockedSettings: - self.logger.debug("Overriding setting '%s' with value '%s' from sample '%s' to sample '%s' in app '%s'" \ - % (settingname, sourcesetting, overridesample._origName, s.name, s.app)) + if (destsetting is None or destsetting == getattr(self, settingname)) \ + and sourcesetting is not None and sourcesetting != getattr(self, settingname) \ + and settingname not in s._lockedSettings: + self.logger.debug("Overriding setting '%s' with value '%s' from sample '%s' to " % + (settingname, sourcesetting, overridesample._origName) + + "sample '%s' in app '%s'" % (s.name, s.app)) setattr(s, settingname, sourcesetting) except AttributeError: pass @@ -649,7 +665,8 @@ def parse(self): # We've added replay mode, so lets loop through the samples again and set the earliest and latest # settings for any samples that were set to replay mode if s.perDayVolume: - self.logger.info("Stanza contains per day volume, changing rater and generator to perdayvolume instead of count") + self.logger.info( + "Stanza contains per day volume, changing rater and generator to perdayvolume instead of count") s.rater = 'perdayvolume' s.count = 1 s.generator = 'perdayvolumegenerator' @@ -676,7 +693,7 @@ def parse(self): if s.autotimestamp: at = self.autotimestamps - line_puncts = [ ] + line_puncts = [] # Check for _time field, if it exists, add a timestamp to support it if len(s.sampleDict) > 0: @@ -712,7 +729,8 @@ def parse(self): t.replacement = x[1] try: - self.logger.debug("Trying regex '%s' for format '%s' on '%s'" % (x[0], x[1], e[s.timeField])) + self.logger.debug( + "Trying regex '%s' for format '%s' on '%s'" % (x[0], x[1], e[s.timeField])) ts = s.getTSFromEvent(e['_raw'], t) if type(ts) == datetime.datetime: found_token = False @@ -722,10 +740,11 @@ def parse(self): found_token = True break if not found_token: - self.logger.debug("Found timestamp '%s', extending token with format '%s'" % (x[0], x[1])) + self.logger.debug( + "Found timestamp '%s', extending token with format '%s'" % (x[0], x[1])) s.tokens.append(t) # Drop this pattern from ones we try in the future - at = [ z for z in at if z[0] != x[0] ] + at = [z for z in at if z[0] != x[0]] break except ValueError: pass @@ -749,23 +768,23 @@ def _validateSetting(self, stanza, key, value): self.logger.debug("Validating setting for '%s' with value '%s' in stanza '%s'" % (key, value, stanza)) if key.find('token.') > -1: results = re.match('token\.(\d+)\.(\w+)', key) - if results != None: + if results is not None: groups = results.groups() if groups[1] not in self._validTokenTypes: - self.logger.error("Could not parse token index '%s' token type '%s' in stanza '%s'" % \ - (groups[0], groups[1], stanza)) - raise ValueError("Could not parse token index '%s' token type '%s' in stanza '%s'" % \ - (groups[0], groups[1], stanza)) + self.logger.error("Could not parse token index '%s' token type '%s' in stanza '%s'" % + (groups[0], groups[1], stanza)) + raise ValueError("Could not parse token index '%s' token type '%s' in stanza '%s'" % + (groups[0], groups[1], stanza)) if groups[1] == 'replacementType': if value not in self._validReplacementTypes: - self.logger.error("Invalid replacementType '%s' for token index '%s' in stanza '%s'" % \ - (value, groups[0], stanza)) - raise ValueError("Could not parse token index '%s' token type '%s' in stanza '%s'" % \ - (groups[0], groups[1], stanza)) + self.logger.error("Invalid replacementType '%s' for token index '%s' in stanza '%s'" % + (value, groups[0], stanza)) + raise ValueError("Could not parse token index '%s' token type '%s' in stanza '%s'" % + (groups[0], groups[1], stanza)) return (int(groups[0]), groups[1]) elif key.find('host.') > -1: results = re.match('host\.(\w+)', key) - if results != None: + if results is not None: groups = results.groups() if groups[0] not in self._validHostTokens: self.logger.error("Could not parse host token type '%s' in stanza '%s'" % (groups[0], stanza)) @@ -811,8 +830,9 @@ def _validateSetting(self, stanza, key, value): self.logger.debug("Calling function for setting '%s' with value '%s'" % (key, value)) value = complexSetting(value) elif isinstance(complexSetting, list): - if not value in complexSetting: - self.logger.error("Setting '%s' is invalid for value '%s' in stanza '%s'" % (key, value, stanza)) + if value not in complexSetting: + self.logger.error( + "Setting '%s' is invalid for value '%s' in stanza '%s'" % (key, value, stanza)) raise ValueError("Setting '%s' is invalid for value '%s' in stanza '%s'" % (key, value, stanza)) else: # Notifying only if the setting isn't valid and continuing on @@ -832,7 +852,7 @@ def _validateTimezone(self, value): mod = 100 else: mod = -100 - value = datetime.timedelta(hours=int(int(value) / 100.0), minutes=int(value) % mod ) + value = datetime.timedelta(hours=int(int(value) / 100.0), minutes=int(value) % mod) except: self.logger.error("Could not parse timezone {}".format(value)) raise ValueError("Could not parse timezone {}".format(value)) @@ -855,7 +875,6 @@ def _buildConfDict(self): """Build configuration dictionary that we will use """ # Abstracts grabbing configuration from Splunk or directly from Configuration Files - if self.splunkEmbedded and not STANDALONE: self.logger.info('Retrieving eventgen configurations from /configs/eventgen') import splunk.entity as entity @@ -866,9 +885,7 @@ def _buildConfDict(self): conf = ConfigParser() # Make case sensitive conf.optionxform = str - currentdir = os.getcwd() - - conffiles = [ ] + conffiles = [] # 2/1/15 CS Moving to argparse way of grabbing command line parameters if self.configfile: if os.path.exists(self.configfile): @@ -876,26 +893,26 @@ def _buildConfDict(self): # In which case we'll assume it's a splunk app and look for config files in # default and local if os.path.isdir(self.configfile): - conffiles = [os.path.join(self.grandparentdir, 'default', 'eventgen.conf'), - os.path.join(self.configfile, 'default', 'eventgen.conf'), - os.path.join(self.configfile, 'local', 'eventgen.conf')] + conffiles = [ + os.path.join(self.grandparentdir, 'default', 'eventgen.conf'), + os.path.join(self.configfile, 'default', 'eventgen.conf'), + os.path.join(self.configfile, 'local', 'eventgen.conf')] else: - conffiles = [os.path.join(self.grandparentdir, 'default', 'eventgen.conf'), - self.configfile] + conffiles = [os.path.join(self.grandparentdir, 'default', 'eventgen.conf'), self.configfile] if len(conffiles) == 0: - conffiles = [os.path.join(self.grandparentdir, 'default', 'eventgen.conf'), - os.path.join(self.grandparentdir, 'local', 'eventgen.conf')] + conffiles = [ + os.path.join(self.grandparentdir, 'default', 'eventgen.conf'), + os.path.join(self.grandparentdir, 'local', 'eventgen.conf')] self.logger.debug('Reading configuration files for non-splunkembedded: %s' % conffiles) conf.read(conffiles) sections = conf.sections() - ret = { } - orig = { } + ret = {} for section in sections: ret[section] = dict(conf.items(section)) # For compatibility with Splunk's configs, need to add the app name to an eai:acl key - ret[section]['eai:acl'] = { 'app': self.grandparentdir.split(os.sep)[-1] } + ret[section]['eai:acl'] = {'app': self.grandparentdir.split(os.sep)[-1]} self._confDict = ret self.logger.debug("ConfDict returned %s" % pprint.pformat(dict(self._confDict))) diff --git a/splunk_eventgen/lib/eventgenexceptions.py b/splunk_eventgen/lib/eventgenexceptions.py index f954192c..3d0d0d91 100644 --- a/splunk_eventgen/lib/eventgenexceptions.py +++ b/splunk_eventgen/lib/eventgenexceptions.py @@ -3,9 +3,7 @@ """ - class PluginNotLoaded(Exception): - def __init__(self, bindir, libdir, plugindir, name, type, msg="Plugin {} Not Loaded, attempting to load."): """Exception raised when a sample asks for a plugin that is not in the plugin list. This exception triggers an upload reload of plugins that expands the search path of plugins to add. @@ -26,8 +24,8 @@ def __init__(self, bindir, libdir, plugindir, name, type, msg="Plugin {} Not Loa self.type = type super(PluginNotLoaded, self).__init__(msg) -class FailedLoadingPlugin(Exception): +class FailedLoadingPlugin(Exception): def __init__(self, name, msg="Plugin {} Not Found or Failed to load."): """Exception raised when a sample asks for a plugin that can't be found diff --git a/splunk_eventgen/lib/eventgenoutput.py b/splunk_eventgen/lib/eventgenoutput.py index b8454dc6..c2b8c1c1 100644 --- a/splunk_eventgen/lib/eventgenoutput.py +++ b/splunk_eventgen/lib/eventgenoutput.py @@ -1,12 +1,13 @@ from __future__ import division + +import datetime import logging import logging.handlers -from Queue import Full -import json import time -import datetime +from Queue import Full + -#TODO: Figure out why we load plugins from here instead of the base plugin class. +# TODO: Figure out why we load plugins from here instead of the base plugin class. class Output(object): """ Base class which loads output plugins in BASE_DIR/lib/plugins/output and handles queueing @@ -22,11 +23,10 @@ def __init__(self, sample): self._setup_logging() self.output_counter = None - def __str__(self): """Only used for debugging, outputs a pretty printed representation of this output""" # Eliminate recursive going back to parent - temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key != '_c']) + # temp = dict([(key, value) for (key, value) in self.__dict__.items() if key != '_c']) # return pprint.pformat(temp) return "" @@ -55,18 +55,18 @@ def setOutputCounter(self, output_counter): def updateConfig(self, config): self.config = config - #TODO: This is where the actual output plugin is loaded, and pushed out. This should be handled way better... + # TODO: This is where the actual output plugin is loaded, and pushed out. This should be handled way better... self.outputPlugin = self.config.getPlugin('output.' + self._sample.outputMode, self._sample) def send(self, msg): """ Adds msg to the output buffer, flushes if buffer is more than MAXQUEUELENGTH """ - ts = self._sample.timestamp if self._sample.timestamp != None else self._sample.now() - self._queue.append({'_raw': msg, 'index': self._sample.index, - 'source': self._sample.source, 'sourcetype': self._sample.sourcetype, - 'host': self._sample.host, 'hostRegex': self._sample.hostRegex, - '_time': int(time.mktime(ts.timetuple()))}) + ts = self._sample.timestamp if self._sample.timestamp is not None else self._sample.now() + self._queue.append({ + '_raw': msg, 'index': self._sample.index, 'source': self._sample.source, 'sourcetype': + self._sample.sourcetype, 'host': self._sample.host, 'hostRegex': self._sample.hostRegex, '_time': int( + time.mktime(ts.timetuple()))}) if len(self._queue) >= self.MAXQUEUELENGTH: self.flush() @@ -90,7 +90,7 @@ def flush(self, endOfInterval=False): more than maxIntervalsBeforeFlush tunable. """ flushing = False - #TODO: Fix interval flushing somehow with a queue, not sure I even want to support this feature anymore. + # TODO: Fix interval flushing somehow with a queue, not sure I even want to support this feature anymore. '''if endOfInterval: logger.debugv("Sample calling flush, checking increment against maxIntervalsBeforeFlush") c.intervalsSinceFlush[self._sample.name].increment() @@ -104,7 +104,7 @@ def flush(self, endOfInterval=False): logger.debugv("maxQueueLength exceeded, flushing") flushing = True''' - #TODO: This is set this way just for the time being while I decide if we want this feature. + # TODO: This is set this way just for the time being while I decide if we want this feature. flushing = True if flushing: q = self._queue @@ -113,8 +113,10 @@ def flush(self, endOfInterval=False): outputer = self.outputPlugin(self._sample, self.output_counter) outputer.updateConfig(self.config) outputer.set_events(q) - # When an outputQueue is used, it needs to run in a single threaded nature which requires to be put back into the outputqueue so a single thread worker can execute it. - # When an outputQueue is not used, it can be ran by multiple processes or threads. Therefore, no need to put the outputer back into the Queue. Just execute it. + # When an outputQueue is used, it needs to run in a single threaded nature which requires to be put back + # into the outputqueue so a single thread worker can execute it. When an outputQueue is not used, it can be + # ran by multiple processes or threads. Therefore, no need to put the outputer back into the Queue. Just + # execute it. # if outputPlugin must be used for useOutputQueue, use outputQueue regardless of user config useOutputQueue: if self.outputPlugin.useOutputQueue or self.config.useOutputQueue: try: @@ -126,9 +128,10 @@ def flush(self, endOfInterval=False): # TODO: clean out eventsSend and bytesSent if they are not being used in config # self.config.eventsSent.add(len(tmp)) # self.config.bytesSent.add(sum(tmp)) - if self.config.splunkEmbedded and len(tmp)>0: + if self.config.splunkEmbedded and len(tmp) > 0: metrics = logging.getLogger('eventgen_metrics') - metrics.info({'timestamp': datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), - 'sample': self._sample.name, 'events': len(tmp), 'bytes': sum(tmp)}) + metrics.info({ + 'timestamp': datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), 'sample': + self._sample.name, 'events': len(tmp), 'bytes': sum(tmp)}) tmp = None outputer.run() diff --git a/splunk_eventgen/lib/eventgensamples.py b/splunk_eventgen/lib/eventgensamples.py index 7e14fb3f..269ffe43 100644 --- a/splunk_eventgen/lib/eventgensamples.py +++ b/splunk_eventgen/lib/eventgensamples.py @@ -1,14 +1,17 @@ # TODO Move config settings to plugins from __future__ import division, with_statement -import os, sys + +import copy +import csv +import datetime import logging +import os import pprint -import datetime import re -import csv -import copy +import sys import urllib + from timeparser import timeParser @@ -23,7 +26,7 @@ class Sample(object): name = None app = None filePath = None - + # Options which are all valid for a sample disabled = None spoolDir = None @@ -93,11 +96,11 @@ class Sample(object): _lastts = None _earliestParsed = None _latestParsed = None - + def __init__(self, name): self.name = name - self.tokens = [ ] - self._lockedSettings = [ ] + self.tokens = [] + self._lockedSettings = [] self.backfilldone = False self._setup_logging() @@ -106,10 +109,10 @@ def updateConfig(self, config): def __str__(self): """Only used for debugging, outputs a pretty printed representation of this sample""" - filter_list = [ 'sampleLines', 'sampleDict' ] - temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key not in filter_list ]) + filter_list = ['sampleLines', 'sampleDict'] + temp = dict([(key, value) for (key, value) in self.__dict__.items() if key not in filter_list]) return pprint.pformat(temp) - + def __repr__(self): return self.__str__() @@ -128,17 +131,17 @@ def _setup_logging(self): logger = logging.getLogger('eventgen') self.logger = logger - ## Replaces $SPLUNK_HOME w/ correct pathing + # Replaces $SPLUNK_HOME w/ correct pathing def pathParser(self, path): greatgreatgrandparentdir = os.path.dirname(os.path.dirname(self.config.grandparentdir)) sharedStorage = ['$SPLUNK_HOME/etc/apps', '$SPLUNK_HOME/etc/users/', '$SPLUNK_HOME/var/run/splunk'] - ## Replace windows os.sep w/ nix os.sep + # Replace windows os.sep w/ nix os.sep path = path.replace('\\', '/') - ## Normalize path to os.sep + # Normalize path to os.sep path = os.path.normpath(path) - ## Iterate special paths + # Iterate special paths for x in range(0, len(sharedStorage)): sharedPath = os.path.normpath(sharedStorage[x]) @@ -146,17 +149,17 @@ def pathParser(self, path): path.replace('$SPLUNK_HOME', greatgreatgrandparentdir) break - ## Split path + # Split path path = path.split(os.sep) - ## Iterate path segments + # Iterate path segments for x in range(0, len(path)): segment = path[x].lstrip('$') - ## If segement is an environment variable then replace - if os.environ.has_key(segment): + # If segement is an environment variable then replace + if segment in os.environ: path[x] = os.environ[segment] - ## Join path + # Join path path = os.sep.join(path) return path @@ -164,11 +167,11 @@ def pathParser(self, path): # 9/2/15 Adding ability to pass in a token rather than using the tokens from the sample def getTSFromEvent(self, event, passed_token=None): currentTime = None - formats = [ ] + formats = [] # JB: 2012/11/20 - Can we optimize this by only testing tokens of type = *timestamp? # JB: 2012/11/20 - Alternatively, documentation should suggest putting timestamp as token.0. - if passed_token != None: - tokens = [ passed_token ] + if passed_token is not None: + tokens = [passed_token] else: tokens = self.tokens for token in tokens: @@ -182,14 +185,16 @@ def getTSFromEvent(self, event, passed_token=None): timeString = results.group(group) # self.logger.debug("Testing '%s' as a time string against '%s'" % (timeString, timeFormat)) if timeFormat == "%s": - ts = float(timeString) if len(timeString) < 10 else float(timeString) / (10**(len(timeString)-10)) + ts = float(timeString) if len(timeString) < 10 else float(timeString) \ + / (10**(len(timeString) - 10)) # self.logger.debug("Getting time for timestamp '%s'" % ts) currentTime = datetime.datetime.fromtimestamp(ts) else: - # self.logger.debugv("Getting time for timeFormat '%s' and timeString '%s'" % (timeFormat, timeString)) - # Working around Python bug with a non thread-safe strptime. Randomly get AttributeError + # self.logger.debug("Getting time for timeFormat '%s' and timeString '%s'" % + # (timeFormat, timeString)) + # Working around Python bug with a non thread-safe strptime. Randomly get AttributeError # when calling strptime, so if we get that, try again - while currentTime == None: + while currentTime is None: try: # Checking for timezone adjustment if timeString[-5] == "+": @@ -201,11 +206,13 @@ def getTSFromEvent(self, event, passed_token=None): if type(currentTime) == datetime.datetime: break except ValueError: - self.logger.warning("Match found ('%s') but time parse failed. Timeformat '%s' Event '%s'" % (timeString, timeFormat, event)) + self.logger.warning("Match found ('%s') but time parse failed. Timeformat '%s' Event '%s'" % + (timeString, timeFormat, event)) if type(currentTime) != datetime.datetime: # Total fail - if passed_token == None: # If we're running for autotimestamp don't log error - self.logger.warning("Can't find a timestamp (using patterns '%s') in this event: '%s'." % (formats, event)) + if passed_token is None: # If we're running for autotimestamp don't log error + self.logger.warning( + "Can't find a timestamp (using patterns '%s') in this event: '%s'." % (formats, event)) raise ValueError("Can't find a timestamp (using patterns '%s') in this event: '%s'." % (formats, event)) # Check to make sure we parsed a year if currentTime.year == 1900: @@ -216,18 +223,18 @@ def getTSFromEvent(self, event, passed_token=None): # if self.timestamp == None: # self.timestamp = currentTime return currentTime - + def saveState(self): """Saves state of all integer IDs of this sample to a file so when we restart we'll pick them up""" for token in self.tokens: if token.replacementType == 'integerid': - stateFile = open(os.path.join(self.sampleDir, 'state.'+urllib.pathname2url(token.token)), 'w') + stateFile = open(os.path.join(self.sampleDir, 'state.' + urllib.pathname2url(token.token)), 'w') stateFile.write(token.replacement) stateFile.close() def now(self, utcnow=False, realnow=False): # self.logger.info("Getting time (timezone %s)" % (self.timezone)) - if not self.backfilldone and not self.backfillts == None and not realnow: + if not self.backfilldone and self.backfillts is not None and not realnow: return self.backfillts elif self.timezone.days > 0: return datetime.datetime.now() @@ -246,11 +253,12 @@ def get_backfill_time(self, current_time): if self.backfill[-2:] == 'ms': time_unit = 'ms' backfill_time = self.backfill[1:-2] - return self.get_time_difference(current_time=current_time, different_time=backfill_time, sign='-', time_unit=time_unit) + return self.get_time_difference(current_time=current_time, different_time=backfill_time, sign='-', + time_unit=time_unit) else: self.logger.error("Backfill time is not in the past.") return current_time - + def get_time_difference(self, current_time, different_time, sign='-', time_unit='ms'): if time_unit == 'ms': return current_time + (int(sign + '1') * datetime.timedelta(milliseconds=int(different_time))) @@ -263,13 +271,10 @@ def get_time_difference(self, current_time, different_time, sign='-', time_unit= elif time_unit == 'd': return current_time + (int(sign + '1') * datetime.timedelta(days=int(different_time))) - - - def earliestTime(self): # First optimization, we need only store earliest and latest # as an offset of now if they're relative times - if self._earliestParsed != None: + if self._earliestParsed is not None: earliestTime = self.now() - self._earliestParsed self.logger.debug("Using cached earliest time: %s" % earliestTime) else: @@ -280,14 +285,16 @@ def earliestTime(self): temptd = self.now(realnow=True) - tempearliest self._earliestParsed = datetime.timedelta(days=temptd.days, seconds=temptd.seconds) earliestTime = self.now() - self._earliestParsed - self.logger.debug("Calulating earliestParsed as '%s' with earliestTime as '%s' and self.sample.earliest as '%s'" % (self._earliestParsed, earliestTime, tempearliest)) + self.logger.debug( + "Calulating earliestParsed as '%s' with earliestTime as '%s' and self.sample.earliest as '%s'" % + (self._earliestParsed, earliestTime, tempearliest)) else: earliestTime = timeParser(self.earliest, timezone=self.timezone) self.logger.debug("earliestTime as absolute time '%s'" % earliestTime) return earliestTime def latestTime(self): - if self._latestParsed != None: + if self._latestParsed is not None: latestTime = self.now() - self._latestParsed self.logger.debug("Using cached latestTime: %s" % latestTime) else: @@ -298,7 +305,9 @@ def latestTime(self): temptd = self.now(realnow=True) - templatest self._latestParsed = datetime.timedelta(days=temptd.days, seconds=temptd.seconds) latestTime = self.now() - self._latestParsed - self.logger.debug("Calulating latestParsed as '%s' with latestTime as '%s' and self.sample.latest as '%s'" % (self._latestParsed, latestTime, templatest)) + self.logger.debug( + "Calulating latestParsed as '%s' with latestTime as '%s' and self.sample.latest as '%s'" % + (self._latestParsed, latestTime, templatest)) else: latestTime = timeParser(self.latest, timezone=self.timezone) self.logger.debug("latstTime as absolute time '%s'" % latestTime) @@ -316,33 +325,36 @@ def _closeSampleFile(self): self._sampleFH.close() def loadSample(self): + """ + Load sample from disk into self._sample.sampleLines and self._sample.sampleDict, using cached copy if possible + """ if not self.logger: self._setup_logging() - """Load sample from disk into self._sample.sampleLines and self._sample.sampleDict, - using cached copy if possible""" if self.sampletype == 'raw': # 5/27/12 CS Added caching of the sample file - if self.sampleDict == None: + if self.sampleDict is None: self._openSampleFile() if self.breaker == self.config.breaker: self.logger.debug("Reading raw sample '%s' in app '%s'" % (self.name, self.app)) self.sampleLines = self._sampleFH.readlines() - # 1/5/14 CS Moving to using only sampleDict and doing the breaking up into events at load time instead of on every generation + # 1/5/14 CS Moving to using only sampleDict and doing the breaking up into events at load time instead + # of on every generation else: - self.logger.debug("Non-default breaker '%s' detected for sample '%s' in app '%s'" \ - % (self.breaker, self.name, self.app) ) + self.logger.debug("Non-default breaker '%s' detected for sample '%s' in app '%s'" % + (self.breaker, self.name, self.app)) sampleData = self._sampleFH.read() - self.sampleLines = [ ] + self.sampleLines = [] - self.logger.debug("Filling array for sample '%s' in app '%s'; sampleData=%s, breaker=%s" \ - % (self.name, self.app, len(sampleData), self.breaker)) + self.logger.debug("Filling array for sample '%s' in app '%s'; sampleData=%s, breaker=%s" % + (self.name, self.app, len(sampleData), self.breaker)) try: breakerRE = re.compile(self.breaker, re.M) except: - self.logger.error("Line breaker '%s' for sample '%s' in app '%s' could not be compiled; using default breaker" \ - % (self.breaker, self.name, self.app) ) + self.logger.error( + "Line breaker '%s' for sample '%s' in app '%s' could not be compiled; using default breaker" + % (self.breaker, self.name, self.app)) self.breaker = self.config.breaker # Loop through data, finding matches of the regular expression and breaking them up into @@ -365,14 +377,17 @@ def loadSample(self): for line in self.sampleLines: if line and line[-1] != '\n': line = line + '\n' - self.sampleDict.append({ '_raw': line, 'index': self.index, 'host': self.host, 'source': self.source, 'sourcetype': self.sourcetype }) - self.logger.debug('Finished creating sampleDict & sampleLines. Len samplesLines: %d Len sampleDict: %d' % (len(self.sampleLines), len(self.sampleDict))) + self.sampleDict.append({ + '_raw': line, 'index': self.index, 'host': self.host, 'source': self.source, 'sourcetype': + self.sourcetype}) + self.logger.debug('Finished creating sampleDict & sampleLines. Len samplesLines: %d Len sampleDict: %d' + % (len(self.sampleLines), len(self.sampleDict))) elif self.sampletype == 'csv': - if self.sampleDict == None: + if self.sampleDict is None: self._openSampleFile() self.logger.debug("Reading csv sample '%s' in app '%s'" % (self.name, self.app)) - self.sampleDict = [ ] - self.sampleLines = [ ] + self.sampleDict = [] + self.sampleLines = [] # Fix to load large csv files, work with python 2.5 onwards csv.field_size_limit(sys.maxint) csvReader = csv.DictReader(self._sampleFH) @@ -395,14 +410,15 @@ def loadSample(self): else: self.logger.error("Missing _raw in line '%s'" % pprint.pformat(line)) self._closeSampleFile() - self.logger.debug("Finished creating sampleDict & sampleLines for sample '%s'. Len sampleDict: %d" % (self.name, len(self.sampleDict))) + self.logger.debug("Finished creating sampleDict & sampleLines for sample '%s'. Len sampleDict: %d" % + (self.name, len(self.sampleDict))) for i in xrange(0, len(self.sampleDict)): if len(self.sampleDict[i]['_raw']) < 1 or self.sampleDict[i]['_raw'][-1] != '\n': self.sampleDict[i]['_raw'] += '\n' def get_loaded_sample(self): - if self.sampletype != 'csv' and os.path.getsize(self.filePath) > 10000000 : + if self.sampletype != 'csv' and os.path.getsize(self.filePath) > 10000000: self._openSampleFile() return self._sampleFH elif self.sampletype == 'csv': diff --git a/splunk_eventgen/lib/eventgentimer.py b/splunk_eventgen/lib/eventgentimer.py index 6079cbaa..f37d726b 100644 --- a/splunk_eventgen/lib/eventgentimer.py +++ b/splunk_eventgen/lib/eventgentimer.py @@ -1,24 +1,26 @@ +import copy import logging import time -import copy -from timeparser import timeParserTimeMath from Queue import Full +from timeparser import timeParserTimeMath + + class Timer(object): """ - Overall governor in Eventgen. A timer is created for every sample in Eventgen. The Timer has the responsibility - for executing each sample. There are two ways the timer can execute: + Overall governor in Eventgen. A timer is created for every sample in Eventgen. The Timer has the responsibility + for executing each sample. There are two ways the timer can execute: * Queueable * Non-Queueable - For Queueable plugins, we place a work item in the generator queue. Generator workers pick up the item from the generator - queue and do work. This queueing architecture allows for parallel execution of workers. Workers then place items in the - output queue for Output workers to pick up and output. + For Queueable plugins, we place a work item in the generator queue. Generator workers pick up the item from the + generator queue and do work. This queueing architecture allows for parallel execution of workers. Workers then place + items in the output queue for Output workers to pick up and output. - However, for some generators, like the replay generator, we need to keep a single view of state of where we are in the replay. - This means we cannot generate items in parallel. This is why we also offer Non-Queueable plugins. In the case of - Non-Queueable plugins, the Timer class calls the generator method of the plugin directly, tracks the amount of time - the plugin takes to generate and sleeps the remaining interval before calling generate again. + However, for some generators, like the replay generator, we need to keep a single view of state of where we are in + the replay. This means we cannot generate items in parallel. This is why we also offer Non-Queueable plugins. In + the case of Non-Queueable plugins, the Timer class calls the generator method of the plugin directly, tracks the + amount of time the plugin takes to generate and sleeps the remaining interval before calling generate again. """ time = None countdown = None @@ -39,23 +41,24 @@ def __init__(self, time, sample=None, config=None, genqueue=None, outputqueue=No self.countdown = 0 self.executions = 0 self.interval = getattr(self.sample, "interval", config.interval) - #enable the logger + # enable the logger self._setup_logging() self.logger.debug('Initializing timer for %s' % sample.name if sample is not None else "None") # load plugins - if self.sample != None: + if self.sample is not None: rater_class = self.config.getPlugin('rater.' + self.sample.rater, self.sample) self.rater = rater_class(self.sample) self.generatorPlugin = self.config.getPlugin('generator.' + self.sample.generator, self.sample) self.outputPlugin = self.config.getPlugin('output.' + self.sample.outputMode, self.sample) if self.sample.timeMultiple < 0: self.logger.error("Invalid setting for timeMultiple: {}, value should be positive".format( - self.sample.timeMultiple)) + self.sample.timeMultiple)) elif self.sample.timeMultiple != 1: self.interval = self.sample.interval * self.sample.timeMultiple self.logger.debug("Adjusting interval {} with timeMultiple {}, new interval: {}".format( - self.sample.interval, self.sample.timeMultiple, self.interval)) - self.logger.info("Start '%s' generatorWorkers for sample '%s'" % (self.sample.config.generatorWorkers, self.sample.name)) + self.sample.interval, self.sample.timeMultiple, self.interval)) + self.logger.info( + "Start '%s' generatorWorkers for sample '%s'" % (self.sample.config.generatorWorkers, self.sample.name)) # loggers can't be pickled due to the lock object, remove them before we try to pickle anything. def __getstate__(self): @@ -114,9 +117,9 @@ def real_run(self): if self.config.stopping or self.stopping: end = True count = self.rater.rate() - #First run of the generator, see if we have any backfill work to do. + # First run of the generator, see if we have any backfill work to do. if self.countdown <= 0: - + if self.sample.backfill and not self.sample.backfilldone: realtime = self.sample.now(realnow=True) if "-" in self.sample.backfill[0]: @@ -130,9 +133,7 @@ def real_run(self): backfillnumber += char elif char != "-": backfillletter += char - backfillearliest = timeParserTimeMath(plusminus=mathsymbol, - num=backfillnumber, - unit=backfillletter, + backfillearliest = timeParserTimeMath(plusminus=mathsymbol, num=backfillnumber, unit=backfillletter, ret=realtime) while backfillearliest < realtime: et = backfillearliest @@ -140,9 +141,7 @@ def real_run(self): genPlugin = self.generatorPlugin(sample=self.sample) # need to make sure we set the queue right if we're using multiprocessing or thread modes genPlugin.updateConfig(config=self.config, outqueue=self.outputQueue) - genPlugin.updateCounts(count=count, - start_time=et, - end_time=lt) + genPlugin.updateCounts(count=count, start_time=et, end_time=lt) try: self.generatorQueue.put(genPlugin) except Full: @@ -154,10 +153,9 @@ def real_run(self): # Save previous interval count left to avoid perdayvolumegenerator drop small tasks if self.sample.generator == 'perdayvolumegenerator': count = self.rater.rate() + previous_count_left - if count < raw_event_size and count > 0: - self.logger.info( - "current interval size is {}, which is smaller than a raw event size {}. wait for the next turn.".format( - count, raw_event_size)) + if 0 < count < raw_event_size: + self.logger.info("current interval size is {}, which is smaller than a raw event size {}.". + format(count, raw_event_size) + "Wait for the next turn.") previous_count_left = count self.countdown = self.interval self.executions += 1 @@ -172,11 +170,13 @@ def real_run(self): try: if count < 1 and count != -1: - self.logger.info("There is no data to be generated in worker {0} because the count is {1}.".format(self.sample.config.generatorWorkers, count)) + self.logger.info( + "There is no data to be generated in worker {0} because the count is {1}.".format( + self.sample.config.generatorWorkers, count)) else: # Spawn workers at the beginning of job rather than wait for next interval - self.logger.info("Start '%d' generatorWorkers for sample '%s'" % ( - self.sample.config.generatorWorkers, self.sample.name)) + self.logger.info("Start '%d' generatorWorkers for sample '%s'" % + (self.sample.config.generatorWorkers, self.sample.name)) for worker_id in range(self.config.generatorWorkers): # self.generatorPlugin is only an instance, now we need a real plugin. Make a copy of # of the sample in case another generator corrupts it. @@ -186,13 +186,14 @@ def real_run(self): genPlugin = self.generatorPlugin(sample=copy_sample) # Adjust queue for threading mode genPlugin.updateConfig(config=self.config, outqueue=self.outputQueue) - genPlugin.updateCounts(count=count, - start_time=et, - end_time=lt) + genPlugin.updateCounts(count=count, start_time=et, end_time=lt) try: self.generatorQueue.put(genPlugin) - self.logger.info("Worker# {0}: Put {1} MB of events in queue for sample '{2}' with et '{3}' and lt '{4}'".format(worker_id, round((count / 1024.0 / 1024), 4), self.sample.name, et, lt)) + self.logger.info(("Worker# {0}: Put {1} MB of events in queue for sample '{2}'" + + "with et '{3}' and lt '{4}'").format( + worker_id, round((count / 1024.0 / 1024), 4), + self.sample.name, et, lt)) except Full: self.logger.warning("Generator Queue Full. Skipping current generation.") except Exception as e: @@ -212,15 +213,16 @@ def real_run(self): # timer thread if not self.endts: if self.executions >= int(self.end): - self.logger.info("End executions %d reached, ending generation of sample '%s'" % (int(self.end), self.sample.name)) + self.logger.info("End executions %d reached, ending generation of sample '%s'" % (int( + self.end), self.sample.name)) self.stopping = True end = True elif lt >= self.endts: - self.logger.info("End Time '%s' reached, ending generation of sample '%s'" % (self.sample.endts, self.sample.name)) + self.logger.info("End Time '%s' reached, ending generation of sample '%s'" % (self.sample.endts, + self.sample.name)) self.stopping = True end = True else: time.sleep(self.time) self.countdown -= self.time - diff --git a/splunk_eventgen/lib/eventgentimestamp.py b/splunk_eventgen/lib/eventgentimestamp.py index f42fb4d9..d1dca9e5 100644 --- a/splunk_eventgen/lib/eventgentimestamp.py +++ b/splunk_eventgen/lib/eventgentimestamp.py @@ -1,9 +1,9 @@ import datetime -import time import random +import time -class EventgenTimestamp(object): +class EventgenTimestamp(object): @staticmethod def get_random_timestamp(earliest, latest): if type(earliest) != datetime.datetime or type(latest) != datetime.datetime: @@ -23,7 +23,8 @@ def get_random_timestamp_backfill(earliest, latest, sample_earliest, sample_late earliest and latest timestamp gets generated with an interval sample_earliest and sample_latest are the user config key values from eventgen.conf we are using earliest as a pivot time and creating a random variance using sample_earliest and sample_latest. - in this way, we respect an interval passed in by a user and use user input earliest and latest to create a random variance + in this way, we respect an interval passed in by a user and use user input earliest and latest to create a + random variance. ''' if type(earliest) != datetime.datetime or type(latest) != datetime.datetime: raise Exception("Earliest {0} or latest {1} arguments are not datetime objects".format(earliest, latest)) @@ -50,7 +51,8 @@ def get_sequential_timestamp(earliest, latest, slot, total_slot): latest_in_epoch = time.mktime(latest.timetuple()) if earliest_in_epoch > latest_in_epoch: raise Exception("Latest time is earlier than earliest time.") - return datetime.datetime.fromtimestamp(earliest_in_epoch + (latest_in_epoch-earliest_in_epoch)*slot/total_slot) + return datetime.datetime.fromtimestamp(earliest_in_epoch + + (latest_in_epoch - earliest_in_epoch) * slot / total_slot) @staticmethod def _convert_time_difference_to_seconds(time_difference): diff --git a/splunk_eventgen/lib/eventgentoken.py b/splunk_eventgen/lib/eventgentoken.py index 9e883e05..f409d90f 100644 --- a/splunk_eventgen/lib/eventgentoken.py +++ b/splunk_eventgen/lib/eventgentoken.py @@ -1,26 +1,29 @@ -# TODO Handle timestamp generation for modular input output where we set sample.timestamp properly when we do a timestamp replacement +# TODO: Handle timestamp generation for modinput and set sample.timestamp properly for timestamp replacement from __future__ import division, with_statement -import os + +import datetime +import json import logging +import os import pprint import random -import datetime, time import re -import json -import copy -from timeparser import timeParser, timeDelta2secs +import time import urllib import uuid +from timeparser import timeDelta2secs + + class Token(object): """Contains data and methods for replacing a token in a given sample""" token = None replacementType = None replacement = None sample = None - mvhash = { } - + mvhash = {} + _replaytd = None _lastts = None _tokenfile = None @@ -35,24 +38,18 @@ class Token(object): _stringMatch = None _listMatch = None _tokenfilecounter = 0 - + def __init__(self, sample=None): - + # Logger already setup by config, just get an instance self._setup_logging() - - if sample == None: - name = "None" - else: - name = sample.name - self._earliestTime = (None, None) self._latestTime = (None, None) - + def __str__(self): """Only used for debugging, outputs a pretty printed representation of this token""" # Eliminate recursive going back to parent - temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key != 'sample' ]) + temp = dict([(key, value) for (key, value) in self.__dict__.items() if key != 'sample']) return pprint.pformat(temp) def __repr__(self): @@ -75,11 +72,11 @@ def _setup_logging(self): def _match(self, event): """Executes regular expression match and returns the re.Match object""" return re.match(self.token, event) - + def _search(self, event): """Executes regular expression search and returns the re.Match object""" return re.search(self.token, event) - + def _finditer(self, event): """Executes regular expression finditer and returns the re.Match object""" return re.finditer(self.token, event) @@ -87,7 +84,7 @@ def _finditer(self, event): def _findall(self, event): """Executes regular expression finditer and returns the re.Match object""" return re.findall(self.token, event) - + def replace(self, event, et=None, lt=None, s=None, pivot_timestamp=None): """Replaces all instances of this token in provided event and returns event""" if not getattr(self, 'logger', None): @@ -96,10 +93,11 @@ def replace(self, event, et=None, lt=None, s=None, pivot_timestamp=None): tokenMatch = list(self._finditer(event)) if len(tokenMatch) > 0: - replacement = self._getReplacement(event[tokenMatch[0].start(0):tokenMatch[0].end(0)], et, lt, s, pivot_timestamp=pivot_timestamp) + replacement = self._getReplacement(event[tokenMatch[0].start(0):tokenMatch[0].end(0)], et, lt, s, + pivot_timestamp=pivot_timestamp) if replacement is not None or self.replacementType == 'replaytimestamp': # logger.debug("Replacement: '%s'" % replacement) - ## Iterate matches + # Iterate matches for match in tokenMatch: # logger.debug("Match: %s" % (match)) try: @@ -131,8 +129,8 @@ def replace(self, event, et=None, lt=None, s=None, pivot_timestamp=None): self._replaytd = None self._lastts = None return event - - def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, pivot_timestamp=None): + + def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, pivot_timestamp=None): if self.replacementType == 'static': return self.replacement # This logic is done in replay.py @@ -141,86 +139,89 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, elif self.replacementType == 'timestamp': if s.earliest and s.latest: if earliestTime and latestTime: - if latestTime>=earliestTime: + if latestTime >= earliestTime: if pivot_timestamp: replacementTime = pivot_timestamp - elif s.timestamp == None: + elif s.timestamp is None: minDelta = 0 - ## Compute timeDelta as total_seconds + # Compute timeDelta as total_seconds td = latestTime - earliestTime if not type(td) == float: maxDelta = timeDelta2secs(td) else: maxDelta = td - ## Get random timeDelta - randomDelta = datetime.timedelta(seconds=random.randint(minDelta, maxDelta), microseconds=random.randint(0, latestTime.microsecond if latestTime.microsecond > 0 else 999999)) + # Get random timeDelta + randomDelta = datetime.timedelta( + seconds=random.randint(minDelta, maxDelta), microseconds=random.randint( + 0, latestTime.microsecond if latestTime.microsecond > 0 else 999999)) - ## Compute replacmentTime + # Compute replacmentTime replacementTime = latestTime - randomDelta s.timestamp = replacementTime else: replacementTime = s.timestamp - # logger.debug("Generating timestamp for sample '%s' with randomDelta %s, minDelta %s, maxDelta %s, earliestTime %s, latestTime %s, earliest: %s, latest: %s" % (s.name, randomDelta, minDelta, maxDelta, earliestTime, latestTime, s.earliest, s.latest)) - - replacement = self.replacement.replace('%s', str(round(time.mktime(replacementTime.timetuple()))).rstrip('0').rstrip('.')) + replacement = self.replacement.replace( + '%s', + str(round(time.mktime(replacementTime.timetuple()))).rstrip('0').rstrip('.')) replacementTime = replacementTime.strftime(replacement) - ## replacementTime == replacement for invalid strptime specifiers + # replacementTime == replacement for invalid strptime specifiers if replacementTime != self.replacement.replace('%', ''): return replacementTime else: - self.logger.error("Invalid strptime specifier '%s' detected; will not replace" \ - % (self.replacement) ) + self.logger.error( + "Invalid strptime specifier '%s' detected; will not replace" % (self.replacement)) return old - ## earliestTime/latestTime not proper + # earliestTime/latestTime not proper else: - self.logger.error("Earliest specifier '%s', value '%s' is greater than latest specifier '%s', value '%s' for sample '%s'; will not replace" \ - % (s.earliest, earliestTime, s.latest, latestTime, s.name) ) + self.logger.error(("Earliest specifier '%s', value '%s' is greater than latest specifier '%s'" + + "value '%s' for sample '%s'; will not replace") % + (s.earliest, earliestTime, s.latest, latestTime, s.name)) return old - ## earliest/latest not proper + # earliest/latest not proper else: self.logger.error('Earliest or latest specifier were not set; will not replace') return old elif self.replacementType in ('random', 'rated'): - ## Validations: - if self._integerMatch != None: + # Validations: + if self._integerMatch is not None: integerMatch = self._integerMatch else: integerRE = re.compile('integer\[([-]?\d+):([-]?\d+)\]', re.I) integerMatch = integerRE.match(self.replacement) self._integerMatch = integerMatch - - if self._floatMatch != None: + + if self._floatMatch is not None: floatMatch = self._floatMatch else: floatRE = re.compile('float\[(-?\d+|\d+\.(\d+)):(-?\d+|\d+\.(\d+))\]', re.I) floatMatch = floatRE.match(self.replacement) self._floatMatch = floatMatch - if self._stringMatch != None: + if self._stringMatch is not None: stringMatch = self._stringMatch else: stringRE = re.compile('string\((\d+)\)', re.I) stringMatch = stringRE.match(self.replacement) self._stringMatch = stringMatch - if self._hexMatch != None: + if self._hexMatch is not None: hexMatch = self._hexMatch - else: + else: hexRE = re.compile('hex\((\d+)\)', re.I) hexMatch = hexRE.match(self.replacement) self._hexMatch = hexMatch - if self._listMatch != None: + if self._listMatch is not None: listMatch = self._listMatch else: listRE = re.compile('list(\[[^\]]+\])', re.I) listMatch = listRE.match(self.replacement) self._listMatch = listMatch - ## Valid replacements: ipv4 | ipv6 | integer[:] | string() + # Valid replacements: ipv4 | ipv6 | integer[:] | string() if self.replacement.lower() == 'ipv4': x = 0 replacement = '' @@ -245,7 +246,7 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, x = 0 replacement = '' - ## Give me 6 blocks of 2 hex + # Give me 6 blocks of 2 hex while x < 6: y = 0 while y < 2: @@ -271,7 +272,7 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, rateFactor *= s.hourOfDayRate[str(s.now())] except KeyError: import traceback - stack = traceback.format_exc() + stack = traceback.format_exc() self.logger.error("Hour of day rate failed for token %s. Stacktrace %s" % stack) if type(s.dayOfWeekRate) == dict: try: @@ -283,13 +284,14 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, rateFactor *= s.dayOfWeekRate[str(weekday)] except KeyError: import traceback - stack = traceback.format_exc() + stack = traceback.format_exc() self.logger.error("Day of week rate failed. Stacktrace %s" % stack) replacementInt = int(round(replacementInt * rateFactor, 0)) replacement = str(replacementInt) return replacement else: - self.logger.error("Start integer %s greater than end integer %s; will not replace" % (startInt, endInt) ) + self.logger.error( + "Start integer %s greater than end integer %s; will not replace" % (startInt, endInt)) return old elif floatMatch: try: @@ -301,7 +303,7 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, significance = len(floatMatch.group(2)) if endFloat >= startFloat: - floatret = round(random.uniform(startFloat,endFloat), significance) + floatret = round(random.uniform(startFloat, endFloat), significance) if self.replacementType == 'rated': rateFactor = 1.0 now = s.now() @@ -310,7 +312,7 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, rateFactor *= s.hourOfDayRate[str(now.hour)] except KeyError: import traceback - stack = traceback.format_exc() + stack = traceback.format_exc() self.logger.error("Hour of day rate failed for token %s. Stacktrace %s" % stack) if type(s.dayOfWeekRate) == dict: try: @@ -322,13 +324,14 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, rateFactor *= s.dayOfWeekRate[str(weekday)] except KeyError: import traceback - stack = traceback.format_exc() + stack = traceback.format_exc() self.logger.error("Day of week rate failed. Stacktrace %s" % stack) floatret = round(floatret * rateFactor, significance) floatret = str(floatret) return floatret else: - self.logger.error("Start float %s greater than end float %s; will not replace" % (startFloat, endFloat)) + self.logger.error( + "Start float %s greater than end float %s; will not replace" % (startFloat, endFloat)) return old except ValueError: self.logger.error("Could not parse float[%s:%s]" % (floatMatch.group(1), floatMatch.group(4))) @@ -340,14 +343,16 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, elif strLength > 0: replacement = '' while len(replacement) < strLength: - ## Generate a random ASCII between dec 33->126 + # Generate a random ASCII between dec 33->126 replacement += chr(random.randint(33, 126)) - ## Practice safe strings + # Practice safe strings replacement = re.sub('%[0-9a-fA-F]+', '', urllib.quote(replacement)) - + return replacement else: - self.logger.error("Length specifier %s for string replacement must be greater than 0; will not replace" % (strLength) ) + self.logger.error( + "Length specifier %s for string replacement must be greater than 0; will not replace" % + (strLength)) return old elif hexMatch: strLength = int(hexMatch.group(1)) @@ -367,29 +372,32 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, return random.choice(value) else: - self.logger.error("Unknown replacement value '%s' for replacementType '%s'; will not replace" % (self.replacement, self.replacementType) ) + self.logger.error("Unknown replacement value '%s' for replacementType '%s'; will not replace" % + (self.replacement, self.replacementType)) return old elif self.replacementType in ('file', 'mvfile', 'seqfile'): - if self._replacementFile != None: + if self._replacementFile is not None: replacementFile = self._replacementFile replacementColumn = self._replacementColumn else: try: paths = self.replacement.split(':') - if(len(paths) == 1): + if (len(paths) == 1): replacementColumn = 0 else: - try: # When it's not a mvfile, there's no number on the end: + try: # When it's not a mvfile, there's no number on the end: replacementColumn = int(paths[-1]) except (ValueError): replacementColumn = 0 - if(replacementColumn > 0): + if (replacementColumn > 0): # This supports having a drive-letter colon replacementFile = s.pathParser(":".join(paths[0:-1])) else: replacementFile = s.pathParser(self.replacement) - except ValueError, e: - self.logger.error("Replacement string '%s' improperly formatted. Should be /path/to/file or /path/to/file:column" % (self.replacement)) + except ValueError: + self.logger.error( + "Replacement string '%s' improperly formatted. Should be /path/to/file or /path/to/file:column" + % self.replacement) return old self._replacementFile = replacementFile self._replacementColumn = replacementColumn @@ -399,18 +407,20 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, # return the same random pick on every iteration if replacementColumn > 0 and replacementFile in self.mvhash: if replacementColumn > len(self.mvhash[replacementFile]): - self.logger.error("Index for column '%s' in replacement file '%s' is out of bounds" % (replacementColumn, replacementFile)) + self.logger.error("Index for column '%s' in replacement file '%s' is out of bounds" % + (replacementColumn, replacementFile)) return old else: # self.logger.debug("Returning mvhash: %s" % self.mvhash[replacementFile][replacementColumn-1]) - return self.mvhash[replacementFile][replacementColumn-1] + return self.mvhash[replacementFile][replacementColumn - 1] else: # Adding caching of the token file to avoid reading it every iteration - if self._tokenfile != None: + if self._tokenfile is not None: replacementLines = self._tokenfile - ## Otherwise, lets read the file and build our cached results, pick a result and return it + # Otherwise, lets read the file and build our cached results, pick a result and return it else: - # self.logger.debug("replacementFile: %s replacementColumn: %s" % (replacementFile, replacementColumn)) + # self.logger.debug("replacementFile: %s replacementColumn: %s" % + # (replacementFile, replacementColumn)) replacementFile = os.path.abspath(replacementFile) self.logger.debug("Normalized replacement file %s" % replacementFile) if os.path.exists(replacementFile) and os.path.isfile(replacementFile): @@ -419,7 +429,7 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, replacementFH.close() if len(replacementLines) == 0: - self.logger.error("Replacement file '%s' is empty; will not replace" % (replacementFile) ) + self.logger.error("Replacement file '%s' is empty; will not replace" % (replacementFile)) return old else: self._tokenfile = replacementLines @@ -432,16 +442,17 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, self._tokenfilecounter += 1 else: # pick value randomly from replacement file - replacement = replacementLines[random.randint(0, len(replacementLines)-1)].strip() + replacement = replacementLines[random.randint(0, len(replacementLines) - 1)].strip() if replacementColumn > 0: self.mvhash[replacementFile] = replacement.split(',') if replacementColumn > len(self.mvhash[replacementFile]): - self.logger.error("Index for column '%s' in replacement file '%s' is out of bounds" % (replacementColumn, replacementFile)) + self.logger.error("Index for column '%s' in replacement file '%s' is out of bounds" % + (replacementColumn, replacementFile)) return old else: - return self.mvhash[replacementFile][replacementColumn-1] + return self.mvhash[replacementFile][replacementColumn - 1] else: return replacement elif self.replacementType == 'integerid': @@ -450,5 +461,5 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, return temp else: - self.logger.error("Unknown replacementType '%s'; will not replace" % (self.replacementType) ) - return old \ No newline at end of file + self.logger.error("Unknown replacementType '%s'; will not replace" % self.replacementType) + return old diff --git a/splunk_eventgen/lib/generatorplugin.py b/splunk_eventgen/lib/generatorplugin.py index ebca83a2..f8364b2d 100644 --- a/splunk_eventgen/lib/generatorplugin.py +++ b/splunk_eventgen/lib/generatorplugin.py @@ -1,15 +1,20 @@ from __future__ import division + +import datetime import logging import logging.handlers import pprint -import datetime -from timeparser import timeParser -import httplib2, urllib +import time +import urllib from xml.dom import minidom from xml.parsers.expat import ExpatError + +import httplib2 + from eventgenoutput import Output from eventgentimestamp import EventgenTimestamp -import time +from timeparser import timeParser + class GeneratorPlugin(object): sampleLines = None @@ -22,7 +27,7 @@ def __init__(self, sample): def __str__(self): """Only used for debugging, outputs a pretty printed representation of this output""" # Eliminate recursive going back to parent - temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key != '_c']) + # temp = dict([(key, value) for (key, value) in self.__dict__.items() if key != '_c']) # return pprint.pformat(temp) return "" @@ -56,8 +61,8 @@ def build_events(self, eventsDict, startTime, earliest, latest, ignore_tokens=Fa timeDiffFrac = "%d.%06d" % (timeDiff.seconds, timeDiff.microseconds) self.logger.debug("Interval complete, flushing feed") self._out.flush(endOfInterval=True) - self.logger.debug("Generation of sample '%s' in app '%s' completed in %s seconds." % ( - self._sample.name, self._sample.app, timeDiffFrac)) + self.logger.debug("Generation of sample '%s' in app '%s' completed in %s seconds." % + (self._sample.name, self._sample.app, timeDiffFrac)) except Exception as e: self.logger.exception("Exception {} happened.".format(type(e))) raise e @@ -81,60 +86,66 @@ def updateConfig(self, config, outqueue): def updateCounts(self, sample=None, count=None, start_time=None, end_time=None): if sample: - self._sample=sample + self._sample = sample self.count = count self.start_time = start_time self.end_time = end_time def setOutputMetadata(self, event): - # self.logger.debug("Sample Index: %s Host: %s Source: %s Sourcetype: %s" % (self.index, self.host, self.source, self.sourcetype)) - # self.logger.debug("Event Index: %s Host: %s Source: %s Sourcetype: %s" % (sampleDict[x]['index'], sampleDict[x]['host'], sampleDict[x]['source'], sampleDict[x]['sourcetype'])) - if self._sample.sampletype == 'csv' and (event['index'] != self._sample.index or - event['host'] != self._sample.host or - event['source'] != self._sample.source or - event['sourcetype'] != self._sample.sourcetype): + # self.logger.debug("Sample Index: %s Host: %s Source: %s Sourcetype: %s" % + # (self.index, self.host, self.source, self.sourcetype)) + # self.logger.debug("Event Index: %s Host: %s Source: %s Sourcetype: %s" % + # (sampleDict[x]['index'], sampleDict[x]['host'], sampleDict[x]['source'], + # sampleDict[x]['sourcetype'])) + if self._sample.sampletype == 'csv' and (event['index'] != self._sample.index + or event['host'] != self._sample.host + or event['source'] != self._sample.source + or event['sourcetype'] != self._sample.sourcetype): self._sample.index = event['index'] self._sample.host = event['host'] # Allow randomizing the host: - if(self._sample.hostToken): + if self._sample.hostToken: self.host = self._sample.hostToken.replace(self.host) self._sample.source = event['source'] self._sample.sourcetype = event['sourcetype'] - self.logger.debugv("Sampletype CSV. Setting CSV parameters. index: '%s' host: '%s' source: '%s' sourcetype: '%s'" \ - % (self._sample.index, self._sample.host, self._sample.source, self._sample.sourcetype)) + self.logger.debug("Setting CSV parameters. index: '%s' host: '%s' source: '%s' sourcetype: '%s'" % + (self._sample.index, self._sample.host, self._sample.source, self._sample.sourcetype)) def setupBackfill(self): - """Called by non-queueable plugins or by the timer to setup backfill times per config or based on a Splunk Search""" + """ + Called by non-queueable plugins or by the timer to setup backfill times per config or based on a Splunk Search + """ s = self._sample - if s.backfill != None: + if s.backfill is not None: try: s.backfillts = timeParser(s.backfill, timezone=s.timezone) - self.logger.info("Setting up backfill of %s (%s)" % (s.backfill,s.backfillts)) + self.logger.info("Setting up backfill of %s (%s)" % (s.backfill, s.backfillts)) except Exception as ex: self.logger.error("Failed to parse backfill '%s': %s" % (s.backfill, ex)) raise - if s.backfillSearch != None: - if s.backfillSearchUrl == None: + if s.backfillSearch is not None: + if s.backfillSearchUrl is None: try: - s.backfillSearchUrl = c.getSplunkUrl(s)[0] + s.backfillSearchUrl = c.getSplunkUrl(s)[0] # noqa, we update c in the globals() dict except ValueError: - self.logger.error("Backfill Search URL not specified for sample '%s', not running backfill search" % s.name) + self.logger.error( + "Backfill Search URL not specified for sample '%s', not running backfill search" % s.name) if not s.backfillSearch.startswith('search'): s.backfillSearch = 'search ' + s.backfillSearch s.backfillSearch += '| head 1 | table _time' - if s.backfillSearchUrl != None: - self.logger.debug("Searching Splunk URL '%s/services/search/jobs' with search '%s' with sessionKey '%s'" % (s.backfillSearchUrl, s.backfillSearch, s.sessionKey)) - + if s.backfillSearchUrl is not None: + self.logger.debug( + "Searching Splunk URL '%s/services/search/jobs' with search '%s' with sessionKey '%s'" % + (s.backfillSearchUrl, s.backfillSearch, s.sessionKey)) + results = httplib2.Http(disable_ssl_certificate_validation=True).request( - s.backfillSearchUrl + '/services/search/jobs', - 'POST', headers={'Authorization': 'Splunk %s' % s.sessionKey}, - body=urllib.urlencode({'search': s.backfillSearch, - 'earliest_time': s.backfill, - 'exec_mode': 'oneshot'}))[1] + s.backfillSearchUrl + '/services/search/jobs', 'POST', headers={ + 'Authorization': 'Splunk %s' % s.sessionKey}, body=urllib.urlencode({ + 'search': s.backfillSearch, 'earliest_time': s.backfill, 'exec_mode': 'oneshot'}))[1] try: temptime = minidom.parseString(results).getElementsByTagName('text')[0].childNodes[0].nodeValue # self.logger.debug("Time returned from backfill search: %s" % temptime) @@ -145,8 +156,9 @@ def setupBackfill(self): temptime = temptime.split('+')[0] temptime = '-'.join(temptime.split('-')[0:3]) s.backfillts = datetime.datetime.strptime(temptime, '%Y-%m-%dT%H:%M:%S.%f') - self.logger.debug("Backfill search results: '%s' value: '%s' time: '%s'" % (pprint.pformat(results), temptime, s.backfillts)) - except (ExpatError, IndexError): + self.logger.debug("Backfill search results: '%s' value: '%s' time: '%s'" % + (pprint.pformat(results), temptime, s.backfillts)) + except (ExpatError, IndexError): pass if s.end is not None: @@ -157,13 +169,14 @@ def setupBackfill(self): parsed = True except ValueError: self.logger.debug("Failed to parse end '%s' for sample '%s', treating as end time" % (s.end, s.name)) - - if not parsed: + + if not parsed: try: s.endts = timeParser(s.end, timezone=s.timezone) self.logger.info("Ending generation at %s (%s)" % (s.end, s.endts)) except Exception as ex: - self.logger.error("Failed to parse end '%s' for sample '%s', treating as number of executions" % (s.end, s.name)) + self.logger.error( + "Failed to parse end '%s' for sample '%s', treating as number of executions" % (s.end, s.name)) raise def run(self, output_counter=None): @@ -189,7 +202,7 @@ def replace_tokens(self, eventsDict, earliest, latest, ignore_tokens=False): mvhash = {} host = targetevent['host'] if hasattr(self._sample, "sequentialTimestamp") and self._sample.sequentialTimestamp and \ - self._sample.generator != 'perdayvolumegenerator': + self._sample.generator != 'perdayvolumegenerator': pivot_timestamp = EventgenTimestamp.get_sequential_timestamp(earliest, latest, eventcount, total_count) else: pivot_timestamp = EventgenTimestamp.get_random_timestamp(earliest, latest) @@ -212,14 +225,10 @@ def replace_tokens(self, eventsDict, earliest, latest, ignore_tokens=False): time_val = int(time.mktime(pivot_timestamp.timetuple())) except Exception: time_val = int(time.mktime(self._sample.now().timetuple())) - l = {'_raw': event, - 'index': targetevent['index'], - 'host': host, - 'hostRegex': self._sample.hostRegex, - 'source': targetevent['source'], - 'sourcetype': targetevent['sourcetype'], - '_time': time_val} - send_events.append(l) + temp_event = { + '_raw': event, 'index': targetevent['index'], 'host': host, 'hostRegex': self._sample.hostRegex, + 'source': targetevent['source'], 'sourcetype': targetevent['sourcetype'], '_time': time_val} + send_events.append(temp_event) return send_events diff --git a/splunk_eventgen/lib/logutils_src/doc/conf.py b/splunk_eventgen/lib/logutils_src/doc/conf.py index 2f89fe54..17a8499c 100644 --- a/splunk_eventgen/lib/logutils_src/doc/conf.py +++ b/splunk_eventgen/lib/logutils_src/doc/conf.py @@ -14,7 +14,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import os +import sys # If your extensions (or modules documented by autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -35,7 +36,7 @@ source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8' +# source_encoding = 'utf-8' # The master toctree document. master_doc = 'index' @@ -55,39 +56,38 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. -#unused_docs = [] +# unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' - # Options for HTML output # ----------------------- @@ -98,19 +98,19 @@ # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -119,38 +119,38 @@ # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_use_modindex = True +# html_use_modindex = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, the reST sources are included in the HTML build as _sources/. -#html_copy_source = True +# html_copy_source = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = '' +# html_file_suffix = '' html_theme = os.environ.get('DOCS_THEME', 'alabaster') html_theme_path = ['themes'] @@ -158,42 +158,37 @@ # Output file base name for HTML help builder. htmlhelp_basename = 'Logutilsdoc' - # Options for LaTeX output # ------------------------ # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Logutils.tex', ur'Logutils Documentation', - ur'Vinay Sajip', 'manual'), -] + ('index', 'Logutils.tex', ur'Logutils Documentation', ur'Vinay Sajip', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_use_modindex = True - +# latex_use_modindex = True # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'http://docs.python.org/dev': None, -} + 'http://docs.python.org/dev': None, } diff --git a/splunk_eventgen/lib/logutils_src/logutils/__init__.py b/splunk_eventgen/lib/logutils_src/logutils/__init__.py index 963c8df4..2811a987 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/__init__.py +++ b/splunk_eventgen/lib/logutils_src/logutils/__init__.py @@ -15,6 +15,7 @@ __version__ = '0.3.4.1' + class NullHandler(logging.Handler): """ This handler does nothing. It's intended to be used to avoid the @@ -48,6 +49,7 @@ def createLock(self): """ self.lock = None + class PercentStyle(object): default_format = '%(message)s' @@ -62,6 +64,7 @@ def usesTime(self): def format(self, record): return self._fmt % record.__dict__ + class StrFormatStyle(PercentStyle): default_format = '{message}' asctime_format = '{asctime}' @@ -85,11 +88,9 @@ def usesTime(self): def format(self, record): return self._tpl.substitute(**record.__dict__) -_STYLES = { - '%': PercentStyle, - '{': StrFormatStyle, - '$': StringTemplateStyle -} + +_STYLES = {'%': PercentStyle, '{': StrFormatStyle, '$': StringTemplateStyle} + class Formatter(logging.Formatter): """ @@ -97,6 +98,7 @@ class Formatter(logging.Formatter): 3.2 Formatter behaviour with respect to allowing %-, {} or $- formatting. """ + def __init__(self, fmt=None, datefmt=None, style='%'): """ Initialize the formatter with specified format strings. @@ -110,8 +112,7 @@ def __init__(self, fmt=None, datefmt=None, style='%'): :class:`string.Template` formatting in your format string. """ if style not in _STYLES: - raise ValueError('Style must be one of: %s' % ','.join( - _STYLES.keys())) + raise ValueError('Style must be one of: %s' % ','.join(_STYLES.keys())) self._style = _STYLES[style](fmt) self._fmt = self._style._fmt self.datefmt = datefmt @@ -166,6 +167,7 @@ def __str__(self): self.str = self.fmt.format(*self.args, **self.kwargs) return self.str + class DollarMessage(object): def __init__(self, fmt, **kwargs): self.fmt = fmt diff --git a/splunk_eventgen/lib/logutils_src/logutils/adapter.py b/splunk_eventgen/lib/logutils_src/logutils/adapter.py index 92706c0f..220c188d 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/adapter.py +++ b/splunk_eventgen/lib/logutils_src/logutils/adapter.py @@ -2,8 +2,10 @@ # Copyright (C) 2010-2017 Vinay Sajip. See LICENSE.txt for details. # import logging + import logutils + class LoggerAdapter(object): """ An adapter for loggers which makes it easier to specify contextual diff --git a/splunk_eventgen/lib/logutils_src/logutils/colorize.py b/splunk_eventgen/lib/logutils_src/logutils/colorize.py index f95c0366..8f375e5d 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/colorize.py +++ b/splunk_eventgen/lib/logutils_src/logutils/colorize.py @@ -10,6 +10,7 @@ except NameError: unicode = None + class ColorizingStreamHandler(logging.StreamHandler): """ A stream handler which supports colorizing of console streams @@ -28,18 +29,16 @@ class ColorizingStreamHandler(logging.StreamHandler): 'blue': 4, 'magenta': 5, 'cyan': 6, - 'white': 7, - } + 'white': 7, } - #levels to (background, foreground, bold/intense) + # levels to (background, foreground, bold/intense) if os.name == 'nt': level_map = { logging.DEBUG: (None, 'blue', True), logging.INFO: (None, 'white', False), logging.WARNING: (None, 'yellow', True), logging.ERROR: (None, 'red', True), - logging.CRITICAL: ('red', 'white', True), - } + logging.CRITICAL: ('red', 'white', True), } else: "Maps levels to colour/intensity settings." level_map = { @@ -47,8 +46,7 @@ class ColorizingStreamHandler(logging.StreamHandler): logging.INFO: (None, 'black', False), logging.WARNING: (None, 'yellow', False), logging.ERROR: (None, 'red', False), - logging.CRITICAL: ('red', 'white', True), - } + logging.CRITICAL: ('red', 'white', True), } csi = '\x1b[' reset = '\x1b[0m' @@ -78,6 +76,7 @@ def emit(self, record): self.handleError(record) if os.name != 'nt': + def output_colorized(self, message): """ Output a colorized message. @@ -98,14 +97,14 @@ def output_colorized(self, message): ansi_esc = re.compile(r'\x1b\[((?:\d+)(?:;(?:\d+))*)m') nt_color_map = { - 0: 0x00, # black - 1: 0x04, # red - 2: 0x02, # green - 3: 0x06, # yellow - 4: 0x01, # blue - 5: 0x05, # magenta - 6: 0x03, # cyan - 7: 0x07, # white + 0: 0x00, # black + 1: 0x04, # red + 2: 0x02, # green + 3: 0x06, # yellow + 4: 0x01, # blue + 5: 0x05, # magenta + 6: 0x03, # cyan + 7: 0x07, # white } def output_colorized(self, message): @@ -128,7 +127,7 @@ def output_colorized(self, message): fd = getattr(self.stream, 'fileno', None) if fd is not None: fd = fd() - if fd in (1, 2): # stdout or stderr + if fd in (1, 2): # stdout or stderr h = ctypes.windll.kernel32.GetStdHandle(-10 - fd) while parts: text = parts.pop(0) @@ -145,11 +144,11 @@ def output_colorized(self, message): elif 30 <= p <= 37: color |= self.nt_color_map[p - 30] elif p == 1: - color |= 0x08 # foreground intensity on - elif p == 0: # reset to default color + color |= 0x08 # foreground intensity on + elif p == 0: # reset to default color color = 0x07 else: - pass # error condition ignored + pass # error condition ignored ctypes.windll.kernel32.SetConsoleTextAttribute(h, color) def colorize(self, message, record): @@ -173,8 +172,7 @@ def colorize(self, message, record): if bold: params.append('1') if params: - message = ''.join((self.csi, ';'.join(params), - 'm', message, self.reset)) + message = ''.join((self.csi, ';'.join(params), 'm', message, self.reset)) return message def format(self, record): diff --git a/splunk_eventgen/lib/logutils_src/logutils/dictconfig.py b/splunk_eventgen/lib/logutils_src/logutils/dictconfig.py index c774552e..26b8886e 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/dictconfig.py +++ b/splunk_eventgen/lib/logutils_src/logutils/dictconfig.py @@ -4,7 +4,6 @@ import logging.handlers import re import sys -import types try: basestring @@ -17,18 +16,21 @@ IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I) + def valid_ident(s): m = IDENTIFIER.match(s) if not m: raise ValueError('Not a valid Python identifier: %r' % s) return True + # # This function is defined in logging only in recent versions of Python # try: from logging import _checkLevel except ImportError: + def _checkLevel(level): if isinstance(level, int): rv = level @@ -41,10 +43,10 @@ def _checkLevel(level): raise ValueError('Unknown level: %r' % level) rv = levelnames[level] else: - raise TypeError('Level not an integer or a ' - 'valid string: %r' % level) + raise TypeError('Level not an integer or a ' 'valid string: %r' % level) return rv + # The ConvertingXXX classes are wrappers around standard Python containers, # and they serve to convert any suitable values in the container. The # conversion converts base dicts, lists and tuples to their wrapped @@ -54,17 +56,17 @@ def _checkLevel(level): # Each wrapper should have a configurator attribute holding the actual # configurator to use for conversion. + class ConvertingDict(dict): """A converting dictionary wrapper.""" def __getitem__(self, key): value = dict.__getitem__(self, key) result = self.configurator.convert(value) - #If the converted value is different, save for next time + # If the converted value is different, save for next time if value is not result: self[key] = result - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self result.key = key return result @@ -72,11 +74,10 @@ def __getitem__(self, key): def get(self, key, default=None): value = dict.get(self, key, default) result = self.configurator.convert(value) - #If the converted value is different, save for next time + # If the converted value is different, save for next time if value is not result: self[key] = result - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self result.key = key return result @@ -85,22 +86,22 @@ def pop(self, key, default=None): value = dict.pop(self, key, default) result = self.configurator.convert(value) if value is not result: - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self result.key = key return result + class ConvertingList(list): """A converting list wrapper.""" + def __getitem__(self, key): value = list.__getitem__(self, key) result = self.configurator.convert(value) - #If the converted value is different, save for next time + # If the converted value is different, save for next time if value is not result: self[key] = result - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self result.key = key return result @@ -109,23 +110,24 @@ def pop(self, idx=-1): value = list.pop(self, idx) result = self.configurator.convert(value) if value is not result: - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self return result + class ConvertingTuple(tuple): """A converting tuple wrapper.""" + def __getitem__(self, key): value = tuple.__getitem__(self, key) result = self.configurator.convert(value) if value is not result: - if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self result.key = key return result + class BaseConfigurator(object): """ The configurator base class which defines some useful defaults. @@ -139,9 +141,8 @@ class BaseConfigurator(object): DIGIT_PATTERN = re.compile(r'^\d+$') value_converters = { - 'ext' : 'ext_convert', - 'cfg' : 'cfg_convert', - } + 'ext': 'ext_convert', + 'cfg': 'cfg_convert', } # We might want to use a different one, e.g. importlib importer = __import__ @@ -191,7 +192,6 @@ def cfg_convert(self, value): else: rest = rest[m.end():] d = self.config[m.groups()[0]] - #print d, rest while rest: m = self.DOT_PATTERN.match(rest) if m: @@ -204,16 +204,15 @@ def cfg_convert(self, value): d = d[idx] else: try: - n = int(idx) # try as number first (most likely) + n = int(idx) # try as number first (most likely) d = d[n] except TypeError: d = d[idx] if m: rest = rest[m.end():] else: - raise ValueError('Unable to convert ' - '%r at %r' % (value, rest)) - #rest should be empty + raise ValueError('Unable to convert ' '%r at %r' % (value, rest)) + # rest should be empty return d def convert(self, value): @@ -228,8 +227,7 @@ def convert(self, value): elif not isinstance(value, ConvertingList) and isinstance(value, list): value = ConvertingList(value) value.configurator = self - elif not isinstance(value, ConvertingTuple) and\ - isinstance(value, tuple): + elif not isinstance(value, ConvertingTuple) and isinstance(value, tuple): value = ConvertingTuple(value) value.configurator = self elif isinstance(value, basestring): @@ -264,6 +262,7 @@ def as_tuple(self, value): value = tuple(value) return value + def named_handlers_supported(): major, minor = sys.version_info[:2] if major == 2: @@ -274,6 +273,7 @@ def named_handlers_supported(): result = (major > 3) return result + class DictConfigurator(BaseConfigurator): """ Configure logging using a dictionary-like object to describe the @@ -299,8 +299,7 @@ def configure(self): if named_handlers_supported(): for name in handlers: if name not in logging._handlers: - raise ValueError('No handler found with ' - 'name %r' % name) + raise ValueError('No handler found with ' 'name %r' % name) else: try: handler = logging._handlers[name] @@ -310,24 +309,21 @@ def configure(self): handler.setLevel(_checkLevel(level)) except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure handler ' - '%r: %s' % (name, e)) + raise ValueError('Unable to configure handler ' '%r: %s' % (name, e)) loggers = config.get('loggers', EMPTY_DICT) for name in loggers: try: self.configure_logger(name, loggers[name], True) except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure logger ' - '%r: %s' % (name, e)) + raise ValueError('Unable to configure logger ' '%r: %s' % (name, e)) root = config.get('root', None) if root: try: self.configure_root(root, True) except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure root ' - 'logger: %s' % e) + raise ValueError('Unable to configure root ' 'logger: %s' % e) else: disable_existing = config.pop('disable_existing_loggers', True) @@ -338,12 +334,10 @@ def configure(self): formatters = config.get('formatters', EMPTY_DICT) for name in formatters: try: - formatters[name] = self.configure_formatter( - formatters[name]) + formatters[name] = self.configure_formatter(formatters[name]) except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure ' - 'formatter %r: %s' % (name, e)) + raise ValueError('Unable to configure ' 'formatter %r: %s' % (name, e)) # Next, do filters - they don't refer to anything else, either filters = config.get('filters', EMPTY_DICT) for name in filters: @@ -351,8 +345,7 @@ def configure(self): filters[name] = self.configure_filter(filters[name]) except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure ' - 'filter %r: %s' % (name, e)) + raise ValueError('Unable to configure ' 'filter %r: %s' % (name, e)) # Next, do handlers - they refer to formatters and filters # As handlers can refer to other handlers, sort the keys @@ -365,28 +358,20 @@ def configure(self): handlers[name] = handler except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure handler ' - '%r: %s' % (name, e)) + raise ValueError('Unable to configure handler ' '%r: %s' % (name, e)) # Next, do loggers - they refer to handlers and filters - #we don't want to lose the existing loggers, - #since other threads may have pointers to them. - #existing is set to contain all existing loggers, - #and as we go through the new configuration we - #remove any which are configured. At the end, - #what's left in existing is the set of loggers - #which were in the previous configuration but - #which are not in the new configuration. + # We don't want to lose the existing loggers, since other threads may have pointers to them. + # Existing is set to contain all existing loggers, and as we go through the new configuration we + # remove any which are configured. At the end, what's left in existing is the set of loggers + # which were in the previous configuration but which are not in the new configuration. root = logging.root existing = sorted(root.manager.loggerDict.keys()) - #The list needs to be sorted so that we can - #avoid disabling child loggers of explicitly - #named loggers. With a sorted list it is easier - #to find the child loggers. - #We'll keep the list of existing loggers - #which are children of named loggers here... + # The list needs to be sorted so that we can avoid disabling child loggers of explicitly named loggers. + # With a sorted list it is easier to find the child loggers. We'll keep the list of existing loggers + # which are children of named loggers here... child_loggers = [] - #now set up the new ones... + # now set up the new ones... loggers = config.get('loggers', EMPTY_DICT) for name in loggers: if name in existing: @@ -394,7 +379,7 @@ def configure(self): prefixed = name + "." pflen = len(prefixed) num_existing = len(existing) - i = i + 1 # look at the entry after name + i = i + 1 # look at the entry after name while (i < num_existing) and\ (existing[i][:pflen] == prefixed): child_loggers.append(existing[i]) @@ -404,14 +389,11 @@ def configure(self): self.configure_logger(name, loggers[name]) except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure logger ' - '%r: %s' % (name, e)) - - #Disable any old loggers. There's no point deleting - #them as other threads may continue to hold references - #and by disabling them, you stop them doing any logging. - #However, don't disable children of named loggers, as that's - #probably not what was intended by the user. + raise ValueError('Unable to configure logger ' '%r: %s' % (name, e)) + + # Disable any old loggers. There's no point deleting them as other threads may continue to hold + # references and by disabling them, you stop them doing any logging. However, don't disable children of + # named loggers, as that's probably not what was intended by the user. for log in existing: logger = root.manager.loggerDict[log] if log in child_loggers: @@ -428,25 +410,22 @@ def configure(self): self.configure_root(root) except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to configure root ' - 'logger: %s' % e) + raise ValueError('Unable to configure root ' 'logger: %s' % e) finally: logging._releaseLock() def configure_formatter(self, config): """Configure a formatter from a dictionary.""" if '()' in config: - factory = config['()'] # for use in exception handler + factory = config['()'] # for use in exception handler try: result = self.configure_custom(config) except TypeError: te = sys.exc_info()[1] if "'format'" not in str(te): raise - #Name of parameter changed from fmt to format. - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) + # Name of parameter changed from fmt to format. Retry with old name. This is so that code can be used + # with older Python versions (e.g. by Django) config['fmt'] = config.pop('format') config['()'] = factory result = self.configure_custom(config) @@ -482,8 +461,7 @@ def configure_handler(self, config): formatter = self.config['formatters'][formatter] except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to set formatter ' - '%r: %s' % (formatter, e)) + raise ValueError('Unable to set formatter ' '%r: %s' % (formatter, e)) level = config.pop('level', None) filters = config.pop('filters', None) if '()' in config: @@ -493,20 +471,16 @@ def configure_handler(self, config): factory = c else: klass = self.resolve(config.pop('class')) - #Special case for handler which refers to another handler - if issubclass(klass, logging.handlers.MemoryHandler) and\ - 'target' in config: + # Special case for handler which refers to another handler + if issubclass(klass, logging.handlers.MemoryHandler) and 'target' in config: try: config['target'] = self.config['handlers'][config['target']] except StandardError: e = sys.exc_info()[1] - raise ValueError('Unable to set target handler ' - '%r: %s' % (config['target'], e)) - elif issubclass(klass, logging.handlers.SMTPHandler) and\ - 'mailhost' in config: + raise ValueError('Unable to set target handler ' '%r: %s' % (config['target'], e)) + elif issubclass(klass, logging.handlers.SMTPHandler) and 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) - elif issubclass(klass, logging.handlers.SysLogHandler) and\ - 'address' in config: + elif issubclass(klass, logging.handlers.SysLogHandler) and 'address' in config: config['address'] = self.as_tuple(config['address']) factory = klass kwargs = dict([(k, config[k]) for k in config if valid_ident(k)]) @@ -516,10 +490,8 @@ def configure_handler(self, config): te = sys.exc_info()[1] if "'stream'" not in str(te): raise - #The argument name changed from strm to stream - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) + # The argument name changed from strm to stream, so we retry with the old name. This is so that code can be + # used with older Python versions (e.g. by Django) kwargs['strm'] = kwargs.pop('stream') result = factory(**kwargs) if formatter: @@ -547,7 +519,7 @@ def common_logger_config(self, logger, config, incremental=False): if level is not None: logger.setLevel(_checkLevel(level)) if not incremental: - #Remove any existing handlers + # Remove any existing handlers for h in logger.handlers[:]: logger.removeHandler(h) handlers = config.get('handlers', None) @@ -570,8 +542,10 @@ def configure_root(self, config, incremental=False): root = logging.getLogger() self.common_logger_config(root, config, incremental) + dictConfigClass = DictConfigurator + def dictConfig(config): """Configure logging using a dictionary.""" dictConfigClass(config).configure() diff --git a/splunk_eventgen/lib/logutils_src/logutils/http.py b/splunk_eventgen/lib/logutils_src/logutils/http.py index d1fe99d3..2d59dc88 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/http.py +++ b/splunk_eventgen/lib/logutils_src/logutils/http.py @@ -3,6 +3,7 @@ # import logging + class HTTPHandler(logging.Handler): """ A class which sends records to a Web server, using either GET or @@ -18,6 +19,7 @@ class HTTPHandler(logging.Handler): to avoid sending usernames and passwords in cleartext over the wire. """ + def __init__(self, host, url, method="GET", secure=False, credentials=None): """ Initialize an instance. @@ -51,7 +53,8 @@ def emit(self, record): :param record: The record to be emitted. """ try: - import http.client, urllib.parse + import http.client + import urllib.parse host = self.host if self.secure: h = http.client.HTTPSConnection(host) @@ -73,8 +76,7 @@ def emit(self, record): host = host[:i] h.putheader("Host", host) if self.method == "POST": - h.putheader("Content-type", - "application/x-www-form-urlencoded") + h.putheader("Content-type", "application/x-www-form-urlencoded") h.putheader("Content-length", str(len(data))) if self.credentials: import base64 @@ -82,7 +84,7 @@ def emit(self, record): s = 'Basic ' + base64.b64encode(s).strip() h.putheader('Authorization', s) h.endheaders(data if self.method == "POST" else None) - h.getresponse() #can't do anything with the result + h.getresponse() # can't do anything with the result except (KeyboardInterrupt, SystemExit): raise except: diff --git a/splunk_eventgen/lib/logutils_src/logutils/queue.py b/splunk_eventgen/lib/logutils_src/logutils/queue.py index fea91d8d..0a7d22a2 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/queue.py +++ b/splunk_eventgen/lib/logutils_src/logutils/queue.py @@ -20,11 +20,13 @@ version here is for use with earlier Python versions. """ import logging +import threading + try: import Queue as queue except ImportError: import queue -import threading + class QueueHandler(logging.Handler): """ @@ -97,6 +99,7 @@ def emit(self, record): except: self.handleError(record) + class QueueListener(object): """ This class implements an internal threaded listener which watches for @@ -144,7 +147,7 @@ def start(self): t.setDaemon(True) t.start() - def prepare(self , record): + def prepare(self, record): """ Prepare a record for handling. diff --git a/splunk_eventgen/lib/logutils_src/logutils/redis.py b/splunk_eventgen/lib/logutils_src/logutils/redis.py index a8ead302..46641bf2 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/redis.py +++ b/splunk_eventgen/lib/logutils_src/logutils/redis.py @@ -6,11 +6,13 @@ """ from logutils.queue import QueueHandler, QueueListener + try: import cPickle as pickle except ImportError: import pickle + class RedisQueueHandler(QueueHandler): """ A QueueHandler implementation which pushes pickled @@ -23,6 +25,7 @@ class RedisQueueHandler(QueueHandler): :param limit: If specified, the queue is restricted to have only this many elements. """ + def __init__(self, key='python.logging', redis=None, limit=0): if redis is None: from redis import Redis @@ -38,6 +41,7 @@ def enqueue(self, record): if self.limit: self.queue.ltrim(self.key, -self.limit, -1) + class RedisQueueListener(QueueListener): """ A QueueListener implementation which fetches pickled @@ -48,6 +52,7 @@ class RedisQueueListener(QueueListener): :param redis: If specified, this instance is used to communicate with a Redis instance. """ + def __init__(self, *handlers, **kwargs): redis = kwargs.get('redis') if redis is None: diff --git a/splunk_eventgen/lib/logutils_src/logutils/testing.py b/splunk_eventgen/lib/logutils_src/logutils/testing.py index 3c612179..bb8ac3df 100644 --- a/splunk_eventgen/lib/logutils_src/logutils/testing.py +++ b/splunk_eventgen/lib/logutils_src/logutils/testing.py @@ -1,9 +1,9 @@ # # Copyright (C) 2010-2017 Vinay Sajip. See LICENSE.txt for details. # -import logging from logging.handlers import BufferingHandler + class TestHandler(BufferingHandler): """ This handler collects records in a buffer for later inspection by @@ -12,6 +12,7 @@ class TestHandler(BufferingHandler): :param matcher: The :class:`~logutils.testing.Matcher` instance to use for matching. """ + def __init__(self, matcher): # BufferingHandler takes a "capacity" argument # so as to know when to flush. As we're overriding @@ -64,8 +65,8 @@ def matches(self, **kwargs): if self.matcher.matches(d, **kwargs): result = True break - #if not result: - # print('*** matcher failed completely on %d records' % len(self.buffer)) + # if not result: + # print('*** matcher failed completely on %d records' % len(self.buffer)) return result def matchall(self, kwarglist): @@ -96,6 +97,7 @@ def count(self): """ return len(self.buffer) + class Matcher(object): """ This utility class matches a stored dictionary of @@ -129,7 +131,7 @@ def matches(self, d, **kwargs): v = kwargs[k] dv = d.get(k) if not self.match_value(k, dv, v): - #print('*** matcher failed: %s, %r, %r' % (k, dv, v)) + # print('*** matcher failed: %s, %r, %r' % (k, dv, v)) result = False break return result @@ -150,6 +152,6 @@ def match_value(self, k, dv, v): result = (v == dv) else: result = dv.find(v) >= 0 - #if not result: + # if not result: # print('*** matcher failed on %s: %r vs. %r' % (k, dv, v)) return result diff --git a/splunk_eventgen/lib/logutils_src/logutils_src_setup.py b/splunk_eventgen/lib/logutils_src/logutils_src_setup.py index f0891d65..8eb90944 100644 --- a/splunk_eventgen/lib/logutils_src/logutils_src_setup.py +++ b/splunk_eventgen/lib/logutils_src/logutils_src_setup.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- import distutils.core -import logutils -from os.path import join, dirname, abspath import re +from os.path import dirname, join + +import logutils def description(): @@ -16,6 +17,7 @@ def description(): avail, = re.findall(regexp, readme, re.DOTALL) return reqts + avail + class TestCommand(distutils.core.Command): user_options = [] @@ -37,6 +39,7 @@ def initialize_options(self): def finalize_options(self): pass + distutils.core.setup( name='logutils', version=logutils.__version__, @@ -44,7 +47,7 @@ def finalize_options(self): author_email='vinay_sajip@red-dove.com', url='http://code.google.com/p/logutils/', description='Logging utilities', - long_description = description(), + long_description=description(), license='Copyright (C) 2010-2017 by Vinay Sajip. All Rights Reserved. See LICENSE.txt for license.', classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -55,11 +58,8 @@ def finalize_options(self): 'Programming Language :: Python', "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", - 'Topic :: Software Development', - ], + 'Topic :: Software Development', ], packages=['logutils'], cmdclass={ - 'test': TestCommand, - }, - + 'test': TestCommand, }, ) diff --git a/splunk_eventgen/lib/logutils_src/tests/logutil_tests.py b/splunk_eventgen/lib/logutils_src/tests/logutil_tests.py deleted file mode 100644 index e86d1062..00000000 --- a/splunk_eventgen/lib/logutils_src/tests/logutil_tests.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (C) 2008-2017 Vinay Sajip. See LICENSE.txt for details. -# -import sys -from test_testing import LoggingTest -from test_dictconfig import ConfigDictTest -from test_queue import QueueTest -from test_formatter import FormatterTest -from test_messages import MessageTest -from test_colorize import ColorizeTest -try: - from test_redis import RedisQueueTest -except ImportError: - pass - -# The adapter won't work in < 2.5 because the "extra" parameter used by it -# only appeared in 2.5 :-( -if sys.version_info[:2] >= (2, 5): - from test_adapter import AdapterTest -else: - print("LoggerAdapter won't work in Python < 2.5, so its tests are being " - "skipped.") diff --git a/splunk_eventgen/lib/logutils_src/tests/mytest.py b/splunk_eventgen/lib/logutils_src/tests/mytest.py index a5f40d32..ac9cbcc2 100644 --- a/splunk_eventgen/lib/logutils_src/tests/mytest.py +++ b/splunk_eventgen/lib/logutils_src/tests/mytest.py @@ -1,6 +1,7 @@ from __future__ import absolute_import -from logutils.testing import TestHandler, Matcher +from logutils.testing import Matcher, TestHandler + class MyTestHandler(TestHandler): def __init__(self): diff --git a/splunk_eventgen/lib/logutils_src/tests/test_adapter.py b/splunk_eventgen/lib/logutils_src/tests/test_adapter.py index d29bd106..a827f95d 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_adapter.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_adapter.py @@ -2,23 +2,25 @@ # Copyright (C) 2008-2017 Vinay Sajip. See LICENSE.txt for details. # import logging -from logutils.adapter import LoggerAdapter -from logutils.testing import TestHandler, Matcher import unittest +from logutils.adapter import LoggerAdapter +from logutils.testing import Matcher, TestHandler + + class AdapterTest(unittest.TestCase): def setUp(self): self.handler = h = TestHandler(Matcher()) - self.logger = l = logging.getLogger() - l.addHandler(h) - self.adapter = LoggerAdapter(l, {}) + self.logger = temp_logger = logging.getLogger() + temp_logger.addHandler(h) + self.adapter = LoggerAdapter(temp_logger, {}) def tearDown(self): self.logger.removeHandler(self.handler) self.handler.close() def test_simple(self): - "Simple test of logging test harness." + """Simple test of logging test harness.""" # Just as a demo, let's log some messages. # Only one should show up in the log. self.adapter.debug("This won't show up.") @@ -30,20 +32,20 @@ def test_simple(self): self.assertFalse(h.matches(levelno=logging.INFO)) def test_partial(self): - "Test of partial matching in logging test harness." + """Test of partial matching in logging test harness.""" # Just as a demo, let's log some messages. # Only one should show up in the log. self.adapter.debug("This won't show up.") self.adapter.info("Neither will this.") self.adapter.warning("But this will.") h = self.handler - self.assertTrue(h.matches(msg="ut th")) # from "But this will" - self.assertTrue(h.matches(message="ut th")) # from "But this will" + self.assertTrue(h.matches(msg="ut th")) # from "But this will" + self.assertTrue(h.matches(message="ut th")) # from "But this will" self.assertFalse(h.matches(message="either")) self.assertFalse(h.matches(message="won't")) def test_multiple(self): - "Test of matching multiple values in logging test harness." + """Test of matching multiple values in logging test harness.""" # Just as a demo, let's log some messages. # Only one should show up in the log. self.adapter.debug("This won't show up.") @@ -51,19 +53,18 @@ def test_multiple(self): self.adapter.warning("But this will.") self.adapter.error("And so will this.") h = self.handler - self.assertTrue(h.matches(levelno=logging.WARNING, - message='ut th')) - self.assertTrue(h.matches(levelno=logging.ERROR, - message='nd so w')) + self.assertTrue(h.matches(levelno=logging.WARNING, message='ut th')) + self.assertTrue(h.matches(levelno=logging.ERROR, message='nd so w')) self.assertFalse(h.matches(levelno=logging.INFO)) def test_hashandlers(self): - "Test of hasHandlers() functionality." + """Test of hasHandlers() functionality.""" self.assertTrue(self.adapter.hasHandlers()) self.logger.removeHandler(self.handler) self.assertFalse(self.adapter.hasHandlers()) self.logger.addHandler(self.handler) self.assertTrue(self.adapter.hasHandlers()) + if __name__ == '__main__': unittest.main() diff --git a/splunk_eventgen/lib/logutils_src/tests/test_colorize.py b/splunk_eventgen/lib/logutils_src/tests/test_colorize.py index 022b6318..18d1b263 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_colorize.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_colorize.py @@ -2,18 +2,18 @@ # Copyright (C) 2012-2017 Vinay Sajip. See LICENSE.txt for details. # import logging -import logutils.colorize -import os import sys import unittest +import logutils.colorize + if sys.version_info[0] < 3: u = lambda o: unicode(o, 'unicode_escape') else: u = lambda o: o -class ColorizeTest(unittest.TestCase): +class ColorizeTest(unittest.TestCase): def test_colorize(self): logger = logging.getLogger() handler = logutils.colorize.ColorizingStreamHandler() diff --git a/splunk_eventgen/lib/logutils_src/tests/test_dictconfig.py b/splunk_eventgen/lib/logutils_src/tests/test_dictconfig.py index 3aee9841..950bcc6c 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_dictconfig.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_dictconfig.py @@ -2,41 +2,47 @@ # Copyright 2009-2017 by Vinay Sajip. See LICENSE.txt for details. # import logging +import unittest + from logutils.adapter import LoggerAdapter from logutils.dictconfig import dictConfig, named_handlers_supported -from logutils.testing import TestHandler, Matcher -import sys -import unittest +from logutils.testing import Matcher, TestHandler try: StandardError except NameError: StandardError = Exception + class ExceptionFormatter(logging.Formatter): """A special exception formatter.""" + def formatException(self, ei): return "Got a [%s]" % ei[0].__name__ + def formatFunc(format, datefmt=None): return logging.Formatter(format, datefmt) + def testHandler(): return TestHandler(Matcher()) + def handlerFunc(): return logging.StreamHandler() + class CustomHandler(logging.StreamHandler): pass -class ConfigDictTest(unittest.TestCase): +class ConfigDictTest(unittest.TestCase): """Reading logging config from a dictionary.""" def setUp(self): - self.logger = l = logging.getLogger() - self.adapter = LoggerAdapter(l, {}) + self.logger = temp_logger = logging.getLogger() + self.adapter = LoggerAdapter(temp_logger, {}) logger_dict = logging.getLogger().manager.loggerDict logging._acquireLock() @@ -55,7 +61,6 @@ def setUp(self): self.root_logger = logging.getLogger("") self.original_logging_level = self.root_logger.getEffectiveLevel() - def tearDown(self): self.root_logger.setLevel(self.original_logging_level) logging._acquireLock() @@ -89,429 +94,287 @@ def next_message(self): config0 = { 'version': 1, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - } - }, - 'root' : { - 'level' : 'WARNING', - 'handlers' : ['hand1'], - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', }}, + 'root': { + 'level': 'WARNING', + 'handlers': ['hand1'], }, } # config1 adds a little to the standard configuration. config1 = { 'version': 1, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - } - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', }}, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WARNING', }, } # config2 has a subtle configuration error that should be reported config2 = { 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - 'class' : 'logging.StreamHandler', - 'formatter' : 'form1', - 'level' : 'NOTSET', - 'stream' : 'ext://sys.stdbout', - }, - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } - - #As config1 but with a misspelt level on a handler + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': { + 'hand1': { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdbout', }, }, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WARNING', }, } + + # As config1 but with a misspelt level on a handler config2a = { 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - 'class' : 'logging.StreamHandler', - 'formatter' : 'form1', - 'level' : 'NTOSET', - 'stream' : 'ext://sys.stdout', - }, - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } - - - #As config1 but with a misspelt level on a logger + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': { + 'hand1': { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NTOSET', + 'stream': 'ext://sys.stdout', }, }, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WARNING', }, } + + # As config1 but with a misspelt level on a logger config2b = { 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - 'class' : 'logging.StreamHandler', - 'formatter' : 'form1', - 'level' : 'NOTSET', - 'stream' : 'ext://sys.stdout', - }, - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WRANING', - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': { + 'hand1': { + 'class': 'logging.StreamHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', }, }, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WRANING', }, } # config3 has a less subtle configuration error config3 = { 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - 'class' : 'logging.StreamHandler', - 'formatter' : 'misspelled_name', - 'level' : 'NOTSET', - 'stream' : 'ext://sys.stdout', - }, - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': { + 'hand1': { + 'class': 'logging.StreamHandler', + 'formatter': 'misspelled_name', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', }, }, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WARNING', }, } # config4 specifies a custom formatter class to be loaded config4 = { 'version': 1, 'formatters': { - 'form1' : { - '()' : __name__ + '.ExceptionFormatter', - 'format' : '%(levelname)s:%(name)s:%(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - } - }, - 'root' : { - 'level' : 'NOTSET', - 'handlers' : ['hand1'], - }, - } + 'form1': { + '()': __name__ + '.ExceptionFormatter', + 'format': '%(levelname)s:%(name)s:%(message)s', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', }}, + 'root': { + 'level': 'NOTSET', + 'handlers': ['hand1'], }, } # As config4 but using an actual callable rather than a string config4a = { 'version': 1, 'formatters': { - 'form1' : { - '()' : ExceptionFormatter, - 'format' : '%(levelname)s:%(name)s:%(message)s', - }, - 'form2' : { - '()' : __name__ + '.formatFunc', - 'format' : '%(levelname)s:%(name)s:%(message)s', - }, - 'form3' : { - '()' : formatFunc, - 'format' : '%(levelname)s:%(name)s:%(message)s', - }, - }, - 'handlers' : { - 'hand1' : { + 'form1': { + '()': ExceptionFormatter, + 'format': '%(levelname)s:%(name)s:%(message)s', }, + 'form2': { + '()': __name__ + '.formatFunc', + 'format': '%(levelname)s:%(name)s:%(message)s', }, + 'form3': { + '()': formatFunc, + 'format': '%(levelname)s:%(name)s:%(message)s', }, }, + 'handlers': { + 'hand1': { '()': testHandler, - 'formatter': 'form1', - }, - 'hand2' : { - '()' : handlerFunc, - }, - }, - 'root' : { - 'level' : 'NOTSET', - 'handlers' : ['hand1'], - }, - } + 'formatter': 'form1', }, + 'hand2': { + '()': handlerFunc, }, }, + 'root': { + 'level': 'NOTSET', + 'handlers': ['hand1'], }, } # config5 specifies a custom handler class to be loaded config5 = { 'version': 1, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - } - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', }}, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WARNING', }, } # config6 specifies a custom handler class to be loaded # but has bad arguments config6 = { 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - 'class' : __name__ + '.CustomHandler', - 'formatter' : 'form1', - 'level' : 'NOTSET', - 'stream' : 'ext://sys.stdout', - '9' : 'invalid parameter name', - }, - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } - - #config 7 does not define compiler.parser but defines compiler.lexer - #so compiler.parser should be disabled after applying it + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': { + 'hand1': { + 'class': __name__ + '.CustomHandler', + 'formatter': 'form1', + 'level': 'NOTSET', + 'stream': 'ext://sys.stdout', + '9': 'invalid parameter name', }, }, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WARNING', }, } + + # config 7 does not define compiler.parser but defines compiler.lexer + # so compiler.parser should be disabled after applying it config7 = { 'version': 1, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - } - }, - 'loggers' : { - 'compiler.lexer' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', }}, + 'loggers': { + 'compiler.lexer': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'WARNING', }, } config8 = { 'version': 1, - 'disable_existing_loggers' : False, + 'disable_existing_loggers': False, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - } - }, - 'loggers' : { - 'compiler' : { - 'level' : 'DEBUG', - 'handlers' : ['hand1'], - }, - 'compiler.lexer' : { - }, - }, - 'root' : { - 'level' : 'WARNING', - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', }}, + 'loggers': { + 'compiler': { + 'level': 'DEBUG', + 'handlers': ['hand1'], }, + 'compiler.lexer': {}, }, + 'root': { + 'level': 'WARNING', }, } config9 = { 'version': 1, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - } - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'WARNING', - 'handlers' : ['hand1'], - }, - }, - 'root' : { - 'level' : 'NOTSET', - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', }}, + 'loggers': { + 'compiler.parser': { + 'level': 'WARNING', + 'handlers': ['hand1'], }, }, + 'root': { + 'level': 'NOTSET', }, } config9a = { 'version': 1, - 'incremental' : True, - 'handlers' : { - 'hand1' : { - 'level' : 'WARNING', - }, - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'INFO', - }, - }, - } + 'incremental': True, + 'handlers': { + 'hand1': { + 'level': 'WARNING', }, }, + 'loggers': { + 'compiler.parser': { + 'level': 'INFO', }, }, } config9b = { 'version': 1, - 'incremental' : True, - 'handlers' : { - 'hand1' : { - 'level' : 'INFO', - }, - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'INFO', - }, - }, - } - - #As config1 but with a filter added + 'incremental': True, + 'handlers': { + 'hand1': { + 'level': 'INFO', }, }, + 'loggers': { + 'compiler.parser': { + 'level': 'INFO', }, }, } + + # As config1 but with a filter added config10 = { 'version': 1, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'filters' : { - 'filt1' : { - 'name' : 'compiler.parser', - }, - }, - 'handlers' : { - 'hand1' : { - '()': testHandler, - 'formatter': 'form1', - 'filters' : ['filt1'], - } - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'filters' : ['filt1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - 'handlers' : ['hand1'], - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'filters': { + 'filt1': { + 'name': 'compiler.parser', }, }, + 'handlers': {'hand1': { + '()': testHandler, + 'formatter': 'form1', + 'filters': ['filt1'], }}, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'filters': ['filt1'], }, }, + 'root': { + 'level': 'WARNING', + 'handlers': ['hand1'], }, } # As config10, but declaring a handler in a module using # absolute imports config11 = { 'version': 1, 'formatters': { - 'form1' : { - 'format' : '%(levelname)s ++ %(message)s', - }, - }, - 'filters' : { - 'filt1' : { - 'name' : 'compiler.parser', - }, - }, - 'handlers' : { - 'hand1' : { - '()': 'mytest.MyTestHandler', - 'formatter': 'form1', - 'filters' : ['filt1'], - } - }, - 'loggers' : { - 'compiler.parser' : { - 'level' : 'DEBUG', - 'filters' : ['filt1'], - }, - }, - 'root' : { - 'level' : 'WARNING', - 'handlers' : ['hand1'], - }, - } + 'form1': { + 'format': '%(levelname)s ++ %(message)s', }, }, + 'filters': { + 'filt1': { + 'name': 'compiler.parser', }, }, + 'handlers': {'hand1': { + '()': 'mytest.MyTestHandler', + 'formatter': 'form1', + 'filters': ['filt1'], }}, + 'loggers': { + 'compiler.parser': { + 'level': 'DEBUG', + 'filters': ['filt1'], }, }, + 'root': { + 'level': 'WARNING', + 'handlers': ['hand1'], }, } def apply_config(self, conf): dictConfig(conf) @@ -526,9 +389,7 @@ def test_config0_ok(self): logger.error(self.next_message()) h = logger.handlers[0] self.assertEqual(1, h.count) - self.assertTrue(h.matchall([ - dict(levelname='ERROR', message='2') - ])) + self.assertTrue(h.matchall([dict(levelname='ERROR', message='2')])) def test_config1_ok(self, config=config1): # A config defining a sub-parser as well. @@ -539,9 +400,8 @@ def test_config1_ok(self, config=config1): logger.error(self.next_message()) h = logger.handlers[0] self.assertTrue(h.matchall([ - dict(levelname='INFO', message='1'), - dict(levelname='ERROR', message='2'), - ])) + dict(levelname='INFO', message='1'), + dict(levelname='ERROR', message='2'), ])) def test_config2_failure(self): # A simple config which overrides the default settings. @@ -568,8 +428,7 @@ def test_config4_ok(self): raise RuntimeError() except RuntimeError: logging.exception("just testing") - self.assertEquals(h.formatted[0], - "ERROR:root:just testing\nGot a [RuntimeError]") + self.assertEquals(h.formatted[0], "ERROR:root:just testing\nGot a [RuntimeError]") def test_config4a_ok(self): # A config specifying a custom formatter class. @@ -580,8 +439,7 @@ def test_config4a_ok(self): raise RuntimeError() except RuntimeError: logging.exception("just testing") - self.assertEquals(h.formatted[0], - "ERROR:root:just testing\nGot a [RuntimeError]") + self.assertEquals(h.formatted[0], "ERROR:root:just testing\nGot a [RuntimeError]") def test_config5_ok(self): self.test_config1_ok(config=self.config5) @@ -597,9 +455,8 @@ def test_config7_ok(self): logger.error(self.next_message()) h = logger.handlers[0] self.assertTrue(h.matchall([ - dict(levelname='INFO', message='1'), - dict(levelname='ERROR', message='2'), - ])) + dict(levelname='INFO', message='1'), + dict(levelname='ERROR', message='2'), ])) self.apply_config(self.config7) logger = logging.getLogger("compiler.parser") self.assertTrue(logger.disabled) @@ -609,11 +466,10 @@ def test_config7_ok(self): logger.info(self.next_message()) logger.error(self.next_message()) self.assertTrue(h.matchall([ - dict(levelname='INFO', message='3'), - dict(levelname='ERROR', message='4'), - ])) + dict(levelname='INFO', message='3'), + dict(levelname='ERROR', message='4'), ])) - #Same as test_config_7_ok but don't disable old loggers. + # Same as test_config_7_ok but don't disable old loggers. def test_config_8_ok(self): self.apply_config(self.config1) logger = logging.getLogger("compiler.parser") @@ -622,9 +478,8 @@ def test_config_8_ok(self): logger.error(self.next_message()) h = logger.handlers[0] self.assertTrue(h.matchall([ - dict(levelname='INFO', message='1'), - dict(levelname='ERROR', message='2'), - ])) + dict(levelname='INFO', message='1'), + dict(levelname='ERROR', message='2'), ])) self.apply_config(self.config8) logger = logging.getLogger("compiler.parser") self.assertFalse(logger.disabled) @@ -637,22 +492,22 @@ def test_config_8_ok(self): logger.info(self.next_message()) logger.error(self.next_message()) h = toplogger.handlers[0] - self.assertTrue(h.matchall([ - dict(levelname='INFO', message='3'), - dict(levelname='ERROR', message='4'), - dict(levelname='INFO', message='5'), - dict(levelname='ERROR', message='6'), - ])) + self.assertTrue( + h.matchall([ + dict(levelname='INFO', message='3'), + dict(levelname='ERROR', message='4'), + dict(levelname='INFO', message='5'), + dict(levelname='ERROR', message='6'), ])) def test_config_9_ok(self): self.apply_config(self.config9) logger = logging.getLogger("compiler.parser") - #Nothing will be output since both handler and logger are set to WARNING + # Nothing will be output since both handler and logger are set to WARNING logger.info(self.next_message()) h = logger.handlers[0] self.assertEqual(0, h.count) self.apply_config(self.config9a) - #Nothing will be output since both handler is still set to WARNING + # Nothing will be output since both handler is still set to WARNING logger.info(self.next_message()) h = logger.handlers[0] nhs = named_handlers_supported() @@ -661,13 +516,12 @@ def test_config_9_ok(self): else: self.assertEqual(1, h.count) self.apply_config(self.config9b) - #Message should now be output + # Message should now be output logger.info(self.next_message()) if nhs: h = logger.handlers[0] self.assertTrue(h.matchall([ - dict(levelname='INFO', message='3'), - ])) + dict(levelname='INFO', message='3'), ])) else: self.assertEqual(2, h.count) @@ -676,19 +530,18 @@ def test_config_10_ok(self): logger = logging.getLogger("compiler.parser") logger.warning(self.next_message()) logger = logging.getLogger('compiler') - #Not output, because filtered + # Not output, because filtered logger.warning(self.next_message()) logger = logging.getLogger('compiler.lexer') - #Not output, because filtered + # Not output, because filtered logger.warning(self.next_message()) logger = logging.getLogger("compiler.parser.codegen") - #Output, as not filtered + # Output, as not filtered logger.error(self.next_message()) h = logging.getLogger().handlers[0] self.assertTrue(h.matchall([ - dict(levelname='WARNING', message='1'), - dict(levelname='ERROR', message='4'), - ])) + dict(levelname='WARNING', message='1'), + dict(levelname='ERROR', message='4'), ])) def test_config_11_ok(self): self.apply_config(self.config11) diff --git a/splunk_eventgen/lib/logutils_src/tests/test_formatter.py b/splunk_eventgen/lib/logutils_src/tests/test_formatter.py index 0a069c78..011ba234 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_formatter.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_formatter.py @@ -2,11 +2,13 @@ # Copyright (C) 2009-2017 Vinay Sajip. See LICENSE.txt for details. # import logging -import logutils import os import sys import unittest +import logutils + + class FormatterTest(unittest.TestCase): def setUp(self): self.common = { @@ -17,10 +19,8 @@ def setUp(self): 'exc_info': None, 'func': None, 'msg': 'Message with %d %s', - 'args': (2, 'placeholders'), - } - self.variants = { - } + 'args': (2, 'placeholders'), } + self.variants = {} def get_record(self, name=None): result = dict(self.common) @@ -42,6 +42,7 @@ def test_percent(self): self.assertFalse(f.usesTime()) if sys.version_info[:2] >= (2, 6): + def test_braces(self): "Test {}-formatting" r = self.get_record() diff --git a/splunk_eventgen/lib/logutils_src/tests/test_messages.py b/splunk_eventgen/lib/logutils_src/tests/test_messages.py index 17f80bbd..0a221105 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_messages.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_messages.py @@ -1,9 +1,12 @@ -import logutils import sys import unittest +import logutils + + class MessageTest(unittest.TestCase): if sys.version_info[:2] >= (2, 6): + def test_braces(self): "Test whether brace-formatting works." __ = logutils.BraceMessage @@ -19,8 +22,7 @@ class Dummy: dummy = Dummy() dummy.x, dummy.y = 0.0, 1.0 - m = __('Message with coordinates: ({point.x:.2f}, {point.y:.2f})', - point=dummy) + m = __('Message with coordinates: ({point.x:.2f}, {point.y:.2f})', point=dummy) self.assertEqual(str(m), 'Message with coordinates: (0.00, 1.00)') def test_dollars(self): @@ -29,5 +31,4 @@ def test_dollars(self): m = __('Message with $num ${what}', num=2, what='placeholders') self.assertEqual(str(m), 'Message with 2 placeholders') ignored = object() - self.assertRaises(TypeError, __, 'Message with $num ${what}', - ignored, num=2, what='placeholders') + self.assertRaises(TypeError, __, 'Message with $num ${what}', ignored, num=2, what='placeholders') diff --git a/splunk_eventgen/lib/logutils_src/tests/test_queue.py b/splunk_eventgen/lib/logutils_src/tests/test_queue.py index 34152e37..f85074c6 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_queue.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_queue.py @@ -2,19 +2,21 @@ # Copyright (C) 2010-2017 Vinay Sajip. See LICENSE.txt for details. # import logging -from logutils.testing import TestHandler, Matcher -from logutils.queue import QueueHandler, QueueListener, queue import unittest +from logutils.queue import QueueHandler, QueueListener, queue +from logutils.testing import Matcher, TestHandler + + class QueueTest(unittest.TestCase): def setUp(self): self.handler = h = TestHandler(Matcher()) - self.logger = l = logging.getLogger() + self.logger = temp_logger = logging.getLogger() self.queue = q = queue.Queue(-1) self.qh = qh = QueueHandler(q) self.ql = ql = QueueListener(q, h) ql.start() - l.addHandler(qh) + temp_logger.addHandler(qh) def tearDown(self): self.logger.removeHandler(self.qh) @@ -28,9 +30,8 @@ def test_simple(self): self.logger.debug("This won't show up.") self.logger.info("Neither will this.") self.logger.warning("But this will.") - self.ql.stop() #ensure all records have come through. + self.ql.stop() # ensure all records have come through. h = self.handler - #import pdb; pdb.set_trace() self.assertTrue(h.matches(levelno=logging.WARNING)) self.assertFalse(h.matches(levelno=logging.DEBUG)) self.assertFalse(h.matches(levelno=logging.INFO)) @@ -42,10 +43,10 @@ def test_partial(self): self.logger.debug("This won't show up.") self.logger.info("Neither will this.") self.logger.warning("But this will.") - self.ql.stop() #ensure all records have come through. + self.ql.stop() # ensure all records have come through. h = self.handler - self.assertTrue(h.matches(msg="ut th")) # from "But this will" - self.assertTrue(h.matches(message="ut th")) # from "But this will" + self.assertTrue(h.matches(msg="ut th")) # from "But this will" + self.assertTrue(h.matches(message="ut th")) # from "But this will" self.assertFalse(h.matches(message="either")) self.assertFalse(h.matches(message="won't")) @@ -57,13 +58,12 @@ def test_multiple(self): self.logger.info("Neither will this.") self.logger.warning("But this will.") self.logger.error("And so will this.") - self.ql.stop() #ensure all records have come through. + self.ql.stop() # ensure all records have come through. h = self.handler - self.assertTrue(h.matches(levelno=logging.WARNING, - message='ut thi')) - self.assertTrue(h.matches(levelno=logging.ERROR, - message='nd so wi')) + self.assertTrue(h.matches(levelno=logging.WARNING, message='ut thi')) + self.assertTrue(h.matches(levelno=logging.ERROR, message='nd so wi')) self.assertFalse(h.matches(levelno=logging.INFO)) + if __name__ == '__main__': unittest.main() diff --git a/splunk_eventgen/lib/logutils_src/tests/test_redis.py b/splunk_eventgen/lib/logutils_src/tests/test_redis.py index 858192cd..e53bdb5d 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_redis.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_redis.py @@ -2,14 +2,17 @@ # Copyright (C) 2011-2017 Vinay Sajip. See LICENSE.txt for details. # import logging -from logutils.testing import TestHandler, Matcher -from logutils.redis import RedisQueueHandler, RedisQueueListener -from redis import Redis import socket import subprocess import time import unittest +from logutils.redis import RedisQueueHandler, RedisQueueListener +from logutils.testing import Matcher, TestHandler + +from redis import Redis + + class QueueListener(RedisQueueListener): def dequeue(self, block): record = RedisQueueListener.dequeue(self, block) @@ -17,19 +20,18 @@ def dequeue(self, block): record = logging.makeLogRecord(record) return record + class RedisQueueTest(unittest.TestCase): def setUp(self): self.handler = h = TestHandler(Matcher()) - self.logger = l = logging.getLogger() - self.server = subprocess.Popen(['redis-server'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + self.logger = temp_logger = logging.getLogger() + self.server = subprocess.Popen(['redis-server'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.wait_for_server() self.queue = q = Redis() self.qh = qh = RedisQueueHandler(redis=q) self.ql = ql = QueueListener(h, redis=q) ql.start() - l.addHandler(qh) + temp_logger.addHandler(qh) def tearDown(self): self.logger.removeHandler(self.qh) @@ -38,7 +40,7 @@ def tearDown(self): self.server.terminate() def wait_for_server(self): - maxtime = time.time() + 2 # 2 seconds to wait for server + maxtime = time.time() + 2 # 2 seconds to wait for server sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) while time.time() < maxtime: try: @@ -57,9 +59,8 @@ def test_simple(self): self.logger.debug("This won't show up.") self.logger.info("Neither will this.") self.logger.warning("But this will.") - self.ql.stop() #ensure all records have come through. + self.ql.stop() # ensure all records have come through. h = self.handler - #import pdb; pdb.set_trace() self.assertTrue(h.matches(levelno=logging.WARNING)) self.assertFalse(h.matches(levelno=logging.DEBUG)) self.assertFalse(h.matches(levelno=logging.INFO)) @@ -71,10 +72,10 @@ def test_partial(self): self.logger.debug("This won't show up.") self.logger.info("Neither will this.") self.logger.warning("But this will.") - self.ql.stop() #ensure all records have come through. + self.ql.stop() # ensure all records have come through. h = self.handler - self.assertTrue(h.matches(msg="ut th")) # from "But this will" - self.assertTrue(h.matches(message="ut th")) # from "But this will" + self.assertTrue(h.matches(msg="ut th")) # from "But this will" + self.assertTrue(h.matches(message="ut th")) # from "But this will" self.assertFalse(h.matches(message="either")) self.assertFalse(h.matches(message="won't")) @@ -86,13 +87,12 @@ def test_multiple(self): self.logger.info("Neither will this.") self.logger.warning("But this will.") self.logger.error("And so will this.") - self.ql.stop() #ensure all records have come through. + self.ql.stop() # ensure all records have come through. h = self.handler - self.assertTrue(h.matches(levelno=logging.WARNING, - message='ut thi')) - self.assertTrue(h.matches(levelno=logging.ERROR, - message='nd so wi')) + self.assertTrue(h.matches(levelno=logging.WARNING, message='ut thi')) + self.assertTrue(h.matches(levelno=logging.ERROR, message='nd so wi')) self.assertFalse(h.matches(levelno=logging.INFO)) + if __name__ == '__main__': unittest.main() diff --git a/splunk_eventgen/lib/logutils_src/tests/test_testing.py b/splunk_eventgen/lib/logutils_src/tests/test_testing.py index c0b7409a..ef61beb7 100644 --- a/splunk_eventgen/lib/logutils_src/tests/test_testing.py +++ b/splunk_eventgen/lib/logutils_src/tests/test_testing.py @@ -2,21 +2,23 @@ # Copyright (C) 2010-2017 Vinay Sajip. See LICENSE.txt for details. # import logging -from logutils.testing import TestHandler, Matcher import unittest +from logutils.testing import Matcher, TestHandler + + class LoggingTest(unittest.TestCase): def setUp(self): self.handler = h = TestHandler(Matcher()) - self.logger = l = logging.getLogger() - l.addHandler(h) + self.logger = temp_logger = logging.getLogger() + temp_logger.addHandler(h) def tearDown(self): self.logger.removeHandler(self.handler) self.handler.close() def test_simple(self): - "Simple test of logging test harness." + """Simple test of logging test harness.""" # Just as a demo, let's log some messages. # Only one should show up in the log. self.logger.debug("This won't show up.") @@ -28,20 +30,20 @@ def test_simple(self): self.assertFalse(h.matches(levelno=logging.INFO)) def test_partial(self): - "Test of partial matching in logging test harness." + """Test of partial matching in logging test harness.""" # Just as a demo, let's log some messages. # Only one should show up in the log. self.logger.debug("This won't show up.") self.logger.info("Neither will this.") self.logger.warning("But this will.") h = self.handler - self.assertTrue(h.matches(msg="ut th")) # from "But this will" - self.assertTrue(h.matches(message="ut th")) # from "But this will" + self.assertTrue(h.matches(msg="ut th")) # from "But this will" + self.assertTrue(h.matches(message="ut th")) # from "But this will" self.assertFalse(h.matches(message="either")) self.assertFalse(h.matches(message="won't")) def test_multiple(self): - "Test of matching multiple values in logging test harness." + """Test of matching multiple values in logging test harness.""" # Just as a demo, let's log some messages. # Only one should show up in the log. self.logger.debug("This won't show up.") @@ -49,11 +51,10 @@ def test_multiple(self): self.logger.warning("But this will.") self.logger.error("And so will this.") h = self.handler - self.assertTrue(h.matches(levelno=logging.WARNING, - message='ut thi')) - self.assertTrue(h.matches(levelno=logging.ERROR, - message='nd so wi')) + self.assertTrue(h.matches(levelno=logging.WARNING, message='ut thi')) + self.assertTrue(h.matches(levelno=logging.ERROR, message='nd so wi')) self.assertFalse(h.matches(levelno=logging.INFO)) + if __name__ == '__main__': unittest.main() diff --git a/splunk_eventgen/lib/outputcounter.py b/splunk_eventgen/lib/outputcounter.py index 37547f11..2e9b803c 100644 --- a/splunk_eventgen/lib/outputcounter.py +++ b/splunk_eventgen/lib/outputcounter.py @@ -1,11 +1,13 @@ -import time import logging +import time + class OutputCounter(object): ''' This object is used as a global variable for outputer to collect how many events and how much size of raw events egx has generated, and use them to calculate a real-time throughput. ''' + def __init__(self): self.event_size_1_min = 0 self.event_count_1_min = 0 @@ -24,7 +26,8 @@ def update_throughput(self, timestamp): self.current_time = timestamp self.event_count_1_min = 0 self.event_size_1_min = 0 - self.logger.error("Current throughput is {} B/s, {} count/s".format(self.throughput_volume, self.throughput_count)) + self.logger.error("Current throughput is {} B/s, {} count/s".format(self.throughput_volume, + self.throughput_count)) def collect(self, event_count, event_size): timestamp = time.time() diff --git a/splunk_eventgen/lib/outputplugin.py b/splunk_eventgen/lib/outputplugin.py index f1dee615..e3bb2fb3 100644 --- a/splunk_eventgen/lib/outputplugin.py +++ b/splunk_eventgen/lib/outputplugin.py @@ -1,8 +1,10 @@ from __future__ import division + import logging import logging.handlers from collections import deque + class OutputPlugin(object): name = 'OutputPlugin' @@ -12,14 +14,15 @@ def __init__(self, sample, output_counter=None): self._outputMode = sample.outputMode self.events = None self._setup_logging() - self.logger.debug("Starting OutputPlugin for sample '%s' with output '%s'" % (self._sample.name, self._sample.outputMode)) + self.logger.debug( + "Starting OutputPlugin for sample '%s' with output '%s'" % (self._sample.name, self._sample.outputMode)) self._queue = deque([]) self.output_counter = output_counter def __str__(self): """Only used for debugging, outputs a pretty printed representation of this output""" # Eliminate recursive going back to parent - temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key != '_c']) + # temp = dict([(key, value) for (key, value) in self.__dict__.items() if key != '_c']) # return pprint.pformat(temp) return "" @@ -55,4 +58,4 @@ def run(self): def load(): - return OutputPlugin \ No newline at end of file + return OutputPlugin diff --git a/splunk_eventgen/lib/plugins/generator/default.py b/splunk_eventgen/lib/plugins/generator/default.py index 52bafe19..ed106e39 100644 --- a/splunk_eventgen/lib/plugins/generator/default.py +++ b/splunk_eventgen/lib/plugins/generator/default.py @@ -1,11 +1,12 @@ -# TODO Sample object now incredibly overloaded and not threadsafe. Need to make it threadsafe and make it simpler to get a -# copy of whats needed without the whole object. +# TODO: Sample object is incredibly overloaded and not threadsafe. Need to make it simpler to get a copy without the +# whole object get a copy of whats needed without the whole object. from __future__ import division -from generatorplugin import GeneratorPlugin -import datetime, time + +import datetime import random -from eventgentimestamp import EventgenTimestamp + +from generatorplugin import GeneratorPlugin class DefaultGenerator(GeneratorPlugin): @@ -13,27 +14,29 @@ def __init__(self, sample): GeneratorPlugin.__init__(self, sample) def gen(self, count, earliest, latest, samplename=None): - s = self._sample - - self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, count, earliest, latest)) + self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % + (self._sample.name, self._sample.app, count, earliest, latest)) startTime = datetime.datetime.now() # If we're random, fill random events from sampleDict into eventsDict if self._sample.randomizeEvents: - eventsDict = [ ] + eventsDict = [] sdlen = len(self._sample.sampleDict) - self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d events" % (self._sample.name, self._sample.app, count)) - # Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually + self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d events" % + (self._sample.name, self._sample.app, count)) + # Count is -1, replay the whole file, but in randomizeEvents I think we'd want it to actually # just put as many events as there are in the file if count == -1: count = sdlen while len(eventsDict) < count: - eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen-1)]) + eventsDict.append(self._sample.sampleDict[random.randint(0, sdlen - 1)]) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: - eventsDict = [ ] - self.logger.debugv("Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, count)) + eventsDict = [] + self.logger.debugv( + "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % + (self._sample.name, self._sample.app, count)) for x in xrange(count): eventsDict.extend(self._sample.sampleDict) @@ -45,9 +48,11 @@ def gen(self, count, earliest, latest, samplename=None): count = len(self._sample.sampleDict) eventsDict = self._sample.sampleDict[0:count] - ## Continue to fill events array until len(events) == count + # Continue to fill events array until len(events) == count if len(eventsDict) < count: - self.logger.debugv("Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" % (self._sample.name, self._sample.app, len(eventsDict), count) ) + self.logger.debugv( + "Events fill for sample '%s' in app '%s' less than count (%s vs. %s); continuing fill" % + (self._sample.name, self._sample.app, len(eventsDict), count)) self.logger.debugv("Current eventsDict: %s" % eventsDict) # run a modulus on the size of the eventdict to figure out what the last event was. Populate to count # from there. @@ -57,9 +62,11 @@ def gen(self, count, earliest, latest, samplename=None): nextEventToUse = self._sample.sampleDict[len(eventsDict) % len(self._sample.sampleDict)] self.logger.debugv("Next event to add: %s" % nextEventToUse) eventsDict.append(nextEventToUse) - self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) + self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" % + (self._sample.name, self._sample.app, len(eventsDict))) GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest) + def load(): return DefaultGenerator diff --git a/splunk_eventgen/lib/plugins/generator/jinja.py b/splunk_eventgen/lib/plugins/generator/jinja.py index 52f3e63c..65644ff7 100644 --- a/splunk_eventgen/lib/plugins/generator/jinja.py +++ b/splunk_eventgen/lib/plugins/generator/jinja.py @@ -1,20 +1,22 @@ from __future__ import division + import datetime -import time import os import random -try: - import ujson as json -except: - import json as json +import time + from jinja2 import nodes from jinja2.ext import Extension from generatorplugin import GeneratorPlugin +try: + import ujson as json +except: + import json as json -class CantFindTemplate(Exception): +class CantFindTemplate(Exception): def __init__(self, msg): """Exception raised when we / Jinja can't find the template @@ -25,7 +27,6 @@ def __init__(self, msg): class CantProcessTemplate(Exception): - def __init__(self, msg): """Exception raised when we / Jinja can't find the template @@ -41,8 +42,8 @@ class JinjaTime(Extension): @staticmethod def _get_time_slice(earliest, latest, slices, target_slice, slice_type="lower"): """ - This method will take a time block bounded by "earliest and latest", and a slice. It'll then divide the time - in sections and return a tuple with 3 arguments, the lower bound, the higher bound, and the target in the middle. + This method will take a time block bounded by "earliest and latest", and a slice. It'll then divide the time in + sections and return a tuple with 3 arguments, the lower bound, the higher bound, and the target in the middle. :param earliest (in epoch): :param latest (in epoch): :param slices: @@ -62,12 +63,12 @@ def _get_time_slice(earliest, latest, slices, target_slice, slice_type="lower"): if slice_type == "lower": slice_time = slice_start elif slice_type == "middle": - slice_time = slice_start + (slice_size/2) + slice_time = slice_start + (slice_size / 2) elif slice_type == "upper": slice_time = slice_end elif slice_type == "random": - start = int(slice_start*100) - end = int(slice_end*100) + start = int(slice_start * 100) + end = int(slice_end * 100) if start == end: slice_time = end * 0.01 else: @@ -93,7 +94,8 @@ def _time_slice_formatted(self, earliest, latest, count, slices, date_format='%Y def _time_slice_epoch(self, earliest, latest, count, slices, date_format=None): slice_start, slice_end, slice_size, slice_time = \ - self._get_time_slice(earliest=earliest, latest=latest, slices=slices, target_slice=count, slice_type="lower") + self._get_time_slice(earliest=earliest, latest=latest, slices=slices, target_slice=count, + slice_type="lower") return slice_time @staticmethod @@ -106,17 +108,14 @@ def _output_var(var_value, lineno): return nodes.Output([var_value], lineno=lineno) def parse(self, parser): - target_var_name = { - "time_now": "time_now", - "time_slice": "time_target" - } + target_var_name = {"time_now": "time_now", "time_slice": "time_target"} tag = parser.stream.current.value name_base = target_var_name[tag] lineno = parser.stream.next().lineno args, kwargs = self.parse_args(parser) task_list = [] - epoch_name = name_base+"_epoch" - formatted_name = name_base+"_formatted" + epoch_name = name_base + "_epoch" + formatted_name = name_base + "_formatted" target_epoch_method = "_{0}_epoch".format(tag) target_formatted_method = "_{0}_formatted".format(tag) epoch_call = self.call_method(target_epoch_method, args=args, kwargs=kwargs, lineno=lineno) @@ -139,8 +138,7 @@ def parse_args(self, parser): kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) else: if kwargs: - parser.fail('Invalid argument syntax for WrapExtension tag', - parser.stream.current.lineno) + parser.fail('Invalid argument syntax for WrapExtension tag', parser.stream.current.lineno) args.append(parser.parse_expression()) require_comma = True return args, kwargs @@ -212,11 +210,10 @@ def gen(self, count, earliest, latest, samplename=None): if not hasattr(self._sample, "jinja_target_template"): raise CantFindTemplate("Template to load not specified in eventgen conf for stanza. Skipping Stanza") jinja_env = Environment( - loader=FileSystemLoader([target_template_dir, working_dir, template_dir], encoding='utf-8', followlinks=False), - extensions=['jinja2.ext.do', 'jinja2.ext.with_', 'jinja2.ext.loopcontrols', JinjaTime], - line_statement_prefix="#", - line_comment_prefix="##" - ) + loader=FileSystemLoader([target_template_dir, working_dir, template_dir], encoding='utf-8', + followlinks=False), extensions=[ + 'jinja2.ext.do', 'jinja2.ext.with_', 'jinja2.ext.loopcontrols', JinjaTime], + line_statement_prefix="#", line_comment_prefix="##") jinja_loaded_template = jinja_env.get_template(str(self._sample.jinja_target_template)) if hasattr(self._sample, 'jinja_variables'): @@ -227,36 +224,40 @@ def gen(self, count, earliest, latest, samplename=None): jinja_loaded_vars["eventgen_count"] = self.current_count jinja_loaded_vars["eventgen_maxcount"] = self.target_count jinja_loaded_vars["eventgen_earliest"] = self.earliest - self.earliest_epoch = (self.earliest - datetime.datetime(1970,1,1)).total_seconds() + self.earliest_epoch = (self.earliest - datetime.datetime(1970, 1, 1)).total_seconds() jinja_loaded_vars["eventgen_earliest_epoch"] = self.earliest_epoch jinja_loaded_vars["eventgen_latest"] = self.latest - jinja_loaded_vars["eventgen_latest_epoch"] = (self.latest - datetime.datetime(1970,1,1)).total_seconds() - self.latest_epoch = (self.latest - datetime.datetime(1970,1,1)).total_seconds() + jinja_loaded_vars["eventgen_latest_epoch"] = (self.latest - datetime.datetime(1970, 1, 1)).total_seconds() + self.latest_epoch = (self.latest - datetime.datetime(1970, 1, 1)).total_seconds() while self.current_count < self.target_count: self.end_of_cycle = False jinja_loaded_vars["eventgen_count"] = self.current_count jinja_loaded_vars["eventgen_target_time_earliest"], jinja_loaded_vars["eventgen_target_time_latest"], \ - jinja_loaded_vars["eventgen_target_time_slice_size"], jinja_loaded_vars["eventgen_target_time_epoch"] = \ - JinjaTime._get_time_slice(self.earliest_epoch, self.latest_epoch, self.target_count, self.current_count, slice_type="random") + jinja_loaded_vars["eventgen_target_time_slice_size"], \ + jinja_loaded_vars["eventgen_target_time_epoch"] = \ + JinjaTime._get_time_slice(self.earliest_epoch, self.latest_epoch, self.target_count, + self.current_count, slice_type="random") self.jinja_stream = jinja_loaded_template.stream(jinja_loaded_vars) lines_out = [] try: for line in self.jinja_stream: if line != "\n": - #TODO: Time can be supported by self._sample.timestamp, should probably set that up in this logic. + # TODO: Time can be supported by self._sample.timestamp, should probably set that up here. try: target_line = json.loads(line) except ValueError as e: self.logger.error("Unable to parse Jinja's return. Line: {0}".format(line)) self.logger.error("Parse Failure Reason: {0}".format(e.message)) - self.logger.error("Please note, you must meet the requirements for json.loads in python if you have not installed ujson. Native python does not support multi-line events.") + self.logger.error( + "Please note, you must meet the requirements for json.loads in python if you have" + + "not installed ujson. Native python does not support multi-line events.") continue current_line_keys = target_line.keys() if "_time" not in current_line_keys: - #TODO: Add a custom exception here + # TODO: Add a custom exception here raise Exception("No _time field supplied, please add time to your jinja template.") if "_raw" not in current_line_keys: - #TODO: Add a custom exception here + # TODO: Add a custom exception here raise Exception("No _raw field supplied, please add time to your jinja template.") if "host" not in current_line_keys: target_line["host"] = self._sample.host @@ -281,7 +282,7 @@ def gen(self, count, earliest, latest, samplename=None): timeDiffFrac = "%d.%06d" % (timeDiff.seconds, timeDiff.microseconds) self.logger.debugv("Interval complete, flushing feed") self._out.flush(endOfInterval=True) - self.logger.info("Generation of sample '%s' completed in %s seconds." % (self._sample.name, timeDiffFrac) ) + self.logger.info("Generation of sample '%s' completed in %s seconds." % (self._sample.name, timeDiffFrac)) return 0 except Exception as e: self.logger.exception(e) diff --git a/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py b/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py index 7e91948c..c58f7bae 100644 --- a/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py +++ b/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py @@ -1,18 +1,21 @@ from __future__ import division -from generatorplugin import GeneratorPlugin -import datetime, time + +import datetime import random +from generatorplugin import GeneratorPlugin + class PerDayVolumeGenerator(GeneratorPlugin): def __init__(self, sample): GeneratorPlugin.__init__(self, sample) - #TODO: Make this work with replay mode. + # TODO: Make this work with replay mode. def gen(self, count, earliest, latest, samplename=None): # count in this plugin is a measurement of byteself._sample. size = count - self.logger.debug("PerDayVolumeGenerator Called with a Size of: %s with Earliest: %s and Latest: %s" % (size, earliest, latest)) + self.logger.debug("PerDayVolumeGenerator Called with a Size of: %s with Earliest: %s and Latest: %s" % + (size, earliest, latest)) # very similar to the default generator. only difference is we go by size instead of count. try: self._sample.loadSample() @@ -21,7 +24,8 @@ def gen(self, count, earliest, latest, samplename=None): self.logger.error("Error loading sample file for sample '%s'" % self._sample.name) return - self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % (self._sample.name, self._sample.app, size, earliest, latest)) + self.logger.debug("Generating sample '%s' in app '%s' with count %d, et: '%s', lt '%s'" % + (self._sample.name, self._sample.app, size, earliest, latest)) startTime = datetime.datetime.now() # Create a counter for the current byte size of the read in samples @@ -33,15 +37,18 @@ def gen(self, count, earliest, latest, samplename=None): eventsDict = [] if self._sample.randomizeEvents: sdlen = len(updated_sample_dict) - self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d bytes" % (self._sample.name, self._sample.app, size)) + self.logger.debugv("Random filling eventsDict for sample '%s' in app '%s' with %d bytes" % + (self._sample.name, self._sample.app, size)) while currentSize < size: - currentevent = updated_sample_dict[random.randint(0, sdlen-1)] + currentevent = updated_sample_dict[random.randint(0, sdlen - 1)] eventsDict.append(currentevent) currentSize += len(currentevent['_raw']) # If we're bundlelines, create count copies of the sampleDict elif self._sample.bundlelines: - self.logger.debugv("Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % (self._sample.name, self._sample.app, size)) + self.logger.debugv( + "Bundlelines, filling eventsDict for sample '%s' in app '%s' with %d copies of sampleDict" % + (self._sample.name, self._sample.app, size)) while currentSize <= size: sizeofsample = sum(len(sample['_raw']) for sample in updated_sample_dict) eventsDict.extend(updated_sample_dict) @@ -62,19 +69,22 @@ def gen(self, count, earliest, latest, samplename=None): sizeremaining = size - currentreadsize targetlinesize = len(updated_sample_dict[targetline]['_raw']) if size < targetlinesize: - self.logger.error("Size is too small for sample {}. For this interval, we need {} bytes but size of one event is {} bytes.".format(self._sample.name, size, targetlinesize)) + self.logger.error( + "Size is too small for sample {}. We need {} bytes but size of one event is {} bytes.".format( + self._sample.name, size, targetlinesize)) break - if targetlinesize <= sizeremaining or targetlinesize*.9 <= sizeremaining: + if targetlinesize <= sizeremaining or targetlinesize * .9 <= sizeremaining: currentreadsize += targetlinesize eventsDict.append(updated_sample_dict[targetline]) else: break linecount += 1 - self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" % (self._sample.name, self._sample.app, len(eventsDict))) + self.logger.debugv("Events fill complete for sample '%s' in app '%s' length %d" % + (self._sample.name, self._sample.app, len(eventsDict))) # Ignore token replacement here because we completed it at the beginning of event generation GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest, ignore_tokens=True) def load(): - return PerDayVolumeGenerator \ No newline at end of file + return PerDayVolumeGenerator diff --git a/splunk_eventgen/lib/plugins/generator/replay.py b/splunk_eventgen/lib/plugins/generator/replay.py index 8854f392..5f1a6900 100644 --- a/splunk_eventgen/lib/plugins/generator/replay.py +++ b/splunk_eventgen/lib/plugins/generator/replay.py @@ -1,12 +1,12 @@ # TODO Add timestamp detection for common timestamp format from __future__ import division -from generatorplugin import GeneratorPlugin -import datetime, time -import re -from eventgentimestamp import EventgenTimestamp +import datetime +import time +from eventgentimestamp import EventgenTimestamp +from generatorplugin import GeneratorPlugin class ReplayGenerator(GeneratorPlugin): @@ -22,14 +22,12 @@ def __init__(self, sample): self._currentevent = 0 self._timeSinceSleep = datetime.timedelta() - self._times = [ ] - - + self._times = [] def set_time_and_send(self, rpevent, event_time, earliest, latest): # temporary time append rpevent['_raw'] = rpevent['_raw'][:-1] - rpevent['_time'] = (event_time - datetime.datetime(1970,1,1)).total_seconds() + rpevent['_time'] = (event_time - datetime.datetime(1970, 1, 1)).total_seconds() event = rpevent['_raw'] @@ -38,10 +36,10 @@ def set_time_and_send(self, rpevent, event_time, earliest, latest): # picked from a random line in that file mvhash = {} - ## Iterate tokens + # Iterate tokens for token in self._sample.tokens: token.mvhash = mvhash - if token.replacementType in ['timestamp', 'replaytimestamp'] : + if token.replacementType in ['timestamp', 'replaytimestamp']: event = token.replace(event, et=event_time, lt=event_time, s=self._sample) else: event = token.replace(event, s=self._sample) @@ -68,7 +66,8 @@ def gen(self, count, earliest, latest, samplename=None): self.backfill_time = self._sample.get_backfill_time(self.current_time) if not self._sample.backfill or self._sample.backfilldone: - self.backfill_time = EventgenTimestamp.get_random_timestamp_backfill(earliest, latest, self._sample.earliest, self._sample.latest) + self.backfill_time = EventgenTimestamp.get_random_timestamp_backfill( + earliest, latest, self._sample.earliest, self._sample.latest) for line in self._sample.get_loaded_sample(): # Add newline to a raw line if necessary @@ -81,27 +80,29 @@ def gen(self, count, earliest, latest, samplename=None): hostRegex = line.get('hostRegex', self._sample.hostRegex) source = line.get('source', self._sample.source) sourcetype = line.get('sourcetype', self._sample.sourcetype) - rpevent = {'_raw': line['_raw'], 'index': index, 'host': host, 'hostRegex': hostRegex, - 'source': source, 'sourcetype': sourcetype} + rpevent = { + '_raw': line['_raw'], 'index': index, 'host': host, 'hostRegex': hostRegex, 'source': source, + 'sourcetype': sourcetype} except: if line[-1] != '\n': line += '\n' - rpevent = {'_raw': line, 'index': self._sample.index, 'host': self._sample.host, - 'hostRegex': self._sample.hostRegex, - 'source': self._sample.source, 'sourcetype': self._sample.sourcetype} + rpevent = { + '_raw': line, 'index': self._sample.index, 'host': self._sample.host, 'hostRegex': + self._sample.hostRegex, 'source': self._sample.source, 'sourcetype': self._sample.sourcetype} # If timestamp doesn't exist, the sample file should be fixed to include timestamp for every event. try: current_event_timestamp = self._sample.getTSFromEvent(rpevent[self._sample.timeField]) - except Exception as e: + except Exception: try: current_event_timestamp = self._sample.getTSFromEvent(line[self._sample.timeField]) - except Exception as e: + except Exception: try: - self.logger.debug("Sample timeField {} failed to locate. Trying to locate _time field.".format(self._sample.timeField)) + self.logger.debug("Sample timeField {} failed to locate. Trying to locate _time field.".format( + self._sample.timeField)) current_event_timestamp = self._sample.getTSFromEvent(line["_time"]) - except Exception as e: + except Exception: self.logger.exception("Extracting timestamp from an event failed.") continue @@ -130,5 +131,6 @@ def gen(self, count, earliest, latest, samplename=None): self._out.flush(endOfInterval=True) return + def load(): return ReplayGenerator diff --git a/splunk_eventgen/lib/plugins/generator/weblog.py b/splunk_eventgen/lib/plugins/generator/weblog.py index 39cda67f..b6c0e841 100755 --- a/splunk_eventgen/lib/plugins/generator/weblog.py +++ b/splunk_eventgen/lib/plugins/generator/weblog.py @@ -1,7 +1,9 @@ from __future__ import division -from generatorplugin import GeneratorPlugin -import time + import random +import time + +from generatorplugin import GeneratorPlugin class WeblogGenerator(GeneratorPlugin): @@ -30,25 +32,22 @@ def __init__(self, sample): def gen(self, count, earliest, latest, **kwargs): # logger.debug("weblog: external_ips_len: %s webhosts_len: %s useragents_len: %s webserverstatus_len: %s" % \ - # (self.external_ips_len, self.webhosts_len, self.useragents_len, self.webserverstatus_len)) - l = [ { '_raw': ('%s %s - - [%s] ' - + '"GET /product.screen?product_id=HolyGouda&JSESSIONID=SD3SL1FF7ADFF8 HTTP 1.1" ' - + '%s %s "http://shop.buttercupgames.com/cart.do?action=view&itemId=HolyGouda" ' - + '"%s" %s') % \ - (self.external_ips[random.randint(0, self.external_ips_len-1)], - self.webhosts[random.randint(0, self.webhosts_len-1)], - latest.strftime('%d/%b/%Y %H:%M:%S:%f'), - self.webserverstatus[random.randint(0, self.webserverstatus_len-1)], - random.randint(100, 1000), - self.useragents[random.randint(0, self.useragents_len-1)], - random.randint(200, 2000)), - 'index': self._sample.index, - 'sourcetype': self._sample.sourcetype, - 'host': self._sample.host, - 'source': self._sample.source, - '_time': int(time.mktime(latest.timetuple())) } for i in xrange(count) ] - - self._out.bulksend(l) + # (self.external_ips_len, self.webhosts_len, self.useragents_len, self.webserverstatus_len)) + payload = [{ + '_raw': + ('%s %s - - [%s] ' + '"GET /product.screen?product_id=HolyGouda&JSESSIONID=SD3SL1FF7ADFF8 HTTP 1.1" ' + + '%s %s "http://shop.buttercupgames.com/cart.do?action=view&itemId=HolyGouda" ' + '"%s" %s') % + (self.external_ips[random.randint(0, self.external_ips_len - 1)], self.webhosts[random.randint( + 0, self.webhosts_len - 1)], latest.strftime('%d/%b/%Y %H:%M:%S:%f'), + self.webserverstatus[random.randint(0, self.webserverstatus_len - 1)], random.randint(100, 1000), + self.useragents[random.randint(0, self.useragents_len - 1)], random.randint(200, 2000)), 'index': + self._sample.index, 'sourcetype': + self._sample.sourcetype, 'host': + self._sample.host, 'source': + self._sample.source, '_time': + int(time.mktime(latest.timetuple()))} for i in xrange(count)] + + self._out.bulksend(payload) return 0 diff --git a/splunk_eventgen/lib/plugins/generator/windbag.py b/splunk_eventgen/lib/plugins/generator/windbag.py index a8276382..4f2dd00e 100644 --- a/splunk_eventgen/lib/plugins/generator/windbag.py +++ b/splunk_eventgen/lib/plugins/generator/windbag.py @@ -1,8 +1,11 @@ from __future__ import division -from generatorplugin import GeneratorPlugin + import datetime from datetime import timedelta +from generatorplugin import GeneratorPlugin + + class WindbagGenerator(GeneratorPlugin): def __init__(self, sample): GeneratorPlugin.__init__(self, sample) @@ -13,8 +16,8 @@ def gen(self, count, earliest, latest, samplename=None): count = 60 time_interval = timedelta.total_seconds((latest - earliest)) / count for i in xrange(count): - current_time_object = earliest + datetime.timedelta(0, time_interval*(i+1)) - msg = '{0} -0700 WINDBAG Event {1} of {2}'.format(current_time_object, (i+1), count) + current_time_object = earliest + datetime.timedelta(0, time_interval * (i + 1)) + msg = '{0} -0700 WINDBAG Event {1} of {2}'.format(current_time_object, (i + 1), count) self._out.send(msg) return 0 diff --git a/splunk_eventgen/lib/plugins/output/awss3.py b/splunk_eventgen/lib/plugins/output/awss3.py index 0e9210d9..223d90fa 100644 --- a/splunk_eventgen/lib/plugins/output/awss3.py +++ b/splunk_eventgen/lib/plugins/output/awss3.py @@ -1,16 +1,20 @@ from __future__ import division -from outputplugin import OutputPlugin + +import datetime +import logging +import threading +import uuid + import requests + +from outputplugin import OutputPlugin + try: import boto3 import botocore.exceptions boto_imported = True except ImportError: boto_imported = False -import uuid -import datetime -import threading -import logging def threaded(fn): @@ -31,18 +35,14 @@ class AwsS3OutputPlugin(OutputPlugin): name = 'awsS3' useOutputQueue = False # MAXQUEUELENGTH = 100 - validSettings = ['awsS3BucketName', 'awsS3CompressionType', - 'awsS3EventType', 'awsS3ObjectPrefix', - 'awsS3ObjectSuffix', 'awsRegion', 'awsKeyId', - 'awsSecretKey', 'awsS3EventPerKey'] - defaultableSettings = ['awsKeyId', 'awsSecretKey', 'awsS3EventType', - 'awsS3CompressionType', 'awsS3ObjectPrefix', - 'awsS3ObjectSuffix'] + validSettings = [ + 'awsS3BucketName', 'awsS3CompressionType', 'awsS3EventType', 'awsS3ObjectPrefix', 'awsS3ObjectSuffix', + 'awsRegion', 'awsKeyId', 'awsSecretKey', 'awsS3EventPerKey'] + defaultableSettings = [ + 'awsKeyId', 'awsSecretKey', 'awsS3EventType', 'awsS3CompressionType', 'awsS3ObjectPrefix', 'awsS3ObjectSuffix'] def __init__(self, sample, output_counter=None): - - # Override maxQueueLength to EventPerKey so that each flush # will generate one aws key if sample.awsS3EventPerKey: @@ -59,17 +59,15 @@ def __init__(self, sample, output_counter=None): # Bind passed in samples to the outputter. self.awsS3compressiontype = sample.awsS3CompressionType if hasattr( - sample, - 'awsS3CompressionType') and sample.awsS3CompressionType else None - self.awsS3eventtype = sample.awsS3EventType if hasattr( - sample, 'awsS3EventType') and sample.awsS3EventType else 'syslog' + sample, 'awsS3CompressionType') and sample.awsS3CompressionType else None + self.awsS3eventtype = sample.awsS3EventType if hasattr(sample, + 'awsS3EventType') and sample.awsS3EventType else 'syslog' self.awsS3objectprefix = sample.awsS3ObjectPrefix if hasattr( sample, 'awsS3ObjectPrefix') and sample.awsS3ObjectPrefix else "" self.awsS3objectsuffix = sample.awsS3ObjectSuffix if hasattr( sample, 'awsS3ObjectSuffix') and sample.awsS3ObjectSuffix else "" self.awsS3bucketname = sample.awsS3BucketName - self.logger.debug("Setting up the connection pool for %s in %s" % - (self._sample.name, self._app)) + self.logger.debug("Setting up the connection pool for %s in %s" % (self._sample.name, self._app)) self._client = None self._createConnections(sample) self.logger.debug("Finished init of awsS3 plugin.") @@ -77,11 +75,8 @@ def __init__(self, sample, output_counter=None): def _createConnections(self, sample): try: if hasattr(sample, 'awsKeyId') and hasattr(sample, 'awsSecretKey'): - self._client = boto3.client( - "s3", - region_name=sample.awsRegion, - aws_access_key_id=sample.awsKeyId, - aws_secret_access_key=sample.awsSecretKey) + self._client = boto3.client("s3", region_name=sample.awsRegion, aws_access_key_id=sample.awsKeyId, + aws_secret_access_key=sample.awsSecretKey) if self._client is None: msg = ''' [your_eventgen_stanza] @@ -89,10 +84,9 @@ def _createConnections(self, sample): awsSecretKey = YOUR_SECRET_KEY ''' - self.logger.error( - "Failed for init boto3 client: %s, you should define correct 'awsKeyId'\ + self.logger.error("Failed for init boto3 client: %s, you should define correct 'awsKeyId'\ and 'awsSecretKey' in eventgen conf %s" % msg) - raise + raise Exception(msg) else: self._client = boto3.client('s3', region_name=sample.awsRegion) except Exception as e: @@ -109,38 +103,29 @@ def _createConnections(self, sample): ''' self.logger.error("Failed for init boto3 client, you should create " - "'~/.aws/credentials' with credential info %s" % msg) + "'~/.aws/credentials' with credential info %s" % msg) raise self.logger.debug("Init conn done, conn = %s" % self._client) def _sendPayloads(self, payload): - currentreadsize = 0 - currentreadevent = 0 - stringpayload = [] - totalbytesexpected = 0 - totalbytessent = 0 numberevents = len(payload) self.logger.debug("Sending %s events to s3 key" % numberevents) self._transmitEvents(payload) def _transmitEvents(self, payloadstring): - self.logger.debug("Transmission called with payloadstring event number: %d " - % len(payloadstring)) + self.logger.debug("Transmission called with payloadstring event number: %d " % len(payloadstring)) records = "".join(payloadstring) # Different key prefix for different log type if self.awsS3eventtype == 'elbaccesslog': - s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow( - ).strftime("%Y%m%dT%H%MZ") + '_' + str(uuid.uuid1( - )) + self.awsS3objectsuffix + s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow().strftime("%Y%m%dT%H%MZ") + '_' + str( + uuid.uuid1()) + self.awsS3objectsuffix elif self.awsS3eventtype == 's3accesslog': - s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow( - ).strftime("%Y-%m-%d-%H-%M-%S") + '-' + str(uuid.uuid1()).replace( - '-', '').upper()[0:15] + self.awsS3objectsuffix + s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") + '-' + str( + uuid.uuid1()).replace('-', '').upper()[0:15] + self.awsS3objectsuffix else: - s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow( - ).isoformat() + str(uuid.uuid1()) + self.awsS3objectsuffix - self.logger.debugv("Uploading %d events into s3 key: %s " % - (len(records), s3keyname)) + s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow().isoformat() + str( + uuid.uuid1()) + self.awsS3objectsuffix + self.logger.debug("Uploading %d events into s3 key: %s " % (len(records), s3keyname)) if self.awsS3compressiontype == 'gz': import StringIO import gzip @@ -149,14 +134,11 @@ def _transmitEvents(self, payloadstring): f.write(records) records = out.getvalue() try: - response = self._client.put_object(Bucket=self.awsS3bucketname, - Key=s3keyname, - Body=records) - self.logger.debugv("response = %s" % response) + response = self._client.put_object(Bucket=self.awsS3bucketname, Key=s3keyname, Body=records) + self.logger.debug("response = %s" % response) except Exception as e: self.logger.error("Failed for exception: %s" % e) - self.logger.debugv("Failed sending events to payload: %s" % - (payloadstring)) + self.logger.debug("Failed sending events to payload: %s" % (payloadstring)) raise e def flush(self, q): @@ -167,12 +149,10 @@ def flush(self, q): self.logger.debug("Currently being called with %d events" % len(q)) for event in q: if event.get('_raw') is None: - self.logger.error( - 'failure outputting event, does not contain _raw') + self.logger.error('failure outputting event, does not contain _raw') else: payload.append(event['_raw']) - self.logger.debug( - "Finished processing events, sending all to AWS S3") + self.logger.debug("Finished processing events, sending all to AWS S3") self._sendPayloads(payload) except Exception as e: import traceback diff --git a/splunk_eventgen/lib/plugins/output/devnull.py b/splunk_eventgen/lib/plugins/output/devnull.py index 70bbdde6..faf26a81 100755 --- a/splunk_eventgen/lib/plugins/output/devnull.py +++ b/splunk_eventgen/lib/plugins/output/devnull.py @@ -1,7 +1,10 @@ from __future__ import division -from outputplugin import OutputPlugin + import logging +from outputplugin import OutputPlugin + + class DevNullOutputPlugin(OutputPlugin): name = 'devnull' MAXQUEUELENGTH = 1000 @@ -21,6 +24,7 @@ def flush(self, q): def _setup_logging(self): self.logger = logging.getLogger('eventgen_devnullout') + def load(): """Returns an instance of the plugin""" return DevNullOutputPlugin diff --git a/splunk_eventgen/lib/plugins/output/file.py b/splunk_eventgen/lib/plugins/output/file.py index d04b23a1..3287c49d 100644 --- a/splunk_eventgen/lib/plugins/output/file.py +++ b/splunk_eventgen/lib/plugins/output/file.py @@ -1,9 +1,11 @@ # Note as implemented this plugin is not threadsafe, file should only be used with one output worker from __future__ import division -from outputplugin import OutputPlugin -import os + import logging +import os + +from outputplugin import OutputPlugin class FileOutputPlugin(OutputPlugin): @@ -11,28 +13,29 @@ class FileOutputPlugin(OutputPlugin): MAXQUEUELENGTH = 10 useOutputQueue = False - validSettings = [ 'fileMaxBytes', 'fileBackupFiles' ] - intSettings = [ 'fileMaxBytes', 'fileBackupFiles' ] + validSettings = ['fileMaxBytes', 'fileBackupFiles'] + intSettings = ['fileMaxBytes', 'fileBackupFiles'] def __init__(self, sample, output_counter=None): OutputPlugin.__init__(self, sample, output_counter) - if sample.fileName == None: + if sample.fileName is None: self.logger.error('outputMode file but file not specified for sample %s' % self._sample.name) raise ValueError('outputMode file but file not specified for sample %s' % self._sample.name) - + self._file = sample.pathParser(sample.fileName) self._fileMaxBytes = sample.fileMaxBytes self._fileBackupFiles = sample.fileBackupFiles self._fileHandle = open(self._file, 'a') self._fileLength = os.stat(self._file).st_size - self.logger.debug("Configured to log to '%s' with maxBytes '%s' with backupCount '%s'" % \ - (self._file, self._fileMaxBytes, self._fileBackupFiles)) + self.logger.debug("Configured to log to '%s' with maxBytes '%s' with backupCount '%s'" % + (self._file, self._fileMaxBytes, self._fileBackupFiles)) def flush(self, q): if len(q) > 0: - self.logger.debug("Flushing output for sample '%s' in app '%s' for queue '%s'" % (self._sample.name, self._app, self._sample.source)) + self.logger.debug("Flushing output for sample '%s' in app '%s' for queue '%s'" % + (self._sample.name, self._app, self._sample.source)) # Loop through all the messages and build the long string, write once for each flush # This may cause the file exceed the maxFileBytes a little bit but will greatly improve the performance @@ -52,19 +55,22 @@ def flush(self, q): if self._fileLength > self._fileMaxBytes: self._fileHandle.flush() self._fileHandle.close() - if os.path.exists(self._file+'.'+str(self._fileBackupFiles)): - self.logger.debug('File Output: Removing file: %s' % self._file+'.'+str(self._fileBackupFiles)) - os.unlink(self._file+'.'+str(self._fileBackupFiles)) + if os.path.exists(self._file + '.' + str(self._fileBackupFiles)): + self.logger.debug('File Output: Removing file: %s' % self._file + '.' + + str(self._fileBackupFiles)) + os.unlink(self._file + '.' + str(self._fileBackupFiles)) for x in range(1, self._fileBackupFiles)[::-1]: - self.logger.debug('File Output: Checking for file: %s' % self._file+'.'+str(x)) - if os.path.exists(self._file+'.'+str(x)): - self.logger.debug('File Output: Renaming file %s to %s' % (self._file+'.'+str(x), self._file+'.'+str(x+1))) - os.rename(self._file+'.'+str(x), self._file+'.'+str(x+1)) - os.rename(self._file, self._file+'.1') + self.logger.debug('File Output: Checking for file: %s' % self._file + '.' + str(x)) + if os.path.exists(self._file + '.' + str(x)): + self.logger.debug('File Output: Renaming file %s to %s' % (self._file + '.' + str(x), + self._file + '.' + str(x + 1))) + os.rename(self._file + '.' + str(x), self._file + '.' + str(x + 1)) + os.rename(self._file, self._file + '.1') self._fileHandle = open(self._file, 'w') self._fileLength = 0 except IndexError: - self.logger.warning("IndexError when writting for app '%s' sample '%s'" % (self._app, self._sample.name)) + self.logger.warning( + "IndexError when writting for app '%s' sample '%s'" % (self._app, self._sample.name)) if not self._fileHandle.closed: self._fileHandle.flush() @@ -75,6 +81,7 @@ def flush(self, q): def _setup_logging(self): self.logger = logging.getLogger('eventgen') + def load(): """Returns an instance of the plugin""" return FileOutputPlugin diff --git a/splunk_eventgen/lib/plugins/output/httpevent.py b/splunk_eventgen/lib/plugins/output/httpevent.py index 107371c5..81082a1c 100644 --- a/splunk_eventgen/lib/plugins/output/httpevent.py +++ b/splunk_eventgen/lib/plugins/output/httpevent.py @@ -1,8 +1,13 @@ from __future__ import division + +import logging +import random +import urllib + from outputplugin import OutputPlugin + try: import requests - import requests_futures from requests import Session from requests_futures.sessions import FuturesSession from concurrent.futures import ThreadPoolExecutor @@ -12,17 +17,17 @@ import ujson as json except: import json -import random -import urllib -import logging + class NoServers(Exception): - def __init__(self,*args,**kwargs): - Exception.__init__(self,*args,**kwargs) + def __init__(self, *args, **kwargs): + Exception.__init__(self, *args, **kwargs) + class BadConnection(Exception): - def __init__(self,*args,**kwargs): - Exception.__init__(self,*args,**kwargs) + def __init__(self, *args, **kwargs): + Exception.__init__(self, *args, **kwargs) + class HTTPEventOutputPlugin(OutputPlugin): ''' @@ -30,9 +35,9 @@ class HTTPEventOutputPlugin(OutputPlugin): to splunk through the HTTP event input. In order to use this output plugin, you will need to supply an attribute 'httpeventServers' as a valid json object. this json object should look like the following: - + {servers:[{ protocol:http/https, address:127.0.0.1, port:8088, key:12345-12345-123123123123123123}]} - + ''' name = 'httpevent' MAXQUEUELENGTH = 1000 @@ -44,11 +49,11 @@ class HTTPEventOutputPlugin(OutputPlugin): def __init__(self, sample, output_counter=None): OutputPlugin.__init__(self, sample, output_counter) - #TODO: make workers a param that can be set in eventgen.conf + # TODO: make workers a param that can be set in eventgen.conf def _setup_REST_workers(self, session=None, workers=10): - #disable any "requests" warnings + # disable any "requests" warnings requests.packages.urllib3.disable_warnings() - #Bind passed in samples to the outputter. + # Bind passed in samples to the outputter. self.lastsourcetype = None if not session: session = Session() @@ -67,8 +72,8 @@ def _urlencode(value): @staticmethod def _bg_convert_json(sess, resp): ''' - Takes a futures session object, and will set the data to a parsed json output. Use this as a background task - for the sesssion queue. Example: future = session.get('http://httpbin.org/get', background_callback=_bg_convert_json) + Takes a futures session object, and sets the data to a parsed json output. Use this as a background task for the + session queue. Example: future = session.get('http://httpbin.org/get', background_callback=_bg_convert_json) :param sess: futures session object. Automatically called on a background_callback as aruguments. :param resp: futures resp object. Automatically called on a background_callback as aruguments. :return: @@ -83,12 +88,14 @@ def _bg_convert_json(sess, resp): def updateConfig(self, config): OutputPlugin.updateConfig(self, config) try: - if hasattr(self.config, 'httpeventServers') == False: + if hasattr(self.config, 'httpeventServers') is False: if hasattr(self._sample, 'httpeventServers'): self.config.httpeventServers = self._sample.httpeventServers else: - self.logger.error('outputMode httpevent but httpeventServers not specified for sample %s' % self._sample.name) - raise NoServers('outputMode httpevent but httpeventServers not specified for sample %s' % self._sample.name) + self.logger.error( + 'outputMode httpevent but httpeventServers not specified for sample %s' % self._sample.name) + raise NoServers( + 'outputMode httpevent but httpeventServers not specified for sample %s' % self._sample.name) # set default output mode to round robin if hasattr(self.config, 'httpeventOutputMode') and self.config.httpeventOutputMode: self.httpeventoutputmode = config.httpeventOutputMode @@ -106,9 +113,9 @@ def updateConfig(self, config): self.httpeventmaxsize = 10000 self.logger.debug("Currentmax size: %s " % self.httpeventmaxsize) if isinstance(config.httpeventServers, str): - self.httpeventServers = json.loads(config.httpeventServers) + self.httpeventServers = json.loads(config.httpeventServers) else: - self.httpeventServers = config.httpeventServers + self.httpeventServers = config.httpeventServers self.logger.debug("Setting up the connection pool for %s in %s" % (self._sample.name, self._app)) self.createConnections() self.logger.debug("Pool created.") @@ -116,27 +123,40 @@ def updateConfig(self, config): except Exception as e: self.logger.exception(e) - def createConnections(self): self.serverPool = [] if self.httpeventServers: for server in self.httpeventServers.get('servers'): if not server.get('address'): - self.logger.error('requested a connection to a httpevent server, but no address specified for sample %s' % self._sample.name) - raise ValueError('requested a connection to a httpevent server, but no address specified for sample %s' % self._sample.name) + self.logger.error( + 'requested a connection to a httpevent server, but no address specified for sample %s' % + self._sample.name) + raise ValueError( + 'requested a connection to a httpevent server, but no address specified for sample %s' % + self._sample.name) if not server.get('port'): - self.logger.error('requested a connection to a httpevent server, but no port specified for server %s' % server) - raise ValueError('requested a connection to a httpevent server, but no port specified for server %s' % server) + self.logger.error( + 'requested a connection to a httpevent server, but no port specified for server %s' % server) + raise ValueError( + 'requested a connection to a httpevent server, but no port specified for server %s' % server) if not server.get('key'): - self.logger.error('requested a connection to a httpevent server, but no key specified for server %s' % server) - raise ValueError('requested a connection to a httpevent server, but no key specified for server %s' % server) + self.logger.error( + 'requested a connection to a httpevent server, but no key specified for server %s' % server) + raise ValueError( + 'requested a connection to a httpevent server, but no key specified for server %s' % server) if not ((server.get('protocol') == 'http') or (server.get('protocol') == 'https')): - self.logger.error('requested a connection to a httpevent server, but no protocol specified for server %s' % server) - raise ValueError('requested a connection to a httpevent server, but no protocol specified for server %s' % server) - self.logger.debug("Validation Passed, Creating a requests object for server: %s" % server.get('address')) + self.logger.error( + 'requested a connection to a httpevent server, but no protocol specified for server %s' % + server) + raise ValueError( + 'requested a connection to a httpevent server, but no protocol specified for server %s' % + server) + self.logger.debug( + "Validation Passed, Creating a requests object for server: %s" % server.get('address')) setserver = {} - setserver['url'] = "%s://%s:%s/services/collector" % (server.get('protocol'), server.get('address'), server.get('port')) + setserver['url'] = "%s://%s:%s/services/collector" % (server.get('protocol'), server.get('address'), + server.get('port')) setserver['header'] = "Splunk %s" % server.get('key') self.logger.debug("Adding server set to pool, server: %s" % setserver) self.serverPool.append(setserver) @@ -151,15 +171,15 @@ def _sendHTTPEvents(self, payload): numberevents = len(payload) self.logger.debug("Sending %s events to splunk" % numberevents) for line in payload: - self.logger.debugv("line: %s " % line) + self.logger.debug("line: %s " % line) targetline = json.dumps(line) - self.logger.debugv("targetline: %s " % targetline) + self.logger.debug("targetline: %s " % targetline) targetlinesize = len(targetline) totalbytesexpected += targetlinesize if (int(currentreadsize) + int(targetlinesize)) <= int(self.httpeventmaxsize): stringpayload = stringpayload + targetline currentreadsize = currentreadsize + targetlinesize - self.logger.debugv("stringpayload: %s " % stringpayload) + self.logger.debug("stringpayload: %s " % stringpayload) else: self.logger.debug("Max size for payload hit, sending to splunk then continuing.") try: @@ -173,7 +193,9 @@ def _sendHTTPEvents(self, payload): else: try: totalbytessent += len(stringpayload) - self.logger.debug("End of for loop hit for sending events to splunk, total bytes sent: %s ---- out of %s -----" % (totalbytessent, totalbytesexpected)) + self.logger.debug( + "End of for loop hit for sending events to splunk, total bytes sent: %s ---- out of %s -----" % + (totalbytessent, totalbytesexpected)) self._transmitEvents(stringpayload) except Exception as e: self.logger.exception(e) @@ -181,7 +203,7 @@ def _sendHTTPEvents(self, payload): def _transmitEvents(self, payloadstring): targetServer = [] - self.logger.debugv("Transmission called with payloadstring: %s " % payloadstring) + self.logger.debug("Transmission called with payloadstring: %s " % payloadstring) if self.httpeventoutputmode == "mirror": targetServer = self.serverPool else: @@ -194,12 +216,15 @@ def _transmitEvents(self, payloadstring): headers['content-type'] = 'application/json' try: payloadsize = len(payloadstring) - #response = requests.post(url, data=payloadstring, headers=headers, verify=False) - self.active_sessions.append(self.session.post(url=url, data=payloadstring, headers=headers, verify=False)) + # response = requests.post(url, data=payloadstring, headers=headers, verify=False) + self.active_sessions.append( + self.session.post(url=url, data=payloadstring, headers=headers, verify=False)) except Exception as e: self.logger.error("Failed for exception: %s" % e) - self.logger.error("Failed sending events to url: %s sourcetype: %s size: %s" % (url, self.lastsourcetype, payloadsize)) - self.logger.debugv("Failed sending events to url: %s headers: %s payload: %s" % (url, headers, payloadstring)) + self.logger.error("Failed sending events to url: %s sourcetype: %s size: %s" % + (url, self.lastsourcetype, payloadsize)) + self.logger.debug( + "Failed sending events to url: %s headers: %s payload: %s" % (url, headers, payloadstring)) raise e def flush(self, q): @@ -208,39 +233,37 @@ def flush(self, q): if len(q) > 0: try: payload = [] - lastsourcetype = "" - payloadsize = 0 self.logger.debug("Currently being called with %d events" % len(q)) for event in q: - self.logger.debugv("HTTPEvent proccessing event: %s" % event) + self.logger.debug("HTTPEvent proccessing event: %s" % event) payloadFragment = {} - if event.get('_raw') == None or event['_raw'] == "\n": + if event.get('_raw') is None or event['_raw'] == "\n": self.logger.error('failure outputting event, does not contain _raw') else: - self.logger.debugv("Event contains _raw, attempting to process...") + self.logger.debug("Event contains _raw, attempting to process...") payloadFragment['event'] = event['_raw'] if event.get('source'): - self.logger.debugv("Event contains source, adding to httpevent event") + self.logger.debug("Event contains source, adding to httpevent event") payloadFragment['source'] = event['source'] if event.get('sourcetype'): - self.logger.debugv("Event contains sourcetype, adding to httpevent event") + self.logger.debug("Event contains sourcetype, adding to httpevent event") payloadFragment['sourcetype'] = event['sourcetype'] self.lastsourcetype = event['sourcetype'] if event.get('host'): - self.logger.debugv("Event contains host, adding to httpevent event") + self.logger.debug("Event contains host, adding to httpevent event") payloadFragment['host'] = event['host'] if event.get('_time'): # make sure _time can be an epoch timestamp try: float(event.get("_time")) - self.logger.debugv("Event contains _time, adding to httpevent event") + self.logger.debug("Event contains _time, adding to httpevent event") payloadFragment['time'] = event['_time'] except: self.logger.error("Timestamp not in epoch format, ignoring event: {0}".format(event)) if event.get('index'): - self.logger.debugv("Event contains index, adding to httpevent event") + self.logger.debug("Event contains index, adding to httpevent event") payloadFragment['index'] = event['index'] - self.logger.debugv("Full payloadFragment: %s" % json.dumps(payloadFragment)) + self.logger.debug("Full payloadFragment: %s" % json.dumps(payloadFragment)) payload.append(payloadFragment) self.logger.debug("Finished processing events, sending all to splunk") self._sendHTTPEvents(payload) @@ -250,8 +273,10 @@ def flush(self, q): if not response.raise_for_status(): self.logger.debug("Payload successfully sent to httpevent server.") else: - self.logger.error("Server returned an error while trying to send, response code: %s" % response.status_code) - raise BadConnection("Server returned an error while sending, response code: %s" % response.status_code) + self.logger.error("Server returned an error while trying to send, response code: %s" % + response.status_code) + raise BadConnection( + "Server returned an error while sending, response code: %s" % response.status_code) else: self.logger.debug("Ignoring response from HTTP server, leaving httpevent outputter") except Exception as e: @@ -260,6 +285,7 @@ def flush(self, q): def _setup_logging(self): self.logger = logging.getLogger('eventgen_httpeventout') + def load(): """Returns an instance of the plugin""" return HTTPEventOutputPlugin diff --git a/splunk_eventgen/lib/plugins/output/modinput.py b/splunk_eventgen/lib/plugins/output/modinput.py index ac748396..cd2b792b 100644 --- a/splunk_eventgen/lib/plugins/output/modinput.py +++ b/splunk_eventgen/lib/plugins/output/modinput.py @@ -4,10 +4,13 @@ # from eventgenoutputtemplates import OutputTemplate from __future__ import division -from outputplugin import OutputPlugin + +import logging import sys from xml.sax.saxutils import escape -import logging + +from outputplugin import OutputPlugin + class ModInputOutputPlugin(OutputPlugin): name = 'modinput' @@ -45,6 +48,7 @@ def flush(self, q): def _setup_logging(self): self.logger = logging.getLogger('eventgen') + def load(): """Returns an instance of the plugin""" - return ModInputOutputPlugin \ No newline at end of file + return ModInputOutputPlugin diff --git a/splunk_eventgen/lib/plugins/output/s2s.py b/splunk_eventgen/lib/plugins/output/s2s.py index 26f132e3..6798dbcc 100644 --- a/splunk_eventgen/lib/plugins/output/s2s.py +++ b/splunk_eventgen/lib/plugins/output/s2s.py @@ -1,9 +1,10 @@ from __future__ import division -from outputplugin import OutputPlugin -import struct -import socket import logging +import socket +import struct + +from outputplugin import OutputPlugin class S2S: @@ -60,7 +61,7 @@ def _encode_string(self, tosend=''): by a null terminated string. """ tosend = str(tosend) - return struct.pack('!I%ds' % (len(tosend)+1), len(tosend)+1, tosend) + return struct.pack('!I%ds' % (len(tosend) + 1), len(tosend) + 1, tosend) def _encode_key_value(self, key='', value=''): """ @@ -74,30 +75,29 @@ def _encode_event(self, index='main', host='', source='', sourcetype='', _raw='_ # Create signature sig = self._encode_sig() - msg_size = len(struct.pack('!I', 0)) # size of unsigned 32 bit integer, which is the count of map entries + msg_size = len(struct.pack('!I', 0)) # size of unsigned 32 bit integer, which is the count of map entries maps = 1 # May not have these, so set them first encoded_source = False encoded_sourcetype = False encoded_host = False - encoded_index = False # Encode source if len(source) > 0: - encoded_source = self._encode_key_value('MetaData:Source', 'source::'+source) + encoded_source = self._encode_key_value('MetaData:Source', 'source::' + source) maps += 1 msg_size += len(encoded_source) # Encode sourcetype if len(sourcetype) > 0: - encoded_sourcetype = self._encode_key_value('MetaData:Sourcetype', 'sourcetype::'+sourcetype) + encoded_sourcetype = self._encode_key_value('MetaData:Sourcetype', 'sourcetype::' + sourcetype) maps += 1 msg_size += len(encoded_sourcetype) - + # Encode host if len(host) > 0: - encoded_host = self._encode_key_value('MetaData:Host', 'host::'+host) + encoded_host = self._encode_key_value('MetaData:Host', 'host::' + host) maps += 1 msg_size += len(encoded_host) @@ -105,7 +105,7 @@ def _encode_event(self, index='main', host='', source='', sourcetype='', _raw='_ encoded_index = self._encode_key_value('_MetaData:Index', index) maps += 1 msg_size += len(encoded_index) - + # Encode _raw encoded_raw = self._encode_key_value('_raw', _raw) msg_size += len(encoded_raw) @@ -124,7 +124,7 @@ def _encode_event(self, index='main', host='', source='', sourcetype='', _raw='_ msg_size += len(encoded_done) # Encode _time - if _time != None: + if _time is not None: encoded_time = self._encode_key_value('_time', _time) msg_size += len(encoded_time) maps += 1 @@ -163,6 +163,7 @@ def close(self): """ self.s.close() + class S2SOutputPlugin(OutputPlugin): name = 's2s' MAXQUEUELENGTH = 10 diff --git a/splunk_eventgen/lib/plugins/output/splunkstream.py b/splunk_eventgen/lib/plugins/output/splunkstream.py index 39fb8432..e258872c 100644 --- a/splunk_eventgen/lib/plugins/output/splunkstream.py +++ b/splunk_eventgen/lib/plugins/output/splunkstream.py @@ -1,10 +1,14 @@ from __future__ import division -from outputplugin import OutputPlugin -from xml.dom import minidom -from collections import deque -import httplib, httplib2 -import urllib + +import httplib import logging +import urllib +from collections import deque +from xml.dom import minidom + +import httplib2 + +from outputplugin import OutputPlugin class SplunkStreamOutputPlugin(OutputPlugin): @@ -29,24 +33,31 @@ def __init__(self, sample, output_counter=None): from eventgenconfig import Config globals()['c'] = Config() - self._splunkUrl, self._splunkMethod, self._splunkHost, self._splunkPort = c.getSplunkUrl(self._sample) + self._splunkUrl, self._splunkMethod, self._splunkHost, self._splunkPort = c.getSplunkUrl(self._sample) # noqa self._splunkUser = self._sample.splunkUser self._splunkPass = self._sample.splunkPass if not self._sample.sessionKey: try: myhttp = httplib2.Http(disable_ssl_certificate_validation=True) - self.logger.debug("Getting session key from '%s' with user '%s' and pass '%s'" % (self._splunkUrl + '/services/auth/login', self._splunkUser, self._splunkPass)) - response = myhttp.request(self._splunkUrl + '/services/auth/login', 'POST', - headers = {}, body=urllib.urlencode({'username': self._splunkUser, - 'password': self._splunkPass}))[1] - self._sample.sessionKey = minidom.parseString(response).getElementsByTagName('sessionKey')[0].childNodes[0].nodeValue + self.logger.debug("Getting session key from '%s' with user '%s' and pass '%s'" % + (self._splunkUrl + '/services/auth/login', self._splunkUser, self._splunkPass)) + response = myhttp.request( + self._splunkUrl + '/services/auth/login', 'POST', headers={}, body=urllib.urlencode({ + 'username': + self._splunkUser, 'password': + self._splunkPass}))[1] + self._sample.sessionKey = minidom.parseString(response).getElementsByTagName( + 'sessionKey')[0].childNodes[0].nodeValue self.logger.debug("Got new session for splunkstream, sessionKey '%s'" % self._sample.sessionKey) except: - self.logger.error("Error getting session key for non-SPLUNK_EMBEEDED for sample '%s'. Credentials are missing or wrong" % self._sample.name) - raise IOError("Error getting session key for non-SPLUNK_EMBEEDED for sample '%s'. Credentials are missing or wrong" % self._sample.name) + self.logger.error("Error getting session key for non-SPLUNK_EMBEEDED for sample '%s'." % + self._sample.name + " Credentials are missing or wrong") + raise IOError("Error getting session key for non-SPLUNK_EMBEEDED for sample '%s'." % self._sample.name + + "Credentials are missing or wrong") - self.logger.debug("Retrieved session key '%s' for Splunk session for sample %s'" % (self._sample.sessionKey, self._sample.name)) + self.logger.debug("Retrieved session key '%s' for Splunk session for sample %s'" % (self._sample.sessionKey, + self._sample.name)) def flush(self, q): if len(q) > 0: @@ -78,7 +89,8 @@ def flush(self, q): except KeyError: pass - self.logger.debug("Flushing output for sample '%s' in app '%s' for queue '%s'" % (self._sample.name, self._app, self._sample.source)) + self.logger.debug("Flushing output for sample '%s' in app '%s' for queue '%s'" % + (self._sample.name, self._app, self._sample.source)) try: if self._splunkMethod == 'https': connmethod = httplib.HTTPSConnection @@ -111,11 +123,14 @@ def flush(self, q): msg = False splunkhttp.request("POST", url, streamout, headers) - self.logger.debug("POSTing to url %s on %s://%s:%s with sessionKey %s" \ - % (url, self._splunkMethod, self._splunkHost, self._splunkPort, self._sample.sessionKey)) + self.logger.debug( + "POSTing to url %s on %s://%s:%s with sessionKey %s" % + (url, self._splunkMethod, self._splunkHost, self._splunkPort, self._sample.sessionKey)) except httplib.HTTPException, e: - self.logger.error('Error connecting to Splunk for logging for sample %s. Exception "%s" Config: %s' % (self._sample.name, e.args, self)) + self.logger.error( + 'Error connecting to Splunk for logging for sample %s. Exception "%s" Config: %s' % + (self._sample.name, e.args, self)) raise IOError('Error connecting to Splunk for logging for sample %s' % self._sample) try: diff --git a/splunk_eventgen/lib/plugins/output/spool.py b/splunk_eventgen/lib/plugins/output/spool.py index cd478123..60a76d69 100644 --- a/splunk_eventgen/lib/plugins/output/spool.py +++ b/splunk_eventgen/lib/plugins/output/spool.py @@ -4,18 +4,21 @@ # from eventgenoutputtemplates import OutputTemplate from __future__ import division -from outputplugin import OutputPlugin -import time -import os + import logging +import os +import time + +from outputplugin import OutputPlugin + class SpoolOutputPlugin(OutputPlugin): useOutputQueue = True name = 'spool' MAXQUEUELENGTH = 10 - validSettings = [ 'spoolDir', 'spoolFile' ] - defaultableSettings = [ 'spoolDir', 'spoolFile' ] + validSettings = ['spoolDir', 'spoolFile'] + defaultableSettings = ['spoolDir', 'spoolFile'] _spoolDir = None _spoolFile = None @@ -28,7 +31,8 @@ def __init__(self, sample, output_counter=None): def flush(self, q): if len(q) > 0: - self.logger.debug("Flushing output for sample '%s' in app '%s' for queue '%s'" % (self._sample.name, self._app, self._sample.source)) + self.logger.debug("Flushing output for sample '%s' in app '%s' for queue '%s'" % + (self._sample.name, self._app, self._sample.source)) # Keep trying to open destination file as it might be touched by other processes data = ''.join(event['_raw'] for event in q if event.get('_raw')) while True: diff --git a/splunk_eventgen/lib/plugins/output/stdout.py b/splunk_eventgen/lib/plugins/output/stdout.py index 3edc44df..a0ea46e1 100644 --- a/splunk_eventgen/lib/plugins/output/stdout.py +++ b/splunk_eventgen/lib/plugins/output/stdout.py @@ -1,7 +1,10 @@ from __future__ import division -from outputplugin import OutputPlugin + import logging +from outputplugin import OutputPlugin + + class StdOutOutputPlugin(OutputPlugin): useOutputQueue = False name = 'stdout' @@ -17,6 +20,7 @@ def flush(self, q): def _setup_logging(self): self.logger = logging.getLogger('eventgen_stdout') + def load(): """Returns an instance of the plugin""" - return StdOutOutputPlugin \ No newline at end of file + return StdOutOutputPlugin diff --git a/splunk_eventgen/lib/plugins/output/syslogout.py b/splunk_eventgen/lib/plugins/output/syslogout.py index eb07a2e1..b1faad28 100644 --- a/splunk_eventgen/lib/plugins/output/syslogout.py +++ b/splunk_eventgen/lib/plugins/output/syslogout.py @@ -1,22 +1,28 @@ from __future__ import division + +import logging +import logging.handlers + from outputplugin import OutputPlugin -import logging, logging.handlers # Dict of flags to gate adding the syslogHandler only once to the given singleton logger loggerInitialized = {} + class SyslogOutOutputPlugin(OutputPlugin): useOutputQueue = True name = 'syslogout' MAXQUEUELENGTH = 10 - validSettings = [ 'syslogDestinationHost', 'syslogDestinationPort' ] - defaultableSettings = [ 'syslogDestinationHost', 'syslogDestinationPort' ] - intSettings = [ 'syslogDestinationPort' ] + validSettings = ['syslogDestinationHost', 'syslogDestinationPort'] + defaultableSettings = ['syslogDestinationHost', 'syslogDestinationPort'] + intSettings = ['syslogDestinationPort'] def __init__(self, sample, output_counter=None): OutputPlugin.__init__(self, sample, output_counter) - self._syslogDestinationHost = sample.syslogDestinationHost if hasattr(sample, 'syslogDestinationHost') and sample.syslogDestinationHost else '127.0.0.1' - self._syslogDestinationPort = sample.syslogDestinationPort if hasattr(sample, 'syslogDestinationPort') and sample.syslogDestinationPort else 1514 + self._syslogDestinationHost = sample.syslogDestinationHost if hasattr( + sample, 'syslogDestinationHost') and sample.syslogDestinationHost else '127.0.0.1' + self._syslogDestinationPort = sample.syslogDestinationPort if hasattr( + sample, 'syslogDestinationPort') and sample.syslogDestinationPort else 1514 loggerName = 'syslog' + sample.name self._l = logging.getLogger(loggerName) @@ -25,8 +31,9 @@ def __init__(self, sample, output_counter=None): global loggerInitialized # This class is instantiated at least once each interval. Since each logger with a given name is a singleton, # only add the syslog handler once instead of every interval. - if not loggerName in loggerInitialized: - syslogHandler = logging.handlers.SysLogHandler(address=(self._syslogDestinationHost, int(self._syslogDestinationPort))) + if loggerName not in loggerInitialized: + syslogHandler = logging.handlers.SysLogHandler( + address=(self._syslogDestinationHost, int(self._syslogDestinationPort))) self._l.addHandler(syslogHandler) loggerInitialized[loggerName] = True @@ -34,6 +41,7 @@ def flush(self, q): for x in q: self._l.info(x['_raw'].rstrip()) + def load(): """Returns an instance of the plugin""" return SyslogOutOutputPlugin diff --git a/splunk_eventgen/lib/plugins/output/tcpout.py b/splunk_eventgen/lib/plugins/output/tcpout.py index 3432d4bd..4658a00f 100644 --- a/splunk_eventgen/lib/plugins/output/tcpout.py +++ b/splunk_eventgen/lib/plugins/output/tcpout.py @@ -1,7 +1,10 @@ from __future__ import division -from outputplugin import OutputPlugin + import logging +from outputplugin import OutputPlugin + + class TcpOutputPlugin(OutputPlugin): useOutputQueue = False name = 'tcpout' @@ -10,13 +13,14 @@ class TcpOutputPlugin(OutputPlugin): def __init__(self, sample, output_counter=None): OutputPlugin.__init__(self, sample, output_counter) - self._tcpDestinationHost = sample.tcpDestinationHost if hasattr(sample,'tcpDestinationHost') and sample.tcpDestinationHost else '127.0.0.1' - self._tcpDestinationPort = sample.tcpDestinationPort if hasattr(sample,'tcpDestinationPort') and sample.tcpDestinationPort else '3333' + self._tcpDestinationHost = sample.tcpDestinationHost if hasattr( + sample, 'tcpDestinationHost') and sample.tcpDestinationHost else '127.0.0.1' + self._tcpDestinationPort = sample.tcpDestinationPort if hasattr( + sample, 'tcpDestinationPort') and sample.tcpDestinationPort else '3333' import socket # Import socket module self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # Bind to the port def flush(self, q): self.s.connect((self._tcpDestinationHost, int(self._tcpDestinationPort))) self.logger.info("Socket connected to {0}:{1}".format(self._tcpDestinationHost, self._tcpDestinationPort)) @@ -27,6 +31,7 @@ def flush(self, q): def _setup_logging(self): self.logger = logging.getLogger('eventgen') + def load(): """Returns an instance of the plugin""" - return TcpOutputPlugin \ No newline at end of file + return TcpOutputPlugin diff --git a/splunk_eventgen/lib/plugins/output/udpout.py b/splunk_eventgen/lib/plugins/output/udpout.py index 43477d0f..30e74324 100644 --- a/splunk_eventgen/lib/plugins/output/udpout.py +++ b/splunk_eventgen/lib/plugins/output/udpout.py @@ -1,7 +1,10 @@ from __future__ import division -from outputplugin import OutputPlugin + import logging +from outputplugin import OutputPlugin + + class UdpOutputPlugin(OutputPlugin): useOutputQueue = False name = 'udpout' @@ -10,8 +13,10 @@ class UdpOutputPlugin(OutputPlugin): def __init__(self, sample, output_counter=None): OutputPlugin.__init__(self, sample, output_counter) - self._udpDestinationHost = sample.udpDestinationHost if hasattr(sample,'udpDestinationHost') and sample.udpDestinationHost else '127.0.0.1' - self._udpDestinationPort = sample.udpDestinationPort if hasattr(sample,'udpDestinationPort') and sample.udpDestinationPort else '3333' + self._udpDestinationHost = sample.udpDestinationHost if hasattr( + sample, 'udpDestinationHost') and sample.udpDestinationHost else '127.0.0.1' + self._udpDestinationPort = sample.udpDestinationPort if hasattr( + sample, 'udpDestinationPort') and sample.udpDestinationPort else '3333' import socket # Import socket module self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) @@ -25,6 +30,7 @@ def flush(self, q): def _setup_logging(self): self.logger = logging.getLogger('eventgen') + def load(): """Returns an instance of the plugin""" - return UdpOutputPlugin \ No newline at end of file + return UdpOutputPlugin diff --git a/splunk_eventgen/lib/plugins/rater/config.py b/splunk_eventgen/lib/plugins/rater/config.py index d15136bd..846b080d 100644 --- a/splunk_eventgen/lib/plugins/rater/config.py +++ b/splunk_eventgen/lib/plugins/rater/config.py @@ -1,9 +1,9 @@ from __future__ import division + +import datetime import logging import logging.handlers -import datetime import random -import os class ConfigRater(object): @@ -21,7 +21,7 @@ def __init__(self, sample): def __str__(self): """Only used for debugging, outputs a pretty printed representation of this output""" # Eliminate recursive going back to parent - temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key != '_c']) + # temp = dict([(key, value) for (key, value) in self.__dict__.items() if key != '_c']) # return pprint.pformat(temp) return "" @@ -50,21 +50,21 @@ def rate(self): self.logger.error('No sample found for default generator, cannot generate events') self._sample.count = len(self._sample.sampleDict) self._generatorWorkers = int(self._generatorWorkers) - count = self._sample.count/self._generatorWorkers + count = self._sample.count / self._generatorWorkers # 5/8/12 CS We've requested not the whole file, so we should adjust count based on # hourOfDay, dayOfWeek and randomizeCount configs rateFactor = 1.0 if self._sample.randomizeCount: try: - self.logger.debug("randomizeCount for sample '%s' in app '%s' is %s" - % (self._sample.name, self._sample.app, self._sample.randomizeCount)) + self.logger.debug("randomizeCount for sample '%s' in app '%s' is %s" % + (self._sample.name, self._sample.app, self._sample.randomizeCount)) # If we say we're going to be 20% variable, then that means we # can be .1% high or .1% low. Math below does that. randBound = round(self._sample.randomizeCount * 1000, 0) rand = random.randint(0, randBound) - randFactor = 1+((-((randBound / 2) - rand)) / 1000) - self.logger.debug("randFactor for sample '%s' in app '%s' is %s" \ - % (self._sample.name, self._sample.app, randFactor)) + randFactor = 1 + ((-((randBound / 2) - rand)) / 1000) + self.logger.debug( + "randFactor for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, randFactor)) rateFactor *= randFactor except: import traceback @@ -73,12 +73,14 @@ def rate(self): if type(self._sample.hourOfDayRate) == dict: try: rate = self._sample.hourOfDayRate[str(self._sample.now().hour)] - self.logger.debug("hourOfDayRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debug( + "hourOfDayRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback stack = traceback.format_exc() - self.logger.error("Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + self.logger.error( + "Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.dayOfWeekRate) == dict: try: weekday = datetime.date.weekday(self._sample.now()) @@ -87,43 +89,52 @@ def rate(self): else: weekday += 1 rate = self._sample.dayOfWeekRate[str(weekday)] - self.logger.debugv("dayOfWeekRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debug( + "dayOfWeekRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback - stack = traceback.format_exc() - self.logger.error("Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + stack = traceback.format_exc() + self.logger.error( + "Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.minuteOfHourRate) == dict: try: rate = self._sample.minuteOfHourRate[str(self._sample.now().minute)] - self.logger.debugv("minuteOfHourRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debug( + "minuteOfHourRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback - stack = traceback.format_exc() - self.logger.error("Minute of hour rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + stack = traceback.format_exc() + self.logger.error( + "Minute of hour rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.dayOfMonthRate) == dict: try: rate = self._sample.dayOfMonthRate[str(self._sample.now().day)] - self.logger.debugv("dayOfMonthRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debug( + "dayOfMonthRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback - stack = traceback.format_exc() - self.logger.error("Day of Month rate for sample '%s' failed. Stacktrace %s" % (self._sample.name, stack)) + stack = traceback.format_exc() + self.logger.error( + "Day of Month rate for sample '%s' failed. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.monthOfYearRate) == dict: try: rate = self._sample.monthOfYearRate[str(self._sample.now().month)] - self.logger.debugv("monthOfYearRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debug( + "monthOfYearRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback - stack = traceback.format_exc() - self.logger.error("Month Of Year rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + stack = traceback.format_exc() + self.logger.error( + "Month Of Year rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) ret = int(round(count * rateFactor, 0)) if rateFactor != 1.0: self.logger.debug("Original count: %s Rated count: %s Rate factor: %s" % (count, ret, rateFactor)) return ret + def load(): return ConfigRater diff --git a/splunk_eventgen/lib/plugins/rater/perdayvolume.py b/splunk_eventgen/lib/plugins/rater/perdayvolume.py index cfdf8746..3c8d217f 100644 --- a/splunk_eventgen/lib/plugins/rater/perdayvolume.py +++ b/splunk_eventgen/lib/plugins/rater/perdayvolume.py @@ -1,7 +1,8 @@ from __future__ import division -from config import ConfigRater + import datetime import random +from config import ConfigRater class PerDayVolume(ConfigRater): @@ -16,7 +17,7 @@ def __init__(self, sample): self._generatorWorkers = self._sample.config.generatorWorkers def rate(self): - perdayvolume = float(self._sample.perDayVolume)/self._generatorWorkers + perdayvolume = float(self._sample.perDayVolume) / self._generatorWorkers # Convert perdayvolume to bytes from GB perdayvolume = perdayvolume * 1024 * 1024 * 1024 interval = self._sample.interval @@ -28,21 +29,20 @@ def rate(self): perintervalvolume = (perdayvolume / intervalsperday) count = self._sample.count - # 5/8/12 CS We've requested not the whole file, so we should adjust count based on # hourOfDay, dayOfWeek and randomizeCount configs rateFactor = 1.0 - if self._sample.randomizeCount != 0 and self._sample.randomizeCount != None: + if self._sample.randomizeCount != 0 and self._sample.randomizeCount is not None: try: - self.logger.debugv("randomizeCount for sample '%s' in app '%s' is %s" \ - % (self._sample.name, self._sample.app, self._sample.randomizeCount)) + self.logger.debugv("randomizeCount for sample '%s' in app '%s' is %s" % + (self._sample.name, self._sample.app, self._sample.randomizeCount)) # If we say we're going to be 20% variable, then that means we # can be .1% high or .1% low. Math below does that. randBound = round(self._sample.randomizeCount * 1000, 0) rand = random.randint(0, randBound) - randFactor = 1+((-((randBound / 2) - rand)) / 1000) - self.logger.debug("randFactor for sample '%s' in app '%s' is %s" \ - % (self._sample.name, self._sample.app, randFactor)) + randFactor = 1 + ((-((randBound / 2) - rand)) / 1000) + self.logger.debug( + "randFactor for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, randFactor)) rateFactor *= randFactor except: import traceback @@ -51,12 +51,14 @@ def rate(self): if type(self._sample.hourOfDayRate) == dict: try: rate = self._sample.hourOfDayRate[str(self._sample.now().hour)] - self.logger.debugv("hourOfDayRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debugv( + "hourOfDayRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback stack = traceback.format_exc() - self.logger.error("Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + self.logger.error( + "Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.dayOfWeekRate) == dict: try: weekday = datetime.date.weekday(self._sample.now()) @@ -65,46 +67,55 @@ def rate(self): else: weekday += 1 rate = self._sample.dayOfWeekRate[str(weekday)] - self.logger.debugv("dayOfWeekRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debugv( + "dayOfWeekRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback stack = traceback.format_exc() - self.logger.error("Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + self.logger.error( + "Hour of day rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.minuteOfHourRate) == dict: try: rate = self._sample.minuteOfHourRate[str(self._sample.now().minute)] - self.logger.debugv("minuteOfHourRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debugv( + "minuteOfHourRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback stack = traceback.format_exc() - self.logger.error("Minute of hour rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + self.logger.error( + "Minute of hour rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.dayOfMonthRate) == dict: try: rate = self._sample.dayOfMonthRate[str(self._sample.now().day)] - self.logger.debugv("dayOfMonthRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debugv( + "dayOfMonthRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback stack = traceback.format_exc() - self.logger.error("Day of Month rate for sample '%s' failed. Stacktrace %s" % (self._sample.name, stack)) + self.logger.error( + "Day of Month rate for sample '%s' failed. Stacktrace %s" % (self._sample.name, stack)) if type(self._sample.monthOfYearRate) == dict: try: rate = self._sample.monthOfYearRate[str(self._sample.now().month)] - self.logger.debugv("monthOfYearRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) + self.logger.debugv( + "monthOfYearRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate)) rateFactor *= rate except KeyError: import traceback stack = traceback.format_exc() - self.logger.error("Month Of Year rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) + self.logger.error( + "Month Of Year rate failed for sample '%s'. Stacktrace %s" % (self._sample.name, stack)) self.logger.debug("Size per interval: %s, rate factor to adjust by: %s" % (perintervalvolume, rateFactor)) ret = int(round(perintervalvolume * rateFactor, 0)) if rateFactor != 1.0: self.logger.debug("Original count: %s Rated count: %s Rate factor: %s" % (count, ret, rateFactor)) - self.logger.debug("Finished rating, interval: {0}s, generation rate: {1} MB/interval".format(interval, round((ret / 1024 / 1024), 4))) + self.logger.debug("Finished rating, interval: {0}s, generation rate: {1} MB/interval".format( + interval, round((ret / 1024 / 1024), 4))) return ret def load(): - return PerDayVolume \ No newline at end of file + return PerDayVolume diff --git a/splunk_eventgen/lib/timeparser.py b/splunk_eventgen/lib/timeparser.py index 4efeab36..020818b8 100644 --- a/splunk_eventgen/lib/timeparser.py +++ b/splunk_eventgen/lib/timeparser.py @@ -1,86 +1,89 @@ import datetime -import re -import math import logging - +import math +import os +import re # Hack to allow distributing python modules since Splunk doesn't have setuptools # We create the egg outside of Splunk (with a copy of python2.7 and using Python only modules # To avoid being platform specific) and then append the egg path and import the module # If we get a lot of these we'll move the eggs from bin to lib # # python-dateutil acquired from http://labix.org/python-dateutil. BSD Licensed -import sys, os +import sys + path_prepend = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'lib') sys.path.append(path_prepend + '/python_dateutil-1.4.1-py2.7.egg') -import dateutil.parser as dateutil_parser +import dateutil.parser as dateutil_parser # noqa isort:skip # If we're inside eventgen, we'll have a global logger, if not set one up logging.getLogger('eventgen') + # 5-5-2012 CS Replacing TimeParser with our own code to remove Splunk dependency -# Based off spec for relative time identifiers at http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/SearchTimeModifiers#How_to_specify_relative_time_modifiers +# Based off spec for relative time identifiers at http://docs.splunk.com/Documentation/Splunk/latest/SearchReference/SearchTimeModifiers#How_to_specify_relative_time_modifiers # noqa # If we're not relative, we'll try to parse it as an ISO compliant time def timeParser(ts='now', timezone=datetime.timedelta(days=1), now=None, utcnow=None): if ts == 'now': if timezone.days > 0: - if now == None: + if now is None: return datetime.datetime.now() else: return now() else: - if utcnow == None: + if utcnow is None: return datetime.datetime.now() else: return utcnow() + timezone else: if ts[:1] == '+' or ts[:1] == '-': if timezone.days > 0: - if now == None: + if now is None: ret = datetime.datetime.now() else: ret = now() else: - if utcnow == None: + if utcnow is None: ret = datetime.datetime.utcnow() + timezone else: ret = utcnow() + timezone - - unitsre = "(seconds|second|secs|sec|minutes|minute|min|hours|hour|hrs|hr|days|day|weeks|week|w[0-6]|months|month|mon|quarters|quarter|qtrs|qtr|years|year|yrs|yr|s|h|m|d|w|y|w|q)" - reltimere = "(?i)(?P[+-]*)(?P\d{1,})(?P"+unitsre+"{1})(([\@](?P"+unitsre+"{1})((?P[+-])(?P\d+)(?P"+unitsre+"{1}))*)*)" - + + unitsre = "(seconds|second|secs|sec|minutes|minute|min|hours|hour|hrs|hr|days|day|weeks|week|w[0-6]|" + \ + "months|month|mon|quarters|quarter|qtrs|qtr|years|year|yrs|yr|s|h|m|d|w|y|w|q)" + reltimere = "(?i)(?P[+-]*)(?P\d{1,})(?P" + unitsre + "{1})(([\@](?P" + \ + unitsre + "{1})((?P[+-])(?P\d+)(?P" + unitsre + \ + "{1}))*)*)" + results = re.match(reltimere, ts) resultsdict = results.groupdict() - + # Handle first part of the time string - if resultsdict['plusminus'] != None and resultsdict['num'] != None \ - and resultsdict['unit'] != None: + if resultsdict['plusminus'] is not None and resultsdict['num'] is not None \ + and resultsdict['unit'] is not None: ret = timeParserTimeMath(resultsdict['plusminus'], resultsdict['num'], resultsdict['unit'], ret) - + # Now handle snap-to - if resultsdict['snapunit'] != None: + if resultsdict['snapunit'] is not None: if resultsdict['snapunit'] in ('s', 'sec', 'secs', 'second', 'seconds'): - ret = datetime.datetime(ret.year, ret.month, ret.day, ret.hour, \ - ret.minute, ret.second, 0) + ret = datetime.datetime(ret.year, ret.month, ret.day, ret.hour, ret.minute, ret.second, 0) elif resultsdict['snapunit'] in ('m', 'min', 'minute', 'minutes'): - ret = datetime.datetime(ret.year, ret.month, ret.day, ret.hour, \ - ret.minute, 0, 0) + ret = datetime.datetime(ret.year, ret.month, ret.day, ret.hour, ret.minute, 0, 0) elif resultsdict['snapunit'] in ('h', 'hr', 'hrs', 'hour', 'hours'): ret = datetime.datetime(ret.year, ret.month, ret.day, ret.hour, 0, 0, 0) elif resultsdict['snapunit'] in ('d', 'day', 'days'): ret = datetime.datetime(ret.year, ret.month, ret.day, 0, 0, 0, 0) - elif re.match('w[0-6]', resultsdict['snapunit']) != None or \ + elif re.match('w[0-6]', resultsdict['snapunit']) is not None or \ resultsdict['snapunit'] in ('w', 'week', 'weeks'): if resultsdict['snapunit'] in ('w', 'week', 'weeks'): resultsdict['snapunit'] = 'w0' weekdaynum = int(resultsdict['snapunit'][1:2]) - + # Convert python's weekdays to Splunk's retweekday = datetime.date.weekday(ret) if retweekday == 6: retweekday = 0 else: retweekday += 1 - + if weekdaynum <= retweekday: ret = ret + datetime.timedelta(days=(weekdaynum - retweekday)) ret = datetime.datetime(ret.year, ret.month, ret.day, 0, 0, 0, 0) @@ -96,23 +99,24 @@ def timeParser(ts='now', timezone=datetime.timedelta(days=1), now=None, utcnow=N ret = datetime.datetime(ret.year, int(math.floor(ret.month / 3.3 + 1) * 3), 1, 0, 0, 0, 0) elif resultsdict['snapunit'] in ('y', 'yr', 'yrs', 'year', 'years'): ret = datetime.datetime(ret.year, 1, 1, 0, 0, 0, 0) - - if resultsdict['snapplusminus'] != None and resultsdict['snaprelnum'] != None \ - and resultsdict['snaprelunit'] != None: - ret = timeParserTimeMath(resultsdict['snapplusminus'], resultsdict['snaprelnum'], - resultsdict['snaprelunit'], ret) + + if resultsdict['snapplusminus'] is not None and resultsdict['snaprelnum'] is not None \ + and resultsdict['snaprelunit'] is not None: + ret = timeParserTimeMath(resultsdict['snapplusminus'], resultsdict['snaprelnum'], + resultsdict['snaprelunit'], ret) return ret - + else: - raise ValueError('Cannot parse relative time string for %s' %(ts)) + raise ValueError('Cannot parse relative time string for %s' % (ts)) else: - # The spec says we must be a ISO8601 time. This parser should be able to handle + # The spec says we must be a ISO8601 time. This parser should be able to handle # more date formats though, so we can be liberal in what we accept return dateutil_parser.parse(ts) - #except ValueError: + # except ValueError: # raise ValueError("Cannot parse date/time for %s" % (ts)) + def timeParserTimeMath(plusminus, num, unit, ret): try: num = int(num) @@ -126,59 +130,59 @@ def timeParserTimeMath(plusminus, num, unit, ret): elif unit in ('d', 'day', 'days'): td = datetime.timedelta(days=int(num)) elif unit in ('w', 'week', 'weeks'): - td = datetime.timedelta(days=(int(num)*7)) - elif re.match('w[0-6]', unit) != None: - logger.error('Day of week is only available in snap-to. Time string: %s' % (ts)) + td = datetime.timedelta(days=(int(num) * 7)) + elif re.match('w[0-6]', unit) is not None: + logging.error('Day of week is only available in snap-to. Time string: %s' % td) return False # Normalize out all year/quarter/months to months and do the math on that - elif unit in ('mon', 'month', 'months') or \ - unit in ('q', 'qtr', 'qtrs', 'quarter', 'quarters') or \ - unit in ('y', 'yr', 'yrs', 'year', 'years'): + elif unit in ('mon', 'month', 'months') or unit in ('q', 'qtr', 'qtrs', 'quarter', 'quarters') or \ + unit in ('y', 'yr', 'yrs', 'year', 'years'): if unit in ('q', 'qtr', 'qtrs', 'quarter', 'quarters'): num *= 3 elif unit in ('y', 'yr', 'yrs', 'year', 'years'): num *= 12 - + monthnum = int(num) * -1 if plusminus == '-' else int(num) if abs(monthnum) / 12 > 0: - yearnum = int(math.floor(abs(monthnum)/12) * -1 if plusminus == '-' else int(math.floor(abs(monthnum)/12))) - monthnum = int((abs(monthnum) % 12) * -1 if plusminus == '-' else int((abs(monthnum)%12))) - ret = datetime.datetime(ret.year + yearnum, ret.month + monthnum, ret.day, ret.hour, - ret.minute, ret.second, ret.microsecond) + yearnum = int( + math.floor(abs(monthnum) / 12) * -1 if plusminus == '-' else int(math.floor(abs(monthnum) / 12))) + monthnum = int((abs(monthnum) % 12) * -1 if plusminus == '-' else int((abs(monthnum) % 12))) + ret = datetime.datetime(ret.year + yearnum, ret.month + monthnum, ret.day, ret.hour, ret.minute, + ret.second, ret.microsecond) elif monthnum > 0: if ret.month + monthnum > 12: - ret = datetime.datetime(ret.year+1, ((ret.month+monthnum)%12), - ret.day, ret.hour, ret.minute, ret.second, ret.microsecond) + ret = datetime.datetime(ret.year + 1, ((ret.month + monthnum) % 12), ret.day, ret.hour, ret.minute, + ret.second, ret.microsecond) else: - ret = datetime.datetime(ret.year, ret.month+monthnum, ret.day, - ret.hour, ret.minute, ret.second, ret.microsecond) + ret = datetime.datetime(ret.year, ret.month + monthnum, ret.day, ret.hour, ret.minute, ret.second, + ret.microsecond) elif monthnum <= 0: if ret.month + monthnum <= 0: - ret = datetime.datetime(ret.year-1, (12-abs(ret.month+monthnum)), - ret.day, ret.hour, ret.minute, ret.second, ret.microsecond) + ret = datetime.datetime(ret.year - 1, (12 - abs(ret.month + monthnum)), ret.day, ret.hour, + ret.minute, ret.second, ret.microsecond) else: - ret = datetime.datetime(ret.year, ret.month+monthnum, ret.day, - ret.hour, ret.minute, ret.second, ret.microsecond) + ret = datetime.datetime(ret.year, ret.month + monthnum, ret.day, ret.hour, ret.minute, ret.second, + ret.microsecond) except ValueError: - logger.error('Cannot parse relative time string') + logging.error('Cannot parse relative time string') import traceback - stack = traceback.format_exc() - logger.debug('%s', stack) + stack = traceback.format_exc() + logging.debug('%s', stack) return False - - if td != None: + + if td is not None: if plusminus == '-': td = td * -1 ret = ret + td - + # Always chop microseconds to maintain compatibility with Splunk's parser ret = datetime.datetime(ret.year, ret.month, ret.day, ret.hour, ret.minute, ret.second) - + return ret - -## Converts Time Delta object to number of seconds in delta + + +# Converts Time Delta object to number of seconds in delta def timeDelta2secs(timeDiff): deltaSecs = (timeDiff.microseconds + (timeDiff.seconds + timeDiff.days * 24 * 3600) * 10**6) / 10**6 return int(deltaSecs) - \ No newline at end of file diff --git a/splunk_eventgen/logger/logger_config.py b/splunk_eventgen/logger/logger_config.py index d2d5b46f..444ef5f9 100644 --- a/splunk_eventgen/logger/logger_config.py +++ b/splunk_eventgen/logger/logger_config.py @@ -2,26 +2,15 @@ 'version': 1, 'formatters': { 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, + 'class': 'logging.Formatter', 'format': + '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s'}}, 'handlers': { 'console': { 'class': 'logging.StreamHandler', 'level': 'INFO', - 'formatter': 'detailed', - }, - 'main': { - 'class': 'logging.FileHandler', - 'filename': 'eventgen-controller-main.log', - 'mode': 'w', - 'formatter': 'detailed', - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console', 'main'] - }, -} - + 'formatter': 'detailed', }, 'main': { + 'class': 'logging.FileHandler', + 'filename': 'eventgen-controller-main.log', + 'mode': 'w', + 'formatter': 'detailed', }}, + 'root': {'level': 'DEBUG', 'handlers': ['console', 'main']}, } diff --git a/splunk_eventgen/logger/requests_futures/__init__.py b/splunk_eventgen/logger/requests_futures/__init__.py index 9ac9cd31..ac0c4f3e 100755 --- a/splunk_eventgen/logger/requests_futures/__init__.py +++ b/splunk_eventgen/logger/requests_futures/__init__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Requests Futures - """ async requests HTTP library ~~~~~~~~~~~~~~~~~~~~~ @@ -22,8 +21,10 @@ try: # Python 2.7+ from logging import NullHandler except ImportError: + class NullHandler(logging.Handler): def emit(self, record): pass + logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/splunk_eventgen/logger/requests_futures/sessions.py b/splunk_eventgen/logger/requests_futures/sessions.py index 7fba4226..643f4e1d 100755 --- a/splunk_eventgen/logger/requests_futures/sessions.py +++ b/splunk_eventgen/logger/requests_futures/sessions.py @@ -19,9 +19,9 @@ print(response.content) """ -from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from functools import partial -from pickle import dumps, PickleError +from pickle import PickleError, dumps from requests import Session from requests.adapters import DEFAULT_POOLSIZE, HTTPAdapter @@ -38,9 +38,7 @@ def wrap(self, sup, background_callback, *args_, **kwargs_): class FuturesSession(Session): - - def __init__(self, executor=None, max_workers=2, session=None, *args, - **kwargs): + def __init__(self, executor=None, max_workers=2, session=None, *args, **kwargs): """Creates a FuturesSession Notes @@ -56,8 +54,7 @@ def __init__(self, executor=None, max_workers=2, session=None, *args, executor = ThreadPoolExecutor(max_workers=max_workers) # set connection pool size equal to max_workers if needed if max_workers > DEFAULT_POOLSIZE: - adapter_kwargs = dict(pool_connections=max_workers, - pool_maxsize=max_workers) + adapter_kwargs = dict(pool_connections=max_workers, pool_maxsize=max_workers) self.mount('https://', HTTPAdapter(**adapter_kwargs)) self.mount('http://', HTTPAdapter(**adapter_kwargs)) diff --git a/splunk_eventgen/splunk_app/bin/modinput_eventgen.py b/splunk_eventgen/splunk_app/bin/modinput_eventgen.py index db1e6115..15463bc5 100644 --- a/splunk_eventgen/splunk_app/bin/modinput_eventgen.py +++ b/splunk_eventgen/splunk_app/bin/modinput_eventgen.py @@ -1,25 +1,26 @@ #!/usr/bin/env python # encoding: utf-8 -import sys -import logging import argparse +import logging import signal +import sys +from modinput import ModularInput +from modinput.fields import VerbosityField # Set path so libraries will load from splunk.clilib.bundle_paths import make_splunkhome_path -sys.path.insert(0, make_splunkhome_path(['etc', 'apps', 'SA-Eventgen', 'lib'])) -sys.path.insert(0, make_splunkhome_path(['etc', 'apps', 'SA-Eventgen', 'lib', 'splunk_eventgen', 'lib'])) - -from modinput.fields import BooleanField, Field, VerbosityField -from xmloutput import setupLogger, XMLOutputManager -from modinput import ModularInput from splunk_eventgen import eventgen_core from splunk_eventgen.lib import eventgenconfig +from xmloutput import XMLOutputManager, setupLogger + +sys.path.insert(0, make_splunkhome_path(['etc', 'apps', 'SA-Eventgen', 'lib'])) +sys.path.insert(0, make_splunkhome_path(['etc', 'apps', 'SA-Eventgen', 'lib', 'splunk_eventgen', 'lib'])) # Initialize logging logger = setupLogger(logger=None, log_format='%(asctime)s %(levelname)s [Eventgen] %(message)s', level=logging.DEBUG, log_name="modinput_eventgen.log", logger_name="eventgen_app") + class SimpleNamespace(dict): """dot.notation access to dictionary attributes""" __getattr__ = dict.get @@ -29,12 +30,8 @@ class SimpleNamespace(dict): class Eventgen(ModularInput): scheme_args = { - 'title': "SA-Eventgen", - 'description': "This modular input generates data for Splunk.", - 'use_external_validation': "true", - 'streaming_mode': "xml", - 'use_single_instance': "False" - } + 'title': "SA-Eventgen", 'description': "This modular input generates data for Splunk.", + 'use_external_validation': "true", 'streaming_mode': "xml", 'use_single_instance': "False"} def __init__(self): logger.debug("Setting up SA-Eventgen Modular Input") @@ -42,9 +39,8 @@ def __init__(self): self.args = [ VerbosityField("verbosity", "Verbosity", - "Logging Level (DEBUG(10), INFO(20), WARN(30), ERROR(40), CRITICAL(50))", - required_on_create=True, required_on_edit=True) - ] + "Logging Level (DEBUG(10), INFO(20), WARN(30), ERROR(40), CRITICAL(50))", + required_on_create=True, required_on_edit=True)] ModularInput.__init__(self, self.scheme_args, self.args) def create_args(self): @@ -138,14 +134,17 @@ def run(self, stanza, input_config, **kwargs): logger.error("Main code exit, Exception caught: %s" % e) raise e + def handler(signum, frame): logger.info("Eventgen Modinput takes signal {0}. Exiting".format(signum)) sys.exit(0) + def handle_signal(): if not sys.platform.startswith('win') and sys.platform != "cygwin": signal.signal(signal.SIGPIPE, handler) + if __name__ == '__main__': handle_signal() worker = Eventgen() diff --git a/splunk_eventgen/splunk_app/lib/modinput/__init__.py b/splunk_eventgen/splunk_app/lib/modinput/__init__.py index 11542901..1d2da228 100644 --- a/splunk_eventgen/splunk_app/lib/modinput/__init__.py +++ b/splunk_eventgen/splunk_app/lib/modinput/__init__.py @@ -10,14 +10,19 @@ import sys import time import xml.dom -from xml.dom.minidom import Document import xml.sax.saxutils +from xml.dom.minidom import Document import splunk import splunk.clilib import splunk.version -from splunk.models.app import App from splunk.clilib.bundle_paths import get_slaveapps_base_path +from splunk.models.app import App +from xmloutput import setupLogger + +from .fields import (BooleanField, Field, FieldValidationException, + IntervalField) + try: from splunk.clilib.bundle_paths import make_splunkhome_path except ImportError: @@ -28,27 +33,13 @@ else: sys.path.append(make_splunkhome_path(["etc", "apps", "@appname@", "lib"])) -from .fields import BooleanField -from .fields import DurationField -from .fields import Field -from .fields import FieldValidationException -from .fields import FloatField -from .fields import IntegerField -from .fields import IntervalField -from .fields import ListField -from .fields import RangeField -from .fields import RegexField -from .fields import SeverityField - -from xmloutput import setupLogger, XMLOutputManager - # Define logger using the name of the script here, versus in the modular_input class. -#logger = log.setup_logger(name='python_modular_input', level=logging.INFO) -logger = setupLogger(logger=None, log_format='%(asctime)s %(levelname)s [ModularInput] %(message)s', level=logging.INFO, log_name="python_modular_input.log", logger_name="modinput") +# logger = log.setup_logger(name='python_modular_input', level=logging.INFO) +logger = setupLogger(logger=None, log_format='%(asctime)s %(levelname)s [ModularInput] %(message)s', level=logging.INFO, + log_name="python_modular_input.log", logger_name="modinput") class ModularInputConfig(object): - def __init__(self, server_host, server_uri, session_key, checkpoint_dir, configuration): self.server_host = server_host self.server_uri = server_uri @@ -135,14 +126,13 @@ class ModularInput(object): # These arguments cover the standard fields that are always supplied standard_args = [ - BooleanField("disabled", "Disabled", "Whether the modular input is disabled or not"), - Field("host", "Host", "The host that is running the input"), - Field("index", "Index", "The index that data should be sent to"), - IntervalField("interval", "Interval", "The interval the script will be run on"), - Field("name", "Stanza name", "The name of the stanza for this modular input"), - Field("source", "Source", "The source for events created by this modular input"), - Field("sourcetype", "Stanza name", "The name of the stanza for this modular input") - ] + BooleanField("disabled", "Disabled", "Whether the modular input is disabled or not"), + Field("host", "Host", "The host that is running the input"), + Field("index", "Index", "The index that data should be sent to"), + IntervalField("interval", "Interval", "The interval the script will be run on"), + Field("name", "Stanza name", "The name of the stanza for this modular input"), + Field("source", "Source", "The source for events created by this modular input"), + Field("sourcetype", "Stanza name", "The name of the stanza for this modular input")] checkpoint_dir = None @@ -401,7 +391,8 @@ def add_xml_args(self, doc, element_args): def do_validation(self, in_stream=sys.stdin): """ - Get the validation data from standard input and attempt to validate it. Returns true if the arguments validated, false otherwise. + Get the validation data from standard input and attempt to validate it. Returns true if the arguments validated, + false otherwise. Arguments: in_stream -- The stream to get the input from (defaults to standard input) @@ -421,7 +412,7 @@ def validate(self, arguments): Validate the argument dictionary where each key is a stanza. Arguments: - arguments -- The arguments as an dictionary where the key is the stanza and the value is a dictionary of the values. + arguments -- a dictionary where the key is the stanza and the value is a dictionary of the values. """ # Check each stanza @@ -541,23 +532,19 @@ def needs_another_run(cls, checkpoint_dir, stanza, interval, cur_time=None): try: last_ran = cls.last_ran(checkpoint_dir, stanza) - return cls.is_expired(last_ran, interval, cur_time) - - except IOError as e: + except IOError: # The file likely doesn't exist logger.exception("The checkpoint file likely doesn't exist") return True - except ValueError as e: + except ValueError: # The file could not be loaded logger.exception("The checkpoint file could not be loaded") return True except Exception as e: - #Catch all that enforces an extra run + # Catch all that enforces an extra run logger.exception("Unexpected exception caught, enforcing extra run, exception info: " + str(e)) return True - # Default return value - return True @classmethod def time_to_next_run(cls, checkpoint_dir, stanza, duration): @@ -582,18 +569,17 @@ def time_to_next_run(cls, checkpoint_dir, stanza, duration): return time_to_next except IOError: # The file likely doesn't exist - logger.warning("Could not read checkpoint file for last time run, likely does not exist, if this persists debug input immediately") + logger.warning("Could not read checkpoint file for last time run, likely does not exist, if this" + + "persists debug input immediately") return 1 except ValueError: # The file could not be loaded - logger.exception("Could not read checkpoint file for last time run, if this persists debug input immediately") + logger.exception( + "Could not read checkpoint file for last time run, if this persists debug input immediately") return 1 except Exception as e: logger.exception("Unexpected exception caught, enforcing extra run, exception info: " + str(e)) return 1 - # Default return value - logger.info("This really should be impossible, but whatevs if your input is breaking check the duration calculations") - return 1 @classmethod def save_checkpoint(cls, checkpoint_dir, stanza, last_run): @@ -663,7 +649,9 @@ def delete_checkpoint_data(self, filename, checkpoint_dir=None): os.unlink(os.path.join(checkpoint_dir, filename)) return True except IOError: - logger.exception('msg="IOError exception when deleting checkpoint data" checkpoint_dir="{}" filename="{}"'.format(checkpoint_dir, filename)) + logger.exception( + 'msg="IOError exception when deleting checkpoint data" checkpoint_dir="{}" filename="{}"'.format( + checkpoint_dir, filename)) return False def set_checkpoint_data(self, filename, data, checkpoint_dir=None): @@ -693,11 +681,17 @@ def set_checkpoint_data(self, filename, data, checkpoint_dir=None): json.dump(data, fp) success = True except IOError: - logger.exception('msg="IOError exception when saving checkpoint data" checkpoint_dir="{}" filename="{}"'.format(checkpoint_dir, filename)) + logger.exception( + 'msg="IOError exception when saving checkpoint data" checkpoint_dir="{}" filename="{}"'.format( + checkpoint_dir, filename)) except ValueError: - logger.exception('msg="ValueError when saving checkpoint data (perhaps invalid JSON)" checkpoint_dir="{}" filename="{}"'.format(checkpoint_dir, filename)) + logger.exception( + 'msg="ValueError when saving checkpoint data (perhaps invalid JSON)" checkpoint_dir="{}" filename="{}"'. + format(checkpoint_dir, filename)) except Exception: - logger.exception('msg="Unknown exception when saving checkpoint data" checkpoint_dir="{}" filename="{}"'.format(checkpoint_dir, filename)) + logger.exception( + 'msg="Unknown exception when saving checkpoint data" checkpoint_dir="{}" filename="{}"'.format( + checkpoint_dir, filename)) return success def get_checkpoint_data(self, filename, checkpoint_dir=None, raise_known_exceptions=False): @@ -727,11 +721,15 @@ def get_checkpoint_data(self, filename, checkpoint_dir=None, raise_known_excepti with open(checkpoint_path, 'r') as fp: data = json.load(fp) except (IOError, ValueError) as e: - logger.exception('msg="Exception when reading checkpoint data" checkpoint_dir="{}" filename="{}" exception="%s"'.format(checkpoint_dir, filename, e)) + logger.exception( + 'msg="Exception when reading checkpoint data" checkpoint_dir="{}" filename="{}" exception="%s"'.format( + checkpoint_dir, filename, e)) if raise_known_exceptions: raise except Exception: - logger.exception('msg="Unknown exception when reading checkpoint data" checkpoint_dir="{}" filename="{}"'.format(checkpoint_dir, filename)) + logger.exception( + 'msg="Unknown exception when reading checkpoint data" checkpoint_dir="{}" filename="{}"'.format( + checkpoint_dir, filename)) raise return data @@ -741,7 +739,8 @@ def do_run(self, in_stream=sys.stdin, log_exception_and_continue=False): Read the config from standard input and return the configuration. in_stream -- The stream to get the input from (defaults to standard input) - log_exception_and_continue -- If true, exceptions will not be thrown for invalid configurations and instead the stanza will be skipped. + log_exception_and_continue -- If true, exceptions will not be thrown for invalid configurations and instead the + stanza will be skipped. """ # Run the modular import @@ -779,14 +778,12 @@ def do_run(self, in_stream=sys.stdin, log_exception_and_continue=False): # Note: The "duration" parameter emulates the behavior of the "interval" # parameter available on Splunk 6.x and higher, and is mainly used by the # VMWare application. - - # TODO: A run() method may pass results back for optional processing - results = None + # TODO: should we collect/process results from run()? if stanzas: if single_instance: # Run the input across all defined stanzas and exit. - results = self.run(stanzas, self._input_config) + self.run(stanzas, self._input_config) else: # Retrieve the single input stanza. stanza = stanzas[0] @@ -795,7 +792,8 @@ def do_run(self, in_stream=sys.stdin, log_exception_and_continue=False): except ValueError as e: # This should never happen unless the author of the modular input # fails to specify "duration" as an IntegerField. - logger.exception("Input stanza '%s' specified an invalid duration: %s" % (stanza.get('name', 'unknown'), str(e))) + logger.exception( + "Input stanza '%s' specified an invalid duration: %s" % (stanza.get('name', 'unknown'), str(e))) # Exit with non-zero exit code so services/admin/inputstatus correctly reflects script status. sys.exit(1) @@ -805,7 +803,7 @@ def do_run(self, in_stream=sys.stdin, log_exception_and_continue=False): # If there splunk 6.0 and interval field is defined, then ignore duration fields completely if stanza.get("interval", -1) >= 0 and splunk.version.__version__ >= '6.0': # Run the single stanza and exit. - results = self.run(stanza, self._input_config) + self.run(stanza, self._input_config) else: # Run duration field if duration > 0 and self.checkpoint_dir: @@ -819,13 +817,13 @@ def do_run(self, in_stream=sys.stdin, log_exception_and_continue=False): # use the name of the input itself, unhashed. Name collisions would # be a configuration error. self.save_checkpoint(self.checkpoint_dir, stanza_name, int(time.time())) - results = self.run(stanza, ) + self.run(stanza, self._input_config) # Results processing, if any, could occur here. time.sleep(ModularInput.time_to_next_run(self.checkpoint_dir, stanza_name, duration)) else: # Duration is not defined # Run the single stanza and exit for Splunk 5.x - results = self.run(stanza, self._input_config) + self.run(stanza, self._input_config) else: logger.info("No input stanzas defined") @@ -899,13 +897,11 @@ def _parse_args(self, argv): parser = argparse.ArgumentParser(description='Modular input parameters') - mode_args= parser.add_mutually_exclusive_group() + mode_args = parser.add_mutually_exclusive_group() debug_args = parser.add_argument_group() - debug_args.add_argument('--username', action="store", default=None, - help="Splunk username (%s)" % warning_text) - debug_args.add_argument('--password', action="store", default=None, - help="Splunk password (%s)" % warning_text) + debug_args.add_argument('--username', action="store", default=None, help="Splunk username (%s)" % warning_text) + debug_args.add_argument('--password', action="store", default=None, help="Splunk password (%s)" % warning_text) debug_args.add_argument('--infile', type=argparse.FileType(), default=None, help="Filename containing XML modular input configuration (%s)" % warning_text) @@ -936,7 +932,7 @@ def execute(self, in_stream=sys.stdin, out_stream=sys.stdout): logger.info("Modular input: validate arguments called") # Exit with a code of -1 if validation failed - if self.do_validation() == False: + if self.do_validation() is False: sys.exit(-1) else: @@ -959,7 +955,8 @@ def execute(self, in_stream=sys.stdin, out_stream=sys.stdout): try: self.do_run(args.infile, log_exception_and_continue=True) except IOError: - logger.exception("Modular input: modinput configuration could not be read from file %s.", args.infile.name) + logger.exception("Modular input: modinput configuration could not be read from file %s.", + args.infile.name) else: try: self.do_run(in_stream, log_exception_and_continue=True) @@ -1034,4 +1031,4 @@ def is_configured(self, app=None, assume_true_on_error=False): except splunk.RESTException: return assume_true_on_error else: - return assume_true_on_error \ No newline at end of file + return assume_true_on_error diff --git a/splunk_eventgen/splunk_app/lib/modinput/fields.py b/splunk_eventgen/splunk_app/lib/modinput/fields.py index 40d6b37b..681ae6d5 100644 --- a/splunk_eventgen/splunk_app/lib/modinput/fields.py +++ b/splunk_eventgen/splunk_app/lib/modinput/fields.py @@ -11,7 +11,8 @@ class FieldValidationException(Exception): class Field(object): """ - This is the base class that should be used to create field validators. Sub-class this and override to_python if you need custom validation. + This is the base class that should be used to create field validators. Sub-class this and override to_python if you + need custom validation. """ DATA_TYPE_STRING = 'string' @@ -32,7 +33,8 @@ def __init__(self, name, title, description, required_on_create=True, required_o Arguments: name -- Set the name of the field (e.g. "database_server") title -- Set the human readable title (e.g. "Database server") - description -- Set the human readable description of the field (e.g. "The IP or domain name of the database server") + description -- Set the human-readable description of the field + (e.g. "The IP or domain name of the database server") required_on_create -- If "true", the parameter is required on input stanza creation. required_on_edit -- If "true", the parameter is required on input stanza modification. @@ -70,7 +72,8 @@ def to_python(self, value): def to_string(self, value): """ - Convert the field to a string value that can be returned. Should throw a FieldValidationException if the data is invalid. + Convert the field to a string value that can be returned. Should throw a FieldValidationException if the data is + invalid. Arguments: value -- The value to convert @@ -80,7 +83,6 @@ def to_string(self, value): class BooleanField(Field): - def to_python(self, value): Field.to_python(self, value) @@ -93,14 +95,15 @@ def to_python(self, value): elif str(value).strip().lower() in ["false", "f", "0"]: return False - raise FieldValidationException("The value of '%s' for the '%s' parameter is not a valid boolean" % (str(value), self.name)) + raise FieldValidationException( + "The value of '%s' for the '%s' parameter is not a valid boolean" % (str(value), self.name)) def to_string(self, value): - if value == True: + if value is True: return "1" - elif value == False: + elif value is False: return "0" return str(value) @@ -110,7 +113,6 @@ def get_data_type(self): class DelimitedField(Field): - def __init__(self, name, title, description, delim, required_on_create=True, required_on_edit=False): super(DelimitedField, self).__init__(name, title, description, required_on_create, required_on_edit) self._delim = delim @@ -154,17 +156,8 @@ class DurationField(Field): WEEK = 604800 UNITS = { - 'w': WEEK, - 'week': WEEK, - 'd': DAY, - 'day': DAY, - 'h': HOUR, - 'hour': HOUR, - 'm': MINUTE, - 'min': MINUTE, - 'minute': MINUTE, - 's': 1 - } + 'w': WEEK, 'week': WEEK, 'd': DAY, 'day': DAY, 'h': HOUR, 'hour': HOUR, 'm': MINUTE, 'min': MINUTE, 'minute': + MINUTE, 's': 1} def to_python(self, value): Field.to_python(self, value) @@ -174,7 +167,8 @@ def to_python(self, value): # Make sure the duration could be parsed if m is None: - raise FieldValidationException("The value of '%s' for the '%s' parameter is not a valid duration" % (str(value), self.name)) + raise FieldValidationException( + "The value of '%s' for the '%s' parameter is not a valid duration" % (str(value), self.name)) # Get the units and duration d = m.groupdict() @@ -185,11 +179,13 @@ def to_python(self, value): try: duration = int(d['duration']) except ValueError: - raise FieldValidationException("The duration '%s' for the '%s' parameter is not a valid number" % (d['duration'], self.name)) + raise FieldValidationException( + "The duration '%s' for the '%s' parameter is not a valid number" % (d['duration'], self.name)) # Make sure the units are valid if len(units) > 0 and units not in DurationField.UNITS: - raise FieldValidationException("The unit '%s' for the '%s' parameter is not a valid unit of duration" % (units, self.name)) + raise FieldValidationException( + "The unit '%s' for the '%s' parameter is not a valid unit of duration" % (units, self.name)) # Convert the units to seconds if len(units) > 0: @@ -202,7 +198,6 @@ def to_string(self, value): class FloatField(Field): - def to_python(self, value): Field.to_python(self, value) @@ -227,7 +222,6 @@ def get_data_type(self): class IntegerField(Field): - def to_python(self, value): Field.to_python(self, value) @@ -270,7 +264,8 @@ class IntervalField(Field): # Note that we don't check explicitly for correct numeric values for each # cron field. - cron_rx = re.compile(''' + cron_rx = re.compile( + ''' ( \d{1,2} # A digit. |\d{1,2}-\d{1,2} # A range. @@ -279,22 +274,20 @@ class IntervalField(Field): |\* # The asterisk character. |\*/\d{1,2} # An asterisk followed by a step. ) - ''', - re.VERBOSE - ) + ''', re.VERBOSE) def to_python(self, value): try: # Try parsing the string as an integer. - tmp = int(value) - return value + return int(value) except ValueError: # Try parsing the string as a cron schedule. if self.parse_cron(value): return value - raise FieldValidationException("The value of '{}' for the '{}' parameter is not a valid value".format(value, self.name)) + raise FieldValidationException("The value of '{}' for the '{}' parameter is not a valid value".format( + value, self.name)) def get_data_type(self): return Field.DATA_TYPE_STRING @@ -309,14 +302,14 @@ def parse_cron(self, value): class JsonField(Field): - def to_python(self, value): Field.to_python(self, value) try: return json.loads(value) except (TypeError, ValueError): - raise FieldValidationException("The value of '%s' for the '%s' parameter is not a valid JSON object" % (str(value), self.name)) + raise FieldValidationException( + "The value of '%s' for the '%s' parameter is not a valid JSON object" % (str(value), self.name)) def to_string(self, value): return str(value) @@ -326,7 +319,6 @@ def get_data_type(self): class ListField(Field): - def to_python(self, value): Field.to_python(self, value) @@ -345,7 +337,6 @@ def to_string(self, value): class RangeField(Field): - def __init__(self, name, title, description, low, high, required_on_create=True, required_on_edit=False): super(RangeField, self).__init__(name, title, description, required_on_create, required_on_edit) self.low = low @@ -379,7 +370,6 @@ def get_data_type(self): class RegexField(Field): - def to_python(self, value): Field.to_python(self, value) @@ -404,11 +394,7 @@ class SeverityField(Field): # Note: We ignore "FATAL" severity since Python's logging assigns it the # same value as "CRITICAL". - SEVERITIES = {'DEBUG': 10, - 'INFO': 20, - 'WARN': 30, - 'ERROR': 40, - 'CRITICAL': 50} + SEVERITIES = {'DEBUG': 10, 'INFO': 20, 'WARN': 30, 'ERROR': 40, 'CRITICAL': 50} SEVERITIES_BY_INT = {v: k for k, v in SEVERITIES.iteritems()} @@ -421,7 +407,8 @@ def to_python(self, value): # Did not receive a string for some reason. pass - raise FieldValidationException("The value of '{}' for the '{}' parameter is not a valid value".format(value, self.name)) + raise FieldValidationException("The value of '{}' for the '{}' parameter is not a valid value".format( + value, self.name)) def to_string(self, value): if value in SeverityField.SEVERITIES_BY_INT: @@ -432,8 +419,8 @@ def to_string(self, value): def get_data_type(self): return Field.DATA_TYPE_NUMBER -class VerbosityField(Field): +class VerbosityField(Field): def to_python(self, value): Field.to_python(self, value) @@ -455,4 +442,4 @@ def to_string(self, value): return "" def get_data_type(self): - return Field.DATA_TYPE_NUMBER \ No newline at end of file + return Field.DATA_TYPE_NUMBER diff --git a/splunk_eventgen/splunk_app/lib/xmloutput.py b/splunk_eventgen/splunk_app/lib/xmloutput.py index 5f378d0d..d1277166 100644 --- a/splunk_eventgen/splunk_app/lib/xmloutput.py +++ b/splunk_eventgen/splunk_app/lib/xmloutput.py @@ -1,9 +1,9 @@ -import xml.sax.saxutils +import datetime import logging import logging.handlers import sys -import time -import datetime +import xml.sax.saxutils + from splunk.appserver.mrsparkle.lib.util import make_splunkhome_path @@ -18,8 +18,8 @@ def setupLogger(logger=None, log_format='%(asctime)s %(levelname)s [ModInput] %( logger.propagate = False # Prevent the log messages from being duplicated in the python.log file logger.setLevel(level) - file_handler = logging.handlers.RotatingFileHandler(make_splunkhome_path(['var', 'log', 'splunk', log_name]), - maxBytes=2500000, backupCount=5) + file_handler = logging.handlers.RotatingFileHandler( + make_splunkhome_path(['var', 'log', 'splunk', log_name]), maxBytes=2500000, backupCount=5) formatter = logging.Formatter(log_format) file_handler.setFormatter(formatter) @@ -30,22 +30,21 @@ def setupLogger(logger=None, log_format='%(asctime)s %(levelname)s [ModInput] %( return logger -######################################################################## -# COMMUNICATION WITH SPLUNKD -# We provide a class for printing data out to splunkd. Essentially this -# is just a wrapper on using xml formatted data delivery to splunkd -######################################################################## +######################################################################### +# COMMUNICATION WITH SPLUNKD # +# We provide a class for printing data out to splunkd. Essentially this # +# is just a wrapper on using xml formatted data delivery to splunkd # +######################################################################### class XMLOutputManager(object): """ - This guy handles writing data to splunkd with modular input xml - streaming mode. + This guy handles writing data to splunkd with modular input xml streaming mode. """ def __init__(self, out=sys.stdout): """ - Construct an output manager. - kwargs: - out - represents the stream to print to. Defaults to sys.stdout. + Construct an output manager. + kwargs: + out - represents the stream to print to. Defaults to sys.stdout. """ self.stream_initiated = False self.out = out @@ -72,13 +71,13 @@ def sendData(self, buf, unbroken=None, sourcetype=None, source=None, host=None, args: buf - the buffer of data to send (string). REQUIRED. kwargs: - unbroken - this is a boolean indicating the buf passed is unbroken data if this is True. + unbroken - this is a boolean indicating the buf passed is unbroken data if this is True. Defaults to False (buf is a single event). sourcetype - the sourcetype to assign to the event (string). Defaults to input default. source - the source to assign to the event (string). Defaults to input default. host - the host to assign to the event (string). Defaults to input default. - time - the time to assign to the event (string of UTC UNIX timestamp, - miliseconds supported). Defaults to letting splunkd work it out. + time - the time to assign to the event (string of UTC UNIX timestamp, + milliseconds supported). Defaults to letting splunkd work it out. index - the index into which the data should be stored. Defaults to the input default. """ if not unbroken: @@ -107,7 +106,7 @@ def sendDoneKey(self, sourcetype=None, source=None, host=None, time=None, index= """ Let splunkd know that previously sent, unbroken events are now complete and ready for processing. Typically you will send some data, like chunks of a log file - then when you know you are done, say at the end of the log file you will send a + then when you know you are done, say at the end of the log file you will send a done key to indicate that sent data may be processed for the provided source, sourcetype, host, and index kwargs: @@ -135,4 +134,4 @@ def sendDoneKey(self, sourcetype=None, source=None, host=None, time=None, index= # prints XML error data to be consumed by Splunk def printError(self, s): - self.out.write("{0}".format(xml.sax.saxutils.escape(s))) \ No newline at end of file + self.out.write("{0}".format(xml.sax.saxutils.escape(s))) diff --git a/tests/large/test_eventgen_orchestration.py b/tests/large/test_eventgen_orchestration.py index d5bc7cbe..19d35da3 100644 --- a/tests/large/test_eventgen_orchestration.py +++ b/tests/large/test_eventgen_orchestration.py @@ -1,16 +1,18 @@ #!/usr/bin/env python # encoding: utf-8 +import json import os import time -import json +from random import choice +from string import ascii_lowercase + import pytest import requests from docker import APIClient -from random import choice -from string import ascii_lowercase # Code to suppress insecure https warnings from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) FILE_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -18,12 +20,14 @@ IMAGE_NAME = "eventgen:test" NETWORK_NAME = "eg_network_test" + def generate_random_string(): return ''.join(choice(ascii_lowercase) for b in range(20)) + def wait_for_response(url, timeout=60): start, end = time.time(), time.time() - while end-start < timeout: + while end - start < timeout: try: r = requests.get(url) r.raise_for_status() @@ -35,249 +39,260 @@ def wait_for_response(url, timeout=60): @pytest.mark.large class TestEventgenOrchestration(object): - ''' - This test class is used to test the Docker image published as part of this repo. - Specifically, this is testing: - * Eventgen "controller" API and responses - * Eventgen "server" API and responses - * Eventgen controller/server orchestration - ''' - - @classmethod - def setup_class(cls): - # Build the image from scratch - cls.client = APIClient(base_url="unix://var/run/docker.sock") - response = cls.client.build(path=REPO_DIR, dockerfile=os.path.join("dockerfiles", "Dockerfile"), tag=IMAGE_NAME, rm=True, nocache=True, pull=True, stream=False) - for line in response: - print line, - # Create a network for both the controller + server to run in - cls.client.create_network(NETWORK_NAME, driver="bridge", attachable=True) - networking_config = cls.client.create_networking_config({NETWORK_NAME: cls.client.create_endpoint_config()}) - # Start the controller - print 'creating controller' - host_config = cls.client.create_host_config(auto_remove=True, publish_all_ports=True) - container = cls.client.create_container(image=IMAGE_NAME, - command="controller", - host_config=host_config, - networking_config=networking_config) - cls.client.start(container["Id"]) - TestEventgenOrchestration.controller_id = container["Id"] - print container["Id"] - cls.controller_container = cls.client.inspect_container(container["Id"]) - cls.controller_eventgen_webport = cls.controller_container["NetworkSettings"]["Ports"]["9500/tcp"][0]["HostPort"] - cls.controller_rabbitmq_webport = cls.controller_container["NetworkSettings"]["Ports"]["15672/tcp"][0]["HostPort"] - # Start the server - print 'creating server' - container = cls.client.create_container(image=IMAGE_NAME, - command="server", - environment=["EVENTGEN_AMQP_HOST={}".format(cls.controller_container["Id"][:12])], - host_config=host_config, - networking_config=networking_config) - cls.client.start(container["Id"]) - TestEventgenOrchestration.server_id = container["Id"] - print container["Id"] - cls.server_container = cls.client.inspect_container(container["Id"]) - cls.server_eventgen_webport = cls.server_container["NetworkSettings"]["Ports"]["9500/tcp"][0]["HostPort"] - cls.server_rabbitmq_webport = cls.server_container["NetworkSettings"]["Ports"]["15672/tcp"][0]["HostPort"] - # Wait for the controller to be available - print "Waiting for Eventgen Controller to become available." - wait_for_response("http://127.0.0.1:{}".format(cls.controller_eventgen_webport)) - print "Eventgen Controller has become available." - # Wait for the server to be available - print "Waiting for Eventgen Server to become available." - wait_for_response("http://127.0.0.1:{}".format(cls.server_eventgen_webport)) - print "Eventgen Server has become available." - time.sleep(60) - - @classmethod - def teardown_class(cls): - cls.client.remove_container(cls.server_container, v=True, force=True) - cls.client.remove_container(cls.controller_container, v=True, force=True) - cls.client.remove_image(IMAGE_NAME, force=True, noprune=False) - cls.client.remove_network(NETWORK_NAME) - - ### Controller tests ### - - def test_controller_rabbitmq(self): - r = requests.get("http://127.0.0.1:{}".format(self.controller_rabbitmq_webport)) - assert r.status_code == 200 - assert "RabbitMQ" in r.content - - def test_controller_root(self): - r = requests.get("http://127.0.0.1:{}".format(self.controller_eventgen_webport)) - assert r.status_code == 200 - assert "Eventgen Controller" in r.content - assert "Host: " in r.content - assert "You are running Eventgen Controller" in r.content - - def test_controller_index(self): - r = requests.get("http://127.0.0.1:{}/index".format(self.controller_eventgen_webport)) - assert r.status_code == 200 - assert "Eventgen Controller" in r.content - assert "Host: " in r.content - assert "You are running Eventgen Controller" in r.content - - def test_controller_status(self): - max_retry = 5 - current_retry = 1 - output = {} - while not output and current_retry <= max_retry: - response = requests.get("http://127.0.0.1:{}/status".format(self.controller_eventgen_webport), timeout=10) - if response.status_code == 200: - output = json.loads(response.content) - current_retry += 1 - time.sleep(10) - assert output - - def test_controller_start(self): - r = requests.post("http://127.0.0.1:{}/start".format(self.controller_eventgen_webport)) - assert r.status_code == 200 - assert "Start event dispatched to all" in r.content - - def test_controller_start_with_target(self): - r = requests.post("http://127.0.0.1:{}/start/{}".format(self.controller_eventgen_webport, TestEventgenOrchestration.server_id[:12])) - assert r.status_code == 200 - assert "Start event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content - - def test_controller_stop(self): - r = requests.post("http://127.0.0.1:{}/stop".format(self.controller_eventgen_webport)) - assert r.status_code == 200 - assert "Stop event dispatched to all" in r.content - - def test_controller_stop_with_target(self): - r = requests.post("http://127.0.0.1:{}/stop/{}".format(self.controller_eventgen_webport, TestEventgenOrchestration.server_id[:12])) - assert r.status_code == 200 - assert "Stop event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content - - def test_controller_restart(self): - r = requests.post("http://127.0.0.1:{}/stop".format(self.controller_eventgen_webport)) - assert r.status_code == 200 - assert "Stop event dispatched to all" in r.content - - def test_controller_restart_with_target(self): - r = requests.post("http://127.0.0.1:{}/stop/{}".format(self.controller_eventgen_webport, TestEventgenOrchestration.server_id[:12])) - assert r.status_code == 200 - assert "Stop event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content - - def test_controller_bundle_invalid_request(self): - r = requests.post("http://127.0.0.1:{}/bundle".format(self.controller_eventgen_webport)) - assert r.status_code == 400 - assert "Please pass in a valid object with bundle URL" in r.content - - def test_controller_bundle_with_url(self): - r = requests.post("http://127.0.0.1:{}/bundle".format(self.controller_eventgen_webport), json={"url": "http://server.com/bundle.tgz"}) - assert r.status_code == 200 - assert "Bundle event dispatched to all with url http://server.com/bundle.tgz" in r.content - - def test_controller_bundle_with_url_and_target(self): - r = requests.post("http://127.0.0.1:{}/bundle/{}".format(self.controller_eventgen_webport, TestEventgenOrchestration.server_id[:12]), json={"url": "http://server.com/bundle.tgz"}) - assert r.status_code == 200 - assert "Bundle event dispatched to {} with url http://server.com/bundle.tgz".format(TestEventgenOrchestration.server_id[:12]) in r.content - - @pytest.mark.skip(reason="Change in implementation") - def test_controller_get_volume(self): - max_retry = 5 - current_retry = 1 - output = {} - while not output and current_retry <= max_retry: - response = requests.get("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport), timeout=10) - if response.status_code == 200: - output = json.loads(response.content) - current_retry += 1 - time.sleep(10) - assert output[TestEventgenOrchestration.server_id[:12]] == 0.0 - - def test_controller_set_volume_invalid_request(self): - r = requests.post("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport)) - assert r.status_code == 400 - assert "Please pass in a valid object with volume" in r.content - - def test_controller_set_volume_with_volume(self): - r = requests.post("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport), json={"perDayVolume": 10}) - assert r.status_code == 200 - assert "set_volume event dispatched to all" in r.content - - def test_controller_set_volume_with_volume_and_target(self): - r = requests.post("http://127.0.0.1:{}/volume/{}".format(self.controller_eventgen_webport, TestEventgenOrchestration.server_id[:12]), json={"perDayVolume": 10}) - assert r.status_code == 200 - assert "set_volume event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content - - ### Server tests ### - - def test_server_root(self): - r = requests.get("http://127.0.0.1:{}".format(self.server_eventgen_webport)) - assert r.status_code == 200 - assert "Host: " in r.content - assert "Eventgen Status" in r.content - assert "Eventgen Config file path" in r.content - assert "Total volume:" in r.content - assert "Worker Queue Status" in r.content - assert "Sample Queue Status" in r.content - assert "Output Queue Status" in r.content - - def test_server_index(self): - r = requests.get("http://127.0.0.1:{}/index".format(self.server_eventgen_webport)) - assert r.status_code == 200 - assert "Host: " in r.content - assert "Eventgen Status" in r.content - assert "Eventgen Config file path" in r.content - assert "Total volume:" in r.content - assert "Worker Queue Status" in r.content - assert "Sample Queue Status" in r.content - assert "Output Queue Status" in r.content - - def test_server_status(self): - r = requests.get("http://127.0.0.1:{}/status".format(self.server_eventgen_webport)) - assert r.status_code == 200 - output = json.loads(r.content) - assert output - assert output['EVENTGEN_STATUS'] == 0 - - def test_server_get_and_set_conf(self): - r = requests.get("http://127.0.0.1:{}/conf".format(self.server_eventgen_webport)) - assert r.status_code == 200 - assert json.loads(r.content) == {} - config_json = {"windbag": {"outputMode": "stdout"}} - r = requests.post("http://127.0.0.1:{}/conf".format(self.server_eventgen_webport), json=config_json) - assert r.status_code == 200 - assert json.loads(r.content) == config_json - - def test_server_start(self): - r = requests.post("http://127.0.0.1:{}/start".format(self.server_eventgen_webport), timeout=5) - assert r.status_code == 200 - assert json.loads(r.content) == "Eventgen has successfully started." - - def test_server_restart(self): - r = requests.post("http://127.0.0.1:{}/restart".format(self.server_eventgen_webport)) - assert r.status_code == 200 - assert json.loads(r.content) == "Eventgen restarted." - - def test_server_stop(self): - r = requests.post("http://127.0.0.1:{}/stop".format(self.server_eventgen_webport)) - assert r.status_code == 200 - assert json.loads(r.content) == "Eventgen is stopped." - - def test_server_bundle(self): - r = requests.post("http://127.0.0.1:{}/bundle".format(self.server_eventgen_webport)) - assert r.status_code == 400 - assert "Please pass in a valid object with bundle URL" in r.content - - def test_server_get_and_set_volume(self): - # Must initialize a stanza with the perDayVolume setting before hitting the /volume endpoint - r = requests.put("http://127.0.0.1:{}/conf".format(self.server_eventgen_webport), json={"windbag": {}}) - assert r.status_code == 200 - assert json.loads(r.content) - r = requests.post("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport), json={"perDayVolume": 10}) - assert r.status_code == 200 - assert json.loads(r.content) - r = requests.get("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport)) - assert r.status_code == 200 - output = json.loads(r.content) - assert output == 10.0 - r = requests.post("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport), json={"perDayVolume": 150}) - assert r.status_code == 200 - assert json.loads(r.content) - r = requests.get("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport)) - assert r.status_code == 200 - output = json.loads(r.content) - assert output == 150.0 + """ + This test class is used to test the Docker image published as part of this repo. + Specifically, this is testing: + * Eventgen "controller" API and responses + * Eventgen "server" API and responses + * Eventgen controller/server orchestration + """ + + @classmethod + def setup_class(cls): + # Build the image from scratch + cls.client = APIClient(base_url="unix://var/run/docker.sock") + response = cls.client.build(path=REPO_DIR, dockerfile=os.path.join("dockerfiles", "Dockerfile"), tag=IMAGE_NAME, + rm=True, nocache=True, pull=True, stream=False) + for line in response: + print line, + # Create a network for both the controller + server to run in + cls.client.create_network(NETWORK_NAME, driver="bridge", attachable=True) + networking_config = cls.client.create_networking_config({NETWORK_NAME: cls.client.create_endpoint_config()}) + # Start the controller + print 'creating controller' + host_config = cls.client.create_host_config(auto_remove=True, publish_all_ports=True) + container = cls.client.create_container(image=IMAGE_NAME, command="controller", host_config=host_config, + networking_config=networking_config) + cls.client.start(container["Id"]) + TestEventgenOrchestration.controller_id = container["Id"] + print container["Id"] + cls.controller_container = cls.client.inspect_container(container["Id"]) + cls.controller_eventgen_webport = cls.controller_container["NetworkSettings"]["Ports"]["9500/tcp"][0][ + "HostPort"] + cls.controller_rabbitmq_webport = cls.controller_container["NetworkSettings"]["Ports"]["15672/tcp"][0][ + "HostPort"] + # Start the server + print 'creating server' + container = cls.client.create_container( + image=IMAGE_NAME, command="server", environment=[ + "EVENTGEN_AMQP_HOST={}".format(cls.controller_container["Id"][:12])], host_config=host_config, + networking_config=networking_config) + cls.client.start(container["Id"]) + TestEventgenOrchestration.server_id = container["Id"] + print container["Id"] + cls.server_container = cls.client.inspect_container(container["Id"]) + cls.server_eventgen_webport = cls.server_container["NetworkSettings"]["Ports"]["9500/tcp"][0]["HostPort"] + cls.server_rabbitmq_webport = cls.server_container["NetworkSettings"]["Ports"]["15672/tcp"][0]["HostPort"] + # Wait for the controller to be available + print "Waiting for Eventgen Controller to become available." + wait_for_response("http://127.0.0.1:{}".format(cls.controller_eventgen_webport)) + print "Eventgen Controller has become available." + # Wait for the server to be available + print "Waiting for Eventgen Server to become available." + wait_for_response("http://127.0.0.1:{}".format(cls.server_eventgen_webport)) + print "Eventgen Server has become available." + time.sleep(60) + + @classmethod + def teardown_class(cls): + cls.client.remove_container(cls.server_container, v=True, force=True) + cls.client.remove_container(cls.controller_container, v=True, force=True) + cls.client.remove_image(IMAGE_NAME, force=True, noprune=False) + cls.client.remove_network(NETWORK_NAME) + + # Controller tests # + + def test_controller_rabbitmq(self): + r = requests.get("http://127.0.0.1:{}".format(self.controller_rabbitmq_webport)) + assert r.status_code == 200 + assert "RabbitMQ" in r.content + + def test_controller_root(self): + r = requests.get("http://127.0.0.1:{}".format(self.controller_eventgen_webport)) + assert r.status_code == 200 + assert "Eventgen Controller" in r.content + assert "Host: " in r.content + assert "You are running Eventgen Controller" in r.content + + def test_controller_index(self): + r = requests.get("http://127.0.0.1:{}/index".format(self.controller_eventgen_webport)) + assert r.status_code == 200 + assert "Eventgen Controller" in r.content + assert "Host: " in r.content + assert "You are running Eventgen Controller" in r.content + + def test_controller_status(self): + max_retry = 5 + current_retry = 1 + output = {} + while not output and current_retry <= max_retry: + response = requests.get("http://127.0.0.1:{}/status".format(self.controller_eventgen_webport), timeout=10) + if response.status_code == 200: + output = json.loads(response.content) + current_retry += 1 + time.sleep(10) + assert output + + def test_controller_start(self): + r = requests.post("http://127.0.0.1:{}/start".format(self.controller_eventgen_webport)) + assert r.status_code == 200 + assert "Start event dispatched to all" in r.content + + def test_controller_start_with_target(self): + r = requests.post("http://127.0.0.1:{}/start/{}".format(self.controller_eventgen_webport, + TestEventgenOrchestration.server_id[:12])) + assert r.status_code == 200 + assert "Start event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content + + def test_controller_stop(self): + r = requests.post("http://127.0.0.1:{}/stop".format(self.controller_eventgen_webport)) + assert r.status_code == 200 + assert "Stop event dispatched to all" in r.content + + def test_controller_stop_with_target(self): + r = requests.post("http://127.0.0.1:{}/stop/{}".format(self.controller_eventgen_webport, + TestEventgenOrchestration.server_id[:12])) + assert r.status_code == 200 + assert "Stop event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content + + def test_controller_restart(self): + r = requests.post("http://127.0.0.1:{}/stop".format(self.controller_eventgen_webport)) + assert r.status_code == 200 + assert "Stop event dispatched to all" in r.content + + def test_controller_restart_with_target(self): + r = requests.post("http://127.0.0.1:{}/stop/{}".format(self.controller_eventgen_webport, + TestEventgenOrchestration.server_id[:12])) + assert r.status_code == 200 + assert "Stop event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content + + def test_controller_bundle_invalid_request(self): + r = requests.post("http://127.0.0.1:{}/bundle".format(self.controller_eventgen_webport)) + assert r.status_code == 400 + assert "Please pass in a valid object with bundle URL" in r.content + + def test_controller_bundle_with_url(self): + r = requests.post("http://127.0.0.1:{}/bundle".format(self.controller_eventgen_webport), json={ + "url": "http://server.com/bundle.tgz"}) + assert r.status_code == 200 + assert "Bundle event dispatched to all with url http://server.com/bundle.tgz" in r.content + + def test_controller_bundle_with_url_and_target(self): + r = requests.post( + "http://127.0.0.1:{}/bundle/{}".format(self.controller_eventgen_webport, + TestEventgenOrchestration.server_id[:12]), json={ + "url": "http://server.com/bundle.tgz"}) + assert r.status_code == 200 + assert "Bundle event dispatched to {} with url http://server.com/bundle.tgz".format( + TestEventgenOrchestration.server_id[:12]) in r.content + + @pytest.mark.skip(reason="Change in implementation") + def test_controller_get_volume(self): + max_retry = 5 + current_retry = 1 + output = {} + while not output and current_retry <= max_retry: + response = requests.get("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport), timeout=10) + if response.status_code == 200: + output = json.loads(response.content) + current_retry += 1 + time.sleep(10) + assert output[TestEventgenOrchestration.server_id[:12]] == 0.0 + + def test_controller_set_volume_invalid_request(self): + r = requests.post("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport)) + assert r.status_code == 400 + assert "Please pass in a valid object with volume" in r.content + + def test_controller_set_volume_with_volume(self): + r = requests.post("http://127.0.0.1:{}/volume".format(self.controller_eventgen_webport), json={ + "perDayVolume": 10}) + assert r.status_code == 200 + assert "set_volume event dispatched to all" in r.content + + def test_controller_set_volume_with_volume_and_target(self): + r = requests.post( + "http://127.0.0.1:{}/volume/{}".format(self.controller_eventgen_webport, + TestEventgenOrchestration.server_id[:12]), json={"perDayVolume": 10}) + assert r.status_code == 200 + assert "set_volume event dispatched to {}".format(TestEventgenOrchestration.server_id[:12]) in r.content + + # Server tests # + + def test_server_root(self): + r = requests.get("http://127.0.0.1:{}".format(self.server_eventgen_webport)) + assert r.status_code == 200 + assert "Host: " in r.content + assert "Eventgen Status" in r.content + assert "Eventgen Config file path" in r.content + assert "Total volume:" in r.content + assert "Worker Queue Status" in r.content + assert "Sample Queue Status" in r.content + assert "Output Queue Status" in r.content + + def test_server_index(self): + r = requests.get("http://127.0.0.1:{}/index".format(self.server_eventgen_webport)) + assert r.status_code == 200 + assert "Host: " in r.content + assert "Eventgen Status" in r.content + assert "Eventgen Config file path" in r.content + assert "Total volume:" in r.content + assert "Worker Queue Status" in r.content + assert "Sample Queue Status" in r.content + assert "Output Queue Status" in r.content + + def test_server_status(self): + r = requests.get("http://127.0.0.1:{}/status".format(self.server_eventgen_webport)) + assert r.status_code == 200 + output = json.loads(r.content) + assert output + assert output['EVENTGEN_STATUS'] == 0 + + def test_server_get_and_set_conf(self): + r = requests.get("http://127.0.0.1:{}/conf".format(self.server_eventgen_webport)) + assert r.status_code == 200 + assert json.loads(r.content) == {} + config_json = {"windbag": {"outputMode": "stdout"}} + r = requests.post("http://127.0.0.1:{}/conf".format(self.server_eventgen_webport), json=config_json) + assert r.status_code == 200 + assert json.loads(r.content) == config_json + + def test_server_start(self): + r = requests.post("http://127.0.0.1:{}/start".format(self.server_eventgen_webport), timeout=5) + assert r.status_code == 200 + assert json.loads(r.content) == "Eventgen has successfully started." + + def test_server_restart(self): + r = requests.post("http://127.0.0.1:{}/restart".format(self.server_eventgen_webport)) + assert r.status_code == 200 + assert json.loads(r.content) == "Eventgen restarted." + + def test_server_stop(self): + r = requests.post("http://127.0.0.1:{}/stop".format(self.server_eventgen_webport)) + assert r.status_code == 200 + assert json.loads(r.content) == "Eventgen is stopped." + + def test_server_bundle(self): + r = requests.post("http://127.0.0.1:{}/bundle".format(self.server_eventgen_webport)) + assert r.status_code == 400 + assert "Please pass in a valid object with bundle URL" in r.content + + def test_server_get_and_set_volume(self): + # Must initialize a stanza with the perDayVolume setting before hitting the /volume endpoint + r = requests.put("http://127.0.0.1:{}/conf".format(self.server_eventgen_webport), json={"windbag": {}}) + assert r.status_code == 200 + assert json.loads(r.content) + r = requests.post("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport), json={"perDayVolume": 10}) + assert r.status_code == 200 + assert json.loads(r.content) + r = requests.get("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport)) + assert r.status_code == 200 + output = json.loads(r.content) + assert output == 10.0 + r = requests.post("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport), json={"perDayVolume": 150}) + assert r.status_code == 200 + assert json.loads(r.content) + r = requests.get("http://127.0.0.1:{}/volume".format(self.server_eventgen_webport)) + assert r.status_code == 200 + output = json.loads(r.content) + assert output == 150.0 diff --git a/tests/medium/plugins/test_file_output.py b/tests/medium/plugins/test_file_output.py index d1b7a92a..f290f640 100644 --- a/tests/medium/plugins/test_file_output.py +++ b/tests/medium/plugins/test_file_output.py @@ -3,7 +3,9 @@ import os import sys + from mock import patch + from splunk_eventgen.__main__ import parse_args from splunk_eventgen.eventgen_core import EventGenerator @@ -11,7 +13,6 @@ class TestFileOutputPlugin(object): - def test_output_data_to_file(self): configfile = "tests/sample_eventgen_conf/medium_test/eventgen.conf.fileoutput" testargs = ["eventgen", "generate", configfile] diff --git a/tests/medium/plugins/test_jinja_generator.py b/tests/medium/plugins/test_jinja_generator.py index beab0b82..7a4201d7 100644 --- a/tests/medium/plugins/test_jinja_generator.py +++ b/tests/medium/plugins/test_jinja_generator.py @@ -1,6 +1,8 @@ import os import sys + from mock import patch + from splunk_eventgen.__main__ import parse_args from splunk_eventgen.eventgen_core import EventGenerator @@ -9,7 +11,6 @@ class TestJinjaGenerator(object): - def test_jinja_generator_to_file(self): configfile = "tests/sample_eventgen_conf/jinja/eventgen.conf.jinja_basic" testargs = ["eventgen", "generate", configfile] diff --git a/tests/medium/plugins/test_syslog_output.py b/tests/medium/plugins/test_syslog_output.py index eb5b8c45..831296c4 100644 --- a/tests/medium/plugins/test_syslog_output.py +++ b/tests/medium/plugins/test_syslog_output.py @@ -3,7 +3,9 @@ import os import sys + from mock import MagicMock, patch + from splunk_eventgen.__main__ import parse_args from splunk_eventgen.eventgen_core import EventGenerator from splunk_eventgen.lib.plugins.output.syslogout import SyslogOutOutputPlugin @@ -12,12 +14,11 @@ class TestSyslogOutputPlugin(object): - def test_output_data_to_syslog(self): configfile = "tests/sample_eventgen_conf/medium_test/eventgen.conf.syslogoutput" testargs = ["eventgen", "generate", configfile] with patch.object(sys, 'argv', testargs): - with patch('logging.getLogger') as mock_log: + with patch('logging.getLogger'): pargs = parse_args() assert pargs.subcommand == 'generate' assert pargs.configfile == configfile diff --git a/tests/medium/plugins/test_tcp_output.py b/tests/medium/plugins/test_tcp_output.py index e62ecdaa..e3ea1320 100644 --- a/tests/medium/plugins/test_tcp_output.py +++ b/tests/medium/plugins/test_tcp_output.py @@ -3,7 +3,9 @@ import os import sys + from mock import MagicMock, patch + from splunk_eventgen.__main__ import parse_args from splunk_eventgen.eventgen_core import EventGenerator from splunk_eventgen.lib.plugins.output.tcpout import TcpOutputPlugin @@ -12,7 +14,6 @@ class TestTcpOutputPlugin(object): - def test_output_data_to_tcp_port(self): configfile = "tests/sample_eventgen_conf/medium_test/eventgen.conf.tcpoutput" testargs = ["eventgen", "generate", configfile] @@ -35,4 +36,3 @@ def test_output_data_to_tcp_port(self): eventgen.start() tcpoutput.s.connect.assert_called_with(('127.0.0.1', 9999)) assert tcpoutput.s.send.call_count == 5 - diff --git a/tests/medium/plugins/test_udp_output.py b/tests/medium/plugins/test_udp_output.py index aec78fa6..a7cbde26 100644 --- a/tests/medium/plugins/test_udp_output.py +++ b/tests/medium/plugins/test_udp_output.py @@ -3,7 +3,9 @@ import os import sys + from mock import MagicMock, patch + from splunk_eventgen.__main__ import parse_args from splunk_eventgen.eventgen_core import EventGenerator from splunk_eventgen.lib.plugins.output.udpout import UdpOutputPlugin @@ -12,7 +14,6 @@ class TestUdpOutputPlugin(object): - def test_output_data_to_udp_port(self): configfile = "tests/sample_eventgen_conf/medium_test/eventgen.conf.udpoutput" testargs = ["eventgen", "generate", configfile] diff --git a/tests/run_tests.py b/tests/run_tests.py index 871c29c2..8d433456 100644 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -1,7 +1,8 @@ -import pytest -import sys -import time import os +import sys + +import pytest + SMALL = 'tests/small' MEDIUM = 'tests/medium' LARGE = 'tests/large' @@ -11,13 +12,12 @@ ENV = os.environ PATH = sys.path -# Set to 1 is debugging is a problem. Normally, it is 8, which should match the cores/hyperthreads of most of our systems. +# Normally, it is 8, which should match the cores/hyperthreads of most of our systems. NUM_TEST_WORKERS_LARGE = '8' - """ How to run the tests: 1. python run_tests.py -2. python run_tests.py {SMALL_TESTS_TO_RUN} {MEDIUM_TESTS_TO_RUN} {LARGE_TESTS_TO_RUN} {XLARGE_TESTS_TO_RUN} {optional RUN_DESTROY} +2. python run_tests.py {SMALL_TEST_PATH} {MEDIUM_TEST_PATH} {LARGE_TEST_PATH} {XLARGE_TEST_PATH} {optional RUN_DESTROY} - You can pass 'None' as a value to either to ignore those tests - To run a specific folder, file, pass it in as a value. ex * python run_tests.py None None tests/large/test_destroy.py None @@ -46,7 +46,9 @@ if SMALL: sys.path = PATH os.environ = ENV - args = [ "--cov=splunk_eventgen", "--cov-config=tests/.coveragerc", "--cov-report=term", "--cov-report=html", SMALL, "--junitxml=tests/test-reports/tests_small_results.xml"] + args = [ + "--cov=splunk_eventgen", "--cov-config=tests/.coveragerc", "--cov-report=term", "--cov-report=html", SMALL, + "--junitxml=tests/test-reports/tests_small_results.xml"] return_codes.append(pytest.main(args)) # Run medium tests diff --git a/tests/small/test_main.py b/tests/small/test_main.py index 6dda5007..3f68254d 100644 --- a/tests/small/test_main.py +++ b/tests/small/test_main.py @@ -1,23 +1,25 @@ #!/usr/bin/env python2 -import pytest import os import sys -from mock import MagicMock, call, patch, mock_open + +import pytest +from mock import MagicMock, patch + +from splunk_eventgen.__main__ import parse_cli_vars, parse_env_vars FILE_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(FILE_DIR, "..", "..", "..")) sys.path.insert(0, os.path.join(FILE_DIR, "..", "..", "..", "splunk_eventgen")) -from splunk_eventgen.__main__ import parse_cli_vars, parse_env_vars -@pytest.mark.parametrize(('config'), - [ - # Empty config - ({}), - # Some elements already defined - function should override - ({"AMQP_HOST": "guest", "AMQP_PASS": "guest"}) - ]) +@pytest.mark.parametrize( + ('config'), + [ + # Empty config + ({}), + # Some elements already defined - function should override + ({"AMQP_HOST": "guest", "AMQP_PASS": "guest"})]) def test_parse_cli_vars(config): args = MagicMock() args.amqp_uri = "pyamqp://user:pass@host:port" @@ -28,32 +30,34 @@ def test_parse_cli_vars(config): args.amqp_pass = "world" args.web_server_address = "0.0.0.:1111" obj = parse_cli_vars(config, args) - assert obj == { "AMQP_URI": "pyamqp://user:pass@host:port", - "AMQP_HOST": "hostname", - "AMQP_PORT": 8001, - "AMQP_WEBPORT": 8000 , - "AMQP_USER": "hello", - "AMQP_PASS": "world", - "WEB_SERVER_ADDRESS": "0.0.0.:1111" } + assert obj == { + "AMQP_URI": "pyamqp://user:pass@host:port", "AMQP_HOST": "hostname", "AMQP_PORT": 8001, "AMQP_WEBPORT": 8000, + "AMQP_USER": "hello", "AMQP_PASS": "world", "WEB_SERVER_ADDRESS": "0.0.0.:1111"} -@pytest.mark.parametrize(('env_vars'), - [ - # No environment vars defined - ({}), - # All environemnt vars defined - ({"EVENTGEN_AMQP_URI": "test", "EVENTGEN_AMQP_HOST": "host", "EVENTGEN_AMQP_PORT": 8000, "EVENTGEN_AMQP_WEBPORT": 8001, "EVENTGEN_AMQP_USER": "hello", "EVENTGEN_AMQP_PASS": "world", "EVENTGEN_WEB_SERVER_ADDR": "0.0.0.0:1111"}) - ]) + +@pytest.mark.parametrize( + ('env_vars'), + [ + # No environment vars defined + ({}), + # All environemnt vars defined + ({ + "EVENTGEN_AMQP_URI": "test", "EVENTGEN_AMQP_HOST": "host", "EVENTGEN_AMQP_PORT": 8000, + "EVENTGEN_AMQP_WEBPORT": 8001, "EVENTGEN_AMQP_USER": "hello", "EVENTGEN_AMQP_PASS": "world", + "EVENTGEN_WEB_SERVER_ADDR": "0.0.0.0:1111"})]) def test_parse_env_vars(env_vars): with patch("splunk_eventgen.__main__.os") as mock_os: mock_os.environ = env_vars obj = parse_env_vars() - assert obj.keys() == ['AMQP_WEBPORT', 'AMQP_USER', 'AMQP_PASS', 'AMQP_PORT', 'AMQP_URI', 'WEB_SERVER_ADDRESS', 'AMQP_HOST'] + assert obj.keys() == [ + 'AMQP_WEBPORT', 'AMQP_USER', 'AMQP_PASS', 'AMQP_PORT', 'AMQP_URI', 'WEB_SERVER_ADDRESS', 'AMQP_HOST'] if env_vars: # If enviroment vars are defined, let's make sure they are set instead of default values assert obj["WEB_SERVER_ADDRESS"] == "0.0.0.0:1111" assert obj["AMQP_HOST"] == "host" assert obj["AMQP_PORT"] == 8000 + def test_parse_env_vars_and_parse_cli_vars(): ''' This test checks the layering effect of both parsing CLI and env vars. @@ -67,7 +71,7 @@ def test_parse_env_vars_and_parse_cli_vars(): assert obj["AMQP_PORT"] == 5672 assert obj["AMQP_PASS"] == "guest" assert obj["AMQP_USER"] == "guest" - assert obj["AMQP_URI"] == None + assert obj["AMQP_URI"] is None assert obj["WEB_SERVER_ADDRESS"] == "0.0.0.0:9500" args = MagicMock() args.amqp_uri = "pyamqp://user:pass@host:port" @@ -78,11 +82,7 @@ def test_parse_env_vars_and_parse_cli_vars(): # Purposely defining None vars here for these CLI args - in this case, environment vars will be used args.amqp_user = None args.amqp_pass = None - newobj = parse_cli_vars(obj, args) - assert obj == { "AMQP_URI": "pyamqp://user:pass@host:port", - "AMQP_HOST": "hostname", - "AMQP_PORT": 8001, - "AMQP_WEBPORT": 8000 , - "AMQP_USER": "guest", - "AMQP_PASS": "guest", - "WEB_SERVER_ADDRESS": "0.0.0.:1111" } + parse_cli_vars(obj, args) + assert obj == { + "AMQP_URI": "pyamqp://user:pass@host:port", "AMQP_HOST": "hostname", "AMQP_PORT": 8001, "AMQP_WEBPORT": + 8000, "AMQP_USER": "guest", "AMQP_PASS": "guest", "WEB_SERVER_ADDRESS": "0.0.0.:1111"}