Configurable extra plugin paths

Closes #2

JsonStats/Utils.py

@@ -1,3 +1,8 @@
+import fnmatch
+import os
+import os.path
+import re
+import sys
 try:
     import json
 except:
@@ -19,3 +24,55 @@ def dump_sorted_json_string(input, **kwargs):
     * `json.dumps` docs - http://docs.python.org/2.7/library/json.html#json.dumps
     """
     return json.dumps(input, sort_keys=True, separators=(',', ': '), **kwargs)
+
+
+def load_extra_plugins(pathspec):
+    """
+    Load extra fact plugins from a user specified directory
+
+    `pathspec` - A string of either: a single directory path, or a
+    compound (colon separated) list of paths
+    """
+    # Collect names of loaded plugins so we have something to test
+    # success/failure against
+    loaded_plugins = []
+
+    paths = pathspec.split(':')
+    for path in paths:
+        loaded_plugins.extend(_load_plugins_from_dir(path))
+
+    return loaded_plugins
+
+
+def _load_plugins_from_dir(path):
+    """
+    Load plugins from a given path
+    """
+    plugins = []
+    loaded_plugins = []
+
+    # Expand out any ~'s
+    full_path = os.path.expanduser(path)
+
+    # Check if the dir exists, by asking for forgiveness
+    try:
+        filtered = fnmatch.filter(os.listdir(path), '*.py')
+        plugins.extend(filtered)
+    except OSError:
+        # Uhoh, the path we were given doesn't exist/can't be read...
+        pass
+    else:
+        # We read in the plugin dir, so lets add it to the load path
+        sys.path.insert(1, path)
+
+    # Import the thing(s)
+    for plugin in plugins:
+        match = re.search('(?P<name>.*)(?P<ext>\.py$)', plugin)
+        if match:
+            try:
+                __import__(match.group('name'), globals(), locals(), [], -1)
+            except:
+                pass
+            else:
+                loaded_plugins.append(match.group('name'))
+    return loaded_plugins

README.md

@@ -138,11 +138,13 @@ of all RPM packages installed on the system.
 
     Options:
       -h, --help            show this help message and exit
-      -p PORT, --port=PORT  Port to listen on. (Default: 8008)
+      -p PORT, --port=PORT  Port to listen on (Default: 8008)
       -l LISTEN, --listen=LISTEN
-                            Address to listen on. (Default: 0.0.0.0)
-      --logdir=LOGDIR       Directory to log access requests to. (Default:
+                            Address to listen on (Default: 0.0.0.0)
+      --logdir=LOGDIR       Directory to log access requests to (Default:
                             ./logs/)
+      -e PLUGIN_PATHSPEC --extra-plugins=PLUGIN_PATHSPEC
+                            Path to directory with additional plugins
 
 
 # More Information
@@ -167,21 +169,11 @@ service configuration file in `/etc/sysconfig/jsonstatsd`
 
 
     ######################################################################
-    # Listen on all interfaces by default. You might also enjoy: 127.0.0.1
-    # to just listen locally
-    INTERFACE=0.0.0.0
-
-    ######################################################################
-    # Port to listen on
-    PORT=8008
-
+    # See 'man 1 jsonstatsd' or 'jsonstatsd --help' for descriptions of
+    # all the available options
     ######################################################################
-    # Customize the logging directory
-    LOGDIR=/var/log/jsonstatsd
 
-    ######################################################################
-    # Single line with all options for the SysV init script
-    OPTIONS="--listen $INTERFACE --port $PORT --logdir $LOGDIR"
+    OPTIONS="--listen 0.0.0.0 --port 8008 --logdir /var/log/jsonstatsd"
 
 
 ## Logging

bin/jsonstatsd

@@ -6,6 +6,7 @@ import re
 import urllib
 import syslog
 import errno
+import JsonStats.Utils
 
 
 ######################################################################
@@ -203,7 +204,8 @@ if __name__ == "__main__":
 
     # Using optparse since argparse is not available in 2.5
     from optparse import OptionParser
-    parser = OptionParser()
+    epilog = "Note: use colons to to specify multiple extra-plugin paths"
+    parser = OptionParser(epilog=epilog)
     parser.add_option('-p', '--port', dest='port', default=8008, type='int',
                       help='Port to listen on. (Default: 8008)')
     parser.add_option(
@@ -212,10 +214,15 @@ if __name__ == "__main__":
     parser.add_option(
         '--logdir', dest='logdir', default='./logs/',
         help='Directory to log access requests to. (Default: ./logs/)')
+    parser.add_option(
+        '--extra-plugins', '-e', dest='extra_plugins', metavar='PLUGIN_PATHSPEC',
+        help='Path to directory with additional plugins')
 
     (options, args) = parser.parse_args()
 
     os.environ['JSONSTATS_LOG_DIR'] = options.logdir
+    if options.extra_plugins:
+        JsonStats.Utils.load_extra_plugins(options.extra_plugins)
 
     try:
         # import StatsApp class after the CLI options are parsed. This

docs/man/man1/jsonstatsd.1

@@ -1,13 +1,13 @@
 '\" t
 .\"     Title: jsonstatsd
-.\"    Author: [see the "AUTHOR" section]
+.\"    Author: :doctype:manpage
 .\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
-.\"      Date: 12/13/2013
+.\"      Date: 02/16/2014
 .\"    Manual: RESTful interface to query system facts
-.\"    Source: jsonstats 1.0.1
+.\"    Source: jsonstats 1.0.2
 .\"  Language: English
 .\"
-.TH "JSONSTATSD" "1" "12/13/2013" "jsonstats 1\&.0\&.1" "RESTful interface to query sys"
+.TH "JSONSTATSD" "1" "02/16/2014" "jsonstats 1\&.0\&.2" "RESTful interface to query sys"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -42,15 +42,15 @@ jsonstatsd [\-h] [\-p PORT] [\-l INTERFACE] [\-\-logdir DIR]
 Show a help message and exit\&.
 .RE
 .PP
-\fB\-p PORT\fR, \fB\-\-port PORT\fR
+\fB\-p PORT\fR, \fB\-\-port=PORT\fR
 .RS 4
 Specify the
 \fBPORT\fR
 to listen on\&. Default:
 \fI8008\fR
 .RE
 .PP
-\fB\-l INTERFACE\fR, \fB\-\-listen INTERFACE\fR
+\fB\-l INTERFACE\fR, \fB\-\-listen=INTERFACE\fR
 .RS 4
 Specify the
 \fBINTERFACE\fR
@@ -62,13 +62,19 @@ Give
 to only listen for local requests\&.
 .RE
 .PP
-\fB\-\-logdir DIRECTORY\fR
+\fB\-\-logdir=DIRECTORY\fR
 .RS 4
 The
 \fBDIRECTORY\fR
 to store application logs in\&. Default:
 \fI\&./logs/\fR
 .RE
+.PP
+\fB\-e PLUGIN_PATHSPEC\fR, \fB\-\-extra\-plugins=PLUGIN_PATHSPEC\fR
+.RS 4
+Path to one or more directories holding custom fact plugins\&. Separate multiple paths with colon characters\&. For example:
+\fB\-\-extra\-plugins=/path1:/another/path\fR
+.RE
 .SH "EXAMPLE"
 .sp
 Fetch the data
@@ -110,7 +116,7 @@ $ curl localhost:8008 | python \-m json\&.tool
         "emacs\-color\-theme": "6\&.6\&.0",
         "emacs\-common": "24\&.1",
         "emacs\-common\-w3m": "1\&.4\&.435",
-        "jsonstats": "1\&.0\&.1",
+        "jsonstats": "1\&.0\&.2",
         "talook": "1\&.2\&.09
         \&.\&.\&.
     }
@@ -143,3 +149,9 @@ jsonstats is released under the terms of the MIT License\&.
 \fBIntroducing JSON\fR: http://www\&.json\&.org/
 .sp
 \fBThe jsonstats Homepage\fR: https://github\&.com/RHInception/jsonstats/
+.SH "AUTHOR"
+.PP
+\fB:doctype:manpage\fR
+.RS 4
+Author.
+.RE

docs/man/man1/jsonstatsd.1.asciidoc.in

@@ -33,20 +33,26 @@ OPTIONS
 
 Show a help message and exit.
 
-*-p PORT*, *--port PORT*::
+*-p PORT*, *--port=PORT*::
 
 Specify the *PORT* to listen on. Default: '8008'
 
-*-l INTERFACE*, *--listen INTERFACE*::
+*-l INTERFACE*, *--listen=INTERFACE*::
 
 Specify the *INTERFACE* to listen on. Default: '0.0.0.0'
 +
 Give *127.0.0.1* to only listen for local requests.
 
-*--logdir DIRECTORY*::
+*--logdir=DIRECTORY*::
 
 The *DIRECTORY* to store application logs in. Default: './logs/'
 
+*-e PLUGIN_PATHSPEC*, *--extra-plugins=PLUGIN_PATHSPEC*::
+
+Path to one or more directories holding custom fact plugins. Separate
+multiple paths with colon characters. For example:
+*--extra-plugins=/path1:/another/path*
+
 
 EXAMPLE
 -------

etc/sysconfig/jsonstatsd

@@ -1,16 +1,6 @@
 ######################################################################
-# Listen on all interfaces by default. You might also enjoy: 127.0.0.1
-# to just listen locally
-INTERFACE=0.0.0.0
-
-######################################################################
-# Port to listen on
-PORT=8008
-
+# See 'man 1 jsonstatsd' or 'jsonstatsd --help' for descriptions of
+# all the available options
 ######################################################################
-# Customize the logging directory
-LOGDIR=/var/log/jsonstatsd
 
-######################################################################
-# Single line with all options for the SysV init script
-OPTIONS="--listen $INTERFACE --port $PORT --logdir $LOGDIR"
+OPTIONS="--listen 0.0.0.0 --port 8008 --logdir /var/log/jsonstatsd"

lib/systemd/system/jsonstatsd.service

@@ -4,7 +4,7 @@ After=syslog.target network.target auditd.service
 
 [Service]
 EnvironmentFile=/etc/sysconfig/jsonstatsd
-ExecStart=/usr/bin/jsonstatsd --listen $INTERFACE --port $PORT --logdir $LOGDIR
+ExecStart=/usr/bin/jsonstatsd $OPTIONS
 User=jsonstatsd
 
 [Install]

test/files/ExtraPlugin.py

@@ -0,0 +1,15 @@
+"""
+Dummy plugin used for the extra_plugins test
+"""
+
+from JsonStats.FetchStats import Fetcher
+class ExtraPlugin(Fetcher):
+    def __init__(self):
+        self.context = 'extraplugin'
+        self._load_data()
+    def _load_data(self):
+        self._loaded(True)
+    def dump(self):
+        return {}
+    def dump_json(self):
+        return self.json.dumps(self.dump())

test/files/ExtraPlugin2.py

@@ -0,0 +1,15 @@
+"""
+Dummy plugin #2 used for the extra_plugins compound-path test
+"""
+
+from JsonStats.FetchStats import Fetcher
+class ExtraPlugin2(Fetcher):
+    def __init__(self):
+        self.context = 'extraplugin2'
+        self._load_data()
+    def _load_data(self):
+        self._loaded(True)
+    def dump(self):
+        return {}
+    def dump_json(self):
+        return self.json.dumps(self.dump())

test/test_loadextraplugins.py

@@ -0,0 +1,54 @@
+"""
+Test for user-defined plugin locations
+"""
+
+
+from . import TestCase
+import tempfile
+from JsonStats.FetchStats import Fetcher
+from JsonStats.Utils import load_extra_plugins
+import shutil
+
+class TestLoadExtraPlugins(TestCase):
+    import os
+    import os.path
+
+    def setUp(self):
+        # We create a directory to put our 'extra' plugin file into
+        self._extra_dir = tempfile.mkdtemp('extra_plugin')
+        self._extra_plugin = self.os.path.join(self._extra_dir, 'ExtraPlugin.py')
+
+        # And again so we can test compound pathspec's
+        self._extra_dir2 = tempfile.mkdtemp('extra_plugin2')
+        self._extra_plugin2 = self.os.path.join(self._extra_dir2, 'ExtraPlugin2.py')
+
+        self._extra_plugin_dirs = (self._extra_dir, self._extra_dir2)
+
+        # Put the extra plugin2 into the temp dirs
+        shutil.copy('test/files/ExtraPlugin.py', self._extra_plugin)
+        shutil.copy('test/files/ExtraPlugin2.py', self._extra_plugin2)
+
+    def test_load_extra_plugins(self):
+        """
+        Verify that we can load plugins from user-defined locations
+        """
+        loaded = load_extra_plugins(self._extra_dir)
+        total_loaded = len(loaded)
+        self.assertEqual(total_loaded, 1, msg="Loaded %d plugins (%s), expected 1" % (total_loaded, ', '.join(loaded)))
+
+    def test_load_compound_path(self):
+        """
+        Verify that we can load plugins from a compound pathspec
+        """
+        compound_pathspec = "%s:%s" % self._extra_plugin_dirs
+        loaded = load_extra_plugins(compound_pathspec)
+        total_loaded = len(loaded)
+        self.assertEqual(total_loaded, 2, msg="Loaded %d plugins (%s), expected 2" % (total_loaded, ', '.join(loaded)))
+
+    def tearDown(self):
+        # Cleanup all of our temporary files
+        for d in self._extra_plugin_dirs:
+            for f in self.os.listdir(d):
+                ff = self.os.path.join(d, f)
+                self.os.remove(ff)
+            self.os.rmdir(d)

test/test_pluginmount.py

@@ -33,4 +33,4 @@ def test_get_plugins(self):
         plugins = self.fetcher.get_plugins()
         plugin_names = map(lambda x: str(x), plugins)
         discovered = len(plugins)
-        self.assertEqual(discovered, 1, msg="Discovered %d plugins (%s), expected 1" % (discovered, ', '.join(plugin_names)))
+        self.assertGreaterEqual(discovered, 1, msg="Discovered %d plugins (%s), expected >=1" % (discovered, ', '.join(plugin_names)))