diff --git a/base-buildout.cfg b/base-buildout.cfg index e3b70a1a5..5dbda7ed0 100644 --- a/base-buildout.cfg +++ b/base-buildout.cfg @@ -74,7 +74,7 @@ extra_options = [extra-stuff] recipe = plone.recipe.command -command = mkdir -p ${buildout:directory}/var/{log,run} && +command = mkdir -p ${buildout:directory}/var/log && mkdir -p ${buildout:directory}/src/rockstor/logs && usermod -a -G root nginx && systemctl disable nginx @@ -146,3 +146,37 @@ cmds = ${buildout:directory}/bin/django collectstatic --noinput -i admin -v 0 recipe = collective.recipe.template input = ${buildout:directory}/conf/docker.service.in output = ${buildout:directory}/conf/docker.service + +[js-libraries] +recipe = hexagonit.recipe.download +strip-top-level-dir = true +destination = ${buildout:directory}/static/js/lib +on-update = true +ignore-existing = true + +[init-gunicorn] +recipe = collective.recipe.template +bind = 127.0.0.1 +port = 8000 +workers = 1 +user = rocky +pidfile = /run/gunicorn.pid + +[supervisord-conf] +recipe = collective.recipe.template +host = 127.0.0.1 +port = 9001 +logdir = ${buildout:directory}/var/log +logfile = ${supervisord-conf:logdir}/supervisord.log +input = ${buildout:directory}/conf/supervisord.conf.in +output = ${buildout:directory}/etc/supervisord.conf +pidfile = /run/supervisord.pid + +[django-settings-conf] +recipe = collective.recipe.template +tapport = 10000 +sinkport = 10001 +schedulerport = 10001 +reppubport = 10002 +reprecvport = 10003 +input = ${buildout:directory}/conf/settings.conf.in \ No newline at end of file diff --git a/buildout.cfg b/buildout.cfg index 88ac131b4..d01c1aa32 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -44,23 +44,11 @@ input = ${buildout:directory}/conf/nginx.conf.in output = ${buildout:directory}/etc/nginx/nginx.conf [init-gunicorn] -recipe = collective.recipe.template -bind = 127.0.0.1 -port = 8000 -workers = 1 -user = rocky logfile = ${buildout:directory}/var/log/gunicorn.log -pidfile = ${buildout:directory}/var/run/gunicorn.pid input = ${buildout:directory}/conf/gunicorn.in output = ${buildout:directory}/etc/init.d/gunicorn [supervisord-conf] -recipe = collective.recipe.template -host = 127.0.0.1 -port = 9001 -logdir = ${buildout:directory}/var/log -logfile = ${supervisord-conf:logdir}/supervisord.log -pidfile = ${buildout:directory}/var/run/supervisord.pid gunicorn_cmd = ${buildout:directory}/bin/gunicorn --bind=${init-gunicorn:bind}:${init-gunicorn:port} --pid=${init-gunicorn:pidfile} --workers=${init-gunicorn:workers} --log-file=${init-gunicorn:logfile} --pythonpath=${buildout:directory}/src/rockstor --settings=settings --timeout=120 --graceful-timeout=120 wsgi:application smart_manager_cmd = ${buildout:directory}/bin/sm replicad_cmd = ${buildout:directory}/bin/replicad @@ -69,11 +57,9 @@ dc_cmd = ${buildout:directory}/bin/data-collector sm_cmd = ${buildout:directory}/bin/service-monitor jd_cmd = ${buildout:directory}/bin/job-dispatcher ztask_cmd = ${buildout:directory}/bin/django ztaskd --noreload -l DEBUG --replayfailed -f ${supervisord-conf:logdir}/ztask.log -input = ${buildout:directory}/conf/supervisord.conf.in -output = ${buildout:directory}/etc/supervisord.conf +dc2_cmd = ${buildout:directory}/bin/dc2 [django-settings-conf] -recipe = collective.recipe.template rootdir = ${buildout:directory}/src/rockstor datastore = ${django-settings-conf:rootdir}/storageadmin/datastore smartdb = ${django-settings-conf:rootdir}/smart_manager/smartdb @@ -84,24 +70,13 @@ template_dir2 = ${django-settings-conf:rootdir}/templates/admin smb_conf = ${buildout:directory}/conf/smb.conf logfile = ${buildout:directory}/var/log/rockstor.log taplib = ${django-settings-conf:rootdir}/smart_manager/taplib -tapport = 10000 -sinkport = 10001 -input = ${buildout:directory}/conf/settings.conf.in output = ${django-settings-conf:rootdir}/settings.py -schedulerport = 10001 -reppubport = 10002 -reprecvport = 10003 debug = True kernel = '4.1.0-1.el7.elrepo.x86_64' [js-libraries] -recipe = hexagonit.recipe.download -url = http://rockstor.com/downloads/js/lib.tgz -md5sum = a949705f5af85db40c92ea1ef8337479 -strip-top-level-dir = true -destination = ${buildout:directory}/static/js/lib -on-update = true -ignore-existing = true +url = http://rockstor.com/downloads/jslibs/testing/lib.tgz +md5sum = ee540ba3904d90f49781d3c994ac8220 [postgres-setup] recipe = plone.recipe.command diff --git a/conf/nginx.conf.in b/conf/nginx.conf.in index 2459bcae6..b65c3ac3e 100644 --- a/conf/nginx.conf.in +++ b/conf/nginx.conf.in @@ -79,5 +79,12 @@ http { proxy_read_timeout 120; proxy_pass http://${init-gunicorn:bind}:${init-gunicorn:port}/; } + location /socket.io { + proxy_pass http://127.0.0.1:8001/socket.io; + proxy_redirect off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } } } diff --git a/conf/supervisord.conf.in b/conf/supervisord.conf.in index 00a06dc80..5579d00bc 100644 --- a/conf/supervisord.conf.in +++ b/conf/supervisord.conf.in @@ -181,3 +181,24 @@ stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) stderr_logfile=${supervisord-conf:logdir}/supervisord_%(program_name)s_stderr.log ; stderr log path, NONE for none; default AUTO stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) stderr_logfile_backups=10 ; # of stderr logfile backups (default 10) + +; dc2 daemon +[program:dc2] +environment=DJANGO_SETTINGS_MODULE=settings +command=${supervisord-conf:dc2_cmd} ; the program (relative uses PATH, can take args) +process_name=%(program_name)s ; process_name expr (default %(program_name)s) +numprocs=1 ; number of processes copies to start (def 1) +priority=200 +autostart=false ; start at supervisord start (default: true) +autorestart=unexpected ; whether/when to restart (default: unexpected) +startsecs=2 ; number of secs prog must stay running (def. 1) +startretries=3 ; max # of serial start failures (default 3) +exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) +stopsignal=TERM ; signal used to kill process (default TERM) +stopwaitsecs=5 ; max num secs to wait b4 SIGKILL (default 10) +stdout_logfile=${supervisord-conf:logdir}/supervisord_%(program_name)s_stdout.log ; stdout log path, NONE for none; default AUTO +stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) +stderr_logfile=${supervisord-conf:logdir}/supervisord_%(program_name)s_stderr.log ; stderr log path, NONE for none; default AUTO +stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +stderr_logfile_backups=10 ; # of stderr logfile backups (default 10) \ No newline at end of file diff --git a/prod-buildout.cfg b/prod-buildout.cfg index 84f29bf39..5c735b5fc 100644 --- a/prod-buildout.cfg +++ b/prod-buildout.cfg @@ -43,23 +43,11 @@ input = ${buildout:directory}/conf/nginx-prod.conf.in output = ${buildout:directory}/etc/nginx/nginx.conf [init-gunicorn] -recipe = collective.recipe.template -bind = 127.0.0.1 -port = 8000 -workers = 1 -user = rocky logfile = ${buildout:depdir}/var/log/gunicorn.log -pidfile = ${buildout:depdir}/var/run/gunicorn.pid input = ${buildout:directory}/conf/gunicorn.in output = ${buildout:directory}/etc/init.d/gunicorn [supervisord-conf] -recipe = collective.recipe.template -host = 127.0.0.1 -port = 9001 -logdir = ${buildout:depdir}/var/log -logfile = ${supervisord-conf:logdir}/supervisord.log -pidfile = ${buildout:depdir}/var/run/supervisord.pid gunicorn_cmd = ${buildout:depdir}/bin/gunicorn --bind=${init-gunicorn:bind}:${init-gunicorn:port} --pid=${init-gunicorn:pidfile} --workers=${init-gunicorn:workers} --log-file=${init-gunicorn:logfile} --pythonpath=${buildout:depdir}/src/rockstor --settings=settings --timeout=120 --graceful-timeout=120 wsgi:application smart_manager_cmd = ${buildout:depdir}/bin/sm replicad_cmd = ${buildout:depdir}/bin/replicad @@ -67,11 +55,8 @@ ts_cmd = ${buildout:depdir}/bin/task-scheduler dc_cmd = ${buildout:depdir}/bin/data-collector sm_cmd = ${buildout:depdir}/bin/service-monitor ztask_cmd = ${buildout:depdir}/bin/django ztaskd --noreload -f ${supervisord-conf:logdir}/ztask.log -input = ${buildout:directory}/conf/supervisord-prod.conf.in -output = ${buildout:directory}/etc/supervisord.conf [django-settings-conf] -recipe = collective.recipe.template rootdir = ${buildout:depdir}/src/rockstor datastore = ${django-settings-conf:rootdir}/storageadmin/datastore smartdb = ${django-settings-conf:rootdir}/smart_manager/smartdb @@ -81,13 +66,7 @@ template_dir2 = ${django-settings-conf:rootdir}/templates/admin smb_conf = ${buildout:depdir}/conf/smb.conf logfile = ${buildout:depdir}/var/log/rockstor.log taplib = ${django-settings-conf:rootdir}/smart_manager/taplib -tapport = 10000 -sinkport = 10001 -input = ${buildout:directory}/conf/settings.conf.in output = ${buildout:directory}/src/rockstor/settings.py -schedulerport = 10001 -reppubport = 10002 -reprecvport = 10003 debug = False kernel = '4.1.0-1.el7.elrepo.x86_64' @@ -96,13 +75,8 @@ recipe = zc.recipe.egg:scripts eggs = zc.sourcerelease [js-libraries] -recipe = hexagonit.recipe.download -url = http://rockstor.com/downloads/js/lib.tgz -md5sum = a949705f5af85db40c92ea1ef8337479 -strip-top-level-dir = true -destination = ${buildout:directory}/static/js/lib -on-update = true -ignore-existing = true +url = http://rockstor.com/downloads/jslibs/production/lib.tgz +md5sum = 491e4fa5fb5358521b05aeeb68a43353 [docker] recipe = plone.recipe.command diff --git a/setup.py b/setup.py index 1209fc135..bd8bc9b92 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ 'qgroup-clean = scripts.qgroup_clean:main', 'rockon-json = scripts.rockon_util:main', 'flash-optimize = scripts.flash_optimize:main', + 'dc2 = smart_manager.dc2:main', ], }, @@ -67,7 +68,10 @@ 'six == 1.7.3', 'django-ztask == 0.1.5', 'mock == 1.0.1', - 'coverage' + 'coverage', + 'gevent-socketio', + 'psycogreen', + 'psutil', ] ) diff --git a/src/rockstor/smart_manager/dc2.py b/src/rockstor/smart_manager/dc2.py new file mode 100644 index 000000000..fa6c3a158 --- /dev/null +++ b/src/rockstor/smart_manager/dc2.py @@ -0,0 +1,137 @@ +from gevent import monkey +monkey.patch_all() + +import gevent +from socketio.server import SocketIOServer +from socketio import socketio_manage +from socketio.namespace import BaseNamespace +from socketio.mixins import BroadcastMixin + +from django.conf import settings +from system.osi import (uptime, kernel_info) + +from system.services import service_status +import logging +logger = logging.getLogger(__name__) + + +class ServicesNamespace(BaseNamespace, BroadcastMixin): + + # Called before the recv_connect function + def initialize(self): + logger.debug('Services have been initialized') + + def recv_connect(self): + logger.debug("Services has connected") + self.emit('services:connected', { + 'key': 'services:connected', 'data': 'connected' + }) + self.spawn(self.send_service_statuses) + + def recv_disconnect(self): + logger.debug("Services have disconnected") + + def send_service_statuses(self): + # Iterate through the collection and assign the values accordingly + services = ('nfs', 'smb', 'ntpd', 'winbind', 'netatalk', + 'snmpd', 'docker', 'smartd', 'replication', + 'nis', 'ldap', 'sftp', 'data-collector', 'smartd', + 'service-monitor', 'docker', 'task-scheduler') + while True: + data = {} + for service in services: + data[service] = {} + output, error, return_code = service_status(service) + if (return_code == 0): + data[service]['running'] = return_code + else: + data[service]['running'] = return_code + + self.emit('services:get_services', { + 'data': data, 'key': 'services:get_services' + }) + gevent.sleep(5) + + +class SysinfoNamespace(BaseNamespace, BroadcastMixin): + start = False + supported_kernel = settings.SUPPORTED_KERNEL_VERSION + + # Called before the connection is established + def initialize(self): + logger.debug("Sysinfo has been initialized") + + # This function is run once on every connection + def recv_connect(self): + logger.debug("Sysinfo has connected") + self.emit("sysinfo:sysinfo", { + "key": "sysinfo:connected", "data": "connected" + }) + self.start = True + gevent.spawn(self.send_uptime) + gevent.spawn(self.send_kernel_info) + + # Run on every disconnect + def recv_disconnect(self): + logger.debug("Sysinfo has disconnected") + self.start = False + + def send_uptime(self): + # Seems redundant + while self.start: + self.emit('sysinfo:uptime', { + 'data': uptime(), 'key': 'sysinfo:uptime' + }) + gevent.sleep(30) + + def send_kernel_info(self): + try: + self.emit('sysinfo:kernel_info', { + 'data': kernel_info(self.supported_kernel), + 'key': 'sysinfo:kernel_info' + }) + except Exception as e: + logger.debug('kernel error') + # Emit an event to the front end to capture error report + self.emit('sysinfo:kernel_error', {'error': str(e)}) + self.error('unsupported_kernel', str(e)) + + +class Application(object): + def __init__(self): + self.buffer = [] + + def __call__(self, environ, start_response): + path = environ['PATH_INFO'].strip('/') or 'index.html' + + if path.startswith('/static') or path == 'index.html': + try: + data = open(path).read() + except Exception: + return not_found(start_response) + + if path.endswith(".js"): + content_type = "text/javascript" + elif path.endswith(".css"): + content_type = "text/css" + elif path.endswith(".swf"): + content_type = "application/x-shockwave-flash" + else: + content_type = "text/html" + + start_response('200 OK', [('Content-Type', content_type)]) + return [data] + if path.startswith("socket.io"): + socketio_manage(environ, {'/services': ServicesNamespace, + '/sysinfo': SysinfoNamespace}) + + +def not_found(start_response): + start_response('404 Not Found', []) + return ['

Not found

'] + + +def main(): + logger.debug('Listening on port http://127.0.0.1:8080 and on port 10843 (flash policy server)') + SocketIOServer(('127.0.0.1', 8001), Application(), + resource="socket.io", policy_server=True).serve_forever() diff --git a/src/rockstor/storageadmin/static/storageadmin/css/style.css b/src/rockstor/storageadmin/static/storageadmin/css/style.css index 6bbacb02a..6f1be17bd 100644 --- a/src/rockstor/storageadmin/static/storageadmin/css/style.css +++ b/src/rockstor/storageadmin/static/storageadmin/css/style.css @@ -1660,11 +1660,6 @@ input[type="radio"].selectedRoot { border-radius: 7px; } -/* Rockstor update modal dialog */ -#update-modal { - margin-left: -380px; - height: 300px; -} /* Rockstor shutdown modal dialog */ #shutdown-modal { @@ -2546,4 +2541,10 @@ This file is generated by `grunt build`, do not edit it by hand. padding: 10px 15px; position:relative; font-weight:bold; +} + +#uptime { + float: right; + padding-left: 20px; + color: white; } \ No newline at end of file diff --git a/src/rockstor/storageadmin/static/storageadmin/js/rockstor.js b/src/rockstor/storageadmin/static/storageadmin/js/rockstor.js index e04c61cbe..f489fef82 100644 --- a/src/rockstor/storageadmin/static/storageadmin/js/rockstor.js +++ b/src/rockstor/storageadmin/static/storageadmin/js/rockstor.js @@ -426,6 +426,7 @@ function fetchLoadAvg() { }); } +/* Deprecated in favor of websockets function fetchKernelInfo() { $.ajax({ url: '/api/commands/kernel', @@ -441,42 +442,7 @@ function fetchKernelInfo() { } }); } - -function displayLoadAvg(data) { - var n = parseInt(data.results[0]['uptime']); - var load_1 = parseFloat(data.results[0]['load_1']); - var load_5 = parseFloat(data.results[0]['load_5']); - var load_15 = parseFloat(data.results[0]['load_15']); - var secs = n % 60; - var mins = Math.round(n/60) % 60; - var hrs = Math.round(n / (60*60)) % 24; - var days = Math.round(n / (60*60*24)) % 365; - var yrs = Math.round(n / (60*60*24*365)); - var str = 'Uptime: '; - if (RockStorGlobals.kernel) { - str = 'Linux: ' + RockStorGlobals.kernel + '         ' + str; - } - if (yrs == 1) { - str += yrs + ' year, '; - } else if (yrs > 1) { - str += yrs + ' years, '; - } - if (days == 1) { - str += days + ' day, '; - } else if (days > 1) { - str += days + ' days, '; - } - if (hrs < 10) { - str += '0'; - } - str += hrs + ':'; - if (mins < 10) { - str += '0'; - } - str += mins; - str += '         Load: ' + load_1 + ', ' + load_5 + ', ' + load_15; - $('#appliance-loadavg').html(str); -} +*/ function fetchServerTime() { RockStorGlobals.serverTimeTimer = window.setInterval(function() { diff --git a/src/rockstor/storageadmin/static/storageadmin/js/router.js b/src/rockstor/storageadmin/static/storageadmin/js/router.js index 99a99406f..61ff20851 100644 --- a/src/rockstor/storageadmin/static/storageadmin/js/router.js +++ b/src/rockstor/storageadmin/static/storageadmin/js/router.js @@ -122,18 +122,12 @@ var AppRouter = Backbone.Router.extend({ if (RockStorGlobals.currentAppliance == null) { setApplianceName(); } - if (!RockStorGlobals.loadAvgDisplayed) { - updateLoadAvg(); - } if (!RockStorGlobals.serverTimeFetched) { fetchServerTime(); } if (!RockStorGlobals.browserChecked) { checkBrowser(); } - if (!RockStorGlobals.kernel) { - fetchKernelInfo(); - } // set a timer to get current rockstor version and checkif there is an // update available @@ -835,12 +829,6 @@ $(document).ready(function() { $('#globalerrmsg').empty(); }); - // Initialize websocket connection - // logger.debug('connecting to websocket'); - // RockStorSocket.socket = io.connect('https://' + document.location.host + ':' + NGINX_WEBSOCKET_PORT, - // {secure: true} - // ); - // RockStorSocket.socket.on('sm_data', RockStorSocket.msgHandler); // Initialize global error popup $('#global-err-overlay').overlay({load: false}); @@ -888,4 +876,57 @@ $(document).ready(function() { $('#donate-modal').modal('hide'); }); + /********** Websockets **************/ + // These are global websocket events + + // Grab the far right of the breadcrumb (under nav) + var $loadavg = $('#appliance-loadavg'); + + var kernelInfo = function(data) { + $loadavg.text('Linux: ' + data); + }; + + var displayLoadAvg = function(data) { + var n = parseInt(data); + var secs = n % 60; + var mins = Math.round(n/60) % 60; + var hrs = Math.round(n / (60*60)) % 24; + var days = Math.round(n / (60*60*24)) % 365; + var yrs = Math.round(n / (60*60*24*365)); + var str = 'Uptime: '; + if (yrs == 1) { + str += yrs + ' year, '; + } else if (yrs > 1) { + str += yrs + ' years, '; + } + if (days == 1) { + str += days + ' day, '; + } else if (days > 1) { + str += days + ' days, '; + } + if (hrs < 10) { + str += '0'; + } + str += hrs + ':'; + if (mins < 10) { + str += '0'; + } + str += mins; + $('#uptime').text(str); + }; + + RockStorSocket.addListener(kernelInfo, this, 'sysinfo:kernel_info'); + RockStorSocket.addListener(displayLoadAvg, this, 'sysinfo:uptime'); + + RockStorSocket.sysinfo.on('kernel_error', function(data) { + // Handling errors just by emitting message via the server + if (data.error.indexOf('kernel') !== -1) { + // Put an alert at the top of the page + $('#browsermsg').html('
' + data.error + '
'); + } + + }); + }); + + diff --git a/src/rockstor/storageadmin/static/storageadmin/js/socket_listen.js b/src/rockstor/storageadmin/static/storageadmin/js/socket_listen.js index b004a8450..39de88946 100644 --- a/src/rockstor/storageadmin/static/storageadmin/js/socket_listen.js +++ b/src/rockstor/storageadmin/static/storageadmin/js/socket_listen.js @@ -25,26 +25,35 @@ */ var RockStorSocket = {}; -RockStorSocket.handlerMap = {} // initialize handler array +RockStorSocket.handlerMap = {}; // initialize handler array -RockStorSocket.addListener = function(fn, fn_this, key) { - RockStorSocket.handlerMap[key] = {fn: fn, fn_this: fn_this}; -} +// Connect globally to sysinfo as the information is in the breadcrumb +RockStorSocket.sysinfo = io.connect('/sysinfo', {'secure': true, 'force new connection': true}); + +// Add the function and context to be fired when a message comes in +RockStorSocket.addListener = function(fn, fn_this, namespace) { + RockStorSocket.handlerMap[namespace] = {fn: fn, fn_this: fn_this}; + var key = namespace.split(':')[0]; + RockStorSocket[key].on(namespace, RockStorSocket.msgHandler); +}; + +// Disconnect everything RockStorSocket.removeAllListeners = function() { - _.each(_.keys(RockStorSocket.handlerMap), function(key) { - delete RockStorSocket.handlerMap[key]; - }); -} + _.each(_.keys(RockStorSocket.handlerMap), function(key) { + delete RockStorSocket.handlerMap[key]; + }); +}; + +RockStorSocket.removeOneListener = function(socketName) { + RockStorSocket[socketName].disconnect(); +}; +// Fire appropriate callback given message RockStorSocket.msgHandler = function(data) { - logger.debug('received msg'); - msg_data = JSON.parse(data.msg); - _.each(_.keys(RockStorSocket.handlerMap), function(key) { - if (!_.isNull(msg_data[key]) && !_.isUndefined(msg_data[key])) { - var obj = RockStorSocket.handlerMap[key]; - obj.fn.call(obj.fn_this, msg_data[key]); + var obj = RockStorSocket.handlerMap[data.key]; + if (!_.isNull(obj) && !_.isUndefined(obj)) { + obj.fn.call(obj.fn_this, data.data); } - }); -} +}; diff --git a/src/rockstor/storageadmin/static/storageadmin/js/templates/common/navbar.jst b/src/rockstor/storageadmin/static/storageadmin/js/templates/common/navbar.jst index d4141ae35..864ec2ab4 100644 --- a/src/rockstor/storageadmin/static/storageadmin/js/templates/common/navbar.jst +++ b/src/rockstor/storageadmin/static/storageadmin/js/templates/common/navbar.jst @@ -19,9 +19,10 @@
    • +
  • Shop