Added a periodic task that checks uptime in erizoJS (lynckia#1369)

Arri98 · Feb 25, 2019 · df5f2b7 · df5f2b7
1 parent bd55ddf
commit df5f2b7
Show file tree

Hide file tree

Showing 4 changed files with 100 additions and 1 deletion.
diff --git a/erizo_controller/erizoJS/erizoJS.js b/erizo_controller/erizoJS/erizoJS.js
@@ -23,6 +23,9 @@ global.config.erizo.turnport = global.config.erizo.turnport || 0;
 global.config.erizo.turnusername = global.config.erizo.turnusername || '';
 global.config.erizo.turnpass = global.config.erizo.turnpass || '';
 global.config.erizo.networkinterface = global.config.erizo.networkinterface || '';
+global.config.erizo.activeUptimeLimit = global.config.erizo.activeUptimeLimit || 7;
+global.config.erizo.maxTimeSinceLastOperation = global.config.erizo.maxTimeSinceLastOperation || 3;
+global.config.erizo.checkUptimeInterval = global.config.erizo.checkUptimeInterval || 1800;
 global.mediaConfig = mediaConfig || {};
 // Parse command line arguments
 const getopt = new Getopt([

diff --git a/erizo_controller/erizoJS/erizoJSController.js b/erizo_controller/erizoJS/erizoJSController.js
@@ -24,6 +24,37 @@ exports.ErizoJSController = (threadPool, ioThreadPool) => {
   const WARN_NOT_FOUND = 404;
   const WARN_CONFLICT = 409;
 
+  const MAX_INACTIVE_UPTIME = global.config.erizo.activeUptimeLimit * 24 * 60 * 60 * 1000;
+  const MAX_TIME_SINCE_LAST_OP =
+    global.config.erizo.maxTimeSinceLastOperation * 60 * 60 * 1000;
+
+  const uptimeStats = {
+    startTime: 0,
+    lastOperation: 0,
+  };
+
+  const checkUptimeStats = () => {
+    const now = new Date();
+    const timeSinceStart = now.getTime() - uptimeStats.startTime.getTime();
+    const timeSinceLastOperation = now.getTime() - uptimeStats.lastOperation.getTime();
+    log.debug(`Checking uptime - timeSinceStart: ${timeSinceStart}, timeSinceLastOperation ${timeSinceLastOperation}, max since start ${MAX_INACTIVE_UPTIME}, max since opf ${MAX_TIME_SINCE_LAST_OP}`);
+    if (timeSinceStart > MAX_INACTIVE_UPTIME &&
+      timeSinceLastOperation > MAX_TIME_SINCE_LAST_OP) {
+      log.error(`Shutting down ErizoJS - uptime: ${timeSinceStart}, timeSinceLastOperation ${timeSinceLastOperation}`);
+      process.exit(0);
+    }
+  };
+
+  const updateUptimeInfo = () => {
+    uptimeStats.lastOperation = new Date();
+    if (uptimeStats.startTime === 0) {
+      log.info('Start checking uptime, interval', global.config.erizo.checkUptimeInterval);
+      uptimeStats.startTime = new Date();
+      setInterval(checkUptimeStats, global.config.erizo.checkUptimeInterval * 1000);
+    }
+  };
+
+
   that.publishers = publishers;
   that.ioThreadPool = io;
 
@@ -94,6 +125,7 @@ exports.ErizoJSController = (threadPool, ioThreadPool) => {
   };
 
   that.addExternalInput = (streamId, url, callbackRpc) => {
+    updateUptimeInfo();
     if (publishers[streamId] === undefined) {
       const client = getOrCreateClient(url);
       publishers[streamId] = new ExternalInput(url, streamId, threadPool);
@@ -113,6 +145,7 @@ exports.ErizoJSController = (threadPool, ioThreadPool) => {
   };
 
   that.addExternalOutput = (streamId, url, options) => {
+    updateUptimeInfo();
     if (publishers[streamId]) publishers[streamId].addExternalOutput(url, options);
   };
 
@@ -144,6 +177,7 @@ exports.ErizoJSController = (threadPool, ioThreadPool) => {
    * of the OneToManyProcessor.
    */
   that.addPublisher = (clientId, streamId, options, callbackRpc) => {
+    updateUptimeInfo();
     let publisher;
     log.info('addPublisher, clientId', clientId, 'streamId', streamId);
     const client = getOrCreateClient(clientId, options.singlePC);
@@ -189,6 +223,7 @@ exports.ErizoJSController = (threadPool, ioThreadPool) => {
    * OneToManyProcessor.
    */
   that.addSubscriber = (clientId, streamId, options, callbackRpc) => {
+    updateUptimeInfo();
     const publisher = publishers[streamId];
     if (publisher === undefined) {
       log.warn('message: addSubscriber to unknown publisher, ' +

diff --git a/erizo_controller/test/erizoJS/erizoJSController.js b/erizo_controller/test/erizoJS/erizoJSController.js
@@ -14,6 +14,9 @@ describe('Erizo JS Controller', () => {
   let licodeConfigMock;
   let erizoApiMock;
   let controller;
+  const kActiveUptimeLimit = 1;  // in days
+  const kMaxTimeSinceLastOperation = 1; // in hours
+  const kCheckUptimeInterval = 1; // in seconds
 
   beforeEach('Mock Process', () => {
     this.originalExit = process.exit;
@@ -26,7 +29,13 @@ describe('Erizo JS Controller', () => {
 
 
   beforeEach(() => {
-    global.config = { logger: { configFile: true } };
+    global.config = { logger: { configFile: true },
+      erizo: {
+        activeUptimeLimit: kActiveUptimeLimit,
+        maxTimeSinceLastOperation: kMaxTimeSinceLastOperation,
+        checkUptimeInterval: kCheckUptimeInterval,
+      },
+    };
     licodeConfigMock = mocks.start(mocks.licodeConfig);
     amqperMock = mocks.start(mocks.amqper);
     erizoApiMock = mocks.start(mocks.erizoAPI);
@@ -55,6 +64,51 @@ describe('Erizo JS Controller', () => {
     expect(controller.removeSubscriptions).not.to.be.undefined;
   });
 
+  describe('Uptime cleanup', () => {
+    let clock;
+    const kActiveUptimeLimitMs = kActiveUptimeLimit * 3600 * 24 * 1000;
+    const kMaxTimeSinceLastOperationMs = kMaxTimeSinceLastOperation * 3600 * 1000;
+    const kCheckUptimeIntervalMs = kCheckUptimeInterval * 1000;
+    let callback;
+    const kArbitraryStreamId = 'pubStreamId1';
+    const kArbitraryClientId = 'pubClientid1';
+
+    beforeEach(() => {
+      callback = sinon.stub();
+      clock = sinon.useFakeTimers();
+      global.config.erizo = {
+        activeUptimeLimit: kActiveUptimeLimit,
+        maxTimeSinceLastOperation: kMaxTimeSinceLastOperation,
+        checkUptimeInterval: kCheckUptimeInterval,
+      };
+      global.config.erizoController = { report: {
+        connection_events: true,
+        rtcp_stats: true } };
+    });
+
+    afterEach(() => {
+      clock.restore();
+    });
+
+    it('should not stop inactive processes', () => {
+      clock.tick(kActiveUptimeLimitMs + kCheckUptimeIntervalMs);
+      expect(process.exit.callCount).to.equal(0);
+    });
+    it('should exit when the max active time is up and there is no new activity', () => {
+      controller.addPublisher(kArbitraryClientId, kArbitraryStreamId, {}, callback);
+      clock.tick(kActiveUptimeLimitMs + kCheckUptimeIntervalMs);
+      expect(process.exit.callCount).to.equal(1);
+    });
+    it('should not exit when the max active time is up but there is new activity', () => {
+      const kArbitraryTimeMargin = kMaxTimeSinceLastOperationMs - 1000;
+      controller.addPublisher(kArbitraryClientId, kArbitraryStreamId, {}, callback);
+      clock.tick(kActiveUptimeLimitMs - kArbitraryTimeMargin);
+      controller.addPublisher(kArbitraryClientId, kArbitraryStreamId, {}, callback);
+      clock.tick(kArbitraryTimeMargin + kCheckUptimeIntervalMs);
+      expect(process.exit.callCount).to.equal(0);
+    });
+  });
+
   describe('Add External Input', () => {
     let callback;
     const kArbitraryStreamId = 'streamId1';

diff --git a/scripts/licode_default.js b/scripts/licode_default.js
@@ -159,6 +159,13 @@ config.erizo.numWorkers = 24;
 // Number of workers what will be used for IO (including ICE logic)
 config.erizo.numIOWorkers = 1;
 
+// the max amount of time in days a process is allowed to be up after the first publisher is added
+config.erizo.activeUptimeLimit = 7;
+// the max time in hours since last publish or subscribe operation where a erizoJS process can be killed
+config.erizo.maxTimeSinceLastOperation = 3;
+// Interval to check uptime in seconds
+config.erizo.checkUptimeInterval = 1800;
+
 //STUN server IP address and port to be used by the server.
 //if '' is used, the address is discovered locally
 //Please note this is only needed if your server does not have a public IP