diff --git a/.eslintrc.json b/.eslintrc.json index 177cfc49..7f739042 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -112,9 +112,26 @@ ], "strict": [ "off" + ], + "new-cap": [ + "off" ] }, "overrides": [ + { + "files": ["bin/**/*Jce.js"], + "rules": { + "no-var": [ + "off" + ], + "no-unused-vars": [ + "off" + ], + "no-use-before-define": [ + "off" + ] + } + }, { "files": ["bin/tsw/wwwroot/*.js"], "env": { @@ -131,7 +148,7 @@ ] } }, { - "files": ["bin/lib/util/mail/tmpl.js", "bin/tsw/util/**/tmpl.js"], + "files": ["bin/**/tmpl.js"], "rules": { "no-unused-vars": [ "error", diff --git a/.travis.yml b/.travis.yml index fdbc59bd..94d9c343 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: node_js node_js: - "8" - "9" + - "10.5.0" script: - commitlint-travis diff --git a/bin/proxy/http.proxy.js b/bin/proxy/http.proxy.js index 889b942d..b034a79e 100644 --- a/bin/proxy/http.proxy.js +++ b/bin/proxy/http.proxy.js @@ -87,7 +87,7 @@ process.on('message', function(message) { return; } - logger.info(`cpu: ${serverInfo.cpu} ${message.cmd}`); + logger.info(`receive message, cmd: ${message.cmd}`); methodMap[message.cmd](message); }); @@ -251,8 +251,7 @@ function listen(cpu) { const finish = function() { - // 开始发送心跳 - logger.info('start heart beat'); + logger.info('start heartbeat'); startHeartBeat(); @@ -268,11 +267,13 @@ function listen(cpu) { uid: user_00 }); } + websocket.start_listen(); logger.info('cpu: ${cpu}, afterStartup...', serverInfo); if (typeof config.afterStartup === 'function') { + logger.info('cpu: ${cpu}, config.afterStartup fired', serverInfo); config.afterStartup(serverInfo.cpu); } }; @@ -356,18 +357,16 @@ function heartBeat() { } global.cpuUsed = cpuUtil.getCpuUsed(serverInfo.cpu); + if (global.cpuUsed >= 80) { global.cpuUsed80 = ~~global.cpuUsed80 + 1; } else { global.cpuUsed80 = 0; } - const cpuUsed = global.cpuUsed; - tnm2.Attr_API_Set('AVG_TSW_CPU_USED', cpuUsed); - // 高负载告警 if (global.cpuUsed80 === 4 && !config.isTest && !isWin32Like) { - afterCpu80(cpuUsed); + afterCpu80(global.cpuUsed); } const currMemory = process.memoryUsage(); diff --git a/bin/proxy/http.route.js b/bin/proxy/http.route.js index 093c7095..767ff8e2 100644 --- a/bin/proxy/http.route.js +++ b/bin/proxy/http.route.js @@ -481,7 +481,10 @@ function doRoute(req, res) { }; })(res.writeHead); - const mod_act = contextMod.currentContext().mod_act || httpModAct.getModAct(req); + let mod_act = contextMod.currentContext().mod_act || httpModAct.getModAct(req); + if (typeof mod_act !== 'string' || !mod_act) { + mod_act = null; + } contextMod.currentContext().mod_act = mod_act; if (alpha.isAlpha(req)) { @@ -606,7 +609,7 @@ function doRoute(req, res) { dcapi.report({ key: 'EVENT_TSW_HTTP_IP_BLOCK', - toIp: clientIp || '127.0.0.1', + toIp: clientIp, code: 0, isFail: 0, delay: 100 diff --git a/bin/proxy/master.js b/bin/proxy/master.js index 36a81c48..cf4ebfc6 100644 --- a/bin/proxy/master.js +++ b/bin/proxy/master.js @@ -18,12 +18,11 @@ const { debugOptions } = process.binding('config'); const methodMap = {}; const workerMap = {}; const cpuMap = []; -const isDeaded = false; +const tnm2 = require('api/tnm2'); +const network = require('util/network.js'); -// 阻止进程因异常而退出 process.on('uncaughtException', function(e) { - // Mac和linux权限不足时给予提醒 if (/\blisten EACCES\b/.test(e.message) && config.httpPort < 1024 && (serverOS.isOSX || serverOS.isLinux)) { logger.error('This is OSX/Linux, you may need to use "sudo" prefix to start server.\n'); } @@ -35,7 +34,7 @@ process.on('uncaughtException', function(e) { process.on('warning', function(warning) { const key = String(warning); const errStr = warning && warning.stack || String(warning); - const content = `

错误堆栈

${errStr}

`; + const content = `

stack

${errStr}

`; if (warning.message && warning.message.indexOf('N-API') > -1) { logger.warn(warning.message); @@ -169,11 +168,12 @@ function closeWorker(worker) { closeTimeWait = Math.max(closeTimeWait, config.timeout.keepAlive); closeTimeWait = Math.min(60000, closeTimeWait) || 10000; - - if (worker.hasClose) { + if (worker.isClosing) { return; } + worker.isClosing = true; + if (workerMap[cpu] === worker) { delete workerMap[cpu]; } @@ -189,68 +189,58 @@ function closeWorker(worker) { try { process.kill(pid, 9); } catch (e) { - logger.info(`kill worker fail ${e.message}`); + logger.info(`kill worker message: ${e.message}`); } - worker.destroy(); - closed = true; + worker.destroy(); }; })(worker); setTimeout(closeFn, closeTimeWait); - if (worker.exitedAfterDisconnect) { - worker.hasClose = true; - return; - } - try { worker.disconnect(closeFn); } catch (e) { logger.info(e.stack); } - } // 重启worker function restartWorker(worker) { - const cpu = getToBindCpu(worker); - if (worker.hasRestart) { return; } - logger.info('worker${cpu} pid=${pid} close. restart new worker again.', { + worker.hasRestart = true; + const cpu = getToBindCpu(worker); + + cpuMap[cpu] = 0; + + logger.info('worker${cpu} pid=${pid} closed. restart new worker again.', { pid: worker.process.pid, cpu: cpu }); - setTimeout(function() { - closeWorker(worker); - }, 10000); - - cpuMap[cpu] = 0; - - worker.hasRestart = true; cluster.fork(process.env).cpuid = cpu; + + closeWorker(worker); } -// 定时检测子进程存活，发现15秒没响应的就干掉 +// 定时检测子进程存活，15秒未响应的采取措施 function checkWorkerAlive() { + const checkWorkerAliveTimeout = 5000; + let checkWorkerAliveCount = 0; setInterval(function() { - - let key, - worker, - cpuid; - + checkWorkerAliveCount += 1; const nowDate = new Date(); const now = nowDate.getTime(); + let key; for (key in workerMap) { - worker = workerMap[key]; - cpuid = worker.cpuid; + const worker = workerMap[key]; + const cpuid = worker.cpuid; worker.lastLiveTime = worker.lastLiveTime || now; if (!worker.startTime) { @@ -258,7 +248,7 @@ function checkWorkerAlive() { } // 无响应进程处理 - if (now - worker.lastLiveTime > 15000 && cpuMap[cpuid] === 1) { + if (now - worker.lastLiveTime > checkWorkerAliveTimeout * 3 && cpuMap[cpuid] === 1) { logger.error('worker${cpu} pid=${pid} miss heartBeat, kill it', { pid: worker.process.pid, @@ -266,6 +256,7 @@ function checkWorkerAlive() { }); restartWorker(worker); + continue; } // 内存超限进程处理 @@ -300,33 +291,53 @@ function checkWorkerAlive() { require('api/keyman/runtimeAdd.js').hello(); } - }, 5000); + // after 1 minute + if (checkWorkerAliveCount % 12 === 0) { + const info = network.getNetInfo(); + + // network report + tnm2.Attr_API('SUM_TSW_NET_EXTERNAL_RXBIT', info.external.receive.bytes); + tnm2.Attr_API('SUM_TSW_NET_EXTERNAL_RXPCK', info.external.receive.packets); + tnm2.Attr_API('SUM_TSW_NET_EXTERNAL_TXBIT', info.external.transmit.bytes); + tnm2.Attr_API('SUM_TSW_NET_EXTERNAL_TXPCK', info.external.transmit.packets); + tnm2.Attr_API('SUM_TSW_NET_INTERNAL_RXBIT', info.internal.receive.bytes); + tnm2.Attr_API('SUM_TSW_NET_INTERNAL_RXPCK', info.internal.receive.packets); + tnm2.Attr_API('SUM_TSW_NET_INTERNAL_TXBIT', info.internal.transmit.bytes); + tnm2.Attr_API('SUM_TSW_NET_INTERNAL_TXPCK', info.internal.transmit.packets); + tnm2.Attr_API('SUM_TSW_NET_LOCAL_RXBIT', info.local.receive.bytes); + tnm2.Attr_API('SUM_TSW_NET_LOCAL_RXPCK', info.local.receive.packets); + tnm2.Attr_API('SUM_TSW_NET_LOCAL_TXBIT', info.local.transmit.bytes); + tnm2.Attr_API('SUM_TSW_NET_LOCAL_TXPCK', info.local.transmit.packets); + + // cpu load report + cpuUtil.getCpuLoadAsync().then(function(data) { + if (!data) { + return; + } + + tnm2.Attr_API_Set('AVG_TSW_CPU_LOAD_1', data.L1); + tnm2.Attr_API_Set('AVG_TSW_CPU_LOAD_5', data.L5); + tnm2.Attr_API_Set('AVG_TSW_CPU_LOAD_15', data.L15); + }); + } + + tnm2.Attr_API_Set('AVG_TSW_CPU_USED', global.cpuUsed); + }, checkWorkerAliveTimeout); } // 获取需要绑定的CPU编号 function getToBindCpu(worker) { - - let cpu = 0;// 如果只有一个cpu或者都占用了 - let i; - if (typeof worker.cpuid !== 'undefined') { - cpu = worker.cpuid; - return cpu; - } else { + return worker.cpuid; + } - for (i = 0; i < cpuMap.length; i++) { - const c = cpuMap[i]; - if (c === 0) { - cpu = i; - worker.cpuid = cpu; - break; - } + for (const cpu of cpuMap) { + if (cpuMap[cpu] === 0) { + worker.cpuid = cpu; + cpuMap[cpu] = 1; + return cpu; } } - - cpuMap[i] = 1; - - return cpu; } @@ -392,7 +403,7 @@ function masterEventHandler() { return; } - logger.info('worker${cpu} pid=${pid} has disconnected. restart new worker again.', { + logger.info('worker${cpu} pid=${pid} disconnect event fired. restart new worker again.', { pid: worker.process.pid, cpu: cpu }); @@ -409,7 +420,7 @@ function masterEventHandler() { return; } - logger.info('worker${cpu} pid=${pid} has been killed. restart new worker again.', { + logger.info('worker${cpu} pid=${pid} exit event fired. restart new worker again.', { pid: worker.process.pid, cpu: cpu }); @@ -418,52 +429,36 @@ function masterEventHandler() { }); process.on('reload', function(GET) { - - let timeout = 1000, - cpu = 0, - key, - worker; - - if (isDeaded) { - process.exit(0); - } - logger.info('reload'); - for (key in workerMap) { - worker = workerMap[key]; - try { - - cpu = getToBindCpu(worker); + for (const key in workerMap) { + const worker = workerMap[key]; + const cpu = getToBindCpu(worker); + let timeout = 1000; - if (config.isTest || config.devMode) { - timeout = (cpu % 8) * 1000; - } else { - timeout = (cpu % 8) * 3000; - } + if (config.isTest || config.devMode) { + timeout = (cpu % 8) * 1000; + } else { + timeout = (cpu % 8) * 3000; + } - setTimeout((function(worker, cpu) { - return function() { - if (!worker.exitedAfterDisconnect) { - logger.info('cpu${cpu} send restart message', { - cpu: cpu - }); - worker.send({ from: 'master', cmd: 'restart' }); - } - restartWorker(worker); - }; - })(worker, cpu), timeout); - - logger.info('cpu${cpu} reload after ${timeout}ms', { - cpu: cpu, - timeout: timeout - }); + setTimeout((function(worker, cpu) { + return function() { + if (!worker.exitedAfterDisconnect) { + logger.info('cpu${cpu} send restart message', { + cpu: cpu + }); + worker.send({ from: 'master', cmd: 'restart' }); + } + restartWorker(worker); + }; + })(worker, cpu), timeout); - } catch (e) { - logger.error(e.stack); - } + logger.info('cpu${cpu} reload after ${timeout}ms', { + cpu: cpu, + timeout: timeout + }); } - }); process.on('sendCmd2workerOnce', function(data) { @@ -473,10 +468,6 @@ function masterEventHandler() { const CMD = data.CMD; const GET = data.GET; - if (isDeaded) { - process.exit(0); - } - logger.info('sendCmd2workerOnce CMD: ${CMD}', { CMD }); diff --git a/bin/proxy/websocket.js b/bin/proxy/websocket.js index b822c018..0ca02ee2 100644 --- a/bin/proxy/websocket.js +++ b/bin/proxy/websocket.js @@ -36,11 +36,14 @@ function wsFiller(ws, req) { ws.reportIndex = 1; ws.send = function(message) { - logger.debug('server send message : ${message}', { - message - }); - - ws.__tempSend(message); + if (ws.readyState == WebSocket.OPEN) { + logger.debug('server send message : ${message}', { + message + }); + ws.__tempSend(message); + } else { + logger.warn('send message fail! WebSocket readyState is : ' + ws.readyState); + } }; ws.logKey = Math.random(); diff --git a/bin/tsw/ajax/ajax.js b/bin/tsw/ajax/ajax.js index 7bb9405b..065b1496 100644 --- a/bin/tsw/ajax/ajax.js +++ b/bin/tsw/ajax/ajax.js @@ -187,7 +187,6 @@ Ajax.prototype.doRequest = function(opt) { let tid = null, currAgent = false, - request, key, v, obj; @@ -541,7 +540,7 @@ Ajax.prototype.doRequest = function(opt) { currAgent = false; } - request = (opt.protocol === 'https:' ? https : http).request({ + const request = (opt.protocol === 'https:' ? https : http).request({ agent: currAgent, host: opt.proxyIp || opt.ip || opt.host, port: opt.proxyPort || opt.port, @@ -561,7 +560,7 @@ Ajax.prototype.doRequest = function(opt) { } const onError = (err) => { - logger.error(logPre + err.stack); + logger.error(logPre + 'socket error: ' + err.stack); clean(); this.emit('fail'); }; @@ -573,9 +572,7 @@ Ajax.prototype.doRequest = function(opt) { const onLookup = (err, address, family, host) => { if (err) { - logger.error(logPre + err.stack); - clean(); - this.emit('fail'); + logger.error(logPre + 'lookup error: ' + err.stack); return; } this.remoteIp = address; @@ -597,11 +594,6 @@ Ajax.prototype.doRequest = function(opt) { defer.always(function() { clearTimeout(tid); - request.removeAllListeners(); - - // request.abort(); 长连接不能开 - // request.destroy(); 长连接不能开 - request = null; tid = null; }); @@ -614,7 +606,8 @@ Ajax.prototype.doRequest = function(opt) { }, opt.timeout); request.once('error', function(err) { - request.emit('fail', err); + logger.error(logPre + 'request error: ' + err.stack); + this.emit('fail', err); }); request.once('fail', function(err) { @@ -928,6 +921,11 @@ Ajax.prototype.doRequest = function(opt) { this.emit('done'); }); + pipe.once('error', function(err) { + logger.debug(logPre + ' decode error: ' + err.stack); + this.emit('done'); + }); + pipe.once('end', function() { const cost = Date.now() - pipe.timeStart; diff --git a/bin/tsw/api/tnm2/index.js b/bin/tsw/api/tnm2/index.js index f89e6ae5..0aff6422 100644 --- a/bin/tsw/api/tnm2/index.js +++ b/bin/tsw/api/tnm2/index.js @@ -45,17 +45,17 @@ if (isFirstLoad) { * 平均值类型上报 */ this.Attr_API_Set = function (attr, iValue) { - cacheOrRepoet(attr, iValue); + cacheOrReport(attr, iValue); }; /** * 叠加类型上报 */ this.Attr_API = function (attr, iValue) { - cacheOrRepoet(attr, iValue); + cacheOrReport(attr, iValue); }; -const cacheOrRepoet = function(attr, iValue) { +const cacheOrReport = function(attr, iValue) { if (!mapping[attr]) { return; diff --git a/bin/tsw/api/tnm2/mapping.json b/bin/tsw/api/tnm2/mapping.json index 1336ed97..3d3261cd 100644 --- a/bin/tsw/api/tnm2/mapping.json +++ b/bin/tsw/api/tnm2/mapping.json @@ -47,12 +47,26 @@ "SUM_TSW_WEBSOCKET_MESSAGE" : 1253121, "SUM_TSW_WEBSOCKET_ERROR" : 1253122, "SUM_TSW_WEBSOCKET_CLOSE" : 1253126, - "SUM_TSW_WEBSOCKET_TIMEOUT" : 1253127, - "SUM_TSW_WEBSOCKET_RESPONSE": 1253128, - "SUM_TSW_WEBSOCKET_PUSH" : 1253129, + + "AVG_TSW_CPU_LOAD_1" : 1280713, + "AVG_TSW_CPU_LOAD_5" : 1280714, + "AVG_TSW_CPU_LOAD_15" : 1280716, + + "SUM_TSW_NET_EXTERNAL_RXBIT": 1280718, + "SUM_TSW_NET_EXTERNAL_RXPCK": 1280719, + "SUM_TSW_NET_EXTERNAL_TXBIT": 1280720, + "SUM_TSW_NET_EXTERNAL_TXPCK": 1280722, + "SUM_TSW_NET_INTERNAL_RXBIT": 1280724, + "SUM_TSW_NET_INTERNAL_RXPCK": 1280725, + "SUM_TSW_NET_INTERNAL_TXBIT": 1280727, + "SUM_TSW_NET_INTERNAL_TXPCK": 1280729, + "SUM_TSW_NET_LOCAL_RXBIT" : 1280730, + "SUM_TSW_NET_LOCAL_RXPCK" : 1280731, + "SUM_TSW_NET_LOCAL_TXBIT" : 1280732, + "SUM_TSW_NET_LOCAL_TXPCK" : 1280733, "AVG_TSW_CPU_USED" : 1154903, "AVG_TSW_MEMORY_RSS" : 1154904, "AVG_TSW_MEMORY_HEAP" : 1154905, "AVG_TSW_MEMORY_EXTERNAL" : 1154906 -} \ No newline at end of file +} diff --git a/bin/tsw/default/config.default.js b/bin/tsw/default/config.default.js index 09604665..8a8acd37 100644 --- a/bin/tsw/default/config.default.js +++ b/bin/tsw/default/config.default.js @@ -122,7 +122,7 @@ this.timeout = { post: 30000, get: 10000, keepAlive: 10000, - dns: 3000 + dns: 1000 }; diff --git a/bin/tsw/loader/seajs/lib/sea-node.js b/bin/tsw/loader/seajs/lib/sea-node.js index d152e606..13654cae 100644 --- a/bin/tsw/loader/seajs/lib/sea-node.js +++ b/bin/tsw/loader/seajs/lib/sea-node.js @@ -14,21 +14,20 @@ const moduleStack = []; Module._resolveFilename = function(request, parent, isMain, options = {}) { - let res; - //request = request.replace(/\?.*$/, '') // remove timestamp etc. + // request = request.replace(/\?.*$/, '') // remove timestamp etc. - //性能优化 + // 性能优化 // do not use cache when `options` has `paths` property // in v8.9.0 require.resolve case - if(parent.resolveFilenameCache && !options.paths) { - if(parent.resolveFilenameCache[request]) { + if (parent.resolveFilenameCache && !options.paths) { + if (parent.resolveFilenameCache[request]) { return parent.resolveFilenameCache[request]; } - }else{ + } else { parent.resolveFilenameCache = {}; } - res = _resolveFilename(request, parent, isMain, options); + const res = _resolveFilename(request, parent, isMain, options); parent.resolveFilenameCache[request] = res; diff --git a/bin/tsw/runtime/Dns.hack.js b/bin/tsw/runtime/Dns.hack.js index d10caecf..8be9ee81 100644 --- a/bin/tsw/runtime/Dns.hack.js +++ b/bin/tsw/runtime/Dns.hack.js @@ -38,6 +38,8 @@ if (!global[__filename]) { let timeoutError; let isCalled = false; + logger.debug(`dns lookup for ${hostname}`); + const callbackWrap = function(err, address, family) { if (isCalled) return; @@ -52,10 +54,12 @@ if (!global[__filename]) { isFail = 1; } + const cost = Date.now() - start; + if (err) { - logger.error('dns lookup error: ' + err.stack); + logger.error(`dns lookup [${cost}ms] error: ${err.stack}`); } else { - logger.debug(`dns lookup: ${hostname} --> ${address}`); + logger.debug(`dns lookup ${cost}ms: ${hostname} --> ${address}`); } dcapi.report({ @@ -63,7 +67,7 @@ if (!global[__filename]) { toIp: '127.0.0.1', code: code, isFail: isFail, - delay: Date.now() - start + delay: cost }); isCalled = true; diff --git a/bin/tsw/runtime/capturer.js b/bin/tsw/runtime/capturer.js index a67c93ff..d9a50857 100644 --- a/bin/tsw/runtime/capturer.js +++ b/bin/tsw/runtime/capturer.js @@ -199,7 +199,8 @@ process.nextTick(function() { } }; - response.on('data', data); + // response.on('data', data); + httpUtil.captureStream(response, data); response.once('close', function() { logger.debug(logPre + 'close'); @@ -222,7 +223,7 @@ process.nextTick(function() { }; const onError = function(err) { - logger.error(err.stack); + logger.error(logPre + 'request error: ' + err.stack); finish(); }; @@ -238,7 +239,7 @@ process.nextTick(function() { } const onError = function(err) { - logger.error(logPre + err.stack); + logger.error(logPre + 'socket error: ' + err.stack); clean(); finish(); }; @@ -254,14 +255,13 @@ process.nextTick(function() { const onLookup = function(err, address, family, host) { timeLookup = Date.now(); + const cost = timeLookup - timeStart; + logger.debug(`${logPre}dns lookup ${host} -> ${address || 'null'}, cost ${cost}ms`); + if (err) { - logger.error(logPre + err.stack); - clean(); - finish(); - return; + logger.error(logPre + 'lookup error: ' + err.stack); + // 会触发socket的error事件，这里不能clean } - const cost = timeLookup - timeStart; - logger.debug(`${logPre}dns lookup ${host} -> ${address}, cost ${cost}ms`); }; const clean = function() { diff --git a/bin/tsw/runtime/overloadProtection.js b/bin/tsw/runtime/overloadProtection.js index 348cab59..4756e5d1 100644 --- a/bin/tsw/runtime/overloadProtection.js +++ b/bin/tsw/runtime/overloadProtection.js @@ -7,9 +7,10 @@ */ 'use strict'; +const cluster = require('cluster'); // 过载保护 -process.nextTick(function() { +cluster.isMaster && process.nextTick(function() { const logger = require('logger'); const tnm2 = require('api/tnm2'); diff --git a/bin/tsw/serverInfo.js b/bin/tsw/serverInfo.js index 1969bcd5..b01d7573 100644 --- a/bin/tsw/serverInfo.js +++ b/bin/tsw/serverInfo.js @@ -37,7 +37,7 @@ function getLinuxLocalIpv4() { return; } - if (tmp === '127.0.0.1') { + if (tmp.startsWith('127.')) { return; } diff --git a/bin/tsw/util/auto-report/logReport.js b/bin/tsw/util/auto-report/logReport.js index 550e0d96..5f434903 100644 --- a/bin/tsw/util/auto-report/logReport.js +++ b/bin/tsw/util/auto-report/logReport.js @@ -676,7 +676,7 @@ function reportLog() { res._body = Buffer.from(format.formatBuffer(res._body)); } - logJson = logJson || logger.getJson(); + logJson = logJson || logger.getJson() || {}; logJson.curr = { protocol: 'HTTP', host: req.headers.host, diff --git a/bin/tsw/util/cpu.echo.js b/bin/tsw/util/cpu.echo.js new file mode 100644 index 00000000..90124f76 --- /dev/null +++ b/bin/tsw/util/cpu.echo.js @@ -0,0 +1,24 @@ +/* ! + * Tencent is pleased to support the open source community by making Tencent Server Web available. + * Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const plug = require('plug'); +const cpu = plug('util/cpu.js'); + +if (process.mainModule === module) { + setInterval(function() { + /* eslint-disable no-console */ + const info = cpu.getCpuUsed(); + + cpu.getCpuLoadAsync().then((res) => { + console.log(info, res); + }); + /* eslint-enable no-console */ + }, 3000); +} + diff --git a/bin/tsw/util/cpu.js b/bin/tsw/util/cpu.js index 08895385..2f57c957 100644 --- a/bin/tsw/util/cpu.js +++ b/bin/tsw/util/cpu.js @@ -50,9 +50,7 @@ this.getCpuUsed = function(cpu) { fs.readFile('/proc/stat', function(err, buffer) { if (err) { - /* eslint-disable no-console */ - console.error(err.stack); - /* eslint-enable no-console */ + logger.error(err.stack); return; } @@ -212,10 +210,45 @@ this.parseTaskset = function(str) { return res; }; -if (process.mainModule === module) { - setInterval(function() { - /* eslint-disable no-console */ - console.log('cpu: ' + module.exports.getCpuUsed()); - /* eslint-enable no-console */ - }, 1000); -} +this.getCpuLoadAsync = function() { + const cpuNum = this.cpus().length; + return new Promise((resolve) => { + cp.exec('uptime', { + encoding: 'utf8', + timeout: 5000 + }, function(err, data, errData) { + if (err) { + logger.error(err); + return resolve(null); + } + + if (errData) { + logger.error(errData); + return resolve(null); + } + + if (!data) { + logger.error('empty data'); + return resolve(null); + } + + const tmp = /([ 0-9\.]+),([ 0-9\.]+),([ 0-9\.]+)$/m.exec(data); + + if (!tmp) { + logger.error('no match data'); + return resolve(null); + } + + const result = { + 'L1': parseInt(tmp[1] * 100 / cpuNum, 10), + 'L5': parseInt(tmp[2] * 100 / cpuNum, 10), + 'L15': parseInt(tmp[3] * 100 / cpuNum, 10) + }; + + return resolve(result); + }); + }); +}; + + +this.getCpuUsed(); diff --git a/bin/tsw/util/h5-test/is-test.js b/bin/tsw/util/h5-test/is-test.js index 27b00b6e..90ceba43 100644 --- a/bin/tsw/util/h5-test/is-test.js +++ b/bin/tsw/util/h5-test/is-test.js @@ -309,6 +309,10 @@ module.exports.isTestUser = function(req, res) { }).done(function(d) { }).fail(function(d) { logger.error('h5test proxy fail...'); + if (res.headersSent) { + res.end(); + return; + } res.setHeader('Content-Type', 'text/html; charset=UTF-8'); res.writeHead(500); res.end(); diff --git a/bin/tsw/util/http.js b/bin/tsw/util/http.js index f090c834..60a8e343 100644 --- a/bin/tsw/util/http.js +++ b/bin/tsw/util/http.js @@ -97,7 +97,13 @@ this.captureIncomingMessageBody = function(req) { logger.debug('capture IncomingMessage body on'); - req.on('data', data); + // req.on('data', data); + + // 直接监听data事件会有问题： + // request流是消费型的，前面监听的data监听器会优先消费缓存中已经接收到的数据， + // 导致当业务侧在异步绑定data事件监听器的时候会丢失前面已经接收的数据 + this.captureStream(req, data); + req.once('end', function() { logger.debug('receive end'); this.removeListener('data', data); @@ -107,6 +113,31 @@ this.captureIncomingMessageBody = function(req) { }); }; +this.captureStream = function(stream, handler) { + const oriPush = stream.push; + + // stream.readableBuffer for node >=9, stream._readableState.buffer for node < 9 + const bufferData = stream.readableBuffer || stream._readableState.buffer; + + let head = bufferData.head; + + while (head) { + handler(head.data); + + head = head.next; + } + + stream.push = (chunk, encoding) => { + try { + chunk && handler(chunk); + } catch (e) { + logger.debug(`captrue stream chunk error ${e.message}`); + } + + return oriPush.call(stream, chunk, encoding); + }; +}; + this.captureBody = this.captureServerResponseBody = function(res) { if (res._capturing) { diff --git a/bin/tsw/util/logger/callInfo.js b/bin/tsw/util/logger/callInfo.js index 6a25047e..006044d1 100644 --- a/bin/tsw/util/logger/callInfo.js +++ b/bin/tsw/util/logger/callInfo.js @@ -5,7 +5,7 @@ * http://opensource.org/licenses/MIT * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -'no use strict'; +// 'no use strict'; this.getCallInfo = function(level) { diff --git a/bin/tsw/util/network.echo.js b/bin/tsw/util/network.echo.js new file mode 100644 index 00000000..ca5d2cbf --- /dev/null +++ b/bin/tsw/util/network.echo.js @@ -0,0 +1,21 @@ +/* ! + * Tencent is pleased to support the open source community by making Tencent Server Web available. + * Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const plug = require('plug'); +const network = plug('util/network.js'); + +if (process.mainModule === module) { + setInterval(function() { + /* eslint-disable no-console */ + const info = network.getNetInfo(); + console.log(info); + /* eslint-enable no-console */ + }, 5000); +} + diff --git a/bin/tsw/util/network.js b/bin/tsw/util/network.js new file mode 100644 index 00000000..67d8faa3 --- /dev/null +++ b/bin/tsw/util/network.js @@ -0,0 +1,171 @@ +/* ! + * Tencent is pleased to support the open source community by making Tencent Server Web available. + * Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; + +const fs = require('fs'); +const os = require('os'); +const isInnerIP = require('util/http.isInnerIP.js'); +const { isWin32Like } = require('./isWindows.js'); +const logger = require('logger'); +let cache; + +if (!global[__filename]) { + cache = { + time: 0, + total: null, + curr: createEmpty() + }; + global[__filename] = cache; +} else { + cache = global[__filename]; +} + +this.getNetworkInterfacesTypes = function() { + const networkInterfaces = os.networkInterfaces(); + const types = {}; + + Object.keys(networkInterfaces).forEach(function(key) { + const eth = networkInterfaces[key]; + const address = eth && eth[0] && eth[0].address; + + if (!address) { + return; + } + + if (eth[0].family !== 'IPv4') { + return; + } + + const tmp = isInnerIP.isInnerIP(address); + + if (!tmp) { + types[key] = 'external'; + } + + if (address.startsWith('127.')) { + types[key] = 'local'; + } else { + types[key] = 'internal'; + } + + }); + + return types; +}; + +this.getNetInfo = function() { + const now = Date.now(); + + if (isWin32Like) { + return cache.curr; + } + + if (now - cache.time < 3000) { + return cache.curr; + } + + const cost = now - cache.time; + cache.time = now; + + const types = this.getNetworkInterfacesTypes(); + + fs.readFile('/proc/net/dev', function(err, buffer) { + + if (err) { + logger.error(err.stack); + return; + } + + const lines = buffer.toString('UTF-8').split('\n'); + const sum = createEmpty(); + const incr = createEmpty(); + const curr = createEmpty(); + + lines.forEach((v, i) => { + const tmp = v.split(/\W+/); + + if (!tmp[1]) { + return; + } + + const key = tmp[1]; + const type = types[key]; + + if (!type) { + return; + } + + sum[type].receive.bytes += parseInt(tmp[2], 10) || 0; + sum[type].receive.packets += parseInt(tmp[3], 10) || 0; + sum[type].transmit.bytes += parseInt(tmp[10], 10) || 0; + sum[type].transmit.packets += parseInt(tmp[11], 10) || 0; + }); + + if (!cache.total) { + // init total first + cache.total = sum; + } + + Object.keys(sum).forEach(function(type) { + incr[type].receive.bytes = sum[type].receive.bytes - cache.total[type].receive.bytes; + incr[type].receive.packets = sum[type].receive.packets - cache.total[type].receive.packets; + incr[type].transmit.bytes = sum[type].transmit.bytes - cache.total[type].transmit.bytes; + incr[type].transmit.packets = sum[type].transmit.packets - cache.total[type].transmit.packets; + + curr[type].receive.bytes = Math.floor(incr[type].receive.bytes * 8 / (cost / 1000)); // bps + curr[type].receive.packets = Math.floor(incr[type].receive.packets / (cost / 1000)); + curr[type].transmit.bytes = Math.floor(incr[type].transmit.bytes * 8 / (cost / 1000)); // bps + curr[type].transmit.packets = Math.floor(incr[type].transmit.packets / (cost / 1000)); + }); + + cache.curr = curr; + cache.total = sum; + }); + + return cache.curr; +}; + +function createEmpty() { + return { + external: { + receive: { + bytes: 0, + packets: 0, + }, + transmit: { + bytes: 0, + packets: 0 + } + }, + internal: { + receive: { + bytes: 0, + packets: 0 + }, + transmit: { + bytes: 0, + packets: 0 + } + }, + local: { + receive: { + bytes: 0, + packets: 0 + }, + transmit: { + bytes: 0, + packets: 0 + } + } + }; +} + +this.getNetInfo(); +setTimeout(() => { + this.getNetInfo(); +}, 3000); diff --git a/conf/config.js b/conf/config.js index 8e42f98f..cd1031ea 100644 --- a/conf/config.js +++ b/conf/config.js @@ -1,4 +1,18 @@ -module.exports = require('../examples/framework/config.js'); +/* ! + * Tencent is pleased to support the open source community by making Tencent Server Web available. + * Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + * http://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +'use strict'; -//module.exports = require('../examples/skyMode/config.js'); +const processArgs = plug('util/process.args.js'); +let configPath = '../examples/framework/config.js'; + +if (typeof processArgs.config === 'string') { + configPath = processArgs.config; +} + +module.exports = require(configPath); diff --git a/package.json b/package.json index 32d51c54..6c1a7e38 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "TSW", - "version": "1.1.4", + "version": "1.1.5", "description": "A Node.js infrastructure which is designed for improving the efficiency of locating problems, providing multiple functions for front-end developers", "scripts": { "lint": "eslint examples bin test --fix", diff --git a/test/bin/tsw/util/cpu.test.js b/test/bin/tsw/util/cpu.test.js index de95b6f8..241fc6ff 100644 --- a/test/bin/tsw/util/cpu.test.js +++ b/test/bin/tsw/util/cpu.test.js @@ -95,7 +95,7 @@ describe('测试获取cpu信息的接口', () => { it('# getCpuUsed ', () => { const cpuUsed = cpuUtil.getCpuUsed(); - expect(cpuUsed).to.equal(0); + expect(cpuUsed >= 0).to.equal(true); }); }); diff --git a/test/bin/tsw/util/network.test.js b/test/bin/tsw/util/network.test.js new file mode 100644 index 00000000..884b07e3 --- /dev/null +++ b/test/bin/tsw/util/network.test.js @@ -0,0 +1,28 @@ +const chai = require('chai'); +const expect = chai.expect; +const plug = require('plug'); +const network = plug('util/network.js'); +const logger = plug('logger'); + +logger.setLogLevel('error'); + +describe('测试获取network信息的接口', () => { + + it('# getNetInfo ', () => { + + const info = network.getNetInfo(); + expect(info.external.receive.bytes >= 0).to.equal(true); + expect(info.external.receive.packets >= 0).to.equal(true); + expect(info.external.transmit.bytes >= 0).to.equal(true); + expect(info.external.transmit.packets >= 0).to.equal(true); + expect(info.internal.receive.bytes >= 0).to.equal(true); + expect(info.internal.receive.packets >= 0).to.equal(true); + expect(info.internal.transmit.bytes >= 0).to.equal(true); + expect(info.internal.transmit.packets >= 0).to.equal(true); + expect(info.local.receive.bytes >= 0).to.equal(true); + expect(info.local.receive.packets >= 0).to.equal(true); + expect(info.local.transmit.bytes >= 0).to.equal(true); + expect(info.local.transmit.packets >= 0).to.equal(true); + + }); +});