From 15ebc877740a97758d8dbf7939cbc85c74799bf2 Mon Sep 17 00:00:00 2001 From: RobinzZH Date: Fri, 6 Jul 2018 14:49:58 +0800 Subject: [PATCH 01/37] feat(package.json): update package.json for R1.1.6 (cherry picked from commit 31e511c76f6363d781737efc6d7f02d5bed471db) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6c1a7e38..5efe72bd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "TSW", - "version": "1.1.5", + "version": "1.1.6", "description": "A Node.js infrastructure which is designed for improving the efficiency of locating problems, providing multiple functions for front-end developers", "scripts": { "lint": "eslint examples bin test --fix", From 6288737866af3c29b2fef363a8c3cd0b4db85e80 Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Fri, 6 Jul 2018 14:58:51 +0800 Subject: [PATCH 02/37] chore(http): ignore errors after a stream was destroyed --- bin/proxy/http.route.js | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/proxy/http.route.js b/bin/proxy/http.route.js index 767ff8e2..2377db09 100644 --- a/bin/proxy/http.route.js +++ b/bin/proxy/http.route.js @@ -707,6 +707,7 @@ function onerror(req, res, err) { const errorIgnore = { 'socket hang up': 'ignore', + 'Cannot call write after a stream was destroyed': 'ignore', 'Cannot read property \'asyncReset\' of null': 'ignore', 'Cannot read property \'resume\' of null': 'ignore', 'write ECONNRESET': 'ignore', From 90671833d69aa1d9144e4130c55618d94e85b5c4 Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Mon, 9 Jul 2018 19:20:50 +0800 Subject: [PATCH 03/37] feat(memcached): add ckv monitor --- bin/tsw/api/tnm2/mapping.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/tsw/api/tnm2/mapping.json b/bin/tsw/api/tnm2/mapping.json index 3d3261cd..4a499b15 100644 --- a/bin/tsw/api/tnm2/mapping.json +++ b/bin/tsw/api/tnm2/mapping.json @@ -48,6 +48,9 @@ "SUM_TSW_WEBSOCKET_ERROR" : 1253122, "SUM_TSW_WEBSOCKET_CLOSE" : 1253126, + "AVG_TSW_CKV_QUEUE_COST" : 1253127, + "AVG_TSW_CKV_COST" : 1253129, + "AVG_TSW_CPU_LOAD_1" : 1280713, "AVG_TSW_CPU_LOAD_5" : 1280714, "AVG_TSW_CPU_LOAD_15" : 1280716, From 209640527dc5621789428269e1d70ba4d5235b51 Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Mon, 9 Jul 2018 20:10:37 +0800 Subject: [PATCH 04/37] fix(h5-test): use local instead of context --- bin/tsw/util/h5-test/add.js | 13 +++++++------ bin/tsw/util/h5-test/del.js | 10 +++++----- bin/tsw/util/h5-test/get.js | 10 +++++----- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/bin/tsw/util/h5-test/add.js b/bin/tsw/util/h5-test/add.js index 2657aa46..89716195 100644 --- a/bin/tsw/util/h5-test/add.js +++ b/bin/tsw/util/h5-test/add.js @@ -50,13 +50,13 @@ module.exports.addTestUser = function(uin, val) { logger.debug('addTestUser:' + uin); val = val || true; const memcached = isTest.cmem(); - let keyText = isTest.keyBitmap(); const defer = Deferred.create(); - let appid = ''; + const appid = context.appid || ''; + const appkey = context.appkey; + let keyText = isTest.keyBitmap(); - if (context.appid && context.appkey) { + if (appid && appkey) { // 开平过来的 - appid = context.appid; keyText = `${keyText}.${appid}`; } @@ -76,11 +76,12 @@ module.exports.addTestUser = function(uin, val) { return defer.reject('memcached not exists'); } + memcached.get(keyText, function(err, data) { if (appid && typeof data === 'string') { // 解密 - data = post.decode(context.appid, context.appkey, data); + data = post.decode(appid, appkey, data); } const expire = 24 * 60 * 60; @@ -104,7 +105,7 @@ module.exports.addTestUser = function(uin, val) { if (appid) { // 加密 - text = post.encode(context.appid, context.appkey, text); + text = post.encode(appid, appkey, text); } memcached.set(keyText, text, expire, function(err, ret) { diff --git a/bin/tsw/util/h5-test/del.js b/bin/tsw/util/h5-test/del.js index 2b8130e6..c529fd00 100644 --- a/bin/tsw/util/h5-test/del.js +++ b/bin/tsw/util/h5-test/del.js @@ -52,9 +52,10 @@ const returnJson = function(json) { module.exports.deleteTestUser = function(uin) { logger.debug('deleteTestUser:' + uin); const memcached = isTest.cmem(); - let keyText = isTest.keyBitmap(); const defer = Deferred.create(); - let appid = ''; + const appid = context.appid || ''; + const appkey = context.appkey; + let keyText = isTest.keyBitmap(); if (!uin) { return defer.reject(); @@ -66,7 +67,6 @@ module.exports.deleteTestUser = function(uin) { if (context.appid && context.appkey) { // 开平过来的 - appid = context.appid; keyText = `${keyText}.${appid}`; } @@ -78,7 +78,7 @@ module.exports.deleteTestUser = function(uin) { if (appid && typeof data === 'string') { // 解密 - data = post.decode(context.appid, context.appkey, data); + data = post.decode(appid, appkey, data); } const expire = 24 * 60 * 60; @@ -104,7 +104,7 @@ module.exports.deleteTestUser = function(uin) { if (appid) { // 加密 - text = post.encode(context.appid, context.appkey, text); + text = post.encode(appid, appkey, text); } memcached.set(keyText, text, expire, function(err, ret) { diff --git a/bin/tsw/util/h5-test/get.js b/bin/tsw/util/h5-test/get.js index 19bf1c28..7b173cbe 100644 --- a/bin/tsw/util/h5-test/get.js +++ b/bin/tsw/util/h5-test/get.js @@ -37,13 +37,13 @@ module.exports.getTestUser = function() { // 从内存中读取testTargetMap const memcached = isTest.cmem(); - let keyText = isTest.keyBitmap(); const defer = Deferred.create(); - let appid = ''; + const appid = context.appid || ''; + const appkey = context.appkey; + let keyText = isTest.keyBitmap(); - if (context.appid && context.appkey) { + if (appid && appkey) { // 开平过来的 - appid = context.appid; keyText = `${keyText}.${appid}`; } @@ -55,7 +55,7 @@ module.exports.getTestUser = function() { if (appid && typeof data === 'string') { // 解密 - data = post.decode(context.appid, context.appkey, data); + data = post.decode(appid, appkey, data); } if (err) { From b22f8e0b2118fba6c8ca0f5a347212a27e10a23e Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Mon, 9 Jul 2018 20:19:01 +0800 Subject: [PATCH 05/37] feat(h5-test): add ckv report --- bin/tsw/api/tnm2/index.js | 2 +- bin/tsw/api/tnm2/mapping.json | 5 +++-- bin/tsw/pool/cmem.l5.js | 9 +++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/bin/tsw/api/tnm2/index.js b/bin/tsw/api/tnm2/index.js index 0aff6422..df369312 100644 --- a/bin/tsw/api/tnm2/index.js +++ b/bin/tsw/api/tnm2/index.js @@ -9,7 +9,7 @@ const serverInfo = require('serverInfo.js'); -const mapping = require('./mapping.json'); +const mapping = require('api/tnm2/mapping.json'); const url = require('url'); const Deferred = require('util/Deferred'); const cluster = require('cluster'); diff --git a/bin/tsw/api/tnm2/mapping.json b/bin/tsw/api/tnm2/mapping.json index 4a499b15..3236d9b1 100644 --- a/bin/tsw/api/tnm2/mapping.json +++ b/bin/tsw/api/tnm2/mapping.json @@ -48,8 +48,9 @@ "SUM_TSW_WEBSOCKET_ERROR" : 1253122, "SUM_TSW_WEBSOCKET_CLOSE" : 1253126, - "AVG_TSW_CKV_QUEUE_COST" : 1253127, - "AVG_TSW_CKV_COST" : 1253129, + "SUM_TSW_CKV_CMD" : 1253127, + "AVG_TSW_CKV_QUEUE_COST" : 1253128, + "AVG_TSW_CKV_CMD_COST" : 1253129, "AVG_TSW_CPU_LOAD_1" : 1280713, "AVG_TSW_CPU_LOAD_5" : 1280714, diff --git a/bin/tsw/pool/cmem.l5.js b/bin/tsw/pool/cmem.l5.js index 41ecf0ad..ed8c3841 100644 --- a/bin/tsw/pool/cmem.l5.js +++ b/bin/tsw/pool/cmem.l5.js @@ -11,6 +11,7 @@ const logger = require('logger'); const Queue = require('util/Queue'); const dcapi = require('api/libdcapi/dcapi.js'); +const tnm2 = require('api/tnm2'); const L5 = require('api/L5/L5.api.js'); const { isWin32Like } = require('util/isWindows.js'); let cache = global[__filename]; @@ -83,7 +84,13 @@ function queueWrap(memcached) { const servers = memcached.servers && memcached.servers[0]; const start = Date.now(); + tnm2.Attr_API('SUM_TSW_CKV_CMD', 1); + queue.queue(function() { + const startQueue = Date.now(); + const costQueue = startQueue - start; + + tnm2.Attr_API_Set('AVG_TSW_CKV_QUEUE_COST', costQueue); const fn = (function(queryCompiler) { return function() { @@ -119,6 +126,8 @@ function queueWrap(memcached) { } } + tnm2.Attr_API_Set('AVG_TSW_CKV_CMD_COST', Date.now() - startQueue); + dcapi.report({ key: 'EVENT_TSW_MEMCACHED', toIp: toIp, From 0cc071c632bc0bf628c6920b1aab8bed9f35e1d9 Mon Sep 17 00:00:00 2001 From: wayenli Date: Tue, 10 Jul 2018 06:40:54 +0800 Subject: [PATCH 06/37] fix(config.js): fix config file path --- bin/proxy/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/proxy/config.js b/bin/proxy/config.js index b498d563..4bac0158 100644 --- a/bin/proxy/config.js +++ b/bin/proxy/config.js @@ -46,7 +46,7 @@ if (isFirstLoad) { if (fs.existsSync('/etc/tsw.config.js')) { - cache.config = require('/usr/local/node_modules/config.js'); + cache.config = require('/etc/tsw.config.js'); } else if (fs.existsSync('/usr/local/node_modules/config.js')) { cache.config = require('/usr/local/node_modules/config.js'); } else if (fs.existsSync('/data/release/node_modules/config.js')) { From caf1e626102ce45e9f48b45d1b8c23fa3b3d7511 Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Wed, 11 Jul 2018 20:00:28 +0800 Subject: [PATCH 07/37] chore(logger): logger add limit --- bin/tsw/util/auto-report/logReport.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/bin/tsw/util/auto-report/logReport.js b/bin/tsw/util/auto-report/logReport.js index 5f434903..7a5c3c90 100644 --- a/bin/tsw/util/auto-report/logReport.js +++ b/bin/tsw/util/auto-report/logReport.js @@ -291,6 +291,30 @@ module.exports.receiveCloud = function(req, res) { return returnJson('get appkey error'); } + if (data.logText) { + return returnJson('logText is invalid'); + } + + if (typeof data.logText !== 'string') { + return returnJson('logText is not string'); + } + + if (data.logText.length >= 64 * 1024) { + return returnJson('logText is large than 64KB'); + } + + if (!data.logJson) { + return returnJson('logJson is invalid'); + } + + if (typeof data.logJson !== 'string') { + return returnJson('logJson is not string'); + } + + if (data.logText.length >= 1024 * 1024) { + return returnJson('logJson is large than 1MB'); + } + const appid = context.appid; const appkey = context.appkey; const reportKey = [appid, data.key].join('/'); From a985a38c2cdff1645ed2bc181fb535aaec4df097 Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Wed, 11 Jul 2018 20:25:04 +0800 Subject: [PATCH 08/37] fix(runtime): bugfix for find bad code --- bin/tsw/runtime/fs.hack.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/tsw/runtime/fs.hack.js b/bin/tsw/runtime/fs.hack.js index 1da48087..1ac74443 100644 --- a/bin/tsw/runtime/fs.hack.js +++ b/bin/tsw/runtime/fs.hack.js @@ -36,7 +36,7 @@ if (global[__filename]) { sum = cache.map[key]; if (sum % TIMES_LIMIT === 0 && !config.devMode) { - logger.warn('[sync]${name} callee ${sum} times on file: ${file} \n${stack}', { + logger.warn('[sync]${name} called ${sum} times on file: ${file} \n${stack}', { name: name, file: file, sum: sum, @@ -56,7 +56,7 @@ if (global[__filename]) { sum = cache.map[key]; if (sum % TIMES_LIMIT === 0 && !config.devMode) { - logger.warn('[sync]${name} callee ${sum} times on file: ${file} \n${stack}', { + logger.warn('[sync]${name} called ${sum} times on file: ${file} \n${stack}', { name: name, file: file, sum: sum, @@ -75,8 +75,8 @@ if (global[__filename]) { cache.map[key] = ~~cache.map[key] + 1; sum = cache.map[key]; - if (sum > TIMES_LIMIT && !config.devMode) { - logger.warn('[sync]${name} callee ${sum} times on file: ${file} \n${stack}', { + if (sum % TIMES_LIMIT === 0 && !config.devMode) { + logger.warn('[sync]${name} called ${sum} times on file: ${file} \n${stack}', { name: name, file: file, sum: sum, @@ -96,7 +96,7 @@ if (global[__filename]) { sum = cache.map[key]; if (sum % TIMES_LIMIT === 0 && !config.devMode) { - logger.warn('[sync]${name} callee ${sum} times on file: ${file} \n${stack}', { + logger.warn('[sync]${name} called ${sum} times on file: ${file} \n${stack}', { name: name, file: file, sum: sum, @@ -115,8 +115,8 @@ if (global[__filename]) { cache.map[key] = ~~cache.map[key] + 1; sum = cache.map[key]; - if (sum > TIMES_LIMIT && !config.devMode) { - logger.warn('[sync]${name} callee ${sum} times on file: ${file} \n${stack}', { + if (sum % TIMES_LIMIT === 0 && !config.devMode) { + logger.warn('[sync]${name} called ${sum} times on file: ${file} \n${stack}', { name: name, file: file, sum: sum, @@ -136,7 +136,7 @@ if (global[__filename]) { sum = cache.map[key]; if (sum % TIMES_LIMIT === 0 && !config.devMode) { - logger.warn('[sync]${name} callee ${sum} times on file: ${file} \n${stack}', { + logger.warn('[sync]${name} called ${sum} times on file: ${file} \n${stack}', { name: name, file: file, sum: sum, From 1c221644b73471dc4e4d03373ee57a42ffec8d94 Mon Sep 17 00:00:00 2001 From: RobinzZH Date: Wed, 11 Jul 2018 20:39:34 +0800 Subject: [PATCH 09/37] fix(cmem.l5): use instances cache instead of poolsize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 多并发使用连接池发现有队列拥堵现象，使用实例缓存先解决拥堵产生的高延时问题 --- bin/tsw/pool/cmem.l5.js | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/bin/tsw/pool/cmem.l5.js b/bin/tsw/pool/cmem.l5.js index ed8c3841..c2216e45 100644 --- a/bin/tsw/pool/cmem.l5.js +++ b/bin/tsw/pool/cmem.l5.js @@ -24,9 +24,9 @@ if (!cache) { module.exports = function(opt) { /** - * 这里像这样写的目的主要是为了进行测试 - 因为在使用sinon.js时， 如果你exports的是一个function，你就无法进行stub， - */ + * 这里像这样写的目的主要是为了进行测试 + 因为在使用sinon.js时， 如果你exports的是一个function，你就无法进行stub， + */ return module.exports.getCmem(opt); }; @@ -57,14 +57,29 @@ module.exports.getCmem = function(opt) { return null; } + return fromCache(opt); +}; + +const fromCache = (opt) => { const key = [opt.modid, opt.cmd, opt.host].join(':'); if (!cache[key]) { const Memcached = require('memcached'); - cache[key] = queueWrap(new Memcached(opt.host, opt)); + const poolSize = opt.poolSize || 1; + const queueWrapList = []; + const option = Object.assign({}, opt, { + poolSize: 1 + }); + for (let i = 0; i < poolSize; i++) { + queueWrapList.push(queueWrap(new Memcached(opt.host, option))); + } + queueWrapList.curr = 0; + cache[key] = queueWrapList; + } else { + cache[key].curr = (cache[key].curr + 1) % cache[key].length; } - return cache[key]; + return cache[key][cache[key].curr]; }; From b93751278930f9dcdfc33af0f73b17fdaffd22c1 Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Wed, 11 Jul 2018 21:08:07 +0800 Subject: [PATCH 10/37] feat(master): add event report for worker --- bin/proxy/master.js | 3 +++ bin/tsw/api/tnm2/mapping.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/bin/proxy/master.js b/bin/proxy/master.js index cf4ebfc6..21003f46 100644 --- a/bin/proxy/master.js +++ b/bin/proxy/master.js @@ -368,6 +368,8 @@ function masterEventHandler() { cpu: cpu }); + tnm2.Attr_API('SUM_TSW_WORKER_FORK', 1); + // 绑定cpu cpuUtil.taskset(cpu, currWorker.process.pid); @@ -430,6 +432,7 @@ function masterEventHandler() { process.on('reload', function(GET) { logger.info('reload'); + tnm2.Attr_API('SUM_TSW_WORKER_RELOAD', 1); for (const key in workerMap) { const worker = workerMap[key]; diff --git a/bin/tsw/api/tnm2/mapping.json b/bin/tsw/api/tnm2/mapping.json index 3236d9b1..2bddc518 100644 --- a/bin/tsw/api/tnm2/mapping.json +++ b/bin/tsw/api/tnm2/mapping.json @@ -48,6 +48,9 @@ "SUM_TSW_WEBSOCKET_ERROR" : 1253122, "SUM_TSW_WEBSOCKET_CLOSE" : 1253126, + "SUM_TSW_WORKER_RELOAD" : 1285174, + "SUM_TSW_WORKER_FORK" : 1285175, + "SUM_TSW_CKV_CMD" : 1253127, "AVG_TSW_CKV_QUEUE_COST" : 1253128, "AVG_TSW_CKV_CMD_COST" : 1253129, From 3eb9b54cf1ecb1454a8aefbb9e1132b9a288640d Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Thu, 12 Jul 2018 11:05:36 +0800 Subject: [PATCH 11/37] fix(logreport): fixbug for logText assert --- bin/tsw/util/auto-report/logReport.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/tsw/util/auto-report/logReport.js b/bin/tsw/util/auto-report/logReport.js index 7a5c3c90..874e2172 100644 --- a/bin/tsw/util/auto-report/logReport.js +++ b/bin/tsw/util/auto-report/logReport.js @@ -291,12 +291,12 @@ module.exports.receiveCloud = function(req, res) { return returnJson('get appkey error'); } - if (data.logText) { - return returnJson('logText is invalid'); + if (!data.logText) { + return returnJson('logText is required'); } if (typeof data.logText !== 'string') { - return returnJson('logText is not string'); + return returnJson('logText is not a string'); } if (data.logText.length >= 64 * 1024) { @@ -304,11 +304,11 @@ module.exports.receiveCloud = function(req, res) { } if (!data.logJson) { - return returnJson('logJson is invalid'); + return returnJson('logJson is required'); } if (typeof data.logJson !== 'string') { - return returnJson('logJson is not string'); + return returnJson('logJson is not a string'); } if (data.logText.length >= 1024 * 1024) { From eadd99233ca4844ab8c8ac757ef43a23c8c90c91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?bdliu=28=E5=88=98=E5=BD=AC=E5=BE=B7=29?= Date: Mon, 16 Jul 2018 15:55:54 +0800 Subject: [PATCH 12/37] chore(runtime): add a doc link in CCFinder mail --- bin/tsw/runtime/CCFinder.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/tsw/runtime/CCFinder.js b/bin/tsw/runtime/CCFinder.js index 4cd88245..fedb72d4 100644 --- a/bin/tsw/runtime/CCFinder.js +++ b/bin/tsw/runtime/CCFinder.js @@ -250,7 +250,8 @@ this.check = function (req, res) { 'to': config.mailTo, 'cc': config.mailCC, 'title': title, - 'content': '

服务器IP：' + serverInfo.intranetIp + '

' + 'content': '

IP聚集相关信息，详情请参阅： [TSW] IP聚集文档

' + +'

服务器IP：' + serverInfo.intranetIp + '

' + '

恶意IP：' + max.ip + '

' + '

自动拉黑：' + (config.CCIPLimitAutoBlock ? '是' : '否') + '

' + '

IP聚集度：' + cache.ipCacheLast.StdX10 + '%

' From 7e5d4e9953fd4e201dce3c15da0ec916bbfff4f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?bdliu=28=E5=88=98=E5=BD=AC=E5=BE=B7=29?= Date: Mon, 16 Jul 2018 16:10:16 +0800 Subject: [PATCH 13/37] chore(runtime): add new runtime type 'CPU alert' --- bin/proxy/http.proxy.js | 1 + bin/tsw/api/keyman/runtimeType.json | 2 ++ 2 files changed, 3 insertions(+) diff --git a/bin/proxy/http.proxy.js b/bin/proxy/http.proxy.js index b034a79e..aa6755d8 100644 --- a/bin/proxy/http.proxy.js +++ b/bin/proxy/http.proxy.js @@ -407,6 +407,7 @@ function afterCpu80(cpuUsed) { mail.SendMail(key, 'js', 600, { 'to': config.mailTo, 'cc': config.mailCC, + 'runtimeType':'CPU', 'msgInfo': `${business.module}[CPU]${serverInfo.intranetIp}单核CPU${serverInfo.cpu}使用率为：${cpuUsed}，超过80%`, 'title': `${business.module}[CPU]${serverInfo.intranetIp}单核CPU${serverInfo.cpu}使用率为：${cpuUsed}，超过80%`, 'content': content, diff --git a/bin/tsw/api/keyman/runtimeType.json b/bin/tsw/api/keyman/runtimeType.json index a5833f14..aa37b7df 100644 --- a/bin/tsw/api/keyman/runtimeType.json +++ b/bin/tsw/api/keyman/runtimeType.json @@ -7,6 +7,8 @@ "Css404" : "内联CSS不存在", "LargeJs" : "内联JS过大", "Js404" : "内联JS不存在", + "CPU" : "CPU告警", "hello" : "hello" + } From 92581286ac772c43c103b90bc9c185b8815e95ac Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Thu, 12 Jul 2018 19:13:57 +0800 Subject: [PATCH 14/37] chore(alpha): move require to function --- bin/tsw/util/auto-report/alpha.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/tsw/util/auto-report/alpha.js b/bin/tsw/util/auto-report/alpha.js index ac0e3fc6..ba36d54e 100644 --- a/bin/tsw/util/auto-report/alpha.js +++ b/bin/tsw/util/auto-report/alpha.js @@ -8,10 +8,7 @@ 'use strict'; -const config = require('config'); -const logger = require('logger'); const { isWin32Like } = require('util/isWindows'); -const TSW = require('api/keyman'); if (!global[__filename]) { global[__filename] = {}; @@ -27,6 +24,10 @@ this.add = function(uin) { this.isAlpha = function(req) { + const TSW = require('api/keyman'); + const logger = require('logger'); + const config = require('config'); + let uin; if (typeof req === 'object') { @@ -63,6 +64,7 @@ this.isAlpha = function(req) { }; this.getUin = function(req) { + const config = require('config'); let uin; const window = context.window || {}; From e55915387efcc7a784d54b911d7d77fed6734340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?bdliu=28=E5=88=98=E5=BD=AC=E5=BE=B7=29?= Date: Mon, 16 Jul 2018 17:40:11 +0800 Subject: [PATCH 15/37] chore(runtime): add Memory over load alert --- bin/proxy/master.js | 15 ++++++++++++--- bin/tsw/api/keyman/runtimeType.json | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/proxy/master.js b/bin/proxy/master.js index 21003f46..bfbcc09d 100644 --- a/bin/proxy/master.js +++ b/bin/proxy/master.js @@ -14,12 +14,14 @@ const cluster = require('cluster'); const cpuUtil = require('util/cpu.js'); const fs = require('fs'); const serverOS = require('util/isWindows.js'); +const mail = require('util/mail/mail.js'); const { debugOptions } = process.binding('config'); const methodMap = {}; const workerMap = {}; const cpuMap = []; const tnm2 = require('api/tnm2'); const network = require('util/network.js'); +const serverInfo = require('serverInfo.js'); process.on('uncaughtException', function(e) { @@ -262,11 +264,9 @@ function checkWorkerAlive() { // 内存超限进程处理 if (worker.lastMessage) { const currMemory = worker.lastMessage.memoryUsage; - - // logger.debug(currMemory); + const key = `memoryLimit.v1:${serverInfo.intranetIp}`; if (currMemory && currMemory.rss > config.memoryLimit) { - logger.error('worker${cpu} pid=${pid} memoryUsage ${memoryUsage}, hit memoryLimit: ${memoryLimit}, kill it', { memoryUsage: currMemory.rss, memoryLimit: config.memoryLimit, @@ -276,6 +276,15 @@ function checkWorkerAlive() { logger.error(worker.lastMessage); + mail.SendMail(key, 'js', 600, { + 'to': config.mailTo, + 'cc': config.mailCC, + 'runtimeType':'Memory', + 'msgInfo': `${serverInfo.intranetIp} 内存超限，服务已重启。请开发人员关注是否存在内存泄露`, + 'title': `${serverInfo.intranetIp} 内存超限告警`, + 'content': `

${serverInfo.intranetIp} 内存超限，服务已重启。请开发人员关注是否存在内存泄露

` + }); + restartWorker(worker); } } diff --git a/bin/tsw/api/keyman/runtimeType.json b/bin/tsw/api/keyman/runtimeType.json index aa37b7df..7186ce1d 100644 --- a/bin/tsw/api/keyman/runtimeType.json +++ b/bin/tsw/api/keyman/runtimeType.json @@ -8,7 +8,7 @@ "LargeJs" : "内联JS过大", "Js404" : "内联JS不存在", "CPU" : "CPU告警", + "Memory" : "内存超载", "hello" : "hello" - } From 273a1c9fdff843c94e58add0989533b5ec9e78c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?bdliu=28=E5=88=98=E5=BD=AC=E5=BE=B7=29?= Date: Tue, 17 Jul 2018 20:10:27 +0800 Subject: [PATCH 16/37] fix(runtime): let url string instead of Tag a --- bin/tsw/runtime/CCFinder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tsw/runtime/CCFinder.js b/bin/tsw/runtime/CCFinder.js index fedb72d4..48a9144f 100644 --- a/bin/tsw/runtime/CCFinder.js +++ b/bin/tsw/runtime/CCFinder.js @@ -250,7 +250,7 @@ this.check = function (req, res) { 'to': config.mailTo, 'cc': config.mailCC, 'title': title, - 'content': '

IP聚集相关信息，详情请参阅： [TSW] IP聚集文档

' + 'content': '

IP聚集相关信息，详情请参阅文档： https://tswjs.org/doc/api/ipCCFinder

' +'

服务器IP：' + serverInfo.intranetIp + '

' + '

恶意IP：' + max.ip + '

' + '

自动拉黑：' + (config.CCIPLimitAutoBlock ? '是' : '否') + '

' From ff9646010b6474de873445ba68e883e95f841181 Mon Sep 17 00:00:00 2001 From: youkunhuang Date: Wed, 18 Jul 2018 16:06:54 +0800 Subject: [PATCH 17/37] fix(logreport): fixbug logText --> logJson --- bin/tsw/util/auto-report/logReport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tsw/util/auto-report/logReport.js b/bin/tsw/util/auto-report/logReport.js index 874e2172..1c8af091 100644 --- a/bin/tsw/util/auto-report/logReport.js +++ b/bin/tsw/util/auto-report/logReport.js @@ -311,7 +311,7 @@ module.exports.receiveCloud = function(req, res) { return returnJson('logJson is not a string'); } - if (data.logText.length >= 1024 * 1024) { + if (data.logJson.length >= 1024 * 1024) { return returnJson('logJson is large than 1MB'); } From 2f94faa7ee5eb1fb69bc788af62e9408f0425e96 Mon Sep 17 00:00:00 2001 From: tarotlwei Date: Wed, 18 Jul 2018 16:55:41 +0800 Subject: [PATCH 18/37] fix(websocket log): clear logJson --- bin/tsw/util/auto-report/logReport.js | 2 +- bin/tsw/util/logger/logger.js | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/tsw/util/auto-report/logReport.js b/bin/tsw/util/auto-report/logReport.js index 5f434903..40fae4aa 100644 --- a/bin/tsw/util/auto-report/logReport.js +++ b/bin/tsw/util/auto-report/logReport.js @@ -676,7 +676,7 @@ function reportLog() { res._body = Buffer.from(format.formatBuffer(res._body)); } - logJson = logJson || logger.getJson() || {}; + logJson = logJson || logger.getJson(isWebSocket) || {}; logJson.curr = { protocol: 'HTTP', host: req.headers.host, diff --git a/bin/tsw/util/logger/logger.js b/bin/tsw/util/logger/logger.js index b3d0acd5..16e0e84f 100644 --- a/bin/tsw/util/logger/logger.js +++ b/bin/tsw/util/logger/logger.js @@ -80,7 +80,7 @@ Logger.prototype = { contextMod.currentContext().log = null; }, - getJson: function() { + getJson: function(cleared) { const log = this.getLog(); let json = { curr: {}, @@ -97,6 +97,10 @@ Logger.prototype = { log.json = json; } + if (cleared) { + log.json = null; + } + return json; }, From 207936217a51079815075610163f6a7ded89d99f Mon Sep 17 00:00:00 2001 From: muyan Date: Wed, 18 Jul 2018 17:13:50 +0800 Subject: [PATCH 19/37] docs(contributing): add contributing.md --- CONTRIBUTING.md | 54 ++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f64cc8df..a81e268d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,28 +1,27 @@ -# Contributing to TSW -Welcome to [report Issues](https://github.com/Tencent/TSW/issues) or [pull requests](https://github.com/Tencent/TSW/pulls). It's recommended to read the following Contributing Guide first before contributing. +# 为TSW做出贡献 +欢迎您 [提出问题](https://github.com/Tencent/TSW/issues) 或 [pull requests](https://github.com/Tencent/TSW/pulls)， 建议您在为TSW做出贡献前先阅读以下TSW贡献指南。 ## Issues -We use Github Issues to track public bugs and feature requests. +我们通过Github Issues来收集问题和功能相关的需求。 -### Search Known Issues First -Please search the existing issues to see if any similar issue or feature request has already been filed. You should make sure your issue isn't redundant. +### 首先查看已知的问题 +在您准备提出问题以前，请先查看现有的Github Issues是否已有其他人提出过相似的功能或问题，以确保您提出的问题是有效的。 -### Reporting New Issues -If you open an issue, the more information the better. Such as detailed description, code blocks of your problem. +### 提交问题 +问题的表述应当尽可能的详细，可以包含相关的代码块。 ## Pull Requests -We strongly welcome your pull request to make TSW better. +我们期待您通过PR（Pull Requests）让TSW变的更加完善。 -### Branch Management -There are two main branches here: +### 分支管理 +仓库一共包含两个分支: -1. `master` branch. - 1. **Don't submit any PR on `master` branch.** -2. `dev` branch. - 1. It is our stable developing branch. After full testing, `dev` will be merged to `master` branch for the next release. - 2. **You are recommended to submit bugfix or feature PR on `dev` branch.** +1. `master` 分支 + 1. **请勿在master分支上提交任何PR。** +2. `dev` 分支 + 1. `dev`分支作为稳定的开发分支，经过测试后会在下一个版本合并到`master`分支。 + 2. **PR应该在`dev`分支上提交。** -Normal bugfix or feature request should be submitted to `dev` branch. After full testing, we will merge them to `master` branch for the next release. ``` master @@ -32,18 +31,17 @@ dev feature/bugfix PR ``` -### Make Pull Requests -The code team will monitor all pull request, we run some code check and test on it. After all tests passed, we will accecpt this PR. But it won't merge to `master` branch at once, which have some delay. +### PR流程 +TSW团队会查看所有的PR，我们会运行一些代码检查和测试，一经测试通过，我们会接受这次PR，但不会立即将代码合并到master分支上，会有一点延迟。 -Before submitting a pull request, please make sure the followings are done: +当您准备拉请求，请确保已经完成以下几个步骤: -1. Fork the repo and create your branch from `master`. -2. Update code or documentation if you have changed APIs. -3. Add the copyright notice to the top of any new files you've added. -4. Check your code lints and checkstyles. -5. Test and test again your code. -6. Now, you can submit your pull request on `dev`. +1. 将仓库fork下来并基于`master`分支创建您的开发分支。 +2. 如果您更改了APIs请更新代码及文档。 +3. 在您添加的每一个新文件头部加上版权声明。 +4. 检查您的代码语法及格式。 +5. 反复测试。 +6. 现在，您可以开始在`dev`分支上PR了。 -## License -By contributing to TSW, you agree that your contributions will be licensed -under its [MIT LICENSE](https://github.com/Tencent/TSW/blob/master/LICENSE) \ No newline at end of file +## 许可证 +通过为TSW做出贡献，代表您同意将其版权归为TSW所有，TSW的开源协议为[MIT LICENSE](https://github.com/Tencent/TSW/blob/master/LICENSE) \ No newline at end of file From 2b4e656ec068040728a8f86bc5ef9f9d3a7e6584 Mon Sep 17 00:00:00 2001 From: muyan Date: Wed, 18 Jul 2018 18:58:15 +0800 Subject: [PATCH 20/37] docs(contributing): add contributing_en.md --- CONTRIBUTING.md | 2 +- CONTRIBUTING_en.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING_en.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a81e268d..5e3da97e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ feature/bugfix PR ### PR流程 TSW团队会查看所有的PR，我们会运行一些代码检查和测试，一经测试通过，我们会接受这次PR，但不会立即将代码合并到master分支上，会有一点延迟。 -当您准备拉请求，请确保已经完成以下几个步骤: +当您准备pull requests时，请确保已经完成以下几个步骤: 1. 将仓库fork下来并基于`master`分支创建您的开发分支。 2. 如果您更改了APIs请更新代码及文档。 diff --git a/CONTRIBUTING_en.md b/CONTRIBUTING_en.md new file mode 100644 index 00000000..f64cc8df --- /dev/null +++ b/CONTRIBUTING_en.md @@ -0,0 +1,49 @@ +# Contributing to TSW +Welcome to [report Issues](https://github.com/Tencent/TSW/issues) or [pull requests](https://github.com/Tencent/TSW/pulls). It's recommended to read the following Contributing Guide first before contributing. + +## Issues +We use Github Issues to track public bugs and feature requests. + +### Search Known Issues First +Please search the existing issues to see if any similar issue or feature request has already been filed. You should make sure your issue isn't redundant. + +### Reporting New Issues +If you open an issue, the more information the better. Such as detailed description, code blocks of your problem. + +## Pull Requests +We strongly welcome your pull request to make TSW better. + +### Branch Management +There are two main branches here: + +1. `master` branch. + 1. **Don't submit any PR on `master` branch.** +2. `dev` branch. + 1. It is our stable developing branch. After full testing, `dev` will be merged to `master` branch for the next release. + 2. **You are recommended to submit bugfix or feature PR on `dev` branch.** + +Normal bugfix or feature request should be submitted to `dev` branch. After full testing, we will merge them to `master` branch for the next release. + +``` +master + ↑ +dev + ↑ +feature/bugfix PR +``` + +### Make Pull Requests +The code team will monitor all pull request, we run some code check and test on it. After all tests passed, we will accecpt this PR. But it won't merge to `master` branch at once, which have some delay. + +Before submitting a pull request, please make sure the followings are done: + +1. Fork the repo and create your branch from `master`. +2. Update code or documentation if you have changed APIs. +3. Add the copyright notice to the top of any new files you've added. +4. Check your code lints and checkstyles. +5. Test and test again your code. +6. Now, you can submit your pull request on `dev`. + +## License +By contributing to TSW, you agree that your contributions will be licensed +under its [MIT LICENSE](https://github.com/Tencent/TSW/blob/master/LICENSE) \ No newline at end of file From 07947d131e07258df8fed5c1cf59a42edf0c5224 Mon Sep 17 00:00:00 2001 From: muyan Date: Wed, 18 Jul 2018 20:37:11 +0800 Subject: [PATCH 21/37] feat(logview): add navigation in logview --- CONTRIBUTING.md | 2 +- README_en.md | 2 +- bin/tsw/util/auto-report/src/view.tmpl.html | 4 +++- bin/tsw/util/auto-report/tmpl.js | 6 ++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5e3da97e..9cb55c51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ feature/bugfix PR ### PR流程 TSW团队会查看所有的PR，我们会运行一些代码检查和测试，一经测试通过，我们会接受这次PR，但不会立即将代码合并到master分支上，会有一点延迟。 -当您准备pull requests时，请确保已经完成以下几个步骤: +当您准备PR时，请确保已经完成以下几个步骤: 1. 将仓库fork下来并基于`master`分支创建您的开发分支。 2. 如果您更改了APIs请更新代码及文档。 diff --git a/README_en.md b/README_en.md index d16a75dd..f68aadc0 100644 --- a/README_en.md +++ b/README_en.md @@ -69,7 +69,7 @@ docker run -v configure_dir:/data/release/node_modules -p 8080:80 tsw Find a bug or have a feature request? Please read the [issues](https://github.com/Tencent/TSW/issues) guidelines and search for existing and closed issues. -If your problem or idea is not addressed yet, please read through our [contributing guidelines](./CONTRIBUTING.md) and open a new [issues](https://github.com/Tencent/TSW/issues). +If your problem or idea is not addressed yet, please read through our [contributing guidelines](./CONTRIBUTING_en.md) and open a new [issues](https://github.com/Tencent/TSW/issues). ## License diff --git a/bin/tsw/util/auto-report/src/view.tmpl.html b/bin/tsw/util/auto-report/src/view.tmpl.html index 4a8a315a..3ce6dd68 100644 --- a/bin/tsw/util/auto-report/src/view.tmpl.html +++ b/bin/tsw/util/auto-report/src/view.tmpl.html @@ -74,7 +74,6 @@
- 下载全部抓包 Fiddler下载 Whistle下载(mac推荐) @@ -82,6 +81,9 @@ <%if(appid){%> 实时监控 测试环境 + 个人主页 + runtime + 应用详情 <%}else{%> 临时染色 <%}%> diff --git a/bin/tsw/util/auto-report/tmpl.js b/bin/tsw/util/auto-report/tmpl.js index 2629ea02..8c55ccff 100644 --- a/bin/tsw/util/auto-report/tmpl.js +++ b/bin/tsw/util/auto-report/tmpl.js @@ -293,11 +293,9 @@ define(function(require, exports, module) { groupArr = tmp; var XSS = plug('util/xss.js'); - __p.push('\n\n \n TSW云抓包™\n \n \n\n\n
\n 下载全部抓包\n Fiddler下载\n Whistle下载(mac推荐)\n \n '); + __p.push('\n\n \n TSW云抓包™\n \n \n\n\n
\n Fiddler下载\n Whistle下载(mac推荐)\n \n '); if (appid) { - __p.push(' 实时监控\n 测试环境'); + __p.push(' 实时监控\n 测试环境\n 个人主页\n runtime\n 应用详情'); } else { __p.push(' 临时染色'); }__p.push('