From 7d715ec24c18cf16091c35c0363d1fb5f7ad15e7 Mon Sep 17 00:00:00 2001 From: viskey98 Date: Sat, 14 Jul 2018 10:35:01 +0530 Subject: [PATCH 1/4] JSON and Nonetype error fixes --- tanner/api/api.py | 2 +- tanner/session_analyzer.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tanner/api/api.py b/tanner/api/api.py index 46b09f44..7ad0d61e 100644 --- a/tanner/api/api.py +++ b/tanner/api/api.py @@ -51,7 +51,7 @@ async def return_snare_info(self, uuid, count=-1): if not query_res: return 'Invalid SNARE UUID' for (i, val) in enumerate(query_res): - query_res[i] = json.loads(val) + query_res[i] = json.loads(val.decode("utf-8")) return query_res async def return_session_info(self, sess_uuid, snare_uuid=None): diff --git a/tanner/session_analyzer.py b/tanner/session_analyzer.py index 96b4c72a..d9bc2fec 100644 --- a/tanner/session_analyzer.py +++ b/tanner/session_analyzer.py @@ -20,7 +20,7 @@ async def analyze(self, session_key, redis_client): await asyncio.sleep(1, loop=self._loop) try: session = await redis_client.execute('get', session_key) - session = json.loads(session) + session = json.loads(session.decode("utf-8")) except (aioredis.ProtocolError, TypeError, ValueError) as error: self.logger.error('Can\'t get session for analyze: %s', error) else: @@ -143,16 +143,17 @@ async def detect_crawler(self, stats, bots_owner, crawler_hosts): if stats['requests_in_second'] > 10: if stats['referer'] is not None: return (0.0, 0.5) - if stats['user_agent'] in bots_owner: + if stats['user_agent'] is not None and stats['user_agent'] in bots_owner: return (0.85, 0.15) return (0.5, 0.85) - if stats['user_agent'] in bots_owner: + if stats['user_agent'] is not None and stats['user_agent'] in bots_owner: hostname, _, _ = await self._loop.run_in_executor( None, socket.gethostbyaddr, stats['peer_ip'] ) - for crawler_host in crawler_hosts: - if crawler_host in hostname: - return (0.75, 0.15) + if hostname is not None: + for crawler_host in crawler_hosts: + if crawler_host in hostname: + return (0.75, 0.15) return (0.25, 0.15) return (0.0, 0.0) @@ -161,13 +162,14 @@ async def detect_attacker(self, stats, bots_owner, crawler_hosts, attacks): return 1.0 if stats['requests_in_second'] > 10: return 0.0 - if stats['user_agent'] in bots_owner: + if stats['user_agent'] is not None and stats['user_agent'] in bots_owner: hostname, _, _ = await self._loop.run_in_executor( None, socket.gethostbyaddr, stats['peer_ip'] ) - for crawler_host in crawler_hosts: - if crawler_host in hostname: - return 0.25 + if hostname is not None: + for crawler_host in crawler_hosts: + if crawler_host in hostname: + return 0.25 return 0.75 if stats['hidden_links'] > 0: return 0.5 From 774a4e27cc4e8e53bcf2578256b49304a962d43f Mon Sep 17 00:00:00 2001 From: viskey98 Date: Tue, 17 Jul 2018 23:02:12 +0530 Subject: [PATCH 2/4] Suggested changes --- tanner/session_analyzer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tanner/session_analyzer.py b/tanner/session_analyzer.py index d9bc2fec..5bbb04fc 100644 --- a/tanner/session_analyzer.py +++ b/tanner/session_analyzer.py @@ -19,8 +19,7 @@ async def analyze(self, session_key, redis_client): session = None await asyncio.sleep(1, loop=self._loop) try: - session = await redis_client.execute('get', session_key) - session = json.loads(session.decode("utf-8")) + session = await redis_client.get(session_key, encoding='utf-8') except (aioredis.ProtocolError, TypeError, ValueError) as error: self.logger.error('Can\'t get session for analyze: %s', error) else: From 5c7f72283f9d87e38df94b017f4a6f8e8c08ff1c Mon Sep 17 00:00:00 2001 From: viskey98 Date: Wed, 18 Jul 2018 18:39:30 +0530 Subject: [PATCH 3/4] Suggested changes --- tanner/api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tanner/api/api.py b/tanner/api/api.py index 7ad0d61e..fe966798 100644 --- a/tanner/api/api.py +++ b/tanner/api/api.py @@ -43,7 +43,7 @@ async def return_snare_info(self, uuid, count=-1): query_res = [] try: query_res = await self.redis_client.zrevrangebyscore( - uuid, offset=0, count=count + uuid, offset=0, count=count, encoding='utf-8' ) except aioredis.ProtocolError as connection_error: self.logger.error('Can not connect to redis %s', connection_error) From 32df9fef72922c3ca1080610205b0de84ceacfac Mon Sep 17 00:00:00 2001 From: viskey98 Date: Wed, 18 Jul 2018 18:40:57 +0530 Subject: [PATCH 4/4] Minor fix --- tanner/api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tanner/api/api.py b/tanner/api/api.py index fe966798..acab5912 100644 --- a/tanner/api/api.py +++ b/tanner/api/api.py @@ -51,7 +51,7 @@ async def return_snare_info(self, uuid, count=-1): if not query_res: return 'Invalid SNARE UUID' for (i, val) in enumerate(query_res): - query_res[i] = json.loads(val.decode("utf-8")) + query_res[i] = json.loads(val) return query_res async def return_session_info(self, sess_uuid, snare_uuid=None):