From 4a86e17e5cb76a3e8d71aaf84bf19345e1a919fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 27 Jul 2016 10:45:56 +0200 Subject: [PATCH 001/164] downgrading log level to avoid spam --- src/lib/conf_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/conf_handling.py b/src/lib/conf_handling.py index 424b33ffd..b3ddd725a 100644 --- a/src/lib/conf_handling.py +++ b/src/lib/conf_handling.py @@ -140,7 +140,7 @@ def _get_fd(self, mode): for path in self.paths: try: fd = open(path, mode) - logger.warn('will use conf from %r', path) + logger.info('will use conf from %r', path) return fd except PermissionError: if os.path.exists(path): From f1a1cb905e1dd1ce5770feaecedcfa96ed350596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 27 Jul 2016 10:46:42 +0200 Subject: [PATCH 002/164] reset_feeds: only reseting active feed --- src/manager.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/manager.py b/src/manager.py index 99e4f34b3..99df12f03 100755 --- a/src/manager.py +++ b/src/manager.py @@ -45,12 +45,19 @@ def fetch(limit=100, retreive_all=False): @manager.command def reset_feeds(): - contr = FeedController() - step = timedelta(seconds=3600 / contr.read().count()) + from web.models import User + fcontr = FeedController(ignore_context=True) now = datetime.utcnow() - for i, feed in enumerate(contr.read() - .order_by(contr._db_cls.last_retrieved)): - contr.update({'id': feed.id}, + last_conn_max = now - timedelta(days=30) + + feeds = list(fcontr.read().join(User).filter(User.is_active == True, + User.last_connection >= last_conn_max)\ + .with_entities(fcontr._db_cls.user_id)\ + .distinct()) + + step = timedelta(seconds=3600 / fcontr.read().count()) + for i, feed in enumerate(feeds): + fcontr.update({'id': feed[0]}, {'etag': '', 'last_modified': '', 'last_retrieved': now - i * step}) From 789dbc33f8c9a404b94140809e39548f3f737f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 29 Jul 2016 16:32:27 +0200 Subject: [PATCH 003/164] setting global timeout for every crawler --- src/crawler/http_crawler.py | 3 ++- src/crawler/lib/article_utils.py | 3 ++- src/lib/conf_handling.py | 1 + src/tests/base.py | 1 + src/web/controllers/icon.py | 7 ++++++- src/web/lib/feed_utils.py | 2 +- src/web/lib/utils.py | 4 +++- 7 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 6623cbd38..9edc1eb55 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -49,7 +49,7 @@ def query_jarr(self, method, urn, data=None): data = {} method = getattr(self.session, method) future = method("%s%s/%s" % (self.url, conf.API_ROOT.strip('/'), urn), - auth=self.auth, + auth=self.auth, timeout=conf.CRAWLER_TIMEOUT, data=json.dumps(data, default=default_handler), headers={'Content-Type': 'application/json', 'User-Agent': conf.CRAWLER_USER_AGENT}) @@ -259,6 +259,7 @@ def callback(self, response): logger.debug('%r %r - fetching resources', feed['id'], feed['title']) future = self.session.get(feed['link'], + timeout=conf.CRAWLER_TIMEOUT, headers=self.prepare_headers(feed)) self._futures.append(future) diff --git a/src/crawler/lib/article_utils.py b/src/crawler/lib/article_utils.py index 8958b8f00..b9a921186 100644 --- a/src/crawler/lib/article_utils.py +++ b/src/crawler/lib/article_utils.py @@ -73,7 +73,8 @@ def get_article_link(entry): if conf.CRAWLER_RESOLV and article_link: try: # resolves URL behind proxies (like feedproxy.google.com) - response = requests.get(article_link, verify=False, timeout=5.0) + response = requests.get(article_link, verify=False, + timeout=conf.CRAWLER_TIMEOUT) article_link = response.url except Exception as error: logger.warning("Unable to get the real URL of %s. Error: %s", diff --git a/src/lib/conf_handling.py b/src/lib/conf_handling.py index b3ddd725a..13f594781 100644 --- a/src/lib/conf_handling.py +++ b/src/lib/conf_handling.py @@ -48,6 +48,7 @@ 'choices': ABS_CHOICES, 'edit': False}, {'key': 'USER_AGENT', 'edit': False, 'default': 'https://github.com/jaesivsm/JARR'}, + {'key': 'TIMEOUT', 'edit': False, 'default': 30}, ]}, {'prefix': 'PLUGINS', 'options': [ {'key': 'READABILITY_KEY', 'default': '', diff --git a/src/tests/base.py b/src/tests/base.py index 7807001a8..75a04e5ba 100644 --- a/src/tests/base.py +++ b/src/tests/base.py @@ -75,6 +75,7 @@ def _api(self, method_name, *urn_parts, **kwargs): del kwargs['user'] urn = path.join(conf.API_ROOT, *map(str, urn_parts)) + kwargs.pop('timeout', None) # removing timeout non supported by flask resp = method(urn, **kwargs) if resp.data and resp.content_type == 'application/json': resp.json = lambda *a, **kw: json.loads(resp.data.decode('utf8')) diff --git a/src/web/controllers/icon.py b/src/web/controllers/icon.py index 07c4a4ef1..6a6880164 100644 --- a/src/web/controllers/icon.py +++ b/src/web/controllers/icon.py @@ -1,5 +1,6 @@ import base64 import requests +from bootstrap import conf from web.models import Icon from .abstract import AbstractController @@ -10,7 +11,11 @@ class IconController(AbstractController): def _build_from_url(self, attrs): if 'url' in attrs and 'content' not in attrs: - resp = requests.get(attrs['url'], verify=False) + try: + resp = requests.get(attrs['url'], verify=False, + timemout=conf.CRAWLER_TIMEOUT) + except Exception: + return attrs attrs.update({'url': resp.url, 'mimetype': resp.headers.get('content-type', None), 'content': base64.b64encode(resp.content).decode('utf8')}) diff --git a/src/web/lib/feed_utils.py b/src/web/lib/feed_utils.py index fab691930..18b369819 100644 --- a/src/web/lib/feed_utils.py +++ b/src/web/lib/feed_utils.py @@ -33,7 +33,7 @@ def metawrapper(*args, **kwargs): @escape_keys('title', 'description') def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): requests_kwargs = {'headers': {'User-Agent': conf.CRAWLER_USER_AGENT}, - 'verify': False} + 'verify': False, 'timeout': conf.CRAWLER_TIMEOUT} if url is None and fp_parsed is not None: url = fp_parsed.get('url') if url is not None and fp_parsed is None: diff --git a/src/web/lib/utils.py b/src/web/lib/utils.py index 3f6764db3..0f318aa2d 100644 --- a/src/web/lib/utils.py +++ b/src/web/lib/utils.py @@ -4,6 +4,7 @@ import logging import requests from hashlib import md5 +from bootstrap import conf from flask import request, url_for logger = logging.getLogger(__name__) @@ -49,7 +50,8 @@ def try_get_icon_url(url, *splits): response = None # if html in content-type, we assume it's a fancy 404 page try: - response = requests.get(rb_url, verify=False, timeout=10) + response = requests.get(rb_url, verify=False, + timeout=conf.CRAWLER_TIMEOUT) content_type = response.headers.get('content-type', '') except Exception: pass From 378700b9490af1884517641c803d1d2be32db7c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 17 Aug 2016 15:27:29 +0200 Subject: [PATCH 004/164] fixing article with not title --- src/crawler/lib/article_utils.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/crawler/lib/article_utils.py b/src/crawler/lib/article_utils.py index b9a921186..b78face2f 100644 --- a/src/crawler/lib/article_utils.py +++ b/src/crawler/lib/article_utils.py @@ -3,6 +3,7 @@ import requests import dateutil.parser from datetime import datetime, timezone +from bs4 import BeautifulSoup, SoupStrainer from bootstrap import conf from web.lib.utils import to_hash @@ -47,14 +48,13 @@ def construct_article(entry, feed): break content = get_article_content(entry) - link = get_article_link(entry) + link, title = get_article_details(entry) content = clean_urls(content, link) return {'feed_id': feed['id'], 'user_id': feed['user_id'], 'entry_id': extract_id(entry).get('entry_id', None), - 'link': link, 'content': content, - 'title': html.unescape(entry.get('title', 'No title')), + 'link': link, 'content': content, 'title': title, 'readed': False, 'like': False, 'retrieved_date': now, 'date': date or now} @@ -68,15 +68,23 @@ def get_article_content(entry): return content -def get_article_link(entry): +def get_article_details(entry): article_link = entry.get('link') - if conf.CRAWLER_RESOLV and article_link: + article_title = html.unescape(entry.get('title', '')) + if conf.CRAWLER_RESOLV and article_link or not article_title: try: # resolves URL behind proxies (like feedproxy.google.com) - response = requests.get(article_link, verify=False, - timeout=conf.CRAWLER_TIMEOUT) - article_link = response.url + response = requests.get(article_link, verify=False, timeout=5.0) except Exception as error: - logger.warning("Unable to get the real URL of %s. Error: %s", - article_link, error) - return article_link + logger.warning("Unable to get the real URL of %s. Won't fix link " + "or title. Error: %s", article_link, error) + return article_link, article_title + article_link = response.url + if not article_title: + bs_parsed = BeautifulSoup(response.content, 'html.parser', + parse_only=SoupStrainer('head')) + try: + article_title = bs_parsed.find_all('title')[0].text + except IndexError: # no title + pass + return article_link, article_title From 532684ad26c5de03fe3b7774cf3227cf7ee753ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 17 Aug 2016 15:57:54 +0200 Subject: [PATCH 005/164] refact, not using requests.get, always applying user-agent and other options --- requirements.txt | 2 +- src/crawler/http_crawler.py | 2 +- src/crawler/lib/article_utils.py | 5 ++--- src/{web => }/lib/utils.py | 9 +++++++-- src/web/controllers/feed.py | 2 +- src/web/controllers/icon.py | 6 ++---- src/web/lib/feed_utils.py | 10 +++------- src/web/lib/view_utils.py | 2 +- src/web/utils.py | 2 +- src/web/views/admin.py | 2 +- src/web/views/common.py | 2 +- src/web/views/home.py | 2 +- 12 files changed, 22 insertions(+), 24 deletions(-) rename src/{web => }/lib/utils.py (90%) diff --git a/requirements.txt b/requirements.txt index f8bfa4494..42251c87a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ opml==0.5 python-dateutil==2.4.2 python-postmark==0.4.7 rauth==0.7.2 -requests==2.10.0 +requests==2.11.0 requests-futures==0.9.7 SQLAlchemy==1.0.11 WTForms==2.1 diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 9edc1eb55..c3b4154b3 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -22,7 +22,7 @@ from datetime import datetime, timedelta from concurrent.futures import wait, ThreadPoolExecutor from requests_futures.sessions import FuturesSession -from web.lib.utils import default_handler, to_hash +from lib.utils import default_handler, to_hash from web.lib.feed_utils import construct_feed_from from crawler.lib.article_utils import extract_id, construct_article diff --git a/src/crawler/lib/article_utils.py b/src/crawler/lib/article_utils.py index b78face2f..ea1353fd4 100644 --- a/src/crawler/lib/article_utils.py +++ b/src/crawler/lib/article_utils.py @@ -1,12 +1,11 @@ import html import logging -import requests import dateutil.parser from datetime import datetime, timezone from bs4 import BeautifulSoup, SoupStrainer from bootstrap import conf -from web.lib.utils import to_hash +from lib.utils import to_hash, jarr_get from web.lib.article_cleaner import clean_urls logger = logging.getLogger(__name__) @@ -74,7 +73,7 @@ def get_article_details(entry): if conf.CRAWLER_RESOLV and article_link or not article_title: try: # resolves URL behind proxies (like feedproxy.google.com) - response = requests.get(article_link, verify=False, timeout=5.0) + response = jarr_get(article_link) except Exception as error: logger.warning("Unable to get the real URL of %s. Won't fix link " "or title. Error: %s", article_link, error) diff --git a/src/web/lib/utils.py b/src/lib/utils.py similarity index 90% rename from src/web/lib/utils.py rename to src/lib/utils.py index 0f318aa2d..90b06c592 100644 --- a/src/web/lib/utils.py +++ b/src/lib/utils.py @@ -50,8 +50,7 @@ def try_get_icon_url(url, *splits): response = None # if html in content-type, we assume it's a fancy 404 page try: - response = requests.get(rb_url, verify=False, - timeout=conf.CRAWLER_TIMEOUT) + response = jarr_get(rb_url) content_type = response.headers.get('content-type', '') except Exception: pass @@ -79,3 +78,9 @@ def clear_string(data): def redirect_url(default='home'): return request.args.get('next') or request.referrer or url_for(default) + + +def jarr_get(url): + return requests.get(url, verify=False, allow_redirects=True, + timeout=conf.CRAWLER_TIMEOUT, + headers={'User-Agent': conf.CRAWLER_USER_AGENT}) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index 48da2266c..033bfa977 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -6,7 +6,7 @@ from .abstract import AbstractController from .icon import IconController from web.models import User, Feed -from web.lib.utils import clear_string +from lib.utils import clear_string logger = logging.getLogger(__name__) DEFAULT_LIMIT = 5 diff --git a/src/web/controllers/icon.py b/src/web/controllers/icon.py index 6a6880164..a9a56930f 100644 --- a/src/web/controllers/icon.py +++ b/src/web/controllers/icon.py @@ -1,6 +1,5 @@ import base64 -import requests -from bootstrap import conf +from lib.utils import jarr_get from web.models import Icon from .abstract import AbstractController @@ -12,8 +11,7 @@ class IconController(AbstractController): def _build_from_url(self, attrs): if 'url' in attrs and 'content' not in attrs: try: - resp = requests.get(attrs['url'], verify=False, - timemout=conf.CRAWLER_TIMEOUT) + resp = jarr_get(attrs['url']) except Exception: return attrs attrs.update({'url': resp.url, diff --git a/src/web/lib/feed_utils.py b/src/web/lib/feed_utils.py index 18b369819..5abeae239 100644 --- a/src/web/lib/feed_utils.py +++ b/src/web/lib/feed_utils.py @@ -1,12 +1,10 @@ import html import urllib import logging -import requests import feedparser from bs4 import BeautifulSoup, SoupStrainer -from bootstrap import conf -from web.lib.utils import try_keys, try_get_icon_url, rebuild_url +from lib.utils import try_keys, try_get_icon_url, rebuild_url, jarr_get logger = logging.getLogger(__name__) logging.captureWarnings(True) @@ -32,13 +30,11 @@ def metawrapper(*args, **kwargs): @escape_keys('title', 'description') def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): - requests_kwargs = {'headers': {'User-Agent': conf.CRAWLER_USER_AGENT}, - 'verify': False, 'timeout': conf.CRAWLER_TIMEOUT} if url is None and fp_parsed is not None: url = fp_parsed.get('url') if url is not None and fp_parsed is None: try: - response = requests.get(url, **requests_kwargs) + response = jarr_get(url) fp_parsed = feedparser.parse(response.content, request_headers=response.headers) except Exception as error: @@ -72,7 +68,7 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): return feed try: - response = requests.get(feed['site_link'], **requests_kwargs) + response = jarr_get(feed['site_link']) except Exception as error: logger.warn('failed to retreive %r: %r', feed['site_link'], error) return feed diff --git a/src/web/lib/view_utils.py b/src/web/lib/view_utils.py index d4c119da6..1d8c6aed1 100644 --- a/src/web/lib/view_utils.py +++ b/src/web/lib/view_utils.py @@ -1,6 +1,6 @@ from functools import wraps from flask import request, Response, make_response -from web.lib.utils import to_hash +from lib.utils import to_hash def etag_match(func): diff --git a/src/web/utils.py b/src/web/utils.py index fa1005bd4..f6e21f9f1 100755 --- a/src/web/utils.py +++ b/src/web/utils.py @@ -26,7 +26,7 @@ from bootstrap import conf from web import controllers from web.models import Article -from web.lib.utils import clear_string +from lib.utils import clear_string logger = logging.getLogger(__name__) diff --git a/src/web/views/admin.py b/src/web/views/admin.py index 5dd5abf7d..3212b7a12 100644 --- a/src/web/views/admin.py +++ b/src/web/views/admin.py @@ -6,8 +6,8 @@ from flask.ext.babel import gettext, format_timedelta from flask.ext.login import login_required, current_user +from lib.utils import redirect_url from web.views.common import admin_permission -from web.lib.utils import redirect_url from web.controllers import UserController, FeedController, ArticleController logger = logging.getLogger(__name__) diff --git a/src/web/views/common.py b/src/web/views/common.py index 987e94a82..9806fadc4 100644 --- a/src/web/views/common.py +++ b/src/web/views/common.py @@ -4,7 +4,7 @@ from flask.ext.login import login_user from flask.ext.principal import (Identity, Permission, RoleNeed, session_identity_loader, identity_changed) -from web.lib.utils import default_handler +from lib.utils import default_handler admin_role = RoleNeed('admin') api_role = RoleNeed('api') diff --git a/src/web/views/home.py b/src/web/views/home.py index cabdd89de..d6674db96 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -9,7 +9,7 @@ from babel.dates import format_datetime, format_timedelta from bootstrap import conf -from web.lib.utils import redirect_url +from lib.utils import redirect_url from web.lib.article_cleaner import clean_urls from web import utils from web.lib.view_utils import etag_match From da5969c7dc27d2f004b342042b80e1c48b7b2de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Tue, 6 Sep 2016 16:49:28 +0200 Subject: [PATCH 006/164] push new articles in batch per feed --- src/crawler/http_crawler.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index c3b4154b3..13dee8eb5 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -103,15 +103,16 @@ def callback(self, response): logger.debug('%r %r - %d entries were not matched ' 'and will be created', self.feed['id'], self.feed['title'], len(results)) + entries = [] for id_to_create in results: article_created = True - entry = construct_article( + entries.append(construct_article( self.entries[tuple(sorted(id_to_create.items()))], - self.feed) + self.feed)) logger.info('%r %r - creating %r for %r - %r', self.feed['id'], - self.feed['title'], entry['title'], - entry['user_id'], id_to_create) - self.query_jarr('post', 'article', entry) + self.feed['title'], entries[-1]['title'], + entries[-1]['user_id'], id_to_create) + self.query_jarr('post', 'articles', entries) logger.debug('%r %r - updating feed etag %r last_mod %r', self.feed['id'], self.feed['title'], From 6fa2294469a67f478ea8b12f3887066409b498dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Tue, 6 Sep 2016 16:58:05 +0200 Subject: [PATCH 007/164] icon needs different delete method --- src/web/controllers/icon.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/web/controllers/icon.py b/src/web/controllers/icon.py index a9a56930f..7a4bc9d4d 100644 --- a/src/web/controllers/icon.py +++ b/src/web/controllers/icon.py @@ -1,4 +1,5 @@ import base64 +from bootstrap import db from lib.utils import jarr_get from web.models import Icon from .abstract import AbstractController @@ -24,3 +25,9 @@ def create(self, **attrs): def update(self, filters, attrs): return super().update(filters, self._build_from_url(attrs)) + + def delete(self, url): + obj = self.get(url=url) + db.session.delete(obj) + db.session.commit() + return obj From 03d3f776b6969719ad21f0e597585c90a1a648db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 9 Sep 2016 15:59:38 +0200 Subject: [PATCH 008/164] fixing broken images from readability parsed content close #51 --- src/web/lib/article_cleaner.py | 9 +++++++-- src/web/views/home.py | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/web/lib/article_cleaner.py b/src/web/lib/article_cleaner.py index 4c3fbd2e3..9e4e9c09d 100644 --- a/src/web/lib/article_cleaner.py +++ b/src/web/lib/article_cleaner.py @@ -1,17 +1,22 @@ -from urllib.parse import urlparse, urlunparse, ParseResult +from urllib.parse import unquote, urlparse, urlunparse, ParseResult from bs4 import BeautifulSoup from bootstrap import is_secure_served HTTPS_IFRAME_DOMAINS = ('vimeo.com', 'youtube.com', 'youtu.be') -def clean_urls(article_content, article_link): +def clean_urls(article_content, article_link, fix_readability=False): parsed_article_url = urlparse(article_link) parsed_content = BeautifulSoup(article_content, 'html.parser') for img in parsed_content.find_all('img'): if 'src' not in img.attrs: continue + # bug reported to readability, fixing it here for now + if fix_readability: + splited_src = unquote(img.attrs['src']).split(', ') + if len(splited_src) > 1: + img.attrs['src'] = splited_src[0].split()[0] if is_secure_served() and 'srcset' in img.attrs: # removing active content when serving over https del img.attrs['srcset'] diff --git a/src/web/views/home.py b/src/web/views/home.py index d6674db96..60ae79507 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -163,7 +163,8 @@ def get_article(article_id, parse=False): article['readability_parsed'] = False else: article['readability_parsed'] = True - article['content'] = clean_urls(new_content, article['link']) + article['content'] = clean_urls(new_content, article['link'], + fix_readability=True) new_attr = {'readability_parsed': True, 'content': new_content} contr.update({'id': article['id']}, new_attr) return article From 018a852c2d50806b5443577981efc321cabc3767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 2 Oct 2016 12:03:18 +0200 Subject: [PATCH 009/164] feed construction was kinda broken --- src/web/lib/feed_utils.py | 50 +++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/web/lib/feed_utils.py b/src/web/lib/feed_utils.py index 5abeae239..5033c1764 100644 --- a/src/web/lib/feed_utils.py +++ b/src/web/lib/feed_utils.py @@ -2,14 +2,15 @@ import urllib import logging import feedparser +from bootstrap import conf from bs4 import BeautifulSoup, SoupStrainer -from lib.utils import try_keys, try_get_icon_url, rebuild_url, jarr_get +from lib.utils import try_get_icon_url, rebuild_url, jarr_get logger = logging.getLogger(__name__) logging.captureWarnings(True) -ACCEPTED_MIMETYPES = ('application/rss+xml', 'application/rdf+xml', - 'application/atom+xml', 'application/xml', 'text/xml') +FEED_MIMETYPES = ('application/atom+xml', 'application/rss+xml', + 'application/rdf+xml', 'application/xml', 'text/xml') def is_parsing_ok(parsed_feed): @@ -28,15 +29,22 @@ def metawrapper(*args, **kwargs): return wrapper +def _browse_feedparser_feed(feed, check): + if feed.get('feed', {}).get('links') is None: + return + for link in feed['feed']['links']: + if check(link): + return link['href'] + + @escape_keys('title', 'description') def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): if url is None and fp_parsed is not None: url = fp_parsed.get('url') if url is not None and fp_parsed is None: try: - response = jarr_get(url) - fp_parsed = feedparser.parse(response.content, - request_headers=response.headers) + fp_parsed = feedparser.parse(url, + request_headers={'User-Agent': conf.CRAWLER_USER_AGENT}) except Exception as error: logger.warn('failed to retreive that url: %r', error) fp_parsed = {'bozo': True} @@ -44,14 +52,30 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None, query_site=True): feed = feed or {} feed_split = urllib.parse.urlsplit(url) site_split = None + feed['site_link'] = url + feed['link'] = _browse_feedparser_feed(fp_parsed, + lambda link: link['type'] in FEED_MIMETYPES) + + if not is_parsing_ok(fp_parsed) and feed.get('link'): + try: + fp_parsed = feedparser.parse(feed['link'], + request_headers={'User-Agent': conf.CRAWLER_USER_AGENT}) + except Exception as error: + logger.warn('failed to retreive that url: %r', error) + fp_parsed = {'bozo': True} + url = feed['link'] + if is_parsing_ok(fp_parsed): feed['link'] = url - feed['site_link'] = try_keys(fp_parsed['feed'], 'href', 'link') - feed['title'] = fp_parsed['feed'].get('title') - feed['description'] = try_keys(fp_parsed['feed'], 'subtitle', 'title') - feed['icon_url'] = try_keys(fp_parsed['feed'], 'icon') - else: - feed['site_link'] = url + feed['site_link'] = fp_parsed['feed'].get('link') \ + or _browse_feedparser_feed(fp_parsed, + lambda link: link['rel'] == 'alternate' + and link['type'] == 'text/html') + feed['title'] = fp_parsed['feed'].get('title_detail', {}).get('value') + feed['description'] = fp_parsed['feed']\ + .get('subtitle_detail', {}).get('value') + feed['icon_url'] = _browse_feedparser_feed(fp_parsed, + lambda link: 'icon' in link['rel']) if feed.get('site_link'): feed['site_link'] = rebuild_url(feed['site_link'], feed_split) @@ -109,7 +133,7 @@ def wrapper(elem): del feed['icon_url'] if not feed.get('link'): - for type_ in ACCEPTED_MIMETYPES: + for type_ in FEED_MIMETYPES: alternates = bs_parsed.find_all(check_keys( rel=['alternate'], type=[type_])) if len(alternates) >= 1: From 5d8ac70352500fb90a523d89a93dc178b3960143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 2 Oct 2016 18:41:43 +0200 Subject: [PATCH 010/164] big feature: clustering close #14 --- .travis.yml | 3 +- install.py | 8 +- package.json | 18 +- requirements.postgres.txt | 2 +- requirements.txt | 23 +-- src/bootstrap.py | 4 +- src/crawler/classic_crawler.py | 1 - src/crawler/http_crawler.py | 17 +- src/crawler/lib/article_utils.py | 1 - src/lib/utils.py | 2 +- src/manager.py | 6 +- .../versions/9e3fecc9d031_oauth_linuxfr.py | 2 +- .../versions/a7f62d50d366_clustering.py | 141 +++++++++++++ src/runserver.py | 4 +- src/scripts/probes.py | 2 +- src/tests/api/article_test.py | 35 +--- src/tests/api/category_test.py | 6 +- src/tests/api/cluster_test.py | 60 ++++++ src/tests/api/feed_test.py | 18 +- src/tests/base.py | 2 +- src/tests/controllers/article_test.py | 103 +++++----- src/tests/controllers/cluster_test.py | 120 +++++++++++ src/tests/crawler_test.py | 18 +- src/tests/fixtures/filler.py | 44 ++-- src/tests/ui_test.py | 40 +++- src/web/controllers/__init__.py | 3 +- src/web/controllers/abstract.py | 50 +++-- src/web/controllers/article.py | 31 +-- src/web/controllers/category.py | 5 - src/web/controllers/cluster.py | 183 +++++++++++++++++ src/web/controllers/feed.py | 6 +- src/web/controllers/icon.py | 3 +- src/web/controllers/user.py | 4 +- src/web/forms.py | 6 +- src/web/js/actions/MenuActions.js | 2 +- src/web/js/actions/MiddlePanelActions.js | 35 ++-- src/web/js/actions/RightPanelActions.js | 25 ++- src/web/js/components/MainApp.react.js | 9 +- src/web/js/components/Menu.react.js | 31 ++- src/web/js/components/MiddlePanel.react.js | 163 ++++++++++----- src/web/js/components/Navbar.react.js | 22 +- src/web/js/components/RightPanel.react.js | 139 ++++++------- src/web/js/constants/JarrConstants.js | 4 +- src/web/js/stores/MenuStore.js | 131 ++++++------ src/web/js/stores/MiddlePanelStore.js | 121 +++++------ src/web/js/stores/RightPanelStore.js | 45 ++++- src/web/lib/feed_utils.py | 22 +- src/web/lib/view_utils.py | 21 ++ src/web/models/__init__.py | 15 +- src/web/models/article.py | 61 +++--- src/web/models/category.py | 22 +- src/web/models/cluster.py | 72 +++++++ src/web/models/feed.py | 52 ++--- src/web/models/icon.py | 11 +- src/web/models/relationships.py | 13 ++ src/web/models/right_mixin.py | 5 +- src/web/models/user.py | 62 +++--- src/web/static/css/one-page-app.css | 18 +- src/web/static/js/articles.js | 191 ------------------ src/web/templates/categories.html | 2 +- src/web/templates/feed_list.html | 2 +- src/web/views/__init__.py | 5 +- src/web/views/admin.py | 13 +- src/web/views/api/__init__.py | 4 +- src/web/views/api/article.py | 2 +- src/web/views/api/category.py | 2 +- src/web/views/api/cluster.py | 26 +++ src/web/views/api/common.py | 18 +- src/web/views/api/feed.py | 2 +- src/web/views/article.py | 23 +-- src/web/views/category.py | 4 +- src/web/views/cluster.py | 15 ++ src/web/views/common.py | 4 +- src/web/views/feed.py | 12 +- src/web/views/home.py | 120 +++++------ src/web/views/icon.py | 11 +- src/web/views/session_mgmt.py | 12 +- src/web/views/user.py | 6 +- src/web/views/views.py | 2 +- submodules/bootstrap | 2 +- 80 files changed, 1553 insertions(+), 997 deletions(-) create mode 100644 src/migrations/versions/a7f62d50d366_clustering.py create mode 100644 src/tests/api/cluster_test.py create mode 100644 src/tests/controllers/cluster_test.py create mode 100644 src/web/controllers/cluster.py create mode 100644 src/web/models/cluster.py create mode 100644 src/web/models/relationships.py delete mode 100644 src/web/static/js/articles.js create mode 100644 src/web/views/api/cluster.py create mode 100644 src/web/views/cluster.py diff --git a/.travis.yml b/.travis.yml index ba9251be8..b4675d2c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,8 @@ install: script: - nosetests --with-coverage --cover-package=bootstrap,runserver,web,crawler,plugins src/tests/ - - pep8 --ignore=E126,E127,E128,E302,E712,W503 src/web/ src/crawler/ src/plugins/ src/lib/ + # ignoring aligning / line breaking non-sens + - pep8 --ignore=E126,E127,E128,W503 src/web/ src/crawler/ src/plugins/ src/lib/ after_success: - coveralls diff --git a/install.py b/install.py index 8315c5f3f..a72032230 100755 --- a/install.py +++ b/install.py @@ -150,9 +150,11 @@ def install_python_deps(args): conf.reload() except Exception: pass - if not args.no_requirements: - install_postgres = 'postgres' in getattr( - conf, 'SQLALCHEMY_DATABASE_URI', '') + if args.no_requirements: + return + + install_postgres = 'postgres' in getattr( + conf, 'SQLALCHEMY_DATABASE_URI', '') print('installing python dependencies...') base_cmd = ['install', '--quiet', '--upgrade', '-r'] diff --git a/package.json b/package.json index 8d60be4c6..0a0303b75 100644 --- a/package.json +++ b/package.json @@ -17,22 +17,22 @@ }, "main": "src/web/js/app.js", "dependencies": { - "bootstrap": "^3.3.6", - "classnames": "^2.1.3", + "bootstrap": "^3.3.7", + "classnames": "^2.2.5", "flux": "^2.0.1", - "jquery": "^2.2.0", + "jquery": "^3.1.0", "keymirror": "~0.1.0", - "object-assign": "^1.0.0", + "object-assign": "^4.1.0", "react": "^0.14.6", "react-bootstrap": "^0.28.0", "react-dom": "^0.14.6" }, "devDependencies": { - "browserify": "^6.2.0", - "envify": "^3.0.0", - "reactify": "^0.15.2", - "uglify-js": "~2.4.15", - "watchify": "^2.1.1" + "browserify": "^13.1.0", + "envify": "^3.4.1", + "reactify": "^1.1.1", + "uglify-js": "~2.7.3", + "watchify": "^3.7.0" }, "scripts": { "start": "watchify -o src/web/static/js/bundle.min.js -v -d src/web/js/app.js", diff --git a/requirements.postgres.txt b/requirements.postgres.txt index 3d49e0822..59117830d 100644 --- a/requirements.postgres.txt +++ b/requirements.postgres.txt @@ -1 +1 @@ -psycopg2==2.6.1 +psycopg2==2.6.2 diff --git a/requirements.txt b/requirements.txt index 42251c87a..a47e433b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,22 @@ -aiohttp==0.21.0 -alembic==0.8.4 -beautifulsoup4==4.4.1 +aiohttp==0.22.5 +alembic==0.8.7 +beautifulsoup4==4.5.1 feedparser==5.2.1 -Flask==0.10.1 -Flask-Babel==0.9 +Flask==0.11.1 +Flask-Babel==0.11.1 Flask-Login==0.3.2 -Flask-Migrate==1.7.0 +Flask-Migrate==2.0.0 Flask-Principal==0.4.0 Flask-RESTful==0.3.5 Flask-Script==2.0.5 Flask-SQLAlchemy==2.1 Flask-SSLify==0.1.5 Flask-WTF==0.12 -lxml==3.5.0 +lxml==3.6.4 opml==0.5 -python-dateutil==2.4.2 -python-postmark==0.4.7 +python-dateutil==2.5.3 +python-postmark==0.4.9 rauth==0.7.2 -requests==2.11.0 +requests==2.11.1 requests-futures==0.9.7 -SQLAlchemy==1.0.11 -WTForms==2.1 +SQLAlchemy==1.0.15 diff --git a/src/bootstrap.py b/src/bootstrap.py index e91f8f48c..6b0f61f03 100644 --- a/src/bootstrap.py +++ b/src/bootstrap.py @@ -4,11 +4,10 @@ # required imports and code exection for basic functionning import os -import json import logging from urllib.parse import urlparse from flask import Flask -from flask.ext.sqlalchemy import SQLAlchemy +from flask_sqlalchemy import SQLAlchemy from lib.conf_handling import ConfObject @@ -50,6 +49,7 @@ def set_logging(log_path=None, log_level=logging.INFO, modules=(), if os.environ.get('JARR_TESTING', False) == 'true': application.debug = True application.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:' + conf.SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:' application.config['TESTING'] = True conf.CRAWLER_NBWORKER = 1 else: diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py index cf88b11f9..569401ea6 100644 --- a/src/crawler/classic_crawler.py +++ b/src/crawler/classic_crawler.py @@ -104,7 +104,6 @@ async def insert_database(user, feed): is_updated = True if existing_article.content != content: existing_article.content = content - existing_article.readed = False is_updated = True if is_updated: art_contr.update({'entry_id': existing_article.entry_id}, diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 13dee8eb5..437000960 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -23,7 +23,7 @@ from concurrent.futures import wait, ThreadPoolExecutor from requests_futures.sessions import FuturesSession from lib.utils import default_handler, to_hash -from web.lib.feed_utils import construct_feed_from +from web.lib.feed_utils import construct_feed_from, is_parsing_ok from crawler.lib.article_utils import extract_id, construct_article logger = logging.getLogger(__name__) @@ -119,21 +119,24 @@ def callback(self, response): self.headers.get('etag', ''), self.headers.get('last-modified', '')) - up_feed = {'error_count': 0, 'last_error': None, - 'etag': self.headers.get('etag', ''), + up_feed = {'etag': self.headers.get('etag', ''), 'last_modified': self.headers.get('last-modified', strftime('%a, %d %b %Y %X %Z', gmtime()))} + + if not is_parsing_ok(self.parsed_feed): + up_feed['last_error'] = str(self.parsed_feed['bozo_exception']) + up_feed['error_count'] = self.feed['error_count'] + 1 + return self.query_jarr('put', 'feed/%d' % self.feed['id'], up_feed) + fresh_feed = construct_feed_from(url=self.feed['link'], fp_parsed=self.parsed_feed) if fresh_feed.get('description'): fresh_feed['description'] \ = html.unescape(fresh_feed['description']) - for key in ('description', 'site_link', 'icon_url'): + for key in 'description', 'site_link', 'icon_url': if fresh_feed.get(key) and fresh_feed[key] != self.feed.get(key): up_feed[key] = fresh_feed[key] - if not self.feed.get('title'): - up_feed['title'] = html.unescape(fresh_feed.get('title', '')) up_feed['user_id'] = self.feed['user_id'] # re-getting that feed earlier since new entries appeared if article_created: @@ -222,7 +225,7 @@ def callback(self, response): self.feed['id'], self.feed['title'], len(ids), ids) future = self.query_jarr('get', 'articles/challenge', {'ids': ids}) updater = JarrUpdater(self.feed, entries, response.headers, - parsed_response, self.auth) + parsed_response, self.auth) future.add_done_callback(updater.callback) diff --git a/src/crawler/lib/article_utils.py b/src/crawler/lib/article_utils.py index ea1353fd4..d906d4eb5 100644 --- a/src/crawler/lib/article_utils.py +++ b/src/crawler/lib/article_utils.py @@ -54,7 +54,6 @@ def construct_article(entry, feed): 'user_id': feed['user_id'], 'entry_id': extract_id(entry).get('entry_id', None), 'link': link, 'content': content, 'title': title, - 'readed': False, 'like': False, 'retrieved_date': now, 'date': date or now} diff --git a/src/lib/utils.py b/src/lib/utils.py index 90b06c592..187f7e636 100644 --- a/src/lib/utils.py +++ b/src/lib/utils.py @@ -16,7 +16,7 @@ def default_handler(obj, role='admin'): return obj.isoformat() if hasattr(obj, 'dump'): return obj.dump(role=role) - if isinstance(obj, (set, frozenset, types.GeneratorType)): + if isinstance(obj, (set, frozenset, filter, types.GeneratorType)): return list(obj) if isinstance(obj, BaseException): return str(obj) diff --git a/src/manager.py b/src/manager.py index 99df12f03..bb6c96da2 100755 --- a/src/manager.py +++ b/src/manager.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- import logging from datetime import datetime, timedelta -from flask.ext.script import Manager -from flask.ext.migrate import Migrate, MigrateCommand +from flask_script import Manager +from flask_migrate import Migrate, MigrateCommand from bootstrap import application, db, conf import web.models @@ -68,7 +68,7 @@ def fetch_asyncio(user_id, feed_id): import asyncio with application.app_context(): - from flask.ext.login import current_user + from flask_login import current_user from crawler import classic_crawler ucontr = UserController() users = [] diff --git a/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py b/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py index d9758bcf9..178d7de68 100644 --- a/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py +++ b/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py @@ -1,4 +1,4 @@ -"""empty message +"""adding linuxfr as an oauth provider Revision ID: 9e3fecc9d031 Revises: 122ac0c356c diff --git a/src/migrations/versions/a7f62d50d366_clustering.py b/src/migrations/versions/a7f62d50d366_clustering.py new file mode 100644 index 000000000..44d38f7e3 --- /dev/null +++ b/src/migrations/versions/a7f62d50d366_clustering.py @@ -0,0 +1,141 @@ +"""adding tables to support clustering + +Revision ID: a7f62d50d366 +Revises: 9e3fecc9d031 +Create Date: 2016-07-31 19:45:19.889247 + +""" + +# revision identifiers, used by Alembic. +revision = 'a7f62d50d366' +down_revision = '9e3fecc9d031' +branch_labels = None +depends_on = None + +from datetime import datetime +from bootstrap import conf +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('cluster', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('cluster_type', sa.String(), nullable=True), + sa.Column('main_link', sa.String(), nullable=True), + sa.Column('read', sa.Boolean(), nullable=True), + sa.Column('liked', sa.Boolean(), nullable=True), + sa.Column('created_date', sa.DateTime(), nullable=True), + sa.Column('main_date', sa.DateTime(), nullable=True), + sa.Column('main_feed_title', sa.String(), nullable=True), + sa.Column('main_title', sa.String(), nullable=True), + sa.Column('main_article_id', sa.Integer(), nullable=True), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['user.id']), + sa.PrimaryKeyConstraint('id')) + op.create_table('cluster_as_category', + sa.Column('cluster_id', sa.Integer(), nullable=True), + sa.Column('category_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['category_id'], ['category.id']), + sa.ForeignKeyConstraint(['cluster_id'], ['cluster.id'])) + op.create_table('cluster_as_feed', + sa.Column('cluster_id', sa.Integer(), nullable=True), + sa.Column('feed_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['cluster_id'], ['cluster.id']), + sa.ForeignKeyConstraint(['feed_id'], ['feed.id'])) + op.add_column('category', + sa.Column('cluster_on_title', sa.Boolean(), default=False)) + op.add_column('article', + sa.Column('cluster_id', sa.Integer(), nullable=True)) + + from web.models import Cluster, Feed, Article + from web.models.relationships import cluster_as_feed, cluster_as_category + if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI: + op.create_foreign_key(None, 'article', 'cluster', + ['cluster_id'], ['id']) + op.create_foreign_key(None, 'cluster', 'article', + ['main_article_id'], ['id']) + + op.execute('CREATE INDEX cluster_uid_date ON cluster ' + '(user_id, main_date DESC NULLS LAST);') + op.execute('CREATE INDEX cluster_liked_uid_date ON cluster ' + '(liked, user_id, main_date DESC NULLS LAST);') + op.execute('CREATE INDEX cluster_read_uid_date ON cluster ' + '(read, user_id, main_date DESC NULLS LAST);') + + print('%s - Creating clusters' % datetime.now().isoformat()) + op.execute(""" + INSERT INTO cluster (main_link, user_id, main_date, read, liked) + SELECT link, user_id, MIN(date), BOOL_AND(readed), BOOL_OR("like") + FROM article GROUP BY link, user_id;""") + + print('%s - Updating clusters with main article infos' + % datetime.now().isoformat()) + op.execute(sa.update(Cluster) + .where(sa.and_(Cluster.main_link == Article.link, + Cluster.user_id == Article.user_id, + Article.feed_id == Feed.id, + Cluster.main_date == Article.date)) + .values(main_title=Article.title, + main_feed_title=Feed.title, + main_article_id=Article.id)) + op.execute("""UPDATE article SET cluster_id = cluster.id + FROM cluster WHERE cluster.main_link = article.link + AND article.user_id = cluster.user_id;""") + + else: + print('%s - Creating clusters' % datetime.now().isoformat()) + op.execute(""" + INSERT INTO cluster (main_link, user_id, main_date, read, liked) + SELECT link, user_id, MIN(date), SUM(readed) = COUNT(id), + SUM("like") > COUNT(id) + FROM article GROUP BY link, user_id;""") + + WHERE = """WHERE cluster.main_link = article.link + AND cluster.user_id = article.user_id + AND cluster.main_date = article.date""" + + print('%s - Updating clusters with main article infos' + % datetime.now().isoformat()) + op.execute("UPDATE cluster SET " + "main_title = (SELECT article.title FROM article %(WHERE)s), " + "main_article_id = (SELECT article.id FROM article %(WHERE)s), " + "main_feed_title = (SELECT feed.title FROM article, feed %(WHERE)s " + "AND article.feed_id = feed.id);" % {'WHERE': WHERE}) + + print('%s - Updating articles' % datetime.now().isoformat()) + op.execute("""UPDATE article + SET cluster_id = (SELECT cluster.id FROM cluster + WHERE cluster.main_link = article.link + AND article.user_id = cluster.user_id); + """) + + print('%s - feeding cluster_as_feed' % datetime.now().isoformat()) + op.execute(""" +INSERT INTO cluster_as_feed (cluster_id, feed_id) +SELECT article.cluster_id, article.feed_id + FROM article GROUP BY article.cluster_id, article.feed_id; +""") + + print('%s - feeding cluster_as_category' % datetime.now().isoformat()) + op.execute(""" +INSERT INTO cluster_as_category (cluster_id, category_id) +SELECT article.cluster_id, article.category_id + FROM article GROUP BY article.cluster_id, article.category_id; +""") + + with op.batch_alter_table('article') as batch_op: + batch_op.drop_column('readed') + batch_op.drop_column('like') + + +def downgrade(): + op.add_column('article', + sa.Column('readed', sa.Boolean(), default=False), + sa.Column('like', sa.Boolean(), default=False)) + op.drop_constraint(None, 'article', type_='foreignkey') + with op.batch_alter_table('article') as batch_op: + batch_op.drop_column('cluster_id') + op.drop_table('cluster_as_feed') + op.drop_table('cluster_as_category') + op.drop_table('cluster') diff --git a/src/runserver.py b/src/runserver.py index 67c015e59..04f53e389 100755 --- a/src/runserver.py +++ b/src/runserver.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import calendar from flask import request -from flask.ext.babel import Babel +from flask_babel import Babel from bootstrap import conf, application if conf.ON_HEROKU: @@ -32,7 +32,7 @@ def get_timezone(): with application.app_context(): from web import views application.register_blueprint(views.articles_bp) - application.register_blueprint(views.article_bp) + application.register_blueprint(views.cluster_bp) application.register_blueprint(views.feeds_bp) application.register_blueprint(views.feed_bp) application.register_blueprint(views.categories_bp) diff --git a/src/scripts/probes.py b/src/scripts/probes.py index 988c6af17..c6624ad26 100644 --- a/src/scripts/probes.py +++ b/src/scripts/probes.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 import sys from datetime import datetime, timedelta -from flask.ext.script import Command, Option +from flask_script import Command, Option from web.controllers import FeedController, ArticleController from web.models import User diff --git a/src/tests/api/article_test.py b/src/tests/api/article_test.py index 2d3e6ada3..49dab0fc7 100644 --- a/src/tests/api/article_test.py +++ b/src/tests/api/article_test.py @@ -30,36 +30,7 @@ def test_api_list(self): resp = self._api('get', self.urns, data={'limit': 200}, user='admin') self.assertEquals(200, resp.status_code) - self.assertEquals(18, len(resp.json())) - - def test_api_update_many(self): - resp = self._api('put', self.urns, user='user1', - data=[[1, {'like': True}], - [2, {'readed': True}]]) - self.assertEquals(['ok', 'ok'], resp.json()) - self.assertEquals(200, resp.status_code) - resp = self._api('get', self.urn, 1, user='user1') - self.assertEquals(200, resp.status_code) - self.assertTrue(resp.json()['like']) - - resp = self._api('get', self.urn, 2, user='user1') - self.assertEquals(200, resp.status_code) - self.assertTrue(resp.json()['readed']) - - resp = self._api('put', self.urns, user='user1', - data=[[1, {'like': False}], - [15, {'readed': True}]]) - self.assertEquals(206, resp.status_code) - self.assertEquals(['ok', 'nok'], resp.json()) - - resp = self._api('put', self.urns, user='user1', - data=[[16, {'readed': True}], - [17, {'readed': True}]]) - self.assertEquals(500, resp.status_code) - self.assertEquals(['nok', 'nok'], resp.json()) - - resp = self._api('get', self.urn, 17, user='user1') - self.assertEquals(404, resp.status_code) + self.assertEquals(36, len(resp.json())) def test_article_challenge_method(self): articles = self._api('get', self.urns, user='user1').json() @@ -76,11 +47,11 @@ def test_article_challenge_method(self): # user2 doesn't know user1 article, will consider them as knew resp = self._api('get', 'articles/challenge', user='user2', data={'ids': [{'id': art['id']} for art in articles]}) - self.assertEquals(9, len(resp.json())) + self.assertEquals(10, len(resp.json())) # fake ids won't be recognised either and considered as new resp = self._api('get', 'articles/challenge', user='user2', data={'ids': [{'entry_id': art['id']} for art in articles]}) - self.assertEquals(9, len(resp.json())) + self.assertEquals(10, len(resp.json())) def test_api_creation(self): resp = self._api('post', self.urn, user='user1', data={'feed_id': 1}) diff --git a/src/tests/api/category_test.py b/src/tests/api/category_test.py index b07f4741d..0f7967c1c 100644 --- a/src/tests/api/category_test.py +++ b/src/tests/api/category_test.py @@ -10,7 +10,7 @@ def test_api_list(self): resp = self._api('get', self.urns, data={'order_by': '-id'}, user='user1') self.assertEquals(200, resp.status_code) - self.assertEquals(2, len(resp.json())) + self.assertEquals(4, len(resp.json())) self.assertTrue(resp.json()[0]['id'] > resp.json()[-1]['id']) resp = self._api('get', self.urns, data={'limit': 1}, user='user1') @@ -19,11 +19,11 @@ def test_api_list(self): resp = self._api('get', self.urns, user='admin') self.assertEquals(200, resp.status_code) - self.assertEquals(4, len(resp.json())) + self.assertEquals(8, len(resp.json())) resp = self._api('get', self.urns, data={'limit': 200}, user='admin') self.assertEquals(200, resp.status_code) - self.assertEquals(4, len(resp.json())) + self.assertEquals(8, len(resp.json())) def test_api_update_many(self): resp = self._api('put', self.urns, user='user1', diff --git a/src/tests/api/cluster_test.py b/src/tests/api/cluster_test.py new file mode 100644 index 000000000..084a66473 --- /dev/null +++ b/src/tests/api/cluster_test.py @@ -0,0 +1,60 @@ +from tests.base import JarrFlaskCommon +from tests.api.common import ApiCommon + + +class ClusterApiTest(JarrFlaskCommon, ApiCommon): + urn = 'cluster' + urns = 'clusters' + + def test_api_list(self): + resp = self._api('get', self.urns, + data={'order_by': '-id'}, + user='user1') + self.assertEquals(200, resp.status_code) + self.assertEquals(9, len(resp.json())) + self.assertTrue(resp.json()[0]['id'] > resp.json()[-1]['id']) + + resp = self._api('get', self.urns, user='user1') + self.assertEquals(200, resp.status_code) + self.assertEquals(9, len(resp.json())) + + resp = self._api('get', self.urns, data={'limit': 1}, user='user1') + self.assertEquals(200, resp.status_code) + self.assertEquals(1, len(resp.json())) + + resp = self._api('get', self.urns, user='admin') + self.assertEquals(200, resp.status_code) + self.assertEquals(10, len(resp.json())) + + resp = self._api('get', self.urns, data={'limit': 200}, user='admin') + self.assertEquals(200, resp.status_code) + self.assertEquals(18, len(resp.json())) + + def test_api_update_many(self): + resp = self._api('put', self.urns, user='user1', + data=[[1, {'liked': True}], + [2, {'read': True}]]) + self.assertEquals(['ok', 'ok'], resp.json()) + self.assertEquals(200, resp.status_code) + resp = self._api('get', self.urn, 1, user='user1') + self.assertEquals(200, resp.status_code) + self.assertTrue(resp.json()['liked']) + + resp = self._api('get', self.urn, 2, user='user1') + self.assertEquals(200, resp.status_code) + self.assertTrue(resp.json()['read']) + + resp = self._api('put', self.urns, user='user1', + data=[[1, {'liked': False}], + [15, {'read': True}]]) + self.assertEquals(206, resp.status_code) + self.assertEquals(['ok', 'nok'], resp.json()) + + resp = self._api('put', self.urns, user='user1', + data=[[16, {'read': True}], + [17, {'read': True}]]) + self.assertEquals(500, resp.status_code) + self.assertEquals(['nok', 'nok'], resp.json()) + + resp = self._api('get', self.urn, 17, user='user1') + self.assertEquals(404, resp.status_code) diff --git a/src/tests/api/feed_test.py b/src/tests/api/feed_test.py index e20dc1b53..c6d403275 100644 --- a/src/tests/api/feed_test.py +++ b/src/tests/api/feed_test.py @@ -11,7 +11,7 @@ def test_api_list(self): resp = self._api('get', self.urns, data={'order_by': '-id'}, user='user1') self.assertEquals(200, resp.status_code) - self.assertEquals(3, len(resp.json())) + self.assertEquals(6, len(resp.json())) self.assertTrue(resp.json()[0]['id'] > resp.json()[-1]['id']) resp = self._api('get', self.urns, @@ -25,11 +25,11 @@ def test_api_list(self): resp = self._api('get', self.urns, user='admin') self.assertEquals(200, resp.status_code) - self.assertEquals(6, len(resp.json())) + self.assertEquals(10, len(resp.json())) resp = self._api('get', self.urns, data={'limit': 200}, user='admin') self.assertEquals(200, resp.status_code) - self.assertEquals(6, len(resp.json())) + self.assertEquals(12, len(resp.json())) def test_api_update_many(self): resp = self._api('put', self.urns, user='user1', @@ -73,22 +73,24 @@ def test_feed_list_fetchable(self): self.assertEquals(403, resp.status_code) UserController().update({'login__in': ['admin', 'user1']}, {'is_api': True}) - resp = self._api('get', 'feeds/fetchable', user='user1') - self.assertEquals(3, len(resp.json())) + resp = self._api('get', 'feeds/fetchable', user='user1', + data={'limit': 100}) + self.assertEquals(6, len(resp.json())) self.assertEquals(200, resp.status_code) resp = self._api('get', 'feeds/fetchable', user='user1') self.assertEquals(204, resp.status_code) - resp = self._api('get', 'feeds/fetchable', user='admin') - self.assertEquals(3, len(resp.json())) + resp = self._api('get', 'feeds/fetchable', user='admin', + data={'limit': 100}) + self.assertEquals(6, len(resp.json())) self.assertEquals(200, resp.status_code) resp = self._api('get', 'feeds/fetchable', user='admin') self.assertEquals(204, resp.status_code) resp = self._api('get', 'feeds/fetchable', user='user1', data={'refresh_rate': 0}) - self.assertEquals(3, len(resp.json())) + self.assertEquals(5, len(resp.json())) resp = self._api('get', 'feeds/fetchable', user='admin', data={'refresh_rate': 0}) self.assertEquals(5, len(resp.json())) diff --git a/src/tests/base.py b/src/tests/base.py index 75a04e5ba..65b87e444 100644 --- a/src/tests/base.py +++ b/src/tests/base.py @@ -8,7 +8,7 @@ from base64 import b64encode from runserver import application from tests.fixtures.filler import populate_db, reset_db -from flask.ext.login import login_user, logout_user +from flask_login import login_user, logout_user from werkzeug.exceptions import NotFound from bootstrap import conf diff --git a/src/tests/controllers/article_test.py b/src/tests/controllers/article_test.py index a36b96916..17f330808 100644 --- a/src/tests/controllers/article_test.py +++ b/src/tests/controllers/article_test.py @@ -1,5 +1,6 @@ from tests.base import BaseJarrTest -from web.controllers import UserController, ArticleController, FeedController +from web.controllers import (UserController, ArticleController, FeedController, + ClusterController) class ArticleControllerTest(BaseJarrTest): @@ -10,97 +11,97 @@ def test_article_rights(self): self._test_controller_rights(article, UserController().get(id=article['user_id'])) - def test_article_get_unread(self): - self.assertEquals({1: 3, 2: 3, 3: 3}, - ArticleController(2).count_by_feed(readed=False)) - self.assertEquals({4: 3, 5: 3, 6: 3}, - ArticleController(3).count_by_feed(readed=False)) - def test_create_using_filters(self): feed_ctr = FeedController(2) feed1 = feed_ctr.read()[0].dump() feed2 = feed_ctr.read()[1].dump() feed3 = feed_ctr.read()[2].dump() - feed_ctr.update({'id': feed1['id']}, - {'filters': [{"type": "simple match", - "pattern": "no see pattern", - "action on": "match", - "action": "mark as read"}]}) feed_ctr.update({'id': feed3['id']}, {'filters': [{"type": "regex", "pattern": ".*(pattern1|pattern2).*", "action on": "no match", "action": "mark as favorite"}, {"type": "simple match", - "pattern": "no see pattern", + "pattern": "pattern3", "action on": "match", "action": "mark as read"}]}) + feed_ctr.update({'id': feed1['id']}, + {'filters': [{"type": "simple match", + "pattern": "pattern3", + "action on": "match", + "action": "mark as read"}]}) + art1 = ArticleController(2).create( - entry_id="thisisnotatest", + entry_id="will be read and faved 1", feed_id=feed1['id'], - title="garbage no see pattern garbage", + title="garbage pattern1 pattern3 garbage", content="doesn't matter", - link="doesn't matter either") + link="cluster1") + self.assertTrue(art1.cluster.read) + self.assertFalse(art1.cluster.liked) + art2 = ArticleController(2).create( - entry_id="thisisnotatesteither", + entry_id="will be ignored 2", feed_id=feed1['id'], title="garbage see pattern garbage", content="doesn't matter2", - link="doesn't matter either2") + link="is ignored 2") + self.assertFalse(art2.cluster.read) + self.assertFalse(art2.cluster.liked) art3 = ArticleController(2).create( - entry_id="thisisnotatest", + entry_id="will be read 3", user_id=2, feed_id=feed2['id'], - title="garbage no see pattern garbage", + title="garbage pattern3 garbage", content="doesn't matter", - link="doesn't matter either") + link="doesn't matter either3") + self.assertFalse(art3.cluster.read) + self.assertFalse(art3.cluster.liked) + art4 = ArticleController(2).create( - entry_id="thisisnotatesteither", + entry_id="will be ignored 4", user_id=2, feed_id=feed2['id'], title="garbage see pattern garbage", content="doesn't matter2", - link="doesn't matter either2") + link="doesn't matter either4") + self.assertFalse(art4.cluster.read) + self.assertFalse(art4.cluster.liked) art5 = ArticleController(2).create( - entry_id="thisisnotatest", + entry_id="will be faved 5", feed_id=feed3['id'], - title="garbage pattern1 garbage", + title="garbage anti-attern3 garbage", content="doesn't matter", - link="doesn't matter either") + link="cluster1") + self.assertTrue(art5.cluster.read, + "should be read because it clustered") + self.assertTrue(art5.cluster.liked) + art6 = ArticleController(2).create( - entry_id="thisisnotatesteither", + entry_id="will be faved 6", feed_id=feed3['id'], - title="garbage pattern2 garbage", + title="garbage pattern1 garbage", content="doesn't matter2", - link="doesn't matter either2") + link="doesn't matter 6") + self.assertFalse(art6.cluster.read) + self.assertFalse(art6.cluster.liked) + art7 = ArticleController(2).create( - entry_id="thisisnotatesteither", + entry_id="will be read 7", feed_id=feed3['id'], - title="garbage no see pattern3 garbage", + title="garbage pattern3 garbage", content="doesn't matter3", - link="doesn't matter either3") + link="doesn't matter either7") + self.assertTrue(art7.cluster.read) + self.assertTrue(art7.cluster.liked) + art8 = ArticleController(2).create( - entry_id="thisisnotatesteither", + entry_id="will be ignored", feed_id=feed3['id'], title="garbage pattern4 garbage", content="doesn't matter4", - link="doesn't matter either4") - - self.assertTrue(art1.readed) - self.assertFalse(art1.like) - self.assertFalse(art2.readed) - self.assertFalse(art2.like) - self.assertFalse(art3.readed) - self.assertFalse(art3.like) - self.assertFalse(art4.readed) - self.assertFalse(art4.like) - self.assertFalse(art5.readed) - self.assertFalse(art5.like) - self.assertFalse(art6.readed) - self.assertFalse(art6.like) - self.assertTrue(art7.readed) - self.assertTrue(art7.like) - self.assertFalse(art8.readed) - self.assertTrue(art8.like) + link="doesn't matter either8") + self.assertFalse(art8.cluster.read) + self.assertTrue(art8.cluster.liked) diff --git a/src/tests/controllers/cluster_test.py b/src/tests/controllers/cluster_test.py new file mode 100644 index 000000000..2bdd66f0a --- /dev/null +++ b/src/tests/controllers/cluster_test.py @@ -0,0 +1,120 @@ +from random import randint +from datetime import timedelta +from tests.base import BaseJarrTest +from web.controllers import (ArticleController, ClusterController, + FeedController, CategoryController) + + +class ClusterControllerTest(BaseJarrTest): + _contr_cls = ClusterController + + def test_article_get_unread(self): + self.assertEquals({1: 3, 2: 3, 3: 3, 7: 3, 8: 3, 9: 3}, + ClusterController(2).count_by_feed(read=False)) + self.assertEquals({4: 3, 5: 3, 6: 3, 10: 3, 11: 3, 12: 3}, + ClusterController(3).count_by_feed(read=False)) + + def test_adding_to_cluster_by_link(self): + acontr = ArticleController() + ccontr = ClusterController() + cluster = ccontr.get(id=6) + ccontr.update({'id': 6}, {'read': True}) + article = cluster.articles[0] + articles_count = len(cluster.articles) + feed = FeedController(cluster.user_id).read( + user_id=article.user_id, + id__ne=article.feed_id).first() + suffix = str(randint(0, 9999)) + acontr.create( + user_id=article.user_id, + feed_id=feed.id, + entry_id=article.entry_id + suffix, + link=article.link, + title=article.title + suffix, + content=article.content + suffix, + date=article.date + timedelta(1), + retrieved_date=article.retrieved_date + timedelta(1), + ) + cluster = ccontr.get(id=6) + self.assertEquals(articles_count + 1, len(cluster.articles)) + self.assertTrue(cluster.read) + + def test_adding_to_cluster_by_title(self): + article = ArticleController().get(category_id=1) + acontr = ArticleController(article.user_id) + ccontr = ClusterController(article.user_id) + cluster = article.cluster + articles_count = len(cluster.articles) + suffix = str(randint(0, 9999)) + feed = FeedController(article.user_id).create(link=suffix, + user_id=article.user_id, category_id=article.category_id, + title=suffix) + + # testing with non activated category + acontr.create( + user_id=article.user_id, + feed_id=feed.id, + entry_id=article.entry_id + suffix, + link=article.link + suffix, + title=article.title, + content=article.content + suffix, + date=article.date, + retrieved_date=article.retrieved_date, + ) + cluster = ccontr.get(id=cluster.id) + self.assertEquals(articles_count, len(cluster.articles)) + + # testing with activated category + CategoryController().update({'id': article.category_id}, + {'cluster_on_title': True}) + acontr.create( + user_id=article.user_id, + feed_id=feed.id, + category_id=article.category_id, + entry_id=article.entry_id + suffix, + link=article.link + suffix + suffix, + title=article.title, + content=article.content + suffix, + date=article.date, + retrieved_date=article.retrieved_date, + ) + cluster = ccontr.get(id=cluster.id) + self.assertEquals(articles_count + 1, len(cluster.articles)) + + def test_no_mixup(self): + acontr = ArticleController() + ccontr = ClusterController() + total_clusters = len(list(ccontr.read())) + total_articles = len(list(acontr.read())) + for cluster in ccontr.read(): + self.assertEquals(2, len(cluster.articles)) + + for article in acontr.read(): + acontr.create( + entry_id=article.entry_id, + feed_id=article.feed_id, + title=article.title, + content=article.content, + link=article.link) + + self.assertEquals(2 * total_articles, len(list(acontr.read()))) + self.assertEquals(total_clusters, len(list(ccontr.read()))) + + for cluster in ccontr.read(): + self.assertEquals(4, len(cluster.articles)) + self.assertEquals(1, + len(set([a.user_id for a in cluster.articles]))) + + main_article = acontr.read().first() + for article in acontr.read(): + acontr.create( + user_id=main_article.user_id, + feed_id=main_article.feed_id, + entry_id=article.entry_id, + title=article.title, + content=article.content, + link=article.link) + + for cluster in ccontr.read(): + self.assertEquals(1, + len(set([a.user_id for a in cluster.articles]))) diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index a83663238..1e6fc09fe 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -78,12 +78,12 @@ def _reset_feeds_freshness(self, **kwargs): def test_http_crawler_add_articles(self): scheduler = CrawlerScheduler('admin', 'admin') resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(18, len(resp.json())) + self.assertEquals(36, len(resp.json())) scheduler.run() scheduler.wait() resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(143, len(resp.json())) + self.assertEquals(161, len(resp.json())) for art in resp.json(): self.assertFalse('srcset=' in art['content']) @@ -93,30 +93,30 @@ def test_http_crawler_add_articles(self): scheduler.run() scheduler.wait() resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(143, len(resp.json())) + self.assertEquals(161, len(resp.json())) def test_no_add_on_304(self): scheduler = CrawlerScheduler('admin', 'admin') self.resp_status_code = 304 resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(18, len(resp.json())) + self.assertEquals(36, len(resp.json())) scheduler.run() scheduler.wait() resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(18, len(resp.json())) + self.assertEquals(36, len(resp.json())) def test_matching_etag(self): self._reset_feeds_freshness(etag='fake etag') self.resp_headers = {'etag': 'fake etag'} resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(18, len(resp.json())) + self.assertEquals(36, len(resp.json())) scheduler = CrawlerScheduler('admin', 'admin') scheduler.run() scheduler.wait() resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(18, len(resp.json())) + self.assertEquals(36, len(resp.json())) self._reset_feeds_freshness(etag='jarr/fake etag') self.resp_headers = {'etag': 'jarr/fake etag'} @@ -124,7 +124,7 @@ def test_matching_etag(self): scheduler.run() scheduler.wait() resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(18, len(resp.json())) + self.assertEquals(36, len(resp.json())) self._reset_feeds_freshness(etag='jarr/fake etag') self.resp_headers = {'etag': '########################'} @@ -132,4 +132,4 @@ def test_matching_etag(self): scheduler.run() scheduler.wait() resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') - self.assertEquals(143, len(resp.json())) + self.assertEquals(161, len(resp.json())) diff --git a/src/tests/fixtures/filler.py b/src/tests/fixtures/filler.py index bd9152a8a..31533f19e 100644 --- a/src/tests/fixtures/filler.py +++ b/src/tests/fixtures/filler.py @@ -14,23 +14,35 @@ def populate_db(): password=name) for name in ["user1", "user2"]] - article_total = 0 - for user in (user1, user2): - for i in range(3): - cat_id = None - if i: - cat_id = ccontr.create(user_id=user.id, - name="category%d" % i).id - feed = fcontr.create(link="feed%d" % i, user_id=user.id, + def to_name(u, c=None, f=None, a=None): + string = u.login + if c: + string += " cat%s" % c + if f is not None: + string += " feed%s" % f + if a is not None: + string += " art%s" % a + return string + + for _ in range(2): + article_total = 0 + for user in (user1, user2): + for i in range(3): + cat_id = None + if i: + cat_id = ccontr.create(user_id=user.id, + name=to_name(user, i)).id + feed = fcontr.create(link="feed%d" % i, user_id=user.id, category_id=cat_id, - title="%s feed%d" % (user.login, i)) - for j in range(3): - entry = "%s %s article%d" % (user.login, feed.title, j) - article_total += 1 - acontr.create(entry_id=entry, - link='http://test.te/%d' % article_total, - feed_id=feed.id, user_id=user.id, category_id=cat_id, - title=entry, content="content %d" % article_total) + title=to_name(user, i, i)) + for j in range(3): + entry = to_name(user, i, i, j) + article_total += 1 + acontr.create(entry_id=entry, + link='http://test.te/%d' % article_total, + feed_id=feed.id, user_id=user.id, + category_id=cat_id, title=entry, + content="content %d" % article_total) def reset_db(): db_empty() diff --git a/src/tests/ui_test.py b/src/tests/ui_test.py index 6c6038e21..fe0ffe899 100644 --- a/src/tests/ui_test.py +++ b/src/tests/ui_test.py @@ -19,11 +19,13 @@ def test_middle_panel(self): resp = self.app.get('/middle_panel') self.assertEquals(200, resp.status_code) self.assertEquals(9, - len(json.loads(resp.data.decode('utf8'))['articles'])) + len(json.loads(resp.data.decode('utf8'))['clusters'])) resp = self.app.get('/middle_panel?filter=unread') self.assertEquals(200, resp.status_code) self.assertEquals(9, - len(json.loads(resp.data.decode('utf8'))['articles'])) + len(json.loads(resp.data.decode('utf8'))['clusters'])) + + def test_search(self): resp = self.app.get('/middle_panel?query=test') self.assertEquals(200, resp.status_code) resp = self.app.get('/middle_panel?query=test&search_title=true') @@ -33,27 +35,45 @@ def test_middle_panel(self): resp = self.app.get('/middle_panel?query=test' '&search_title=true&search_content=true') self.assertEquals(200, resp.status_code) - resp = self.app.get('/middle_panel?filed_type=feed&filter_id=1') - self.assertEquals(200, resp.status_code) - resp = self.app.get('/middle_panel?filter_type=category&filter_id=0') + + def test_middle_panel_filtered_on_category(self): + cat_id = 1 + resp = self.app.get( + '/middle_panel?filter_type=category_id&filter_id=%d' % cat_id) self.assertEquals(200, resp.status_code) + clusters = json.loads(resp.data.decode('utf8'))['clusters'] + for cluster in clusters: + self.assertTrue(cat_id in cluster['categories_id'], + "%d not in %r" % (cat_id, cluster['categories_id'])) + self.assertEquals(3, len(clusters)) + + def test_middle_panel_filtered_on_feed(self): + feed_id = 3 + resp = self.app.get( + '/middle_panel?filter_type=feed_id&filter_id=%d' % feed_id) + clusters = json.loads(resp.data.decode('utf8'))['clusters'] + for cluster in clusters: + self.assertTrue(feed_id in cluster['feeds_id'], + "%d not in %r" % (feed_id, cluster['feeds_id'])) + self.assertEquals(3, len(clusters)) + self.assertEquals(200, resp.status_code) # marking all as read - # marking all as read + def test_mark_all_as_read(self): resp = self.app.put('/mark_all_as_read', data='{}', headers={'Content-Type': 'application/json'}) self.assertEquals(200, resp.status_code) resp = self.app.get('/middle_panel?filter=unread') self.assertEquals(200, resp.status_code) self.assertEquals(0, - len(json.loads(resp.data.decode('utf8'))['articles'])) + len(json.loads(resp.data.decode('utf8'))['clusters'])) - def test_getart(self): - resp = self.app.get('/getart/1', + def test_getclu(self): + resp = self.app.get('/getclu/1', headers={'Content-Type': 'application/json'}) self.assertEquals(200, resp.status_code) self.app.get('/logout') self.app.post('/login', data={'login': 'user2', 'password': 'user2'}) - resp = self.app.get('/getart/1', + resp = self.app.get('/getclu/1', headers={'Content-Type': 'application/json'}) self.assertEquals(404, resp.status_code) diff --git a/src/web/controllers/__init__.py b/src/web/controllers/__init__.py index a1b89ea8d..cda7f37bd 100644 --- a/src/web/controllers/__init__.py +++ b/src/web/controllers/__init__.py @@ -3,7 +3,8 @@ from .article import ArticleController from .user import UserController from .icon import IconController +from .cluster import ClusterController __all__ = ['FeedController', 'CategoryController', 'ArticleController', - 'UserController', 'IconController'] + 'UserController', 'IconController', 'ClusterController'] diff --git a/src/web/controllers/abstract.py b/src/web/controllers/abstract.py index 57194ee13..d4579717e 100644 --- a/src/web/controllers/abstract.py +++ b/src/web/controllers/abstract.py @@ -3,7 +3,7 @@ from bootstrap import db from datetime import datetime from collections import defaultdict -from sqlalchemy import and_, or_, func +from sqlalchemy import and_, or_ from werkzeug.exceptions import Forbidden, NotFound logger = logging.getLogger(__name__) @@ -25,7 +25,8 @@ def __init__(self, user_id=None, ignore_context=False): except TypeError: self.user_id = user_id - def _to_filters(self, **filters): + @classmethod + def _to_filters(cls, **filters): """ Will translate filters to sqlalchemy filter. This method will also apply user_id restriction if available. @@ -37,26 +38,26 @@ def _to_filters(self, **filters): db_filters = set() for key, value in filters.items(): if key == '__or__': - db_filters.add(or_(*[and_(*self._to_filters(**sub_filter)) + db_filters.add(or_(*[and_(*cls._to_filters(**sub_filter)) for sub_filter in value])) elif key.endswith('__gt'): - db_filters.add(getattr(self._db_cls, key[:-4]) > value) + db_filters.add(getattr(cls._db_cls, key[:-4]) > value) elif key.endswith('__lt'): - db_filters.add(getattr(self._db_cls, key[:-4]) < value) + db_filters.add(getattr(cls._db_cls, key[:-4]) < value) elif key.endswith('__ge'): - db_filters.add(getattr(self._db_cls, key[:-4]) >= value) + db_filters.add(getattr(cls._db_cls, key[:-4]) >= value) elif key.endswith('__le'): - db_filters.add(getattr(self._db_cls, key[:-4]) <= value) + db_filters.add(getattr(cls._db_cls, key[:-4]) <= value) elif key.endswith('__ne'): - db_filters.add(getattr(self._db_cls, key[:-4]) != value) + db_filters.add(getattr(cls._db_cls, key[:-4]) != value) elif key.endswith('__in'): - db_filters.add(getattr(self._db_cls, key[:-4]).in_(value)) + db_filters.add(getattr(cls._db_cls, key[:-4]).in_(value)) elif key.endswith('__like'): - db_filters.add(getattr(self._db_cls, key[:-6]).like(value)) + db_filters.add(getattr(cls._db_cls, key[:-6]).like(value)) elif key.endswith('__ilike'): - db_filters.add(getattr(self._db_cls, key[:-7]).ilike(value)) + db_filters.add(getattr(cls._db_cls, key[:-7]).ilike(value)) else: - db_filters.add(getattr(self._db_cls, key) == value) + db_filters.add(getattr(cls._db_cls, key) == value) return db_filters def _get(self, **filters): @@ -98,10 +99,12 @@ def create(self, **attrs): def read(self, **filters): return self._get(**filters) - def update(self, filters, attrs): + def update(self, filters, attrs, return_objs=False): assert attrs, "attributes to update must not be empty" result = self._get(**filters).update(attrs, synchronize_session=False) db.session.commit() + if return_objs: + return self._get(**filters) return result def delete(self, obj_id): @@ -117,26 +120,30 @@ def _has_right_on(self, obj): return self.user_id is None \ or getattr(obj, self._user_id_key, None) == self.user_id - def _count_by(self, elem_to_group_by, filters): - if self.user_id: - filters['user_id'] = self.user_id - return dict(db.session.query(elem_to_group_by, func.count('id')) - .filter(*self._to_filters(**filters)) - .group_by(elem_to_group_by).all()) + @classmethod + def _extra_columns(cls, role, right=None): + return {} @classmethod def _get_attrs_desc(cls, role, right=None): result = defaultdict(dict) if role == 'admin': - columns = cls._db_cls.__table__.columns.keys() + columns = set(cls._db_cls.__table__.columns.keys())\ + .union(cls._db_cls.fields_base_read())\ + .union(cls._db_cls.fields_base_write())\ + .union(cls._db_cls.fields_api_read())\ + .union(cls._db_cls.fields_api_write()) else: assert role in {'base', 'api'}, 'unknown role %r' % role assert right in {'read', 'write'}, \ "right must be 'read' or 'write' with role %r" % role columns = getattr(cls._db_cls, 'fields_%s_%s' % (role, right))() for column in columns: + try: + db_col = getattr(cls._db_cls, column).property.columns[0] + except AttributeError: + continue result[column] = {} - db_col = getattr(cls._db_cls, column).property.columns[0] try: result[column]['type'] = db_col.type.python_type except NotImplementedError: @@ -149,4 +156,5 @@ def _get_attrs_desc(cls, role, right=None): result[column]['type'] = lambda x: dateutil.parser.parse(x) elif db_col.default: result[column]['default'] = db_col.default.arg + result.update(cls._extra_columns(role, right)) return result diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index d1c04b31d..61beee1d2 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -23,21 +23,24 @@ def challenge(self, ids): continue yield id_ - def count_by_category(self, **filters): - return self._count_by(Article.category_id, filters) - def count_by_feed(self, **filters): - return self._count_by(Article.feed_id, filters) + if self.user_id: + filters['user_id'] = self.user_id + return dict(db.session.query(Article.feed_id, func.count('id')) + .filter(*self._to_filters(**filters)) + .group_by(Article.feed_id).all()) def count_by_user_id(self, **filters): last_conn_max = datetime.utcnow() - timedelta(days=30) return dict(db.session.query(Article.user_id, func.count(Article.id)) .filter(*self._to_filters(**filters)) - .join(User).filter(User.is_active == True, + .join(User).filter(User.is_active.__eq__(True), User.last_connection >= last_conn_max) .group_by(Article.user_id).all()) def create(self, **attrs): + from web.controllers.cluster import ClusterController + cluster_contr = ClusterController(self.user_id) # handling special denorm for article rights assert 'feed_id' in attrs, "must provide feed_id when creating article" feed = FeedController( @@ -48,6 +51,7 @@ def create(self, **attrs): attrs['user_id'], attrs['category_id'] = feed.user_id, feed.category_id # handling feed's filters + cluster_read, cluster_liked = None, False for filter_ in feed.filters or []: match = False if filter_.get('type') == 'regex': @@ -61,17 +65,19 @@ def create(self, **attrs): continue if filter_.get('action') == 'mark as read': - attrs['readed'] = True + cluster_read = True logger.warn("article %s will be created as read", attrs['link']) elif filter_.get('action') == 'mark as favorite': - attrs['like'] = True + cluster_liked = True logger.warn("article %s will be created as liked", attrs['link']) - return super().create(**attrs) + article = super().create(**attrs) + cluster_contr.clusterize(article, cluster_read, cluster_liked) + return article - def update(self, filters, attrs): + def update(self, filters, attrs, *args, **kwargs): user_id = attrs.get('user_id', self.user_id) if 'feed_id' in attrs: feed = FeedController().get(id=attrs['feed_id']) @@ -82,7 +88,7 @@ def update(self, filters, attrs): cat = CategoryController().get(id=attrs['category_id']) assert self.user_id is None or cat.user_id == user_id, \ "no right on cat %r" % cat.id - return super().update(filters, attrs) + return super().update(filters, attrs, *args, **kwargs) def get_history(self, year=None, month=None): "Sort articles by year and month." @@ -101,8 +107,3 @@ def get_history(self, year=None, month=None): else: articles_counter[article.date.year] += 1 return articles_counter, articles - - def read_light(self, **filters): - return super().read(**filters).with_entities(Article.id, Article.title, - Article.readed, Article.like, Article.feed_id, Article.date, - Article.category_id).order_by(Article.date.desc()) diff --git a/src/web/controllers/category.py b/src/web/controllers/category.py index fef5ca81b..01ec4e3c4 100644 --- a/src/web/controllers/category.py +++ b/src/web/controllers/category.py @@ -5,8 +5,3 @@ class CategoryController(AbstractController): _db_cls = Category - - def delete(self, obj_id): - FeedController(self.user_id).update({'category_id': obj_id}, - {'category_id': None}) - return super().delete(obj_id) diff --git a/src/web/controllers/cluster.py b/src/web/controllers/cluster.py new file mode 100644 index 000000000..d96013a08 --- /dev/null +++ b/src/web/controllers/cluster.py @@ -0,0 +1,183 @@ +import logging +from bootstrap import conf, db + +from sqlalchemy import func, Integer, or_, and_ +from sqlalchemy.sql import select, exists +from sqlalchemy.dialects.postgres import ARRAY +from werkzeug.exceptions import NotFound +from .abstract import AbstractController +from web.models import Cluster, Article +from web.models.relationships import cluster_as_feed, cluster_as_category +from web.controllers.article import ArticleController + +logger = logging.getLogger(__name__) + + +class ClusterController(AbstractController): + _db_cls = Cluster + + def _get_cluster_by_link(self, article): + return self.read(user_id=article.user_id, + main_link=article.link).first() + + def _get_cluster_by_title(self, article): + if article.category and article.category.cluster_on_title: + try: + article = ArticleController(self.user_id).get( + user_id=article.user_id, + category_id=article.category_id, + title__ilike=article.title) + except NotFound: + return + return article.cluster + + def _create_from_article(self, article, + cluster_read=None, cluster_liked=False): + cluster = Cluster() + cluster.user_id = article.user_id + cluster.main_link = article.link + cluster.main_date = article.date + cluster.main_feed_title = article.feed.title + cluster.main_title = article.title + cluster.main_article_id = article.id + cluster.read = bool(cluster_read) + cluster.liked = cluster_liked + self._enrich_cluster(cluster, article, cluster_read, cluster_liked) + + def _enrich_cluster(self, cluster, article, + cluster_read=None, cluster_liked=False): + article.cluster = cluster + # a cluster + if cluster_read is not None: + cluster.read = cluster.read and cluster_read + # once one article is liked the cluster is liked + cluster.liked = cluster.liked or cluster_liked + if cluster.main_date > article.date: + cluster.main_title = article.title + cluster.main_date = article.date + cluster.main_feed_title = article.feed.title + cluster.main_article_id = article.id + cluster.feeds.append(article.feed) + if article.category_id: + cluster.categories.append(article.category) + db.session.add(cluster) + db.session.add(article) + db.session.commit() + + def clusterize(self, article, cluster_read=None, cluster_liked=False): + """Will add given article to a fitting cluster or create a cluster + fitting that article.""" + cluster = self._get_cluster_by_link(article) + if not cluster: + cluster = self._get_cluster_by_title(article) + if cluster: + return self._enrich_cluster(cluster, article, + cluster_read, cluster_liked) + return self._create_from_article(article, cluster_read, cluster_liked) + + def join_read(self, feed_id=None, category_id=None, **filters): + art_filters = {} + if self.user_id: + filters['user_id'] = self.user_id + + for key in {'__or__', 'title__ilike', 'content__ilike'}\ + .intersection(filters): + art_filters[key] = filters.pop(key) + + if art_filters: + art_contr = ArticleController(self.user_id) + filters['id__in'] = {line[0] for line in art_contr + .read(**art_filters).with_entities(Article.cluster_id)} + + if not filters['id__in']: + return + + caf_cols = cluster_as_feed.columns + cac_cols = cluster_as_category.columns + fields = {key: getattr(Cluster, key) for key in ('main_title', 'id', + 'liked', 'read', 'main_article_id', 'main_feed_title', + 'main_date', 'main_link')} + sqla_fields = list(fields.values()) + selected_fields = list(fields.values()) + + if 'sqlite' in conf.SQLALCHEMY_DATABASE_URI: + selected_fields.append( + func.group_concat(caf_cols['feed_id']).label('feeds_id')) + if category_id: + selected_fields.append(func.group_concat( + cac_cols['category_id']).label('categories_id')) + else: + selected_fields.append(func.array_agg(caf_cols['feed_id'], + type_=ARRAY(Integer)).label('feeds_id')) + if category_id: + selected_fields.append(func.array_agg(cac_cols['category_id'], + type_=ARRAY(Integer)).label('categories_id')) + + # DESC of what's going on below : + # base query with the above fields and the aggregations + query = db.session.query(*selected_fields) + + # adding parent filter, but we can't just filter on one id, because + # we'll miss all the other parent of the cluster + if feed_id: + cluster_has_feed = exists(select([caf_cols['feed_id']]) + .where(and_(caf_cols['cluster_id'] == Cluster.id, + caf_cols['feed_id'] == feed_id)) + .correlate(Cluster)) + query = query.join(cluster_as_feed, + caf_cols['cluster_id'] == Cluster.id)\ + .filter(cluster_has_feed) + else: + query = query.join(cluster_as_feed, + caf_cols['cluster_id'] == Cluster.id) + if category_id: + # joining only if filtering on categories to lighten the query + # as every article doesn't obligatorily have a category > outerjoin + cluster_has_category = exists(select([cac_cols['category_id']]) + .where(and_(cac_cols['cluster_id'] == Cluster.id, + cac_cols['category_id'] == category_id)) + .correlate(Cluster)) + query = query.join(cluster_as_category, + cac_cols['cluster_id'] == Cluster.id)\ + .filter(cluster_has_category) + + # applying common filter (read / liked) + # grouping all the fields so that agreg works on distant ids + query = query.group_by(*sqla_fields)\ + .filter(*self._to_filters(**filters)) + + for clu in query.order_by(Cluster.main_date.desc()).limit(1000): + row = {} + for key in fields: + row[key] = getattr(clu, key) + if 'sqlite' in conf.SQLALCHEMY_DATABASE_URI: + row['feeds_id'] = set(map(int, clu.feeds_id.split(','))) + if category_id and clu.categories_id: + row['categories_id'] = set( + map(int, clu.categories_id.split(','))) + elif category_id: + row['categories_id'] = [0] + else: + row['feeds_id'] = set(clu.feeds_id) + if category_id: + row['categories_id'] = set(clu.categories_id) + yield row + + @classmethod + def _extra_columns(cls, role, right): + return {'articles': {'type': list}} + + def count_by_feed(self, **filters): + return self._count_by(cluster_as_feed.columns['feed_id'], **filters) + + def count_by_category(self, **filters): + return self._count_by(cluster_as_category.columns['category_id'], + **filters) + + def _count_by(self, group_on, **filters): + if self.user_id: + filters['user_id'] = self.user_id + return dict(db.session.query(group_on, func.count('cluster_id')) + .outerjoin(Cluster) + .filter(*self._to_filters(**filters)) + .group_by(group_on).all()) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index 033bfa977..4dc13357b 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -48,7 +48,7 @@ def list_late(self, delta, max_error=conf.FEED_ERROR_MAX, __or__=[{'last_retrieved__lt': feed_last_retrieved}, {'last_retrieved__lt': min_wait, 'id__in': new_art_feed}]) - .join(User).filter(User.is_active == True, + .join(User).filter(User.is_active.__eq__(True), User.last_connection >= last_conn_max) .order_by(Feed.last_retrieved)) if limit: @@ -117,11 +117,11 @@ def create(self, **attrs): self.__clean_feed_fields(attrs) return super().create(**attrs) - def update(self, filters, attrs): + def update(self, filters, attrs, *args, **kwargs): self._ensure_icon(attrs) self.__clean_feed_fields(attrs) if 'category_id' in attrs: for feed in self.read(**filters): self.__get_art_contr().update({'feed_id': feed.id}, {'category_id': attrs['category_id']}) - return super().update(filters, attrs) + return super().update(filters, attrs, *args, **kwargs) diff --git a/src/web/controllers/icon.py b/src/web/controllers/icon.py index 7a4bc9d4d..60b7c2430 100644 --- a/src/web/controllers/icon.py +++ b/src/web/controllers/icon.py @@ -24,7 +24,8 @@ def create(self, **attrs): return super().create(**self._build_from_url(attrs)) def update(self, filters, attrs): - return super().update(filters, self._build_from_url(attrs)) + attrs = self._build_from_url(attrs) + return super().update(filters, attrs, *args, **kwargs) def delete(self, url): obj = self.get(url=url) diff --git a/src/web/controllers/user.py b/src/web/controllers/user.py index 54213987f..e0c2caf9a 100644 --- a/src/web/controllers/user.py +++ b/src/web/controllers/user.py @@ -23,6 +23,6 @@ def create(self, **attrs): self._handle_password(attrs) return super().create(**attrs) - def update(self, filters, attrs): + def update(self, filters, attrs, *args, **kwargs): self._handle_password(attrs) - return super().update(filters, attrs) + return super().update(filters, attrs, *args, **kwargs) diff --git a/src/web/forms.py b/src/web/forms.py index 34ee57221..1071ea07f 100644 --- a/src/web/forms.py +++ b/src/web/forms.py @@ -1,10 +1,10 @@ -from flask.ext.wtf import Form +from flask_wtf import Form from flask import url_for, redirect -from flask.ext.babel import lazy_gettext +from flask_babel import lazy_gettext from werkzeug.exceptions import NotFound from wtforms import TextField, PasswordField, BooleanField, \ SubmitField, SelectField, validators, HiddenField -from flask.ext.wtf.html5 import EmailField +from flask_wtf.html5 import EmailField from web import utils from web.controllers import UserController diff --git a/src/web/js/actions/MenuActions.js b/src/web/js/actions/MenuActions.js index c514d3e66..69ceed89e 100644 --- a/src/web/js/actions/MenuActions.js +++ b/src/web/js/actions/MenuActions.js @@ -9,7 +9,7 @@ var MenuActions = { // PARENT FILTERS reload: function(set_filter, setFilterFunc, id) { jquery.getJSON('/menu', function(payload) { - var old_all_unread_count = MenuStore._datas['all_unread_count']; + var old_all_unread_count = MenuStore.all_unread_count; JarrDispatcher.dispatch({ type: ActionTypes.RELOAD_MENU, feeds: payload.feeds, diff --git a/src/web/js/actions/MiddlePanelActions.js b/src/web/js/actions/MiddlePanelActions.js index efae516a1..cd9107d63 100644 --- a/src/web/js/actions/MiddlePanelActions.js +++ b/src/web/js/actions/MiddlePanelActions.js @@ -23,8 +23,7 @@ var shouldFetch = function(filters) { } var reloadIfNecessaryAndDispatch = function(dispath_payload) { if(shouldFetch(dispath_payload)) { - var filters = MiddlePanelStore.getRequestFilter( - dispath_payload.display_search); + var filters = MiddlePanelStore.getRequestFilter(dispath_payload.display_search); MiddlePanelStore.filter_whitelist.map(function(key) { if(key in dispath_payload) { filters[key] = dispath_payload[key]; @@ -38,7 +37,7 @@ var reloadIfNecessaryAndDispatch = function(dispath_payload) { } jquery.getJSON('/middle_panel', filters, function(payload) { - dispath_payload.articles = payload.articles; + dispath_payload.clusters = payload.clusters; dispath_payload.filters = filters; JarrDispatcher.dispatch(dispath_payload); _last_fetched_with = MiddlePanelStore.getRequestFilter(); @@ -97,38 +96,38 @@ var MiddlePanelActions = { filter: filter, }); }, - changeRead: function(category_id, feed_id, article_id, new_value){ + changeRead: function(cluster_id, new_value){ jquery.ajax({type: 'PUT', contentType: 'application/json', - data: JSON.stringify({readed: new_value}), - url: "api/v2.0/article/" + article_id, - success: function () { + data: JSON.stringify({read: new_value}), + url: "api/v2.0/cluster/" + cluster_id, + success: function (payload) { JarrDispatcher.dispatch({ type: ActionTypes.CHANGE_ATTR, attribute: 'read', value_bool: new_value, value_num: new_value ? -1 : 1, - articles: [{article_id: article_id, - category_id: category_id, - feed_id: feed_id}], + clusters: [{cluster_id: cluster_id, + categories_id: payload.categories_id, + feeds_id: payload.feeds_id}], }); }, }); }, - changeLike: function(category_id, feed_id, article_id, new_value){ + changeLike: function(cluster_id, new_value){ jquery.ajax({type: 'PUT', contentType: 'application/json', - data: JSON.stringify({like: new_value}), - url: "api/v2.0/article/" + article_id, - success: function () { + data: JSON.stringify({liked: new_value}), + url: "api/v2.0/cluster/" + cluster_id, + success: function (payload) { JarrDispatcher.dispatch({ type: ActionTypes.CHANGE_ATTR, attribute: 'liked', value_bool: new_value, value_num: new_value ? -1 : 1, - articles: [{article_id: article_id, - category_id: category_id, - feed_id: feed_id}], + clusters: [{cluster_id: cluster_id, + categories_id: payload.categories_id, + feeds_id: payload.feeds_id}], }); }, }); @@ -142,7 +141,7 @@ var MiddlePanelActions = { success: function (payload) { JarrDispatcher.dispatch({ type: ActionTypes.MARK_ALL_AS_READ, - articles: payload.articles, + clusters: payload.clusters, }); }, }); diff --git a/src/web/js/actions/RightPanelActions.js b/src/web/js/actions/RightPanelActions.js index 5d78e0013..9f3707825 100644 --- a/src/web/js/actions/RightPanelActions.js +++ b/src/web/js/actions/RightPanelActions.js @@ -4,21 +4,38 @@ var ActionTypes = require('../constants/JarrConstants'); var MenuActions = require('../actions/MenuActions'); var RightPanelActions = { - loadArticle: function(article_id, was_read_before, to_parse) { + loadParent: function(parent_type, parent_id) { + JarrDispatcher.dispatch({ + type: ActionTypes.LOAD_PARENT, + filter_type: parent_type, + filter_id: parent_id + }); + }, + loadCluster: function(cluster_id, was_read_before, to_parse, article_id) { var suffix = ''; if(to_parse) { suffix = '/parse'; + if(article_id) { + suffix += '/' + article_id + } } - jquery.getJSON('/getart/' + article_id + suffix, + jquery.getJSON('/getclu/' + cluster_id + suffix, function(payload) { JarrDispatcher.dispatch({ - type: ActionTypes.LOAD_ARTICLE, - article: payload, + type: ActionTypes.LOAD_CLUSTER, + cluster: payload, was_read_before: was_read_before, + article_id: article_id, }); } ); }, + loadArticle: function(article_id) { + JarrDispatcher.dispatch({ + type: ActionTypes.LOAD_ARTICLE, + article_id: article_id, + }); + }, _apiReq: function(meth, id, obj_type, data, success_callback) { var args = {type: meth, contentType: 'application/json', url: "api/v2.0/" + obj_type + "/" + id} diff --git a/src/web/js/components/MainApp.react.js b/src/web/js/components/MainApp.react.js index ffb145890..7944a00b3 100644 --- a/src/web/js/components/MainApp.react.js +++ b/src/web/js/components/MainApp.react.js @@ -1,6 +1,5 @@ var React = require('react'); -var Col = require('react-bootstrap/lib/Col'); -var Grid = require('react-bootstrap/lib/Grid'); +var Grid = require('react-bootstrap').Grid; var JarrNavBar = require('./Navbar.react'); var Menu = require('./Menu.react'); @@ -14,11 +13,7 @@ var MainApp = React.createClass({ - - - - + diff --git a/src/web/js/components/Menu.react.js b/src/web/js/components/Menu.react.js index 7168743d0..a8fde5f09 100644 --- a/src/web/js/components/Menu.react.js +++ b/src/web/js/components/Menu.react.js @@ -1,9 +1,9 @@ var React = require('react'); -var Col = require('react-bootstrap/lib/Col'); -var Badge = require('react-bootstrap/lib/Badge'); -var Button = require('react-bootstrap/lib/Button'); -var ButtonGroup = require('react-bootstrap/lib/ButtonGroup'); -var Glyphicon = require('react-bootstrap/lib/Glyphicon'); +var Col = require('react-bootstrap').Col; +var Badge = require('react-bootstrap').Badge; +var Button = require('react-bootstrap').Button; +var ButtonGroup = require('react-bootstrap').ButtonGroup; +var Glyphicon = require('react-bootstrap').Glyphicon; var MenuStore = require('../stores/MenuStore'); var MenuActions = require('../actions/MenuActions'); @@ -20,21 +20,25 @@ var FeedItem = React.createClass({ render: function() { var icon = null; var badge_unread = null; + // handling icon, and replacement in case of no-icon if(this.props.icon_url){ icon = (); } else { icon = ; } + // handling unread badge if(this.props.unread){ badge_unread = {this.props.unread}; } + // handling it's the selected feed in the menu var classes = "nav-feed"; if(this.props.active) { classes += " bg-primary"; } - if(this.props.error_count >= MenuStore._datas.max_error) { + // handling error count displaying + if(this.props.error_count >= MenuStore.max_error) { classes += " bg-danger"; - } else if(this.props.error_count > MenuStore._datas.error_threshold) { + } else if(this.props.error_count > MenuStore.error_threshold) { classes += " bg-warning"; } var title = {this.props.title}; @@ -43,6 +47,7 @@ var FeedItem = React.createClass({ ); }, + // filtering on said feed handleClick: function() { MiddlePanelActions.setFeedFilter(this.props.feed_id); }, @@ -54,6 +59,7 @@ var Category = React.createClass({ active_id: React.PropTypes.number}, render: function() { var classes = "nav-cat"; + // handling this category being the selected one in the menu if((this.props.active_type == 'category_id' || this.props.category_id == null) && this.props.active_id == this.props.category_id) { @@ -64,12 +70,14 @@ var Category = React.createClass({ ); }, + // filtering on said category handleClick: function(evnt) { // hack to avoid selection when clicking on folding icon if(!evnt.target.classList.contains('glyphicon')) { if(this.props.category_id != null) { MiddlePanelActions.setCategoryFilter(this.props.category_id); } else { + // handling selecting the "all category" item > removing all filters MiddlePanelActions.removeParentFilter(); } } @@ -95,7 +103,7 @@ var CategoryGroup = React.createClass({ } }, render: function() { - // hidden the no category if empty + // hidden the "no / 0 category" if empty if(!this.props.cat_id && !this.props.feeds.length) { return
    ; } @@ -107,7 +115,7 @@ var CategoryGroup = React.createClass({ var feeds = this.props.feeds.filter(function(feed) { if (filter == 'unread' && feed.unread <= 0) { return false; - } else if (filter == 'error' && feed.error_count <= MenuStore._datas.error_threshold) { + } else if (filter == 'error' && feed.error_count <= MenuStore.error_threshold) { return false; } return true; @@ -125,9 +133,11 @@ var CategoryGroup = React.createClass({ var feeds = []; } var unread = null; + // displaying unread count if(this.props.unread) { unread = {this.props.unread}; } + // folding icon on the right of the category var ctrl = ( ); @@ -142,6 +152,7 @@ var CategoryGroup = React.createClass({
); }, + // handling folding toggleFolding: function(evnt) { this.setState({folded: !this.state.folded}); evnt.stopPropagation(); @@ -241,7 +252,7 @@ var Menu = React.createClass({ var feeds = []; var unread = 0; this.state.categories[cat_id].feeds.map(function(feed_id) { - if(this.state.feeds[feed_id].error_count > MenuStore._datas.error_threshold) { + if(this.state.feeds[feed_id].error_count > MenuStore.error_threshold) { feed_in_error = true; } unread += this.state.feeds[feed_id].unread; diff --git a/src/web/js/components/MiddlePanel.react.js b/src/web/js/components/MiddlePanel.react.js index 95003f2f5..60cd473f5 100644 --- a/src/web/js/components/MiddlePanel.react.js +++ b/src/web/js/components/MiddlePanel.react.js @@ -1,10 +1,13 @@ var React = require('react'); -var Row = require('react-bootstrap/lib/Row'); -var Button = require('react-bootstrap/lib/Button'); -var ButtonGroup = require('react-bootstrap/lib/ButtonGroup'); -var Glyphicon = require('react-bootstrap/lib/Glyphicon'); +var Col = require('react-bootstrap').Col; +var Row = require('react-bootstrap').Row; +var Panel = require('react-bootstrap').Panel; +var Button = require('react-bootstrap').Button; +var ButtonGroup = require('react-bootstrap').ButtonGroup; +var Glyphicon = require('react-bootstrap').Glyphicon; +var MenuStore = require('../stores/MenuStore'); var MiddlePanelStore = require('../stores/MiddlePanelStore'); var MiddlePanelActions = require('../actions/MiddlePanelActions'); var RightPanelActions = require('../actions/RightPanelActions'); @@ -12,15 +15,16 @@ var RightPanelActions = require('../actions/RightPanelActions'); var JarrTime = require('./time.react'); var TableLine = React.createClass({ - propTypes: {article_id: React.PropTypes.number.isRequired, + propTypes: {cluster_id: React.PropTypes.number.isRequired, + main_article_id: React.PropTypes.number.isRequired, feed_title: React.PropTypes.string.isRequired, - icon_url: React.PropTypes.string, title: React.PropTypes.string.isRequired, rel_date: React.PropTypes.string.isRequired, date: React.PropTypes.string.isRequired, read: React.PropTypes.bool.isRequired, selected: React.PropTypes.bool.isRequired, liked: React.PropTypes.bool.isRequired, + feeds_id: React.PropTypes.array.isRequired, }, getInitialState: function() { return {read: this.props.read, liked: this.props.liked, @@ -28,16 +32,17 @@ var TableLine = React.createClass({ }, render: function() { var liked = this.state.liked ? 'l' : ''; - var icon = null; - if(this.props.icon_url){ - icon = (); - } else { - icon = ; - } - var title = ( - {icon} {this.props.feed_title} + {this.props.feeds_id.map(function(feed_id) { + var feed = MenuStore.feeds[feed_id]; + if(feed && feed.icon_url) { + return ; + } + return ; + })} + {this.props.feed_title} ); var read = (); @@ -48,10 +53,9 @@ var TableLine = React.createClass({ if(this.props.selected) { clsses += " active"; } - return (
+ return (
{title} - +
{read} {liked} {this.props.title}
); @@ -63,22 +67,20 @@ var TableLine = React.createClass({ }, toogleRead: function(evnt) { this.setState({read: !this.state.read}, function() { - MiddlePanelActions.changeRead(this.props.category_id, - this.props.feed_id, this.props.article_id, this.state.read); + MiddlePanelActions.changeRead(this.props.cluster_id, this.state.read); }.bind(this)); evnt.stopPropagation(); }, toogleLike: function(evnt) { this.setState({liked: !this.state.liked}, function() { - MiddlePanelActions.changeLike(this.props.category_id, - this.props.feed_id, this.props.article_id, this.state.liked); + MiddlePanelActions.changeLike(this.props.cluster_id, this.state.liked); }.bind(this)); evnt.stopPropagation(); }, - loadArticle: function() { + loadCluster: function() { this.setState({selected: true, read: true}, function() { - RightPanelActions.loadArticle( - this.props.article_id, this.props.read); + RightPanelActions.loadCluster( + this.props.cluster_id, this.props.read); }.bind(this)); }, stopPropagation: function(evnt) { @@ -88,9 +90,9 @@ var TableLine = React.createClass({ var MiddlePanelSearchRow = React.createClass({ getInitialState: function() { - return {query: MiddlePanelStore._datas.query, - search_title: MiddlePanelStore._datas.search_title, - search_content: MiddlePanelStore._datas.search_content, + return {query: MiddlePanelStore.query, + search_title: MiddlePanelStore.search_title, + search_content: MiddlePanelStore.search_content, }; }, render: function() { @@ -143,10 +145,49 @@ var MiddlePanelSearchRow = React.createClass({ }, }); +var MiddlePanelParentFilterRow = React.createClass({ + getInitialState: function() { + return {id: MenuStore.active_id, + type: MenuStore.active_type, + }; + }, + render: function() { + var cn; + var img; + var content = "Selected "; + if (this.state.type == 'feed_id') { + var feed = MenuStore.feeds[this.state.id]; + img = ; + content += "Feed: " + feed.title; + } else if (this.state.type == 'category_id') { + content += "Category: " + MenuStore.categories[this.state.id].name; + } else { + cn = "hidden"; + } + return ( + {img} + {content} + ); + }, + showParent: function(evnt) { + RightPanelActions.loadParent(this.state.type, this.state.id); + }, + componentDidMount: function() { + MenuStore.addChangeListener(this._onChange); + }, + componentWillUnmount: function() { + MenuStore.removeChangeListener(this._onChange); + }, + _onChange: function() { + this.setState({id: MenuStore.active_id, + type: MenuStore.active_type}); + }, +}); + var MiddlePanelFilter = React.createClass({ getInitialState: function() { - return {filter: MiddlePanelStore._datas.filter, - display_search: MiddlePanelStore._datas.display_search}; + return {filter: MiddlePanelStore.filter, + display_search: MiddlePanelStore.display_search}; }, render: function() { var search_row = null; @@ -157,18 +198,18 @@ var MiddlePanelFilter = React.createClass({ @@ -219,31 +260,30 @@ var MiddlePanelFilter = React.createClass({ }, }); -var MiddlePanel = React.createClass({ +var ClusterList = React.createClass({ getInitialState: function() { - return {filter: MiddlePanelStore._datas.filter, articles: []}; + return {filter: MiddlePanelStore.filter, clusters: []}; }, render: function() { return (
- {this.state.articles.map(function(article){ - var key = "a" + article.article_id; - if(article.read) {key+="r";} - if(article.liked) {key+="l";} - if(article.selected) {key+="s";} + {this.state.clusters.map(function(cluster){ + var key = "clu" + cluster.id; + if(cluster.read) {key+="r";} + if(cluster.liked) {key+="l";} + if(cluster.selected) {key+="s";} return ();})} + feed_title={cluster.main_feed_title} />);})}
); @@ -256,10 +296,21 @@ var MiddlePanel = React.createClass({ MiddlePanelStore.removeChangeListener(this._onChange); }, _onChange: function() { - this.setState({filter: MiddlePanelStore._datas.filter, - articles: MiddlePanelStore.getArticles()}); + this.setState({filter: MiddlePanelStore.filter, + clusters: MiddlePanelStore.getClusters()}); + }, +}); + +var MiddlePanel = React.createClass({ + render: function() { + return ( + + + + + ); }, }); -module.exports = {MiddlePanel: MiddlePanel, - MiddlePanelFilter: MiddlePanelFilter}; +module.exports = MiddlePanel; diff --git a/src/web/js/components/Navbar.react.js b/src/web/js/components/Navbar.react.js index dcd57bf84..b295e274f 100644 --- a/src/web/js/components/Navbar.react.js +++ b/src/web/js/components/Navbar.react.js @@ -1,20 +1,20 @@ var React = require('react'); -var Glyphicon = require('react-bootstrap/lib/Glyphicon'); -var Nav = require('react-bootstrap/lib/Nav'); -var NavItem = require('react-bootstrap/lib/NavItem'); -var Navbar = require('react-bootstrap/lib/Navbar'); -var NavDropdown = require('react-bootstrap/lib/NavDropdown'); -var MenuItem = require('react-bootstrap/lib/MenuItem'); -var Modal = require('react-bootstrap/lib/Modal'); -var Button = require('react-bootstrap/lib/Button'); -var Input = require('react-bootstrap/lib/Input'); +var Glyphicon = require('react-bootstrap').Glyphicon; +var Nav = require('react-bootstrap').Nav; +var NavItem = require('react-bootstrap').NavItem; +var Navbar = require('react-bootstrap').Navbar; +var NavDropdown = require('react-bootstrap').NavDropdown; +var MenuItem = require('react-bootstrap').MenuItem; +var Modal = require('react-bootstrap').Modal; +var Button = require('react-bootstrap').Button; +var Input = require('react-bootstrap').Input; var MenuStore = require('../stores/MenuStore'); JarrNavBar = React.createClass({ getInitialState: function() { - return {is_admin: MenuStore._datas.is_admin, - crawling_method: MenuStore._datas.crawling_method, + return {is_admin: MenuStore.is_admin, + crawling_method: MenuStore.crawling_method, showModal: false, modalType: null}; }, buttonFetch: function() { diff --git a/src/web/js/components/RightPanel.react.js b/src/web/js/components/RightPanel.react.js index 5dee0fefa..42a526e03 100644 --- a/src/web/js/components/RightPanel.react.js +++ b/src/web/js/components/RightPanel.react.js @@ -1,9 +1,11 @@ var React = require('react'); -var Col = require('react-bootstrap/lib/Col'); -var Glyphicon = require('react-bootstrap/lib/Glyphicon'); -var Button = require('react-bootstrap/lib/Button'); -var ButtonGroup = require('react-bootstrap/lib/ButtonGroup'); -var Modal = require('react-bootstrap/lib/Modal'); +var Col = require('react-bootstrap').Col; +var Nav = require('react-bootstrap').Nav; +var Modal = require('react-bootstrap').Modal; +var Button = require('react-bootstrap').Button; +var NavItem = require('react-bootstrap').NavItem; +var Glyphicon = require('react-bootstrap').Glyphicon; +var ButtonGroup = require('react-bootstrap').ButtonGroup; var RightPanelActions = require('../actions/RightPanelActions'); var RightPanelStore = require('../stores/RightPanelStore'); @@ -201,7 +203,8 @@ var Article = React.createClass({ reloadParsed: function() { if(this.props.obj.readability_available && !this.props.obj.readability_parsed) { - RightPanelActions.loadArticle(this.props.obj.id, true, true); + RightPanelActions.loadCluster(this.props.obj.cluster_id, + true, true, this.props.obj.id); } }, }); @@ -276,7 +279,10 @@ var Feed = React.createClass({ rows.push(
Filters
); for(var i in this.state.obj.filters) { rows.push(
- When {this.state.obj.filters[i]['action on']} on "{this.state.obj.filters[i].pattern}" ({this.state.obj.filters[i].type}) => {this.state.obj.filters[i].action} + When {this.state.obj.filters[i]['action on']} + on "{this.state.obj.filters[i].pattern}" + ({this.state.obj.filters[i].type}) + "=" {this.state.obj.filters[i].action}
); } return
{rows}
; @@ -285,12 +291,12 @@ var Feed = React.createClass({ var content = null; if(this.state.edit_mode) { var categ_options = []; - for(var index in MenuStore._datas.categories_order) { - var cat_id = MenuStore._datas.categories_order[index]; + for(var index in MenuStore.categories_order) { + var cat_id = MenuStore.categories_order[index]; categ_options.push( ); } content = (; } else if (this.state.modalType == 'addCategory') { heading = 'Add a new category'; - action = '/category/create'; body = ; } return ( -
+ {heading} @@ -61,8 +61,20 @@ JarrNavBar = React.createClass({
); }, + handleModalChange: function(evnt) { + this.setState({modalValue: evnt.target.value}); + }, + submit: function(evnt) { + if(this.state.modalType == 'addCategory') { + MenuActions.addCategory(this.state.modalValue); + } else { + MenuActions.addFeed(this.state.modalValue); + } + evnt.preventDefault() + this.close(); + }, close: function() { - this.setState({showModal: false, modalType: null}); + this.setState({showModal: false, modalType: null, modalValue: null}); }, openAddFeed: function() { this.setState({showModal: true, modalType: 'addFeed'}); diff --git a/src/web/views/api/common.py b/src/web/views/api/common.py index 0a005b201..4e0cfc8df 100644 --- a/src/web/views/api/common.py +++ b/src/web/views/api/common.py @@ -114,7 +114,10 @@ class PyAggResourceNew(PyAggAbstractResource): @api_permission.require(http_exception=403) def post(self): """Create a single new object""" - return self.controller.create(**self.reqparse_args(right='write')), 201 + attrs = self.reqparse_args(right='write') + if not attrs.get('user_id'): + attrs['user_id'] = current_user.id + return self.controller.create(**attrs), 201 class PyAggResourceExisting(PyAggAbstractResource): @@ -178,6 +181,8 @@ class Proxy: try: Proxy.json = attrs args = self.reqparse_args('write', req=Proxy, default=False) + if not args.get('user_id'): + args['user_id'] = current_user.id obj = self.controller.create(**args) results.append(obj) except Exception as error: From 13254e2b6e45e24b989878bf7a73e0d0d499da2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 2 Oct 2016 22:26:06 +0200 Subject: [PATCH 012/164] removing dead code (close #50) --- src/crawler/classic_crawler.py | 154 -------------------------- src/manager.py | 35 +----- src/web/controllers/feed.py | 17 --- src/web/js/components/Navbar.react.js | 8 -- src/web/templates/layout.html | 3 - src/web/utils.py | 102 ----------------- src/web/views/feed.py | 17 --- src/web/views/home.py | 18 --- 8 files changed, 2 insertions(+), 352 deletions(-) delete mode 100644 src/crawler/classic_crawler.py diff --git a/src/crawler/classic_crawler.py b/src/crawler/classic_crawler.py deleted file mode 100644 index 569401ea6..000000000 --- a/src/crawler/classic_crawler.py +++ /dev/null @@ -1,154 +0,0 @@ -import ssl -import asyncio -import logging -import feedparser -import dateutil.parser -from bootstrap import conf -from datetime import datetime -from sqlalchemy import or_ - -from bootstrap import db -from web.models import User -from web.controllers import FeedController, ArticleController -from web.lib.feed_utils import construct_feed_from, is_parsing_ok -from crawler.lib.article_utils import construct_article, extract_id, \ - get_article_content - -logger = logging.getLogger(__name__) - -sem = asyncio.Semaphore(5) - -try: - _create_unverified_https_context = ssl._create_unverified_context -except AttributeError: - # Legacy Python that doesn't verify HTTPS certificates by default - pass -else: - # Handle target environment that doesn't support HTTPS verification - ssl._create_default_https_context = _create_unverified_https_context - - -async def get(*args, **kwargs): - # kwargs["connector"] = aiohttp.TCPConnector(verify_ssl=False) - try: - data = feedparser.parse(args[0]) - return data - except Exception as e: - raise e - - -async def parse_feed(user, feed): - """ - Fetch a feed. - Update the feed and return the articles. - """ - parsed_feed = None - up_feed = {} - articles = [] - with (await sem): - try: - parsed_feed = await get(feed.link) - except Exception as e: - up_feed['last_error'] = str(e) - up_feed['error_count'] = feed.error_count + 1 - finally: - up_feed['last_retrieved'] = datetime.utcnow() - if parsed_feed is None: - FeedController().update({'id': feed.id}, up_feed) - return - - if not is_parsing_ok(parsed_feed): - up_feed['last_error'] = str(parsed_feed['bozo_exception']) - up_feed['error_count'] = feed.error_count + 1 - FeedController().update({'id': feed.id}, up_feed) - return - if parsed_feed['entries'] != []: - articles = parsed_feed['entries'] - - up_feed['error_count'] = 0 - up_feed['last_error'] = "" - - # Feed informations - construct_feed_from(feed.link, parsed_feed).update(up_feed) - if feed.title and 'title' in up_feed: - # do not override the title set by the user - del up_feed['title'] - FeedController().update({'id': feed.id}, up_feed) - - return articles - - -async def insert_database(user, feed): - - articles = await parse_feed(user, feed) - if None is articles: - return [] - - logger.debug('inserting articles for {}'.format(feed.title)) - - logger.info("Database insertion...") - new_articles = [] - art_contr = ArticleController(user.id) - for article in articles: - existing_article_req = art_contr.read(feed_id=feed.id, - **extract_id(article)) - exist = existing_article_req.count() != 0 - if exist: - existing_article = existing_article_req.first() - is_updated = False - logger.debug("Article %r (%r) already in the database.", - article['title'], article['link']) - content = get_article_content(article) - if existing_article.title != article['title']: - existing_article.title = article['title'] - is_updated = True - if existing_article.content != content: - existing_article.content = content - is_updated = True - if is_updated: - art_contr.update({'entry_id': existing_article.entry_id}, - existing_article.dump()) - continue - article = construct_article(article, feed.dump()) - try: - new_articles.append(art_contr.create(**article)) - logger.info("New article % (%r) added.", - article['title'], article['link']) - except Exception: - logger.exception("Error when inserting article in database:") - continue - return new_articles - - -async def init_process(user, feed): - # Fetch the feed and insert new articles in the database - articles = await insert_database(user, feed) - logger.debug('inserted articles for %s', feed.title) - return articles - - -def retrieve_feed(loop, user, feed_id=None): - """ - Launch the processus. - """ - logger.info("Starting to retrieve feeds.") - - # Get the list of feeds to fetch - user = User.query.filter(User.email == user.email).first() - feeds = [feed for feed in user.feeds if - feed.error_count <= conf.FEED_ERROR_MAX and feed.enabled] - if feed_id is not None: - feeds = [feed for feed in feeds if feed.id == feed_id] - - if feeds == []: - return - - # Launch the process for all the feeds - tasks = [asyncio.ensure_future(init_process(user, feed)) for feed in feeds] - - try: - loop.run_until_complete(asyncio.wait(tasks)) - except Exception: - logger.exception('an error occured') - - logger.info("All articles retrieved. End of the processus.") diff --git a/src/manager.py b/src/manager.py index bb6c96da2..5a332983a 100755 --- a/src/manager.py +++ b/src/manager.py @@ -51,8 +51,8 @@ def reset_feeds(): last_conn_max = now - timedelta(days=30) feeds = list(fcontr.read().join(User).filter(User.is_active == True, - User.last_connection >= last_conn_max)\ - .with_entities(fcontr._db_cls.user_id)\ + User.last_connection >= last_conn_max) + .with_entities(fcontr._db_cls.user_id) .distinct()) step = timedelta(seconds=3600 / fcontr.read().count()) @@ -62,37 +62,6 @@ def reset_feeds(): 'last_retrieved': now - i * step}) -@manager.command -def fetch_asyncio(user_id, feed_id): - "Crawl the feeds with asyncio." - import asyncio - - with application.app_context(): - from flask_login import current_user - from crawler import classic_crawler - ucontr = UserController() - users = [] - try: - users = [ucontr.get(user_id)] - except: - users = ucontr.read() - finally: - if users == []: - users = ucontr.read() - - try: - feed_id = int(feed_id) - except: - feed_id = None - - loop = asyncio.get_event_loop() - for user in users: - if user.is_active: - logger.warn("Fetching articles for " + user.login) - classic_crawler.retrieve_feed(loop, current_user, feed_id) - loop.close() - - manager.add_command('probe_articles', ArticleProbe()) manager.add_command('probe_feeds', FeedProbe()) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index 4dc13357b..364710da9 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -64,23 +64,6 @@ def list_fetchable(self, max_error=conf.FEED_ERROR_MAX, {'last_retrieved': now}) return feeds - def get_duplicates(self, feed_id): - """ - Compare a list of documents by pair. - Pairs of duplicates are sorted by "retrieved date". - """ - feed = self.get(id=feed_id) - duplicates = [] - for pair in itertools.combinations(feed.articles, 2): - date1, date2 = pair[0].date, pair[1].date - if clear_string(pair[0].title) == clear_string(pair[1].title) \ - and (date1 - date2) < timedelta(days=1): - if pair[0].retrieved_date < pair[1].retrieved_date: - duplicates.append((pair[0], pair[1])) - else: - duplicates.append((pair[1], pair[0])) - return feed, duplicates - def get_inactives(self, nb_days): today = datetime.utcnow() inactives = [] diff --git a/src/web/js/components/Navbar.react.js b/src/web/js/components/Navbar.react.js index 11c145014..9fec44174 100644 --- a/src/web/js/components/Navbar.react.js +++ b/src/web/js/components/Navbar.react.js @@ -18,13 +18,6 @@ JarrNavBar = React.createClass({ crawling_method: MenuStore.crawling_method, showModal: false, modalType: null, modalValue: null}; }, - buttonFetch: function() { - if(this.state.is_admin && this.state.crawling_method != 'http') { - return ( - Fetch - ); - } - }, sectionAdmin: function() { if(this.state.is_admin) { return ( @@ -93,7 +86,6 @@ JarrNavBar = React.createClass({
); }, diff --git a/src/web/js/components/Notifications.react.js b/src/web/js/components/Notifications.react.js new file mode 100644 index 000000000..10f188dc6 --- /dev/null +++ b/src/web/js/components/Notifications.react.js @@ -0,0 +1,47 @@ +var React = require('react'); +var NotificationsStore = require('../stores/NotificationsStore'); +var NotificationSystem = require('react-notification-system'); + + +var Notifications = React.createClass({ + _notificationSystem: null, + addNotification: function(notif) { + this._notificationSystem.addNotification({ + message: notif.message, + level: notif.level, + autoDismiss: 30, + onRemove: this.removeNotification, + }); + }, + removeNotification: function(notif) { + for(var idx in NotificationsStore.notifs) { + if(NotificationsStore.notifs[idx].read == false + && NotificationsStore.notifs[idx].level == notif.level + && NotificationsStore.notifs[idx].message == notif.message) { + NotificationsStore.notifs[idx].read = true; + break; + } + + } + }, + render: function() { + return ; + }, + componentDidMount: function() { + this._notificationSystem = this.refs.notificationSystem; + NotificationsStore.addChangeListener(this._onChange); + }, + componentWillUnmount: function() { + NotificationsStore.removeChangeListener(this._onChange); + }, + _onChange: function() { + for(var idx in NotificationsStore.notifs) { + if(!NotificationsStore.notifs[idx].read) { + this.addNotification(NotificationsStore.notifs[idx]); + NotificationsStore.notifs[idx].read = true; + } + } + }, +}); + +module.exports = Notifications; diff --git a/src/web/js/stores/NotificationsStore.js b/src/web/js/stores/NotificationsStore.js new file mode 100644 index 000000000..8f9613745 --- /dev/null +++ b/src/web/js/stores/NotificationsStore.js @@ -0,0 +1,50 @@ +var JarrDispatcher = require('../dispatcher/JarrDispatcher'); +var ActionTypes = require('../constants/JarrConstants'); +var EventEmitter = require('events').EventEmitter; + +var CHANGE_EVENT = 'change_menu'; +var assign = require('object-assign'); + + +var NotificationsStore = assign({}, EventEmitter.prototype, { + notifs: [], + + addNotifications: function(notifications) { + var count = this.notifs.length; + for(var idx in notifications) { + this.notifs.push({ + key: parseInt(idx) + count, + read: false, + level: notifications[idx].level, + message: notifications[idx].message, + }); + } + }, + getNotifications: function() { + this.notifs = this.notifs.filter(function(notif) {return !notif.read;}); + return this.notifs; + }, + emitChange: function(all_folded) { + if (all_folded) { + this.all_folded = all_folded; + } else { + this.all_folded = null; + } + this.emit(CHANGE_EVENT); + }, + addChangeListener: function(callback) { + this.on(CHANGE_EVENT, callback); + }, + removeChangeListener: function(callback) { + this.removeListener(CHANGE_EVENT, callback); + }, +}); + +NotificationsStore.dispatchToken = JarrDispatcher.register(function(action) { + if(action.notifications) { + NotificationsStore.addNotifications(action.notifications); + NotificationsStore.emitChange(); + } +}); + +module.exports = NotificationsStore; diff --git a/src/web/lib/view_utils.py b/src/web/lib/view_utils.py index 676a513c5..642470fce 100644 --- a/src/web/lib/view_utils.py +++ b/src/web/lib/view_utils.py @@ -1,12 +1,14 @@ import pytz from functools import wraps from datetime import datetime -from flask import request, Response, make_response +from flask import request, Response, make_response, get_flashed_messages from flask_babel import get_locale from babel.dates import format_datetime, format_timedelta from web.views.common import jsonify from lib.utils import to_hash +ACCEPTED_LEVELS = {'success', 'info', 'warning', 'error'} + def etag_match(func): @wraps(func) @@ -41,7 +43,14 @@ def _iter_on_rows(rows, now, locale): yield row +def get_notifications(): + for msg in get_flashed_messages(with_categories=True): + yield {'level': msg[0] if msg[0] in ACCEPTED_LEVELS else 'info', + 'message': msg[1]} + + @jsonify def clusters_to_json(clusters): return {'clusters': _iter_on_rows(clusters, - datetime.utcnow(), get_locale())} + datetime.utcnow(), get_locale()), + 'notifications': get_notifications()} diff --git a/src/web/models/article.py b/src/web/models/article.py index 3e4e2fed2..c336ce9c3 100644 --- a/src/web/models/article.py +++ b/src/web/models/article.py @@ -17,12 +17,15 @@ class Article(db.Model, RightMixin): retrieved_date = Column(DateTime, default=datetime.utcnow) readability_parsed = Column(Boolean, default=False) - # relationships - user_id = Column(Integer, ForeignKey('user.id')) - feed_id = Column(Integer, ForeignKey('feed.id')) - category_id = Column(Integer, ForeignKey('category.id')) + # foreign keys + user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE')) + feed_id = Column(Integer, ForeignKey('feed.id', ondelete='CASCADE')) + category_id = Column(Integer, + ForeignKey('category.id', ondelete='CASCADE')) cluster_id = Column(Integer, ForeignKey('cluster.id')) + # relationships + user = relationship('User', back_populates='articles') cluster = relationship('Cluster', back_populates='articles', foreign_keys=[cluster_id]) category = relationship('Category', back_populates='articles', diff --git a/src/web/models/category.py b/src/web/models/category.py index f8c21883d..c9ab0ade0 100644 --- a/src/web/models/category.py +++ b/src/web/models/category.py @@ -10,13 +10,18 @@ class Category(db.Model, RightMixin): name = Column(String) cluster_on_title = Column(Boolean, default=False) + # foreign keys + user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE')) + # relationships - user_id = Column(Integer, ForeignKey('user.id')) - feeds = relationship('Feed', backref='category', + user = relationship('User', back_populates='categories') + feeds = relationship('Feed', back_populates='category', cascade='all,delete-orphan') articles = relationship('Article', back_populates='category', cascade='all,delete-orphan') - clusters = relationship('Article', back_populates='category') + clusters = relationship('Cluster', back_populates='categories', + foreign_keys='[Article.category_id, Article.cluster_id]', + secondary='article') # index idx_category_uid = Index('user_id') diff --git a/src/web/models/cluster.py b/src/web/models/cluster.py index da400cbf0..f531b92e0 100644 --- a/src/web/models/cluster.py +++ b/src/web/models/cluster.py @@ -21,16 +21,22 @@ class Cluster(db.Model, RightMixin): main_title = Column(String) main_link = Column(String, default=None) - # relationship + # foreign keys main_article_id = Column(Integer, ForeignKey('article.id')) - user_id = Column(Integer, ForeignKey('user.id')) + user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE')) + + # relationships + user = relationship('User', back_populates='clusters') + main_article = relationship('Article', uselist=False, + foreign_keys=[main_article_id]) articles = relationship('Article', back_populates='cluster', - foreign_keys=[Article.cluster_id], - cascade='all,delete-orphan', - order_by=Article.date.asc()) - feeds = relationship('Article', back_populates='cluster', - foreign_keys=[Article.feed_id, Article.cluster_id]) - categories = relationship('Article', back_populates='cluster', + foreign_keys=[Article.cluster_id], + order_by=Article.date.asc()) + feeds = relationship('Feed', back_populates='clusters', + secondary='article', + foreign_keys=[Article.feed_id, Article.cluster_id]) + categories = relationship('Category', back_populates='clusters', + secondary='article', foreign_keys=[Article.cluster_id, Article.category_id]) # index diff --git a/src/web/models/feed.py b/src/web/models/feed.py index 5fcb63438..da47b21a9 100644 --- a/src/web/models/feed.py +++ b/src/web/models/feed.py @@ -29,13 +29,20 @@ class Feed(db.Model, RightMixin): last_error = Column(String, default="") error_count = Column(Integer, default=0) - # relationships + # foreign keys icon_url = Column(String, ForeignKey('icon.url'), default=None) - user_id = Column(Integer, ForeignKey('user.id')) - category_id = Column(Integer, ForeignKey('category.id')) + user_id = Column(Integer, ForeignKey('user.id', ondelete='CASCADE')) + category_id = Column(Integer, + ForeignKey('category.id', ondelete='CASCADE')) + + # relationships + user = relationship('User', back_populates='feeds') + category = relationship('Category', back_populates='feeds') articles = relationship('Article', back_populates='feed', cascade='all,delete-orphan') - clusters = relationship('Article', back_populates='feed') + clusters = relationship('Cluster', back_populates='feeds', + foreign_keys='[Article.feed_id, Article.cluster_id]', + secondary='article') # index idx_feed_uid_cid = Index('user_id', 'category_id') diff --git a/src/web/models/user.py b/src/web/models/user.py index 7348ee418..0d567821c 100644 --- a/src/web/models/user.py +++ b/src/web/models/user.py @@ -6,10 +6,6 @@ from bootstrap import db from web.models.right_mixin import RightMixin -from web.models.category import Category -from web.models.feed import Feed -from web.models.article import Article -from web.models.cluster import Cluster class User(db.Model, UserMixin, RightMixin): @@ -37,18 +33,18 @@ class User(db.Model, UserMixin, RightMixin): linuxfr_identity = Column(String) # relationships - categories = relationship('Category', backref='user', + categories = relationship('Category', back_populates='user', cascade='all, delete-orphan', - foreign_keys=[Category.user_id]) - feeds = relationship('Feed', backref='user', + foreign_keys='[Category.user_id]') + feeds = relationship('Feed', back_populates='user', cascade='all, delete-orphan', - foreign_keys=[Feed.user_id]) - articles = relationship('Article', backref='user', + foreign_keys='[Feed.user_id]') + articles = relationship('Article', back_populates='user', cascade='all, delete-orphan', - foreign_keys=[Article.user_id]) - clusters = relationship('Cluster', backref='user', + foreign_keys='[Article.user_id]') + clusters = relationship('Cluster', back_populates='user', cascade='all, delete-orphan', - foreign_keys=[Cluster.user_id]) + foreign_keys='[Cluster.user_id]') # api whitelists @staticmethod diff --git a/src/web/templates/feed_list.html b/src/web/templates/feed_list.html index b0ea6437f..62072fe35 100644 --- a/src/web/templates/feed_list.html +++ b/src/web/templates/feed_list.html @@ -36,7 +36,6 @@

{{_("No feed")}}

- {% endfor %} diff --git a/src/web/views/admin.py b/src/web/views/admin.py index 1103635b2..baa2b3d4b 100644 --- a/src/web/views/admin.py +++ b/src/web/views/admin.py @@ -50,7 +50,7 @@ def user(user_id=None): unread_counts=clu_contr.count_by_feed(read=False)) else: - flash(gettext('This user does not exist.'), 'warn') + flash(gettext('This user does not exist.'), 'warning') return redirect(redirect_url()) @@ -67,7 +67,7 @@ def toggle_user(user_id=None): {'is_active': not user.is_active}) if not user_changed: - flash(gettext('This user does not exist.'), 'danger') + flash(gettext('This user does not exist.'), 'error') return redirect(url_for('admin.dashboard')) else: diff --git a/src/web/views/feed.py b/src/web/views/feed.py index 050250858..ce68ad057 100644 --- a/src/web/views/feed.py +++ b/src/web/views/feed.py @@ -1,5 +1,4 @@ import logging -import requests.exceptions from werkzeug.exceptions import BadRequest from flask import Blueprint, render_template, flash, \ @@ -31,21 +30,37 @@ def feeds(): @feed_bp.route('/bookmarklet', methods=['GET', 'POST']) @login_required def bookmarklet(): + def check_feeds(link, site_link): + filters = [] + if link: + filters.append({'link': link}) + if link: + filters.append({'site_link': site_link}) + filters = {'__or__': filters} if len(filters) > 1 else filters[0] + feed_exists = feed_contr.read(**filters).first() + if feed_exists: + flash(gettext("Didn't add feed: feed already exists."), + "warning") + return feed_exists + feed_contr = FeedController(current_user.id) url = (request.args if request.method == 'GET' else request.form)\ .get('url', None) + if not url: flash(gettext("Couldn't add feed: url missing."), "error") raise BadRequest("url is missing") - feed_exists = list(feed_contr.read(__or__=[{'link': url}, - {'site_link': url}])) - if feed_exists: - flash(gettext("Didn't add feed: feed already exists."), - "warning") - return redirect(url_for('home', at='f', ai=feed_exists[0].id)) + existing_feed = check_feeds(url, url) + if existing_feed: + return redirect(url_for('home', at='f', ai=existing_feed.id)) feed = construct_feed_from(url) + + existing_feed = check_feeds(feed.get('link'), feed.get('site_link')) + if existing_feed: + return redirect(url_for('home', at='f', ai=existing_feed.id)) + if not feed.get('link'): feed['enabled'] = False flash(gettext("Couldn't find a feed url, you'll need to find a Atom or" @@ -59,9 +74,6 @@ def bookmarklet(): @feeds_bp.route('/inactives', methods=['GET']) @login_required def inactives(): - """ - List of inactive feeds. - """ nb_days = int(request.args.get('nb_days', 365)) inactives = FeedController(current_user.id).get_inactives(nb_days) return render_template('inactives.html', diff --git a/src/web/views/home.py b/src/web/views/home.py index 49187fb27..2f2583386 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -9,7 +9,7 @@ from bootstrap import conf from web.lib.article_cleaner import clean_urls -from web.lib.view_utils import etag_match, clusters_to_json +from web.lib.view_utils import etag_match, clusters_to_json, get_notifications from web.views.common import jsonify from web.controllers import (UserController, CategoryController, @@ -71,6 +71,7 @@ def get_menu(): 'max_error': conf.FEED_ERROR_MAX, 'error_threshold': conf.FEED_ERROR_THRESHOLD, 'is_admin': current_user.is_admin, + 'notifications': get_notifications(), 'all_unread_count': 0} @@ -144,7 +145,7 @@ def get_cluster(cluster_id, parse=False, article_id=None): new_content = clean_urls(new_content, article['link'], fix_readability=True) except Exception as error: - flash("Readability failed with %r" % error, "danger") + flash("Readability failed with %r" % error, "warning") article['readability_parsed'] = False else: article['readability_parsed'] = True @@ -153,6 +154,7 @@ def get_cluster(cluster_id, parse=False, article_id=None): {'readability_parsed': True, 'content': new_content}) for article in cluster.articles: article['readability_available'] = readability_available + cluster['notifications'] = get_notifications() return cluster diff --git a/src/web/views/user.py b/src/web/views/user.py index b883ab9a4..dae1222ef 100644 --- a/src/web/views/user.py +++ b/src/web/views/user.py @@ -45,7 +45,7 @@ def opml_import(): try: subscriptions = opml.from_string(data.read()) except: - flash(gettext("Couldn't parse file"), 'danger') + flash(gettext("Couldn't parse file"), 'error') return redirect(request.referrer) ccontr = CategoryController(current_user.id) diff --git a/src/web/views/views.py b/src/web/views/views.py index 7588ea998..3ce3e1804 100644 --- a/src/web/views/views.py +++ b/src/web/views/views.py @@ -21,7 +21,7 @@ def authentication_required(error): def authentication_failed(error): if conf.API_ROOT in request.url: return error - flash(gettext('Forbidden.'), 'danger') + flash(gettext('Forbidden.'), 'error') return redirect(url_for('login')) From 8dbfcfe4e51716a540c4b19348b13df6aec60a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sat, 8 Oct 2016 13:05:49 +0200 Subject: [PATCH 020/164] fixing updating unread count --- src/web/js/actions/MiddlePanelActions.js | 55 +++++++----------------- src/web/js/stores/MenuStore.js | 37 ++++++++++++++-- src/web/js/stores/MiddlePanelStore.js | 2 +- 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/web/js/actions/MiddlePanelActions.js b/src/web/js/actions/MiddlePanelActions.js index cd9107d63..28dffc060 100644 --- a/src/web/js/actions/MiddlePanelActions.js +++ b/src/web/js/actions/MiddlePanelActions.js @@ -3,48 +3,25 @@ var ActionTypes = require('../constants/JarrConstants'); var jquery = require('jquery'); var MiddlePanelStore = require('../stores/MiddlePanelStore'); -var _last_fetched_with = {}; -var shouldFetch = function(filters) { - return true; // FIXME disabling intelligent fetch for now, no caching better that bad one -// if(filters.filter != null // undefined means unchanged -// && (_last_fetched_with.filter != 'all' -// || _last_fetched_with.filter != filters.filter)) { -// return true; -// } -// if(_last_fetched_with.filter_type != null) { -// if(_last_fetched_with.filter_type != filters.filter_type) { -// return true; -// } -// if(_last_fetched_with.filter_id != filters.filter_id) { -// return true; -// } -// } -// return false; -} var reloadIfNecessaryAndDispatch = function(dispath_payload) { - if(shouldFetch(dispath_payload)) { - var filters = MiddlePanelStore.getRequestFilter(dispath_payload.display_search); - MiddlePanelStore.filter_whitelist.map(function(key) { - if(key in dispath_payload) { - filters[key] = dispath_payload[key]; - } - if(filters[key] == null) { - delete filters[key]; - } - }); - if('display_search' in filters) { - delete filters['display_search']; + var filters = MiddlePanelStore.getRequestFilter(dispath_payload.display_search); + MiddlePanelStore.filter_whitelist.map(function(key) { + if(key in dispath_payload) { + filters[key] = dispath_payload[key]; } - jquery.getJSON('/middle_panel', filters, - function(payload) { - dispath_payload.clusters = payload.clusters; - dispath_payload.filters = filters; - JarrDispatcher.dispatch(dispath_payload); - _last_fetched_with = MiddlePanelStore.getRequestFilter(); - }); - } else { - JarrDispatcher.dispatch(dispath_payload); + if(filters[key] == null) { + delete filters[key]; + } + }); + if('display_search' in filters) { + delete filters['display_search']; } + jquery.getJSON('/middle_panel', filters, + function(payload) { + dispath_payload.clusters = payload.clusters; + dispath_payload.filters = filters; + JarrDispatcher.dispatch(dispath_payload); + }); } diff --git a/src/web/js/stores/MenuStore.js b/src/web/js/stores/MenuStore.js index bf41b0fe0..4e6eb6412 100644 --- a/src/web/js/stores/MenuStore.js +++ b/src/web/js/stores/MenuStore.js @@ -13,7 +13,7 @@ var MenuStore = assign({}, EventEmitter.prototype, { active_type: null, active_id: null, is_admin: false, - crawling_method: 'classic', + crawling_method: 'http', all_unread_count: -1, max_error: 0, error_threshold: 0, @@ -76,9 +76,39 @@ MenuStore.dispatchToken = JarrDispatcher.register(function(action) { MenuStore.emitChange(); break; case ActionTypes.PARENT_FILTER: - if(MenuStore.setActive(action.filter_type, action.filter_id)) { - MenuStore.emitChange(); + var changed = MenuStore.setActive(action.filter_type, action.filter_id); + if(action.filters && action.clusters && !action.filters.query + && action.filters.filter == 'unread') { + var new_unread = {}; + action.clusters.map(function(cluster) { + cluster.feeds_id.map(function(feed_id) { + if(!(feed_id in new_unread)) { + new_unread[feed_id] = 0; + } + if(!cluster.read) { + new_unread[feed_id] += 1; + } + }); + }); + for(var feed_id in new_unread) { + var cat_id = MenuStore.feeds[feed_id].category_id; + var old_unread = MenuStore.feeds[feed_id].unread; + if(old_unread > new_unread[feed_id]) { + continue; + } + changed = true; + if((MenuStore.active_type == 'feed_id' + && MenuStore.active_id == feed_id) + || (MenuStore.active_type == 'category_id' + && MenuStore.active_id == cat_id) + || (MenuStore.active_type == null)) { + MenuStore.feeds[feed_id].unread = new_unread[feed_id]; + MenuStore.categories[cat_id].unread -= old_unread; + MenuStore.categories[cat_id].unread += new_unread[feed_id]; + } + } } + if(changed) {MenuStore.emitChange();} break; case ActionTypes.MENU_FILTER: if (MenuStore.setFilter(action.filter)) { @@ -109,7 +139,6 @@ MenuStore.dispatchToken = JarrDispatcher.register(function(action) { MenuStore.readCluster(cluster, -1); } }); - MenuStore.emitChange(); break; default: diff --git a/src/web/js/stores/MiddlePanelStore.js b/src/web/js/stores/MiddlePanelStore.js index ff8661b77..a3472a35b 100644 --- a/src/web/js/stores/MiddlePanelStore.js +++ b/src/web/js/stores/MiddlePanelStore.js @@ -103,7 +103,7 @@ MiddlePanelStore.dispatchToken = JarrDispatcher.register(function(action) { var val = action.value_bool; action.clusters.map(function(cluster) { for (var i in MiddlePanelStore.clusters) { - if(MiddlePanelStore.clusters[i].id == cluster.id) { + if(MiddlePanelStore.clusters[i].id == cluster.cluster_id) { if (MiddlePanelStore.clusters[i][attr] != val) { MiddlePanelStore.clusters[i][attr] = val; // avoiding redraw if not filter, display won't change anyway From c1d94620c859bd69d3b8198ab5287145204b5f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sat, 8 Oct 2016 16:40:35 +0200 Subject: [PATCH 021/164] updating feed title triggers the denorm on cluster --- src/bootstrap.py | 1 + src/tests/controllers/feed_test.py | 22 +++++++++++++ src/web/controllers/article.py | 7 ++-- src/web/controllers/cluster.py | 12 +++---- src/web/controllers/feed.py | 37 ++++++++++++++++++---- src/web/js/actions/MenuActions.js | 3 +- src/web/js/actions/RightPanelActions.js | 31 ++++++++++++++++-- src/web/js/components/MiddlePanel.react.js | 23 ++++++++++---- 8 files changed, 108 insertions(+), 28 deletions(-) diff --git a/src/bootstrap.py b/src/bootstrap.py index eacd89eda..bb2914a0a 100644 --- a/src/bootstrap.py +++ b/src/bootstrap.py @@ -55,6 +55,7 @@ def set_logging(log_path=None, log_level=logging.INFO, modules=(), else: application.debug = conf.LOG_LEVEL <= logging.DEBUG +SQLITE_ENGINE = 'sqlite' in conf.SQLALCHEMY_DATABASE_URI PARSED_PLATFORM_URL = urlparse(conf.PLATFORM_URL) application.config['SERVER_NAME'] = PARSED_PLATFORM_URL.netloc application.config['PREFERRED_URL_SCHEME'] = PARSED_PLATFORM_URL.scheme diff --git a/src/tests/controllers/feed_test.py b/src/tests/controllers/feed_test.py index 01ae24fad..9055e85a9 100644 --- a/src/tests/controllers/feed_test.py +++ b/src/tests/controllers/feed_test.py @@ -19,3 +19,25 @@ def test_feed_rights(self): ArticleController().read(feed_id=feed['id']).count()) self._test_controller_rights(feed, UserController().get(id=feed['user_id'])) + + def test_update_cluster_on_change_title(self): + feed = FeedController(2).read()[0] + for cluster in feed.clusters: + self.assertEquals(feed['title'], cluster['main_feed_title']) + FeedController(2).update({'id': feed.id}, {'title': 'updated title'}) + + feed = FeedController(2).get(id=feed.id) + self.assertEquals('updated title', feed.title) + for cluster in feed.clusters: + self.assertEquals(feed.title, cluster.main_feed_title) + + def test_admin_update_cluster_on_change_title(self): + feed = FeedController(2).read()[0] + for cluster in feed.clusters: + self.assertEquals(feed['title'], cluster['main_feed_title']) + FeedController().update({'id': feed.id}, {'title': 'updated title'}) + + feed = FeedController().get(id=feed.id) + self.assertEquals('updated title', feed.title) + for cluster in feed.clusters: + self.assertEquals(feed.title, cluster.main_feed_title) diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index 56d037e49..20a6c67a2 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -115,14 +115,11 @@ def remove_from_all_clusters(self, article_id): """ from web.controllers import ClusterController clu_ctrl = ClusterController(self.user_id) - # FIXME : update all articles - article = self.get(id=article_id) - cluster = article.cluster + cluster = self.get(id=article_id).cluster if len(cluster.articles) == 1: clu_ctrl.delete(cluster.id) return False - clu_ctrl.update({'id': cluster.id}, - {'main_article_id': cluster.articles[1].id}) + clu_ctrl._enrich_cluster(cluster, cluster.articles[1]) return True def delete(self, obj_id): diff --git a/src/web/controllers/cluster.py b/src/web/controllers/cluster.py index 365bbf128..93e0b7894 100644 --- a/src/web/controllers/cluster.py +++ b/src/web/controllers/cluster.py @@ -1,5 +1,5 @@ import logging -from bootstrap import conf, db +from bootstrap import db, SQLITE_ENGINE from sqlalchemy import func, Integer, and_ from sqlalchemy.orm import aliased @@ -11,7 +11,6 @@ from web.controllers.article import ArticleController logger = logging.getLogger(__name__) -SQLITE_ENGINE = 'sqlite' in conf.SQLALCHEMY_DATABASE_URI class ClusterController(AbstractController): @@ -46,14 +45,15 @@ def _create_from_article(self, article, self._enrich_cluster(cluster, article, cluster_read, cluster_liked) def _enrich_cluster(self, cluster, article, - cluster_read=None, cluster_liked=False): + cluster_read=None, cluster_liked=False, + force_article_as_main=False): article.cluster = cluster # a cluster if cluster_read is not None: cluster.read = cluster.read and cluster_read # once one article is liked the cluster is liked cluster.liked = cluster.liked or cluster_liked - if cluster.main_date > article.date: + if cluster.main_date > article.date or force_article_as_main: cluster.main_title = article.title cluster.main_date = article.date cluster.main_feed_title = article.feed.title @@ -105,7 +105,7 @@ def join_read(self, feed_id=None, **filters): if filter_on_category: selected_fields.append(func.group_concat( art_cat_alias.category_id).label('categories_id')) - else: + else: # pragma: no cover selected_fields.append(func.array_agg(art_feed_alias.feed_id, type_=ARRAY(Integer)).label('feeds_id')) if filter_on_category: @@ -162,7 +162,7 @@ def join_read(self, feed_id=None, **filters): map(int, clu.categories_id.split(','))) elif filter_on_category: row['categories_id'] = [0] - else: + else: # pragma: no cover row['feeds_id'] = set(clu.feeds_id) if filter_on_category: row['categories_id'] = set(clu.categories_id) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index d44cf7a27..1c975dfd3 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -1,10 +1,12 @@ import logging +from sqlalchemy import and_ +from sqlalchemy.sql import select, update from datetime import datetime, timedelta -from bootstrap import conf -from .abstract import AbstractController -from .icon import IconController -from web.models import User, Feed +from bootstrap import db, conf, SQLITE_ENGINE +from web.controllers.abstract import AbstractController +from web.controllers.icon import IconController +from web.models import User, Feed, Article, Cluster logger = logging.getLogger(__name__) DEFAULT_LIMIT = 5 @@ -101,10 +103,31 @@ def create(self, **attrs): def update(self, filters, attrs, *args, **kwargs): self._ensure_icon(attrs) self.__clean_feed_fields(attrs) - if 'category_id' in attrs: + if {'title', 'category_id'}.intersection(attrs): for feed in self.read(**filters): - self.__get_art_contr().update({'feed_id': feed.id}, - {'category_id': attrs['category_id']}, *args, **kwargs) + if 'category_id' in attrs: + self.__get_art_contr().update({'feed_id': feed.id}, + {'category_id': attrs['category_id']}) + if 'title' in attrs: + # sqlite doesn't support join on update, but they're REALLY + # more efficient so we'll use them anyway with postgres + if self.user_id: + where_clause = and_(Article.user_id == self.user_id, + Article.feed_id == feed.id) + else: + where_clause = Article.feed_id == feed.id + if SQLITE_ENGINE: + stmt = select([Article.id]).where(where_clause) + stmt = update(Cluster)\ + .where(Cluster.main_article_id.in_(stmt))\ + .values(main_feed_title=attrs['title']) + else: # pragma: no cover + stmt = update(Cluster)\ + .where(and_( + Article.id == Cluster.main_article_id, + where_clause))\ + .values(dict(main_feed_title=attrs['title'])) + db.session.execute(stmt) return super().update(filters, attrs, *args, **kwargs) def delete(self, obj_id): diff --git a/src/web/js/actions/MenuActions.js b/src/web/js/actions/MenuActions.js index 9cb804cb1..755d43d9d 100644 --- a/src/web/js/actions/MenuActions.js +++ b/src/web/js/actions/MenuActions.js @@ -16,7 +16,8 @@ var MenuActions = { * purpose of setting filter and that the setFilterFunc is not * some event passed by react */ - if(set_filter == 'set_filter' && typeof setFilterFunc == 'function' && id) { + if(set_filter == 'set_filter' && typeof setFilterFunc == 'function' + && (id || id == 0)) { setFilterFunc(id); /* old_all_unread_count will be -1 on first iteration, * so it won't be triggered twice */ diff --git a/src/web/js/actions/RightPanelActions.js b/src/web/js/actions/RightPanelActions.js index 92308b707..a78f01105 100644 --- a/src/web/js/actions/RightPanelActions.js +++ b/src/web/js/actions/RightPanelActions.js @@ -2,6 +2,9 @@ var jquery = require('jquery'); var JarrDispatcher = require('../dispatcher/JarrDispatcher'); var ActionTypes = require('../constants/JarrConstants'); var MenuActions = require('../actions/MenuActions'); +var MenuStore = require('../stores/MenuStore'); +var MiddlePanelActions = require('../actions/MiddlePanelActions'); + var RightPanelActions = { loadParent: function(parent_type, parent_id) { @@ -45,10 +48,34 @@ var RightPanelActions = { jquery.ajax(args); }, putObj: function(id, obj_type, fields) { - this._apiReq('PUT', id, obj_type, fields, MenuActions.reload); + function callback() { + MenuActions.reload(); + if('title' in fields && obj_type == 'feed') { + MiddlePanelActions.reload(); + } + } + this._apiReq('PUT', id, obj_type, fields, callback); }, delObj: function(id, obj_type, fields) { - this._apiReq('DELETE', id, obj_type, null, MenuActions.reload); + var future_active_type; + var future_active_id = 0; + if(obj_type == 'feed') { + future_active_type = 'category_id'; + future_active_id = MenuStore.feeds[id].category_id; + } + function callback() { + var cmd; + var reload_callback; + if(obj_type == 'feed') { + cmd = 'set_filter'; + reload_callback = MiddlePanelActions.setCategoryFilter; + } else if (obj_type == 'category') { + cmd = 'set_filter'; + reload_callback = MiddlePanelActions.removeParentFilter; + } + MenuActions.reload(cmd, reload_callback, future_active_id); + } + this._apiReq('DELETE', id, obj_type, null, callback); }, resetErrors: function(feed_id) { this._apiReq('PUT', feed_id, 'feed', {error_count: 0, last_error: ''}, diff --git a/src/web/js/components/MiddlePanel.react.js b/src/web/js/components/MiddlePanel.react.js index 60cd473f5..aa1075b27 100644 --- a/src/web/js/components/MiddlePanel.react.js +++ b/src/web/js/components/MiddlePanel.react.js @@ -32,7 +32,7 @@ var TableLine = React.createClass({ }, render: function() { var liked = this.state.liked ? 'l' : ''; - var title = ( {this.props.feeds_id.map(function(feed_id) { @@ -149,6 +149,8 @@ var MiddlePanelParentFilterRow = React.createClass({ getInitialState: function() { return {id: MenuStore.active_id, type: MenuStore.active_type, + title: null, + icon_url: null, }; }, render: function() { @@ -156,11 +158,10 @@ var MiddlePanelParentFilterRow = React.createClass({ var img; var content = "Selected "; if (this.state.type == 'feed_id') { - var feed = MenuStore.feeds[this.state.id]; - img = ; - content += "Feed: " + feed.title; + content += "Feed: " + this.state.title; + img = ; } else if (this.state.type == 'category_id') { - content += "Category: " + MenuStore.categories[this.state.id].name; + content += "Category: " + this.state.title; } else { cn = "hidden"; } @@ -179,8 +180,16 @@ var MiddlePanelParentFilterRow = React.createClass({ MenuStore.removeChangeListener(this._onChange); }, _onChange: function() { - this.setState({id: MenuStore.active_id, - type: MenuStore.active_type}); + var new_state = {id: MenuStore.active_id, title: null, + type: MenuStore.active_type, icon_url: null}; + if (new_state.type == 'feed_id' && new_state.id in MenuStore.feeds) { + new_state.title = MenuStore.feeds[new_state.id].title; + new_state.icon_url = MenuStore.feeds[new_state.id].icon_url; + } else if (new_state.type == 'category_id' + && new_state.id in MenuStore.categories) { + new_state.title = MenuStore.categories[new_state.id].name; + } + this.setState(new_state); }, }); From ea2c0d2fd6bc156ad1bb6ec8549e29e0d3c26357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 9 Oct 2016 20:04:44 +0200 Subject: [PATCH 022/164] basic tag support (close #54) --- src/crawler/http_crawler.py | 25 ++-- src/lib/article_utils.py | 119 +++++++++++++++--- .../versions/9462d9753423_tag_handling.py | 29 +++++ .../versions/a7f62d50d366_clustering.py | 25 ++-- src/tests/api/article_test.py | 14 ++- src/tests/controllers/article_test.py | 29 +++++ src/tests/crawler_test.py | 18 +++ src/tests/fixtures/filler.py | 6 +- src/web/controllers/abstract.py | 4 + src/web/controllers/article.py | 41 +++--- src/web/controllers/cluster.py | 3 +- src/web/js/components/RightPanel.react.js | 94 +++++++------- src/web/js/stores/MenuStore.js | 6 +- src/web/models/__init__.py | 3 +- src/web/models/article.py | 12 +- src/web/models/right_mixin.py | 6 + src/web/models/tag.py | 18 +++ src/web/views/views.py | 1 - 18 files changed, 334 insertions(+), 119 deletions(-) create mode 100644 src/migrations/versions/9462d9753423_tag_handling.py create mode 100644 src/web/models/tag.py diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index d89dd3747..2bcc41858 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -24,7 +24,7 @@ from requests_futures.sessions import FuturesSession from lib.utils import default_handler, to_hash from lib.feed_utils import construct_feed_from, is_parsing_ok -from lib.article_utils import extract_id, construct_article +from lib.article_utils import construct_article, process_filters, FiltersAction logger = logging.getLogger(__name__) logging.captureWarnings(True) @@ -181,9 +181,9 @@ def callback(self, response): 'feed; bumping error count to %r', self.feed['id'], self.feed['title'], error_count) future = self.query_jarr('put', 'feed/%d' % self.feed['id'], - {'error_count': error_count, - 'last_error': str(error), - 'user_id': self.feed['user_id']}) + {'error_count': error_count, + 'last_error': str(error), + 'user_id': self.feed['user_id']}) return if response.status_code == 304: @@ -215,13 +215,24 @@ def callback(self, response): self.feed['id'], self.feed['title']) ids, entries = [], {} + feedparser parsed_response = feedparser.parse(response.content) for entry in parsed_response['entries']: - entry_ids = {'entry_id': extract_id(entry), - 'feed_id': self.feed['id'], - 'user_id': self.feed['user_id']} + entry_ids = construct_article(entry, self.feed, + {'entry_id', 'feed_id', 'user_id', 'tags'}) + skipped, _, _ = process_filters(self.feed['filters'], entry_ids, + {FiltersAction.SKIP}) + if skipped: + logger.debug('%r %r - skipping article', + self.feed['id'], self.feed['title']) + continue + del entry_ids['tags'] entries[tuple(sorted(entry_ids.items()))] = entry ids.append(entry_ids) + if not ids: + logger.info('%r %r - all articles skipped, adding nothing', + self.feed['id'], self.feed['title']) + return logger.debug('%r %r - found %d entries %r', self.feed['id'], self.feed['title'], len(ids), ids) future = self.query_jarr('get', 'articles/challenge', {'ids': ids}) diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index e5c1992ec..29f19b2c2 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -1,5 +1,7 @@ +import re import html import logging +from enum import Enum import dateutil.parser from datetime import datetime, timezone from bs4 import BeautifulSoup, SoupStrainer @@ -17,29 +19,37 @@ def extract_id(entry): return entry.get('entry_id') or entry.get('id') or entry['link'] -def construct_article(entry, feed): +def construct_article(entry, feed, fields=None): "Safe method to transorm a feedparser entry into an article" now = datetime.utcnow() - date = None - for date_key in ('published', 'created', 'date'): - if entry.get(date_key): - try: - date = dateutil.parser.parse(entry[date_key])\ - .astimezone(timezone.utc) - except Exception: - pass - else: - break + article = {} - content = get_article_content(entry) - link, title = get_article_details(entry) - content = clean_urls(content, link) - - return {'feed_id': feed['id'], - 'user_id': feed['user_id'], - 'entry_id': extract_id(entry), - 'link': link, 'content': content, 'title': title, - 'retrieved_date': now, 'date': date or now} + def push_in_article(key, value): + if fields is None or key in fields: + article[key] = value + push_in_article('feed_id', feed['id']) + push_in_article('user_id', feed['user_id']) + push_in_article('entry_id', extract_id(entry)) + push_in_article('retrieved_date', now) + if fields is None or 'date' in fields: + for date_key in ('published', 'created', 'updated'): + if entry.get(date_key): + try: + article['date'] = dateutil.parser.parse(entry[date_key])\ + .astimezone(timezone.utc) + except Exception: + pass + else: + break + push_in_article('content', get_article_content(entry)) + if fields is None or {'link', 'title'}.intersection(fields): + link, title = get_article_details(entry) + push_in_article('link', link) + push_in_article('title', title) + push_in_article('content', clean_urls(article['content'], link)) + push_in_article('tags', [tag.get('term').strip() + for tag in entry.get('tags', [])]) + return article def get_article_content(entry): @@ -71,3 +81,72 @@ def get_article_details(entry): except IndexError: # no title pass return article_link, article_title or 'No title' + + +class FiltersAction(Enum): + READ = 'mark as read' + LIKED = 'mark as favorite' + SKIP = 'skipped' + + +class FiltersType(Enum): + REGEX = 'regex' + MATCH = 'simple match' + EXACT_MATCH = 'exact match' + TAG_MATCH = 'tag match' + TAG_CONTAINS = 'tag contains' + + +class FiltersTrigger(Enum): + MATCH = 'match' + NO_MATCH = 'no match' + + +def process_filters(filters, article, only_actions=None): + skipped, read, liked = False, None, False + filters = filters or [] + if only_actions is None: + only_actions = set(FiltersAction) + for filter_ in filters: + match = False + try: + pattern = filter_.get('pattern', '') + filter_type = FiltersType(filter_.get('type')) + filter_action = FiltersAction(filter_.get('action')) + filter_trigger = FiltersTrigger(filter_.get('action on')) + if filter_type is not FiltersType.REGEX: + pattern = pattern.lower() + except ValueError: + continue + if filter_action not in only_actions: + logger.debug('ignoring filter %r' % filter_) + continue + + title = article.get('title').lower() + tags = [tag.lower() for tag in article.get('tags', [])] + if filter_type is FiltersType.REGEX: + match = re.match(pattern, title) + elif filter_type is FiltersType.MATCH: + match = pattern in title + elif filter_type is FiltersType.EXACT_MATCH: + match = pattern == title + elif filter_type is FiltersType.TAG_MATCH: + match = pattern in tags + elif filter_type is FiltersType.TAG_CONTAINS: + match = any(pattern in tag for tag in tags) + take_action = match and filter_trigger is FiltersTrigger.MATCH \ + or not match and filter_trigger is FiltersTrigger.NO_MATCH + + if not take_action: + continue + + if filter_action is FiltersAction.READ: + read = True + elif filter_action is FiltersAction.LIKED: + liked = True + elif filter_action is FiltersAction.SKIP: + skipped = True + + if skipped or read or liked: + logger.info("%r applied on %r", filter_action.value, article['link']) + return skipped, read, liked diff --git a/src/migrations/versions/9462d9753423_tag_handling.py b/src/migrations/versions/9462d9753423_tag_handling.py new file mode 100644 index 000000000..8f5434c5f --- /dev/null +++ b/src/migrations/versions/9462d9753423_tag_handling.py @@ -0,0 +1,29 @@ +"""adding tag handling capacities + +Revision ID: 9462d9753423 +Revises: 835c03754c69 +Create Date: 2016-10-08 23:07:57.425931 + +""" + +# revision identifiers, used by Alembic. +revision = '9462d9753423' +down_revision = '835c03754c69' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('tag', + sa.Column('text', sa.String(), nullable=False), + sa.Column('article_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['article_id'], ['article.id'], + ondelete='CASCADE'), + sa.PrimaryKeyConstraint('text', 'article_id') + ) + +def downgrade(): + op.drop_table('tag') diff --git a/src/migrations/versions/a7f62d50d366_clustering.py b/src/migrations/versions/a7f62d50d366_clustering.py index 646e1348d..567149dab 100644 --- a/src/migrations/versions/a7f62d50d366_clustering.py +++ b/src/migrations/versions/a7f62d50d366_clustering.py @@ -13,7 +13,7 @@ depends_on = None from datetime import datetime -from bootstrap import conf +from bootstrap import SQLITE_ENGINE from alembic import op import sqlalchemy as sa @@ -39,7 +39,7 @@ def upgrade(): sa.Column('cluster_id', sa.Integer(), nullable=True)) from web.models import Cluster, Feed, Article - if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI: + if not SQLITE_ENGINE: op.create_foreign_key(None, 'article', 'cluster', ['cluster_id'], ['id']) op.create_foreign_key(None, 'cluster', 'article', @@ -98,26 +98,27 @@ def upgrade(): batch_op.drop_column('readed') batch_op.drop_column('like') - if 'sqlite' not in conf.SQLALCHEMY_DATABASE_URI: - print('%s - creating index 0/5' % datetime.now().isoformat()) - op.execute('CREATE INDEX article_uid_cluid ON article ' - '(user_id, cluster_id);') - print('%s - creating index 1/5' % datetime.now().isoformat()) + if not SQLITE_ENGINE: + print('%s - creating index 0/6' % datetime.now().isoformat()) + op.execute('CREATE INDEX article_cluid ON article (cluster_id);') + print('%s - creating index 1/6' % datetime.now().isoformat()) op.execute('CREATE INDEX article_uid_cid_cluid ON article' '(user_id, category_id, cluster_id);') - print('%s - creating index 2/5' % datetime.now().isoformat()) + print('%s - creating index 2/6' % datetime.now().isoformat()) op.execute('CREATE INDEX article_uid_fid_cluid ON article' '(user_id, feed_id, cluster_id);') - print('%s - creating index 3/5' % datetime.now().isoformat()) + print('%s - creating index 3/6' % datetime.now().isoformat()) op.execute('CREATE INDEX cluster_uid_date ON cluster ' '(user_id, main_date DESC NULLS LAST);') - print('%s - creating index 4/5' % datetime.now().isoformat()) + print('%s - creating index 4/6' % datetime.now().isoformat()) op.execute('CREATE INDEX cluster_liked_uid_date ON cluster ' '(liked, user_id, main_date DESC NULLS LAST);') - print('%s - creating index 5/5' % datetime.now().isoformat()) + print('%s - creating index 5/6' % datetime.now().isoformat()) op.execute('CREATE INDEX cluster_read_uid_date ON cluster ' '(read, user_id, main_date DESC NULLS LAST);') - + print('%s - creating index 6/6' % datetime.now().isoformat()) + op.execute('CREATE INDEX cluster_uid_mlink ON cluster ' + '(user_id, main_link);') def downgrade(): op.add_column('article', diff --git a/src/tests/api/article_test.py b/src/tests/api/article_test.py index 49dab0fc7..e214fe125 100644 --- a/src/tests/api/article_test.py +++ b/src/tests/api/article_test.py @@ -57,16 +57,26 @@ def test_api_creation(self): resp = self._api('post', self.urn, user='user1', data={'feed_id': 1}) self.assertEquals(403, resp.status_code) UserController().update({'login': 'user1'}, {'is_api': True}) - resp = self._api('post', self.urn, user='user1', data={'feed_id': 1}) + + resp = self._api('post', self.urn, user='user1', + data={'feed_id': 1, 'tags': ['tag1', 'tag2']}) + content = resp.json() self.assertEquals(201, resp.status_code) - self.assertEquals(2, resp.json()['user_id']) + self.assertEquals(2, content['user_id']) + self.assertEquals(['tag1', 'tag2'], content['tags']) + + resp = self._api('get', "%s/%s" % (self.urn, content['id'])) + self.assertEquals(['tag1', 'tag2'], resp.json()['tags']) + resp = self._api('post', self.urn, user='user1', data={'feed_id': 1}) self.assertEquals(2, resp.json()['user_id']) self.assertEquals(201, resp.status_code) + resp = self._api('post', self.urn, user='user2', data={'user_id': 2, 'feed_id': 1}) self.assertEquals(403, resp.status_code) UserController().update({'login': 'user2'}, {'is_api': True}) + resp = self._api('post', self.urn, user='user2', data={'user_id': 2, 'feed_id': 1}) self.assertEquals(404, resp.status_code) diff --git a/src/tests/controllers/article_test.py b/src/tests/controllers/article_test.py index b32b86435..8a344d005 100644 --- a/src/tests/controllers/article_test.py +++ b/src/tests/controllers/article_test.py @@ -29,6 +29,15 @@ def test_create_using_filters(self): "pattern": "pattern3", "action on": "match", "action": "mark as read"}]}) + feed_ctr.update({'id': feed2['id']}, + {'filters': [{"type": "tag match", + "pattern": "pattern4", + "action on": "match", + "action": "skipped"}, + {"type": "tag contains", + "pattern": "pattern5", + "action on": "match", + "action": "skipped"}]}) art1 = ArticleController(2).create( entry_id="will be read and faved 1", @@ -104,3 +113,23 @@ def test_create_using_filters(self): link="doesn't matter either8") self.assertFalse(art8.cluster.read) self.assertTrue(art8.cluster.liked) + + art9 = ArticleController(2).create( + entry_id="unique9", + feed_id=feed2['id'], + title="garbage", tags=['garbage', 'pattern4'], + content="doesn't matterç", + link="doesn't matter either9") + self.assertIsNone(art9) + self.assertEquals(0, + ArticleController(2).read(entry_id='unique9').count()) + + art10 = ArticleController(2).create( + entry_id="will be ignored", + feed_id=feed2['id'], + title="garbage", tags=['pattern5 garbage', 'garbage'], + content="doesn't matter10", + link="doesn't matter either10") + self.assertIsNone(art10) + self.assertEquals(0, + ArticleController(2).read(entry_id='unique10').count()) diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index d307d7408..71cdd7ed5 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -107,6 +107,24 @@ def test_no_add_on_304(self): resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') self.assertEquals(36, len(resp.json())) + @patch('crawler.http_crawler.JarrUpdater.callback') + def test_no_add_feed_skip(self, jarr_updated_callback): + scheduler = CrawlerScheduler('admin', 'admin') + self.resp_status_code = 304 + resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') + self.assertEquals(36, len(resp.json())) + FeedController().update({}, {'filters': [{"type": "tag contains", + "action on": "match", + "pattern": "pattern5", + "action": "skipped"}]}) + + scheduler.run() + scheduler.wait(**self.wait_params) + self.assertFalse(jarr_updated_callback.called, + "all articles should have been skipped") + resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') + self.assertEquals(36, len(resp.json())) + def test_matching_etag(self): self._reset_feeds_freshness(etag='fake etag') self.resp_headers = {'etag': 'fake etag'} diff --git a/src/tests/fixtures/filler.py b/src/tests/fixtures/filler.py index 643b7bce1..d6abba09e 100644 --- a/src/tests/fixtures/filler.py +++ b/src/tests/fixtures/filler.py @@ -14,7 +14,7 @@ def populate_db(): password=name) for name in ["user1", "user2"]] - def to_name(u, c=None, f=None, a=None): + def to_name(u, c=None, f=None, a=None, *args): string = u.login if c: string += " cat%s" % c @@ -22,7 +22,7 @@ def to_name(u, c=None, f=None, a=None): string += " feed%s" % f if a is not None: string += " art%s" % a - return string + return string + ''.join(args) for k in range(2): article_total = 0 @@ -41,6 +41,8 @@ def to_name(u, c=None, f=None, a=None): acontr.create(entry_id=entry, link='http://test.te/%d' % article_total, feed_id=feed.id, user_id=user.id, + tags=[to_name(user, i, i, j, '1'), + to_name(user, i, i, j, '2')], category_id=cat_id, title=entry, content="content %d" % article_total) diff --git a/src/web/controllers/abstract.py b/src/web/controllers/abstract.py index db84f3728..7d2bcbdc5 100644 --- a/src/web/controllers/abstract.py +++ b/src/web/controllers/abstract.py @@ -4,6 +4,7 @@ from datetime import datetime from collections import defaultdict from sqlalchemy import and_, or_ +from sqlalchemy.ext.associationproxy import AssociationProxy from werkzeug.exceptions import Forbidden, NotFound logger = logging.getLogger(__name__) @@ -144,6 +145,9 @@ def _get_attrs_desc(cls, role, right=None): "right must be 'read' or 'write' with role %r" % role columns = getattr(cls._db_cls, 'fields_%s_%s' % (role, right))() for column in columns: + if isinstance(getattr(cls._db_cls, column), AssociationProxy): + result[column] = {'type': list, 'default': list} + continue try: db_col = getattr(cls._db_cls, column).property.columns[0] except AttributeError: diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index 20a6c67a2..c22116efe 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -1,4 +1,3 @@ -import re import logging import sqlalchemy from sqlalchemy import func @@ -9,6 +8,7 @@ from .abstract import AbstractController from web.controllers import CategoryController, FeedController from web.models import User, Article +from lib.article_utils import process_filters logger = logging.getLogger(__name__) @@ -50,31 +50,11 @@ def create(self, **attrs): "no right on feed %r" % feed.id attrs['user_id'], attrs['category_id'] = feed.user_id, feed.category_id - # handling feed's filters - cluster_read, cluster_liked = None, False - for filter_ in feed.filters or []: - match = False - if filter_.get('type') == 'regex': - match = re.match(filter_['pattern'], attrs.get('title', '')) - elif filter_.get('type') == 'simple match': - match = filter_['pattern'] in attrs.get('title', '') - take_action = match and filter_.get('action on') == 'match' \ - or not match and filter_.get('action on') == 'no match' - - if not take_action: - continue - - if filter_.get('action') == 'mark as read': - cluster_read = True - logger.info("article article will be created as read %r", - attrs['link']) - elif filter_.get('action') == 'mark as favorite': - cluster_liked = True - logger.info("article article will be created as liked %r", - attrs['link']) - + skipped, read, liked = process_filters(feed.filters, attrs) + if skipped: + return None article = super().create(**attrs) - cluster_contr.clusterize(article, cluster_read, cluster_liked) + cluster_contr.clusterize(article, read, liked) return article def update(self, filters, attrs, *args, **kwargs): @@ -122,7 +102,14 @@ def remove_from_all_clusters(self, article_id): clu_ctrl._enrich_cluster(cluster, cluster.articles[1]) return True - def delete(self, obj_id): + def delete(self, obj_id, commit=True): still_delete_article = self.remove_from_all_clusters(obj_id) if still_delete_article: - return super().delete(obj_id) + obj = self.get(id=obj_id) + for tag in obj.tag_objs: + db.session.delete(tag) + db.session.delete(obj) + if commit: + db.session.flush() + db.session.commit() + return obj diff --git a/src/web/controllers/cluster.py b/src/web/controllers/cluster.py index 93e0b7894..946503337 100644 --- a/src/web/controllers/cluster.py +++ b/src/web/controllers/cluster.py @@ -131,7 +131,8 @@ def join_read(self, feed_id=None, **filters): .filter(cluster_has_feed) else: query = query.join(art_feed_alias, - art_feed_alias.cluster_id == Cluster.id) + and_(art_feed_alias.user_id == self.user_id, + art_feed_alias.cluster_id == Cluster.id)) if filter_on_category: # joining only if filtering on categories to lighten the query # as every article doesn't obligatorily have a category > outerjoin diff --git a/src/web/js/components/RightPanel.react.js b/src/web/js/components/RightPanel.react.js index 42a526e03..f2e1416e1 100644 --- a/src/web/js/components/RightPanel.react.js +++ b/src/web/js/components/RightPanel.react.js @@ -72,47 +72,52 @@ var PanelMixin = { var items = []; var key; if(!this.state.edit_mode) { - this.fields.filter(function(field) { - return field.type != 'ignore'; - }).map(function(field) { - key = this.getKey('dt', field.key); - items.push(
{field.title}
); - key = this.getKey('dd', field.key); - if(field.type == 'string') { - items.push(
{this.props.obj[field.key]}
); - } else if(field.type == 'bool') { - if(this.props.obj[field.key]) { - items.push(
); - } else { - items.push(
); - } - } else if (field.type == 'link') { - items.push(
- - {this.props.obj[field.key]} - -
); - } - }.bind(this)); + this.fields.filter(function(field) { + return field.type != 'ignore'; + }).map(function(field) { + if(field.type == 'list' && this.props.obj[field.key].length == 0) { + return; + } + key = this.getKey('dt', field.key); + items.push(
{field.title}
); + key = this.getKey('dd', field.key); + if(field.type == 'string') { + items.push(
{this.props.obj[field.key]}
); + } else if(field.type == 'list') { + items.push(
{this.props.obj[field.key].join(', ')}
); + } else if(field.type == 'bool') { + if(this.props.obj[field.key]) { + items.push(
); + } else { + items.push(
); + } + } else if (field.type == 'link') { + items.push(
+ + {this.props.obj[field.key]} + +
); + } + }.bind(this)); } else { - this.fields.filter(function(field) { - return field.type != 'ignore'; - }).map(function(field) { - key = this.getKey('dd', field.key); - items.push(
{field.title}
); - key = this.getKey('dt', field.key); - var input = null; - if(field.type == 'string' || field.type == 'link') { - input = (); - } else if (field.type == 'bool') { - input = (); - } - items.push(
{input}
); - }.bind(this)); + this.fields.filter(function(field) { + return field.type != 'ignore'; + }).map(function(field) { + key = this.getKey('dd', field.key); + items.push(
{field.title}
); + key = this.getKey('dt', field.key); + var input = null; + if(field.type == 'string' || field.type == 'link') { + input = (); + } else if (field.type == 'bool') { + input = (); + } + items.push(
{input}
); + }.bind(this)); } return (
{items}
); }, @@ -175,6 +180,7 @@ var Article = React.createClass({ isRemovable: function() {return true;}, fields: [{'title': 'Date', 'type': 'string', 'key': 'date'}, {'title': 'Original link', 'type': 'link', 'key': 'link'}, + {'title': 'Tags', 'type': 'list', 'key': 'tags'}, ], obj_type: 'article', getTitle: function() {return this.props.obj.title;}, @@ -247,14 +253,18 @@ var Feed = React.createClass({ ); }, diff --git a/src/web/js/stores/MenuStore.js b/src/web/js/stores/MenuStore.js index 4e6eb6412..3e7095dca 100644 --- a/src/web/js/stores/MenuStore.js +++ b/src/web/js/stores/MenuStore.js @@ -40,9 +40,9 @@ var MenuStore = assign({}, EventEmitter.prototype, { readCluster: function(cluster, value) { cluster.feeds_id.map(function(feed_id) { this.feeds[feed_id].unread += value; - }.bind(this)); - cluster.categories_id.map(function(category_id) { - this.categories[category_id].unread += value; + if(this.feeds[feed_id].category_id) { + this.categories[this.feeds[feed_id].category_id].unread += value; + } }.bind(this)); }, emitChange: function(all_folded) { diff --git a/src/web/models/__init__.py b/src/web/models/__init__.py index 10cb05e21..9962cb730 100644 --- a/src/web/models/__init__.py +++ b/src/web/models/__init__.py @@ -4,8 +4,9 @@ from .icon import Icon from .category import Category from .cluster import Cluster +from .tag import Tag -__all__ = ['Feed', 'User', 'Article', 'Icon', 'Category', 'Cluster'] +__all__ = ['Feed', 'User', 'Article', 'Icon', 'Category', 'Cluster', 'Tag'] def db_empty(db): diff --git a/src/web/models/article.py b/src/web/models/article.py index c336ce9c3..0e28b0068 100644 --- a/src/web/models/article.py +++ b/src/web/models/article.py @@ -2,6 +2,7 @@ from sqlalchemy import (Column, Index, ForeignKey, Integer, String, Boolean, DateTime) from sqlalchemy.orm import relationship +from sqlalchemy.ext.associationproxy import association_proxy from bootstrap import db from web.models.right_mixin import RightMixin @@ -32,6 +33,11 @@ class Article(db.Model, RightMixin): foreign_keys=[category_id]) feed = relationship('Feed', back_populates='articles', foreign_keys=[feed_id]) + tag_objs = relationship('Tag', back_populates='article', + cascade='all,delete-orphan', + lazy=False, + foreign_keys='[Tag.article_id]') + tags = association_proxy('tag_objs', 'text') # index article_uid_cluid = Index('user_id', 'cluster_id') @@ -49,7 +55,11 @@ def _fields_base_write(): @staticmethod def _fields_base_read(): return {'id', 'entry_id', 'link', 'title', 'content', 'date', - 'retrieved_date', 'user_id'} + 'retrieved_date', 'user_id', 'tags'} + + @staticmethod + def _fields_api_write(): + return {'tags'} def __repr__(self): return " Date: Mon, 10 Oct 2016 22:40:36 +0200 Subject: [PATCH 023/164] fixing littles post deployement bug * waiting on crawler improved * ignoring empty entries * folding all categories fixed * small fix in the feed creation --- src/crawler/http_crawler.py | 31 ++++++++++++++----------- src/lib/article_utils.py | 16 +++++++------ src/manager.py | 5 ++-- src/tests/crawler_test.py | 2 +- src/tests/views/admin_test.py | 2 -- src/web/js/stores/MenuStore.js | 2 +- src/web/js/stores/NotificationsStore.js | 7 +----- src/web/views/feed.py | 2 +- 8 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 2bcc41858..263bc9941 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -56,25 +56,29 @@ def query_jarr(self, method, urn, data=None): self._futures.append(future) return future - def wait(self, max_wait=300, checks=10, wait_for=5): - checked, second_waited = 0, 0 + def wait(self, max_wait=600, wait_for=2): start = datetime.now() max_wait_delta = timedelta(seconds=max_wait) while True: time.sleep(wait_for) - if datetime.now() - start > max_wait_delta: - logger.warn('Exiting after %d seconds', second_waited) + # checking not thread is still running + # some thread are running and we are not behind + if datetime.now() - start <= max_wait_delta \ + and any(fu.running() for fu in self._futures): + # let's wait and see if it's not okay next time + continue + # all thread are done, let's exit + if all(fu.done() for fu in self._futures): break + # some thread are still running and we're gonna future.wait on 'em + wait_minus_passed = max_wait - (datetime.now() - start).seconds + if wait_minus_passed > 0: + max_wait = wait_minus_passed try: # no idea why wait throw ValueError around - not_done = len(wait(self._futures, timeout=max_wait).not_done) + wait(self._futures, timeout=max_wait) except ValueError: - not_done = 1 - if not_done != 0: - checked = 0 - continue - checked += 1 - if checked >= checks: - break + logger.exception('something bad happened:') + break class JarrUpdater(AbstractCrawler): @@ -215,9 +219,10 @@ def callback(self, response): self.feed['id'], self.feed['title']) ids, entries = [], {} - feedparser parsed_response = feedparser.parse(response.content) for entry in parsed_response['entries']: + if not entry: + continue entry_ids = construct_article(entry, self.feed, {'entry_id', 'feed_id', 'user_id', 'tags'}) skipped, _, _ = process_filters(self.feed['filters'], entry_ids, diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index 29f19b2c2..a0d71db66 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -11,6 +11,7 @@ from web.lib.article_cleaner import clean_urls logger = logging.getLogger(__name__) +PROCESSED_DATE_KEYS = {'published', 'created', 'updated'} def extract_id(entry): @@ -25,14 +26,14 @@ def construct_article(entry, feed, fields=None): article = {} def push_in_article(key, value): - if fields is None or key in fields: + if not fields or key in fields: article[key] = value push_in_article('feed_id', feed['id']) push_in_article('user_id', feed['user_id']) push_in_article('entry_id', extract_id(entry)) push_in_article('retrieved_date', now) - if fields is None or 'date' in fields: - for date_key in ('published', 'created', 'updated'): + if not fields or 'date' in fields: + for date_key in PROCESSED_DATE_KEYS: if entry.get(date_key): try: article['date'] = dateutil.parser.parse(entry[date_key])\ @@ -69,8 +70,8 @@ def get_article_details(entry): # resolves URL behind proxies (like feedproxy.google.com) response = jarr_get(article_link) except Exception as error: - logger.warning("Unable to get the real URL of %s. Won't fix link " - "or title. Error: %s", article_link, error) + logger.warn("Unable to get the real URL of %s. Won't fix link " + "or title. Error: %s", article_link, error) return article_link, article_title or 'No title' article_link = response.url if not article_title: @@ -122,7 +123,7 @@ def process_filters(filters, article, only_actions=None): logger.debug('ignoring filter %r' % filter_) continue - title = article.get('title').lower() + title = article.get('title', '').lower() tags = [tag.lower() for tag in article.get('tags', [])] if filter_type is FiltersType.REGEX: match = re.match(pattern, title) @@ -148,5 +149,6 @@ def process_filters(filters, article, only_actions=None): skipped = True if skipped or read or liked: - logger.info("%r applied on %r", filter_action.value, article['link']) + logger.info("%r applied on %r", filter_action.value, + article.get('link') or article.get('title')) return skipped, read, liked diff --git a/src/manager.py b/src/manager.py index 1f2a6da64..1f148f650 100755 --- a/src/manager.py +++ b/src/manager.py @@ -38,9 +38,8 @@ def fetch(limit=100, retreive_all=False): "Crawl the feeds with the client crawler." from crawler.http_crawler import CrawlerScheduler scheduler = CrawlerScheduler(conf.CRAWLER_LOGIN, conf.CRAWLER_PASSWD) - with scheduler.pool: - scheduler.run(limit=limit, retreive_all=retreive_all) - scheduler.wait() + scheduler.run(limit=limit, retreive_all=retreive_all) + scheduler.wait() @manager.command diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index 71cdd7ed5..e89423725 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -13,7 +13,7 @@ class CrawlerTest(JarrFlaskCommon): def setUp(self): super().setUp() - self.wait_params = {'checks': 3, 'wait_for': 2, 'max_wait': 60} + self.wait_params = {'wait_for': 5, 'max_wait': 30} UserController().update({'login': 'admin'}, {'is_api': True}) self._is_secure_served \ = patch('web.lib.article_cleaner.is_secure_served') diff --git a/src/tests/views/admin_test.py b/src/tests/views/admin_test.py index 218128447..06823cfc4 100644 --- a/src/tests/views/admin_test.py +++ b/src/tests/views/admin_test.py @@ -1,5 +1,3 @@ -from io import BytesIO -from mock import patch from tests.base import JarrFlaskCommon from web.controllers import UserController from flask_principal import PermissionDenied diff --git a/src/web/js/stores/MenuStore.js b/src/web/js/stores/MenuStore.js index 3e7095dca..4b4e1eff7 100644 --- a/src/web/js/stores/MenuStore.js +++ b/src/web/js/stores/MenuStore.js @@ -46,7 +46,7 @@ var MenuStore = assign({}, EventEmitter.prototype, { }.bind(this)); }, emitChange: function(all_folded) { - if (all_folded) { + if (all_folded == true || all_folded == false) { this.all_folded = all_folded; } else { this.all_folded = null; diff --git a/src/web/js/stores/NotificationsStore.js b/src/web/js/stores/NotificationsStore.js index 8f9613745..d113414a6 100644 --- a/src/web/js/stores/NotificationsStore.js +++ b/src/web/js/stores/NotificationsStore.js @@ -24,12 +24,7 @@ var NotificationsStore = assign({}, EventEmitter.prototype, { this.notifs = this.notifs.filter(function(notif) {return !notif.read;}); return this.notifs; }, - emitChange: function(all_folded) { - if (all_folded) { - this.all_folded = all_folded; - } else { - this.all_folded = null; - } + emitChange: function() { this.emit(CHANGE_EVENT); }, addChangeListener: function(callback) { diff --git a/src/web/views/feed.py b/src/web/views/feed.py index ce68ad057..ce970126d 100644 --- a/src/web/views/feed.py +++ b/src/web/views/feed.py @@ -34,7 +34,7 @@ def check_feeds(link, site_link): filters = [] if link: filters.append({'link': link}) - if link: + if site_link: filters.append({'site_link': site_link}) filters = {'__or__': filters} if len(filters) > 1 else filters[0] feed_exists = feed_contr.read(**filters).first() From 6757146660bba02589fe6daeaf89777279c54252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Mon, 10 Oct 2016 23:30:08 +0200 Subject: [PATCH 024/164] fixing skipping from crawler --- src/crawler/http_crawler.py | 3 ++- src/lib/article_utils.py | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 263bc9941..69c0d527d 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -224,7 +224,8 @@ def callback(self, response): if not entry: continue entry_ids = construct_article(entry, self.feed, - {'entry_id', 'feed_id', 'user_id', 'tags'}) + {'title', 'entry_id', 'feed_id', 'user_id', 'tags'}, + fetch=False) skipped, _, _ = process_filters(self.feed['filters'], entry_ids, {FiltersAction.SKIP}) if skipped: diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index a0d71db66..dd1444385 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -20,7 +20,7 @@ def extract_id(entry): return entry.get('entry_id') or entry.get('id') or entry['link'] -def construct_article(entry, feed, fields=None): +def construct_article(entry, feed, fields=None, fetch=True): "Safe method to transorm a feedparser entry into an article" now = datetime.utcnow() article = {} @@ -44,10 +44,11 @@ def push_in_article(key, value): break push_in_article('content', get_article_content(entry)) if fields is None or {'link', 'title'}.intersection(fields): - link, title = get_article_details(entry) + link, title = get_article_details(entry, fetch) push_in_article('link', link) push_in_article('title', title) - push_in_article('content', clean_urls(article['content'], link)) + if 'content' in article: + push_in_article('content', clean_urls(article['content'], link)) push_in_article('tags', [tag.get('term').strip() for tag in entry.get('tags', [])]) return article @@ -62,10 +63,10 @@ def get_article_content(entry): return content -def get_article_details(entry): +def get_article_details(entry, fetch=True): article_link = entry.get('link') article_title = html.unescape(entry.get('title', '')) - if conf.CRAWLER_RESOLV and article_link or not article_title: + if fetch and conf.CRAWLER_RESOLV and article_link or not article_title: try: # resolves URL behind proxies (like feedproxy.google.com) response = jarr_get(article_link) From fe60cb05cc3ccc81a6f42683ed094367b6f2ca85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Tue, 11 Oct 2016 17:09:55 +0200 Subject: [PATCH 025/164] handling url without scheme in feed --- src/bootstrap.py | 2 +- src/crawler/http_crawler.py | 10 +++++++--- src/lib/article_utils.py | 22 +++++++++++++++++++--- src/lib/utils.py | 10 ++++++---- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/bootstrap.py b/src/bootstrap.py index bb2914a0a..1765ca7e0 100644 --- a/src/bootstrap.py +++ b/src/bootstrap.py @@ -28,7 +28,7 @@ def set_logging(log_path=None, log_level=logging.INFO, modules=(), log_format='%(asctime)s %(levelname)s %(message)s'): if not modules: - modules = ('root', 'bootstrap', 'runserver', + modules = ('root', 'bootstrap', 'runserver', 'lib', 'web', 'crawler', 'manager', 'plugins') if log_path: handler = logging.FileHandler(log_path) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 69c0d527d..d1d5c03a7 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -56,9 +56,10 @@ def query_jarr(self, method, urn, data=None): self._futures.append(future) return future - def wait(self, max_wait=600, wait_for=2): - start = datetime.now() + def wait(self, max_wait=600, wait_for=2, checks=2): + start, checked = datetime.now(), 0 max_wait_delta = timedelta(seconds=max_wait) + time.sleep(wait_for * 3) while True: time.sleep(wait_for) # checking not thread is still running @@ -67,6 +68,9 @@ def wait(self, max_wait=600, wait_for=2): and any(fu.running() for fu in self._futures): # let's wait and see if it's not okay next time continue + if checks == checked: + checked += 1 + continue # all thread are done, let's exit if all(fu.done() for fu in self._futures): break @@ -155,7 +159,7 @@ def callback(self, response): return # meaningless if no new article has been published logger.info('%r %r - pushing feed attrs %r', self.feed['id'], self.feed['title'], - {key: "%s -> %s" % (up_feed[key], self.feed.get(key)) + {key: "%r -> %r" % (self.feed.get(key), up_feed[key]) for key in up_feed if up_feed[key] != self.feed.get(key)}) self.query_jarr('put', 'feed/%d' % self.feed['id'], up_feed) diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index dd1444385..2009be509 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -3,6 +3,8 @@ import logging from enum import Enum import dateutil.parser +from urllib.parse import urlsplit, urlunsplit, SplitResult +from requests.exceptions import MissingSchema from datetime import datetime, timezone from bs4 import BeautifulSoup, SoupStrainer @@ -69,10 +71,24 @@ def get_article_details(entry, fetch=True): if fetch and conf.CRAWLER_RESOLV and article_link or not article_title: try: # resolves URL behind proxies (like feedproxy.google.com) - response = jarr_get(article_link) + response = jarr_get(article_link, timeout=5) + except MissingSchema: + split, failed = urlsplit(article_link), False + for scheme in 'https', 'http': + new_link = urlunsplit(SplitResult(scheme, *split[1:])) + try: + response = jarr_get(new_link, timeout=5) + except Exception as error: + failed = True + continue + failed = False + article_link = new_link + break + if failed: + return article_link, article_title or 'No title' except Exception as error: - logger.warn("Unable to get the real URL of %s. Won't fix link " - "or title. Error: %s", article_link, error) + logger.info("Unable to get the real URL of %s. Won't fix " + "link or title. Error: %s", article_link, error) return article_link, article_title or 'No title' article_link = response.url if not article_title: diff --git a/src/lib/utils.py b/src/lib/utils.py index 187f7e636..e6226eb32 100644 --- a/src/lib/utils.py +++ b/src/lib/utils.py @@ -80,7 +80,9 @@ def redirect_url(default='home'): return request.args.get('next') or request.referrer or url_for(default) -def jarr_get(url): - return requests.get(url, verify=False, allow_redirects=True, - timeout=conf.CRAWLER_TIMEOUT, - headers={'User-Agent': conf.CRAWLER_USER_AGENT}) +def jarr_get(url, **kwargs): + request_kwargs = {'verify': False, 'allow_redirects': True, + 'timeout': conf.CRAWLER_TIMEOUT, + 'headers': {'User-Agent': conf.CRAWLER_USER_AGENT}} + request_kwargs.update(kwargs) + return requests.get(url, **request_kwargs) From dd84cf37f93ba2b520340c844be354298d95eac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 12 Oct 2016 01:01:03 +0200 Subject: [PATCH 026/164] improving feed deletion, no more browsing all the articles --- src/web/controllers/feed.py | 38 ++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index 1c975dfd3..af90df4b1 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -1,6 +1,6 @@ import logging from sqlalchemy import and_ -from sqlalchemy.sql import select, update +from sqlalchemy.sql import select, update, delete from datetime import datetime, timedelta from bootstrap import db, conf, SQLITE_ENGINE @@ -131,7 +131,39 @@ def update(self, filters, attrs, *args, **kwargs): return super().update(filters, attrs, *args, **kwargs) def delete(self, obj_id): + from web.controllers.cluster import ClusterController + feed = self.get(id=obj_id) actrl = self.__get_art_contr() - for article in self.get(id=obj_id).articles: - actrl.remove_from_all_clusters(article.id) + clu_ctrl = ClusterController(self.user_id) + + # removing back ref from cluster to article + clu_ctrl.update({'main_article_id__in': actrl.read(feed_id=obj_id) + .with_entities('id')}, + {'main_article_id': None}) + + def select_art(col): + return select([col]).where(and_(Cluster.id == Article.cluster_id, + Article.user_id == feed.user_id))\ + .order_by(Article.date.asc()).limit(1) + + # removing articles + db.session.execute(delete(Article).where( + and_(Article.feed_id == feed.id, + Article.user_id == feed.user_id))) + + # reclustering + clu_ctrl.update({'main_article_id': None}, + {'main_title': select_art(Article.title), + 'main_article_id': select_art(Article.id), + 'main_feed_title': select([Feed.title]) + .where(and_( + Cluster.id == Article.cluster_id, + Article.user_id == feed.user_id, + Feed.id == Article.feed_id, + Feed.user_id == feed.user_id)) + .order_by(Article.date.asc()).limit(1)}) + # removing remaing clusters + db.session.execute(delete(Cluster).where( + and_(Cluster.user_id == feed.user_id, + Cluster.main_article_id.__eq__(None)))) return super().delete(obj_id) From f39e2a9840a9ea9d66f2ff2c780635e07653987e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 12 Oct 2016 01:48:11 +0200 Subject: [PATCH 027/164] testing missing scheme handling --- src/crawler/http_crawler.py | 28 +++++++--------- src/lib/article_utils.py | 16 ++++++++- src/tests/crawler_test.py | 8 +++++ src/tests/libs/article_utils_test.py | 49 +++++++++++++++++++++++----- 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index d1d5c03a7..914504c50 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -24,7 +24,7 @@ from requests_futures.sessions import FuturesSession from lib.utils import default_handler, to_hash from lib.feed_utils import construct_feed_from, is_parsing_ok -from lib.article_utils import construct_article, process_filters, FiltersAction +from lib.article_utils import construct_article, get_skip_and_ids logger = logging.getLogger(__name__) logging.captureWarnings(True) @@ -59,7 +59,7 @@ def query_jarr(self, method, urn, data=None): def wait(self, max_wait=600, wait_for=2, checks=2): start, checked = datetime.now(), 0 max_wait_delta = timedelta(seconds=max_wait) - time.sleep(wait_for * 3) + time.sleep(wait_for * 2) while True: time.sleep(wait_for) # checking not thread is still running @@ -122,11 +122,6 @@ def callback(self, response): entries[-1]['user_id'], id_to_create) self.query_jarr('post', 'articles', entries) - logger.debug('%r %r - updating feed etag %r last_mod %r', - self.feed['id'], self.feed['title'], - self.headers.get('etag', ''), - self.headers.get('last-modified', '')) - up_feed = {'etag': self.headers.get('etag', ''), 'last_modified': self.headers.get('last-modified', strftime('%a, %d %b %Y %X %Z', gmtime()))} @@ -150,6 +145,9 @@ def callback(self, response): # re-getting that feed earlier since new entries appeared if article_created: up_feed['last_retrieved'] = datetime.utcnow() + else: + logger.info('%r %r - all article matched in db, adding nothing', + self.feed['id'], self.feed['title']) diff_keys = {key for key in up_feed if up_feed[key] != self.feed.get(key)} @@ -224,24 +222,22 @@ def callback(self, response): ids, entries = [], {} parsed_response = feedparser.parse(response.content) + skipped_list = [] for entry in parsed_response['entries']: if not entry: continue - entry_ids = construct_article(entry, self.feed, - {'title', 'entry_id', 'feed_id', 'user_id', 'tags'}, - fetch=False) - skipped, _, _ = process_filters(self.feed['filters'], entry_ids, - {FiltersAction.SKIP}) + skipped, entry_ids = get_skip_and_ids(entry, self.feed) if skipped: + skipped_list.append(entry_ids) logger.debug('%r %r - skipping article', self.feed['id'], self.feed['title']) continue - del entry_ids['tags'] entries[tuple(sorted(entry_ids.items()))] = entry ids.append(entry_ids) - if not ids: - logger.info('%r %r - all articles skipped, adding nothing', - self.feed['id'], self.feed['title']) + if not ids and skipped_list: + logger.debug('%r %r - nothing to add (skipped %r) %r', + self.feed['id'], self.feed['title'], skipped_list, + parsed_response) return logger.debug('%r %r - found %d entries %r', self.feed['id'], self.feed['title'], len(ids), ids) diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index 2009be509..c719217c1 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -139,7 +139,12 @@ def process_filters(filters, article, only_actions=None): if filter_action not in only_actions: logger.debug('ignoring filter %r' % filter_) continue - + if filter_action in {FiltersType.REGEX, FiltersType.MATCH, + FiltersType.EXACT_MATCH} and 'title' not in article: + continue + if filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS} \ + and 'tags' not in article: + continue title = article.get('title', '').lower() tags = [tag.lower() for tag in article.get('tags', [])] if filter_type is FiltersType.REGEX: @@ -169,3 +174,12 @@ def process_filters(filters, article, only_actions=None): logger.info("%r applied on %r", filter_action.value, article.get('link') or article.get('title')) return skipped, read, liked + + +def get_skip_and_ids(entry, feed): + entry_ids = construct_article(entry, feed, + {'entry_id', 'feed_id', 'user_id'}, fetch=False) + skipped, _, _ = process_filters(feed['filters'], + construct_article(entry, feed, {'title', 'tags'}, fetch=False), + {FiltersAction.SKIP}) + return skipped, entry_ids diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index e89423725..e487d44de 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -114,6 +114,14 @@ def test_no_add_feed_skip(self, jarr_updated_callback): resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') self.assertEquals(36, len(resp.json())) FeedController().update({}, {'filters': [{"type": "tag contains", + "action on": "match", + "pattern": "pattern5", + "action": "skipped"}, + {"type": "simple match", + "action on": "match", + "pattern": "pattern5", + "action": "mark as read"}, + {"type": "regex", "action on": "match", "pattern": "pattern5", "action": "skipped"}]}) diff --git a/src/tests/libs/article_utils_test.py b/src/tests/libs/article_utils_test.py index ee584e74d..f914f02c6 100644 --- a/src/tests/libs/article_utils_test.py +++ b/src/tests/libs/article_utils_test.py @@ -1,30 +1,63 @@ from tests.base import JarrFlaskCommon import json from mock import patch +from requests.exceptions import MissingSchema from lib.article_utils import construct_article class ConstructArticleTest(JarrFlaskCommon): + response_url = '//www.pariszigzag.fr/paris-insolite-secret/'\ + 'les-plus-belles-boulangeries-de-paris' - @patch('lib.article_utils.jarr_get') - def test_missing_title(self, jarr_get): + def setUp(self): + self._jarr_get_patch = patch('lib.article_utils.jarr_get') + self.jarr_get_patch = self._jarr_get_patch.start() + + def tearDown(self): + self._jarr_get_patch.stop() + + @staticmethod + def get_entry(): + with open('src/tests/fixtures/article.json') as fd: + return json.load(fd) + + @staticmethod + def get_response(scheme='http:'): class Response: @property def url(self): - return 'http://www.pariszigzag.fr/paris-insolite-secret/'\ - 'les-plus-belles-boulangeries-de-paris' + return scheme + ConstructArticleTest.response_url + @property def content(self): with open('src/tests/fixtures/article.html') as fd: return fd.read() - jarr_get.return_value = Response() - with open('src/tests/fixtures/article.json') as fd: - entry = json.load(fd) + return Response() + + def test_missing_title(self): + self.jarr_get_patch.return_value = self.get_response('http:') + article = construct_article(self.get_entry(), {'id': 1, 'user_id': 1}) + self.assertEquals('http://www.pariszigzag.fr/?p=56413', + article['entry_id']) + self.assertEquals('http:' + self.response_url, article['link']) + self.assertEquals('Les plus belles boulangeries de Paris', + article['title']) + self.assertEquals(1, article['user_id']) + self.assertEquals(1, article['feed_id']) + + def test_missing_scheme(self): + response = self.get_response('https:') + self.jarr_get_patch.side_effect = [MissingSchema, response] + entry = self.get_entry() + entry['link'] = entry['link'][5:] article = construct_article(entry, {'id': 1, 'user_id': 1}) + + self.assertEquals(2, self.jarr_get_patch.call_count) + self.assertEquals(response.url, self.jarr_get_patch.call_args[0][0]) self.assertEquals('http://www.pariszigzag.fr/?p=56413', article['entry_id']) - self.assertEquals(Response().url, article['link']) + self.assertEquals(response.url, article['link']) self.assertEquals('Les plus belles boulangeries de Paris', article['title']) self.assertEquals(1, article['user_id']) From 1b1a56cef416d0001f3c58e0e10b134b172d1dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 12 Oct 2016 09:55:22 +0200 Subject: [PATCH 028/164] adding tests --- src/tests/controllers/feed_test.py | 26 ++++++++++++++++++++++++++ src/tests/fixtures/filler.py | 23 +++++++++++++---------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/tests/controllers/feed_test.py b/src/tests/controllers/feed_test.py index 9055e85a9..9561296ec 100644 --- a/src/tests/controllers/feed_test.py +++ b/src/tests/controllers/feed_test.py @@ -13,6 +13,32 @@ def test_delete(self): self.assertEquals(0, ClusterController(2).read().count()) self.assertEquals(0, ArticleController(2).read().count()) + def test_delete_main_cluster_handling(self): + clu = ClusterController().get(id=10) + old_title = clu.main_title + old_feed_title, old_art_id = clu.main_feed_title, clu.main_article_id + self.assertEquals(2, len(clu.articles)) + FeedController(clu.user_id).delete(clu.main_article.feed_id) + new_cluster = ClusterController(clu.user_id).get(id=clu.id) + self.assertEquals(1, len(new_cluster.articles)) + self.assertNotEquals(old_title, new_cluster.main_title) + self.assertNotEquals(old_feed_title, new_cluster.main_feed_title) + self.assertNotEquals(old_art_id, new_cluster.main_article_id) + + def test_delete_cluster_handling(self): + clu = ClusterController().get(id=10) + old_title = clu.main_title + old_feed_title, old_art_id = clu.main_feed_title, clu.main_article_id + self.assertEquals(2, len(clu.articles)) + FeedController(clu.user_id).delete( # deleting not main article + next(art.feed_id for art in clu.articles + if art.id != clu.main_article_id)) + new_cluster = ClusterController(clu.user_id).get(id=clu.id) + self.assertEquals(1, len(new_cluster.articles)) + self.assertEquals(old_title, new_cluster.main_title) + self.assertEquals(old_feed_title, new_cluster.main_feed_title) + self.assertEquals(old_art_id, new_cluster.main_article_id) + def test_feed_rights(self): feed = FeedController(2).read()[0].dump() self.assertTrue(3, diff --git a/src/tests/fixtures/filler.py b/src/tests/fixtures/filler.py index d6abba09e..e384da592 100644 --- a/src/tests/fixtures/filler.py +++ b/src/tests/fixtures/filler.py @@ -1,3 +1,4 @@ +from datetime import datetime, timedelta from manager import db_create, db_empty from web.controllers import UserController, CategoryController, \ FeedController, ArticleController @@ -13,19 +14,20 @@ def populate_db(): user1, user2 = [ucontr.create(login=name, email="%s@test.te" % name, password=name) for name in ["user1", "user2"]] - - def to_name(u, c=None, f=None, a=None, *args): - string = u.login - if c: - string += " cat%s" % c - if f is not None: - string += " feed%s" % f - if a is not None: - string += " art%s" % a - return string + ''.join(args) + now = datetime.now() for k in range(2): article_total = 0 + + def to_name(u, c=None, f=None, a=None, *args): + string = "i%d %s" % (k, u.login) + if c: + string += " cat%s" % c + if f is not None: + string += " feed%s" % f + if a is not None: + string += " art%s" % a + return string + ''.join(args) for user in (user1, user2): for i in range(3): cat_id = None @@ -44,6 +46,7 @@ def to_name(u, c=None, f=None, a=None, *args): tags=[to_name(user, i, i, j, '1'), to_name(user, i, i, j, '2')], category_id=cat_id, title=entry, + date=now + timedelta(seconds=k), content="content %d" % article_total) def reset_db(): From 3a5b1d4bafd960a34caaad6624a386bdc51cd238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 12 Oct 2016 10:35:17 +0200 Subject: [PATCH 029/164] fix for reading cluster when not admin --- src/web/models/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/models/cluster.py b/src/web/models/cluster.py index f531b92e0..a13c1ed39 100644 --- a/src/web/models/cluster.py +++ b/src/web/models/cluster.py @@ -72,7 +72,7 @@ def _fields_base_write(): def _fields_base_read(): return {'id', 'user_id', 'categories_id', 'feeds_id', 'main_link', 'main_title', 'main_feed_title', 'main_date', - 'created_date', 'cluster_type', 'articles'} + 'created_date', 'cluster_type', 'articles', 'main_article_id'} def __repr__(self): return "" \ From ccb27e52ab947363323e4109904d59bc919471a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 12 Oct 2016 11:49:49 +0200 Subject: [PATCH 030/164] getting rid of dead code --- src/web/var/english-stop-words.txt | 311 ------------------ src/web/var/french-stop-words.txt | 176 ---------- src/web/var/generate-top-words-list.sh | 8 - .../stop_words/english-stop-words-list.txt | 1 - .../var/stop_words/french-stop-words-list.txt | 1 - 5 files changed, 497 deletions(-) delete mode 100644 src/web/var/english-stop-words.txt delete mode 100644 src/web/var/french-stop-words.txt delete mode 100755 src/web/var/generate-top-words-list.sh delete mode 100644 src/web/var/stop_words/english-stop-words-list.txt delete mode 100644 src/web/var/stop_words/french-stop-words-list.txt diff --git a/src/web/var/english-stop-words.txt b/src/web/var/english-stop-words.txt deleted file mode 100644 index 497a1f965..000000000 --- a/src/web/var/english-stop-words.txt +++ /dev/null @@ -1,311 +0,0 @@ - - | An English stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | Many of the forms below are quite rare (e.g. "yourselves") but included for - | completeness. - - | PRONOUNS FORMS - | 1st person sing - -i | subject, always in upper case of course - -me | object -my | possessive adjective - | the possessive pronoun `mine' is best suppressed, because of the - | sense of coal-mine etc. -myself | reflexive - | 1st person plural -we | subject - -| us | object - | care is required here because US = United States. It is usually - | safe to remove it if it is in lower case. -our | possessive adjective -ours | possessive pronoun -ourselves | reflexive - | second person (archaic `thou' forms not included) -you | subject and object -your | possessive adjective -yours | possessive pronoun -yourself | reflexive (singular) -yourselves | reflexive (plural) - | third person singular -he | subject -him | object -his | possessive adjective and pronoun -himself | reflexive - -she | subject -her | object and possessive adjective -hers | possessive pronoun -herself | reflexive - -it | subject and object -its | possessive adjective -itself | reflexive - | third person plural -they | subject -them | object -their | possessive adjective -theirs | possessive pronoun -themselves | reflexive - | other forms (demonstratives, interrogatives) -what -which -who -whom -this -that -these -those - - | VERB FORMS (using F.R. Palmer's nomenclature) - | BE -am | 1st person, present -is | -s form (3rd person, present) -are | present -was | 1st person, past -were | past -be | infinitive -been | past participle -being | -ing form - | HAVE -have | simple -has | -s form -had | past -having | -ing form - | DO -do | simple -does | -s form -did | past -doing | -ing form - - | The forms below are, I believe, best omitted, because of the significant - | homonym forms: - - | He made a WILL - | old tin CAN - | merry month of MAY - | a smell of MUST - | fight the good fight with all thy MIGHT - - | would, could, should, ought might however be included - - | | AUXILIARIES - | | WILL - |will - -would - - | | SHALL - |shall - -should - - | | CAN - |can - -could - - | | MAY - |may - |might - | | MUST - |must - | | OUGHT - -ought - - | COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing - | pronoun + verb - -i'm -you're -he's -she's -it's -we're -they're -i've -you've -we've -they've -i'd -you'd -he'd -she'd -we'd -they'd -i'll -you'll -he'll -she'll -we'll -they'll - - | verb + negation - -isn't -aren't -wasn't -weren't -hasn't -haven't -hadn't -doesn't -don't -didn't - - | auxiliary + negation - -won't -wouldn't -shan't -shouldn't -can't -cannot -couldn't -mustn't - - | miscellaneous forms - -let's -that's -who's -what's -here's -there's -when's -where's -why's -how's - - | rarer forms - - | daren't needn't - - | doubtful forms - - | oughtn't mightn't - - | ARTICLES -a -an -the - - | THE REST (Overlap among prepositions, conjunctions, adverbs etc is so - | high, that classification is pointless.) -and -but -if -or -because -as -until -while - -of -at -by -for -with -about -against -between -into -through -during -before -after -above -below -to -from -up -down -in -out -on -off -over -under - -again -further -then -once - -here -there -when -where -why -how - -all -any -both -each -few -more -most -other -some -such - -no -nor -not -only -own -same -so -than -too -very - - | Just for the record, the following words are among the commonest in English - - | one - | every - | least - | less - | many - | now - | ever - | never - | say - | says - | said - | also - | get - | go - | goes - | just - | made - | make - | put - | see - | seen - | whether - | like - | well - | back - | even - | still - | way - | take - | since - | another - | however - | two - | three - | four - | five - | first - | second - | new - | old - | high - | long \ No newline at end of file diff --git a/src/web/var/french-stop-words.txt b/src/web/var/french-stop-words.txt deleted file mode 100644 index 08a2f5d74..000000000 --- a/src/web/var/french-stop-words.txt +++ /dev/null @@ -1,176 +0,0 @@ - - | A French stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -au | a + le -aux | a + les -avec | with -ce | this -ces | these -dans | with -de | of -des | de + les -du | de + le -elle | she -en | `of them' etc -et | and -eux | them -il | he -je | I -la | the -le | the -leur | their -lui | him -ma | my (fem) -mais | but -me | me -même | same; as in moi-même (myself) etc -mes | me (pl) -moi | me -mon | my (masc) -ne | not -nos | our (pl) -notre | our -nous | we -on | one -ou | where -par | by -pas | not -pour | for -qu | que before vowel -que | that -qui | who -sa | his, her (fem) -se | oneself -ses | his (pl) -son | his, her (masc) -sur | on -ta | thy (fem) -te | thee -tes | thy (pl) -toi | thee -ton | thy (masc) -tu | thou -un | a -une | a -vos | your (pl) -votre | your -vous | you - - | single letter forms - -c | c' -d | d' -j | j' -l | l' -à | to, at -m | m' -n | n' -s | s' -t | t' -y | there - - | forms of être (not including the infinitive): -été -étée -étées -étés -étant -suis -es -est -sommes -êtes -sont -serai -seras -sera -serons -serez -seront -serais -serait -serions -seriez -seraient -étais -était -étions -étiez -étaient -fus -fut -fûmes -fûtes -furent -sois -soit -soyons -soyez -soient -fusse -fusses -fût -fussions -fussiez -fussent - - | forms of avoir (not including the infinitive): -ayant -eu -eue -eues -eus -ai -as -avons -avez -ont -aurai -auras -aura -aurons -aurez -auront -aurais -aurait -aurions -auriez -auraient -avais -avait -avions -aviez -avaient -eut -eûmes -eûtes -eurent -aie -aies -ait -ayons -ayez -aient -eusse -eusses -eût -eussions -eussiez -eussent - - | Later additions (from Jean-Christophe Deschamps) -ceci | this -celà | that -cet | this -cette | this -ici | here -ils | they -les | the (pl) -leurs | their (pl) -quel | which -quels | which -quelle | which -quelles | which -sans | without -soi | oneself \ No newline at end of file diff --git a/src/web/var/generate-top-words-list.sh b/src/web/var/generate-top-words-list.sh deleted file mode 100755 index 2a87e1472..000000000 --- a/src/web/var/generate-top-words-list.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if test $# != 2 ; then - echo No input files given 1>&2 - exit 1 -fi - -awk 'BEGIN{FS = " "} { if ($1 ~ /^[A-Za-z]/) {print $1}}' $1 | sort | tr '\n' ';' > $2 \ No newline at end of file diff --git a/src/web/var/stop_words/english-stop-words-list.txt b/src/web/var/stop_words/english-stop-words-list.txt deleted file mode 100644 index caa26aaf7..000000000 --- a/src/web/var/stop_words/english-stop-words-list.txt +++ /dev/null @@ -1 +0,0 @@ -a;about;above;after;again;against;all;am;an;and;any;are;aren't;as;at;be;because;been;before;being;below;between;both;but;by;cannot;can't;could;couldn't;did;didn't;do;does;doesn't;doing;don't;down;during;each;few;for;from;further;had;hadn't;has;hasn't;have;haven't;having;he;he'd;he'll;her;here;here's;hers;herself;he's;him;himself;his;how;how's;i;i'd;if;i'll;i'm;in;into;is;isn't;it;its;it's;itself;i've;let's;me;more;most;mustn't;my;myself;no;nor;not;of;off;on;once;only;or;other;ought;our;ours;ourselves;out;over;own;same;shan't;she;she'd;she'll;she's;should;shouldn't;slashdot;so;some;such;than;that;that's;the;their;theirs;them;themselves;then;there;there's;these;they;they'd;they'll;they're;they've;this;those;through;to;too;under;until;up;very;was;wasn't;we;we'd;we'll;were;we're;weren't;we've;what;what's;when;when's;where;where's;which;while;who;whom;who's;why;why's;with;won't;would;wouldn't;writes;you;you'd;you'll;your;you're;yours;yourself;yourselves;you've; diff --git a/src/web/var/stop_words/french-stop-words-list.txt b/src/web/var/stop_words/french-stop-words-list.txt deleted file mode 100644 index a6a36c799..000000000 --- a/src/web/var/stop_words/french-stop-words-list.txt +++ /dev/null @@ -1 +0,0 @@ -à;ai;aie;aient;aies;ait;as;au;aura;aurai;auraient;aurais;aurait;auras;aurez;auriez;aurions;aurons;auront;aux;avaient;avais;avait;avec;avez;aviez;avions;avons;ayant;ayez;ayons;c;ce;ceci;celà;ces;cet;cette;d;dans;de;des;du;elle;en;es;est;et;étaient;étais;était;étant;été;étée;étées;êtes;étés;étiez;étions;eu;eue;eues;eûmes;eurent;eus;eusse;eussent;eusses;eussiez;eussions;eut;eût;eûtes;eux;fûmes;furent;fus;fusse;fussent;fusses;fussiez;fussions;fut;fût;fûtes;ici;il;ils;j;je;l;la;le;les;leur;leurs;lui;m;ma;mais;me;même;mes;moi;mon;n;ne;nos;notre;nous;on;ont;ou;par;pas;pour;qu;que;quel;quelle;quelles;quels;qui;s;sa;sans;se;sera;serai;seraient;serais;serait;seras;serez;seriez;serions;serons;seront;ses;soi;soient;sois;soit;sommes;son;sont;soyez;soyons;suis;sur;t;ta;te;tes;toi;ton;tu;toujours;un;une;vos;votre;vous;y; From cadd132c87ee05058e9d196151fa47565bb41c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Wed, 12 Oct 2016 17:41:16 +0200 Subject: [PATCH 031/164] miscellaneous fixes * fixing building feed: * won't override link * will update all other attributes * won't erase icon_url with broken ones * feed dict instance isn't modified anymore * improving the crawler waiting method... again * improving the crawler feed updating * adding test coverage * fixing use cases when adding feed through bookmarklet * fixing clustering: not clustering article too far away in time --- src/crawler/http_crawler.py | 92 ++++++++++++++-------------- src/lib/feed_utils.py | 87 +++++++++++++------------- src/lib/utils.py | 2 +- src/tests/crawler_test.py | 2 +- src/tests/libs/article_utils_test.py | 11 ++-- src/tests/libs/feed_utils_test.py | 4 +- src/web/controllers/cluster.py | 5 ++ src/web/controllers/feed.py | 2 +- src/web/views/feed.py | 2 +- 9 files changed, 110 insertions(+), 97 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 914504c50..daa1318a7 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -12,7 +12,6 @@ to create the missing entries """ -import html import time import json import logging @@ -28,12 +27,12 @@ logger = logging.getLogger(__name__) logging.captureWarnings(True) +FUTURES = [] class AbstractCrawler: pool = ThreadPoolExecutor(max_workers=conf.CRAWLER_NBWORKER) session = FuturesSession(executor=pool) - _futures = [] def __init__(self, auth): self.auth = auth @@ -53,10 +52,10 @@ def query_jarr(self, method, urn, data=None): data=json.dumps(data, default=default_handler), headers={'Content-Type': 'application/json', 'User-Agent': conf.CRAWLER_USER_AGENT}) - self._futures.append(future) + FUTURES.append(future) return future - def wait(self, max_wait=600, wait_for=2, checks=2): + def wait(self, max_wait=600, wait_for=2, checks=10): start, checked = datetime.now(), 0 max_wait_delta = timedelta(seconds=max_wait) time.sleep(wait_for * 2) @@ -65,21 +64,21 @@ def wait(self, max_wait=600, wait_for=2, checks=2): # checking not thread is still running # some thread are running and we are not behind if datetime.now() - start <= max_wait_delta \ - and any(fu.running() for fu in self._futures): + and any(fu.running() for fu in FUTURES): # let's wait and see if it's not okay next time continue - if checks == checked: + if checks > checked: checked += 1 continue # all thread are done, let's exit - if all(fu.done() for fu in self._futures): + if all(fu.done() for fu in FUTURES): break # some thread are still running and we're gonna future.wait on 'em wait_minus_passed = max_wait - (datetime.now() - start).seconds if wait_minus_passed > 0: max_wait = wait_minus_passed try: # no idea why wait throw ValueError around - wait(self._futures, timeout=max_wait) + wait(FUTURES, timeout=max_wait) except ValueError: logger.exception('something bad happened:') break @@ -114,51 +113,53 @@ def callback(self, response): entries = [] for id_to_create in results: article_created = True - entries.append(construct_article( + entry = construct_article( self.entries[tuple(sorted(id_to_create.items()))], - self.feed)) + self.feed) logger.info('%r %r - creating %r for %r - %r', self.feed['id'], - self.feed['title'], entries[-1]['title'], - entries[-1]['user_id'], id_to_create) + self.feed['title'], entry.get('title'), + entry.get('user_id'), id_to_create) + entries.append(entry) self.query_jarr('post', 'articles', entries) - up_feed = {'etag': self.headers.get('etag', ''), - 'last_modified': self.headers.get('last-modified', - strftime('%a, %d %b %Y %X %Z', gmtime()))} - if not is_parsing_ok(self.parsed_feed): - up_feed['last_error'] = str(self.parsed_feed.get('bozo_exception')) - up_feed['error_count'] = self.feed['error_count'] + 1 - return self.query_jarr('put', 'feed/%d' % self.feed['id'], up_feed) - - fresh_feed = construct_feed_from(url=self.feed['link'], - fp_parsed=self.parsed_feed, - feed=self.feed) - if fresh_feed.get('description'): - fresh_feed['description'] \ - = html.unescape(fresh_feed['description']) - - for key in 'description', 'site_link', 'icon_url': - if fresh_feed.get(key) and fresh_feed[key] != self.feed.get(key): - up_feed[key] = fresh_feed[key] + logger.info('%r %r - parsing failed, bumping error count', + self.feed['id'], self.feed['title']) + return self.query_jarr('put', 'feed/%d' % self.feed['id'], + {'last_error': str(self.parsed_feed.get('bozo_exception')), + 'error_count': self.feed['error_count'] + 1}) + + up_feed = construct_feed_from(url=self.feed['link'], + fp_parsed=self.parsed_feed, + feed=self.feed) + + up_feed['etag'] = self.headers.get('etag', '') + up_feed['last_modified'] = self.headers.get('last-modified', + strftime('%a, %d %b %Y %X %Z', gmtime())) + + up_feed.pop('link', None) # not updating feed url up_feed['user_id'] = self.feed['user_id'] - # re-getting that feed earlier since new entries appeared - if article_created: - up_feed['last_retrieved'] = datetime.utcnow() - else: + up_feed['last_error'] = None + up_feed['error_count'] = 0 + if not article_created: logger.info('%r %r - all article matched in db, adding nothing', self.feed['id'], self.feed['title']) - diff_keys = {key for key in up_feed - if up_feed[key] != self.feed.get(key)} - if not diff_keys: + # updating only changed attrs + up_feed = {key: value for key, value in up_feed.items() + if up_feed[key] != self.feed.get(key)} + if not up_feed: + logger.info('%r %r - nothing to update in feed attrs', + self.feed['id'], self.feed['title']) return # no change in the feed, no update - if not article_created and diff_keys == {'last_modified', 'etag'}: + if not article_created and set(up_feed) == {'last_modified', 'etag'}: + logger.info('%r %r - feed changed attrs are meaningless, ignoring', + self.feed['id'], self.feed['title']) return # meaningless if no new article has been published logger.info('%r %r - pushing feed attrs %r', self.feed['id'], self.feed['title'], {key: "%r -> %r" % (self.feed.get(key), up_feed[key]) - for key in up_feed if up_feed[key] != self.feed.get(key)}) + for key in up_feed}) self.query_jarr('put', 'feed/%d' % self.feed['id'], up_feed) @@ -173,7 +174,7 @@ def clean_feed(self): """Will reset the errors counters on a feed that have known errors""" if self.feed.get('error_count') or self.feed.get('last_error'): self.query_jarr('put', 'feed/%d' % self.feed['id'], - {'error_count': 0, 'last_error': ''}) + {'error_count': 0, 'last_error': None}) def callback(self, response): """will fetch the feed and interprete results (304, etag) or will @@ -183,9 +184,10 @@ def callback(self, response): response.raise_for_status() except Exception as error: error_count = self.feed['error_count'] + 1 - logger.warn('%r %r - an error occured while fetching ' - 'feed; bumping error count to %r', - self.feed['id'], self.feed['title'], error_count) + if self.feed['error_count'] > conf.FEED_ERROR_THRESHOLD: + logger.warn('%r %r - an error occured while fetching ' + 'feed; bumping error count to %r', + self.feed['id'], self.feed['title'], error_count) future = self.query_jarr('put', 'feed/%d' % self.feed['id'], {'error_count': error_count, 'last_error': str(error), @@ -221,7 +223,7 @@ def callback(self, response): self.feed['id'], self.feed['title']) ids, entries = [], {} - parsed_response = feedparser.parse(response.content) + parsed_response = feedparser.parse(response.content.strip()) skipped_list = [] for entry in parsed_response['entries']: if not entry: @@ -283,7 +285,7 @@ def callback(self, response): future = self.session.get(feed['link'], timeout=conf.CRAWLER_TIMEOUT, headers=self.prepare_headers(feed)) - self._futures.append(future) + FUTURES.append(future) feed_crwlr = FeedCrawler(feed, self.auth) future.add_done_callback(feed_crwlr.callback) diff --git a/src/lib/feed_utils.py b/src/lib/feed_utils.py index ff1885b23..005d3a443 100644 --- a/src/lib/feed_utils.py +++ b/src/lib/feed_utils.py @@ -2,6 +2,7 @@ import urllib import logging import feedparser +from copy import deepcopy from bootstrap import conf from bs4 import BeautifulSoup, SoupStrainer @@ -25,7 +26,7 @@ def _escape_title_and_desc(feed): def _browse_feedparser_feed(feed, check, default=None): - if feed.get('feed', {}).get('links') is None: + if not feed.get('feed', {}).get('links'): return default for link in feed['feed']['links']: if check(link): @@ -33,6 +34,16 @@ def _browse_feedparser_feed(feed, check, default=None): return default +def get_parsed_feed(url): + try: + fp_parsed = feedparser.parse(url, + request_headers={'User-Agent': conf.CRAWLER_USER_AGENT}) + except Exception as error: + logger.warn('failed to retreive that url: %r', error) + fp_parsed = {'bozo': 1, 'feed': {}, 'entries': []} + return fp_parsed + + def construct_feed_from(url=None, fp_parsed=None, feed=None): """ Will try to construct the most complete feed dict possible. @@ -41,45 +52,29 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None): fp_parsed: a feedparser object previously obtained feed: an existing feed dict, will be updated """ - feed = feed or {'link': url, 'site_link': url} - if not url and fp_parsed: + feed = deepcopy(feed) if feed else {} + if not url and hasattr(fp_parsed, 'get') and fp_parsed.get('href'): url = fp_parsed.get('href') # we'll try to obtain our first parsing from feedparser if url and not fp_parsed: - try: - fp_parsed = feedparser.parse(url, - request_headers={'User-Agent': conf.CRAWLER_USER_AGENT}) - except Exception as error: - logger.warn('failed to retreive that url: %r', error) - fp_parsed = {'bozo': 1, 'feed': {}, 'entries': []} + fp_parsed = get_parsed_feed(url) assert url is not None and fp_parsed is not None - if not is_parsing_ok(fp_parsed): - feed['link'] = None - - # updating link - feed['link'] = _browse_feedparser_feed(fp_parsed, - lambda link: link['type'] in FEED_MIMETYPES, - default=feed.get('link')) + if is_parsing_ok(fp_parsed): + feed['link'] = url + else: + # trying to get link url from data parsed by feedparser + feed['link'] = _browse_feedparser_feed(fp_parsed, + lambda link: link['type'] in FEED_MIMETYPES, + default=feed.get('link')) + if feed['link'] and feed['link'] != url: + # trying newly got url + fp_parsed = get_parsed_feed(feed['link']) + if not is_parsing_ok(fp_parsed): # didn't work, no link found + feed['link'] = None + feed['site_link'] = url - # parsing failed but we obtained a new link to try - if not is_parsing_ok(fp_parsed) and feed.get('link') != url: - try: - fp_parsed = feedparser.parse(feed['link'], - request_headers={'User-Agent': conf.CRAWLER_USER_AGENT}) - except Exception as error: - logger.warn('failed to retreive that url: %r', error) - fp_parsed = {'bozo': 1, 'feed': {}, 'entries': []} - url = feed['link'] - - if not is_parsing_ok(fp_parsed): - feed['link'] = None - - # extracting maximum values from parsed feed - feed['link'] = _browse_feedparser_feed(fp_parsed, - lambda link: link['type'] in FEED_MIMETYPES, - default=feed.get('link') or url) if fp_parsed['feed'].get('link'): feed['site_link'] = fp_parsed['feed']['link'] else: @@ -100,17 +95,9 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None): elif fp_parsed['feed'].get('subtitle_detail', {}).get('value'): feed['description'] = fp_parsed['feed']['subtitle_detail']['value'] - feed['icon_url'] = _browse_feedparser_feed(fp_parsed, - lambda link: 'icon' in link['rel'], default=feed.get('icon_url')) - if 'icon_url' not in feed: - del feed['icon_url'] - # trying to make up for missing values feed_split = urllib.parse.urlsplit(url) site_split = None - if not feed.get('site_link') and not feed.get('link'): - feed['site_link'] = url - if feed.get('site_link'): feed['site_link'] = rebuild_url(feed['site_link'], feed_split) site_split = urllib.parse.urlsplit(feed['site_link']) @@ -121,6 +108,24 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None): if feed['icon_url'] is None: del feed['icon_url'] + old_icon_url = feed.get('icon_url') + feed['icon_url'] = _browse_feedparser_feed(fp_parsed, + lambda link: 'icon' in link['rel']) + if feed['icon_url'] and feed['icon_url'] != old_icon_url: + feed['icon_url'] = try_get_icon_url(feed['icon_url'], + site_split, feed_split) + + if not feed['icon_url'] and fp_parsed.get('feed', {}).get('icon'): + feed['icon_url'] = try_get_icon_url(fp_parsed['feed']['icon'], + site_split, feed_split) + if not feed['icon_url'] and fp_parsed.get('feed', {}).get('logo'): + feed['icon_url'] = try_get_icon_url(fp_parsed['feed']['logo'], + site_split, feed_split) + if not feed['icon_url'] and old_icon_url: + feed['icon_url'] = old_icon_url + elif not feed['icon_url']: + del feed['icon_url'] + nothing_to_fill = all(bool(feed.get(key)) for key in ('link', 'title', 'icon_url')) # here we have all we want or we do not have the main url, diff --git a/src/lib/utils.py b/src/lib/utils.py index e6226eb32..16a1a3526 100644 --- a/src/lib/utils.py +++ b/src/lib/utils.py @@ -38,7 +38,7 @@ def rebuild_url(url, base_split): new_split = urllib.parse.SplitResult( scheme=split.scheme or base_split.scheme, netloc=split.netloc or base_split.netloc, - path=split.path, query='', fragment='') + path=split.path, query=split.query, fragment=split.fragment) return urllib.parse.urlunsplit(new_split) diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index e487d44de..5cb713dc8 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -13,7 +13,7 @@ class CrawlerTest(JarrFlaskCommon): def setUp(self): super().setUp() - self.wait_params = {'wait_for': 5, 'max_wait': 30} + self.wait_params = {'wait_for': 1, 'max_wait': 10, 'checks': 1} UserController().update({'login': 'admin'}, {'is_api': True}) self._is_secure_served \ = patch('web.lib.article_cleaner.is_secure_served') diff --git a/src/tests/libs/article_utils_test.py b/src/tests/libs/article_utils_test.py index f914f02c6..60ce3b8d1 100644 --- a/src/tests/libs/article_utils_test.py +++ b/src/tests/libs/article_utils_test.py @@ -1,11 +1,11 @@ -from tests.base import JarrFlaskCommon +import unittest import json from mock import patch from requests.exceptions import MissingSchema from lib.article_utils import construct_article -class ConstructArticleTest(JarrFlaskCommon): +class ConstructArticleTest(unittest.TestCase): response_url = '//www.pariszigzag.fr/paris-insolite-secret/'\ 'les-plus-belles-boulangeries-de-paris' @@ -46,14 +46,15 @@ def test_missing_title(self): self.assertEquals(1, article['feed_id']) def test_missing_scheme(self): - response = self.get_response('https:') - self.jarr_get_patch.side_effect = [MissingSchema, response] + response = self.get_response('http:') + self.jarr_get_patch.side_effect = [ + MissingSchema, MissingSchema, response] entry = self.get_entry() entry['link'] = entry['link'][5:] article = construct_article(entry, {'id': 1, 'user_id': 1}) - self.assertEquals(2, self.jarr_get_patch.call_count) + self.assertEquals(3, self.jarr_get_patch.call_count) self.assertEquals(response.url, self.jarr_get_patch.call_args[0][0]) self.assertEquals('http://www.pariszigzag.fr/?p=56413', article['entry_id']) diff --git a/src/tests/libs/feed_utils_test.py b/src/tests/libs/feed_utils_test.py index 4a42bdf42..5ddc769ab 100644 --- a/src/tests/libs/feed_utils_test.py +++ b/src/tests/libs/feed_utils_test.py @@ -1,8 +1,8 @@ -from tests.base import JarrFlaskCommon +import unittest from lib.feed_utils import construct_feed_from -class ConstructFeedFromTest(JarrFlaskCommon): +class ConstructFeedFromTest(unittest.TestCase): @property def jdh_feed(self): diff --git a/src/web/controllers/cluster.py b/src/web/controllers/cluster.py index 946503337..5ad934225 100644 --- a/src/web/controllers/cluster.py +++ b/src/web/controllers/cluster.py @@ -1,4 +1,5 @@ import logging +from datetime import timedelta from bootstrap import db, SQLITE_ENGINE from sqlalchemy import func, Integer, and_ @@ -18,12 +19,16 @@ class ClusterController(AbstractController): def _get_cluster_by_link(self, article): return self.read(user_id=article.user_id, + main_date__lt=article.date + timedelta(days=7), + main_date__gt=article.date - timedelta(days=7), main_link=article.link).first() def _get_cluster_by_title(self, article): if article.category and article.category.cluster_on_title: try: article = ArticleController(self.user_id).get( + date__lt=article.date + timedelta(days=7), + date__gt=article.date - timedelta(days=7), user_id=article.user_id, category_id=article.category_id, title__ilike=article.title) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index af90df4b1..cc00e510c 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -92,7 +92,7 @@ def __clean_feed_fields(self, attrs): if attrs.get('category_id') == 0: attrs['category_id'] = None if 'filters' in attrs: - attrs['filters'] = [filter_ for filter_ in attrs['filters'] + attrs['filters'] = [filter_ for filter_ in (attrs['filters'] or []) if type(filter_) is dict] def create(self, **attrs): diff --git a/src/web/views/feed.py b/src/web/views/feed.py index ce970126d..b40eb9dcc 100644 --- a/src/web/views/feed.py +++ b/src/web/views/feed.py @@ -57,7 +57,7 @@ def check_feeds(link, site_link): feed = construct_feed_from(url) - existing_feed = check_feeds(feed.get('link'), feed.get('site_link')) + existing_feed = check_feeds(feed.get('link'), feed.get('link')) if existing_feed: return redirect(url_for('home', at='f', ai=existing_feed.id)) From 64b8c5d0a882c07d2a4f9b13119511f6ed121aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 21 Oct 2016 13:25:50 +0200 Subject: [PATCH 032/164] sorting imports --- src/bootstrap.py | 5 +++-- src/crawler/http_crawler.py | 16 +++++++++------- src/lib/article_utils.py | 9 +++++---- src/lib/conf_handling.py | 6 +++--- src/lib/feed_utils.py | 9 +++++---- src/lib/utils.py | 8 +++++--- src/manager.py | 7 ++++--- src/migrations/env.py | 14 ++++++++------ .../versions/122ac0c356c_new_users.py | 7 ++++--- ...fe_changed_the_type_of_the_column_last_.py | 6 ++++-- .../versions/19bdaa6208e_add_icon_column.py | 2 +- ...389c22_remove_email_notification_column.py | 5 +++-- ...a207_mv_icons_from_feed_tbl_to_icon_tbl.py | 2 +- .../versions/3f83bfe93fc_adding_category.py | 5 +++-- .../422da2d0234_adding_filters_field.py | 2 +- .../48f561c0ce6_add_column_entry_id.py | 5 +++-- .../493abdb2b73_readability_integration.py | 5 +++-- ...835c03754c69_enhancing_comparison_style.py | 3 ++- .../versions/9462d9753423_tag_handling.py | 2 +- .../versions/9e3fecc9d031_oauth_linuxfr.py | 5 +++-- .../versions/a7f62d50d366_clustering.py | 6 ++++-- ...ea_adding_feed_and_user_attributes_for_.py | 5 +++-- src/plugins/readability.py | 2 +- src/runserver.py | 4 +++- src/scripts/probes.py | 4 +++- src/tests/api/article_test.py | 2 +- src/tests/api/category_test.py | 2 +- src/tests/api/cluster_test.py | 2 +- src/tests/api/feed_test.py | 2 +- src/tests/base.py | 8 +++++--- src/tests/controllers/article_test.py | 2 +- src/tests/controllers/category_test.py | 4 ++-- src/tests/controllers/cluster_test.py | 7 ++++--- src/tests/controllers/feed_test.py | 4 ++-- src/tests/crawler_test.py | 8 +++++--- src/tests/fixtures/filler.py | 5 +++-- src/tests/libs/article_utils_test.py | 4 +++- src/tests/libs/feed_utils_test.py | 1 + src/tests/views/admin_test.py | 4 ++-- src/tests/views/home_test.py | 4 +++- src/tests/views/user_test.py | 7 ++++--- src/web/controllers/abstract.py | 8 +++++--- src/web/controllers/article.py | 12 +++++++----- src/web/controllers/category.py | 3 ++- src/web/controllers/cluster.py | 14 ++++++++------ src/web/controllers/feed.py | 9 +++++---- src/web/controllers/icon.py | 2 ++ src/web/controllers/user.py | 7 +++++-- src/web/forms.py | 12 +++++++----- src/web/lib/article_cleaner.py | 4 +++- src/web/lib/view_utils.py | 12 +++++++----- src/web/models/article.py | 8 +++++--- src/web/models/category.py | 4 ++-- src/web/models/cluster.py | 6 ++++-- src/web/models/feed.py | 6 ++++-- src/web/models/icon.py | 3 ++- src/web/models/tag.py | 1 + src/web/models/user.py | 5 +++-- src/web/utils.py | 2 +- src/web/views/admin.py | 12 ++++++------ src/web/views/api/article.py | 11 ++++++----- src/web/views/api/category.py | 7 +++---- src/web/views/api/cluster.py | 4 ++-- src/web/views/api/common.py | 9 +++++---- src/web/views/api/feed.py | 11 ++++------- src/web/views/article.py | 2 +- src/web/views/cluster.py | 3 ++- src/web/views/common.py | 8 +++++--- src/web/views/feed.py | 11 +++++------ src/web/views/home.py | 19 ++++++++----------- src/web/views/icon.py | 2 ++ src/web/views/session_mgmt.py | 17 ++++++++--------- src/web/views/user.py | 17 ++++++++--------- src/web/views/views.py | 5 +++-- 74 files changed, 265 insertions(+), 201 deletions(-) diff --git a/src/bootstrap.py b/src/bootstrap.py index 1765ca7e0..83608396b 100644 --- a/src/bootstrap.py +++ b/src/bootstrap.py @@ -3,13 +3,14 @@ # required imports and code exection for basic functionning -import os import logging +import os from urllib.parse import urlparse + from flask import Flask from flask_sqlalchemy import SQLAlchemy -from lib.conf_handling import ConfObject +from lib.conf_handling import ConfObject conf = ConfObject() # handling on the fly migration to new conf style diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index daa1318a7..de7dc9c29 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -12,18 +12,20 @@ to create the missing entries """ -import time import json import logging -import feedparser -from bootstrap import conf -from time import strftime, gmtime +import time +from concurrent.futures import ThreadPoolExecutor, wait from datetime import datetime, timedelta -from concurrent.futures import wait, ThreadPoolExecutor +from time import gmtime, strftime + +import feedparser from requests_futures.sessions import FuturesSession -from lib.utils import default_handler, to_hash -from lib.feed_utils import construct_feed_from, is_parsing_ok + +from bootstrap import conf from lib.article_utils import construct_article, get_skip_and_ids +from lib.feed_utils import construct_feed_from, is_parsing_ok +from lib.utils import default_handler, to_hash logger = logging.getLogger(__name__) logging.captureWarnings(True) diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index c719217c1..4c7742d48 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -1,12 +1,13 @@ -import re import html import logging +import re +from datetime import datetime, timezone from enum import Enum +from urllib.parse import SplitResult, urlsplit, urlunsplit + import dateutil.parser -from urllib.parse import urlsplit, urlunsplit, SplitResult -from requests.exceptions import MissingSchema -from datetime import datetime, timezone from bs4 import BeautifulSoup, SoupStrainer +from requests.exceptions import MissingSchema from bootstrap import conf from lib.utils import jarr_get diff --git a/src/lib/conf_handling.py b/src/lib/conf_handling.py index 13f594781..4f864a8a9 100644 --- a/src/lib/conf_handling.py +++ b/src/lib/conf_handling.py @@ -1,8 +1,8 @@ -import os import json -import random import logging -from os.path import abspath, join, dirname +import os +import random +from os.path import abspath, dirname, join logger = logging.getLogger(__name__) ROOT = abspath(join(dirname(globals()['__file__']), '../../..')) diff --git a/src/lib/feed_utils.py b/src/lib/feed_utils.py index 005d3a443..7f7ac57e1 100644 --- a/src/lib/feed_utils.py +++ b/src/lib/feed_utils.py @@ -1,12 +1,13 @@ import html -import urllib import logging -import feedparser +import urllib from copy import deepcopy -from bootstrap import conf + +import feedparser from bs4 import BeautifulSoup, SoupStrainer -from lib.utils import try_get_icon_url, rebuild_url, jarr_get +from bootstrap import conf +from lib.utils import jarr_get, rebuild_url, try_get_icon_url logger = logging.getLogger(__name__) logging.captureWarnings(True) diff --git a/src/lib/utils.py b/src/lib/utils.py index 16a1a3526..71faaa2b9 100644 --- a/src/lib/utils.py +++ b/src/lib/utils.py @@ -1,12 +1,14 @@ +import logging import re import types import urllib -import logging -import requests from hashlib import md5 -from bootstrap import conf + +import requests from flask import request, url_for +from bootstrap import conf + logger = logging.getLogger(__name__) diff --git a/src/manager.py b/src/manager.py index 1f148f650..8dd0d3b58 100755 --- a/src/manager.py +++ b/src/manager.py @@ -2,13 +2,14 @@ # -*- coding: utf-8 -*- import logging from datetime import datetime, timedelta -from flask_script import Manager + from flask_migrate import Migrate, MigrateCommand +from flask_script import Manager -from bootstrap import application, db, conf import web.models -from web.controllers import FeedController, UserController +from bootstrap import application, conf, db from scripts.probes import ArticleProbe, FeedProbe +from web.controllers import FeedController, UserController logger = logging.getLogger(__name__) Migrate(application, db) diff --git a/src/migrations/env.py b/src/migrations/env.py index 597778f9a..347667300 100644 --- a/src/migrations/env.py +++ b/src/migrations/env.py @@ -1,8 +1,15 @@ import logging -from bootstrap import set_logging + from alembic import context +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app from sqlalchemy import engine_from_config, pool +from bootstrap import set_logging + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -14,11 +21,6 @@ log_level=logging.WARNING) set_logging(modules=('alembic'), log_format=log_format) -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -from flask import current_app config.set_main_option('sqlalchemy.url', current_app.config.get('SQLALCHEMY_DATABASE_URI')) target_metadata = current_app.extensions['migrate'].db.metadata diff --git a/src/migrations/versions/122ac0c356c_new_users.py b/src/migrations/versions/122ac0c356c_new_users.py index 6237615d2..ffa7a57a2 100644 --- a/src/migrations/versions/122ac0c356c_new_users.py +++ b/src/migrations/versions/122ac0c356c_new_users.py @@ -12,10 +12,11 @@ branch_labels = None depends_on = None -from bootstrap import conf -from alembic import op import sqlalchemy as sa -from sqlalchemy.sql import table, column +from alembic import op +from sqlalchemy.sql import column, table + +from bootstrap import conf def get_tables(): diff --git a/src/migrations/versions/17dcb75f3fe_changed_the_type_of_the_column_last_.py b/src/migrations/versions/17dcb75f3fe_changed_the_type_of_the_column_last_.py index 2f5a756a5..309988240 100644 --- a/src/migrations/versions/17dcb75f3fe_changed_the_type_of_the_column_last_.py +++ b/src/migrations/versions/17dcb75f3fe_changed_the_type_of_the_column_last_.py @@ -11,9 +11,11 @@ down_revision = 'cde34831ea' from datetime import datetime -from bootstrap import conf -from alembic import op + import sqlalchemy as sa +from alembic import op + +from bootstrap import conf def upgrade(): diff --git a/src/migrations/versions/19bdaa6208e_add_icon_column.py b/src/migrations/versions/19bdaa6208e_add_icon_column.py index 2efa376f9..35149d530 100644 --- a/src/migrations/versions/19bdaa6208e_add_icon_column.py +++ b/src/migrations/versions/19bdaa6208e_add_icon_column.py @@ -10,8 +10,8 @@ revision = '19bdaa6208e' down_revision = '422da2d0234' -from alembic import op import sqlalchemy as sa +from alembic import op def upgrade(): diff --git a/src/migrations/versions/1b750a389c22_remove_email_notification_column.py b/src/migrations/versions/1b750a389c22_remove_email_notification_column.py index bcb6e084c..240525231 100644 --- a/src/migrations/versions/1b750a389c22_remove_email_notification_column.py +++ b/src/migrations/versions/1b750a389c22_remove_email_notification_column.py @@ -10,9 +10,10 @@ revision = '1b750a389c22' down_revision = '48f561c0ce6' -from bootstrap import conf -from alembic import op import sqlalchemy as sa +from alembic import op + +from bootstrap import conf def upgrade(): diff --git a/src/migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py b/src/migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py index a1b7c157d..b12ecfbb0 100644 --- a/src/migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py +++ b/src/migrations/versions/25ca960a207_mv_icons_from_feed_tbl_to_icon_tbl.py @@ -10,8 +10,8 @@ revision = '25ca960a207' down_revision = '19bdaa6208e' -from alembic import op import sqlalchemy as sa +from alembic import op from bootstrap import conf diff --git a/src/migrations/versions/3f83bfe93fc_adding_category.py b/src/migrations/versions/3f83bfe93fc_adding_category.py index 126acd4dd..3fecf9afa 100644 --- a/src/migrations/versions/3f83bfe93fc_adding_category.py +++ b/src/migrations/versions/3f83bfe93fc_adding_category.py @@ -9,9 +9,10 @@ revision = '3f83bfe93fc' down_revision = '25ca960a207' -from bootstrap import conf -from alembic import op import sqlalchemy as sa +from alembic import op + +from bootstrap import conf def upgrade(): diff --git a/src/migrations/versions/422da2d0234_adding_filters_field.py b/src/migrations/versions/422da2d0234_adding_filters_field.py index bcbdf042d..b40d8cd87 100644 --- a/src/migrations/versions/422da2d0234_adding_filters_field.py +++ b/src/migrations/versions/422da2d0234_adding_filters_field.py @@ -10,8 +10,8 @@ revision = '422da2d0234' down_revision = '17dcb75f3fe' -from alembic import op import sqlalchemy as sa +from alembic import op def upgrade(): diff --git a/src/migrations/versions/48f561c0ce6_add_column_entry_id.py b/src/migrations/versions/48f561c0ce6_add_column_entry_id.py index 6ca8bca94..ca7de975c 100644 --- a/src/migrations/versions/48f561c0ce6_add_column_entry_id.py +++ b/src/migrations/versions/48f561c0ce6_add_column_entry_id.py @@ -12,9 +12,10 @@ branch_labels = None depends_on = None -from bootstrap import conf -from alembic import op import sqlalchemy as sa +from alembic import op + +from bootstrap import conf def upgrade(): diff --git a/src/migrations/versions/493abdb2b73_readability_integration.py b/src/migrations/versions/493abdb2b73_readability_integration.py index 3665dc600..eadc5aa0d 100644 --- a/src/migrations/versions/493abdb2b73_readability_integration.py +++ b/src/migrations/versions/493abdb2b73_readability_integration.py @@ -9,9 +9,10 @@ # revision identifiers, used by Alembic. revision = '493abdb2b73' down_revision = '3f83bfe93fc' -from bootstrap import conf -from alembic import op import sqlalchemy as sa +from alembic import op + +from bootstrap import conf def upgrade(): diff --git a/src/migrations/versions/835c03754c69_enhancing_comparison_style.py b/src/migrations/versions/835c03754c69_enhancing_comparison_style.py index 9924401b7..87fdf06a4 100644 --- a/src/migrations/versions/835c03754c69_enhancing_comparison_style.py +++ b/src/migrations/versions/835c03754c69_enhancing_comparison_style.py @@ -12,8 +12,9 @@ branch_labels = None depends_on = None -from alembic import op import sqlalchemy as sa +from alembic import op + from web.models.article import Article diff --git a/src/migrations/versions/9462d9753423_tag_handling.py b/src/migrations/versions/9462d9753423_tag_handling.py index 8f5434c5f..18735bddc 100644 --- a/src/migrations/versions/9462d9753423_tag_handling.py +++ b/src/migrations/versions/9462d9753423_tag_handling.py @@ -12,8 +12,8 @@ branch_labels = None depends_on = None -from alembic import op import sqlalchemy as sa +from alembic import op def upgrade(): diff --git a/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py b/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py index 178d7de68..5c0f442c6 100644 --- a/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py +++ b/src/migrations/versions/9e3fecc9d031_oauth_linuxfr.py @@ -12,9 +12,10 @@ branch_labels = None depends_on = None -from bootstrap import conf -from alembic import op import sqlalchemy as sa +from alembic import op + +from bootstrap import conf def upgrade(): diff --git a/src/migrations/versions/a7f62d50d366_clustering.py b/src/migrations/versions/a7f62d50d366_clustering.py index 567149dab..df03f1f0a 100644 --- a/src/migrations/versions/a7f62d50d366_clustering.py +++ b/src/migrations/versions/a7f62d50d366_clustering.py @@ -13,9 +13,11 @@ depends_on = None from datetime import datetime -from bootstrap import SQLITE_ENGINE -from alembic import op + import sqlalchemy as sa +from alembic import op + +from bootstrap import SQLITE_ENGINE def upgrade(): diff --git a/src/migrations/versions/cde34831ea_adding_feed_and_user_attributes_for_.py b/src/migrations/versions/cde34831ea_adding_feed_and_user_attributes_for_.py index 63c93bf40..a76f58669 100644 --- a/src/migrations/versions/cde34831ea_adding_feed_and_user_attributes_for_.py +++ b/src/migrations/versions/cde34831ea_adding_feed_and_user_attributes_for_.py @@ -9,11 +9,12 @@ # revision identifiers, used by Alembic. revision = 'cde34831ea' down_revision = '1b750a389c22' -from bootstrap import conf from datetime import datetime -from alembic import op import sqlalchemy as sa +from alembic import op + +from bootstrap import conf def upgrade(): diff --git a/src/plugins/readability.py b/src/plugins/readability.py index 9bcfc9767..0c1d50b4a 100644 --- a/src/plugins/readability.py +++ b/src/plugins/readability.py @@ -1,7 +1,7 @@ import logging -import requests from urllib.parse import urlencode +import requests logger = logging.getLogger(__name__) READABILITY_PARSER = 'https://www.readability.com/api/content/v1/parser?' diff --git a/src/runserver.py b/src/runserver.py index 5734d9826..31c5a1fe0 100755 --- a/src/runserver.py +++ b/src/runserver.py @@ -1,9 +1,11 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- import calendar + from flask import request from flask_babel import Babel -from bootstrap import conf, application + +from bootstrap import application, conf if conf.ON_HEROKU: from flask_sslify import SSLify diff --git a/src/scripts/probes.py b/src/scripts/probes.py index e4b6b93a6..d9c4f815a 100644 --- a/src/scripts/probes.py +++ b/src/scripts/probes.py @@ -1,10 +1,12 @@ #!/usr/bin/python3 import sys from datetime import datetime, timedelta + from flask_script import Command, Option -from web.controllers import FeedController, ArticleController +from web.controllers import ArticleController, FeedController from web.models import User + DEFAULT_HEADERS = {'Content-Type': 'application/json', 'User-Agent': 'munin'} LATE_AFTER = 60 FETCH_RATE = 3 diff --git a/src/tests/api/article_test.py b/src/tests/api/article_test.py index e214fe125..0fef3560a 100644 --- a/src/tests/api/article_test.py +++ b/src/tests/api/article_test.py @@ -1,5 +1,5 @@ -from tests.base import JarrFlaskCommon from tests.api.common import ApiCommon +from tests.base import JarrFlaskCommon from web.controllers import UserController diff --git a/src/tests/api/category_test.py b/src/tests/api/category_test.py index 0f7967c1c..31063596d 100644 --- a/src/tests/api/category_test.py +++ b/src/tests/api/category_test.py @@ -1,5 +1,5 @@ -from tests.base import JarrFlaskCommon from tests.api.common import ApiCommon +from tests.base import JarrFlaskCommon class CategoryApiTest(JarrFlaskCommon, ApiCommon): diff --git a/src/tests/api/cluster_test.py b/src/tests/api/cluster_test.py index 084a66473..583af2353 100644 --- a/src/tests/api/cluster_test.py +++ b/src/tests/api/cluster_test.py @@ -1,5 +1,5 @@ -from tests.base import JarrFlaskCommon from tests.api.common import ApiCommon +from tests.base import JarrFlaskCommon class ClusterApiTest(JarrFlaskCommon, ApiCommon): diff --git a/src/tests/api/feed_test.py b/src/tests/api/feed_test.py index c6d403275..c6837fdd2 100644 --- a/src/tests/api/feed_test.py +++ b/src/tests/api/feed_test.py @@ -1,5 +1,5 @@ -from tests.base import JarrFlaskCommon from tests.api.common import ApiCommon +from tests.base import JarrFlaskCommon from web.controllers import UserController diff --git a/src/tests/base.py b/src/tests/base.py index 65b87e444..051654dfb 100644 --- a/src/tests/base.py +++ b/src/tests/base.py @@ -4,14 +4,16 @@ import json import logging import unittest -from os import path from base64 import b64encode -from runserver import application -from tests.fixtures.filler import populate_db, reset_db +from os import path + from flask_login import login_user, logout_user from werkzeug.exceptions import NotFound from bootstrap import conf +from runserver import application +from tests.fixtures.filler import populate_db, reset_db + logger = logging.getLogger('web') diff --git a/src/tests/controllers/article_test.py b/src/tests/controllers/article_test.py index 8a344d005..b5de78711 100644 --- a/src/tests/controllers/article_test.py +++ b/src/tests/controllers/article_test.py @@ -1,5 +1,5 @@ from tests.base import BaseJarrTest -from web.controllers import UserController, ArticleController, FeedController +from web.controllers import ArticleController, FeedController, UserController class ArticleControllerTest(BaseJarrTest): diff --git a/src/tests/controllers/category_test.py b/src/tests/controllers/category_test.py index 3bf309669..1062aff54 100644 --- a/src/tests/controllers/category_test.py +++ b/src/tests/controllers/category_test.py @@ -1,6 +1,6 @@ from tests.base import BaseJarrTest -from web.controllers import (UserController, CategoryController, - FeedController, ArticleController) +from web.controllers import (ArticleController, CategoryController, + FeedController, UserController) class CategoryControllerTest(BaseJarrTest): diff --git a/src/tests/controllers/cluster_test.py b/src/tests/controllers/cluster_test.py index 1cd1e05f9..fe18bdd69 100644 --- a/src/tests/controllers/cluster_test.py +++ b/src/tests/controllers/cluster_test.py @@ -1,8 +1,9 @@ -from random import randint from datetime import timedelta +from random import randint + from tests.base import BaseJarrTest -from web.controllers import (ArticleController, ClusterController, - FeedController, CategoryController) +from web.controllers import (ArticleController, CategoryController, + ClusterController, FeedController) class ClusterControllerTest(BaseJarrTest): diff --git a/src/tests/controllers/feed_test.py b/src/tests/controllers/feed_test.py index 9561296ec..2be589af6 100644 --- a/src/tests/controllers/feed_test.py +++ b/src/tests/controllers/feed_test.py @@ -1,6 +1,6 @@ from tests.base import BaseJarrTest -from web.controllers import (UserController, FeedController, - ArticleController, ClusterController) +from web.controllers import (ArticleController, ClusterController, + FeedController, UserController) class FeedControllerTest(BaseJarrTest): diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index 5cb713dc8..8b92e2bb5 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -1,11 +1,13 @@ -from tests.base import JarrFlaskCommon import logging -from mock import Mock, patch from datetime import datetime +from mock import Mock, patch + from bootstrap import conf from crawler.http_crawler import CrawlerScheduler -from web.controllers import UserController, FeedController +from tests.base import JarrFlaskCommon +from web.controllers import FeedController, UserController + logger = logging.getLogger('web') diff --git a/src/tests/fixtures/filler.py b/src/tests/fixtures/filler.py index e384da592..7b992f5cf 100644 --- a/src/tests/fixtures/filler.py +++ b/src/tests/fixtures/filler.py @@ -1,7 +1,8 @@ from datetime import datetime, timedelta + from manager import db_create, db_empty -from web.controllers import UserController, CategoryController, \ - FeedController, ArticleController +from web.controllers import (ArticleController, CategoryController, + FeedController, UserController) def populate_db(): diff --git a/src/tests/libs/article_utils_test.py b/src/tests/libs/article_utils_test.py index 60ce3b8d1..d99505b02 100644 --- a/src/tests/libs/article_utils_test.py +++ b/src/tests/libs/article_utils_test.py @@ -1,7 +1,9 @@ -import unittest import json +import unittest + from mock import patch from requests.exceptions import MissingSchema + from lib.article_utils import construct_article diff --git a/src/tests/libs/feed_utils_test.py b/src/tests/libs/feed_utils_test.py index 5ddc769ab..a06b9ab07 100644 --- a/src/tests/libs/feed_utils_test.py +++ b/src/tests/libs/feed_utils_test.py @@ -1,4 +1,5 @@ import unittest + from lib.feed_utils import construct_feed_from diff --git a/src/tests/views/admin_test.py b/src/tests/views/admin_test.py index 06823cfc4..dc5993b33 100644 --- a/src/tests/views/admin_test.py +++ b/src/tests/views/admin_test.py @@ -1,6 +1,7 @@ +from flask_principal import PermissionDenied + from tests.base import JarrFlaskCommon from web.controllers import UserController -from flask_principal import PermissionDenied class BaseUiTest(JarrFlaskCommon): @@ -79,4 +80,3 @@ def test_toggle_user(self): self.assertEquals(302, resp.status_code) self.assertFalse(self.uctrl.get(id=self.user.id).is_active) self.logout() - diff --git a/src/tests/views/home_test.py b/src/tests/views/home_test.py index e21320cd2..9ae9e1687 100644 --- a/src/tests/views/home_test.py +++ b/src/tests/views/home_test.py @@ -1,7 +1,9 @@ import json + from mock import patch + from tests.base import JarrFlaskCommon -from web.controllers import UserController, FeedController +from web.controllers import FeedController, UserController class BaseUiTest(JarrFlaskCommon): diff --git a/src/tests/views/user_test.py b/src/tests/views/user_test.py index ee44f64ca..3aec045a6 100644 --- a/src/tests/views/user_test.py +++ b/src/tests/views/user_test.py @@ -1,9 +1,10 @@ from io import BytesIO + from mock import patch + from tests.base import JarrFlaskCommon -from web.controllers import (UserController, CategoryController, - FeedController, - ClusterController, ArticleController) +from web.controllers import (ArticleController, CategoryController, + ClusterController, FeedController, UserController) class BaseUiTest(JarrFlaskCommon): diff --git a/src/web/controllers/abstract.py b/src/web/controllers/abstract.py index 7d2bcbdc5..44c04e6cc 100644 --- a/src/web/controllers/abstract.py +++ b/src/web/controllers/abstract.py @@ -1,12 +1,14 @@ import logging -import dateutil.parser -from bootstrap import db -from datetime import datetime from collections import defaultdict +from datetime import datetime + +import dateutil.parser from sqlalchemy import and_, or_ from sqlalchemy.ext.associationproxy import AssociationProxy from werkzeug.exceptions import Forbidden, NotFound +from bootstrap import db + logger = logging.getLogger(__name__) diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index c22116efe..941732ca2 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -1,14 +1,16 @@ import logging -import sqlalchemy -from sqlalchemy import func from collections import Counter from datetime import datetime, timedelta +import sqlalchemy +from sqlalchemy import func + from bootstrap import db -from .abstract import AbstractController -from web.controllers import CategoryController, FeedController -from web.models import User, Article from lib.article_utils import process_filters +from web.controllers import CategoryController, FeedController +from web.models import Article, User + +from .abstract import AbstractController logger = logging.getLogger(__name__) diff --git a/src/web/controllers/category.py b/src/web/controllers/category.py index 1196e6cba..27cbbe812 100644 --- a/src/web/controllers/category.py +++ b/src/web/controllers/category.py @@ -1,6 +1,7 @@ -from .abstract import AbstractController from web.models import Category +from .abstract import AbstractController + class CategoryController(AbstractController): _db_cls = Category diff --git a/src/web/controllers/cluster.py b/src/web/controllers/cluster.py index 5ad934225..ca00f118a 100644 --- a/src/web/controllers/cluster.py +++ b/src/web/controllers/cluster.py @@ -1,15 +1,17 @@ import logging from datetime import timedelta -from bootstrap import db, SQLITE_ENGINE -from sqlalchemy import func, Integer, and_ -from sqlalchemy.orm import aliased -from sqlalchemy.sql import select, exists +from sqlalchemy import Integer, and_, func from sqlalchemy.dialects.postgres import ARRAY +from sqlalchemy.orm import aliased +from sqlalchemy.sql import exists, select from werkzeug.exceptions import NotFound -from .abstract import AbstractController -from web.models import Cluster, Article + +from bootstrap import SQLITE_ENGINE, db from web.controllers.article import ArticleController +from web.models import Article, Cluster + +from .abstract import AbstractController logger = logging.getLogger(__name__) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index cc00e510c..65058df7e 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -1,12 +1,13 @@ import logging -from sqlalchemy import and_ -from sqlalchemy.sql import select, update, delete from datetime import datetime, timedelta -from bootstrap import db, conf, SQLITE_ENGINE +from sqlalchemy import and_ +from sqlalchemy.sql import delete, select, update + +from bootstrap import SQLITE_ENGINE, conf, db from web.controllers.abstract import AbstractController from web.controllers.icon import IconController -from web.models import User, Feed, Article, Cluster +from web.models import Article, Cluster, Feed, User logger = logging.getLogger(__name__) DEFAULT_LIMIT = 5 diff --git a/src/web/controllers/icon.py b/src/web/controllers/icon.py index 149a69b1b..1c0d47624 100644 --- a/src/web/controllers/icon.py +++ b/src/web/controllers/icon.py @@ -1,7 +1,9 @@ import base64 + from bootstrap import db from lib.utils import jarr_get from web.models import Icon + from .abstract import AbstractController diff --git a/src/web/controllers/user.py b/src/web/controllers/user.py index 8e70e7a46..97c98e482 100644 --- a/src/web/controllers/user.py +++ b/src/web/controllers/user.py @@ -1,8 +1,11 @@ import logging -from werkzeug import generate_password_hash, check_password_hash -from .abstract import AbstractController + +from werkzeug import check_password_hash, generate_password_hash + from web.models import User +from .abstract import AbstractController + logger = logging.getLogger(__name__) diff --git a/src/web/forms.py b/src/web/forms.py index 1071ea07f..cfb2c9b07 100644 --- a/src/web/forms.py +++ b/src/web/forms.py @@ -1,13 +1,15 @@ -from flask_wtf import Form -from flask import url_for, redirect +from flask import redirect, url_for from flask_babel import lazy_gettext -from werkzeug.exceptions import NotFound -from wtforms import TextField, PasswordField, BooleanField, \ - SubmitField, SelectField, validators, HiddenField +from flask_wtf import Form from flask_wtf.html5 import EmailField +from werkzeug.exceptions import NotFound +from wtforms import (BooleanField, HiddenField, PasswordField, SelectField, + SubmitField, TextField, validators) from web import utils from web.controllers import UserController + + # from flask_wtf import RecaptchaField diff --git a/src/web/lib/article_cleaner.py b/src/web/lib/article_cleaner.py index 9e4e9c09d..6cd484fa4 100644 --- a/src/web/lib/article_cleaner.py +++ b/src/web/lib/article_cleaner.py @@ -1,5 +1,7 @@ -from urllib.parse import unquote, urlparse, urlunparse, ParseResult +from urllib.parse import ParseResult, unquote, urlparse, urlunparse + from bs4 import BeautifulSoup + from bootstrap import is_secure_served HTTPS_IFRAME_DOMAINS = ('vimeo.com', 'youtube.com', 'youtu.be') diff --git a/src/web/lib/view_utils.py b/src/web/lib/view_utils.py index 642470fce..0a7d675d2 100644 --- a/src/web/lib/view_utils.py +++ b/src/web/lib/view_utils.py @@ -1,11 +1,13 @@ -import pytz -from functools import wraps from datetime import datetime -from flask import request, Response, make_response, get_flashed_messages -from flask_babel import get_locale +from functools import wraps + +import pytz from babel.dates import format_datetime, format_timedelta -from web.views.common import jsonify +from flask import Response, get_flashed_messages, make_response, request +from flask_babel import get_locale + from lib.utils import to_hash +from web.views.common import jsonify ACCEPTED_LEVELS = {'success', 'info', 'warning', 'error'} diff --git a/src/web/models/article.py b/src/web/models/article.py index 0e28b0068..9a61c4a4d 100644 --- a/src/web/models/article.py +++ b/src/web/models/article.py @@ -1,8 +1,10 @@ from datetime import datetime -from sqlalchemy import (Column, Index, ForeignKey, - Integer, String, Boolean, DateTime) -from sqlalchemy.orm import relationship + +from sqlalchemy import (Boolean, Column, DateTime, ForeignKey, Index, Integer, + String) from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.orm import relationship + from bootstrap import db from web.models.right_mixin import RightMixin diff --git a/src/web/models/category.py b/src/web/models/category.py index c9ab0ade0..4fe6846be 100644 --- a/src/web/models/category.py +++ b/src/web/models/category.py @@ -1,6 +1,6 @@ -from sqlalchemy import (Index, Column, ForeignKey, - Integer, String, Boolean) +from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, String from sqlalchemy.orm import relationship + from bootstrap import db from web.models.right_mixin import RightMixin diff --git a/src/web/models/cluster.py b/src/web/models/cluster.py index a13c1ed39..8d432f3fb 100644 --- a/src/web/models/cluster.py +++ b/src/web/models/cluster.py @@ -1,7 +1,9 @@ from datetime import datetime -from sqlalchemy import (Column, ForeignKey, Index, - Boolean, String, Integer, DateTime) + +from sqlalchemy import (Boolean, Column, DateTime, ForeignKey, Index, Integer, + String) from sqlalchemy.orm import relationship + from bootstrap import db from web.models.article import Article from web.models.right_mixin import RightMixin diff --git a/src/web/models/feed.py b/src/web/models/feed.py index da47b21a9..cb87d2d88 100644 --- a/src/web/models/feed.py +++ b/src/web/models/feed.py @@ -1,7 +1,9 @@ from datetime import datetime -from sqlalchemy import (Index, Column, ForeignKey, - String, Boolean, Integer, DateTime, PickleType) + +from sqlalchemy import (Boolean, Column, DateTime, ForeignKey, Index, Integer, + PickleType, String) from sqlalchemy.orm import relationship, validates + from bootstrap import db from web.models.right_mixin import RightMixin diff --git a/src/web/models/icon.py b/src/web/models/icon.py index dd257b592..183aefe0e 100644 --- a/src/web/models/icon.py +++ b/src/web/models/icon.py @@ -1,7 +1,8 @@ -from bootstrap import db from sqlalchemy import Column, String from sqlalchemy.orm import relationship +from bootstrap import db + class Icon(db.Model): url = Column(String, primary_key=True) diff --git a/src/web/models/tag.py b/src/web/models/tag.py index a56e00a85..b346581e4 100644 --- a/src/web/models/tag.py +++ b/src/web/models/tag.py @@ -1,5 +1,6 @@ from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.orm import relationship + from bootstrap import db diff --git a/src/web/models/user.py b/src/web/models/user.py index 0d567821c..9bae3e40c 100644 --- a/src/web/models/user.py +++ b/src/web/models/user.py @@ -1,8 +1,9 @@ import re from datetime import datetime -from sqlalchemy import Column, String, Boolean, Integer, DateTime -from sqlalchemy.orm import validates, relationship + from flask_login import UserMixin +from sqlalchemy import Boolean, Column, DateTime, Integer, String +from sqlalchemy.orm import relationship, validates from bootstrap import db from web.models.right_mixin import RightMixin diff --git a/src/web/utils.py b/src/web/utils.py index 968f57fb8..49f0a43b5 100755 --- a/src/web/utils.py +++ b/src/web/utils.py @@ -6,7 +6,7 @@ import logging from collections import Counter -from urllib.parse import urlparse, urljoin +from urllib.parse import urljoin, urlparse import sqlalchemy from flask import request diff --git a/src/web/views/admin.py b/src/web/views/admin.py index baa2b3d4b..c0853219d 100644 --- a/src/web/views/admin.py +++ b/src/web/views/admin.py @@ -1,14 +1,14 @@ import logging -from sqlalchemy import desc from datetime import datetime -from flask import (Blueprint, render_template, redirect, - flash, url_for, request) -from flask_babel import gettext, format_timedelta -from flask_login import login_required, current_user + +from flask import Blueprint, flash, redirect, render_template, request, url_for +from flask_babel import format_timedelta, gettext +from flask_login import current_user, login_required +from sqlalchemy import desc from lib.utils import redirect_url +from web.controllers import ClusterController, FeedController, UserController from web.views.common import admin_permission -from web.controllers import UserController, FeedController, ClusterController logger = logging.getLogger(__name__) admin_bp = Blueprint('admin', __name__, url_prefix='/admin') diff --git a/src/web/views/api/article.py b/src/web/views/api/article.py index 6e895e127..38cfd7143 100644 --- a/src/web/views/api/article.py +++ b/src/web/views/api/article.py @@ -1,13 +1,14 @@ -from bootstrap import conf -import dateutil.parser from datetime import datetime + +import dateutil.parser from flask import current_app from flask_restful import Api -from web.views.common import api_permission +from bootstrap import conf from web.controllers import ArticleController -from web.views.api.common import (PyAggAbstractResource, - PyAggResourceNew, PyAggResourceExisting, PyAggResourceMulti) +from web.views.api.common import (PyAggAbstractResource, PyAggResourceExisting, + PyAggResourceMulti, PyAggResourceNew) +from web.views.common import api_permission class ArticleNewAPI(PyAggResourceNew): diff --git a/src/web/views/api/category.py b/src/web/views/api/category.py index 5d23bbf8d..4e0ab8f90 100644 --- a/src/web/views/api/category.py +++ b/src/web/views/api/category.py @@ -1,11 +1,10 @@ -from bootstrap import conf from flask import current_app from flask_restful import Api +from bootstrap import conf from web.controllers.category import CategoryController -from web.views.api.common import (PyAggResourceNew, - PyAggResourceExisting, - PyAggResourceMulti) +from web.views.api.common import (PyAggResourceExisting, PyAggResourceMulti, + PyAggResourceNew) class CategoryNewAPI(PyAggResourceNew): diff --git a/src/web/views/api/cluster.py b/src/web/views/api/cluster.py index 3af2416e1..d3661d3b2 100644 --- a/src/web/views/api/cluster.py +++ b/src/web/views/api/cluster.py @@ -3,8 +3,8 @@ from bootstrap import conf from web.controllers import ClusterController -from web.views.api.common import ( - PyAggResourceNew, PyAggResourceExisting, PyAggResourceMulti) +from web.views.api.common import (PyAggResourceExisting, PyAggResourceMulti, + PyAggResourceNew) class ClusterNewAPI(PyAggResourceNew): diff --git a/src/web/views/api/common.py b/src/web/views/api/common.py index 4e0cfc8df..5761ebffc 100644 --- a/src/web/views/api/common.py +++ b/src/web/views/api/common.py @@ -20,14 +20,15 @@ """ import logging from functools import wraps -from werkzeug.exceptions import Unauthorized, BadRequest, Forbidden, NotFound + from flask import request -from flask_restful import Resource, reqparse from flask_login import current_user +from flask_restful import Resource, reqparse +from werkzeug.exceptions import BadRequest, Forbidden, NotFound, Unauthorized -from web.views.common import admin_permission, api_permission, \ - login_user_bundle, jsonify from web.controllers import UserController +from web.views.common import (admin_permission, api_permission, jsonify, + login_user_bundle) logger = logging.getLogger(__name__) diff --git a/src/web/views/api/feed.py b/src/web/views/api/feed.py index 254d9401a..2a7564933 100644 --- a/src/web/views/api/feed.py +++ b/src/web/views/api/feed.py @@ -1,14 +1,11 @@ -from bootstrap import conf from flask import current_app from flask_restful import Api +from bootstrap import conf +from web.controllers.feed import DEFAULT_LIMIT, FeedController +from web.views.api.common import (PyAggAbstractResource, PyAggResourceExisting, + PyAggResourceMulti, PyAggResourceNew) from web.views.common import api_permission -from web.controllers.feed import FeedController, DEFAULT_LIMIT - -from web.views.api.common import PyAggAbstractResource, \ - PyAggResourceNew, \ - PyAggResourceExisting, \ - PyAggResourceMulti class FeedNewAPI(PyAggResourceNew): diff --git a/src/web/views/article.py b/src/web/views/article.py index d40e3db46..f48ee97fd 100644 --- a/src/web/views/article.py +++ b/src/web/views/article.py @@ -1,5 +1,5 @@ from flask import Blueprint, render_template -from flask_login import login_required, current_user +from flask_login import current_user, login_required from web.controllers import ArticleController diff --git a/src/web/views/cluster.py b/src/web/views/cluster.py index 80fe264a3..9f140f71f 100644 --- a/src/web/views/cluster.py +++ b/src/web/views/cluster.py @@ -1,5 +1,6 @@ from flask import Blueprint, redirect -from flask_login import login_required, current_user +from flask_login import current_user, login_required + from web.controllers import ClusterController cluster_bp = Blueprint('cluster', __name__, url_prefix='/cluster') diff --git a/src/web/views/common.py b/src/web/views/common.py index d7251579f..2034a62a8 100644 --- a/src/web/views/common.py +++ b/src/web/views/common.py @@ -1,9 +1,11 @@ import json from functools import wraps -from flask import current_app, Response + +from flask import Response, current_app from flask_login import login_user -from flask_principal import (Identity, Permission, RoleNeed, - session_identity_loader, identity_changed) +from flask_principal import (Identity, Permission, RoleNeed, identity_changed, + session_identity_loader) + from lib.utils import default_handler admin_role = RoleNeed('admin') diff --git a/src/web/views/feed.py b/src/web/views/feed.py index b40eb9dcc..f67456953 100644 --- a/src/web/views/feed.py +++ b/src/web/views/feed.py @@ -1,14 +1,13 @@ import logging -from werkzeug.exceptions import BadRequest -from flask import Blueprint, render_template, flash, \ - redirect, request, url_for +from flask import Blueprint, flash, redirect, render_template, request, url_for from flask_babel import gettext -from flask_login import login_required, current_user +from flask_login import current_user, login_required +from werkzeug.exceptions import BadRequest -from web.lib.view_utils import etag_match from lib.feed_utils import construct_feed_from -from web.controllers import FeedController, ClusterController +from web.controllers import ClusterController, FeedController +from web.lib.view_utils import etag_match logger = logging.getLogger(__name__) feeds_bp = Blueprint('feeds', __name__, url_prefix='/feeds') diff --git a/src/web/views/home.py b/src/web/views/home.py index 2f2583386..136ee3e93 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -1,23 +1,20 @@ -import pytz import logging from datetime import datetime -from flask import current_app, render_template, request, flash, url_for -from flask_login import login_required, current_user -from flask_babel import get_locale +import pytz from babel.dates import format_datetime, format_timedelta +from flask import current_app, flash, render_template, request, url_for +from flask_babel import get_locale +from flask_login import current_user, login_required from bootstrap import conf +from plugins import readability +from web.controllers import (ArticleController, CategoryController, + ClusterController, FeedController, UserController) from web.lib.article_cleaner import clean_urls -from web.lib.view_utils import etag_match, clusters_to_json, get_notifications +from web.lib.view_utils import clusters_to_json, etag_match, get_notifications from web.views.common import jsonify -from web.controllers import (UserController, CategoryController, - FeedController, ArticleController, - ClusterController) - -from plugins import readability - localize = pytz.utc.localize logger = logging.getLogger(__name__) diff --git a/src/web/views/icon.py b/src/web/views/icon.py index fd41513b6..892c1de46 100644 --- a/src/web/views/icon.py +++ b/src/web/views/icon.py @@ -1,5 +1,7 @@ import base64 + from flask import Blueprint, Response, request + from web.controllers import IconController from web.lib.view_utils import etag_match diff --git a/src/web/views/session_mgmt.py b/src/web/views/session_mgmt.py index fae7a5aac..4ffb432c3 100644 --- a/src/web/views/session_mgmt.py +++ b/src/web/views/session_mgmt.py @@ -1,21 +1,20 @@ import json import logging -from werkzeug.exceptions import NotFound -from flask import (render_template, flash, session, request, - url_for, redirect, current_app) +from flask import (current_app, flash, redirect, render_template, request, + session, url_for) from flask_babel import gettext -from flask_login import LoginManager, logout_user, \ - login_required, current_user -from flask_principal import (Principal, AnonymousIdentity, UserNeed, +from flask_login import LoginManager, current_user, login_required, logout_user +from flask_principal import (AnonymousIdentity, Principal, UserNeed, identity_changed, identity_loaded, session_identity_loader) +from rauth import OAuth1Service, OAuth2Service +from werkzeug.exceptions import NotFound from bootstrap import conf -from web.views.common import admin_role, api_role, login_user_bundle from web.controllers import UserController -from web.forms import SignupForm, SigninForm -from rauth import OAuth1Service, OAuth2Service +from web.forms import SigninForm, SignupForm +from web.views.common import admin_role, api_role, login_user_bundle Principal(current_app) # Create a permission with a single Need, in this case a RoleNeed. diff --git a/src/web/views/user.py b/src/web/views/user.py index dae1222ef..7875c5755 100644 --- a/src/web/views/user.py +++ b/src/web/views/user.py @@ -1,20 +1,19 @@ -import opml import random from datetime import datetime -from werkzeug.exceptions import NotFound, Forbidden -from flask import (Blueprint, render_template, redirect, - flash, url_for, request, make_response) -from flask_principal import Permission, UserNeed + +import opml +from flask import (Blueprint, flash, make_response, redirect, render_template, + request, url_for) from flask_babel import gettext from flask_login import current_user, login_required, logout_user +from flask_principal import Permission, UserNeed +from werkzeug.exceptions import Forbidden, NotFound from bootstrap import conf from lib import emails +from web.controllers import CategoryController, FeedController, UserController +from web.forms import PasswordModForm, ProfileForm, RecoverPasswordForm from web.views.common import admin_permission, login_user_bundle -from web.controllers import (UserController, CategoryController, - FeedController) - -from web.forms import ProfileForm, PasswordModForm, RecoverPasswordForm users_bp = Blueprint('users', __name__, url_prefix='/users') user_bp = Blueprint('user', __name__, url_prefix='/user') diff --git a/src/web/views/views.py b/src/web/views/views.py index cf8787eb1..db50cb7bf 100644 --- a/src/web/views/views.py +++ b/src/web/views/views.py @@ -1,6 +1,7 @@ import logging -from flask import (request, render_template, flash, - url_for, redirect, current_app) + +from flask import (current_app, flash, redirect, render_template, request, + url_for) from flask_babel import gettext from bootstrap import conf From 0f2c3afdd02d225003d1f2643ebae3c0057d0408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 23 Oct 2016 22:43:50 +0200 Subject: [PATCH 033/164] more dead code removal --- src/web/static/js/bootstrap.js | 2317 ---------------------------- src/web/static/js/bootstrap.min.js | 8 +- src/web/static/js/feed.js | 22 - src/web/templates/layout.html | 4 +- 4 files changed, 2 insertions(+), 2349 deletions(-) delete mode 100644 src/web/static/js/bootstrap.js mode change 100644 => 120000 src/web/static/js/bootstrap.min.js delete mode 100644 src/web/static/js/feed.js diff --git a/src/web/static/js/bootstrap.js b/src/web/static/js/bootstrap.js deleted file mode 100644 index 1c88b71e8..000000000 --- a/src/web/static/js/bootstrap.js +++ /dev/null @@ -1,2317 +0,0 @@ -/*! - * Bootstrap v3.3.4 (http://getbootstrap.com) - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */ - -if (typeof jQuery === 'undefined') { - throw new Error('Bootstrap\'s JavaScript requires jQuery') -} - -+function ($) { - 'use strict'; - var version = $.fn.jquery.split(' ')[0].split('.') - if ((version[0] < 2 && version[1] < 9) || (version[0] == 1 && version[1] == 9 && version[2] < 1)) { - throw new Error('Bootstrap\'s JavaScript requires jQuery version 1.9.1 or higher') - } -}(jQuery); - -/* ======================================================================== - * Bootstrap: transition.js v3.3.4 - * http://getbootstrap.com/javascript/#transitions - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // CSS TRANSITION SUPPORT (Shoutout: http://www.modernizr.com/) - // ============================================================ - - function transitionEnd() { - var el = document.createElement('bootstrap') - - var transEndEventNames = { - WebkitTransition : 'webkitTransitionEnd', - MozTransition : 'transitionend', - OTransition : 'oTransitionEnd otransitionend', - transition : 'transitionend' - } - - for (var name in transEndEventNames) { - if (el.style[name] !== undefined) { - return { end: transEndEventNames[name] } - } - } - - return false // explicit for ie8 ( ._.) - } - - // http://blog.alexmaccaw.com/css-transitions - $.fn.emulateTransitionEnd = function (duration) { - var called = false - var $el = this - $(this).one('bsTransitionEnd', function () { called = true }) - var callback = function () { if (!called) $($el).trigger($.support.transition.end) } - setTimeout(callback, duration) - return this - } - - $(function () { - $.support.transition = transitionEnd() - - if (!$.support.transition) return - - $.event.special.bsTransitionEnd = { - bindType: $.support.transition.end, - delegateType: $.support.transition.end, - handle: function (e) { - if ($(e.target).is(this)) return e.handleObj.handler.apply(this, arguments) - } - } - }) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: alert.js v3.3.4 - * http://getbootstrap.com/javascript/#alerts - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // ALERT CLASS DEFINITION - // ====================== - - var dismiss = '[data-dismiss="alert"]' - var Alert = function (el) { - $(el).on('click', dismiss, this.close) - } - - Alert.VERSION = '3.3.4' - - Alert.TRANSITION_DURATION = 150 - - Alert.prototype.close = function (e) { - var $this = $(this) - var selector = $this.attr('data-target') - - if (!selector) { - selector = $this.attr('href') - selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7 - } - - var $parent = $(selector) - - if (e) e.preventDefault() - - if (!$parent.length) { - $parent = $this.closest('.alert') - } - - $parent.trigger(e = $.Event('close.bs.alert')) - - if (e.isDefaultPrevented()) return - - $parent.removeClass('in') - - function removeElement() { - // detach from parent, fire event then clean up data - $parent.detach().trigger('closed.bs.alert').remove() - } - - $.support.transition && $parent.hasClass('fade') ? - $parent - .one('bsTransitionEnd', removeElement) - .emulateTransitionEnd(Alert.TRANSITION_DURATION) : - removeElement() - } - - - // ALERT PLUGIN DEFINITION - // ======================= - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.alert') - - if (!data) $this.data('bs.alert', (data = new Alert(this))) - if (typeof option == 'string') data[option].call($this) - }) - } - - var old = $.fn.alert - - $.fn.alert = Plugin - $.fn.alert.Constructor = Alert - - - // ALERT NO CONFLICT - // ================= - - $.fn.alert.noConflict = function () { - $.fn.alert = old - return this - } - - - // ALERT DATA-API - // ============== - - $(document).on('click.bs.alert.data-api', dismiss, Alert.prototype.close) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: button.js v3.3.4 - * http://getbootstrap.com/javascript/#buttons - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // BUTTON PUBLIC CLASS DEFINITION - // ============================== - - var Button = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, Button.DEFAULTS, options) - this.isLoading = false - } - - Button.VERSION = '3.3.4' - - Button.DEFAULTS = { - loadingText: 'loading...' - } - - Button.prototype.setState = function (state) { - var d = 'disabled' - var $el = this.$element - var val = $el.is('input') ? 'val' : 'html' - var data = $el.data() - - state = state + 'Text' - - if (data.resetText == null) $el.data('resetText', $el[val]()) - - // push to event loop to allow forms to submit - setTimeout($.proxy(function () { - $el[val](data[state] == null ? this.options[state] : data[state]) - - if (state == 'loadingText') { - this.isLoading = true - $el.addClass(d).attr(d, d) - } else if (this.isLoading) { - this.isLoading = false - $el.removeClass(d).removeAttr(d) - } - }, this), 0) - } - - Button.prototype.toggle = function () { - var changed = true - var $parent = this.$element.closest('[data-toggle="buttons"]') - - if ($parent.length) { - var $input = this.$element.find('input') - if ($input.prop('type') == 'radio') { - if ($input.prop('checked') && this.$element.hasClass('active')) changed = false - else $parent.find('.active').removeClass('active') - } - if (changed) $input.prop('checked', !this.$element.hasClass('active')).trigger('change') - } else { - this.$element.attr('aria-pressed', !this.$element.hasClass('active')) - } - - if (changed) this.$element.toggleClass('active') - } - - - // BUTTON PLUGIN DEFINITION - // ======================== - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.button') - var options = typeof option == 'object' && option - - if (!data) $this.data('bs.button', (data = new Button(this, options))) - - if (option == 'toggle') data.toggle() - else if (option) data.setState(option) - }) - } - - var old = $.fn.button - - $.fn.button = Plugin - $.fn.button.Constructor = Button - - - // BUTTON NO CONFLICT - // ================== - - $.fn.button.noConflict = function () { - $.fn.button = old - return this - } - - - // BUTTON DATA-API - // =============== - - $(document) - .on('click.bs.button.data-api', '[data-toggle^="button"]', function (e) { - var $btn = $(e.target) - if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn') - Plugin.call($btn, 'toggle') - e.preventDefault() - }) - .on('focus.bs.button.data-api blur.bs.button.data-api', '[data-toggle^="button"]', function (e) { - $(e.target).closest('.btn').toggleClass('focus', /^focus(in)?$/.test(e.type)) - }) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: carousel.js v3.3.4 - * http://getbootstrap.com/javascript/#carousel - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // CAROUSEL CLASS DEFINITION - // ========================= - - var Carousel = function (element, options) { - this.$element = $(element) - this.$indicators = this.$element.find('.carousel-indicators') - this.options = options - this.paused = null - this.sliding = null - this.interval = null - this.$active = null - this.$items = null - - this.options.keyboard && this.$element.on('keydown.bs.carousel', $.proxy(this.keydown, this)) - - this.options.pause == 'hover' && !('ontouchstart' in document.documentElement) && this.$element - .on('mouseenter.bs.carousel', $.proxy(this.pause, this)) - .on('mouseleave.bs.carousel', $.proxy(this.cycle, this)) - } - - Carousel.VERSION = '3.3.4' - - Carousel.TRANSITION_DURATION = 600 - - Carousel.DEFAULTS = { - interval: 5000, - pause: 'hover', - wrap: true, - keyboard: true - } - - Carousel.prototype.keydown = function (e) { - if (/input|textarea/i.test(e.target.tagName)) return - switch (e.which) { - case 37: this.prev(); break - case 39: this.next(); break - default: return - } - - e.preventDefault() - } - - Carousel.prototype.cycle = function (e) { - e || (this.paused = false) - - this.interval && clearInterval(this.interval) - - this.options.interval - && !this.paused - && (this.interval = setInterval($.proxy(this.next, this), this.options.interval)) - - return this - } - - Carousel.prototype.getItemIndex = function (item) { - this.$items = item.parent().children('.item') - return this.$items.index(item || this.$active) - } - - Carousel.prototype.getItemForDirection = function (direction, active) { - var activeIndex = this.getItemIndex(active) - var willWrap = (direction == 'prev' && activeIndex === 0) - || (direction == 'next' && activeIndex == (this.$items.length - 1)) - if (willWrap && !this.options.wrap) return active - var delta = direction == 'prev' ? -1 : 1 - var itemIndex = (activeIndex + delta) % this.$items.length - return this.$items.eq(itemIndex) - } - - Carousel.prototype.to = function (pos) { - var that = this - var activeIndex = this.getItemIndex(this.$active = this.$element.find('.item.active')) - - if (pos > (this.$items.length - 1) || pos < 0) return - - if (this.sliding) return this.$element.one('slid.bs.carousel', function () { that.to(pos) }) // yes, "slid" - if (activeIndex == pos) return this.pause().cycle() - - return this.slide(pos > activeIndex ? 'next' : 'prev', this.$items.eq(pos)) - } - - Carousel.prototype.pause = function (e) { - e || (this.paused = true) - - if (this.$element.find('.next, .prev').length && $.support.transition) { - this.$element.trigger($.support.transition.end) - this.cycle(true) - } - - this.interval = clearInterval(this.interval) - - return this - } - - Carousel.prototype.next = function () { - if (this.sliding) return - return this.slide('next') - } - - Carousel.prototype.prev = function () { - if (this.sliding) return - return this.slide('prev') - } - - Carousel.prototype.slide = function (type, next) { - var $active = this.$element.find('.item.active') - var $next = next || this.getItemForDirection(type, $active) - var isCycling = this.interval - var direction = type == 'next' ? 'left' : 'right' - var that = this - - if ($next.hasClass('active')) return (this.sliding = false) - - var relatedTarget = $next[0] - var slideEvent = $.Event('slide.bs.carousel', { - relatedTarget: relatedTarget, - direction: direction - }) - this.$element.trigger(slideEvent) - if (slideEvent.isDefaultPrevented()) return - - this.sliding = true - - isCycling && this.pause() - - if (this.$indicators.length) { - this.$indicators.find('.active').removeClass('active') - var $nextIndicator = $(this.$indicators.children()[this.getItemIndex($next)]) - $nextIndicator && $nextIndicator.addClass('active') - } - - var slidEvent = $.Event('slid.bs.carousel', { relatedTarget: relatedTarget, direction: direction }) // yes, "slid" - if ($.support.transition && this.$element.hasClass('slide')) { - $next.addClass(type) - $next[0].offsetWidth // force reflow - $active.addClass(direction) - $next.addClass(direction) - $active - .one('bsTransitionEnd', function () { - $next.removeClass([type, direction].join(' ')).addClass('active') - $active.removeClass(['active', direction].join(' ')) - that.sliding = false - setTimeout(function () { - that.$element.trigger(slidEvent) - }, 0) - }) - .emulateTransitionEnd(Carousel.TRANSITION_DURATION) - } else { - $active.removeClass('active') - $next.addClass('active') - this.sliding = false - this.$element.trigger(slidEvent) - } - - isCycling && this.cycle() - - return this - } - - - // CAROUSEL PLUGIN DEFINITION - // ========================== - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.carousel') - var options = $.extend({}, Carousel.DEFAULTS, $this.data(), typeof option == 'object' && option) - var action = typeof option == 'string' ? option : options.slide - - if (!data) $this.data('bs.carousel', (data = new Carousel(this, options))) - if (typeof option == 'number') data.to(option) - else if (action) data[action]() - else if (options.interval) data.pause().cycle() - }) - } - - var old = $.fn.carousel - - $.fn.carousel = Plugin - $.fn.carousel.Constructor = Carousel - - - // CAROUSEL NO CONFLICT - // ==================== - - $.fn.carousel.noConflict = function () { - $.fn.carousel = old - return this - } - - - // CAROUSEL DATA-API - // ================= - - var clickHandler = function (e) { - var href - var $this = $(this) - var $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) // strip for ie7 - if (!$target.hasClass('carousel')) return - var options = $.extend({}, $target.data(), $this.data()) - var slideIndex = $this.attr('data-slide-to') - if (slideIndex) options.interval = false - - Plugin.call($target, options) - - if (slideIndex) { - $target.data('bs.carousel').to(slideIndex) - } - - e.preventDefault() - } - - $(document) - .on('click.bs.carousel.data-api', '[data-slide]', clickHandler) - .on('click.bs.carousel.data-api', '[data-slide-to]', clickHandler) - - $(window).on('load', function () { - $('[data-ride="carousel"]').each(function () { - var $carousel = $(this) - Plugin.call($carousel, $carousel.data()) - }) - }) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: collapse.js v3.3.4 - * http://getbootstrap.com/javascript/#collapse - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // COLLAPSE PUBLIC CLASS DEFINITION - // ================================ - - var Collapse = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, Collapse.DEFAULTS, options) - this.$trigger = $('[data-toggle="collapse"][href="#' + element.id + '"],' + - '[data-toggle="collapse"][data-target="#' + element.id + '"]') - this.transitioning = null - - if (this.options.parent) { - this.$parent = this.getParent() - } else { - this.addAriaAndCollapsedClass(this.$element, this.$trigger) - } - - if (this.options.toggle) this.toggle() - } - - Collapse.VERSION = '3.3.4' - - Collapse.TRANSITION_DURATION = 350 - - Collapse.DEFAULTS = { - toggle: true - } - - Collapse.prototype.dimension = function () { - var hasWidth = this.$element.hasClass('width') - return hasWidth ? 'width' : 'height' - } - - Collapse.prototype.show = function () { - if (this.transitioning || this.$element.hasClass('in')) return - - var activesData - var actives = this.$parent && this.$parent.children('.panel').children('.in, .collapsing') - - if (actives && actives.length) { - activesData = actives.data('bs.collapse') - if (activesData && activesData.transitioning) return - } - - var startEvent = $.Event('show.bs.collapse') - this.$element.trigger(startEvent) - if (startEvent.isDefaultPrevented()) return - - if (actives && actives.length) { - Plugin.call(actives, 'hide') - activesData || actives.data('bs.collapse', null) - } - - var dimension = this.dimension() - - this.$element - .removeClass('collapse') - .addClass('collapsing')[dimension](0) - .attr('aria-expanded', true) - - this.$trigger - .removeClass('collapsed') - .attr('aria-expanded', true) - - this.transitioning = 1 - - var complete = function () { - this.$element - .removeClass('collapsing') - .addClass('collapse in')[dimension]('') - this.transitioning = 0 - this.$element - .trigger('shown.bs.collapse') - } - - if (!$.support.transition) return complete.call(this) - - var scrollSize = $.camelCase(['scroll', dimension].join('-')) - - this.$element - .one('bsTransitionEnd', $.proxy(complete, this)) - .emulateTransitionEnd(Collapse.TRANSITION_DURATION)[dimension](this.$element[0][scrollSize]) - } - - Collapse.prototype.hide = function () { - if (this.transitioning || !this.$element.hasClass('in')) return - - var startEvent = $.Event('hide.bs.collapse') - this.$element.trigger(startEvent) - if (startEvent.isDefaultPrevented()) return - - var dimension = this.dimension() - - this.$element[dimension](this.$element[dimension]())[0].offsetHeight - - this.$element - .addClass('collapsing') - .removeClass('collapse in') - .attr('aria-expanded', false) - - this.$trigger - .addClass('collapsed') - .attr('aria-expanded', false) - - this.transitioning = 1 - - var complete = function () { - this.transitioning = 0 - this.$element - .removeClass('collapsing') - .addClass('collapse') - .trigger('hidden.bs.collapse') - } - - if (!$.support.transition) return complete.call(this) - - this.$element - [dimension](0) - .one('bsTransitionEnd', $.proxy(complete, this)) - .emulateTransitionEnd(Collapse.TRANSITION_DURATION) - } - - Collapse.prototype.toggle = function () { - this[this.$element.hasClass('in') ? 'hide' : 'show']() - } - - Collapse.prototype.getParent = function () { - return $(this.options.parent) - .find('[data-toggle="collapse"][data-parent="' + this.options.parent + '"]') - .each($.proxy(function (i, element) { - var $element = $(element) - this.addAriaAndCollapsedClass(getTargetFromTrigger($element), $element) - }, this)) - .end() - } - - Collapse.prototype.addAriaAndCollapsedClass = function ($element, $trigger) { - var isOpen = $element.hasClass('in') - - $element.attr('aria-expanded', isOpen) - $trigger - .toggleClass('collapsed', !isOpen) - .attr('aria-expanded', isOpen) - } - - function getTargetFromTrigger($trigger) { - var href - var target = $trigger.attr('data-target') - || (href = $trigger.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') // strip for ie7 - - return $(target) - } - - - // COLLAPSE PLUGIN DEFINITION - // ========================== - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.collapse') - var options = $.extend({}, Collapse.DEFAULTS, $this.data(), typeof option == 'object' && option) - - if (!data && options.toggle && /show|hide/.test(option)) options.toggle = false - if (!data) $this.data('bs.collapse', (data = new Collapse(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - var old = $.fn.collapse - - $.fn.collapse = Plugin - $.fn.collapse.Constructor = Collapse - - - // COLLAPSE NO CONFLICT - // ==================== - - $.fn.collapse.noConflict = function () { - $.fn.collapse = old - return this - } - - - // COLLAPSE DATA-API - // ================= - - $(document).on('click.bs.collapse.data-api', '[data-toggle="collapse"]', function (e) { - var $this = $(this) - - if (!$this.attr('data-target')) e.preventDefault() - - var $target = getTargetFromTrigger($this) - var data = $target.data('bs.collapse') - var option = data ? 'toggle' : $this.data() - - Plugin.call($target, option) - }) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: dropdown.js v3.3.4 - * http://getbootstrap.com/javascript/#dropdowns - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // DROPDOWN CLASS DEFINITION - // ========================= - - var backdrop = '.dropdown-backdrop' - var toggle = '[data-toggle="dropdown"]' - var Dropdown = function (element) { - $(element).on('click.bs.dropdown', this.toggle) - } - - Dropdown.VERSION = '3.3.4' - - Dropdown.prototype.toggle = function (e) { - var $this = $(this) - - if ($this.is('.disabled, :disabled')) return - - var $parent = getParent($this) - var isActive = $parent.hasClass('open') - - clearMenus() - - if (!isActive) { - if ('ontouchstart' in document.documentElement && !$parent.closest('.navbar-nav').length) { - // if mobile we use a backdrop because click events don't delegate - $('
').insertAfter($(this)).on('click', clearMenus) - } - - var relatedTarget = { relatedTarget: this } - $parent.trigger(e = $.Event('show.bs.dropdown', relatedTarget)) - - if (e.isDefaultPrevented()) return - - $this - .trigger('focus') - .attr('aria-expanded', 'true') - - $parent - .toggleClass('open') - .trigger('shown.bs.dropdown', relatedTarget) - } - - return false - } - - Dropdown.prototype.keydown = function (e) { - if (!/(38|40|27|32)/.test(e.which) || /input|textarea/i.test(e.target.tagName)) return - - var $this = $(this) - - e.preventDefault() - e.stopPropagation() - - if ($this.is('.disabled, :disabled')) return - - var $parent = getParent($this) - var isActive = $parent.hasClass('open') - - if ((!isActive && e.which != 27) || (isActive && e.which == 27)) { - if (e.which == 27) $parent.find(toggle).trigger('focus') - return $this.trigger('click') - } - - var desc = ' li:not(.disabled):visible a' - var $items = $parent.find('[role="menu"]' + desc + ', [role="listbox"]' + desc) - - if (!$items.length) return - - var index = $items.index(e.target) - - if (e.which == 38 && index > 0) index-- // up - if (e.which == 40 && index < $items.length - 1) index++ // down - if (!~index) index = 0 - - $items.eq(index).trigger('focus') - } - - function clearMenus(e) { - if (e && e.which === 3) return - $(backdrop).remove() - $(toggle).each(function () { - var $this = $(this) - var $parent = getParent($this) - var relatedTarget = { relatedTarget: this } - - if (!$parent.hasClass('open')) return - - $parent.trigger(e = $.Event('hide.bs.dropdown', relatedTarget)) - - if (e.isDefaultPrevented()) return - - $this.attr('aria-expanded', 'false') - $parent.removeClass('open').trigger('hidden.bs.dropdown', relatedTarget) - }) - } - - function getParent($this) { - var selector = $this.attr('data-target') - - if (!selector) { - selector = $this.attr('href') - selector = selector && /#[A-Za-z]/.test(selector) && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7 - } - - var $parent = selector && $(selector) - - return $parent && $parent.length ? $parent : $this.parent() - } - - - // DROPDOWN PLUGIN DEFINITION - // ========================== - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.dropdown') - - if (!data) $this.data('bs.dropdown', (data = new Dropdown(this))) - if (typeof option == 'string') data[option].call($this) - }) - } - - var old = $.fn.dropdown - - $.fn.dropdown = Plugin - $.fn.dropdown.Constructor = Dropdown - - - // DROPDOWN NO CONFLICT - // ==================== - - $.fn.dropdown.noConflict = function () { - $.fn.dropdown = old - return this - } - - - // APPLY TO STANDARD DROPDOWN ELEMENTS - // =================================== - - $(document) - .on('click.bs.dropdown.data-api', clearMenus) - .on('click.bs.dropdown.data-api', '.dropdown form', function (e) { e.stopPropagation() }) - .on('click.bs.dropdown.data-api', toggle, Dropdown.prototype.toggle) - .on('keydown.bs.dropdown.data-api', toggle, Dropdown.prototype.keydown) - .on('keydown.bs.dropdown.data-api', '[role="menu"]', Dropdown.prototype.keydown) - .on('keydown.bs.dropdown.data-api', '[role="listbox"]', Dropdown.prototype.keydown) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: modal.js v3.3.4 - * http://getbootstrap.com/javascript/#modals - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // MODAL CLASS DEFINITION - // ====================== - - var Modal = function (element, options) { - this.options = options - this.$body = $(document.body) - this.$element = $(element) - this.$dialog = this.$element.find('.modal-dialog') - this.$backdrop = null - this.isShown = null - this.originalBodyPad = null - this.scrollbarWidth = 0 - this.ignoreBackdropClick = false - - if (this.options.remote) { - this.$element - .find('.modal-content') - .load(this.options.remote, $.proxy(function () { - this.$element.trigger('loaded.bs.modal') - }, this)) - } - } - - Modal.VERSION = '3.3.4' - - Modal.TRANSITION_DURATION = 300 - Modal.BACKDROP_TRANSITION_DURATION = 150 - - Modal.DEFAULTS = { - backdrop: true, - keyboard: true, - show: true - } - - Modal.prototype.toggle = function (_relatedTarget) { - return this.isShown ? this.hide() : this.show(_relatedTarget) - } - - Modal.prototype.show = function (_relatedTarget) { - var that = this - var e = $.Event('show.bs.modal', { relatedTarget: _relatedTarget }) - - this.$element.trigger(e) - - if (this.isShown || e.isDefaultPrevented()) return - - this.isShown = true - - this.checkScrollbar() - this.setScrollbar() - this.$body.addClass('modal-open') - - this.escape() - this.resize() - - this.$element.on('click.dismiss.bs.modal', '[data-dismiss="modal"]', $.proxy(this.hide, this)) - - this.$dialog.on('mousedown.dismiss.bs.modal', function () { - that.$element.one('mouseup.dismiss.bs.modal', function (e) { - if ($(e.target).is(that.$element)) that.ignoreBackdropClick = true - }) - }) - - this.backdrop(function () { - var transition = $.support.transition && that.$element.hasClass('fade') - - if (!that.$element.parent().length) { - that.$element.appendTo(that.$body) // don't move modals dom position - } - - that.$element - .show() - .scrollTop(0) - - that.adjustDialog() - - if (transition) { - that.$element[0].offsetWidth // force reflow - } - - that.$element - .addClass('in') - .attr('aria-hidden', false) - - that.enforceFocus() - - var e = $.Event('shown.bs.modal', { relatedTarget: _relatedTarget }) - - transition ? - that.$dialog // wait for modal to slide in - .one('bsTransitionEnd', function () { - that.$element.trigger('focus').trigger(e) - }) - .emulateTransitionEnd(Modal.TRANSITION_DURATION) : - that.$element.trigger('focus').trigger(e) - }) - } - - Modal.prototype.hide = function (e) { - if (e) e.preventDefault() - - e = $.Event('hide.bs.modal') - - this.$element.trigger(e) - - if (!this.isShown || e.isDefaultPrevented()) return - - this.isShown = false - - this.escape() - this.resize() - - $(document).off('focusin.bs.modal') - - this.$element - .removeClass('in') - .attr('aria-hidden', true) - .off('click.dismiss.bs.modal') - .off('mouseup.dismiss.bs.modal') - - this.$dialog.off('mousedown.dismiss.bs.modal') - - $.support.transition && this.$element.hasClass('fade') ? - this.$element - .one('bsTransitionEnd', $.proxy(this.hideModal, this)) - .emulateTransitionEnd(Modal.TRANSITION_DURATION) : - this.hideModal() - } - - Modal.prototype.enforceFocus = function () { - $(document) - .off('focusin.bs.modal') // guard against infinite focus loop - .on('focusin.bs.modal', $.proxy(function (e) { - if (this.$element[0] !== e.target && !this.$element.has(e.target).length) { - this.$element.trigger('focus') - } - }, this)) - } - - Modal.prototype.escape = function () { - if (this.isShown && this.options.keyboard) { - this.$element.on('keydown.dismiss.bs.modal', $.proxy(function (e) { - e.which == 27 && this.hide() - }, this)) - } else if (!this.isShown) { - this.$element.off('keydown.dismiss.bs.modal') - } - } - - Modal.prototype.resize = function () { - if (this.isShown) { - $(window).on('resize.bs.modal', $.proxy(this.handleUpdate, this)) - } else { - $(window).off('resize.bs.modal') - } - } - - Modal.prototype.hideModal = function () { - var that = this - this.$element.hide() - this.backdrop(function () { - that.$body.removeClass('modal-open') - that.resetAdjustments() - that.resetScrollbar() - that.$element.trigger('hidden.bs.modal') - }) - } - - Modal.prototype.removeBackdrop = function () { - this.$backdrop && this.$backdrop.remove() - this.$backdrop = null - } - - Modal.prototype.backdrop = function (callback) { - var that = this - var animate = this.$element.hasClass('fade') ? 'fade' : '' - - if (this.isShown && this.options.backdrop) { - var doAnimate = $.support.transition && animate - - this.$backdrop = $('
') - .appendTo(this.$body) - - this.$element.on('click.dismiss.bs.modal', $.proxy(function (e) { - if (this.ignoreBackdropClick) { - this.ignoreBackdropClick = false - return - } - if (e.target !== e.currentTarget) return - this.options.backdrop == 'static' - ? this.$element[0].focus() - : this.hide() - }, this)) - - if (doAnimate) this.$backdrop[0].offsetWidth // force reflow - - this.$backdrop.addClass('in') - - if (!callback) return - - doAnimate ? - this.$backdrop - .one('bsTransitionEnd', callback) - .emulateTransitionEnd(Modal.BACKDROP_TRANSITION_DURATION) : - callback() - - } else if (!this.isShown && this.$backdrop) { - this.$backdrop.removeClass('in') - - var callbackRemove = function () { - that.removeBackdrop() - callback && callback() - } - $.support.transition && this.$element.hasClass('fade') ? - this.$backdrop - .one('bsTransitionEnd', callbackRemove) - .emulateTransitionEnd(Modal.BACKDROP_TRANSITION_DURATION) : - callbackRemove() - - } else if (callback) { - callback() - } - } - - // these following methods are used to handle overflowing modals - - Modal.prototype.handleUpdate = function () { - this.adjustDialog() - } - - Modal.prototype.adjustDialog = function () { - var modalIsOverflowing = this.$element[0].scrollHeight > document.documentElement.clientHeight - - this.$element.css({ - paddingLeft: !this.bodyIsOverflowing && modalIsOverflowing ? this.scrollbarWidth : '', - paddingRight: this.bodyIsOverflowing && !modalIsOverflowing ? this.scrollbarWidth : '' - }) - } - - Modal.prototype.resetAdjustments = function () { - this.$element.css({ - paddingLeft: '', - paddingRight: '' - }) - } - - Modal.prototype.checkScrollbar = function () { - var fullWindowWidth = window.innerWidth - if (!fullWindowWidth) { // workaround for missing window.innerWidth in IE8 - var documentElementRect = document.documentElement.getBoundingClientRect() - fullWindowWidth = documentElementRect.right - Math.abs(documentElementRect.left) - } - this.bodyIsOverflowing = document.body.clientWidth < fullWindowWidth - this.scrollbarWidth = this.measureScrollbar() - } - - Modal.prototype.setScrollbar = function () { - var bodyPad = parseInt((this.$body.css('padding-right') || 0), 10) - this.originalBodyPad = document.body.style.paddingRight || '' - if (this.bodyIsOverflowing) this.$body.css('padding-right', bodyPad + this.scrollbarWidth) - } - - Modal.prototype.resetScrollbar = function () { - this.$body.css('padding-right', this.originalBodyPad) - } - - Modal.prototype.measureScrollbar = function () { // thx walsh - var scrollDiv = document.createElement('div') - scrollDiv.className = 'modal-scrollbar-measure' - this.$body.append(scrollDiv) - var scrollbarWidth = scrollDiv.offsetWidth - scrollDiv.clientWidth - this.$body[0].removeChild(scrollDiv) - return scrollbarWidth - } - - - // MODAL PLUGIN DEFINITION - // ======================= - - function Plugin(option, _relatedTarget) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.modal') - var options = $.extend({}, Modal.DEFAULTS, $this.data(), typeof option == 'object' && option) - - if (!data) $this.data('bs.modal', (data = new Modal(this, options))) - if (typeof option == 'string') data[option](_relatedTarget) - else if (options.show) data.show(_relatedTarget) - }) - } - - var old = $.fn.modal - - $.fn.modal = Plugin - $.fn.modal.Constructor = Modal - - - // MODAL NO CONFLICT - // ================= - - $.fn.modal.noConflict = function () { - $.fn.modal = old - return this - } - - - // MODAL DATA-API - // ============== - - $(document).on('click.bs.modal.data-api', '[data-toggle="modal"]', function (e) { - var $this = $(this) - var href = $this.attr('href') - var $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) // strip for ie7 - var option = $target.data('bs.modal') ? 'toggle' : $.extend({ remote: !/#/.test(href) && href }, $target.data(), $this.data()) - - if ($this.is('a')) e.preventDefault() - - $target.one('show.bs.modal', function (showEvent) { - if (showEvent.isDefaultPrevented()) return // only register focus restorer if modal will actually get shown - $target.one('hidden.bs.modal', function () { - $this.is(':visible') && $this.trigger('focus') - }) - }) - Plugin.call($target, option, this) - }) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: tooltip.js v3.3.4 - * http://getbootstrap.com/javascript/#tooltip - * Inspired by the original jQuery.tipsy by Jason Frame - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // TOOLTIP PUBLIC CLASS DEFINITION - // =============================== - - var Tooltip = function (element, options) { - this.type = null - this.options = null - this.enabled = null - this.timeout = null - this.hoverState = null - this.$element = null - - this.init('tooltip', element, options) - } - - Tooltip.VERSION = '3.3.4' - - Tooltip.TRANSITION_DURATION = 150 - - Tooltip.DEFAULTS = { - animation: true, - placement: 'top', - selector: false, - template: '', - trigger: 'hover focus', - title: '', - delay: 0, - html: false, - container: false, - viewport: { - selector: 'body', - padding: 0 - } - } - - Tooltip.prototype.init = function (type, element, options) { - this.enabled = true - this.type = type - this.$element = $(element) - this.options = this.getOptions(options) - this.$viewport = this.options.viewport && $(this.options.viewport.selector || this.options.viewport) - - if (this.$element[0] instanceof document.constructor && !this.options.selector) { - throw new Error('`selector` option must be specified when initializing ' + this.type + ' on the window.document object!') - } - - var triggers = this.options.trigger.split(' ') - - for (var i = triggers.length; i--;) { - var trigger = triggers[i] - - if (trigger == 'click') { - this.$element.on('click.' + this.type, this.options.selector, $.proxy(this.toggle, this)) - } else if (trigger != 'manual') { - var eventIn = trigger == 'hover' ? 'mouseenter' : 'focusin' - var eventOut = trigger == 'hover' ? 'mouseleave' : 'focusout' - - this.$element.on(eventIn + '.' + this.type, this.options.selector, $.proxy(this.enter, this)) - this.$element.on(eventOut + '.' + this.type, this.options.selector, $.proxy(this.leave, this)) - } - } - - this.options.selector ? - (this._options = $.extend({}, this.options, { trigger: 'manual', selector: '' })) : - this.fixTitle() - } - - Tooltip.prototype.getDefaults = function () { - return Tooltip.DEFAULTS - } - - Tooltip.prototype.getOptions = function (options) { - options = $.extend({}, this.getDefaults(), this.$element.data(), options) - - if (options.delay && typeof options.delay == 'number') { - options.delay = { - show: options.delay, - hide: options.delay - } - } - - return options - } - - Tooltip.prototype.getDelegateOptions = function () { - var options = {} - var defaults = this.getDefaults() - - this._options && $.each(this._options, function (key, value) { - if (defaults[key] != value) options[key] = value - }) - - return options - } - - Tooltip.prototype.enter = function (obj) { - var self = obj instanceof this.constructor ? - obj : $(obj.currentTarget).data('bs.' + this.type) - - if (self && self.$tip && self.$tip.is(':visible')) { - self.hoverState = 'in' - return - } - - if (!self) { - self = new this.constructor(obj.currentTarget, this.getDelegateOptions()) - $(obj.currentTarget).data('bs.' + this.type, self) - } - - clearTimeout(self.timeout) - - self.hoverState = 'in' - - if (!self.options.delay || !self.options.delay.show) return self.show() - - self.timeout = setTimeout(function () { - if (self.hoverState == 'in') self.show() - }, self.options.delay.show) - } - - Tooltip.prototype.leave = function (obj) { - var self = obj instanceof this.constructor ? - obj : $(obj.currentTarget).data('bs.' + this.type) - - if (!self) { - self = new this.constructor(obj.currentTarget, this.getDelegateOptions()) - $(obj.currentTarget).data('bs.' + this.type, self) - } - - clearTimeout(self.timeout) - - self.hoverState = 'out' - - if (!self.options.delay || !self.options.delay.hide) return self.hide() - - self.timeout = setTimeout(function () { - if (self.hoverState == 'out') self.hide() - }, self.options.delay.hide) - } - - Tooltip.prototype.show = function () { - var e = $.Event('show.bs.' + this.type) - - if (this.hasContent() && this.enabled) { - this.$element.trigger(e) - - var inDom = $.contains(this.$element[0].ownerDocument.documentElement, this.$element[0]) - if (e.isDefaultPrevented() || !inDom) return - var that = this - - var $tip = this.tip() - - var tipId = this.getUID(this.type) - - this.setContent() - $tip.attr('id', tipId) - this.$element.attr('aria-describedby', tipId) - - if (this.options.animation) $tip.addClass('fade') - - var placement = typeof this.options.placement == 'function' ? - this.options.placement.call(this, $tip[0], this.$element[0]) : - this.options.placement - - var autoToken = /\s?auto?\s?/i - var autoPlace = autoToken.test(placement) - if (autoPlace) placement = placement.replace(autoToken, '') || 'top' - - $tip - .detach() - .css({ top: 0, left: 0, display: 'block' }) - .addClass(placement) - .data('bs.' + this.type, this) - - this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element) - - var pos = this.getPosition() - var actualWidth = $tip[0].offsetWidth - var actualHeight = $tip[0].offsetHeight - - if (autoPlace) { - var orgPlacement = placement - var $container = this.options.container ? $(this.options.container) : this.$element.parent() - var containerDim = this.getPosition($container) - - placement = placement == 'bottom' && pos.bottom + actualHeight > containerDim.bottom ? 'top' : - placement == 'top' && pos.top - actualHeight < containerDim.top ? 'bottom' : - placement == 'right' && pos.right + actualWidth > containerDim.width ? 'left' : - placement == 'left' && pos.left - actualWidth < containerDim.left ? 'right' : - placement - - $tip - .removeClass(orgPlacement) - .addClass(placement) - } - - var calculatedOffset = this.getCalculatedOffset(placement, pos, actualWidth, actualHeight) - - this.applyPlacement(calculatedOffset, placement) - - var complete = function () { - var prevHoverState = that.hoverState - that.$element.trigger('shown.bs.' + that.type) - that.hoverState = null - - if (prevHoverState == 'out') that.leave(that) - } - - $.support.transition && this.$tip.hasClass('fade') ? - $tip - .one('bsTransitionEnd', complete) - .emulateTransitionEnd(Tooltip.TRANSITION_DURATION) : - complete() - } - } - - Tooltip.prototype.applyPlacement = function (offset, placement) { - var $tip = this.tip() - var width = $tip[0].offsetWidth - var height = $tip[0].offsetHeight - - // manually read margins because getBoundingClientRect includes difference - var marginTop = parseInt($tip.css('margin-top'), 10) - var marginLeft = parseInt($tip.css('margin-left'), 10) - - // we must check for NaN for ie 8/9 - if (isNaN(marginTop)) marginTop = 0 - if (isNaN(marginLeft)) marginLeft = 0 - - offset.top = offset.top + marginTop - offset.left = offset.left + marginLeft - - // $.fn.offset doesn't round pixel values - // so we use setOffset directly with our own function B-0 - $.offset.setOffset($tip[0], $.extend({ - using: function (props) { - $tip.css({ - top: Math.round(props.top), - left: Math.round(props.left) - }) - } - }, offset), 0) - - $tip.addClass('in') - - // check to see if placing tip in new offset caused the tip to resize itself - var actualWidth = $tip[0].offsetWidth - var actualHeight = $tip[0].offsetHeight - - if (placement == 'top' && actualHeight != height) { - offset.top = offset.top + height - actualHeight - } - - var delta = this.getViewportAdjustedDelta(placement, offset, actualWidth, actualHeight) - - if (delta.left) offset.left += delta.left - else offset.top += delta.top - - var isVertical = /top|bottom/.test(placement) - var arrowDelta = isVertical ? delta.left * 2 - width + actualWidth : delta.top * 2 - height + actualHeight - var arrowOffsetPosition = isVertical ? 'offsetWidth' : 'offsetHeight' - - $tip.offset(offset) - this.replaceArrow(arrowDelta, $tip[0][arrowOffsetPosition], isVertical) - } - - Tooltip.prototype.replaceArrow = function (delta, dimension, isVertical) { - this.arrow() - .css(isVertical ? 'left' : 'top', 50 * (1 - delta / dimension) + '%') - .css(isVertical ? 'top' : 'left', '') - } - - Tooltip.prototype.setContent = function () { - var $tip = this.tip() - var title = this.getTitle() - - $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title) - $tip.removeClass('fade in top bottom left right') - } - - Tooltip.prototype.hide = function (callback) { - var that = this - var $tip = $(this.$tip) - var e = $.Event('hide.bs.' + this.type) - - function complete() { - if (that.hoverState != 'in') $tip.detach() - that.$element - .removeAttr('aria-describedby') - .trigger('hidden.bs.' + that.type) - callback && callback() - } - - this.$element.trigger(e) - - if (e.isDefaultPrevented()) return - - $tip.removeClass('in') - - $.support.transition && $tip.hasClass('fade') ? - $tip - .one('bsTransitionEnd', complete) - .emulateTransitionEnd(Tooltip.TRANSITION_DURATION) : - complete() - - this.hoverState = null - - return this - } - - Tooltip.prototype.fixTitle = function () { - var $e = this.$element - if ($e.attr('title') || typeof ($e.attr('data-original-title')) != 'string') { - $e.attr('data-original-title', $e.attr('title') || '').attr('title', '') - } - } - - Tooltip.prototype.hasContent = function () { - return this.getTitle() - } - - Tooltip.prototype.getPosition = function ($element) { - $element = $element || this.$element - - var el = $element[0] - var isBody = el.tagName == 'BODY' - - var elRect = el.getBoundingClientRect() - if (elRect.width == null) { - // width and height are missing in IE8, so compute them manually; see https://github.com/twbs/bootstrap/issues/14093 - elRect = $.extend({}, elRect, { width: elRect.right - elRect.left, height: elRect.bottom - elRect.top }) - } - var elOffset = isBody ? { top: 0, left: 0 } : $element.offset() - var scroll = { scroll: isBody ? document.documentElement.scrollTop || document.body.scrollTop : $element.scrollTop() } - var outerDims = isBody ? { width: $(window).width(), height: $(window).height() } : null - - return $.extend({}, elRect, scroll, outerDims, elOffset) - } - - Tooltip.prototype.getCalculatedOffset = function (placement, pos, actualWidth, actualHeight) { - return placement == 'bottom' ? { top: pos.top + pos.height, left: pos.left + pos.width / 2 - actualWidth / 2 } : - placement == 'top' ? { top: pos.top - actualHeight, left: pos.left + pos.width / 2 - actualWidth / 2 } : - placement == 'left' ? { top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left - actualWidth } : - /* placement == 'right' */ { top: pos.top + pos.height / 2 - actualHeight / 2, left: pos.left + pos.width } - - } - - Tooltip.prototype.getViewportAdjustedDelta = function (placement, pos, actualWidth, actualHeight) { - var delta = { top: 0, left: 0 } - if (!this.$viewport) return delta - - var viewportPadding = this.options.viewport && this.options.viewport.padding || 0 - var viewportDimensions = this.getPosition(this.$viewport) - - if (/right|left/.test(placement)) { - var topEdgeOffset = pos.top - viewportPadding - viewportDimensions.scroll - var bottomEdgeOffset = pos.top + viewportPadding - viewportDimensions.scroll + actualHeight - if (topEdgeOffset < viewportDimensions.top) { // top overflow - delta.top = viewportDimensions.top - topEdgeOffset - } else if (bottomEdgeOffset > viewportDimensions.top + viewportDimensions.height) { // bottom overflow - delta.top = viewportDimensions.top + viewportDimensions.height - bottomEdgeOffset - } - } else { - var leftEdgeOffset = pos.left - viewportPadding - var rightEdgeOffset = pos.left + viewportPadding + actualWidth - if (leftEdgeOffset < viewportDimensions.left) { // left overflow - delta.left = viewportDimensions.left - leftEdgeOffset - } else if (rightEdgeOffset > viewportDimensions.width) { // right overflow - delta.left = viewportDimensions.left + viewportDimensions.width - rightEdgeOffset - } - } - - return delta - } - - Tooltip.prototype.getTitle = function () { - var title - var $e = this.$element - var o = this.options - - title = $e.attr('data-original-title') - || (typeof o.title == 'function' ? o.title.call($e[0]) : o.title) - - return title - } - - Tooltip.prototype.getUID = function (prefix) { - do prefix += ~~(Math.random() * 1000000) - while (document.getElementById(prefix)) - return prefix - } - - Tooltip.prototype.tip = function () { - return (this.$tip = this.$tip || $(this.options.template)) - } - - Tooltip.prototype.arrow = function () { - return (this.$arrow = this.$arrow || this.tip().find('.tooltip-arrow')) - } - - Tooltip.prototype.enable = function () { - this.enabled = true - } - - Tooltip.prototype.disable = function () { - this.enabled = false - } - - Tooltip.prototype.toggleEnabled = function () { - this.enabled = !this.enabled - } - - Tooltip.prototype.toggle = function (e) { - var self = this - if (e) { - self = $(e.currentTarget).data('bs.' + this.type) - if (!self) { - self = new this.constructor(e.currentTarget, this.getDelegateOptions()) - $(e.currentTarget).data('bs.' + this.type, self) - } - } - - self.tip().hasClass('in') ? self.leave(self) : self.enter(self) - } - - Tooltip.prototype.destroy = function () { - var that = this - clearTimeout(this.timeout) - this.hide(function () { - that.$element.off('.' + that.type).removeData('bs.' + that.type) - }) - } - - - // TOOLTIP PLUGIN DEFINITION - // ========================= - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.tooltip') - var options = typeof option == 'object' && option - - if (!data && /destroy|hide/.test(option)) return - if (!data) $this.data('bs.tooltip', (data = new Tooltip(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - var old = $.fn.tooltip - - $.fn.tooltip = Plugin - $.fn.tooltip.Constructor = Tooltip - - - // TOOLTIP NO CONFLICT - // =================== - - $.fn.tooltip.noConflict = function () { - $.fn.tooltip = old - return this - } - -}(jQuery); - -/* ======================================================================== - * Bootstrap: popover.js v3.3.4 - * http://getbootstrap.com/javascript/#popovers - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // POPOVER PUBLIC CLASS DEFINITION - // =============================== - - var Popover = function (element, options) { - this.init('popover', element, options) - } - - if (!$.fn.tooltip) throw new Error('Popover requires tooltip.js') - - Popover.VERSION = '3.3.4' - - Popover.DEFAULTS = $.extend({}, $.fn.tooltip.Constructor.DEFAULTS, { - placement: 'right', - trigger: 'click', - content: '', - template: '' - }) - - - // NOTE: POPOVER EXTENDS tooltip.js - // ================================ - - Popover.prototype = $.extend({}, $.fn.tooltip.Constructor.prototype) - - Popover.prototype.constructor = Popover - - Popover.prototype.getDefaults = function () { - return Popover.DEFAULTS - } - - Popover.prototype.setContent = function () { - var $tip = this.tip() - var title = this.getTitle() - var content = this.getContent() - - $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title) - $tip.find('.popover-content').children().detach().end()[ // we use append for html objects to maintain js events - this.options.html ? (typeof content == 'string' ? 'html' : 'append') : 'text' - ](content) - - $tip.removeClass('fade top bottom left right in') - - // IE8 doesn't accept hiding via the `:empty` pseudo selector, we have to do - // this manually by checking the contents. - if (!$tip.find('.popover-title').html()) $tip.find('.popover-title').hide() - } - - Popover.prototype.hasContent = function () { - return this.getTitle() || this.getContent() - } - - Popover.prototype.getContent = function () { - var $e = this.$element - var o = this.options - - return $e.attr('data-content') - || (typeof o.content == 'function' ? - o.content.call($e[0]) : - o.content) - } - - Popover.prototype.arrow = function () { - return (this.$arrow = this.$arrow || this.tip().find('.arrow')) - } - - - // POPOVER PLUGIN DEFINITION - // ========================= - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.popover') - var options = typeof option == 'object' && option - - if (!data && /destroy|hide/.test(option)) return - if (!data) $this.data('bs.popover', (data = new Popover(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - var old = $.fn.popover - - $.fn.popover = Plugin - $.fn.popover.Constructor = Popover - - - // POPOVER NO CONFLICT - // =================== - - $.fn.popover.noConflict = function () { - $.fn.popover = old - return this - } - -}(jQuery); - -/* ======================================================================== - * Bootstrap: scrollspy.js v3.3.4 - * http://getbootstrap.com/javascript/#scrollspy - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // SCROLLSPY CLASS DEFINITION - // ========================== - - function ScrollSpy(element, options) { - this.$body = $(document.body) - this.$scrollElement = $(element).is(document.body) ? $(window) : $(element) - this.options = $.extend({}, ScrollSpy.DEFAULTS, options) - this.selector = (this.options.target || '') + ' .nav li > a' - this.offsets = [] - this.targets = [] - this.activeTarget = null - this.scrollHeight = 0 - - this.$scrollElement.on('scroll.bs.scrollspy', $.proxy(this.process, this)) - this.refresh() - this.process() - } - - ScrollSpy.VERSION = '3.3.4' - - ScrollSpy.DEFAULTS = { - offset: 10 - } - - ScrollSpy.prototype.getScrollHeight = function () { - return this.$scrollElement[0].scrollHeight || Math.max(this.$body[0].scrollHeight, document.documentElement.scrollHeight) - } - - ScrollSpy.prototype.refresh = function () { - var that = this - var offsetMethod = 'offset' - var offsetBase = 0 - - this.offsets = [] - this.targets = [] - this.scrollHeight = this.getScrollHeight() - - if (!$.isWindow(this.$scrollElement[0])) { - offsetMethod = 'position' - offsetBase = this.$scrollElement.scrollTop() - } - - this.$body - .find(this.selector) - .map(function () { - var $el = $(this) - var href = $el.data('target') || $el.attr('href') - var $href = /^#./.test(href) && $(href) - - return ($href - && $href.length - && $href.is(':visible') - && [[$href[offsetMethod]().top + offsetBase, href]]) || null - }) - .sort(function (a, b) { return a[0] - b[0] }) - .each(function () { - that.offsets.push(this[0]) - that.targets.push(this[1]) - }) - } - - ScrollSpy.prototype.process = function () { - var scrollTop = this.$scrollElement.scrollTop() + this.options.offset - var scrollHeight = this.getScrollHeight() - var maxScroll = this.options.offset + scrollHeight - this.$scrollElement.height() - var offsets = this.offsets - var targets = this.targets - var activeTarget = this.activeTarget - var i - - if (this.scrollHeight != scrollHeight) { - this.refresh() - } - - if (scrollTop >= maxScroll) { - return activeTarget != (i = targets[targets.length - 1]) && this.activate(i) - } - - if (activeTarget && scrollTop < offsets[0]) { - this.activeTarget = null - return this.clear() - } - - for (i = offsets.length; i--;) { - activeTarget != targets[i] - && scrollTop >= offsets[i] - && (offsets[i + 1] === undefined || scrollTop < offsets[i + 1]) - && this.activate(targets[i]) - } - } - - ScrollSpy.prototype.activate = function (target) { - this.activeTarget = target - - this.clear() - - var selector = this.selector + - '[data-target="' + target + '"],' + - this.selector + '[href="' + target + '"]' - - var active = $(selector) - .parents('li') - .addClass('active') - - if (active.parent('.dropdown-menu').length) { - active = active - .closest('li.dropdown') - .addClass('active') - } - - active.trigger('activate.bs.scrollspy') - } - - ScrollSpy.prototype.clear = function () { - $(this.selector) - .parentsUntil(this.options.target, '.active') - .removeClass('active') - } - - - // SCROLLSPY PLUGIN DEFINITION - // =========================== - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.scrollspy') - var options = typeof option == 'object' && option - - if (!data) $this.data('bs.scrollspy', (data = new ScrollSpy(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - var old = $.fn.scrollspy - - $.fn.scrollspy = Plugin - $.fn.scrollspy.Constructor = ScrollSpy - - - // SCROLLSPY NO CONFLICT - // ===================== - - $.fn.scrollspy.noConflict = function () { - $.fn.scrollspy = old - return this - } - - - // SCROLLSPY DATA-API - // ================== - - $(window).on('load.bs.scrollspy.data-api', function () { - $('[data-spy="scroll"]').each(function () { - var $spy = $(this) - Plugin.call($spy, $spy.data()) - }) - }) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: tab.js v3.3.4 - * http://getbootstrap.com/javascript/#tabs - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // TAB CLASS DEFINITION - // ==================== - - var Tab = function (element) { - this.element = $(element) - } - - Tab.VERSION = '3.3.4' - - Tab.TRANSITION_DURATION = 150 - - Tab.prototype.show = function () { - var $this = this.element - var $ul = $this.closest('ul:not(.dropdown-menu)') - var selector = $this.data('target') - - if (!selector) { - selector = $this.attr('href') - selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7 - } - - if ($this.parent('li').hasClass('active')) return - - var $previous = $ul.find('.active:last a') - var hideEvent = $.Event('hide.bs.tab', { - relatedTarget: $this[0] - }) - var showEvent = $.Event('show.bs.tab', { - relatedTarget: $previous[0] - }) - - $previous.trigger(hideEvent) - $this.trigger(showEvent) - - if (showEvent.isDefaultPrevented() || hideEvent.isDefaultPrevented()) return - - var $target = $(selector) - - this.activate($this.closest('li'), $ul) - this.activate($target, $target.parent(), function () { - $previous.trigger({ - type: 'hidden.bs.tab', - relatedTarget: $this[0] - }) - $this.trigger({ - type: 'shown.bs.tab', - relatedTarget: $previous[0] - }) - }) - } - - Tab.prototype.activate = function (element, container, callback) { - var $active = container.find('> .active') - var transition = callback - && $.support.transition - && (($active.length && $active.hasClass('fade')) || !!container.find('> .fade').length) - - function next() { - $active - .removeClass('active') - .find('> .dropdown-menu > .active') - .removeClass('active') - .end() - .find('[data-toggle="tab"]') - .attr('aria-expanded', false) - - element - .addClass('active') - .find('[data-toggle="tab"]') - .attr('aria-expanded', true) - - if (transition) { - element[0].offsetWidth // reflow for transition - element.addClass('in') - } else { - element.removeClass('fade') - } - - if (element.parent('.dropdown-menu').length) { - element - .closest('li.dropdown') - .addClass('active') - .end() - .find('[data-toggle="tab"]') - .attr('aria-expanded', true) - } - - callback && callback() - } - - $active.length && transition ? - $active - .one('bsTransitionEnd', next) - .emulateTransitionEnd(Tab.TRANSITION_DURATION) : - next() - - $active.removeClass('in') - } - - - // TAB PLUGIN DEFINITION - // ===================== - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.tab') - - if (!data) $this.data('bs.tab', (data = new Tab(this))) - if (typeof option == 'string') data[option]() - }) - } - - var old = $.fn.tab - - $.fn.tab = Plugin - $.fn.tab.Constructor = Tab - - - // TAB NO CONFLICT - // =============== - - $.fn.tab.noConflict = function () { - $.fn.tab = old - return this - } - - - // TAB DATA-API - // ============ - - var clickHandler = function (e) { - e.preventDefault() - Plugin.call($(this), 'show') - } - - $(document) - .on('click.bs.tab.data-api', '[data-toggle="tab"]', clickHandler) - .on('click.bs.tab.data-api', '[data-toggle="pill"]', clickHandler) - -}(jQuery); - -/* ======================================================================== - * Bootstrap: affix.js v3.3.4 - * http://getbootstrap.com/javascript/#affix - * ======================================================================== - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - * ======================================================================== */ - - -+function ($) { - 'use strict'; - - // AFFIX CLASS DEFINITION - // ====================== - - var Affix = function (element, options) { - this.options = $.extend({}, Affix.DEFAULTS, options) - - this.$target = $(this.options.target) - .on('scroll.bs.affix.data-api', $.proxy(this.checkPosition, this)) - .on('click.bs.affix.data-api', $.proxy(this.checkPositionWithEventLoop, this)) - - this.$element = $(element) - this.affixed = null - this.unpin = null - this.pinnedOffset = null - - this.checkPosition() - } - - Affix.VERSION = '3.3.4' - - Affix.RESET = 'affix affix-top affix-bottom' - - Affix.DEFAULTS = { - offset: 0, - target: window - } - - Affix.prototype.getState = function (scrollHeight, height, offsetTop, offsetBottom) { - var scrollTop = this.$target.scrollTop() - var position = this.$element.offset() - var targetHeight = this.$target.height() - - if (offsetTop != null && this.affixed == 'top') return scrollTop < offsetTop ? 'top' : false - - if (this.affixed == 'bottom') { - if (offsetTop != null) return (scrollTop + this.unpin <= position.top) ? false : 'bottom' - return (scrollTop + targetHeight <= scrollHeight - offsetBottom) ? false : 'bottom' - } - - var initializing = this.affixed == null - var colliderTop = initializing ? scrollTop : position.top - var colliderHeight = initializing ? targetHeight : height - - if (offsetTop != null && scrollTop <= offsetTop) return 'top' - if (offsetBottom != null && (colliderTop + colliderHeight >= scrollHeight - offsetBottom)) return 'bottom' - - return false - } - - Affix.prototype.getPinnedOffset = function () { - if (this.pinnedOffset) return this.pinnedOffset - this.$element.removeClass(Affix.RESET).addClass('affix') - var scrollTop = this.$target.scrollTop() - var position = this.$element.offset() - return (this.pinnedOffset = position.top - scrollTop) - } - - Affix.prototype.checkPositionWithEventLoop = function () { - setTimeout($.proxy(this.checkPosition, this), 1) - } - - Affix.prototype.checkPosition = function () { - if (!this.$element.is(':visible')) return - - var height = this.$element.height() - var offset = this.options.offset - var offsetTop = offset.top - var offsetBottom = offset.bottom - var scrollHeight = $(document.body).height() - - if (typeof offset != 'object') offsetBottom = offsetTop = offset - if (typeof offsetTop == 'function') offsetTop = offset.top(this.$element) - if (typeof offsetBottom == 'function') offsetBottom = offset.bottom(this.$element) - - var affix = this.getState(scrollHeight, height, offsetTop, offsetBottom) - - if (this.affixed != affix) { - if (this.unpin != null) this.$element.css('top', '') - - var affixType = 'affix' + (affix ? '-' + affix : '') - var e = $.Event(affixType + '.bs.affix') - - this.$element.trigger(e) - - if (e.isDefaultPrevented()) return - - this.affixed = affix - this.unpin = affix == 'bottom' ? this.getPinnedOffset() : null - - this.$element - .removeClass(Affix.RESET) - .addClass(affixType) - .trigger(affixType.replace('affix', 'affixed') + '.bs.affix') - } - - if (affix == 'bottom') { - this.$element.offset({ - top: scrollHeight - height - offsetBottom - }) - } - } - - - // AFFIX PLUGIN DEFINITION - // ======================= - - function Plugin(option) { - return this.each(function () { - var $this = $(this) - var data = $this.data('bs.affix') - var options = typeof option == 'object' && option - - if (!data) $this.data('bs.affix', (data = new Affix(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - var old = $.fn.affix - - $.fn.affix = Plugin - $.fn.affix.Constructor = Affix - - - // AFFIX NO CONFLICT - // ================= - - $.fn.affix.noConflict = function () { - $.fn.affix = old - return this - } - - - // AFFIX DATA-API - // ============== - - $(window).on('load', function () { - $('[data-spy="affix"]').each(function () { - var $spy = $(this) - var data = $spy.data() - - data.offset = data.offset || {} - - if (data.offsetBottom != null) data.offset.bottom = data.offsetBottom - if (data.offsetTop != null) data.offset.top = data.offsetTop - - Plugin.call($spy, data) - }) - }) - -}(jQuery); diff --git a/src/web/static/js/bootstrap.min.js b/src/web/static/js/bootstrap.min.js deleted file mode 100644 index c8f82e592..000000000 --- a/src/web/static/js/bootstrap.min.js +++ /dev/null @@ -1,7 +0,0 @@ -/*! - * Bootstrap v3.3.4 (http://getbootstrap.com) - * Copyright 2011-2015 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */ -if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){return a(b.target).is(this)?b.handleObj.handler.apply(this,arguments):void 0}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.4",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a(f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.4",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")&&(c.prop("checked")&&this.$element.hasClass("active")?a=!1:b.find(".active").removeClass("active")),a&&c.prop("checked",!this.$element.hasClass("active")).trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active"));a&&this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target);d.hasClass("btn")||(d=d.closest(".btn")),b.call(d,"toggle"),c.preventDefault()}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.4",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();break;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));return a>this.$items.length-1||0>a?void 0:this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){return this.sliding?void 0:this.slide("next")},c.prototype.prev=function(){return this.sliding?void 0:this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offsetWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.4",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expanded",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){b&&3===b.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=c(d),f={relatedTarget:this};e.hasClass("open")&&(e.trigger(b=a.Event("hide.bs.dropdown",f)),b.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger("hidden.bs.dropdown",f)))}))}function c(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.4",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=c(e),g=f.hasClass("open");if(b(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a('
').insertAfter(a(this)).on("click",b);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger("shown.bs.dropdown",h)}return!1}},g.prototype.keydown=function(b){if(/(38|40|27|32)/.test(b.which)&&!/input|textarea/i.test(b.target.tagName)){var d=a(this);if(b.preventDefault(),b.stopPropagation(),!d.is(".disabled, :disabled")){var e=c(d),g=e.hasClass("open");if(!g&&27!=b.which||g&&27==b.which)return 27==b.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find('[role="menu"]'+h+', [role="listbox"]'+h);if(i.length){var j=i.index(b.target);38==b.which&&j>0&&j--,40==b.which&&j').appendTo(this.$body),this.$element.on("click.dismiss.bs.modal",a.proxy(function(a){return this.ignoreBackdropClick?void(this.ignoreBackdropClick=!1):void(a.target===a.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus():this.hide()))},this)),f&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!b)return;f?this.$backdrop.one("bsTransitionEnd",b).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):b()}else if(!this.isShown&&this.$backdrop){this.$backdrop.removeClass("in");var g=function(){d.removeBackdrop(),b&&b()};a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one("bsTransitionEnd",g).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):g()}else b&&b()},c.prototype.handleUpdate=function(){this.adjustDialog()},c.prototype.adjustDialog=function(){var a=this.$element[0].scrollHeight>document.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth
',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(this.options.viewport.selector||this.options.viewport),this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c&&c.$tip&&c.$tip.is(":visible")?void(c.hoverState="in"):(c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),clearTimeout(c.timeout),c.hoverState="out",c.options.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide()},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0].offsetHeight;if(j){var n=h,o=this.options.container?a(this.options.container):this.$element.parent(),p=this.getPosition(o);h="bottom"==h&&k.bottom+m>p.bottom?"top":"top"==h&&k.top-mp.width?"left":"left"==h&&k.left-lg.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;jg.width&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){return this.$tip=this.$tip||a(this.options.template)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){this.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type)})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;(e||!/destroy|hide/.test(b))&&(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.4",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:''}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.4",b.DEFAULTS={offset:10},b.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b=e[a]&&(void 0===e[a+1]||b .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};c.VERSION="3.3.4",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return c>e?"top":!1;if("bottom"==this.affixed)return null!=c?e+this.unpin<=f.top?!1:"bottom":a-d>=e+g?!1:"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&c>=e?"top":null!=d&&i+j>=a-d?"bottom":!1},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=a(document.body).height();"object"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery); \ No newline at end of file diff --git a/src/web/static/js/bootstrap.min.js b/src/web/static/js/bootstrap.min.js new file mode 120000 index 000000000..55cac00d5 --- /dev/null +++ b/src/web/static/js/bootstrap.min.js @@ -0,0 +1 @@ +../../../../submodules/bootstrap/dist/js/bootstrap.min.js \ No newline at end of file diff --git a/src/web/static/js/feed.js b/src/web/static/js/feed.js deleted file mode 100644 index ceef58fcb..000000000 --- a/src/web/static/js/feed.js +++ /dev/null @@ -1,22 +0,0 @@ -$('.container').on('click', '#add-feed-filter-row', function() { - $('#filters-container').append( - '
' - + ' ' - + ' ' - + ' ' - + ' ' - + ' ' - + '
'); -}); -$('.container').on('click', '.del-feed-filter-row', function() { - $(this).parent().remove(); -}); diff --git a/src/web/templates/layout.html b/src/web/templates/layout.html index 233cb13d7..cf6b97296 100644 --- a/src/web/templates/layout.html +++ b/src/web/templates/layout.html @@ -105,9 +105,7 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + + +Ceci n'est pas Old Boy - Owlboy (suite) - Benzaie Live - YouTube + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ + +
+
+ +
+ + +
+
+
+
+ +FR
+
+ +
+
+
+ + +
+
+
+ +
+ +
+
+
+
+ En savoir plus
+ View this message in English +
+YouTube s'affiche actuellement en Français. +Vous pouvez modifier ce paramètre ci-dessous. +
+
+
+ Learn more
+
+ You're viewing YouTube in French. + You can change this preference below. +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+ + + +
+
+
+ +
+
+
+ +
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+

+ + + + + Ceci n'est pas Old Boy - Owlboy (suite) - Benzaie Live + + +

+
+
+ +
+ + + + + BenzaieLive + + + + + + + + +220 162220 k +
+ +
+
+ + +
+
+ + +
+ +
+
+ + + + + +
+
18 123 vues
+
+
+
+
+
+ + + + + + +
+
+ + + +
+
+
+
+ + +
+
+
+ +
+ +
+
+ + +
+
+ +
+
+ + +
+
+ + +
+
+ Pour évaluer une vidéo, vous devez la louer. +
+ +
+ +
+
+ Cette fonctionnalité n'est pas disponible pour le moment. Veuillez réessayer ultérieurement. +
+
+ + +
+ + +
+ +
Ajoutée le 19 nov. 2016

-- Watch live at https://www.twitch.tv/benzaie

+ +
+
+ +
+ + + +
+
+ + +
+ +
+ + +
+
+
+ + + +
+
+ +
+ +
+
+
+Publicité +
+
+
+
+ + +
+
+
+
+
+ + + +Lorsque cette fonctionnalité est activée, une vidéo issue des suggestions est automatiquement lancée à la suite de la lecture en cours. + + + +
+

+ À suivre +

+
+ +
+ +
+
+ + +
+
+
+ +
+
+ +
+
+ +
+
+
+
+
+ +
+ +
+ + +
+ +
+
+ + +
+
+ +
+ + +
+
+ + +
+
+ + +
+ pour ajouter cette vidéo à la liste "À regarder plus tard". + +
+
+

+Ajouter à +

+
+
+ + +
+
+ + + + + + + \ No newline at end of file diff --git a/src/tests/fixtures/article-2.json b/src/tests/fixtures/article-2.json new file mode 100644 index 000000000..82ae6cf48 --- /dev/null +++ b/src/tests/fixtures/article-2.json @@ -0,0 +1 @@ +{"author_detail": {"name": "BenzaieLive", "href": "http://www.youtube.com/channel/UCOWsWZTiXkbvQvtWO9RA0gA"}, "id": "yt:video:scbrjaqM3Oc", "media_starrating": {"average": "4.91", "max": "5", "count": "469", "min": "1"}, "updated_parsed": [2016, 11, 22, 5, 21, 55, 1, 327, 0], "media_community": "", "yt_videoid": "scbrjaqM3Oc", "media_content": [{"height": "390", "url": "https://www.youtube.com/v/scbrjaqM3Oc?version=3", "width": "640", "type": "application/x-shockwave-flash"}], "media_statistics": {"views": "19361"}, "link": "http://www.youtube.com/watch?v=scbrjaqM3Oc", "title": "Ceci n'est pas Old Boy - Owlboy (suite) - Benzaie Live", "title_detail": {"type": "text/plain", "language": null, "base": "https://www.youtube.com/feeds/videos.xml?channel_id=UCOWsWZTiXkbvQvtWO9RA0gA", "value": "Ceci n'est pas Old Boy - Owlboy (suite) - Benzaie Live"}, "guidislink": true, "author": "BenzaieLive", "summary": "-- Watch live at https://www.twitch.tv/benzaie", "links": [{"rel": "alternate", "href": "http://www.youtube.com/watch?v=scbrjaqM3Oc", "type": "text/html"}], "href": "", "yt_channelid": "UCOWsWZTiXkbvQvtWO9RA0gA", "media_thumbnail": [{"height": "360", "url": "https://i4.ytimg.com/vi/scbrjaqM3Oc/hqdefault.jpg", "width": "480"}], "summary_detail": {"type": "text/html", "language": null, "base": "https://www.youtube.com/feeds/videos.xml?channel_id=UCOWsWZTiXkbvQvtWO9RA0gA", "value": "-- Watch live at https://www.twitch.tv/benzaie"}, "published_parsed": [2016, 11, 19, 10, 8, 7, 5, 324, 0], "updated": "2016-11-22T05:21:55+00:00", "authors": [{"name": "BenzaieLive", "href": "http://www.youtube.com/channel/UCOWsWZTiXkbvQvtWO9RA0gA"}], "published": "2016-11-19T10:08:07+00:00"} \ No newline at end of file diff --git a/src/tests/libs/article_utils_test.py b/src/tests/libs/article_utils_test.py index d99505b02..f190beb67 100644 --- a/src/tests/libs/article_utils_test.py +++ b/src/tests/libs/article_utils_test.py @@ -2,6 +2,7 @@ import unittest from mock import patch +from requests import Response from requests.exceptions import MissingSchema from lib.article_utils import construct_article @@ -18,27 +19,37 @@ def setUp(self): def tearDown(self): self._jarr_get_patch.stop() - @staticmethod - def get_entry(): + @property + def entry(self): with open('src/tests/fixtures/article.json') as fd: return json.load(fd) + @property + def entry2(self): + with open('src/tests/fixtures/article-2.json') as fd: + return json.load(fd) + @staticmethod def get_response(scheme='http:'): - class Response: - @property - def url(self): - return scheme + ConstructArticleTest.response_url + resp = Response() + resp.url = scheme + ConstructArticleTest.response_url + resp.encoding = 'utf8' + with open('src/tests/fixtures/article.html') as fd: + resp._content = fd.read() + return resp - @property - def content(self): - with open('src/tests/fixtures/article.html') as fd: - return fd.read() - return Response() + @property + def response2(self): + resp = Response() + resp.url = 'https://www.youtube.com/watch?v=scbrjaqM3Oc' + resp.encoding = 'utf8' + with open('src/tests/fixtures/article-2.html') as fd: + resp._content = fd.read() + return resp def test_missing_title(self): self.jarr_get_patch.return_value = self.get_response('http:') - article = construct_article(self.get_entry(), {'id': 1, 'user_id': 1}) + article = construct_article(self.entry, {'id': 1, 'user_id': 1}) self.assertEquals('http://www.pariszigzag.fr/?p=56413', article['entry_id']) self.assertEquals('http:' + self.response_url, article['link']) @@ -51,7 +62,7 @@ def test_missing_scheme(self): response = self.get_response('http:') self.jarr_get_patch.side_effect = [ MissingSchema, MissingSchema, response] - entry = self.get_entry() + entry = self.entry entry['link'] = entry['link'][5:] article = construct_article(entry, {'id': 1, 'user_id': 1}) @@ -65,3 +76,16 @@ def test_missing_scheme(self): article['title']) self.assertEquals(1, article['user_id']) self.assertEquals(1, article['feed_id']) + + def test_tags(self): + import bootstrap + bootstrap.conf.CRAWLER_RESOLV = True + self.jarr_get_patch.return_value = self.response2 + article = construct_article(self.entry2, {'id': 1, 'user_id': 1}) + self.assertEquals('yt:video:scbrjaqM3Oc', article['entry_id']) + self.assertEquals(self.response2.url, article['link']) + self.assertEquals("Ceci n'est pas Old Boy - Owlboy (suite) - " + "Benzaie Live", article['title']) + self.assertEquals(1, article['user_id']) + self.assertEquals(1, article['feed_id']) + self.assertEquals({'twitch', 'games'}, article['tags']) diff --git a/src/tests/libs/feed_utils_test.py b/src/tests/libs/feed_utils_test.py index a06b9ab07..1c87ef17e 100644 --- a/src/tests/libs/feed_utils_test.py +++ b/src/tests/libs/feed_utils_test.py @@ -30,7 +30,8 @@ def test_joies_du_code(self): self.assertEquals( {'description': "Les joies du code, l'instant GIF des développeurs !", - 'icon_url': 'http://ljdchost.com/ljdc-theme/favicons/favicon.ico?v=9BK2m20LWn', + 'icon_url': 'http://ljdchost.com/ljdc-theme/favicons' + '/favicon.ico?v=9BK2m20LWn', 'link': 'http://lesjoiesducode.tumblr.com/rss', 'site_link': 'http://lesjoiesducode.fr/', 'title': 'Les joies du code'}, @@ -39,7 +40,6 @@ def test_joies_du_code(self): def test_apod(self): self.assertEquals( {'icon_url': 'http://apod.nasa.gov/favicon.ico', - 'link': None, 'site_link': 'http://apod.nasa.gov/', 'title': 'Astronomy Picture of the Day'}, construct_feed_from('http://apod.nasa.gov/')) diff --git a/src/tests/libs/html_parsing_test.py b/src/tests/libs/html_parsing_test.py new file mode 100644 index 000000000..3c4516aa1 --- /dev/null +++ b/src/tests/libs/html_parsing_test.py @@ -0,0 +1,65 @@ +import unittest +import urllib + +from mock import patch +from requests import Response + +from lib.html_parsing import (extract_title, extract_tags, extract_icon_url, + extract_feed_link) + + +class HTMLParsingTest(unittest.TestCase): + + @property + def article(self): + resp = Response() + resp.url = 'http://www.pariszigzag.fr/paris-insolite-secret/'\ + 'les-plus-belles-boulangeries-de-paris' + resp.encoding = 'utf8' + with open('src/tests/fixtures/article.html', 'rb') as fd: + resp._content = fd.read() + return resp + + @property + def article2(self): + resp = Response() + resp.url = 'https://www.youtube.com/watch?v=scbrjaqM3Oc' + resp.encoding = 'utf8' + with open('src/tests/fixtures/article-2.html', 'rb') as fd: + resp._content = fd.read() + return resp + + def test_extract_tags(self): + self.assertEquals(set(), extract_tags(self.article)) + self.assertEquals({'twitch', 'games'}, extract_tags(self.article2)) + + def test_extract_title(self): + self.assertEquals('Les plus belles boulangeries de Paris', + extract_title(self.article)) + self.assertEquals("Ceci n'est pas Old Boy - Owlboy (suite) " + "- Benzaie Live - YouTube", + extract_title(self.article2)) + + def test_extract_feed_link(self): + feed_split = urllib.parse.urlsplit(self.article.url) + self.assertEquals(self.article.url + '/feed', + extract_feed_link(self.article, feed_split)) + + yt_feed_link = 'http://www.youtube.com/oembed?url=https%3A%2F%2F'\ + 'www.youtube.com%2Fwatch%3Fv%3DscbrjaqM3Oc&format=xml' + feed_split = urllib.parse.urlsplit(self.article2.url) + self.assertEquals(yt_feed_link, + extract_feed_link(self.article2, feed_split)) + + @patch('lib.html_parsing.try_get_icon_url') + def test_extract_icon_url(self, get_icon_patch): + def return_first_val(*a, **kwargs): + return a[0] + get_icon_patch.side_effect = return_first_val + split = urllib.parse.urlsplit(self.article.url) + self.assertEquals('http://www.pariszigzag.fr/wp-content/themes' + '/paris_zigzag_2016/favicon.ico', + extract_icon_url(self.article, split, split)) + split = urllib.parse.urlsplit(self.article2.url) + self.assertEquals('https://s.ytimg.com/yts/img/favicon-vflz7uhzw.ico', + extract_icon_url(self.article2, split, split)) From 77e265d656ad192a283c3f61f29dc8ceb5864e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Thu, 24 Nov 2016 18:37:03 +0100 Subject: [PATCH 046/164] adding fix for relative urls in articles --- src/tests/libs/article_cleaner_test.py | 91 +++++++++++++++ src/tests/libs/article_utils_test.py | 147 ++++++++++++------------- src/web/lib/article_cleaner.py | 101 ++++++++++------- 3 files changed, 222 insertions(+), 117 deletions(-) create mode 100644 src/tests/libs/article_cleaner_test.py diff --git a/src/tests/libs/article_cleaner_test.py b/src/tests/libs/article_cleaner_test.py new file mode 100644 index 000000000..f190beb67 --- /dev/null +++ b/src/tests/libs/article_cleaner_test.py @@ -0,0 +1,91 @@ +import json +import unittest + +from mock import patch +from requests import Response +from requests.exceptions import MissingSchema + +from lib.article_utils import construct_article + + +class ConstructArticleTest(unittest.TestCase): + response_url = '//www.pariszigzag.fr/paris-insolite-secret/'\ + 'les-plus-belles-boulangeries-de-paris' + + def setUp(self): + self._jarr_get_patch = patch('lib.article_utils.jarr_get') + self.jarr_get_patch = self._jarr_get_patch.start() + + def tearDown(self): + self._jarr_get_patch.stop() + + @property + def entry(self): + with open('src/tests/fixtures/article.json') as fd: + return json.load(fd) + + @property + def entry2(self): + with open('src/tests/fixtures/article-2.json') as fd: + return json.load(fd) + + @staticmethod + def get_response(scheme='http:'): + resp = Response() + resp.url = scheme + ConstructArticleTest.response_url + resp.encoding = 'utf8' + with open('src/tests/fixtures/article.html') as fd: + resp._content = fd.read() + return resp + + @property + def response2(self): + resp = Response() + resp.url = 'https://www.youtube.com/watch?v=scbrjaqM3Oc' + resp.encoding = 'utf8' + with open('src/tests/fixtures/article-2.html') as fd: + resp._content = fd.read() + return resp + + def test_missing_title(self): + self.jarr_get_patch.return_value = self.get_response('http:') + article = construct_article(self.entry, {'id': 1, 'user_id': 1}) + self.assertEquals('http://www.pariszigzag.fr/?p=56413', + article['entry_id']) + self.assertEquals('http:' + self.response_url, article['link']) + self.assertEquals('Les plus belles boulangeries de Paris', + article['title']) + self.assertEquals(1, article['user_id']) + self.assertEquals(1, article['feed_id']) + + def test_missing_scheme(self): + response = self.get_response('http:') + self.jarr_get_patch.side_effect = [ + MissingSchema, MissingSchema, response] + entry = self.entry + entry['link'] = entry['link'][5:] + + article = construct_article(entry, {'id': 1, 'user_id': 1}) + + self.assertEquals(3, self.jarr_get_patch.call_count) + self.assertEquals(response.url, self.jarr_get_patch.call_args[0][0]) + self.assertEquals('http://www.pariszigzag.fr/?p=56413', + article['entry_id']) + self.assertEquals(response.url, article['link']) + self.assertEquals('Les plus belles boulangeries de Paris', + article['title']) + self.assertEquals(1, article['user_id']) + self.assertEquals(1, article['feed_id']) + + def test_tags(self): + import bootstrap + bootstrap.conf.CRAWLER_RESOLV = True + self.jarr_get_patch.return_value = self.response2 + article = construct_article(self.entry2, {'id': 1, 'user_id': 1}) + self.assertEquals('yt:video:scbrjaqM3Oc', article['entry_id']) + self.assertEquals(self.response2.url, article['link']) + self.assertEquals("Ceci n'est pas Old Boy - Owlboy (suite) - " + "Benzaie Live", article['title']) + self.assertEquals(1, article['user_id']) + self.assertEquals(1, article['feed_id']) + self.assertEquals({'twitch', 'games'}, article['tags']) diff --git a/src/tests/libs/article_utils_test.py b/src/tests/libs/article_utils_test.py index f190beb67..c645894fb 100644 --- a/src/tests/libs/article_utils_test.py +++ b/src/tests/libs/article_utils_test.py @@ -1,91 +1,80 @@ -import json import unittest from mock import patch -from requests import Response -from requests.exceptions import MissingSchema -from lib.article_utils import construct_article +from web.lib.article_cleaner import clean_urls +SAMPLE = """ + + +test + + +""" -class ConstructArticleTest(unittest.TestCase): - response_url = '//www.pariszigzag.fr/paris-insolite-secret/'\ - 'les-plus-belles-boulangeries-de-paris' - def setUp(self): - self._jarr_get_patch = patch('lib.article_utils.jarr_get') - self.jarr_get_patch = self._jarr_get_patch.start() - - def tearDown(self): - self._jarr_get_patch.stop() - - @property - def entry(self): - with open('src/tests/fixtures/article.json') as fd: - return json.load(fd) - - @property - def entry2(self): - with open('src/tests/fixtures/article-2.json') as fd: - return json.load(fd) +class ArticleCleanerTest(unittest.TestCase): - @staticmethod - def get_response(scheme='http:'): - resp = Response() - resp.url = scheme + ConstructArticleTest.response_url - resp.encoding = 'utf8' - with open('src/tests/fixtures/article.html') as fd: - resp._content = fd.read() - return resp - - @property - def response2(self): - resp = Response() - resp.url = 'https://www.youtube.com/watch?v=scbrjaqM3Oc' - resp.encoding = 'utf8' - with open('src/tests/fixtures/article-2.html') as fd: - resp._content = fd.read() - return resp - - def test_missing_title(self): - self.jarr_get_patch.return_value = self.get_response('http:') - article = construct_article(self.entry, {'id': 1, 'user_id': 1}) - self.assertEquals('http://www.pariszigzag.fr/?p=56413', - article['entry_id']) - self.assertEquals('http:' + self.response_url, article['link']) - self.assertEquals('Les plus belles boulangeries de Paris', - article['title']) - self.assertEquals(1, article['user_id']) - self.assertEquals(1, article['feed_id']) + def setUp(self): + self.sample = SAMPLE.split('\n') + self.url = 'https://test.te' - def test_missing_scheme(self): - response = self.get_response('http:') - self.jarr_get_patch.side_effect = [ - MissingSchema, MissingSchema, response] - entry = self.entry - entry['link'] = entry['link'][5:] + @patch('web.lib.article_cleaner.is_secure_served') + def test_clean_clear(self, is_secure_served): + is_secure_served.return_value = False + result = clean_urls(SAMPLE, self.url).split('\n') + self.assertEqual('', + result[0]) + self.assertEqual(self.sample[1], result[1]) # unchanged + self.assertEqual(self.sample[2], result[2]) # unchanged + self.assertEqual(self.sample[3], result[3]) # unchanged + self.assertEqual(self.sample[4], result[4]) # unchanged + self.assertEqual(self.sample[6], result[6]) # unchanged + self.assertEqual('' % self.url, result[7]) - article = construct_article(entry, {'id': 1, 'user_id': 1}) + @patch('web.lib.article_cleaner.is_secure_served') + def test_clean_https(self, is_secure_served): + is_secure_served.return_value = True + result = clean_urls(SAMPLE, self.url).split('\n') + self.assertEqual('', + result[0]) + self.assertEqual(self.sample[1], result[1]) # unchanged + self.assertEqual(self.sample[2], result[2]) # unchanged + self.assertEqual(self.sample[3], result[3]) # unchanged + self.assertEqual( + ' + +""" + + +class MercuryIntegrationTest(unittest.TestCase): + + def setUp(self): + self.inte = MercuryIntegration() + + def test_match_feed_creation(self): + self.assertFalse(self.inte.match_feed_creation()) + + @property + def base_objs(self): + user = Mock(readability_key=False) + feed = Mock(readability_auto_parse=True) + + class Article(dict): + id, link, readability_parsed = 1, 'http://whatever.com/', True + cluster = Mock() + cluster.main_article = Article() + cluster.main_article['readability_parsed'] = True + return user, feed, cluster + + def test_match_article_parsing(self): + self.assertFalse(self.inte.match_article_parsing( + None, None, None, mercury_may_parse=False)) + + user, feed, cluster = self.base_objs + + self.assertFalse(self.inte.match_article_parsing( + user, feed, cluster, mercury_may_parse=True)) + + user.readability_key = True + + self.assertFalse(self.inte.match_article_parsing( + user, feed, cluster, mercury_may_parse=True)) + + cluster.main_article.readability_parsed = False + + self.assertTrue(self.inte.match_article_parsing( + user, feed, cluster, mercury_may_parse=True)) + + feed.readability_auto_parse = False + + self.assertTrue(self.inte.match_article_parsing(user, feed, cluster, + mercury_may_parse=True, mercury_parse=True)) + + self.assertFalse(self.inte.match_article_parsing( + user, feed, cluster, mercury_may_parse=True)) + + @patch('lib.integrations.mercury.jarr_get') + @patch('lib.integrations.mercury.flash') + @patch('lib.integrations.mercury.ArticleController') + def test_parsing(self, actrl, flash, jarr_get): + jarr_get.return_value.json.return_value = {} + user, feed, cluster = self.base_objs + cluster.main_article['readability_parsed'] = False + self.assertEqual(cluster.main_article, + self.inte.article_parsing(user, feed, cluster)) + self.assertEqual('Mercury responded with {}(1)', flash.call_args[0][0]) + self.assertFalse(cluster.main_article['readability_parsed']) + + jarr_get.return_value.json.return_value = {'garbage': 'garbage'} + self.assertEqual(cluster.main_article, + self.inte.article_parsing(user, feed, cluster)) + self.assertEqual('Mercury responded without content', + flash.call_args[0][0]) + self.assertFalse(cluster.main_article['readability_parsed']) + + jarr_get.return_value.json.return_value = {'content': 'content'} + new_article = self.inte.article_parsing(user, feed, cluster) + self.assertEqual('content', new_article['content']) + self.assertEqual('content', cluster.main_article['content']) + self.assertTrue(cluster.main_article['readability_parsed']) diff --git a/src/web/views/home.py b/src/web/views/home.py index 6295f5324..e7c5b74be 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -1,16 +1,15 @@ import logging from babel.dates import format_datetime, format_timedelta -from flask import current_app, flash, render_template, request, url_for +from flask import current_app, render_template, request, url_for from flask_babel import get_locale from flask_login import current_user, login_required from bootstrap import conf from lib.utils import utc_now -from plugins import readability -from web.controllers import (ArticleController, CategoryController, - ClusterController, FeedController, UserController) -from web.lib.article_cleaner import clean_urls +from lib import integrations +from web.controllers import (CategoryController, ClusterController, + FeedController, UserController) from web.lib.view_utils import clusters_to_json, etag_match, get_notifications from web.views.common import jsonify @@ -112,7 +111,6 @@ def get_middle_panel(): @jsonify def get_cluster(cluster_id, parse=False, article_id=None): locale = get_locale() - artc = ArticleController(current_user.id) cluc = ClusterController(current_user.id) cluster = cluc.get(id=cluster_id) if not cluster.read: @@ -125,28 +123,9 @@ def get_cluster(cluster_id, parse=False, article_id=None): readability_available = bool(current_user.readability_key or conf.PLUGINS_READABILITY_KEY) cluster['main_date'] = format_datetime(cluster.main_date, locale=locale) - if parse or (not cluster.main_article.readability_parsed - and feed.readability_auto_parse and readability_available): - if article_id: - article = next(article for article in cluster.articles - if article.id == article_id) - else: - article = cluster.main_article - try: - new_content = readability.parse(article.link, - current_user.readability_key - or conf.PLUGINS_READABILITY_KEY) - new_content = clean_urls(new_content, article['link'], - fix_readability=True) - except Exception as error: - error = str(error).split(conf.PLUGINS_READABILITY_KEY)[0] - flash("Readability failed with %r" % error, "warning") - article['readability_parsed'] = False - else: - article['readability_parsed'] = True - article['content'] = new_content - artc.update({'id': article.id}, - {'readability_parsed': True, 'content': new_content}) + integrations.dispatch('article_parsing', current_user, feed, cluster, + mercury_may_parse=True, # enabling mercury parsing + mercury_parse=parse, article_id=article_id) for article in cluster.articles: article['readability_available'] = readability_available article['date'] = format_datetime(article.date, locale=locale) From ef2ec457acfae474d1333b527fccb843baeb7a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Thu, 1 Dec 2016 14:45:06 +0100 Subject: [PATCH 050/164] fixing various codeclimate warning --- .codeclimate.yml | 2 +- install.py | 89 +++++++++-------- src/crawler/lib/headers_handling.py | 49 +++++----- src/lib/article_utils.py | 111 +++++++++++----------- src/lib/feed_utils.py | 110 +++++++++++++-------- src/web/controllers/abstract.py | 81 ++++++++-------- src/web/controllers/cluster.py | 121 ++++++++++++------------ src/web/js/stores/IconStore.js | 4 +- src/web/js/stores/NotificationsStore.js | 14 +-- submodules/bootstrap | 2 +- 10 files changed, 322 insertions(+), 261 deletions(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index ee7ff26c7..3b521a98e 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -17,7 +17,7 @@ engines: radon: enabled: true config: - threshold: "C" + threshold: "D" ratings: paths: - "**.css" diff --git a/install.py b/install.py index a72032230..c42734f27 100755 --- a/install.py +++ b/install.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import imp from sys import stderr, path from subprocess import Popen, PIPE from argparse import ArgumentParser @@ -9,6 +8,9 @@ from lib import conf_handling +LOGIN_OPTION_NAME = "CRAWLER_LOGIN" +PASSWORD_OPTION_NAME = "CRAWLER_PASSWD" + def parse_args(): parser = ArgumentParser("""This script is aimed to bootstrap JARR. @@ -45,20 +47,25 @@ def can_npm(): return bool(Popen(['which', 'npm'], stdout=PIPE).communicate()[0]) +def _print_line_informations(choices, default): + if choices: + print('[%s]' % '/'.join([chc.upper() if chc == default + or choices[chc] == default else chc + for chc in choices]), end=' ') + if default not in {'', None}: + if choices and default not in choices: + for new_default, value in choices.items(): + if value == default: + default = new_default + break + print('(default: %r)' % default, end=' ') + return default + + def ask(text, choices=[], default=None, cast=None): while True: print(text, end=' ') - if choices: - print('[%s]' % '/'.join([chc.upper() if chc == default - or choices[chc] == default else chc - for chc in choices]), end=' ') - if default not in {'', None}: - if choices and default not in choices: - for new_default, value in choices.items(): - if value == default: - default = new_default - break - print('(default: %r)' % default, end=' ') + default = _print_line_informations(choices, default) print(':', end=' ') result = input() @@ -83,9 +90,8 @@ def ask(text, choices=[], default=None, cast=None): return result -def build_conf(test=False, creds={}): +def _get_conf(test, creds): conf, could_import_conf = None, False - logins, passwords = {'CRAWLER_LOGIN'}, {'CRAWLER_PASSWD'} if not test: try: conf = conf_handling.ConfObject() @@ -99,6 +105,36 @@ def build_conf(test=False, creds={}): else: creds['login'] = conf.CRAWLER_LOGIN creds['password'] = conf.CRAWLER_PASSWD + return conf, could_import_conf + + +def _get_default(option, name, test, conf, could_import_conf): + if test and 'test' in option: + return option['test'] + elif could_import_conf and hasattr(conf, name): + return getattr(conf, name) + elif name == 'BUNDLE_JS' and can_npm(): + return 'local' + return option.get('default') + + +def _get_value(option, name, default, edit, test, creds): + if (edit and not test) or (name == 'BUNDLE_JS' and not can_npm()): + value = ask(option['ask'], choices=option.get('choices', []), + default=default) + else: + value = default + if 'type' in option: + value = option['type'](value) + if name == LOGIN_OPTION_NAME and creds.get('login'): + value = creds['login'] + if name == PASSWORD_OPTION_NAME and creds.get('password'): + value = creds['password'] + return value + + +def build_conf(test=False, creds={}): + conf, could_import_conf = _get_conf(test, creds) for section in conf_handling.SECTIONS: section_edit = section.get('edit', True) @@ -111,30 +147,9 @@ def build_conf(test=False, creds={}): title(prefix) for option in section['options']: edit = section_edit and option.get('edit', True) - name = conf_handling.get_key(section, option) - - if test and 'test' in option: - default = option['test'] - elif could_import_conf and hasattr(conf, name): - default = getattr(conf, name) - elif name == 'BUNDLE_JS' and can_npm(): - default = 'local' - else: - default = option.get('default') - - if (edit and not test) \ - or (name == 'BUNDLE_JS' and not can_npm()): - value = ask(option['ask'], choices=option.get('choices', []), - default=default) - else: - value = default - if 'type' in option: - value = option['type'](value) - if name in logins and creds.get('login'): - value = creds['login'] - if name in passwords and creds.get('password'): - value = creds['password'] + default = _get_default(option, name, test, conf, could_import_conf) + value = _get_value(option, name, default, edit, test, creds) yield prefix, name, value print() diff --git a/src/crawler/lib/headers_handling.py b/src/crawler/lib/headers_handling.py index fa743e79a..59b48612b 100644 --- a/src/crawler/lib/headers_handling.py +++ b/src/crawler/lib/headers_handling.py @@ -22,6 +22,28 @@ def rfc_1123_utc(time_obj=None, delta=None): return time_obj.strftime(RFC_1123_FORMAT) +def _extract_max_age(headers, feed_info, now): + if 'max-age' in headers.get('cache-control', ''): + try: + max_age = int(MAX_AGE_RE.search(headers['cache-control']).group(1)) + feed_info['expires'] = now + timedelta(seconds=max_age) + except Exception: + pass + + +def _extract_expires(headers, feed_info): + if headers.get('expires'): + try: + expires = dateutil.parser.parse(headers['expires']) + if expires.tzinfo: + expires = expires.astimezone(timezone.utc) + else: + expires = expires.replace(tzinfo=timezone.utc) + feed_info['expires'] = expires + except Exception: + pass + + def extract_feed_info(headers): """providing the headers of a feed response, will calculate the headers needed for basic cache control. @@ -37,32 +59,13 @@ def extract_feed_info(headers): feed_info = {'etag': headers.get('etag', ''), 'last_modified': headers.get('last-modified', rfc_1123_utc())} - msg = "didn't found expiring mechanism, expiring at %r" - if 'max-age' in headers.get('cache-control', ''): - msg = 'found Cache-Control "max-age" header, expiring at %r' - try: - max_age = int(MAX_AGE_RE.search(headers['cache-control']).group(1)) - except Exception: - pass - else: - feed_info['expires'] = now + timedelta(seconds=max_age) - if 'expires' not in feed_info and headers.get('expires'): - msg = "found Expires header, expiring at %r" - try: - expires = dateutil.parser.parse(headers['expires']) - if expires.tzinfo: - expires = expires.astimezone(timezone.utc) - else: - expires = expires.replace(tzinfo=timezone.utc) - except Exception: - pass - else: - feed_info['expires'] = expires - + _extract_max_age(headers, feed_info, now) + if 'expires' not in feed_info: + _extract_expires(headers, feed_info) if not feed_info.get('expires'): feed_info['expires'] = now + timedelta( seconds=conf.FEED_DEFAULT_EXPIRES) - logger.info(msg, feed_info['expires'].isoformat()) + if max_expires < feed_info['expires']: logger.info("expiring too late, forcing expiring at %r", max_expires.isoformat()) diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index 9f1fa352c..0a2724bb1 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -65,32 +65,31 @@ def get_article_content(entry): return content +def _fetch_article(link): + try: + # resolves URL behind proxies (like feedproxy.google.com) + return jarr_get(link, timeout=5) + except MissingSchema: + split = urlsplit(link) + for scheme in 'https', 'http': + new_link = urlunsplit(SplitResult(scheme, *split[1:])) + try: + return jarr_get(new_link, timeout=5) + except Exception as error: + continue + except Exception as error: + logger.info("Unable to get the real URL of %s. Won't fix " + "link or title. Error: %s", link, error) + + def get_article_details(entry, fetch=True): article_link = entry.get('link') article_title = html.unescape(entry.get('title', '')) tags = {tag.get('term').strip() for tag in entry.get('tags', []) if tag.get('term').strip()} if fetch and conf.CRAWLER_RESOLV and article_link or not article_title: - try: - # resolves URL behind proxies (like feedproxy.google.com) - response = jarr_get(article_link, timeout=5) - except MissingSchema: - split, failed = urlsplit(article_link), False - for scheme in 'https', 'http': - new_link = urlunsplit(SplitResult(scheme, *split[1:])) - try: - response = jarr_get(new_link, timeout=5) - except Exception as error: - failed = True - continue - failed = False - article_link = new_link - break - if failed: - return article_link, article_title or 'No title', tags - except Exception as error: - logger.info("Unable to get the real URL of %s. Won't fix " - "link or title. Error: %s", article_link, error) + response = _fetch_article(article_link) + if response is None: return article_link, article_title or 'No title', tags article_link = response.url if not article_title: @@ -118,47 +117,53 @@ class FiltersTrigger(Enum): NO_MATCH = 'no match' +def _is_filter_to_skip(filter_action, only_actions, article): + if filter_action not in only_actions: + return True + if filter_action in {FiltersType.REGEX, FiltersType.MATCH, + FiltersType.EXACT_MATCH} and 'title' not in article: + return True + if filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS} \ + and 'tags' not in article: + return True + return False + + +def _is_filter_matching(filter_, article): + pattern = filter_.get('pattern', '') + filter_type = FiltersType(filter_.get('type')) + filter_trigger = FiltersTrigger(filter_.get('action on')) + if filter_type is not FiltersType.REGEX: + pattern = pattern.lower() + title = article.get('title', '').lower() + tags = [tag.lower() for tag in article.get('tags', [])] + if filter_type is FiltersType.REGEX: + match = re.match(pattern, title) + elif filter_type is FiltersType.MATCH: + match = pattern in title + elif filter_type is FiltersType.EXACT_MATCH: + match = pattern == title + elif filter_type is FiltersType.TAG_MATCH: + match = pattern in tags + elif filter_type is FiltersType.TAG_CONTAINS: + match = any(pattern in tag for tag in tags) + return match and filter_trigger is FiltersTrigger.MATCH \ + or not match and filter_trigger is FiltersTrigger.NO_MATCH + + def process_filters(filters, article, only_actions=None): skipped, read, liked = False, None, False filters = filters or [] if only_actions is None: only_actions = set(FiltersAction) for filter_ in filters: - match = False - try: - pattern = filter_.get('pattern', '') - filter_type = FiltersType(filter_.get('type')) - filter_action = FiltersAction(filter_.get('action')) - filter_trigger = FiltersTrigger(filter_.get('action on')) - if filter_type is not FiltersType.REGEX: - pattern = pattern.lower() - except ValueError: - continue - if filter_action not in only_actions: + filter_action = FiltersAction(filter_.get('action')) + + if _is_filter_to_skip(filter_action, only_actions, article): logger.debug('ignoring filter %r' % filter_) continue - if filter_action in {FiltersType.REGEX, FiltersType.MATCH, - FiltersType.EXACT_MATCH} and 'title' not in article: - continue - if filter_action in {FiltersType.TAG_MATCH, FiltersType.TAG_CONTAINS} \ - and 'tags' not in article: - continue - title = article.get('title', '').lower() - tags = [tag.lower() for tag in article.get('tags', [])] - if filter_type is FiltersType.REGEX: - match = re.match(pattern, title) - elif filter_type is FiltersType.MATCH: - match = pattern in title - elif filter_type is FiltersType.EXACT_MATCH: - match = pattern == title - elif filter_type is FiltersType.TAG_MATCH: - match = pattern in tags - elif filter_type is FiltersType.TAG_CONTAINS: - match = any(pattern in tag for tag in tags) - take_action = match and filter_trigger is FiltersTrigger.MATCH \ - or not match and filter_trigger is FiltersTrigger.NO_MATCH - - if not take_action: + + if not _is_filter_matching(filter_, article): continue if filter_action is FiltersAction.READ: diff --git a/src/lib/feed_utils.py b/src/lib/feed_utils.py index 177260189..2641f223d 100644 --- a/src/lib/feed_utils.py +++ b/src/lib/feed_utils.py @@ -2,6 +2,7 @@ import logging import urllib from copy import deepcopy +from functools import lru_cache import feedparser @@ -53,24 +54,17 @@ def get_parsed_feed(url): return fp_parsed -@correct_feed_values -def construct_feed_from(url=None, fp_parsed=None, feed=None): - """ - Will try to construct the most complete feed dict possible. +@lru_cache(maxsize=None) +def get_splits(url, site_link=None): + # trying to make up for missing values + feed_split = urllib.parse.urlsplit(url) + site_split = None + if site_link: + site_split = urllib.parse.urlsplit(site_link) + return site_split, feed_split - url: an url of a feed or a site that might be hosting a feed - fp_parsed: a feedparser object previously obtained - feed: an existing feed dict, will be updated - """ - feed = deepcopy(feed) if feed else {} - if not url and hasattr(fp_parsed, 'get') and fp_parsed.get('href'): - url = fp_parsed.get('href') - - # we'll try to obtain our first parsing from feedparser - if url and not fp_parsed: - fp_parsed = get_parsed_feed(url) - assert url is not None and fp_parsed is not None +def _extract_links(url, feed, fp_parsed): if is_parsing_ok(fp_parsed): feed['link'] = url else: @@ -93,25 +87,18 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None): and link['type'] == 'text/html')) feed['site_link'] = site_link or feed.get('site_link') feed['site_link'] = feed['site_link'] or feed.get('link') + return fp_parsed - if not feed.get('title'): # not overriding user pref for title - if fp_parsed['feed'].get('title'): - feed['title'] = fp_parsed['feed'].get('title') - elif fp_parsed['feed'].get('title_detail', {}).get('value'): - feed['title'] = fp_parsed['feed']['title_detail']['value'] - if fp_parsed['feed'].get('summary'): - feed['description'] = fp_parsed['feed']['summary'] - elif fp_parsed['feed'].get('subtitle_detail', {}).get('value'): - feed['description'] = fp_parsed['feed']['subtitle_detail']['value'] +def _update_feed_w_parsed(fkey, simple_key, value_key, feed, fp_parsed): + if fp_parsed['feed'].get(simple_key): + feed[fkey] = fp_parsed['feed'].get(simple_key) + elif fp_parsed['feed'].get(value_key, {}).get('value'): + feed[fkey] = fp_parsed['feed'][value_key]['value'] - # trying to make up for missing values - feed_split = urllib.parse.urlsplit(url) - site_split = None - if feed.get('site_link'): - feed['site_link'] = rebuild_url(feed['site_link'], feed_split) - site_split = urllib.parse.urlsplit(feed['site_link']) +def _check_and_fix_icon(url, feed, fp_parsed): + site_split, feed_split = get_splits(url, feed.get('site_link')) new_icon_urls = [fp_parsed.get('feed', {}).get('icon')] \ + list(_browse_feedparser_feed(fp_parsed, lambda link: 'icon' in link['rel'])) @@ -124,16 +111,12 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None): feed['icon_url'] = icon_url break - nothing_to_fill = all(bool(feed.get(key)) - for key in ('link', 'title', 'icon_url')) - # here we have all we want or we do not have the main url, - # either way we're leaving - if not feed.get('site_link') or nothing_to_fill: - return feed - # trying to parse the page of the site for some rel link in the header +def _fetch_url_and_enhance_feed(url, feed): + """trying to parse the page of the site for some rel link in the header""" + site_split, feed_split = get_splits(url, feed.get('site_link')) try: - response = jarr_get(feed['site_link']) + response = jarr_get(url) except Exception as error: logger.warn('failed to retreive %r: %r', feed['site_link'], error) return feed @@ -147,3 +130,52 @@ def construct_feed_from(url=None, fp_parsed=None, feed=None): if not feed.get('link'): feed['link'] = extract_feed_link(response, feed_split) return feed + + +def _is_processing_complete(feed, site_link_necessary=False): + all_filled = all(bool(feed.get(key)) + for key in ('link', 'title', 'icon_url')) + # here we have all we want or we do not have the main url, + # either way we're leaving + return (site_link_necessary and not feed.get('site_link')) or all_filled + + +@correct_feed_values +def construct_feed_from(url=None, fp_parsed=None, feed=None): + """ + Will try to construct the most complete feed dict possible. + + url: an url of a feed or a site that might be hosting a feed + fp_parsed: a feedparser object previously obtained + feed: an existing feed dict, will be updated + """ + feed = deepcopy(feed) if feed else {} + if not url and hasattr(fp_parsed, 'get') and fp_parsed.get('href'): + url = fp_parsed.get('href') + + # we'll try to obtain our first parsing from feedparser + if url and not fp_parsed: + fp_parsed = get_parsed_feed(url) + assert url is not None and fp_parsed is not None + + fp_parsed = _extract_links(url, feed, fp_parsed) + + if not feed.get('title'): # not overriding user pref for title + _update_feed_w_parsed('title', 'title', 'title_detail', + feed, fp_parsed) + _update_feed_w_parsed('description', 'summary', 'subtitle_detail', + feed, fp_parsed) + + if _is_processing_complete(feed): + return feed + + if feed.get('site_link'): + feed['site_link'] = rebuild_url(feed['site_link'], + get_splits(url, feed['site_link'])[1]) + + _check_and_fix_icon(url, feed, fp_parsed) + + if _is_processing_complete(feed, site_link_necessary=True): + return feed + + return _fetch_url_and_enhance_feed(feed['site_link'], feed) diff --git a/src/web/controllers/abstract.py b/src/web/controllers/abstract.py index c78a21e7e..0e95dc198 100644 --- a/src/web/controllers/abstract.py +++ b/src/web/controllers/abstract.py @@ -36,6 +36,18 @@ def __init__(self, user_id=None, ignore_context=False): except TypeError: self.user_id = user_id + @staticmethod + def _to_comparison(key, model): + "extract from the key the method used by sqla for comparison" + if '__' not in key: + return getattr(model, key).__eq__ + attr, ope = key.rsplit('__', 1) + if ope == 'in': + return getattr(model, attr).in_ + elif ope not in {'like', 'ilike'}: + ope = '__%s__' % ope + return getattr(getattr(model, attr), ope) + @classmethod def _to_filters(cls, **filters): """ @@ -44,31 +56,15 @@ def _to_filters(cls, **filters): each parameters of the function is treated as an equality unless the name of the parameter ends with either "__gt", "__lt", "__ge", "__le", - "__ne", "__in" ir "__like". + "__ne", "__in", "__like" or "__ilike". """ db_filters = set() for key, value in filters.items(): if key == '__or__': db_filters.add(or_(*[and_(*cls._to_filters(**sub_filter)) for sub_filter in value])) - elif key.endswith('__gt'): - db_filters.add(getattr(cls._db_cls, key[:-4]) > value) - elif key.endswith('__lt'): - db_filters.add(getattr(cls._db_cls, key[:-4]) < value) - elif key.endswith('__ge'): - db_filters.add(getattr(cls._db_cls, key[:-4]) >= value) - elif key.endswith('__le'): - db_filters.add(getattr(cls._db_cls, key[:-4]) <= value) - elif key.endswith('__ne'): - db_filters.add(getattr(cls._db_cls, key[:-4]) != value) - elif key.endswith('__in'): - db_filters.add(getattr(cls._db_cls, key[:-4]).in_(value)) - elif key.endswith('__like'): - db_filters.add(getattr(cls._db_cls, key[:-6]).like(value)) - elif key.endswith('__ilike'): - db_filters.add(getattr(cls._db_cls, key[:-7]).ilike(value)) else: - db_filters.add(getattr(cls._db_cls, key) == value) + db_filters.add(cls._to_comparison(key, cls._db_cls)(value)) return db_filters def _get(self, **filters): @@ -141,20 +137,37 @@ def _extra_columns(cls, role, right=None): return {} @classmethod - def _get_attrs_desc(cls, role, right=None): - result = defaultdict(dict) + def _get_columns(cls, role, right): if role == 'admin': - columns = set(cls._db_cls.__table__.columns.keys())\ + return set(cls._db_cls.__table__.columns.keys())\ .union(cls._db_cls.fields_base_read())\ .union(cls._db_cls.fields_base_write())\ .union(cls._db_cls.fields_api_read())\ .union(cls._db_cls.fields_api_write()) - else: - assert role in {'base', 'api'}, 'unknown role %r' % role - assert right in {'read', 'write'}, \ - "right must be 'read' or 'write' with role %r" % role - columns = getattr(cls._db_cls, 'fields_%s_%s' % (role, right))() - for column in columns: + assert role in {'base', 'api'}, 'unknown role %r' % role + assert right in {'read', 'write'}, \ + "right must be 'read' or 'write' with role %r" % role + return getattr(cls._db_cls, 'fields_%s_%s' % (role, right))() + + @staticmethod + def _db_col_to_dict(db_col): + dict_col = {} + try: + dict_col['type'] = db_col.type.python_type + except NotImplementedError: + if db_col.default: + dict_col['type'] = db_col.default.arg.__class__ + if issubclass(dict_col['type'], datetime): + dict_col['default'] = utc_now() + dict_col['type'] = cast_to_utc + elif db_col.default: + dict_col['default'] = db_col.default.arg + return dict_col + + @classmethod + def _get_attrs_desc(cls, role, right=None): + result = {} + for column in cls._get_columns(role, right): if isinstance(getattr(cls._db_cls, column), AssociationProxy): result[column] = {'type': list, 'default': list} continue @@ -162,18 +175,6 @@ def _get_attrs_desc(cls, role, right=None): db_col = getattr(cls._db_cls, column).property.columns[0] except AttributeError: continue - result[column] = {} - try: - result[column]['type'] = db_col.type.python_type - except NotImplementedError: - if db_col.default: - result[column]['type'] = db_col.default.arg.__class__ - if column not in result: - continue - if issubclass(result[column]['type'], datetime): - result[column]['default'] = utc_now() - result[column]['type'] = cast_to_utc - elif db_col.default: - result[column]['default'] = db_col.default.arg + result[column] = cls._db_col_to_dict(db_col) result.update(cls._extra_columns(role, right)) return result diff --git a/src/web/controllers/cluster.py b/src/web/controllers/cluster.py index 07488019e..2bba9da32 100644 --- a/src/web/controllers/cluster.py +++ b/src/web/controllers/cluster.py @@ -80,13 +80,8 @@ def clusterize(self, article, cluster_read=None, cluster_liked=False): cluster_read, cluster_liked) return self._create_from_article(article, cluster_read, cluster_liked) - def join_read(self, feed_id=None, **filters): + def _preprocess_per_article_filters(self, filters): art_filters = {} - filter_on_category = 'category_id' in filters - category_id = filters.pop('category_id', None) - if self.user_id: - filters['user_id'] = self.user_id - for key in {'__or__', 'title__ilike', 'content__ilike'}\ .intersection(filters): art_filters[key] = filters.pop(key) @@ -96,69 +91,36 @@ def join_read(self, feed_id=None, **filters): filters['id__in'] = {line[0] for line in art_contr .read(**art_filters).with_entities(Article.cluster_id)} - if not filters['id__in']: - return - - fields = {key: getattr(Cluster, key) for key in ('main_title', 'id', - 'liked', 'read', 'main_article_id', 'main_feed_title', - 'main_date', 'main_link')} - sqla_fields = list(fields.values()) + @staticmethod + def _get_selected(fields, art_f_alias, art_c_alias, filter_on_category): selected_fields = list(fields.values()) - - art_feed_alias, art_cat_alias = aliased(Article), aliased(Article) if SQLITE_ENGINE: selected_fields.append(func.group_concat( - art_feed_alias.feed_id).label('feeds_id')) + art_f_alias.feed_id).label('feeds_id')) if filter_on_category: selected_fields.append(func.group_concat( - art_cat_alias.category_id).label('categories_id')) + art_c_alias.category_id).label('categories_id')) else: # pragma: no cover - selected_fields.append(func.array_agg(art_feed_alias.feed_id, + selected_fields.append(func.array_agg(art_f_alias.feed_id, type_=ARRAY(Integer)).label('feeds_id')) if filter_on_category: selected_fields.append(func.array_agg( - art_cat_alias.category_id, + art_c_alias.category_id, type_=ARRAY(Integer)).label('categories_id')) - - # DESC of what's going on below : - # base query with the above fields and the aggregations - query = db.session.query(*selected_fields) - - # adding parent filter, but we can't just filter on one id, because - # we'll miss all the other parent of the cluster - if feed_id: - cluster_has_feed = exists(select([art_feed_alias.feed_id]) - .where(and_(art_feed_alias.cluster_id == Cluster.id, - art_feed_alias.user_id == self.user_id, - art_feed_alias.feed_id == feed_id)) - .correlate(Cluster).limit(1)) - query = query.join(art_feed_alias, - and_(art_feed_alias.user_id == self.user_id, - art_feed_alias.cluster_id == Cluster.id))\ - .filter(cluster_has_feed) - else: - query = query.join(art_feed_alias, - and_(art_feed_alias.user_id == self.user_id, - art_feed_alias.cluster_id == Cluster.id)) - if filter_on_category: - # joining only if filtering on categories to lighten the query - # as every article doesn't obligatorily have a category > outerjoin - category_filter = exists( - select([art_cat_alias.category_id]) - .where(and_(art_cat_alias.cluster_id == Cluster.id, - art_cat_alias.user_id == self.user_id, - art_cat_alias.category_id == category_id)) - .correlate(Cluster).limit(1)) - query = query.join(art_cat_alias, - and_(art_cat_alias.user_id == self.user_id, - art_cat_alias.cluster_id == Cluster.id))\ - .filter(category_filter) - - # applying common filter (read / liked) - # grouping all the fields so that agreg works on distant ids - query = query.group_by(*sqla_fields)\ - .filter(*self._to_filters(**filters)) - + return selected_fields + + def _join_on_exist(self, query, alias, attr, value): + val_col = getattr(alias, attr) + exist_query = exists(select([val_col]) + .where(and_(alias.cluster_id == Cluster.id, + alias.user_id == self.user_id, val_col == value)) + .correlate(Cluster).limit(1)) + return query.join(alias, and_(alias.user_id == self.user_id, + alias.cluster_id == Cluster.id))\ + .filter(exist_query) + + @staticmethod + def _iter_on_query(query, fields, filter_on_category): for clu in query.order_by(Cluster.main_date.desc()).limit(1000): row = {} for key in fields: @@ -176,6 +138,47 @@ def join_read(self, feed_id=None, **filters): row['categories_id'] = set(clu.categories_id) yield row + def join_read(self, feed_id=None, **filters): + filter_on_cat = 'category_id' in filters + cat_id = filters.pop('category_id', None) + if self.user_id: + filters['user_id'] = self.user_id + + self._preprocess_per_article_filters(filters) + if 'id__in' in filters and not filters['id__in']: + # filtering by article but did not found anything + return + + fields = {key: getattr(Cluster, key) for key in ('main_title', 'id', + 'liked', 'read', 'main_article_id', 'main_feed_title', + 'main_date', 'main_link')} + sqla_fields = list(fields.values()) + art_feed_alias, art_cat_alias = aliased(Article), aliased(Article) + # DESC of what's going on below : + # base query with the above fields and the aggregations + query = db.session.query(*self._get_selected(fields, + art_feed_alias, art_cat_alias, filter_on_cat)) + + # adding parent filter, but we can't just filter on one id, because + # we'll miss all the other parent of the cluster + if feed_id: + query = self._join_on_exist(query, art_feed_alias, + 'feed_id', feed_id) + else: + query = query.join(art_feed_alias, + and_(art_feed_alias.user_id == self.user_id, + art_feed_alias.cluster_id == Cluster.id)) + if filter_on_cat: + # joining only if filtering on categories to lighten the query + # as every article doesn't obligatorily have a category > outerjoin + query = self._join_on_exist(query, art_cat_alias, + 'category_id', cat_id) + + # applying common filter (read / liked) + # grouping all the fields so that agreg works on distant ids + yield from self._iter_on_query(query.group_by(*sqla_fields) + .filter(*self._to_filters(**filters)), fields, filter_on_cat) + def delete(self, obj_id): from web.controllers import ArticleController self.update({'id': obj_id}, {'main_article_id': None}, commit=False) diff --git a/src/web/js/stores/IconStore.js b/src/web/js/stores/IconStore.js index 115d70bb1..62abc4489 100644 --- a/src/web/js/stores/IconStore.js +++ b/src/web/js/stores/IconStore.js @@ -8,7 +8,7 @@ var assign = require('object-assign'); var IconStore = assign({}, EventEmitter.prototype, { icons: {}, getIcon: function(feed_id) { - var feed_id = feed_id.toString(); + feed_id = feed_id.toString(); if(feed_id in this.icons) { return this.icons[feed_id]; } @@ -22,7 +22,7 @@ var IconStore = assign({}, EventEmitter.prototype, { } var feed_id = feeds[idx].id.toString(); if(feeds[idx].id.toString() in this.icons - && feeds[idx].icon_url == this.icons[feed_id]) { + && feeds[idx].icon_url === this.icons[feed_id]) { continue; // icons already in } this.icons[feed_id] = feeds[idx].icon_url; diff --git a/src/web/js/stores/NotificationsStore.js b/src/web/js/stores/NotificationsStore.js index 21321233c..15a933d40 100644 --- a/src/web/js/stores/NotificationsStore.js +++ b/src/web/js/stores/NotificationsStore.js @@ -12,12 +12,14 @@ var NotificationsStore = assign({}, EventEmitter.prototype, { addNotifications: function(notifications) { var count = this.notifs.length; for(var idx in notifications) { - this.notifs.push({ - key: parseInt(idx, 10) + count, - read: false, - level: notifications[idx].level, - message: notifications[idx].message - }); + if(notifications[idx].level && notifications[idx].message) { + this.notifs.push({ + key: parseInt(idx, 10) + count, + read: false, + level: notifications[idx].level, + message: notifications[idx].message + }); + } } }, getNotifications: function() { diff --git a/submodules/bootstrap b/submodules/bootstrap index 7e2741eb1..9b7d1406b 160000 --- a/submodules/bootstrap +++ b/submodules/bootstrap @@ -1 +1 @@ -Subproject commit 7e2741eb1074afc66bdf2fb5c2b65d476cf7f37e +Subproject commit 9b7d1406b922300c5d01e99026e4fffc67445185 From e36763d9a2aab1b02071393908d638b956852788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 9 Dec 2016 14:17:59 +0100 Subject: [PATCH 051/164] removed default expires values if expires is set to None, a value is calculated on the feed average rate on the last duration of FEED_MAX_EXPIRES --- src/crawler/lib/headers_handling.py | 15 +++--- src/lib/conf_handling.py | 2 - src/lib/html_parsing.py | 6 +-- src/manager.py | 2 +- src/tests/libs/headers_handling_test.py | 23 ++++----- src/web/controllers/feed.py | 68 ++++++++++++++++--------- 6 files changed, 63 insertions(+), 53 deletions(-) diff --git a/src/crawler/lib/headers_handling.py b/src/crawler/lib/headers_handling.py index 59b48612b..2ed231917 100644 --- a/src/crawler/lib/headers_handling.py +++ b/src/crawler/lib/headers_handling.py @@ -62,20 +62,17 @@ def extract_feed_info(headers): _extract_max_age(headers, feed_info, now) if 'expires' not in feed_info: _extract_expires(headers, feed_info) - if not feed_info.get('expires'): - feed_info['expires'] = now + timedelta( - seconds=conf.FEED_DEFAULT_EXPIRES) - if max_expires < feed_info['expires']: + if not feed_info.get('expires'): + feed_info['expires'] = None + elif max_expires < feed_info['expires']: logger.info("expiring too late, forcing expiring at %r", max_expires.isoformat()) feed_info['expires'] = max_expires - if feed_info['expires'] < min_expires: - min_ex_plus_buffer = min_expires \ - + timedelta(seconds=conf.FEED_MIN_EXPIRES / 2) + elif feed_info['expires'] < min_expires: logger.info("expiring too early, forcing expiring at %r", - min_ex_plus_buffer.isoformat()) - feed_info['expires'] = min_ex_plus_buffer + min_expires.isoformat()) + feed_info['expires'] = min_expires + timedelta(minutes=5) return feed_info diff --git a/src/lib/conf_handling.py b/src/lib/conf_handling.py index c688126a8..16309c706 100644 --- a/src/lib/conf_handling.py +++ b/src/lib/conf_handling.py @@ -104,8 +104,6 @@ 'type': int, 'default': 3, 'edit': False}, {'key': 'MIN_EXPIRES', 'default': 60 * 10, 'type': int, 'edit': False}, - {'key': 'DEFAULT_EXPIRES', - 'default': 60 * 60, 'type': int, 'edit': False}, {'key': 'MAX_EXPIRES', 'default': 60 * 60 * 4, 'type': int, 'edit': False}, {'key': 'STOP_FETCH', 'default': 30, 'type': int, 'edit': False, diff --git a/src/lib/html_parsing.py b/src/lib/html_parsing.py index fda7e8778..cc2a732d8 100644 --- a/src/lib/html_parsing.py +++ b/src/lib/html_parsing.py @@ -73,9 +73,9 @@ def extract_tags(response): tags = set() keywords = soup.find_all('meta', {'name': 'keywords'}) if keywords: - tags = set(map(str.strip, sum([keyword.attrs['content'].split(',') - for keyword in keywords], []))) - tags = tags.union({meta.attrs['content'] + tags = set(sum([keyword.attrs.get('content', '').split(',') + for keyword in keywords], [])) + tags = tags.union({meta.attrs.get('content', '') for meta in soup.find_all('meta', {'property': 'article:tag'})}) return {tag.strip() for tag in tags if tag.strip()} diff --git a/src/manager.py b/src/manager.py index 2be813530..6ec7a1956 100755 --- a/src/manager.py +++ b/src/manager.py @@ -58,7 +58,7 @@ def reset_feeds(): .with_entities(fcontr._db_cls.id) .distinct()) - step = timedelta(seconds=conf.FEED_DEFAULT_EXPIRES / len(feeds)) + step = timedelta(seconds=conf.FEED_MIN_EXPIRES * 3 / len(feeds)) for i, feed in enumerate(feeds): fcontr.update({'id': feed[0]}, {'etag': '', 'last_modified': '', diff --git a/src/tests/libs/headers_handling_test.py b/src/tests/libs/headers_handling_test.py index d7d0f924b..1dd80bbad 100644 --- a/src/tests/libs/headers_handling_test.py +++ b/src/tests/libs/headers_handling_test.py @@ -17,22 +17,17 @@ def assert_in_range(val, ref, sec_range=1): class HeadersHandlingTest(unittest.TestCase): def test_defaulting(self): - def default_expire(): - return utc_now() + timedelta(seconds=conf.FEED_DEFAULT_EXPIRES) + self.assertEquals(None, extract_feed_info({})['expires']) - assert_in_range(extract_feed_info({})['expires'], - utc_now() + timedelta(seconds=conf.FEED_DEFAULT_EXPIRES)) + self.assertEquals(None, + extract_feed_info({'cache-control': ''})['expires']) + self.assertEquals(None, + extract_feed_info({'cache-control': 'garbage'})['expires']) - assert_in_range(extract_feed_info({'cache-control': ''})['expires'], - utc_now() + timedelta(seconds=conf.FEED_DEFAULT_EXPIRES)) - assert_in_range( - extract_feed_info({'cache-control': 'garbage'})['expires'], - utc_now() + timedelta(seconds=conf.FEED_DEFAULT_EXPIRES)) - - assert_in_range(extract_feed_info({'expires': ''})['expires'], - utc_now() + timedelta(seconds=conf.FEED_DEFAULT_EXPIRES)) - assert_in_range(extract_feed_info({'expires': 'garbage'})['expires'], - utc_now() + timedelta(seconds=conf.FEED_DEFAULT_EXPIRES)) + self.assertEquals(None, + extract_feed_info({'expires': ''})['expires']) + self.assertEquals(None, + extract_feed_info({'expires': 'garbage'})['expires']) def test_extract_max_age(self): max_age = conf.FEED_MAX_EXPIRES / 2 diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index 4907d0cbf..f4ef5631a 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) DEFAULT_LIMIT = 0 +DEFAULT_ART_SPAN_TIME = timedelta(seconds=conf.FEED_MAX_EXPIRES) class FeedController(AbstractController): @@ -100,34 +101,53 @@ def create(self, **attrs): self.__clean_feed_fields(attrs) return super().create(**attrs) + def __denorm_cat_id_on_articles(self, feed, attrs): + if 'category_id' in attrs: + self.__get_art_contr().update({'feed_id': feed.id}, + {'category_id': attrs['category_id']}) + + def __denorm_title_on_clusters(self, feed, attrs): + if 'title' in attrs: + # sqlite doesn't support join on update, but they're REALLY + # more efficient so we'll use them anyway with postgres + if self.user_id: + where_clause = and_(Article.user_id == self.user_id, + Article.feed_id == feed.id) + else: + where_clause = Article.feed_id == feed.id + if SQLITE_ENGINE: + stmt = select([Article.id]).where(where_clause) + stmt = update(Cluster)\ + .where(Cluster.main_article_id.in_(stmt))\ + .values(main_feed_title=attrs['title']) + else: # pragma: no cover + stmt = update(Cluster)\ + .where(and_( + Article.id == Cluster.main_article_id, + where_clause))\ + .values(dict(main_feed_title=attrs['title'])) + db.session.execute(stmt) + + def __update_default_expires(self, feed, attrs): + now = utc_now() + if 'expires' in attrs and not attrs['expires']: + span_time = timedelta(seconds=conf.FEED_MAX_EXPIRES) + art_count = self.__get_art_contr().read(feed_id=feed.id, + retrieved_date__gt=now - span_time).count() + attrs['expires'] = now + (span_time / (art_count or 1)) + def update(self, filters, attrs, *args, **kwargs): self._ensure_icon(attrs) self.__clean_feed_fields(attrs) - if {'title', 'category_id'}.intersection(attrs): + stuff_to_denorm = bool({'title', 'category_id'}.intersection(attrs)) + attrs_to_default = 'expires' in attrs and not attrs['expires'] + if stuff_to_denorm or attrs_to_default: for feed in self.read(**filters): - if 'category_id' in attrs: - self.__get_art_contr().update({'feed_id': feed.id}, - {'category_id': attrs['category_id']}) - if 'title' in attrs: - # sqlite doesn't support join on update, but they're REALLY - # more efficient so we'll use them anyway with postgres - if self.user_id: - where_clause = and_(Article.user_id == self.user_id, - Article.feed_id == feed.id) - else: - where_clause = Article.feed_id == feed.id - if SQLITE_ENGINE: - stmt = select([Article.id]).where(where_clause) - stmt = update(Cluster)\ - .where(Cluster.main_article_id.in_(stmt))\ - .values(main_feed_title=attrs['title']) - else: # pragma: no cover - stmt = update(Cluster)\ - .where(and_( - Article.id == Cluster.main_article_id, - where_clause))\ - .values(dict(main_feed_title=attrs['title'])) - db.session.execute(stmt) + if stuff_to_denorm: + self.__denorm_cat_id_on_articles(feed, attrs) + self.__denorm_title_on_clusters(feed, attrs) + if attrs_to_default: + self.__update_default_expires(feed, attrs) return super().update(filters, attrs, *args, **kwargs) def delete(self, obj_id): From 8a74b2f49314f780864f39d04ddaea4695633c21 Mon Sep 17 00:00:00 2001 From: Daniel Aleksandersen Date: Thu, 8 Dec 2016 14:23:52 +0100 Subject: [PATCH 052/164] Add support for feed deltas Add `A-IM: feed` header when either `If-Modified-Since` or `If-None-Match` is also set. --- src/crawler/lib/headers_handling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crawler/lib/headers_handling.py b/src/crawler/lib/headers_handling.py index 2ed231917..2d1b0b648 100644 --- a/src/crawler/lib/headers_handling.py +++ b/src/crawler/lib/headers_handling.py @@ -83,6 +83,8 @@ def prepare_headers(feed): headers['If-Modified-Since'] = feed['last_modified'] if feed.get('etag') and 'jarr' not in feed['etag']: headers['If-None-Match'] = feed['etag'] + if 'If-Modified-Since' in headers or 'If-None-Match' in headers: + headers['A-IM'] = 'feed' logger.debug('%r %r - calculated headers %r', feed['id'], feed['title'], headers) return headers From c32725a07857667c6578349d67b81de7c252c502 Mon Sep 17 00:00:00 2001 From: Daniel Aleksandersen Date: Thu, 8 Dec 2016 14:25:24 +0100 Subject: [PATCH 053/164] Set Accept header to prefer known feed formats Changes header from `*/*` to a list of known feed formats. --- src/crawler/lib/headers_handling.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/crawler/lib/headers_handling.py b/src/crawler/lib/headers_handling.py index 2d1b0b648..14390e7c7 100644 --- a/src/crawler/lib/headers_handling.py +++ b/src/crawler/lib/headers_handling.py @@ -78,7 +78,10 @@ def extract_feed_info(headers): def prepare_headers(feed): """For a known feed, will construct some header dictionnary""" - headers = {'User-Agent': conf.CRAWLER_USER_AGENT} + headers = {'User-Agent': conf.CRAWLER_USER_AGENT, + 'Accept': 'application/atom+xml,application/rss+xml,' + 'application/rdf+xml;q=0.8,application/xml;q=0.5,' + 'text/xml;q=0.5,*/*;q=0.2'} if feed.get('last_modified'): headers['If-Modified-Since'] = feed['last_modified'] if feed.get('etag') and 'jarr' not in feed['etag']: From e40775d6a028e60eae0fdecbfbde19aa6fc54ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 9 Dec 2016 18:18:02 +0100 Subject: [PATCH 054/164] regrouping const about mimetypes, using it when constructing feed --- src/crawler/lib/headers_handling.py | 10 +++++----- src/lib/const.py | 9 +++++++++ src/lib/feed_utils.py | 5 +++-- src/lib/html_parsing.py | 12 ++++++++++-- src/lib/integrations/mercury.py | 3 +-- src/lib/utils.py | 8 +++++--- src/tests/libs/headers_handling_test.py | 2 +- 7 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 src/lib/const.py diff --git a/src/crawler/lib/headers_handling.py b/src/crawler/lib/headers_handling.py index 14390e7c7..acc6a509c 100644 --- a/src/crawler/lib/headers_handling.py +++ b/src/crawler/lib/headers_handling.py @@ -4,6 +4,7 @@ import re from bootstrap import conf +from lib.const import FEED_ACCEPT_HEADERS from lib.utils import to_hash, utc_now logger = logging.getLogger(__name__) @@ -70,18 +71,17 @@ def extract_feed_info(headers): max_expires.isoformat()) feed_info['expires'] = max_expires elif feed_info['expires'] < min_expires: + min_exp_w_buffer = now + timedelta(seconds=conf.FEED_MIN_EXPIRES * 1.2) logger.info("expiring too early, forcing expiring at %r", - min_expires.isoformat()) - feed_info['expires'] = min_expires + timedelta(minutes=5) + min_exp_w_buffer.isoformat()) + feed_info['expires'] = min_exp_w_buffer return feed_info def prepare_headers(feed): """For a known feed, will construct some header dictionnary""" headers = {'User-Agent': conf.CRAWLER_USER_AGENT, - 'Accept': 'application/atom+xml,application/rss+xml,' - 'application/rdf+xml;q=0.8,application/xml;q=0.5,' - 'text/xml;q=0.5,*/*;q=0.2'} + 'Accept': FEED_ACCEPT_HEADERS} if feed.get('last_modified'): headers['If-Modified-Since'] = feed['last_modified'] if feed.get('etag') and 'jarr' not in feed['etag']: diff --git a/src/lib/const.py b/src/lib/const.py new file mode 100644 index 000000000..d41bb80b9 --- /dev/null +++ b/src/lib/const.py @@ -0,0 +1,9 @@ +"""Shared constants.""" + +MIMETYPES = (('application/atom+xml', 1), ('application/rss+xml', 0.9), + ('application/rdf+xml', 0.8), ('application/xml', 0.5), + ('text/xml', 0.5), ('*/*', 0.2)) + +FEED_ACCEPT_HEADERS = ','.join(mtype + (';q=%0.1f' % qual if qual < 1 else '') + for mtype, qual in MIMETYPES) +FEED_MIMETYPES = [mtype for mtype, quality in MIMETYPES if quality >= 0.5] diff --git a/src/lib/feed_utils.py b/src/lib/feed_utils.py index 2641f223d..2a53a704e 100644 --- a/src/lib/feed_utils.py +++ b/src/lib/feed_utils.py @@ -7,8 +7,9 @@ import feedparser from bootstrap import conf +from lib.const import FEED_ACCEPT_HEADERS, FEED_MIMETYPES from lib.utils import jarr_get, rebuild_url -from lib.html_parsing import (extract_title, extract_icon_url, FEED_MIMETYPES, +from lib.html_parsing import (extract_title, extract_icon_url, extract_feed_link, try_get_icon_url) logger = logging.getLogger(__name__) @@ -116,7 +117,7 @@ def _fetch_url_and_enhance_feed(url, feed): """trying to parse the page of the site for some rel link in the header""" site_split, feed_split = get_splits(url, feed.get('site_link')) try: - response = jarr_get(url) + response = jarr_get(url, headers={'Accept': FEED_ACCEPT_HEADERS}) except Exception as error: logger.warn('failed to retreive %r: %r', feed['site_link'], error) return feed diff --git a/src/lib/html_parsing.py b/src/lib/html_parsing.py index cc2a732d8..236750ace 100644 --- a/src/lib/html_parsing.py +++ b/src/lib/html_parsing.py @@ -2,11 +2,10 @@ from bs4 import BeautifulSoup, SoupStrainer +from lib.const import FEED_MIMETYPES from lib.utils import jarr_get, rebuild_url CHARSET_TAG = b' Date: Tue, 27 Dec 2016 00:43:48 +0100 Subject: [PATCH 055/164] =?UTF-8?q?Don=E2=80=99t=20leak=20active=20user=20?= =?UTF-8?q?name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevent the login form being abused for confirming valid user names. Let attackers keep guess at usernames as well as passwords. --- src/web/forms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/web/forms.py b/src/web/forms.py index 34ee57221..0a173985a 100644 --- a/src/web/forms.py +++ b/src/web/forms.py @@ -78,16 +78,15 @@ def validate(self): try: user = ucontr.get(login=self.login.data) except NotFound: - self.login.errors.append('Wrong login') validated = False else: if not user.is_active: - self.login.errors.append('User is desactivated') validated = False if not ucontr.check_password(user, self.password.data): - self.password.errors.append('Wrong password') validated = False self.user = user + if not validated: + self.login.errors.append('Invalid credentials.') return validated From 80900fa08f50f0ae3e65c8894730c8981b0434b5 Mon Sep 17 00:00:00 2001 From: Daniel Aleksandersen Date: Sat, 31 Dec 2016 05:00:37 +0100 Subject: [PATCH 056/164] Use input type email Enhanced experience when using soft-keyboards. --- src/web/templates/signup.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/templates/signup.html b/src/web/templates/signup.html index 7a3dbe8dc..def4f0aaf 100644 --- a/src/web/templates/signup.html +++ b/src/web/templates/signup.html @@ -10,7 +10,7 @@ {% endfor %} - {{form.email(class_="form-control", type="text", id="inputEmail", placeholder="Email", required=True, autofocus=True)}} + {{form.email(class_="form-control", type="email", id="inputEmail", placeholder="Email", required=True, autofocus=True)}} {% for error in form.email.errors %} {{ error }}
 {% endfor %} From 7cb4a975d7eec3ad6939e828c3aeb64d14f24fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Mon, 2 Jan 2017 15:59:12 +0100 Subject: [PATCH 057/164] all tags stipped and to lower case to avoid collision --- src/lib/article_utils.py | 5 +++-- src/lib/html_parsing.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/article_utils.py b/src/lib/article_utils.py index 0a2724bb1..a9cfeac5a 100644 --- a/src/lib/article_utils.py +++ b/src/lib/article_utils.py @@ -85,8 +85,9 @@ def _fetch_article(link): def get_article_details(entry, fetch=True): article_link = entry.get('link') article_title = html.unescape(entry.get('title', '')) - tags = {tag.get('term').strip() for tag in entry.get('tags', []) - if tag.get('term').strip()} + tags = {tag.get('term', '').lower().strip() + for tag in entry.get('tags', []) + if tag.get('term', '').strip()} if fetch and conf.CRAWLER_RESOLV and article_link or not article_title: response = _fetch_article(article_link) if response is None: diff --git a/src/lib/html_parsing.py b/src/lib/html_parsing.py index 236750ace..0546e8089 100644 --- a/src/lib/html_parsing.py +++ b/src/lib/html_parsing.py @@ -86,7 +86,7 @@ def extract_tags(response): tags = tags.union({meta.attrs.get('content', '') for meta in soup.find_all('meta', {'property': 'article:tag'})}) - return {tag.strip() for tag in tags if tag.strip()} + return {tag.lower().strip() for tag in tags if tag.strip()} def _check_keys(**kwargs): From 43477a31bb15489d6406a4bfbce31069bf8b9155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Mon, 2 Jan 2017 16:00:33 +0100 Subject: [PATCH 058/164] better expires handling server side --- src/tests/libs/feed_utils_test.py | 6 +++--- src/web/controllers/feed.py | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/tests/libs/feed_utils_test.py b/src/tests/libs/feed_utils_test.py index 1c87ef17e..7db9315a8 100644 --- a/src/tests/libs/feed_utils_test.py +++ b/src/tests/libs/feed_utils_test.py @@ -39,14 +39,14 @@ def test_joies_du_code(self): def test_apod(self): self.assertEquals( - {'icon_url': 'http://apod.nasa.gov/favicon.ico', + {'icon_url': 'https://apod.nasa.gov/favicon.ico', 'site_link': 'http://apod.nasa.gov/', 'title': 'Astronomy Picture of the Day'}, construct_feed_from('http://apod.nasa.gov/')) self.assertEquals( {'description': 'Astronomy Picture of the Day', - 'icon_url': 'http://apod.nasa.gov/favicon.ico', + 'icon_url': 'https://apod.nasa.gov/favicon.ico', 'link': 'http://apod.nasa.gov/apod.rss', - 'site_link': 'http://antwrp.gsfc.nasa.gov/', + 'site_link': 'https://apod.nasa.gov/', 'title': 'APOD'}, construct_feed_from('http://apod.nasa.gov/apod.rss')) diff --git a/src/web/controllers/feed.py b/src/web/controllers/feed.py index f4ef5631a..e21edfc69 100644 --- a/src/web/controllers/feed.py +++ b/src/web/controllers/feed.py @@ -1,4 +1,5 @@ -from datetime import timedelta +from datetime import timedelta, datetime +import dateutil.parser import logging from sqlalchemy import and_ @@ -130,11 +131,20 @@ def __denorm_title_on_clusters(self, feed, attrs): def __update_default_expires(self, feed, attrs): now = utc_now() - if 'expires' in attrs and not attrs['expires']: + if 'expires' in attrs: + expires = [] + if attrs['expires']: + if not isinstance(attrs['expires'], datetime): + expires.append(dateutil.parser.parse(attrs['expires'])) + else: + expires.append(attrs['expires']) + span_time = timedelta(seconds=conf.FEED_MAX_EXPIRES) art_count = self.__get_art_contr().read(feed_id=feed.id, retrieved_date__gt=now - span_time).count() - attrs['expires'] = now + (span_time / (art_count or 1)) + expires.append(now + (span_time / (art_count or 1))) + + attrs['expires'] = min(expires) def update(self, filters, attrs, *args, **kwargs): self._ensure_icon(attrs) From 5a717aa55253d56536f90af6469d4fbb5ab50b23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 6 Jan 2017 09:49:23 +0100 Subject: [PATCH 059/164] safer parsing --- src/lib/html_parsing.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/html_parsing.py b/src/lib/html_parsing.py index 0546e8089..500411372 100644 --- a/src/lib/html_parsing.py +++ b/src/lib/html_parsing.py @@ -55,11 +55,17 @@ def get_soup(content, header_encoding='utf8'): try this one before parsing with the one in args. """ strainer = SoupStrainer('head') + decoded_content = None if not isinstance(content, str): encodings = [_extract_charset(content, strainer), header_encoding] \ if CHARSET_TAG in content else [header_encoding] - content = _try_encodings(content, encodings) - return BeautifulSoup(content, 'html.parser', parse_only=strainer) + decoded_content = _try_encodings(content, encodings) + for cnt in decoded_content, content: + if cnt: + try: + return BeautifulSoup(cnt, 'html.parser', parse_only=strainer) + except Exception as error: + logger.warn('something went wrong when parsing: %r', error) def extract_title(response, og_prop='og;title'): From 2894dcd2cc354caa352b0fe60adcf244cf82b211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 6 Jan 2017 10:02:22 +0100 Subject: [PATCH 060/164] fix on recovery mech --- src/lib/emails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/emails.py b/src/lib/emails.py index 1f73809d4..3124f2ef3 100644 --- a/src/lib/emails.py +++ b/src/lib/emails.py @@ -46,5 +46,5 @@ def send(to="", bcc="", subject="", plaintext=""): if conf.NOTIFICATION_STARTTLS: smtp.starttls() smtp.ehlo() - smtp.login(conf.NOTIFICATION_USERNAME, conf.NOTIFICATION_PASSWORD) + smtp.login(conf.NOTIFICATION_EMAIL, conf.NOTIFICATION_PASSWORD) smtp.sendmail(conf.NOTIFICATION_EMAIL, [msg['To']], msg.as_string()) From 7956aeac70211d619e50ca61f29f2e8fd8008430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Fri, 6 Jan 2017 16:22:28 +0100 Subject: [PATCH 061/164] adding Date to recovery email --- src/crawler/lib/headers_handling.py | 14 +------------- src/lib/emails.py | 22 ++-------------------- src/lib/utils.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/src/crawler/lib/headers_handling.py b/src/crawler/lib/headers_handling.py index acc6a509c..0fef0af4b 100644 --- a/src/crawler/lib/headers_handling.py +++ b/src/crawler/lib/headers_handling.py @@ -5,22 +5,10 @@ from bootstrap import conf from lib.const import FEED_ACCEPT_HEADERS -from lib.utils import to_hash, utc_now +from lib.utils import to_hash, utc_now, rfc_1123_utc logger = logging.getLogger(__name__) MAX_AGE_RE = re.compile('max-age=([0-9]+)') -RFC_1123_FORMAT = '%a, %d %b %Y %X %Z' - - -def rfc_1123_utc(time_obj=None, delta=None): - """return time obj or now formated in the RFC1123 style. Add time delta if - present. - """ - if time_obj is None: - time_obj = utc_now() - if delta is not None: - time_obj += delta - return time_obj.strftime(RFC_1123_FORMAT) def _extract_max_age(headers, feed_info, now): diff --git a/src/lib/emails.py b/src/lib/emails.py index 3124f2ef3..0c62f8b63 100644 --- a/src/lib/emails.py +++ b/src/lib/emails.py @@ -1,30 +1,11 @@ #! /usr/bin/env python -# -*- coding: utf-8 -*- - -# jarr - A Web based news aggregator. -# Copyright (C) 2010-2016 Cédric Bonhomme - https://www.JARR-aggregator.org -# -# For more information : https://github.com/JARR-aggregator/JARR/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - import logging import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from bootstrap import conf +from lib.utils import rfc_1123_utc logger = logging.getLogger(__name__) @@ -37,6 +18,7 @@ def send(to="", bcc="", subject="", plaintext=""): msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = conf.NOTIFICATION_EMAIL + msg['Date'] = rfc_1123_utc() msg['To'] = to msg.attach(MIMEText(plaintext, 'plain', 'utf-8')) diff --git a/src/lib/utils.py b/src/lib/utils.py index be6deb834..ce329badd 100644 --- a/src/lib/utils.py +++ b/src/lib/utils.py @@ -12,12 +12,24 @@ from bootstrap import conf logger = logging.getLogger(__name__) +RFC_1123_FORMAT = '%a, %d %b %Y %X %Z' def utc_now(): return pytz.utc.localize(datetime.utcnow()) +def rfc_1123_utc(time_obj=None, delta=None): + """return time obj or now formated in the RFC1123 style. Add time delta if + present. + """ + if time_obj is None: + time_obj = utc_now() + if delta is not None: + time_obj += delta + return time_obj.strftime(RFC_1123_FORMAT) + + def default_handler(obj, role='admin'): """JSON handler for default query formatting""" if hasattr(obj, 'isoformat'): From caf1a3cc149bd44b9ca43d48d68170e1ccb64d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Thu, 16 Feb 2017 14:00:45 +0100 Subject: [PATCH 062/164] handling failed html parsing --- src/lib/html_parsing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/html_parsing.py b/src/lib/html_parsing.py index 500411372..a66571014 100644 --- a/src/lib/html_parsing.py +++ b/src/lib/html_parsing.py @@ -84,6 +84,8 @@ def extract_tags(response): """From a requests.Response objects will return the tags (keywords + open graphs ones).""" soup = get_soup(response.content, response.encoding) + if not soup: + return {} tags = set() keywords = soup.find_all('meta', {'name': 'keywords'}) if keywords: @@ -111,6 +113,8 @@ def wrapper(elem): def extract_icon_url(response, site_split, feed_split): soup = get_soup(response.content, response.encoding) + if not soup: + return icons = soup.find_all(_check_keys(rel=['icon', 'shortcut'])) if not len(icons): icons = soup.find_all(_check_keys(rel=['icon'])) From 77afc763cbfa5c8e141f76bff273ef1402b80416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Thu, 16 Feb 2017 14:01:18 +0100 Subject: [PATCH 063/164] enhancing probes and reset scripts --- src/manager.py | 14 ++++---------- src/scripts/probes.py | 23 +++++++++++++++++++---- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/manager.py b/src/manager.py index 6ec7a1956..4e2da0cb1 100755 --- a/src/manager.py +++ b/src/manager.py @@ -48,19 +48,13 @@ def fetch(limit=0, retreive_all=False): @manager.command def reset_feeds(): """Will reschedule all active feeds to be fetched in the next two hours""" - from web.models import User fcontr = FeedController(ignore_context=True) now = utc_now() - last_conn_max = now - timedelta(days=30) + feeds = [feed[0] for feed in fcontr.with_entities(fcontr._db_cls.id)] - feeds = list(fcontr.read().join(User).filter(User.is_active.__eq__(True), - User.last_connection >= last_conn_max) - .with_entities(fcontr._db_cls.id) - .distinct()) - - step = timedelta(seconds=conf.FEED_MIN_EXPIRES * 3 / len(feeds)) - for i, feed in enumerate(feeds): - fcontr.update({'id': feed[0]}, + step = timedelta(seconds=conf.FEED_MAX_EXPIRES / len(feeds)) + for i, feed_id in enumerate(feeds): + fcontr.update({'id': feed_id}, {'etag': '', 'last_modified': '', 'last_retrieved': datetime(1970, 1, 1, tzinfo=timezone.utc), 'expires': now + i * step}) diff --git a/src/scripts/probes.py b/src/scripts/probes.py index 3b5dbd8d0..970eda9b5 100644 --- a/src/scripts/probes.py +++ b/src/scripts/probes.py @@ -58,11 +58,26 @@ def execute(self): class FeedLatenessProbe(AbstractMuninPlugin): - split_number = 5 - colours = 'F90026', 'FB7601', 'FCC402', 'E6FC02', '99FD03', '05FF0B' + split_number = 12 colour_expired = '3B0B17' colour_in_a_while = '2E2EFE' + @staticmethod + def colors(nb_steps): + red, green, blue = 255, 0, 0 + steps = 255 * 2 / (nb_steps - 1) + yield '{:02X}{:02X}{:02X}'.format(red, green, blue) + for i in range(nb_steps): + if green < 255: + green += steps + if green > 255: + red -= green - 255 + green = 255 + else: + red -= steps + yield '{:02X}{:02X}{:02X}'.format(int(red if red < 255 else 255), + int(green if green > 0 else 0), int(blue)) + def iter_on_splits(self): offset = 2 min_delta = timedelta(0) @@ -89,7 +104,7 @@ def _to_hour(td): def config(self): print("graph_title JARR - Feeds lateness repartition") print("graph_vlabel feeds counts") - colours = iter(self.colours) + colors = self.colors(self.split_number) for i, range_start, range_end in self.iter_on_splits(): print("feeds_%s.draw AREASTACK" % i) print("feeds_%s.min 0" % i) @@ -105,7 +120,7 @@ def config(self): "than %s" % self._to_hour(range_start)) else: - print("feeds_%s.colour %s" % (i, next(colours))) + print("feeds_%s.colour %s" % (i, next(colors))) print("feeds_%s.label feeds in split %d" % (i, i)) print("feeds_%s.info feeds expiring in between %s and %s" % (i, self._to_hour(range_start), self._to_hour(range_end))) From d979055a5ac1dc00670809ebedfd92b8c2e74057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Thu, 16 Feb 2017 14:01:42 +0100 Subject: [PATCH 064/164] fixing filtering on 'No Category' on postgres --- src/web/controllers/cluster.py | 10 +++++----- src/web/js/components/Menu.react.js | 3 --- src/web/views/home.py | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/web/controllers/cluster.py b/src/web/controllers/cluster.py index 2bba9da32..4216e8648 100644 --- a/src/web/controllers/cluster.py +++ b/src/web/controllers/cluster.py @@ -135,7 +135,7 @@ def _iter_on_query(query, fields, filter_on_category): else: # pragma: no cover row['feeds_id'] = set(clu.feeds_id) if filter_on_category: - row['categories_id'] = set(clu.categories_id) + row['categories_id'] = {i or 0 for i in clu.categories_id} yield row def join_read(self, feed_id=None, **filters): @@ -149,10 +149,10 @@ def join_read(self, feed_id=None, **filters): # filtering by article but did not found anything return - fields = {key: getattr(Cluster, key) for key in ('main_title', 'id', - 'liked', 'read', 'main_article_id', 'main_feed_title', - 'main_date', 'main_link')} - sqla_fields = list(fields.values()) + returned_keys = ('main_title', 'id', 'liked', 'read', + 'main_article_id', 'main_feed_title', 'main_date', 'main_link') + fields = {key: getattr(Cluster, key) for key in returned_keys} + sqla_fields = [getattr(Cluster, key) for key in returned_keys] art_feed_alias, art_cat_alias = aliased(Article), aliased(Article) # DESC of what's going on below : # base query with the above fields and the aggregations diff --git a/src/web/js/components/Menu.react.js b/src/web/js/components/Menu.react.js index a590a33e1..6e259020d 100644 --- a/src/web/js/components/Menu.react.js +++ b/src/web/js/components/Menu.react.js @@ -21,9 +21,6 @@ var FeedItem = React.createClass({ }, render: function() { var badge_unread = null; - console.log(this.props.title); - console.log(this.props.feed_id); - console.log(IconStore.getIcon(this.props.feed_id)); var icon = ; // handling unread badge if(this.props.unread){ diff --git a/src/web/views/home.py b/src/web/views/home.py index e7c5b74be..8414b0b19 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -100,8 +100,8 @@ def _get_filters(in_dict): @login_required @etag_match def get_middle_panel(): - return clusters_to_json(ClusterController(current_user.id).join_read( - **_get_filters(request.args))) + clu_contr = ClusterController(current_user.id) + return clusters_to_json(clu_contr.join_read(**_get_filters(request.args))) @current_app.route('/getclu/') From bf7bdce90da2d752747205eb6cf962663793fb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Thu, 16 Feb 2017 14:56:32 +0100 Subject: [PATCH 065/164] fixing test relying on external api --- src/tests/libs/feed_utils_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/libs/feed_utils_test.py b/src/tests/libs/feed_utils_test.py index 7db9315a8..c2cef4528 100644 --- a/src/tests/libs/feed_utils_test.py +++ b/src/tests/libs/feed_utils_test.py @@ -8,7 +8,8 @@ class ConstructFeedFromTest(unittest.TestCase): @property def jdh_feed(self): return {'icon_url': 'https://www.journalduhacker.net/' - 'assets/jdh-ico-2c6c8060958bf86c28b20d0c83f1bbc5.ico', + 'assets/jdh-ico-311c23d65a3a9928889718838e2626c0665d83712d' + '488713c9a6c2ba2c676c0e.ico', 'link': 'https://www.journalduhacker.net/rss', 'site_link': 'https://www.journalduhacker.net/', 'title': 'Journal du hacker'} From fcc23130d70e87318e24efb12237704eebc8172a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Thu, 16 Feb 2017 15:02:31 +0100 Subject: [PATCH 066/164] style fix --- src/scripts/probes.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/scripts/probes.py b/src/scripts/probes.py index 970eda9b5..a3a99d534 100644 --- a/src/scripts/probes.py +++ b/src/scripts/probes.py @@ -64,19 +64,19 @@ class FeedLatenessProbe(AbstractMuninPlugin): @staticmethod def colors(nb_steps): - red, green, blue = 255, 0, 0 - steps = 255 * 2 / (nb_steps - 1) - yield '{:02X}{:02X}{:02X}'.format(red, green, blue) - for i in range(nb_steps): - if green < 255: - green += steps - if green > 255: - red -= green - 255 - green = 255 - else: - red -= steps - yield '{:02X}{:02X}{:02X}'.format(int(red if red < 255 else 255), - int(green if green > 0 else 0), int(blue)) + red, green, blue = 255, 0, 0 + steps = 255 * 2 / (nb_steps - 1) + yield '{:02X}{:02X}{:02X}'.format(red, green, blue) + for i in range(nb_steps): + if green < 255: + green += steps + if green > 255: + red -= green - 255 + green = 255 + else: + red -= steps + yield '{:02X}{:02X}{:02X}'.format(int(red if red < 255 else 255), + int(green if green > 0 else 0), int(blue)) def iter_on_splits(self): offset = 2 From 0beb397843769c98b3fc1824066b9e00e2ab27c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 5 Mar 2017 22:18:15 +0100 Subject: [PATCH 067/164] adding style for underlining wrong data in sign up form --- src/web/static/css/customized-bootstrap.css | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/web/static/css/customized-bootstrap.css b/src/web/static/css/customized-bootstrap.css index 591ae5102..3b52ec2a4 100644 --- a/src/web/static/css/customized-bootstrap.css +++ b/src/web/static/css/customized-bootstrap.css @@ -31,3 +31,9 @@ a { width: 200px; display: inline; } + +input:not(:placeholder-shown):invalid { + color: red; + border-color: red; + box-shadow-color: red; +} From c1924dd842aa9776d0d7847d9cbe57dbb3273062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 5 Mar 2017 22:25:25 +0100 Subject: [PATCH 068/164] not displaying unix time for last retrieved date when feed was never fetched close #71 --- src/lib/const.py | 3 ++- src/web/models/feed.py | 5 +---- src/web/views/home.py | 14 ++++++++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/lib/const.py b/src/lib/const.py index d41bb80b9..71fe867f3 100644 --- a/src/lib/const.py +++ b/src/lib/const.py @@ -1,5 +1,6 @@ -"""Shared constants.""" +from datetime import datetime, timezone +UNIX_START = datetime(1970, 1, 1, tzinfo=timezone.utc) MIMETYPES = (('application/atom+xml', 1), ('application/rss+xml', 0.9), ('application/rdf+xml', 0.8), ('application/xml', 0.5), ('text/xml', 0.5), ('*/*', 0.2)) diff --git a/src/web/models/feed.py b/src/web/models/feed.py index d72ec5943..62391f657 100644 --- a/src/web/models/feed.py +++ b/src/web/models/feed.py @@ -1,4 +1,3 @@ -from datetime import datetime, timezone from sqlalchemy import (Boolean, Column, ForeignKey, Index, Integer, PickleType, String) @@ -6,13 +5,11 @@ from bootstrap import db from lib.utils import utc_now +from lib.const import UNIX_START from web.models.utc_datetime_type import UTCDateTime from web.models.right_mixin import RightMixin -UNIX_START = datetime(1970, 1, 1, tzinfo=timezone.utc) - - class Feed(db.Model, RightMixin): id = Column(Integer, primary_key=True) title = Column(String, default="") diff --git a/src/web/views/home.py b/src/web/views/home.py index 8414b0b19..9d2199f24 100644 --- a/src/web/views/home.py +++ b/src/web/views/home.py @@ -6,6 +6,7 @@ from flask_login import current_user, login_required from bootstrap import conf +from lib.const import UNIX_START from lib.utils import utc_now from lib import integrations from web.controllers import (CategoryController, ClusterController, @@ -45,12 +46,17 @@ def get_menu(): for feed in FeedController(current_user.id).read(): feed['created_rel'] = format_timedelta( feed.created_date - now, add_direction=True, locale=locale) - feed['last_rel'] = format_timedelta( - feed.last_retrieved - now, add_direction=True, locale=locale) feed['created_date'] = format_datetime(feed.created_date, locale=locale) - feed['last_retrieved'] = format_datetime(feed.last_retrieved, - locale=locale) + if feed.last_retrieved == UNIX_START: + feed['last_rel'] = 'Never fetched' + feed['last_retrieved'] = '' + else: + feed['last_rel'] = format_timedelta( + feed.last_retrieved - now, + add_direction=True, locale=locale) + feed['last_retrieved'] = format_datetime(feed.last_retrieved, + locale=locale) feed['category_id'] = feed.category_id or 0 feed['unread'] = cnt_by_feed.get(feed.id, 0) if not feed.filters: From b335591e8f2455b47a96df44927b30bb6483fdbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 5 Mar 2017 22:42:06 +0100 Subject: [PATCH 069/164] not loading entire article object when challenging --- src/web/controllers/article.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/controllers/article.py b/src/web/controllers/article.py index d5e3c81b7..87925b274 100644 --- a/src/web/controllers/article.py +++ b/src/web/controllers/article.py @@ -22,7 +22,7 @@ class ArticleController(AbstractController): def challenge(self, ids): """Will return each id that wasn't found in the database.""" for id_ in ids: - if self.read(**id_).first(): + if self.read(**id_).with_entities(self._db_cls.id).first(): continue yield id_ From 90767643dbfbe9b955df103412d1b7ba2459e839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Mon, 6 Mar 2017 14:32:35 +0100 Subject: [PATCH 070/164] light factorizing and cleaning crawler --- src/crawler/http_crawler.py | 157 ++++++++++++++-------------- src/crawler/lib/headers_handling.py | 19 +--- src/tests/crawler_test.py | 2 +- 3 files changed, 80 insertions(+), 98 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 430f95418..077abeb50 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -24,7 +24,7 @@ from bootstrap import conf from lib.article_utils import construct_article, get_skip_and_ids from lib.feed_utils import construct_feed_from, is_parsing_ok -from lib.utils import default_handler, utc_now +from lib.utils import default_handler, to_hash, utc_now from crawler.lib import headers_handling logger = logging.getLogger(__name__) @@ -86,14 +86,24 @@ def wait(self, max_wait=600, wait_for=2, checks=10): break -class JarrUpdater(AbstractCrawler): +class AbstractFeedCrawler(AbstractCrawler): - def __init__(self, feed, entries, headers, parsed_feed, auth): + def __init__(self, feed, auth): self.feed = feed + self.base_msg = '%r %r - ' % (self.feed['id'], self.feed['title']) + super().__init__(auth) + + def log(self, level, message, *args, **kwargs): + return getattr(logger, level)(self.base_msg + message, *args, **kwargs) + + +class JarrUpdater(AbstractFeedCrawler): + + def __init__(self, feed, entries, headers, parsed_feed, auth): self.entries = entries self.headers = headers self.parsed_feed = parsed_feed - super().__init__(auth) + super().__init__(feed, auth) def callback(self, response): """Will process the result from the challenge, creating missing article @@ -109,72 +119,72 @@ def callback(self, response): return if response.status_code != 204: results = response.json() - logger.debug('%r %r - %d entries were not matched ' - 'and will be created', - self.feed['id'], self.feed['title'], len(results)) + self.log('debug', "%d entries wern't matched and will be created", + len(results)) entries = [] for id_to_create in results: article_created = True entry = construct_article( self.entries[tuple(sorted(id_to_create.items()))], self.feed) - logger.info('%r %r - creating %r for %r - %r', self.feed['id'], - self.feed['title'], entry.get('title'), - entry.get('user_id'), id_to_create) + self.log('info', 'creating %r for %r - %r', entry.get('title'), + entry.get('user_id'), id_to_create) entries.append(entry) self.query_jarr('post', 'articles', entries) - if not is_parsing_ok(self.parsed_feed): - logger.info('%r %r - parsing failed, bumping error count', - self.feed['id'], self.feed['title']) - up_feed = headers_handling.extract_feed_info(self.headers) - up_feed['last_error'] = str(self.parsed_feed.get('bozo_exception')) - up_feed['error_count'] = self.feed['error_count'] + 1 - up_feed['last_retrieved'] = utc_now() - - return self.query_jarr('put', 'feed/%d' % self.feed['id'], up_feed) - - up_feed = construct_feed_from(url=self.feed['link'], - fp_parsed=self.parsed_feed, - feed=self.feed) - up_feed.update(headers_handling.extract_feed_info(self.headers)) - - up_feed.pop('link', None) # not updating feed url - up_feed['user_id'] = self.feed['user_id'] - up_feed['last_error'] = None - up_feed['error_count'] = 0 - up_feed['last_retrieved'] = utc_now() if not article_created: - logger.info('%r %r - all article matched in db, adding nothing', - self.feed['id'], self.feed['title']) - - # updating only changed attrs - up_feed = {key: value for key, value in up_feed.items() - if up_feed[key] != self.feed.get(key)} - logger.info('%r %r - pushing feed attrs %r', - self.feed['id'], self.feed['title'], - {key: "%r -> %r" % (self.feed.get(key), up_feed[key]) - for key in up_feed}) - - self.query_jarr('put', 'feed/%d' % self.feed['id'], up_feed) + self.log('info', 'all article matched in db, adding nothing') -class FeedCrawler(AbstractCrawler): +class FeedCrawler(AbstractFeedCrawler): - def __init__(self, feed, auth): - self.feed = feed - super().__init__(auth) - - def clean_feed(self, headers): + def clean_feed(self, response, parsed_feed=None): """Will reset the errors counters on a feed that have known errors""" - info = headers_handling.extract_feed_info(headers) + info = headers_handling.extract_feed_info(response.headers) info.update({'error_count': 0, 'last_error': None, 'last_retrieved': utc_now()}) + + if parsed_feed is not None: # updating feed with retrieved info + constructed = construct_feed_from( + self.feed['link'], parsed_feed, self.feed) + for key in 'description', 'site_link', 'icon_url': + if up_feed.get(key): + info[key] = constructed[key] + info = {key: value for key, value in info.items() if self.feed.get(key) != value} + if info: self.query_jarr('put', 'feed/%d' % self.feed['id'], info) + def set_feed_error(self, error=None, parsed_feed=None): + error_count = self.feed['error_count'] + 1 + if error: + last_error = str(error) + elif parsed_feed: + last_error = str(parsed_feed.get('bozo_exception', '')) + if self.feed['error_count'] > conf.FEED_ERROR_THRESHOLD: + self.log('warn', 'an error occured while fetching feed; ' + 'bumping error count to %r', error_count) + info = {'error_count': error_count, 'last_error': last_error, + 'user_id': self.feed['user_id'], 'last_retrieved': utc_now()} + info.update(headers_handling.extract_feed_info({})) + future = self.query_jarr('put', 'feed/%d' % self.feed['id'], info) + + def response_match_cache(self, response): + if 'etag' not in response.headers: + self.log('debug', 'manually generating etag') + response.headers['etag'] = 'jarr/"%s"' % to_hash(response.text) + if response.headers['etag'] and self.feed['etag'] \ + and response.headers['etag'] == self.feed['etag']: + if 'jarr' in self.feed['etag']: + msg = "calculated hash matches (%d)" + else: + msg = "feed responded with same etag (%d)" + self.log('info', msg, response.status_code) + return True + return False + def callback(self, response): """will fetch the feed and interprete results (304, etag) or will challenge jarr to compare gotten entries with existing ones""" @@ -182,55 +192,44 @@ def callback(self, response): response = response.result() response.raise_for_status() except Exception as error: - error_count = self.feed['error_count'] + 1 - if self.feed['error_count'] > conf.FEED_ERROR_THRESHOLD: - logger.warn('%r %r - an error occured while fetching ' - 'feed; bumping error count to %r', - self.feed['id'], self.feed['title'], error_count) - info = {'error_count': error_count, 'last_error': str(error), - 'user_id': self.feed['user_id'], - 'last_retrieved': utc_now()} - info.update(headers_handling.extract_feed_info({})) - future = self.query_jarr('put', 'feed/%d' % self.feed['id'], info) + self.set_feed_error(error=error) return if response.status_code == 304: - logger.info("%r %r - feed responded with 304", - self.feed['id'], self.feed['title']) - self.clean_feed(response.headers) + self.log('info', 'feed responded with 304') + self.clean_feed(response) return - if headers_handling.response_match_cache(response, self.feed): - self.clean_feed(response.headers) + if self.response_match_cache(response): + self.clean_feed(response) return else: - logger.debug('%r %r - etag mismatch %r != %r', - self.feed['id'], self.feed['title'], - response.headers['etag'], self.feed['etag']) - logger.info('%r %r - cache validation failed, challenging entries', - self.feed['id'], self.feed['title']) + self.log('debug', 'etag mismatch %r != %r', + response.headers['etag'], self.feed['etag']) + self.log('info', 'cache validation failed, challenging entries') - ids, entries = [], {} parsed_response = feedparser.parse(response.content.strip()) - skipped_list = [] + if is_parsing_ok(parsed_response): + self.clean_feed(response) + else: + self.set_feed_error(parsed_feed=self.parsed_feed) + return + + ids, entries, skipped_list = [], {}, [] for entry in parsed_response['entries']: if not entry: continue skipped, entry_ids = get_skip_and_ids(entry, self.feed) if skipped: skipped_list.append(entry_ids) - logger.debug('%r %r - skipping article', - self.feed['id'], self.feed['title']) + self.log('debug', 'skipping article') continue entries[tuple(sorted(entry_ids.items()))] = entry ids.append(entry_ids) if not ids and skipped_list: - logger.debug('%r %r - nothing to add (skipped %r) %r', - self.feed['id'], self.feed['title'], skipped_list, - parsed_response) - self.clean_feed(response.headers) + self.log('debug', 'nothing to add (skipped %r) %r', + skipped_list, parsed_response) return - logger.debug('%r %r - found %d entries %r', - self.feed['id'], self.feed['title'], len(ids), ids) + self.log('debug', 'found %d entries %r', len(ids), ids) future = self.query_jarr('get', 'articles/challenge', {'ids': ids}) updater = JarrUpdater(self.feed, entries, response.headers, parsed_response, self.auth) diff --git a/src/crawler/lib/headers_handling.py b/src/crawler/lib/headers_handling.py index 0fef0af4b..cd5021b2d 100644 --- a/src/crawler/lib/headers_handling.py +++ b/src/crawler/lib/headers_handling.py @@ -5,7 +5,7 @@ from bootstrap import conf from lib.const import FEED_ACCEPT_HEADERS -from lib.utils import to_hash, utc_now, rfc_1123_utc +from lib.utils import utc_now, rfc_1123_utc logger = logging.getLogger(__name__) MAX_AGE_RE = re.compile('max-age=([0-9]+)') @@ -79,20 +79,3 @@ def prepare_headers(feed): logger.debug('%r %r - calculated headers %r', feed['id'], feed['title'], headers) return headers - - -def response_match_cache(response, feed): - if 'etag' not in response.headers: - logger.debug('%r %r - manually generating etag', - feed['id'], feed['title']) - response.headers['etag'] = 'jarr/"%s"' % to_hash(response.text) - if response.headers['etag'] and feed['etag'] \ - and response.headers['etag'] == feed['etag']: - if 'jarr' in feed['etag']: - logger.info("%r %r - calculated hash matches (%d)", - feed['id'], feed['title'], response.status_code) - else: - logger.info("%r %r - feed responded with same etag (%d)", - feed['id'], feed['title'], response.status_code) - return True - return False diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index f85a4b5da..f2f431675 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -9,9 +9,9 @@ from bootstrap import conf from crawler.http_crawler import CrawlerScheduler from web.controllers import FeedController, UserController +from lib.const import UNIX_START logger = logging.getLogger('web') -UNIX_START = datetime(1970, 1, 1, tzinfo=timezone.utc) class CrawlerTest(JarrFlaskCommon): From a16b78afcb9eb0b4523b395722f20de2a52c58e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 12 Mar 2017 19:35:52 +0100 Subject: [PATCH 071/164] supporting permanent redirection also bumping test coverage a bit (fix #66) --- src/crawler/http_crawler.py | 9 ++- src/lib/utils.py | 3 + src/tests/crawler_test.py | 116 +++++++++++++++++++++++++++++------ src/tests/views/user_test.py | 25 ++++++++ src/web/views/api/common.py | 4 +- src/web/views/user.py | 52 +++++----------- 6 files changed, 151 insertions(+), 58 deletions(-) diff --git a/src/crawler/http_crawler.py b/src/crawler/http_crawler.py index 077abeb50..017a0454c 100644 --- a/src/crawler/http_crawler.py +++ b/src/crawler/http_crawler.py @@ -148,12 +148,19 @@ def clean_feed(self, response, parsed_feed=None): constructed = construct_feed_from( self.feed['link'], parsed_feed, self.feed) for key in 'description', 'site_link', 'icon_url': - if up_feed.get(key): + if constructed.get(key): info[key] = constructed[key] info = {key: value for key, value in info.items() if self.feed.get(key) != value} + # updating link on permanent move /redirect + if response.history and self.feed['link'] != response.url and any( + resp.status_code in {301, 308} for resp in response.history): + self.log('warn', 'feed moved from %r to %r', + self.feed['link'], response.url) + info['link'] = response.url + if info: self.query_jarr('put', 'feed/%d' % self.feed['id'], info) diff --git a/src/lib/utils.py b/src/lib/utils.py index ce329badd..ae4fa7f16 100644 --- a/src/lib/utils.py +++ b/src/lib/utils.py @@ -8,6 +8,7 @@ import requests from flask import request, url_for +from werkzeug.exceptions import HTTPException from bootstrap import conf @@ -38,6 +39,8 @@ def default_handler(obj, role='admin'): return obj.dump(role=role) if isinstance(obj, (set, frozenset, filter, types.GeneratorType)): return list(obj) + if isinstance(obj, HTTPException): + return "%d: %s" % (obj.code, obj.name) if isinstance(obj, BaseException): return str(obj) raise TypeError("Object of type %s with value of %r " diff --git a/src/tests/crawler_test.py b/src/tests/crawler_test.py index f2f431675..bff876ab8 100644 --- a/src/tests/crawler_test.py +++ b/src/tests/crawler_test.py @@ -1,14 +1,16 @@ -from tests.base import JarrFlaskCommon +from tests.base import BaseJarrTest, JarrFlaskCommon import logging +import unittest from datetime import datetime, timezone import feedparser from mock import Mock, patch from bootstrap import conf -from crawler.http_crawler import CrawlerScheduler +from crawler.http_crawler import CrawlerScheduler, FeedCrawler from web.controllers import FeedController, UserController +from lib.utils import to_hash from lib.const import UNIX_START logger = logging.getLogger('web') @@ -39,23 +41,13 @@ def setUp(self): def _api_req(method, url, **kwargs): if url.startswith('feed') and len(url) == 6: - class Proxy: - status_code = self.resp_status_code - headers = self.resp_headers - - @classmethod - def raise_for_status(cls): - if self.resp_raise is None: - return - raise self.resp_raise - - @property - def content(self): - with open(atom_file_path) as f: - return f.read() - - text = content - return Proxy() + with open(atom_file_path) as f: + content = f.read() + resp = Mock(status_code=self.resp_status_code, + headers=self.resp_headers, + content=content, text=content, history=[]) + resp.raise_for_status.return_value = self.resp_raise + return resp url = url.split(conf.API_ROOT)[1].strip('/') kwargs.pop('allow_redirects', None) @@ -171,3 +163,89 @@ def test_matching_etag(self): scheduler.wait(**self.wait_params) resp = self._api('get', 'articles', data={'limit': 1000}, user='admin') self.assertEquals(36 + self.new_entries_cnt, len(resp.json())) + + +class CrawlerMethodsTest(unittest.TestCase): + + def setUp(self): + super().setUp() + self.feed = {'user_id': 1, 'id': 1, 'title': 'title', + 'description': 'description', + 'etag': '', 'error_count': 5, 'link': 'link'} + self.resp = Mock(text='text', headers={}, status_code=304, history=[]) + self.crawler = FeedCrawler(self.feed, Mock()) + + def test_etag_matching_w_constructed_etag(self): + self.feed['etag'] = 'jarr/"%s"' % to_hash('text') + self.assertTrue(self.crawler.response_match_cache(self.resp)) + + def test_etag_no_matching_wo_etag(self): + self.assertFalse(self.crawler.response_match_cache(self.resp)) + + def test_etag_matching(self): + self.resp.headers['etag'] = self.feed['etag'] = 'etag' + self.assertTrue(self.crawler.response_match_cache(self.resp)) + + def test_set_feed_error_w_error(self): + original_error_count = self.feed['error_count'] + self.crawler.query_jarr = Mock() + self.crawler.set_feed_error(Exception('an error')) + call = self.crawler.query_jarr.mock_calls[0][1] + + self.assertEquals('put', call[0]) + self.assertEquals('feed/%d' % self.feed['id'], call[1]) + self.assertEquals(original_error_count + 1, call[2]['error_count']) + self.assertEquals('an error', call[2]['last_error']) + + def test_set_feed_error_w_parsed(self): + original_error_count = self.feed['error_count'] + self.crawler.query_jarr = Mock() + self.crawler.set_feed_error(parsed_feed={'bozo_exception': 'an error'}) + call = self.crawler.query_jarr.mock_calls[0][1] + self.assertEquals('put', call[0]) + self.assertEquals('feed/%d' % self.feed['id'], call[1]) + self.assertEquals(original_error_count + 1, call[2]['error_count']) + self.assertEquals('an error', call[2]['last_error']) + + def test_clean_feed(self): + self.crawler.query_jarr = Mock() + self.crawler.clean_feed(self.resp) + call = self.crawler.query_jarr.mock_calls[0][1] + + self.assertEquals('put', call[0]) + self.assertEquals('feed/%d' % self.feed['id'], call[1]) + self.assertTrue('link' not in call[2]) + self.assertTrue('title' not in call[2]) + self.assertTrue('description' not in call[2]) + self.assertTrue('site_link' not in call[2]) + self.assertTrue('icon_url' not in call[2]) + + def test_clean_feed_update_link(self): + self.crawler.query_jarr = Mock() + self.resp.history.append(Mock(status_code=301)) + self.resp.url = 'new_link' + self.crawler.clean_feed(self.resp) + call = self.crawler.query_jarr.mock_calls[0][1] + + self.assertEquals('put', call[0]) + self.assertEquals('feed/%d' % self.feed['id'], call[1]) + self.assertEquals('new_link', call[2]['link']) + self.assertTrue('title' not in call[2]) + self.assertTrue('description' not in call[2]) + self.assertTrue('site_link' not in call[2]) + self.assertTrue('icon_url' not in call[2]) + + @patch('crawler.http_crawler.construct_feed_from') + def test_clean_feed_w_constructed(self, construct_feed_mock): + construct_feed_mock.return_value = {'description': 'new description'} + self.crawler.query_jarr = Mock() + self.crawler.clean_feed(self.resp, True) + call = self.crawler.query_jarr.mock_calls[0][1] + + self.assertEquals('put', call[0]) + self.assertEquals('feed/%d' % self.feed['id'], call[1]) + self.assertEquals('new description', call[2]['description']) + self.assertTrue('link' not in call[2]) + self.assertTrue('title' not in call[2]) + self.assertTrue('site_link' not in call[2]) + self.assertTrue('icon_url' not in call[2]) diff --git a/src/tests/views/user_test.py b/src/tests/views/user_test.py index 3aec045a6..2061eed2b 100644 --- a/src/tests/views/user_test.py +++ b/src/tests/views/user_test.py @@ -150,3 +150,28 @@ def test_password_recovery(self, mock_emails_send): 'password': 'new_password'}, follow_redirects=True) self.assertEquals(200, self.app.get('/').status_code) + + def test_password_update(self): + old_password = self.user.password + self.app.post('/user/password_update/%d' % self.user.id, + data={'password': 'new_pass', 'password_conf': 'no matching'}) + user = self.uctrl.get(id=self.user.id) + self.assertEquals(user.password, old_password) + self.app.post('/user/password_update/%d' % self.user.id, + data={'password': 'new_pass', 'password_conf': 'new_pass'}) + user = self.uctrl.get(id=self.user.id) + self.assertNotEquals(user.password, old_password) + + def test_profile_update(self): + self.app.post('/user/password_update/%d' % self.user.id, + data={'email': 'not an email', 'is_admin': True}) + user = self.uctrl.get(id=self.user.id) + self.assertFalse(user.is_admin) + self.assertNotEquals(user.email, 'not an email') + + self.uctrl.update({'id': self.user.id}, {'is_admin': True}) + self.app.post('/user/password_update/%d' % self.user.id, + data={'email': 'a.valid@email.fake', 'is_admin': True}) + user = self.uctrl.get(id=self.user.id) + self.assertTrue(user.is_admin) + self.assertNotEquals(user.email, 'a.valid@email.fake') diff --git a/src/web/views/api/common.py b/src/web/views/api/common.py index 5761ebffc..19dd0f07a 100644 --- a/src/web/views/api/common.py +++ b/src/web/views/api/common.py @@ -188,7 +188,7 @@ class Proxy: results.append(obj) except Exception as error: fail_count += 1 - results.append(str(error)) + results.append(error) if fail_count == len(results): # all failed => 500 status = 500 elif fail_count: # some failed => 206 @@ -215,7 +215,7 @@ class Proxy: else: results.append('nok') except Exception as error: - results.append(str(error)) + results.append(error) if results.count('ok') == 0: # all failed => 500 status = 500 elif results.count('ok') != len(results): # some failed => 206 diff --git a/src/web/views/user.py b/src/web/views/user.py index 6703a2b2f..1b48d79b2 100644 --- a/src/web/views/user.py +++ b/src/web/views/user.py @@ -19,6 +19,18 @@ user_bp = Blueprint('user', __name__, url_prefix='/user') +def get_contr_and_id(user_id): + if user_id and admin_permission.can(): + return user_id, UserController() + elif user_id and Permission(UserNeed(user_id)).can(): + return user_id, UserController(user_id) + elif user_id: + flash(gettext('You do not have rights on this user'), 'danger') + raise Forbidden(gettext('You do not have rights on this user')) + else: + return current_user.id, UserController(current_user.id) + + @user_bp.route('/opml/export', methods=['GET']) @login_required def opml_export(): @@ -87,17 +99,7 @@ def opml_import(): @user_bp.route('/profile/', methods=['GET']) @login_required def profile(user_id=None): - ucontr = None - if user_id and admin_permission.can(): - ucontr = UserController() - elif user_id and Permission(UserNeed(user_id)).can(): - ucontr = UserController(user_id) - elif user_id: - flash(gettext('You do not have rights on this user'), 'danger') - raise Forbidden(gettext('You do not have rights on this user')) - else: - ucontr = UserController(current_user.id) - user_id = current_user.id + user_id, ucontr = get_contr_and_id(user_id) user = ucontr.get(id=user_id) profile_form, pass_form = ProfileForm(obj=user), PasswordModForm() return render_template('profile.html', user=user, @@ -108,14 +110,7 @@ def profile(user_id=None): @user_bp.route('/password_update/', methods=['POST']) @login_required def password_update(user_id): - ucontr = None - if admin_permission.can(): - ucontr = UserController() - elif Permission(UserNeed(user_id)).can(): - ucontr = UserController(user_id) - else: - flash(gettext('You do not have rights on this user'), 'danger') - raise Forbidden(gettext('You do not have rights on this user')) + user_id, ucontr = get_contr_and_id(user_id) user = ucontr.get(id=user_id) profile_form, pass_form = ProfileForm(obj=user), PasswordModForm() if pass_form.validate(): @@ -133,14 +128,7 @@ def password_update(user_id): @user_bp.route('/profile_update/', methods=['POST']) @login_required def profile_update(user_id): - ucontr = None - if admin_permission.can(): - ucontr = UserController() - elif Permission(UserNeed(user_id)).can(): - ucontr = UserController(user_id) - else: - flash(gettext('You do not have rights on this user'), 'danger') - raise Forbidden(gettext('You do not have rights on this user')) + user_id, ucontr = get_contr_and_id(user_id) user = ucontr.get(id=user_id) profile_form, pass_form = ProfileForm(obj=user), PasswordModForm() if profile_form.validate(): @@ -164,15 +152,7 @@ def profile_update(user_id): @user_bp.route('/delete_account/', methods=['GET']) @login_required def delete(user_id): - ucontr = None - if admin_permission.can(): - ucontr = UserController() - elif Permission(UserNeed(user_id)).can(): - ucontr = UserController(user_id) - logout_user() - else: - flash(gettext('You do not have rights on this user'), 'danger') - raise Forbidden(gettext('You do not have rights on this user')) + user_id, ucontr = get_contr_and_id(user_id) ucontr.delete(user_id) flash(gettext('Deletion successful'), 'success') if admin_permission.can(): From 15946a46c243916acd9c2d7146f224c859b3f541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 12 Mar 2017 19:48:44 +0100 Subject: [PATCH 072/164] style fix (close #63) --- src/web/js/components/MiddlePanel.react.js | 2 +- src/web/js/components/RightPanel.react.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/web/js/components/MiddlePanel.react.js b/src/web/js/components/MiddlePanel.react.js index fd1661a23..d1239bfe2 100644 --- a/src/web/js/components/MiddlePanel.react.js +++ b/src/web/js/components/MiddlePanel.react.js @@ -230,7 +230,7 @@ var MiddlePanelFilter = React.createClass({ diff --git a/src/web/js/components/RightPanel.react.js b/src/web/js/components/RightPanel.react.js index 521a49875..a46b2c87c 100644 --- a/src/web/js/components/RightPanel.react.js +++ b/src/web/js/components/RightPanel.react.js @@ -40,7 +40,7 @@ var PanelMixin = { if(this.isRemovable()) { rem_button = (); } btn_grp = ( From 6239fed7c33f0ec256296528214bb0006cd0e09d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 12 Mar 2017 21:05:33 +0100 Subject: [PATCH 073/164] fixing test --- src/tests/views/user_test.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tests/views/user_test.py b/src/tests/views/user_test.py index 2061eed2b..d24bfd398 100644 --- a/src/tests/views/user_test.py +++ b/src/tests/views/user_test.py @@ -163,15 +163,15 @@ def test_password_update(self): self.assertNotEquals(user.password, old_password) def test_profile_update(self): - self.app.post('/user/password_update/%d' % self.user.id, - data={'email': 'not an email', 'is_admin': True}) - user = self.uctrl.get(id=self.user.id) - self.assertFalse(user.is_admin) - self.assertNotEquals(user.email, 'not an email') + data = {'email': 'not an email', 'is_admin': True} + self.app.post('/user/profile_update/%d' % self.user2.id, data=data) + user2 = self.uctrl.get(id=self.user2.id) + self.assertFalse(user2.is_admin) + self.assertNotEquals(user2.email, 'not an email') self.uctrl.update({'id': self.user.id}, {'is_admin': True}) - self.app.post('/user/password_update/%d' % self.user.id, - data={'email': 'a.valid@email.fake', 'is_admin': True}) - user = self.uctrl.get(id=self.user.id) - self.assertTrue(user.is_admin) - self.assertNotEquals(user.email, 'a.valid@email.fake') + data['email'], data['login'] = 'a.valid@email.fake', self.user2.login + self.app.post('/user/profile_update/%d' % self.user2.id, data=data) + user2 = self.uctrl.get(id=self.user2.id) + self.assertTrue(user2.is_admin) + self.assertEquals(user2.email, data['email']) From a40b69de626ebef2eebe5862d72ce590086d432d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Schmidts?= Date: Sun, 12 Mar 2017 23:03:37 +0100 Subject: [PATCH 074/164] removing readability mention in the txt --- src/lib/conf_handling.py | 2 +- src/web/js/components/RightPanel.react.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/conf_handling.py b/src/lib/conf_handling.py index 16309c706..05788fb61 100644 --- a/src/lib/conf_handling.py +++ b/src/lib/conf_handling.py @@ -52,7 +52,7 @@ ]}, {'prefix': 'PLUGINS', 'options': [ {'key': 'READABILITY_KEY', 'default': '', - 'ask': 'Enter your readability key if you have one'}, + 'ask': 'Enter your Mercury key key if you have one'}, ]}, {'prefix': 'AUTH', 'options': [ {'key': 'ALLOW_SIGNUP', 'default': True, 'type': bool, diff --git a/src/web/js/components/RightPanel.react.js b/src/web/js/components/RightPanel.react.js index a46b2c87c..b2044ebc0 100644 --- a/src/web/js/components/RightPanel.react.js +++ b/src/web/js/components/RightPanel.react.js @@ -195,9 +195,9 @@ var Article = React.createClass({ } var title; if(this.props.obj.readability_parsed) { - title = "Click this to download a more readable version of this article through redability"; + title = "Click here to fetch and parse the article's URL"; } else { - title = "This article has been downloaded through redability"; + title = "This article has been fetched and parsed"; } return (