Update subliminal to 2.0.rc1 (b48c0c9) VANILLA!

Adds shooter.cn provider
Removes itasa provider until it can be finalized into upstream since it downloads incorrect subtitles #1457
Removes napiprojekt Diaoul/subliminal#536
Replaces #1315

lib/subliminal/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 __title__ = 'subliminal'
-__version__ = '2.0.dev0'
+__version__ = '2.0.rc1'
 __short_version__ = '.'.join(__version__.split('.')[:2])
 __author__ = 'Antoine Bertin'
 __license__ = 'MIT'

lib/subliminal/cli.py

@@ -19,7 +19,7 @@
 from six.moves import configparser
 
 from subliminal import (AsyncProviderPool, Episode, Movie, Video, __version__, check_video, compute_score, get_scores,
-                        provider_manager, refine, region, save_subtitles, scan_video, scan_videos)
+                        provider_manager, refine, refiner_manager, region, save_subtitles, scan_video, scan_videos)
 from subliminal.core import ARCHIVE_EXTENSIONS, search_external_subtitles
 
 logger = logging.getLogger(__name__)
@@ -62,6 +62,7 @@ def __init__(self, path):
         self.config.add_section('general')
         self.config.set('general', 'languages', json.dumps(['en']))
         self.config.set('general', 'providers', json.dumps(sorted([p.name for p in provider_manager])))
+        self.config.set('general', 'refiners', json.dumps(sorted([r.name for r in refiner_manager])))
         self.config.set('general', 'single', str(0))
         self.config.set('general', 'embedded_subtitles', str(1))
         self.config.set('general', 'age', str(int(timedelta(weeks=2).total_seconds())))
@@ -93,6 +94,14 @@ def providers(self):
     def providers(self, value):
         self.config.set('general', 'providers', json.dumps(sorted([p.lower() for p in value])))
 
+    @property
+    def refiners(self):
+        return json.loads(self.config.get('general', 'refiners'))
+
+    @refiners.setter
+    def refiners(self, value):
+        self.config.set('general', 'refiners', json.dumps([r.lower() for r in value]))
+
     @property
     def single(self):
         return self.config.getboolean('general', 'single')
@@ -196,6 +205,8 @@ def convert(self, value, param, ctx):
 
 PROVIDER = click.Choice(sorted(provider_manager.names()))
 
+REFINER = click.Choice(sorted(refiner_manager.names()))
+
 dirs = AppDirs('subliminal')
 cache_file = 'subliminal.dbm'
 config_file = 'config.ini'
@@ -204,7 +215,6 @@ def convert(self, value, param, ctx):
 @click.group(context_settings={'max_content_width': 100}, epilog='Suggestions and bug reports are greatly appreciated: '
              'https://github.com/Diaoul/subliminal/')
 @click.option('--addic7ed', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD', help='Addic7ed configuration.')
-@click.option('--itasa', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD', help='ItaSA configuration.')
 @click.option('--legendastv', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD', help='LegendasTV configuration.')
 @click.option('--opensubtitles', type=click.STRING, nargs=2, metavar='USERNAME PASSWORD',
               help='OpenSubtitles configuration.')
@@ -214,7 +224,7 @@ def convert(self, value, param, ctx):
 @click.option('--debug', is_flag=True, help='Print useful information for debugging subliminal and for reporting bugs.')
 @click.version_option(__version__)
 @click.pass_context
-def subliminal(ctx, addic7ed, itasa, legendastv, opensubtitles, subscenter, cache_dir, debug):
+def subliminal(ctx, addic7ed, legendastv, opensubtitles, subscenter, cache_dir, debug):
     """Subtitles, faster than your thoughts."""
     # create cache directory
     try:
@@ -230,7 +240,6 @@ def subliminal(ctx, addic7ed, itasa, legendastv, opensubtitles, subscenter, cach
     # configure logging
     if debug:
         handler = logging.StreamHandler()
-        # TODO: change format to something nicer (use colorlogs + funcName)
         handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
         logging.getLogger('subliminal').addHandler(handler)
         logging.getLogger('subliminal').setLevel(logging.DEBUG)
@@ -239,8 +248,6 @@ def subliminal(ctx, addic7ed, itasa, legendastv, opensubtitles, subscenter, cach
     ctx.obj = {'provider_configs': {}}
     if addic7ed:
         ctx.obj['provider_configs']['addic7ed'] = {'username': addic7ed[0], 'password': addic7ed[1]}
-    if itasa:
-        ctx.obj['provider_configs']['itasa'] = {'username': itasa[0], 'password': itasa[1]}
     if legendastv:
         ctx.obj['provider_configs']['legendastv'] = {'username': legendastv[0], 'password': legendastv[1]}
     if opensubtitles:
@@ -266,6 +273,7 @@ def cache(ctx, clear_subliminal):
 @click.option('-l', '--language', type=LANGUAGE, required=True, multiple=True, help='Language as IETF code, '
               'e.g. en, pt-BR (can be used multiple times).')
 @click.option('-p', '--provider', type=PROVIDER, multiple=True, help='Provider to use (can be used multiple times).')
+@click.option('-r', '--refiner', type=REFINER, multiple=True, help='Refiner to use (can be used multiple times).')
 @click.option('-a', '--age', type=AGE, help='Filter videos newer than AGE, e.g. 12h, 1w2d.')
 @click.option('-d', '--directory', type=click.STRING, metavar='DIR', help='Directory where to save subtitles, '
               'default is next to the video file.')
@@ -283,8 +291,8 @@ def cache(ctx, clear_subliminal):
 @click.option('-v', '--verbose', count=True, help='Increase verbosity.')
 @click.argument('path', type=click.Path(), required=True, nargs=-1)
 @click.pass_obj
-def download(obj, provider, language, age, directory, encoding, single, force, hearing_impaired,
-             min_score, max_workers, archives, verbose, path):
+def download(obj, provider, refiner, language, age, directory, encoding, single, force, hearing_impaired, min_score,
+             max_workers, archives, verbose, path):
     """Download best subtitles.
 
     PATH can be an directory containing videos, a video file path or a video file name. It can be used multiple times.
@@ -314,15 +322,14 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
                     continue
                 if not force:
                     video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
-                refine(video, embedded_subtitles=not force)
+                refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force)
                 videos.append(video)
                 continue
 
             # directories
             if os.path.isdir(p):
                 try:
-                    scanned_videos = scan_videos(p, age=age, archives=archives, subtitles=not force,
-                                                 subtitles_dir=directory)
+                    scanned_videos = scan_videos(p, age=age, archives=archives)
                 except:
                     logger.exception('Unexpected error while collecting directory path %s', p)
                     errored_paths.append(p)
@@ -332,7 +339,7 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
                         if not force:
                             video.subtitle_languages |= set(search_external_subtitles(video.name,
                                                                                       directory=directory).values())
-                        refine(video, embedded_subtitles=not force)
+                        refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force)
                         videos.append(video)
                     else:
                         ignored_videos.append(video)
@@ -348,7 +355,7 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
             if check_video(video, languages=language, age=age, undefined=single):
                 if not force:
                     video.subtitle_languages |= set(search_external_subtitles(video.name, directory=directory).values())
-                refine(video, embedded_subtitles=not force)
+                refine(video, episode_refiners=refiner, movie_refiners=refiner, embedded_subtitles=not force)
                 videos.append(video)
             else:
                 ignored_videos.append(video)
@@ -394,7 +401,10 @@ def download(obj, provider, language, age, directory, encoding, single, force, h
                                                       hearing_impaired=hearing_impaired, only_one=single)
                 downloaded_subtitles[v] = subtitles
 
-    # TODO: warn about discarded providers
+        if p.discarded_providers:
+            click.secho('Some providers have been discarded due to unexpected errors: %s' %
+                        ', '.join(p.discarded_providers), fg='yellow')
+
     # save subtitles
     total_subtitles = 0
     for v, subtitles in downloaded_subtitles.items():

lib/subliminal/converters/legendastv.py

@@ -4,25 +4,11 @@
 from ..exceptions import ConfigurationError
 
 
-class LegendasTvConverter(LanguageReverseConverter):
+class LegendasTVConverter(LanguageReverseConverter):
     def __init__(self):
-        self.from_legendastv = {1:  ('por', 'BR'),
-                                2:  ('eng',),
-                                3:  ('spa',),
-                                4:  ('fra',),
-                                5:  ('deu',),
-                                6:  ('jpn',),
-                                7:  ('dan',),
-                                8:  ('nor',),
-                                9:  ('swe',),
-                                10: ('por',),
-                                11: ('ara',),
-                                12: ('ces',),
-                                13: ('zho',),
-                                14: ('kor',),
-                                15: ('bul',),
-                                16: ('ita',),
-                                17: ('pol',)}
+        self.from_legendastv = {1: ('por', 'BR'), 2: ('eng',), 3: ('spa',), 4: ('fra',), 5: ('deu',), 6: ('jpn',),
+                                7: ('dan',), 8: ('nor',), 9: ('swe',), 10: ('por',), 11: ('ara',), 12: ('ces',),
+                                13: ('zho',), 14: ('kor',), 15: ('bul',), 16: ('ita',), 17: ('pol',)}
         self.to_legendastv = {v: k for k, v in self.from_legendastv.items()}
         self.codes = set(self.from_legendastv.keys())
 

lib/subliminal/converters/shooter.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+from babelfish import LanguageReverseConverter
+
+from ..exceptions import ConfigurationError
+
+
+class ShooterConverter(LanguageReverseConverter):
+    def __init__(self):
+        self.from_shooter = {'chn': ('zho',), 'eng': ('eng',)}
+        self.to_shooter = {v: k for k, v in self.from_shooter.items()}
+        self.codes = set(self.from_shooter.keys())
+
+    def convert(self, alpha3, country=None, script=None):
+        if (alpha3,) in self.to_shooter:
+            return self.to_shooter[(alpha3,)]
+
+        raise ConfigurationError('Unsupported language for shooter: %s, %s, %s' % (alpha3, country, script))
+
+    def reverse(self, shooter):
+        if shooter in self.from_shooter:
+            return self.from_shooter[shooter]
+
+        raise ConfigurationError('Unsupported language code for shooter: %s' % shooter)

lib/subliminal/core.py

@@ -17,7 +17,7 @@
 from .extensions import provider_manager, refiner_manager
 from .score import compute_score as default_compute_score
 from .subtitle import SUBTITLE_EXTENSIONS, get_subtitle_path
-from .utils import hash_itasa, hash_napiprojekt, hash_opensubtitles, hash_thesubdb
+from .utils import hash_napiprojekt, hash_opensubtitles, hash_shooter, hash_thesubdb
 from .video import VIDEO_EXTENSIONS, Episode, Movie, Video
 
 #: Supported archive extensions
@@ -383,8 +383,8 @@ def scan_video(path):
     video.size = os.path.getsize(path)
     if video.size > 10485760:
         logger.debug('Size is %d', video.size)
-        video.hashes['itasa'] = hash_itasa(path)
         video.hashes['opensubtitles'] = hash_opensubtitles(path)
+        video.hashes['shooter'] = hash_shooter(path)
         video.hashes['thesubdb'] = hash_thesubdb(path)
         video.hashes['napiprojekt'] = hash_napiprojekt(path)
         logger.debug('Computed hashes %r', video.hashes)
@@ -441,7 +441,7 @@ def scan_archive(path):
     return video
 
 
-def scan_videos(path, age=None, archives=True, **kwargs):
+def scan_videos(path, age=None, archives=True):
     """Scan `path` for videos and their subtitles.
 
     See :func:`refine` to find additional information for the video.
@@ -517,7 +517,7 @@ def scan_videos(path, age=None, archives=True, **kwargs):
     return videos
 
 
-def refine(video, episode_refiners=('metadata', 'tvdb', 'omdb'), movie_refiners=('metadata', 'omdb'), **kwargs):
+def refine(video, episode_refiners=None, movie_refiners=None, **kwargs):
     """Refine a video using :ref:`refiners`.
 
     .. note::
@@ -528,14 +528,14 @@ def refine(video, episode_refiners=('metadata', 'tvdb', 'omdb'), movie_refiners=
     :type video: :class:`~subliminal.video.Video`
     :param tuple episode_refiners: refiners to use for episodes.
     :param tuple movie_refiners: refiners to use for movies.
-    :param \*\*kwargs: parameters for refiners.
+    :param \*\*kwargs: additional parameters for the :func:`~subliminal.refiners.refine` functions.
 
     """
     refiners = ()
     if isinstance(video, Episode):
-        refiners = episode_refiners or ()
+        refiners = episode_refiners or ('metadata', 'tvdb', 'omdb')
     elif isinstance(video, Movie):
-        refiners = movie_refiners or ()
+        refiners = movie_refiners or ('metadata', 'omdb')
     for refiner in refiners:
         logger.info('Refining video with %s', refiner)
         try:
@@ -549,14 +549,13 @@ def list_subtitles(videos, languages, pool_class=ProviderPool, **kwargs):
 
     The `videos` must pass the `languages` check of :func:`check_video`.
 
-    All other parameters are passed onwards to the provided `pool_class` constructor.
-
     :param videos: videos to list subtitles for.
     :type videos: set of :class:`~subliminal.video.Video`
     :param languages: languages to search for.
     :type languages: set of :class:`~babelfish.language.Language`
     :param pool_class: class to use as provider pool.
-    :type: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :type pool_class: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :param \*\*kwargs: additional parameters for the provided `pool_class` constructor.
     :return: found subtitles per video.
     :rtype: dict of :class:`~subliminal.video.Video` to list of :class:`~subliminal.subtitle.Subtitle`
 
@@ -589,12 +588,11 @@ def list_subtitles(videos, languages, pool_class=ProviderPool, **kwargs):
 def download_subtitles(subtitles, pool_class=ProviderPool, **kwargs):
     """Download :attr:`~subliminal.subtitle.Subtitle.content` of `subtitles`.
 
-    All other parameters are passed onwards to the `pool_class` constructor.
-
     :param subtitles: subtitles to download.
     :type subtitles: list of :class:`~subliminal.subtitle.Subtitle`
     :param pool_class: class to use as provider pool.
-    :type: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :type pool_class: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :param \*\*kwargs: additional parameters for the provided `pool_class` constructor.
 
     """
     with pool_class(**kwargs) as pool:
@@ -609,8 +607,6 @@ def download_best_subtitles(videos, languages, min_score=0, hearing_impaired=Fal
 
     The `videos` must pass the `languages` and `undefined` (`only_one`) checks of :func:`check_video`.
 
-    All other parameters are passed onwards to the `pool_class` constructor.
-
     :param videos: videos to download subtitles for.
     :type videos: set of :class:`~subliminal.video.Video`
     :param languages: languages to download.
@@ -621,7 +617,8 @@ def download_best_subtitles(videos, languages, min_score=0, hearing_impaired=Fal
     :param compute_score: function that takes `subtitle` and `video` as positional arguments,
         `hearing_impaired` as keyword argument and returns the score.
     :param pool_class: class to use as provider pool.
-    :type: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :type pool_class: :class:`ProviderPool`, :class:`AsyncProviderPool` or similar
+    :param \*\*kwargs: additional parameters for the provided `pool_class` constructor.
     :return: downloaded subtitles per video.
     :rtype: dict of :class:`~subliminal.video.Video` to list of :class:`~subliminal.subtitle.Subtitle`
 

lib/subliminal/extensions.py

@@ -50,8 +50,8 @@ def _find_entry_points(self, namespace):
     def register(self, entry_point):
         """Register an extension
 
-        :param str entry_point: extension to register (entry point syntax)
-        :raise: ValueError if already registered
+        :param str entry_point: extension to register (entry point syntax).
+        :raise: ValueError if already registered.
 
         """
         if entry_point in self.registered_extensions:
@@ -70,7 +70,7 @@ def register(self, entry_point):
     def unregister(self, entry_point):
         """Unregister a provider
 
-        :param str entry_point: provider to unregister (entry point syntax)
+        :param str entry_point: provider to unregister (entry point syntax).
 
         """
         if entry_point not in self.registered_extensions:
@@ -89,8 +89,10 @@ def unregister(self, entry_point):
 #: Provider manager
 provider_manager = RegistrableExtensionManager('subliminal.providers', [
     'addic7ed = subliminal.providers.addic7ed:Addic7edProvider',
+    'legendastv = subliminal.providers.legendastv:LegendasTVProvider',
     'opensubtitles = subliminal.providers.opensubtitles:OpenSubtitlesProvider',
     'podnapisi = subliminal.providers.podnapisi:PodnapisiProvider',
+    'shooter = subliminal.providers.shooter:ShooterProvider',
     'subscenter = subliminal.providers.subscenter:SubsCenterProvider',
     'thesubdb = subliminal.providers.thesubdb:TheSubDBProvider',
     'tvsubtitles = subliminal.providers.tvsubtitles:TVsubtitlesProvider'

lib/subliminal/providers/__init__.py

@@ -26,7 +26,7 @@ class ParserBeautifulSoup(BeautifulSoup):
     """A ``bs4.BeautifulSoup`` that picks the first parser available in `parsers`.
 
     :param markup: markup for the ``bs4.BeautifulSoup``.
-    :param list parsers: parser names, in order of preference
+    :param list parsers: parser names, in order of preference.
 
     """
     def __init__(self, markup, parsers, **kwargs):