Refactoring some stuff

README.md

@@ -1,10 +1,3 @@
-<!-- <div align="center" id="top"> 
-  <img src="./.github/app.gif" alt="metatube" />
-
-  &#xa0;
-
-  <a href="https://metatube.netlify.app">Demo</a>
-</div> -->
 <h1 align="center">MetaTube</h1>
 
 <p align="center">
@@ -265,8 +258,9 @@ Made with :heart: by <a href="https://github.com/JVT038" target="_blank">JVT038<
 - [X] Dark mode support
 - [X] Fix error `Synchronous XMLHttpRequest on the main thread is deprecated because of its detrimental effects to the end user’s experience. For more help http://xhr.spec.whatwg.org/` in overview
 - [X] Make sure the search for downloaded song field works
+- [x] Make sure the progress bar works properly in a Docker container, because it doesn't work properly rn
 
-### Not finished (I'll never finish this)
+### Not finished (I'll probably never finish this lol)
 
 - [ ] Add it to the PyPi library
 - [ ] Add support for sites other than YouTube
@@ -289,8 +283,10 @@ Made with :heart: by <a href="https://github.com/JVT038" target="_blank">JVT038<
 - [ ] Have a proper versioning system, because it's impossible to keep track of versions rn
 - [ ] Cache and store the segments and other video data, so next time of loading a video will be faster
 - [ ] Send websocket requests to one specific device / client only, to prevent duplicate websocket requests
-- [ ] Make sure the progress bar works properly in a Docker container, because it doesn't work properly rn.
-- [ ] Use proper queues and threading during download instead of the weird ping-pong system between the client and the server.
+- [ ] Use proper queues and threading during download instead of the weird ping-pong system between the client and the server.*
+- [ ] Add unit tests for the download, metadata logic, template / database stuff, config detection, automatic migrations
+
+* in progress
 &#xa0;
 
 ## Disclaimer

cli_to_api.py

@@ -0,0 +1,51 @@
+'''
+Source: https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py
+Allow direct execution
+'''
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import yt_dlp
+import yt_dlp.options
+
+create_parser = yt_dlp.options.create_parser
+
+
+def parse_patched_options(opts):
+    patched_parser = create_parser()
+    patched_parser.defaults.update({
+        'ignoreerrors': False,
+        'retries': 0,
+        'fragment_retries': 0,
+        'extract_flat': False,
+        'concat_playlist': 'never',
+    })
+    yt_dlp.options.create_parser = lambda: patched_parser
+    try:
+        return yt_dlp.parse_options(opts)
+    finally:
+        yt_dlp.options.create_parser = create_parser
+
+
+default_opts = parse_patched_options([]).ydl_opts
+
+
+def cli_to_api(opts, cli_defaults=False):
+    opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
+
+    diff = {k: v for k, v in opts.items() if default_opts[k] != v}
+    if 'postprocessors' in diff:
+        diff['postprocessors'] = [pp for pp in diff['postprocessors']
+                                  if pp not in default_opts['postprocessors']]
+    return diff
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    print('\nThe arguments passed translate to:\n')
+    pprint(cli_to_api(sys.argv[1:]))
+    print('\nCombining these with the CLI defaults gives:\n')
+    pprint(cli_to_api(sys.argv[1:], True))

metatube/overview/routes.py

@@ -1,3 +1,4 @@
+from platform import release
 import shutil
 from magic import Magic
 from metatube.overview import bp
@@ -112,24 +113,24 @@ def searchmetadata(data):
         socketio.start_background_task(Genius.searchsong, data, token)
 
 @socketio.on('ytdl_download')
-def download(data):
-    url = data["url"]
-    ext = data["ext"] or 'mp3'
-    output_folder = data["output_folder"] or '/downloads'
-    output_type = data["type"] or 'Audio'
-    output_format = data["output_format"] or f'%(title)s.%(ext)s'
-    bitrate = data["bitrate"] or '192'
-    skipfragments = data["skipfragments"] or {}
-    proxy_data = data["proxy_data"] or {'proxy_type': 'None'}
+def download(fileData, metadata):
+    url = fileData["url"]
+    ext = fileData["ext"] or 'mp3'
+    output_folder = fileData["output_folder"] or '/downloads'
+    output_type = fileData["type"] or 'Audio'
+    output_format = fileData["output_format"] or f'%(title)s.%(ext)s'
+    bitrate = fileData["bitrate"] or '192'
+    skipfragments = fileData["skipfragments"] or {}
+    proxy_data = fileData["proxy_data"] or {'proxy_type': 'None'}
 
-    width = data["width"] or 1920
-    height = data["height"] or 1080
+    width = fileData["width"] or 1920
+    height = fileData["height"] or 1080
     ffmpeg = Config.get_ffmpeg()
     hw_transcoding = Config.get_hwt()
     vaapi_device = hw_transcoding.split(';')[1] if 'vaapi' in hw_transcoding else ''
     verbose = strtobool(str(env.LOGGER))
-    logger.info('Request to download %s', data["url"])
-    ytdl_options = yt.get_options(url, ext, output_folder, output_type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose)
+    logger.info('Request to download %s', fileData["url"])
+    ytdl_options = yt.get_options(ext, output_folder, output_type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose, metadata)
     if ytdl_options is not False:
         socketio.start_background_task(yt.start_download, url, ytdl_options)
         # socketio.start_background_task(yt.download, url, ytdl_options)
@@ -182,7 +183,11 @@ def fetchgeniusalbum(input_id):
     genius.fetchalbum(input_id)    
 
 @socketio.on('mergedata')
-def mergedata(filepath, release_id, metadata, cover, source):
+def mergedata(metadata, filepath):
+    release_id = metadata["release_id"]
+    cover = metadata["cover"]
+    source = metadata["metadata_source"]
+
     if Database.checktrackid(release_id) is None and Database.checktrackid(metadata.get('trackid', '')) is None:
 
         metadata_user = metadata

metatube/static/JS/overview.js

@@ -700,13 +700,72 @@ $(document).ready(function() {
         $("tr#" + rowid).find('.finditembtn').remove();
     }
 
+    function getMetadata() {          
+        let release_id = $(".audiocol-checkbox:checked").parent().parent().attr('id');
+        let people = {};
+        let metadata_source = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').find('span.metadatasource').text() : "Unavailable";
+        let cover = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').children('img').attr('src') : "Unavailable";
+
+        if(metadata_source == 'Unavailable') {
+            // The priority order is: Spotify -> Deezer -> Musibrainz
+            var trackid = $("#spotify_trackid").length > 0 ? $("#spotify_trackid").val() : ($("#deezer_releaseid").val().length > 0 ? $("#deezer_trackid").val() : $("#mbp_trackid").val());
+            var albumid = $("#spotify_albumid").length > 0 ? $("#spotify_albumid").val() : ($("#deezer_albumid").val().length > 0 ? $("#deezer_albumid").val() : $("#mbp_albumid").val());
+        } else if(metadata_source == 'Spotify') {
+            var trackid = $("#spotify_trackid").val();
+            var albumid = $("#spotify_albumid").val();   
+        } else if(metadata_source == 'Musicbrainz') {
+            var trackid = $("#mbp_releaseid").val();
+            var albumid = $("#mbp_albumid").val();
+        } else if(metadata_source == 'Deezer') {
+            var trackid = $("#deezer_trackid").val();
+            var albumid = $("#deezer_albumid").val();
+        } else if(metadata_source == 'Genius') { 
+            var trackid = $("#genius_songid").val();
+        }
+
+        $.each($('.artist_relations'), function() {
+            if($(this).val().trim().length < 1 || $(this).parent().siblings().find('.artist_relations').val().trim().length < 1) {
+                return;
+            } else {
+                // Get ID by removing all letters from the ID, so the number remains
+                let id = $(this).parents('.personrow').attr('id').replace(/[a-zA-Z]/g, '');
+                if(this.id.replace(/[0-9]/g, '') == 'artist_relations_name') {
+                    people[id].name = $(this).val();
+                } else {
+                    people[id].type = $(this).val();
+                }
+            }
+        });
+
+        let artists = $("#md_artists").val().split(';');
+        let albumartists = $("#md_album_artists").val().split(';');
+        return {
+            'trackid': trackid,
+            'albumid': albumid,
+            'title': $("#md_title").val(),
+            'artists': JSON.stringify(artists),
+            'album': $("#md_album").val(),
+            'album_artists': JSON.stringify(albumartists),
+            'album_tracknr': $("#md_album_tracknr").val(),
+            'album_releasedate': $("#md_album_releasedate").val(),
+            'cover': $("#md_cover").val(),
+            'people': JSON.stringify(people),
+            'cover': cover,
+            'release_id': release_id,
+            'metadata_source': metadata_source
+        };
+    }
+
     function getPhases() {
         return $("#segments_check").is(':checked') ? 4 : 5;
     }
 
+    function getProgress() {
+        return $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progressedit") : $("#progress");
+    }
+
     function setProgress(percentage) { 
-        let progress = $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progressedit") : $("#progress");
-        progress.attr({
+        getProgress().attr({
             'aria-valuenow': percentage + "%",
             'style': 'width: ' + parseInt(percentage) + '%'
         }).text(percentage + "%");
@@ -1402,7 +1461,7 @@ $(document).ready(function() {
                 'width': width,
                 'height': height
             }
-            socket.emit('ytdl_download', data, function(ack) {
+            socket.emit('ytdl_download', data, getMetadata(), function(ack) {
                 if(ack == "OK") {
                     $("#editmetadata, #downloadbtn, #searchmetadataview, #404p, #defaultview, #resetviewbtn, #geniusbtn, #audiocol, #savemetadata, #metadataview, #geniuscol").addClass('d-none');
                     $("#progressview").removeClass('d-none');
@@ -1529,58 +1588,7 @@ $(document).ready(function() {
             progress_text.text('Adding metadata...');
             var filepath = msg.filepath;
             if($("#edititemmodal").css('display').toLowerCase() == 'none') {
-
-                let release_id = $(".audiocol-checkbox:checked").parent().parent().attr('id');
-                let people = {};
-                let metadata_source = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').find('span.metadatasource').text() : "Unavailable";
-                let cover = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').children('img').attr('src') : "Unavailable";
-
-                if(metadata_source == 'Unavailable') {
-                    // The priority order is: Spotify -> Deezer -> Musibrainz
-                    var trackid = $("#spotify_trackid").length > 0 ? $("#spotify_trackid").val() : ($("#deezer_releaseid").val().length > 0 ? $("#deezer_trackid").val() : $("#mbp_trackid").val());
-                    var albumid = $("#spotify_albumid").length > 0 ? $("#spotify_albumid").val() : ($("#deezer_albumid").val().length > 0 ? $("#deezer_albumid").val() : $("#mbp_albumid").val());
-                } else if(metadata_source == 'Spotify') {
-                    var trackid = $("#spotify_trackid").val();
-                    var albumid = $("#spotify_albumid").val();   
-                } else if(metadata_source == 'Musicbrainz') {
-                    var trackid = $("#mbp_releaseid").val();
-                    var albumid = $("#mbp_albumid").val();
-                } else if(metadata_source == 'Deezer') {
-                    var trackid = $("#deezer_trackid").val();
-                    var albumid = $("#deezer_albumid").val();
-                } else if(metadata_source == 'Genius') { 
-                    var trackid = $("#genius_songid").val();
-                }
-
-                $.each($('.artist_relations'), function() {
-                    if($(this).val().trim().length < 1 || $(this).parent().siblings().find('.artist_relations').val().trim().length < 1) {
-                        return;
-                    } else {
-                        // Get ID by removing all letters from the ID, so the number remains
-                        let id = $(this).parents('.personrow').attr('id').replace(/[a-zA-Z]/g, '');
-                        if(this.id.replace(/[0-9]/g, '') == 'artist_relations_name') {
-                            people[id].name = $(this).val();
-                        } else {
-                            people[id].type = $(this).val();
-                        }
-                    }
-                });
-
-                let artists = $("#md_artists").val().split(';');
-                let albumartists = $("#md_album_artists").val().split(';');
-                let metadata = {
-                    'trackid': trackid,
-                    'albumid': albumid,
-                    'title': $("#md_title").val(),
-                    'artists': JSON.stringify(artists),
-                    'album': $("#md_album").val(),
-                    'album_artists': JSON.stringify(albumartists),
-                    'album_tracknr': $("#md_album_tracknr").val(),
-                    'album_releasedate': $("#md_album_releasedate").val(),
-                    'cover': $("#md_cover").val(),
-                    'people': JSON.stringify(people)
-                };
-                socket.emit('mergedata', filepath, release_id, metadata, cover, metadata_source);
+                socket.emit('mergedata', getMetadata(), filepath);
             } else {
                 let itemid = $("#edititemmodal").attr('itemid');
                 socket.emit('editfilerequest', filepath, itemid);
@@ -1612,9 +1620,9 @@ $(document).ready(function() {
 
     socket.on('downloaderror', function(msg) {
         progress_text.text(msg.message);
-        progress.attr('aria-valuenow', 100);
-        progress.html('ERROR <i class="fi-cwluxl-smiley-sad-wide" aria-hidden="true"></i>');
-        progress.css('width', '100%');
+        getProgress().attr('aria-valuenow', 100);
+        getProgress().html('ERROR <i class="fi-cwluxl-smiley-sad-wide" aria-hidden="true"></i>');
+        getProgress().css('width', '100%');
         progress_text.text(msg.message);
         if($("#edititemmodal").css('display').toLowerCase() != 'block') {
             $("#resetviewbtn").removeClass('d-none');

metatube/youtube.py

@@ -1,10 +1,11 @@
 import yt_dlp, json, os
 from yt_dlp.postprocessor.ffmpeg import FFmpegPostProcessorError
+from yt_dlp.postprocessor.metadataparser import MetadataParserPP
 from youtubesearchpython import VideosSearch
 from threading import Thread
 from urllib.error import URLError
 from yt_dlp.utils import ExtractorError, DownloadError, PostProcessingError
-from metatube import sockets, logger, socketio
+from metatube import sockets, logger
 from metatube.sponsorblock import segments as findsegments
 from jinja2 import Environment, PackageLoader, select_autoescape
 import asyncio
@@ -101,7 +102,7 @@ def postprocessor_hook(d):
             sockets.finished_postprocessor(d['postprocessor'], d['info_dict']['filepath'])
 
     @staticmethod
-    def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose):
+    def get_options(ext, output_folder, type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose, metadata):
         proxy = json.loads(proxy_data)
         filepath = os.path.join(output_folder, output_format)
         segments = json.loads(skipfragments)
@@ -168,6 +169,21 @@ def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragm
                 'key': 'ModifyChapters',
                 'remove_ranges': ranges
             })
+
+        '''
+        --parse-metadata example in CLI:
+        yt-dlp orJSJGHjBLI -x --audio-format mp3 --add-metadata -o "%(track,title)s - %(artist)s.%(ext)s" --parse-metadata " Bad Habits: %(title)s" --parse-metadata "Ed Sheeran:%(artist)s"
+        '''
+        # postprocessors.append({
+        #     'actions': [
+        #         (MetadataParserPP.interpretter, " " + metadata['title'], ' %(title)s'),
+        #         (MetadataParserPP.interpretter, metadata['album'], '%(album)s'),
+        #         (MetadataParserPP.interpretter, ';'.join(json.loads(metadata['album_artists'])), '%(album_artist)s'),
+        #         (MetadataParserPP.interpretter, metadata['album_tracknr'], '%(track_number)s'),
+        #     ],
+        #     'key': 'MetadataParser',
+        #     'when': 'pre_process'
+        # })
 
         ytdl_options = {
             'format': format,

requirements.txt

@@ -1,20 +1,20 @@
 mutagen==1.47.0
 requests==2.31.0
-yt-dlp==2023.11.16
+yt-dlp==2023.12.30
 gevent==23.9.1
 gevent-websocket==0.10.1
-Flask==3.0.0
+Flask==3.0.1
 Flask-SocketIO==5.3.6
-Flask-Migrate==4.0.4
+Flask-Migrate==4.0.5
 Flask-SQLAlchemy==3.1.1
 musicbrainzngs==0.7.1
 sponsorblock.py==0.2.2
 python-dateutil==2.8.2
-python-dotenv==1.0.0
+python-dotenv==1.0.1
 python-magic==0.4.27
 ffmpeg-python==0.2.0
 youtube-search-python==1.6.6
 spotipy==2.23.0
 urllib3==2.1.0
-deezer-python==5.8.1
+deezer-python==6.1.1
 lyricsgenius==3.0.1