From 6a372d7312bac5416ddd8d19482ea57eaa42aef2 Mon Sep 17 00:00:00 2001
From: JVT038 <47184046+JVT038@users.noreply.github.com>
Date: Sun, 28 Jan 2024 15:10:38 +0100
Subject: [PATCH 1/3] Implemented some proper threading finally!!
---
metatube.py | 2 +-
metatube/__init__.py | 6 +-
metatube/database.py | 29 +++-
metatube/metadata.py | 52 ++++--
metatube/overview/routes.py | 14 +-
metatube/settings/routes.py | 2 +
metatube/sockets.py | 33 +++-
metatube/static/JS/overview.js | 287 ++++++++++++++++++---------------
metatube/youtube.py | 166 +++++++++++++------
9 files changed, 376 insertions(+), 215 deletions(-)
diff --git a/metatube.py b/metatube.py
index cb37bfac..d66dcde4 100644
--- a/metatube.py
+++ b/metatube.py
@@ -14,6 +14,6 @@
get_hub().NOT_ERROR += (KeyboardInterrupt,)
try:
print(u'Starting the webserver on http://%s:%s...'%(host, port))
- socketio.run(app, str(host), int(port), log_output=strtobool(str(log_output)))
+ socketio.run(app, str(host), int(port), log_output=strtobool(str(log_output))) # type: ignore
except KeyboardInterrupt:
print('Stopped server because of KeyboardInterrupt')
\ No newline at end of file
diff --git a/metatube/__init__.py b/metatube/__init__.py
index 23e06e0f..3dad96ab 100644
--- a/metatube/__init__.py
+++ b/metatube/__init__.py
@@ -1,3 +1,5 @@
+from gevent import monkey
+monkey.patch_all()
from flask import Flask, json
from flask.logging import default_handler
from flask_sqlalchemy import SQLAlchemy
@@ -34,12 +36,12 @@ def create_app(config_class=Config):
)
app.register_error_handler(Exception, error)
app.logger.removeHandler(default_handler)
- app.logger.addHandler(logger)
+ app.logger.addHandler(console)
console.setLevel(int(app.config["LOG_LEVEL"]))
socket_log = logger if strtobool(str(app.config["SOCKET_LOG"])) == 1 else False
db.init_app(app)
migrate.init_app(app, db, compare_type=True, ping_interval=60)
- socketio.init_app(app, json=json, engineio_logger=socket_log, logger=socket_log)
+ socketio.init_app(app, json=json, engineio_logger=socket_log, logger=socket_log, async_mode='gevent')
app.register_blueprint(bp_overview)
app.register_blueprint(bp_settings)
if app.config.get('INIT_DB') == True:
diff --git a/metatube/database.py b/metatube/database.py
index 449498ec..20dadc3e 100644
--- a/metatube/database.py
+++ b/metatube/database.py
@@ -19,9 +19,11 @@ def ffmpeg(self, ffmpeg_path):
db.session.commit()
logger.info('Set FFmpeg path to %s', ffmpeg_path)
+ @staticmethod
def get_ffmpeg():
return Config.query.get(1).ffmpeg_directory
+ @staticmethod
def get_hwt():
return Config.query.get(1).hardware_transcoding
@@ -32,7 +34,6 @@ def set_amount(self, amount):
def set_spotify(self, spotify):
self.spotify_api = spotify
- print(spotify)
db.session.commit()
logger.info('Changed the Spotify API settings')
@@ -51,15 +52,19 @@ def set_hwtranscoding(self, hw_transcoding):
db.session.commit()
logger.info('Set hardware transcoding to %s', hw_transcoding)
+ @staticmethod
def get_metadata_sources():
return Config.query.get(1).metadata_sources
+ @staticmethod
def get_spotify():
return Config.query.get(1).spotify_api
+ @staticmethod
def get_genius():
return Config.query.get(1).genius_api
+ @staticmethod
def get_max():
return Config.query.get(1).amount
@@ -80,12 +85,15 @@ class Templates(db.Model):
proxy_address = db.Column(db.String(128))
proxy_port = db.Column(db.Integer)
+ @staticmethod
def check_existing(value):
return True if Templates.query.filter_by(name = value).count() > 0 else False
+ @staticmethod
def counttemplates():
return Templates.query.count()
+ @staticmethod
def add(data):
row = Templates(
name = data["name"],
@@ -101,15 +109,17 @@ def add(data):
proxy_password = data["proxy"]["password"],
proxy_address = data["proxy"]["address"],
proxy_port = data["proxy"]["port"]
- )
+ ) # type: ignore
db.session.add(row)
db.session.commit()
logger.info('Added template %s', data["name"])
return row.id
+ @staticmethod
def fetchtemplate(input_id):
return Templates.query.filter_by(id = input_id).first()
+ @staticmethod
def fetchalltemplates():
return Templates.query.all()
@@ -117,7 +127,8 @@ def delete(self):
logger.info('Deleting template %s', self.name)
db.session.delete(self)
db.session.commit()
-
+
+ @staticmethod
def searchdefault():
return Templates.query.filter_by(default = True).first()
@@ -159,30 +170,38 @@ class Database(db.Model):
audio_id = db.Column(db.String(128))
youtube_id = db.Column(db.String(16), unique=True)
+ @staticmethod
def searchrecords(query):
return Database.query.filter(Database.name.like(query + "%")).all()
+ @staticmethod
def itemtodict(item):
dict = {}
for column in item.__table__.columns:
dict[column.name] = str(getattr(item, column.name))
return dict
+ @staticmethod
def getrecords():
return Database.query.all()
+ @staticmethod
def fetchitem(input_id):
return Database.query.filter_by(id = input_id).first()
+ @staticmethod
def checkfile(filepath_input):
return Database.query.filter_by(filepath = filepath_input).first()
+ @staticmethod
def checkyt(youtube_id_input):
return Database.query.filter_by(youtube_id = youtube_id_input).first()
+ @staticmethod
def checktrackid(release_id_input):
return Database.query.filter_by(audio_id = release_id_input).first()
+ @staticmethod
def insert(data):
row = Database(
filepath = data["filepath"],
@@ -193,7 +212,7 @@ def insert(data):
cover = data["image"],
audio_id = data["track_id"],
youtube_id = data["ytid"]
- )
+ ) # type: ignore
db.session.add(row)
db.session.commit()
logger.info('Inserted item %s into database', data["name"])
@@ -213,7 +232,7 @@ def update(self, data):
logger.info('Updated item %s', data["name"])
data["date"] = data["date"].strftime('%d-%m-%Y')
sockets.overview({'msg': 'changed_metadata_db', 'data': data})
-
+
def updatefilepath(self, filepath):
self.filepath = filepath
db.session.commit()
diff --git a/metatube/metadata.py b/metatube/metadata.py
index 461217a2..0ae252ae 100644
--- a/metatube/metadata.py
+++ b/metatube/metadata.py
@@ -17,7 +17,8 @@
from datetime import datetime
import requests, base64, os
-class MetaData:
+class MetaData:
+ @staticmethod
def getresponse(data):
return {
'filepath': os.path.join(Config.BASE_DIR, data["filename"]),
@@ -30,6 +31,7 @@ def getresponse(data):
'track_id': data["track_id"]
}
+ @staticmethod
def getmusicbrainzdata(filename, metadata_user, metadata_source, cover_source):
logger.info('Getting Musicbrainz metadata')
album = metadata_source["release"]["release-group"]["title"] if len(metadata_user["album"]) < 1 else metadata_user["album"]
@@ -62,7 +64,7 @@ def getmusicbrainzdata(filename, metadata_user, metadata_source, cover_source):
magic = Magic(mime=True)
cover_mime_type = magic.from_buffer(image)
except Exception:
- sockets.downloadprogress({'status': 'error', 'message': 'Cover URL is invalid!'})
+ sockets.metadata_error('Cover URL is invalid!')
return False
else:
cover_mime_type = "image/png"
@@ -117,6 +119,7 @@ def getmusicbrainzdata(filename, metadata_user, metadata_source, cover_source):
}
return data
+ @staticmethod
def getspotifydata(filename, metadata_user, metadata_source):
logger.info('Getting Spotify metadata')
album = metadata_source["album"]["name"] if len(metadata_user["album"]) < 1 else metadata_user["album"]
@@ -142,7 +145,7 @@ def getspotifydata(filename, metadata_user, metadata_source):
magic = Magic(mime=True)
cover_mime_type = magic.from_buffer(image)
except Exception:
- sockets.downloadprogress({'status': 'error', 'message': 'Cover URL is invalid!'})
+ sockets.metadata_error('Cover URL is invalid!')
return False
else:
cover_mime_type = "image/png"
@@ -169,6 +172,7 @@ def getspotifydata(filename, metadata_user, metadata_source):
}
return data
+ @staticmethod
def getdeezerdata(filename, metadata_user, metadata_source):
album = metadata_source["album"]["title"] if len(metadata_user["album"]) < 1 else metadata_user["album"]
trackid = str(metadata_source["id"]) if len(metadata_user["trackid"]) < 1 else str(metadata_user["trackid"])
@@ -193,7 +197,7 @@ def getdeezerdata(filename, metadata_user, metadata_source):
magic = Magic(mime=True)
cover_mime_type = magic.from_buffer(image)
except Exception:
- sockets.downloadprogress({'status': 'error', 'message': 'Cover URL is invalid!'})
+ sockets.metadata_error('Cover URL is invalid!')
return False
else:
file = open(cover_path, 'rb')
@@ -220,6 +224,7 @@ def getdeezerdata(filename, metadata_user, metadata_source):
}
return data
+ @staticmethod
def getgeniusdata(filename, metadata_user, metadata_source, lyrics):
logger.info('Getting Genius metadata')
album = metadata_source["song"]["album"]["name"] if len(metadata_user["album"]) < 1 else metadata_user["album"]
@@ -243,7 +248,7 @@ def getgeniusdata(filename, metadata_user, metadata_source, lyrics):
magic = Magic(mime=True)
cover_mime_type = magic.from_buffer(image)
except Exception:
- sockets.downloadprogress({'status': 'error', 'message': 'Cover URL is invalid!'})
+ sockets.metadata_error('Cover URL is invalid!')
return False
else:
cover_mime_type = "image/png"
@@ -272,6 +277,7 @@ def getgeniusdata(filename, metadata_user, metadata_source, lyrics):
}
return data
+ @staticmethod
def onlyuserdata(filename, metadata_user):
if metadata_user["cover"] != '':
try:
@@ -281,7 +287,7 @@ def onlyuserdata(filename, metadata_user):
magic = Magic(mime=True)
cover_mime_type = magic.from_buffer(image)
except Exception:
- sockets.downloadprogress({'status': 'error', 'message': 'Cover URL is invalid!'})
+ sockets.metadata_error('Cover URL is invalid!')
return False
else:
cover_path = os.path.join(Config.BASE_DIR, 'metatube/static/images/empty_cover.png')
@@ -309,7 +315,8 @@ def onlyuserdata(filename, metadata_user):
'genres': ""
}
return data
-
+
+ @staticmethod
def mergeaudiodata(data):
'''
Valid fields for EasyID3:
@@ -377,10 +384,8 @@ def mergeaudiodata(data):
elif data.get('source', '') == 'Deezer':
audio.RegisterTXXXKey('deezer_trackid', data["track_id"])
audio.RegisterTXXXKey('deezer_albumid', data["album_id"])
-
if 'lyrics' in data:
- audio.RegisterTextKey('lyrics', "USLT")
-
+ audio.RegisterTextKey('lyrics', "USLT")
elif data["extension"] == 'FLAC':
audio = FLAC(data["filename"])
elif data["extension"] == 'AAC':
@@ -389,6 +394,8 @@ def mergeaudiodata(data):
audio = OggOpus(data["filename"])
elif data["extension"] == 'OGG':
audio = OggVorbis(data["filename"])
+ else:
+ return
audio["album"] = data["album"]
audio["artist"] = data["artists"]
@@ -436,11 +443,14 @@ def mergeaudiodata(data):
sockets.overview({'msg': 'changed_metadata', 'data': response})
elif data["goal"] == 'add':
logger.info('Finished adding metadata to %s', data["title"])
- sockets.downloadprogress({'status':'finished_metadata', 'data': response})
-
+ sockets.finished_metadata(response)
+
+ @staticmethod
def mergeid3data(data):
if data["extension"] == 'WAV':
audio = WAVE(data["filename"])
+ else:
+ return
try:
audio.add_tags()
except Exception:
@@ -464,10 +474,14 @@ def mergeid3data(data):
sockets.overview({'msg': 'changed_metadata', 'data': response})
elif data["goal"] == 'add':
logger.info('Finished adding metadata to %s', data["title"])
- sockets.downloadprogress({'status':'finished_metadata', 'data': response})
+ sockets.finished_metadata(response)
+
+ @staticmethod
def mergevideodata(data):
if data["extension"] in ['M4A', 'MP4']:
video = MP4(data["filename"])
+ else:
+ return
dateobj = datetime.strptime(data["release_date"], '%Y-%m-%d') if len(data["release_date"]) > 0 else datetime.now().date()
year = dateobj.year
# iTunes metadata list / key values: https://mutagen.readthedocs.io/en/latest/api/mp4.html?highlight=M4A#mutagen.mp4.MP4Tags
@@ -492,8 +506,9 @@ def mergevideodata(data):
sockets.overview({'msg': 'changed_metadata', 'data': response})
elif data["goal"] == 'add':
logger.info('Finished adding metadata to %s', data["title"])
- sockets.downloadprogress({'status':'finished_metadata', 'data': response})
+ sockets.finished_metadata(response)
+ @staticmethod
def readaudiometadata(filename):
logger.info('Reading metadata of %s', filename)
extension = filename.split('.')[len(filename.split('.')) - 1].upper()
@@ -512,6 +527,8 @@ def readaudiometadata(filename):
elif extension == 'OGG':
audio = OggVorbis(filename)
data = OggVorbis(filename)
+ else:
+ return
response = {
'title': audio.get('title', [''])[0],
@@ -537,10 +554,13 @@ def readaudiometadata(filename):
return response
+ @staticmethod
def readvideometadata(filename):
extension = filename.split('.')[len(filename.split('.')) - 1].upper()
if extension in ['M4A', 'MP4']:
video = MP4(filename)
+ else:
+ return
# Bitrate calculation: https://www.reddit.com/r/headphones/comments/3xju4s/comment/cy5dn8h/?utm_source=share&utm_medium=web2x&context=3
# Mutagen MP4 stream info: https://mutagen.readthedocs.io/en/latest/api/mp4.html#mutagen.mp4.MP4Info
@@ -559,14 +579,18 @@ def readvideometadata(filename):
}
return response
+ @staticmethod
def FLV(filename):
pass
+ @staticmethod
def WEBM(filename):
pass
+ @staticmethod
def MKV(filename):
pass
+ @staticmethod
def AVI(filename):
pass
\ No newline at end of file
diff --git a/metatube/overview/routes.py b/metatube/overview/routes.py
index cb7e1c3e..be91d5d3 100644
--- a/metatube/overview/routes.py
+++ b/metatube/overview/routes.py
@@ -24,6 +24,7 @@
import requests
import random
import string
+import time
@bp.route('/')
def index():
@@ -84,7 +85,8 @@ def search(query):
else:
sockets.searchvideo('This video has already been downloaded!')
else:
- asyncio.run(yt.search(query))
+ # asyncio.run(yt.search(query))
+ socketio.start_background_task(yt.search, query)
else:
sockets.searchvideo('Enter an URL!')
@@ -130,8 +132,8 @@ def download(data):
logger.info('Request to download %s', data["url"])
ytdl_options = yt.get_options(url, ext, output_folder, output_type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose)
if ytdl_options is not False:
- yt_instance = yt()
- yt_instance.get_video(url, ytdl_options)
+ socketio.start_background_task(yt.start_download, url, ytdl_options)
+ # socketio.start_background_task(yt.download, url, ytdl_options)
return 'OK'
@socketio.on('fetchmbprelease')
@@ -174,7 +176,7 @@ def fetchgeniussong(input_id):
sockets.foundgeniussong(song)
@socketio.on('fetchgeniusalbum')
-def fetchgeniussong(input_id):
+def fetchgeniusalbum(input_id):
logger.info('Request for Genius album with id %s', input_id)
token = Config.get_genius()
genius = Genius(token)
@@ -207,6 +209,8 @@ def mergedata(filepath, release_id, metadata, cover, source):
data = MetaData.getgeniusdata(filepath, metadata_user, metadata_source, lyrics)
elif source == 'Unavailable':
data = MetaData.onlyuserdata(filepath, metadata_user)
+ else:
+ return
if data is not False:
data["goal"] = 'add'
data["extension"] = extension
@@ -231,7 +235,7 @@ def mergedata(filepath, release_id, metadata, cover, source):
'image': cover_source,
'track_id': release_id
}
- sockets.downloadprogress({'status': 'metadata_unavailable', 'data': data})
+ sockets.metadata_error(data)
logger.debug('Metadata unavailable for file %s', data["filepath"])
else:
sockets.searchvideo(f'{source} item has already been downloaded!')
diff --git a/metatube/settings/routes.py b/metatube/settings/routes.py
index 4ad6b6e2..3bc5a4f1 100644
--- a/metatube/settings/routes.py
+++ b/metatube/settings/routes.py
@@ -113,6 +113,8 @@ def defaulttemplate(id):
@socketio.on('updatesettings')
def updatesettings(ffmpeg_path, amount, hardware_transcoding, metadata_sources, extradata):
db_config = Config.query.get(1)
+ if db_config is None:
+ return
response = ""
if db_config.ffmpeg_directory != ffmpeg_path:
diff --git a/metatube/sockets.py b/metatube/sockets.py
index f86979cb..f18db195 100644
--- a/metatube/sockets.py
+++ b/metatube/sockets.py
@@ -1,8 +1,5 @@
from metatube import socketio
-def downloadprogress(message):
- socketio.emit('downloadprogress', message)
-
def downloadsettings(message):
socketio.emit('downloadsettings', message)
@@ -61,4 +58,32 @@ def deezersearch(data):
socketio.emit('deezer_response', data)
def deezertrack(data):
- socketio.emit('deezer_track', data)
\ No newline at end of file
+ socketio.emit('deezer_track', data)
+
+def downloadprogress(downloaded_bytes, total_bytes):
+ socketio.emit('downloadprogress', {
+ 'status': 'downloading',
+ 'downloaded_bytes': downloaded_bytes,
+ 'total_bytes': total_bytes
+ })
+
+def postprocessing(postprocessor):
+ socketio.emit('postprocessing', {'postprocessor': postprocessor})
+
+def finished_postprocessor(postprocessor, filepath):
+ socketio.emit('finished_postprocessor', {
+ 'postprocessor': postprocessor,
+ 'filepath': filepath
+ })
+
+def finished_download():
+ socketio.emit('finished_download')
+
+def finished_metadata(response):
+ socketio.emit('finished_metadata', {'status':'finished_metadata', 'data': response})
+
+def metadata_error(error):
+ socketio.emit('downloaderror', {'status': 'error', 'message': error})
+
+def downloaderrors(message):
+ socketio.emit('downloaderror', message)
\ No newline at end of file
diff --git a/metatube/static/JS/overview.js b/metatube/static/JS/overview.js
index f8b1b0ae..3f8e037d 100644
--- a/metatube/static/JS/overview.js
+++ b/metatube/static/JS/overview.js
@@ -1,4 +1,5 @@
var socket = io();
+var progress_text;
$(document).ready(function() {
ap = new APlayer({
container: document.getElementById('audioplayer'),
@@ -699,6 +700,18 @@ $(document).ready(function() {
$("tr#" + rowid).find('.finditembtn').remove();
}
+ function getPhases() {
+ return $("#segments_check").is(':checked') ? 4 : 5;
+ }
+
+ function setProgress(percentage) {
+ let progress = $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progressedit") : $("#progress");
+ progress.attr({
+ 'aria-valuenow': percentage + "%",
+ 'style': 'width: ' + parseInt(percentage) + '%'
+ }).text(percentage + "%");
+ }
+
$(window).resize(function() {
if ($(window).width() < 700) {
$(".youtuberesult").children('li').removeClass('media');
@@ -1394,6 +1407,7 @@ $(document).ready(function() {
$("#editmetadata, #downloadbtn, #searchmetadataview, #404p, #defaultview, #resetviewbtn, #geniusbtn, #audiocol, #savemetadata, #metadataview, #geniuscol").addClass('d-none');
$("#progressview").removeClass('d-none');
$("#searchlog").empty();
+ progress_text = $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progresstextedit") : $("#progresstext");
}
});
}
@@ -1471,151 +1485,162 @@ $(document).ready(function() {
});
socket.on('downloadprogress', function(msg) {
- var progress = $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progressedit") : $("#progress");
- function setprogress(percentage) {
- progress.attr({
- 'aria-valuenow': percentage + "%",
- 'style': 'width: ' + parseInt(percentage) + '%'
- }).text(percentage + "%");
- }
-
$("#editmetadata, #nextbtn, #defaultview, #ytcol").addClass('d-none');
$("#progressview").removeClass('d-none');
$("#searchlog").empty();
- var progress_text = $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progresstextedit") : $("#progresstext");
- let phases = $("#segments_check").is(':checked') ? 4 : 5;
-
- if(msg.status == 'downloading') {
- if(msg.total_bytes != 'Unknown') {
- if((msg.downloaded_bytes / msg.total_byes) == 1) {
- progress_text.text("Extracting audio...");
- setprogress(100 / phases);
- } else {
- progress_text.text("Downloading...");
- let percentage = Math.round(((msg.downloaded_bytes / msg.total_bytes) * 100) / phases);
- setprogress(percentage);
- }
+ // var progress_text = $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progresstextedit") : $("#progresstext");
+
+ if(msg.total_bytes != 'Unknown') {
+ if((msg.downloaded_bytes / msg.total_byes) == 1) {
+ progress_text.text("Extracting audio...");
+ setProgress(100 / getPhases());
} else {
progress_text.text("Downloading...");
+ let percentage = Math.round(((msg.downloaded_bytes / msg.total_bytes) * 100) / getPhases());
+ setProgress(percentage);
}
- }
- else if(msg.status == 'finished_ytdl') {
- let percentage = 100 / phases;
- setprogress(percentage);
- progress_text.text('Extracting audio...');
- } else if(msg.status == 'finished_ffmpeg') {
- if(msg.postprocessor == 'ExtractAudio') {
- let percentage = (100 / phases) * 2;
- setprogress(percentage);
- progress_text.text('Cutting segments from the video... ');
- } else if(msg.postprocessor == 'ModifyChapters') {
- let percentage = (100 / phases) * 3;
- setprogress(percentage);
- progress_text.text('Moving the files to its destination... ');
- } else if(msg.postprocessor == 'MoveFiles') {
- let percentage = (100 / phases) * (phases - 1);
- setprogress(percentage);
- progress_text.text('Adding metadata...');
- var filepath = msg.filepath;
- if($("#edititemmodal").css('display').toLowerCase() == 'none') {
-
- let release_id = $(".audiocol-checkbox:checked").parent().parent().attr('id');
- let people = {};
- let metadata_source = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').find('span.metadatasource').text() : "Unavailable";
- let cover = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').children('img').attr('src') : "Unavailable";
+ } else {
+ progress_text.text("Downloading...");
+ }
+ });
+
+ socket.on('finished_download', function() {
+ let percentage = 100 / getPhases();
+ setProgress(percentage);
+ progress_text.text('Extracting audio...');
+ });
+
+ socket.on('postprocessing', function(msg) {
+ if(msg.postprocessor == 'ModifyChapters') {
+ let percentage = (100 / getPhases()) * 2;
+ setProgress(percentage);
+ progress_text.text('Cutting segments from the video... ');
+ } else if(msg.postprocessor == 'MoveFiles') {
+ let percentage = (100 / getPhases()) * 3;
+ setProgress(percentage);
+ progress_text.text('Moving the files to its destination... ');
+ }
+ });
+
+ socket.on('finished_postprocessor', function(msg) {
+ // if(msg.postprocessor == 'ExtractAudio') {
+ // let percentage = (100 / getPhases()) * 2;
+ // setProgress(percentage);
+ // progress_text.text('Cutting segments from the video... ');
+ // } else if(msg.postprocessor == 'ModifyChapters') {
+ // let percentage = (100 / getPhases()) * 3;
+ // setProgress(percentage);
+ // progress_text.text('Moving the files to its destination... ');
+ // }
+ if(msg.postprocessor == 'MoveFiles') {
+ let percentage = (100 / getPhases()) * (getPhases() - 1);
+ setProgress(percentage);
+ progress_text.text('Adding metadata...');
+ var filepath = msg.filepath;
+ if($("#edititemmodal").css('display').toLowerCase() == 'none') {
+
+ let release_id = $(".audiocol-checkbox:checked").parent().parent().attr('id');
+ let people = {};
+ let metadata_source = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').find('span.metadatasource').text() : "Unavailable";
+ let cover = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').children('img').attr('src') : "Unavailable";
+
+ if(metadata_source == 'Unavailable') {
+ // The priority order is: Spotify -> Deezer -> Musibrainz
+ var trackid = $("#spotify_trackid").length > 0 ? $("#spotify_trackid").val() : ($("#deezer_releaseid").val().length > 0 ? $("#deezer_trackid").val() : $("#mbp_trackid").val());
+ var albumid = $("#spotify_albumid").length > 0 ? $("#spotify_albumid").val() : ($("#deezer_albumid").val().length > 0 ? $("#deezer_albumid").val() : $("#mbp_albumid").val());
+ } else if(metadata_source == 'Spotify') {
+ var trackid = $("#spotify_trackid").val();
+ var albumid = $("#spotify_albumid").val();
+ } else if(metadata_source == 'Musicbrainz') {
+ var trackid = $("#mbp_releaseid").val();
+ var albumid = $("#mbp_albumid").val();
+ } else if(metadata_source == 'Deezer') {
+ var trackid = $("#deezer_trackid").val();
+ var albumid = $("#deezer_albumid").val();
+ } else if(metadata_source == 'Genius') {
+ var trackid = $("#genius_songid").val();
+ }
- if(metadata_source == 'Unavailable') {
- // The priority order is: Spotify -> Deezer -> Musibrainz
- var trackid = $("#spotify_trackid").length > 0 ? $("#spotify_trackid").val() : ($("#deezer_releaseid").val().length > 0 ? $("#deezer_trackid").val() : $("#mbp_trackid").val());
- var albumid = $("#spotify_albumid").length > 0 ? $("#spotify_albumid").val() : ($("#deezer_albumid").val().length > 0 ? $("#deezer_albumid").val() : $("#mbp_albumid").val());
- } else if(metadata_source == 'Spotify') {
- var trackid = $("#spotify_trackid").val();
- var albumid = $("#spotify_albumid").val();
- } else if(metadata_source == 'Musicbrainz') {
- var trackid = $("#mbp_releaseid").val();
- var albumid = $("#mbp_albumid").val();
- } else if(metadata_source == 'Deezer') {
- var trackid = $("#deezer_trackid").val();
- var albumid = $("#deezer_albumid").val();
- } else if(metadata_source == 'Genius') {
- var trackid = $("#genius_songid").val();
- }
-
- $.each($('.artist_relations'), function() {
- if($(this).val().trim().length < 1 || $(this).parent().siblings().find('.artist_relations').val().trim().length < 1) {
- return;
+ $.each($('.artist_relations'), function() {
+ if($(this).val().trim().length < 1 || $(this).parent().siblings().find('.artist_relations').val().trim().length < 1) {
+ return;
+ } else {
+ // Get ID by removing all letters from the ID, so the number remains
+ let id = $(this).parents('.personrow').attr('id').replace(/[a-zA-Z]/g, '');
+ if(this.id.replace(/[0-9]/g, '') == 'artist_relations_name') {
+ people[id].name = $(this).val();
} else {
- // Get ID by removing all letters from the ID, so the number remains
- let id = $(this).parents('.personrow').attr('id').replace(/[a-zA-Z]/g, '');
- if(this.id.replace(/[0-9]/g, '') == 'artist_relations_name') {
- people[id].name = $(this).val();
- } else {
- people[id].type = $(this).val();
- }
+ people[id].type = $(this).val();
}
- });
-
- let artists = $("#md_artists").val().split(';');
- let albumartists = $("#md_album_artists").val().split(';');
- let metadata = {
- 'trackid': trackid,
- 'albumid': albumid,
- 'title': $("#md_title").val(),
- 'artists': JSON.stringify(artists),
- 'album': $("#md_album").val(),
- 'album_artists': JSON.stringify(albumartists),
- 'album_tracknr': $("#md_album_tracknr").val(),
- 'album_releasedate': $("#md_album_releasedate").val(),
- 'cover': $("#md_cover").val(),
- 'people': JSON.stringify(people)
- };
- socket.emit('mergedata', filepath, release_id, metadata, cover, metadata_source);
- } else {
- let itemid = $("#edititemmodal").attr('itemid');
- socket.emit('editfilerequest', filepath, itemid);
- }
- }
- } else if(msg.status == 'finished_metadata') {
- setprogress("100");
- progress_text.text('Finished adding metadata!');
- msg.data["ytid"] = $("#thumbnail_yt").attr('ytid');
- try {
- socket.emit('insertitem', msg.data);
- $("#downloadfilebtn").removeClass('d-none');
- $("#downloadfilebtn").attr('filepath', msg.data["filepath"]);
- } catch (error) {
- console.error(error);
+ }
+ });
+
+ let artists = $("#md_artists").val().split(';');
+ let albumartists = $("#md_album_artists").val().split(';');
+ let metadata = {
+ 'trackid': trackid,
+ 'albumid': albumid,
+ 'title': $("#md_title").val(),
+ 'artists': JSON.stringify(artists),
+ 'album': $("#md_album").val(),
+ 'album_artists': JSON.stringify(albumartists),
+ 'album_tracknr': $("#md_album_tracknr").val(),
+ 'album_releasedate': $("#md_album_releasedate").val(),
+ 'cover': $("#md_cover").val(),
+ 'people': JSON.stringify(people)
+ };
+ socket.emit('mergedata', filepath, release_id, metadata, cover, metadata_source);
+ } else {
+ let itemid = $("#edititemmodal").attr('itemid');
+ socket.emit('editfilerequest', filepath, itemid);
}
- } else if(msg.status == 'metadata_unavailable') {
- msg.data["ytid"] = $("#thumbnail_yt").attr('ytid');
- progress_text.text('Metadata has NOT been added, because metadata is not supported for the selected extension');
- setprogress("100");
+ }
+ });
+
+ socket.on('finished_metadata', function(msg) {
+ setProgress("100");
+ progress_text.text('Finished adding metadata!');
+ msg.data["ytid"] = $("#thumbnail_yt").attr('ytid');
+ try {
+ socket.emit('insertitem', msg.data);
$("#downloadfilebtn").removeClass('d-none');
$("#downloadfilebtn").attr('filepath', msg.data["filepath"]);
- socket.emit('insertitem', msg.data);
- } else if(msg.status == 'error') {
- progress_text.text(msg.message);
- progress.attr('aria-valuenow', 100);
- progress.html('ERROR ');
- progress.css('width', '100%');
- progress_text.text(msg.message);
- if($("#edititemmodal").css('display').toLowerCase() != 'block') {
- $("#resetviewbtn").removeClass('d-none');
- }
- // if($("#edititemmodal").css('display').toLowerCase() == 'block') {
- // $("#progresstextedit").text(msg.message);
- // $("#progressedit").attr('aria-valuenow', 100);
- // $("#progressedit").html('ERROR ');
- // $("#progressedit").css('width', '100%');
- // } else {
- // progress_text.text(msg.message);
- // $("#progress").attr('aria-valuenow', 100);
- // $("#progress").html('ERROR ');
- // $("#progress").css('width', '100%');
- // $("#resetviewbtn").removeClass('d-none');
- // }
+ } catch (error) {
+ console.error(error);
}
});
+
+ socket.on('metadata_unavailable', function(msg) {
+ msg.data["ytid"] = $("#thumbnail_yt").attr('ytid');
+ progress_text.text('Metadata has NOT been added, because metadata is not supported for the selected extension');
+ setProgress("100");
+ $("#downloadfilebtn").removeClass('d-none');
+ $("#downloadfilebtn").attr('filepath', msg.data["filepath"]);
+ socket.emit('insertitem', msg.data);
+ });
+
+ socket.on('downloaderror', function(msg) {
+ progress_text.text(msg.message);
+ progress.attr('aria-valuenow', 100);
+ progress.html('ERROR ');
+ progress.css('width', '100%');
+ progress_text.text(msg.message);
+ if($("#edititemmodal").css('display').toLowerCase() != 'block') {
+ $("#resetviewbtn").removeClass('d-none');
+ }
+ // if($("#edititemmodal").css('display').toLowerCase() == 'block') {
+ // $("#progresstextedit").text(msg.message);
+ // $("#progressedit").attr('aria-valuenow', 100);
+ // $("#progressedit").html('ERROR ');
+ // $("#progressedit").css('width', '100%');
+ // } else {
+ // progress_text.text(msg.message);
+ // $("#progress").attr('aria-valuenow', 100);
+ // $("#progress").html('ERROR ');
+ // $("#progress").css('width', '100%');
+ // $("#resetviewbtn").removeClass('d-none');
+ // }
+ })
socket.on('ytdl_response', (video, downloadform, metadataform) => {
console.info('Got YouTube info');
diff --git a/metatube/youtube.py b/metatube/youtube.py
index 2dd153cf..13912aac 100644
--- a/metatube/youtube.py
+++ b/metatube/youtube.py
@@ -1,14 +1,20 @@
import yt_dlp, json, os
from yt_dlp.postprocessor.ffmpeg import FFmpegPostProcessorError
-from youtubesearchpython.__future__ import VideosSearch
+from youtubesearchpython import VideosSearch
from threading import Thread
from urllib.error import URLError
from yt_dlp.utils import ExtractorError, DownloadError, PostProcessingError
from metatube import sockets, logger, socketio
from metatube.sponsorblock import segments as findsegments
from jinja2 import Environment, PackageLoader, select_autoescape
+import asyncio
+from functools import partial
+from queue import LifoQueue, Empty
+from time import sleep
+
class YouTube:
+ @staticmethod
def is_supported(url):
extractors = yt_dlp.extractor.gen_extractors()
for e in extractors:
@@ -16,6 +22,7 @@ def is_supported(url):
return True
return False
+ @staticmethod
def fetch_url(url, verbose):
if YouTube.is_supported(url):
ytdl_options = {'logger': logger, 'verbose': verbose}
@@ -27,7 +34,8 @@ def fetch_url(url, verbose):
return str(e)
else:
raise ValueError("Invalid URL!")
-
+
+ @staticmethod
def verifytemplate(template, info_dict, verbose):
ytdl_options = {'logger': logger, 'verbose': verbose}
with yt_dlp.YoutubeDL(ytdl_options) as ytdl:
@@ -36,72 +44,96 @@ def verifytemplate(template, info_dict, verbose):
return filename
except Exception as e:
return str(e)
-
- async def search(query):
+
+ @staticmethod
+ def search(query: str):
logger.info('Searching YouTube for \'%s\'', query)
search = VideosSearch(query)
- result = await search.next()
+ result = search.result()
sockets.youtubesearch(result)
- def __download(self, url: list, ytdl_options: dict):
+ @staticmethod
+ async def download(url: list, queue: LifoQueue, ytdl_options: dict):
+ download_hook_partial = partial(YouTube.download_hook, queue)
+ # postprocess_hook_partial = partial(YouTube.postprocessor_hook, queue)
+ ytdl_options['progress_hooks'] = [download_hook_partial]
+ # ytdl_options['postprocessor_hooks'] = [postprocess_hook_partial]
+ # ytdl_options['progress_hooks'] = [YouTube.download_hook]
+ ytdl_options['postprocessor_hooks'] = [YouTube.postprocessor_hook]
with yt_dlp.YoutubeDL(ytdl_options) as ytdl:
try:
- ytdl.download(url)
+ return ytdl.download(url)
except KeyError as e:
logger.error('%s key did not exist', str(e))
- sockets.downloadprogress({'status': 'error', 'message': 'The output template was incorrect. Check logs for more info.'})
+ sockets.downloaderrors({'status': 'error', 'message': 'The output template was incorrect. Check logs for more info.'})
+ return None
except ExtractorError as e:
logger.error('Extractor error: %s', str(e))
- sockets.downloadprogress({'status': 'error', 'message': 'An extractor error has occured. Check logs for more info.'})
+ sockets.downloaderrors({'status': 'error', 'message': 'An extractor error has occured. Check logs for more info.'})
+ return None
except FFmpegPostProcessorError as e:
logger.error('FFmpegPostProcessor error: %s', str(e))
- sockets.downloadprogress({'status': 'error', 'message': 'An processing error involving FFmpeg has occured. Check logs for more info.'})
+ sockets.downloaderrors({'status': 'error', 'message': 'An processing error involving FFmpeg has occured. Check logs for more info.'})
+ return None
except PostProcessingError as e:
logger.error('Postprocessor error: %s', str(e))
- sockets.downloadprogress({'status': 'error', 'message': 'A processing error has occured. Check logs for more info.'})
+ sockets.downloaderrors({'status': 'error', 'message': 'A processing error has occured. Check logs for more info.'})
+ return None
except DownloadError as e:
logger.error('Downloading error: %s', str(e))
- sockets.downloadprogress({'status': 'error', 'message': 'A downloading error has occured. Check logs for more info.'})
+ sockets.downloaderrors({'status': 'error', 'message': 'A downloading error has occured. Check logs for more info.'})
+ return None
except URLError as e:
logger.error('Network connection error: %s', str(e))
- sockets.downloadprogress({'status': 'error', 'message': 'A network error occured. Check logs for more info.'})
+ sockets.downloaderrors({'status': 'error', 'message': 'A network error occured. Check logs for more info.'})
+ return None
except Exception as e:
logger.exception('Error during downloading video: %s', str(e))
- sockets.downloadprogress({'status': 'error', 'message': 'Something has gone wrong. Check logs for more info'})
+ sockets.downloaderrors({'status': 'error', 'message': 'Something has gone wrong. Check logs for more info'})
+ return None
- def download_hook(d):
- if d['status'] == 'finished':
- socketio.emit('downloadprogress', {'status': 'finished_ytdl'})
- # sockets.downloadprogress({'status': 'finished_ytdl'})
- elif d['status'] == 'downloading':
- if "total_bytes_estimate" in d:
- socketio.emit('downloadprogress', {
- 'status': 'downloading',
- 'downloaded_bytes': d['downloaded_bytes'],
- 'total_bytes': d['total_bytes_estimate']
- })
- elif 'total_bytes' in d:
- socketio.emit('downloadprogress', {
- 'status': 'downloading',
- 'downloaded_bytes': d['downloaded_bytes'],
- 'total_bytes': d['total_bytes']
- })
- else:
- socketio.emit('downloadprogress', {
- 'status': 'downloading',
- 'total_bytes': 'Unknown'
- })
-
+ @staticmethod
+ def download_hook(queue: LifoQueue, d):
+ queue.put(d)
+ # if d['status'] == 'finished':
+ # socketio.emit('downloadprogress', {'status': 'finished_ytdl'})
+ # # sockets.downloadprogress({'status': 'finished_ytdl'})
+ # elif d['status'] == 'downloading':
+ # if "total_bytes_estimate" in d:
+ # socketio.emit('downloadprogress', {
+ # 'status': 'downloading',
+ # 'downloaded_bytes': d['downloaded_bytes'],
+ # 'total_bytes': d['total_bytes_estimate']
+ # })
+ # elif 'total_bytes' in d:
+ # socketio.emit('downloadprogress', {
+ # 'status': 'downloading',
+ # 'downloaded_bytes': d['downloaded_bytes'],
+ # 'total_bytes': d['total_bytes']
+ # })
+ # else:
+ # socketio.emit('downloadprogress', {
+ # 'status': 'downloading',
+ # 'total_bytes': 'Unknown'
+ # })
+
+ @staticmethod
+ # def postprocessor_hook(queue: LifoQueue, d):
def postprocessor_hook(d):
- if d['status'] == 'finished':
- socketio.emit('downloadprogress', {
- 'status': 'finished_ffmpeg',
- 'filepath': d['info_dict']['filepath'],
- 'postprocessor': d["postprocessor"]
- })
- logger.info("Finished postprocessor %s", d["postprocessor"])
+ # queue.put(d)
+ if d['status'] == 'processing':
+ sockets.postprocessing(d['postprocessor'])
+ elif d['status'] == 'finished':
+ sockets.finished_postprocessor(d['postprocessor'], d['info_dict']['filepath'])
+ # socketio.emit('downloadprogress', {
+ # 'status': 'finished_ffmpeg',
+ # 'filepath': d['info_dict']['filepath'],
+ # 'postprocessor': d["postprocessor"]
+ # })
+ # logger.info("Finished postprocessor %s", d["postprocessor"])
# sockets.downloadprogress({'status': 'finished_ffmpeg', 'filepath': d['info_dict']['filepath'], 'postprocessor': d["postprocessor"]})
-
+
+ @staticmethod
def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose):
proxy = json.loads(proxy_data)
filepath = os.path.join(output_folder, output_format)
@@ -169,15 +201,13 @@ def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragm
'key': 'ModifyChapters',
'remove_ranges': ranges
})
-
+
ytdl_options = {
'format': format,
'merge_output_format': ext,
'postprocessors': postprocessors,
'postprocessor_args': postprocessor_args,
'ffmpeg_location': ffmpeg,
- 'progress_hooks': [YouTube.download_hook],
- 'postprocessor_hooks': [YouTube.postprocessor_hook],
'logger': logger,
'outtmpl': filepath,
'noplaylist': True,
@@ -193,11 +223,41 @@ def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragm
proxy_string += proxy["proxy_address"].strip() + ":" + proxy["proxy_port"].strip()
ytdl_options["proxy"] = proxy_string
return ytdl_options
-
- def get_video(self, url, ytdl_options):
- # Thread(target=self.__download, args=(url, ytdl_options), name="YouTube-DLP download").start()
- socketio.start_background_task(self.__download, url, ytdl_options)
-
+
+ @staticmethod
+ def start_download(url, ytdl_options):
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ queue = LifoQueue()
+ coros = [YouTube.download(url, queue, ytdl_options)]
+ future = asyncio.gather(*coros)
+ thread = Thread(target=YouTube.loop_in_thread, args=[loop, future])
+ thread.start()
+ # While the future isn't finished yet continue
+ while not future.done():
+ try:
+ # Get the latest status update from the que and print it
+ d = queue.get_nowait()
+ if d['status'] == 'downloading':
+ if "total_bytes_estimate" in d:
+ downloaded_bytes = d['downloaded_bytes'] or 'Unknown'
+ total_bytes = d['total_bytes_estimate'] or d['total_bytes'] or 'Unknown'
+ sockets.downloadprogress(downloaded_bytes, total_bytes)
+ elif d['status'] == 'processing':
+ sockets.postprocessing(d['postprocessor'])
+ elif d['status'] == 'finished':
+ sockets.finished_download()
+ except Empty:
+ pass
+ finally:
+ # Sleep between checking for updates
+ sleep(0.1)
+
+ @staticmethod
+ def loop_in_thread(loop, future):
+ loop.run_until_complete(future)
+
+ @staticmethod
def fetch_video(video, templates, metadata_sources, defaulttemplate):
sb = findsegments(video["webpage_url"])
segments = sb if type(sb) == list else 'error'
From d5d630aefab627c1b126c0f0b0c19b3531811fba Mon Sep 17 00:00:00 2001
From: JVT038 <47184046+JVT038@users.noreply.github.com>
Date: Sun, 28 Jan 2024 15:12:21 +0100
Subject: [PATCH 2/3] Removed some commented code
---
metatube/overview/routes.py | 1 -
metatube/static/JS/overview.js | 21 ---------------------
metatube/youtube.py | 33 ---------------------------------
3 files changed, 55 deletions(-)
diff --git a/metatube/overview/routes.py b/metatube/overview/routes.py
index be91d5d3..548b69d1 100644
--- a/metatube/overview/routes.py
+++ b/metatube/overview/routes.py
@@ -85,7 +85,6 @@ def search(query):
else:
sockets.searchvideo('This video has already been downloaded!')
else:
- # asyncio.run(yt.search(query))
socketio.start_background_task(yt.search, query)
else:
sockets.searchvideo('Enter an URL!')
diff --git a/metatube/static/JS/overview.js b/metatube/static/JS/overview.js
index 3f8e037d..4cf7a586 100644
--- a/metatube/static/JS/overview.js
+++ b/metatube/static/JS/overview.js
@@ -1523,15 +1523,6 @@ $(document).ready(function() {
});
socket.on('finished_postprocessor', function(msg) {
- // if(msg.postprocessor == 'ExtractAudio') {
- // let percentage = (100 / getPhases()) * 2;
- // setProgress(percentage);
- // progress_text.text('Cutting segments from the video... ');
- // } else if(msg.postprocessor == 'ModifyChapters') {
- // let percentage = (100 / getPhases()) * 3;
- // setProgress(percentage);
- // progress_text.text('Moving the files to its destination... ');
- // }
if(msg.postprocessor == 'MoveFiles') {
let percentage = (100 / getPhases()) * (getPhases() - 1);
setProgress(percentage);
@@ -1628,18 +1619,6 @@ $(document).ready(function() {
if($("#edititemmodal").css('display').toLowerCase() != 'block') {
$("#resetviewbtn").removeClass('d-none');
}
- // if($("#edititemmodal").css('display').toLowerCase() == 'block') {
- // $("#progresstextedit").text(msg.message);
- // $("#progressedit").attr('aria-valuenow', 100);
- // $("#progressedit").html('ERROR ');
- // $("#progressedit").css('width', '100%');
- // } else {
- // progress_text.text(msg.message);
- // $("#progress").attr('aria-valuenow', 100);
- // $("#progress").html('ERROR ');
- // $("#progress").css('width', '100%');
- // $("#resetviewbtn").removeClass('d-none');
- // }
})
socket.on('ytdl_response', (video, downloadform, metadataform) => {
diff --git a/metatube/youtube.py b/metatube/youtube.py
index 13912aac..2461981e 100644
--- a/metatube/youtube.py
+++ b/metatube/youtube.py
@@ -55,10 +55,7 @@ def search(query: str):
@staticmethod
async def download(url: list, queue: LifoQueue, ytdl_options: dict):
download_hook_partial = partial(YouTube.download_hook, queue)
- # postprocess_hook_partial = partial(YouTube.postprocessor_hook, queue)
ytdl_options['progress_hooks'] = [download_hook_partial]
- # ytdl_options['postprocessor_hooks'] = [postprocess_hook_partial]
- # ytdl_options['progress_hooks'] = [YouTube.download_hook]
ytdl_options['postprocessor_hooks'] = [YouTube.postprocessor_hook]
with yt_dlp.YoutubeDL(ytdl_options) as ytdl:
try:
@@ -95,43 +92,13 @@ async def download(url: list, queue: LifoQueue, ytdl_options: dict):
@staticmethod
def download_hook(queue: LifoQueue, d):
queue.put(d)
- # if d['status'] == 'finished':
- # socketio.emit('downloadprogress', {'status': 'finished_ytdl'})
- # # sockets.downloadprogress({'status': 'finished_ytdl'})
- # elif d['status'] == 'downloading':
- # if "total_bytes_estimate" in d:
- # socketio.emit('downloadprogress', {
- # 'status': 'downloading',
- # 'downloaded_bytes': d['downloaded_bytes'],
- # 'total_bytes': d['total_bytes_estimate']
- # })
- # elif 'total_bytes' in d:
- # socketio.emit('downloadprogress', {
- # 'status': 'downloading',
- # 'downloaded_bytes': d['downloaded_bytes'],
- # 'total_bytes': d['total_bytes']
- # })
- # else:
- # socketio.emit('downloadprogress', {
- # 'status': 'downloading',
- # 'total_bytes': 'Unknown'
- # })
@staticmethod
- # def postprocessor_hook(queue: LifoQueue, d):
def postprocessor_hook(d):
- # queue.put(d)
if d['status'] == 'processing':
sockets.postprocessing(d['postprocessor'])
elif d['status'] == 'finished':
sockets.finished_postprocessor(d['postprocessor'], d['info_dict']['filepath'])
- # socketio.emit('downloadprogress', {
- # 'status': 'finished_ffmpeg',
- # 'filepath': d['info_dict']['filepath'],
- # 'postprocessor': d["postprocessor"]
- # })
- # logger.info("Finished postprocessor %s", d["postprocessor"])
- # sockets.downloadprogress({'status': 'finished_ffmpeg', 'filepath': d['info_dict']['filepath'], 'postprocessor': d["postprocessor"]})
@staticmethod
def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose):
From 7dd27c3409d6c3b128788af9243492b89044e8f4 Mon Sep 17 00:00:00 2001
From: JVT038 <47184046+JVT038@users.noreply.github.com>
Date: Sun, 28 Jan 2024 17:08:26 +0100
Subject: [PATCH 3/3] Refactoring some stuff
---
README.md | 16 ++---
cli_to_api.py | 51 ++++++++++++++
metatube/overview/routes.py | 33 +++++----
metatube/static/JS/overview.js | 124 ++++++++++++++++++---------------
metatube/youtube.py | 20 +++++-
requirements.txt | 10 +--
6 files changed, 165 insertions(+), 89 deletions(-)
create mode 100644 cli_to_api.py
diff --git a/README.md b/README.md
index 8e764761..7020e09e 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,3 @@
-
MetaTube
@@ -265,8 +258,9 @@ Made with :heart: by JVT038<
- [X] Dark mode support
- [X] Fix error `Synchronous XMLHttpRequest on the main thread is deprecated because of its detrimental effects to the end user’s experience. For more help http://xhr.spec.whatwg.org/` in overview
- [X] Make sure the search for downloaded song field works
+- [x] Make sure the progress bar works properly in a Docker container, because it doesn't work properly rn
-### Not finished (I'll never finish this)
+### Not finished (I'll probably never finish this lol)
- [ ] Add it to the PyPi library
- [ ] Add support for sites other than YouTube
@@ -289,8 +283,10 @@ Made with :heart: by JVT038<
- [ ] Have a proper versioning system, because it's impossible to keep track of versions rn
- [ ] Cache and store the segments and other video data, so next time of loading a video will be faster
- [ ] Send websocket requests to one specific device / client only, to prevent duplicate websocket requests
-- [ ] Make sure the progress bar works properly in a Docker container, because it doesn't work properly rn.
-- [ ] Use proper queues and threading during download instead of the weird ping-pong system between the client and the server.
+- [ ] Use proper queues and threading during download instead of the weird ping-pong system between the client and the server.*
+- [ ] Add unit tests for the download, metadata logic, template / database stuff, config detection, automatic migrations
+
+* in progress
## Disclaimer
diff --git a/cli_to_api.py b/cli_to_api.py
new file mode 100644
index 00000000..89958b84
--- /dev/null
+++ b/cli_to_api.py
@@ -0,0 +1,51 @@
+'''
+Source: https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py
+Allow direct execution
+'''
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import yt_dlp
+import yt_dlp.options
+
+create_parser = yt_dlp.options.create_parser
+
+
+def parse_patched_options(opts):
+ patched_parser = create_parser()
+ patched_parser.defaults.update({
+ 'ignoreerrors': False,
+ 'retries': 0,
+ 'fragment_retries': 0,
+ 'extract_flat': False,
+ 'concat_playlist': 'never',
+ })
+ yt_dlp.options.create_parser = lambda: patched_parser
+ try:
+ return yt_dlp.parse_options(opts)
+ finally:
+ yt_dlp.options.create_parser = create_parser
+
+
+default_opts = parse_patched_options([]).ydl_opts
+
+
+def cli_to_api(opts, cli_defaults=False):
+ opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
+
+ diff = {k: v for k, v in opts.items() if default_opts[k] != v}
+ if 'postprocessors' in diff:
+ diff['postprocessors'] = [pp for pp in diff['postprocessors']
+ if pp not in default_opts['postprocessors']]
+ return diff
+
+
+if __name__ == '__main__':
+ from pprint import pprint
+
+ print('\nThe arguments passed translate to:\n')
+ pprint(cli_to_api(sys.argv[1:]))
+ print('\nCombining these with the CLI defaults gives:\n')
+ pprint(cli_to_api(sys.argv[1:], True))
\ No newline at end of file
diff --git a/metatube/overview/routes.py b/metatube/overview/routes.py
index 548b69d1..98d13863 100644
--- a/metatube/overview/routes.py
+++ b/metatube/overview/routes.py
@@ -1,3 +1,4 @@
+from platform import release
import shutil
from magic import Magic
from metatube.overview import bp
@@ -112,24 +113,24 @@ def searchmetadata(data):
socketio.start_background_task(Genius.searchsong, data, token)
@socketio.on('ytdl_download')
-def download(data):
- url = data["url"]
- ext = data["ext"] or 'mp3'
- output_folder = data["output_folder"] or '/downloads'
- output_type = data["type"] or 'Audio'
- output_format = data["output_format"] or f'%(title)s.%(ext)s'
- bitrate = data["bitrate"] or '192'
- skipfragments = data["skipfragments"] or {}
- proxy_data = data["proxy_data"] or {'proxy_type': 'None'}
+def download(fileData, metadata):
+ url = fileData["url"]
+ ext = fileData["ext"] or 'mp3'
+ output_folder = fileData["output_folder"] or '/downloads'
+ output_type = fileData["type"] or 'Audio'
+ output_format = fileData["output_format"] or f'%(title)s.%(ext)s'
+ bitrate = fileData["bitrate"] or '192'
+ skipfragments = fileData["skipfragments"] or {}
+ proxy_data = fileData["proxy_data"] or {'proxy_type': 'None'}
- width = data["width"] or 1920
- height = data["height"] or 1080
+ width = fileData["width"] or 1920
+ height = fileData["height"] or 1080
ffmpeg = Config.get_ffmpeg()
hw_transcoding = Config.get_hwt()
vaapi_device = hw_transcoding.split(';')[1] if 'vaapi' in hw_transcoding else ''
verbose = strtobool(str(env.LOGGER))
- logger.info('Request to download %s', data["url"])
- ytdl_options = yt.get_options(url, ext, output_folder, output_type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose)
+ logger.info('Request to download %s', fileData["url"])
+ ytdl_options = yt.get_options(ext, output_folder, output_type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose, metadata)
if ytdl_options is not False:
socketio.start_background_task(yt.start_download, url, ytdl_options)
# socketio.start_background_task(yt.download, url, ytdl_options)
@@ -182,7 +183,11 @@ def fetchgeniusalbum(input_id):
genius.fetchalbum(input_id)
@socketio.on('mergedata')
-def mergedata(filepath, release_id, metadata, cover, source):
+def mergedata(metadata, filepath):
+ release_id = metadata["release_id"]
+ cover = metadata["cover"]
+ source = metadata["metadata_source"]
+
if Database.checktrackid(release_id) is None and Database.checktrackid(metadata.get('trackid', '')) is None:
metadata_user = metadata
diff --git a/metatube/static/JS/overview.js b/metatube/static/JS/overview.js
index 4cf7a586..fe249b38 100644
--- a/metatube/static/JS/overview.js
+++ b/metatube/static/JS/overview.js
@@ -700,13 +700,72 @@ $(document).ready(function() {
$("tr#" + rowid).find('.finditembtn').remove();
}
+ function getMetadata() {
+ let release_id = $(".audiocol-checkbox:checked").parent().parent().attr('id');
+ let people = {};
+ let metadata_source = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').find('span.metadatasource').text() : "Unavailable";
+ let cover = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').children('img').attr('src') : "Unavailable";
+
+ if(metadata_source == 'Unavailable') {
+ // The priority order is: Spotify -> Deezer -> Musibrainz
+ var trackid = $("#spotify_trackid").length > 0 ? $("#spotify_trackid").val() : ($("#deezer_releaseid").val().length > 0 ? $("#deezer_trackid").val() : $("#mbp_trackid").val());
+ var albumid = $("#spotify_albumid").length > 0 ? $("#spotify_albumid").val() : ($("#deezer_albumid").val().length > 0 ? $("#deezer_albumid").val() : $("#mbp_albumid").val());
+ } else if(metadata_source == 'Spotify') {
+ var trackid = $("#spotify_trackid").val();
+ var albumid = $("#spotify_albumid").val();
+ } else if(metadata_source == 'Musicbrainz') {
+ var trackid = $("#mbp_releaseid").val();
+ var albumid = $("#mbp_albumid").val();
+ } else if(metadata_source == 'Deezer') {
+ var trackid = $("#deezer_trackid").val();
+ var albumid = $("#deezer_albumid").val();
+ } else if(metadata_source == 'Genius') {
+ var trackid = $("#genius_songid").val();
+ }
+
+ $.each($('.artist_relations'), function() {
+ if($(this).val().trim().length < 1 || $(this).parent().siblings().find('.artist_relations').val().trim().length < 1) {
+ return;
+ } else {
+ // Get ID by removing all letters from the ID, so the number remains
+ let id = $(this).parents('.personrow').attr('id').replace(/[a-zA-Z]/g, '');
+ if(this.id.replace(/[0-9]/g, '') == 'artist_relations_name') {
+ people[id].name = $(this).val();
+ } else {
+ people[id].type = $(this).val();
+ }
+ }
+ });
+
+ let artists = $("#md_artists").val().split(';');
+ let albumartists = $("#md_album_artists").val().split(';');
+ return {
+ 'trackid': trackid,
+ 'albumid': albumid,
+ 'title': $("#md_title").val(),
+ 'artists': JSON.stringify(artists),
+ 'album': $("#md_album").val(),
+ 'album_artists': JSON.stringify(albumartists),
+ 'album_tracknr': $("#md_album_tracknr").val(),
+ 'album_releasedate': $("#md_album_releasedate").val(),
+ 'cover': $("#md_cover").val(),
+ 'people': JSON.stringify(people),
+ 'cover': cover,
+ 'release_id': release_id,
+ 'metadata_source': metadata_source
+ };
+ }
+
function getPhases() {
return $("#segments_check").is(':checked') ? 4 : 5;
}
+ function getProgress() {
+ return $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progressedit") : $("#progress");
+ }
+
function setProgress(percentage) {
- let progress = $("#edititemmodal").css('display').toLowerCase() != 'none' ? $("#progressedit") : $("#progress");
- progress.attr({
+ getProgress().attr({
'aria-valuenow': percentage + "%",
'style': 'width: ' + parseInt(percentage) + '%'
}).text(percentage + "%");
@@ -1402,7 +1461,7 @@ $(document).ready(function() {
'width': width,
'height': height
}
- socket.emit('ytdl_download', data, function(ack) {
+ socket.emit('ytdl_download', data, getMetadata(), function(ack) {
if(ack == "OK") {
$("#editmetadata, #downloadbtn, #searchmetadataview, #404p, #defaultview, #resetviewbtn, #geniusbtn, #audiocol, #savemetadata, #metadataview, #geniuscol").addClass('d-none');
$("#progressview").removeClass('d-none');
@@ -1529,58 +1588,7 @@ $(document).ready(function() {
progress_text.text('Adding metadata...');
var filepath = msg.filepath;
if($("#edititemmodal").css('display').toLowerCase() == 'none') {
-
- let release_id = $(".audiocol-checkbox:checked").parent().parent().attr('id');
- let people = {};
- let metadata_source = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').find('span.metadatasource').text() : "Unavailable";
- let cover = $("#audiocol").length > 0 ? $(".audiocol-checkbox:checked").parents('li').children('img').attr('src') : "Unavailable";
-
- if(metadata_source == 'Unavailable') {
- // The priority order is: Spotify -> Deezer -> Musibrainz
- var trackid = $("#spotify_trackid").length > 0 ? $("#spotify_trackid").val() : ($("#deezer_releaseid").val().length > 0 ? $("#deezer_trackid").val() : $("#mbp_trackid").val());
- var albumid = $("#spotify_albumid").length > 0 ? $("#spotify_albumid").val() : ($("#deezer_albumid").val().length > 0 ? $("#deezer_albumid").val() : $("#mbp_albumid").val());
- } else if(metadata_source == 'Spotify') {
- var trackid = $("#spotify_trackid").val();
- var albumid = $("#spotify_albumid").val();
- } else if(metadata_source == 'Musicbrainz') {
- var trackid = $("#mbp_releaseid").val();
- var albumid = $("#mbp_albumid").val();
- } else if(metadata_source == 'Deezer') {
- var trackid = $("#deezer_trackid").val();
- var albumid = $("#deezer_albumid").val();
- } else if(metadata_source == 'Genius') {
- var trackid = $("#genius_songid").val();
- }
-
- $.each($('.artist_relations'), function() {
- if($(this).val().trim().length < 1 || $(this).parent().siblings().find('.artist_relations').val().trim().length < 1) {
- return;
- } else {
- // Get ID by removing all letters from the ID, so the number remains
- let id = $(this).parents('.personrow').attr('id').replace(/[a-zA-Z]/g, '');
- if(this.id.replace(/[0-9]/g, '') == 'artist_relations_name') {
- people[id].name = $(this).val();
- } else {
- people[id].type = $(this).val();
- }
- }
- });
-
- let artists = $("#md_artists").val().split(';');
- let albumartists = $("#md_album_artists").val().split(';');
- let metadata = {
- 'trackid': trackid,
- 'albumid': albumid,
- 'title': $("#md_title").val(),
- 'artists': JSON.stringify(artists),
- 'album': $("#md_album").val(),
- 'album_artists': JSON.stringify(albumartists),
- 'album_tracknr': $("#md_album_tracknr").val(),
- 'album_releasedate': $("#md_album_releasedate").val(),
- 'cover': $("#md_cover").val(),
- 'people': JSON.stringify(people)
- };
- socket.emit('mergedata', filepath, release_id, metadata, cover, metadata_source);
+ socket.emit('mergedata', getMetadata(), filepath);
} else {
let itemid = $("#edititemmodal").attr('itemid');
socket.emit('editfilerequest', filepath, itemid);
@@ -1612,9 +1620,9 @@ $(document).ready(function() {
socket.on('downloaderror', function(msg) {
progress_text.text(msg.message);
- progress.attr('aria-valuenow', 100);
- progress.html('ERROR ');
- progress.css('width', '100%');
+ getProgress().attr('aria-valuenow', 100);
+ getProgress().html('ERROR ');
+ getProgress().css('width', '100%');
progress_text.text(msg.message);
if($("#edititemmodal").css('display').toLowerCase() != 'block') {
$("#resetviewbtn").removeClass('d-none');
diff --git a/metatube/youtube.py b/metatube/youtube.py
index 2461981e..d79dc482 100644
--- a/metatube/youtube.py
+++ b/metatube/youtube.py
@@ -1,10 +1,11 @@
import yt_dlp, json, os
from yt_dlp.postprocessor.ffmpeg import FFmpegPostProcessorError
+from yt_dlp.postprocessor.metadataparser import MetadataParserPP
from youtubesearchpython import VideosSearch
from threading import Thread
from urllib.error import URLError
from yt_dlp.utils import ExtractorError, DownloadError, PostProcessingError
-from metatube import sockets, logger, socketio
+from metatube import sockets, logger
from metatube.sponsorblock import segments as findsegments
from jinja2 import Environment, PackageLoader, select_autoescape
import asyncio
@@ -101,7 +102,7 @@ def postprocessor_hook(d):
sockets.finished_postprocessor(d['postprocessor'], d['info_dict']['filepath'])
@staticmethod
- def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose):
+ def get_options(ext, output_folder, type, output_format, bitrate, skipfragments, proxy_data, ffmpeg, hw_transcoding, vaapi_device, width, height, verbose, metadata):
proxy = json.loads(proxy_data)
filepath = os.path.join(output_folder, output_format)
segments = json.loads(skipfragments)
@@ -168,6 +169,21 @@ def get_options(url, ext, output_folder, type, output_format, bitrate, skipfragm
'key': 'ModifyChapters',
'remove_ranges': ranges
})
+
+ '''
+ --parse-metadata example in CLI:
+ yt-dlp orJSJGHjBLI -x --audio-format mp3 --add-metadata -o "%(track,title)s - %(artist)s.%(ext)s" --parse-metadata " Bad Habits: %(title)s" --parse-metadata "Ed Sheeran:%(artist)s"
+ '''
+ # postprocessors.append({
+ # 'actions': [
+ # (MetadataParserPP.interpretter, " " + metadata['title'], ' %(title)s'),
+ # (MetadataParserPP.interpretter, metadata['album'], '%(album)s'),
+ # (MetadataParserPP.interpretter, ';'.join(json.loads(metadata['album_artists'])), '%(album_artist)s'),
+ # (MetadataParserPP.interpretter, metadata['album_tracknr'], '%(track_number)s'),
+ # ],
+ # 'key': 'MetadataParser',
+ # 'when': 'pre_process'
+ # })
ytdl_options = {
'format': format,
diff --git a/requirements.txt b/requirements.txt
index a0faae64..6b084cd0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,20 +1,20 @@
mutagen==1.47.0
requests==2.31.0
-yt-dlp==2023.11.16
+yt-dlp==2023.12.30
gevent==23.9.1
gevent-websocket==0.10.1
-Flask==3.0.0
+Flask==3.0.1
Flask-SocketIO==5.3.6
-Flask-Migrate==4.0.4
+Flask-Migrate==4.0.5
Flask-SQLAlchemy==3.1.1
musicbrainzngs==0.7.1
sponsorblock.py==0.2.2
python-dateutil==2.8.2
-python-dotenv==1.0.0
+python-dotenv==1.0.1
python-magic==0.4.27
ffmpeg-python==0.2.0
youtube-search-python==1.6.6
spotipy==2.23.0
urllib3==2.1.0
-deezer-python==5.8.1
+deezer-python==6.1.1
lyricsgenius==3.0.1
\ No newline at end of file