diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ac2a8b1..b24b3ae 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,10 +5,15 @@ on: branches: - feature/* - master + pull_request: + branches: + - master jobs: tests: uses: nzbgetcom/nzbget-extensions/.github/workflows/python-tests.yml@main with: - python-versions: "3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12" - supported-python-versions: "3.9 3.10 3.11 3.12" + python-versions: "3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12" + supported-python-versions: "3.8 3.9 3.10 3.11 3.12" + test-script: tests.py + debug: true diff --git a/.gitignore b/.gitignore index d9630b4..b68009a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ .vscode __ __pycache__ +tmp diff --git a/README.md b/README.md index 29fdd25..b5e3aa9 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,8 @@ ## NZBGet Versions -- pre-release v23+ [v3.0](https://github.com/nzbgetcom/Extension-FakeDetector/releases/tag/v3.0) -- stable v22 [v2.0](https://github.com/nzbgetcom/Extension-FakeDetector/releases/tag/v2.0) -- legacy v21 [v2.0](https://github.com/nzbgetcom/Extension-FakeDetector/releases/tag/v2.0) +- stable v23+ [v3.1](https://github.com/nzbgetcom/Extension-FakeDetector/releases/tag/v3.1) +- legacy v22 [v2.0](https://github.com/nzbgetcom/Extension-FakeDetector/releases/tag/v2.0) > **Note:** This script is compatible with python 3.8.x and above. If you need support for Python 2.x or older Python3.x versions please use [v1.7](https://github.com/nzbgetcom/Extension-FakeDetector/releases/tag/v1.7) release. diff --git a/main.py b/main.py index a639e70..b1f41f8 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ # Copyright (C) 2014-2016 Andrey Prygunkov # Copyright (C) 2014 Clinton Hall # Copyright (C) 2014 JVM +# Copyright (C) 2024 Denis # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,390 +33,450 @@ import shlex import traceback +sys.stdout.reconfigure(encoding="utf-8") + # Exit codes used by NZBGet for post-processing scripts. # Queue-scripts don't have any special exit codes. -POSTPROCESS_SUCCESS=93 -POSTPROCESS_NONE=95 -POSTPROCESS_ERROR=94 - -mediaExtensions = ['.mkv', '.avi', '.divx', '.xvid', '.mov', '.wmv', '.mp4', '.mpg', '.mpeg', '.vob', '.iso', '.m4v'] -bannedMediaExtensions = os.environ.get('NZBPO_BANNEDEXTENSIONS').replace(' ', '').split(',') +POSTPROCESS_SUCCESS = 93 +POSTPROCESS_NONE = 95 +POSTPROCESS_ERROR = 94 + +mediaExtensions = [ + ".mkv", + ".avi", + ".divx", + ".xvid", + ".mov", + ".wmv", + ".mp4", + ".mpg", + ".mpeg", + ".vob", + ".iso", + ".m4v", +] +bannedMediaExtensions = ( + os.environ.get("NZBPO_BANNEDEXTENSIONS").replace(" ", "").split(",") +) verbose = False + # Start up checks def start_check(): - # Check if the script is called from a compatible NZBGet version (as queue-script or as pp-script) - if not ('NZBNA_EVENT' in os.environ or 'NZBPP_DIRECTORY' in os.environ) or not 'NZBOP_ARTICLECACHE' in os.environ: - print('*** NZBGet queue script ***') - print('This script is supposed to be called from nzbget (14.0 or later).') - sys.exit(1) - - # This script processes only certain queue events. - # For compatibility with newer NZBGet versions it ignores event types it doesn't know - if os.environ.get('NZBNA_EVENT') not in ['NZB_ADDED', 'FILE_DOWNLOADED', 'NZB_DOWNLOADED', None]: - sys.exit(0) - - # If nzb was already marked as bad don't do any further detection - if os.environ.get('NZBPP_STATUS') == 'FAILURE/BAD': - if os.environ.get('NZBPR_PPSTATUS_FAKE') == 'yes': - # Print the message again during post-processing to add it into the post-processing log - # (which is then can be used by notification scripts such as EMail.py) - # Pp-parameter "NZBPR_PPSTATUS_FAKEBAN" contains more details (saved previously by our script) - if os.environ.get('NZBPR_PPSTATUS_FAKEBAN') == None: - print('[WARNING] Download has media files and executables') - else: - print('[WARNING] Download contains banned extension ' + os.environ.get('NZBPR_PPSTATUS_FAKEBAN')) - clean_up() - sys.exit(POSTPROCESS_SUCCESS) - - # If called via "Post-process again" from history details dialog the download may not exist anymore - if 'NZBPP_DIRECTORY' in os.environ and not os.path.exists(os.environ.get('NZBPP_DIRECTORY')): - print('Destination directory doesn\'t exist, exiting') - clean_up() - sys.exit(POSTPROCESS_NONE) - - # If nzb is already failed, don't do any further detection - if os.environ.get('NZBPP_TOTALSTATUS') == 'FAILURE': - clean_up() - sys.exit(POSTPROCESS_NONE) + # Check if the script is called from a compatible NZBGet version (as queue-script or as pp-script) + if ( + not ("NZBNA_EVENT" in os.environ or "NZBPP_DIRECTORY" in os.environ) + or not "NZBOP_ARTICLECACHE" in os.environ + ): + print("*** NZBGet queue script ***") + print("This script is supposed to be called from nzbget (14.0 or later).") + sys.exit(1) + + # This script processes only certain queue events. + # For compatibility with newer NZBGet versions it ignores event types it doesn't know + if os.environ.get("NZBNA_EVENT") not in [ + "NZB_ADDED", + "FILE_DOWNLOADED", + "NZB_DOWNLOADED", + None, + ]: + sys.exit(0) + + # If nzb was already marked as bad don't do any further detection + if os.environ.get("NZBPP_STATUS") == "FAILURE/BAD": + if os.environ.get("NZBPR_PPSTATUS_FAKE") == "yes": + # Print the message again during post-processing to add it into the post-processing log + # (which is then can be used by notification scripts such as EMail.py) + # Pp-parameter "NZBPR_PPSTATUS_FAKEBAN" contains more details (saved previously by our script) + if os.environ.get("NZBPR_PPSTATUS_FAKEBAN") == None: + print("[WARNING] Download has media files and executables") + else: + print( + "[WARNING] Download contains banned extension " + + os.environ.get("NZBPR_PPSTATUS_FAKEBAN") + ) + clean_up() + sys.exit(POSTPROCESS_SUCCESS) + + # If called via "Post-process again" from history details dialog the download may not exist anymore + if "NZBPP_DIRECTORY" in os.environ and not os.path.exists( + os.environ.get("NZBPP_DIRECTORY") + ): + print("Destination directory doesn't exist, exiting") + clean_up() + sys.exit(POSTPROCESS_NONE) + + # If nzb is already failed, don't do any further detection + if os.environ.get("NZBPP_TOTALSTATUS") == "FAILURE": + clean_up() + sys.exit(POSTPROCESS_NONE) + # Check if media files present in the list of files def contains_media(list): - for item in list: - if os.path.splitext(item)[1] in mediaExtensions: - return True - else: - continue - return False + for item in list: + if os.path.splitext(item)[1] in mediaExtensions: + return True + else: + continue + return False + # Check if banned media files present in the list of files def contains_banned_media(list): - for item in list: - if os.path.splitext(item)[1] in bannedMediaExtensions: - print('[INFO] Found file with banned extension: ' + item) - return os.path.splitext(item)[1] - else: - continue - return '' + for item in list: + if os.path.splitext(item)[1] in bannedMediaExtensions: + print("[INFO] Found file with banned extension: " + item) + return os.path.splitext(item)[1] + else: + continue + return "" + # Check if executable files present in the list of files # Exception: rename.bat (.sh, .exe) are ignored, sometimes valid posts include them. def contains_executable(list): - exExtensions = [ '.exe', '.bat', '.sh' ] - allowNames = [ 'rename', 'Rename' ] - excludePath = [ r'reverse', r'spiegelen' ] - for item in list: - ep = False - for ap in excludePath: - if re.search(ap, item, re.I): - ep = True - break - if ep: - continue - name, ext = os.path.splitext(item) - if os.path.split(name)[1] != "": - name = os.path.split(name)[1] - if ext == '.exe' or (ext in exExtensions and not name in allowNames): - print('[INFO] Found executable %s' % item) - return True - else: - continue - return False + exExtensions = [".exe", ".bat", ".sh"] + allowNames = ["rename", "Rename"] + excludePath = [r"reverse", r"spiegelen"] + for item in list: + ep = False + for ap in excludePath: + if re.search(ap, item, re.I): + ep = True + break + if ep: + continue + name, ext = os.path.splitext(item) + if os.path.split(name)[1] != "": + name = os.path.split(name)[1] + if ext == ".exe" or (ext in exExtensions and not name in allowNames): + print("[INFO] Found executable %s" % item) + return True + else: + continue + return False + # Finds untested files, comparing all files and processed files in tmp_file def get_latest_file(dir): - try: - with open(tmp_file_name) as tmp_file: - tested = tmp_file.read().splitlines() - files = os.listdir(dir) - return list(set(files)-set(tested)) - except: - # tmp_file doesn't exist, all files need testing - temp_folder = os.path.dirname(tmp_file_name) - if not os.path.exists(temp_folder): - os.makedirs(temp_folder) - print('[DETAIL] Created folder ' + temp_folder) - with open(tmp_file_name, "w") as tmp_file: - tmp_file.write('') - print('[DETAIL] Created temp file ' + tmp_file_name) - return os.listdir(dir) + try: + with open(tmp_file_name) as tmp_file: + tested = tmp_file.read().splitlines() + files = os.listdir(dir) + return list(set(files) - set(tested)) + except: + # tmp_file doesn't exist, all files need testing + temp_folder = os.path.dirname(tmp_file_name) + if not os.path.exists(temp_folder): + os.makedirs(temp_folder) + print("[DETAIL] Created folder " + temp_folder) + with open(tmp_file_name, "w") as tmp_file: + tmp_file.write("") + print("[DETAIL] Created temp file " + tmp_file_name) + return os.listdir(dir) + # Saves tested files so to not test again def save_tested(data): - with open(tmp_file_name, "a") as tmp_file: - tmp_file.write(data) + with open(tmp_file_name, "a") as tmp_file: + tmp_file.write(data) + # Extract path to unrar from NZBGet's global option "UnrarCmd"; # Since v15 "UnrarCmd" may contain extra parameters passed to unrar; # We have to strip these parameters because we need only the path to unrar. # Returns path to unrar executable. def unrar(): - exe_name = 'unrar.exe' if os.name == 'nt' else 'unrar' - UnrarCmd = os.environ['NZBOP_UNRARCMD'] - if os.path.isfile(UnrarCmd) and UnrarCmd.lower().endswith(exe_name): - return UnrarCmd - args = shlex.split(UnrarCmd) - for arg in args: - if arg.lower().endswith(exe_name): - return arg - # We were unable to determine the path to unrar; - # Let's use the exe name with a hope it's in the search path - return exe_name + exe_name = "unrar.exe" if os.name == "nt" else "unrar" + UnrarCmd = os.environ["NZBOP_UNRARCMD"] + if os.path.isfile(UnrarCmd) and UnrarCmd.lower().endswith(exe_name): + return UnrarCmd + args = shlex.split(UnrarCmd) + for arg in args: + if arg.lower().endswith(exe_name): + return arg + # We were unable to determine the path to unrar; + # Let's use the exe name with a hope it's in the search path + return exe_name + # List contents of rar-files (without unpacking). # That's how we detect fakes during download, when the download is not completed yet. def list_all_rars(dir): - files = get_latest_file(dir) - tested = '' - out = '' - for file in files: - # avoid .tmp files as corrupt - if not "tmp" in file: - try: - command = [unrar(), "vb", dir + '/' + file] - if verbose: - print('command: %s' % command) - proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out_tmp, err = proc.communicate() - out += out_tmp.decode() - result = proc.returncode - if verbose: - print(out_tmp) - except Exception as e: - print('[ERROR] Failed %s: %s' % (file, e)) - if verbose: - traceback.print_exc() - tested += file + '\n' - save_tested(tested) - return out.splitlines() + files = get_latest_file(dir) + tested = "" + out = "" + for file in files: + # avoid .tmp files as corrupt + if not "tmp" in file: + try: + command = [unrar(), "vb", dir + "/" + file] + if verbose: + print("command: %s" % command) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + out_tmp, err = proc.communicate() + out += out_tmp.decode() + result = proc.returncode + if verbose: + print(out_tmp) + except Exception as e: + print("[ERROR] Failed %s: %s" % (file, e)) + if verbose: + traceback.print_exc() + tested += file + "\n" + save_tested(tested) + return out.splitlines() + # Detect fake nzbs. Returns True if a fake is detected. def detect_fake(name, dir): - # Fake detection: - # If download contains media files AND executables we consider it a fake. - # QUEUE mode (called during download and before unpack): - # - if directory contains archives list their content and use the file - # names for detection; - # POST-PROCESSING mode (called after unpack): - # - scan directroy content and use file names for detection; - # - TODO: check video files using ffprobe. - # - # It's actually not necessary to check the mode (QUEUE or POST-PROCESSING), we always do all checks. - - filelist = [] - dir = os.path.normpath(dir) - filelist.extend([ o for o in os.listdir(dir) if os.path.isfile(os.path.join(dir, o)) ]) - dirlist = [ os.path.join(dir, o) for o in os.listdir(dir) if os.path.isdir(os.path.join(dir, o)) ] - filelist.extend(list_all_rars(dir)) - for subdir in dirlist: - filelist.extend(list_all_rars(subdir)) - if contains_media(filelist) and contains_executable(filelist): - print('[WARNING] Download has media files and executables') - # Remove info about banned extension from pp-parameter "NZBPR_PPSTATUS_FAKEBAN" - # (in a case it was saved previously) - print('[NZB] NZBPR_PPSTATUS_FAKEBAN=') - return True - banned_ext = contains_banned_media(filelist) - if banned_ext != '': - print('[WARNING] Download contains banned extension ' + banned_ext) - # Save details about banned extension in pp-parameter "NZBPR_PPSTATUS_FAKEBAN" - print('[NZB] NZBPR_PPSTATUS_FAKEBAN=' + banned_ext) - return True - return False + # Fake detection: + # If download contains media files AND executables we consider it a fake. + # QUEUE mode (called during download and before unpack): + # - if directory contains archives list their content and use the file + # names for detection; + # POST-PROCESSING mode (called after unpack): + # - scan directroy content and use file names for detection; + # - TODO: check video files using ffprobe. + # + # It's actually not necessary to check the mode (QUEUE or POST-PROCESSING), we always do all checks. + + filelist = [] + dir = os.path.normpath(dir) + filelist.extend( + [o for o in os.listdir(dir) if os.path.isfile(os.path.join(dir, o))] + ) + dirlist = [ + os.path.join(dir, o) + for o in os.listdir(dir) + if os.path.isdir(os.path.join(dir, o)) + ] + filelist.extend(list_all_rars(dir)) + for subdir in dirlist: + filelist.extend(list_all_rars(subdir)) + if contains_media(filelist) and contains_executable(filelist): + print("[WARNING] Download has media files and executables") + # Remove info about banned extension from pp-parameter "NZBPR_PPSTATUS_FAKEBAN" + # (in a case it was saved previously) + print("[NZB] NZBPR_PPSTATUS_FAKEBAN=") + return True + banned_ext = contains_banned_media(filelist) + if banned_ext != "": + print("[WARNING] Download contains banned extension " + banned_ext) + # Save details about banned extension in pp-parameter "NZBPR_PPSTATUS_FAKEBAN" + print("[NZB] NZBPR_PPSTATUS_FAKEBAN=" + banned_ext) + return True + return False + # Establish connection to NZBGet via RPC-API def connect_to_nzbget(): - # First we need to know connection info: host, port and password of NZBGet server. - # NZBGet passes all configuration options to scripts as environment variables. - host = os.environ['NZBOP_CONTROLIP'] - if host == '0.0.0.0': host = '127.0.0.1' - port = os.environ['NZBOP_CONTROLPORT'] - username = os.environ['NZBOP_CONTROLUSERNAME'] - password = os.environ['NZBOP_CONTROLPASSWORD'] - - # Build an URL for XML-RPC requests - # TODO: encode username and password in URL-format - xmlRpcUrl = 'http://%s:%s@%s:%s/xmlrpc' % (username, password, host, port) - - # Create remote server object - nzbget = ServerProxy(xmlRpcUrl) - return nzbget + # First we need to know connection info: host, port and password of NZBGet server. + # NZBGet passes all configuration options to scripts as environment variables. + host = os.environ["NZBOP_CONTROLIP"] + if host == "0.0.0.0": + host = "127.0.0.1" + port = os.environ["NZBOP_CONTROLPORT"] + username = os.environ["NZBOP_CONTROLUSERNAME"] + password = os.environ["NZBOP_CONTROLPASSWORD"] + + # Build an URL for XML-RPC requests + # TODO: encode username and password in URL-format + xmlRpcUrl = "http://%s:%s@%s:%s/xmlrpc" % (username, password, host, port) + + # Create remote server object + nzbget = ServerProxy(xmlRpcUrl) + return nzbget + # Connect to NZBGet and call an RPC-API-method without using of python's XML-RPC. # XML-RPC is easy to use but it is slow for large amount of data def call_nzbget_direct(url_command): - # First we need to know connection info: host, port and password of NZBGet server. - # NZBGet passes all configuration options to scripts as environment variables. - host = os.environ['NZBOP_CONTROLIP'] - if host == '0.0.0.0': host = '127.0.0.1' - port = os.environ['NZBOP_CONTROLPORT'] - username = os.environ['NZBOP_CONTROLUSERNAME'] - password = os.environ['NZBOP_CONTROLPASSWORD'] + # First we need to know connection info: host, port and password of NZBGet server. + # NZBGet passes all configuration options to scripts as environment variables. + host = os.environ["NZBOP_CONTROLIP"] + if host == "0.0.0.0": + host = "127.0.0.1" + port = os.environ["NZBOP_CONTROLPORT"] + username = os.environ["NZBOP_CONTROLUSERNAME"] + password = os.environ["NZBOP_CONTROLPASSWORD"] - # Building http-URL to call the method - httpUrl = 'http://%s:%s/jsonrpc/%s' % (host, port, url_command) - request = urllib.request.Request(httpUrl) + # Building http-URL to call the method + httpUrl = "http://%s:%s/jsonrpc/%s" % (host, port, url_command) + request = urllib.request.Request(httpUrl) - credentials = ('%s:%s' % (username, password)).encode('utf8') - base64string = base64.b64encode(credentials).decode('utf8') - request.add_header("Authorization", "Basic %s" % base64string) + credentials = ("%s:%s" % (username, password)).encode("utf8") + base64string = base64.b64encode(credentials).decode("utf8") + request.add_header("Authorization", "Basic %s" % base64string) - # Load data from NZBGet - response = urllib.request.urlopen(request) - data = response.read().decode('utf-8') + # Load data from NZBGet + response = urllib.request.urlopen(request) + data = response.read().decode("utf-8") + + # "data" is a JSON raw-string + return data - # "data" is a JSON raw-string - return data # Reorder inner files for earlier fake detection def sort_inner_files(): - nzb_id = int(os.environ.get('NZBNA_NZBID')) - - # Building command-URL to call method "listfiles" passing three parameters: (0, 0, nzb_id) - url_command = 'listfiles?1=0&2=0&3=%i' % nzb_id - data = call_nzbget_direct(url_command) - - # The "data" is a raw json-string. We could use json.loads(data) to - # parse it but json-module is slow. We parse it on our own. - - # Iterate through the list of files to find the last rar-file. - # The last is the one with the highest XX in ".partXX.rar" or ".rXX" - regex1 = re.compile(r'.*\.part(\d+)\.rar', re.IGNORECASE) - regex2 = re.compile(r'.*\.r(\d+)', re.IGNORECASE) - file_num = None - file_id = None - file_name = None - - for line in data.splitlines(): - if line.startswith('"ID" : '): - cur_id = int(line[7:len(line)-1]) - if line.startswith('"Filename" : "'): - cur_name = line[14:len(line)-2] - match = regex1.match(cur_name) or regex2.match(cur_name) - if (match): - cur_num = int(match.group(1)) - if not file_num or cur_num > file_num: - file_num = cur_num - file_id = cur_id - file_name = cur_name - - # Move the last rar-file to the top of file list - if (file_id): - print('[INFO] Moving last rar-file to the top: %s' % file_name) - # Create remote server object - nzbget = connect_to_nzbget() - # Using RPC-method "editqueue" of XML-RPC-object "nzbget". - # we could use direct http access here too but the speed isn't - # an issue here and XML-RPC is easier to use. - nzbget.editqueue('FileMoveTop', 0, '', [file_id]) - else: - print('[INFO] Skipping sorting since could not find any rar-files') + nzb_id = int(os.environ.get("NZBNA_NZBID")) + + # Building command-URL to call method "listfiles" passing three parameters: (0, 0, nzb_id) + url_command = "listfiles?1=0&2=0&3=%i" % nzb_id + data = call_nzbget_direct(url_command) + + # The "data" is a raw json-string. We could use json.loads(data) to + # parse it but json-module is slow. We parse it on our own. + + # Iterate through the list of files to find the last rar-file. + # The last is the one with the highest XX in ".partXX.rar" or ".rXX" + regex1 = re.compile(r".*\.part(\d+)\.rar", re.IGNORECASE) + regex2 = re.compile(r".*\.r(\d+)", re.IGNORECASE) + file_num = None + file_id = None + file_name = None + + for line in data.splitlines(): + if line.startswith('"ID" : '): + cur_id = int(line[7 : len(line) - 1]) + if line.startswith('"Filename" : "'): + cur_name = line[14 : len(line) - 2] + match = regex1.match(cur_name) or regex2.match(cur_name) + if match: + cur_num = int(match.group(1)) + if not file_num or cur_num > file_num: + file_num = cur_num + file_id = cur_id + file_name = cur_name + + # Move the last rar-file to the top of file list + if file_id: + print("[INFO] Moving last rar-file to the top: %s" % file_name) + # Create remote server object + nzbget = connect_to_nzbget() + # Using RPC-method "editqueue" of XML-RPC-object "nzbget". + # we could use direct http access here too but the speed isn't + # an issue here and XML-RPC is easier to use. + nzbget.editqueue("FileMoveTop", 0, "", [file_id]) + else: + print("[INFO] Skipping sorting since could not find any rar-files") + # Remove current and any old temp files def clean_up(): - nzb_id = os.environ.get('NZBPP_NZBID') - temp_folder = os.environ.get('NZBOP_TEMPDIR') + '/FakeDetector' - - nzbids = [] - files = os.listdir(temp_folder) - - if len(files) > 1: - # Create the list of nzbs in download queue - data = call_nzbget_direct('listgroups?1=0') - # The "data" is a raw json-string. We could use json.loads(data) to - # parse it but json-module is slow. We parse it on our own. - for line in data.splitlines(): - if line.startswith('"NZBID" : '): - cur_id = int(line[10:len(line)-1]) - nzbids.append(str(cur_id)) - - old_temp_files = list(set(files)-set(nzbids)) - if nzb_id in files and nzb_id not in old_temp_files: - old_temp_files.append(nzb_id) - - for temp_id in old_temp_files: - temp_file = temp_folder + '/' + str(temp_id) - try: - print('[DETAIL] Removing temp file ' + temp_file) - os.remove(temp_file) - except: - print('[ERROR] Could not remove temp file ' + temp_file) + nzb_id = os.environ.get("NZBPP_NZBID") + temp_folder = os.environ.get("NZBOP_TEMPDIR") + "/FakeDetector" + + nzbids = [] + files = os.listdir(temp_folder) + + if len(files) > 1: + # Create the list of nzbs in download queue + data = call_nzbget_direct("listgroups?1=0") + # The "data" is a raw json-string. We could use json.loads(data) to + # parse it but json-module is slow. We parse it on our own. + for line in data.splitlines(): + if line.startswith('"NZBID" : '): + cur_id = int(line[10 : len(line) - 1]) + nzbids.append(str(cur_id)) + + old_temp_files = list(set(files) - set(nzbids)) + if nzb_id in files and nzb_id not in old_temp_files: + old_temp_files.append(nzb_id) + + for temp_id in old_temp_files: + temp_file = temp_folder + "/" + str(temp_id) + try: + print("[DETAIL] Removing temp file " + temp_file) + os.remove(temp_file) + except: + print("[ERROR] Could not remove temp file " + temp_file) + # Script body def main(): - # Globally define directory for storing list of tested files - global tmp_file_name - - # Do start up check - start_check() - - # That's how we determine if the download is still runnning or is completely downloaded. - # We don't use this info in the fake detector (yet). - Downloading = os.environ.get('NZBNA_EVENT') == 'FILE_DOWNLOADED' - - # Depending on the mode in which the script was called (queue-script - # or post-processing-script) a different set of parameters (env. vars) - # is passed. They also have different prefixes: - # - NZBNA_ in queue-script mode; - # - NZBPP_ in pp-script mode. - Prefix = 'NZBNA_' if 'NZBNA_EVENT' in os.environ else 'NZBPP_' - - # Read context (what nzb is currently being processed) - Category = os.environ[Prefix + 'CATEGORY'] - Directory = os.environ[Prefix + 'DIRECTORY'] - NzbName = os.environ[Prefix + 'NZBNAME'] - - # Directory for storing list of tested files - tmp_file_name = os.environ.get('NZBOP_TEMPDIR') + '/FakeDetector/' + os.environ.get(Prefix + 'NZBID') - - # When nzb is added to queue - reorder inner files for earlier fake detection. - # Also it is possible that nzb was added with a category which doesn't have - # FakeDetector listed in the PostScript. In this case FakeDetector was not called - # when adding nzb to queue but it is being called now and we can reorder - # files now. - if os.environ.get('NZBNA_EVENT') == 'NZB_ADDED' or \ - (os.environ.get('NZBNA_EVENT') == 'FILE_DOWNLOADED' and \ - os.environ.get('NZBPR_FAKEDETECTOR_SORTED') != 'yes'): - print('[INFO] Sorting inner files for earlier fake detection for %s' % NzbName) - sys.stdout.flush() - sort_inner_files() - print('[NZB] NZBPR_FAKEDETECTOR_SORTED=yes') - if os.environ.get('NZBNA_EVENT') == 'NZB_ADDED': - sys.exit(POSTPROCESS_NONE) - - print('[DETAIL] Detecting fake for %s' % NzbName) - sys.stdout.flush() - - if detect_fake(NzbName, Directory): - # A fake is detected - # - # Add post-processing parameter "PPSTATUS_FAKE" for nzb-file. - # Scripts running after fake detector can check the parameter like this: - # if os.environ.get('NZBPR_PPSTATUS_FAKE') == 'yes': - # print('Marked as fake by another script') - print('[NZB] NZBPR_PPSTATUS_FAKE=yes') - - # Special command telling NZBGet to mark nzb as bad. The nzb will - # be removed from queue and become status "FAILURE/BAD". - print('[NZB] MARK=BAD') - else: - # Not a fake or at least doesn't look like a fake (yet). - # - # When nzb is downloaded again (using "Download again" from history) - # it may have been marked by our script as a fake. Since now the script - # doesn't consider nzb as fake we remove the old marking. That's - # of course a rare case that someone will redownload a fake but - # at least during debugging of fake detector we do that all the time. - if os.environ.get('NZBPR_PPSTATUS_FAKE') == 'yes': - print('[NZB] NZBPR_PPSTATUS_FAKE=') - - print('[DETAIL] Detecting completed for %s' % NzbName) - sys.stdout.flush() - - # Remove temp files in PP - if Prefix == 'NZBPP_': - clean_up() + # Globally define directory for storing list of tested files + global tmp_file_name + + # Do start up check + start_check() + + # That's how we determine if the download is still runnning or is completely downloaded. + # We don't use this info in the fake detector (yet). + Downloading = os.environ.get("NZBNA_EVENT") == "FILE_DOWNLOADED" + + # Depending on the mode in which the script was called (queue-script + # or post-processing-script) a different set of parameters (env. vars) + # is passed. They also have different prefixes: + # - NZBNA_ in queue-script mode; + # - NZBPP_ in pp-script mode. + Prefix = "NZBNA_" if "NZBNA_EVENT" in os.environ else "NZBPP_" + + # Read context (what nzb is currently being processed) + Category = os.environ[Prefix + "CATEGORY"] + Directory = os.environ[Prefix + "DIRECTORY"] + NzbName = os.environ[Prefix + "NZBNAME"] + + # Directory for storing list of tested files + tmp_file_name = ( + os.environ.get("NZBOP_TEMPDIR") + + "/FakeDetector/" + + os.environ.get(Prefix + "NZBID") + ) + + # When nzb is added to queue - reorder inner files for earlier fake detection. + # Also it is possible that nzb was added with a category which doesn't have + # FakeDetector listed in the PostScript. In this case FakeDetector was not called + # when adding nzb to queue but it is being called now and we can reorder + # files now. + if os.environ.get("NZBNA_EVENT") == "NZB_ADDED" or ( + os.environ.get("NZBNA_EVENT") == "FILE_DOWNLOADED" + and os.environ.get("NZBPR_FAKEDETECTOR_SORTED") != "yes" + ): + print("[INFO] Sorting inner files for earlier fake detection for %s" % NzbName) + sys.stdout.flush() + sort_inner_files() + print("[NZB] NZBPR_FAKEDETECTOR_SORTED=yes") + if os.environ.get("NZBNA_EVENT") == "NZB_ADDED": + sys.exit(POSTPROCESS_NONE) + + print("[DETAIL] Detecting fake for %s" % NzbName) + sys.stdout.flush() + + if detect_fake(NzbName, Directory): + # A fake is detected + # + # Add post-processing parameter "PPSTATUS_FAKE" for nzb-file. + # Scripts running after fake detector can check the parameter like this: + # if os.environ.get('NZBPR_PPSTATUS_FAKE') == 'yes': + # print('Marked as fake by another script') + print("[NZB] NZBPR_PPSTATUS_FAKE=yes") + + # Special command telling NZBGet to mark nzb as bad. The nzb will + # be removed from queue and become status "FAILURE/BAD". + print("[NZB] MARK=BAD") + else: + # Not a fake or at least doesn't look like a fake (yet). + # + # When nzb is downloaded again (using "Download again" from history) + # it may have been marked by our script as a fake. Since now the script + # doesn't consider nzb as fake we remove the old marking. That's + # of course a rare case that someone will redownload a fake but + # at least during debugging of fake detector we do that all the time. + if os.environ.get("NZBPR_PPSTATUS_FAKE") == "yes": + print("[NZB] NZBPR_PPSTATUS_FAKE=") + + print("[DETAIL] Detecting completed for %s" % NzbName) + sys.stdout.flush() + + # Remove temp files in PP + if Prefix == "NZBPP_": + clean_up() + # Execute main script function main() diff --git a/manifest.json b/manifest.json index 2f14e17..ff60f7f 100644 --- a/manifest.json +++ b/manifest.json @@ -4,7 +4,8 @@ "homepage": "https://github.com/nzbgetcom/Extension-FakeDetector", "kind": "QUEUE/POST-PROCESSING", "displayName": "Fake Detector", - "version": "3.0.0", + "version": "3.1", + "nzbgetMinVersion": "23.0", "author": "Andrey Prygunkov", "license": "GNU", "about": "Detects nzbs with fake media files.", diff --git a/tests.py b/tests.py index 7695e28..f99d8ca 100644 --- a/tests.py +++ b/tests.py @@ -21,211 +21,196 @@ import sys from os.path import dirname import os -import traceback import subprocess import json import http.server import xmlrpc.server import threading import json +import unittest +import shutil -POSTPROCESS_SUCCESS=93 -POSTPROCESS_NONE=95 -POSTPROCESS_ERROR=94 +SUCCESS = 93 +NONE = 95 +ERROR = 94 root_dir = dirname(__file__) -test_data_dir = root_dir + '/test_data' -tmp_dir = root_dir + '/__' -host = '127.0.0.1' -username = 'TestUser' -password = 'TestPassword' -port = '6789' - -os.makedirs(tmp_dir + '/FakeDetector') - -def RUN_TESTS(): - TEST('Should ignore incompatibale event', TEST_IGNORE_INCOMPATIBALE_EVENT) - TEST('Should do nothing if nzb was marked as bad or containes banned extension', TEST_DO_NOTHING) - TEST('Should skip sorting part0x.rar files if File ID not found on NZB_ADDED NZBNA_EVENT', TEST_SKIP_SORTING_RAR_FILES) - TEST('Should sort part0x.rar files on FILE_DOWNLOADED NZBNA_EVENT',TEST_SORT_FILES) - TEST('Should detect fake (media, executable) or banned files',TEST_DETECT_FAKE_FILES) - clean_up() +test_data_dir = root_dir + "/test_data" +tmp_dir = root_dir + "/tmp" +host = "127.0.0.1" +username = "TestUser" +password = "TestPassword" +port = "6789" + + +def get_python(): + if os.name == "nt": + return "python" + return "python3" + + +def clean_up(): + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + class RequestEmpty(http.server.BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(b'{}') + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b"{}") + class RequestWithFileId(http.server.BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - f = open(test_data_dir + '/nzbget_response.json') - data = json.load(f) - self.send_header("Content-Type", "application/json") - self.end_headers() - formatted = json.dumps(data, separators=(',\n', ' : '), indent=0) - self.wfile.write(formatted.encode('utf-8')) - f.close() - - def do_POST(self): - self.log_request() - self.send_response(200) - self.send_header("Content-Type", "text/xml") - self.end_headers() - data = '' - response = xmlrpc.client.dumps((data,), allow_none=False, encoding=None) - self.wfile.write(response.encode('utf-8')) - -def TEST(statement: str, test_func): - print('\n********************************************************') - print('TEST:', statement) - print('--------------------------------------------------------') - - try: - test_func() - print(test_func.__name__, '...SUCCESS') - except Exception as e: - print(test_func.__name__, '...FAILED') - traceback.print_exc() - finally: - print('********************************************************\n') - -def get_python(): - if os.name == 'nt': - return 'python' - return 'python3' + def do_GET(self): + self.send_response(200) + f = open(test_data_dir + "/nzbget_response.json") + data = json.load(f) + self.send_header("Content-Type", "application/json") + self.end_headers() + formatted = json.dumps(data, separators=(",\n", " : "), indent=0) + self.wfile.write(formatted.encode("utf-8")) + f.close() + + def do_POST(self): + self.log_request() + self.send_response(200) + self.send_header("Content-Type", "text/xml") + self.end_headers() + data = '' + response = xmlrpc.client.dumps((data,), allow_none=False, encoding=None) + self.wfile.write(response.encode("utf-8")) -def clean_up(): - for root, dirs, files in os.walk(tmp_dir, topdown=False): - for name in files: - file_path = os.path.join(root, name) - os.remove(file_path) - for name in dirs: - dir_path = os.path.join(root, name) - os.rmdir(dir_path) - os.rmdir(tmp_dir) def run_script(): - sys.stdout.flush() - proc = subprocess.Popen([get_python(), root_dir + '/main.py'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ.copy()) - out, err = proc.communicate() - proc.pid - ret_code = proc.returncode - return (out.decode(), int(ret_code), err.decode()) + sys.stdout.flush() + proc = subprocess.Popen( + [get_python(), root_dir + "/main.py"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=os.environ.copy(), + ) + out, err = proc.communicate() + proc.pid + ret_code = proc.returncode + return (out.decode(), int(ret_code), err.decode()) + def set_defaults_env(): - # NZBGet global options - os.environ['NZBOP_SCRIPTDIR'] = 'test' - os.environ['NZBOP_ARTICLECACHE'] = '64' - os.environ['NZBOP_TEMPDIR'] = tmp_dir - os.environ['NZBOP_CONTROLPORT'] = port - os.environ['NZBOP_CONTROLIP'] = host - os.environ['NZBOP_CONTROLUSERNAME'] = username - os.environ['NZBOP_CONTROLPASSWORD'] = password - - # script options - os.environ['NZBPO_BANNEDEXTENSIONS'] = '.mkv,.mp4' - - os.environ['NZBPP_DIRECTORY'] = tmp_dir - os.environ['NZBPP_NZBNAME'] = 'test' - os.environ['NZBPP_PARSTATUS'] = '2' - os.environ['NZBPP_UNPACKSTATUS'] = '2' - os.environ['NZBPP_CATEGORY'] = '' - os.environ['NZBPP_NZBID'] = '8' - - os.environ['NZBPR__DNZB_USENZBNAME'] = 'no' - os.environ['NZBPR__DNZB_PROPERNAME'] = '' - os.environ['NZBPR__DNZB_EPISODENAME'] = '' - - os.environ['NZBNA_EVENT'] = 'NZB_ADDED' - os.environ.pop('NZBPR_PPSTATUS_FAKEBAN', None) - os.environ.pop('NZBPP_STATUS', None) - -def TEST_IGNORE_INCOMPATIBALE_EVENT(): - set_defaults_env() - os.environ['NZBNA_EVENT'] = '' - res = run_script() - assert(res[1] == 0) - -def TEST_DO_NOTHING(): - set_defaults_env() - os.environ['NZBPP_STATUS'] = 'FAILURE/BAD' - os.environ['NZBPR_PPSTATUS_FAKE'] = 'yes' - [out, code, err] = run_script() - assert(code == POSTPROCESS_SUCCESS) - - os.environ.pop('NZBPR_PPSTATUS_FAKEBAN', None) - [out, code, err] = run_script() - assert('[WARNING] Download has media files and executables' in out) - assert(code == POSTPROCESS_SUCCESS) - - os.environ['NZBPR_PPSTATUS_FAKEBAN'] = '.mp4' - [out, code, err] = run_script() - assert('[WARNING] Download contains banned extension ' + os.environ.get('NZBPR_PPSTATUS_FAKEBAN') in out) - assert(code == POSTPROCESS_SUCCESS) - -def TEST_SKIP_SORTING_RAR_FILES(): - set_defaults_env() - os.environ['NZBNA_NZBNAME'] = 'nzb_test_file' - os.environ['NZBNA_CATEGORY'] = 'movies' - os.environ['NZBNA_NZBID'] = '8' - os.environ['NZBNA_DIRECTORY'] = test_data_dir - os.environ['NZBNA_EVENT'] = 'NZB_ADDED' - os.environ['NZBPR_FAKEDETECTOR_SORTED'] = 'no' - - server = http.server.HTTPServer((host, int(port)), RequestEmpty) - thread = threading.Thread(target=server.serve_forever) - thread.start() - [out, code, err] = run_script() - server.shutdown() - thread.join() - assert('[INFO] Skipping sorting since could not find any rar-files' in out) - assert('NZB] NZBPR_FAKEDETECTOR_SORTED=yes' in out) - assert(code == POSTPROCESS_NONE) - -def TEST_SORT_FILES(): - set_defaults_env() - file_name = 'nzb_test_file' - os.environ['NZBNA_NZBNAME'] = file_name - os.environ['NZBNA_CATEGORY'] = 'movies' - os.environ['NZBNA_NZBID'] = '8' - os.environ['NZBNA_DIRECTORY'] = test_data_dir - os.environ['NZBNA_EVENT'] = 'FILE_DOWNLOADED' - os.environ['NZBPR_FAKEDETECTOR_SORTED'] = 'no' - - server = http.server.HTTPServer((host, int(port)), RequestWithFileId) - thread = threading.Thread(target=server.serve_forever) - thread.start() - [out, code, err] = run_script() - server.shutdown() - thread.join() - assert('[DETAIL] Detecting completed for %s' % file_name in out) - assert('NZB] NZBPR_FAKEDETECTOR_SORTED=yes' in out) - assert(code == POSTPROCESS_SUCCESS) - -def TEST_DETECT_FAKE_FILES(): - set_defaults_env() - file_name = 'nzb_test_file' - os.environ['NZBNA_NZBNAME'] = file_name - os.environ['NZBNA_CATEGORY'] = 'movies' - os.environ['NZBNA_NZBID'] = '8' - os.environ['NZBPR_PPSTATUS_FAKEBAN'] = '.nzb,.json,.mp4' - os.environ['NZBNA_DIRECTORY'] = test_data_dir - os.environ['NZBNA_EVENT'] = 'FILE_DOWNLOADED' - os.environ['NZBPR_FAKEDETECTOR_SORTED'] = 'no' - - server = http.server.HTTPServer((host, int(port)), RequestWithFileId) - thread = threading.Thread(target=server.serve_forever) - thread.start() - [out, code, err] = run_script() - server.shutdown() - thread.join() - assert('[WARNING] Download has media files and executables' in out) - assert('[NZB] NZBPR_PPSTATUS_FAKE=yes' in out) - assert('[NZB] MARK=BAD' in out) - assert('[DETAIL] Detecting completed for %s' % file_name in out) - assert(code == POSTPROCESS_SUCCESS) - -RUN_TESTS() + # NZBGet global options + os.environ["NZBOP_SCRIPTDIR"] = "test" + os.environ["NZBOP_ARTICLECACHE"] = "64" + os.environ["NZBOP_TEMPDIR"] = tmp_dir + os.environ["NZBOP_CONTROLPORT"] = port + os.environ["NZBOP_CONTROLIP"] = host + os.environ["NZBOP_CONTROLUSERNAME"] = username + os.environ["NZBOP_CONTROLPASSWORD"] = password + + # script options + os.environ["NZBPO_BANNEDEXTENSIONS"] = ".mkv,.mp4" + + os.environ["NZBPP_DIRECTORY"] = tmp_dir + os.environ["NZBPP_NZBNAME"] = "test" + os.environ["NZBPP_PARSTATUS"] = "2" + os.environ["NZBPP_UNPACKSTATUS"] = "2" + os.environ["NZBPP_CATEGORY"] = "" + os.environ["NZBPP_NZBID"] = "8" + + os.environ["NZBPR__DNZB_USENZBNAME"] = "no" + os.environ["NZBPR__DNZB_PROPERNAME"] = "" + os.environ["NZBPR__DNZB_EPISODENAME"] = "" + + os.environ["NZBNA_EVENT"] = "NZB_ADDED" + os.environ.pop("NZBPR_PPSTATUS_FAKEBAN", None) + os.environ.pop("NZBPP_STATUS", None) + + os.makedirs(tmp_dir + "/FakeDetector") + + +class Tests(unittest.TestCase): + def test_ignore_incompitable_event(self): + set_defaults_env() + os.environ["NZBNA_EVENT"] = "" + res = run_script() + clean_up() + self.assertEqual(res[1], 0) + + def test_skip_sorting_rar_files(self): + set_defaults_env() + os.environ["NZBNA_NZBNAME"] = "nzb_test_file" + os.environ["NZBNA_CATEGORY"] = "movies" + os.environ["NZBNA_NZBID"] = "8" + os.environ["NZBNA_DIRECTORY"] = test_data_dir + os.environ["NZBNA_EVENT"] = "NZB_ADDED" + os.environ["NZBPR_FAKEDETECTOR_SORTED"] = "no" + + server = http.server.HTTPServer((host, int(port)), RequestEmpty) + thread = threading.Thread(target=server.serve_forever) + thread.start() + [out, code, err] = run_script() + server.shutdown() + server.server_close() + thread.join() + clean_up() + self.assertEqual(code, NONE) + + def test_do_nothing(self): + set_defaults_env() + os.environ["NZBPP_STATUS"] = "FAILURE/BAD" + os.environ["NZBPR_PPSTATUS_FAKE"] = "yes" + [out, code, err] = run_script() + clean_up() + self.assertEqual(code, SUCCESS) + + def test_detect_fake_files(self): + set_defaults_env() + file_name = "nzb_test_file" + os.environ["NZBNA_NZBNAME"] = file_name + os.environ["NZBNA_CATEGORY"] = "movies" + os.environ["NZBNA_NZBID"] = "8" + os.environ["NZBPR_PPSTATUS_FAKEBAN"] = ".nzb,.json,.mp4" + os.environ["NZBNA_DIRECTORY"] = test_data_dir + os.environ["NZBNA_EVENT"] = "FILE_DOWNLOADED" + os.environ["NZBPR_FAKEDETECTOR_SORTED"] = "no" + + server = http.server.HTTPServer((host, int(port)), RequestWithFileId) + thread = threading.Thread(target=server.serve_forever) + thread.start() + [out, code, err] = run_script() + server.shutdown() + server.server_close() + thread.join() + clean_up() + self.assertEqual(code, SUCCESS) + + def test_sort_files(self): + set_defaults_env() + file_name = "nzb_test_file" + os.environ["NZBNA_NZBNAME"] = file_name + os.environ["NZBNA_CATEGORY"] = "movies" + os.environ["NZBNA_NZBID"] = "8" + os.environ["NZBNA_DIRECTORY"] = test_data_dir + os.environ["NZBNA_EVENT"] = "FILE_DOWNLOADED" + os.environ["NZBPR_FAKEDETECTOR_SORTED"] = "no" + + server = http.server.HTTPServer((host, int(port)), RequestWithFileId) + thread = threading.Thread(target=server.serve_forever) + thread.start() + [out, code, err] = run_script() + server.shutdown() + server.server_close() + thread.join() + clean_up() + self.assertEqual(code, SUCCESS) + + def test_manifest(self): + with open(root_dir + "/manifest.json", encoding="utf-8") as file: + try: + json.loads(file.read()) + except ValueError as e: + self.fail("manifest.json is not valid.") + + +if __name__ == "__main__": + unittest.main()