diff --git a/src/run.py b/src/run.py index bdb162c..4e99333 100644 --- a/src/run.py +++ b/src/run.py @@ -13,16 +13,21 @@ fatal, which, exists, - err, - sanitize_slurm_env + err, + sanitize_slurm_env, ) from . import version as __version__ -def init(repo_path, output_path, links=[], required=['workflow', 'bin', 'resources', 'config']): +def init( + repo_path, + output_path, + links=[], + required=["workflow", "bin", "resources", "config"], +): """Initialize the output directory. If user provides a output - directory path that already exists on the filesystem as a file + directory path that already exists on the filesystem as a file (small chance of happening but possible), a OSError is raised. If the output directory PATH already EXISTS, it will not try to create the directory. @param repo_path : @@ -40,25 +45,28 @@ def init(repo_path, output_path, links=[], required=['workflow', 'bin', 'resourc os.makedirs(output_path) elif exists(output_path) and os.path.isfile(output_path): - # Provided Path for pipeline + # Provided Path for pipeline # output directory exists as file - raise OSError("""\n\tFatal: Failed to create provided pipeline output directory! + raise OSError( + """\n\tFatal: Failed to create provided pipeline output directory! User provided --output PATH already exists on the filesystem as a file. Please run {} again with a different --output PATH. - """.format(sys.argv[0]) + """.format( + sys.argv[0] + ) ) # Copy over templates are other required resources - copy_safe(source = repo_path, target = output_path, resources = required) + copy_safe(source=repo_path, target=output_path, resources=required) - # Create renamed symlinks for each rawdata + # Create renamed symlinks for each rawdata # file provided as input to the pipeline - inputs = sym_safe(input_data = links, target = output_path) + inputs = sym_safe(input_data=links, target=output_path) return inputs -def copy_safe(source, target, resources = []): +def copy_safe(source, target, resources=[]): """Private function: Given a list paths it will recursively copy each to the target location. If a target path already exists, it will NOT over-write the existing paths data. @@ -88,7 +96,7 @@ def sym_safe(input_data, target): @return input_fastqs list[]: List of renamed input FastQs """ - input_fastqs = [] # store renamed fastq file names + input_fastqs = [] # store renamed fastq file names for file in input_data: filename = os.path.basename(file) renamed = os.path.join(target, rename(filename)) @@ -124,11 +132,10 @@ def rename(filename): ".R2.(?P...).f(ast)?q.gz$": ".R2.fastq.gz", # Matches: _[12].fastq.gz, _[12].fq.gz, _[12]_fastq_gz, etc. "_1.f(ast)?q.gz$": ".R1.fastq.gz", - "_2.f(ast)?q.gz$": ".R2.fastq.gz" + "_2.f(ast)?q.gz$": ".R2.fastq.gz", } - if (filename.endswith('.R1.fastq.gz') or - filename.endswith('.R2.fastq.gz')): + if filename.endswith(".R1.fastq.gz") or filename.endswith(".R2.fastq.gz"): # Filename is already in the correct format return filename @@ -139,10 +146,11 @@ def rename(filename): # regex matches with a pattern in extensions converted = True filename = re.sub(regex, new_ext, filename) - break # only rename once + break # only rename once if not converted: - raise NameError("""\n\tFatal: Failed to rename provided input '{}'! + raise NameError( + """\n\tFatal: Failed to rename provided input '{}'! Cannot determine the extension of the user provided input file. Please rename the file list above before trying again. Here is example of acceptable input file extensions: @@ -151,7 +159,9 @@ def rename(filename): sampleName_1.fastq.gz sampleName_2.fastq.gz Please also check that your input files are gzipped? If they are not, please gzip them before proceeding again. - """.format(filename, sys.argv[0]) + """.format( + filename, sys.argv[0] + ) ) return filename @@ -170,61 +180,63 @@ def setup(sub_args, ifiles, repo_path, output_path): """ # Check for mixed inputs, # inputs which are a mixture - # of FastQ and BAM files + # of FastQ and BAM files mixed_inputs(ifiles) - # Resolves PATH to reference file - # template or a user generated - # reference genome built via build + # Resolves PATH to reference file + # template or a user generated + # reference genome built via build # subcommand - genome_config = os.path.join(output_path,'config','genome.json') + genome_config = os.path.join(output_path, "config", "genome.json") # if sub_args.genome.endswith('.json'): - # Provided a custom reference genome generated by build pipline - # genome_config = os.path.abspath(sub_args.genome) + # Provided a custom reference genome generated by build pipline + # genome_config = os.path.abspath(sub_args.genome) required = { # Base configuration file - "base": os.path.join(output_path,'config','config.json'), + "base": os.path.join(output_path, "config", "config.json"), # Template for project-level information - "project": os.path.join(output_path,'config','containers.json'), + "project": os.path.join(output_path, "config", "containers.json"), # Template for genomic reference files # User provided argument --genome is used to select the template "genome": genome_config, # Template for tool information - "tools": os.path.join(output_path,'config', 'modules.json'), + "tools": os.path.join(output_path, "config", "modules.json"), } - # Create the global or master config + # Create the global or master config # file for pipeline, config.json - config = join_jsons(required.values()) # uses templates in config/*.json - config['project'] = {} + config = join_jsons(required.values()) # uses templates in config/*.json + config["project"] = {} config = add_user_information(config) config = add_rawdata_information(sub_args, config, ifiles) - # Resolves if an image needs to be pulled + # Resolves if an image needs to be pulled # from an OCI registry or a local SIF exists config = image_cache(sub_args, config, output_path) # Add other runtime info for debugging - config['project']['version'] = __version__ - config['project']['workpath'] = os.path.abspath(sub_args.output) - config['project']['binpath'] = os.path.abspath(os.path.join(config['project']['workpath'], 'bin')) + config["project"]["version"] = __version__ + config["project"]["workpath"] = os.path.abspath(sub_args.output) + config["project"]["binpath"] = os.path.abspath( + os.path.join(config["project"]["workpath"], "bin") + ) git_hash = git_commit_hash(repo_path) - config['project']['git_commit_hash'] = git_hash # Add latest git commit hash - config['project']['pipeline_path'] = repo_path # Add path to installation + config["project"]["git_commit_hash"] = git_hash # Add latest git commit hash + config["project"]["pipeline_path"] = repo_path # Add path to installation # Add all cli options for data provenance for opt, v in vars(sub_args).items(): - if opt == 'func': + if opt == "func": # Pass over sub command's handler continue elif not isinstance(v, (list, dict)): # CLI value can be converted to a string v = str(v) - config['options'][opt] = v + config["options"][opt] = v # initiate a few workflow vars - config['options']['peak_type_base'] = ["protTSS"] + config["options"]["peak_type_base"] = ["protTSS"] return config @@ -232,28 +244,28 @@ def unpacked(nested_dict): """Generator to recursively retrieves all values in a nested dictionary. @param nested_dict dict[]: Nested dictionary to unpack - @yields value in dictionary + @yields value in dictionary """ - # Iterate over all values of + # Iterate over all values of # given dictionary for key, value in nested_dict.items(): # Check if value is of dict type # also exclude certain directories - dontcheck = ('userhome',) - # we exclude the /home directory so it does not interfere with + dontcheck = ("userhome",) + # we exclude the /home directory so it does not interfere with # container system if isinstance(value, dict) and key not in dontcheck: - # If value is dict then iterate + # If value is dict then iterate # over all its values recursively for v in unpacked(value): yield v else: - # If value is not dict type + # If value is not dict type # then yield the value yield value -def get_fastq_screen_paths(fastq_screen_confs, match = 'DATABASE', file_index = -1): +def get_fastq_screen_paths(fastq_screen_confs, match="DATABASE", file_index=-1): """Parses fastq_screen.conf files to get the paths of each fastq_screen database. This path contains bowtie2 indices for reference genome to screen against. The paths are added as singularity bind points. @@ -268,11 +280,11 @@ def get_fastq_screen_paths(fastq_screen_confs, match = 'DATABASE', file_index = """ databases = [] for file in fastq_screen_confs: - with open(file, 'r') as fh: + with open(file, "r") as fh: for line in fh: if line.startswith(match): - db_path = line.strip().split()[file_index] - databases.append(db_path) + db_path = line.strip().split()[file_index] + databases.append(db_path) return databases @@ -295,25 +307,27 @@ def resolve_additional_bind_paths(search_paths): # Skip over resources with remote URI and # skip over strings that are not file PATHS as # build command creates absolute resource PATHS - if ref.lower().startswith('sftp://') or \ - ref.lower().startswith('s3://') or \ - ref.lower().startswith('gs://') or \ - not ref.lower().startswith(os.sep): + if ( + ref.lower().startswith("sftp://") + or ref.lower().startswith("s3://") + or ref.lower().startswith("gs://") + or not ref.lower().startswith(os.sep) + ): continue # Break up path into directory tokens path_list = os.path.abspath(ref).split(os.sep) - try: # Create composite index from first two directories + try: # Create composite index from first two directories # Avoids issues created by shared /gpfs/ PATHS index = path_list[1:3] index = tuple(index) except IndexError: - index = path_list[1] # ref startswith / + index = path_list[1] # ref startswith / if index not in indexed_paths: indexed_paths[index] = [] - # Create an INDEX to find common PATHS for each root - # child directory like /scratch or /data. This prevents - # issues when trying to find the common path betweeen + # Create an INDEX to find common PATHS for each root + # child directory like /scratch or /data. This prevents + # issues when trying to find the common path betweeen # these two different directories (resolves to /) indexed_paths[index].append(str(os.sep).join(path_list)) @@ -322,7 +336,7 @@ def resolve_additional_bind_paths(search_paths): p = os.path.dirname(os.path.commonprefix(paths)) if p == os.sep: # Aviods adding / to bind list when - # given /tmp or /scratch as input + # given /tmp or /scratch as input p = os.path.commonprefix(paths) common_paths.append(p) @@ -336,7 +350,7 @@ def bind(sub_args, config): @param configfile dict[]: Config dictionary generated by setup command. @return bindpaths list[]: - List of singularity/docker bind paths + List of singularity/docker bind paths """ bindpaths = [] for value in unpacked(config): @@ -351,10 +365,12 @@ def bind(sub_args, config): # Bind input file paths, working # directory, and other reference # genome paths - rawdata_bind_paths = [os.path.realpath(p) for p in config['project']['datapath'].split(',')] - working_directory = os.path.realpath(config['project']['workpath']) + rawdata_bind_paths = [ + os.path.realpath(p) for p in config["project"]["datapath"].split(",") + ] + working_directory = os.path.realpath(config["project"]["workpath"]) genome_bind_paths = resolve_additional_bind_paths(bindpaths) - bindpaths = [working_directory] + rawdata_bind_paths + genome_bind_paths + bindpaths = [working_directory] + rawdata_bind_paths + genome_bind_paths bindpaths = list(set([p for p in bindpaths if p != os.sep])) return bindpaths @@ -371,16 +387,17 @@ def mixed_inputs(ifiles): fastqs = False bams = False for file in ifiles: - if file.endswith('.R1.fastq.gz') or file.endswith('.R2.fastq.gz'): - fastqs = True + if file.endswith(".R1.fastq.gz") or file.endswith(".R2.fastq.gz"): + fastqs = True fq_files.append(file) - elif file.endswith('.bam'): + elif file.endswith(".bam"): bams = True bam_files.append(file) if fastqs and bams: # User provided a mix of FastQs and BAMs - raise TypeError("""\n\tFatal: Detected a mixture of --input data types. + raise TypeError( + """\n\tFatal: Detected a mixture of --input data types. A mixture of BAM and FastQ files were provided; however, the pipeline does NOT support processing a mixture of input FastQ and BAM files. Input FastQ Files: @@ -393,9 +410,12 @@ def mixed_inputs(ifiles): for your project, please run the set of FastQ and BAM files separately (in two separate output directories). If you feel like this functionality should exist, feel free to open an issue on Github. - """.format(" ".join(fq_files), " ".join(bam_files), sys.argv[0]) + """.format( + " ".join(fq_files), " ".join(bam_files), sys.argv[0] + ) ) + def add_user_information(config): """Adds username and user's home directory to config. @params config : @@ -404,17 +424,17 @@ def add_user_information(config): Updated config dictionary containing user information (username and home directory) """ # Get PATH to user's home directory - # Method is portable across unix-like + # Method is portable across unix-like # OS and Windows home = os.path.expanduser("~") # Get username from home directory PATH username = os.path.split(home)[-1] - # Update config with home directory and + # Update config with home directory and # username - config['project']['userhome'] = home - config['project']['username'] = username + config["project"]["userhome"] = home + config["project"]["username"] = username return config @@ -434,17 +454,17 @@ def add_sample_metadata(input_files, config, group=None): """ import re - # TODO: Add functionality for basecase + # TODO: Add functionality for basecase # when user has samplesheet added = [] - config['samples'] = [] + config["samples"] = [] for file in input_files: # Split sample name on file extension - sample = re.split('\.R[12]\.fastq\.gz', os.path.basename(file))[0] + sample = re.split("\.R[12]\.fastq\.gz", os.path.basename(file))[0] if sample not in added: # Only add PE sample information once added.append(sample) - config['samples'].append(sample) + config["samples"].append(sample) return config @@ -468,17 +488,17 @@ def add_rawdata_information(sub_args, config, ifiles): # or single-end # Updates config['project']['nends'] where # 1 = single-end, 2 = paired-end, -1 = bams - convert = {1: 'single-end', 2: 'paired-end', -1: 'bam'} + convert = {1: "single-end", 2: "paired-end", -1: "bam"} nends = get_nends(ifiles) # Checks PE data for both mates (R1 and R2) - config['project']['nends'] = nends - config['project']['filetype'] = convert[nends] + config["project"]["nends"] = nends + config["project"]["filetype"] = convert[nends] # Finds the set of rawdata directories to bind - rawdata_paths = get_rawdata_bind_paths(input_files = sub_args.input) - config['project']['datapath'] = ','.join(rawdata_paths) + rawdata_paths = get_rawdata_bind_paths(input_files=sub_args.input) + config["project"]["datapath"] = ",".join(rawdata_paths) # Add each sample's basename - config = add_sample_metadata(input_files = ifiles, config = config) + config = add_sample_metadata(input_files=ifiles, config=config) return config @@ -496,24 +516,32 @@ def image_cache(sub_args, config, repo_path): @return config : Updated config dictionary containing user information (username and home directory) """ - images = os.path.join(repo_path, 'config','containers.json') + images = os.path.join(repo_path, "config", "containers.json") - # Read in config for docker image uris - with open(images, 'r') as fh: + # Read in config for docker image uris + with open(images, "r") as fh: data = json.load(fh) - # Check if local sif exists - for image, uri in data['images'].items(): + # Check if local sif exists + for image, uri in data["images"].items(): if sub_args.sif_cache: - sif = os.path.join(sub_args.sif_cache, '{}.sif'.format(os.path.basename(uri).replace(':', '_'))) + sif = os.path.join( + sub_args.sif_cache, + "{}.sif".format(os.path.basename(uri).replace(":", "_")), + ) if not exists(sif): - # If local sif does not exist on in cache, - # print warning and default to pulling from + # If local sif does not exist on in cache, + # print warning and default to pulling from # URI in config/containers.json - print('Warning: Local image "{}" does not exist in singularity cache'.format(sif), file=sys.stderr) + print( + 'Warning: Local image "{}" does not exist in singularity cache'.format( + sif + ), + file=sys.stderr, + ) else: - # Change pointer to image from Registry URI + # Change pointer to image from Registry URI # to local SIF - data['images'][image] = sif + data["images"][image] = sif config.update(data) @@ -534,22 +562,22 @@ def get_nends(ifiles): bam_files = False nends_status = 1 for file in ifiles: - if file.endswith('.bam'): + if file.endswith(".bam"): bam_files = True nends_status = -1 break - elif file.endswith('.R2.fastq.gz'): + elif file.endswith(".R2.fastq.gz"): paired_end = True nends_status = 2 - break # dataset is paired-end + break # dataset is paired-end - # Check to see if both mates (R1 and R2) + # Check to see if both mates (R1 and R2) # are present paired-end data if paired_end: - nends = {} # keep count of R1 and R2 for each sample + nends = {} # keep count of R1 and R2 for each sample for file in ifiles: # Split sample name on file extension - sample = re.split('\.R[12]\.fastq\.gz', os.path.basename(file))[0] + sample = re.split("\.R[12]\.fastq\.gz", os.path.basename(file))[0] if sample not in nends: nends[sample] = 0 @@ -564,7 +592,8 @@ def get_nends(ifiles): # functionality could be added later but it may not # be the best idea due to batch effects/potentially # adding unwanted sources of technical variation. - raise NameError("""\n\tFatal: Detected pair-end data but user failed to provide + raise NameError( + """\n\tFatal: Detected pair-end data but user failed to provide both mates (R1 and R2) for the following samples:\n\t\t{}\n Please check that the basename for each sample is consistent across mates. Here is an example of a consistent basename across mates: @@ -577,7 +606,9 @@ def get_nends(ifiles): paired-end samples and single-end samples separately (in two separate output directories). If you feel like this functionality should exist, feel free to open an issue on Github. - """.format(missing_mates, sys.argv[0]) + """.format( + missing_mates, sys.argv[0] + ) ) return nends_status @@ -601,7 +632,9 @@ def get_rawdata_bind_paths(input_files): return bindpaths -def dryrun(outdir, config='config.json', snakefile=os.path.join('workflow', 'Snakefile')): +def dryrun( + outdir, config="config.json", snakefile=os.path.join("workflow", "Snakefile") +): """Dryruns the pipeline to ensure there are no errors prior to runnning. @param outdir : Pipeline output PATH @@ -612,37 +645,53 @@ def dryrun(outdir, config='config.json', snakefile=os.path.join('workflow', 'Sna # Setting cores to dummy high number so # displays the true number of cores a rule # will use, it uses the min(--cores CORES, N) - dryrun_output = subprocess.check_output([ - 'snakemake', '-npr', - '-s', str(snakefile), - '--verbose', - # '--debug-dag', - '--use-singularity', - '--rerun-incomplete', - '--cores', str(256), - '--configfile={}'.format(config) - ], cwd = outdir, - stderr=subprocess.STDOUT) + dryrun_output = subprocess.check_output( + [ + "snakemake", + "-npr", + "-s", + str(snakefile), + "--verbose", + "--debug-dag", + "--use-singularity", + "--rerun-incomplete", + "--cores", + str(256), + "--configfile={}".format(config), + ], + cwd=outdir, + stderr=subprocess.STDOUT, + ) except OSError as e: # Catch: OSError: [Errno 2] No such file or directory # Occurs when command returns a non-zero exit-code - if e.errno == 2 and not which('snakemake'): + if e.errno == 2 and not which("snakemake"): # Failure caused because snakemake is NOT in $PATH - err('\n\x1b[6;37;41mError: Are snakemake AND singularity in your $PATH?\x1b[0m') - fatal('\x1b[6;37;41mPlease check before proceeding again!\x1b[0m') + err( + "\n\x1b[6;37;41mError: Are snakemake AND singularity in your $PATH?\x1b[0m" + ) + fatal("\x1b[6;37;41mPlease check before proceeding again!\x1b[0m") else: # Failure caused by unknown cause, raise error raise e except subprocess.CalledProcessError as e: print(e, e.output.decode("utf-8")) - raise(e) + raise (e) return dryrun_output -def runner(mode, outdir, alt_cache, logger, additional_bind_paths = None, - threads=2, jobname='pl:master', submission_script='run.sh', - tmp_dir = '/lscratch/$SLURM_JOBID/'): +def runner( + mode, + outdir, + alt_cache, + logger, + additional_bind_paths=None, + threads=2, + jobname="pl:master", + submission_script="run.sh", + tmp_dir="/lscratch/$SLURM_JOBID/", +): """Runs the pipeline via selected executor: local or slurm. If 'local' is selected, the pipeline is executed locally on a compute node/instance. If 'slurm' is selected, jobs will be submited to the cluster using SLURM job scheduler. @@ -666,77 +715,88 @@ def runner(mode, outdir, alt_cache, logger, additional_bind_paths = None, @return masterjob : """ # Add additional singularity bind PATHs - # to mount the local filesystem to the - # containers filesystem, NOTE: these + # to mount the local filesystem to the + # containers filesystem, NOTE: these # PATHs must be an absolute PATHs outdir = os.path.abspath(outdir) - # Add any default PATHs to bind to - # the container's filesystem, like + # Add any default PATHs to bind to + # the container's filesystem, like # tmp directories, /lscratch addpaths = [] - temp = os.path.dirname(tmp_dir.rstrip('/')) + temp = os.path.dirname(tmp_dir.rstrip("/")) if temp == os.sep: - temp = tmp_dir.rstrip('/') - if outdir not in additional_bind_paths.split(','): + temp = tmp_dir.rstrip("/") + if outdir not in additional_bind_paths.split(","): addpaths.append(outdir) - if temp not in additional_bind_paths.split(','): + if temp not in additional_bind_paths.split(","): addpaths.append(temp) - bindpaths = ','.join(addpaths) - - # Set ENV variable 'SINGULARITY_CACHEDIR' + bindpaths = ",".join(addpaths) + + # Set ENV variable 'SINGULARITY_CACHEDIR' # to output directory - my_env = {}; my_env.update(os.environ) + my_env = {} + my_env.update(os.environ) cache = os.path.join(outdir, ".singularity") - my_env['SINGULARITY_CACHEDIR'] = cache - my_env['APPTAINER_CACHEDIR'] = cache + my_env["SINGULARITY_CACHEDIR"] = cache + my_env["APPTAINER_CACHEDIR"] = cache if alt_cache: - # Override the pipeline's default + # Override the pipeline's default # cache location - my_env['SINGULARITY_CACHEDIR'] = alt_cache - my_env['APPTAINER_CACHEDIR'] = alt_cache + my_env["SINGULARITY_CACHEDIR"] = alt_cache + my_env["APPTAINER_CACHEDIR"] = alt_cache cache = alt_cache my_env = sanitize_slurm_env(my_env) - + if additional_bind_paths: # Add Bind PATHs for outdir and tmp dir if bindpaths: bindpaths = ",{}".format(bindpaths) - bindpaths = "{}{}".format(additional_bind_paths,bindpaths) + bindpaths = "{}{}".format(additional_bind_paths, bindpaths) - if not exists(os.path.join(outdir, 'logfiles')): + if not exists(os.path.join(outdir, "logfiles")): # Create directory for logfiles - os.makedirs(os.path.join(outdir, 'logfiles')) + os.makedirs(os.path.join(outdir, "logfiles")) - # Create .singularity directory for + # Create .singularity directory for # installations of snakemake without # setuid which creates a sandbox in # the SINGULARITY_CACHEDIR if not exists(cache): - # Create directory for sandbox + # Create directory for sandbox # and image layers os.makedirs(cache) # Run on compute node or instance # without submitting jobs to a scheduler - if mode == 'local': + if mode == "local": # Run pipeline's main process - # Look into later: it maybe worth + # Look into later: it maybe worth # replacing Popen subprocess with a direct # snakemake API call: https://snakemake.readthedocs.io/en/stable/api_reference/snakemake.html - masterjob = subprocess.Popen([ - 'snakemake', '-pr', '--rerun-incomplete', - '--use-singularity', - '--singularity-args', "\\-c \\-B '{}'".format(bindpaths), - '--cores', str(threads), - '--configfile=config.json' - ], cwd = outdir, stderr=subprocess.STDOUT, stdout=logger, env=my_env) + masterjob = subprocess.Popen( + [ + "snakemake", + "-pr", + "--rerun-incomplete", + "--use-singularity", + "--singularity-args", + "\\-c \\-B '{}'".format(bindpaths), + "--cores", + str(threads), + "--configfile=config.json", + ], + cwd=outdir, + stderr=subprocess.STDOUT, + stdout=logger, + env=my_env, + ) # Submitting jobs to cluster via SLURM's job scheduler - elif mode == 'slurm': + elif mode == "slurm": # Run pipeline's main process - # Look into later: it maybe worth + # Look into later: it maybe worth # replacing Popen subprocess with a direct # snakemake API call: https://snakemake.readthedocs.io/en/stable/api_reference/snakemake.html # CLUSTER_OPTS="'sbatch --gres {cluster.gres} --cpus-per-task {cluster.threads} -p {cluster.partition} \ @@ -750,11 +810,25 @@ def runner(mode, outdir, alt_cache, logger, additional_bind_paths = None, # --cluster "${CLUSTER_OPTS}" --keep-going --restart-times 3 -j 500 \ # --rerun-incomplete --stats "$3"/logfiles/runtime_statistics.json \ # --keep-remote --local-cores 30 2>&1 | tee -a "$3"/logfiles/master.log - masterjob = subprocess.Popen([ - str(submission_script), mode, - '-j', jobname, '-b', str(bindpaths), - '-o', str(outdir), '-c', str(cache), - '-t', "'{}'".format(tmp_dir) - ], cwd = outdir, stderr=subprocess.STDOUT, stdout=logger, env=my_env) + masterjob = subprocess.Popen( + [ + str(submission_script), + mode, + "-j", + jobname, + "-b", + str(bindpaths), + "-o", + str(outdir), + "-c", + str(cache), + "-t", + "'{}'".format(tmp_dir), + ], + cwd=outdir, + stderr=subprocess.STDOUT, + stdout=logger, + env=my_env, + ) return masterjob