diff --git a/jobbergate-api/jobbergate_api/apps/job_submissions/properties_parser.py b/jobbergate-api/jobbergate_api/apps/job_submissions/properties_parser.py new file mode 100644 index 000000000..fa8dff1fd --- /dev/null +++ b/jobbergate-api/jobbergate_api/apps/job_submissions/properties_parser.py @@ -0,0 +1,320 @@ +""" +Parser for Slurm REST API parameters from SBATCH parameters at the job script file. +""" +from argparse import ArgumentParser +from dataclasses import dataclass, field +from itertools import chain +from typing import Any, Dict, Iterator, List, Sequence, Union, cast + +from bidict import bidict +from loguru import logger + +from jobbergate_api.apps.job_scripts.job_script_files import JobScriptFiles +from jobbergate_api.apps.job_submissions.schemas import JobProperties + +_IDENTIFICATION_FLAG = "#SBATCH" +_INLINE_COMMENT_MARK = "#" + + +def _flagged_line(line: str) -> bool: + """ + Identify if a provided line starts with the identification flag. + """ + return line.startswith(_IDENTIFICATION_FLAG) + + +def _clean_line(line: str) -> str: + """ + Clean the provided line. + + It includes removing the identification flag at the beginning of the line, + then remove the inline comment mark and anything after it, and finally strip white spaces at both sides. + """ + return line.lstrip(_IDENTIFICATION_FLAG).split(_INLINE_COMMENT_MARK)[0].strip() + + +def _split_line(line: str) -> List[str]: + """ + Split the provided line at the equal and with spaces characters. + + This procedure is important because it is the way argparse expects to receive the parameters. + """ + return line.replace("=", " ").split() + + +def _clean_jobscript(jobscript: str) -> Iterator[str]: + """ + Transform a job script string. + + It is done by filtering only the lines that start with + the identification flag and mapping a cleaning procedure to them in order + to remove the identification flag, remove inline comments, and strip extra + white spaces. Finally, split each pair of parameter/value and chain them + in a single iterator. + """ + jobscript_filtered = filter(_flagged_line, jobscript.splitlines()) + jobscript_cleaned = map(_clean_line, jobscript_filtered) + jobscript_splitted = map(_split_line, jobscript_cleaned) + return chain.from_iterable(jobscript_splitted) + + +@dataclass(frozen=True) +class SbatchToSlurm: + """ + Store the information for each parameter, including its name at Slurm API and SBATCH. + + Besides that, any extra argument this parameter needs when added to + the parser. This information is used to build the jobscript/SBATCH parser + and the two-way mapping between Slurm API and SBATCH names. + """ + + slurmrestd_var_name: str + sbatch: str + sbatch_short: str = "" + argparser_param: dict = field(default_factory=dict) + + +sbatch_to_slurm = [ + SbatchToSlurm("account", "--account", "-A"), + SbatchToSlurm("account_gather_freqency", "--acctg-freq"), + SbatchToSlurm("array", "--array", "-a"), + SbatchToSlurm("batch_features", "--batch"), + SbatchToSlurm("burst_buffer", "--bb"), + SbatchToSlurm("", "--bbf"), + SbatchToSlurm("begin_time", "--begin", "-b"), + SbatchToSlurm("current_working_directory", "--chdir", "-D"), + SbatchToSlurm("cluster_constraints", "--cluster-constraint"), + SbatchToSlurm("", "--clusters", "-M"), + SbatchToSlurm("comment", "--comment"), + SbatchToSlurm("constraints", "--constraint", "-C"), + SbatchToSlurm("", "--container"), + SbatchToSlurm("", "--contiguous", "", dict(action="store_const", const=True)), + SbatchToSlurm("core_specification", "--core-spec", "-S", dict(type=int)), + SbatchToSlurm("cores_per_socket", "--cores-per-socket", "", dict(type=int)), + SbatchToSlurm("cpu_binding", "--cpu-bind"), + SbatchToSlurm("cpu_frequency", "--cpu-freq"), + SbatchToSlurm("cpus_per_gpu", "--cpus-per-gpu"), + SbatchToSlurm("cpus_per_task", "--cpus-per-task", "-c", dict(type=int)), + SbatchToSlurm("deadline", "--deadline"), + SbatchToSlurm("delay_boot", "--delay-boot", "", dict(type=int)), + SbatchToSlurm("dependency", "--dependency", "-d"), + SbatchToSlurm("distribution", "--distribution", "-m"), + SbatchToSlurm("standard_error", "--error", "-e"), + SbatchToSlurm("", "--exclude", "-x"), + SbatchToSlurm( + "exclusive", + "--exclusive", + "", + dict( + type=str, + choices={"user", "mcs", "exclusive", "oversubscribe"}, + nargs="?", + const="exclusive", + ), + ), + SbatchToSlurm("", "--export"), + SbatchToSlurm("", "--export-file"), + SbatchToSlurm("", "--extra-node-info", "-B"), + SbatchToSlurm("get_user_environment", "--get-user-env", "", dict(type=int)), + SbatchToSlurm("", "--gid"), + SbatchToSlurm("gpu_binding", "--gpu-bind"), + SbatchToSlurm("gpu_frequency", "--gpu-freq"), + SbatchToSlurm("gpus", "--gpus", "-G"), + SbatchToSlurm("gpus_per_node", "--gpus-per-node"), + SbatchToSlurm("gpus_per_socket", "--gpus-per-socket"), + SbatchToSlurm("gpus_per_task", "--gpus-per-task"), + SbatchToSlurm("gres", "--gres"), + SbatchToSlurm("gres_flags", "--gres-flags"), + SbatchToSlurm("", "--hint"), + SbatchToSlurm("hold", "--hold", "-H", dict(action="store_const", const=True)), + SbatchToSlurm("", "--ignore-pbs", "", dict(action="store_const", const=True)), + SbatchToSlurm("standard_input", "--input", "-i"), + SbatchToSlurm("name", "--job-name", "-J"), + # kill_on_invalid_dependency is an invalid key for Slurm API according to our tests + # SbatchToSlurm( + # "kill_on_invalid_dependency", "--kill-on-invalid-dep", "", dict(type=int) + # ), + SbatchToSlurm("licenses", "--licenses", "-L"), + SbatchToSlurm("mail_type", "--mail-type"), + SbatchToSlurm("mail_user", "--mail-user"), + SbatchToSlurm("mcs_label", "--mcs-label"), + SbatchToSlurm("memory_per_node", "--mem"), + SbatchToSlurm("memory_binding", "--mem-bind"), + SbatchToSlurm("memory_per_cpu", "--mem-per-cpu"), + SbatchToSlurm("memory_per_gpu", "--mem-per-gpu"), + SbatchToSlurm("minimum_cpus_per_node", "--mincpus", "", dict(type=int)), + SbatchToSlurm("", "--network"), + SbatchToSlurm("nice", "--nice"), + SbatchToSlurm("no_kill", "--no-kill", "-k", dict(action="store_const", const=True)), + SbatchToSlurm("", "--no-requeue", "", dict(action="store_false", dest="requeue")), + SbatchToSlurm("", "--nodefile", "-F"), + SbatchToSlurm("", "--nodelist", "-w"), + SbatchToSlurm("nodes", "--nodes", "-N"), + SbatchToSlurm("tasks", "--ntasks", "-n", dict(type=int)), + SbatchToSlurm("tasks_per_core", "--ntasks-per-core", "", dict(type=int)), + SbatchToSlurm("", "--ntasks-per-gpu"), + SbatchToSlurm("tasks_per_node", "--ntasks-per-node", "", dict(type=int)), + SbatchToSlurm("tasks_per_socket", "--ntasks-per-socket", "", dict(type=int)), + SbatchToSlurm("open_mode", "--open-mode"), + SbatchToSlurm("standard_output", "--output", "-o"), + SbatchToSlurm("", "--overcommit", "-O", dict(action="store_const", const=True)), + SbatchToSlurm( + "", + "--oversubscribe", + "-s", + dict(action="store_const", const="oversubscribe", dest="exclusive"), + ), + SbatchToSlurm("", "--parsable", "", dict(action="store_const", const=True)), + SbatchToSlurm("partition", "--partition", "-p"), + SbatchToSlurm("", "--power"), + SbatchToSlurm("priority", "--priority"), + SbatchToSlurm("", "--profile"), + SbatchToSlurm("", "--propagate"), + SbatchToSlurm("qos", "--qos", "-q"), + SbatchToSlurm("", "--quiet", "-Q", dict(action="store_const", const=True)), + SbatchToSlurm("", "--reboot", "", dict(action="store_const", const=True)), + SbatchToSlurm("requeue", "--requeue", "", dict(action="store_const", const=True)), + SbatchToSlurm("reservation", "--reservation"), + SbatchToSlurm("signal", "--signal"), + SbatchToSlurm("sockets_per_node", "--sockets-per-node", "", dict(type=int)), + SbatchToSlurm("spread_job", "--spread-job", "", dict(action="store_const", const=True)), + SbatchToSlurm("", "--switches"), + SbatchToSlurm("", "--test-only", "", dict(action="store_const", const=True)), + SbatchToSlurm("thread_specification", "--thread-spec", "", dict(type=int)), + SbatchToSlurm("threads_per_core", "--threads-per-core", "", dict(type=int)), + SbatchToSlurm("time_limit", "--time", "-t"), + SbatchToSlurm("time_minimum", "--time-min"), + SbatchToSlurm("", "--tmp"), + SbatchToSlurm("", "--uid"), + SbatchToSlurm("", "--usage", "", dict(action="store_const", const=True)), + SbatchToSlurm("minimum_nodes", "--use-min-nodes", "", dict(action="store_const", const=True)), + SbatchToSlurm("", "--verbose", "-v", dict(action="store_const", const=True)), + SbatchToSlurm("", "--version", "-V", dict(action="store_const", const=True)), + SbatchToSlurm("", "--wait", "-W", dict(action="store_const", const=True)), + SbatchToSlurm("wait_all_nodes", "--wait-all-nodes", "", dict(type=int)), + SbatchToSlurm("wckey", "--wckey"), + SbatchToSlurm("", "--wrap"), +] + + +class ArgumentParserCustomExit(ArgumentParser): + """ + Custom implementation of the built-in class for argument parsing. + + The sys.exit triggered by the original code is replaced by a ValueError, + besides some friendly logging messages. + """ + + def exit(self, status=0, message=None): + """ + Raise ValueError when parsing invalid parameters or if the type of their values is not correct. + """ + log_message = f"Argparse exit status {status}: {message}" + if status: + logger.error(log_message) + else: + logger.info(log_message) + raise ValueError(message) + + +def build_parser() -> ArgumentParser: + """ + Build an ArgumentParser to handle all SBATCH parameters declared at sbatch_to_slurm. + """ + parser = ArgumentParserCustomExit() + for item in sbatch_to_slurm: + args = (i for i in (item.sbatch_short, item.sbatch) if i) + parser.add_argument(*args, **item.argparser_param) + # make --requeue and --no-requeue work together, with default to None + parser.set_defaults(requeue=None) + return parser + + +def build_mapping_sbatch_to_slurm() -> bidict: + """ + Create a mapper to translate in both ways between the names expected by Slurm REST API and SBATCH. + """ + mapping: bidict = bidict() + + for item in sbatch_to_slurm: + if item.slurmrestd_var_name: + sbatch_name = item.sbatch.lstrip("-").replace("-", "_") + mapping[sbatch_name] = item.slurmrestd_var_name + + return mapping + + +def jobscript_to_dict(jobscript: str) -> Dict[str, Union[str, bool]]: + """ + Extract the SBATCH params from a given job script. + + It returns them in a dictionary for mapping the parameter names to their values. + + Raise ValueError if any of the parameters are unknown to the parser. + """ + parsed_args, unknown_arg = parser.parse_known_args( + cast(Sequence[str], _clean_jobscript(jobscript)), + ) + + if unknown_arg: + raise ValueError("Unrecognized SBATCH arguments: {}".format(" ".join(unknown_arg))) + + sbatch_params = {key: value for key, value in vars(parsed_args).items() if value is not None} + + logger.debug(f"SBATCH params parsed from job script: {sbatch_params}") + + return sbatch_params + + +def convert_sbatch_to_slurm_api(input: Dict[str, Any]) -> Dict[str, Any]: + """ + Take a dictionary containing key-value pairing of SBATCH parameter name space to Slurm API namespace. + + Notice the values should not be affected. + + Raise KeyError if any of the keys are unknown to the mapper. + """ + mapped = {} + unknown_keys = [] + + for sbatch_name, value in input.items(): + try: + slurm_name = mapping_sbatch_to_slurm[sbatch_name] + except KeyError: + unknown_keys.append(sbatch_name) + else: + mapped[slurm_name] = value + + if unknown_keys: + error_message = "Impossible to convert from SBATCH to Slurm REST API: {}" + raise KeyError(error_message.format(", ".join(unknown_keys))) + + logger.debug(f"Slurm API params mapped from SBATCH: {mapped}") + + return mapped + + +def get_job_parameters(jobscript: str) -> Dict[str, Any]: + """ + Parse all SBATCH parameters from a job script, map their names to Slurm API parameters. + + They are returned as a key-value pairing dictionary. + """ + return convert_sbatch_to_slurm_api(jobscript_to_dict(jobscript)) + + +def get_job_properties_from_job_script(job_script_id: int, **kwargs) -> JobProperties: + """ + Get the job properties for Slurm REST API from a job script file, given its id. + + Extra keyword arguments can be used to overwrite any parameter from the + job script, like name or current_working_directory. + """ + job_script_files = JobScriptFiles.get_from_s3(job_script_id) + slurm_parameters = get_job_parameters(job_script_files.main_file) + merged_parameters = {**slurm_parameters, **kwargs} + return JobProperties.parse_obj(merged_parameters) + + +parser = build_parser() +mapping_sbatch_to_slurm = build_mapping_sbatch_to_slurm() diff --git a/jobbergate-api/jobbergate_api/apps/job_submissions/routers.py b/jobbergate-api/jobbergate_api/apps/job_submissions/routers.py index ccd76b4c1..a42e631bf 100644 --- a/jobbergate-api/jobbergate_api/apps/job_submissions/routers.py +++ b/jobbergate-api/jobbergate_api/apps/job_submissions/routers.py @@ -17,6 +17,7 @@ searchable_fields, sortable_fields, ) +from jobbergate_api.apps.job_submissions.properties_parser import get_job_properties_from_job_script from jobbergate_api.apps.job_submissions.schemas import ( ActiveJobSubmission, JobSubmissionCreateRequest, @@ -66,17 +67,17 @@ async def job_submission_create( detail=message, ) - create_dict = dict( + new_job_submission_data: Dict[str, Any] = dict( **job_submission.dict(exclude_unset=True), job_submission_owner_email=identity_claims.email, status=JobSubmissionStatus.CREATED, ) if job_submission.client_id is None: - create_dict.update(client_id=client_id) + new_job_submission_data.update(client_id=client_id) - exec_dir = create_dict.pop("execution_directory", None) + exec_dir = new_job_submission_data.pop("execution_directory", None) if exec_dir is not None: - create_dict.update(execution_directory=str(exec_dir)) + new_job_submission_data.update(execution_directory=str(exec_dir)) select_query = job_scripts_table.select().where(job_scripts_table.c.id == job_submission.job_script_id) logger.trace(f"job_scripts select_query = {render_sql(select_query)}") @@ -90,13 +91,23 @@ async def job_submission_create( detail=message, ) + try: + job_properties = get_job_properties_from_job_script( + job_submission.job_script_id, **job_submission.execution_parameters.dict(exclude_unset=True) + ) + new_job_submission_data["execution_parameters"] = job_properties.dict(exclude_unset=True) + except Exception as e: + message = f"Error extracting execution parameters from job script: {str(e)}" + logger.error(message) + raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=message) + logger.debug("Inserting job-submission") async with database.transaction(): try: insert_query = job_submissions_table.insert().returning(job_submissions_table) logger.trace(f"job_submissions insert_query = {render_sql(insert_query)}") - job_submission_data = await database.fetch_one(query=insert_query, values=create_dict) + job_submission_data = await database.fetch_one(query=insert_query, values=new_job_submission_data) except INTEGRITY_CHECK_EXCEPTIONS as e: logger.error(f"Reverting database transaction: {str(e)}") diff --git a/jobbergate-api/jobbergate_api/apps/job_submissions/schemas.py b/jobbergate-api/jobbergate_api/apps/job_submissions/schemas.py index 32170279a..70e3dc395 100644 --- a/jobbergate-api/jobbergate_api/apps/job_submissions/schemas.py +++ b/jobbergate-api/jobbergate_api/apps/job_submissions/schemas.py @@ -284,7 +284,7 @@ class JobSubmissionCreateRequest(BaseModel): job_script_id: int execution_directory: Optional[Path] client_id: Optional[str] - execution_parameters: Optional[JobProperties] + execution_parameters: JobProperties = Field(default_factory=dict) class Config: schema_extra = job_submission_meta_mapper diff --git a/jobbergate-api/jobbergate_api/tests/apps/job_submissions/test_properties_parser.py b/jobbergate-api/jobbergate_api/tests/apps/job_submissions/test_properties_parser.py new file mode 100644 index 000000000..8bc12380a --- /dev/null +++ b/jobbergate-api/jobbergate_api/tests/apps/job_submissions/test_properties_parser.py @@ -0,0 +1,665 @@ +""" +Test the properties parser module. +""" +import contextlib +import inspect +from argparse import ArgumentParser +from pathlib import Path +from typing import MutableMapping +from unittest import mock + +import pytest +from bidict import bidict +from pydantic import ValidationError + +from jobbergate_api.apps.job_scripts.job_script_files import JobScriptFiles +from jobbergate_api.apps.job_submissions.properties_parser import ( + _IDENTIFICATION_FLAG, + _INLINE_COMMENT_MARK, + ArgumentParserCustomExit, + _clean_jobscript, + _clean_line, + _flagged_line, + _split_line, + build_mapping_sbatch_to_slurm, + build_parser, + convert_sbatch_to_slurm_api, + get_job_parameters, + get_job_properties_from_job_script, + jobscript_to_dict, + sbatch_to_slurm, +) +from jobbergate_api.apps.job_submissions.schemas import JobProperties + + +def test_identification_flag(): + """ + Check if the value of the identification flag is the same codded in the tests. + """ + assert _IDENTIFICATION_FLAG == "#SBATCH" + + +def test_inline_comment_mark(): + """ + Check if the inline comment mark is the same codded in the tests. + """ + assert _INLINE_COMMENT_MARK == "#" + + +@pytest.mark.parametrize( + "line, desired_value", + ( + ("#SBATCH -abc", True), + ("##SBATCH -abc", False), + ("run python application", False), + ("# A comment", False), + ), +) +def test_flagged_line(line, desired_value): + """ + Check if the flagged lines are identified properly. + """ + actual_value = _flagged_line(line) + assert actual_value == desired_value + + +@pytest.mark.parametrize( + "line, desired_value", + ( + ("#SBATCH", ""), + ("#SBATCH#SBATCH", ""), + ("#SBATCH -abc # A comment", "-abc"), + ("#SBATCH --abc=0 # A comment", "--abc=0"), + ("#SBATCH --abc 0 # A comment", "--abc 0"), + ("#SBATCH -a 0 # A comment", "-a 0"), + ("#SBATCH -a=0 # A comment", "-a=0"), + ("#SBATCH -a = 0 # A comment", "-a = 0"), + ("#SBATCH -a=C&Aaccount # A comment", "-a=C&Aaccount"), + ), +) +def test_clean_line(line, desired_value): + """ + Check if the provided lines are cleaned properly. + + I.e., identification flag, inline comments, and comment mark are all removed. + """ + actual_value = _clean_line(line) + assert actual_value == desired_value + + +@pytest.mark.parametrize( + "line, desired_value", + ( + ("--help", ["--help"]), + ("--abc=0", ["--abc", "0"]), + ("-J job_name", ["-J", "job_name"]), + ("-v -J job_name", ["-v", "-J", "job_name"]), + ("-J job_name -v", ["-J", "job_name", "-v"]), + ("-a=0", ["-a", "0"]), + ("-a = 0", ["-a", "0"]), + ("-a 0", ["-a", "0"]), + ), +) +def test_split_line(line, desired_value): + """ + Check if the provided lines are splitted properly at white spaces and equal character. + + This procedure is important because it is the way argparse expects to receive the parameters. + """ + actual_value = _split_line(line) + assert actual_value == desired_value + + +@pytest.fixture +def dummy_slurm_script(): + """ + Provide a dummy job script for testing. + """ + # TODO: DRY, integrate this one with conftest/dummy_template_source + return inspect.cleandoc( + """ + #!/bin/bash + #SBATCH # Empty line + #SBATCH --no-kill --no-requeue # Flagged params + #SBATCH -n 4 -A # Multiple args per line + #SBATCH --job-name=serial_job_test # Job name + #SBATCH --mail-type=END,FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) + #SBATCH --mail-user=email@somewhere.com # Where to send mail + #SBATCH --mem=1gb # Job memory request + #SBATCH --time=00:05:00 # Time limit hrs:min:sec + #SBATCH --output=serial_test_%j.log # Standard output and error log + #SBATCH --wait-all-nodes=0 # + pwd; hostname; date + + module load python + + echo "Running plot script on a single CPU core" + + python /data/training/SLURM/plot_template.py + + date + """ + ) + + +def test_clean_jobscript(dummy_slurm_script): + """ + Check if all sbatch parameters are correctly extracted from a job script. + + This operation combines many of the functions tested above (filter, + clean, and slit the parameters on each line). + """ + desired_list = [ + "--no-kill", + "--no-requeue", + "-n", + "4", + "-A", + "", + "--job-name", + "serial_job_test", + "--mail-type", + "END,FAIL", + "--mail-user", + "email@somewhere.com", + "--mem", + "1gb", + "--time", + "00:05:00", + "--output", + "serial_test_%j.log", + "--wait-all-nodes", + "0", + ] + actual_list = list(_clean_jobscript(dummy_slurm_script)) + assert actual_list == desired_list + + +@pytest.mark.parametrize("item", sbatch_to_slurm) +class TestSbatchToSlurmList: + """ + Test all the fields of the objects SbatchToSlurm that are stored in the list sbatch_to_slurm. + """ + + def test_slurmrestd_var_name__is_string(self, item): + """ + Check if the field slurmrestd_var_name is a string. + """ + assert isinstance(item.slurmrestd_var_name, str) + + def test_slurmrestd_var_name__only_contains_letters(self, item): + """ + Check if the field slurmrestd_var_name contains only underscores and letters. + """ + if item.slurmrestd_var_name: + assert item.slurmrestd_var_name.replace("_", "").isalpha() + + def test_sbatch__is_string(self, item): + """ + Check if the field sbatch is a string. + """ + assert isinstance(item.sbatch, str) + + def test_sbatch__starts_with_double_hyphen(self, item): + """ + Check if the field sbatch starts with a double hyphen. + """ + assert item.sbatch.startswith("--") + + def test_sbatch__is_not_empty(self, item): + """ + Check if the field sbatch is not empty by asserting that it has more than three characters. + + Notice it is supposed to start with a double hyphen. + """ + assert len(item.sbatch) >= 3 + + def test_sbatch__only_contains_letters(self, item): + """ + Check if the field sbatch contains only hyphens and letters. + """ + assert item.sbatch.replace("-", "").isalpha() + + def test_sbatch_short__is_string(self, item): + """ + Check if the optional field sbatch_short is a string. + """ + assert isinstance(item.sbatch_short, str) + + def test_sbatch_short__starts_with_hyphen(self, item): + """ + Check if the optional field sbatch_short starts with a hyphen. + """ + if item.sbatch_short: + assert item.sbatch_short.startswith("-") + + def test_sbatch_short__is_not_empty(self, item): + """ + Check if of the optional field sbatch_short is equal to two, since it should be a hyphen and a letter. + """ + if item.sbatch_short: + assert len(item.sbatch_short) == 2 + + def test_sbatch_short__only_contains_letters(self, item): + """ + Check if the optional field sbatch_short contains only hyphens and letters. + """ + if item.sbatch_short: + assert item.sbatch_short.replace("-", "").isalpha() + + def test_argparser_param__is_mutable_mapping(self, item): + """ + Check if the field argparser_param is a MutableMapping. + """ + assert isinstance(item.argparser_param, MutableMapping) + + def test_argparser_param__is_valid_for_parser(self, item): + """ + Check if the field argparser_param can be added in a parser. + """ + if item.argparser_param: + args = (i for i in (item.sbatch_short, item.sbatch) if i) + parser = ArgumentParserCustomExit() + parser.add_argument(*args, **item.argparser_param) + + +@pytest.mark.parametrize("field", ["slurmrestd_var_name", "sbatch", "sbatch_short"]) +def test_sbatch_to_slurm_list__contains_only_unique_values(field): + """ + Test that any given field has no duplicated values for all parameters stored at sbatch_to_slurm. + + This aims to avoid ambiguity at the SBATCH argparser and the two-way mapping between + SBATCH and Slurm Rest API namespaces. + """ + list_of_values = [getattr(i, field) for i in sbatch_to_slurm if bool(getattr(i, field))] + + assert len(list_of_values) == len(set(list_of_values)) + + +class TestArgumentParserCustomExit: + """ + Test the custom error handling implemented over the built-in argument parser. + """ + + @pytest.fixture(scope="module") + def parser(self): + """ + Support the tests in this class bt providing an instance of the parser. + """ + parser = ArgumentParserCustomExit() + parser.add_argument("--foo", type=int) + parser.add_argument("--bar", action="store_true") + return parser + + def test_argument_parser_success(self, parser): + """ + Test the base case, where the arguments are successfully parsed and converted to the expected type. + """ + args = parser.parse_args("--foo=10 --bar".split()) + assert {"foo": 10, "bar": True} == vars(args) + + def test_argument_parser_raise_value_error_when_value_is_missing(self, parser): + """ + Test that ValueError is raised when the value for one parameter is missing. + """ + with pytest.raises(ValueError, match="error: argument --foo: expected one argument"): + parser.parse_args("--foo".split()) + + def test_argument_parser_raise_value_error_in_case_of_wrong_type(self, parser): + """ + Test that ValueError is raised when the value for some parameter is incompatible with its type. + """ + with pytest.raises(ValueError, match="error: argument --foo: invalid int value:"): + parser.parse_args("--foo some_text".split()) + + +def test_build_parser(): + """ + Test if build_parser runs with no problem and returns the correct type. + """ + parser = build_parser() + assert isinstance(parser, ArgumentParser) + + +def test_build_mapping_sbatch_to_slurm(): + """ + Test if build_mapping_sbatch_to_slurm runs with no problem and returns the correct type. + """ + mapping = build_mapping_sbatch_to_slurm() + assert isinstance(mapping, bidict) + + +def test_jobscript_to_dict__success(dummy_slurm_script): + """ + Test if the SBATCH parameters are properly extracted from a job script. + + They are expected to be returned a dictionary mapping parameters to their value. + """ + desired_dict = { + "no_kill": True, + "requeue": False, + "account": "", + "job_name": "serial_job_test", + "mail_type": "END,FAIL", + "mail_user": "email@somewhere.com", + "mem": "1gb", + "ntasks": 4, + "output": "serial_test_%j.log", + "time": "00:05:00", + "wait_all_nodes": 0, + } + + actual_dict = jobscript_to_dict(dummy_slurm_script) + + assert actual_dict == desired_dict + + +def test_jobscript_to_dict__raises_exception_for_unknown_parameter(): + """ + Test if jobscript_to_dict raises a ValueError when facing unknown parameters. + """ + with pytest.raises(ValueError, match="Unrecognized SBATCH arguments:"): + jobscript_to_dict("#SBATCH --foo\n#SBATCH --bar=0") + + +@pytest.mark.parametrize( + "item", + filter(lambda i: getattr(i, "slurmrestd_var_name", False), sbatch_to_slurm), +) +def test_convert_sbatch_to_slurm_api__success(item): + """ + Test if the keys in a dictionary are properly renamed from SBATCH to Slurm Rest API namespace. + + Notice the values should not be affected. + """ + desired_dict = {item.slurmrestd_var_name: None} + + sbatch_name = item.sbatch.lstrip("-").replace("-", "_") + actual_dict = convert_sbatch_to_slurm_api({sbatch_name: None}) + + assert actual_dict == desired_dict + + +def test_convert_sbatch_to_slurm_api__raises_exception_for_unknown_parameter(): + """ + Test the conversion of dict keys from SBATCH to Slurm Rest API raises KeyError when facing unknown names. + """ + with pytest.raises(KeyError, match="Impossible to convert from SBATCH to Slurm REST API:"): + convert_sbatch_to_slurm_api(dict(foo=0, bar=1)) + + +def test_get_job_parameters(dummy_slurm_script): + """ + Test if all SBATCH parameters are properly extracted from a given job script. + + The name of each of them is mapped to Slurm Rest API namespace and returned + in a dictionary. Notice get_job_parameters accepts extra keywords that can be + used to overwrite the values from the job script. + """ + desired_dict = { + "no_kill": True, + "requeue": False, + "account": "", + "name": "serial_job_test", + "mail_type": "END,FAIL", + "mail_user": "email@somewhere.com", + "memory_per_node": "1gb", + "tasks": 4, + "standard_output": "serial_test_%j.log", + "time_limit": "00:05:00", + "wait_all_nodes": 0, + } + + actual_dict = get_job_parameters(dummy_slurm_script) + + assert desired_dict == actual_dict + + +@pytest.fixture +def mock_job_script_files(): + """ + Create a dummy job script file with the given content and mock JobScriptFiles.get_from_s3. + """ + + @contextlib.contextmanager + def _helper(main_file_content): + main_file_name = Path("jobbergate.sh") + job_script_files = JobScriptFiles( + main_file_path=main_file_name, + files={main_file_name: main_file_content}, + ) + with mock.patch( + "jobbergate_api.apps.job_submissions.properties_parser.JobScriptFiles.get_from_s3" + ) as mocked: + mocked.return_value = job_script_files + yield mocked + + return _helper + + +class TestGetJobPropertiesFromJobScript: + """ + Test the get_job_properties_from_job_script function. + + It covers job properties obtained from the job script and from the users when creating a job submission. + """ + + def test_base_case(self, mock_job_script_files): + """ + Base case, not SBATCH parameters on the job script and no extra parameters. + """ + job_script_id = 1 + desired_job_properties = JobProperties() + + with mock_job_script_files("almost-empty-file") as mocked: + actual_job_properties = get_job_properties_from_job_script(job_script_id) + mocked.assert_called_once_with(job_script_id) + + assert actual_job_properties == desired_job_properties + + def test_properties_only_from_file(self, mock_job_script_files): + """ + Job properties are obtained only from the job script. + """ + job_script_id = 1 + desired_job_properties = JobProperties(exclusive="exclusive", name="test-test") + + with mock_job_script_files("#SBATCH --exclusive\n#SBATCH -J test-test") as mocked: + actual_job_properties = get_job_properties_from_job_script(job_script_id) + mocked.assert_called_once_with(job_script_id) + + assert actual_job_properties == desired_job_properties + + def test_properties_only_from_arguments(self, mock_job_script_files): + """ + Job properties are obtained only from the extra arguments. + """ + job_script_id = 1 + + job_properties = dict(exclusive="exclusive", name="test-test") + desired_job_properties = JobProperties.parse_obj(job_properties) + + with mock_job_script_files("almost-empty-file") as mocked: + actual_job_properties = get_job_properties_from_job_script( + job_script_id, + **job_properties, + ) + mocked.assert_called_once_with(job_script_id) + + assert actual_job_properties == desired_job_properties + + def test_properties_priority_when_both_are_provided(self, mock_job_script_files): + """ + Job properties are obtained from the job script and from the extra arguments. + + The ones from the extra arguments have priority over the ones from the job script. + """ + job_script_id = 1 + + job_properties = dict(exclusive="exclusive", name="high-priority-test") + desired_job_properties = JobProperties.parse_obj(job_properties) + + with mock_job_script_files("#SBATCH --exclusive\n#SBATCH -J test-test") as mocked: + actual_job_properties = get_job_properties_from_job_script( + job_script_id, + **job_properties, + ) + mocked.assert_called_once_with(job_script_id) + + assert actual_job_properties == desired_job_properties + + def test_properties_fails_with_unknown_field(self, mock_job_script_files): + """ + The function fails when the extra arguments contain unknown fields. + """ + job_script_id = 1 + + with mock_job_script_files("almost-empty-file") as mocked: + with pytest.raises(ValidationError, match="1 validation error for JobProperties"): + get_job_properties_from_job_script(job_script_id, foo="bar") + mocked.assert_called_once_with(job_script_id) + + +class TestBidictMapping: + """ + Integration test with the requirement bidict (used for two-way mapping). + """ + + @pytest.fixture + def dummy_mapping(self): + """ + Provide a dummy dictionary used for testing. + """ + return {f"key_{i}": f"value_{i}" for i in range(5)} + + def test_bidict__is_mutable_mapping(self): + """ + Check if bidict implements all the necessary protocols to be a MutableMapping. + """ + assert issubclass(bidict, MutableMapping) + + def test_bidict__can_be_compared_to_a_dictionary(self, dummy_mapping): + """ + Check if bidict can be really comparable to a dictionary. + """ + assert dummy_mapping == bidict(dummy_mapping) + + def test_bidict__can_be_compared_to_a_dictionary_inverse(self, dummy_mapping): + """ + Check if bidict can be comparable to a dictionary. + + This time checking its inverse capability (i.e., swapping keys and values). + """ + desired_value = {value: key for key, value in dummy_mapping.items()} + + assert desired_value == bidict(dummy_mapping).inverse + + +class TestExclusiveParameter: + """ + --exclusive is a special SBATCH parameter that can be used in some different ways. + + See the details: + 1. --exclusive as a flag, meaning the value 'exclusive' should be recovered for slurmd. + 2. --exclusive=, meaning the value should be recovered for slurmd. + According to the Slurm documentation, the value can be 'user' or 'mcs'. + 3. On top of that, 'exclusive' is expected to be set to 'oversubscribe' when the + flag --oversubscribe is used. + + Note: When both are used, the last of them takes precedence. + """ + + def test_empty_jobscript(self): + """ + Base case: no --exclusive parameter at all. + """ + jobscript = "" + + desired_dict = {} + + actual_dict = jobscript_to_dict(jobscript) + + assert actual_dict == desired_dict + + def test_exclusive_as_a_flag(self): + """ + Test the first scenario: --exclusive as a flag. + """ + jobscript = "#SBATCH --exclusive" + + desired_dict = {"exclusive": "exclusive"} + + actual_dict = jobscript_to_dict(jobscript) + + assert actual_dict == desired_dict + + @pytest.mark.parametrize("exclusive_value", ["user", "mcs", "exclusive", "oversubscribe"]) + def test_exclusive_with_string_value(self, exclusive_value): + """ + Test the second scenario: --exclusive=. + """ + jobscript = f"#SBATCH --exclusive={exclusive_value}" + + desired_dict = {"exclusive": exclusive_value} + + actual_dict = jobscript_to_dict(jobscript) + + assert actual_dict == desired_dict + + def test_exclusive_with_incorrect_value(self): + """ + Test the second scenario with an incorrect value, ValueError should be raised. + """ + jobscript = "#SBATCH --exclusive=test-test" + + with pytest.raises(ValueError, match="invalid choice: 'test-test'"): + jobscript_to_dict(jobscript) + + def test_oversubscribe(self): + """ + Test the third scenario: --oversubscribe as a flag. + """ + jobscript = "#SBATCH --oversubscribe" + + desired_dict = {"exclusive": "oversubscribe"} + + actual_dict = jobscript_to_dict(jobscript) + + assert actual_dict == desired_dict + + def test_oversubscribe_with_incorrect_value(self): + """ + Test the second scenario with an incorrect value, ValueError should be raised. + """ + jobscript = "#SBATCH --oversubscribe=test-test" + + with pytest.raises(ValueError, match="Unrecognized SBATCH arguments: test-test"): + jobscript_to_dict(jobscript) + + def test_both_exclusive_and_oversubscribe_1(self): + """ + Test that when both are used, the last of them takes precedence. + + In this case, --exclusive is the last one. + """ + jobscript = "#SBATCH --oversubscribe\n#SBATCH --exclusive" + + desired_dict = {"exclusive": "exclusive"} + + actual_dict = jobscript_to_dict(jobscript) + + assert actual_dict == desired_dict + + def test_both_exclusive_and_oversubscribe_2(self): + """ + Test that when both are used, the last of them takes precedence. + + In this case, --oversubscribe is the last one. + """ + jobscript = "#SBATCH --exclusive\n#SBATCH --oversubscribe" + + desired_dict = {"exclusive": "oversubscribe"} + + actual_dict = jobscript_to_dict(jobscript) + + assert actual_dict == desired_dict diff --git a/jobbergate-api/jobbergate_api/tests/apps/job_submissions/test_routers.py b/jobbergate-api/jobbergate_api/tests/apps/job_submissions/test_routers.py index 689236a4f..bbdd6dd84 100644 --- a/jobbergate-api/jobbergate_api/tests/apps/job_submissions/test_routers.py +++ b/jobbergate-api/jobbergate_api/tests/apps/job_submissions/test_routers.py @@ -2,6 +2,7 @@ Tests for the /job-submissions/ endpoint. """ import pathlib +from unittest import mock import pytest from fastapi import status @@ -11,7 +12,7 @@ from jobbergate_api.apps.job_scripts.models import job_scripts_table from jobbergate_api.apps.job_submissions.constants import JobSubmissionStatus from jobbergate_api.apps.job_submissions.models import job_submissions_table -from jobbergate_api.apps.job_submissions.schemas import JobSubmissionResponse +from jobbergate_api.apps.job_submissions.schemas import JobProperties, JobSubmissionResponse from jobbergate_api.apps.permissions import Permissions from jobbergate_api.storage import database @@ -58,8 +59,16 @@ async def test_create_job_submission__with_client_id_in_token( create_data.pop("status", None) create_data.pop("client_id", None) - with time_frame() as window: - response = await client.post("/jobbergate/job-submissions/", json=create_data) + with mock.patch( + "jobbergate_api.apps.job_submissions.routers.get_job_properties_from_job_script" + ) as mocked: + mocked.return_value = JobProperties.parse_obj(create_data["execution_parameters"]) + with time_frame() as window: + response = await client.post("/jobbergate/job-submissions/", json=create_data) + mocked.assert_called_once_with( + inserted_job_script_id, + **create_data["execution_parameters"], + ) assert response.status_code == status.HTTP_201_CREATED @@ -70,7 +79,16 @@ async def test_create_job_submission__with_client_id_in_token( assert len(id_rows) == 1 job_submission = JobSubmissionResponse(**response.json()) - assert job_submission.id == id_rows[0][0] + + # Check that the response correspond to the entry in the database + job_submission_raw_data = await database.fetch_one( + query=job_submissions_table.select().where(job_submissions_table.c.id == id_rows[0][0]) + ) + assert job_submission_raw_data is not None + assert job_submission == JobSubmissionResponse(**job_submission_raw_data) # type: ignore + + # Check that each field is correctly set + assert job_submission.id == job_submission_raw_data.get("id") assert job_submission.job_submission_name == "sub1" assert job_submission.job_submission_owner_email == "owner1@org.com" assert job_submission.job_submission_description is None @@ -117,20 +135,27 @@ async def test_create_job_submission__with_client_id_in_request_body( Permissions.JOB_SUBMISSIONS_EDIT, client_id="dummy-cluster-client", ) - with time_frame() as window: - response = await client.post( - "/jobbergate/job-submissions/", - json=fill_job_submission_data( - job_script_id=inserted_job_script_id, - job_submission_name="sub1", - job_submission_owner_email="owner1@org.com", - client_id="silly-cluster-client", - execution_parameters={ - "name": "job-submission-name", - "comment": "I am a comment", - }, - ), - ) + + execution_parameters = { + "name": "job-submission-name", + "comment": "I am a comment", + } + + with mock.patch( + "jobbergate_api.apps.job_submissions.routers.get_job_properties_from_job_script" + ) as mocked: + mocked.return_value = JobProperties.parse_obj(execution_parameters) + with time_frame() as window: + response = await client.post( + "/jobbergate/job-submissions/", + json=fill_job_submission_data( + job_script_id=inserted_job_script_id, + job_submission_name="sub1", + job_submission_owner_email="owner1@org.com", + client_id="silly-cluster-client", + execution_parameters=execution_parameters, + ), + ) assert response.status_code == status.HTTP_201_CREATED @@ -141,7 +166,16 @@ async def test_create_job_submission__with_client_id_in_request_body( assert len(id_rows) == 1 job_submission = JobSubmissionResponse(**response.json()) - assert job_submission.id == id_rows[0][0] + + # Check that the response correspond to the entry in the database + job_submission_raw_data = await database.fetch_one( + query=job_submissions_table.select().where(job_submissions_table.c.id == id_rows[0][0]) + ) + assert job_submission_raw_data is not None + assert job_submission == JobSubmissionResponse(**job_submission_raw_data) # type: ignore + + # Check that each field is correctly set + assert job_submission.id == job_submission_raw_data.get("id") assert job_submission.job_submission_name == "sub1" assert job_submission.job_submission_owner_email == "owner1@org.com" assert job_submission.job_submission_description is None @@ -199,8 +233,16 @@ async def test_create_job_submission__with_execution_directory( create_data.pop("status", None) create_data.pop("client_id", None) - with time_frame() as window: - response = await client.post("/jobbergate/job-submissions/", json=create_data) + with mock.patch( + "jobbergate_api.apps.job_submissions.routers.get_job_properties_from_job_script" + ) as mocked: + mocked.return_value = JobProperties.parse_obj(create_data["execution_parameters"]) + with time_frame() as window: + response = await client.post("/jobbergate/job-submissions/", json=create_data) + mocked.assert_called_once_with( + inserted_job_script_id, + **create_data["execution_parameters"], + ) assert response.status_code == status.HTTP_201_CREATED @@ -211,7 +253,15 @@ async def test_create_job_submission__with_execution_directory( assert len(id_rows) == 1 job_submission = JobSubmissionResponse(**response.json()) - assert job_submission.id == id_rows[0][0] + + # Check that the response correspond to the entry in the database + job_submission_raw_data = await database.fetch_one( + query=job_submissions_table.select().where(job_submissions_table.c.id == id_rows[0][0]) + ) + assert job_submission_raw_data is not None + assert job_submission == JobSubmissionResponse(**job_submission_raw_data) # type: ignore + + assert job_submission.id == job_submission_raw_data.get("id") assert job_submission.job_submission_name == "sub1" assert job_submission.job_submission_owner_email == "owner1@org.com" assert job_submission.job_submission_description is None diff --git a/jobbergate-api/poetry.lock b/jobbergate-api/poetry.lock index f8253678f..f61a2cbf7 100644 --- a/jobbergate-api/poetry.lock +++ b/jobbergate-api/poetry.lock @@ -28,8 +28,8 @@ idna = ">=2.8" sniffio = ">=1.1" [package.extras] -doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] -test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=6.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"] +doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=6.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] trio = ["trio (>=0.16)"] [[package]] @@ -78,7 +78,7 @@ optional = false python-versions = ">=3.7" [package.extras] -tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] +tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] [[package]] name = "asyncpg" @@ -89,9 +89,9 @@ optional = false python-versions = ">=3.5.0" [package.extras] -dev = ["Cython (>=0.29.20,<0.30.0)", "pytest (>=3.6.0)", "Sphinx (>=1.7.3,<1.8.0)", "sphinxcontrib-asyncio (>=0.2.0,<0.3.0)", "sphinx-rtd-theme (>=0.2.4,<0.3.0)", "pycodestyle (>=2.5.0,<2.6.0)", "flake8 (>=3.7.9,<3.8.0)", "uvloop (>=0.14.0,<0.15.0)"] -docs = ["Sphinx (>=1.7.3,<1.8.0)", "sphinxcontrib-asyncio (>=0.2.0,<0.3.0)", "sphinx-rtd-theme (>=0.2.4,<0.3.0)"] -test = ["pycodestyle (>=2.5.0,<2.6.0)", "flake8 (>=3.7.9,<3.8.0)", "uvloop (>=0.14.0,<0.15.0)"] +dev = ["Cython (>=0.29.20,<0.30.0)", "Sphinx (>=1.7.3,<1.8.0)", "flake8 (>=3.7.9,<3.8.0)", "pycodestyle (>=2.5.0,<2.6.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.2.4,<0.3.0)", "sphinxcontrib-asyncio (>=0.2.0,<0.3.0)", "uvloop (>=0.14.0,<0.15.0)"] +docs = ["Sphinx (>=1.7.3,<1.8.0)", "sphinx-rtd-theme (>=0.2.4,<0.3.0)", "sphinxcontrib-asyncio (>=0.2.0,<0.3.0)"] +test = ["flake8 (>=3.7.9,<3.8.0)", "pycodestyle (>=2.5.0,<2.6.0)", "uvloop (>=0.14.0,<0.15.0)"] [[package]] name = "atomicwrites" @@ -110,10 +110,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] name = "auto-name-enum" @@ -131,6 +131,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "bidict" +version = "0.22.0" +description = "The bidirectional mapping library for Python." +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "black" version = "22.3.0" @@ -221,7 +229,7 @@ optional = false python-versions = ">=3.5.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "cli-helpers" @@ -292,12 +300,12 @@ python-versions = ">=3.6" cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx_rtd_theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] [[package]] name = "databases" @@ -313,9 +321,9 @@ sqlalchemy = ">=1.4,<1.5" [package.extras] mysql = ["aiomysql"] -mysql_asyncmy = ["asyncmy"] +mysql-asyncmy = ["asyncmy"] postgresql = ["asyncpg"] -postgresql_aiopg = ["aiopg"] +postgresql-aiopg = ["aiopg"] sqlite = ["aiosqlite"] [[package]] @@ -343,8 +351,8 @@ optional = false python-versions = ">=3.6,<4.0" [package.extras] -dnssec = ["cryptography (>=2.6,<37.0)"] curio = ["curio (>=1.2,<2.0)", "sniffio (>=1.1,<2.0)"] +dnssec = ["cryptography (>=2.6,<37.0)"] doh = ["h2 (>=4.1.0)", "httpx (>=0.21.1)", "requests (>=2.23.0,<3.0.0)", "requests-toolbelt (>=0.9.1,<0.10.0)"] idna = ["idna (>=2.1,<4.0)"] trio = ["trio (>=0.14,<0.20)"] @@ -365,7 +373,7 @@ websocket-client = ">=0.32.0" [package.extras] ssh = ["paramiko (>=2.4.2)"] -tls = ["pyOpenSSL (>=17.5.0)", "cryptography (>=3.4.7)", "idna (>=2.0.0)"] +tls = ["cryptography (>=3.4.7)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] [[package]] name = "ecdsa" @@ -407,10 +415,10 @@ pydantic = ">=1.6.2,<1.7 || >1.7,<1.7.1 || >1.7.1,<1.7.2 || >1.7.2,<1.7.3 || >1. starlette = "0.14.2" [package.extras] -all = ["requests (>=2.24.0,<3.0.0)", "aiofiles (>=0.5.0,<0.8.0)", "jinja2 (>=2.11.2,<3.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "itsdangerous (>=1.1.0,<2.0.0)", "pyyaml (>=5.3.1,<6.0.0)", "graphene (>=2.1.8,<3.0.0)", "ujson (>=4.0.1,<5.0.0)", "orjson (>=3.2.1,<4.0.0)", "email_validator (>=1.1.1,<2.0.0)", "uvicorn[standard] (>=0.12.0,<0.16.0)", "async_exit_stack (>=1.0.1,<2.0.0)", "async_generator (>=1.10,<2.0.0)"] -dev = ["python-jose[cryptography] (>=3.3.0,<4.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "autoflake (>=1.4.0,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "uvicorn[standard] (>=0.12.0,<0.16.0)", "graphene (>=2.1.8,<3.0.0)"] -doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=7.1.9,<8.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "typer-cli (>=0.0.12,<0.0.13)", "pyyaml (>=5.3.1,<6.0.0)"] -test = ["pytest (>=6.2.4,<7.0.0)", "pytest-cov (>=2.12.0,<4.0.0)", "pytest-asyncio (>=0.14.0,<0.16.0)", "mypy (==0.910)", "flake8 (>=3.8.3,<4.0.0)", "black (==21.9b0)", "isort (>=5.0.6,<6.0.0)", "requests (>=2.24.0,<3.0.0)", "httpx (>=0.14.0,<0.19.0)", "email_validator (>=1.1.1,<2.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "peewee (>=3.13.3,<4.0.0)", "databases[sqlite] (>=0.3.2,<0.6.0)", "orjson (>=3.2.1,<4.0.0)", "ujson (>=4.0.1,<5.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "aiofiles (>=0.5.0,<0.8.0)", "flask (>=1.1.2,<2.0.0)", "async_exit_stack (>=1.0.1,<2.0.0)", "async_generator (>=1.10,<2.0.0)", "types-ujson (==0.1.1)", "types-orjson (==3.6.0)", "types-dataclasses (==0.1.7)"] +all = ["aiofiles (>=0.5.0,<0.8.0)", "async_exit_stack (>=1.0.1,<2.0.0)", "async_generator (>=1.10,<2.0.0)", "email_validator (>=1.1.1,<2.0.0)", "graphene (>=2.1.8,<3.0.0)", "itsdangerous (>=1.1.0,<2.0.0)", "jinja2 (>=2.11.2,<3.0.0)", "orjson (>=3.2.1,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "pyyaml (>=5.3.1,<6.0.0)", "requests (>=2.24.0,<3.0.0)", "ujson (>=4.0.1,<5.0.0)", "uvicorn[standard] (>=0.12.0,<0.16.0)"] +dev = ["autoflake (>=1.4.0,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "graphene (>=2.1.8,<3.0.0)", "passlib[bcrypt] (>=1.7.2,<2.0.0)", "python-jose[cryptography] (>=3.3.0,<4.0.0)", "uvicorn[standard] (>=0.12.0,<0.16.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-markdownextradata-plugin (>=0.1.7,<0.3.0)", "mkdocs-material (>=7.1.9,<8.0.0)", "pyyaml (>=5.3.1,<6.0.0)", "typer-cli (>=0.0.12,<0.0.13)"] +test = ["aiofiles (>=0.5.0,<0.8.0)", "async_exit_stack (>=1.0.1,<2.0.0)", "async_generator (>=1.10,<2.0.0)", "black (==21.9b0)", "databases[sqlite] (>=0.3.2,<0.6.0)", "email_validator (>=1.1.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "flask (>=1.1.2,<2.0.0)", "httpx (>=0.14.0,<0.19.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "orjson (>=3.2.1,<4.0.0)", "peewee (>=3.13.3,<4.0.0)", "pytest (>=6.2.4,<7.0.0)", "pytest-asyncio (>=0.14.0,<0.16.0)", "pytest-cov (>=2.12.0,<4.0.0)", "python-multipart (>=0.0.5,<0.0.6)", "requests (>=2.24.0,<3.0.0)", "sqlalchemy (>=1.3.18,<1.5.0)", "types-dataclasses (==0.1.7)", "types-orjson (==3.6.0)", "types-ujson (==0.1.1)", "ujson (>=4.0.1,<5.0.0)"] [[package]] name = "file-storehouse" @@ -480,7 +488,7 @@ optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" [package.extras] -docs = ["sphinx"] +docs = ["Sphinx"] [[package]] name = "h11" @@ -555,9 +563,9 @@ python-versions = ">=3.7" zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "importlib-resources" @@ -571,8 +579,8 @@ python-versions = ">=3.7" zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] name = "iniconfig" @@ -601,6 +609,7 @@ pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} pickleshare = "*" prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0" pygments = "*" +setuptools = ">=18.5" traitlets = ">=4.2" [package.extras] @@ -609,10 +618,10 @@ doc = ["Sphinx (>=1.3)"] kernel = ["ipykernel"] nbconvert = ["nbconvert"] nbformat = ["nbformat"] -notebook = ["notebook", "ipywidgets"] +notebook = ["ipywidgets", "notebook"] parallel = ["ipyparallel"] qtconsole = ["qtconsole"] -test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.17)"] +test = ["ipykernel", "nbformat", "nose (>=0.10.1)", "numpy (>=1.17)", "pygments", "requests", "testpath"] [[package]] name = "isort" @@ -623,10 +632,10 @@ optional = false python-versions = ">=3.6.1,<4.0" [package.extras] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] -requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] [[package]] name = "jedi" @@ -678,7 +687,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "tox (>=3.9.0)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "black (>=19.10b0)", "isort (>=5.1.1)", "Sphinx (>=4.1.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)"] +dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"] [[package]] name = "mako" @@ -692,7 +701,7 @@ python-versions = ">=3.7" MarkupSafe = ">=0.9.2" [package.extras] -babel = ["babel"] +babel = ["Babel"] lingua = ["lingua"] testing = ["pytest"] @@ -806,7 +815,7 @@ python-versions = "*" [package.extras] argon2 = ["argon2-cffi (>=18.2.0)"] bcrypt = ["bcrypt (>=3.1.0)"] -build_docs = ["sphinx (>=1.6)", "sphinxcontrib-fulltoc (>=1.2.0)", "cloud-sptheme (>=1.10.1)"] +build-docs = ["cloud-sptheme (>=1.10.1)", "sphinx (>=1.6)", "sphinxcontrib-fulltoc (>=1.2.0)"] totp = ["cryptography"] [[package]] @@ -1130,7 +1139,7 @@ pytest = ">=4.6" toml = "*" [package.extras] -testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] [[package]] name = "pytest-env" @@ -1212,8 +1221,8 @@ rsa = "*" [package.extras] cryptography = ["cryptography (>=3.4.0)"] -pycrypto = ["pycrypto (>=2.6.0,<2.7.0)", "pyasn1"] -pycryptodome = ["pycryptodome (>=3.3.1,<4.0.0)", "pyasn1"] +pycrypto = ["pyasn1", "pycrypto (>=2.6.0,<2.7.0)"] +pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] [[package]] name = "python-multipart" @@ -1274,7 +1283,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<5)"] [[package]] name = "respx" @@ -1358,11 +1367,11 @@ celery = ["celery (>=3)"] chalice = ["chalice (>=1.16.0)"] django = ["django (>=1.8)"] falcon = ["falcon (>=1.4)"] -flask = ["flask (>=0.11)", "blinker (>=1.1)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)"] httpx = ["httpx (>=0.16.0)"] -pure_eval = ["pure-eval", "executing", "asttokens"] +pure-eval = ["asttokens", "executing", "pure-eval"] pyspark = ["pyspark (>=2.4.4)"] -quart = ["quart (>=0.16.1)", "blinker (>=1.1)"] +quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] rq = ["rq (>=0.6)"] sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] @@ -1379,6 +1388,19 @@ python-versions = ">=3.6" [package.extras] test = ["pytest (>=6.1,<6.2)"] +[[package]] +name = "setuptools" +version = "65.6.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "six" version = "1.16.0" @@ -1426,25 +1448,25 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} [package.extras] -aiomysql = ["greenlet (!=0.4.17)", "aiomysql"] -aiosqlite = ["typing_extensions (!=3.10.0.1)", "greenlet (!=0.4.17)", "aiosqlite"] +aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["greenlet (!=0.4.17)", "asyncmy (>=0.2.3,!=0.2.4)"] -mariadb_connector = ["mariadb (>=1.0.1)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1)"] mssql = ["pyodbc"] -mssql_pymssql = ["pymssql"] -mssql_pyodbc = ["pyodbc"] -mypy = ["sqlalchemy2-stubs", "mypy (>=0.910)"] -mysql = ["mysqlclient (>=1.4.0,<2)", "mysqlclient (>=1.4.0)"] -mysql_connector = ["mysql-connector-python"] -oracle = ["cx_oracle (>=7,<8)", "cx_oracle (>=7)"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] +mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] -postgresql_asyncpg = ["greenlet (!=0.4.17)", "asyncpg"] -postgresql_pg8000 = ["pg8000 (>=1.16.6)"] -postgresql_psycopg2binary = ["psycopg2-binary"] -postgresql_psycopg2cffi = ["psycopg2cffi"] -pymysql = ["pymysql (<1)", "pymysql"] -sqlcipher = ["sqlcipher3-binary"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.16.6)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql", "pymysql (<1)"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlalchemy-stubs" @@ -1540,8 +1562,8 @@ click = ">=7.1.1,<9.0.0" [package.extras] all = ["colorama (>=0.4.3,<0.5.0)", "shellingham (>=1.3.0,<2.0.0)"] dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)"] -doc = ["mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "mdx-include (>=1.4.1,<2.0.0)"] -test = ["shellingham (>=1.3.0,<2.0.0)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "coverage (>=5.2,<6.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "mypy (==0.910)", "black (>=22.3.0,<23.0.0)", "isort (>=5.0.6,<6.0.0)"] +doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "shellingham (>=1.3.0,<2.0.0)"] [[package]] name = "typing-extensions" @@ -1560,8 +1582,8 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] -brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -1578,7 +1600,7 @@ click = ">=7.0" h11 = ">=0.8" [package.extras] -standard = ["websockets (>=9.1)", "httptools (>=0.2.0,<0.3.0)", "watchgod (>=0.6)", "python-dotenv (>=0.13)", "PyYAML (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "colorama (>=0.4)"] +standard = ["PyYAML (>=5.1)", "colorama (>=0.4)", "httptools (>=0.2.0,<0.3.0)", "python-dotenv (>=0.13)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchgod (>=0.6)", "websockets (>=9.1)"] [[package]] name = "virtualenv" @@ -1596,7 +1618,7 @@ six = ">=1.9.0,<2" [package.extras] docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=21.3)"] -testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "packaging (>=20.0)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)"] [[package]] name = "wcwidth" @@ -1628,7 +1650,7 @@ optional = false python-versions = ">=3.5" [package.extras] -dev = ["pytest (>=4.6.2)", "black (>=19.3b0)"] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [[package]] name = "yarl" @@ -1651,13 +1673,13 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "5efe7e854001be8049ec6e16c19959952dfd175c53fba791052e325422cb01f7" +content-hash = "07928c39e2bc417c0a8774db67830fa89dc0a6da5ae5af948d08c78e0fe21abf" [metadata.files] alembic = [ @@ -1672,7 +1694,10 @@ appnope = [ {file = "appnope-0.1.3-py2.py3-none-any.whl", hash = "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e"}, {file = "appnope-0.1.3.tar.gz", hash = "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24"}, ] -armasec = [] +armasec = [ + {file = "armasec-0.11.0-py3-none-any.whl", hash = "sha256:2eaddb6efd3ec37326240a98661144b660f7b8a412f21a37c2ac9e35099235c1"}, + {file = "armasec-0.11.0.tar.gz", hash = "sha256:613ee28ce11fa1de1d8c13b33f8d77b8d5194eaf4dc2b4a59547b32a79d78d9b"}, +] asgi-lifespan = [ {file = "asgi-lifespan-1.0.1.tar.gz", hash = "sha256:9a33e7da2073c4764bc79bd6136501d6c42f60e3d2168ba71235e84122eadb7f"}, {file = "asgi_lifespan-1.0.1-py3-none-any.whl", hash = "sha256:9ea969dc5eb5cf08e52c08dce6f61afcadd28112e72d81c972b1d8eb8691ab53"}, @@ -1706,11 +1731,18 @@ attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] -auto-name-enum = [] +auto-name-enum = [ + {file = "auto-name-enum-2.0.0.tar.gz", hash = "sha256:aa38d9c2730b95b0bb700cd93fb5f65a2a982f6d79bbbcd0c64e5bcfe8f3d5b1"}, + {file = "auto_name_enum-2.0.0-py3-none-any.whl", hash = "sha256:fb9fe25985d7ef890e7b419d242b9ec145162ae1252121f8a5e3e8cca0e6d87a"}, +] backcall = [ {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, ] +bidict = [ + {file = "bidict-0.22.0-py3-none-any.whl", hash = "sha256:415126d23a0c81e1a8c584a8fb1f6905ea090c772571803aeee0a2242e8e7ba0"}, + {file = "bidict-0.22.0.tar.gz", hash = "sha256:5c826b3e15e97cc6e615de295756847c282a79b79c5430d3bfc909b1ac9f5bd8"}, +] black = [ {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, @@ -1736,8 +1768,14 @@ black = [ {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, ] -boto3 = [] -botocore = [] +boto3 = [ + {file = "boto3-1.24.38-py3-none-any.whl", hash = "sha256:bcf97fd7c494f4e2bbbe2511625500654179c0a6b3bea977d46f97af764e85a4"}, + {file = "boto3-1.24.38.tar.gz", hash = "sha256:f4c6b025f392c934338c7f01badfddbd0d3cf2397ff5df35c31409798dce33f5"}, +] +botocore = [ + {file = "botocore-1.27.38-py3-none-any.whl", hash = "sha256:46a0264ff3335496bd9cb404f83ec0d8eb7bfdef8f74a830c13e6a6b9612adea"}, + {file = "botocore-1.27.38.tar.gz", hash = "sha256:56a7682564ea57ceecfef5648f77b77e0543b9c904212fc9ef4416517d24fa45"}, +] certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, @@ -2315,18 +2353,7 @@ py-docker-gadgets = [ {file = "py_docker_gadgets-0.1.1-py3-none-any.whl", hash = "sha256:199ced9284d25628e699b435bac61431aeef4aa20533a16e0f711ee748a25105"}, ] pyasn1 = [ - {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"}, - {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"}, - {file = "pyasn1-0.4.8-py2.6.egg", hash = "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00"}, - {file = "pyasn1-0.4.8-py2.7.egg", hash = "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8"}, {file = "pyasn1-0.4.8-py2.py3-none-any.whl", hash = "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d"}, - {file = "pyasn1-0.4.8-py3.1.egg", hash = "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86"}, - {file = "pyasn1-0.4.8-py3.2.egg", hash = "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7"}, - {file = "pyasn1-0.4.8-py3.3.egg", hash = "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576"}, - {file = "pyasn1-0.4.8-py3.4.egg", hash = "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12"}, - {file = "pyasn1-0.4.8-py3.5.egg", hash = "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2"}, - {file = "pyasn1-0.4.8-py3.6.egg", hash = "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359"}, - {file = "pyasn1-0.4.8-py3.7.egg", hash = "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776"}, {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, ] pycodestyle = [ @@ -2420,7 +2447,10 @@ python-dateutil = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] -python-dotenv = [] +python-dotenv = [ + {file = "python-dotenv-0.20.0.tar.gz", hash = "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f"}, + {file = "python_dotenv-0.20.0-py3-none-any.whl", hash = "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"}, +] python-http-client = [ {file = "python_http_client-3.3.7-py3-none-any.whl", hash = "sha256:ad371d2bbedc6ea15c26179c6222a78bc9308d272435ddf1d5c84f068f249a36"}, {file = "python_http_client-3.3.7.tar.gz", hash = "sha256:bf841ee45262747e00dec7ee9971dfb8c7d83083f5713596488d67739170cea0"}, @@ -2461,6 +2491,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -2539,6 +2576,10 @@ setproctitle = [ {file = "setproctitle-1.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:4fc5bebd34f451dc87d2772ae6093adea1ea1dc29afc24641b250140decd23bb"}, {file = "setproctitle-1.2.2.tar.gz", hash = "sha256:7dfb472c8852403d34007e01d6e3c68c57eb66433fb8a5c77b13b89a160d97df"}, ] +setuptools = [ + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/jobbergate-api/pyproject.toml b/jobbergate-api/pyproject.toml index 4e2f09119..a9a77da9e 100644 --- a/jobbergate-api/pyproject.toml +++ b/jobbergate-api/pyproject.toml @@ -42,6 +42,7 @@ sendgrid = "^6.9.7" file-storehouse = "0.5.0" py-buzz = "^3.2.1" toml = "^0.10.2" +bidict = "^0.22.0" [tool.poetry.dev-dependencies] black = "^22" @@ -49,7 +50,7 @@ pre-commit = "^2.9.2" pytest = "^6.2" pytest-asyncio = "^0.12" pytest-cov = "^2.8" -python-status = "i^1.0" +python-status = "^1.0" requests = "^2.23.0" nest_asyncio = "^1.3.3" coverage = "^5.1"