diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fea0e204..cc9062b2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,13 +16,11 @@ repos: - id: flake8 args: - --ignore=E203,W503 - - --max-line-length=120 + - --max-line-length=100 files: '\.py$' additional_dependencies: [ flake8 ] - - - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.4.1 hooks: diff --git a/import_specifications/clients/authclient.py b/import_specifications/clients/authclient.py index bc492fb2..a7a354bb 100644 --- a/import_specifications/clients/authclient.py +++ b/import_specifications/clients/authclient.py @@ -46,9 +46,7 @@ def add_valid_token(self, token, user): with self._lock: self._cache[token] = [user, _time.time()] if len(self._cache) > self._maxsize: - sorted_items = sorted( - list(self._cache.items()), key=(lambda v: v[1][1]) - ) + sorted_items = sorted(list(self._cache.items()), key=(lambda v: v[1][1])) for i, (t, _) in enumerate(sorted_items): if i <= self._halfmax: del self._cache[t] diff --git a/import_specifications/clients/narrative_method_store_client.py b/import_specifications/clients/narrative_method_store_client.py index 3655b0b2..250830b4 100644 --- a/import_specifications/clients/narrative_method_store_client.py +++ b/import_specifications/clients/narrative_method_store_client.py @@ -49,9 +49,7 @@ def ver(self, context=None): Returns the current running version of the NarrativeMethodStore. :returns: instance of String """ - return self._client.call_method( - "NarrativeMethodStore.ver", [], self._service_ver, context - ) + return self._client.call_method("NarrativeMethodStore.ver", [], self._service_ver, context) def status(self, context=None): """ diff --git a/import_specifications/generate_import_template.py b/import_specifications/generate_import_template.py index 9acf92e4..86e58339 100755 --- a/import_specifications/generate_import_template.py +++ b/import_specifications/generate_import_template.py @@ -32,9 +32,7 @@ def parse_args(): - parser = argparse.ArgumentParser( - description="Generate a bulk import template for an app" - ) + parser = argparse.ArgumentParser(description="Generate a bulk import template for an app") parser.add_argument( "app_id", help="The app ID to process, for example kb_uploadmethods/import_sra_as_reads_from_staging", @@ -67,9 +65,7 @@ def is_file_input(param): if param["field_type"] != "dynamic_dropdown": return False if "dynamic_dropdown_options" not in param: - raise ValueError( - "Missing dynamic_dropdown_options field for dynamic_dropdown input" - ) + raise ValueError("Missing dynamic_dropdown_options field for dynamic_dropdown input") return param["dynamic_dropdown_options"].get("data_source") == "ftp_staging" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..c1009e68 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,77 @@ +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +# Same as Black. +line-length = 100 +indent-width = 4 + +target-version = "py39" + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" diff --git a/scripts/prune_acls.py b/scripts/prune_acls.py index ee8a4504..44934e9f 100644 --- a/scripts/prune_acls.py +++ b/scripts/prune_acls.py @@ -34,9 +34,7 @@ client = globus_sdk.NativeAppAuthClient(cf["client_id"]) try: - transfer_authorizer = globus_sdk.RefreshTokenAuthorizer( - cf["transfer_token"], client - ) + transfer_authorizer = globus_sdk.RefreshTokenAuthorizer(cf["transfer_token"], client) globus_transfer_client = globus_sdk.TransferClient(authorizer=transfer_authorizer) auth_authorizer = globus_sdk.RefreshTokenAuthorizer(cf["auth_token"], client) globus_auth_client = globus_sdk.AuthClient(authorizer=auth_authorizer) @@ -54,9 +52,7 @@ def remove_directory(directory): logging.info("About to delete {}".format(directory)) # shutil.rmtree(directory) except OSError as error: - logging.error( - "Couldn't delete {} {} {}".format(directory, error.message, error.filename) - ) + logging.error("Couldn't delete {} {} {}".format(directory, error.message, error.filename)) def remove_acl(acl): @@ -81,9 +77,7 @@ def main(): old_acls = get_old_acls() - logging.info( - "{}:ATTEMPTING TO DELETE {} OLD ACLS".format(current_time, len(old_acls)) - ) + logging.info("{}:ATTEMPTING TO DELETE {} OLD ACLS".format(current_time, len(old_acls))) for acl in old_acls: remove_acl(acl.acl) remove_directory(acl.dir) diff --git a/staging_service/app.py b/staging_service/app.py index d014a55d..1b078f04 100644 --- a/staging_service/app.py +++ b/staging_service/app.py @@ -93,13 +93,7 @@ def _file_type_resolver(path: PathPy) -> FileTypeResolution: if ftype in _IMPSPEC_FILE_TO_PARSER: return FileTypeResolution(parser=_IMPSPEC_FILE_TO_PARSER[ftype]) else: - ext = ( - fi["suffix"] - if fi["suffix"] - else path.suffix[1:] - if path.suffix - else path.name - ) + ext = fi["suffix"] if fi["suffix"] else path.suffix[1:] if path.suffix else path.name return FileTypeResolution(unsupported_type=ext) @@ -124,9 +118,7 @@ async def bulk_specification(request: web.Request) -> web.json_response: res = parse_import_specifications( tuple(list(paths)), _file_type_resolver, - lambda e: logging.error( - "Unexpected error while parsing import specs", exc_info=e - ), + lambda e: logging.error("Unexpected error while parsing import specs", exc_info=e), ) if res.results: types = {dt: result.result for dt, result in res.results.items()} @@ -192,9 +184,7 @@ async def write_bulk_specification(request: web.Request) -> web.json_response: folder = data.get("output_directory") type_ = data.get("output_file_type") if type(folder) != str: # noqa E721 - return _createJSONErrorResponse( - "output_directory is required and must be a string" - ) + return _createJSONErrorResponse("output_directory is required and must be a string") writer = _IMPSPEC_FILE_TO_WRITER.get(type_) if not writer: return _createJSONErrorResponse(f"Invalid output_file_type: {type_}") @@ -219,12 +209,8 @@ async def add_acl_concierge(request: web.Request): user_dir = Path.validate_path(username).full_path concierge_path = f"{Path._CONCIERGE_PATH}/{username}/" aclm = AclManager() - result = aclm.add_acl_concierge( - shared_directory=user_dir, concierge_path=concierge_path - ) - result["msg"] = ( - f"Requesting Globus Perms for the following globus dir: {concierge_path}" - ) + result = aclm.add_acl_concierge(shared_directory=user_dir, concierge_path=concierge_path) + result["msg"] = f"Requesting Globus Perms for the following globus dir: {concierge_path}" result["link"] = ( f"https://app.globus.org/file-manager?destination_id={aclm.endpoint_id}&destination_path={concierge_path}" ) @@ -298,9 +284,7 @@ async def list_files(request: web.Request): username = await authorize_request(request) path = Path.validate_path(username, request.match_info.get("path", "")) if not os.path.exists(path.full_path): - raise web.HTTPNotFound( - text="path {path} does not exist".format(path=path.user_path) - ) + raise web.HTTPNotFound(text="path {path} does not exist".format(path=path.user_path)) elif os.path.isfile(path.full_path): raise web.HTTPBadRequest( text="{path} is a file not a directory".format(path=path.full_path) @@ -325,17 +309,13 @@ async def download_files(request: web.Request): username = await authorize_request(request) path = Path.validate_path(username, request.match_info.get("path", "")) if not os.path.exists(path.full_path): - raise web.HTTPNotFound( - text="path {path} does not exist".format(path=path.user_path) - ) + raise web.HTTPNotFound(text="path {path} does not exist".format(path=path.user_path)) elif not os.path.isfile(path.full_path): raise web.HTTPBadRequest( text="{path} is a directory not a file".format(path=path.full_path) ) # hard coding the mime type to force download - return web.FileResponse( - path.full_path, headers={"content-type": "application/octet-stream"} - ) + return web.FileResponse(path.full_path, headers={"content-type": "application/octet-stream"}) @routes.get("/similar/{path:.+}") @@ -346,9 +326,7 @@ async def similar_files(request: web.Request): username = await authorize_request(request) path = Path.validate_path(username, request.match_info["path"]) if not os.path.exists(path.full_path): - raise web.HTTPNotFound( - text="path {path} does not exist".format(path=path.user_path) - ) + raise web.HTTPNotFound(text="path {path} does not exist".format(path=path.user_path)) elif os.path.isdir(path.full_path): raise web.HTTPBadRequest( text="{path} is a directory not a file".format(path=path.full_path) @@ -400,9 +378,7 @@ async def get_metadata(request: web.Request): username = await authorize_request(request) path = Path.validate_path(username, request.match_info["path"]) if not os.path.exists(path.full_path): - raise web.HTTPNotFound( - text="path {path} does not exist".format(path=path.user_path) - ) + raise web.HTTPNotFound(text="path {path} does not exist".format(path=path.user_path)) return web.json_response(await some_metadata(path)) @@ -430,9 +406,7 @@ async def upload_files_chunked(request: web.Request): counter = 0 user_file = None destPath = None - while ( - counter < 100 - ): # TODO this is arbitrary to keep an attacker from creating infinite loop + while counter < 100: # TODO this is arbitrary to keep an attacker from creating infinite loop # This loop handles the null parts that come in inbetween destpath and file part = await reader.next() @@ -498,9 +472,7 @@ async def define_UPA(request: web.Request): path = Path.validate_path(username, request.match_info["path"]) if not os.path.exists(path.full_path or not os.path.isfile(path.full_path)): # TODO the security model here is to not care if someone wants to put in a false upa - raise web.HTTPNotFound( - text="no file found found on path {}".format(path.user_path) - ) + raise web.HTTPNotFound(text="no file found found on path {}".format(path.user_path)) if not request.has_body: raise web.HTTPBadRequest(text="must provide UPA field in body") body = await request.post() @@ -510,9 +482,7 @@ async def define_UPA(request: web.Request): raise web.HTTPBadRequest(text="must provide UPA field in body") await add_upa(path, UPA) return web.Response( - text="succesfully updated UPA {UPA} for file {path}".format( - UPA=UPA, path=path.user_path - ) + text="succesfully updated UPA {UPA} for file {path}".format(UPA=UPA, path=path.user_path) ) @@ -537,9 +507,7 @@ async def delete(request: web.Request): if os.path.exists(path.metadata_path): shutil.rmtree(path.metadata_path) else: - raise web.HTTPNotFound( - text="could not delete {path}".format(path=path.user_path) - ) + raise web.HTTPNotFound(text="could not delete {path}".format(path=path.user_path)) return web.Response(text="successfully deleted {path}".format(path=path.user_path)) @@ -591,13 +559,9 @@ async def decompress(request: web.Request): # 2 could try again after doign an automatic rename scheme (add nubmers to end) # 3 just overwrite and force destination = os.path.dirname(path.full_path) - if ( - upper_file_extension == ".tar" and file_extension == ".gz" - ) or file_extension == ".tgz": + if (upper_file_extension == ".tar" and file_extension == ".gz") or file_extension == ".tgz": await run_command("tar", "xzf", path.full_path, "-C", destination) - elif upper_file_extension == ".tar" and ( - file_extension == ".bz" or file_extension == ".bz2" - ): + elif upper_file_extension == ".tar" and (file_extension == ".bz" or file_extension == ".bz2"): await run_command("tar", "xjf", path.full_path, "-C", destination) elif file_extension == ".zip" or file_extension == ".ZIP": await run_command("unzip", path.full_path, "-d", destination) @@ -608,9 +572,7 @@ async def decompress(request: web.Request): elif file_extension == ".bz2" or file_extension == "bzip2": await run_command("bzip2", "-d", path.full_path) else: - raise web.HTTPBadRequest( - text="cannot decompress a {ext} file".format(ext=file_extension) - ) + raise web.HTTPBadRequest(text="cannot decompress a {ext} file".format(ext=file_extension)) return web.Response(text="succesfully decompressed " + path.user_path) diff --git a/staging_service/app_error_formatter.py b/staging_service/app_error_formatter.py index d76ae06c..70a67918 100644 --- a/staging_service/app_error_formatter.py +++ b/staging_service/app_error_formatter.py @@ -33,11 +33,7 @@ "file": file1, "tab": tab1, }, - ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE: lambda msg, - file1, - tab1, - file2, - tab2: { + ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE: lambda msg, file1, tab1, file2, tab2: { "type": "multiple_specifications_for_data_type", "message": msg, "file_1": file1, @@ -71,7 +67,5 @@ def format_import_spec_errors( if e.source_2: file2 = str(path_translations[e.source_2.file]) tab2 = e.source_2.tab - errs.append( - _IMPORT_SPEC_ERROR_FORMATTERS[e.error](e.message, file1, tab1, file2, tab2) - ) + errs.append(_IMPORT_SPEC_ERROR_FORMATTERS[e.error](e.message, file1, tab1, file2, tab2)) return errs diff --git a/staging_service/auth2Client.py b/staging_service/auth2Client.py index a7917eab..414ba946 100644 --- a/staging_service/auth2Client.py +++ b/staging_service/auth2Client.py @@ -41,9 +41,7 @@ def add_valid_token(self, token, user, expire_time): token = hashlib.sha256(token.encode("utf8")).hexdigest() self._cache[token] = [user, _time.time(), expire_time] if len(self._cache) > self._maxsize: - for i, (t, _) in enumerate( - sorted(self._cache.items(), key=lambda v: v[1][1]) - ): + for i, (t, _) in enumerate(sorted(self._cache.items(), key=lambda v: v[1][1])): if i <= self._halfmax: del self._cache[t] else: @@ -69,9 +67,7 @@ async def get_user(self, token): if user: return user async with aiohttp.ClientSession() as session: - async with session.get( - self._authurl, headers={"Authorization": token} - ) as resp: + async with session.get(self._authurl, headers={"Authorization": token}) as resp: ret = await resp.json() if not resp.reason == "OK": raise aiohttp.web.HTTPUnauthorized( diff --git a/staging_service/autodetect/Mappings.py b/staging_service/autodetect/Mappings.py index 86a06124..51d27091 100644 --- a/staging_service/autodetect/Mappings.py +++ b/staging_service/autodetect/Mappings.py @@ -69,9 +69,7 @@ def _flatten(some_list): # longer term there's probably a better way to do this but this is quick def _add_gzip(extension_list): - return _flatten( - [[ext + comp for comp in _COMPRESSION_EXT] for ext in extension_list] - ) + return _flatten([[ext + comp for comp in _COMPRESSION_EXT] for ext in extension_list]) file_format_to_extension_mapping = { @@ -114,7 +112,5 @@ def _add_gzip(extension_list): for ext in extensions: if ext in extension_to_file_format_mapping: type2 = extension_to_file_format_mapping[ext] - raise ValueError( - f"Duplicate entry for extension {ext} in {type_} and {type2}" - ) + raise ValueError(f"Duplicate entry for extension {ext} in {type_} and {type2}") extension_to_file_format_mapping[ext] = type_ diff --git a/staging_service/import_specifications/file_parser.py b/staging_service/import_specifications/file_parser.py index 1fea399d..c9cf7651 100644 --- a/staging_service/import_specifications/file_parser.py +++ b/staging_service/import_specifications/file_parser.py @@ -102,9 +102,7 @@ def __post_init__(self): for attr in attrs: if not getattr(self, attr): # grammar sucks but this is not expected to be seen by end users so meh - raise ValueError( - f"{', '.join(attrs)} is required for a {self.error.name} error" - ) + raise ValueError(f"{', '.join(attrs)} is required for a {self.error.name} error") @dataclass(frozen=True) @@ -174,9 +172,7 @@ class FileTypeResolution: def __post_init__(self): if not (bool(self.parser) ^ bool(self.unsupported_type)): # xnor - raise ValueError( - "Exectly one of parser or unsupported_type must be supplied" - ) + raise ValueError("Exectly one of parser or unsupported_type must be supplied") def parse_import_specifications( diff --git a/staging_service/import_specifications/file_writers.py b/staging_service/import_specifications/file_writers.py index 1c40fd9e..35b5423c 100644 --- a/staging_service/import_specifications/file_writers.py +++ b/staging_service/import_specifications/file_writers.py @@ -90,13 +90,9 @@ def _check_import_specification(types: dict[str, dict[str, list[Any]]]): _check_string(datatype, "A data type") spec = types[datatype] if not isinstance(type(spec), dict): # noqa: E721 - raise ImportSpecWriteException( - f"The value for data type {datatype} must be a mapping" - ) + raise ImportSpecWriteException(f"The value for data type {datatype} must be a mapping") if _ORDER_AND_DISPLAY not in spec: - raise ImportSpecWriteException( - f"Data type {datatype} missing {_ORDER_AND_DISPLAY} key" - ) + raise ImportSpecWriteException(f"Data type {datatype} missing {_ORDER_AND_DISPLAY} key") _check_is_sequence( spec[_ORDER_AND_DISPLAY], f"Data type {datatype} {_ORDER_AND_DISPLAY} value" ) @@ -110,10 +106,7 @@ def _check_import_specification(types: dict[str, dict[str, list[Any]]]): param_ids = set() for i, id_display in enumerate(spec[_ORDER_AND_DISPLAY]): - err = ( - f"Invalid {_ORDER_AND_DISPLAY} entry for datatype {datatype} " - + f"at index {i} " - ) + err = f"Invalid {_ORDER_AND_DISPLAY} entry for datatype {datatype} " + f"at index {i} " _check_is_sequence(id_display, err + "- the entry") if len(id_display) != 2: raise ImportSpecWriteException(err + "- expected 2 item list") @@ -130,14 +123,9 @@ def _check_import_specification(types: dict[str, dict[str, list[Any]]]): err + f" does not have the same keys as {_ORDER_AND_DISPLAY}" ) for pid, v in datarow.items(): - if ( - v is not None - and not isinstance(v, numbers.Number) - and not isinstance(v, str) - ): + if v is not None and not isinstance(v, numbers.Number) and not isinstance(v, str): raise ImportSpecWriteException( - err - + f"'s value for parameter {pid} is not a number or a string" + err + f"'s value for parameter {pid} is not a number or a string" ) @@ -149,9 +137,7 @@ def _check_string(tocheck: Any, errprefix: str): def _check_is_sequence(tocheck: Any, errprefix: str): - if not ( - isinstance(tocheck, collections.abc.Sequence) and not isinstance(tocheck, str) - ): + if not (isinstance(tocheck, collections.abc.Sequence) and not isinstance(tocheck, str)): raise ImportSpecWriteException(errprefix + " is not a list") @@ -171,9 +157,7 @@ def write_tsv(folder: Path, types: dict[str, dict[str, list[Any]]]) -> dict[str, return _write_xsv(folder, types, _EXT_TSV, _SEP_TSV) -def _write_xsv( - folder: Path, types: dict[str, dict[str, list[Any]]], ext: str, sep: str -): +def _write_xsv(folder: Path, types: dict[str, dict[str, list[Any]]], ext: str, sep: str): _check_write_args(folder, types) res = {} for datatype in types: @@ -208,9 +192,7 @@ def _check_write_args(folder: Path, types: dict[str, dict[str, list[Any]]]): _check_import_specification(types) -def write_excel( - folder: Path, types: dict[str, dict[str, list[Any]]] -) -> dict[str, Path]: +def write_excel(folder: Path, types: dict[str, dict[str, list[Any]]]) -> dict[str, Path]: """ Writes import specifications to an Excel files. All the writers in this module have the same function signatures; see the module level documentation. @@ -255,9 +237,7 @@ def _expand_excel_columns_to_max_width(sheet: Worksheet): # https://stackoverflow.com/a/40935194/643675 for column_cells in sheet.columns: length = max(len(_as_text(cell.value)) for cell in column_cells) - sheet.column_dimensions[ - get_column_letter(column_cells[0].column) - ].width = length + sheet.column_dimensions[get_column_letter(column_cells[0].column)].width = length def _as_text(value): diff --git a/staging_service/import_specifications/individual_parsers.py b/staging_service/import_specifications/individual_parsers.py index cc975494..8b6ffeff 100644 --- a/staging_service/import_specifications/individual_parsers.py +++ b/staging_service/import_specifications/individual_parsers.py @@ -179,9 +179,7 @@ def _parse_xsv(path: Path, sep: str) -> ParseResults: try: filetype = magic.from_file(str(path), mime=True) if filetype not in _MAGIC_TEXT_FILES: - return _error( - Error(ErrorType.PARSE_FAIL, "Not a text file: " + filetype, spcsrc) - ) + return _error(Error(ErrorType.PARSE_FAIL, "Not a text file: " + filetype, spcsrc)) with open(path, newline="") as input_: rdr = csv.reader(input_, delimiter=sep) # let parser handle quoting dthd = _csv_next(rdr, 1, None, spcsrc, "Missing data type / version header") @@ -204,24 +202,15 @@ def _parse_xsv(path: Path, sep: str) -> ParseResults: ) ) results.append( - frozendict( - { - param_ids[j]: _normalize_xsv(row[j]) - for j in range(len(row)) - } - ) + frozendict({param_ids[j]: _normalize_xsv(row[j]) for j in range(len(row))}) ) if not results: - raise _ParseException( - Error(ErrorType.PARSE_FAIL, "No non-header data in file", spcsrc) - ) + raise _ParseException(Error(ErrorType.PARSE_FAIL, "No non-header data in file", spcsrc)) return ParseResults(frozendict({datatype: ParseResult(spcsrc, tuple(results))})) except FileNotFoundError: return _error(Error(ErrorType.FILE_NOT_FOUND, source_1=spcsrc)) except IsADirectoryError: - return _error( - Error(ErrorType.PARSE_FAIL, "The given path is a directory", spcsrc) - ) + return _error(Error(ErrorType.PARSE_FAIL, "The given path is a directory", spcsrc)) except _ParseException as e: return _error(e.args[0]) @@ -257,9 +246,7 @@ def _process_excel_row( def _process_excel_tab( excel: pandas.ExcelFile, spcsrc: SpecificationSource ) -> (Opt[str], Opt[ParseResult]): - df = excel.parse( - sheet_name=spcsrc.tab, na_values=_EXCEL_MISSING_VALUES, keep_default_na=False - ) + df = excel.parse(sheet_name=spcsrc.tab, na_values=_EXCEL_MISSING_VALUES, keep_default_na=False) if df.shape[0] < 3: # might as well not error check headers in sheets with no data return (None, None) # at this point we know that at least 4 lines are present - expecting the data type header, @@ -275,9 +262,7 @@ def _process_excel_tab( row = _process_excel_row(row, i, columns, spcsrc) if any(map(lambda x: not pandas.isna(x), row)): # skip empty rows results.append( - frozendict( - {param_ids[j]: _normalize_pandas(row[j]) for j in range(len(row))} - ) + frozendict({param_ids[j]: _normalize_pandas(row[j]) for j in range(len(row))}) ) return datatype, ParseResult(spcsrc, tuple(results)) @@ -316,9 +301,7 @@ def parse_excel(path: Path) -> ParseResults: except FileNotFoundError: return _error(Error(ErrorType.FILE_NOT_FOUND, source_1=spcsrc)) except IsADirectoryError: - return _error( - Error(ErrorType.PARSE_FAIL, "The given path is a directory", spcsrc) - ) + return _error(Error(ErrorType.PARSE_FAIL, "The given path is a directory", spcsrc)) except ValueError as e: if "Excel file format cannot be determined" in str(e): return _error( diff --git a/staging_service/metadata.py b/staging_service/metadata.py index 8aad3f17..64820bdc 100644 --- a/staging_service/metadata.py +++ b/staging_service/metadata.py @@ -115,9 +115,7 @@ async def _only_source(path: Path): return data["source"] -async def dir_info( - path: Path, show_hidden: bool, query: str = "", recurse=True -) -> list: +async def dir_info(path: Path, show_hidden: bool, query: str = "", recurse=True) -> list: """ only call this on a validated full path """ @@ -132,9 +130,7 @@ async def dir_info( if query == "" or specific_path.user_path.find(query) != -1: response.append(await stat_data(specific_path)) if recurse: - response.extend( - await dir_info(specific_path, show_hidden, query, recurse) - ) + response.extend(await dir_info(specific_path, show_hidden, query, recurse)) if entry.is_file(): if query == "" or specific_path.user_path.find(query) != -1: data = await stat_data(specific_path) diff --git a/staging_service/utils.py b/staging_service/utils.py index c3984052..8a33f9e2 100644 --- a/staging_service/utils.py +++ b/staging_service/utils.py @@ -31,12 +31,10 @@ async def run_command(*args): if process.returncode == 0: return stdout.decode().strip() else: - error_msg = ( - "command {cmd} failed\nreturn code: {returncode}\nerror: {error}".format( - cmd=" ".join(args), - returncode=process.returncode, - error=stderr.decode().strip(), - ) + error_msg = "command {cmd} failed\nreturn code: {returncode}\nerror: {error}".format( + cmd=" ".join(args), + returncode=process.returncode, + error=stderr.decode().strip(), ) raise HTTPInternalServerError(text=error_msg) @@ -100,21 +98,13 @@ def __init__(self): client = globus_sdk.NativeAppAuthClient(cf["client_id"]) try: - transfer_authorizer = globus_sdk.RefreshTokenAuthorizer( - cf["transfer_token"], client - ) - self.globus_transfer_client = globus_sdk.TransferClient( - authorizer=transfer_authorizer - ) - auth_authorizer = globus_sdk.RefreshTokenAuthorizer( - cf["auth_token"], client - ) + transfer_authorizer = globus_sdk.RefreshTokenAuthorizer(cf["transfer_token"], client) + self.globus_transfer_client = globus_sdk.TransferClient(authorizer=transfer_authorizer) + auth_authorizer = globus_sdk.RefreshTokenAuthorizer(cf["auth_token"], client) self.globus_auth_client = globus_sdk.AuthClient(authorizer=auth_authorizer) except globus_sdk.GlobusAPIError as error: logging.error(str(error.code) + error.raw_text) - raise HTTPInternalServerError( - text=str("Invalid Token Specified in globus.cfg file") - ) + raise HTTPInternalServerError(text=str("Invalid Token Specified in globus.cfg file")) def _get_globus_identities(self, shared_directory: str): """ @@ -124,18 +114,14 @@ def _get_globus_identities(self, shared_directory: str): globus_id_filename = "{}.globus_id".format(shared_directory) with open(globus_id_filename, "r") as fp: ident = fp.read() - return self.globus_auth_client.get_identities( - usernames=ident.split("\n")[0] - ) + return self.globus_auth_client.get_identities(usernames=ident.split("\n")[0]) def _get_globus_identity(self, globus_id_filename: str): """ Get the first identity for the username in the .globus_id file """ try: - return self._get_globus_identities(globus_id_filename)["identities"][0][ - "id" - ] + return self._get_globus_identities(globus_id_filename)["identities"][0]["id"] except FileNotFoundError as error: response = { "success": False, @@ -188,9 +174,7 @@ def _add_acl(self, user_identity_id: str, shared_directory_basename: str): } logging.info(response) - logging.info( - "Shared %s with %s\n" % (shared_directory_basename, user_identity_id) - ) + logging.info("Shared %s with %s\n" % (shared_directory_basename, user_identity_id)) logging.info(response) return response @@ -207,18 +191,14 @@ def _add_acl(self, user_identity_id: str, shared_directory_basename: str): if error.code == "Exists": raise HTTPOk(text=json.dumps(response), content_type="application/json") - raise HTTPInternalServerError( - text=json.dumps(response), content_type="application/json" - ) + raise HTTPInternalServerError(text=json.dumps(response), content_type="application/json") def _remove_acl(self, user_identity_id: str): """ Get all ACLS and attempt to remove the correct ACL for the given user_identity """ try: - acls = self.globus_transfer_client.endpoint_acl_list(self.endpoint_id)[ - "DATA" - ] + acls = self.globus_transfer_client.endpoint_acl_list(self.endpoint_id)["DATA"] for acl in acls: if user_identity_id == acl["principal"]: if "id" in acl and acl["id"] is not None: diff --git a/tests/import_specifications/test_file_parser.py b/tests/import_specifications/test_file_parser.py index 6c809d51..ecb49ddf 100644 --- a/tests/import_specifications/test_file_parser.py +++ b/tests/import_specifications/test_file_parser.py @@ -112,9 +112,7 @@ def test_Error_init_w_PARSE_FAIL_success(): def test_Error_init_w_INCORRECT_COLUMN_COUNT_success(): - e = Error( - ErrorType.INCORRECT_COLUMN_COUNT, message="42", source_1=spcsrc("somefile") - ) + e = Error(ErrorType.INCORRECT_COLUMN_COUNT, message="42", source_1=spcsrc("somefile")) assert e.error == ErrorType.INCORRECT_COLUMN_COUNT assert e.message == "42" @@ -177,14 +175,12 @@ def test_Error_init_fail(): error_init_fail(ErrorType.PARSE_FAIL, None, spcsrc("wooo"), None, ValueError(err)) error_init_fail(ErrorType.PARSE_FAIL, "msg", None, None, ValueError(err)) err = "message, source_1 is required for a INCORRECT_COLUMN_COUNT error" - error_init_fail( - ErrorType.INCORRECT_COLUMN_COUNT, None, spcsrc("whee"), None, ValueError(err) - ) - error_init_fail( - ErrorType.INCORRECT_COLUMN_COUNT, "msg", None, None, ValueError(err) - ) + error_init_fail(ErrorType.INCORRECT_COLUMN_COUNT, None, spcsrc("whee"), None, ValueError(err)) + error_init_fail(ErrorType.INCORRECT_COLUMN_COUNT, "msg", None, None, ValueError(err)) ms = ErrorType.MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE - err = "message, source_1, source_2 is required for a MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE error" + err = ( + "message, source_1, source_2 is required for a MULTIPLE_SPECIFICATIONS_FOR_DATA_TYPE error" + ) error_init_fail(ms, None, None, None, ValueError(err)) error_init_fail(ms, None, spcsrc("foo"), spcsrc("bar"), ValueError(err)) error_init_fail(ms, "msg", None, spcsrc("bar"), ValueError(err)) @@ -219,9 +215,7 @@ def test_ParseResult_init_success(): def test_ParseResult_init_fail(): parseResult_init_fail(None, None, ValueError("source is required")) - parseResult_init_fail( - None, (frozendict({"foo": "bar"}),), ValueError("source is required") - ) + parseResult_init_fail(None, (frozendict({"foo": "bar"}),), ValueError("source is required")) parseResult_init_fail(spcsrc("foo"), None, ValueError("result is required")) @@ -286,9 +280,7 @@ def parseResults_init_fail( assert_exception_correct(got.value, expected) -def _ftr( - parser: Callable[[Path], ParseResults] = None, notype: str = None -) -> FileTypeResolution: +def _ftr(parser: Callable[[Path], ParseResults] = None, notype: str = None) -> FileTypeResolution: return FileTypeResolution(parser, notype) @@ -326,9 +318,7 @@ def test_parse_import_specifications_success(): ) ) - res = parse_import_specifications( - (Path("myfile.xlsx"), Path("somefile.csv")), resolver, logger - ) + res = parse_import_specifications((Path("myfile.xlsx"), Path("somefile.csv")), resolver, logger) assert res == ParseResults( frozendict( @@ -371,9 +361,7 @@ def test_parse_import_specification_resolver_exception(): # test that other errors aren't included in the result parser1.return_value = ParseResults(errors=tuple([Error(ErrorType.OTHER, "foo")])) - res = parse_import_specifications( - (Path("myfile.xlsx"), Path("somefile.csv")), resolver, logger - ) + res = parse_import_specifications((Path("myfile.xlsx"), Path("somefile.csv")), resolver, logger) assert res == ParseResults(errors=tuple([Error(ErrorType.OTHER, "crapsticks")])) diff --git a/tests/import_specifications/test_file_writers.py b/tests/import_specifications/test_file_writers.py index 31e62b5c..d0195567 100644 --- a/tests/import_specifications/test_file_writers.py +++ b/tests/import_specifications/test_file_writers.py @@ -238,10 +238,7 @@ def test_file_writers_fail(): "data": [], } }, - E( - "Invalid order_and_display entry for datatype t at index 2 - " - + "expected 2 item list" - ), + E("Invalid order_and_display entry for datatype t at index 2 - " + "expected 2 item list"), ) file_writers_fail( p, @@ -251,10 +248,7 @@ def test_file_writers_fail(): "data": [], } }, - E( - "Invalid order_and_display entry for datatype t at index 0 - " - + "expected 2 item list" - ), + E("Invalid order_and_display entry for datatype t at index 0 - " + "expected 2 item list"), ) for parm in [None, " \t ", 1]: file_writers_fail( @@ -316,10 +310,7 @@ def test_file_writers_fail(): "data": [{"foo": 2, "whee": 3}, {"foo": 1, "whee": []}], } }, - E( - "Data type ty data row 1's value for parameter whee " - + "is not a number or a string" - ), + E("Data type ty data row 1's value for parameter whee " + "is not a number or a string"), ) diff --git a/tests/import_specifications/test_individual_parsers.py b/tests/import_specifications/test_individual_parsers.py index 21934aa1..2c4a8a24 100644 --- a/tests/import_specifications/test_individual_parsers.py +++ b/tests/import_specifications/test_individual_parsers.py @@ -45,9 +45,7 @@ def test_xsv_parse_success(temp_dir: Path): _xsv_parse_success(temp_dir, "\t", parse_tsv) -def _xsv_parse_success( - temp_dir: Path, sep: str, parser: Callable[[Path], ParseResults] -): +def _xsv_parse_success(temp_dir: Path, sep: str, parser: Callable[[Path], ParseResults]): s = sep input_ = temp_dir / str(uuid.uuid4()) with open(input_, "w") as test_file: @@ -121,9 +119,7 @@ def test_xsv_parse_success_nan_inf(temp_dir: Path): _xsv_parse_success_nan_inf(temp_dir, "\t", parse_tsv) -def _xsv_parse_success_nan_inf( - temp_dir: Path, sep: str, parser: Callable[[Path], ParseResults] -): +def _xsv_parse_success_nan_inf(temp_dir: Path, sep: str, parser: Callable[[Path], ParseResults]): s = sep input_ = temp_dir / str(uuid.uuid4()) with open(input_, "w") as test_file: @@ -212,9 +208,7 @@ def _xsv_parse_success_with_numeric_headers( SpecificationSource(input_), tuple( [ - frozendict( - {"1": "val3", "2.0": "val4", "3": 1, "4.1": 8.9} - ), + frozendict({"1": "val3", "2.0": "val4", "3": 1, "4.1": 8.9}), ] ), ) @@ -301,9 +295,7 @@ def test_xsv_parse_fail_no_file(temp_dir: Path): res = parse_csv(input_) assert res == ParseResults( - errors=tuple( - [Error(ErrorType.FILE_NOT_FOUND, source_1=SpecificationSource(input_))] - ) + errors=tuple([Error(ErrorType.FILE_NOT_FOUND, source_1=SpecificationSource(input_))]) ) @@ -357,9 +349,7 @@ def _xsv_parse_fail( test_file.writelines(lines) res = parser(input_) - expected = ParseResults( - errors=tuple([Error(err_type, message, SpecificationSource(input_))]) - ) + expected = ParseResults(errors=tuple([Error(err_type, message, SpecificationSource(input_))])) assert res == expected @@ -377,16 +367,12 @@ def test_xsv_parse_fail_bad_datatype_header(temp_dir: Path): def test_xsv_parse_fail_bad_version(temp_dir: Path): err = "Schema version 87 is larger than maximum processable version 1" - _xsv_parse_fail( - temp_dir, ["Data type: foo; Columns: 22; Version: 87"], parse_csv, err - ) + _xsv_parse_fail(temp_dir, ["Data type: foo; Columns: 22; Version: 87"], parse_csv, err) def test_xsv_parse_fail_missing_column_headers(temp_dir: Path): err = "Missing 2nd header line" - _xsv_parse_fail( - temp_dir, ["Data type: foo; Columns: 3; Version: 1\n"], parse_csv, err - ) + _xsv_parse_fail(temp_dir, ["Data type: foo; Columns: 3; Version: 1\n"], parse_csv, err) err = "Missing 3rd header line" lines = ["Data type: foo; Columns: 3; Version: 1\n", "head1, head2, head3\n"] @@ -516,18 +502,10 @@ def test_excel_parse_success(): "type1": ParseResult( SpecificationSource(ex, "tab1"), ( - frozendict( - {"header1": "foo", "header2": 1, "header3": 6.7} - ), - frozendict( - {"header1": "bar", "header2": 2, "header3": 8.9} - ), - frozendict( - {"header1": "baz", "header2": None, "header3": 3.4} - ), - frozendict( - {"header1": "bat", "header2": 4, "header3": None} - ), + frozendict({"header1": "foo", "header2": 1, "header3": 6.7}), + frozendict({"header1": "bar", "header2": 2, "header3": 8.9}), + frozendict({"header1": "baz", "header2": None, "header3": 3.4}), + frozendict({"header1": "bat", "header2": 4, "header3": None}), ), ), "type2": ParseResult( @@ -618,9 +596,7 @@ def _excel_parse_fail( def test_excel_parse_fail_no_file(): f = _get_test_file("testtabs3full2nodata1empty0.xls") - _excel_parse_fail( - f, errors=[Error(ErrorType.FILE_NOT_FOUND, source_1=SpecificationSource(f))] - ) + _excel_parse_fail(f, errors=[Error(ErrorType.FILE_NOT_FOUND, source_1=SpecificationSource(f))]) def test_excel_parse_fail_directory(temp_dir): @@ -628,15 +604,11 @@ def test_excel_parse_fail_directory(temp_dir): f = temp_dir / d os.makedirs(f, exist_ok=True) err = "The given path is a directory" - _excel_parse_fail( - f, errors=[Error(ErrorType.PARSE_FAIL, err, SpecificationSource(f))] - ) + _excel_parse_fail(f, errors=[Error(ErrorType.PARSE_FAIL, err, SpecificationSource(f))]) def test_excel_parse_fail_empty_file(temp_dir: Path): - _xsv_parse_fail( - temp_dir, [], parse_excel, "Not a supported Excel file type", extension=".xls" - ) + _xsv_parse_fail(temp_dir, [], parse_excel, "Not a supported Excel file type", extension=".xls") def test_excel_parse_fail_non_excel_file(temp_dir: Path): @@ -655,9 +627,7 @@ def test_excel_parse_fail_non_excel_file(temp_dir: Path): def test_excel_parse_1emptytab(): - _excel_parse_fail( - _get_test_file("testtabs1empty.xls"), "No non-header data in file" - ) + _excel_parse_fail(_get_test_file("testtabs1empty.xls"), "No non-header data in file") def test_excel_parse_fail_bad_datatype_header(): diff --git a/tests/test_app.py b/tests/test_app.py index f4c80769..de147af5 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -71,9 +71,7 @@ async def mock_auth(*args, **kwargs): async def mock_globus_id(*args, **kwargs): return ["testuser@globusid.org"] - globus._get_globus_ids = ( - mock_globus_id # TODO this doesn't allow testing of this fn does it - ) + globus._get_globus_ids = mock_globus_id # TODO this doesn't allow testing of this fn does it return application @@ -135,22 +133,11 @@ def remove_dir(self, path): @given(username_first_strat, username_strat) def test_path_cases(username_first, username_rest): username = username_first + username_rest - assert ( - username + "/foo/bar" == utils.Path.validate_path(username, "foo/bar").user_path - ) - assert ( - username + "/baz" - == utils.Path.validate_path(username, "foo/../bar/../baz").user_path - ) - assert ( - username + "/bar" - == utils.Path.validate_path(username, "foo/../../../../bar").user_path - ) + assert username + "/foo/bar" == utils.Path.validate_path(username, "foo/bar").user_path + assert username + "/baz" == utils.Path.validate_path(username, "foo/../bar/../baz").user_path + assert username + "/bar" == utils.Path.validate_path(username, "foo/../../../../bar").user_path assert username + "/foo" == utils.Path.validate_path(username, "./foo").user_path - assert ( - username + "/foo/bar" - == utils.Path.validate_path(username, "../foo/bar").user_path - ) + assert username + "/foo/bar" == utils.Path.validate_path(username, "../foo/bar").user_path assert username + "/foo" == utils.Path.validate_path(username, "/../foo").user_path assert username + "/" == utils.Path.validate_path(username, "/foo/..").user_path assert username + "/foo" == utils.Path.validate_path(username, "/foo/.").user_path @@ -162,10 +149,7 @@ def test_path_cases(username_first, username_rest): assert username + "/" == utils.Path.validate_path(username, "").user_path assert username + "/" == utils.Path.validate_path(username, "foo/..").user_path assert username + "/" == utils.Path.validate_path(username, "/..../").user_path - assert ( - username + "/stuff.ext" - == utils.Path.validate_path(username, "/stuff.ext").user_path - ) + assert username + "/stuff.ext" == utils.Path.validate_path(username, "/stuff.ext").user_path @given(username_first_strat, username_strat, st.text()) @@ -222,9 +206,7 @@ async def test_jbi_metadata(): with FileUtil() as fs: fs.make_dir(os.path.join(username, "test")) fs.make_file(os.path.join(username, "test", "test_jgi.fastq"), txt) - fs.make_file( - os.path.join(username, "test", ".test_jgi.fastq.jgi"), jbi_metadata - ) + fs.make_file(os.path.join(username, "test", ".test_jgi.fastq.jgi"), jbi_metadata) res1 = await cli.get( os.path.join("jgi-metadata", "test", "test_jgi.fastq"), headers={"Authorization": ""}, @@ -435,9 +417,7 @@ async def test_mv(): fs.make_file(os.path.join(username, "test", "test_file_1"), txt) # list current test directory - res1 = await cli.get( - os.path.join("list", "test"), headers={"Authorization": ""} - ) + res1 = await cli.get(os.path.join("list", "test"), headers={"Authorization": ""}) assert res1.status == 200 json_text = await res1.text() json = decoder.decode(json_text) @@ -454,9 +434,7 @@ async def test_mv(): assert "successfully moved" in json_text # relist test directory - res3 = await cli.get( - os.path.join("list", "test"), headers={"Authorization": ""} - ) + res3 = await cli.get(os.path.join("list", "test"), headers={"Authorization": ""}) assert res3.status == 200 json_text = await res3.text() json = decoder.decode(json_text) @@ -509,9 +487,7 @@ async def test_delete(): fs.make_file(os.path.join(username, "test", "test_file_1"), txt) # list current test directory - res1 = await cli.get( - os.path.join("list", "test"), headers={"Authorization": ""} - ) + res1 = await cli.get(os.path.join("list", "test"), headers={"Authorization": ""}) assert res1.status == 200 json_text = await res1.text() json = decoder.decode(json_text) @@ -527,9 +503,7 @@ async def test_delete(): assert "successfully deleted" in json_text # relist test directory - res3 = await cli.get( - os.path.join("list", "test"), headers={"Authorization": ""} - ) + res3 = await cli.get(os.path.join("list", "test"), headers={"Authorization": ""}) assert res3.status == 200 json_text = await res3.text() json = decoder.decode(json_text) @@ -556,9 +530,7 @@ async def test_list(): fs.make_dir(os.path.join(username, "test")) fs.make_file(os.path.join(username, "test", "test_file_1"), txt) fs.make_dir(os.path.join(username, "test", "test_sub_dir")) - fs.make_file( - os.path.join(username, "test", "test_sub_dir", "test_file_2"), txt - ) + fs.make_file(os.path.join(username, "test", "test_sub_dir", "test_file_2"), txt) res1 = await cli.get("list/..", headers={"Authorization": ""}) assert res1.status == 404 res2 = await cli.get( @@ -594,9 +566,7 @@ async def test_list(): assert sum(file_folder_count) == 2 # testing sub-directory - res5 = await cli.get( - os.path.join("list", "test"), headers={"Authorization": ""} - ) + res5 = await cli.get(os.path.join("list", "test"), headers={"Authorization": ""}) assert res5.status == 200 json_text = await res5.text() json = decoder.decode(json_text) @@ -613,9 +583,7 @@ async def test_list(): json_text = await res6.text() json = decoder.decode(json_text) - file_names = [ - file_json["name"] for file_json in json if not file_json["isFolder"] - ] + file_names = [file_json["name"] for file_json in json if not file_json["isFolder"]] assert ".test_file_1" not in file_names assert ".globus_id" not in file_names assert len(file_names) == 2 @@ -627,9 +595,7 @@ async def test_list(): assert res7.status == 200 json_text = await res7.text() json = decoder.decode(json_text) - file_names = [ - file_json["name"] for file_json in json if not file_json["isFolder"] - ] + file_names = [file_json["name"] for file_json in json if not file_json["isFolder"]] assert ".test_file_1" in file_names assert ".globus_id" in file_names assert len(file_names) == 4 @@ -674,21 +640,15 @@ async def test_similar(): fs.make_dir(os.path.join(username, "test")) fs.make_file(os.path.join(username, "test", "test_file_1.fq"), txt) fs.make_dir(os.path.join(username, "test", "test_sub_dir")) - fs.make_file( - os.path.join(username, "test", "test_sub_dir", "test_file_2.fq"), txt - ) + fs.make_file(os.path.join(username, "test", "test_sub_dir", "test_file_2.fq"), txt) fs.make_file( os.path.join(username, "test", "test_sub_dir", "test_file_right.fq"), txt, ) - fs.make_file( - os.path.join(username, "test", "test_sub_dir", "my_files"), txt - ) + fs.make_file(os.path.join(username, "test", "test_sub_dir", "my_files"), txt) # testing similar file name - res1 = await cli.get( - "similar/test/test_file_1.fq", headers={"Authorization": ""} - ) + res1 = await cli.get("similar/test/test_file_1.fq", headers={"Authorization": ""}) assert res1.status == 200 json_text = await res1.text() json = decoder.decode(json_text) @@ -697,9 +657,7 @@ async def test_similar(): assert json[1].get("name") in ["test_file_2.fq", "test_file_right.fq"] # testing non-existing file - res2 = await cli.get( - "similar/test/non-existing", headers={"Authorization": ""} - ) + res2 = await cli.get("similar/test/non-existing", headers={"Authorization": ""}) assert res2.status == 404 # testing path is a directory @@ -715,15 +673,11 @@ async def test_existence(): fs.make_dir(os.path.join(username, "test")) fs.make_file(os.path.join(username, "test", "test_file_1"), txt) fs.make_dir(os.path.join(username, "test", "test_sub_dir")) - fs.make_file( - os.path.join(username, "test", "test_sub_dir", "test_file_2"), txt - ) + fs.make_file(os.path.join(username, "test", "test_sub_dir", "test_file_2"), txt) fs.make_dir(os.path.join(username, "test", "test_sub_dir", "test_file_1")) fs.make_dir(os.path.join(username, "test", "test_sub_dir", "test_sub_dir")) fs.make_file( - os.path.join( - username, "test", "test_sub_dir", "test_sub_dir", "test_file_1" - ), + os.path.join(username, "test", "test_sub_dir", "test_sub_dir", "test_file_1"), txt, ) @@ -744,9 +698,7 @@ async def test_existence(): assert json["isFolder"] is False # testing existence of folder - res3 = await cli.get( - "existence/test_sub_dir", headers={"Authorization": ""} - ) + res3 = await cli.get("existence/test_sub_dir", headers={"Authorization": ""}) assert res3.status == 200 json_text = await res3.text() json = decoder.decode(json_text) @@ -809,9 +761,7 @@ async def test_upload(): files = {"destPath": "/", "uploads": open(f, "rb")} - res2 = await cli.post( - os.path.join("upload"), headers={"Authorization": ""}, data=files - ) + res2 = await cli.post(os.path.join("upload"), headers={"Authorization": ""}, data=files) assert res2.status == 200 @@ -892,9 +842,7 @@ async def test_directory_decompression(contents): assert not os.path.exists(d2) assert not os.path.exists(f3) assert os.path.exists(compressed) - resp = await cli.patch( - "/decompress/" + name, headers={"Authorization": ""} - ) + resp = await cli.patch("/decompress/" + name, headers={"Authorization": ""}) assert resp.status == 200 text = await resp.text() assert "succesfully decompressed" in text @@ -1003,10 +951,7 @@ async def test_importer_mappings(): resp = await cli.get(f"importer_mappings/?{qsd}") assert resp.status == 400 text = await resp.text() - assert ( - f"must provide file_list field. Your provided qs: {unquote(qsd)}" - in text - ) + assert f"must provide file_list field. Your provided qs: {unquote(qsd)}" in text async def test_bulk_specification_success(): @@ -1059,9 +1004,7 @@ async def test_bulk_specification_success(): ) df.to_excel(exw, sheet_name="sloths", header=False, index=False) - resp = await cli.get( - f"bulk_specification/?files={tsv} , {csv}, {excel} " - ) + resp = await cli.get(f"bulk_specification/?files={tsv} , {csv}, {excel} ") jsn = await resp.json() assert jsn == { "types": { @@ -1077,9 +1020,7 @@ async def test_bulk_specification_success(): {"bat_name": "George", "wing_count": 42}, {"bat_name": "Fred", "wing_count": 1.5}, ], - "tree_sloths": [ - {"entity_id": "That which ends all", "preferred_food": "ꔆ"} - ], + "tree_sloths": [{"entity_id": "That which ends all", "preferred_food": "ꔆ"}], }, "files": { "genomes": {"file": "testuser/genomes.tsv", "tab": None}, @@ -1109,9 +1050,7 @@ async def test_bulk_specification_fail_no_files(): async def test_bulk_specification_fail_not_found(): async with AppClient(config) as cli: with FileUtil() as fu: - fu.make_dir( - "testuser/otherfolder" - ) # testuser is hardcoded in the auth mock + fu.make_dir("testuser/otherfolder") # testuser is hardcoded in the auth mock base = Path(fu.base_dir) / "testuser" tsv = "otherfolder/genomes.tsv" with open(base / tsv, "w") as f: @@ -1126,9 +1065,7 @@ async def test_bulk_specification_fail_not_found(): resp = await cli.get(f"bulk_specification/?files={tsv},somefile.csv") jsn = await resp.json() assert jsn == { - "errors": [ - {"type": "cannot_find_file", "file": "testuser/somefile.csv"} - ] + "errors": [{"type": "cannot_find_file", "file": "testuser/somefile.csv"}] } assert resp.status == 404 @@ -1140,9 +1077,7 @@ async def test_bulk_specification_fail_parse_fail(): """ async with AppClient(config) as cli: with FileUtil() as fu: - fu.make_dir( - "testuser/otherfolder" - ) # testuser is hardcoded in the auth mock + fu.make_dir("testuser/otherfolder") # testuser is hardcoded in the auth mock base = Path(fu.base_dir) / "testuser" tsv = "otherfolder/genomes.tsv" # this one is fine @@ -1378,9 +1313,7 @@ async def test_bulk_specification_fail_multiple_specs_per_type(): ) df.to_excel(exw, sheet_name="sloths", header=False, index=False) - resp = await cli.get( - f"bulk_specification/?files={tsv},{csv1},{csv2},{excel}" - ) + resp = await cli.get(f"bulk_specification/?files={tsv},{csv1},{csv2},{excel}") jsn = await resp.json() err = "Data type breakfastcereals appears in two importer specification sources" assert jsn == { @@ -1591,9 +1524,7 @@ async def test_write_bulk_specification_success_excel(): "reads": "testuser/import_specification.xlsx", }, } - wb = openpyxl.load_workbook( - Path(fu.base_dir) / "testuser/import_specification.xlsx" - ) + wb = openpyxl.load_workbook(Path(fu.base_dir) / "testuser/import_specification.xlsx") assert wb.sheetnames == ["genome", "reads"] check_excel_contents( wb, @@ -1643,10 +1574,7 @@ async def test_write_bulk_specification_fail_large_input(): resp = await cli.post("write_bulk_specification/", json="a" * (1024 * 1024 - 2)) txt = await resp.text() # this seems to be a built in (somewhat inaccurate) server feature - assert ( - txt - == "Maximum request body size 1048576 exceeded, actual body size 1048576" - ) + assert txt == "Maximum request body size 1048576 exceeded, actual body size 1048576" assert resp.status == 413 diff --git a/tests/test_autodetect.py b/tests/test_autodetect.py index 6e4d58d2..2cc4fda3 100644 --- a/tests/test_autodetect.py +++ b/tests/test_autodetect.py @@ -72,9 +72,7 @@ def test_reasonable_filenames(): expected_suffix = filename_variant.split(".", heading_dotcount + 1)[-1] assert ( possible_importers - == AutoDetectUtils._MAPPINGS["types"][expected_suffix.lower()][ - "mappings" - ] + == AutoDetectUtils._MAPPINGS["types"][expected_suffix.lower()]["mappings"] ), filename_variant assert fileinfo == { "prefix": filename_variant[: -len(expected_suffix) - 1], @@ -186,9 +184,7 @@ def test_specific_filenames(): ] for filename, importers in test_data: - assert ( - AutoDetectUtils.determine_possible_importers(filename) == importers - ), filename + assert AutoDetectUtils.determine_possible_importers(filename) == importers, filename def test_sra_mappings(): @@ -197,9 +193,7 @@ def test_sra_mappings(): :return: """ sra_file = "test.sra" - possible_importers, fileinfo = AutoDetectUtils.determine_possible_importers( - filename=sra_file - ) + possible_importers, fileinfo = AutoDetectUtils.determine_possible_importers(filename=sra_file) assert possible_importers == [ { "id": "sra_reads", @@ -216,9 +210,7 @@ def test_zip_mappings(): :return: """ gz_file = "test.tar.gz" - possible_importers, fileinfo = AutoDetectUtils.determine_possible_importers( - filename=gz_file - ) + possible_importers, fileinfo = AutoDetectUtils.determine_possible_importers(filename=gz_file) assert possible_importers == [ { "id": "decompress", @@ -238,9 +230,7 @@ def test_get_mappings(): Basic test of the get mappings logic. Most of the logic is in determine_possible_importers which is throughly tested above. """ - assert AutoDetectUtils.get_mappings( - ["filename", "file.name.Gz", "some.dots.gff3.gz"] - ) == { + assert AutoDetectUtils.get_mappings(["filename", "file.name.Gz", "some.dots.gff3.gz"]) == { "mappings": [ None, [