Skip to content

Commit

Permalink
Merge pull request #1355 from Sage-Bionetworks/feature-black
Browse files Browse the repository at this point in the history 
ran black on all modules
  • Loading branch information
@andrewelamb
andrewelamb authored Jan 23, 2024
2 parents 2ade117 + f111661 commit 48d7f96
Show file tree
Hide file tree
Showing 27 changed files with 3,358 additions and 2,395 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ jobs:
run: |
# ran only on certain files for now
# add here when checked
poetry run black schematic/configuration/*.py schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py --check
poetry run black schematic --check
#----------------------------------------------
# type checking/enforcement
Expand Down
2 changes: 1 addition & 1 deletion schematic/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
# dict() -> new empty dictionary
CONTEXT_SETTINGS = dict(help_option_names=["--help", "-h"]) # help options


# invoke_without_command=True -> forces the application not to show aids before losing them with a --h
@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True)
@click_log.simple_verbosity_option(logger)
Expand All @@ -40,6 +41,5 @@ def main():
main.add_command(viz_cli) # add viz commands



if __name__ == "__main__":
main()
81 changes: 48 additions & 33 deletions schematic/manifest/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@
from schematic.store.synapse import SynapseStorage
from schematic.configuration.configuration import CONFIG

logger = logging.getLogger('schematic')
logger = logging.getLogger("schematic")
click_log.basic_config(logger)

CONTEXT_SETTINGS = dict(help_option_names=["--help", "-h"]) # help options


# invoke_without_command=True -> forces the application not to show aids before losing them with a --h
@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True)
@click_log.simple_verbosity_option(logger)
Expand Down Expand Up @@ -62,7 +63,9 @@ def manifest(ctx, config): # use as `schematic manifest ...`
help=query_dict(manifest_commands, ("manifest", "get", "data_type")),
)
@click.option(
"-p", "--path_to_data_model", help=query_dict(manifest_commands, ("manifest", "get", "path_to_data_model"))
"-p",
"--path_to_data_model",
help=query_dict(manifest_commands, ("manifest", "get", "path_to_data_model")),
)
@click.option(
"-d",
Expand Down Expand Up @@ -99,7 +102,7 @@ def manifest(ctx, config): # use as `schematic manifest ...`
@click.option(
"-av",
"--alphabetize_valid_values",
default = 'ascending',
default="ascending",
help=query_dict(manifest_commands, ("manifest", "get", "alphabetize_valid_values")),
)
@click.pass_obj
Expand All @@ -122,18 +125,18 @@ def get_manifest(
# Optional parameters that need to be passed to ManifestGenerator()
# If CLI parameters are None they are gotten from the CONFIG object and logged
if data_type is None:
data_type = CONFIG.manifest_data_type
data_type = CONFIG.manifest_data_type
log_value_from_config("data_type", data_type)
if path_to_data_model is None:
path_to_data_model = CONFIG.model_location
path_to_data_model = CONFIG.model_location
log_value_from_config("path_to_data_model", path_to_data_model)
if title is None:
title = CONFIG.manifest_title
title = CONFIG.manifest_title
log_value_from_config("title", title)

data_model_parser = DataModelParser(path_to_data_model = path_to_data_model)
data_model_parser = DataModelParser(path_to_data_model=path_to_data_model)

#Parse Model
# Parse Model
logger.info("Parsing data model.")
parsed_data_model = data_model_parser.parse_model()

Expand All @@ -148,7 +151,7 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):
# create object of type ManifestGenerator
manifest_generator = ManifestGenerator(
path_to_data_model=path_to_data_model,
graph = graph_data_model,
graph=graph_data_model,
title=t,
root=data_type,
use_annotations=use_annotations,
Expand All @@ -157,7 +160,7 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):

# call get_manifest() on manifest_generator
# if output_xlsx gets specified, output_format = "excel"
if output_xlsx:
if output_xlsx:
output_format = "excel"
# if file name is in the path, and that file does not exist
if not os.path.exists(output_xlsx):
Expand All @@ -170,27 +173,31 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):
)
else:
raise ValueError(
f"{output_xlsx} does not exists. Please try a valid file path"
)
f"{output_xlsx} does not exists. Please try a valid file path"
)
else:
# Check if base path itself exists.
if not os.path.exists(os.path.dirname(output_xlsx)):
raise ValueError(
f"{output_xlsx} does not exists. Please try a valid file path"
f"{output_xlsx} does not exists. Please try a valid file path"
)
output_path = output_xlsx
else:
else:
output_format = None
output_path = None

result = manifest_generator.get_manifest(
dataset_id=dataset_id, sheet_url=sheet_url, json_schema=json_schema, output_format = output_format, output_path = output_path
dataset_id=dataset_id,
sheet_url=sheet_url,
json_schema=json_schema,
output_format=output_format,
output_path=output_path,
)

if sheet_url:
logger.info("Find the manifest template using this Google Sheet URL:")
click.echo(result)
if output_csv is None and output_xlsx is None:
if output_csv is None and output_xlsx is None:
prefix, _ = os.path.splitext(path_to_data_model)
prefix_root, prefix_ext = os.path.splitext(prefix)
if prefix_ext == ".model":
Expand All @@ -204,37 +211,41 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):
return result
export_manifest_csv(file_path=output_csv, manifest=result)
logger.info(
f"Find the manifest template using this CSV file path: {output_csv}"
)
f"Find the manifest template using this CSV file path: {output_csv}"
)
return result

if type(data_type) is str:
data_type = [data_type]

if data_type[0] == 'all manifests':
if data_type[0] == "all manifests":
# Feed graph into the data model graph explorer
dmge = DataModelGraphExplorer(graph_data_model)
component_digraph = dmge.get_digraph_by_edge_type('requiresComponent')
component_digraph = dmge.get_digraph_by_edge_type("requiresComponent")
components = component_digraph.nodes()
for component in components:
t = f'{title}.{component}.manifest'
result = create_single_manifest(data_type = component)
t = f"{title}.{component}.manifest"
result = create_single_manifest(data_type=component)
else:
for dt in data_type:
if len(data_type) > 1 and not output_xlsx:
t = f'{title}.{dt}.manifest'
elif output_xlsx:
t = f"{title}.{dt}.manifest"
elif output_xlsx:
if ".xlsx" or ".xls" in output_xlsx:
title_with_extension = os.path.basename(output_xlsx)
t = title_with_extension.split('.')[0]
t = title_with_extension.split(".")[0]
else:
t = title
result = create_single_manifest(data_type = dt, output_csv=output_csv, output_xlsx=output_xlsx)
result = create_single_manifest(
data_type=dt, output_csv=output_csv, output_xlsx=output_xlsx
)

return result


@manifest.command(
"migrate", short_help=query_dict(manifest_commands, ("manifest", "migrate", "short_help"))
"migrate",
short_help=query_dict(manifest_commands, ("manifest", "migrate", "short_help")),
)
@click_log.simple_verbosity_option(logger)
# define the optional arguments
Expand Down Expand Up @@ -281,18 +292,22 @@ def migrate_manifests(
Running CLI with manifest migration options.
"""
if jsonld is None:
jsonld = CONFIG.model_location
jsonld = CONFIG.model_location
log_value_from_config("jsonld", jsonld)

full_scope = project_scope + [archive_project]
synStore = SynapseStorage(project_scope = full_scope)
synStore = SynapseStorage(project_scope=full_scope)

for project in project_scope:
if not return_entities:
logging.info("Re-uploading manifests as tables")
synStore.upload_annotated_project_manifests_to_synapse(project, jsonld, dry_run)
synStore.upload_annotated_project_manifests_to_synapse(
project, jsonld, dry_run
)
if archive_project:
logging.info("Migrating entitites")
synStore.move_entities_to_new_project(project, archive_project, return_entities, dry_run)

return
synStore.move_entities_to_new_project(
project, archive_project, return_entities, dry_run
)

return
Loading

0 comments on commit 48d7f96

Please sign in to comment.