Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ran black on all modules #1355

Merged
merged 2 commits into from
Jan 23, 2024
Merged

ran black on all modules #1355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a686757
ran black on all modules
andrewelamb Jan 22, 2024
f111661
fixed black github action
andrewelamb Jan 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ jobs:
run: |
# ran only on certain files for now
# add here when checked
poetry run black schematic/configuration/*.py schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py --check
poetry run black schematic --check
#----------------------------------------------
# type checking/enforcement
Expand Down
2 changes: 1 addition & 1 deletion schematic/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
# dict() -> new empty dictionary
CONTEXT_SETTINGS = dict(help_option_names=["--help", "-h"]) # help options


# invoke_without_command=True -> forces the application not to show aids before losing them with a --h
@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True)
@click_log.simple_verbosity_option(logger)
Expand All @@ -40,6 +41,5 @@ def main():
main.add_command(viz_cli) # add viz commands



if __name__ == "__main__":
main()
81 changes: 48 additions & 33 deletions schematic/manifest/commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@
from schematic.store.synapse import SynapseStorage
from schematic.configuration.configuration import CONFIG

logger = logging.getLogger('schematic')
logger = logging.getLogger("schematic")
click_log.basic_config(logger)

CONTEXT_SETTINGS = dict(help_option_names=["--help", "-h"]) # help options


# invoke_without_command=True -> forces the application not to show aids before losing them with a --h
@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True)
@click_log.simple_verbosity_option(logger)
Expand Down Expand Up @@ -62,7 +63,9 @@ def manifest(ctx, config): # use as `schematic manifest ...`
help=query_dict(manifest_commands, ("manifest", "get", "data_type")),
)
@click.option(
"-p", "--path_to_data_model", help=query_dict(manifest_commands, ("manifest", "get", "path_to_data_model"))
"-p",
"--path_to_data_model",
help=query_dict(manifest_commands, ("manifest", "get", "path_to_data_model")),
)
@click.option(
"-d",
Expand Down Expand Up @@ -99,7 +102,7 @@ def manifest(ctx, config): # use as `schematic manifest ...`
@click.option(
"-av",
"--alphabetize_valid_values",
default = 'ascending',
default="ascending",
help=query_dict(manifest_commands, ("manifest", "get", "alphabetize_valid_values")),
)
@click.pass_obj
Expand All @@ -122,18 +125,18 @@ def get_manifest(
# Optional parameters that need to be passed to ManifestGenerator()
# If CLI parameters are None they are gotten from the CONFIG object and logged
if data_type is None:
data_type = CONFIG.manifest_data_type
data_type = CONFIG.manifest_data_type
log_value_from_config("data_type", data_type)
if path_to_data_model is None:
path_to_data_model = CONFIG.model_location
path_to_data_model = CONFIG.model_location
log_value_from_config("path_to_data_model", path_to_data_model)
if title is None:
title = CONFIG.manifest_title
title = CONFIG.manifest_title
log_value_from_config("title", title)

data_model_parser = DataModelParser(path_to_data_model = path_to_data_model)
data_model_parser = DataModelParser(path_to_data_model=path_to_data_model)

#Parse Model
# Parse Model
logger.info("Parsing data model.")
parsed_data_model = data_model_parser.parse_model()

Expand All @@ -148,7 +151,7 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):
# create object of type ManifestGenerator
manifest_generator = ManifestGenerator(
path_to_data_model=path_to_data_model,
graph = graph_data_model,
graph=graph_data_model,
title=t,
root=data_type,
use_annotations=use_annotations,
Expand All @@ -157,7 +160,7 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):

# call get_manifest() on manifest_generator
# if output_xlsx gets specified, output_format = "excel"
if output_xlsx:
if output_xlsx:
output_format = "excel"
# if file name is in the path, and that file does not exist
if not os.path.exists(output_xlsx):
Expand All @@ -170,27 +173,31 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):
)
else:
raise ValueError(
f"{output_xlsx} does not exists. Please try a valid file path"
)
f"{output_xlsx} does not exists. Please try a valid file path"
)
else:
# Check if base path itself exists.
if not os.path.exists(os.path.dirname(output_xlsx)):
raise ValueError(
f"{output_xlsx} does not exists. Please try a valid file path"
f"{output_xlsx} does not exists. Please try a valid file path"
)
output_path = output_xlsx
else:
else:
output_format = None
output_path = None

result = manifest_generator.get_manifest(
dataset_id=dataset_id, sheet_url=sheet_url, json_schema=json_schema, output_format = output_format, output_path = output_path
dataset_id=dataset_id,
sheet_url=sheet_url,
json_schema=json_schema,
output_format=output_format,
output_path=output_path,
)

if sheet_url:
logger.info("Find the manifest template using this Google Sheet URL:")
click.echo(result)
if output_csv is None and output_xlsx is None:
if output_csv is None and output_xlsx is None:
prefix, _ = os.path.splitext(path_to_data_model)
prefix_root, prefix_ext = os.path.splitext(prefix)
if prefix_ext == ".model":
Expand All @@ -204,37 +211,41 @@ def create_single_manifest(data_type, output_csv=None, output_xlsx=None):
return result
export_manifest_csv(file_path=output_csv, manifest=result)
logger.info(
f"Find the manifest template using this CSV file path: {output_csv}"
)
f"Find the manifest template using this CSV file path: {output_csv}"
)
return result

if type(data_type) is str:
data_type = [data_type]

if data_type[0] == 'all manifests':
if data_type[0] == "all manifests":
# Feed graph into the data model graph explorer
dmge = DataModelGraphExplorer(graph_data_model)
component_digraph = dmge.get_digraph_by_edge_type('requiresComponent')
component_digraph = dmge.get_digraph_by_edge_type("requiresComponent")
components = component_digraph.nodes()
for component in components:
t = f'{title}.{component}.manifest'
result = create_single_manifest(data_type = component)
t = f"{title}.{component}.manifest"
result = create_single_manifest(data_type=component)
else:
for dt in data_type:
if len(data_type) > 1 and not output_xlsx:
t = f'{title}.{dt}.manifest'
elif output_xlsx:
t = f"{title}.{dt}.manifest"
elif output_xlsx:
if ".xlsx" or ".xls" in output_xlsx:
title_with_extension = os.path.basename(output_xlsx)
t = title_with_extension.split('.')[0]
t = title_with_extension.split(".")[0]
else:
t = title
result = create_single_manifest(data_type = dt, output_csv=output_csv, output_xlsx=output_xlsx)
result = create_single_manifest(
data_type=dt, output_csv=output_csv, output_xlsx=output_xlsx
)

return result


@manifest.command(
"migrate", short_help=query_dict(manifest_commands, ("manifest", "migrate", "short_help"))
"migrate",
short_help=query_dict(manifest_commands, ("manifest", "migrate", "short_help")),
)
@click_log.simple_verbosity_option(logger)
# define the optional arguments
Expand Down Expand Up @@ -281,18 +292,22 @@ def migrate_manifests(
Running CLI with manifest migration options.
"""
if jsonld is None:
jsonld = CONFIG.model_location
jsonld = CONFIG.model_location
log_value_from_config("jsonld", jsonld)

full_scope = project_scope + [archive_project]
synStore = SynapseStorage(project_scope = full_scope)
synStore = SynapseStorage(project_scope=full_scope)

for project in project_scope:
if not return_entities:
logging.info("Re-uploading manifests as tables")
synStore.upload_annotated_project_manifests_to_synapse(project, jsonld, dry_run)
synStore.upload_annotated_project_manifests_to_synapse(
project, jsonld, dry_run
)
if archive_project:
logging.info("Migrating entitites")
synStore.move_entities_to_new_project(project, archive_project, return_entities, dry_run)

return
synStore.move_entities_to_new_project(
project, archive_project, return_entities, dry_run
)

return
Loading
Loading