Skip to content

Commit

Permalink
Merge pull request #7995 from uc-cdis/midrc-prod-update
Browse files Browse the repository at this point in the history 
transferring etl and gitops changes to prod
  • Loading branch information
@DanBiber
DanBiber authored Oct 14, 2024
2 parents d9237e2 + d8a2905 commit 718d287
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 8 deletions.
22 changes: 20 additions & 2 deletions data.midrc.org/etlMapping.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ mappings:
path: imaging_studies.mr_series_files
fn: count
parent_props:
- path: datasets[dataset_submitter_id:submitter_id]
- path: datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor]
nested_props:
- name: case_annotations
path: annotations
Expand All @@ -42,6 +42,7 @@ mappings:
props:
- name: airspace_disease_grading
- name: class_covid19_pneumonia
- name: annotation_name
- name: midrc_mRALE_score
- name: annotation_method
- name: annotator_id
Expand Down Expand Up @@ -137,6 +138,8 @@ mappings:
- name: test_method
- name: test_name
- name: test_result_text
parent_props:
- path: cases.datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor]
- name: midrc_annotation
doc_type: annotation
type: aggregator
Expand All @@ -147,6 +150,7 @@ mappings:
- name: case_ids
- name: airspace_disease_grading
- name: class_covid19_pneumonia
- name: annotation_name
- name: midrc_mRALE_score
- name: annotation_method
- name: annotator_id
Expand Down Expand Up @@ -181,6 +185,7 @@ mappings:
props:
- name: airspace_disease_grading
- name: class_covid19_pneumonia
- name: annotation_name
- name: midrc_mRALE_score
- name: annotation_method
- name: annotator_id
Expand All @@ -199,7 +204,7 @@ mappings:
path: mr_series_files
fn: count
parent_props:
- path: cases[sex,race,age_at_index,index_event,zip,covid19_positive,ethnicity].datasets[dataset_submitter_id:submitter_id]
- path: cases[sex,race,age_at_index,index_event,zip,covid19_positive,ethnicity].datasets[dataset_submitter_id:submitter_id,license,data_url_doi,data_contributor]
joining_props:
- index: data_file
join_on: _imaging_study_id
Expand Down Expand Up @@ -278,6 +283,7 @@ mappings:
- name: manufacturer
- name: manufacturer_model_name
- name: modality
- name: number_of_instances
- name: number_of_temporal_positions
- name: patient_position
- name: pixel_spacing
Expand All @@ -298,6 +304,18 @@ mappings:
- name: _dataset_id
src: id
fn: set
- name: dataset_submitter_id
src: submitter_id
fn: set
- name: license
src: license
fn: set
- name: data_url_doi
src: data_url_doi
fn: set
- name: data_contributor
src: data_contributor
fn: set
annotation:
props:
- name: _annotation_id
Expand Down
106 changes: 100 additions & 6 deletions data.midrc.org/portal/gitops.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@
"public": true,
"description": "The Jupyter notebooks contained in this notebook viewer pull data from various sources to generate and output useful tables, charts, graphs, and models. Each notebook is static, meaning the data being used by the notebooks is not updated in real time. Executable versions of these notebooks can be downloaded from <a href='https://github.com/MIDRC/tutorial_notebooks'> MIDRC's GitHub Repository</a>. If you’d like to browse the machine learning-related algorithms and code being developed by MIDRC, please visit <a href='https://midrc.org/algorithms'>the algorithms page</a> on our website.",
"resources": [
{
"title": "Cohort Building Using LOINC Terms",
"description": "This notebook demonstrates how to build a cohort of patients based on LOINC terms derived by harmonizing original DICOM study descriptions, modalities, and body parts examined using the LOINC playbook. Imaging study DICOM data element harmonization using the LOINC playbook is the result of an effort by MIDRC's Technology and Development Project 3b (TDP3b) and the Collaborative Research Project 12 (CRP-12). Links to the GitHub repository where the code for performing LOINC harmonization is maintained is included in the notebook.",
"imageUrl": "/dashboard/Public/notebooks/Cohort_Building_Using_LOINC_Terms.png",
"link": "/dashboard/Public/notebooks/Cohort_Building_Using_LOINC_Terms.html"
},
{
"title": "Build a Patient Cohort and Access Chest CT Scans and Annotations",
"description": "This notebook demonstrates how to build a cohort of MIDRC patients based on clinical and demographic data and then access all Chest CT scans and any related annotations.",
Expand Down Expand Up @@ -444,6 +450,7 @@
{
"title": "Annotations",
"fields": [
"imaging_study_annotations.annotation_name",
"imaging_study_annotations.midrc_mRALE_score",
"imaging_study_annotations.annotation_method"
]
Expand All @@ -464,6 +471,9 @@
"imaging_studies.age_at_imaging",
"imaging_studies.body_part_examined",
"imaging_studies.days_to_study",
"license",
"data_url_doi",
"data_contributor",
"imaging_studies.loinc_code",
"imaging_studies.loinc_contrast",
"imaging_studies.loinc_long_common_name",
Expand Down Expand Up @@ -497,6 +507,7 @@
"conditions.condition_code",
"conditions.days_to_condition_start",
"conditions.days_to_condition_end",
"imaging_study_annotations.annotation_name",
"imaging_study_annotations.midrc_mRALE_score",
"imaging_study_annotations.annotation_method",
"imaging_study_annotations.annotator_id",
Expand All @@ -507,7 +518,11 @@
"_cr_series_file_count",
"_mr_series_file_count",
"project_id"
]
],
"linkFields": [
"license",
"data_url_doi"
]
},
"dropdowns": {
"download-table": {
Expand Down Expand Up @@ -716,6 +731,10 @@
"field": "imaging_study_annotations.class_covid19_pneumonia",
"name": "Class COVID-19 Pneumonia"
},
{
"field": "imaging_study_annotations.annotation_name",
"name": "Annotation Name"
},
{
"field": "imaging_study_annotations.midrc_mRALE_score",
"name": "MIDRC mRALE Score"
Expand Down Expand Up @@ -751,6 +770,18 @@
{
"field": "_mr_series_file_count",
"name": "MR Series Count"
},
{
"field": "license",
"name": "License"
},
{
"field": "data_url_doi",
"name": "DOI"
},
{
"field": "data_contributor",
"name": "Data Contributor"
}
],
"manifestMapping": {
Expand All @@ -773,6 +804,7 @@
{
"title": "Imaging Study Annotations",
"fields": [
"annotation_name",
"midrc_mRALE_score",
"annotation_method",
"annotator_id",
Expand Down Expand Up @@ -890,8 +922,15 @@
"test_name",
"test_days_from_index",
"submitter_id",
"project_id"
]
"project_id",
"license",
"data_url_doi",
"data_contributor"
],
"linkFields": [
"license",
"data_url_doi"
]
},
"dropdowns": {
"download-table": {
Expand Down Expand Up @@ -943,6 +982,18 @@
{
"field": "case_ids",
"name": "Case ID"
},
{
"field": "license",
"name": "License"
},
{
"field": "data_url_doi",
"name": "DOI"
},
{
"field": "data_contributor",
"name": "Data Contributor"
}
],
"accessibleFieldCheckList": [
Expand Down Expand Up @@ -970,6 +1021,7 @@
"body_part_examined",
"days_from_study_to_pos_covid_test",
"days_from_study_to_neg_covid_test",
"image_data_modified",
"project_id"
]
},
Expand Down Expand Up @@ -1009,6 +1061,7 @@
{
"title": "Annotations",
"fields": [
"imaging_study_annotations.annotation_name",
"imaging_study_annotations.midrc_mRALE_score",
"imaging_study_annotations.annotation_method",
"imaging_study_annotations.annotator_id",
Expand All @@ -1028,6 +1081,9 @@
"days_to_study",
"study_year",
"study_year_shifted",
"license",
"data_url_doi",
"data_contributor",
"loinc_long_common_name",
"loinc_method",
"loinc_system",
Expand All @@ -1043,6 +1099,7 @@
"race",
"ethnicity",
"zip",
"imaging_study_annotations.annotation_name",
"imaging_study_annotations.midrc_mRALE_score",
"imaging_study_annotations.annotation_method",
"imaging_study_annotations.annotator_id",
Expand All @@ -1066,7 +1123,9 @@
"dicomViewerId": "submitter_id",
"dicomViewerUrl": "ohif-viewer",
"linkFields": [
"submitter_id"
"submitter_id",
"license",
"data_url_doi"
]
},
"dropdowns": {
Expand Down Expand Up @@ -1147,6 +1206,10 @@
"field": "imaging_study_annotations.class_covid19_pneumonia",
"name": "Class COVID-19 Pneumonia"
},
{
"field": "imaging_study_annotations.annotation_name",
"name": "Annotation Name"
},
{
"field": "imaging_study_annotations.midrc_mRALE_score",
"name": "MIDRC mRALE Score"
Expand Down Expand Up @@ -1179,6 +1242,18 @@
"field": "_mr_series_file_count",
"name": "MR Series Count"
},
{
"field": "license",
"name": "License"
},
{
"field": "data_url_doi",
"name": "DOI"
},
{
"field": "data_contributor",
"name": "Data Contributor"
},
{
"field": "loinc_long_common_name",
"name": "Long Common Name (Study description)"
Expand Down Expand Up @@ -1229,7 +1304,7 @@
"data_format",
"data_category",
"data_type",
"image_data_modified",
"image_data_modified",
"image_data_modification_name",
"image_data_modification_method"
]
Expand Down Expand Up @@ -1276,6 +1351,7 @@
"manufacturer",
"manufacturer_model_name",
"modality",
"number_of_instances",
"number_of_temporal_positions",
"patient_position",
"pixel_spacing",
Expand Down Expand Up @@ -1346,6 +1422,7 @@
"manufacturer",
"manufacturer_model_name",
"modality",
"number_of_instances",
"number_of_temporal_positions",
"patient_position",
"pixel_spacing",
Expand All @@ -1367,6 +1444,9 @@
"study_year_shifted",
"days_from_study_to_pos_covid_test",
"days_from_study_to_neg_covid_test",
"license",
"data_url_doi",
"data_contributor",
"loinc_code",
"loinc_contrast",
"loinc_long_common_name",
Expand Down Expand Up @@ -1398,7 +1478,9 @@
"dicomViewerId": "submitter_id",
"dicomViewerUrl": "ohif-viewer",
"linkFields": [
"submitter_id"
"submitter_id",
"license",
"data_url_doi"
]
},
"dropdowns": {
Expand Down Expand Up @@ -1470,6 +1552,18 @@
{
"field": "series_UID",
"name": "Series UID"
},
{
"field": "license",
"name": "License"
},
{
"field": "data_url_doi",
"name": "DOI"
},
{
"field": "data_contributor",
"name": "Data Contributor"
}
],
"manifestMapping": {
Expand Down

0 comments on commit 718d287

Please sign in to comment.