Skip to content

Commit

Permalink
Update registered exports (#558)
Browse files Browse the repository at this point in the history
* Tweak module files

* Update Makefile

* Cleanup

* Run make

* Update workflow

* Created prelim yaml file for PortalPublication (#542)

* Created prelim yaml file for PortalPublication

* Rebuild NF.jsonld, json

* Add compilation step

* Created a parent publication model and an nf specific model. Anh nyuget edits were also incorporated

* Rebuild NF.jsonld, json

* Updated PortalPublication with new Enum for StudyUse

* Rebuild NF.jsonld, json

* fixed string vs int for pmid

* Fixed string vs int for pmid

* Rebuild NF.jsonld, json

* Update make step

* Rebuild NF.jsonld, json

* Made requested changes to PortalPublication model, along with fixed typos

* Rebuild NF.jsonld, json

* Update Makefile, make final revisions, recompile

* Rebuild NF.jsonld, json

---------

Co-authored-by: nf-osi[bot] <nf-osi@sagebionetworks.org>
Co-authored-by: Anh Nguyet Vu <32753274+anngvu@users.noreply.github.com>
Co-authored-by: Anh Nguyet Vu <anngvu@gmail.com>

* Finalize changes

* Add script

* Rebuild NF.jsonld, json

* Update workflow

* Tweak display labels

* Fixes

* Rebuild NF.jsonld, json

* Fix missing license values

* Update ref in docs

---------

Co-authored-by: aditya-nath-sage <aditya.nath@sagebase.org>
Co-authored-by: nf-osi[bot] <nf-osi@sagebionetworks.org>
  • Loading branch information
3 people authored Nov 27, 2024
1 parent 4c2a051 commit 12a9e43
Show file tree
Hide file tree
Showing 34 changed files with 2,372 additions and 2,487 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/main-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ jobs:
make PortalDataset
make PortalStudy
make Superdataset
make PortalPublication
make Protocol
# Set up supported python.
- uses: actions/setup-python@v5
Expand Down
18 changes: 1 addition & 17 deletions .github/workflows/propagate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,20 +39,4 @@ jobs:
- name: Push to Synapse schema registry only JSON schemas that changed
run: |
CHANGED=$(git diff --name-only HEAD HEAD~1 registered-json-schemas)
for FILE in $CHANGED
do
REQUEST_BODY=$(jq '. + { concreteType: "org.sagebionetworks.repo.model.schema.CreateSchemaRequest", dryRun: false }' $FILE)
RESPONSE=$(curl -X POST https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/start \
-H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN" \
-H "Content-Type: application/json" \
-d "$REQUEST_BODY")
# Will error if token not within response
TOKEN=$( echo $RESPONSE | jq -e -r '.token')
sleep 1
STATUS=$(curl "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/get/$TOKEN" \
-H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN")
echo $STATUS | jq -e '.newVersionInfo'
done
.github/workflows/register-schema.sh $CHANGED
34 changes: 34 additions & 0 deletions .github/workflows/register-schema.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash

for FILE in $1
do
# Synapse does not allow additionalProperties in schema, so remove that first
jq 'del(.additionalProperties)' $FILE > temp.json
REQUEST_BODY=$(jq '{schema : ., concreteType: "org.sagebionetworks.repo.model.schema.CreateSchemaRequest", dryRun: false }' temp.json)
RESPONSE=$(curl -X POST https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/start \
-H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN" \
-H "Content-Type: application/json" \
-d "$REQUEST_BODY")
rm temp.json

# Will error if token not within response
echo $RESPONSE
TOKEN=$( echo $RESPONSE | jq -e -r '.token')
sleep 1
STATUS=$(curl "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/get/$TOKEN" \
-H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN")
while true; do
if echo "$STATUS" | jq -e '.jobState == "PROCESSING"' > /dev/null; then
sleep 1
STATUS=$(curl "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/get/$TOKEN" \
-H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN")
elif echo "$STATUS" | jq -e '.concreteType == "org.sagebionetworks.repo.model.ErrorResponse"' > /dev/null; then
echo "Error: $(echo $STATUS | jq -r '.reason')"
exit 1
else
echo "Success:"
echo "$STATUS" | jq -e '.newVersionInfo'
break
fi
done
done
42 changes: 26 additions & 16 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,10 @@ PortalDataset:
yq '.slots |= with_entries(select(.value.in_subset[] == "portal"))' modules/props.yaml > relevant_props.yaml
yq ea '. as $$item ireduce ({}; . * $$item )' modules/Data/Data.yaml modules/Assay/Assay.yaml modules/Sample/Species.yaml modules/DCC/Portal.yaml > relevant_enums.yaml
cat header.yaml relevant_props.yaml relevant_enums.yaml modules/Template/PortalDataset.yaml > temp.yaml
gen-json-schema --inline --no-metadata --not-closed temp.yaml > tmp.json
rm relevant_props.yaml relevant_enums.yaml temp.yaml
gen-json-schema --inline --no-metadata --title-from=title --not-closed temp.yaml > tmp.json
json-dereference -s tmp.json -o tmp.json
jq '{ schema: { "$$schema": "http://json-schema.org/draft-07/schema#", "$$id": "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portaldataset", properties: ."$$defs".PortalDataset.properties, required: ."$$defs".PortalDataset.required }}' tmp.json > registered-json-schemas/PortalDataset.json
rm tmp.json
# There is a LinkML bug that needs some titles to be adjusted manually
yq -iP '.schema.properties.accessType.title="accessType"' registered-json-schemas/PortalDataset.json -o json
yq -iP '.schema.properties.assay.title="assay"' registered-json-schemas/PortalDataset.json -o json
yq -iP '.schema.properties.dataType.title="dataType"' registered-json-schemas/PortalDataset.json -o json
yq -iP '.schema.properties.fundingAgency.title="fundingAgency"' registered-json-schemas/PortalDataset.json -o json
yq -iP '.schema.properties.license.title="license"' registered-json-schemas/PortalDataset.json -o json
yq -iP '.schema.properties.manifestation.title="manifestation"' registered-json-schemas/PortalDataset.json -o json
yq -iP '.schema.properties.species.title="species"' registered-json-schemas/PortalDataset.json -o json
jq '."$$defs".PortalDataset | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portaldataset"' tmp.json > registered-json-schemas/PortalDataset.json
rm -f relevant_props.yaml relevant_enums.yaml temp.yaml tmp.json
@echo "--- Saved registered-json-schemas/PortalDataset.json ---"


Expand All @@ -34,14 +25,33 @@ PortalStudy:
yq eval-all '. as $$item ireduce ({}; . * $$item )' modules/Data/Data.yaml modules/DCC/Portal.yaml modules/Other/Organization.yaml > relevant_enums.yaml
cat header.yaml relevant_enums.yaml modules/Template/PortalStudy.yaml > temp.yaml
gen-json-schema --inline --no-metadata --not-closed temp.yaml > tmp.json
rm relevant_enums.yaml temp.yaml
json-dereference -s tmp.json -o tmp.json
jq '{ schema: { "$$schema": "http://json-schema.org/draft-07/schema#", "$$id": "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portalstudy", properties: ."$$defs".PortalStudy.properties, required: ."$$defs".PortalStudy.required }}' tmp.json > registered-json-schemas/PortalStudy.json
rm tmp.json
jq '."$$defs".PortalStudy | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portalstudy"' tmp.json > registered-json-schemas/PortalStudy.json
rm -f relevant_enums.yaml temp.yaml tmp.json
@echo "--- Saved registered-json-schemas/PortalStudy.json ---"

Protocol:
yq '.slots |= with_entries(select(.value.in_subset[] == "portal" or .value.in_subset[] == "registered"))' modules/props.yaml > relevant_props.yaml
yq ea '. as $$item ireduce ({}; . * $$item )' modules/Data/Data.yaml modules/DCC/Portal.yaml modules/Assay/Assay.yaml > relevant_enums.yaml
cat header.yaml relevant_props.yaml relevant_enums.yaml modules/Template/Protocol.yaml > temp.yaml
gen-json-schema --inline --no-metadata --not-closed --title-from=title temp.yaml > tmp.json
json-dereference -s tmp.json -o tmp.json
jq '."$$defs".Protocol | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-protocol"' tmp.json > registered-json-schemas/Protocol.json
rm -f relevant_props.yaml relevant_enums.yaml temp.yaml tmp.json
@echo "--- Saved registered-json-schemas/Protocol.json ---"

Superdataset:
jq '.schema += input | del('.schema.required') | .schema["$$id"] = "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-superdataset"' registered-json-schemas/PortalDataset.json registered-json-schemas/super_rules.json > registered-json-schemas/Superdataset.json
jq '. += input | del(.required) | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-superdataset"' registered-json-schemas/PortalDataset.json registered-json-schemas/super_rules.json > registered-json-schemas/Superdataset.json
@echo "--- Saved registered-json-schemas/Superdataset.json ---"

PortalPublication:
yq ea '. as $$item ireduce ({}; . * $$item )' modules/DCC/Portal.yaml modules/Other/PublicationEnum.yaml > relevant_enums.yaml
cat header.yaml modules/Template/PortalPublication.yaml relevant_enums.yaml > temp.yaml
gen-json-schema --top-class=PortalPublication --no-metadata --not-closed --title-from=title temp.yaml > tmp.json
json-dereference -s tmp.json -o tmp.json
jq '."$$defs".PortalPublication | ."$$id"="https://repo-prod.prod.sagebase.org/repo/v1/schema/type/registered/org.synapse.nf-portalpublication"' tmp.json > registered-json-schemas/PortalPublication.json
rm -f tmp.json temp.yaml relevant_enums.yaml
@echo "--- Saved registered-json-schemas/PortalPublication.json ---"

PortalPublication:
yq ea '. as $$item ireduce ({}; . * $$item )' modules/DCC/Portal.yaml modules/Other/PublicationEnum.yaml > relevant_enums.yaml
Expand Down
Loading

0 comments on commit 12a9e43

Please sign in to comment.