From 42b2539d4558c234e74ac99871158cf37b1d79a8 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Thu, 30 Apr 2020 11:42:03 -0500 Subject: [PATCH] Feat/bhcdatastaging april release (#1446) * update services * update workspace * add sower * add export buttons * add pub page * downgrade sheepdog * remove pub page for now --- dataprep.braincommons.org/manifest.json | 124 ++++++++++++++++--- dataprep.braincommons.org/portal/gitops.json | 14 +++ 2 files changed, 120 insertions(+), 18 deletions(-) diff --git a/dataprep.braincommons.org/manifest.json b/dataprep.braincommons.org/manifest.json index 84d11f5dc5..17c32a80c6 100644 --- a/dataprep.braincommons.org/manifest.json +++ b/dataprep.braincommons.org/manifest.json @@ -7,24 +7,26 @@ "autodeploy": "yes" }, "versions": { - "arborist": "quay.io/cdis/arborist:2.3.2", + "arborist": "quay.io/cdis/arborist:2020.04", + "ambassador": "quay.io/datawire/ambassador:0.60.3", "aws-es-proxy": "abutaha/aws-es-proxy:0.8", "dashboard": "quay.io/cdis/gen3-statics:1.0.0", - "ambassador": "quay.io/datawire/ambassador:0.60.3", - "hatchery": "quay.io/cdis/hatchery:0.1.0", - "fence": "quay.io/cdis/fence:4.14.0", - "indexd": "quay.io/cdis/indexd:2.1.0", - "peregrine": "quay.io/cdis/peregrine:2.1.1", - "pidgin": "quay.io/cdis/pidgin:1.1.0", - "revproxy": "quay.io/cdis/nginx:1.17.6-ctds-1.0.1", - "sheepdog": "quay.io/cdis/sheepdog:2.1.2", - "portal": "quay.io/cdis/data-portal:2.23.4", "fluentd": "fluent/fluentd-kubernetes-daemonset:v1.2-debian-cloudwatch", - "ssjdispatcher": "quay.io/cdis/ssjdispatcher:0.0.6", - "spark": "quay.io/cdis/gen3-spark:1.0.0", - "tube": "quay.io/cdis/tube:0.3.22", - "guppy": "quay.io/cdis/guppy:0.3.10", - "wts": "quay.io/cdis/workspace-token-service:0.1.0" + "guppy": "quay.io/cdis/guppy:2020.04", + "hatchery": "quay.io/cdis/hatchery:2020.04", + "fence": "quay.io/cdis/fence:2020.04", + "indexd": "quay.io/cdis/indexd:2020.04", + "manifestservice": "quay.io/cdis/manifestservice:2020.04", + "peregrine": "quay.io/cdis/peregrine:2020.04", + "pidgin": "quay.io/cdis/pidgin:2020.04", + "portal": "quay.io/cdis/data-portal:2.25.2", + "revproxy": "quay.io/cdis/nginx:1.17.6-ctds-1.0.1", + "sheepdog": "quay.io/cdis/sheepdog:3.0.0", + "sower": "quay.io/cdis/sower:2020.04", + "ssjdispatcher": "quay.io/cdis/ssjdispatcher:2020.04", + "spark": "quay.io/cdis/gen3-spark:2020.04", + "tube": "quay.io/cdis/tube:2020.04", + "wts": "quay.io/cdis/workspace-token-service:2020.04" }, "arborist": { "deployment_version": "2" @@ -69,7 +71,9 @@ "args": [ "--NotebookApp.base_url=/lw-workspace/proxy/", "--NotebookApp.password=''", - "--NotebookApp.token=''" + "--NotebookApp.token=''", + "--NotebookApp.notebook_dir='/home/jovyan/pd'", + "--NotebookApp.quit_button=False" ], "command": [ "start-notebook.sh" @@ -96,7 +100,9 @@ "args": [ "--NotebookApp.base_url=/lw-workspace/proxy/", "--NotebookApp.password=''", - "--NotebookApp.token=''" + "--NotebookApp.token=''", + "--NotebookApp.notebook_dir='/home/jovyan/pd'", + "--NotebookApp.quit_button=False" ], "command": [ "start-notebook.sh" @@ -123,7 +129,9 @@ "args": [ "--NotebookApp.base_url=/lw-workspace/proxy/", "--NotebookApp.password=''", - "--NotebookApp.token=''" + "--NotebookApp.token=''", + "--NotebookApp.notebook_dir='/home/jovyan/pd'", + "--NotebookApp.quit_button=False" ], "command": [ "start-notebook.sh" @@ -139,9 +147,89 @@ "user-uid": 1000, "fs-gid": 100, "user-volume-location": "/home/jovyan/pd" + }, + { + "target-port": 8787, + "cpu-limit": "1.0", + "memory-limit": "1024Mi", + "name": "R Studio", + "image": "quay.io/cdis/rstudio:master", + "env": {"DISABLE_AUTH": "true"}, + "args": [], + "path-rewrite": "/", + "use-tls": "false", + "ready-probe": "/", + "user-volume-location": "/home/rstudio/pd", + "fs-gid": 100 } ] }, + "sower": [ + { + "name": "pelican-export", + "action": "export", + "container": { + "name": "job-task", + "image": "quay.io/cdis/pelican-export:2020.04", + "pull_policy": "Always", + "env": [ + { + "name": "DICTIONARY_URL", + "valueFrom": { + "configMapKeyRef": { + "name": "manifest-global", + "key": "dictionary_url" + } + } + }, + { + "name": "GEN3_HOSTNAME", + "valueFrom": { + "configMapKeyRef": { + "name": "manifest-global", + "key": "hostname" + } + } + }, + { + "name": "ROOT_NODE", + "value": "case" + } + ], + "volumeMounts": [ + { + "name": "pelican-creds-volume", + "readOnly": true, + "mountPath": "/pelican-creds.json", + "subPath": "config.json" + }, + { + "name": "peregrine-creds-volume", + "readOnly": true, + "mountPath": "/peregrine-creds.json", + "subPath": "creds.json" + } + ], + "cpu-limit": "1", + "memory-limit": "12Gi" + }, + "volumes": [ + { + "name": "pelican-creds-volume", + "secret": { + "secretName": "pelicanservice-g3auto" + } + }, + { + "name": "peregrine-creds-volume", + "secret": { + "secretName": "peregrine-creds" + } + } + ], + "restart_policy": "Never" + } + ], "global": { "environment": "bhcdatastaging", "hostname": "dataprep.braincommons.org", diff --git a/dataprep.braincommons.org/portal/gitops.json b/dataprep.braincommons.org/portal/gitops.json index 1f52576e59..56940fca67 100644 --- a/dataprep.braincommons.org/portal/gitops.json +++ b/dataprep.braincommons.org/portal/gitops.json @@ -291,6 +291,13 @@ "leftIcon": "datafile", "rightIcon": "download", "fileName": "manifest.json" + }, + { + "enabled": true, + "type": "export-to-workspace", + "title": "Export File Manifest to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" } ] }, @@ -350,6 +357,13 @@ "leftIcon": "datafile", "rightIcon": "download", "fileName": "file-manifest.json" + }, + { + "enabled": true, + "type": "export-files-to-workspace", + "title": "Export File Manifest to Workspace", + "leftIcon": "datafile", + "rightIcon": "download" } ], "dropdowns": {}