Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automated project now working #127

Merged
merged 1 commit into from
Sep 23, 2024
Merged

Automated project now working #127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 31 additions & 20 deletions bootstrap/ic-user-projects/create-projects-and-resources-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,13 @@ spec:
MINIO_ROOT_USER=$(oc get secret minio-root-user -n ic-shared-minio -o template --template '{{.data.MINIO_ROOT_USER|base64decode}}')
MINIO_ROOT_PASSWORD=$(oc get secret minio-root-user -n ic-shared-minio -o template --template '{{.data.MINIO_ROOT_PASSWORD|base64decode}}')
MINIO_HOST=https://$(oc get route minio-s3 -n ic-shared-minio -o template --template '{{.spec.host}}')
MINIO_ROUTE=$(oc get route minio-s3 -n ic-shared-minio -o template --template '{{.spec.host}}')
DASHBOARD_ROUTE=https://$(oc get route rhods-dashboard -n redhat-ods-applications -o jsonpath='{.spec.host}')

# Define some variables
WORKBENCH_NAME="my-workbench"
WORKBENCH_IMAGE="ic-workbench:2.1.2"
PIPELINE_ENGINE="Tekton"
WORKBENCH_IMAGE="s2i-generic-data-science-notebook:2024.1"
PIPELINE_ENGINE="Argo"

for i in $(seq 1 $user_count);
do
Expand Down Expand Up @@ -135,7 +136,7 @@ spec:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ds-pipeline-user-access-pipelines-definition
name: ds-pipeline-user-access-dspa
EOF

# Create Data Science Connections
Expand All @@ -155,7 +156,7 @@ spec:
- |-
echo "Minio user: $MINIO_ROOT_USER"
echo "Minio pass: $MINIO_ROOT_PASSWORD"
echo "Internal service url: http://minio.ic-shared-minio.svc.cluster.local:9000/"
echo "Minio route: $MINIO_HOST"
cat << EOF | oc apply -f-
apiVersion: v1
kind: Secret
Expand All @@ -172,7 +173,7 @@ spec:
AWS_ACCESS_KEY_ID: $MINIO_ROOT_USER
AWS_SECRET_ACCESS_KEY: $MINIO_ROOT_PASSWORD
AWS_DEFAULT_REGION: us
AWS_S3_ENDPOINT: http://minio.ic-shared-minio.svc:9000
AWS_S3_ENDPOINT: $MINIO_HOST
AWS_S3_BUCKET: $USER_NAME
EOF
command:
Expand All @@ -192,42 +193,52 @@ spec:
metadata:
finalizers:
- datasciencepipelinesapplications.opendatahub.io/finalizer
name: pipelines-definition
name: dspa
namespace: $USER_PROJECT
spec:
apiServer:
applyTektonCustomResource: true
archiveLogs: false
autoUpdatePipelineDefaultVersion: true
collectMetrics: true
caBundleFileMountPath: ''
stripEOF: true
dbConfigConMaxLifetimeSec: 120
applyTektonCustomResource: true
caBundleFileName: ''
deploy: true
enableOauth: true
artifactSignedURLExpirySeconds: 60
enableSamplePipeline: false
injectDefaultScript: true
stripEOF: true
autoUpdatePipelineDefaultVersion: true
archiveLogs: false
terminateStatus: Cancelled
enableOauth: true
trackArtifacts: true
collectMetrics: true
injectDefaultScript: true
database:
disableHealthCheck: false
mariaDB:
deploy: true
pipelineDBName: mlpipeline
pvcSize: 10Gi
username: mlpipeline
dspVersion: v2
objectStorage:
disableHealthCheck: false
enableExternalRoute: false
externalStorage:
basePath: ''
bucket: $USER_NAME
host: minio.ic-shared-minio.svc.cluster.local:9000
host: $MINIO_ROUTE
port: ''
region: us
s3CredentialsSecret:
accessKey: AWS_ACCESS_KEY_ID
secretKey: AWS_SECRET_ACCESS_KEY
secretName: aws-connection-shared-minio---pipelines
scheme: http
secure: false
scheme: https
secure: true
persistenceAgent:
deploy: true
numWorkers: 2
podToPodTLS: true
scheduledWorkflow:
cronScheduleTimezone: UTC
deploy: true
Expand All @@ -248,13 +259,13 @@ spec:
- args:
- -ec
- |-
echo -n 'Waiting for ds-pipeline-pipelines-definition route'
while ! oc get route ds-pipeline-pipelines-definition 2>/dev/null; do
echo -n 'Waiting for ds-pipeline-dspa route'
while ! oc get route ds-pipeline-dspa 2>/dev/null; do
echo -n .
sleep 5
done; echo

PIPELINE_ROUTE=https://\$(oc get route ds-pipeline-pipelines-definition -o jsonpath='{.spec.host}')
PIPELINE_ROUTE=https://\$(oc get route ds-pipeline-dspa -o jsonpath='{.spec.host}')

cat << EOF | oc apply -f-
apiVersion: v1
Expand All @@ -266,7 +277,7 @@ spec:
odh_dsp.json: '{"display_name": "Data Science Pipeline", "metadata": {"tags": [],
"display_name": "Data Science Pipeline", "engine": "$PIPELINE_ENGINE", "auth_type": "KUBERNETES_SERVICE_ACCOUNT_TOKEN",
"api_endpoint": "\$PIPELINE_ROUTE",
"public_api_endpoint": "$DASHBOARD_ROUTE/pipelineRuns/$USER_PROJECT/pipelineRun/view/",
"public_api_endpoint": "$DASHBOARD_ROUTE/experiments/$USER_PROJECT",
"cos_auth_type": "KUBERNETES_SECRET", "cos_secret": "aws-connection-shared-minio---pipelines",
"cos_endpoint": "$MINIO_HOST", "cos_bucket": "$USER_NAME",
"cos_username": "$MINIO_ROOT_USER", "cos_password": "$MINIO_ROOT_PASSWORD",
Expand Down
19 changes: 10 additions & 9 deletions bootstrap/ic-user-projects/create-projects-and-resources.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@ done; echo
MINIO_ROOT_USER=$(oc get secret minio-root-user -n ic-shared-minio -o template --template '{{.data.MINIO_ROOT_USER|base64decode}}')
MINIO_ROOT_PASSWORD=$(oc get secret minio-root-user -n ic-shared-minio -o template --template '{{.data.MINIO_ROOT_PASSWORD|base64decode}}')
MINIO_HOST=https://$(oc get route minio-s3 -n ic-shared-minio -o template --template '{{.spec.host}}')
MINIO_ROUTE=$(oc get route minio-s3 -n ic-shared-minio -o template --template '{{.spec.host}}')
DASHBOARD_ROUTE=https://$(oc get route rhods-dashboard -n redhat-ods-applications -o jsonpath='{.spec.host}')

# Define some variables
WORKBENCH_NAME="my-workbench"
WORKBENCH_IMAGE="ic-workbench:2.1.2"
PIPELINE_ENGINE="Tekton"
PIPELINE_ENGINE="Argo"

for i in $(seq 1 $user_count);
do
Expand Down Expand Up @@ -113,7 +114,7 @@ subjects:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ds-pipeline-user-access-pipelines-definition
name: ds-pipeline-user-access-dspa
EOF

# Create Data Science Connections
Expand Down Expand Up @@ -195,14 +196,14 @@ spec:
objectStorage:
externalStorage:
bucket: $USER_NAME
host: minio.ic-shared-minio.svc.cluster.local:9000
host: $MINIO_ROUTE
port: ''
s3CredentialsSecret:
accessKey: AWS_ACCESS_KEY_ID
secretKey: AWS_SECRET_ACCESS_KEY
secretName: aws-connection-shared-minio---pipelines
scheme: http
secure: false
scheme: https
secure: true
persistenceAgent:
deploy: true
numWorkers: 2
Expand All @@ -226,13 +227,13 @@ spec:
- args:
- -ec
- |-
echo -n 'Waiting for ds-pipeline-pipelines-definition route'
while ! oc get route ds-pipeline-pipelines-definition 2>/dev/null; do
echo -n 'Waiting for ds-pipeline-dspa route'
while ! oc get route ds-pipeline-dspa 2>/dev/null; do
echo -n .
sleep 5
done; echo

PIPELINE_ROUTE=https://\$(oc get route ds-pipeline-pipelines-definition -o jsonpath='{.spec.host}')
PIPELINE_ROUTE=https://\$(oc get route ds-pipeline-dspa -o jsonpath='{.spec.host}')

cat << EOF | oc apply -f-
apiVersion: v1
Expand All @@ -244,7 +245,7 @@ spec:
odh_dsp.json: '{"display_name": "Data Science Pipeline", "metadata": {"tags": [],
"display_name": "Data Science Pipeline", "engine": "$PIPELINE_ENGINE", "auth_type": "KUBERNETES_SERVICE_ACCOUNT_TOKEN",
"api_endpoint": "\$PIPELINE_ROUTE",
"public_api_endpoint": "$DASHBOARD_ROUTE/pipelineRuns/$USER_PROJECT/pipelineRun/view/",
"public_api_endpoint": "$DASHBOARD_ROUTE/experiments/$USER_PROJECT",
"cos_auth_type": "KUBERNETES_SECRET", "cos_secret": "aws-connection-shared-minio---pipelines",
"cos_endpoint": "$MINIO_HOST", "cos_bucket": "$USER_NAME",
"cos_username": "$MINIO_ROOT_USER", "cos_password": "$MINIO_ROOT_PASSWORD",
Expand Down
Loading