From 5d8114744824941fb89c70f247ea8e6537b02eb9 Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Fri, 4 Jun 2021 14:21:59 -0400 Subject: [PATCH 1/4] fix: Update sagemaker permission boundary to allow for stopping sagemaker instances --- .../service-catalog/sagemaker-notebook-instance.cfn.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml index 1ad5d7d950..dcf254c66a 100644 --- a/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml +++ b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml @@ -73,6 +73,11 @@ Resources: Action: - 'sts:AssumeRole' Resource: 'arn:aws:iam::*:role/swb-*' + - Effect: Allow + Action: + - sagemaker:DescribeNotebookInstance + - sagemaker:StopNotebookInstance + Resource: '*' IAMRole: Type: 'AWS::IAM::Role' Properties: @@ -175,7 +180,7 @@ Resources: chmod a+x /usr/local/bin/autostop.py IDLE_TIME=`expr ${AutoStopIdleTimeInMinutes} \* 60` echo "Starting the SageMaker autostop script in cron" - (crontab -l 2>/dev/null; echo "5 * * * * /usr/bin/python /usr/local/bin/autostop.py --time $IDLE_TIME --ignore-connections") | crontab - + (crontab -l 2>/dev/null; echo "5 * * * * /usr/bin/python /usr/local/bin/autostop.py --time $IDLE_TIME --ignore-connections > /var/log/autostop.log") | crontab - fi Outputs: From 244bd43f20f08781aabdfcbc57a3239a80190378 Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Fri, 4 Jun 2021 15:02:15 -0400 Subject: [PATCH 2/4] Run every 5 minutes --- .../service-catalog/sagemaker-notebook-instance.cfn.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml index dcf254c66a..5e559191b0 100644 --- a/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml +++ b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml @@ -180,7 +180,7 @@ Resources: chmod a+x /usr/local/bin/autostop.py IDLE_TIME=`expr ${AutoStopIdleTimeInMinutes} \* 60` echo "Starting the SageMaker autostop script in cron" - (crontab -l 2>/dev/null; echo "5 * * * * /usr/bin/python /usr/local/bin/autostop.py --time $IDLE_TIME --ignore-connections > /var/log/autostop.log") | crontab - + (crontab -l 2>/dev/null; echo "*/5 * * * * /usr/bin/python /usr/local/bin/autostop.py --time $IDLE_TIME --ignore-connections >> /var/log/autostop.log") | crontab - fi Outputs: From e9165b2ebb8c40b45e322c021e3f5890fd02a4f7 Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Sat, 5 Jun 2021 13:46:49 -0400 Subject: [PATCH 3/4] 1 minute interval for envStatusPollHandler and check autostop --- .../service-catalog/sagemaker-notebook-instance.cfn.yml | 2 +- main/solution/backend/config/infra/functions.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml index 5e559191b0..f5d85fc1e7 100644 --- a/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml +++ b/addons/addon-base-raas/packages/base-raas-cfn-templates/src/templates/service-catalog/sagemaker-notebook-instance.cfn.yml @@ -180,7 +180,7 @@ Resources: chmod a+x /usr/local/bin/autostop.py IDLE_TIME=`expr ${AutoStopIdleTimeInMinutes} \* 60` echo "Starting the SageMaker autostop script in cron" - (crontab -l 2>/dev/null; echo "*/5 * * * * /usr/bin/python /usr/local/bin/autostop.py --time $IDLE_TIME --ignore-connections >> /var/log/autostop.log") | crontab - + (crontab -l 2>/dev/null; echo "*/1 * * * * /usr/bin/python /usr/local/bin/autostop.py --time $IDLE_TIME --ignore-connections >> /var/log/autostop.log") | crontab - fi Outputs: diff --git a/main/solution/backend/config/infra/functions.yml b/main/solution/backend/config/infra/functions.yml index 4e008e862c..57ee15522d 100644 --- a/main/solution/backend/config/infra/functions.yml +++ b/main/solution/backend/config/infra/functions.yml @@ -33,7 +33,7 @@ envStatusPollHandler: description: Handles status polling for sc environments events: - schedule: - rate: rate(3 minutes) + rate: rate(1 minute) description: 'Invokes the lambda function that polls and synchronize environment status.' environment: APP_CUSTOM_USER_AGENT: ${self:custom.settings.customUserAgent} From 13c8d031aa51cc14f9d7e175f64c0f89434030e4 Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Mon, 7 Jun 2021 09:43:12 -0400 Subject: [PATCH 4/4] Trigger notification