From b0ab4788004a11b6b23d647e65f3a1a48d324ba8 Mon Sep 17 00:00:00 2001 From: August Kilponen Date: Thu, 30 Jan 2025 10:21:10 +0200 Subject: [PATCH] =?UTF-8?q?Tarkitus=20onko=20edellinen=20ECS-ajo=20(DBT=20?= =?UTF-8?q?tai=20Lampi=20export)=20viel=C3=A4=20k=C3=A4ynniss=C3=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cdk/lib/ecs-stack.ts | 20 ++++++++++++++++++++ dbt-container/install.sh | 3 ++- dbt-container/run.sh | 12 ++++++++++++ lampi-siirtaja-container/install.sh | 17 +++-------------- lampi-siirtaja-container/run.sh | 12 ++++++++++++ 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/cdk/lib/ecs-stack.ts b/cdk/lib/ecs-stack.ts index 6b7ffba..2109532 100644 --- a/cdk/lib/ecs-stack.ts +++ b/cdk/lib/ecs-stack.ts @@ -2,6 +2,7 @@ import * as cdk from 'aws-cdk-lib'; import * as appscaling from 'aws-cdk-lib/aws-applicationautoscaling'; import * as cloudwatch from 'aws-cdk-lib/aws-cloudwatch'; import * as cloudwatchActions from 'aws-cdk-lib/aws-cloudwatch-actions'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as ecr from 'aws-cdk-lib/aws-ecr'; import * as ecs from 'aws-cdk-lib/aws-ecs'; @@ -34,6 +35,18 @@ export class EcsStack extends cdk.Stack { const config: Config = props.config; + const ecsProsessiOnKaynnissaTable = new dynamodb.TableV2( + this, + 'ecsProsessiOnKaynnissa', + { + tableName: 'ecsProsessiOnKaynnissa', + partitionKey: { + name: 'prosessi', + type: dynamodb.AttributeType.STRING, + }, + } + ); + const addActionsToAlarm = (alarm: cloudwatch.Alarm) => { alarm.addAlarmAction( new cloudwatchActions.SnsAction(props.slackAlarmIntegrationSnsTopic) @@ -176,6 +189,9 @@ export class EcsStack extends cdk.Stack { dbtRunnerScheduledFargateTask.taskDefinition.taskRole ); dbtLogsBucket.grantReadWrite(dbtRunnerScheduledFargateTask.taskDefinition.taskRole); + ecsProsessiOnKaynnissaTable.grantReadWriteData( + dbtRunnerScheduledFargateTask.taskDefinition.taskRole + ); dbtRunnerScheduledFargateTask.taskDefinition .obtainExecutionRole() @@ -487,6 +503,10 @@ export class EcsStack extends cdk.Stack { lampiSiirtajaScheduledFargateTask.taskDefinition.taskRole ); + ecsProsessiOnKaynnissaTable.grantReadWriteData( + lampiSiirtajaScheduledFargateTask.taskDefinition.taskRole + ); + lampiSiirtajaScheduledFargateTask.taskDefinition.addToExecutionRolePolicy( new iam.PolicyStatement({ actions: [ diff --git a/dbt-container/install.sh b/dbt-container/install.sh index df98094..b95050f 100644 --- a/dbt-container/install.sh +++ b/dbt-container/install.sh @@ -14,7 +14,8 @@ apk --no-cache add \ python3 \ py3-pip \ aws-cli \ - tzdata + tzdata \ + jq ln -sf /usr/bin/python3 /usr/bin/python ln -sf /usr/bin/pip3 /usr/bin/pip diff --git a/dbt-container/run.sh b/dbt-container/run.sh index c1dbbf5..155a723 100755 --- a/dbt-container/run.sh +++ b/dbt-container/run.sh @@ -9,6 +9,15 @@ start=$(date +%s) cd dbt . venv/bin/activate +IS_RUNNING=$(aws dynamodb execute-statement --statement "SELECT onKaynnissa FROM ecsProsessiOnKaynnissa WHERE prosessi='dbt-scheduled-task'" | jq ' .Items.[0].onKaynnissa.S ') +if [ $IS_RUNNING == "true" ] ; then + echo "ERROR: Edellinen ajo on vielä käynnissä." + exit 1 +fi + +echo "Merkitetään DynamoDB:hen että prosessi on ajossa" +aws dynamodb execute-statement --statement "UPDATE ecsProsessiOnKaynnissa SET onKaynnissa='true' WHERE prosessi='dbt-scheduled-task' RETURNING ALL NEW *" + dbt seed -s raw_taulut --target=prod dbt run-operation create_raw_tables --target=prod @@ -41,4 +50,7 @@ CURRENT_TIME="$(TZ=Europe/Helsinki date +%Y-%m-%d_%H:%M:%S%Z)" echo "$CURRENT_TIME" aws s3 cp ./logs s3://$DBT_LOGS_BUCKET/$CURRENT_TIME --recursive --include 'logs/dbt.log*' --content-type 'text/plain' +echo "Merkitetään DynamoDB:hen että prosessi ei ole enää ajossa" +aws dynamodb execute-statement --statement "UPDATE ecsProsessiOnKaynnissa SET onKaynnissa='false' WHERE prosessi='dbt-scheduled-task' RETURNING ALL NEW *" + exit 0 diff --git a/lampi-siirtaja-container/install.sh b/lampi-siirtaja-container/install.sh index 14fb67b..6a7386e 100644 --- a/lampi-siirtaja-container/install.sh +++ b/lampi-siirtaja-container/install.sh @@ -9,20 +9,9 @@ case "$(uname -m)" in esac echo $ARCHITECTURE -#echo "Installing needed software" -#apk --no-cache add \ -# python3 \ -# py3-pip \ -# libpq-dev \ -# g++ \ -# make +echo "Installing needed software" +apk --no-cache add \ + jq echo "Listing contents of /root folder" ls -Al /root - -#echo "Listing contents of /root/lampi-siirtaja folder" -#ls -Al /root/lampi-siirtaja - -#echo "Installing dependencies with npm" -#cd /root/lampi-siirtaja -#npm ci diff --git a/lampi-siirtaja-container/run.sh b/lampi-siirtaja-container/run.sh index cf03490..cb49e4f 100755 --- a/lampi-siirtaja-container/run.sh +++ b/lampi-siirtaja-container/run.sh @@ -4,6 +4,15 @@ set -e echo "Running Lampi-siirtäjä..." +IS_RUNNING=$(aws dynamodb execute-statement --statement "SELECT onKaynnissa FROM ecsProsessiOnKaynnissa WHERE prosessi='lampi-scheduled-task'" | jq ' .Items.[0].onKaynnissa.S ') +if [ $IS_RUNNING == "true" ] ; then + echo "ERROR: Edellinen ajo on vielä käynnissä." + exit 1 +fi + +echo "Merkitetään DynamoDB:hen että prosessi on ajossa" +aws dynamodb execute-statement --statement "UPDATE ecsProsessiOnKaynnissa SET onKaynnissa='true' WHERE prosessi='lampi-scheduled-task' RETURNING ALL NEW *" + start=$(date +%s) cd /root @@ -11,4 +20,7 @@ java -jar ovara-lampi-siirtaja.jar echo "Ajon kesto `expr $(date +%s) - ${start}` s" +echo "Merkitetään DynamoDB:hen että prosessi ei ole enää ajossa" +aws dynamodb execute-statement --statement "UPDATE ecsProsessiOnKaynnissa SET onKaynnissa='false' WHERE prosessi='lampi-scheduled-task' RETURNING ALL NEW *" + exit 0