DBT:n lokien ja dokumentaation kopiointi S3-ämpäriin

Opetushallitus · Jan 29, 2025 · ce770f0 · ce770f0
1 parent e36c621
commit ce770f0
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 1 deletion.
diff --git a/cdk/lib/ecs-stack.ts b/cdk/lib/ecs-stack.ts
@@ -105,6 +105,22 @@ export class EcsStack extends cdk.Stack {
       dbtRunnerImageVersion
     );
 
+    const ovaraDokumentaatioBucket = s3.Bucket.fromBucketName(
+      this,
+      `${config.environment}-ovara-dokumentaatio`,
+      `${config.environment}-ovara-dokumentaatio`
+    );
+    const dbtLogsBucket = new s3.Bucket(this, `${config.environment}-dbt-logs`, {
+      bucketName: `${config.environment}-dbt-logs`,
+      objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_ENFORCED,
+      blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
+      serverAccessLogsBucket: new s3.Bucket(
+        this,
+        `${config.environment}-dbt-logs-bucket-server-access-logs`
+      ),
+      versioned: false,
+    });
+
     const dbtRunnerSchedule = appscaling.Schedule.cron(config.dbtCron);
     const dbtProcessingEnabled = config.dbtProcessingEnabled?.toLowerCase() === 'true';
     const dbtRunnerScheduledFargateTask = new ecsPatterns.ScheduledFargateTask(
@@ -122,6 +138,8 @@ export class EcsStack extends cdk.Stack {
             POSTGRES_HOST_PROD: `raportointi.db.${config.publicHostedZone}`,
             DBT_PORT_PROD: '5432',
             DBT_USERNAME_PROD: 'app',
+            OVARA_DOC_BUCKET: ovaraDokumentaatioBucket.bucketName,
+            DBT_LOGS_BUCKET: dbtLogsBucket.bucketName,
           },
           secrets: {
             DBT_PASSWORD_PROD: ecs.Secret.fromSsmParameter(
@@ -154,6 +172,11 @@ export class EcsStack extends cdk.Stack {
       })
     );
 
+    ovaraDokumentaatioBucket.grantReadWrite(
+      dbtRunnerScheduledFargateTask.taskDefinition.taskRole
+    );
+    dbtLogsBucket.grantReadWrite(dbtRunnerScheduledFargateTask.taskDefinition.taskRole);
+
     dbtRunnerScheduledFargateTask.taskDefinition
       .obtainExecutionRole()
       .grantAssumeRole(props.githubActionsDeploymentRole);

diff --git a/dbt-container/install.sh b/dbt-container/install.sh
@@ -12,7 +12,9 @@ echo $ARCHITECTURE
 echo "Installing needed software"
 apk --no-cache add \
   python3 \
-  py3-pip
+  py3-pip \
+  aws-cli \
+  tzdata
 
 ln -sf /usr/bin/python3 /usr/bin/python
 ln -sf /usr/bin/pip3 /usr/bin/pip

diff --git a/dbt-container/run.sh b/dbt-container/run.sh
@@ -28,4 +28,17 @@ start=$(date +%s)
 dbt run-operation tempdata_cleanup --target=prod
 echo "Siivouksen kesto `expr $(date +%s) - ${start}` s"
 
+echo "Generoidaan dokumentaatio"
+dbt docs generate --target=prod
+
+echo "Kopioidaan dokumentaatio S3:een"
+aws s3 cp ./target/catalog.json s3://$OVARA_DOC_BUCKET/dbt/catalog.json
+aws s3 cp ./target/index.html s3://$OVARA_DOC_BUCKET/dbt/index.html
+aws s3 cp ./target/manifest.json s3://$OVARA_DOC_BUCKET/dbt/manifest.json
+
+echo "Kopioidaan lokit S3:een"
+CURRENT_TIME="$(TZ=Europe/Helsinki date +%Y-%m-%d_%H:%M:%S%Z)"
+echo "$CURRENT_TIME"
+aws s3 cp ./logs s3://$DBT_LOGS_BUCKET/$CURRENT_TIME --recursive --include 'logs/dbt.log*'
+
 exit 0