From ee13321650adaf9f4f6d0679d3f6388328c6a7e3 Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Mon, 19 Feb 2024 17:18:52 -0300 Subject: [PATCH 1/8] Node v18; CDK @1.204.0 (#100) --- .nvmrc | 1 + README.md | 4 +- constraints.txt | 122 ++++++++++++++++++++++++------------------------ 3 files changed, 65 insertions(+), 62 deletions(-) create mode 100644 .nvmrc diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..7eae4e2 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +18.0.0 diff --git a/README.md b/README.md index 027d334..a05ed13 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,12 @@ Some lambdas require extra pip packages to be installed in the lambda directory Deployment uses AWS CDK. Requirements: -* node: Use [nvm](https://heynode.com/tutorial/install-nodejs-locally-nvm/) to make sure a supported node is being used, tested with 16.16.0 +* node: Use [nvm](https://heynode.com/tutorial/install-nodejs-locally-nvm/) to make sure a supported node is being used, tested with 18.0.0 * AWS credentials configured To install and check AWS CDK: ```bash -$ npm install -g aws-cdk +$ npm install -g aws-cdk@1.204.0 $ cdk --version $ cdk bootstrap # Deploys the CDK toolkit stack into an AWS environment diff --git a/constraints.txt b/constraints.txt index 016c29a..b9b9b72 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,70 +1,71 @@ astroid==2.11.7 attrs==21.4.0 -aws-cdk.assets==1.168.0 -aws-cdk.aws-acmpca==1.168.0 -aws-cdk.aws-apigateway==1.168.0 -aws-cdk.aws-applicationautoscaling==1.168.0 -aws-cdk.aws-autoscaling==1.168.0 -aws-cdk.aws-autoscaling-common==1.168.0 -aws-cdk.aws-autoscaling-hooktargets==1.168.0 -aws-cdk.aws-certificatemanager==1.168.0 -aws-cdk.aws-cloudformation==1.168.0 -aws-cdk.aws-cloudfront==1.168.0 -aws-cdk.aws-cloudwatch==1.168.0 -aws-cdk.aws-cloudwatch-actions==1.168.0 -aws-cdk.aws-codebuild==1.168.0 -aws-cdk.aws-codecommit==1.168.0 -aws-cdk.aws-codeguruprofiler==1.168.0 -aws-cdk.aws-codepipeline==1.168.0 -aws-cdk.aws-codestarnotifications==1.168.0 -aws-cdk.aws-cognito==1.168.0 -aws-cdk.aws-dynamodb==1.168.0 -aws-cdk.aws-ec2==1.168.0 -aws-cdk.aws-ecr==1.168.0 -aws-cdk.aws-ecr-assets==1.168.0 -aws-cdk.aws-ecs==1.168.0 -aws-cdk.aws-efs==1.168.0 -aws-cdk.aws-elasticloadbalancing==1.168.0 -aws-cdk.aws-elasticloadbalancingv2==1.168.0 -aws-cdk.aws-elasticsearch==1.168.0 -aws-cdk.aws-events==1.168.0 -aws-cdk.aws-events-targets==1.168.0 -aws-cdk.aws-globalaccelerator==1.168.0 -aws-cdk.aws-iam==1.168.0 -aws-cdk.aws-kinesis==1.168.0 -aws-cdk.aws-kinesisfirehose==1.168.0 -aws-cdk.aws-kms==1.168.0 -aws-cdk.aws-lambda==1.168.0 -aws-cdk.aws-lambda-event-sources==1.168.0 -aws-cdk.aws-logs==1.168.0 -aws-cdk.aws-route53==1.168.0 -aws-cdk.aws-route53-targets==1.168.0 -aws-cdk.aws-s3==1.168.0 -aws-cdk.aws-s3-assets==1.168.0 -aws-cdk.aws-s3-deployment==1.168.0 -aws-cdk.aws-s3-notifications==1.168.0 -aws-cdk.aws-sam==1.168.0 -aws-cdk.aws-secretsmanager==1.168.0 -aws-cdk.aws-servicediscovery==1.168.0 -aws-cdk.aws-signer==1.168.0 -aws-cdk.aws-sns==1.168.0 -aws-cdk.aws-sns-subscriptions==1.168.0 -aws-cdk.aws-sqs==1.168.0 -aws-cdk.aws-ssm==1.168.0 -aws-cdk.aws-stepfunctions==1.168.0 -aws-cdk.aws-synthetics==1.168.0 -aws-cdk.cloud-assembly-schema==1.168.0 -aws-cdk.core==1.168.0 -aws-cdk.custom-resources==1.168.0 -aws-cdk.cx-api==1.168.0 -aws-cdk.lambda-layer-awscli==1.168.0 -aws-cdk.region-info==1.168.0 +aws-cdk.assets==1.204.0 +aws-cdk.aws-acmpca==1.204.0 +aws-cdk.aws-apigateway==1.204.0 +aws-cdk.aws-applicationautoscaling==1.204.0 +aws-cdk.aws-autoscaling==1.204.0 +aws-cdk.aws-autoscaling-common==1.204.0 +aws-cdk.aws-autoscaling-hooktargets==1.204.0 +aws-cdk.aws-certificatemanager==1.204.0 +aws-cdk.aws-cloudformation==1.204.0 +aws-cdk.aws-cloudfront==1.204.0 +aws-cdk.aws-cloudwatch==1.204.0 +aws-cdk.aws-cloudwatch-actions==1.204.0 +aws-cdk.aws-codebuild==1.204.0 +aws-cdk.aws-codecommit==1.204.0 +aws-cdk.aws-codeguruprofiler==1.204.0 +aws-cdk.aws-codepipeline==1.204.0 +aws-cdk.aws-codestarnotifications==1.204.0 +aws-cdk.aws-cognito==1.204.0 +aws-cdk.aws-dynamodb==1.204.0 +aws-cdk.aws-ec2==1.204.0 +aws-cdk.aws-ecr==1.204.0 +aws-cdk.aws-ecr-assets==1.204.0 +aws-cdk.aws-ecs==1.204.0 +aws-cdk.aws-efs==1.204.0 +aws-cdk.aws-elasticloadbalancing==1.204.0 +aws-cdk.aws-elasticloadbalancingv2==1.204.0 +aws-cdk.aws-elasticsearch==1.204.0 +aws-cdk.aws-events==1.204.0 +aws-cdk.aws-events-targets==1.204.0 +aws-cdk.aws-globalaccelerator==1.204.0 +aws-cdk.aws-iam==1.204.0 +aws-cdk.aws-kinesis==1.204.0 +aws-cdk.aws-kinesisfirehose==1.204.0 +aws-cdk.aws-kms==1.204.0 +aws-cdk.aws-lambda==1.204.0 +aws-cdk.aws-lambda-event-sources==1.204.0 +aws-cdk.aws-logs==1.204.0 +aws-cdk.aws-route53==1.204.0 +aws-cdk.aws-route53-targets==1.204.0 +aws-cdk.aws-s3==1.204.0 +aws-cdk.aws-s3-assets==1.204.0 +aws-cdk.aws-s3-deployment==1.204.0 +aws-cdk.aws-s3-notifications==1.204.0 +aws-cdk.aws-sam==1.204.0 +aws-cdk.aws-secretsmanager==1.204.0 +aws-cdk.aws-servicediscovery==1.204.0 +aws-cdk.aws-signer==1.204.0 +aws-cdk.aws-sns==1.204.0 +aws-cdk.aws-sns-subscriptions==1.204.0 +aws-cdk.aws-sqs==1.204.0 +aws-cdk.aws-ssm==1.204.0 +aws-cdk.aws-stepfunctions==1.204.0 +aws-cdk.aws-synthetics==1.204.0 +aws-cdk.cloud-assembly-schema==1.204.0 +aws-cdk.core==1.204.0 +aws-cdk.custom-resources==1.204.0 +aws-cdk.cx-api==1.204.0 +aws-cdk.lambda-layer-awscli==1.204.0 +aws-cdk.region-info==1.204.0 aws-requests-auth==0.4.3 awscli==1.25.48 awscli-local==0.20 boto3==1.24.48 botocore==1.27.48 cattrs==22.1.0 +-e git+ssh://git@github.com/fredliporace/cbers-2-stac.git@2e9c09f56916c00ce61a7bea531dca2a359a0e21#egg=cbers_2_stac certifi==2022.6.15 cfgv==3.3.1 charset-normalizer==2.1.0 @@ -83,10 +84,11 @@ filelock==3.8.0 identify==2.5.3 idna==3.3 importlib-metadata==1.7.0 +importlib-resources==5.12.0 iniconfig==1.1.1 isort==5.10.1 jmespath==1.0.1 -jsii==1.63.2 +jsii==1.91.0 jsonschema==3.2.0 lazy-object-proxy==1.7.1 localstack-client==1.36 From bc1b5806fe1e264b30716e15f88037065f895178 Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Mon, 19 Feb 2024 17:30:50 -0300 Subject: [PATCH 2/8] use `Tags.of(scope).add()` (#100) --- stack/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/app.py b/stack/app.py index 15e6b23..76bd7fc 100644 --- a/stack/app.py +++ b/stack/app.py @@ -906,7 +906,7 @@ def __init__( "cost_center": settings.cost_center, }.items(): if value: - core.Tag.add(app, key, value) + core.Tags.of(app).add(key, value) stackname = f"{settings.name}-{settings.stage}" CBERS2STACStack( From 85d5155e8899d4a335bbb3735b04079f3cd95b15 Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Mon, 19 Feb 2024 17:42:27 -0300 Subject: [PATCH 3/8] @aws-cdk/aws-lambda.Code#asset is deprecated. use `fromAsset` (#100) --- stack/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/app.py b/stack/app.py index 76bd7fc..c4951ef 100644 --- a/stack/app.py +++ b/stack/app.py @@ -359,7 +359,7 @@ def create_all_layers(self) -> None: layer_dir="./cbers2stac/layers/common", prefix="python", ) - common_layer_asset = aws_lambda.Code.asset("./stack/common.zip") + common_layer_asset = aws_lambda.Code.from_asset("./stack/common.zip") self.layers_["common_layer"] = aws_lambda.LayerVersion( self, "common_layer", From abcb0298b7e8fe50d64c53ba51622051ee66a6a6 Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Mon, 19 Feb 2024 18:01:59 -0300 Subject: [PATCH 4/8] aws-elasticsearch.* is deprecated. use opensearchservice (#100) --- constraints.txt | 3 ++- setup.py | 1 + stack/app.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/constraints.txt b/constraints.txt index b9b9b72..8b60c49 100644 --- a/constraints.txt +++ b/constraints.txt @@ -37,6 +37,7 @@ aws-cdk.aws-kms==1.204.0 aws-cdk.aws-lambda==1.204.0 aws-cdk.aws-lambda-event-sources==1.204.0 aws-cdk.aws-logs==1.204.0 +aws-cdk.aws-opensearchservice==1.204.0 aws-cdk.aws-route53==1.204.0 aws-cdk.aws-route53-targets==1.204.0 aws-cdk.aws-s3==1.204.0 @@ -65,7 +66,7 @@ awscli-local==0.20 boto3==1.24.48 botocore==1.27.48 cattrs==22.1.0 --e git+ssh://git@github.com/fredliporace/cbers-2-stac.git@2e9c09f56916c00ce61a7bea531dca2a359a0e21#egg=cbers_2_stac +-e git+ssh://git@github.com/fredliporace/cbers-2-stac.git@85d5155e8899d4a335bbb3735b04079f3cd95b15#egg=cbers_2_stac certifi==2022.6.15 cfgv==3.3.1 charset-normalizer==2.1.0 diff --git a/setup.py b/setup.py index 9c7eaa6..047ac57 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ "aws-cdk.aws-events-targets", "aws-cdk.aws-apigateway", "aws-cdk.aws-elasticsearch", + "aws-cdk.aws-opensearchservice", "aws-cdk.aws-synthetics", ], } diff --git a/stack/app.py b/stack/app.py index c4951ef..fe0b422 100644 --- a/stack/app.py +++ b/stack/app.py @@ -10,10 +10,10 @@ from aws_cdk import aws_cloudwatch as cloudwatch from aws_cdk import aws_cloudwatch_actions as cw_actions from aws_cdk import aws_dynamodb as dynamodb -from aws_cdk import aws_elasticsearch as elasticsearch from aws_cdk import aws_events, aws_events_targets from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda +from aws_cdk import aws_opensearchservice as elasticsearch from aws_cdk import aws_s3 as s3 from aws_cdk import aws_s3_assets as s3_assets from aws_cdk import aws_s3_deployment as s3_deployment @@ -682,7 +682,7 @@ def create_es_domain(self) -> None: self, id="cbers2stac", # This is the version currently used by localstack - version=elasticsearch.ElasticsearchVersion.V7_7, + version=elasticsearch.EngineVersion.ELASTICSEARCH_7_7, ebs=elasticsearch.EbsOptions( enabled=True, volume_size=settings.es_volume_size ), From 6cf28e8d517a010cff6df6ea3e6cb87d87d9dd7e Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Mon, 19 Feb 2024 18:06:21 -0300 Subject: [PATCH 5/8] Remove link from constraints (#100) --- constraints.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/constraints.txt b/constraints.txt index 8b60c49..57ed080 100644 --- a/constraints.txt +++ b/constraints.txt @@ -66,7 +66,6 @@ awscli-local==0.20 boto3==1.24.48 botocore==1.27.48 cattrs==22.1.0 --e git+ssh://git@github.com/fredliporace/cbers-2-stac.git@85d5155e8899d4a335bbb3735b04079f3cd95b15#egg=cbers_2_stac certifi==2022.6.15 cfgv==3.3.1 charset-normalizer==2.1.0 From d1c2112df3425b0f0a62f8c9194491b56f8c4904 Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Wed, 21 Feb 2024 18:08:26 -0300 Subject: [PATCH 6/8] ES domain removal policy back to destroy (#100) --- stack/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stack/app.py b/stack/app.py index fe0b422..591c301 100644 --- a/stack/app.py +++ b/stack/app.py @@ -698,6 +698,7 @@ def create_es_domain(self) -> None: ) ], removal_policy=core.RemovalPolicy.DESTROY, + enable_version_upgrade=False, ) # Add environment for lambdas From 579c44d4b2a6aa864bd5c4598ed62e1c1d1c6ded Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Wed, 21 Feb 2024 18:58:29 -0300 Subject: [PATCH 7/8] Migration to CDK2 (#100) --- README.md | 6 ++-- cdk.json | 5 +++- setup.py | 23 +------------- stack/app.py | 84 +++++++++++++++++++++++++--------------------------- 4 files changed, 49 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index a05ed13..34f483a 100644 --- a/README.md +++ b/README.md @@ -34,15 +34,15 @@ Some lambdas require extra pip packages to be installed in the lambda directory ## CDK bootstrap -Deployment uses AWS CDK. +Deployment uses AWS CDK2. Requirements: * node: Use [nvm](https://heynode.com/tutorial/install-nodejs-locally-nvm/) to make sure a supported node is being used, tested with 18.0.0 * AWS credentials configured -To install and check AWS CDK: +To install and check AWS CDK (tested with CDK 2.129.0): ```bash -$ npm install -g aws-cdk@1.204.0 +$ npm install -g aws-cdk $ cdk --version $ cdk bootstrap # Deploys the CDK toolkit stack into an AWS environment diff --git a/cdk.json b/cdk.json index 68fd5c3..b31901b 100644 --- a/cdk.json +++ b/cdk.json @@ -1,3 +1,6 @@ { - "app": "(export PYTHONPATH=. && python stack/app.py)" + "app": "(export PYTHONPATH=. && python stack/app.py)", + "context":{ + "@aws-cdk/customresources:installLatestAwsSdkDefault":false + } } diff --git a/setup.py b/setup.py index 047ac57..72abbb7 100644 --- a/setup.py +++ b/setup.py @@ -34,28 +34,7 @@ # Used in process_new_scene_queue lambda. "utm", ], - "deploy": [ - "pydantic[dotenv]", - "aws-cdk.core", - "aws-cdk.aws-sqs", - "aws-cdk.aws-sns", - "aws-cdk.aws-sns-subscriptions", - "aws-cdk.aws-cloudwatch", - "aws-cdk.aws-cloudwatch-actions", - "aws-cdk.aws-lambda", - "aws-cdk.aws-s3", - "aws-cdk.aws-s3-deployment", - "aws-cdk.aws-s3-assets", - "aws-cdk.aws-iam", - "aws-cdk.aws-dynamodb", - "aws-cdk.aws-lambda-event-sources", - "aws-cdk.aws-events", - "aws-cdk.aws-events-targets", - "aws-cdk.aws-apigateway", - "aws-cdk.aws-elasticsearch", - "aws-cdk.aws-opensearchservice", - "aws-cdk.aws-synthetics", - ], + "deploy": ["pydantic[dotenv]", "aws-cdk-lib>=2.0.0", "constructs>=10.0.0",], } ENTRY_POINTS = """ diff --git a/stack/app.py b/stack/app.py index 591c301..21be234 100644 --- a/stack/app.py +++ b/stack/app.py @@ -5,7 +5,7 @@ ) from typing import Any, Dict, List -# from aws_cdk import aws_s3_notifications as s3n +from aws_cdk import App, CfnOutput, Duration, Fn, RemovalPolicy, Stack, Tags from aws_cdk import aws_apigateway as apigateway from aws_cdk import aws_cloudwatch as cloudwatch from aws_cdk import aws_cloudwatch_actions as cw_actions @@ -21,9 +21,9 @@ from aws_cdk import aws_sns_subscriptions as sns_subscriptions from aws_cdk import aws_sqs as sqs from aws_cdk import aws_synthetics as synthetics -from aws_cdk import core from aws_cdk.aws_cloudwatch import ComparisonOperator from aws_cdk.aws_lambda_event_sources import SqsEventSource +from constructs import Construct from cbers2stac.layers.common.dbtable import DBTable from cbers2stac.local.create_static_catalog_structure import ( @@ -34,7 +34,7 @@ settings = StackSettings() -class CBERS2STACStack(core.Stack): +class CBERS2STACStack(Stack): """CBERS2STACStack""" lambdas_env_: Dict[str, str] = {} @@ -115,7 +115,7 @@ def create_all_queues(self) -> None: # subscribe to CBERS 4/4A quicklook notification topics self.create_queue( id="process_new_scenes_queue_dlq", - retention_period=core.Duration.seconds(1209600), + retention_period=Duration.seconds(1209600), ) process_new_scenes_queue_alarm = cloudwatch.Alarm( self, @@ -132,8 +132,8 @@ def create_all_queues(self) -> None: ) self.create_queue( id="new_scenes_queue", - visibility_timeout=core.Duration.seconds(385), - retention_period=core.Duration.seconds(1209600), + visibility_timeout=Duration.seconds(385), + retention_period=Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=1, queue=self.queues_["process_new_scenes_queue_dlq"] ), @@ -180,8 +180,8 @@ def create_all_queues(self) -> None: self.create_queue( id="catalog_prefix_update_queue", - visibility_timeout=core.Duration.seconds(60), - retention_period=core.Duration.seconds(1209600), + visibility_timeout=Duration.seconds(60), + retention_period=Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["dead_letter_queue"] ), @@ -190,7 +190,7 @@ def create_all_queues(self) -> None: # Reconcile queue for INPE's XML metadata self.create_queue( id="consume_reconcile_queue_dlq", - retention_period=core.Duration.seconds(1209600), + retention_period=Duration.seconds(1209600), ) consume_reconcile_queue_alarm = cloudwatch.Alarm( self, @@ -207,8 +207,8 @@ def create_all_queues(self) -> None: ) self.create_queue( id="reconcile_queue", - visibility_timeout=core.Duration.seconds(1000), - retention_period=core.Duration.seconds(1209600), + visibility_timeout=Duration.seconds(1000), + retention_period=Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["consume_reconcile_queue_dlq"] ), @@ -217,7 +217,7 @@ def create_all_queues(self) -> None: # Reconcile queue for STAC items self.create_queue( id="consume_stac_reconcile_queue_dlq", - retention_period=core.Duration.seconds(1209600), + retention_period=Duration.seconds(1209600), ) consume_stac_reconcile_queue_alarm = cloudwatch.Alarm( self, @@ -234,8 +234,8 @@ def create_all_queues(self) -> None: ) self.create_queue( id="stac_reconcile_queue", - visibility_timeout=core.Duration.seconds(1000), - retention_period=core.Duration.seconds(1209600), + visibility_timeout=Duration.seconds(1000), + retention_period=Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["consume_stac_reconcile_queue_dlq"], @@ -246,8 +246,8 @@ def create_all_queues(self) -> None: # topic with new stac items self.create_queue( id="insert_into_elasticsearch_queue", - visibility_timeout=core.Duration.seconds(180), - retention_period=core.Duration.seconds(1209600), + visibility_timeout=Duration.seconds(180), + retention_period=Duration.seconds(1209600), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["dead_letter_queue"] ), @@ -272,8 +272,8 @@ def create_all_queues(self) -> None: # This queue subscribe only to new item topics self.create_queue( id="backup_insert_into_elasticsearch_queue", - visibility_timeout=core.Duration.seconds(180), - retention_period=core.Duration.days(settings.backup_queue_retention_days), + visibility_timeout=Duration.seconds(180), + retention_period=Duration.days(settings.backup_queue_retention_days), dead_letter_queue=sqs.DeadLetterQueue( max_receive_count=3, queue=self.queues_["dead_letter_queue"] ), @@ -287,7 +287,7 @@ def create_all_queues(self) -> None: # Queue for corrupted XML entries, see #89 self.create_queue( - id="corrupted_xml_queue", retention_period=core.Duration.days(14), + id="corrupted_xml_queue", retention_period=Duration.days(14), ) def create_all_topics(self) -> None: @@ -303,7 +303,7 @@ def create_all_topics(self) -> None: ) # Public STAC item topic for new STAC items self.topics_["stac_item_topic"] = sns.Topic(self, "stac_item_topic") - core.CfnOutput( + CfnOutput( self, "stac_item_topic_output", value=self.topics_["stac_item_topic"].topic_arn, @@ -391,7 +391,7 @@ def create_all_lambdas(self) -> None: "MESSAGE_BATCH_SIZE": "1", }, }, - timeout=core.Duration.seconds(55), + timeout=Duration.seconds(55), dead_letter_queue=self.queues_["process_new_scenes_queue_dlq"], layers=[self.layers_["common_layer"]], description="Process new scenes from quicklook queue", @@ -415,7 +415,7 @@ def create_all_lambdas(self) -> None: ].queue_url }, }, - timeout=core.Duration.seconds(900), + timeout=Duration.seconds(900), dead_letter_queue=self.queues_["dead_letter_queue"], layers=[self.layers_["common_layer"]], description="Generate levels into output table from input table", @@ -429,7 +429,7 @@ def create_all_lambdas(self) -> None: handler="code.trigger_handler", runtime=aws_lambda.Runtime.PYTHON_3_7, environment={**self.lambdas_env_,}, - timeout=core.Duration.seconds(55), + timeout=Duration.seconds(55), dead_letter_queue=self.queues_["dead_letter_queue"], layers=[self.layers_["common_layer"]], description="Update catalog from prefix", @@ -451,7 +451,7 @@ def create_all_lambdas(self) -> None: **self.lambdas_env_, **{"RECONCILE_QUEUE": self.queues_["reconcile_queue"].queue_url}, }, - timeout=core.Duration.seconds(300), + timeout=Duration.seconds(300), dead_letter_queue=self.queues_["dead_letter_queue"], layers=[self.layers_["common_layer"]], description="Populates reconcile queue with S3 keys from a common prefix", @@ -468,7 +468,7 @@ def create_all_lambdas(self) -> None: **self.lambdas_env_, **{"NEW_SCENES_QUEUE": self.queues_["new_scenes_queue"].queue_url}, }, - timeout=core.Duration.seconds(900), + timeout=Duration.seconds(900), dead_letter_queue=self.queues_["consume_reconcile_queue_dlq"], layers=[self.layers_["common_layer"]], description="Consume dirs from reconcile queue, populating " @@ -513,7 +513,7 @@ def create_all_lambdas(self) -> None: runtime=aws_lambda.Runtime.PYTHON_3_7, environment={**self.lambdas_env_,}, layers=[self.layers_["common_layer"]], - timeout=core.Duration.seconds(30), + timeout=Duration.seconds(30), dead_letter_queue=self.queues_["dead_letter_queue"], description="Create Elasticsearch stac index", ) @@ -530,7 +530,7 @@ def create_all_lambdas(self) -> None: **{"ES_STRIPPED": "YES", "BULK_CALLS": "1", "BULK_SIZE": "10"}, }, layers=[self.layers_["common_layer"]], - timeout=core.Duration.seconds(30), + timeout=Duration.seconds(30), dead_letter_queue=self.queues_["dead_letter_queue"], # Concurrent executions tuned to work with t2.small.elasticsearch reserved_concurrent_executions=5, @@ -551,7 +551,7 @@ def create_all_lambdas(self) -> None: runtime=aws_lambda.Runtime.PYTHON_3_7, environment=self.lambdas_env_, layers=[self.layers_["common_layer"]], - timeout=core.Duration.seconds(900), + timeout=Duration.seconds(900), description="Reindex STAC items from a prefix", ) # Batch size changed from 5 to 2 to reduce the lambda work and increase @@ -568,7 +568,7 @@ def create_all_lambdas(self) -> None: handler="code.populate_stac_reconcile_queue_handler", runtime=aws_lambda.Runtime.PYTHON_3_7, environment={**self.lambdas_env_,}, - timeout=core.Duration.seconds(300), + timeout=Duration.seconds(300), dead_letter_queue=self.queues_["dead_letter_queue"], layers=[self.layers_["common_layer"]], description="Populates reconcile queue with STAC items from a common prefix", @@ -591,7 +591,7 @@ def create_api_lambdas(self) -> None: runtime=aws_lambda.Runtime.PYTHON_3_7, environment={**self.lambdas_env_,}, layers=[self.layers_["common_layer"]], - timeout=core.Duration.seconds(30), + timeout=Duration.seconds(30), dead_letter_queue=self.queues_["api_dead_letter_queue"], description="Implement / endpoint (landing page)", ) @@ -604,7 +604,7 @@ def create_api_lambdas(self) -> None: runtime=aws_lambda.Runtime.PYTHON_3_7, environment={**self.lambdas_env_,}, layers=[self.layers_["common_layer"]], - timeout=core.Duration.seconds(55), + timeout=Duration.seconds(55), dead_letter_queue=self.queues_["api_dead_letter_queue"], description="Implement /search endpoint", ) @@ -623,9 +623,7 @@ def create_api_gateway(self) -> None: "openapi_asset", path="cbers2stac/openapi/core-item-search-query-integrated.yaml", ) - data = core.Fn.transform( - "AWS::Include", {"Location": openapi_asset.s3_object_url} - ) + data = Fn.transform("AWS::Include", {"Location": openapi_asset.s3_object_url}) definition = apigateway.AssetApiDefinition.from_inline(data) apigw = apigateway.SpecRestApi( self, @@ -640,12 +638,12 @@ def create_api_gateway(self) -> None: self, "canary_artifacts", auto_delete_objects=True, - removal_policy=core.RemovalPolicy.DESTROY, + removal_policy=RemovalPolicy.DESTROY, ) canary = synthetics.Canary( self, "SearchEndpointCanary", - schedule=synthetics.Schedule.rate(core.Duration.hours(1)), + schedule=synthetics.Schedule.rate(Duration.hours(1)), runtime=synthetics.Runtime.SYNTHETICS_PYTHON_SELENIUM_1_0, test=synthetics.Test.custom( code=synthetics.Code.from_asset("cbers2stac/canary", exclude=["*~"]), @@ -659,7 +657,7 @@ def create_api_gateway(self) -> None: canary_alarm = cloudwatch.Alarm( self, "CanaryAlarm", - metric=canary.metric_failed(period=core.Duration.hours(1)), + metric=canary.metric_failed(period=Duration.hours(1)), evaluation_periods=1, threshold=0, comparison_operator=ComparisonOperator.GREATER_THAN_THRESHOLD, @@ -697,7 +695,7 @@ def create_es_domain(self) -> None: # No need to specify resource, the domain is implicit ) ], - removal_policy=core.RemovalPolicy.DESTROY, + removal_policy=RemovalPolicy.DESTROY, enable_version_upgrade=False, ) @@ -709,7 +707,7 @@ def create_es_domain(self) -> None: def __init__( self, - scope: core.Construct, + scope: Construct, stack_id: str, description: str, env: Dict[str, str], @@ -773,7 +771,7 @@ def __init__( block_public_acls=False, restrict_public_buckets=False, ), - removal_policy=core.RemovalPolicy.DESTROY, + removal_policy=RemovalPolicy.DESTROY, ) self.lambdas_env_.update({"STAC_BUCKET": stac_working_bucket.bucket_name}) self.lambdas_perms_.append( @@ -821,7 +819,7 @@ def __init__( name=DBTable.pk_attr_name_, type=dynamodb.AttributeType.STRING ), billing_mode=dynamodb.BillingMode.PAY_PER_REQUEST, - removal_policy=core.RemovalPolicy.DESTROY, + removal_policy=RemovalPolicy.DESTROY, ) self.lambdas_env_.update( {"CATALOG_UPDATE_TABLE": catalog_update_table.table_name} @@ -899,7 +897,7 @@ def __init__( self.create_es_domain() -app = core.App() +app = App() # Tag infrastructure for key, value in { @@ -907,7 +905,7 @@ def __init__( "cost_center": settings.cost_center, }.items(): if value: - core.Tags.of(app).add(key, value) + Tags.of(app).add(key, value) stackname = f"{settings.name}-{settings.stage}" CBERS2STACStack( From 621ab6fbef4a8fd672b95d81c9d4b84e448e8cfd Mon Sep 17 00:00:00 2001 From: Frederico Liporace Date: Wed, 21 Feb 2024 19:01:10 -0300 Subject: [PATCH 8/8] Updating constrains for CDK2 (#100) --- constraints.txt | 66 ++++--------------------------------------------- 1 file changed, 5 insertions(+), 61 deletions(-) diff --git a/constraints.txt b/constraints.txt index 57ed080..c21602b 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,65 +1,9 @@ astroid==2.11.7 attrs==21.4.0 -aws-cdk.assets==1.204.0 -aws-cdk.aws-acmpca==1.204.0 -aws-cdk.aws-apigateway==1.204.0 -aws-cdk.aws-applicationautoscaling==1.204.0 -aws-cdk.aws-autoscaling==1.204.0 -aws-cdk.aws-autoscaling-common==1.204.0 -aws-cdk.aws-autoscaling-hooktargets==1.204.0 -aws-cdk.aws-certificatemanager==1.204.0 -aws-cdk.aws-cloudformation==1.204.0 -aws-cdk.aws-cloudfront==1.204.0 -aws-cdk.aws-cloudwatch==1.204.0 -aws-cdk.aws-cloudwatch-actions==1.204.0 -aws-cdk.aws-codebuild==1.204.0 -aws-cdk.aws-codecommit==1.204.0 -aws-cdk.aws-codeguruprofiler==1.204.0 -aws-cdk.aws-codepipeline==1.204.0 -aws-cdk.aws-codestarnotifications==1.204.0 -aws-cdk.aws-cognito==1.204.0 -aws-cdk.aws-dynamodb==1.204.0 -aws-cdk.aws-ec2==1.204.0 -aws-cdk.aws-ecr==1.204.0 -aws-cdk.aws-ecr-assets==1.204.0 -aws-cdk.aws-ecs==1.204.0 -aws-cdk.aws-efs==1.204.0 -aws-cdk.aws-elasticloadbalancing==1.204.0 -aws-cdk.aws-elasticloadbalancingv2==1.204.0 -aws-cdk.aws-elasticsearch==1.204.0 -aws-cdk.aws-events==1.204.0 -aws-cdk.aws-events-targets==1.204.0 -aws-cdk.aws-globalaccelerator==1.204.0 -aws-cdk.aws-iam==1.204.0 -aws-cdk.aws-kinesis==1.204.0 -aws-cdk.aws-kinesisfirehose==1.204.0 -aws-cdk.aws-kms==1.204.0 -aws-cdk.aws-lambda==1.204.0 -aws-cdk.aws-lambda-event-sources==1.204.0 -aws-cdk.aws-logs==1.204.0 -aws-cdk.aws-opensearchservice==1.204.0 -aws-cdk.aws-route53==1.204.0 -aws-cdk.aws-route53-targets==1.204.0 -aws-cdk.aws-s3==1.204.0 -aws-cdk.aws-s3-assets==1.204.0 -aws-cdk.aws-s3-deployment==1.204.0 -aws-cdk.aws-s3-notifications==1.204.0 -aws-cdk.aws-sam==1.204.0 -aws-cdk.aws-secretsmanager==1.204.0 -aws-cdk.aws-servicediscovery==1.204.0 -aws-cdk.aws-signer==1.204.0 -aws-cdk.aws-sns==1.204.0 -aws-cdk.aws-sns-subscriptions==1.204.0 -aws-cdk.aws-sqs==1.204.0 -aws-cdk.aws-ssm==1.204.0 -aws-cdk.aws-stepfunctions==1.204.0 -aws-cdk.aws-synthetics==1.204.0 -aws-cdk.cloud-assembly-schema==1.204.0 -aws-cdk.core==1.204.0 -aws-cdk.custom-resources==1.204.0 -aws-cdk.cx-api==1.204.0 -aws-cdk.lambda-layer-awscli==1.204.0 -aws-cdk.region-info==1.204.0 +aws-cdk-lib==2.111.0 +aws-cdk.asset-awscli-v1==2.2.201 +aws-cdk.asset-kubectl-v20==2.1.2 +aws-cdk.asset-node-proxy-agent-v6==2.0.1 aws-requests-auth==0.4.3 awscli==1.25.48 awscli-local==0.20 @@ -70,7 +14,7 @@ certifi==2022.6.15 cfgv==3.3.1 charset-normalizer==2.1.0 colorama==0.4.4 -constructs==3.4.64 +constructs==10.3.0 coverage==6.4.3 decorator==5.1.1 dill==0.3.5.1