diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88a29906a98..d465071f771 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,3 +47,9 @@ repos: hooks: - id: actionlint-docker args: [-pyflakes=] + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: 3420134c37197c21edffc7e6093b14ffae8402f2 # v1.81.0 + hooks: + - id: terraform_fmt + args: + - --args=-recursive diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 471224964d5..bfd65d0a1f6 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -5,8 +5,7 @@ description: Utility -The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which -are safe to retry. +The idempotency utility provides a simple solution to convert your Lambda functions into idempotent operations which are safe to retry. ## Key features @@ -18,11 +17,9 @@ are safe to retry. ## Terminology -The property of idempotency means that an operation does not cause additional side effects if it is called more than -once with the same input parameters. +The property of idempotency means that an operation does not cause additional side effects if it is called more than once with the same input parameters. -**Idempotent operations will return the same result when they are called multiple -times with the same parameters**. This makes idempotent operations safe to retry. +**Idempotent operations will return the same result when they are called multiple times with the same parameters**. This makes idempotent operations safe to retry. **Idempotency key** is a hash representation of either the entire event or a specific configured subset of the event, and invocation results are **JSON serialized** and stored in your persistence storage layer. @@ -59,7 +56,7 @@ classDiagram Your Lambda function IAM Role must have `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem` and `dynamodb:DeleteItem` IAM permissions before using this feature. ???+ note - If you're using our example [AWS Serverless Application Model (SAM)](#required-resources), it already adds the required permissions. + If you're using our example [AWS Serverless Application Model (SAM)](#required-resources), [AWS Cloud Development Kit (CDK)](#required-resources), or [Terraform](#required-resources) it already adds the required permissions. ### Required resources @@ -79,15 +76,21 @@ If you're not [changing the default configuration for the DynamoDB persistence l ???+ tip "Tip: You can share a single state table for all functions" You can reuse the same DynamoDB table to store idempotency state. We add `module_name` and [qualified name for classes and functions](https://peps.python.org/pep-3155/){target="_blank"} in addition to the idempotency key as a hash key. -=== "sam.yaml" +=== "AWS Serverless Application Model (SAM) example" - ```yaml hl_lines="6-14 24-31" title="AWS Serverless Application Model (SAM) example" - --8<-- "examples/idempotency/sam.yaml" + ```yaml hl_lines="6-14 24-31" + --8<-- "examples/idempotency/templates/sam.yaml" ``` -=== "cdk.py" +=== "AWS Cloud Development Kit (CDK)" - ```python hl_lines="10 13 16 19-21" title="AWS Cloud Development Kit (CDK) Construct example" - --8<-- "examples/idempotency/cdk.py" + ```python hl_lines="10 13 16 19-21" + --8<-- "examples/idempotency/templates/cdk.py" + ``` + +=== "Terraform" + + ```terraform hl_lines="14-26 64-70" + --8<-- "examples/idempotency/templates/terraform.tf" ``` ???+ warning "Warning: Large responses with DynamoDB persistence layer" @@ -110,36 +113,16 @@ You can quickly start by initializing the `DynamoDBPersistenceLayer` class and u !!! tip "See [Choosing a payload subset for idempotency](#choosing-a-payload-subset-for-idempotency) for more elaborate use cases." -=== "app.py" - - ```python hl_lines="1-3 5 7 14" - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - - @idempotent(persistence_store=persistence_layer) - def handler(event, context): - payment = create_subscription_payment( - user=event['user'], - product=event['product_id'] - ) - ... - return { - "payment_id": payment.id, - "message": "success", - "statusCode": 200, - } +=== "Idempotent decorator" + + ```python hl_lines="4-7 10 24" + --8<-- "examples/idempotency/src/getting_started_with_idempotency.py" ``` -=== "Example event" +=== "Sample event" - ```json - { - "username": "xyz", - "product_id": "123456789" - } + ```json + --8<-- "examples/idempotency/src/getting_started_with_idempotency_payload.json" ``` After processing this request successfully, a second request containing the exact same payload above will now return the same response, ensuring our customer isn't charged twice. @@ -157,79 +140,16 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo ???+ warning "Limitation" Make sure to call your decorated function using keyword arguments. -=== "dataclass_sample.py" - - ```python hl_lines="3-4 23 33" - from dataclasses import dataclass - - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) - - dynamodb = DynamoDBPersistenceLayer(table_name="idem") - config = IdempotencyConfig( - event_key_jmespath="order_id", # see Choosing a payload subset section - use_local_cache=True, - ) +=== "Using Dataclasses" - @dataclass - class OrderItem: - sku: str - description: str - - @dataclass - class Order: - item: OrderItem - order_id: int - - - @idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) - def process_order(order: Order): - return f"processed order {order.order_id}" - - def lambda_handler(event, context): - config.register_lambda_context(context) # see Lambda timeouts section - order_item = OrderItem(sku="fake", description="sample") - order = Order(item=order_item, order_id="fake-id") - - # `order` parameter must be called as a keyword argument to work - process_order(order=order) + ```python hl_lines="3-7 11 26 37" + --8<-- "examples/idempotency/src/working_with_idempotent_function_dataclass.py" ``` -=== "parser_pydantic_sample.py" - - ```python hl_lines="1-2 22 32" - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) - from aws_lambda_powertools.utilities.parser import BaseModel - - dynamodb = DynamoDBPersistenceLayer(table_name="idem") - config = IdempotencyConfig( - event_key_jmespath="order_id", # see Choosing a payload subset section - use_local_cache=True, - ) +=== "Using Pydantic" - - class OrderItem(BaseModel): - sku: str - description: str - - - class Order(BaseModel): - item: OrderItem - order_id: int - - - @idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) - def process_order(order: Order): - return f"processed order {order.order_id}" - - def lambda_handler(event, context): - config.register_lambda_context(context) # see Lambda timeouts section - order_item = OrderItem(sku="fake", description="sample") - order = Order(item=order_item, order_id="fake-id") - - # `order` parameter must be called as a keyword argument to work - process_order(order=order) + ```python hl_lines="1-5 10 23 34" + --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic.py" ``` #### Batch integration @@ -241,71 +161,16 @@ You can can easily integrate with [Batch utility](batch.md){target="_blank"} via Depending on your use case, it might be more accurate [to choose another field](#choosing-a-payload-subset-for-idempotency) your producer intentionally set to define uniqueness. -=== "batch_sample.py" - - ```python hl_lines="3-4 10 15 21 25-26 29 31" - from aws_lambda_powertools.utilities.batch import BatchProcessor, EventType - from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function) - - - processor = BatchProcessor(event_type=EventType.SQS) - dynamodb = DynamoDBPersistenceLayer(table_name="idem") - config = IdempotencyConfig( - event_key_jmespath="messageId", # see Choosing a payload subset section - use_local_cache=True, - ) - - - @idempotent_function(data_keyword_argument="record", config=config, persistence_store=dynamodb) - def record_handler(record: SQSRecord): - return {"message": record.body} - - - def lambda_handler(event, context): - config.register_lambda_context(context) # see Lambda timeouts section +=== "Integration with Batch Processor" - # with Lambda context registered for Idempotency - # we can now kick in the Bach processing logic - batch = event["Records"] - with processor(records=batch, handler=record_handler): - # in case you want to access each record processed by your record_handler - # otherwise ignore the result variable assignment - processed_messages = processor.process() - - return processor.response() + ```python hl_lines="2 12 16 20 31 35 37" + --8<-- "examples/idempotency/src/integrate_idempotency_with_batch_processor.py" ``` -=== "batch_event.json" +=== "Sample event" ```json hl_lines="4" - { - "Records": [ - { - "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", - "receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", - "body": "Test message.", - "attributes": { - "ApproximateReceiveCount": "1", - "SentTimestamp": "1545082649183", - "SenderId": "AIDAIENQZJOLO23YVJ4VO", - "ApproximateFirstReceiveTimestamp": "1545082649185" - }, - "messageAttributes": { - "testAttr": { - "stringValue": "100", - "binaryValue": "base64Str", - "dataType": "Number" - } - }, - "md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue", - "awsRegion": "us-east-2" - } - ] - } + --8<-- "examples/idempotency/src/integrate_idempotency_with_batch_processor_payload.json" ``` ### Choosing a payload subset for idempotency @@ -321,7 +186,7 @@ In this example, we have a Lambda handler that creates a payment for a user subs Imagine the function executes successfully, but the client never receives the response due to a connection issue. It is safe to retry in this instance, as the idempotent decorator will return a previously saved response. -**What we want here** is to instruct Idempotency to use `user` and `product_id` fields from our incoming payload as our idempotency key. +**What we want here** is to instruct Idempotency to use `user_id` and `product_id` fields from our incoming payload as our idempotency key. If we were to treat the entire request as our idempotency key, a simple HTTP header change would cause our customer to be charged twice. ???+ tip "Deserializing JSON strings in payloads for increased accuracy." @@ -330,68 +195,16 @@ If we were to treat the entire request as our idempotency key, a simple HTTP hea To alter this behaviour, we can use the [JMESPath built-in function](jmespath_functions.md#powertools_json-function){target="_blank"} `powertools_json()` to treat the payload as a JSON object (dict) rather than a string. -=== "payment.py" - - ```python hl_lines="2-4 10 12 15 20" - import json - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - - # Deserialize JSON string under the "body" key - # then extract "user" and "product_id" data - config = IdempotencyConfig(event_key_jmespath="powertools_json(body).[user, product_id]") - - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - body = json.loads(event['body']) - payment = create_subscription_payment( - user=body['user'], - product=body['product_id'] - ) - ... - return { - "payment_id": payment.id, - "message": "success", - "statusCode": 200 - } +=== "Payment function" + + ```python hl_lines="5-9 16 30" + --8<-- "examples/idempotency/src/working_with_payload_subset.py" ``` -=== "Example event" +=== "Sample event" ```json hl_lines="28" - { - "version":"2.0", - "routeKey":"ANY /createpayment", - "rawPath":"/createpayment", - "rawQueryString":"", - "headers": { - "Header1": "value1", - "Header2": "value2" - }, - "requestContext":{ - "accountId":"123456789012", - "apiId":"api-id", - "domainName":"id.execute-api.us-east-1.amazonaws.com", - "domainPrefix":"id", - "http":{ - "method":"POST", - "path":"/createpayment", - "protocol":"HTTP/1.1", - "sourceIp":"ip", - "userAgent":"agent" - }, - "requestId":"id", - "routeKey":"ANY /createpayment", - "stage":"$default", - "time":"10/Feb/2021:13:40:43 +0000", - "timeEpoch":1612964443723 - }, - "body":"{\"user\":\"xyz\",\"product_id\":\"123456789\"}", - "isBase64Encoded":false - } + --8<-- "examples/idempotency/src/working_with_payload_subset_payload.json" ``` ### Lambda timeouts @@ -413,26 +226,11 @@ Powertools for AWS Lambda (Python) calculates and includes the remaining invocat Here is an example on how you register the Lambda context in your handler: -```python hl_lines="8 16" title="Registering the Lambda context" -from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, idempotent_function -) - -persistence_layer = DynamoDBPersistenceLayer(table_name="...") - -config = IdempotencyConfig() - -@idempotent_function(data_keyword_argument="record", persistence_store=persistence_layer, config=config) -def record_handler(record: SQSRecord): - return {"message": record["body"]} +=== "Registering the Lambda context" - -def lambda_handler(event, context): - config.register_lambda_context(context) - - return record_handler(event) -``` + ```python hl_lines="11 20" + --8<-- "examples/idempotency/src/working_with_lambda_timeout.py" + ``` ### Handling exceptions @@ -461,24 +259,11 @@ If you are using `idempotent_function`, any unhandled exceptions that are raised If an Exception is raised _outside_ the scope of the decorated function and after your function has been called, the persistent record will not be affected. In this case, idempotency will be maintained for your decorated function. Example: -```python hl_lines="2-4 8-10" title="Exception not affecting idempotency record sample" -def lambda_handler(event, context): - # If an exception is raised here, no idempotent record will ever get created as the - # idempotent function does not get called - do_some_stuff() - - result = call_external_service(data={"user": "user1", "id": 5}) - - # This exception will not cause the idempotent record to be deleted, since it - # happens after the decorated function has been successfully called - raise Exception - +=== "Handling exceptions" -@idempotent_function(data_keyword_argument="data", config=config, persistence_store=dynamodb) -def call_external_service(data: dict, **kwargs): - result = requests.post('http://example.com', json={"user": data['user'], "transaction_id": data['id']} - return result.json() -``` + ```python hl_lines="18-22 28 31" + --8<-- "examples/idempotency/src/working_with_exceptions.py" + ``` ???+ warning **We will raise `IdempotencyPersistenceLayerError`** if any of the calls to the persistence layer fail unexpectedly. @@ -687,19 +472,11 @@ sequenceDiagram This persistence layer is built-in, and you can either use an existing DynamoDB table or create a new one dedicated for idempotency state (recommended). -```python hl_lines="5-10" title="Customizing DynamoDBPersistenceLayer to suit your table structure" -from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer - -persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - key_attr="idempotency_key", - expiry_attr="expires_at", - in_progress_expiry_attr="in_progress_expires_at", - status_attr="current_status", - data_attr="result_data", - validation_key_attr="validation_key", -) -``` +=== "Customizing DynamoDBPersistenceLayer to suit your table structure" + + ```python hl_lines="7-15" + --8<-- "examples/idempotency/src/customize_persistence_layer.py" + ``` When using DynamoDB as a persistence layer, you can alter the attribute names by passing these parameters when initializing the persistence layer: @@ -747,21 +524,17 @@ This is a locking mechanism for correctness. Since we don't know the result from You can enable in-memory caching with the **`use_local_cache`** parameter: -```python hl_lines="8 11" title="Caching idempotent transactions in-memory to prevent multiple calls to storage" -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent -) +=== "Caching idempotent transactions in-memory to prevent multiple calls to storage" -persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") -config = IdempotencyConfig( - event_key_jmespath="body", - use_local_cache=True, -) + ```python hl_lines="11" + --8<-- "examples/idempotency/src/working_with_local_cache.py" + ``` -@idempotent(config=config, persistence_store=persistence_layer) -def handler(event, context): - ... -``` +=== "Sample event" + + ```json + --8<-- "examples/idempotency/src/working_with_local_cache_payload.json" + ``` When enabled, the default is to cache a maximum of 256 records in each Lambda execution environment - You can change it with the **`local_cache_max_items`** parameter. @@ -773,21 +546,17 @@ In most cases, it is not desirable to store the idempotency records forever. Rat You can change this window with the **`expires_after_seconds`** parameter: -```python hl_lines="8 11" title="Adjusting idempotency record expiration" -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent -) +=== "Adjusting idempotency record expiration" -persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") -config = IdempotencyConfig( - event_key_jmespath="body", - expires_after_seconds=5*60, # 5 minutes -) + ```python hl_lines="11" + --8<-- "examples/idempotency/src/working_with_record_expiration.py" + ``` -@idempotent(config=config, persistence_store=persistence_layer) -def handler(event, context): - ... -``` +=== "Sample event" + + ```json + --8<-- "examples/idempotency/src/working_with_record_expiration_payload.json" + ``` This will mark any records older than 5 minutes as expired, and [your function will be executed as normal if it is invoked with a matching payload](#expired-idempotency-records). @@ -811,66 +580,25 @@ By default, we will return the same result as it returned before, however in thi With **`payload_validation_jmespath`**, you can provide an additional JMESPath expression to specify which part of the event body should be validated against previous idempotent invocations -=== "app.py" - - ```python hl_lines="7 11 18 25" - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - config = IdempotencyConfig( - event_key_jmespath="[userDetail, productId]", - payload_validation_jmespath="amount" - ) - persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - # Creating a subscription payment is a side - # effect of calling this function! - payment = create_subscription_payment( - user=event['userDetail']['username'], - product=event['product_id'], - amount=event['amount'] - ) - ... - return { - "message": "success", - "statusCode": 200, - "payment_id": payment.id, - "amount": payment.amount - } +=== "Payload validation" + + ```python hl_lines="12 20 28" + --8<-- "examples/idempotency/src/working_with_validation_payload.py" ``` -=== "Example Event 1" - - ```json hl_lines="8" - { - "userDetail": { - "username": "User1", - "user_email": "user@example.com" - }, - "productId": 1500, - "charge_type": "subscription", - "amount": 500 - } +=== "Sample event 1" + + ```json hl_lines="2 5" + --8<-- "examples/idempotency/src/working_with_validation_payload_payload1.json" ``` -=== "Example Event 2" - - ```json hl_lines="8" - { - "userDetail": { - "username": "User1", - "user_email": "user@example.com" - }, - "productId": 1500, - "charge_type": "subscription", - "amount": 1 - } +=== "Sample event 2" + + ```json hl_lines="2 5" + --8<-- "examples/idempotency/src/working_with_validation_payload_payload2.json" ``` -In this example, the **`userDetail`** and **`productId`** keys are used as the payload to generate the idempotency key, as per **`event_key_jmespath`** parameter. +In this example, the **`user_id`** and **`product_id`** keys are used as the payload to generate the idempotency key, as per **`event_key_jmespath`** parameter. ???+ note If we try to send the same request but with a different amount, we will raise **`IdempotencyValidationError`**. @@ -888,50 +616,22 @@ This means that we will raise **`IdempotencyKeyError`** if the evaluation of **` ???+ warning To prevent errors, transactions will not be treated as idempotent if **`raise_on_no_idempotency_key`** is set to `False` and the evaluation of **`event_key_jmespath`** is `None`. Therefore, no data will be fetched, stored, or deleted in the idempotency storage layer. -=== "app.py" +=== "Idempotency key required" - ```python hl_lines="9-10 13" - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") - - # Requires "user"."uid" and "order_id" to be present - config = IdempotencyConfig( - event_key_jmespath="[user.uid, order_id]", - raise_on_no_idempotency_key=True, - ) - - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - pass + ```python hl_lines="11" + --8<-- "examples/idempotency/src/working_with_idempotency_key_required.py" ``` === "Success Event" ```json hl_lines="3 6" - { - "user": { - "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", - "name": "Foo" - }, - "order_id": 10000 - } + --8<-- "examples/idempotency/src/working_with_idempotency_key_required_payload_success.json" ``` === "Failure Event" - Notice that `order_id` is now accidentally within `user` key - ```json hl_lines="3 5" - { - "user": { - "uid": "DE0D000E-1234-10D1-991E-EAC1DD1D52C8", - "name": "Joe Bloggs", - "order_id": 10000 - }, - } + --8<-- "examples/idempotency/src/working_with_idempotency_key_required_payload_error.json" ``` ### Customizing boto configuration @@ -940,42 +640,20 @@ The **`boto_config`** and **`boto3_session`** parameters enable you to pass in a === "Custom session" - ```python hl_lines="1 6 9 14" - import boto3 - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - boto3_session = boto3.session.Session() - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - boto3_session=boto3_session - ) + ```python hl_lines="1 11 13" + --8<-- "examples/idempotency/src/working_with_custom_session.py" + ``` - config = IdempotencyConfig(event_key_jmespath="body") +=== "Custom config" - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - ... + ```python hl_lines="1 11 13" + --8<-- "examples/idempotency/src/working_with_custom_config.py" ``` -=== "Custom config" - ```python hl_lines="1 7 10" - from botocore.config import Config - from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent - ) - - config = IdempotencyConfig(event_key_jmespath="body") - boto_config = Config() - persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - boto_config=boto_config - ) - - @idempotent(config=config, persistence_store=persistence_layer) - def handler(event, context): - ... +=== "Sample Event" + + ```json + --8<-- "examples/idempotency/src/working_with_custom_config_payload.json" ``` ### Using a DynamoDB table with a composite primary key @@ -986,162 +664,42 @@ With this setting, we will save the idempotency key in the sort key instead of t You can optionally set a static value for the partition key using the `static_pk_value` parameter. -```python hl_lines="5" title="Reusing a DynamoDB table that uses a composite primary key" -from aws_lambda_powertools.utilities.idempotency import DynamoDBPersistenceLayer, idempotent +=== "Reusing a DynamoDB table that uses a composite primary key" -persistence_layer = DynamoDBPersistenceLayer( - table_name="IdempotencyTable", - sort_key_attr='sort_key') + ```python hl_lines="7" + --8<-- "examples/idempotency/src/working_with_composite_key.py" + ``` +=== "Sample Event" -@idempotent(persistence_store=persistence_layer) -def handler(event, context): - return {"message": "success": "id": event['body']['id]} -``` + ```json + --8<-- "examples/idempotency/src/working_with_composite_key_payload.json" + ``` The example function above would cause data to be stored in DynamoDB like this: -| id | sort_key | expiration | status | data | -| ---------------------------- | -------------------------------- | ---------- | ----------- | ------------------------------------ | -| idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"id": 12391, "message": "success"} | -| idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"id": 527212, "message": "success"} | -| idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | +| id | sort_key | expiration | status | data | +| ---------------------------- | -------------------------------- | ---------- | ----------- | ----------------------------------------- | +| idempotency#MyLambdaFunction | 1e956ef7da78d0cb890be999aecc0c9e | 1636549553 | COMPLETED | {"user_id": 12391, "message": "success"} | +| idempotency#MyLambdaFunction | 2b2cdb5f86361e97b4383087c1ffdf27 | 1636549571 | COMPLETED | {"user_id": 527212, "message": "success"} | +| idempotency#MyLambdaFunction | f091d2527ad1c78f05d54cc3f363be80 | 1636549585 | IN_PROGRESS | | ### Bring your own persistent store This utility provides an abstract base class (ABC), so that you can implement your choice of persistent storage layer. -You can inherit from the `BasePersistenceLayer` class and implement the abstract methods `_get_record`, `_put_record`, -`_update_record` and `_delete_record`. - -```python hl_lines="8-13 57 65 74 96 124" title="Excerpt DynamoDB Persistence Layer implementation for reference" -import datetime -import logging -from typing import Any, Dict, Optional - -import boto3 -from botocore.config import Config - -from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer -from aws_lambda_powertools.utilities.idempotency.exceptions import ( - IdempotencyItemAlreadyExistsError, - IdempotencyItemNotFoundError, -) -from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord - -logger = logging.getLogger(__name__) - - -class DynamoDBPersistenceLayer(BasePersistenceLayer): - def __init__( - self, - table_name: str, - key_attr: str = "id", - expiry_attr: str = "expiration", - status_attr: str = "status", - data_attr: str = "data", - validation_key_attr: str = "validation", - boto_config: Optional[Config] = None, - boto3_session: Optional[boto3.session.Session] = None, - ): - boto_config = boto_config or Config() - session = boto3_session or boto3.session.Session() - self._ddb_resource = session.resource("dynamodb", config=boto_config) - self.table_name = table_name - self.table = self._ddb_resource.Table(self.table_name) - self.key_attr = key_attr - self.expiry_attr = expiry_attr - self.status_attr = status_attr - self.data_attr = data_attr - self.validation_key_attr = validation_key_attr - super(DynamoDBPersistenceLayer, self).__init__() - - def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: - """ - Translate raw item records from DynamoDB to DataRecord - - Parameters - ---------- - item: Dict[str, Union[str, int]] - Item format from dynamodb response - - Returns - ------- - DataRecord - representation of item - - """ - return DataRecord( - idempotency_key=item[self.key_attr], - status=item[self.status_attr], - expiry_timestamp=item[self.expiry_attr], - response_data=item.get(self.data_attr), - payload_hash=item.get(self.validation_key_attr), - ) - - def _get_record(self, idempotency_key) -> DataRecord: - response = self.table.get_item(Key={self.key_attr: idempotency_key}, ConsistentRead=True) - - try: - item = response["Item"] - except KeyError: - raise IdempotencyItemNotFoundError - return self._item_to_data_record(item) - - def _put_record(self, data_record: DataRecord) -> None: - item = { - self.key_attr: data_record.idempotency_key, - self.expiry_attr: data_record.expiry_timestamp, - self.status_attr: data_record.status, - } - - if self.payload_validation_enabled: - item[self.validation_key_attr] = data_record.payload_hash - - now = datetime.datetime.now() - try: - logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}") - self.table.put_item( - Item=item, - ConditionExpression=f"attribute_not_exists({self.key_attr}) OR {self.expiry_attr} < :now", - ExpressionAttributeValues={":now": int(now.timestamp())}, - ) - except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException: - logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") - raise IdempotencyItemAlreadyExistsError - - def _update_record(self, data_record: DataRecord): - logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}") - update_expression = "SET #response_data = :response_data, #expiry = :expiry, #status = :status" - expression_attr_values = { - ":expiry": data_record.expiry_timestamp, - ":response_data": data_record.response_data, - ":status": data_record.status, - } - expression_attr_names = { - "#response_data": self.data_attr, - "#expiry": self.expiry_attr, - "#status": self.status_attr, - } - - if self.payload_validation_enabled: - update_expression += ", #validation_key = :validation_key" - expression_attr_values[":validation_key"] = data_record.payload_hash - expression_attr_names["#validation_key"] = self.validation_key_attr - - kwargs = { - "Key": {self.key_attr: data_record.idempotency_key}, - "UpdateExpression": update_expression, - "ExpressionAttributeValues": expression_attr_values, - "ExpressionAttributeNames": expression_attr_names, - } - - self.table.update_item(**kwargs) - - def _delete_record(self, data_record: DataRecord) -> None: - logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}") - self.table.delete_item(Key={self.key_attr: data_record.idempotency_key},) -``` +You can create your own persistent store from scratch by inheriting the `BasePersistenceLayer` class, and implementing `_get_record()`, `_put_record()`, `_update_record()` and `_delete_record()`. + +* **`_get_record()`** – Retrieves an item from the persistence store using an idempotency key and returns it as a `DataRecord` instance. +* **`_put_record()`** – Adds a `DataRecord` to the persistence store if it doesn't already exist with that key. Raises an `ItemAlreadyExists` exception if a non-expired entry already exists. +* **`_update_record()`** – Updates an item in the persistence store. +* **`_delete_record()`** – Removes an item from the persistence store. + +=== "Bring your own persistent store" + + ```python hl_lines="8 18 65 74 96 124" + --8<-- "examples/idempotency/src/bring_your_own_persistent_store.py" + ``` ???+ danger Pay attention to the documentation for each - you may need to perform additional checks inside these methods to ensure the idempotency guarantees remain intact. @@ -1164,21 +722,17 @@ The idempotency utility can be used with the `validator` decorator. Ensure that Make sure to account for this behaviour, if you set the `event_key_jmespath`. -```python hl_lines="9 10" title="Using Idempotency with JSONSchema Validation utility" -from aws_lambda_powertools.utilities.validation import validator, envelopes -from aws_lambda_powertools.utilities.idempotency import ( - IdempotencyConfig, DynamoDBPersistenceLayer, idempotent -) +=== "Using Idempotency with JSONSchema Validation utility" -config = IdempotencyConfig(event_key_jmespath="[message, username]") -persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + ```python hl_lines="13" + --8<-- "examples/idempotency/src/integrate_idempotency_with_validator.py" + ``` -@validator(envelope=envelopes.API_GATEWAY_HTTP) -@idempotent(config=config, persistence_store=persistence_layer) -def lambda_handler(event, context): - cause_some_side_effects(event['username') - return {"message": event['message'], "statusCode": 200} -``` +=== "Sample Event" + + ```json hl_lines="60" + --8<-- "examples/idempotency/src/integrate_idempotency_with_validator_payload.json" + ``` ???+ tip "Tip: JMESPath Powertools for AWS Lambda (Python) functions are also available" Built-in functions known in the validation utility like `powertools_json`, `powertools_base64`, `powertools_base64_gzip` are also available to use in this utility. @@ -1192,115 +746,32 @@ The idempotency utility provides several routes to test your code. When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED` with a truthy value. If you prefer setting this for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html){target="_blank"} fixture: -=== "tests.py" - - ```python hl_lines="24-25" - from dataclasses import dataclass - - import pytest - - import app - - - @pytest.fixture - def lambda_context(): - @dataclass - class LambdaContext: - function_name: str = "test" - memory_limit_in_mb: int = 128 - invoked_function_arn: str = "arn:aws:lambda:eu-west-1:809313241:function:test" - aws_request_id: str = "52fdfc07-2182-154f-163f-5f0f9a621d72" - - def get_remaining_time_in_millis(self) -> int: - return 5 - - return LambdaContext() +=== "test_disabling_idempotency_utility.py" + ```python hl_lines="3 4 23 24" + --8<-- "examples/idempotency/tests/test_disabling_idempotency_utility.py" + ``` - def test_idempotent_lambda_handler(monkeypatch, lambda_context): - # Set POWERTOOLS_IDEMPOTENCY_DISABLED before calling decorated functions - monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", 1) +=== "app_test_disabling_idempotency_utility.py" - result = handler({}, lambda_context) - ... - ``` -=== "app.py" - - ```python - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") - - @idempotent(persistence_store=persistence_layer) - def handler(event, context): - print('expensive operation') - return { - "payment_id": 12345, - "message": "success", - "statusCode": 200, - } + ```python hl_lines="10" + --8<-- "examples/idempotency/tests/app_test_disabling_idempotency_utility.py" ``` ### Testing with DynamoDB Local To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html){target="_blank"}, you can replace the `DynamoDB client` used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url. -=== "tests.py" - - ```python hl_lines="24-27" - from dataclasses import dataclass - - import boto3 - import pytest - - import app - - - @pytest.fixture - def lambda_context(): - @dataclass - class LambdaContext: - function_name: str = "test" - memory_limit_in_mb: int = 128 - invoked_function_arn: str = "arn:aws:lambda:eu-west-1:809313241:function:test" - aws_request_id: str = "52fdfc07-2182-154f-163f-5f0f9a621d72" - - def get_remaining_time_in_millis(self) -> int: - return 5 - - return LambdaContext() +=== "test_with_dynamodb_local.py" - def test_idempotent_lambda(lambda_context): - # Configure the boto3 to use the endpoint for the DynamoDB Local instance - dynamodb_local_client = boto3.client("dynamodb", endpoint_url='http://localhost:8000') - app.persistence_layer.client = dynamodb_local_client - - # If desired, you can use a different DynamoDB Local table name than what your code already uses - # app.persistence_layer.table_name = "another table name" - - result = app.handler({'testkey': 'testvalue'}, lambda_context) - assert result['payment_id'] == 12345 + ```python hl_lines="3-5 25 26" + --8<-- "examples/idempotency/tests/test_with_dynamodb_local.py" ``` -=== "app.py" - - ```python - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") +=== "app_test_dynamodb_local.py" - @idempotent(persistence_store=persistence_layer) - def handler(event, context): - print('expensive operation') - return { - "payment_id": 12345, - "message": "success", - "statusCode": 200, - } + ```python hl_lines="10" + --8<-- "examples/idempotency/tests/app_test_dynamodb_local.py" ``` ### How do I mock all DynamoDB I/O operations @@ -1308,58 +779,16 @@ To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/ The idempotency utility lazily creates the dynamodb [Table](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table){target="_blank"} which it uses to access DynamoDB. This means it is possible to pass a mocked Table resource, or stub various methods. -=== "tests.py" - - ```python hl_lines="26-29" - from dataclasses import dataclass - from unittest.mock import MagicMock - - import boto3 - import pytest - - import app +=== "test_with_io_operations.py" - - @pytest.fixture - def lambda_context(): - @dataclass - class LambdaContext: - function_name: str = "test" - memory_limit_in_mb: int = 128 - invoked_function_arn: str = "arn:aws:lambda:eu-west-1:809313241:function:test" - aws_request_id: str = "52fdfc07-2182-154f-163f-5f0f9a621d72" - - def get_remaining_time_in_millis(self) -> int: - return 5 - - return LambdaContext() - - - def test_idempotent_lambda(lambda_context): - mock_client = MagicMock() - app.persistence_layer.client = mock_client - result = app.handler({'testkey': 'testvalue'}, lambda_context) - mock_client.put_item.assert_called() - ... + ```python hl_lines="4 5 24 25 27" + --8<-- "examples/idempotency/tests/test_with_io_operations.py" ``` -=== "app.py" - - ```python - from aws_lambda_powertools.utilities.idempotency import ( - DynamoDBPersistenceLayer, idempotent - ) - - persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency") +=== "app_test_io_operations.py" - @idempotent(persistence_store=persistence_layer) - def handler(event, context): - print('expensive operation') - return { - "payment_id": 12345, - "message": "success", - "statusCode": 200, - } + ```python hl_lines="10" + --8<-- "examples/idempotency/tests/app_test_io_operations.py" ``` ## Extra resources diff --git a/examples/idempotency/src/bring_your_own_persistent_store.py b/examples/idempotency/src/bring_your_own_persistent_store.py new file mode 100644 index 00000000000..b6170f0d8fb --- /dev/null +++ b/examples/idempotency/src/bring_your_own_persistent_store.py @@ -0,0 +1,128 @@ +import datetime +import logging +from typing import Any, Dict, Optional + +import boto3 +from botocore.config import Config + +from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyItemAlreadyExistsError, + IdempotencyItemNotFoundError, +) +from aws_lambda_powertools.utilities.idempotency.persistence.base import DataRecord + +logger = logging.getLogger(__name__) + + +class MyOwnPersistenceLayer(BasePersistenceLayer): + def __init__( + self, + table_name: str, + key_attr: str = "id", + expiry_attr: str = "expiration", + status_attr: str = "status", + data_attr: str = "data", + validation_key_attr: str = "validation", + boto_config: Optional[Config] = None, + boto3_session: Optional[boto3.session.Session] = None, + ): + boto_config = boto_config or Config() + session = boto3_session or boto3.session.Session() + self._ddb_resource = session.resource("dynamodb", config=boto_config) + self.table_name = table_name + self.table = self._ddb_resource.Table(self.table_name) + self.key_attr = key_attr + self.expiry_attr = expiry_attr + self.status_attr = status_attr + self.data_attr = data_attr + self.validation_key_attr = validation_key_attr + super(MyOwnPersistenceLayer, self).__init__() + + def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord: + """ + Translate raw item records from DynamoDB to DataRecord + + Parameters + ---------- + item: Dict[str, Union[str, int]] + Item format from dynamodb response + + Returns + ------- + DataRecord + representation of item + + """ + return DataRecord( + idempotency_key=item[self.key_attr], + status=item[self.status_attr], + expiry_timestamp=item[self.expiry_attr], + response_data=item.get(self.data_attr, ""), + payload_hash=item.get(self.validation_key_attr, ""), + ) + + def _get_record(self, idempotency_key) -> DataRecord: + response = self.table.get_item(Key={self.key_attr: idempotency_key}, ConsistentRead=True) + + try: + item = response["Item"] + except KeyError: + raise IdempotencyItemNotFoundError + return self._item_to_data_record(item) + + def _put_record(self, data_record: DataRecord) -> None: + item = { + self.key_attr: data_record.idempotency_key, + self.expiry_attr: data_record.expiry_timestamp, + self.status_attr: data_record.status, + } + + if self.payload_validation_enabled: + item[self.validation_key_attr] = data_record.payload_hash + + now = datetime.datetime.now() + try: + logger.debug(f"Putting record for idempotency key: {data_record.idempotency_key}") + self.table.put_item( + Item=item, + ConditionExpression=f"attribute_not_exists({self.key_attr}) OR {self.expiry_attr} < :now", + ExpressionAttributeValues={":now": int(now.timestamp())}, + ) + except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException: + logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}") + raise IdempotencyItemAlreadyExistsError + + def _update_record(self, data_record: DataRecord): + logger.debug(f"Updating record for idempotency key: {data_record.idempotency_key}") + update_expression = "SET #response_data = :response_data, #expiry = :expiry, #status = :status" + expression_attr_values = { + ":expiry": data_record.expiry_timestamp, + ":response_data": data_record.response_data, + ":status": data_record.status, + } + expression_attr_names = { + "#response_data": self.data_attr, + "#expiry": self.expiry_attr, + "#status": self.status_attr, + } + + if self.payload_validation_enabled: + update_expression += ", #validation_key = :validation_key" + expression_attr_values[":validation_key"] = data_record.payload_hash + expression_attr_names["#validation_key"] = self.validation_key_attr + + kwargs = { + "Key": {self.key_attr: data_record.idempotency_key}, + "UpdateExpression": update_expression, + "ExpressionAttributeValues": expression_attr_values, + "ExpressionAttributeNames": expression_attr_names, + } + + self.table.update_item(**kwargs) + + def _delete_record(self, data_record: DataRecord) -> None: + logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}") + self.table.delete_item( + Key={self.key_attr: data_record.idempotency_key}, + ) diff --git a/examples/idempotency/src/customize_persistence_layer.py b/examples/idempotency/src/customize_persistence_layer.py new file mode 100644 index 00000000000..26409191ca9 --- /dev/null +++ b/examples/idempotency/src/customize_persistence_layer.py @@ -0,0 +1,20 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer( + table_name="IdempotencyTable", + key_attr="idempotency_key", + expiry_attr="expires_at", + in_progress_expiry_attr="in_progress_expires_at", + status_attr="current_status", + data_attr="result_data", + validation_key_attr="validation_key", +) + + +@idempotent(persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return event diff --git a/examples/idempotency/src/getting_started_with_idempotency.py b/examples/idempotency/src/getting_started_with_idempotency.py new file mode 100644 index 00000000000..0754f42c6b3 --- /dev/null +++ b/examples/idempotency/src/getting_started_with_idempotency.py @@ -0,0 +1,38 @@ +from dataclasses import dataclass, field +from uuid import uuid4 + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + +@dataclass +class Payment: + user_id: str + product_id: str + payment_id: str = field(default_factory=lambda: f"{uuid4()}") + + +class PaymentError(Exception): + ... + + +@idempotent(persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext): + try: + payment: Payment = create_subscription_payment(event) + return { + "payment_id": payment.payment_id, + "message": "success", + "statusCode": 200, + } + except Exception as exc: + raise PaymentError(f"Error creating payment {str(exc)}") + + +def create_subscription_payment(event: dict) -> Payment: + return Payment(**event) diff --git a/examples/idempotency/src/getting_started_with_idempotency_payload.json b/examples/idempotency/src/getting_started_with_idempotency_payload.json new file mode 100644 index 00000000000..74a7ec55962 --- /dev/null +++ b/examples/idempotency/src/getting_started_with_idempotency_payload.json @@ -0,0 +1,4 @@ +{ + "user_id": "xyz", + "product_id": "123456789" +} diff --git a/examples/idempotency/src/integrate_idempotency_with_batch_processor.py b/examples/idempotency/src/integrate_idempotency_with_batch_processor.py new file mode 100644 index 00000000000..957cefb3202 --- /dev/null +++ b/examples/idempotency/src/integrate_idempotency_with_batch_processor.py @@ -0,0 +1,37 @@ +from aws_lambda_powertools import Logger +from aws_lambda_powertools.utilities.batch import BatchProcessor, EventType +from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +logger = Logger() +processor = BatchProcessor(event_type=EventType.SQS) + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig( + event_key_jmespath="messageId", # see Choosing a payload subset section +) + + +@idempotent_function(data_keyword_argument="record", config=config, persistence_store=dynamodb) +def record_handler(record: SQSRecord): + return {"message": record.body} + + +def lambda_handler(event: SQSRecord, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + + # with Lambda context registered for Idempotency + # we can now kick in the Bach processing logic + batch = event["Records"] + with processor(records=batch, handler=record_handler): + # in case you want to access each record processed by your record_handler + # otherwise ignore the result variable assignment + processed_messages = processor.process() + logger.info(processed_messages) + + return processor.response() diff --git a/examples/idempotency/src/integrate_idempotency_with_batch_processor_payload.json b/examples/idempotency/src/integrate_idempotency_with_batch_processor_payload.json new file mode 100644 index 00000000000..73a5029d61a --- /dev/null +++ b/examples/idempotency/src/integrate_idempotency_with_batch_processor_payload.json @@ -0,0 +1,26 @@ +{ + "Records": [ + { + "messageId": "059f36b4-87a3-44ab-83d2-661975830a7d", + "receiptHandle": "AQEBwJnKyrHigUMZj6rYigCgxlaS3SLy0a...", + "body": "Test message.", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1545082649183", + "SenderId": "AIDAIENQZJOLO23YVJ4VO", + "ApproximateFirstReceiveTimestamp": "1545082649185" + }, + "messageAttributes": { + "testAttr": { + "stringValue": "100", + "binaryValue": "base64Str", + "dataType": "Number" + } + }, + "md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:my-queue", + "awsRegion": "us-east-2" + } + ] +} diff --git a/examples/idempotency/src/integrate_idempotency_with_validator.py b/examples/idempotency/src/integrate_idempotency_with_validator.py new file mode 100644 index 00000000000..af833951446 --- /dev/null +++ b/examples/idempotency/src/integrate_idempotency_with_validator.py @@ -0,0 +1,16 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext +from aws_lambda_powertools.utilities.validation import envelopes, validator + +config = IdempotencyConfig(event_key_jmespath='["message", "username"]') +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + +@validator(envelope=envelopes.API_GATEWAY_HTTP) +@idempotent(config=config, persistence_store=persistence_layer) +def lambda_handler(event, context: LambdaContext): + return {"message": event["message"], "statusCode": 200} diff --git a/examples/idempotency/src/integrate_idempotency_with_validator_payload.json b/examples/idempotency/src/integrate_idempotency_with_validator_payload.json new file mode 100644 index 00000000000..9de632b8e3d --- /dev/null +++ b/examples/idempotency/src/integrate_idempotency_with_validator_payload.json @@ -0,0 +1,69 @@ +{ + "version": "2.0", + "routeKey": "$default", + "rawPath": "/my/path", + "rawQueryString": "parameter1=value1¶meter1=value2¶meter2=value", + "cookies": [ + "cookie1", + "cookie2" + ], + "headers": { + "Header1": "value1", + "Header2": "value1,value2" + }, + "queryStringParameters": { + "parameter1": "value1,value2", + "parameter2": "value" + }, + "requestContext": { + "accountId": "123456789012", + "apiId": "api-id", + "authentication": { + "clientCert": { + "clientCertPem": "CERT_CONTENT", + "subjectDN": "www.example.com", + "issuerDN": "Example issuer", + "serialNumber": "a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1", + "validity": { + "notBefore": "May 28 12:30:02 2019 GMT", + "notAfter": "Aug 5 09:36:04 2021 GMT" + } + } + }, + "authorizer": { + "jwt": { + "claims": { + "claim1": "value1", + "claim2": "value2" + }, + "scopes": [ + "scope1", + "scope2" + ] + } + }, + "domainName": "id.execute-api.us-east-1.amazonaws.com", + "domainPrefix": "id", + "http": { + "method": "POST", + "path": "/my/path", + "protocol": "HTTP/1.1", + "sourceIp": "192.168.0.1/32", + "userAgent": "agent" + }, + "requestId": "id", + "routeKey": "$default", + "stage": "$default", + "time": "12/Mar/2020:19:03:58 +0000", + "timeEpoch": 1583348638390 + }, + "body": "{\"message\": \"hello world\", \"username\": \"tom\"}", + "pathParameters": { + "parameter1": "value1" + }, + "isBase64Encoded": false, + "stageVariables": { + "stageVariable1": "value1", + "stageVariable2": "value2" + } +} diff --git a/examples/idempotency/src/working_with_composite_key.py b/examples/idempotency/src/working_with_composite_key.py new file mode 100644 index 00000000000..f1b70cba99a --- /dev/null +++ b/examples/idempotency/src/working_with_composite_key.py @@ -0,0 +1,13 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable", sort_key_attr="sort_key") + + +@idempotent(persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext) -> dict: + user_id: str = event.get("body", "")["user_id"] + return {"message": "success", "user_id": user_id} diff --git a/examples/idempotency/src/working_with_composite_key_payload.json b/examples/idempotency/src/working_with_composite_key_payload.json new file mode 100644 index 00000000000..d2b720442a1 --- /dev/null +++ b/examples/idempotency/src/working_with_composite_key_payload.json @@ -0,0 +1,3 @@ +{ + "body": "{\"user_id\":\"xyz\",\"product_id\":\"123456789\"}" +} diff --git a/examples/idempotency/src/working_with_custom_config.py b/examples/idempotency/src/working_with_custom_config.py new file mode 100644 index 00000000000..30539f88f3c --- /dev/null +++ b/examples/idempotency/src/working_with_custom_config.py @@ -0,0 +1,20 @@ +from botocore.config import Config + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +# See: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore-config +boto_config = Config() + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable", boto_config=boto_config) + +config = IdempotencyConfig(event_key_jmespath="body") + + +@idempotent(persistence_store=persistence_layer, config=config) +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return event diff --git a/examples/idempotency/src/working_with_custom_config_payload.json b/examples/idempotency/src/working_with_custom_config_payload.json new file mode 100644 index 00000000000..d2b720442a1 --- /dev/null +++ b/examples/idempotency/src/working_with_custom_config_payload.json @@ -0,0 +1,3 @@ +{ + "body": "{\"user_id\":\"xyz\",\"product_id\":\"123456789\"}" +} diff --git a/examples/idempotency/src/working_with_custom_session.py b/examples/idempotency/src/working_with_custom_session.py new file mode 100644 index 00000000000..aae89f8a3fe --- /dev/null +++ b/examples/idempotency/src/working_with_custom_session.py @@ -0,0 +1,20 @@ +import boto3 + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +# See: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#module-boto3.session +boto3_session = boto3.session.Session() + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable", boto3_session=boto3_session) + +config = IdempotencyConfig(event_key_jmespath="body") + + +@idempotent(persistence_store=persistence_layer, config=config) +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return event diff --git a/examples/idempotency/src/working_with_exceptions.py b/examples/idempotency/src/working_with_exceptions.py new file mode 100644 index 00000000000..9b495c01ce4 --- /dev/null +++ b/examples/idempotency/src/working_with_exceptions.py @@ -0,0 +1,36 @@ +import requests + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + +config = IdempotencyConfig() + + +def lambda_handler(event: dict, context: LambdaContext): + # If an exception is raised here, no idempotent record will ever get created as the + # idempotent function does not get called + try: + endpoint = "https://jsonplaceholder.typicode.com/comments/" # change this endpoint to force an exception + requests.get(endpoint) + except Exception as exc: + return str(exc) + + call_external_service(data={"user": "user1", "id": 5}) + + # This exception will not cause the idempotent record to be deleted, since it + # happens after the decorated function has been successfully called + raise Exception + + +@idempotent_function(data_keyword_argument="data", config=config, persistence_store=persistence_layer) +def call_external_service(data: dict): + result: requests.Response = requests.post( + "https://jsonplaceholder.typicode.com/comments/", json={"user": data["user"], "transaction_id": data["id"]} + ) + return result.json() diff --git a/examples/idempotency/src/working_with_idempotency_key_required.py b/examples/idempotency/src/working_with_idempotency_key_required.py new file mode 100644 index 00000000000..347740ab4a3 --- /dev/null +++ b/examples/idempotency/src/working_with_idempotency_key_required.py @@ -0,0 +1,17 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig( + event_key_jmespath='["user.uid", "order_id"]', + raise_on_no_idempotency_key=True, +) + + +@idempotent(config=config, persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext) -> dict: + return event diff --git a/examples/idempotency/src/working_with_idempotency_key_required_payload_error.json b/examples/idempotency/src/working_with_idempotency_key_required_payload_error.json new file mode 100644 index 00000000000..43f5ccad8e5 --- /dev/null +++ b/examples/idempotency/src/working_with_idempotency_key_required_payload_error.json @@ -0,0 +1,7 @@ +{ + "user": { + "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", + "name": "Foo", + "order_id": 10000 + } +} diff --git a/examples/idempotency/src/working_with_idempotency_key_required_payload_success.json b/examples/idempotency/src/working_with_idempotency_key_required_payload_success.json new file mode 100644 index 00000000000..ce85eb38989 --- /dev/null +++ b/examples/idempotency/src/working_with_idempotency_key_required_payload_success.json @@ -0,0 +1,7 @@ +{ + "user": { + "uid": "BB0D045C-8878-40C8-889E-38B3CB0A61B1", + "name": "Foo" + }, + "order_id": 10000 +} diff --git a/examples/idempotency/src/working_with_idempotent_function_dataclass.py b/examples/idempotency/src/working_with_idempotent_function_dataclass.py new file mode 100644 index 00000000000..e56c0b42029 --- /dev/null +++ b/examples/idempotency/src/working_with_idempotent_function_dataclass.py @@ -0,0 +1,37 @@ +from dataclasses import dataclass + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath="order_id") # see Choosing a payload subset section + + +@dataclass +class OrderItem: + sku: str + description: str + + +@dataclass +class Order: + item: OrderItem + order_id: int + + +@idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) +def process_order(order: Order): + return f"processed order {order.order_id}" + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id=1) + + # `order` parameter must be called as a keyword argument to work + process_order(order=order) diff --git a/examples/idempotency/src/working_with_idempotent_function_pydantic.py b/examples/idempotency/src/working_with_idempotent_function_pydantic.py new file mode 100644 index 00000000000..5dfd42ae0a8 --- /dev/null +++ b/examples/idempotency/src/working_with_idempotent_function_pydantic.py @@ -0,0 +1,34 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.parser import BaseModel +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath="order_id") # see Choosing a payload subset section + + +class OrderItem(BaseModel): + sku: str + description: str + + +class Order(BaseModel): + item: OrderItem + order_id: int + + +@idempotent_function(data_keyword_argument="order", config=config, persistence_store=dynamodb) +def process_order(order: Order): + return f"processed order {order.order_id}" + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id=1) + + # `order` parameter must be called as a keyword argument to work + process_order(order=order) diff --git a/examples/idempotency/src/working_with_lambda_timeout.py b/examples/idempotency/src/working_with_lambda_timeout.py new file mode 100644 index 00000000000..82b8130b6b7 --- /dev/null +++ b/examples/idempotency/src/working_with_lambda_timeout.py @@ -0,0 +1,22 @@ +from aws_lambda_powertools.utilities.data_classes.sqs_event import SQSRecord +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + +config = IdempotencyConfig() + + +@idempotent_function(data_keyword_argument="record", persistence_store=persistence_layer, config=config) +def record_handler(record: SQSRecord): + return {"message": record["body"]} + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) + + return record_handler(event) diff --git a/examples/idempotency/src/working_with_local_cache.py b/examples/idempotency/src/working_with_local_cache.py new file mode 100644 index 00000000000..82f39dff2ef --- /dev/null +++ b/examples/idempotency/src/working_with_local_cache.py @@ -0,0 +1,17 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig( + event_key_jmespath="body", + use_local_cache=True, +) + + +@idempotent(config=config, persistence_store=persistence_layer) +def lambda_handler(event, context: LambdaContext): + return event diff --git a/examples/idempotency/src/working_with_local_cache_payload.json b/examples/idempotency/src/working_with_local_cache_payload.json new file mode 100644 index 00000000000..d2b720442a1 --- /dev/null +++ b/examples/idempotency/src/working_with_local_cache_payload.json @@ -0,0 +1,3 @@ +{ + "body": "{\"user_id\":\"xyz\",\"product_id\":\"123456789\"}" +} diff --git a/examples/idempotency/src/working_with_payload_subset.py b/examples/idempotency/src/working_with_payload_subset.py new file mode 100644 index 00000000000..9fcc828fe1d --- /dev/null +++ b/examples/idempotency/src/working_with_payload_subset.py @@ -0,0 +1,45 @@ +import json +from dataclasses import dataclass, field +from uuid import uuid4 + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + +# Deserialize JSON string under the "body" key +# then extract "user" and "product_id" data +config = IdempotencyConfig(event_key_jmespath='powertools_json(body).["user_id", "product_id"]') + + +@dataclass +class Payment: + user_id: str + product_id: str + payment_id: str = field(default_factory=lambda: f"{uuid4()}") + + +class PaymentError(Exception): + ... + + +@idempotent(config=config, persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext): + try: + payment_info: str = event.get("body", "") + payment: Payment = create_subscription_payment(json.loads(payment_info)) + return { + "payment_id": payment.payment_id, + "message": "success", + "statusCode": 200, + } + except Exception as exc: + raise PaymentError(f"Error creating payment {str(exc)}") + + +def create_subscription_payment(event: dict) -> Payment: + return Payment(**event) diff --git a/examples/idempotency/src/working_with_payload_subset_payload.json b/examples/idempotency/src/working_with_payload_subset_payload.json new file mode 100644 index 00000000000..03fd9737163 --- /dev/null +++ b/examples/idempotency/src/working_with_payload_subset_payload.json @@ -0,0 +1,30 @@ +{ + "version": "2.0", + "routeKey": "ANY /createpayment", + "rawPath": "/createpayment", + "rawQueryString": "", + "headers": { + "Header1": "value1", + "Header2": "value2" + }, + "requestContext": { + "accountId": "123456789012", + "apiId": "api-id", + "domainName": "id.execute-api.us-east-1.amazonaws.com", + "domainPrefix": "id", + "http": { + "method": "POST", + "path": "/createpayment", + "protocol": "HTTP/1.1", + "sourceIp": "ip", + "userAgent": "agent" + }, + "requestId": "id", + "routeKey": "ANY /createpayment", + "stage": "$default", + "time": "10/Feb/2021:13:40:43 +0000", + "timeEpoch": 1612964443723 + }, + "body": "{\"user_id\":\"xyz\",\"product_id\":\"123456789\"}", + "isBase64Encoded": false +} diff --git a/examples/idempotency/src/working_with_record_expiration.py b/examples/idempotency/src/working_with_record_expiration.py new file mode 100644 index 00000000000..738b4749ebc --- /dev/null +++ b/examples/idempotency/src/working_with_record_expiration.py @@ -0,0 +1,17 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig( + event_key_jmespath="body", + expires_after_seconds=5 * 60, # 5 minutes +) + + +@idempotent(config=config, persistence_store=persistence_layer) +def lambda_handler(event, context: LambdaContext): + return event diff --git a/examples/idempotency/src/working_with_record_expiration_payload.json b/examples/idempotency/src/working_with_record_expiration_payload.json new file mode 100644 index 00000000000..d2b720442a1 --- /dev/null +++ b/examples/idempotency/src/working_with_record_expiration_payload.json @@ -0,0 +1,3 @@ +{ + "body": "{\"user_id\":\"xyz\",\"product_id\":\"123456789\"}" +} diff --git a/examples/idempotency/src/working_with_validation_payload.py b/examples/idempotency/src/working_with_validation_payload.py new file mode 100644 index 00000000000..d81e7d183bd --- /dev/null +++ b/examples/idempotency/src/working_with_validation_payload.py @@ -0,0 +1,42 @@ +from dataclasses import dataclass, field +from uuid import uuid4 + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath='["user_id", "product_id"]', payload_validation_jmespath="amount") + + +@dataclass +class Payment: + user_id: str + product_id: str + charge_type: str + amount: int + payment_id: str = field(default_factory=lambda: f"{uuid4()}") + + +class PaymentError(Exception): + ... + + +@idempotent(config=config, persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext): + try: + payment: Payment = create_subscription_payment(event) + return { + "payment_id": payment.payment_id, + "message": "success", + "statusCode": 200, + } + except Exception as exc: + raise PaymentError(f"Error creating payment {str(exc)}") + + +def create_subscription_payment(event: dict) -> Payment: + return Payment(**event) diff --git a/examples/idempotency/src/working_with_validation_payload_payload1.json b/examples/idempotency/src/working_with_validation_payload_payload1.json new file mode 100644 index 00000000000..7f94aa04a07 --- /dev/null +++ b/examples/idempotency/src/working_with_validation_payload_payload1.json @@ -0,0 +1,6 @@ +{ + "user_id": 1, + "product_id": 1500, + "charge_type": "subscription", + "amount": 500 +} diff --git a/examples/idempotency/src/working_with_validation_payload_payload2.json b/examples/idempotency/src/working_with_validation_payload_payload2.json new file mode 100644 index 00000000000..f400627f891 --- /dev/null +++ b/examples/idempotency/src/working_with_validation_payload_payload2.json @@ -0,0 +1,6 @@ +{ + "user_id": 1, + "product_id": 1500, + "charge_type": "subscription", + "amount": 10 +} diff --git a/examples/idempotency/cdk.py b/examples/idempotency/templates/cdk.py similarity index 100% rename from examples/idempotency/cdk.py rename to examples/idempotency/templates/cdk.py diff --git a/examples/idempotency/sam.yaml b/examples/idempotency/templates/sam.yaml similarity index 96% rename from examples/idempotency/sam.yaml rename to examples/idempotency/templates/sam.yaml index ee9b7540de9..8443a0914d7 100644 --- a/examples/idempotency/sam.yaml +++ b/examples/idempotency/templates/sam.yaml @@ -17,7 +17,7 @@ Resources: HelloWorldFunction: Type: AWS::Serverless::Function Properties: - Runtime: python3.9 + Runtime: python3.10 Handler: app.py Policies: - Statement: diff --git a/examples/idempotency/templates/terraform.tf b/examples/idempotency/templates/terraform.tf new file mode 100644 index 00000000000..1572dfefa1f --- /dev/null +++ b/examples/idempotency/templates/terraform.tf @@ -0,0 +1,79 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } +} + +provider "aws" { + region = "us-east-1" # Replace with your desired AWS region +} + +resource "aws_dynamodb_table" "IdempotencyTable" { + name = "IdempotencyTable" + billing_mode = "PAY_PER_REQUEST" + hash_key = "id" + attribute { + name = "id" + type = "S" + } + ttl { + attribute_name = "expiration" + enabled = true + } +} + +resource "aws_lambda_function" "IdempotencyFunction" { + function_name = "IdempotencyFunction" + role = aws_iam_role.IdempotencyFunctionRole.arn + runtime = "python3.10" + handler = "app.lambda_handler" + filename = "lambda.zip" + +} + +resource "aws_iam_role" "IdempotencyFunctionRole" { + name = "IdempotencyFunctionRole" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "" + Effect = "Allow" + Principal = { + Service = "lambda.amazonaws.com" + } + Action = "sts:AssumeRole" + }, + ] + }) +} + +resource "aws_iam_policy" "LambdaDynamoDBPolicy" { + name = "LambdaDynamoDBPolicy" + description = "IAM policy for Lambda function to access DynamoDB" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowDynamodbReadWrite" + Effect = "Allow" + Action = [ + "dynamodb:PutItem", + "dynamodb:GetItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + ] + Resource = aws_dynamodb_table.IdempotencyTable.arn + }, + ] + }) +} + +resource "aws_iam_role_policy_attachment" "IdempotencyFunctionRoleAttachment" { + role = aws_iam_role.IdempotencyFunctionRole.name + policy_arn = aws_iam_policy.LambdaDynamoDBPolicy.arn +} diff --git a/examples/idempotency/tests/app_test_disabling_idempotency_utility.py b/examples/idempotency/tests/app_test_disabling_idempotency_utility.py new file mode 100644 index 00000000000..0405ea6e729 --- /dev/null +++ b/examples/idempotency/tests/app_test_disabling_idempotency_utility.py @@ -0,0 +1,17 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + +@idempotent(persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext): + print("expensive operation") + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } diff --git a/examples/idempotency/tests/app_test_dynamodb_local.py b/examples/idempotency/tests/app_test_dynamodb_local.py new file mode 100644 index 00000000000..0405ea6e729 --- /dev/null +++ b/examples/idempotency/tests/app_test_dynamodb_local.py @@ -0,0 +1,17 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + +@idempotent(persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext): + print("expensive operation") + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } diff --git a/examples/idempotency/tests/app_test_io_operations.py b/examples/idempotency/tests/app_test_io_operations.py new file mode 100644 index 00000000000..0405ea6e729 --- /dev/null +++ b/examples/idempotency/tests/app_test_io_operations.py @@ -0,0 +1,17 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + idempotent, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +persistence_layer = DynamoDBPersistenceLayer(table_name="IdempotencyTable") + + +@idempotent(persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext): + print("expensive operation") + return { + "payment_id": 12345, + "message": "success", + "statusCode": 200, + } diff --git a/examples/idempotency/tests/test_disabling_idempotency_utility.py b/examples/idempotency/tests/test_disabling_idempotency_utility.py new file mode 100644 index 00000000000..f33174cde3d --- /dev/null +++ b/examples/idempotency/tests/test_disabling_idempotency_utility.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass + +import app_test_disabling_idempotency_utility +import pytest + + +@pytest.fixture +def lambda_context(): + @dataclass + class LambdaContext: + function_name: str = "test" + memory_limit_in_mb: int = 128 + invoked_function_arn: str = "arn:aws:lambda:eu-west-1:809313241:function:test" + aws_request_id: str = "52fdfc07-2182-154f-163f-5f0f9a621d72" + + def get_remaining_time_in_millis(self) -> int: + return 5 + + return LambdaContext() + + +def test_idempotent_lambda_handler(monkeypatch, lambda_context): + # Set POWERTOOLS_IDEMPOTENCY_DISABLED before calling decorated functions + monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", 1) + + result = app_test_disabling_idempotency_utility.lambda_handler({}, lambda_context) + + assert result diff --git a/examples/idempotency/tests/test_with_dynamodb_local.py b/examples/idempotency/tests/test_with_dynamodb_local.py new file mode 100644 index 00000000000..eaa77a9dddd --- /dev/null +++ b/examples/idempotency/tests/test_with_dynamodb_local.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass + +import app_test_dynamodb_local +import boto3 +import pytest + + +@pytest.fixture +def lambda_context(): + @dataclass + class LambdaContext: + function_name: str = "test" + memory_limit_in_mb: int = 128 + invoked_function_arn: str = "arn:aws:lambda:eu-west-1:809313241:function:test" + aws_request_id: str = "52fdfc07-2182-154f-163f-5f0f9a621d72" + + def get_remaining_time_in_millis(self) -> int: + return 5 + + return LambdaContext() + + +def test_idempotent_lambda(lambda_context): + # Configure the boto3 to use the endpoint for the DynamoDB Local instance + dynamodb_local_client = boto3.client("dynamodb", endpoint_url="http://localhost:8000") + app_test_dynamodb_local.persistence_layer.client = dynamodb_local_client + + # If desired, you can use a different DynamoDB Local table name than what your code already uses + # app.persistence_layer.table_name = "another table name" # noqa: E800 + + result = app_test_dynamodb_local.handler({"testkey": "testvalue"}, lambda_context) + assert result["payment_id"] == 12345 diff --git a/examples/idempotency/tests/test_with_io_operations.py b/examples/idempotency/tests/test_with_io_operations.py new file mode 100644 index 00000000000..9d455906889 --- /dev/null +++ b/examples/idempotency/tests/test_with_io_operations.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from unittest.mock import MagicMock + +import app_test_io_operations +import pytest + + +@pytest.fixture +def lambda_context(): + @dataclass + class LambdaContext: + function_name: str = "test" + memory_limit_in_mb: int = 128 + invoked_function_arn: str = "arn:aws:lambda:eu-west-1:809313241:function:test" + aws_request_id: str = "52fdfc07-2182-154f-163f-5f0f9a621d72" + + def get_remaining_time_in_millis(self) -> int: + return 5 + + return LambdaContext() + + +def test_idempotent_lambda(lambda_context): + mock_client = MagicMock() + app_test_io_operations.persistence_layer.client = mock_client + result = app_test_io_operations.handler({"testkey": "testvalue"}, lambda_context) + mock_client.put_item.assert_called() + assert result