Merge branch 'scene-intelligence-with-rosbag-on-aws' of github.com:aw…

…slabs/autonomous-driving-data-framework into scene-intelligence-with-rosbag-on-aws

modules/sensor-extraction/ros-to-png/app.py

@@ -23,11 +23,18 @@ def _param(name: str) -> str:
 resized_height = os.getenv(_param("RESIZED_HEIGHT"))
 removal_policy = os.getenv(_param("REMOVAL_POLICY"), "")
 
+batch_config = {
+    "retries": retries,
+    "timeout_seconds": timeout_seconds,
+    "vcpus": vcpus,
+    "memory_limit_mib": memory_limit_mib,
+}
+
 if resized_width:
-    resized_width = int(resized_width)  # type: ignore
+    batch_config["resized_width"] = int(resized_width)
 
 if resized_height:
-    resized_height = int(resized_height)  # type: ignore
+    batch_config["resized_height"] = int(resized_height)
 
 if not full_access_policy:
     raise ValueError("S3 Full Access Policy ARN is missing.")
@@ -57,14 +64,9 @@ def generate_description() -> str:
         account=os.environ["CDK_DEFAULT_ACCOUNT"],
         region=os.environ["CDK_DEFAULT_REGION"],
     ),
-    retries=retries,
-    timeout_seconds=timeout_seconds,
-    vcpus=vcpus,
-    memory_limit_mib=memory_limit_mib,
     s3_access_policy=full_access_policy,
+    batch_config=batch_config,
     removal_policy=RemovalPolicy.RETAIN if removal_policy.upper() == "RETAIN" else RemovalPolicy.DESTROY,
-    resized_width=resized_width,  # type: ignore
-    resized_height=resized_height,  # type: ignore
     stack_description=generate_description(),
 )
 

modules/sensor-extraction/ros-to-png/stack.py

@@ -3,7 +3,7 @@
 
 import logging
 import os
-from typing import Any, cast
+from typing import Any, Dict, cast
 
 import aws_cdk.aws_batch_alpha as batch
 import aws_cdk.aws_ecr as ecr
@@ -24,15 +24,11 @@ def __init__(
         self,
         scope: Construct,
         id: str,
+        *,
         deployment_name: str,
         module_name: str,
         s3_access_policy: str,
-        retries: int,
-        timeout_seconds: int,
-        vcpus: int,
-        memory_limit_mib: int,
-        resized_width: int,
-        resized_height: int,
+        batch_config: Dict[str, Any],
         stack_description: str,
         removal_policy: RemovalPolicy = RemovalPolicy.DESTROY,
         **kwargs: Any,
@@ -107,23 +103,28 @@ def __init__(
             max_session_duration=Duration.hours(12),
         )
 
+        batch_env = {
+            "AWS_DEFAULT_REGION": self.region,
+            "AWS_ACCOUNT_ID": self.account,
+            "DEBUG": "true",
+        }
+        if batch_config.get("resized_width"):
+            batch_env["RESIZE_WIDTH"] = str(batch_config["resized_width"])
+
+        if batch_config.get("resized_height"):
+            batch_env["RESIZE_HEIGHT"] = str(batch_config["resized_height"])
+
         self.batch_job = batch.JobDefinition(
             self,
             "batch-job-def-from-ecr",
             container=batch.JobDefinitionContainer(
                 image=ecs.ContainerImage.from_ecr_repository(repo, "latest"),
                 command=["bash", "entrypoint.sh"],
-                environment={
-                    "AWS_DEFAULT_REGION": self.region,
-                    "AWS_ACCOUNT_ID": self.account,
-                    "DEBUG": "true",
-                    "RESIZE_WIDTH": str(resized_width),
-                    "RESIZE_HEIGHT": str(resized_height),
-                },
+                environment=batch_env,
                 job_role=role,
                 execution_role=role,
-                memory_limit_mib=memory_limit_mib,
-                vcpus=vcpus,
+                memory_limit_mib=batch_config["memory_limit_mib"],
+                vcpus=batch_config["vcpus"],
                 volumes=[
                     ecs.Volume(
                         name="scratch",
@@ -142,8 +143,8 @@ def __init__(
             ),
             job_definition_name=self.repository_name,
             platform_capabilities=[batch.PlatformCapabilities.EC2],
-            retry_attempts=retries,
-            timeout=Duration.seconds(timeout_seconds),
+            retry_attempts=batch_config["retries"],
+            timeout=Duration.seconds(batch_config["timeout_seconds"]),
         )
 
         Aspects.of(self).add(cdk_nag.AwsSolutionsChecks())

modules/sensor-extraction/ros-to-png/tests/test_stack.py

@@ -28,18 +28,57 @@ def test_synthesize_stack(stack_defaults):
     dep_name = "test-deployment"
     mod_name = "test-module"
 
+    batch_config = {
+        "retries": 1,
+        "timeout_seconds": 1800,
+        "vcpus": 2,
+        "memory_limit_mib": 8192,
+        "resized_width": 1280,
+        "resized_height": 720,
+    }
+
+    ros_to_png = stack.RosToPngBatchJob(
+        scope=app,
+        id=f"addf-{dep_name}-{mod_name}",
+        deployment_name=dep_name,
+        module_name=mod_name,
+        s3_access_policy="arn:aws:iam::123456789012:policy/addf-buckets-us-west-2-123-full-access",
+        batch_config=batch_config,
+        stack_description="Testing",
+        env=cdk.Environment(
+            account=os.environ["CDK_DEFAULT_ACCOUNT"],
+            region=os.environ["CDK_DEFAULT_REGION"],
+        ),
+    )
+
+    template = Template.from_stack(ros_to_png)
+    template.resource_count_is("AWS::ECR::Repository", 1)
+    template.resource_count_is("AWS::Lambda::Function", 2)
+    template.resource_count_is("AWS::Batch::JobDefinition", 1)
+    template.resource_count_is("AWS::IAM::Role", 3)
+
+
+def test_synthesize_stack_without_resize(stack_defaults):
+    import stack
+
+    app = cdk.App()
+    dep_name = "test-deployment"
+    mod_name = "test-module"
+
+    batch_config = {
+        "retries": 1,
+        "timeout_seconds": 1800,
+        "vcpus": 2,
+        "memory_limit_mib": 8192,
+    }
+
     ros_to_png = stack.RosToPngBatchJob(
         scope=app,
         id=f"addf-{dep_name}-{mod_name}",
         deployment_name=dep_name,
         module_name=mod_name,
         s3_access_policy="arn:aws:iam::123456789012:policy/addf-buckets-us-west-2-123-full-access",
-        retries=1,
-        timeout_seconds=1800,
-        vcpus=2,
-        memory_limit_mib=8192,
-        resized_width=1280,
-        resized_height=720,
+        batch_config=batch_config,
         stack_description="Testing",
         env=cdk.Environment(
             account=os.environ["CDK_DEFAULT_ACCOUNT"],