feat(glue-alpha): include extraJars parameter in pyspark jobs

aws · Jan 30, 2025 · 399e8df · 399e8df
1 parent c40a9e2
commit 399e8df
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 1 deletion.
diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-etl-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-etl-job.ts
@@ -38,6 +38,13 @@ export interface PySparkEtlJobProps extends JobProperties {
    */
   readonly extraFiles?: Code[];
 
+  /**
+   * Extra Jars S3 URL (optional)
+   * S3 URL where additional jar dependencies are located
+   * @default - no extra jar files
+   */
+  readonly extraJars?: Code[];
+
   /**
    * Specifies whether job run queuing is enabled for the job runs for this job.
    * A value of true means job run queuing is enabled for the job runs.
@@ -156,6 +163,9 @@ export class PySparkEtlJob extends Job {
     if (props.extraFiles && props.extraFiles.length > 0) {
       args['--extra-files'] = props.extraFiles.map(code => this.codeS3ObjectUrl(code)).join(',');
     }
+    if (props.extraJars && props.extraJars?.length > 0) {
+      args['--extra-jars'] = props.extraJars.map(code => this.codeS3ObjectUrl(code)).join(',');
+    }
 
     return args;
   }

diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-flex-etl-job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/jobs/pyspark-flex-etl-job.ts
@@ -47,6 +47,13 @@ export interface PySparkFlexEtlJobProps extends JobProperties {
    */
   readonly extraFiles?: Code[];
 
+  /**
+   * Extra Jars S3 URL (optional)
+   * S3 URL where additional jar dependencies are located
+   * @default - no extra jar files
+   */
+  readonly extraJars?: Code[];
+
 }
 
 /**
@@ -157,6 +164,9 @@ export class PySparkFlexEtlJob extends Job {
     if (props.extraFiles && props.extraFiles.length > 0) {
       args['--extra-files'] = props.extraFiles.map(code => this.codeS3ObjectUrl(code)).join(',');
     }
+    if (props.extraJars && props.extraJars?.length > 0) {
+      args['--extra-jars'] = props.extraJars.map(code => this.codeS3ObjectUrl(code)).join(',');
+    }
 
     return args;
   }

diff --git a/packages/@aws-cdk/aws-glue-alpha/test/pyspark-etl-jobs.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/pyspark-etl-jobs.test.ts
@@ -365,7 +365,7 @@ describe('Job', () => {
     });
   });
 
-  describe('Create PySpark ETL Job with extraPythonFiles and extraFiles', () => {
+  describe('Create PySpark ETL Job with extraPythonFiles, extraFiles and extraJars', () => {
     beforeEach(() => {
       job = new glue.PySparkEtlJob(stack, 'PySparkETLJob', {
         role,
@@ -381,6 +381,11 @@ describe('Job', () => {
             s3.Bucket.fromBucketName(stack, 'extraFilesBucket', 'extra-files-bucket'),
             'prefix/file.txt'),
         ],
+        extraJars: [
+          glue.Code.fromBucket(
+            s3.Bucket.fromBucketName(stack, 'extraJarsBucket', 'extra-jars-bucket'),
+            'prefix/file.jar'),
+        ],
       });
     });
 
@@ -408,6 +413,7 @@ describe('Job', () => {
           '--enable-continuous-cloudwatch-log': 'true',
           '--extra-py-files': 's3://extra-python-files-bucket/prefix/file.py',
           '--extra-files': 's3://extra-files-bucket/prefix/file.txt',
+          '--extra-jars': 's3://extra-jars-bucket/prefix/file.jar',
         }),
       });
     });