From 24248dfd9047788fcf84caf50e443f8734584f3c Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 02:26:16 +0000
Subject: [PATCH 01/26] Add Python and Glue version parameters, add check mode

Available Python and Glue version can be found here:
https://docs.aws.amazon.com/glue/latest/dg/add-job.html

Example:

```
community.aws.aws_glue__jobs:
  - name: my-job
    description: My test job
    command_script_location: my-s3-bucket/script.py
    command_python_version: 3
    glue_version: "2.0"
    role: MyGlueJobRole
    state: present
```
---
 .../480-aws_glue_job-python-glue-version.yml  |   4 +
 plugins/modules/aws_glue_job.py               |  48 +++-
 .../integration/targets/aws_glue_job/aliases  |   1 +
 .../targets/aws_glue_job/defaults/main.yml    |   5 +
 .../targets/aws_glue_job/tasks/main.yml       | 271 ++++++++++++++++++
 5 files changed, 323 insertions(+), 6 deletions(-)
 create mode 100644 changelogs/fragments/480-aws_glue_job-python-glue-version.yml
 create mode 100644 tests/integration/targets/aws_glue_job/aliases
 create mode 100644 tests/integration/targets/aws_glue_job/defaults/main.yml
 create mode 100644 tests/integration/targets/aws_glue_job/tasks/main.yml

diff --git a/changelogs/fragments/480-aws_glue_job-python-glue-version.yml b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
new file mode 100644
index 00000000000..3f62ed5a3b6
--- /dev/null
+++ b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
@@ -0,0 +1,4 @@
+minor_changes:
+  - aws_glue_job - Added ``glue_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
+  - aws_glue_job - Added ``command_python_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
+  - aws_glue_job - Added support for check mode (https://github.com/ansible-collections/community.aws/pull/480).
diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index dac91ecc794..f7e7641c8f3 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -28,6 +28,11 @@
       - The name of the job command. This must be 'glueetl'.
     default: glueetl
     type: str
+  command_python_version:
+    description:
+      - Python version being used to execute a Python shell job. Allowed values are '2' or '3'.
+    default: 2
+    type: str
   command_script_location:
     description:
       - The S3 path to a script that executes a job.
@@ -47,6 +52,12 @@
     description:
       - Description of the job being defined.
     type: str
+  glue_version:
+    description:
+      - AWS Glue version. This determines the available version of Apache Scala and Python as described here
+        U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
+    default: 0.9
+    type: str
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when
@@ -138,6 +149,11 @@
             returned: when state is present
             type: str
             sample: mybucket/myscript.py
+        python_version:
+            description: Specifies the Python version.
+            returned: when state is present
+            type: str
+            sample: 3
 connections:
     description: The connections used for this job.
     returned: when state is present
@@ -158,6 +174,11 @@
     returned: when state is present
     type: str
     sample: My first Glue job
+glue_version:
+    description: Glue version.
+    returned: when state is present
+    type: str
+    sample: 2.0
 job_name:
     description: The name of the AWS Glue job.
     returned: always
@@ -252,8 +273,11 @@ def _compare_glue_job_params(user_params, current_params):
 
     if 'AllocatedCapacity' in user_params and user_params['AllocatedCapacity'] != current_params['AllocatedCapacity']:
         return True
-    if 'Command' in user_params and user_params['Command']['ScriptLocation'] != current_params['Command']['ScriptLocation']:
-        return True
+    if 'Command' in user_params:
+        if user_params['Command']['ScriptLocation'] != current_params['Command']['ScriptLocation']:
+            return True
+        if user_params['Command']['PythonVersion'] != current_params['Command']['PythonVersion']:
+            return True
     if 'Connections' in user_params and set(user_params['Connections']) != set(current_params['Connections']):
         return True
     if 'DefaultArguments' in user_params and set(user_params['DefaultArguments']) != set(current_params['DefaultArguments']):
@@ -262,6 +286,8 @@ def _compare_glue_job_params(user_params, current_params):
         return True
     if 'ExecutionProperty' in user_params and user_params['ExecutionProperty']['MaxConcurrentRuns'] != current_params['ExecutionProperty']['MaxConcurrentRuns']:
         return True
+    if 'GlueVersion' in user_params and user_params['GlueVersion'] != current_params['GlueVersion']:
+        return True
     if 'MaxRetries' in user_params and user_params['MaxRetries'] != current_params['MaxRetries']:
         return True
     if 'Timeout' in user_params and user_params['Timeout'] != current_params['Timeout']:
@@ -294,12 +320,16 @@ def create_or_update_glue_job(connection, module, glue_job):
         params['AllocatedCapacity'] = module.params.get("allocated_capacity")
     if module.params.get("command_script_location") is not None:
         params['Command'] = {'Name': module.params.get("command_name"), 'ScriptLocation': module.params.get("command_script_location")}
+        if module.params.get("command_python_version") is not None:
+            params['Command']['PythonVersion'] = module.params.get("command_python_version")
     if module.params.get("connections") is not None:
         params['Connections'] = {'Connections': module.params.get("connections")}
     if module.params.get("default_arguments") is not None:
         params['DefaultArguments'] = module.params.get("default_arguments")
     if module.params.get("description") is not None:
         params['Description'] = module.params.get("description")
+    if module.params.get("glue_version") is not None:
+        params['GlueVersion'] = module.params.get("glue_version")
     if module.params.get("max_concurrent_runs") is not None:
         params['ExecutionProperty'] = {'MaxConcurrentRuns': module.params.get("max_concurrent_runs")}
     if module.params.get("max_retries") is not None:
@@ -320,13 +350,15 @@ def create_or_update_glue_job(connection, module, glue_job):
                 # Update job needs slightly modified params
                 update_params = {'JobName': params['Name'], 'JobUpdate': copy.deepcopy(params)}
                 del update_params['JobUpdate']['Name']
-                connection.update_job(**update_params)
+                if not module.check_mode:
+                    connection.update_job(**update_params)
                 changed = True
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e)
     else:
         try:
-            connection.create_job(**params)
+            if not module.check_mode:
+                connection.create_job(**params)
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -352,7 +384,8 @@ def delete_glue_job(connection, module, glue_job):
 
     if glue_job:
         try:
-            connection.delete_job(JobName=glue_job['Name'])
+            if not module.check_mode:
+                connection.delete_job(JobName=glue_job['Name'])
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -366,10 +399,12 @@ def main():
         dict(
             allocated_capacity=dict(type='int'),
             command_name=dict(type='str', default='glueetl'),
+            command_python_version=dict(type='str', default='2'),
             command_script_location=dict(type='str'),
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
+            glue_version=dict(type='str', default='0.9'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),
@@ -385,7 +420,8 @@ def main():
     module = AnsibleAWSModule(argument_spec=argument_spec,
                               required_if=[
                                   ('state', 'present', ['role', 'command_script_location'])
-                              ]
+                              ],
+                              supports_check_mode=True
                               )
 
     connection = module.client('glue')
diff --git a/tests/integration/targets/aws_glue_job/aliases b/tests/integration/targets/aws_glue_job/aliases
new file mode 100644
index 00000000000..4ef4b2067d0
--- /dev/null
+++ b/tests/integration/targets/aws_glue_job/aliases
@@ -0,0 +1 @@
+cloud/aws
diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
new file mode 100644
index 00000000000..26557a1137a
--- /dev/null
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+glue_job_name: '{{ resource_prefix }}'
+glue_job_role_name: 'ansible-test-{{ resource_prefix }}-glue-job'
+glue_job_description: Test job
+glue_job_command_script_location: test-s3-bucket-glue/job.py
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
new file mode 100644
index 00000000000..d3122a2a4fd
--- /dev/null
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -0,0 +1,271 @@
+---
+- name: aws_glue_job integration tests
+  collections:
+    - amazon.aws
+  module_defaults:
+    group/aws:
+      aws_access_key: "{{ aws_access_key }}"
+      aws_secret_key: "{{ aws_secret_key }}"
+      security_token: "{{ security_token | default(omit) }}"
+      region: "{{ aws_region }}"
+  block:
+    # AWS CLI is needed until there's a module to get info about Glue jobs
+    - name: Install AWS CLI
+      pip:
+        name: awscli
+        state: present
+
+    - name: Create minimal Glue job role
+      iam_role:
+        name: "{{ glue_job_role_name }}"
+        trust_policy:
+          Version: "2008-10-17"
+          Statement:
+            - Action: "sts:AssumeRole"
+              Effect: Allow
+              Principal:
+                Service: glue.amazonaws.com
+        create_instance_profile: false
+        managed_policies:
+          - "arn:aws:iam::aws:policy/AWSGlueServiceRole"
+
+    - name: Create Glue job (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      check_mode: true
+      register: glue_job_check
+
+    - name: Verity that Glue job was not created in check mode
+      assert:
+        that:
+          - glue_job_check.changed
+          - not glue_job_check.description
+
+    - name: Create Glue job
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      register: glue_job
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query
+
+    - name: Convert it to an object
+      set_fact:
+        job_info: "{{ job_info_query.stdout | from_json }}"
+
+    - name: Verity that Glue job was created
+      assert:
+        that:
+          - glue_job.changed"
+          - glue_job.description == job_info["Job"]["Description"]
+          - glue_job.role == job_info["Job"]["Role"]
+          - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
+          - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
+          - glue_job.glue_version == job_info["Job"]["GlueVersion"]
+
+    - name: Create Glue job (idempotent) (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      check_mode: true
+      register: glue_job_idempotent_check
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_idempotent_check
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_idempotent_check: "{{ job_info_query_idempotent_check.stdout | from_json }}"
+
+    - name: Verity that Glue job was not modified in check mode
+      assert:
+        that:
+          - not glue_job_idempotent_check.changed
+          - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
+          - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
+          - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
+          - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+
+    - name: Create Glue job (idempotent)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 3
+        glue_version: "2.0"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      register: glue_job_idempotent
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_idempotent
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_idempotent: "{{ job_info_query_idempotent.stdout | from_json }}"
+
+    - name: Verity that Glue job was not modified
+      assert:
+        that:
+          - not glue_job_idempotent.changed
+          - job_info["Job"]["Name"] == job_info_idempotent["Job"]["Name"]
+          - job_info["Job"]["Description"] == job_info_idempotent["Job"]["Description"]
+          - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
+          - job_info["Job"]["GlueVersion"] == job_info_idempotent["Job"]["GlueVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+
+    - name: Update Glue job (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 2
+        glue_version: "0.9"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      check_mode: true
+      register: glue_job_update_check
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_update_check
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_update_check: "{{ job_info_query_update_check.stdout | from_json }}"
+
+    - name: Verity that Glue job was not modified in check mode
+      assert:
+        that:
+          - glue_job_update_check.changed
+          - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
+          - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
+          - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update_check.command.python_version == "3"
+          - glue_job_update_check.glue_version == "2.0"
+
+    - name: Update Glue job
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        description: "{{ glue_job_description }}"
+        command_script_location: "{{ glue_job_command_script_location }}"
+        command_python_version: 2
+        glue_version: "0.9"
+        role: "{{ glue_job_role_name }}"
+        state: present
+      register: glue_job_update
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_update
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_update: "{{ job_info_query_update.stdout | from_json }}"
+
+    - name: Verity that Glue job was modified
+      assert:
+        that:
+          - glue_job_update.changed
+          - glue_job_update.description == job_info_update["Job"]["Description"]
+          - glue_job_update.role == job_info_update["Job"]["Role"]
+          - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update.command.python_version == job_info_update["Job"]["Command"]["PythonVersion"]
+          - glue_job_update.glue_version == job_info_update["Job"]["Command"]["GlueVersion"]
+
+    - name: Delete Glue job (check mode)
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        state: absent
+      check_mode: true
+      register: glue_job_delete_check
+
+    - name: Get info on Glue job
+      command: "aws glue get-job --job-name {{ glue_job_name }}"
+      environment:
+        AWS_ACCESS_KEY_ID: "{{ aws_access_key }}"
+        AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}"
+        AWS_SESSION_TOKEN: "{{ security_token | default('') }}"
+        AWS_DEFAULT_REGION: "{{ aws_region }}"
+      register: job_info_query_delete_check
+
+    - name: Convert it to an object
+      set_fact:
+        job_info_delete_check: "{{ job_info_query_delete_check.stdout | from_json }}"
+
+    - name: Verity that Glue job was not deleted in check mode
+      assert:
+        that:
+          - glue_job_delete_check.changed
+          - job_info["Job"]["Name"] == job_info_delete_check["Job"]["Name"]
+
+    - name: Delete Glue job
+      aws_glue_job:
+        name: "{{ glue_job_name }}"
+        state: absent
+      register: glue_job_delete
+
+    - name: Verity that Glue job was deleted
+      assert:
+        that:
+          - glue_job_delete.changed
+
+  always:
+    - name: Delete Glue job
+      aws_glue_connection:
+        name: "{{ glue_job_name }}"
+        state: absent
+      ignore_errors: true
+    - name: Delete Glue job role
+      iam_role:
+        name: "{{ glue_job_role_name }}"
+        state: absent
+      ignore_errors: true

From fe1160d988421486d0a5d22ffa4b15df05636a45 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 02:50:08 +0000
Subject: [PATCH 02/26] Fix parameter name - assume_role_policy_document

---
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index d3122a2a4fd..2c3c686b065 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -18,7 +18,7 @@
     - name: Create minimal Glue job role
       iam_role:
         name: "{{ glue_job_role_name }}"
-        trust_policy:
+        assume_role_policy_document:
           Version: "2008-10-17"
           Statement:
             - Action: "sts:AssumeRole"

From 223f8164a2869b9ca668869934e452bcea49974a Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 03:31:26 +0000
Subject: [PATCH 03/26] Attempt with a different policy that appears to be
 allowed

---
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 2c3c686b065..677c8ce3bb3 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -27,7 +27,7 @@
                 Service: glue.amazonaws.com
         create_instance_profile: false
         managed_policies:
-          - "arn:aws:iam::aws:policy/AWSGlueServiceRole"
+          - "arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess"
 
     - name: Create Glue job (check mode)
       aws_glue_job:

From b6ab6b7617686c47aa5205bd34e61789a7b1158e Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 09:49:45 -0400
Subject: [PATCH 04/26] Remove default value for command_python_version

Co-authored-by: Mark Chappell <mchappel@redhat.com>
---
 plugins/modules/aws_glue_job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index f7e7641c8f3..16b67951d20 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -399,7 +399,7 @@ def main():
         dict(
             allocated_capacity=dict(type='int'),
             command_name=dict(type='str', default='glueetl'),
-            command_python_version=dict(type='str', default='2'),
+            command_python_version=dict(type='str'),
             command_script_location=dict(type='str'),
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),

From 4731d67f90bc7cd07a149e7aeee8d8e8de8e1379 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 13:51:24 +0000
Subject: [PATCH 05/26] Remove default values for command_python_version and
 glue_version

This is to avoid inadvertently breaking existing playbooks. Defaults
will be set by AWS.
---
 plugins/modules/aws_glue_job.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 16b67951d20..ce4792ece5a 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -31,7 +31,6 @@
   command_python_version:
     description:
       - Python version being used to execute a Python shell job. Allowed values are '2' or '3'.
-    default: 2
     type: str
   command_script_location:
     description:
@@ -56,7 +55,6 @@
     description:
       - AWS Glue version. This determines the available version of Apache Scala and Python as described here
         U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
-    default: 0.9
     type: str
   max_concurrent_runs:
     description:
@@ -404,7 +402,7 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
-            glue_version=dict(type='str', default='0.9'),
+            glue_version=dict(type='str'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),

From 30f64615b82504d5c05d3408a7c52ddd25fa8551 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 14:48:49 +0000
Subject: [PATCH 06/26] Update IAM role name for Glue job to make sure it's
 shorter than 64 characters

---
 .../integration/targets/aws_glue_job/defaults/main.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
index 26557a1137a..51ed2fddc62 100644
--- a/tests/integration/targets/aws_glue_job/defaults/main.yml
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -1,5 +1,11 @@
 ---
-glue_job_name: '{{ resource_prefix }}'
-glue_job_role_name: 'ansible-test-{{ resource_prefix }}-glue-job'
+glue_job_name: "{{ resource_prefix }}"
 glue_job_description: Test job
 glue_job_command_script_location: test-s3-bucket-glue/job.py
+# IAM role names have to be less than 64 characters
+# The 8 digit identifier at the end of resource_prefix helps determine during
+# which test something was created and allows tests to be run in parallel
+# Shippable resource_prefixes are in the format shippable-123456-123, so in those cases
+# we need both sets of digits to keep the resource name unique
+unique_id: "{{ resource_prefix | regex_search('(\\d+-?)(\\d+)$') }}"
+glue_job_role_name: "ansible-test-{{ unique_id }}-glue-job"

From ed546fbce00b0f17bf9ea43ae59925cdfd7cbedb Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sun, 21 Mar 2021 23:31:23 +0000
Subject: [PATCH 07/26] Add support for tags

---
 .../480-aws_glue_job-python-glue-version.yml  |  1 +
 plugins/modules/aws_glue_job.py               | 80 +++++++++++++++++--
 .../targets/aws_glue_job/tasks/main.yml       | 70 +++++++++-------
 3 files changed, 118 insertions(+), 33 deletions(-)

diff --git a/changelogs/fragments/480-aws_glue_job-python-glue-version.yml b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
index 3f62ed5a3b6..a277d0bcc3d 100644
--- a/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
+++ b/changelogs/fragments/480-aws_glue_job-python-glue-version.yml
@@ -2,3 +2,4 @@ minor_changes:
   - aws_glue_job - Added ``glue_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
   - aws_glue_job - Added ``command_python_version`` parameter (https://github.com/ansible-collections/community.aws/pull/480).
   - aws_glue_job - Added support for check mode (https://github.com/ansible-collections/community.aws/pull/480).
+  - aws_glue_job - Added support for tags (https://github.com/ansible-collections/community.aws/pull/480).
diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index ce4792ece5a..11e38ce8464 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -81,6 +81,12 @@
     required: true
     choices: [ 'present', 'absent' ]
     type: str
+  tags:
+    description:
+      - A hash/dictionary of tags to be applied to the job.
+      - Remove completely or specify an empty dictionary to remove all tags.
+    default: {}
+    type: dict
   timeout:
     description:
       - The job timeout in minutes.
@@ -112,7 +118,10 @@
 
 # Create an AWS Glue job
 - community.aws.aws_glue_job:
-    command_script_location: s3bucket/script.py
+    command_script_location: "s3://s3bucket/script.py"
+    default_arguments:
+      "--extra-py-files": s3://s3bucket/script-package.zip
+      "--TempDir": "s3://s3bucket/temp/"
     name: my-glue-job
     role: my-iam-role
     state: present
@@ -232,8 +241,28 @@
 
 from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
+from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
+
+
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+def _get_account_info(module):
+    """
+    Return the account information (account ID and partition) we are currently working on.
+    """
+    account_id = None
+    partition = None
+    sts_client = module.client('sts')
+    caller_identity = sts_client.get_caller_identity()
+    account_id = caller_identity.get('Account')
+    partition = caller_identity.get('Arn').split(':')[1]
+    return account_id, partition
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def _get_glue_job(connection, module, glue_job_name):
     """
     Get an AWS Glue job based on name. If not found, return None.
@@ -300,6 +329,44 @@ def _compare_glue_job_params(user_params, current_params):
     return False
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+def ensure_tags(connection, module, glue_job):
+    changed = False
+
+    account_id, partition = _get_account_info(module)
+    arn = 'arn:{0}:glue:{1}:{2}:job/{3}'.format(partition, module.region, account_id, module.params.get('name'))
+
+    try:
+        response = connection.get_tags(ResourceArn=arn)
+    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+        module.fail_json_aws(e, msg='Unable to get tags for Glue job %s' % module.params.get('name'))
+
+    if module.params.get('tags') is None:
+        return False
+
+    existing_tags = response.get('Tags')
+    tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, module.params.get('tags'), True)
+
+    if tags_to_remove:
+        changed = True
+        if not module.check_mode:
+            try:
+                connection.untag_resource(ResourceArn=arn, TagsToRemove=tags_to_remove)
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
+
+    if tags_to_add:
+        changed = True
+        if not module.check_mode:
+            try:
+                connection.tag_resource(ResourceArn=arn, TagsToAdd=tags_to_add)
+            except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
+                module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
+
+    return changed
+
+
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def create_or_update_glue_job(connection, module, glue_job):
     """
     Create or update an AWS Glue job
@@ -361,13 +428,14 @@ def create_or_update_glue_job(connection, module, glue_job):
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
 
-    # If changed, get the Glue job again
-    if changed:
-        glue_job = _get_glue_job(connection, module, params['Name'])
+    glue_job = _get_glue_job(connection, module, params['Name'])
+
+    changed |= ensure_tags(connection, module, glue_job)
 
     module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job))
 
 
+@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def delete_glue_job(connection, module, glue_job):
     """
     Delete an AWS Glue job
@@ -402,16 +470,16 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
-            glue_version=dict(type='str'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),
             role=dict(type='str'),
             state=dict(required=True, choices=['present', 'absent'], type='str'),
-            timeout=dict(type='int'),
             glue_version=dict(type='str'),
             worker_type=dict(choices=['Standard', 'G.1X', 'G.2X'], type='str'),
             number_of_workers=dict(type='int'),
+            tags=dict(type='dict', default={}),
+            timeout=dict(type='int')
         )
     )
 
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 677c8ce3bb3..b906539a0b4 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -32,11 +32,14 @@
     - name: Create Glue job (check mode)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       check_mode: true
       register: glue_job_check
@@ -50,11 +53,14 @@
     - name: Create Glue job
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       register: glue_job
 
@@ -75,20 +81,23 @@
       assert:
         that:
           - glue_job.changed"
-          - glue_job.description == job_info["Job"]["Description"]
-          - glue_job.role == job_info["Job"]["Role"]
-          - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
           - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
+          - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
+          - glue_job.description == job_info["Job"]["Description"]
           - glue_job.glue_version == job_info["Job"]["GlueVersion"]
+          - glue_job.role == job_info["Job"]["Role"]
 
     - name: Create Glue job (idempotent) (check mode)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       check_mode: true
       register: glue_job_idempotent_check
@@ -111,20 +120,23 @@
         that:
           - not glue_job_idempotent_check.changed
           - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
           - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
-          - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
 
     - name: Create Glue job (idempotent)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 3
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
+          Product: Glue
         state: present
       register: glue_job_idempotent
 
@@ -146,20 +158,22 @@
         that:
           - not glue_job_idempotent.changed
           - job_info["Job"]["Name"] == job_info_idempotent["Job"]["Name"]
+          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
           - job_info["Job"]["Description"] == job_info_idempotent["Job"]["Description"]
-          - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent["Job"]["GlueVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
 
     - name: Update Glue job (check mode)
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 2
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
         state: present
       check_mode: true
       register: glue_job_update_check
@@ -181,20 +195,22 @@
       assert:
         that:
           - glue_job_update_check.changed
-          - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
-          - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
-          - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
           - glue_job_update_check.command.python_version == "3"
+          - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
           - glue_job_update_check.glue_version == "2.0"
+          - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
 
     - name: Update Glue job
       aws_glue_job:
         name: "{{ glue_job_name }}"
-        description: "{{ glue_job_description }}"
-        command_script_location: "{{ glue_job_command_script_location }}"
         command_python_version: 2
+        command_script_location: "{{ glue_job_command_script_location }}"
+        description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
+        tags:
+          Environment: Test
         state: present
       register: glue_job_update
 
@@ -215,11 +231,11 @@
       assert:
         that:
           - glue_job_update.changed
-          - glue_job_update.description == job_info_update["Job"]["Description"]
-          - glue_job_update.role == job_info_update["Job"]["Role"]
-          - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
           - glue_job_update.command.python_version == job_info_update["Job"]["Command"]["PythonVersion"]
+          - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update.description == job_info_update["Job"]["Description"]
           - glue_job_update.glue_version == job_info_update["Job"]["Command"]["GlueVersion"]
+          - glue_job_update.role == job_info_update["Job"]["Role"]
 
     - name: Delete Glue job (check mode)
       aws_glue_job:

From 1053b7041ff07539d8bb7abca1d832a782a6b06f Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Mon, 22 Mar 2021 16:42:44 +0000
Subject: [PATCH 08/26] Properly detect changes in DefaultArguments parameter

Add tests for DefaultArguments.
---
 plugins/modules/aws_glue_job.py               |  4 +--
 .../targets/aws_glue_job/defaults/main.yml    |  3 ++-
 .../targets/aws_glue_job/tasks/main.yml       | 25 ++++++++++++++++---
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 11e38ce8464..24272210f98 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -305,9 +305,9 @@ def _compare_glue_job_params(user_params, current_params):
             return True
         if user_params['Command']['PythonVersion'] != current_params['Command']['PythonVersion']:
             return True
-    if 'Connections' in user_params and set(user_params['Connections']) != set(current_params['Connections']):
+    if 'Connections' in user_params and user_params['Connections'] != current_params['Connections']:
         return True
-    if 'DefaultArguments' in user_params and set(user_params['DefaultArguments']) != set(current_params['DefaultArguments']):
+    if 'DefaultArguments' in user_params and user_params['DefaultArguments'] != current_params['DefaultArguments']:
         return True
     if 'Description' in user_params and user_params['Description'] != current_params['Description']:
         return True
diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
index 51ed2fddc62..19c44066b20 100644
--- a/tests/integration/targets/aws_glue_job/defaults/main.yml
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -1,7 +1,8 @@
 ---
 glue_job_name: "{{ resource_prefix }}"
 glue_job_description: Test job
-glue_job_command_script_location: test-s3-bucket-glue/job.py
+glue_job_command_script_location: "s3://test-s3-bucket-glue/job.py"
+glue_job_temp_dir: "s3://test-s3-bucket-glue/temp/"
 # IAM role names have to be less than 64 characters
 # The 8 digit identifier at the end of resource_prefix helps determine during
 # which test something was created and allows tests to be run in parallel
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index b906539a0b4..3f6d8833c40 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -34,6 +34,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -55,6 +57,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -83,6 +87,7 @@
           - glue_job.changed"
           - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
           - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
+          - glue_job.default_arguments == job_info["Job"]["DefaultArguments"]
           - glue_job.description == job_info["Job"]["Description"]
           - glue_job.glue_version == job_info["Job"]["GlueVersion"]
           - glue_job.role == job_info["Job"]["Role"]
@@ -92,6 +97,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -122,6 +129,7 @@
           - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
           - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
           - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["DefaultArguments"] == job_info_idempotent_check["Job"]["DefaultArguments"]
           - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]
           - job_info["Job"]["Role"] == job_info_idempotent_check["Job"]["Role"]
@@ -131,6 +139,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 3
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}"
         description: "{{ glue_job_description }}"
         glue_version: "2.0"
         role: "{{ glue_job_role_name }}"
@@ -158,8 +168,9 @@
         that:
           - not glue_job_idempotent.changed
           - job_info["Job"]["Name"] == job_info_idempotent["Job"]["Name"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]["PythonVersion"] == job_info_idempotent["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]["ScriptLocation"] == job_info_idempotent["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["DefaultArguments"] == job_info_idempotent["Job"]["DefaultArguments"]
           - job_info["Job"]["Description"] == job_info_idempotent["Job"]["Description"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent["Job"]["GlueVersion"]
           - job_info["Job"]["Role"] == job_info_idempotent["Job"]["Role"]
@@ -169,6 +180,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 2
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}subfolder/"
         description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
@@ -195,10 +208,11 @@
       assert:
         that:
           - glue_job_update_check.changed
-          - glue_job_update_check.command.python_version == "3"
+          - glue_job_update_check.command.python_version == job_info_update_check["Job"]["Command"]["PythonVersion"]
           - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update_check.default_arguments == job_info_update_check["Job"]["DefaultArguments"]
           - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
-          - glue_job_update_check.glue_version == "2.0"
+          - glue_job_update_check.glue_version == job_info_update_check["Job"]["Command"]["GlueVersion"]
           - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
 
     - name: Update Glue job
@@ -206,6 +220,8 @@
         name: "{{ glue_job_name }}"
         command_python_version: 2
         command_script_location: "{{ glue_job_command_script_location }}"
+        default_arguments:
+          "--TempDir": "{{ glue_job_temp_dir }}subfolder/"
         description: "{{ glue_job_description }}"
         glue_version: "0.9"
         role: "{{ glue_job_role_name }}"
@@ -233,6 +249,7 @@
           - glue_job_update.changed
           - glue_job_update.command.python_version == job_info_update["Job"]["Command"]["PythonVersion"]
           - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
+          - glue_job_update.default_arguments == job_info_update["Job"]["DefaultArguments"]
           - glue_job_update.description == job_info_update["Job"]["Description"]
           - glue_job_update.glue_version == job_info_update["Job"]["Command"]["GlueVersion"]
           - glue_job_update.role == job_info_update["Job"]["Role"]

From e66d99a00405157eee5c699c43f82649f2cc7ed4 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 26 Mar 2021 19:50:20 +0000
Subject: [PATCH 09/26] Ensure that Role paramter is modified if necessary

Correctly handle response when no Glue job exists in check mode.
---
 plugins/modules/aws_glue_job.py                | 18 +++++++++++-------
 .../targets/aws_glue_job/tasks/main.yml        |  6 +++---
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 24272210f98..5f39eaab6f3 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -317,6 +317,8 @@ def _compare_glue_job_params(user_params, current_params):
         return True
     if 'MaxRetries' in user_params and user_params['MaxRetries'] != current_params['MaxRetries']:
         return True
+    if 'Role' in user_params and user_params['Role'] != current_params['Role']:
+        return True
     if 'Timeout' in user_params and user_params['Timeout'] != current_params['Timeout']:
         return True
     if 'GlueVersion' in user_params and user_params['GlueVersion'] != current_params['GlueVersion']:
@@ -333,18 +335,20 @@ def _compare_glue_job_params(user_params, current_params):
 def ensure_tags(connection, module, glue_job):
     changed = False
 
+    if module.params.get('tags') is None:
+        return False
+
     account_id, partition = _get_account_info(module)
     arn = 'arn:{0}:glue:{1}:{2}:job/{3}'.format(partition, module.region, account_id, module.params.get('name'))
 
     try:
-        response = connection.get_tags(ResourceArn=arn)
+        existing_tags = connection.get_tags(ResourceArn=arn).get('Tags', {})
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg='Unable to get tags for Glue job %s' % module.params.get('name'))
-
-    if module.params.get('tags') is None:
-        return False
+        if module.check_mode:
+            existing_tags = {}
+        else:
+            module.fail_json_aws(e, msg='Unable to get tags for Glue job %s' % module.params.get('name'))
 
-    existing_tags = response.get('Tags')
     tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, module.params.get('tags'), True)
 
     if tags_to_remove:
@@ -432,7 +436,7 @@ def create_or_update_glue_job(connection, module, glue_job):
 
     changed |= ensure_tags(connection, module, glue_job)
 
-    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job))
+    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}))
 
 
 @AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 3f6d8833c40..8e8db677090 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -84,7 +84,7 @@
     - name: Verity that Glue job was created
       assert:
         that:
-          - glue_job.changed"
+          - glue_job.changed
           - glue_job.command.python_version == job_info["Job"]["Command"]["PythonVersion"]
           - glue_job.command.script_location == job_info["Job"]["Command"]["ScriptLocation"]
           - glue_job.default_arguments == job_info["Job"]["DefaultArguments"]
@@ -127,8 +127,8 @@
         that:
           - not glue_job_idempotent_check.changed
           - job_info["Job"]["Name"] == job_info_idempotent_check["Job"]["Name"]
-          - job_info["Job"]["Command"]['PythonVersion'] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
-          - job_info["Job"]["Command"]['ScriptLocation'] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
+          - job_info["Job"]["Command"]["PythonVersion"] == job_info_idempotent_check["Job"]["Command"]["PythonVersion"]
+          - job_info["Job"]["Command"]["ScriptLocation"] == job_info_idempotent_check["Job"]["Command"]["ScriptLocation"]
           - job_info["Job"]["DefaultArguments"] == job_info_idempotent_check["Job"]["DefaultArguments"]
           - job_info["Job"]["Description"] == job_info_idempotent_check["Job"]["Description"]
           - job_info["Job"]["GlueVersion"] == job_info_idempotent_check["Job"]["GlueVersion"]

From 2ddbe8b5c0e13fc3564b7f3133c262ac8fb64d8c Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sat, 27 Mar 2021 21:51:08 +0000
Subject: [PATCH 10/26] Use retries with jittered backoff

Reference:  https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/

Remove duplicate import.
---
 plugins/modules/aws_glue_job.py | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 5f39eaab6f3..1c623832224 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -32,6 +32,7 @@
     description:
       - Python version being used to execute a Python shell job. Allowed values are '2' or '3'.
     type: str
+    version_added: 1.5.0
   command_script_location:
     description:
       - The S3 path to a script that executes a job.
@@ -56,6 +57,7 @@
       - AWS Glue version. This determines the available version of Apache Scala and Python as described here
         U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
     type: str
+    version_added: 1.5.0
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when
@@ -87,6 +89,7 @@
       - Remove completely or specify an empty dictionary to remove all tags.
     default: {}
     type: dict
+    version_added: 1.5.0
   timeout:
     description:
       - The job timeout in minutes.
@@ -243,12 +246,10 @@
 from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def _get_account_info(module):
     """
     Return the account information (account ID and partition) we are currently working on.
@@ -262,7 +263,6 @@ def _get_account_info(module):
     return account_id, partition
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def _get_glue_job(connection, module, glue_job_name):
     """
     Get an AWS Glue job based on name. If not found, return None.
@@ -272,9 +272,8 @@ def _get_glue_job(connection, module, glue_job_name):
     :param glue_job_name: Name of Glue job to get
     :return: boto3 Glue job dict or None if not found
     """
-
     try:
-        return connection.get_job(JobName=glue_job_name)['Job']
+        return connection.get_job(aws_retry=True, JobName=glue_job_name)['Job']
     except is_boto3_error_code('EntityNotFoundException'):
         return None
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:  # pylint: disable=duplicate-except
@@ -289,7 +288,6 @@ def _compare_glue_job_params(user_params, current_params):
     :param current_params: the Glue job parameters currently configured
     :return: True if any parameter is mismatched else False
     """
-
     # Weirdly, boto3 doesn't return some keys if the value is empty e.g. Description
     # To counter this, add the key if it's missing with a blank value
 
@@ -331,7 +329,6 @@ def _compare_glue_job_params(user_params, current_params):
     return False
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def ensure_tags(connection, module, glue_job):
     changed = False
 
@@ -342,7 +339,7 @@ def ensure_tags(connection, module, glue_job):
     arn = 'arn:{0}:glue:{1}:{2}:job/{3}'.format(partition, module.region, account_id, module.params.get('name'))
 
     try:
-        existing_tags = connection.get_tags(ResourceArn=arn).get('Tags', {})
+        existing_tags = connection.get_tags(aws_retry=True, ResourceArn=arn).get('Tags', {})
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         if module.check_mode:
             existing_tags = {}
@@ -355,7 +352,7 @@ def ensure_tags(connection, module, glue_job):
         changed = True
         if not module.check_mode:
             try:
-                connection.untag_resource(ResourceArn=arn, TagsToRemove=tags_to_remove)
+                connection.untag_resource(aws_retry=True, ResourceArn=arn, TagsToRemove=tags_to_remove)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
 
@@ -363,14 +360,13 @@ def ensure_tags(connection, module, glue_job):
         changed = True
         if not module.check_mode:
             try:
-                connection.tag_resource(ResourceArn=arn, TagsToAdd=tags_to_add)
+                connection.tag_resource(aws_retry=True, ResourceArn=arn, TagsToAdd=tags_to_add)
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e, msg='Unable to set tags for Glue job %s' % module.params.get('name'))
 
     return changed
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def create_or_update_glue_job(connection, module, glue_job):
     """
     Create or update an AWS Glue job
@@ -420,14 +416,14 @@ def create_or_update_glue_job(connection, module, glue_job):
                 update_params = {'JobName': params['Name'], 'JobUpdate': copy.deepcopy(params)}
                 del update_params['JobUpdate']['Name']
                 if not module.check_mode:
-                    connection.update_job(**update_params)
+                    connection.update_job(aws_retry=True, **update_params)
                 changed = True
             except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
                 module.fail_json_aws(e)
     else:
         try:
             if not module.check_mode:
-                connection.create_job(**params)
+                connection.create_job(aws_retry=True, **params)
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -439,7 +435,6 @@ def create_or_update_glue_job(connection, module, glue_job):
     module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}))
 
 
-@AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
 def delete_glue_job(connection, module, glue_job):
     """
     Delete an AWS Glue job
@@ -449,13 +444,12 @@ def delete_glue_job(connection, module, glue_job):
     :param glue_job: a dict of AWS Glue job parameters or None
     :return:
     """
-
     changed = False
 
     if glue_job:
         try:
             if not module.check_mode:
-                connection.delete_job(JobName=glue_job['Name'])
+                connection.delete_job(aws_retry=True, JobName=glue_job['Name'])
             changed = True
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
             module.fail_json_aws(e)
@@ -494,7 +488,8 @@ def main():
                               supports_check_mode=True
                               )
 
-    connection = module.client('glue')
+    retry_decorator = AWSRetry.jittered_backoff(retries=10)
+    connection = module.client('glue', retry_decorator=retry_decorator)
 
     state = module.params.get("state")
 

From 31867d4859aa579509d7cba7b20464df8483986b Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 2 Apr 2021 00:47:11 +0000
Subject: [PATCH 11/26] Do not convert keys in DefaultArguments dict to
 snake_case

---
 plugins/modules/aws_glue_job.py                       | 2 +-
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 1c623832224..e8b5563dee3 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -432,7 +432,7 @@ def create_or_update_glue_job(connection, module, glue_job):
 
     changed |= ensure_tags(connection, module, glue_job)
 
-    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}))
+    module.exit_json(changed=changed, **camel_dict_to_snake_dict(glue_job or {}, ignore_list=['DefaultArguments']))
 
 
 def delete_glue_job(connection, module, glue_job):
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 8e8db677090..4daf665730e 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -50,7 +50,7 @@
       assert:
         that:
           - glue_job_check.changed
-          - not glue_job_check.description
+          - "description" not in glue_job_check
 
     - name: Create Glue job
       aws_glue_job:

From d3c29026ee3cb62c208282b439f35e83252d19fd Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 2 Apr 2021 01:07:49 +0000
Subject: [PATCH 12/26] Fix test syntax. Remove duplicate documentation key

---
 plugins/modules/aws_glue_job.py                       | 6 ------
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index e8b5563dee3..6463138ffeb 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -52,12 +52,6 @@
     description:
       - Description of the job being defined.
     type: str
-  glue_version:
-    description:
-      - AWS Glue version. This determines the available version of Apache Scala and Python as described here
-        U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
-    type: str
-    version_added: 1.5.0
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when
diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index 4daf665730e..cc61c870eb8 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -50,7 +50,7 @@
       assert:
         that:
           - glue_job_check.changed
-          - "description" not in glue_job_check
+          - glue_job_check.description is not defined
 
     - name: Create Glue job
       aws_glue_job:

From 2962a2d73dddf7ccab9f01e9524efa042d510d28 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 02:26:16 +0000
Subject: [PATCH 13/26] Add Python and Glue version parameters, add check mode

Available Python and Glue version can be found here:
https://docs.aws.amazon.com/glue/latest/dg/add-job.html

Example:

```
community.aws.aws_glue__jobs:
  - name: my-job
    description: My test job
    command_script_location: my-s3-bucket/script.py
    command_python_version: 3
    glue_version: "2.0"
    role: MyGlueJobRole
    state: present
```
---
 plugins/modules/aws_glue_job.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 6463138ffeb..287093467d2 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -52,6 +52,12 @@
     description:
       - Description of the job being defined.
     type: str
+  glue_version:
+    description:
+      - AWS Glue version. This determines the available version of Apache Scala and Python as described here
+        U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
+    default: 0.9
+    type: str
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when
@@ -462,6 +468,7 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
+            glue_version=dict(type='str', default='0.9'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),

From c9f8b409c0850ec17ff5793368f91aa59017035c Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 13:51:24 +0000
Subject: [PATCH 14/26] Remove default values for command_python_version and
 glue_version

This is to avoid inadvertently breaking existing playbooks. Defaults
will be set by AWS.
---
 plugins/modules/aws_glue_job.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 287093467d2..b4a9091958c 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -56,7 +56,6 @@
     description:
       - AWS Glue version. This determines the available version of Apache Scala and Python as described here
         U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
-    default: 0.9
     type: str
   max_concurrent_runs:
     description:
@@ -468,7 +467,7 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
-            glue_version=dict(type='str', default='0.9'),
+            glue_version=dict(type='str'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),

From fdc258ed3692a30bb86838ac72dab523fe4c64c7 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sun, 21 Mar 2021 23:31:23 +0000
Subject: [PATCH 15/26] Add support for tags

---
 plugins/modules/aws_glue_job.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index b4a9091958c..287093467d2 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -56,6 +56,7 @@
     description:
       - AWS Glue version. This determines the available version of Apache Scala and Python as described here
         U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
+    default: 0.9
     type: str
   max_concurrent_runs:
     description:
@@ -467,7 +468,7 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
-            glue_version=dict(type='str'),
+            glue_version=dict(type='str', default='0.9'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),

From 16a295ad9b4183b1a29067a375a14a98da7e96eb Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sat, 27 Mar 2021 21:51:08 +0000
Subject: [PATCH 16/26] Use retries with jittered backoff

Reference:  https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/

Remove duplicate import.
---
 plugins/modules/aws_glue_job.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 287093467d2..9f51453874d 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -58,6 +58,7 @@
         U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
     default: 0.9
     type: str
+    version_added: 1.5.0
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when

From d264c94e13c9edc24e6a2cbbbe9effc5ac98bcd2 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 2 Apr 2021 00:56:56 +0000
Subject: [PATCH 17/26] Remove duplicate glue_version argument in module
 definition

---
 plugins/modules/aws_glue_job.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 9f51453874d..bfca0bda0d3 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -469,7 +469,6 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
-            glue_version=dict(type='str', default='0.9'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),

From 6952d6e2df8a2a3990a2a223d2d66b09a82aa6d8 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Tue, 16 Mar 2021 14:48:49 +0000
Subject: [PATCH 18/26] Update IAM role name for Glue job to make sure it's
 shorter than 64 characters

---
 tests/integration/targets/aws_glue_job/defaults/main.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
index 19c44066b20..ef24ebaa6ea 100644
--- a/tests/integration/targets/aws_glue_job/defaults/main.yml
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -1,8 +1,12 @@
 ---
 glue_job_name: "{{ resource_prefix }}"
 glue_job_description: Test job
+<<<<<<< HEAD
 glue_job_command_script_location: "s3://test-s3-bucket-glue/job.py"
 glue_job_temp_dir: "s3://test-s3-bucket-glue/temp/"
+=======
+glue_job_command_script_location: test-s3-bucket-glue/job.py
+>>>>>>> Update IAM role name for Glue job to make sure it's shorter than 64 characters
 # IAM role names have to be less than 64 characters
 # The 8 digit identifier at the end of resource_prefix helps determine during
 # which test something was created and allows tests to be run in parallel

From a1be7d87db82d19289bfebb4fd4a648202d4c4c7 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Mon, 22 Mar 2021 16:42:44 +0000
Subject: [PATCH 19/26] Properly detect changes in DefaultArguments parameter

Add tests for DefaultArguments.
---
 tests/integration/targets/aws_glue_job/defaults/main.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/integration/targets/aws_glue_job/defaults/main.yml b/tests/integration/targets/aws_glue_job/defaults/main.yml
index ef24ebaa6ea..19c44066b20 100644
--- a/tests/integration/targets/aws_glue_job/defaults/main.yml
+++ b/tests/integration/targets/aws_glue_job/defaults/main.yml
@@ -1,12 +1,8 @@
 ---
 glue_job_name: "{{ resource_prefix }}"
 glue_job_description: Test job
-<<<<<<< HEAD
 glue_job_command_script_location: "s3://test-s3-bucket-glue/job.py"
 glue_job_temp_dir: "s3://test-s3-bucket-glue/temp/"
-=======
-glue_job_command_script_location: test-s3-bucket-glue/job.py
->>>>>>> Update IAM role name for Glue job to make sure it's shorter than 64 characters
 # IAM role names have to be less than 64 characters
 # The 8 digit identifier at the end of resource_prefix helps determine during
 # which test something was created and allows tests to be run in parallel

From 55c53fb6a491a651208cdd02ae3c2301e43d6285 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 2 Apr 2021 02:07:22 +0000
Subject: [PATCH 20/26] Remove duplicate key in documentation

---
 plugins/modules/aws_glue_job.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index bfca0bda0d3..6463138ffeb 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -52,13 +52,6 @@
     description:
       - Description of the job being defined.
     type: str
-  glue_version:
-    description:
-      - AWS Glue version. This determines the available version of Apache Scala and Python as described here
-        U(https://docs.aws.amazon.com/glue/latest/dg/add-job.html).
-    default: 0.9
-    type: str
-    version_added: 1.5.0
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when

From f88dc4cf9b239b755f51da41510dab595fc434c0 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Thu, 8 Apr 2021 23:44:42 +0000
Subject: [PATCH 21/26] Update JSON path to fix tests

---
 tests/integration/targets/aws_glue_job/tasks/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index cc61c870eb8..b41325a2b99 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -212,7 +212,7 @@
           - glue_job_update_check.command.script_location == job_info_update_check["Job"]["Command"]["ScriptLocation"]
           - glue_job_update_check.default_arguments == job_info_update_check["Job"]["DefaultArguments"]
           - glue_job_update_check.description == job_info_update_check["Job"]["Description"]
-          - glue_job_update_check.glue_version == job_info_update_check["Job"]["Command"]["GlueVersion"]
+          - glue_job_update_check.glue_version == job_info_update_check["Job"]["GlueVersion"]
           - glue_job_update_check.role == job_info_update_check["Job"]["Role"]
 
     - name: Update Glue job
@@ -251,7 +251,7 @@
           - glue_job_update.command.script_location == job_info_update["Job"]["Command"]["ScriptLocation"]
           - glue_job_update.default_arguments == job_info_update["Job"]["DefaultArguments"]
           - glue_job_update.description == job_info_update["Job"]["Description"]
-          - glue_job_update.glue_version == job_info_update["Job"]["Command"]["GlueVersion"]
+          - glue_job_update.glue_version == job_info_update["Job"]["GlueVersion"]
           - glue_job_update.role == job_info_update["Job"]["Role"]
 
     - name: Delete Glue job (check mode)

From 6318759a71d9ee829a00099793b197b2e1b96c75 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Fri, 16 Apr 2021 01:45:02 +0000
Subject: [PATCH 22/26] Minor edit to fix cleanup

---
 tests/integration/targets/aws_glue_job/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/targets/aws_glue_job/tasks/main.yml b/tests/integration/targets/aws_glue_job/tasks/main.yml
index b41325a2b99..307a9befb23 100644
--- a/tests/integration/targets/aws_glue_job/tasks/main.yml
+++ b/tests/integration/targets/aws_glue_job/tasks/main.yml
@@ -293,7 +293,7 @@
 
   always:
     - name: Delete Glue job
-      aws_glue_connection:
+      aws_glue_job:
         name: "{{ glue_job_name }}"
         state: absent
       ignore_errors: true

From e3a80fd15358b7480324d1d0642a263fd566a93c Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sun, 25 Apr 2021 16:23:08 +0000
Subject: [PATCH 23/26] Add `purge_tags` parameter

Use `get_aws_account_info` method to determine AWS partition and accunt ID
instead of reinventing the wheel.

Rearrange module attributes alphabetically.
---
 plugins/modules/aws_glue_job.py | 58 +++++++++++++++------------------
 1 file changed, 27 insertions(+), 31 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 6463138ffeb..8f1e2e12c0a 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -30,7 +30,8 @@
     type: str
   command_python_version:
     description:
-      - Python version being used to execute a Python shell job. Allowed values are '2' or '3'.
+      - Python version being used to execute a Python shell job.
+      - AWS currently supports C('2') or C('3').
     type: str
     version_added: 1.5.0
   command_script_location:
@@ -52,6 +53,11 @@
     description:
       - Description of the job being defined.
     type: str
+  glue_version:
+    description:
+      - Glue version determines the versions of Apache Spark and Python that AWS Glue supports.
+    type: str
+    version_added: 1.5.0
   max_concurrent_runs:
     description:
       - The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when
@@ -66,6 +72,18 @@
       - The name you assign to this job definition. It must be unique in your account.
     required: true
     type: str
+  number_of_workers:
+    description:
+      - The number of workers of a defined workerType that are allocated when a job runs.
+    type: int
+    version_added: 1.5.0
+  purge_tags:
+    description:
+      - If C(true), existing tags will be purged from the resource to match exactly what is defined by I(tags) parameter.
+      - If the I(tags) parameter is not set then tags will not be modified.
+    default: true
+    type: bool
+    version_added: 1.5.0
   role:
     description:
       - The name or ARN of the IAM role associated with this job.
@@ -81,29 +99,18 @@
     description:
       - A hash/dictionary of tags to be applied to the job.
       - Remove completely or specify an empty dictionary to remove all tags.
-    default: {}
     type: dict
     version_added: 1.5.0
   timeout:
     description:
       - The job timeout in minutes.
     type: int
-  glue_version:
-    description:
-      - Glue version determines the versions of Apache Spark and Python that AWS Glue supports.
-    type: str
-    version_added: 1.5.0
   worker_type:
     description:
       - The type of predefined worker that is allocated when a job runs.
     choices: [ 'Standard', 'G.1X', 'G.2X' ]
     type: str
     version_added: 1.5.0
-  number_of_workers:
-    description:
-      - The number of workers of a defined workerType that are allocated when a job runs.
-    type: int
-    version_added: 1.5.0
 extends_documentation_fragment:
 - amazon.aws.aws
 - amazon.aws.ec2
@@ -242,19 +249,7 @@
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict
 from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_aws_tags
-
-
-def _get_account_info(module):
-    """
-    Return the account information (account ID and partition) we are currently working on.
-    """
-    account_id = None
-    partition = None
-    sts_client = module.client('sts')
-    caller_identity = sts_client.get_caller_identity()
-    account_id = caller_identity.get('Account')
-    partition = caller_identity.get('Arn').split(':')[1]
-    return account_id, partition
+from ansible_collections.amazon.aws.plugins.module_utils.iam import get_aws_account_info
 
 
 def _get_glue_job(connection, module, glue_job_name):
@@ -329,7 +324,7 @@ def ensure_tags(connection, module, glue_job):
     if module.params.get('tags') is None:
         return False
 
-    account_id, partition = _get_account_info(module)
+    account_id, partition = get_aws_account_info(module)
     arn = 'arn:{0}:glue:{1}:{2}:job/{3}'.format(partition, module.region, account_id, module.params.get('name'))
 
     try:
@@ -340,7 +335,7 @@ def ensure_tags(connection, module, glue_job):
         else:
             module.fail_json_aws(e, msg='Unable to get tags for Glue job %s' % module.params.get('name'))
 
-    tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, module.params.get('tags'), True)
+    tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, module.params.get('tags'), module.params.get('purge_tags'))
 
     if tags_to_remove:
         changed = True
@@ -462,16 +457,17 @@ def main():
             connections=dict(type='list', elements='str'),
             default_arguments=dict(type='dict'),
             description=dict(type='str'),
+            glue_version=dict(type='str'),
             max_concurrent_runs=dict(type='int'),
             max_retries=dict(type='int'),
             name=dict(required=True, type='str'),
+            number_of_workers=dict(type='int'),
+            purge_tags=dict(type='bool', default=True),
             role=dict(type='str'),
             state=dict(required=True, choices=['present', 'absent'], type='str'),
-            glue_version=dict(type='str'),
-            worker_type=dict(choices=['Standard', 'G.1X', 'G.2X'], type='str'),
-            number_of_workers=dict(type='int'),
             tags=dict(type='dict', default={}),
-            timeout=dict(type='int')
+            timeout=dict(type='int'),
+            worker_type=dict(choices=['Standard', 'G.1X', 'G.2X'], type='str'),
         )
     )
 

From 551a36fd3da51ad1a434b67e3d0e2ece62566806 Mon Sep 17 00:00:00 2001
From: ichekaldin <39010411+ichekaldin@users.noreply.github.com>
Date: Sun, 25 Apr 2021 18:06:15 +0000
Subject: [PATCH 24/26] Do not set defaults for tags

---
 plugins/modules/aws_glue_job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 8f1e2e12c0a..92f5e3a2761 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -465,7 +465,7 @@ def main():
             purge_tags=dict(type='bool', default=True),
             role=dict(type='str'),
             state=dict(required=True, choices=['present', 'absent'], type='str'),
-            tags=dict(type='dict', default={}),
+            tags=dict(type='dict'),
             timeout=dict(type='int'),
             worker_type=dict(choices=['Standard', 'G.1X', 'G.2X'], type='str'),
         )

From dbf539303e8b0e02a2d7db2b22cc5ea859e7bfdd Mon Sep 17 00:00:00 2001
From: Mark Chappell <mchappel@redhat.com>
Date: Fri, 22 Oct 2021 22:49:04 +0200
Subject: [PATCH 25/26] Update version_added

---
 plugins/modules/aws_glue_job.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 92f5e3a2761..17e6f276c01 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -33,7 +33,7 @@
       - Python version being used to execute a Python shell job.
       - AWS currently supports C('2') or C('3').
     type: str
-    version_added: 1.5.0
+    version_added: 2.1.0
   command_script_location:
     description:
       - The S3 path to a script that executes a job.
@@ -83,7 +83,7 @@
       - If the I(tags) parameter is not set then tags will not be modified.
     default: true
     type: bool
-    version_added: 1.5.0
+    version_added: 2.1.0
   role:
     description:
       - The name or ARN of the IAM role associated with this job.
@@ -100,7 +100,7 @@
       - A hash/dictionary of tags to be applied to the job.
       - Remove completely or specify an empty dictionary to remove all tags.
     type: dict
-    version_added: 1.5.0
+    version_added: 2.1.0
   timeout:
     description:
       - The job timeout in minutes.

From 4081f534bf7a7bbb8c69f15f4aa77c202bf684af Mon Sep 17 00:00:00 2001
From: Mark Chappell <mchappel@redhat.com>
Date: Thu, 25 Nov 2021 14:14:17 +0100
Subject: [PATCH 26/26] Bump version_added

---
 plugins/modules/aws_glue_job.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/modules/aws_glue_job.py b/plugins/modules/aws_glue_job.py
index 17e6f276c01..edca5d051d5 100644
--- a/plugins/modules/aws_glue_job.py
+++ b/plugins/modules/aws_glue_job.py
@@ -33,7 +33,7 @@
       - Python version being used to execute a Python shell job.
       - AWS currently supports C('2') or C('3').
     type: str
-    version_added: 2.1.0
+    version_added: 2.2.0
   command_script_location:
     description:
       - The S3 path to a script that executes a job.
@@ -83,7 +83,7 @@
       - If the I(tags) parameter is not set then tags will not be modified.
     default: true
     type: bool
-    version_added: 2.1.0
+    version_added: 2.2.0
   role:
     description:
       - The name or ARN of the IAM role associated with this job.
@@ -100,7 +100,7 @@
       - A hash/dictionary of tags to be applied to the job.
       - Remove completely or specify an empty dictionary to remove all tags.
     type: dict
-    version_added: 2.1.0
+    version_added: 2.2.0
   timeout:
     description:
       - The job timeout in minutes.