feat: added dataproc workflows samples

dataproc/instantiate_inline_workflow_template.py

@@ -0,0 +1,98 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This sample walks a user through instantiating an inline
+# workflow for Cloud Dataproc using the Python client library.
+#
+# This script can be run on its own:
+#   python workflows.py ${PROJECT_ID} ${REGION}
+
+import sys
+# [START dataproc_instantiate_inline_workflow_template]
+from google.cloud import dataproc_v1 as dataproc
+
+
+def instantiate_inline_workflow_template(project_id, region):
+    """This sample walks a user through submitting a workflow
+       for a Cloud Dataproc using the Python client library.
+
+       Args:
+           project_id (string): Project to use for running the workflow.
+           region (string): Region where the workflow resources should live.
+    """
+
+    # Create a client with the endpoint set to the desired region.
+    workflow_template_client = dataproc.WorkflowTemplateServiceClient(
+        client_options={
+            'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region)}
+    )
+
+    parent = workflow_template_client.region_path(project_id, region)
+
+    template = {
+        'jobs': [
+            {
+                'hadoop_job': {
+                    'main_jar_file_uri': 'file:///usr/lib/hadoop-mapreduce/'
+                    'hadoop-mapreduce-examples.jar',
+                    'args': [
+                        'teragen',
+                        '1000',
+                        'hdfs:///gen/'
+                    ]
+                },
+                'step_id': 'teragen'
+            },
+            {
+                'hadoop_job': {
+                    'main_jar_file_uri': 'file:///usr/lib/hadoop-mapreduce/'
+                    'hadoop-mapreduce-examples.jar',
+                    'args': [
+                        'terasort',
+                        'hdfs:///gen/',
+                        'hdfs:///sort/'
+                    ]
+                },
+                'step_id': 'terasort',
+                'prerequisite_step_ids': [
+                    'teragen'
+                ]
+            }],
+        'placement': {
+            'managed_cluster': {
+                'cluster_name': 'my-managed-cluster',
+                'config': {
+                    'gce_cluster_config': {
+                        # Leave 'zone_uri' empty for 'Auto Zone Placement'
+                        # 'zone_uri': ''
+                        'zone_uri': 'us-central1-a'
+                    }
+                }
+            }
+        }
+    }
+
+    # Submit the request to instantiate the workflow from an inline template.
+    operation = workflow_template_client.instantiate_inline_workflow_template(
+        parent, template
+    )
+    operation.result()
+
+    # Output a success message.
+    print('Workflow ran successfully.')
+# [END dataproc_instantiate_inline_workflow_template]
+
+
+if __name__ == "__main__":
+    instantiate_inline_workflow_template(sys.argv[1], sys.argv[2])

dataproc/instantiate_inline_workflow_template_test.py

@@ -0,0 +1,31 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import instantiate_inline_workflow_template
+
+
+PROJECT_ID = os.environ['GCLOUD_PROJECT']
+REGION = 'us-central1'
+
+
+def test_workflows(capsys):
+    # Wrapper function for client library function
+    instantiate_inline_workflow_template.instantiate_inline_workflow_template(
+        PROJECT_ID, REGION
+    )
+
+    out, _ = capsys.readouterr()
+    assert "successfully" in out