From e57d25c32f6a39ecaa3ab03378833345ead2da8a Mon Sep 17 00:00:00 2001 From: Tarun Goyal Date: Tue, 10 Dec 2019 15:23:43 +0530 Subject: [PATCH] SDK-361: Support for cluster image version clusters API (#291) * SDK changes for adding cluster image version support --- qds_sdk/cluster_info_v22.py | 21 +++++++++++++++++++-- qds_sdk/clusterv2.py | 14 ++++++++++++-- qds_sdk/engine.py | 2 +- tests/test_clusterv2.py | 18 ++++++++++++++++++ tests/test_clusterv22.py | 16 ++++++++++++++++ 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py index 1af9cdf2..32aa52d8 100644 --- a/qds_sdk/cluster_info_v22.py +++ b/qds_sdk/cluster_info_v22.py @@ -53,7 +53,9 @@ def set_cluster_info_from_arguments(self, arguments): disable_cluster_pause=arguments.disable_cluster_pause, paused_cluster_timeout_mins=arguments.paused_cluster_timeout_mins, disable_autoscale_node_pause=arguments.disable_autoscale_node_pause, - paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins) + paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins, + parent_cluster_id=arguments.parent_cluster_id, + image_version=arguments.image_version) self.set_composition(master_type=arguments.master_type, master_spot_block_duration=arguments.master_spot_block_duration, @@ -106,7 +108,9 @@ def set_cluster_info(self, disable_cluster_pause=None, paused_cluster_timeout_mins=None, disable_autoscale_node_pause=None, - paused_autoscale_node_timeout_mins=None): + paused_autoscale_node_timeout_mins=None, + parent_cluster_id=None, + image_version=None): """ Args: @@ -180,6 +184,10 @@ def set_cluster_info(self, `paused_autoscale_node_timeout_mins`: Paused autoscale node timeout in mins + `parent_cluster_id`: parent cluster id for HS2 cluster + + `image_version`: cluster image version + Doc: For getting details about arguments http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#parameters @@ -206,6 +214,8 @@ def set_cluster_info(self, self.cluster_info['idle_cluster_timeout'] = idle_cluster_timeout self.cluster_info['rootdisk'] = {} self.cluster_info['rootdisk']['size'] = root_disk_size + self.cluster_info['parent_cluster_id'] = parent_cluster_id + self.cluster_info['cluster_image_version'] = image_version self.set_data_disk(disk_size, disk_count, disk_type, upscaling_config, enable_encryption) self.set_monitoring(enable_ganglia_monitoring, @@ -498,6 +508,13 @@ def cluster_info_parser(argparser, action): dest="root_disk_size", type=int, help="size of the root volume in GB") + cluster_info.add_argument("--parent-cluster-id", + dest="parent_cluster_id", + type=int, + help="Id of the parent cluster this hs2 cluster is attached to") + cluster_info.add_argument("--image-version", + dest="image_version", + help="cluster image version") termination = cluster_info.add_mutually_exclusive_group() termination.add_argument("--disallow-cluster-termination", dest="disallow_cluster_termination", diff --git a/qds_sdk/clusterv2.py b/qds_sdk/clusterv2.py index e0e4cb0c..6e20ef20 100755 --- a/qds_sdk/clusterv2.py +++ b/qds_sdk/clusterv2.py @@ -182,7 +182,8 @@ def set_cluster_info_from_arguments(self, arguments): paused_cluster_timeout_mins=arguments.paused_cluster_timeout_mins, disable_autoscale_node_pause=arguments.disable_autoscale_node_pause, paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins, - parent_cluster_id=arguments.parent_cluster_id) + parent_cluster_id=arguments.parent_cluster_id, + image_version=arguments.image_version) def set_cluster_info(self, disallow_cluster_termination=None, @@ -225,7 +226,8 @@ def set_cluster_info(self, paused_cluster_timeout_mins=None, disable_autoscale_node_pause=None, paused_autoscale_node_timeout_mins=None, - parent_cluster_id=None): + parent_cluster_id=None, + image_version=None): """ Args: @@ -330,6 +332,10 @@ def set_cluster_info(self, `paused_autoscale_node_timeout_mins`: Paused autoscale node timeout in mins + `parent_cluster_id`: parent cluster id for HS2 cluster + + `image_version`: cluster image version + Doc: For getting details about arguments http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#parameters @@ -360,6 +366,7 @@ def set_cluster_info(self, self.cluster_info['rootdisk'] = {} self.cluster_info['rootdisk']['size'] = root_disk_size self.cluster_info['parent_cluster_id'] = parent_cluster_id + self.cluster_info['cluster_image_version'] = image_version self.set_spot_instance_settings(maximum_bid_price_percentage, timeout_for_request, maximum_spot_instance_percentage) @@ -522,6 +529,9 @@ def cluster_info_parser(argparser, action): dest="parent_cluster_id", type=int, help="Id of the parent cluster this hs2 cluster is attached to") + cluster_info.add_argument("--image-version", + dest="image_version", + help="cluster image version") termination = cluster_info.add_mutually_exclusive_group() termination.add_argument("--disallow-cluster-termination", dest="disallow_cluster_termination", diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index 22893df8..24a8289b 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -147,7 +147,7 @@ def engine_parser(argparser): engine_group = argparser.add_argument_group("engine settings") engine_group.add_argument("--flavour", dest="flavour", - choices=["hadoop", "hadoop2", "hs2", "presto", "spark", "sparkstreaming", "hbase", "airflow", "deeplearning"], + choices=["hadoop", "hadoop2", "hs2", "hive", "presto", "spark", "sparkstreaming", "hbase", "airflow", "deeplearning"], default=None, help="Set engine flavour") diff --git a/tests/test_clusterv2.py b/tests/test_clusterv2.py index 9acc5dc5..0c149e50 100644 --- a/tests/test_clusterv2.py +++ b/tests/test_clusterv2.py @@ -572,6 +572,24 @@ def test_image_override(self): 'internal':{'image_uri_overrides': 'test/image1'} }) + def test_image_version_v2(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', + 'test_label', '--flavour', 'hadoop2', '--slave-instance-type', 'c1.xlarge', '--min-nodes', '3', '--image-version', '1.latest'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', + {'engine_config': + {'flavour': 'hadoop2'}, + 'cluster_info': {'label': ['test_label'], + 'min_nodes': 3, + 'slave_instance_type': 'c1.xlarge', + 'cluster_image_version': '1.latest'}}) + + + + def test_spot_block_duration_v2(self): sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label', '--spot-block-duration', '120'] diff --git a/tests/test_clusterv22.py b/tests/test_clusterv22.py index e05128dd..5b1e4ee7 100644 --- a/tests/test_clusterv22.py +++ b/tests/test_clusterv22.py @@ -195,3 +195,19 @@ def test_spot_spot_spot(self): 'maximum_bid_price_percentage': 50}]}, 'autoscaling_nodes': {'nodes': [ {'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}}) + + def test_image_version_v22(self): + sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', + 'test_label', '--flavour', 'hive', '--slave-instance-type', 'c1.xlarge', '--min-nodes', '3', '--image-version', '1.latest'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', + {'engine_config': + {'flavour': 'hive'}, + 'cluster_info': {'label': ['test_label'], + 'min_nodes': 3, + 'slave_instance_type': 'c1.xlarge', + 'cluster_image_version': '1.latest', + 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 1, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', 'maximum_bid_price_percentage': 100}]}}, 'label': ['test_label']}})