Skip to content

Commit

Permalink
SDK-361: Support for cluster image version clusters API (#291)
Browse files Browse the repository at this point in the history 
* SDK changes for adding cluster image version support
  • Loading branch information
@tarrygo @chattarajoy
tarrygo authored and chattarajoy committed Dec 10, 2019
1 parent 99544a9 commit e57d25c
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 5 deletions.
21 changes: 19 additions & 2 deletions qds_sdk/cluster_info_v22.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ def set_cluster_info_from_arguments(self, arguments):
disable_cluster_pause=arguments.disable_cluster_pause,
paused_cluster_timeout_mins=arguments.paused_cluster_timeout_mins,
disable_autoscale_node_pause=arguments.disable_autoscale_node_pause,
paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins)
paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins,
parent_cluster_id=arguments.parent_cluster_id,
image_version=arguments.image_version)

self.set_composition(master_type=arguments.master_type,
master_spot_block_duration=arguments.master_spot_block_duration,
Expand Down Expand Up @@ -106,7 +108,9 @@ def set_cluster_info(self,
disable_cluster_pause=None,
paused_cluster_timeout_mins=None,
disable_autoscale_node_pause=None,
paused_autoscale_node_timeout_mins=None):
paused_autoscale_node_timeout_mins=None,
parent_cluster_id=None,
image_version=None):
"""
Args:
Expand Down Expand Up @@ -180,6 +184,10 @@ def set_cluster_info(self,
`paused_autoscale_node_timeout_mins`: Paused autoscale node timeout in mins
`parent_cluster_id`: parent cluster id for HS2 cluster
`image_version`: cluster image version
Doc: For getting details about arguments
http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#parameters
Expand All @@ -206,6 +214,8 @@ def set_cluster_info(self,
self.cluster_info['idle_cluster_timeout'] = idle_cluster_timeout
self.cluster_info['rootdisk'] = {}
self.cluster_info['rootdisk']['size'] = root_disk_size
self.cluster_info['parent_cluster_id'] = parent_cluster_id
self.cluster_info['cluster_image_version'] = image_version
self.set_data_disk(disk_size, disk_count, disk_type,
upscaling_config, enable_encryption)
self.set_monitoring(enable_ganglia_monitoring,
Expand Down Expand Up @@ -498,6 +508,13 @@ def cluster_info_parser(argparser, action):
dest="root_disk_size",
type=int,
help="size of the root volume in GB")
cluster_info.add_argument("--parent-cluster-id",
dest="parent_cluster_id",
type=int,
help="Id of the parent cluster this hs2 cluster is attached to")
cluster_info.add_argument("--image-version",
dest="image_version",
help="cluster image version")
termination = cluster_info.add_mutually_exclusive_group()
termination.add_argument("--disallow-cluster-termination",
dest="disallow_cluster_termination",
Expand Down
14 changes: 12 additions & 2 deletions qds_sdk/clusterv2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ def set_cluster_info_from_arguments(self, arguments):
paused_cluster_timeout_mins=arguments.paused_cluster_timeout_mins,
disable_autoscale_node_pause=arguments.disable_autoscale_node_pause,
paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins,
parent_cluster_id=arguments.parent_cluster_id)
parent_cluster_id=arguments.parent_cluster_id,
image_version=arguments.image_version)

def set_cluster_info(self,
disallow_cluster_termination=None,
Expand Down Expand Up @@ -225,7 +226,8 @@ def set_cluster_info(self,
paused_cluster_timeout_mins=None,
disable_autoscale_node_pause=None,
paused_autoscale_node_timeout_mins=None,
parent_cluster_id=None):
parent_cluster_id=None,
image_version=None):
"""
Args:
Expand Down Expand Up @@ -330,6 +332,10 @@ def set_cluster_info(self,
`paused_autoscale_node_timeout_mins`: Paused autoscale node timeout in mins
`parent_cluster_id`: parent cluster id for HS2 cluster
`image_version`: cluster image version
Doc: For getting details about arguments
http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#parameters
Expand Down Expand Up @@ -360,6 +366,7 @@ def set_cluster_info(self,
self.cluster_info['rootdisk'] = {}
self.cluster_info['rootdisk']['size'] = root_disk_size
self.cluster_info['parent_cluster_id'] = parent_cluster_id
self.cluster_info['cluster_image_version'] = image_version

self.set_spot_instance_settings(maximum_bid_price_percentage, timeout_for_request,
maximum_spot_instance_percentage)
Expand Down Expand Up @@ -522,6 +529,9 @@ def cluster_info_parser(argparser, action):
dest="parent_cluster_id",
type=int,
help="Id of the parent cluster this hs2 cluster is attached to")
cluster_info.add_argument("--image-version",
dest="image_version",
help="cluster image version")
termination = cluster_info.add_mutually_exclusive_group()
termination.add_argument("--disallow-cluster-termination",
dest="disallow_cluster_termination",
Expand Down
2 changes: 1 addition & 1 deletion qds_sdk/engine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def engine_parser(argparser):
engine_group = argparser.add_argument_group("engine settings")
engine_group.add_argument("--flavour",
dest="flavour",
choices=["hadoop", "hadoop2", "hs2", "presto", "spark", "sparkstreaming", "hbase", "airflow", "deeplearning"],
choices=["hadoop", "hadoop2", "hs2", "hive", "presto", "spark", "sparkstreaming", "hbase", "airflow", "deeplearning"],
default=None,
help="Set engine flavour")

Expand Down
18 changes: 18 additions & 0 deletions tests/test_clusterv2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,24 @@ def test_image_override(self):
'internal':{'image_uri_overrides': 'test/image1'}
})

def test_image_version_v2(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label',
'test_label', '--flavour', 'hadoop2', '--slave-instance-type', 'c1.xlarge', '--min-nodes', '3', '--image-version', '1.latest']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{'engine_config':
{'flavour': 'hadoop2'},
'cluster_info': {'label': ['test_label'],
'min_nodes': 3,
'slave_instance_type': 'c1.xlarge',
'cluster_image_version': '1.latest'}})




def test_spot_block_duration_v2(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--spot-block-duration', '120']
Expand Down
16 changes: 16 additions & 0 deletions tests/test_clusterv22.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,3 +195,19 @@ def test_spot_spot_spot(self):
'maximum_bid_price_percentage': 50}]}, 'autoscaling_nodes': {'nodes': [
{'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None,
'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}})

def test_image_version_v22(self):
sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label',
'test_label', '--flavour', 'hive', '--slave-instance-type', 'c1.xlarge', '--min-nodes', '3', '--image-version', '1.latest']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{'engine_config':
{'flavour': 'hive'},
'cluster_info': {'label': ['test_label'],
'min_nodes': 3,
'slave_instance_type': 'c1.xlarge',
'cluster_image_version': '1.latest',
'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 1, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', 'maximum_bid_price_percentage': 100}]}}, 'label': ['test_label']}})

0 comments on commit e57d25c

Please sign in to comment.