Skip to content

Commit

Permalink
SDK-345: Add custom resource group and start-stop related params for …
Browse files Browse the repository at this point in the history 
…Azure clusters (#268)
  • Loading branch information
@akaranjkar-qu @chattarajoy
akaranjkar-qu authored and chattarajoy committed May 14, 2019
1 parent ac95aaf commit 70b075e
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 4 deletions.
11 changes: 9 additions & 2 deletions qds_sdk/cloud/azure_cloud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ def set_cloud_config(self,
vnet_resource_group_name=None,
master_elastic_ip=None,
master_static_nic_name=None,
master_static_public_ip_name=None):
master_static_public_ip_name=None,
resource_group_name=None):
'''
Args:
Expand Down Expand Up @@ -74,6 +75,7 @@ def set_cloud_config(self,
master_static_public_ip_name: Name of Static Public Ip address that has to be attached
to cluster's master node
resource_group_name: Resource group for cluster
'''

self.set_compute_config(use_account_compute_creds, compute_tenant_id,
Expand All @@ -86,6 +88,7 @@ def set_cloud_config(self,
self.set_storage_config(storage_access_key, storage_account_name,
disk_storage_account_name,
disk_storage_account_resource_group_name)
self.resource_group_name = resource_group_name

def set_compute_config(self,
use_account_compute_creds=None,
Expand Down Expand Up @@ -147,7 +150,8 @@ def set_cloud_config_from_arguments(self, arguments):
subnet_name=arguments.subnet_name,
vnet_resource_group_name=arguments.vnet_resource_group_name,
master_static_nic_name=arguments.master_static_nic_name,
master_static_public_ip_name=arguments.master_static_public_ip_name)
master_static_public_ip_name=arguments.master_static_public_ip_name,
resource_group_name=arguments.resource_group_name)

def create_parser(self, argparser):
# compute settings parser
Expand Down Expand Up @@ -204,6 +208,9 @@ def create_parser(self, argparser):
network_config_group.add_argument("--master-static-public-ip-name",
dest="master_static_public_ip_name",
help="name of public IP to be attached to master node")
network_config_group.add_argument("--resource-group-name",
dest="resource_group_name",
help="resource group for cluster")
# storage config settings parser
storage_config = argparser.add_argument_group("storage config settings")
storage_config.add_argument("--storage-access-key",
Expand Down
68 changes: 66 additions & 2 deletions qds_sdk/clusterv2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,11 @@ def get_cluster_create_clone_update(arguments, action):
image_uri_overrides=arguments.image_uri_overrides,
env_name=arguments.env_name,
python_version=arguments.python_version,
r_version=arguments.r_version)
r_version=arguments.r_version,
disable_cluster_pause=arguments.disable_cluster_pause,
paused_cluster_timeout_mins=arguments.paused_cluster_timeout_mins,
disable_autoscale_node_pause=arguments.disable_autoscale_node_pause,
paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins)

# This will set cloud config settings
cloud_config = Qubole.get_cloud()
Expand Down Expand Up @@ -193,7 +197,11 @@ def set_cluster_info(self,
image_uri_overrides=None,
env_name=None,
python_version=None,
r_version=None):
r_version=None,
disable_cluster_pause=None,
paused_cluster_timeout_mins=None,
disable_autoscale_node_pause=None,
paused_autoscale_node_timeout_mins=None):
"""
Args:
Expand Down Expand Up @@ -290,6 +298,14 @@ def set_cluster_info(self,
`r_version`: Version of R for environment. (For Spark clusters)
`disable_cluster_pause`: Disable cluster pause
`paused_cluster_timeout_mins`: Paused cluster timeout in mins
`disable_autoscale_node_pause`: Disable autoscale node pause
`paused_autoscale_node_timeout_mins`: Paused autoscale node timeout in mins
Doc: For getting details about arguments
http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#parameters
Expand Down Expand Up @@ -327,6 +343,8 @@ def set_cluster_info(self,
self.set_monitoring(enable_ganglia_monitoring, datadog_api_token, datadog_app_token)
self.set_internal(image_uri_overrides)
self.set_env_settings(env_name, python_version, r_version)
self.set_start_stop_settings(disable_cluster_pause, paused_cluster_timeout_mins,
disable_autoscale_node_pause, paused_autoscale_node_timeout_mins)

def set_datadog_setting(self,
datadog_api_token=None,
Expand Down Expand Up @@ -392,6 +410,20 @@ def set_env_settings(self, env_name=None, python_version=None, r_version=None):
self.cluster_info['env_settings']['python_version'] = python_version
self.cluster_info['env_settings']['r_version'] = r_version

def set_start_stop_settings(self,
disable_cluster_pause=None,
paused_cluster_timeout_mins=None,
disable_autoscale_node_pause=None,
paused_autoscale_node_timeout_mins=None):
if disable_cluster_pause is not None:
disable_cluster_pause = int(disable_cluster_pause)
self.cluster_info['disable_cluster_pause'] = disable_cluster_pause
self.cluster_info['paused_cluster_timeout_mins'] = paused_cluster_timeout_mins
if disable_autoscale_node_pause is not None:
disable_autoscale_node_pause = int(disable_autoscale_node_pause)
self.cluster_info['disable_autoscale_node_pause'] = disable_autoscale_node_pause
self.cluster_info['paused_autoscale_node_timeout_mins'] = paused_autoscale_node_timeout_mins

@staticmethod
def list_info_parser(argparser, action):
argparser.add_argument("--id", dest="cluster_id",
Expand Down Expand Up @@ -641,6 +673,38 @@ def cluster_info_parser(argparser, action):
default=None,
help="version of R in environment")

start_stop_group = argparser.add_argument_group("start stop settings")
start_stop_group.add_argument("--disable-cluster-pause",
dest="disable_cluster_pause",
action='store_true',
default=None,
help="disable cluster pause")
start_stop_group.add_argument("--no-disable-cluster-pause",
dest="disable_cluster_pause",
action='store_false',
default=None,
help="disable cluster pause")
start_stop_group.add_argument("--paused-cluster-timeout",
dest="paused_cluster_timeout_mins",
default=None,
type=int,
help="paused cluster timeout in min")
start_stop_group.add_argument("--disable-autoscale-node-pause",
dest="disable_autoscale_node_pause",
action='store_true',
default=None,
help="disable autoscale node pause")
start_stop_group.add_argument("--no-disable-autoscale-node-pause",
dest="disable_autoscale_node_pause",
action='store_false',
default=None,
help="disable autoscale node pause")
start_stop_group.add_argument("--paused-autoscale-node-timeout",
dest="paused_autoscale_node_timeout_mins",
default=None,
type=int,
help="paused autoscale node timeout in min")

class ClusterV2(Resource):

rest_entity_path = "clusters"
Expand Down
53 changes: 53 additions & 0 deletions tests/test_clusterv2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,19 @@ def test_azure_master_static_pip(self):
'master_static_public_ip_name':'pip1'}},
'cluster_info': {'label': ['test_label']}})

def test_azure_resource_group_name(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'AZURE', 'cluster', 'create', '--label', 'test_label',
'--resource-group-name', 'testrg']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{'cloud_config': {
'resource_group_name': 'testrg'
},
'cluster_info': {'label': ['test_label']}})

def test_oracle_opc_compute_config(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'ORACLE_OPC', 'cluster', 'create', '--label', 'test_label',
'--username', 'testusername', '--password', 'testpassword',
Expand Down Expand Up @@ -585,6 +598,46 @@ def test_root_disk_size_invalid_v2(self):
qds.main()


def test_disable_start_stop(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--disable-cluster-pause', '--disable-autoscale-node-pause']
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{
'cluster_info': {
'label': ['test_label'],
'disable_cluster_pause': 1,
'disable_autoscale_node_pause': 1
}
})
def test_start_stop_timeouts(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--no-disable-cluster-pause', '--paused-cluster-timeout', '30',
'--no-disable-autoscale-node-pause', '--paused-autoscale-node-timeout', '60']
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{
'cluster_info': {
'label': ['test_label'],
'disable_cluster_pause': 0,
'paused_cluster_timeout_mins': 30,
'disable_autoscale_node_pause': 0,
'paused_autoscale_node_timeout_mins': 60
}
})

def test_start_stop_timeouts_invalid(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--paused-cluster-timeout', 'invalid_value', '--paused-autoscale-node-timeout', 'invalid_value']
print_command()
with self.assertRaises(SystemExit):
qds.main()


class TestClusterUpdate(QdsCliTestCase):
def test_minimal(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'update', '123']
Expand Down

0 comments on commit 70b075e

Please sign in to comment.