From 946f029e935c41920cbae1358d3a45c1577c94d3 Mon Sep 17 00:00:00 2001 From: juhig_qubole Date: Sun, 3 May 2020 11:24:20 +0530 Subject: [PATCH 1/4] chg --- qds_sdk/cluster_info_v22.py | 26 +++++++++++++++++++------- tests/test_clusterv22.py | 4 ++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py index 43e699f6..1d8cd80c 100644 --- a/qds_sdk/cluster_info_v22.py +++ b/qds_sdk/cluster_info_v22.py @@ -77,7 +77,8 @@ def set_cluster_info_from_arguments(self, arguments): autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request, autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy, - autoscaling_spot_fallback=arguments.autoscaling_spot_fallback) + autoscaling_spot_fallback=arguments.autoscaling_spot_fallback, + autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback) def set_cluster_info(self, disallow_cluster_termination=None, @@ -248,7 +249,8 @@ def set_composition(self, autoscaling_maximum_bid_price_percentage=None, autoscaling_timeout_for_request=None, autoscaling_spot_allocation_strategy=None, - autoscaling_spot_fallback=None): + autoscaling_spot_fallback=None, + autoscaling_spot_block_fallback=None): self.cluster_info["composition"] = {} @@ -274,7 +276,8 @@ def set_composition(self, autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request, autoscaling_spot_allocation_strategy, - autoscaling_spot_fallback) + autoscaling_spot_fallback, + autoscaling_spot_block_fallback) def set_master_config(self, master_type, @@ -323,7 +326,8 @@ def set_autoscaling_config(self, autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request, autoscaling_spot_allocation_strategy, - autoscaling_spot_fallback): + autoscaling_spot_fallback, + autoscaling_spot_block_fallback): self.cluster_info["composition"]["autoscaling_nodes"] = {"nodes": []} if not autoscaling_ondemand_percentage and not autoscaling_spot_block_percentage and not autoscaling_spot_percentage: self.set_autoscaling_ondemand(50) @@ -333,7 +337,8 @@ def set_autoscaling_config(self, self.set_autoscaling_ondemand(autoscaling_ondemand_percentage) if autoscaling_spot_block_percentage: self.set_autoscaling_spot_block(autoscaling_spot_block_percentage, - autoscaling_spot_block_duration) + autoscaling_spot_block_duration, + autoscaling_spot_block_fallback) if autoscaling_spot_percentage: self.set_autoscaling_spot(autoscaling_spot_percentage, autoscaling_maximum_bid_price_percentage, @@ -388,10 +393,11 @@ def set_autoscaling_ondemand(self, autoscaling_ondemand_percentage=None): "percentage": autoscaling_ondemand_percentage, "type": "ondemand"} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(ondemand) - def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, autoscaling_spot_block_duration=120): + def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, autoscaling_spot_block_duration=120,autoscaling_spot_block_fallback=None): spot_block = {"percentage": autoscaling_spot_block_percentage, "type": "spotblock", - "timeout": autoscaling_spot_block_duration} + "timeout": autoscaling_spot_block_duration + "fallback": autoscaling_spot_block_fallback} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot_block) def set_autoscaling_spot(self, autoscaling_spot_percentage=None, @@ -693,6 +699,12 @@ def cluster_info_parser(argparser, action): type=int, default=120, help="spot block duration unit: minutes") + composition_group.add_argument("--autoscaling-spot-block-fallback", + dest="autoscaling_spot_block_fallback", + choices=["ondemand", None], + default=None, + help="whether to fallback to on-demand instances for autoscaling nodes" + + " if spot block instances aren't available") composition_group.add_argument("--autoscaling-maximum-bid-price-percentage", dest="autoscaling_maximum_bid_price_percentage", type=int, diff --git a/tests/test_clusterv22.py b/tests/test_clusterv22.py index ab4c03ec..0cf0b792 100644 --- a/tests/test_clusterv22.py +++ b/tests/test_clusterv22.py @@ -97,7 +97,7 @@ def test_od_od_spotblock(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-spot-block-percentage', - '100', '--autoscaling-spot-block-duration', '60'] + '100', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-block-fallback', 'ondemand'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -105,7 +105,7 @@ def test_od_od_spotblock(self): Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, - 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}}, + 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60, 'fallback': 'ondemand'}]}}, 'label': ['test_label']}}) def test_od_od_spotblockspot(self): From ee5da2751a657498b52bb882b86176d24d135d49 Mon Sep 17 00:00:00 2001 From: juhig_qubole Date: Mon, 4 May 2020 10:28:34 +0530 Subject: [PATCH 2/4] chg --- qds_sdk/cluster_info_v22.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py index 1d8cd80c..d09c4463 100644 --- a/qds_sdk/cluster_info_v22.py +++ b/qds_sdk/cluster_info_v22.py @@ -393,10 +393,11 @@ def set_autoscaling_ondemand(self, autoscaling_ondemand_percentage=None): "percentage": autoscaling_ondemand_percentage, "type": "ondemand"} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(ondemand) - def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, autoscaling_spot_block_duration=120,autoscaling_spot_block_fallback=None): + def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, autoscaling_spot_block_duration=120, + autoscaling_spot_block_fallback=None): spot_block = {"percentage": autoscaling_spot_block_percentage, "type": "spotblock", - "timeout": autoscaling_spot_block_duration + "timeout": autoscaling_spot_block_duration, "fallback": autoscaling_spot_block_fallback} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot_block) From 939fb1d0e5f3406a3a0909a15969c8b0e374c097 Mon Sep 17 00:00:00 2001 From: juhig_qubole Date: Mon, 4 May 2020 10:52:31 +0530 Subject: [PATCH 3/4] chg --- tests/test_clusterv22.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_clusterv22.py b/tests/test_clusterv22.py index 0cf0b792..c188936b 100644 --- a/tests/test_clusterv22.py +++ b/tests/test_clusterv22.py @@ -112,8 +112,8 @@ def test_od_od_spotblockspot(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-spot-block-percentage', - '50', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-percentage', '50', - '--autoscaling-maximum-bid-price-percentage', '50', + '50', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-block-fallback', None, + '--autoscaling-spot-percentage', '50', '--autoscaling-maximum-bid-price-percentage', '50', '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', None, '--autoscaling-spot-allocation-strategy', 'capacityOptimized'] Qubole.cloud = None print_command() @@ -122,7 +122,7 @@ def test_od_od_spotblockspot(self): Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': { - 'nodes': [{'percentage': 50, 'type': 'spotblock', 'timeout': 60}, + 'nodes': [{'percentage': 50, 'type': 'spotblock', 'timeout': 60, 'fallback': None}, {'timeout_for_request': 3, 'percentage': 50, 'type': 'spot', 'fallback': None, 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'capacityOptimized'}]}}, 'label': ['test_label']}}) @@ -164,7 +164,7 @@ def test_spotblock_spotblock_spotblock(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'spotblock', '--master-spot-block-duration', '60', '--min-spot-block-percentage', '100', '--min-spot-block-duration', '60', '--autoscaling-spot-block-percentage', - '100', '--autoscaling-spot-block-duration', '60'] + '100', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-block-fallback', None] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -172,7 +172,7 @@ def test_spotblock_spotblock_spotblock(self): Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}, - 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}}, + 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60, 'fallback': None}]}}, 'label': ['test_label']}}) def test_spot_spot_spot(self): From 65df464445b4d796d7f70c49b32a249518f232e6 Mon Sep 17 00:00:00 2001 From: juhig_qubole Date: Mon, 4 May 2020 11:03:14 +0530 Subject: [PATCH 4/4] chg --- qds_sdk/cluster_info_v22.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py index d09c4463..9f946a3f 100644 --- a/qds_sdk/cluster_info_v22.py +++ b/qds_sdk/cluster_info_v22.py @@ -393,7 +393,8 @@ def set_autoscaling_ondemand(self, autoscaling_ondemand_percentage=None): "percentage": autoscaling_ondemand_percentage, "type": "ondemand"} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(ondemand) - def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, autoscaling_spot_block_duration=120, + def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, + autoscaling_spot_block_duration=120, autoscaling_spot_block_fallback=None): spot_block = {"percentage": autoscaling_spot_block_percentage, "type": "spotblock", @@ -704,8 +705,8 @@ def cluster_info_parser(argparser, action): dest="autoscaling_spot_block_fallback", choices=["ondemand", None], default=None, - help="whether to fallback to on-demand instances for autoscaling nodes" + - " if spot block instances aren't available") + help="whether to fallback to on-demand instances for autoscaling" + + " nodes if spot block instances aren't available") composition_group.add_argument("--autoscaling-maximum-bid-price-percentage", dest="autoscaling_maximum_bid_price_percentage", type=int,