diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py index 43e699f6..9f946a3f 100644 --- a/qds_sdk/cluster_info_v22.py +++ b/qds_sdk/cluster_info_v22.py @@ -77,7 +77,8 @@ def set_cluster_info_from_arguments(self, arguments): autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request, autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy, - autoscaling_spot_fallback=arguments.autoscaling_spot_fallback) + autoscaling_spot_fallback=arguments.autoscaling_spot_fallback, + autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback) def set_cluster_info(self, disallow_cluster_termination=None, @@ -248,7 +249,8 @@ def set_composition(self, autoscaling_maximum_bid_price_percentage=None, autoscaling_timeout_for_request=None, autoscaling_spot_allocation_strategy=None, - autoscaling_spot_fallback=None): + autoscaling_spot_fallback=None, + autoscaling_spot_block_fallback=None): self.cluster_info["composition"] = {} @@ -274,7 +276,8 @@ def set_composition(self, autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request, autoscaling_spot_allocation_strategy, - autoscaling_spot_fallback) + autoscaling_spot_fallback, + autoscaling_spot_block_fallback) def set_master_config(self, master_type, @@ -323,7 +326,8 @@ def set_autoscaling_config(self, autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request, autoscaling_spot_allocation_strategy, - autoscaling_spot_fallback): + autoscaling_spot_fallback, + autoscaling_spot_block_fallback): self.cluster_info["composition"]["autoscaling_nodes"] = {"nodes": []} if not autoscaling_ondemand_percentage and not autoscaling_spot_block_percentage and not autoscaling_spot_percentage: self.set_autoscaling_ondemand(50) @@ -333,7 +337,8 @@ def set_autoscaling_config(self, self.set_autoscaling_ondemand(autoscaling_ondemand_percentage) if autoscaling_spot_block_percentage: self.set_autoscaling_spot_block(autoscaling_spot_block_percentage, - autoscaling_spot_block_duration) + autoscaling_spot_block_duration, + autoscaling_spot_block_fallback) if autoscaling_spot_percentage: self.set_autoscaling_spot(autoscaling_spot_percentage, autoscaling_maximum_bid_price_percentage, @@ -388,10 +393,13 @@ def set_autoscaling_ondemand(self, autoscaling_ondemand_percentage=None): "percentage": autoscaling_ondemand_percentage, "type": "ondemand"} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(ondemand) - def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, autoscaling_spot_block_duration=120): + def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, + autoscaling_spot_block_duration=120, + autoscaling_spot_block_fallback=None): spot_block = {"percentage": autoscaling_spot_block_percentage, "type": "spotblock", - "timeout": autoscaling_spot_block_duration} + "timeout": autoscaling_spot_block_duration, + "fallback": autoscaling_spot_block_fallback} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot_block) def set_autoscaling_spot(self, autoscaling_spot_percentage=None, @@ -693,6 +701,12 @@ def cluster_info_parser(argparser, action): type=int, default=120, help="spot block duration unit: minutes") + composition_group.add_argument("--autoscaling-spot-block-fallback", + dest="autoscaling_spot_block_fallback", + choices=["ondemand", None], + default=None, + help="whether to fallback to on-demand instances for autoscaling" + + " nodes if spot block instances aren't available") composition_group.add_argument("--autoscaling-maximum-bid-price-percentage", dest="autoscaling_maximum_bid_price_percentage", type=int, diff --git a/tests/test_clusterv22.py b/tests/test_clusterv22.py index ab4c03ec..c188936b 100644 --- a/tests/test_clusterv22.py +++ b/tests/test_clusterv22.py @@ -97,7 +97,7 @@ def test_od_od_spotblock(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-spot-block-percentage', - '100', '--autoscaling-spot-block-duration', '60'] + '100', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-block-fallback', 'ondemand'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -105,15 +105,15 @@ def test_od_od_spotblock(self): Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, - 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}}, + 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60, 'fallback': 'ondemand'}]}}, 'label': ['test_label']}}) def test_od_od_spotblockspot(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-spot-block-percentage', - '50', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-percentage', '50', - '--autoscaling-maximum-bid-price-percentage', '50', + '50', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-block-fallback', None, + '--autoscaling-spot-percentage', '50', '--autoscaling-maximum-bid-price-percentage', '50', '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', None, '--autoscaling-spot-allocation-strategy', 'capacityOptimized'] Qubole.cloud = None print_command() @@ -122,7 +122,7 @@ def test_od_od_spotblockspot(self): Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': { - 'nodes': [{'percentage': 50, 'type': 'spotblock', 'timeout': 60}, + 'nodes': [{'percentage': 50, 'type': 'spotblock', 'timeout': 60, 'fallback': None}, {'timeout_for_request': 3, 'percentage': 50, 'type': 'spot', 'fallback': None, 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'capacityOptimized'}]}}, 'label': ['test_label']}}) @@ -164,7 +164,7 @@ def test_spotblock_spotblock_spotblock(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'spotblock', '--master-spot-block-duration', '60', '--min-spot-block-percentage', '100', '--min-spot-block-duration', '60', '--autoscaling-spot-block-percentage', - '100', '--autoscaling-spot-block-duration', '60'] + '100', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-block-fallback', None] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -172,7 +172,7 @@ def test_spotblock_spotblock_spotblock(self): Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}, - 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60}]}}, + 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spotblock', 'timeout': 60, 'fallback': None}]}}, 'label': ['test_label']}}) def test_spot_spot_spot(self):