From 08836fd5718bf1410a77493ea44e932338496d51 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 18:04:26 -0700
Subject: [PATCH 01/14] Ability to set azure cluster composition

---
 bin/qds.py                   |   2 +-
 qds_sdk/cloud/azure_cloud.py |  72 +++++++++-
 qds_sdk/cloud/cloud.py       |   6 +
 qds_sdk/cluster_info_v22.py  | 253 +++++++++++++++++++----------------
 qds_sdk/qubole.py            |   6 +-
 5 files changed, 217 insertions(+), 122 deletions(-)

diff --git a/bin/qds.py b/bin/qds.py
index ea8deb3e..668fd81b 100755
--- a/bin/qds.py
+++ b/bin/qds.py
@@ -587,7 +587,7 @@ def main():
                          help="skip verification of server SSL certificate. Insecure: use with caution.")
 
     optparser.add_option("--cloud_name", dest="cloud_name",
-                         default=os.getenv('CLOUD_PROVIDER'),
+                         default=os.getenv('CLOUD_PROVIDER') or "AWS",
                          help="cloud", choices=["AWS", "AZURE", "ORACLE_BMC", "ORACLE_OPC", "GCP"])
 
     optparser.add_option("--base_retry_delay", dest="base_retry_delay",
diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py
index 435e4993..a1a39444 100755
--- a/qds_sdk/cloud/azure_cloud.py
+++ b/qds_sdk/cloud/azure_cloud.py
@@ -153,6 +153,76 @@ def set_cloud_config_from_arguments(self, arguments):
                               master_static_public_ip_name=arguments.master_static_public_ip_name,
                               resource_group_name=arguments.resource_group_name)
 
+    def set_composition_arguments(self, composition_group):
+      #composition arguments we want to accept for azure
+
+      composition_group.add_argument("--min-ondemand-percentage",
+                              dest="min_ondemand_percentage",
+                              type=int, default =0,
+                              help="Percentage of ondemand nodes in min config.")
+      composition_group.add_argument("--min-spot-percentage",
+                              dest="min_spot_percentage",
+                              type=int, default = 0,
+                              help="Percentage of spot nodes in min config.")
+      composition_group.add_argument("--max-price-percentage",
+                              dest="max_price_percentage",
+                              type=int, default = 100,
+                              help="Percentage of maximum price percentage for spot nodes.")
+      composition_group.add_argument("--min-spot-fallback",
+                                     dest="min_spot_fallback",
+                                     choices=["ondemand", None],
+                                     default=None,
+                                     help="Whether to fallback to on-demand instances for min nodes" +
+                                          " if spot instances aren't available.")
+
+
+      composition_group.add_argument("--autoscaling-ondemand-percentage",
+                              dest="autoscaling_ondemand_percentage",
+                              type=int, default=0,
+                              help="Percentage of ondemand nodes in autoscaling config.")
+      composition_group.add_argument("--autoscaling-spot-percentage",
+                              dest="autoscaling_spot_percentage",
+                              type=int, default=0,
+                              help="Percentage of spot nodes in autoscaling config.")
+      composition_group.add_argument("--autoscaling-spot-fallback",
+                                     dest="autoscaling_spot_fallback",
+                                     choices=["ondemand", None],
+                                     default=None,
+                                     help="Whether to fallback to on-demand instances for autoscaling nodes" +
+                                          " if spot instances aren't available.")
+
+    #Ignore other key-value arguments.
+    def get_composition(self,
+                        min_ondemand_percentage=0,
+                        min_spot_percentage=0,
+                        min_spot_fallback=None,
+                        autoscaling_ondemand_percentage=0,
+                        autoscaling_spot_percentage=0,
+                        autoscaling_spot_fallback=None,
+                        max_price_percentage=100,
+                        **kwargs):
+      composition = {}
+      composition["min_nodes"] = {"nodes": []}
+      if min_ondemand_percentage + min_spot_percentage != 100:
+          raise ValueError("Minimum nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
+                          % (min_ondemand_percentage,min_spot_percentage))
+      if min_ondemand_percentage > 0:
+          composition["min_nodes"]["nodes"].append({"type":"ondemand", "percentage":min_sondemand_percentage})
+      if min_spot_percentage > 0:
+          composition["min_nodes"]["nodes"].append({"type":"spot", "percentage":min_spot_percentage,
+                                                 "fallback": min_spot_fallback, "max_price_percentage": max_price_percentage})
+
+      composition["autoscaling_nodes"] = {"nodes": []}
+      if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
+          raise ValueError("Autoscaling nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
+                          % (autoscaling_ondemand_percentage,autoscaling_spot_percentage))
+      if autoscaling_ondemand_percentage > 0:
+          composition["autoscaling_nodes"]["nodes"].append({"type":"ondemand", "percentage":autoscaling_ondemand_percentage})
+      if autoscaling_spot_percentage > 0:
+          composition["autoscaling_nodes"]["nodes"].append({"type":"spot", "percentage":autoscaling_spot_percentage,
+                                                 "fallback": autoscaling_spot_fallback, "max_price_percentage": max_price_percentage})
+      return composition
+
     def create_parser(self, argparser):
         # compute settings parser
         compute_config = argparser.add_argument_group("compute config settings")
@@ -228,4 +298,4 @@ def create_parser(self, argparser):
         storage_config.add_argument("--disk-storage-account-resource-group-name",
                                     dest="disk_storage_account_resource_group_name",
                                     default=None,
-                                    help="disk storage account resource group for azure cluster")
\ No newline at end of file
+                                    help="disk storage account resource group for azure cluster")
diff --git a/qds_sdk/cloud/cloud.py b/qds_sdk/cloud/cloud.py
index 49053d87..45bd30a9 100755
--- a/qds_sdk/cloud/cloud.py
+++ b/qds_sdk/cloud/cloud.py
@@ -5,3 +5,9 @@ def create_parser(self, argparser):
 
     def set_cloud_config_from_arguments(self, arguments):
         return NotImplemented
+
+    def set_composition_arguments(seld, set_composition_arguments):
+    	pass
+
+    def get_composition(*args, **kwargs):
+    	return {}
diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index 9f946a3f..e13641c8 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -1,6 +1,7 @@
 import json
 
 from qds_sdk import util
+from qds_sdk.qubole import Qubole
 
 
 def str2bool(v):
@@ -56,8 +57,9 @@ def set_cluster_info_from_arguments(self, arguments):
                               paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins,
                               parent_cluster_id=arguments.parent_cluster_id,
                               image_version=arguments.image_version)
-
-        self.set_composition(master_type=arguments.master_type,
+        if Qubole.get_cloud_name() == "aws":
+            # TODO: Need to move to aws cloud.
+            self.set_composition(master_type=arguments.master_type,
                              master_spot_block_duration=arguments.master_spot_block_duration,
                              master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
                              master_timeout_for_request=arguments.master_timeout_for_request,
@@ -79,6 +81,8 @@ def set_cluster_info_from_arguments(self, arguments):
                              autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy,
                              autoscaling_spot_fallback=arguments.autoscaling_spot_fallback,
                              autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback)
+        else:
+            self.set_composition_from_cloud_using_parser(arguments)
 
     def set_cluster_info(self,
                          disallow_cluster_termination=None,
@@ -228,6 +232,13 @@ def set_cluster_info(self,
         self.set_start_stop_settings(disable_cluster_pause, paused_cluster_timeout_mins,
                                      disable_autoscale_node_pause, paused_autoscale_node_timeout_mins)
 
+    def set_composition_from_cloud_using_parser(self, arguments):
+        self.set_composition_for_cluster(**{k:v for k,v in arguments.__dict__.items() if v != None})
+
+    def set_composition_for_cluster(self, **kwargs):
+        cloud = Qubole.get_cloud()
+        self.cluster_info["composition"] = cloud.get_composition(kwargs)
+
     def set_composition(self,
                         master_type="ondemand",
                         master_spot_block_duration=None,
@@ -614,123 +625,127 @@ def cluster_info_parser(argparser, action):
                                   dest="heterogeneous_config",
                                   help="heterogeneous config for the cluster")
 
-        composition_group = argparser.add_argument_group("cluster composition settings")
-        composition_group.add_argument("--master-type",
-                                      dest="master_type",
-                                      choices=["ondemand", "spot", "spotblock"],
-                                      default="ondemand",
-                                      help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
-                                       "default: ondemand")
-        composition_group.add_argument("--master-spot-block-duration",
-                                       dest="master_spot_block_duration",
-                                       type=int,
-                                       default=120,
-                                       help="spot block duration unit: minutes")
-        composition_group.add_argument("--master-maximum-bid-price-percentage",
-                                       dest="master_maximum_bid_price_percentage",
-                                       type=int,
-                                       default=100,
-                                       help="maximum value to bid for master spot instances" +
-                                            " expressed as a percentage of the base" +
-                                            " price for the master instance types")
-        composition_group.add_argument("--master-timeout-for-request",
-                                       dest="master_timeout_for_request",
-                                       type=int,
-                                       default=1,
-                                       help="timeout for a master spot instance request, unit: minutes")
-        composition_group.add_argument("--master-spot-fallback",
-                                       dest="master_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for master nodes" +
-                                            " if spot instances aren't available")
-        composition_group.add_argument("--min-ondemand-percentage",
-                                       dest="min_ondemand_percentage",
-                                       type=int,
-                                       help="percentage of ondemand nodes in min config")
-        composition_group.add_argument("--min-spot-block-percentage",
-                                       dest="min_spot_block_percentage",
-                                       type=int,
-                                       help="percentage of spot block nodes in min config")
-        composition_group.add_argument("--min-spot-percentage",
-                                       dest="min_spot_percentage",
-                                       type=int,
-                                       help="percentage of spot nodes in min config")
-        composition_group.add_argument("--min-spot-block-duration",
-                                       dest="min_spot_block_duration",
-                                       type=int,
-                                       default=120,
-                                       help="spot block duration unit: minutes")
-        composition_group.add_argument("--min-maximum-bid-price-percentage",
-                                       dest="min_maximum_bid_price_percentage",
-                                       type=int,
-                                       default=100,
-                                       help="maximum value to bid for min spot instances" +
-                                            " expressed as a percentage of the base" +
-                                            " price for the master instance types")
-        composition_group.add_argument("--min-timeout-for-request",
-                                       dest="min_timeout_for_request",
-                                       type=int,
-                                       default=1,
-                                       help="timeout for a min spot instance request, unit: minutes")
-        composition_group.add_argument("--min-spot-fallback",
-                                       dest="min_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for min nodes" +
-                                            " if spot instances aren't available")
-        composition_group.add_argument("--min-spot-allocation-strategy",
-                                       dest="min_spot_allocation_strategy",
-                                       choices=["lowestPrice", "capacityOptimized", None],
-                                       default=None,
-                                       help="allocation strategy for min spot nodes")
-        composition_group.add_argument("--autoscaling-ondemand-percentage",
-                                       dest="autoscaling_ondemand_percentage",
-                                       type=int,
-                                       help="percentage of ondemand nodes in autoscaling config")
-        composition_group.add_argument("--autoscaling-spot-block-percentage",
-                                       dest="autoscaling_spot_block_percentage",
-                                       type=int,
-                                       help="percentage of spot block nodes in autoscaling config")
-        composition_group.add_argument("--autoscaling-spot-percentage",
-                                       dest="autoscaling_spot_percentage",
-                                       type=int,
-                                       help="percentage of spot nodes in autoscaling config")
-        composition_group.add_argument("--autoscaling-spot-block-duration",
-                                       dest="autoscaling_spot_block_duration",
-                                       type=int,
-                                       default=120,
-                                       help="spot block duration unit:  minutes")
-        composition_group.add_argument("--autoscaling-spot-block-fallback",
-                                       dest="autoscaling_spot_block_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for autoscaling" +
-                                            " nodes if spot block instances aren't available")
-        composition_group.add_argument("--autoscaling-maximum-bid-price-percentage",
-                                       dest="autoscaling_maximum_bid_price_percentage",
-                                       type=int,
-                                       default=100,
-                                       help="maximum value to bid for autoscaling spot instances" +
-                                            " expressed as a percentage of the base" +
-                                            " price for the master instance types")
-        composition_group.add_argument("--autoscaling-timeout-for-request",
-                                       dest="autoscaling_timeout_for_request",
-                                       type=int,
-                                       default=1,
-                                       help="timeout for a autoscaling spot instance request, unit: minutes")
-        composition_group.add_argument("--autoscaling-spot-fallback",
-                                       dest="autoscaling_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for autoscaling nodes" +
-                                            " if spot instances aren't available")
-        composition_group.add_argument("--autoscaling-spot-allocation-strategy",
-                                       dest="autoscaling_spot_allocation_strategy",
-                                       choices=["lowestPrice", "capacityOptimized", None],
-                                       default=None,
-                                       help="allocation strategy for autoscaling" +
-                                            " spot nodes")
+        composition_group = argparser.add_argument_group("Cluster composition settings")
+        if Qubole.get_cloud_name() == "aws":
+            composition_group.add_argument("--master-type",
+                                          dest="master_type",
+                                          choices=["ondemand", "spot", "spotblock"],
+                                          default="ondemand",
+                                          help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
+                                           "default: ondemand")
+            composition_group.add_argument("--master-spot-block-duration",
+                                           dest="master_spot_block_duration",
+                                           type=int,
+                                           default=120,
+                                           help="spot block duration unit: minutes")
+            composition_group.add_argument("--master-maximum-bid-price-percentage",
+                                           dest="master_maximum_bid_price_percentage",
+                                           type=int,
+                                           default=100,
+                                           help="maximum value to bid for master spot instances" +
+                                                " expressed as a percentage of the base" +
+                                                " price for the master instance types")
+            composition_group.add_argument("--master-timeout-for-request",
+                                           dest="master_timeout_for_request",
+                                           type=int,
+                                           default=1,
+                                           help="timeout for a master spot instance request, unit: minutes")
+            composition_group.add_argument("--master-spot-fallback",
+                                           dest="master_spot_fallback",
+                                           choices=["ondemand", None],
+                                           default=None,
+                                           help="whether to fallback to on-demand instances for master nodes" +
+                                                " if spot instances aren't available")
+            composition_group.add_argument("--min-ondemand-percentage",
+                                           dest="min_ondemand_percentage",
+                                           type=int,
+                                           help="percentage of ondemand nodes in min config")
+            composition_group.add_argument("--min-spot-block-percentage",
+                                           dest="min_spot_block_percentage",
+                                           type=int,
+                                           help="percentage of spot block nodes in min config")
+            composition_group.add_argument("--min-spot-percentage",
+                                           dest="min_spot_percentage",
+                                           type=int,
+                                           help="percentage of spot nodes in min config")
+            composition_group.add_argument("--min-spot-block-duration",
+                                           dest="min_spot_block_duration",
+                                           type=int,
+                                           default=120,
+                                           help="spot block duration unit: minutes")
+            composition_group.add_argument("--min-maximum-bid-price-percentage",
+                                           dest="min_maximum_bid_price_percentage",
+                                           type=int,
+                                           default=100,
+                                           help="maximum value to bid for min spot instances" +
+                                                " expressed as a percentage of the base" +
+                                                " price for the master instance types")
+            composition_group.add_argument("--min-timeout-for-request",
+                                           dest="min_timeout_for_request",
+                                           type=int,
+                                           default=1,
+                                           help="timeout for a min spot instance request, unit: minutes")
+            composition_group.add_argument("--min-spot-fallback",
+                                           dest="min_spot_fallback",
+                                           choices=["ondemand", None],
+                                           default=None,
+                                           help="whether to fallback to on-demand instances for min nodes" +
+                                                " if spot instances aren't available")
+            composition_group.add_argument("--min-spot-allocation-strategy",
+                                           dest="min_spot_allocation_strategy",
+                                           choices=["lowestPrice", "capacityOptimized", None],
+                                           default=None,
+                                           help="allocation strategy for min spot nodes")
+            composition_group.add_argument("--autoscaling-ondemand-percentage",
+                                           dest="autoscaling_ondemand_percentage",
+                                           type=int,
+                                           help="percentage of ondemand nodes in autoscaling config")
+            composition_group.add_argument("--autoscaling-spot-block-percentage",
+                                           dest="autoscaling_spot_block_percentage",
+                                           type=int,
+                                           help="percentage of spot block nodes in autoscaling config")
+            composition_group.add_argument("--autoscaling-spot-percentage",
+                                           dest="autoscaling_spot_percentage",
+                                           type=int,
+                                           help="percentage of spot nodes in autoscaling config")
+            composition_group.add_argument("--autoscaling-spot-block-duration",
+                                           dest="autoscaling_spot_block_duration",
+                                           type=int,
+                                           default=120,
+                                           help="spot block duration unit:  minutes")
+            composition_group.add_argument("--autoscaling-spot-block-fallback",
+                                           dest="autoscaling_spot_block_fallback",
+                                           choices=["ondemand", None],
+                                           default=None,
+                                           help="whether to fallback to on-demand instances for autoscaling" +
+                                                " nodes if spot block instances aren't available")
+            composition_group.add_argument("--autoscaling-maximum-bid-price-percentage",
+                                           dest="autoscaling_maximum_bid_price_percentage",
+                                           type=int,
+                                           default=100,
+                                           help="maximum value to bid for autoscaling spot instances" +
+                                                " expressed as a percentage of the base" +
+                                                " price for the master instance types")
+            composition_group.add_argument("--autoscaling-timeout-for-request",
+                                           dest="autoscaling_timeout_for_request",
+                                           type=int,
+                                           default=1,
+                                           help="timeout for a autoscaling spot instance request, unit: minutes")
+            composition_group.add_argument("--autoscaling-spot-fallback",
+                                           dest="autoscaling_spot_fallback",
+                                           choices=["ondemand", None],
+                                           default=None,
+                                           help="whether to fallback to on-demand instances for autoscaling nodes" +
+                                                " if spot instances aren't available")
+            composition_group.add_argument("--autoscaling-spot-allocation-strategy",
+                                           dest="autoscaling_spot_allocation_strategy",
+                                           choices=["lowestPrice", "capacityOptimized", None],
+                                           default=None,
+                                           help="allocation strategy for autoscaling" +
+                                                " spot nodes")
+        else:
+            Qubole.get_cloud().set_composition_arguments(composition_group)
+
 
         # monitoring settings
         monitoring_group = argparser.add_argument_group("monitoring settings")
diff --git a/qds_sdk/qubole.py b/qds_sdk/qubole.py
index 659516af..5edeab3d 100644
--- a/qds_sdk/qubole.py
+++ b/qds_sdk/qubole.py
@@ -30,7 +30,7 @@ class Qubole:
     version = None
     poll_interval = None
     skip_ssl_cert_check = None
-    cloud_name = None
+    cloud_name = "aws"
     cached_agent = None
     cloud = None
     base_retry_delay = None
@@ -143,3 +143,7 @@ def get_cloud_object(cls, cloud_name):
         elif cloud_name.lower() == "gcp":
             import qds_sdk.cloud.gcp_cloud
             return qds_sdk.cloud.gcp_cloud.GcpCloud()
+
+    @classmethod
+    def get_cloud_name(cls):
+        return Qubole.cloud_name

From 1292faf2c75d30e2436c8bb2d480d27b5ecffd53 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 18:17:06 -0700
Subject: [PATCH 02/14] chg

---
 qds_sdk/cloud/aws_cloud.py   | 118 ++++++++++++++++++++++++++++++++
 qds_sdk/cloud/azure_cloud.py | 113 ++++++++++++++++---------------
 qds_sdk/cloud/cloud.py       |   2 +-
 qds_sdk/cluster_info_v22.py  | 126 ++---------------------------------
 4 files changed, 179 insertions(+), 180 deletions(-)

diff --git a/qds_sdk/cloud/aws_cloud.py b/qds_sdk/cloud/aws_cloud.py
index b4f56156..f2460551 100755
--- a/qds_sdk/cloud/aws_cloud.py
+++ b/qds_sdk/cloud/aws_cloud.py
@@ -107,6 +107,124 @@ def set_cloud_config_from_arguments(self, arguments):
                               bastion_node_public_dns=arguments.bastion_node_public_dns,
                               master_elastic_ip=arguments.master_elastic_ip)
 
+    def set_composition_arguments(self, composition_group)
+        composition_group.add_argument("--master-type",
+                                      dest="master_type",
+                                      choices=["ondemand", "spot", "spotblock"],
+                                      default="ondemand",
+                                      help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
+                                       "default: ondemand")
+        composition_group.add_argument("--master-spot-block-duration",
+                                       dest="master_spot_block_duration",
+                                       type=int,
+                                       default=120,
+                                       help="spot block duration unit: minutes")
+        composition_group.add_argument("--master-maximum-bid-price-percentage",
+                                       dest="master_maximum_bid_price_percentage",
+                                       type=int,
+                                       default=100,
+                                       help="maximum value to bid for master spot instances" +
+                                            " expressed as a percentage of the base" +
+                                            " price for the master instance types")
+        composition_group.add_argument("--master-timeout-for-request",
+                                       dest="master_timeout_for_request",
+                                       type=int,
+                                       default=1,
+                                       help="timeout for a master spot instance request, unit: minutes")
+        composition_group.add_argument("--master-spot-fallback",
+                                       dest="master_spot_fallback",
+                                       choices=["ondemand", None],
+                                       default=None,
+                                       help="whether to fallback to on-demand instances for master nodes" +
+                                            " if spot instances aren't available")
+        composition_group.add_argument("--min-ondemand-percentage",
+                                       dest="min_ondemand_percentage",
+                                       type=int,
+                                       help="percentage of ondemand nodes in min config")
+        composition_group.add_argument("--min-spot-block-percentage",
+                                       dest="min_spot_block_percentage",
+                                       type=int,
+                                       help="percentage of spot block nodes in min config")
+        composition_group.add_argument("--min-spot-percentage",
+                                       dest="min_spot_percentage",
+                                       type=int,
+                                       help="percentage of spot nodes in min config")
+        composition_group.add_argument("--min-spot-block-duration",
+                                       dest="min_spot_block_duration",
+                                       type=int,
+                                       default=120,
+                                       help="spot block duration unit: minutes")
+        composition_group.add_argument("--min-maximum-bid-price-percentage",
+                                       dest="min_maximum_bid_price_percentage",
+                                       type=int,
+                                       default=100,
+                                       help="maximum value to bid for min spot instances" +
+                                            " expressed as a percentage of the base" +
+                                            " price for the master instance types")
+        composition_group.add_argument("--min-timeout-for-request",
+                                       dest="min_timeout_for_request",
+                                       type=int,
+                                       default=1,
+                                       help="timeout for a min spot instance request, unit: minutes")
+        composition_group.add_argument("--min-spot-fallback",
+                                       dest="min_spot_fallback",
+                                       choices=["ondemand", None],
+                                       default=None,
+                                       help="whether to fallback to on-demand instances for min nodes" +
+                                            " if spot instances aren't available")
+        composition_group.add_argument("--min-spot-allocation-strategy",
+                                       dest="min_spot_allocation_strategy",
+                                       choices=["lowestPrice", "capacityOptimized", None],
+                                       default=None,
+                                       help="allocation strategy for min spot nodes")
+        composition_group.add_argument("--autoscaling-ondemand-percentage",
+                                       dest="autoscaling_ondemand_percentage",
+                                       type=int,
+                                       help="percentage of ondemand nodes in autoscaling config")
+        composition_group.add_argument("--autoscaling-spot-block-percentage",
+                                       dest="autoscaling_spot_block_percentage",
+                                       type=int,
+                                       help="percentage of spot block nodes in autoscaling config")
+        composition_group.add_argument("--autoscaling-spot-percentage",
+                                       dest="autoscaling_spot_percentage",
+                                       type=int,
+                                       help="percentage of spot nodes in autoscaling config")
+        composition_group.add_argument("--autoscaling-spot-block-duration",
+                                       dest="autoscaling_spot_block_duration",
+                                       type=int,
+                                       default=120,
+                                       help="spot block duration unit:  minutes")
+        composition_group.add_argument("--autoscaling-spot-block-fallback",
+                                       dest="autoscaling_spot_block_fallback",
+                                       choices=["ondemand", None],
+                                       default=None,
+                                       help="whether to fallback to on-demand instances for autoscaling" +
+                                            " nodes if spot block instances aren't available")
+        composition_group.add_argument("--autoscaling-maximum-bid-price-percentage",
+                                       dest="autoscaling_maximum_bid_price_percentage",
+                                       type=int,
+                                       default=100,
+                                       help="maximum value to bid for autoscaling spot instances" +
+                                            " expressed as a percentage of the base" +
+                                            " price for the master instance types")
+        composition_group.add_argument("--autoscaling-timeout-for-request",
+                                       dest="autoscaling_timeout_for_request",
+                                       type=int,
+                                       default=1,
+                                       help="timeout for a autoscaling spot instance request, unit: minutes")
+        composition_group.add_argument("--autoscaling-spot-fallback",
+                                       dest="autoscaling_spot_fallback",
+                                       choices=["ondemand", None],
+                                       default=None,
+                                       help="whether to fallback to on-demand instances for autoscaling nodes" +
+                                            " if spot instances aren't available")
+        composition_group.add_argument("--autoscaling-spot-allocation-strategy",
+                                       dest="autoscaling_spot_allocation_strategy",
+                                       choices=["lowestPrice", "capacityOptimized", None],
+                                       default=None,
+                                       help="allocation strategy for autoscaling" +
+                                            " spot nodes")
+
     def create_parser(self, argparser):
         # compute settings parser
         compute_config = argparser.add_argument_group("compute config settings")
diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py
index a1a39444..60cee68d 100755
--- a/qds_sdk/cloud/azure_cloud.py
+++ b/qds_sdk/cloud/azure_cloud.py
@@ -154,42 +154,41 @@ def set_cloud_config_from_arguments(self, arguments):
                               resource_group_name=arguments.resource_group_name)
 
     def set_composition_arguments(self, composition_group):
-      #composition arguments we want to accept for azure
-
-      composition_group.add_argument("--min-ondemand-percentage",
-                              dest="min_ondemand_percentage",
-                              type=int, default =0,
-                              help="Percentage of ondemand nodes in min config.")
-      composition_group.add_argument("--min-spot-percentage",
-                              dest="min_spot_percentage",
-                              type=int, default = 0,
-                              help="Percentage of spot nodes in min config.")
-      composition_group.add_argument("--max-price-percentage",
-                              dest="max_price_percentage",
-                              type=int, default = 100,
-                              help="Percentage of maximum price percentage for spot nodes.")
-      composition_group.add_argument("--min-spot-fallback",
-                                     dest="min_spot_fallback",
-                                     choices=["ondemand", None],
-                                     default=None,
-                                     help="Whether to fallback to on-demand instances for min nodes" +
-                                          " if spot instances aren't available.")
-
-
-      composition_group.add_argument("--autoscaling-ondemand-percentage",
-                              dest="autoscaling_ondemand_percentage",
-                              type=int, default=0,
-                              help="Percentage of ondemand nodes in autoscaling config.")
-      composition_group.add_argument("--autoscaling-spot-percentage",
-                              dest="autoscaling_spot_percentage",
-                              type=int, default=0,
-                              help="Percentage of spot nodes in autoscaling config.")
-      composition_group.add_argument("--autoscaling-spot-fallback",
-                                     dest="autoscaling_spot_fallback",
-                                     choices=["ondemand", None],
-                                     default=None,
-                                     help="Whether to fallback to on-demand instances for autoscaling nodes" +
-                                          " if spot instances aren't available.")
+        #composition arguments we want to accept for azure
+        composition_group.add_argument("--min-ondemand-percentage",
+                                       dest="min_ondemand_percentage",
+                                       type=int, default =0,
+                                       help="Percentage of ondemand nodes in min config.")
+        composition_group.add_argument("--min-spot-percentage",
+                                       dest="min_spot_percentage",
+                                       type=int, default = 0,
+                                       help="Percentage of spot nodes in min config.")
+        composition_group.add_argument("--max-price-percentage",
+                                       dest="max_price_percentage",
+                                       type=int, default = 100,
+                                       help="Percentage of maximum price percentage for spot nodes.")
+        composition_group.add_argument("--min-spot-fallback",
+                                       dest="min_spot_fallback",
+                                       choices=["ondemand", None],
+                                       default=None,
+                                       help="Whether to fallback to on-demand instances for min nodes" +
+                                            " if spot instances aren't available.")
+
+
+        composition_group.add_argument("--autoscaling-ondemand-percentage",
+                                       dest="autoscaling_ondemand_percentage",
+                                       type=int, default=0,
+                                       help="Percentage of ondemand nodes in autoscaling config.")
+        composition_group.add_argument("--autoscaling-spot-percentage",
+                                       dest="autoscaling_spot_percentage",
+                                       type=int, default=0,
+                                       help="Percentage of spot nodes in autoscaling config.")
+        composition_group.add_argument("--autoscaling-spot-fallback",
+                                       dest="autoscaling_spot_fallback",
+                                       choices=["ondemand", None],
+                                       default=None,
+                                       help="Whether to fallback to on-demand instances for autoscaling nodes" +
+                                            " if spot instances aren't available.")
 
     #Ignore other key-value arguments.
     def get_composition(self,
@@ -201,27 +200,27 @@ def get_composition(self,
                         autoscaling_spot_fallback=None,
                         max_price_percentage=100,
                         **kwargs):
-      composition = {}
-      composition["min_nodes"] = {"nodes": []}
-      if min_ondemand_percentage + min_spot_percentage != 100:
-          raise ValueError("Minimum nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
-                          % (min_ondemand_percentage,min_spot_percentage))
-      if min_ondemand_percentage > 0:
-          composition["min_nodes"]["nodes"].append({"type":"ondemand", "percentage":min_sondemand_percentage})
-      if min_spot_percentage > 0:
-          composition["min_nodes"]["nodes"].append({"type":"spot", "percentage":min_spot_percentage,
-                                                 "fallback": min_spot_fallback, "max_price_percentage": max_price_percentage})
-
-      composition["autoscaling_nodes"] = {"nodes": []}
-      if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
-          raise ValueError("Autoscaling nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
-                          % (autoscaling_ondemand_percentage,autoscaling_spot_percentage))
-      if autoscaling_ondemand_percentage > 0:
-          composition["autoscaling_nodes"]["nodes"].append({"type":"ondemand", "percentage":autoscaling_ondemand_percentage})
-      if autoscaling_spot_percentage > 0:
-          composition["autoscaling_nodes"]["nodes"].append({"type":"spot", "percentage":autoscaling_spot_percentage,
-                                                 "fallback": autoscaling_spot_fallback, "max_price_percentage": max_price_percentage})
-      return composition
+        composition = {}
+        composition["min_nodes"] = {"nodes": []}
+        if min_ondemand_percentage + min_spot_percentage != 100:
+            raise ValueError("Minimum nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
+                            % (min_ondemand_percentage,min_spot_percentage))
+        if min_ondemand_percentage > 0:
+            composition["min_nodes"]["nodes"].append({"type":"ondemand", "percentage":min_ondemand_percentage})
+        if min_spot_percentage > 0:
+            composition["min_nodes"]["nodes"].append({"type":"spot", "percentage":min_spot_percentage,
+                                                   "fallback": min_spot_fallback, "max_price_percentage": max_price_percentage})
+
+        composition["autoscaling_nodes"] = {"nodes": []}
+        if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
+            raise ValueError("Autoscaling nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
+                            % (autoscaling_ondemand_percentage,autoscaling_spot_percentage))
+        if autoscaling_ondemand_percentage > 0:
+            composition["autoscaling_nodes"]["nodes"].append({"type":"ondemand", "percentage":autoscaling_ondemand_percentage})
+        if autoscaling_spot_percentage > 0:
+            composition["autoscaling_nodes"]["nodes"].append({"type":"spot", "percentage":autoscaling_spot_percentage,
+                                                   "fallback": autoscaling_spot_fallback, "max_price_percentage": max_price_percentage})
+        return composition
 
     def create_parser(self, argparser):
         # compute settings parser
diff --git a/qds_sdk/cloud/cloud.py b/qds_sdk/cloud/cloud.py
index 45bd30a9..d9981ea5 100755
--- a/qds_sdk/cloud/cloud.py
+++ b/qds_sdk/cloud/cloud.py
@@ -6,7 +6,7 @@ def create_parser(self, argparser):
     def set_cloud_config_from_arguments(self, arguments):
         return NotImplemented
 
-    def set_composition_arguments(seld, set_composition_arguments):
+    def set_composition_arguments(self, set_composition_arguments):
     	pass
 
     def get_composition(*args, **kwargs):
diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index e13641c8..067fb68c 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -58,7 +58,7 @@ def set_cluster_info_from_arguments(self, arguments):
                               parent_cluster_id=arguments.parent_cluster_id,
                               image_version=arguments.image_version)
         if Qubole.get_cloud_name() == "aws":
-            # TODO: Need to move to aws cloud.
+            #Need to move to aws cloud.
             self.set_composition(master_type=arguments.master_type,
                              master_spot_block_duration=arguments.master_spot_block_duration,
                              master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
@@ -233,7 +233,8 @@ def set_cluster_info(self,
                                      disable_autoscale_node_pause, paused_autoscale_node_timeout_mins)
 
     def set_composition_from_cloud_using_parser(self, arguments):
-        self.set_composition_for_cluster(**{k:v for k,v in arguments.__dict__.items() if v != None})
+        self.set_composition_for_cluster(**{k:v for k,v in arguments.__dict__.items() \
+                                             if v != None})
 
     def set_composition_for_cluster(self, **kwargs):
         cloud = Qubole.get_cloud()
@@ -626,126 +627,7 @@ def cluster_info_parser(argparser, action):
                                   help="heterogeneous config for the cluster")
 
         composition_group = argparser.add_argument_group("Cluster composition settings")
-        if Qubole.get_cloud_name() == "aws":
-            composition_group.add_argument("--master-type",
-                                          dest="master_type",
-                                          choices=["ondemand", "spot", "spotblock"],
-                                          default="ondemand",
-                                          help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
-                                           "default: ondemand")
-            composition_group.add_argument("--master-spot-block-duration",
-                                           dest="master_spot_block_duration",
-                                           type=int,
-                                           default=120,
-                                           help="spot block duration unit: minutes")
-            composition_group.add_argument("--master-maximum-bid-price-percentage",
-                                           dest="master_maximum_bid_price_percentage",
-                                           type=int,
-                                           default=100,
-                                           help="maximum value to bid for master spot instances" +
-                                                " expressed as a percentage of the base" +
-                                                " price for the master instance types")
-            composition_group.add_argument("--master-timeout-for-request",
-                                           dest="master_timeout_for_request",
-                                           type=int,
-                                           default=1,
-                                           help="timeout for a master spot instance request, unit: minutes")
-            composition_group.add_argument("--master-spot-fallback",
-                                           dest="master_spot_fallback",
-                                           choices=["ondemand", None],
-                                           default=None,
-                                           help="whether to fallback to on-demand instances for master nodes" +
-                                                " if spot instances aren't available")
-            composition_group.add_argument("--min-ondemand-percentage",
-                                           dest="min_ondemand_percentage",
-                                           type=int,
-                                           help="percentage of ondemand nodes in min config")
-            composition_group.add_argument("--min-spot-block-percentage",
-                                           dest="min_spot_block_percentage",
-                                           type=int,
-                                           help="percentage of spot block nodes in min config")
-            composition_group.add_argument("--min-spot-percentage",
-                                           dest="min_spot_percentage",
-                                           type=int,
-                                           help="percentage of spot nodes in min config")
-            composition_group.add_argument("--min-spot-block-duration",
-                                           dest="min_spot_block_duration",
-                                           type=int,
-                                           default=120,
-                                           help="spot block duration unit: minutes")
-            composition_group.add_argument("--min-maximum-bid-price-percentage",
-                                           dest="min_maximum_bid_price_percentage",
-                                           type=int,
-                                           default=100,
-                                           help="maximum value to bid for min spot instances" +
-                                                " expressed as a percentage of the base" +
-                                                " price for the master instance types")
-            composition_group.add_argument("--min-timeout-for-request",
-                                           dest="min_timeout_for_request",
-                                           type=int,
-                                           default=1,
-                                           help="timeout for a min spot instance request, unit: minutes")
-            composition_group.add_argument("--min-spot-fallback",
-                                           dest="min_spot_fallback",
-                                           choices=["ondemand", None],
-                                           default=None,
-                                           help="whether to fallback to on-demand instances for min nodes" +
-                                                " if spot instances aren't available")
-            composition_group.add_argument("--min-spot-allocation-strategy",
-                                           dest="min_spot_allocation_strategy",
-                                           choices=["lowestPrice", "capacityOptimized", None],
-                                           default=None,
-                                           help="allocation strategy for min spot nodes")
-            composition_group.add_argument("--autoscaling-ondemand-percentage",
-                                           dest="autoscaling_ondemand_percentage",
-                                           type=int,
-                                           help="percentage of ondemand nodes in autoscaling config")
-            composition_group.add_argument("--autoscaling-spot-block-percentage",
-                                           dest="autoscaling_spot_block_percentage",
-                                           type=int,
-                                           help="percentage of spot block nodes in autoscaling config")
-            composition_group.add_argument("--autoscaling-spot-percentage",
-                                           dest="autoscaling_spot_percentage",
-                                           type=int,
-                                           help="percentage of spot nodes in autoscaling config")
-            composition_group.add_argument("--autoscaling-spot-block-duration",
-                                           dest="autoscaling_spot_block_duration",
-                                           type=int,
-                                           default=120,
-                                           help="spot block duration unit:  minutes")
-            composition_group.add_argument("--autoscaling-spot-block-fallback",
-                                           dest="autoscaling_spot_block_fallback",
-                                           choices=["ondemand", None],
-                                           default=None,
-                                           help="whether to fallback to on-demand instances for autoscaling" +
-                                                " nodes if spot block instances aren't available")
-            composition_group.add_argument("--autoscaling-maximum-bid-price-percentage",
-                                           dest="autoscaling_maximum_bid_price_percentage",
-                                           type=int,
-                                           default=100,
-                                           help="maximum value to bid for autoscaling spot instances" +
-                                                " expressed as a percentage of the base" +
-                                                " price for the master instance types")
-            composition_group.add_argument("--autoscaling-timeout-for-request",
-                                           dest="autoscaling_timeout_for_request",
-                                           type=int,
-                                           default=1,
-                                           help="timeout for a autoscaling spot instance request, unit: minutes")
-            composition_group.add_argument("--autoscaling-spot-fallback",
-                                           dest="autoscaling_spot_fallback",
-                                           choices=["ondemand", None],
-                                           default=None,
-                                           help="whether to fallback to on-demand instances for autoscaling nodes" +
-                                                " if spot instances aren't available")
-            composition_group.add_argument("--autoscaling-spot-allocation-strategy",
-                                           dest="autoscaling_spot_allocation_strategy",
-                                           choices=["lowestPrice", "capacityOptimized", None],
-                                           default=None,
-                                           help="allocation strategy for autoscaling" +
-                                                " spot nodes")
-        else:
-            Qubole.get_cloud().set_composition_arguments(composition_group)
-
+        Qubole.get_cloud().set_composition_arguments(composition_group)
 
         # monitoring settings
         monitoring_group = argparser.add_argument_group("monitoring settings")

From bdc9eb785048416652c9bd426f686bf1416d390f Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 18:48:25 -0700
Subject: [PATCH 03/14] chg

---
 qds_sdk/cloud/aws_cloud.py   | 10 +++---
 qds_sdk/cloud/azure_cloud.py | 68 +++++++++++++++++-------------------
 qds_sdk/cloud/cloud.py       |  2 +-
 qds_sdk/cluster_info_v22.py  | 46 ++++++++++++------------
 4 files changed, 62 insertions(+), 64 deletions(-)

diff --git a/qds_sdk/cloud/aws_cloud.py b/qds_sdk/cloud/aws_cloud.py
index f2460551..88c20ef3 100755
--- a/qds_sdk/cloud/aws_cloud.py
+++ b/qds_sdk/cloud/aws_cloud.py
@@ -107,12 +107,12 @@ def set_cloud_config_from_arguments(self, arguments):
                               bastion_node_public_dns=arguments.bastion_node_public_dns,
                               master_elastic_ip=arguments.master_elastic_ip)
 
-    def set_composition_arguments(self, composition_group)
+    def set_composition_arguments(self, composition_group):
         composition_group.add_argument("--master-type",
-                                      dest="master_type",
-                                      choices=["ondemand", "spot", "spotblock"],
-                                      default="ondemand",
-                                      help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
+                                       dest="master_type",
+                                       choices=["ondemand", "spot", "spotblock"],
+                                       default="ondemand",
+                                       help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
                                        "default: ondemand")
         composition_group.add_argument("--master-spot-block-duration",
                                        dest="master_spot_block_duration",
diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py
index 60cee68d..30220512 100755
--- a/qds_sdk/cloud/azure_cloud.py
+++ b/qds_sdk/cloud/azure_cloud.py
@@ -153,42 +153,40 @@ def set_cloud_config_from_arguments(self, arguments):
                               master_static_public_ip_name=arguments.master_static_public_ip_name,
                               resource_group_name=arguments.resource_group_name)
 
-    def set_composition_arguments(self, composition_group):
+    def set_composition_arguments(self, comp_group):
         #composition arguments we want to accept for azure
-        composition_group.add_argument("--min-ondemand-percentage",
-                                       dest="min_ondemand_percentage",
-                                       type=int, default =0,
-                                       help="Percentage of ondemand nodes in min config.")
-        composition_group.add_argument("--min-spot-percentage",
-                                       dest="min_spot_percentage",
-                                       type=int, default = 0,
-                                       help="Percentage of spot nodes in min config.")
-        composition_group.add_argument("--max-price-percentage",
-                                       dest="max_price_percentage",
-                                       type=int, default = 100,
-                                       help="Percentage of maximum price percentage for spot nodes.")
-        composition_group.add_argument("--min-spot-fallback",
-                                       dest="min_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="Whether to fallback to on-demand instances for min nodes" +
-                                            " if spot instances aren't available.")
-
-
-        composition_group.add_argument("--autoscaling-ondemand-percentage",
-                                       dest="autoscaling_ondemand_percentage",
-                                       type=int, default=0,
-                                       help="Percentage of ondemand nodes in autoscaling config.")
-        composition_group.add_argument("--autoscaling-spot-percentage",
-                                       dest="autoscaling_spot_percentage",
-                                       type=int, default=0,
-                                       help="Percentage of spot nodes in autoscaling config.")
-        composition_group.add_argument("--autoscaling-spot-fallback",
-                                       dest="autoscaling_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="Whether to fallback to on-demand instances for autoscaling nodes" +
-                                            " if spot instances aren't available.")
+        comp_group.add_argument("--min-ondemand-percentage",
+                                dest="min_ondemand_percentage",
+                                type=int, default =0,
+                                help="Percentage of ondemand nodes in min config.")
+        comp_group.add_argument("--min-spot-percentage",
+                                dest="min_spot_percentage",
+                                type=int, default = 0,
+                                help="Percentage of spot nodes in min config.")
+        comp_group.add_argument("--max-price-percentage",
+                                dest="max_price_percentage",
+                                type=int, default = 100,
+                                help="Percentage of maximum price percentage for spot nodes.")
+        comp_group.add_argument("--min-spot-fallback",
+                                dest="min_spot_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="Whether to fallback to on-demand instances for min nodes" +
+                                      " if spot instances aren't available.")
+        comp_group.add_argument("--autoscaling-ondemand-percentage",
+                                dest="autoscaling_ondemand_percentage",
+                                type=int, default=0,
+                                help="Percentage of ondemand nodes in autoscaling config.")
+        comp_group.add_argument("--autoscaling-spot-percentage",
+                                dest="autoscaling_spot_percentage",
+                                type=int, default=0,
+                                help="Percentage of spot nodes in autoscaling config.")
+        comp_group.add_argument("--autoscaling-spot-fallback",
+                                dest="autoscaling_spot_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="Whether to fallback to on-demand instances for autoscaling nodes" +
+                                     " if spot instances aren't available.")
 
     #Ignore other key-value arguments.
     def get_composition(self,
diff --git a/qds_sdk/cloud/cloud.py b/qds_sdk/cloud/cloud.py
index d9981ea5..a21bf01f 100755
--- a/qds_sdk/cloud/cloud.py
+++ b/qds_sdk/cloud/cloud.py
@@ -9,5 +9,5 @@ def set_cloud_config_from_arguments(self, arguments):
     def set_composition_arguments(self, set_composition_arguments):
     	pass
 
-    def get_composition(*args, **kwargs):
+    def get_composition(self, *args, **kwargs):
     	return {}
diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index 067fb68c..20e4dd24 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -60,27 +60,27 @@ def set_cluster_info_from_arguments(self, arguments):
         if Qubole.get_cloud_name() == "aws":
             #Need to move to aws cloud.
             self.set_composition(master_type=arguments.master_type,
-                             master_spot_block_duration=arguments.master_spot_block_duration,
-                             master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
-                             master_timeout_for_request=arguments.master_timeout_for_request,
-                             master_spot_fallback=arguments.master_spot_fallback,
-                             min_ondemand_percentage=arguments.min_ondemand_percentage,
-                             min_spot_block_percentage=arguments.min_spot_block_percentage,
-                             min_spot_block_duration=arguments.min_spot_block_duration,
-                             min_spot_percentage=arguments.min_spot_percentage,
-                             min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage,
-                             min_timeout_for_request=arguments.min_timeout_for_request,
-                             min_spot_allocation_strategy=arguments.min_spot_allocation_strategy,
-                             min_spot_fallback=arguments.min_spot_fallback,
-                             autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage,
-                             autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage,
-                             autoscaling_spot_percentage=arguments.autoscaling_spot_percentage,
-                             autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration,
-                             autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage,
-                             autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request,
-                             autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy,
-                             autoscaling_spot_fallback=arguments.autoscaling_spot_fallback,
-                             autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback)
+                                 master_spot_block_duration=arguments.master_spot_block_duration,
+                                 master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
+                                 master_timeout_for_request=arguments.master_timeout_for_request,
+                                 master_spot_fallback=arguments.master_spot_fallback,
+                                 min_ondemand_percentage=arguments.min_ondemand_percentage,
+                                 min_spot_block_percentage=arguments.min_spot_block_percentage,
+                                 min_spot_block_duration=arguments.min_spot_block_duration,
+                                 min_spot_percentage=arguments.min_spot_percentage,
+                                 min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage,
+                                 min_timeout_for_request=arguments.min_timeout_for_request,
+                                 min_spot_allocation_strategy=arguments.min_spot_allocation_strategy,
+                                 min_spot_fallback=arguments.min_spot_fallback,
+                                 autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage,
+                                 autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage,
+                                 autoscaling_spot_percentage=arguments.autoscaling_spot_percentage,
+                                 autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration,
+                                 autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage,
+                                 autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request,
+                                 autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy,
+                                 autoscaling_spot_fallback=arguments.autoscaling_spot_fallback,
+                                 autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback)
         else:
             self.set_composition_from_cloud_using_parser(arguments)
 
@@ -233,8 +233,8 @@ def set_cluster_info(self,
                                      disable_autoscale_node_pause, paused_autoscale_node_timeout_mins)
 
     def set_composition_from_cloud_using_parser(self, arguments):
-        self.set_composition_for_cluster(**{k:v for k,v in arguments.__dict__.items() \
-                                             if v != None})
+        self.set_composition_for_cluster(**{k:v for k,v in arguments.__dict__.items()
+                                            if v != None})
 
     def set_composition_for_cluster(self, **kwargs):
         cloud = Qubole.get_cloud()

From 2a30d2fc5a941d09fa16bd5aee21d1165a53943b Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 18:55:24 -0700
Subject: [PATCH 04/14] chg

---
 qds_sdk/cloud/azure_cloud.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py
index 30220512..a7b4ba96 100755
--- a/qds_sdk/cloud/azure_cloud.py
+++ b/qds_sdk/cloud/azure_cloud.py
@@ -200,24 +200,30 @@ def get_composition(self,
                         **kwargs):
         composition = {}
         composition["min_nodes"] = {"nodes": []}
+        min_nodes = composition["min_nodes"]["nodes"]
         if min_ondemand_percentage + min_spot_percentage != 100:
-            raise ValueError("Minimum nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
-                            % (min_ondemand_percentage,min_spot_percentage))
+            raise ValueError("Minimum nodes ondemand+spot percentage should be 100:"
+                             "Ondemand pct: %d Spot pct: %d"
+                             %(min_ondemand_percentage,min_spot_percentage))
         if min_ondemand_percentage > 0:
-            composition["min_nodes"]["nodes"].append({"type":"ondemand", "percentage":min_ondemand_percentage})
+            min_nodes.append({"type":"ondemand", "percentage":min_ondemand_percentage})
         if min_spot_percentage > 0:
-            composition["min_nodes"]["nodes"].append({"type":"spot", "percentage":min_spot_percentage,
-                                                   "fallback": min_spot_fallback, "max_price_percentage": max_price_percentage})
+            min_nodes.append({"type":"spot", "percentage":min_spot_percentage,
+                              "fallback": min_spot_fallback,
+                              "max_price_percentage": max_price_percentage})
 
         composition["autoscaling_nodes"] = {"nodes": []}
+        autoscaling_nodes = composition["autoscaling_nodes"]["nodes"]
         if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
-            raise ValueError("Autoscaling nodes ondemand+spot percentage should be 100: Ondemand pct: %d Spot pct: %d"
-                            % (autoscaling_ondemand_percentage,autoscaling_spot_percentage))
+            raise ValueError("Autoscaling nodes ondemand+spot percentage should be 100:" +
+                             " Ondemand pct: %d Spot pct: %d"
+                             %(autoscaling_ondemand_percentage,autoscaling_spot_percentage))
         if autoscaling_ondemand_percentage > 0:
-            composition["autoscaling_nodes"]["nodes"].append({"type":"ondemand", "percentage":autoscaling_ondemand_percentage})
+            autoscaling_nodes.append({"type":"ondemand", "percentage":autoscaling_ondemand_percentage})
         if autoscaling_spot_percentage > 0:
-            composition["autoscaling_nodes"]["nodes"].append({"type":"spot", "percentage":autoscaling_spot_percentage,
-                                                   "fallback": autoscaling_spot_fallback, "max_price_percentage": max_price_percentage})
+            autoscaling_nodes.append({"type":"spot", "percentage":autoscaling_spot_percentage,
+                                      "fallback": autoscaling_spot_fallback,
+                                      "max_price_percentage": max_price_percentage})
         return composition
 
     def create_parser(self, argparser):

From ccd57559e3b2954fc50d0f55b524f78c473ec45e Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 19:52:09 -0700
Subject: [PATCH 05/14] chg

---
 qds_sdk/cloud/aws_cloud.py   | 234 +++++++++++++++++------------------
 qds_sdk/cloud/azure_cloud.py |  36 +++---
 qds_sdk/cloud/cloud.py       |   4 +-
 3 files changed, 141 insertions(+), 133 deletions(-)

diff --git a/qds_sdk/cloud/aws_cloud.py b/qds_sdk/cloud/aws_cloud.py
index 88c20ef3..d333e0ea 100755
--- a/qds_sdk/cloud/aws_cloud.py
+++ b/qds_sdk/cloud/aws_cloud.py
@@ -107,123 +107,123 @@ def set_cloud_config_from_arguments(self, arguments):
                               bastion_node_public_dns=arguments.bastion_node_public_dns,
                               master_elastic_ip=arguments.master_elastic_ip)
 
-    def set_composition_arguments(self, composition_group):
-        composition_group.add_argument("--master-type",
-                                       dest="master_type",
-                                       choices=["ondemand", "spot", "spotblock"],
-                                       default="ondemand",
-                                       help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
-                                       "default: ondemand")
-        composition_group.add_argument("--master-spot-block-duration",
-                                       dest="master_spot_block_duration",
-                                       type=int,
-                                       default=120,
-                                       help="spot block duration unit: minutes")
-        composition_group.add_argument("--master-maximum-bid-price-percentage",
-                                       dest="master_maximum_bid_price_percentage",
-                                       type=int,
-                                       default=100,
-                                       help="maximum value to bid for master spot instances" +
-                                            " expressed as a percentage of the base" +
-                                            " price for the master instance types")
-        composition_group.add_argument("--master-timeout-for-request",
-                                       dest="master_timeout_for_request",
-                                       type=int,
-                                       default=1,
-                                       help="timeout for a master spot instance request, unit: minutes")
-        composition_group.add_argument("--master-spot-fallback",
-                                       dest="master_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for master nodes" +
-                                            " if spot instances aren't available")
-        composition_group.add_argument("--min-ondemand-percentage",
-                                       dest="min_ondemand_percentage",
-                                       type=int,
-                                       help="percentage of ondemand nodes in min config")
-        composition_group.add_argument("--min-spot-block-percentage",
-                                       dest="min_spot_block_percentage",
-                                       type=int,
-                                       help="percentage of spot block nodes in min config")
-        composition_group.add_argument("--min-spot-percentage",
-                                       dest="min_spot_percentage",
-                                       type=int,
-                                       help="percentage of spot nodes in min config")
-        composition_group.add_argument("--min-spot-block-duration",
-                                       dest="min_spot_block_duration",
-                                       type=int,
-                                       default=120,
-                                       help="spot block duration unit: minutes")
-        composition_group.add_argument("--min-maximum-bid-price-percentage",
-                                       dest="min_maximum_bid_price_percentage",
-                                       type=int,
-                                       default=100,
-                                       help="maximum value to bid for min spot instances" +
-                                            " expressed as a percentage of the base" +
-                                            " price for the master instance types")
-        composition_group.add_argument("--min-timeout-for-request",
-                                       dest="min_timeout_for_request",
-                                       type=int,
-                                       default=1,
-                                       help="timeout for a min spot instance request, unit: minutes")
-        composition_group.add_argument("--min-spot-fallback",
-                                       dest="min_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for min nodes" +
-                                            " if spot instances aren't available")
-        composition_group.add_argument("--min-spot-allocation-strategy",
-                                       dest="min_spot_allocation_strategy",
-                                       choices=["lowestPrice", "capacityOptimized", None],
-                                       default=None,
-                                       help="allocation strategy for min spot nodes")
-        composition_group.add_argument("--autoscaling-ondemand-percentage",
-                                       dest="autoscaling_ondemand_percentage",
-                                       type=int,
-                                       help="percentage of ondemand nodes in autoscaling config")
-        composition_group.add_argument("--autoscaling-spot-block-percentage",
-                                       dest="autoscaling_spot_block_percentage",
-                                       type=int,
-                                       help="percentage of spot block nodes in autoscaling config")
-        composition_group.add_argument("--autoscaling-spot-percentage",
-                                       dest="autoscaling_spot_percentage",
-                                       type=int,
-                                       help="percentage of spot nodes in autoscaling config")
-        composition_group.add_argument("--autoscaling-spot-block-duration",
-                                       dest="autoscaling_spot_block_duration",
-                                       type=int,
-                                       default=120,
-                                       help="spot block duration unit:  minutes")
-        composition_group.add_argument("--autoscaling-spot-block-fallback",
-                                       dest="autoscaling_spot_block_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for autoscaling" +
-                                            " nodes if spot block instances aren't available")
-        composition_group.add_argument("--autoscaling-maximum-bid-price-percentage",
-                                       dest="autoscaling_maximum_bid_price_percentage",
-                                       type=int,
-                                       default=100,
-                                       help="maximum value to bid for autoscaling spot instances" +
-                                            " expressed as a percentage of the base" +
-                                            " price for the master instance types")
-        composition_group.add_argument("--autoscaling-timeout-for-request",
-                                       dest="autoscaling_timeout_for_request",
-                                       type=int,
-                                       default=1,
-                                       help="timeout for a autoscaling spot instance request, unit: minutes")
-        composition_group.add_argument("--autoscaling-spot-fallback",
-                                       dest="autoscaling_spot_fallback",
-                                       choices=["ondemand", None],
-                                       default=None,
-                                       help="whether to fallback to on-demand instances for autoscaling nodes" +
-                                            " if spot instances aren't available")
-        composition_group.add_argument("--autoscaling-spot-allocation-strategy",
-                                       dest="autoscaling_spot_allocation_strategy",
-                                       choices=["lowestPrice", "capacityOptimized", None],
-                                       default=None,
-                                       help="allocation strategy for autoscaling" +
-                                            " spot nodes")
+    def set_composition_arguments(self, comp_group):
+        comp_group.add_argument("--master-type",
+                                dest="master_type",
+                                choices=["ondemand", "spot", "spotblock"],
+                                default="ondemand",
+                                help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
+                                "default: ondemand")
+        comp_group.add_argument("--master-spot-block-duration",
+                                dest="master_spot_block_duration",
+                                type=int,
+                                default=120,
+                                help="spot block duration unit: minutes")
+        comp_group.add_argument("--master-maximum-bid-price-percentage",
+                                dest="master_maximum_bid_price_percentage",
+                                type=int,
+                                default=100,
+                                help="maximum value to bid for master spot instances" +
+                                     " expressed as a percentage of the base" +
+                                     " price for the master instance types")
+        comp_group.add_argument("--master-timeout-for-request",
+                                dest="master_timeout_for_request",
+                                type=int,
+                                default=1,
+                                help="timeout for a master spot instance request, unit: minutes")
+        comp_group.add_argument("--master-spot-fallback",
+                                dest="master_spot_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="whether to fallback to on-demand instances for master nodes" +
+                                     " if spot instances aren't available")
+        comp_group.add_argument("--min-ondemand-percentage",
+                                dest="min_ondemand_percentage",
+                                type=int,
+                                help="percentage of ondemand nodes in min config")
+        comp_group.add_argument("--min-spot-block-percentage",
+                                dest="min_spot_block_percentage",
+                                type=int,
+                                help="percentage of spot block nodes in min config")
+        comp_group.add_argument("--min-spot-percentage",
+                                dest="min_spot_percentage",
+                                type=int,
+                                help="percentage of spot nodes in min config")
+        comp_group.add_argument("--min-spot-block-duration",
+                                dest="min_spot_block_duration",
+                                type=int,
+                                default=120,
+                                help="spot block duration unit: minutes")
+        comp_group.add_argument("--min-maximum-bid-price-percentage",
+                                dest="min_maximum_bid_price_percentage",
+                                type=int,
+                                default=100,
+                                help="maximum value to bid for min spot instances" +
+                                     " expressed as a percentage of the base" +
+                                     " price for the master instance types")
+        comp_group.add_argument("--min-timeout-for-request",
+                                dest="min_timeout_for_request",
+                                type=int,
+                                default=1,
+                                help="timeout for a min spot instance request, unit: minutes")
+        comp_group.add_argument("--min-spot-fallback",
+                                dest="min_spot_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="whether to fallback to on-demand instances for min nodes" +
+                                     " if spot instances aren't available")
+        comp_group.add_argument("--min-spot-allocation-strategy",
+                                dest="min_spot_allocation_strategy",
+                                choices=["lowestPrice", "capacityOptimized", None],
+                                default=None,
+                                help="allocation strategy for min spot nodes")
+        comp_group.add_argument("--autoscaling-ondemand-percentage",
+                                dest="autoscaling_ondemand_percentage",
+                                type=int,
+                                help="percentage of ondemand nodes in autoscaling config")
+        comp_group.add_argument("--autoscaling-spot-block-percentage",
+                                dest="autoscaling_spot_block_percentage",
+                                type=int,
+                                help="percentage of spot block nodes in autoscaling config")
+        comp_group.add_argument("--autoscaling-spot-percentage",
+                                dest="autoscaling_spot_percentage",
+                                type=int,
+                                help="percentage of spot nodes in autoscaling config")
+        comp_group.add_argument("--autoscaling-spot-block-duration",
+                                dest="autoscaling_spot_block_duration",
+                                type=int,
+                                default=120,
+                                help="spot block duration unit:  minutes")
+        comp_group.add_argument("--autoscaling-spot-block-fallback",
+                                dest="autoscaling_spot_block_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="whether to fallback to on-demand instances for autoscaling" +
+                                     " nodes if spot block instances aren't available")
+        comp_group.add_argument("--autoscaling-maximum-bid-price-percentage",
+                                dest="autoscaling_maximum_bid_price_percentage",
+                                type=int,
+                                default=100,
+                                help="maximum value to bid for autoscaling spot instances" +
+                                     " expressed as a percentage of the base" +
+                                     " price for the master instance types")
+        comp_group.add_argument("--autoscaling-timeout-for-request",
+                               dest="autoscaling_timeout_for_request",
+                                type=int,
+                                default=1,
+                                help="timeout for a autoscaling spot instance request, unit: minutes")
+        comp_group.add_argument("--autoscaling-spot-fallback",
+                                 dest="autoscaling_spot_fallback",
+                                 choices=["ondemand", None],
+                                 default=None,
+                                 help="whether to fallback to on-demand instances for autoscaling nodes" +
+                                     " if spot instances aren't available")
+        comp_group.add_argument("--autoscaling-spot-allocation-strategy",
+                                 dest="autoscaling_spot_allocation_strategy",
+                                 choices=["lowestPrice", "capacityOptimized", None],
+                                 default=None,
+                                 help="allocation strategy for autoscaling" +
+                                      " spot nodes")
 
     def create_parser(self, argparser):
         # compute settings parser
diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py
index a7b4ba96..51c94f74 100755
--- a/qds_sdk/cloud/azure_cloud.py
+++ b/qds_sdk/cloud/azure_cloud.py
@@ -166,27 +166,32 @@ def set_composition_arguments(self, comp_group):
         comp_group.add_argument("--max-price-percentage",
                                 dest="max_price_percentage",
                                 type=int, default = 100,
-                                help="Percentage of maximum price percentage for spot nodes.")
+                                help="Percentage of maximum price percentage"
+                                     " for spot nodes.")
         comp_group.add_argument("--min-spot-fallback",
                                 dest="min_spot_fallback",
                                 choices=["ondemand", None],
                                 default=None,
-                                help="Whether to fallback to on-demand instances for min nodes" +
-                                      " if spot instances aren't available.")
+                                help="Whether to fallback to on-demand instances" +
+                                      " for min nodes if spot instances" +
+                                      " aren't available.")
         comp_group.add_argument("--autoscaling-ondemand-percentage",
                                 dest="autoscaling_ondemand_percentage",
                                 type=int, default=0,
-                                help="Percentage of ondemand nodes in autoscaling config.")
+                                help="Percentage of ondemand nodes" +
+                                     "in autoscaling config.")
         comp_group.add_argument("--autoscaling-spot-percentage",
                                 dest="autoscaling_spot_percentage",
                                 type=int, default=0,
-                                help="Percentage of spot nodes in autoscaling config.")
+                                help="Percentage of spot nodes" +
+                                "in autoscaling config.")
         comp_group.add_argument("--autoscaling-spot-fallback",
                                 dest="autoscaling_spot_fallback",
                                 choices=["ondemand", None],
                                 default=None,
-                                help="Whether to fallback to on-demand instances for autoscaling nodes" +
-                                     " if spot instances aren't available.")
+                                help="Whether to fallback to on-demand instances" +
+                                     " for autoscaling nodes if spot instances" +
+                                     " aren't available.")
 
     #Ignore other key-value arguments.
     def get_composition(self,
@@ -202,8 +207,8 @@ def get_composition(self,
         composition["min_nodes"] = {"nodes": []}
         min_nodes = composition["min_nodes"]["nodes"]
         if min_ondemand_percentage + min_spot_percentage != 100:
-            raise ValueError("Minimum nodes ondemand+spot percentage should be 100:"
-                             "Ondemand pct: %d Spot pct: %d"
+            raise ValueError("Minimum nodes ondemand+spot percentage"
+                             " should be 100: Ondemand pct: %d Spot pct: %d"
                              %(min_ondemand_percentage,min_spot_percentage))
         if min_ondemand_percentage > 0:
             min_nodes.append({"type":"ondemand", "percentage":min_ondemand_percentage})
@@ -215,13 +220,16 @@ def get_composition(self,
         composition["autoscaling_nodes"] = {"nodes": []}
         autoscaling_nodes = composition["autoscaling_nodes"]["nodes"]
         if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
-            raise ValueError("Autoscaling nodes ondemand+spot percentage should be 100:" +
-                             " Ondemand pct: %d Spot pct: %d"
-                             %(autoscaling_ondemand_percentage,autoscaling_spot_percentage))
+            raise ValueError("Autoscaling nodes ondemand+spot percentage" +
+                             " should be 100: Ondemand pct: %d Spot pct: %d"
+                             %(autoscaling_ondemand_percentage,
+                               autoscaling_spot_percentage))
         if autoscaling_ondemand_percentage > 0:
-            autoscaling_nodes.append({"type":"ondemand", "percentage":autoscaling_ondemand_percentage})
+            autoscaling_nodes.append({"type":"ondemand",
+                                      "percentage":autoscaling_ondemand_percentage})
         if autoscaling_spot_percentage > 0:
-            autoscaling_nodes.append({"type":"spot", "percentage":autoscaling_spot_percentage,
+            autoscaling_nodes.append({"type":"spot",
+                                      "percentage":autoscaling_spot_percentage,
                                       "fallback": autoscaling_spot_fallback,
                                       "max_price_percentage": max_price_percentage})
         return composition
diff --git a/qds_sdk/cloud/cloud.py b/qds_sdk/cloud/cloud.py
index a21bf01f..b26e49b7 100755
--- a/qds_sdk/cloud/cloud.py
+++ b/qds_sdk/cloud/cloud.py
@@ -7,7 +7,7 @@ def set_cloud_config_from_arguments(self, arguments):
         return NotImplemented
 
     def set_composition_arguments(self, set_composition_arguments):
-    	pass
+        pass
 
     def get_composition(self, *args, **kwargs):
-    	return {}
+        return {}

From 8b42bc153882f29f14064ce4da07d46fb36609e1 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 20:03:39 -0700
Subject: [PATCH 06/14] chg

---
 qds_sdk/cloud/aws_cloud.py   | 20 ++++++++++----------
 qds_sdk/cloud/azure_cloud.py | 20 +++++++++++---------
 qds_sdk/cloud/cloud.py       |  2 +-
 qds_sdk/cluster_info_v22.py  | 20 +++++++++++++-------
 4 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/qds_sdk/cloud/aws_cloud.py b/qds_sdk/cloud/aws_cloud.py
index d333e0ea..4d3cbcc9 100755
--- a/qds_sdk/cloud/aws_cloud.py
+++ b/qds_sdk/cloud/aws_cloud.py
@@ -208,22 +208,22 @@ def set_composition_arguments(self, comp_group):
                                      " expressed as a percentage of the base" +
                                      " price for the master instance types")
         comp_group.add_argument("--autoscaling-timeout-for-request",
-                               dest="autoscaling_timeout_for_request",
+                                dest="autoscaling_timeout_for_request",
                                 type=int,
                                 default=1,
                                 help="timeout for a autoscaling spot instance request, unit: minutes")
         comp_group.add_argument("--autoscaling-spot-fallback",
-                                 dest="autoscaling_spot_fallback",
-                                 choices=["ondemand", None],
-                                 default=None,
-                                 help="whether to fallback to on-demand instances for autoscaling nodes" +
+                                dest="autoscaling_spot_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="whether to fallback to on-demand instances for autoscaling nodes" +
                                      " if spot instances aren't available")
         comp_group.add_argument("--autoscaling-spot-allocation-strategy",
-                                 dest="autoscaling_spot_allocation_strategy",
-                                 choices=["lowestPrice", "capacityOptimized", None],
-                                 default=None,
-                                 help="allocation strategy for autoscaling" +
-                                      " spot nodes")
+                                dest="autoscaling_spot_allocation_strategy",
+                                choices=["lowestPrice", "capacityOptimized", None],
+                                default=None,
+                                help="allocation strategy for autoscaling" +
+                                     " spot nodes")
 
     def create_parser(self, argparser):
         # compute settings parser
diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py
index 51c94f74..0f21a516 100755
--- a/qds_sdk/cloud/azure_cloud.py
+++ b/qds_sdk/cloud/azure_cloud.py
@@ -173,8 +173,8 @@ def set_composition_arguments(self, comp_group):
                                 choices=["ondemand", None],
                                 default=None,
                                 help="Whether to fallback to on-demand instances" +
-                                      " for min nodes if spot instances" +
-                                      " aren't available.")
+                                     " for min nodes if spot instances" +
+                                     " aren't available.")
         comp_group.add_argument("--autoscaling-ondemand-percentage",
                                 dest="autoscaling_ondemand_percentage",
                                 type=int, default=0,
@@ -209,11 +209,13 @@ def get_composition(self,
         if min_ondemand_percentage + min_spot_percentage != 100:
             raise ValueError("Minimum nodes ondemand+spot percentage"
                              " should be 100: Ondemand pct: %d Spot pct: %d"
-                             %(min_ondemand_percentage,min_spot_percentage))
+                             %(min_ondemand_percentage,
+                               min_spot_percentage))
         if min_ondemand_percentage > 0:
-            min_nodes.append({"type":"ondemand", "percentage":min_ondemand_percentage})
+            min_nodes.append({"type": "ondemand",
+                              "percentage": min_ondemand_percentage})
         if min_spot_percentage > 0:
-            min_nodes.append({"type":"spot", "percentage":min_spot_percentage,
+            min_nodes.append({"type": "spot", "percentage": min_spot_percentage,
                               "fallback": min_spot_fallback,
                               "max_price_percentage": max_price_percentage})
 
@@ -225,11 +227,11 @@ def get_composition(self,
                              %(autoscaling_ondemand_percentage,
                                autoscaling_spot_percentage))
         if autoscaling_ondemand_percentage > 0:
-            autoscaling_nodes.append({"type":"ondemand",
-                                      "percentage":autoscaling_ondemand_percentage})
+            autoscaling_nodes.append({"type": "ondemand",
+                                      "percentage": autoscaling_ondemand_percentage})
         if autoscaling_spot_percentage > 0:
-            autoscaling_nodes.append({"type":"spot",
-                                      "percentage":autoscaling_spot_percentage,
+            autoscaling_nodes.append({"type": "spot",
+                                      "percentage": autoscaling_spot_percentage,
                                       "fallback": autoscaling_spot_fallback,
                                       "max_price_percentage": max_price_percentage})
         return composition
diff --git a/qds_sdk/cloud/cloud.py b/qds_sdk/cloud/cloud.py
index b26e49b7..fda9133d 100755
--- a/qds_sdk/cloud/cloud.py
+++ b/qds_sdk/cloud/cloud.py
@@ -10,4 +10,4 @@ def set_composition_arguments(self, set_composition_arguments):
         pass
 
     def get_composition(self, *args, **kwargs):
-        return {}
+        pass
diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index 20e4dd24..53b75325 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -60,10 +60,14 @@ def set_cluster_info_from_arguments(self, arguments):
         if Qubole.get_cloud_name() == "aws":
             #Need to move to aws cloud.
             self.set_composition(master_type=arguments.master_type,
-                                 master_spot_block_duration=arguments.master_spot_block_duration,
-                                 master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
-                                 master_timeout_for_request=arguments.master_timeout_for_request,
-                                 master_spot_fallback=arguments.master_spot_fallback,
+                                 master_spot_block_duration=
+                                    arguments.master_spot_block_duration,
+                                 master_maximum_bid_price_percentage=
+                                    arguments.master_maximum_bid_price_percentage,
+                                 master_timeout_for_request=
+                                    arguments.master_timeout_for_request,
+                                 master_spot_fallback=
+                                    arguments.master_spot_fallback,
                                  min_ondemand_percentage=arguments.min_ondemand_percentage,
                                  min_spot_block_percentage=arguments.min_spot_block_percentage,
                                  min_spot_block_duration=arguments.min_spot_block_duration,
@@ -233,12 +237,14 @@ def set_cluster_info(self,
                                      disable_autoscale_node_pause, paused_autoscale_node_timeout_mins)
 
     def set_composition_from_cloud_using_parser(self, arguments):
-        self.set_composition_for_cluster(**{k:v for k,v in arguments.__dict__.items()
-                                            if v != None})
+        self.set_composition_for_cluster(**{k: v for k, v in arguments.__dict__.items()
+                                            if v is not None})
 
     def set_composition_for_cluster(self, **kwargs):
         cloud = Qubole.get_cloud()
-        self.cluster_info["composition"] = cloud.get_composition(kwargs)
+        composition = cloud.get_composition(kwargs)
+        if composition is not None:
+            self.cluster_info["composition"] = composition
 
     def set_composition(self,
                         master_type="ondemand",

From 8d00ec5f9e2ff317b414b3422a73741d56407602 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 20:11:50 -0700
Subject: [PATCH 07/14] chg

---
 qds_sdk/cloud/aws_cloud.py   |  6 +++---
 qds_sdk/cloud/azure_cloud.py | 18 +++++++++---------
 qds_sdk/cloud/cloud.py       |  4 ++--
 qds_sdk/cluster_info_v22.py  | 14 +++++---------
 4 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/qds_sdk/cloud/aws_cloud.py b/qds_sdk/cloud/aws_cloud.py
index 4d3cbcc9..722c40d5 100755
--- a/qds_sdk/cloud/aws_cloud.py
+++ b/qds_sdk/cloud/aws_cloud.py
@@ -204,9 +204,9 @@ def set_composition_arguments(self, comp_group):
                                 dest="autoscaling_maximum_bid_price_percentage",
                                 type=int,
                                 default=100,
-                                help="maximum value to bid for autoscaling spot instances" +
-                                     " expressed as a percentage of the base" +
-                                     " price for the master instance types")
+                                help="maximum value to bid for autoscaling spot" +
+                                     " instances expressed as a percentage of" +
+                                     " the base price for the master instance types")
         comp_group.add_argument("--autoscaling-timeout-for-request",
                                 dest="autoscaling_timeout_for_request",
                                 type=int,
diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py
index 0f21a516..2214dee0 100755
--- a/qds_sdk/cloud/azure_cloud.py
+++ b/qds_sdk/cloud/azure_cloud.py
@@ -154,18 +154,18 @@ def set_cloud_config_from_arguments(self, arguments):
                               resource_group_name=arguments.resource_group_name)
 
     def set_composition_arguments(self, comp_group):
-        #composition arguments we want to accept for azure
+        # composition arguments we want to accept for azure
         comp_group.add_argument("--min-ondemand-percentage",
                                 dest="min_ondemand_percentage",
-                                type=int, default =0,
+                                type=int, default=0,
                                 help="Percentage of ondemand nodes in min config.")
         comp_group.add_argument("--min-spot-percentage",
                                 dest="min_spot_percentage",
-                                type=int, default = 0,
+                                type=int, default=0,
                                 help="Percentage of spot nodes in min config.")
         comp_group.add_argument("--max-price-percentage",
                                 dest="max_price_percentage",
-                                type=int, default = 100,
+                                type=int, default=100,
                                 help="Percentage of maximum price percentage"
                                      " for spot nodes.")
         comp_group.add_argument("--min-spot-fallback",
@@ -193,7 +193,7 @@ def set_composition_arguments(self, comp_group):
                                      " for autoscaling nodes if spot instances" +
                                      " aren't available.")
 
-    #Ignore other key-value arguments.
+    # Ignore other key-value arguments.
     def get_composition(self,
                         min_ondemand_percentage=0,
                         min_spot_percentage=0,
@@ -209,8 +209,8 @@ def get_composition(self,
         if min_ondemand_percentage + min_spot_percentage != 100:
             raise ValueError("Minimum nodes ondemand+spot percentage"
                              " should be 100: Ondemand pct: %d Spot pct: %d"
-                             %(min_ondemand_percentage,
-                               min_spot_percentage))
+                             % (min_ondemand_percentage,
+                                min_spot_percentage))
         if min_ondemand_percentage > 0:
             min_nodes.append({"type": "ondemand",
                               "percentage": min_ondemand_percentage})
@@ -224,8 +224,8 @@ def get_composition(self,
         if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
             raise ValueError("Autoscaling nodes ondemand+spot percentage" +
                              " should be 100: Ondemand pct: %d Spot pct: %d"
-                             %(autoscaling_ondemand_percentage,
-                               autoscaling_spot_percentage))
+                             % (autoscaling_ondemand_percentage,
+                                autoscaling_spot_percentage))
         if autoscaling_ondemand_percentage > 0:
             autoscaling_nodes.append({"type": "ondemand",
                                       "percentage": autoscaling_ondemand_percentage})
diff --git a/qds_sdk/cloud/cloud.py b/qds_sdk/cloud/cloud.py
index fda9133d..31831283 100755
--- a/qds_sdk/cloud/cloud.py
+++ b/qds_sdk/cloud/cloud.py
@@ -6,8 +6,8 @@ def create_parser(self, argparser):
     def set_cloud_config_from_arguments(self, arguments):
         return NotImplemented
 
-    def set_composition_arguments(self, set_composition_arguments):
+    def set_composition_arguments(self, comp_group):
         pass
 
-    def get_composition(self, *args, **kwargs):
+    def get_composition(self, **kwargs):
         pass
diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index 53b75325..a0b68298 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -58,16 +58,12 @@ def set_cluster_info_from_arguments(self, arguments):
                               parent_cluster_id=arguments.parent_cluster_id,
                               image_version=arguments.image_version)
         if Qubole.get_cloud_name() == "aws":
-            #Need to move to aws cloud.
+            # Need to move to aws cloud.
             self.set_composition(master_type=arguments.master_type,
-                                 master_spot_block_duration=
-                                    arguments.master_spot_block_duration,
-                                 master_maximum_bid_price_percentage=
-                                    arguments.master_maximum_bid_price_percentage,
-                                 master_timeout_for_request=
-                                    arguments.master_timeout_for_request,
-                                 master_spot_fallback=
-                                    arguments.master_spot_fallback,
+                                 master_spot_block_duration=arguments.master_spot_block_duration,
+                                 master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
+                                 master_timeout_for_request=arguments.master_timeout_for_request,
+                                 master_spot_fallback=arguments.master_spot_fallback,
                                  min_ondemand_percentage=arguments.min_ondemand_percentage,
                                  min_spot_block_percentage=arguments.min_spot_block_percentage,
                                  min_spot_block_duration=arguments.min_spot_block_duration,

From 84c37d04fbf1a1bd7cff485030cc9995f314e10f Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 20:22:18 -0700
Subject: [PATCH 08/14] chg

---
 qds_sdk/cluster_info_v22.py | 45 +++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index a0b68298..f5d70663 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -59,28 +59,29 @@ def set_cluster_info_from_arguments(self, arguments):
                               image_version=arguments.image_version)
         if Qubole.get_cloud_name() == "aws":
             # Need to move to aws cloud.
-            self.set_composition(master_type=arguments.master_type,
-                                 master_spot_block_duration=arguments.master_spot_block_duration,
-                                 master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
-                                 master_timeout_for_request=arguments.master_timeout_for_request,
-                                 master_spot_fallback=arguments.master_spot_fallback,
-                                 min_ondemand_percentage=arguments.min_ondemand_percentage,
-                                 min_spot_block_percentage=arguments.min_spot_block_percentage,
-                                 min_spot_block_duration=arguments.min_spot_block_duration,
-                                 min_spot_percentage=arguments.min_spot_percentage,
-                                 min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage,
-                                 min_timeout_for_request=arguments.min_timeout_for_request,
-                                 min_spot_allocation_strategy=arguments.min_spot_allocation_strategy,
-                                 min_spot_fallback=arguments.min_spot_fallback,
-                                 autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage,
-                                 autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage,
-                                 autoscaling_spot_percentage=arguments.autoscaling_spot_percentage,
-                                 autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration,
-                                 autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage,
-                                 autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request,
-                                 autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy,
-                                 autoscaling_spot_fallback=arguments.autoscaling_spot_fallback,
-                                 autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback)
+            self.set_composition(
+                master_type=arguments.master_type,
+                master_spot_block_duration=arguments.master_spot_block_duration,
+                master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
+                master_timeout_for_request=arguments.master_timeout_for_request,
+                master_spot_fallback=arguments.master_spot_fallback,
+                min_ondemand_percentage=arguments.min_ondemand_percentage,
+                min_spot_block_percentage=arguments.min_spot_block_percentage,
+                min_spot_block_duration=arguments.min_spot_block_duration,
+                min_spot_percentage=arguments.min_spot_percentage,
+                min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage,
+                min_timeout_for_request=arguments.min_timeout_for_request,
+                min_spot_allocation_strategy=arguments.min_spot_allocation_strategy,
+                min_spot_fallback=arguments.min_spot_fallback,
+                autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage,
+                autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage,
+                autoscaling_spot_percentage=arguments.autoscaling_spot_percentage,
+                autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration,
+                autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage,
+                autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request,
+                autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy,
+                autoscaling_spot_fallback=arguments.autoscaling_spot_fallback,
+                autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback)
         else:
             self.set_composition_from_cloud_using_parser(arguments)
 

From 066e0b48c981681dccf30e823566c74e8f707b12 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 20:28:50 -0700
Subject: [PATCH 09/14] chg

---
 qds_sdk/cloud/aws_cloud.py | 255 ++++++++++++++++++++-----------------
 1 file changed, 139 insertions(+), 116 deletions(-)

diff --git a/qds_sdk/cloud/aws_cloud.py b/qds_sdk/cloud/aws_cloud.py
index 722c40d5..6a75ef8a 100755
--- a/qds_sdk/cloud/aws_cloud.py
+++ b/qds_sdk/cloud/aws_cloud.py
@@ -108,122 +108,145 @@ def set_cloud_config_from_arguments(self, arguments):
                               master_elastic_ip=arguments.master_elastic_ip)
 
     def set_composition_arguments(self, comp_group):
-        comp_group.add_argument("--master-type",
-                                dest="master_type",
-                                choices=["ondemand", "spot", "spotblock"],
-                                default="ondemand",
-                                help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" +
-                                "default: ondemand")
-        comp_group.add_argument("--master-spot-block-duration",
-                                dest="master_spot_block_duration",
-                                type=int,
-                                default=120,
-                                help="spot block duration unit: minutes")
-        comp_group.add_argument("--master-maximum-bid-price-percentage",
-                                dest="master_maximum_bid_price_percentage",
-                                type=int,
-                                default=100,
-                                help="maximum value to bid for master spot instances" +
-                                     " expressed as a percentage of the base" +
-                                     " price for the master instance types")
-        comp_group.add_argument("--master-timeout-for-request",
-                                dest="master_timeout_for_request",
-                                type=int,
-                                default=1,
-                                help="timeout for a master spot instance request, unit: minutes")
-        comp_group.add_argument("--master-spot-fallback",
-                                dest="master_spot_fallback",
-                                choices=["ondemand", None],
-                                default=None,
-                                help="whether to fallback to on-demand instances for master nodes" +
-                                     " if spot instances aren't available")
-        comp_group.add_argument("--min-ondemand-percentage",
-                                dest="min_ondemand_percentage",
-                                type=int,
-                                help="percentage of ondemand nodes in min config")
-        comp_group.add_argument("--min-spot-block-percentage",
-                                dest="min_spot_block_percentage",
-                                type=int,
-                                help="percentage of spot block nodes in min config")
-        comp_group.add_argument("--min-spot-percentage",
-                                dest="min_spot_percentage",
-                                type=int,
-                                help="percentage of spot nodes in min config")
-        comp_group.add_argument("--min-spot-block-duration",
-                                dest="min_spot_block_duration",
-                                type=int,
-                                default=120,
-                                help="spot block duration unit: minutes")
-        comp_group.add_argument("--min-maximum-bid-price-percentage",
-                                dest="min_maximum_bid_price_percentage",
-                                type=int,
-                                default=100,
-                                help="maximum value to bid for min spot instances" +
-                                     " expressed as a percentage of the base" +
-                                     " price for the master instance types")
-        comp_group.add_argument("--min-timeout-for-request",
-                                dest="min_timeout_for_request",
-                                type=int,
-                                default=1,
-                                help="timeout for a min spot instance request, unit: minutes")
-        comp_group.add_argument("--min-spot-fallback",
-                                dest="min_spot_fallback",
-                                choices=["ondemand", None],
-                                default=None,
-                                help="whether to fallback to on-demand instances for min nodes" +
-                                     " if spot instances aren't available")
-        comp_group.add_argument("--min-spot-allocation-strategy",
-                                dest="min_spot_allocation_strategy",
-                                choices=["lowestPrice", "capacityOptimized", None],
-                                default=None,
-                                help="allocation strategy for min spot nodes")
-        comp_group.add_argument("--autoscaling-ondemand-percentage",
-                                dest="autoscaling_ondemand_percentage",
-                                type=int,
-                                help="percentage of ondemand nodes in autoscaling config")
-        comp_group.add_argument("--autoscaling-spot-block-percentage",
-                                dest="autoscaling_spot_block_percentage",
-                                type=int,
-                                help="percentage of spot block nodes in autoscaling config")
-        comp_group.add_argument("--autoscaling-spot-percentage",
-                                dest="autoscaling_spot_percentage",
-                                type=int,
-                                help="percentage of spot nodes in autoscaling config")
-        comp_group.add_argument("--autoscaling-spot-block-duration",
-                                dest="autoscaling_spot_block_duration",
-                                type=int,
-                                default=120,
-                                help="spot block duration unit:  minutes")
-        comp_group.add_argument("--autoscaling-spot-block-fallback",
-                                dest="autoscaling_spot_block_fallback",
-                                choices=["ondemand", None],
-                                default=None,
-                                help="whether to fallback to on-demand instances for autoscaling" +
-                                     " nodes if spot block instances aren't available")
-        comp_group.add_argument("--autoscaling-maximum-bid-price-percentage",
-                                dest="autoscaling_maximum_bid_price_percentage",
-                                type=int,
-                                default=100,
-                                help="maximum value to bid for autoscaling spot" +
-                                     " instances expressed as a percentage of" +
-                                     " the base price for the master instance types")
-        comp_group.add_argument("--autoscaling-timeout-for-request",
-                                dest="autoscaling_timeout_for_request",
-                                type=int,
-                                default=1,
-                                help="timeout for a autoscaling spot instance request, unit: minutes")
-        comp_group.add_argument("--autoscaling-spot-fallback",
-                                dest="autoscaling_spot_fallback",
-                                choices=["ondemand", None],
-                                default=None,
-                                help="whether to fallback to on-demand instances for autoscaling nodes" +
-                                     " if spot instances aren't available")
-        comp_group.add_argument("--autoscaling-spot-allocation-strategy",
-                                dest="autoscaling_spot_allocation_strategy",
-                                choices=["lowestPrice", "capacityOptimized", None],
-                                default=None,
-                                help="allocation strategy for autoscaling" +
-                                     " spot nodes")
+        comp_group.add_argument(
+            "--master-type",
+            dest="master_type",
+            choices=["ondemand", "spot", "spotblock"],
+            default="ondemand",
+            help="type of master nodes. Valid values are:" +
+                 " ('ondemand', 'spot', 'spotblock')" +
+                 " default: ondemand")
+        comp_group.add_argument(
+            "--master-spot-block-duration",
+            dest="master_spot_block_duration",
+            type=int,
+            default=120,
+            help="spot block duration unit: minutes")
+        comp_group.add_argument(
+            "--master-maximum-bid-price-percentage",
+            dest="master_maximum_bid_price_percentage",
+            type=int,
+            default=100,
+            help="maximum value to bid for master spot instances" +
+                 " expressed as a percentage of the base" +
+                 " price for the master instance types")
+        comp_group.add_argument(
+            "--master-timeout-for-request",
+            dest="master_timeout_for_request",
+            type=int,
+            default=1,
+            help="timeout for a master spot instance request, unit: minutes")
+        comp_group.add_argument(
+            "--master-spot-fallback",
+            dest="master_spot_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for master nodes" +
+                 " if spot instances aren't available")
+        comp_group.add_argument(
+            "--min-ondemand-percentage",
+            dest="min_ondemand_percentage",
+            type=int,
+            help="percentage of ondemand nodes in min config")
+        comp_group.add_argument(
+            "--min-spot-block-percentage",
+            dest="min_spot_block_percentage",
+            type=int,
+            help="percentage of spot block nodes in min config")
+        comp_group.add_argument(
+            "--min-spot-percentage",
+            dest="min_spot_percentage",
+            type=int,
+            help="percentage of spot nodes in min config")
+        comp_group.add_argument(
+            "--min-spot-block-duration",
+            dest="min_spot_block_duration",
+            type=int,
+            default=120,
+            help="spot block duration unit: minutes")
+        comp_group.add_argument(
+            "--min-maximum-bid-price-percentage",
+            dest="min_maximum_bid_price_percentage",
+            type=int,
+            default=100,
+            help="maximum value to bid for min spot instances" +
+                 " expressed as a percentage of the base" +
+                 " price for the master instance types")
+        comp_group.add_argument(
+            "--min-timeout-for-request",
+            dest="min_timeout_for_request",
+            type=int,
+            default=1,
+            help="timeout for a min spot instance request, unit: minutes")
+        comp_group.add_argument(
+            "--min-spot-fallback",
+            dest="min_spot_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for min nodes" +
+                 " if spot instances aren't available")
+        comp_group.add_argument(
+            "--min-spot-allocation-strategy",
+            dest="min_spot_allocation_strategy",
+            choices=["lowestPrice", "capacityOptimized", None],
+            default=None,
+            help="allocation strategy for min spot nodes")
+        comp_group.add_argument(
+            "--autoscaling-ondemand-percentage",
+            dest="autoscaling_ondemand_percentage",
+            type=int,
+            help="percentage of ondemand nodes in autoscaling config")
+        comp_group.add_argument(
+            "--autoscaling-spot-block-percentage",
+            dest="autoscaling_spot_block_percentage",
+            type=int,
+            help="percentage of spot block nodes in autoscaling config")
+        comp_group.add_argument(
+            "--autoscaling-spot-percentage",
+            dest="autoscaling_spot_percentage",
+            type=int,
+            help="percentage of spot nodes in autoscaling config")
+        comp_group.add_argument(
+            "--autoscaling-spot-block-duration",
+            dest="autoscaling_spot_block_duration",
+            type=int,
+            default=120,
+            help="spot block duration unit:  minutes")
+        comp_group.add_argument(
+            "--autoscaling-spot-block-fallback",
+            dest="autoscaling_spot_block_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for autoscaling" +
+                 " nodes if spot block instances aren't available")
+        comp_group.add_argument(
+            "--autoscaling-maximum-bid-price-percentage",
+            dest="autoscaling_maximum_bid_price_percentage",
+            type=int,
+            default=100,
+            help="maximum value to bid for autoscaling spot" +
+                 " instances expressed as a percentage of" +
+                 " the base price for the master instance types")
+        comp_group.add_argument(
+            "--autoscaling-timeout-for-request",
+            dest="autoscaling_timeout_for_request",
+            type=int,
+            default=1,
+            help="timeout for a autoscaling spot instance request, unit: minutes")
+        comp_group.add_argument(
+            "--autoscaling-spot-fallback",
+            dest="autoscaling_spot_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for autoscaling nodes" +
+                 " if spot instances aren't available")
+        comp_group.add_argument(
+            "--autoscaling-spot-allocation-strategy",
+            dest="autoscaling_spot_allocation_strategy",
+            choices=["lowestPrice", "capacityOptimized", None],
+            default=None,
+            help="allocation strategy for autoscaling" +
+                 " spot nodes")
 
     def create_parser(self, argparser):
         # compute settings parser

From e50b74151bbd8afd8425e0d9881e55b62282d3b4 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Fri, 12 Jun 2020 21:00:09 -0700
Subject: [PATCH 10/14] chg

---
 qds_sdk/cluster_info_v22.py |  2 +-
 tests/test_clusterv22.py    | 55 +++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index f5d70663..9a98ea43 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -239,7 +239,7 @@ def set_composition_from_cloud_using_parser(self, arguments):
 
     def set_composition_for_cluster(self, **kwargs):
         cloud = Qubole.get_cloud()
-        composition = cloud.get_composition(kwargs)
+        composition = cloud.get_composition(**kwargs)
         if composition is not None:
             self.cluster_info["composition"] = composition
 
diff --git a/tests/test_clusterv22.py b/tests/test_clusterv22.py
index 928ace03..2b10fc0a 100644
--- a/tests/test_clusterv22.py
+++ b/tests/test_clusterv22.py
@@ -227,6 +227,61 @@ def test_hive_settings(self):
                                                                                   'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]},
                                                                                   'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'},
                                                                                                                   {'percentage': 50, 'type': 'spot', 'maximum_bid_price_percentage': 100, 'timeout_for_request': 1, 'allocation_strategy': None, 'fallback': 'ondemand'}]}}}})
+class TestAzureClusterComposition(QdsCliTestCase):
+    def test_od_od(self):
+        sys.argv = ['qds.py', '--version', 'v2.2', '--cloud','AZURE', 'cluster',
+                    'create', '--label', 'test_label',
+                    '--min-ondemand-percentage', '100',
+                    '--autoscaling-ondemand-percentage', '100']
+        Qubole.cloud = None
+        print_command()
+        Connection._api_call = Mock(return_value={})
+        qds.main()
+        Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': {
+            'composition': {'min_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 100}]},
+                            'autoscaling_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 100}]}},
+            'label': ['test_label']}})
+
+    def test_spot_spot(self):
+        sys.argv = ['qds.py', '--version', 'v2.2', '--cloud','AZURE',  'cluster', 'create',
+                    '--label', 'test_label',
+                    '--min-spot-percentage', '100', '--min-spot-fallback', 'ondemand',
+                    '--max-price-percentage', '50', 
+                    '--autoscaling-spot-percentage', '100', "--autoscaling-spot-fallback", 'ondemand']
+        Qubole.cloud = None
+        print_command()
+        Connection._api_call = Mock(return_value={})
+        qds.main()
+        Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': {
+            'composition': {
+                'min_nodes': {'nodes': [{'percentage': 100, 'type': 'spot',
+                                          'fallback': "ondemand", 'max_price_percentage': 50}]}, 
+                'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spot',
+                                                 'fallback': "ondemand", 'max_price_percentage': 50}]}},
+                'label': ['test_label']}})
+
+
+    def test_od_spot(self):
+        sys.argv = ['qds.py', '--version', 'v2.2', '--cloud','AZURE',  'cluster', 'create',
+                    '--label', 'test_label',
+                    '--min-ondemand-percentage', '50',
+                    '--min-spot-percentage', '50',  '--min-spot-fallback', 'ondemand',
+                    '--max-price-percentage', '50', 
+                    '--autoscaling-ondemand-percentage', '50',
+                    '--autoscaling-spot-percentage', '50', '--autoscaling-spot-fallback', 'ondemand']
+        Qubole.cloud = None
+        print_command()
+        Connection._api_call = Mock(return_value={})
+        qds.main()
+        Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': {
+            'composition': {
+                'min_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 50},
+                                        {'percentage': 50, 'type': 'spot',
+                                          'fallback': "ondemand", 'max_price_percentage': 50}]}, 
+                'autoscaling_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 50},
+                                                {'percentage': 50, 'type': 'spot',
+                                                 'fallback': "ondemand", 'max_price_percentage': 50}]}},
+                'label': ['test_label']}})
 
 if __name__ == '__main__':
     unittest.main()

From 6a76da7bd01fa7d2723392a1ac2d6c07c5dfb5b4 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Sun, 14 Jun 2020 19:18:37 -0700
Subject: [PATCH 11/14] chg

---
 qds_sdk/cluster_info_v22.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index 9a98ea43..fd230221 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -62,7 +62,8 @@ def set_cluster_info_from_arguments(self, arguments):
             self.set_composition(
                 master_type=arguments.master_type,
                 master_spot_block_duration=arguments.master_spot_block_duration,
-                master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
+                master_maximum_bid_price_percentage=\
+                    arguments.master_maximum_bid_price_percentage,
                 master_timeout_for_request=arguments.master_timeout_for_request,
                 master_spot_fallback=arguments.master_spot_fallback,
                 min_ondemand_percentage=arguments.min_ondemand_percentage,

From 99b42e68236a078fd367794f4c85f419b45fc9f0 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Sun, 14 Jun 2020 19:20:13 -0700
Subject: [PATCH 12/14] chg

---
 qds_sdk/cluster_info_v22.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py
index fd230221..9a98ea43 100644
--- a/qds_sdk/cluster_info_v22.py
+++ b/qds_sdk/cluster_info_v22.py
@@ -62,8 +62,7 @@ def set_cluster_info_from_arguments(self, arguments):
             self.set_composition(
                 master_type=arguments.master_type,
                 master_spot_block_duration=arguments.master_spot_block_duration,
-                master_maximum_bid_price_percentage=\
-                    arguments.master_maximum_bid_price_percentage,
+                master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage,
                 master_timeout_for_request=arguments.master_timeout_for_request,
                 master_spot_fallback=arguments.master_spot_fallback,
                 min_ondemand_percentage=arguments.min_ondemand_percentage,

From ca3c99f1ef64e176989bf8d9d25c9534453a5dc9 Mon Sep 17 00:00:00 2001
From: Harsh <harshshah87@gmail.com>
Date: Mon, 15 Jun 2020 14:15:41 -0700
Subject: [PATCH 13/14] Update bin/qds.py

Co-authored-by: Joy Lal Chattaraj <8450903+chattarajoy@users.noreply.github.com>
---
 bin/qds.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/qds.py b/bin/qds.py
index 668fd81b..01cde904 100755
--- a/bin/qds.py
+++ b/bin/qds.py
@@ -587,7 +587,7 @@ def main():
                          help="skip verification of server SSL certificate. Insecure: use with caution.")
 
     optparser.add_option("--cloud_name", dest="cloud_name",
-                         default=os.getenv('CLOUD_PROVIDER') or "AWS",
+                         default=os.getenv('CLOUD_PROVIDER', "AWS"),
                          help="cloud", choices=["AWS", "AZURE", "ORACLE_BMC", "ORACLE_OPC", "GCP"])
 
     optparser.add_option("--base_retry_delay", dest="base_retry_delay",

From 167681b7397982f3113094bd5633fc92572fbb14 Mon Sep 17 00:00:00 2001
From: Harsh Shah <harsh@qubole.com>
Date: Thu, 18 Jun 2020 10:56:37 -0700
Subject: [PATCH 14/14] chg

---
 qds_sdk/qubole.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qds_sdk/qubole.py b/qds_sdk/qubole.py
index 5edeab3d..58b40304 100644
--- a/qds_sdk/qubole.py
+++ b/qds_sdk/qubole.py
@@ -30,7 +30,7 @@ class Qubole:
     version = None
     poll_interval = None
     skip_ssl_cert_check = None
-    cloud_name = "aws"
+    cloud_name = None
     cached_agent = None
     cloud = None
     base_retry_delay = None