Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ability to set azure cluster composition #323

Merged
merged 15 commits into from
Jun 19, 2020
Merged

Ability to set azure cluster composition #323

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
08836fd
Ability to set azure cluster composition
shahharsh87 Jun 13, 2020
1292faf
chg
shahharsh87 Jun 13, 2020
bdc9eb7
chg
shahharsh87 Jun 13, 2020
2a30d2f
chg
shahharsh87 Jun 13, 2020
ccd5755
chg
shahharsh87 Jun 13, 2020
8b42bc1
chg
shahharsh87 Jun 13, 2020
8d00ec5
chg
shahharsh87 Jun 13, 2020
84c37d0
chg
shahharsh87 Jun 13, 2020
066e0b4
chg
shahharsh87 Jun 13, 2020
e50b741
chg
shahharsh87 Jun 13, 2020
6a76da7
chg
shahharsh87 Jun 15, 2020
99b42e6
chg
shahharsh87 Jun 15, 2020
ca3c99f
Update bin/qds.py
harshashah16 Jun 15, 2020
167681b
chg
shahharsh87 Jun 18, 2020
61eb262
Merge branch 'azure-2' of https://github.com/harshshah87/qds-sdk-py i…
shahharsh87 Jun 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bin/qds.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,7 @@ def main():
help="skip verification of server SSL certificate. Insecure: use with caution.")

optparser.add_option("--cloud_name", dest="cloud_name",
default=os.getenv('CLOUD_PROVIDER'),
default=os.getenv('CLOUD_PROVIDER') or "AWS",
harshashah16 marked this conversation as resolved.
Show resolved Hide resolved
help="cloud", choices=["AWS", "AZURE", "ORACLE_BMC", "ORACLE_OPC", "GCP"])

optparser.add_option("--base_retry_delay", dest="base_retry_delay",
Expand Down
141 changes: 141 additions & 0 deletions qds_sdk/cloud/aws_cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,147 @@ def set_cloud_config_from_arguments(self, arguments):
bastion_node_public_dns=arguments.bastion_node_public_dns,
master_elastic_ip=arguments.master_elastic_ip)

def set_composition_arguments(self, comp_group):
comp_group.add_argument(
"--master-type",
dest="master_type",
choices=["ondemand", "spot", "spotblock"],
default="ondemand",
help="type of master nodes. Valid values are:" +
" ('ondemand', 'spot', 'spotblock')" +
" default: ondemand")
comp_group.add_argument(
"--master-spot-block-duration",
dest="master_spot_block_duration",
type=int,
default=120,
help="spot block duration unit: minutes")
comp_group.add_argument(
"--master-maximum-bid-price-percentage",
dest="master_maximum_bid_price_percentage",
type=int,
default=100,
help="maximum value to bid for master spot instances" +
" expressed as a percentage of the base" +
" price for the master instance types")
comp_group.add_argument(
"--master-timeout-for-request",
dest="master_timeout_for_request",
type=int,
default=1,
help="timeout for a master spot instance request, unit: minutes")
comp_group.add_argument(
"--master-spot-fallback",
dest="master_spot_fallback",
choices=["ondemand", None],
default=None,
help="whether to fallback to on-demand instances for master nodes" +
" if spot instances aren't available")
comp_group.add_argument(
"--min-ondemand-percentage",
dest="min_ondemand_percentage",
type=int,
help="percentage of ondemand nodes in min config")
comp_group.add_argument(
"--min-spot-block-percentage",
dest="min_spot_block_percentage",
type=int,
help="percentage of spot block nodes in min config")
comp_group.add_argument(
"--min-spot-percentage",
dest="min_spot_percentage",
type=int,
help="percentage of spot nodes in min config")
comp_group.add_argument(
"--min-spot-block-duration",
dest="min_spot_block_duration",
type=int,
default=120,
help="spot block duration unit: minutes")
comp_group.add_argument(
"--min-maximum-bid-price-percentage",
dest="min_maximum_bid_price_percentage",
type=int,
default=100,
help="maximum value to bid for min spot instances" +
" expressed as a percentage of the base" +
" price for the master instance types")
comp_group.add_argument(
"--min-timeout-for-request",
dest="min_timeout_for_request",
type=int,
default=1,
help="timeout for a min spot instance request, unit: minutes")
comp_group.add_argument(
"--min-spot-fallback",
dest="min_spot_fallback",
choices=["ondemand", None],
default=None,
help="whether to fallback to on-demand instances for min nodes" +
" if spot instances aren't available")
comp_group.add_argument(
"--min-spot-allocation-strategy",
dest="min_spot_allocation_strategy",
choices=["lowestPrice", "capacityOptimized", None],
default=None,
help="allocation strategy for min spot nodes")
comp_group.add_argument(
"--autoscaling-ondemand-percentage",
dest="autoscaling_ondemand_percentage",
type=int,
help="percentage of ondemand nodes in autoscaling config")
comp_group.add_argument(
"--autoscaling-spot-block-percentage",
dest="autoscaling_spot_block_percentage",
type=int,
help="percentage of spot block nodes in autoscaling config")
comp_group.add_argument(
"--autoscaling-spot-percentage",
dest="autoscaling_spot_percentage",
type=int,
help="percentage of spot nodes in autoscaling config")
comp_group.add_argument(
"--autoscaling-spot-block-duration",
dest="autoscaling_spot_block_duration",
type=int,
default=120,
help="spot block duration unit: minutes")
comp_group.add_argument(
"--autoscaling-spot-block-fallback",
dest="autoscaling_spot_block_fallback",
choices=["ondemand", None],
default=None,
help="whether to fallback to on-demand instances for autoscaling" +
" nodes if spot block instances aren't available")
comp_group.add_argument(
"--autoscaling-maximum-bid-price-percentage",
dest="autoscaling_maximum_bid_price_percentage",
type=int,
default=100,
help="maximum value to bid for autoscaling spot" +
" instances expressed as a percentage of" +
" the base price for the master instance types")
comp_group.add_argument(
"--autoscaling-timeout-for-request",
dest="autoscaling_timeout_for_request",
type=int,
default=1,
help="timeout for a autoscaling spot instance request, unit: minutes")
comp_group.add_argument(
"--autoscaling-spot-fallback",
dest="autoscaling_spot_fallback",
choices=["ondemand", None],
default=None,
help="whether to fallback to on-demand instances for autoscaling nodes" +
" if spot instances aren't available")
comp_group.add_argument(
"--autoscaling-spot-allocation-strategy",
dest="autoscaling_spot_allocation_strategy",
choices=["lowestPrice", "capacityOptimized", None],
default=None,
help="allocation strategy for autoscaling" +
" spot nodes")

def create_parser(self, argparser):
# compute settings parser
compute_config = argparser.add_argument_group("compute config settings")
Expand Down
85 changes: 84 additions & 1 deletion qds_sdk/cloud/azure_cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,89 @@ def set_cloud_config_from_arguments(self, arguments):
master_static_public_ip_name=arguments.master_static_public_ip_name,
resource_group_name=arguments.resource_group_name)

def set_composition_arguments(self, comp_group):
# composition arguments we want to accept for azure
comp_group.add_argument("--min-ondemand-percentage",
dest="min_ondemand_percentage",
type=int, default=0,
help="Percentage of ondemand nodes in min config.")
comp_group.add_argument("--min-spot-percentage",
dest="min_spot_percentage",
type=int, default=0,
help="Percentage of spot nodes in min config.")
comp_group.add_argument("--max-price-percentage",
dest="max_price_percentage",
type=int, default=100,
help="Percentage of maximum price percentage"
" for spot nodes.")
comp_group.add_argument("--min-spot-fallback",
dest="min_spot_fallback",
choices=["ondemand", None],
default=None,
help="Whether to fallback to on-demand instances" +
" for min nodes if spot instances" +
" aren't available.")
comp_group.add_argument("--autoscaling-ondemand-percentage",
dest="autoscaling_ondemand_percentage",
type=int, default=0,
help="Percentage of ondemand nodes" +
"in autoscaling config.")
comp_group.add_argument("--autoscaling-spot-percentage",
dest="autoscaling_spot_percentage",
type=int, default=0,
help="Percentage of spot nodes" +
"in autoscaling config.")
comp_group.add_argument("--autoscaling-spot-fallback",
dest="autoscaling_spot_fallback",
choices=["ondemand", None],
default=None,
help="Whether to fallback to on-demand instances" +
" for autoscaling nodes if spot instances" +
" aren't available.")

# Ignore other key-value arguments.
def get_composition(self,
min_ondemand_percentage=0,
min_spot_percentage=0,
min_spot_fallback=None,
autoscaling_ondemand_percentage=0,
autoscaling_spot_percentage=0,
autoscaling_spot_fallback=None,
max_price_percentage=100,
**kwargs):
composition = {}
composition["min_nodes"] = {"nodes": []}
min_nodes = composition["min_nodes"]["nodes"]
if min_ondemand_percentage + min_spot_percentage != 100:
raise ValueError("Minimum nodes ondemand+spot percentage"
" should be 100: Ondemand pct: %d Spot pct: %d"
% (min_ondemand_percentage,
min_spot_percentage))
if min_ondemand_percentage > 0:
min_nodes.append({"type": "ondemand",
"percentage": min_ondemand_percentage})
if min_spot_percentage > 0:
min_nodes.append({"type": "spot", "percentage": min_spot_percentage,
"fallback": min_spot_fallback,
"max_price_percentage": max_price_percentage})

composition["autoscaling_nodes"] = {"nodes": []}
autoscaling_nodes = composition["autoscaling_nodes"]["nodes"]
if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
raise ValueError("Autoscaling nodes ondemand+spot percentage" +
" should be 100: Ondemand pct: %d Spot pct: %d"
% (autoscaling_ondemand_percentage,
autoscaling_spot_percentage))
if autoscaling_ondemand_percentage > 0:
autoscaling_nodes.append({"type": "ondemand",
"percentage": autoscaling_ondemand_percentage})
if autoscaling_spot_percentage > 0:
autoscaling_nodes.append({"type": "spot",
"percentage": autoscaling_spot_percentage,
"fallback": autoscaling_spot_fallback,
"max_price_percentage": max_price_percentage})
return composition

def create_parser(self, argparser):
# compute settings parser
compute_config = argparser.add_argument_group("compute config settings")
Expand Down Expand Up @@ -228,4 +311,4 @@ def create_parser(self, argparser):
storage_config.add_argument("--disk-storage-account-resource-group-name",
dest="disk_storage_account_resource_group_name",
default=None,
help="disk storage account resource group for azure cluster")
help="disk storage account resource group for azure cluster")
6 changes: 6 additions & 0 deletions qds_sdk/cloud/cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,9 @@ def create_parser(self, argparser):

def set_cloud_config_from_arguments(self, arguments):
return NotImplemented

def set_composition_arguments(self, comp_group):
pass
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we

Suggested change
pass
return NotImplemented

?

Copy link
Contributor Author

@harshashah16 harshashah16 Jun 15, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we go with that approach, we have to check for cloud when we set the composition.
Rather than having an if condition to check whether the cloud supports, each cloud can set empty data or implement the function which will sets the value.
I implemented it this way to avoid that if conditions. In future, user has to implement the function and no need to edit the condition to allow cloud to set compositions.


def get_composition(self, **kwargs):
pass
Loading