diff --git a/examples/azure-vm/README.md b/examples/azure-vm/README.md new file mode 100644 index 0000000000..5c1f1d1c44 --- /dev/null +++ b/examples/azure-vm/README.md @@ -0,0 +1,96 @@ + + +# **Getting Started With DeepSparse in an Azure VM** + +![diagram](./azure-vm-pic.png) + +Neural Magic’s DeepSparse is an inference runtime that can be deployed directly from a public Docker image. DeepSparse supports various CPU instance types and sizes, allowing you to quickly deploy the infrastructure that works best for your use case, based on cost and performance. + +If you are interested in configuring and launching an instance with DeepSparse in Python, follow the step-by-step guide below. + +We recommend installing the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) for easy access to Azure's functionalities although it is not required. + +## Step 1: Create a Subscription +Create an [Azure subscription](https://learn.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription) to gain access to a `subscription id`. + + +## Step 2: Install Dependencies + +```bash +git clone https://github.com/neuralmagic/deepsparse.git +cd deepsparse/examples/azure-vm +pip install -r requirements.txt +``` + +## Step 3: Run Script + +The [azure-vm.py](https://github.com/neuralmagic/deepsparse/tree/main/examples/azure-vm/azure-vm.py) script creates an Azure resource group, launches an Ubuntu instance and returns the Public IP address so you can SSH into the instance after it finishes staging. Additionally, it also contains a bash script which automatically downloads Docker and pulls Neural Magic's public DeepSparse image into your instance. + +To execute the script, run the following command and pass in your `subscription id` from step 1, your VMs `location`, `vm-type`, a resources `group name`, your `virtual machine's name` and the `password` for logging in to your instance: + +```bash +python azure-vm.py create-vm --subscription-id --location --vm-type --group-name --vm-name --pw +``` + +To leverage CPU optimized instances, we recommend using the [`Fsv2-series`](https://learn.microsoft.com/en-us/azure/virtual-machines/sizes-compute) instances which contain AVX-512 instructions. Here's an example command for launching a VM in the US East location using a F4s-v2 instance (4 vCPUs and 8GB of RAM): + +```bash +python azure-vm.py create-vm --subscription-id --location eastus --vm-type Standard_F4s_v2 --group-name deepsparse-group --vm-name deepsparse-vm --pw Password123! +``` + +**PRO-TIP**: The password passed into the CLI command must satisfy the following conditions: + +1) Contains an uppercase character. +2) Contains a lowercase character. +3) Contains a numeric digit. +4) Contains a special character. +5) Control characters are not allowed. + +## **Step 4: SSH Into the Instance** + +After running the script, your instance's public IP address will be printed out in the terminal. Pass the IP address into the following CLI command to SSH into your running instance: + +```bash +ssh testuser@ +``` + +After entering your password, get root access: + +```bash +sudo su +``` + +## **Step 5: Run DeepSparse** + +We recommend giving your instance 2-3 mins. to finish executing the bash script. To make sure you have the DeepSparse image imported, run the following command: + +```bash +docker images +``` +You should be able to see a downloaded DeepSparse image, if it shows an empty table, the bash script hasn't completed execution. + +Upon image download, you can now use DeepSparse. Here's an example of benchmarking a pruned-quantized version of BERT trained on SQuAD from Docker: + +```bash +docker run -it deepsparse_docker deepsparse.benchmark zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned95_obs_quant-none -i [64,128] -b 64 -nstreams 1 -s sync +``` + +## **Step 6: Delete Instance and Resource Group** + +```bash +python azure-vm.py delete-vm-rg --subscription-id --group-name --vm-name +``` \ No newline at end of file diff --git a/examples/azure-vm/azure-vm-pic.png b/examples/azure-vm/azure-vm-pic.png new file mode 100644 index 0000000000..9bb1dc2641 Binary files /dev/null and b/examples/azure-vm/azure-vm-pic.png differ diff --git a/examples/azure-vm/azure-vm.py b/examples/azure-vm/azure-vm.py new file mode 100644 index 0000000000..fb8bf8bf01 --- /dev/null +++ b/examples/azure-vm/azure-vm.py @@ -0,0 +1,398 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import base64 + +import click + +from azure.identity import DefaultAzureCredential +from azure.mgmt.compute import ComputeManagementClient +from azure.mgmt.network import NetworkManagementClient +from azure.mgmt.resource import ResourceManagementClient + + +def create_resource_group( + resource_client: ResourceManagementClient, group_name: str, location: str +): + """ + Create a resource group in Azure. + + Args: + resource_client (ResourceManagementClient): Azure Resource client. + group_name (str): Name of the resource group to create. + location (str): Location of the resource group. + """ + resource_client.resource_groups.create_or_update(group_name, {"location": location}) + + +def create_virtual_network( + network_client: NetworkManagementClient, + group_name: str, + network_name: str, + location: str, +): + """ + Create a virtual network in Azure. + + Args: + network_client (NetworkManagementClient): Azure NetworkManagementClient object. + group_name (str): Name of the resource group. + network_name (str): Name of the virtual network to create. + location (str): Location of the virtual network. + """ + network_client.virtual_networks.begin_create_or_update( + group_name, + network_name, + {"location": location, "address_space": {"address_prefixes": ["10.0.0.0/16"]}}, + ).result() + + +def create_subnet( + network_client: NetworkManagementClient, + group_name: str, + network_name: str, + subnet_name: str, +): + """ + Create a subnet in Azure. + + Args: + network_client (NetworkManagementClient): Azure NetworkManagementClient object. + group_name (str): Name of the resource group. + network_name (str): Name of the virtual network. + subnet_name (str): Name of the subnet to create. + + Returns: + The created subnet. + """ + return network_client.subnets.begin_create_or_update( + group_name, network_name, subnet_name, {"address_prefix": "10.0.0.0/24"} + ).result() + + +def create_network_security_group( + network_client: NetworkManagementClient, + group_name: str, + nsg_name: str, + location: str, +): + """ + Create a network security group in Azure. + + Args: + network_client (NetworkManagementClient): Azure NetworkManagementClient object. + group_name (str): Name of the resource group. + nsg_name (str): Name of the network security group to create. + location (str): Location of the network security group. + + Returns: + The created network security group. + """ + nsg_params = { + "location": location, + "security_rules": [ + { + "name": "SSH", + "protocol": "Tcp", + "source_port_range": "*", + "destination_port_range": "22", + "source_address_prefix": "*", + "destination_address_prefix": "*", + "access": "Allow", + "priority": 100, + "direction": "Inbound", + } + ], + } + return network_client.network_security_groups.begin_create_or_update( + group_name, nsg_name, nsg_params + ).result() + + +def create_public_ip_address( + network_client: NetworkManagementClient, + group_name: str, + ip_address_name: str, + location: str, +): + """ + Create a public IP address in Azure. + + Args: + network_client (NetworkManagementClient): Azure NetworkManagementClient object. + group_name (str): Name of the resource group. + ip_address_name (str): Name of the public IP address to create. + location (str): Location of the public IP address. + + Returns: + The created public IP address. + """ + public_ip_address_params = { + "location": location, + "public_ip_allocation_method": "static", + "public_ip_address_version": "ipv4", + } + return network_client.public_ip_addresses.begin_create_or_update( + group_name, ip_address_name, public_ip_address_params + ).result() + + +def create_network_interface( + network_client: NetworkManagementClient, + group_name: str, + interface_name: str, + location: str, + subnet: create_subnet, + public_ip_address: create_public_ip_address, + nsg: create_network_security_group, +): + """ + Create a network interface in Azure. + + Args: + network_client (NetworkManagementClient): Azure NetworkManagementClient object. + group_name (str): Name of the resource group. + interface_name (str): Name of the network interface to create. + location (str): Location of the network interface. + subnet : The subnet object to associate with the network interface. + public_ip_address: The public IP address object to associate with the network. + nsg: The network security group object to associate with the network interface. + + Returns: + The created network interface. + """ + network_interface_params = { + "location": location, + "ip_configurations": [ + { + "name": "MyIpConfig", + "subnet": {"id": subnet.id}, + "public_ip_address": {"id": public_ip_address.id}, + } + ], + "network_security_group": {"id": nsg.id}, + } + return network_client.network_interfaces.begin_create_or_update( + group_name, interface_name, network_interface_params + ).result() + + +def create_virtual_machine( + compute_client: ComputeManagementClient, + group_name: str, + vm_name: str, + location: str, + vm_type: str, + interface_name: str, + subscription_id: str, + PASSWORD: str, + startup_script: str, +): + """ + Create a virtual machine in Azure. + + Args: + compute_client (ComputeManagementClient): Azure ComputeManagementClient object. + group_name (str): Name of the resource group. + vm_name (str): Name of the virtual machine to create. + location (str): Location of the virtual machine. + vm_type (str): Type of virtual machine (size). + interface_name (str): Name of the network interface. + subscription_id (str): Azure subscription ID. + PASSWORD (str): Admin password for the virtual machine. + startup_script (str): Custom startup script for the virtual machine. + + Returns: + The created virtual machine. + """ + return compute_client.virtual_machines.begin_create_or_update( + group_name, + vm_name, + { + "location": location, + "hardware_profile": {"vm_size": vm_type}, + "storage_profile": { + "image_reference": { + "sku": "20_04-lts-gen2", + "publisher": "Canonical", + "version": "latest", + "offer": "0001-com-ubuntu-server-focal", + }, + "os_disk": { + "caching": "ReadWrite", + "managed_disk": {"storage_account_type": "Standard_LRS"}, + "name": "myVMosdisk", + "create_option": "FromImage", + }, + "data_disks": [ + {"disk_size_gb": "1023", "create_option": "Empty", "lun": "0"}, + {"disk_size_gb": "1023", "create_option": "Empty", "lun": "1"}, + ], + }, + "os_profile": { + "admin_username": "testuser", + "computer_name": "myVM", + "admin_password": PASSWORD, + "custom_data": startup_script, + }, + "network_profile": { + "network_interfaces": [ + { + "id": "/subscriptions/" + + subscription_id + + "/resourceGroups/" + + group_name + + "/providers/Microsoft.Network/networkInterfaces/" + + interface_name + + "", + "properties": {"primary": True}, + } + ] + }, + }, + ).result() + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option("--subscription-id", required=True, help="Azure subscription ID") +@click.option("--location", required=True, help="Location") +@click.option("--vm-type", required=True, help="Virtual machine type") +@click.option("--group-name", required=True, help="Resource group name") +@click.option("--vm-name", required=True, help="Virtual machine name") +@click.option("--pw", required=True, help="Virtual machine password") +def create_vm( + subscription_id: str, + location: str, + vm_type: str, + group_name: str, + vm_name: str, + pw: str, +): + """ + Create a new virtual machine in Azure. + + Args: + subscription_id (str): Azure subscription ID. + location (str): Location where the VM will be created. + vm_type (str): Type of virtual machine (size). + group_name (str): Name of the resource group to create. + vm_name (str): Name of the virtual machine to create. + pw (str): Password for the virtual machine. + """ + SUBSCRIPTION_ID = subscription_id + LOCATION = location + VM_TYPE = vm_type + GROUP_NAME = group_name + VIRTUAL_MACHINE_NAME = vm_name + PASSWORD = pw + SUBNET_NAME = "subnetx" + INTERFACE_NAME = "interfacex" + NETWORK_NAME = "networknamex" + IP_ADDRESS_NAME = "ipaddressx" + NSG_NAME = "nsgx" + + startup_script = """#!/bin/bash + apt-get update + apt-get install -y apt-transport-https ca-certificates curl \\ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \\ + sudo apt-key add - + add-apt-repository "deb [arch=amd64] \\ + https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + apt-get update + apt-get install -y docker-ce + docker pull ghcr.io/neuralmagic/deepsparse:1.4.2 + docker tag ghcr.io/neuralmagic/deepsparse:1.4.2 deepsparse_docker + """ + + # Encode startup_script as Base64 + startup_script = base64.b64encode(startup_script.encode()).decode() + + # Create client + resource_client = ResourceManagementClient( + credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID + ) + network_client = NetworkManagementClient( + credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID + ) + compute_client = ComputeManagementClient( + credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID + ) + + create_resource_group(resource_client, GROUP_NAME, LOCATION) + create_virtual_network(network_client, GROUP_NAME, NETWORK_NAME, LOCATION) + subnet = create_subnet(network_client, GROUP_NAME, NETWORK_NAME, SUBNET_NAME) + nsg = create_network_security_group(network_client, GROUP_NAME, NSG_NAME, LOCATION) + public_ip_address = create_public_ip_address( + network_client, GROUP_NAME, IP_ADDRESS_NAME, LOCATION + ) + create_network_interface( + network_client, + GROUP_NAME, + INTERFACE_NAME, + LOCATION, + subnet, + public_ip_address, + nsg, + ) + create_virtual_machine( + compute_client, + GROUP_NAME, + VIRTUAL_MACHINE_NAME, + LOCATION, + VM_TYPE, + INTERFACE_NAME, + SUBSCRIPTION_ID, + PASSWORD, + startup_script, + ) + + print("Your external public IP address:", public_ip_address.ip_address) + + +@cli.command() +@click.option("--subscription-id", required=True, help="Azure subscription ID") +@click.option("--group-name", required=True, help="Resource group name") +@click.option("--vm-name", required=True, help="Virtual machine name") +def delete_vm_rg(subscription_id: str, group_name: str, vm_name: str): + """ + Delete a virtual machine and its associated resource group in Azure. + + Args: + subscription_id (str): Azure subscription ID. + group_name (str): Name of the resource group to delete. + vm_name (str): Name of the virtual machine to delete. + """ + compute_client = ComputeManagementClient( + credential=DefaultAzureCredential(), subscription_id=subscription_id + ) + resource_client = ResourceManagementClient( + credential=DefaultAzureCredential(), subscription_id=subscription_id + ) + + compute_client.virtual_machines.begin_power_off(group_name, vm_name).result() + compute_client.virtual_machines.begin_delete(group_name, vm_name).result() + print("Deleted virtual machine.") + + resource_client.resource_groups.begin_delete(group_name).result() + print("Deleted resource group.") + + +if __name__ == "__main__": + cli() diff --git a/examples/azure-vm/requirements.txt b/examples/azure-vm/requirements.txt new file mode 100644 index 0000000000..19fd16dd6b --- /dev/null +++ b/examples/azure-vm/requirements.txt @@ -0,0 +1,4 @@ +azure-mgmt-compute=>30.0.0 +azure-identity=>1.13.0 +azure-mgmt-resource=>23.0.1 +click=>8.0.4