Skip to content

Commit

Permalink
{Packaging/MSI} Use embeddable python for MSI (#14300)
Browse files Browse the repository at this point in the history
  • Loading branch information
fengzhou-msft authored Jul 30, 2020
1 parent 66ff8d1 commit d23d175
Show file tree
Hide file tree
Showing 7 changed files with 104 additions and 33 deletions.
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@
*.bat eol=crlf
# The az script for Git Bash/Cygwin should be LF
build_scripts/windows/scripts/az eol=lf
# sh scripts should be LF
*.sh eol=lf
13 changes: 6 additions & 7 deletions build_scripts/windows/scripts/build.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ if "%CLI_VERSION%"=="" (
echo Please set the CLI_VERSION environment variable, e.g. 2.0.13
goto ERROR
)
set PYTHON_VERSION=3.6.6
set PYTHON_VERSION=3.6.8

set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip"
set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python366-32.zip"
set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-32.zip"
set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip"

:: Set up the output directory and temp. directories
Expand All @@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR%
set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch
set BUILDING_DIR=%ARTIFACTS_DIR%\cli
set WIX_DIR=%ARTIFACTS_DIR%\wix
set PYTHON_DIR=%ARTIFACTS_DIR%\Python366-32
set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-32
set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change

set REPO_ROOT=%~dp0..\..\..
Expand Down Expand Up @@ -75,10 +75,10 @@ if not exist %PYTHON_DIR% (
mkdir %PYTHON_DIR%
pushd %PYTHON_DIR%
echo Downloading Python.
curl -o Python366-32.zip %PYTHON_DOWNLOAD_URL% -k
unzip -q Python366-32.zip
curl -o Python368-32.zip %PYTHON_DOWNLOAD_URL% -k
unzip -q Python368-32.zip
if %errorlevel% neq 0 goto ERROR
del Python366-32.zip
del Python368-32.zip
echo Python downloaded and extracted successfully.
popd
)
Expand All @@ -98,7 +98,6 @@ for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-nsp
if %errorlevel% neq 0 goto ERROR

%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg
%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --force-reinstall urllib3==1.24.2

pushd %BUILDING_DIR%
%BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py
Expand Down
21 changes: 21 additions & 0 deletions build_scripts/windows/scripts/setup_msi_test.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Prerequisites:
# 1. Install the MSI built with current branch
# 2. Run bash azure-cli\scripts\ci\build.sh with Git Bash first to generate artifacts under azure-cli\artifacts\build so we can use the testsdk and fulltest wheels.

# Elevate to Admin
If (-NOT ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator"))
{
$arguments = "& '" + $myinvocation.mycommand.definition + "'"
Start-Process powershell -Verb runAs -ArgumentList $arguments
}

& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest
& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install pytest-xdist

$testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_testsdk*.whl | Select-Object Name
& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($testsdk.Name)

$fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name
& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name)

& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py
41 changes: 41 additions & 0 deletions build_scripts/windows/scripts/test_msi_package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

# Invoke this script in Powershell with:
# & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' test_msi_package.py <start_mod_name>

import os
import sys
import subprocess

base_dir = 'C:\\Program Files (x86)\\Microsoft SDKs\\Azure\\CLI2\\Lib\\site-packages\\azure\\cli'
root_dir = '{}\\command_modules'.format(base_dir)
mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__']

pytest_base_cmd = ['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN']
pytest_parallel_cmd = pytest_base_cmd + ['-n', 'auto']

for mod_name in mod_list:
try:
start_mod = sys.argv[1]
if mod_name < start_mod:
continue
except:
pass
mod_cmd = ['--junit-xml', '{}\\azure_cli_test_result\\{}.xml'.format(os.path.expanduser('~'), mod_name),
'--pyargs', 'azure.cli.command_modules.{}'.format(mod_name)]
if mod_name in ['botservice', 'network']:
exit_code = subprocess.call(pytest_base_cmd + mod_cmd)
else:
exit_code = subprocess.call(pytest_parallel_cmd + mod_cmd)
if exit_code == 5:
print('No tests found for {}'.format(mod_name))
elif exit_code != 0:
sys.exit(exit_code)

core_dir = '{}\\core'.format(base_dir)
exit_code = subprocess.call(['python', '-m', 'pytest', '-x', '-v', '-p', 'no:warnings', '--log-level', 'WARN',
'--junit-xml', '{}\\azure_cli_test_result\\azure-cli-core.xml'.format(os.path.expanduser('~')), '-n', 'auto', '--import-mode=append', core_dir])
sys.exit(exit_code)
13 changes: 10 additions & 3 deletions src/azure-cli-core/azure/cli/core/tests/test_generic_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,16 @@ def my_get():
def my_set(**kwargs): # pylint:disable=unused-argument
return my_obj

test_type = CliCommandType(operations_tmpl='{}#{{}}'.format(__name__))
setattr(sys.modules[__name__], my_get.__name__, my_get)
setattr(sys.modules[__name__], my_set.__name__, my_set)
test_module = 'azure.cli.core.tests.test_generic_update'
test_type = CliCommandType(operations_tmpl='{}#{{}}'.format(test_module))
try:
setattr(sys.modules[test_module], my_get.__name__, my_get)
setattr(sys.modules[test_module], my_set.__name__, my_set)
except KeyError:
import importlib
loaded_module = importlib.import_module(test_module)
setattr(loaded_module, my_get.__name__, my_get)
setattr(loaded_module, my_set.__name__, my_set)
with self.command_group('', test_type) as g:
g.generic_update_command('genupdate', getter_name='my_get', setter_name='my_set')

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ class AmsSpTests(ScenarioTest):
@StorageAccountPreparer(parameter_name='storage_account_for_create')
@AllowLargeResponse()
def test_ams_sp_create_reset(self, resource_group, storage_account_for_create):
with mock.patch('azure.cli.command_modules.ams._utils._gen_guid', side_effect=self.create_guid):
with mock.patch('azure.cli.command_modules.ams.operations.sp._gen_guid', side_effect=self.create_guid):
amsname = self.create_random_name(prefix='ams', length=12)

self.kwargs.update({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,16 +69,16 @@ def test_ams_streaming_endpoint_create_with_akamai(self, storage_account_for_cre
'scaleUnits': 4,
'tags': 'foo=bar',
'ips': '1.1.1.1 2.2.2.2',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'identifier': 'id1',
'expiration': '2030-12-31T16:00:00-08:00',
'base64Key': 'dGVzdGlkMQ=='
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --ips {ips} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --ips {ips} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('resourceGroup', '{rg}'),
self.check('location', 'North Europe'),
Expand Down Expand Up @@ -124,14 +124,14 @@ def test_ams_streaming_endpoint_update(self, storage_account_for_create):
'maxCacheAge': 11,
'scaleUnits': 5,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'ip': '4.4.4.4'
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('resourceGroup', '{rg}'),
self.check('location', 'Australia East'),
Expand All @@ -155,12 +155,12 @@ def test_ams_streaming_endpoint_update(self, storage_account_for_create):
'description': 'test streaming description2',
'maxCacheAge': 9,
'tags': 'foo2=bar2 foo3=bar3',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml')),
'ips': '1.1.1.1 2.2.2.2 192.168.0.0/28'
})

self.cmd('az ams streaming-endpoint update -g {rg} -a {amsname} -n {streamingEndpointName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --tags {tags} --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint update -g {rg} -a {amsname} -n {streamingEndpointName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --tags {tags} --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('cdnProvider', '{cdnProvider}'),
self.check('cdnProfile', '{cdnProfile}'),
Expand Down Expand Up @@ -201,13 +201,13 @@ def test_ams_streaming_endpoint_create(self, storage_account_for_create):
'maxCacheAge': 11,
'scaleUnits': 6,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('name', '{streamingEndpointName}'),
self.check('resourceGroup', '{rg}'),
self.check('location', 'Canada Central'),
Expand Down Expand Up @@ -240,13 +240,13 @@ def test_ams_streaming_endpoint_show(self, storage_account_for_show):
'maxCacheAge': 11,
'scaleUnits': 7,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"')
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"')

self.cmd('az ams streaming-endpoint show -g {rg} -a {amsname} -n {streamingEndpointName}', checks=[
self.check('name', '{streamingEndpointName}'),
Expand Down Expand Up @@ -288,13 +288,13 @@ def test_ams_streaming_endpoint_delete(self, storage_account_for_delete):
'maxCacheAge': 11,
'scaleUnits': 8,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName1} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"')
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName1} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"')

self.cmd('az ams streaming-endpoint list -g {rg} -a {amsname}', checks=[
self.check('length(@)', 2)
Expand Down Expand Up @@ -325,13 +325,13 @@ def test_ams_streaming_endpoint_scale(self, storage_account_for_scale):
'scaleUnits': 9,
'scaleUnits2': 10,
'tags': 'foo=bar',
'clientAccessPolicy': '@' + self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': '@' + self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
'clientAccessPolicy': self._normalize_filename(_get_test_data_file('clientAccessPolicy.xml')),
'crossDomainPolicy': self._normalize_filename(_get_test_data_file('crossDomainPolicy.xml'))
})

self.cmd('az ams account create -n {amsname} -g {rg} --storage-account {storageAccount} -l {location}')

self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy "{clientAccessPolicy}" --cross-domain-policy "{crossDomainPolicy}"', checks=[
self.cmd('az ams streaming-endpoint create -g {rg} -a {amsname} -n {streamingEndpointName} --availability-set-name {availabilitySetName} --cdn-provider {cdnProvider} --cdn-profile {cdnProfile} --description "{description}" --max-cache-age {maxCacheAge} --scale-units {scaleUnits} --tags "{tags}" --client-access-policy @"{clientAccessPolicy}" --cross-domain-policy @"{crossDomainPolicy}"', checks=[
self.check('scaleUnits', '{scaleUnits}')
])

Expand Down Expand Up @@ -468,4 +468,5 @@ def test_ams_streaming_endpoint_list(self, storage_account_for_create):
# Helper functions

def _normalize_filename(cmd, string):
return '"' + string.replace('\\', '/') + '"'
import platform
return '"' + string.replace('\\', '/') + '"' if platform.system() != 'Windows' else string

0 comments on commit d23d175

Please sign in to comment.