From aaaf0799731c33ffdf6f8b6756f316d9a19d736a Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Mon, 4 Feb 2019 14:26:59 -0800 Subject: [PATCH 01/15] # This is a combination of 4 commits. # This is the 1st commit message: Resolved merge conflict by incorporating both suggestions # This is the commit message #2: indentation fixed # This is the commit message #3: indentation fixed # This is the commit message #4: indentation fixed --- .azure-pipelines/client.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 33692a504c43..b13e21e46b9c 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -112,9 +112,16 @@ jobs: summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' reportDirectory: '$(Build.SourcesDirectory)/htmlcov' + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: and(eq(variables['os.name'],'Windows'),succeeded()) + - job: Test_Alpha_Python timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) + + +- job: Special_Python_Distro_Tests dependsOn: - 'Build_Target_Packages' pool: From 6b78a26900da0bd2ad40e12b45c88a2d57d7773f Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Mon, 4 Feb 2019 14:28:48 -0800 Subject: [PATCH 02/15] Resolved merge conflicts --- .azure-pipelines/client.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index b13e21e46b9c..0136344a7621 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -2,6 +2,7 @@ # 'build_targeting_string' comma separated glob strings selecting the packages that should be built. EG: "azure-keyvault,azure-mgmt-batch" or "azure-*" # 'codecov-python-repository-token' Is the codecov.io repository token. # 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. +# 'EnableDetectorPip': true trigger: - master From be44c31b7e0d10a521eb9b325b5dfb9f28c9897c Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Mon, 4 Feb 2019 14:30:01 -0800 Subject: [PATCH 03/15] Resolved merge conflicts --- .azure-pipelines/client.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 0136344a7621..d85fe7f6ac42 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -2,7 +2,8 @@ # 'build_targeting_string' comma separated glob strings selecting the packages that should be built. EG: "azure-keyvault,azure-mgmt-batch" or "azure-*" # 'codecov-python-repository-token' Is the codecov.io repository token. # 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. -# 'EnableDetectorPip': true +# 'EnableDetectorPip': true . Required for Component Governance Support + trigger: - master From 112b221122f0cf5099ffdf6f8fc453e020ce0376 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Mon, 11 Feb 2019 11:14:47 -0800 Subject: [PATCH 04/15] resolved conflict, updated job analyze with dependency steps --- .azure-pipelines/client.yml | 48 ++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index c92f1c1ef562..30fb9096c062 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -2,6 +2,8 @@ # 'build_targeting_string' comma separated glob strings selecting the packages that should be built. EG: "azure-keyvault,azure-mgmt-batch" or "azure-*" # 'codecov-python-repository-token' Is the codecov.io repository token. # 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. +# 'EnableDetectorPip': true . Required for Component Governance Support + trigger: - master @@ -19,15 +21,9 @@ jobs: versionSpec: 3.6 - script: | - pip install wheel setuptools pathlib Jinja2 twine readme-renderer[md] + pip install wheel setuptools pathlib twine readme-renderer[md] displayName: 'Prep Environment' - - - task: PythonScript@0 - displayName: 'Analyze dependencies' - inputs: - scriptPath: 'scripts/analyze_deps.py' - arguments: '--verbose --out "$(Build.SourcesDirectory)/dependencies.html"' - + - task: PythonScript@0 displayName: 'Generate Packages' inputs: @@ -50,6 +46,35 @@ jobs: displayName: 'Publish Artifacts' condition: succeededOrFailed() +- job: 'Analyze' + dependsOn: + - 'Build_Target_Packages' + + pool: + vmImage: 'vs2017-win2016' + + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' + inputs: + versionSpec: 3.6 + + - script: | + pip install wheel Jinja2 + displayName: 'Prep Environment' + + - task: PythonScript@0 + displayName: 'Analyze dependencies' + enabled: true + inputs: + scriptPath: 'scripts/analyze_deps.py' + arguments: '--verbose --out "$(Build.SourcesDirectory)/dependencies.html"' + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + condition: and(always(), ne(variables['System.TeamProject'], 'public')) + displayName: 'Component Detection' + - job: 'Generic_Tests' dependsOn: - 'Build_Target_Packages' @@ -124,9 +149,16 @@ jobs: summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' reportDirectory: '$(Build.SourcesDirectory)/htmlcov' + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + condition: and(eq(variables['os.name'],'Windows'),succeeded()) + - job: Test_Alpha_Python timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) + + +- job: Special_Python_Distro_Tests dependsOn: - 'Build_Target_Packages' pool: From 4ee707451189084a9b4cc9cdbb30091d0ec51708 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Mon, 11 Feb 2019 18:20:42 -0800 Subject: [PATCH 05/15] Incorporating suggestions --- .azure-pipelines/client.yml | 50 +++++++++++-------------------------- 1 file changed, 15 insertions(+), 35 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 30fb9096c062..905932ae6d1b 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -4,21 +4,20 @@ # 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. # 'EnableDetectorPip': true . Required for Component Governance Support - trigger: - master +variables: + python.version: '3.6' jobs: - job: 'Build_Target_Packages' - pool: vmImage: 'ubuntu-16.04' - steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' + displayName: 'Use Python $(python.version)' inputs: - versionSpec: 3.6 + versionSpec: $(python.version) - script: | pip install wheel setuptools pathlib twine readme-renderer[md] @@ -34,31 +33,20 @@ jobs: twine check $(Build.ArtifactStagingDirectory)/* displayName: 'Verify Readme' - - task: CopyFiles@2 - displayName: 'Move Dependencies Report to Build Artifacts' - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: dependencies.html - TargetFolder: '$(Build.ArtifactStagingDirectory)' - condition: always() - - task: PublishBuildArtifacts@1 displayName: 'Publish Artifacts' condition: succeededOrFailed() - job: 'Analyze' dependsOn: - - 'Build_Target_Packages' - + - 'Build_Target_Packages' pool: - vmImage: 'vs2017-win2016' - - steps: - + vmImage: 'ubuntu-16.04' + steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' + displayName: 'Use Python $(python.version)' inputs: - versionSpec: 3.6 + versionSpec: $(python.version) - script: | pip install wheel Jinja2 @@ -69,9 +57,11 @@ jobs: enabled: true inputs: scriptPath: 'scripts/analyze_deps.py' - arguments: '--verbose --out "$(Build.SourcesDirectory)/dependencies.html"' + arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/dependencies.html"' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR + # builds should be sufficient. condition: and(always(), ne(variables['System.TeamProject'], 'public')) displayName: 'Component Detection' @@ -110,7 +100,6 @@ jobs: python.version: '2.7' pool: vmImage: '$(os.vmImage)' - steps: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' @@ -149,15 +138,10 @@ jobs: summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' reportDirectory: '$(Build.SourcesDirectory)/htmlcov' - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - condition: and(eq(variables['os.name'],'Windows'),succeeded()) - - job: Test_Alpha_Python timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) - - job: Special_Python_Distro_Tests dependsOn: - 'Build_Target_Packages' @@ -177,10 +161,8 @@ jobs: mkdir debug cd debug ../configure --enable-optimizations --prefix=$HOME - make install - + make install cd $(Build.SourcesDirectory) - PATH=/home/vsts/bin:$PATH python3 -m ensurepip python ./scripts/devops_tasks/setup_execute_tests.py -p python3 "$(build_targeting_string)" @@ -207,10 +189,8 @@ jobs: - script: | cd ~/ wget https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2 - tar xf pypy3-v6.0.0-linux64.tar.bz2 - - cd $(Build.SourcesDirectory) - + tar xf pypy3-v6.0.0-linux64.tar.bz2 + cd $(Build.SourcesDirectory) PATH=/home/vsts/pypy3-v6.0.0-linux64/bin/:$PATH pypy3 -m ensurepip python ./scripts/devops_tasks/setup_execute_tests.py -p pypy3 "$(build_targeting_string)" From 13c9a358050e0ca55424961e587557a8923f2046 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Mon, 11 Feb 2019 20:06:52 -0800 Subject: [PATCH 06/15] single quote for variable --- .azure-pipelines/client.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 905932ae6d1b..51d04a0f7f64 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -17,7 +17,7 @@ jobs: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' inputs: - versionSpec: $(python.version) + versionSpec: '$(python.version)' - script: | pip install wheel setuptools pathlib twine readme-renderer[md] @@ -46,7 +46,7 @@ jobs: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' inputs: - versionSpec: $(python.version) + versionSpec: '$(python.version)' - script: | pip install wheel Jinja2 From 346516f58082b11139243ecd86f6e9f6fe7dc87c Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Mon, 11 Feb 2019 21:30:05 -0800 Subject: [PATCH 07/15] added publish step for Analyze Job --- .azure-pipelines/client.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 51d04a0f7f64..055c3dedb0f3 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -65,6 +65,10 @@ jobs: condition: and(always(), ne(variables['System.TeamProject'], 'public')) displayName: 'Component Detection' + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + - job: 'Generic_Tests' dependsOn: - 'Build_Target_Packages' From 871b2d4fb5a0c602a0e8f485736ff52c8d0b19fe Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Tue, 12 Feb 2019 15:25:25 -0800 Subject: [PATCH 08/15] Fixed the code that went wrong during merge --- .azure-pipelines/client.yml | 67 ++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index d85fe7f6ac42..b47af802eda1 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -4,9 +4,10 @@ # 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. # 'EnableDetectorPip': true . Required for Component Governance Support - trigger: - master +variables: + PythonVersion: '3.6' jobs: - job: 'Build_Target_Packages' @@ -16,25 +17,55 @@ jobs: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' + displayName: 'Use Python $(PythonVersion)' inputs: - versionSpec: 3.6 + versionSpec: $(PythonVersion) - script: | - pip install wheel setuptools pathlib Jinja2 + pip install wheel setuptools pathlib Jinja2 twine readme-renderer[md] displayName: 'Prep Environment' + - task: PythonScript@0 + displayName: 'Generate Packages' + inputs: + scriptPath: 'scripts/devops_tasks/build_packages.py' + arguments: '-d "$(Build.ArtifactStagingDirectory)" "$(build_targeting_string)"' + + - script: | + twine check $(Build.ArtifactStagingDirectory)/* + displayName: 'Verify Readme' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + +- job: 'Analyze' + dependsOn: + - 'Build_Target_Packages' + pool: + vmImage: 'ubuntu-16.04' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + + - script: | + pip install wheel Jinja2 + displayName: 'Prep Environment' + - task: PythonScript@0 displayName: 'Analyze dependencies' + enabled: true inputs: scriptPath: 'scripts/analyze_deps.py' arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/dependencies.html"' - - task: PythonScript@0 - displayName: 'Generate Packages' - inputs: - scriptPath: 'scripts/devops_tasks/build_packages.py' - arguments: '-d "$(Build.ArtifactStagingDirectory)" "$(build_targeting_string)"' + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR + # builds should be sufficient. + condition: and(always(), ne(variables['System.TeamProject'], 'public')) + displayName: 'Component Detection' - task: PublishBuildArtifacts@1 displayName: 'Publish Artifacts' @@ -78,14 +109,13 @@ jobs: steps: - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' + displayName: 'Use Python $(PythonVersion)' inputs: - versionSpec: '$(python.version)' + versionSpec: '$(PythonVersion)' - script: | pip install pathlib twine codecov beautifulsoup4 displayName: 'Prep Environment' - - task: PythonScript@0 displayName: 'Setup and Run Tests' inputs: @@ -96,12 +126,10 @@ jobs: coverage xml coverage html displayName: 'Generate Coverage XML' - - script: | codecov -t $(codecov-python-repository-token) displayName: 'Publish Code Cov' condition: ne(variables['codecov-python-repository-token'], '') - - task: PythonScript@0 displayName: 'Inline CSS for Cobertura' inputs: @@ -114,16 +142,9 @@ jobs: summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' reportDirectory: '$(Build.SourcesDirectory)/htmlcov' - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - condition: and(eq(variables['os.name'],'Windows'),succeeded()) - - job: Test_Alpha_Python timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) - - -- job: Special_Python_Distro_Tests dependsOn: - 'Build_Target_Packages' pool: @@ -145,7 +166,6 @@ jobs: make install cd $(Build.SourcesDirectory) - PATH=/home/vsts/bin:$PATH python3 -m ensurepip python ./scripts/devops_tasks/setup_execute_tests.py -p python3 "$(build_targeting_string)" @@ -168,7 +188,6 @@ jobs: - script: | pip install pathlib twine displayName: 'Prep Environment' - - script: | cd ~/ wget https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2 @@ -180,4 +199,4 @@ jobs: pypy3 -m ensurepip python ./scripts/devops_tasks/setup_execute_tests.py -p pypy3 "$(build_targeting_string)" displayName: 'Setup and Run Filtered Tests PyPy3' - continueOnError: true + continueOnError: true \ No newline at end of file From b2cb750fd193192da1a1a1925f95a714e5786c98 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Tue, 12 Feb 2019 16:31:28 -0800 Subject: [PATCH 09/15] Fixed whitesapces and newlines --- .azure-pipelines/client.yml | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index a57db9e59f33..ee89ef8785e4 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -1,33 +1,34 @@ # Variables: # 'build_targeting_string' comma separated glob strings selecting the packages that should be built. EG: "azure-keyvault,azure-mgmt-batch" or "azure-*" # 'codecov-python-repository-token' Is the codecov.io repository token. -# 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. -# 'EnableDetectorPip': true . Required for Component Governance Support - +# 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. trigger: - master + variables: PythonVersion: '3.6' jobs: - job: 'Build_Target_Packages' + pool: vmImage: 'ubuntu-16.04' + steps: - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: $(PythonVersion) + displayName: 'Use Python $(PythonVersion)' - script: | pip install wheel setuptools pathlib twine readme-renderer[md] displayName: 'Prep Environment' - task: PythonScript@0 - displayName: 'Generate Packages' inputs: scriptPath: 'scripts/devops_tasks/build_packages.py' arguments: '-d "$(Build.ArtifactStagingDirectory)" "$(build_targeting_string)"' + displayName: 'Generate Packages' - script: | twine check $(Build.ArtifactStagingDirectory)/* @@ -38,26 +39,29 @@ jobs: condition: succeededOrFailed() - job: 'Analyze' + dependsOn: - 'Build_Target_Packages' + pool: vmImage: 'ubuntu-16.04' - steps: + + steps: - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' inputs: versionSpec: '$(PythonVersion)' + displayName: 'Use Python $(PythonVersion)' - script: | pip install wheel Jinja2 displayName: 'Prep Environment' - - - task: PythonScript@0 - displayName: 'Analyze dependencies' + + - task: PythonScript@0 enabled: true inputs: scriptPath: 'scripts/analyze_deps.py' arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/dependencies.html"' + displayName: 'Analyze dependencies' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR @@ -66,12 +70,14 @@ jobs: displayName: 'Component Detection' - task: PublishBuildArtifacts@1 + condition: succeededOrFailed() displayName: 'Publish Artifacts' - condition: succeededOrFailed() - job: 'Generic_Tests' + dependsOn: - 'Build_Target_Packages' + strategy: matrix: Linux_Python27: @@ -102,8 +108,10 @@ jobs: os.name: 'MacOS' os.vmImage: 'macOS-10.13' python.version: '2.7' + pool: vmImage: '$(os.vmImage)' + steps: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' @@ -143,12 +151,15 @@ jobs: reportDirectory: '$(Build.SourcesDirectory)/htmlcov' - job: Test_Alpha_Python + timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) dependsOn: - 'Build_Target_Packages' + pool: vmImage: 'ubuntu-16.04' + steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.7 For Build Tools' @@ -174,12 +185,15 @@ jobs: continueOnError: true - job: Test_PyPy + timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) dependsOn: - 'Build_Target_Packages' + pool: vmImage: 'ubuntu-16.04' + steps: - task: UsePythonVersion@0 displayName: 'Use Python 3.7 For Build Tools' From d8d2e53c38858eb479dbe5f85aa4aa7bef399aff Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Tue, 12 Feb 2019 17:13:45 -0800 Subject: [PATCH 10/15] testing condition for public builds --- .azure-pipelines/client.yml | 197 +++++------------------------------- 1 file changed, 28 insertions(+), 169 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index ee89ef8785e4..fc33acd63cf0 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -1,216 +1,75 @@ # Variables: # 'build_targeting_string' comma separated glob strings selecting the packages that should be built. EG: "azure-keyvault,azure-mgmt-batch" or "azure-*" # 'codecov-python-repository-token' Is the codecov.io repository token. -# 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. +# 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. +# 'EnableDetectorPip': true . Required for Component Governance Support + trigger: - master -variables: - PythonVersion: '3.6' - jobs: - job: 'Build_Target_Packages' pool: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-16.04' steps: - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' inputs: - versionSpec: $(PythonVersion) - displayName: 'Use Python $(PythonVersion)' + versionSpec: 3.6 - script: | pip install wheel setuptools pathlib twine readme-renderer[md] - displayName: 'Prep Environment' - + displayName: 'Prep Environment' + - task: PythonScript@0 + displayName: 'Generate Packages' inputs: scriptPath: 'scripts/devops_tasks/build_packages.py' arguments: '-d "$(Build.ArtifactStagingDirectory)" "$(build_targeting_string)"' - displayName: 'Generate Packages' - script: | twine check $(Build.ArtifactStagingDirectory)/* displayName: 'Verify Readme' + - task: CopyFiles@2 + displayName: 'Move Dependencies Report to Build Artifacts' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: dependencies.html + TargetFolder: '$(Build.ArtifactStagingDirectory)' + condition: always() + - task: PublishBuildArtifacts@1 displayName: 'Publish Artifacts' condition: succeededOrFailed() - job: 'Analyze' - dependsOn: - 'Build_Target_Packages' - + pool: - vmImage: 'ubuntu-16.04' - + vmImage: 'vs2017-win2016' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6' inputs: - versionSpec: '$(PythonVersion)' - displayName: 'Use Python $(PythonVersion)' + versionSpec: 3.6 - script: | pip install wheel Jinja2 displayName: 'Prep Environment' - - - task: PythonScript@0 + + - task: PythonScript@0 + displayName: 'Analyze dependencies' enabled: true inputs: scriptPath: 'scripts/analyze_deps.py' - arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/dependencies.html"' - displayName: 'Analyze dependencies' + arguments: '--verbose --out "$(Build.SourcesDirectory)/dependencies.html"' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR - # builds should be sufficient. - condition: and(always(), ne(variables['System.TeamProject'], 'public')) + condition: and(always(), ne(variables['Build.Reason'], 'PullRequest')) displayName: 'Component Detection' - - - task: PublishBuildArtifacts@1 - condition: succeededOrFailed() - displayName: 'Publish Artifacts' - -- job: 'Generic_Tests' - - dependsOn: - - 'Build_Target_Packages' - - strategy: - matrix: - Linux_Python27: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '2.7' - Linux_Python34: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.4' - Linux_Python35: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.5' - Linux_Python36: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.6' - Linux_Python37: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.7' - Windows_Python35: - os.name: 'Windows' - os.vmImage: 'vs2017-win2016' - python.version: '3.5' - MacOS_Python27: - os.name: 'MacOS' - os.vmImage: 'macOS-10.13' - python.version: '2.7' - - pool: - vmImage: '$(os.vmImage)' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - - script: | - pip install pathlib twine codecov beautifulsoup4 - displayName: 'Prep Environment' - - - task: PythonScript@0 - displayName: 'Setup and Run Tests' - inputs: - scriptPath: 'scripts/devops_tasks/setup_execute_tests.py' - arguments: '"$(build_targeting_string)"' - - - script: | - coverage xml - coverage html - displayName: 'Generate Coverage XML' - - - script: | - codecov -t $(codecov-python-repository-token) - displayName: 'Publish Code Cov' - condition: ne(variables['codecov-python-repository-token'], '') - - - task: PythonScript@0 - displayName: 'Inline CSS for Cobertura' - inputs: - scriptPath: 'scripts/devops_tasks/inline_css_for_cobertura.py' - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Code Coverage to DevOps' - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' - reportDirectory: '$(Build.SourcesDirectory)/htmlcov' - -- job: Test_Alpha_Python - - timeoutInMinutes: 90 - condition: eq(variables['long_running_tests'], True) - dependsOn: - - 'Build_Target_Packages' - - pool: - vmImage: 'ubuntu-16.04' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7 For Build Tools' - inputs: - versionSpec: '3.7' - - - script: | - sudo apt-get install build-essential libsqlite3-dev sqlite3 bzip2 libbz2-dev zlib1g-dev libssl-dev openssl libgdbm-dev liblzma-dev libreadline-dev libncursesw5-dev libffi-dev uuid-dev - cd ~/ - git clone https://github.com/python/cpython.git - cd cpython - mkdir debug - cd debug - ../configure --enable-optimizations --prefix=$HOME - make install - - cd $(Build.SourcesDirectory) - - PATH=/home/vsts/bin:$PATH - python3 -m ensurepip - python ./scripts/devops_tasks/setup_execute_tests.py -p python3 "$(build_targeting_string)" - displayName: 'Setup - Run Filtered Tests "Nightly" Python' - continueOnError: true - -- job: Test_PyPy - - timeoutInMinutes: 90 - condition: eq(variables['long_running_tests'], True) - dependsOn: - - 'Build_Target_Packages' - - pool: - vmImage: 'ubuntu-16.04' - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7 For Build Tools' - inputs: - versionSpec: '3.7' - - - script: | - pip install pathlib twine - displayName: 'Prep Environment' - - - script: | - cd ~/ - wget https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2 - tar xf pypy3-v6.0.0-linux64.tar.bz2 - cd $(Build.SourcesDirectory) - PATH=/home/vsts/pypy3-v6.0.0-linux64/bin/:$PATH - pypy3 -m ensurepip - python ./scripts/devops_tasks/setup_execute_tests.py -p pypy3 "$(build_targeting_string)" - displayName: 'Setup and Run Filtered Tests PyPy3' - continueOnError: true From 8c610a91678442e2ba06095f50b79cd085b52bb9 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Tue, 12 Feb 2019 17:35:46 -0800 Subject: [PATCH 11/15] whitespaces, comments and CG condition --- .azure-pipelines/client.yml | 188 +++++++++++++++++++++++++++++++----- 1 file changed, 164 insertions(+), 24 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index fc33acd63cf0..3f3928edfb00 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -2,27 +2,29 @@ # 'build_targeting_string' comma separated glob strings selecting the packages that should be built. EG: "azure-keyvault,azure-mgmt-batch" or "azure-*" # 'codecov-python-repository-token' Is the codecov.io repository token. # 'long_running_tests' Is a boolean indicating that the long-running tests should be executed. -# 'EnableDetectorPip': true . Required for Component Governance Support trigger: - master +variables: + PythonVersion: '3.6' + jobs: - job: 'Build_Target_Packages' pool: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-16.04' steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' + displayName: 'Use Python $(PythonVersion)' inputs: - versionSpec: 3.6 + versionSpec: $(PythonVersion) - script: | pip install wheel setuptools pathlib twine readme-renderer[md] - displayName: 'Prep Environment' - + displayName: 'Prep Environment' + - task: PythonScript@0 displayName: 'Generate Packages' inputs: @@ -32,44 +34,182 @@ jobs: - script: | twine check $(Build.ArtifactStagingDirectory)/* displayName: 'Verify Readme' - - - task: CopyFiles@2 - displayName: 'Move Dependencies Report to Build Artifacts' - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: dependencies.html - TargetFolder: '$(Build.ArtifactStagingDirectory)' - condition: always() - - task: PublishBuildArtifacts@1 displayName: 'Publish Artifacts' condition: succeededOrFailed() - job: 'Analyze' + dependsOn: - 'Build_Target_Packages' - + pool: - vmImage: 'vs2017-win2016' - + vmImage: 'ubuntu-16.04' + steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' + displayName: 'Use Python $(PythonVersion)' inputs: - versionSpec: 3.6 + versionSpec: '$(PythonVersion)' - script: | pip install wheel Jinja2 displayName: 'Prep Environment' - + - task: PythonScript@0 displayName: 'Analyze dependencies' enabled: true inputs: scriptPath: 'scripts/analyze_deps.py' - arguments: '--verbose --out "$(Build.SourcesDirectory)/dependencies.html"' + arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/dependencies.html"' - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - condition: and(always(), ne(variables['Build.Reason'], 'PullRequest')) + # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR + # builds should be sufficient. + condition: and(always(), ne(variables['Build.Reason'],'PullRequest')) displayName: 'Component Detection' + + - task: PublishBuildArtifacts@1 + # To Publish artifacts generated by the 'Analyze dependencies' task under 'Analyze' job + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + +- job: 'Generic_Tests' + + dependsOn: + - 'Build_Target_Packages' + + strategy: + matrix: + Linux_Python27: + os.name: 'Linux' + os.vmImage: 'ubuntu-16.04' + python.version: '2.7' + Linux_Python34: + os.name: 'Linux' + os.vmImage: 'ubuntu-16.04' + python.version: '3.4' + Linux_Python35: + os.name: 'Linux' + os.vmImage: 'ubuntu-16.04' + python.version: '3.5' + Linux_Python36: + os.name: 'Linux' + os.vmImage: 'ubuntu-16.04' + python.version: '3.6' + Linux_Python37: + os.name: 'Linux' + os.vmImage: 'ubuntu-16.04' + python.version: '3.7' + Windows_Python35: + os.name: 'Windows' + os.vmImage: 'vs2017-win2016' + python.version: '3.5' + MacOS_Python27: + os.name: 'MacOS' + os.vmImage: 'macOS-10.13' + python.version: '2.7' + + pool: + vmImage: '$(os.vmImage)' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + + - script: | + pip install pathlib twine codecov beautifulsoup4 + displayName: 'Prep Environment' + + - task: PythonScript@0 + displayName: 'Setup and Run Tests' + inputs: + scriptPath: 'scripts/devops_tasks/setup_execute_tests.py' + arguments: '"$(build_targeting_string)"' + + - script: | + coverage xml + coverage html + displayName: 'Generate Coverage XML' + + - script: | + codecov -t $(codecov-python-repository-token) + displayName: 'Publish Code Cov' + condition: ne(variables['codecov-python-repository-token'], '') + + - task: PythonScript@0 + displayName: 'Inline CSS for Cobertura' + inputs: + scriptPath: 'scripts/devops_tasks/inline_css_for_cobertura.py' + + - task: PublishCodeCoverageResults@1 + displayName: 'Publish Code Coverage to DevOps' + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' + reportDirectory: '$(Build.SourcesDirectory)/htmlcov' + +- job: Test_Alpha_Python + + timeoutInMinutes: 90 + condition: eq(variables['long_running_tests'], True) + dependsOn: + - 'Build_Target_Packages' + + pool: + vmImage: 'ubuntu-16.04' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7 For Build Tools' + inputs: + versionSpec: '3.7' + + - script: | + sudo apt-get install build-essential libsqlite3-dev sqlite3 bzip2 libbz2-dev zlib1g-dev libssl-dev openssl libgdbm-dev liblzma-dev libreadline-dev libncursesw5-dev libffi-dev uuid-dev + cd ~/ + git clone https://github.com/python/cpython.git + cd cpython + mkdir debug + cd debug + ../configure --enable-optimizations --prefix=$HOME + make install + cd $(Build.SourcesDirectory) + PATH=/home/vsts/bin:$PATH + python3 -m ensurepip + python ./scripts/devops_tasks/setup_execute_tests.py -p python3 "$(build_targeting_string)" + displayName: 'Setup - Run Filtered Tests "Nightly" Python' + continueOnError: true + +- job: Test_PyPy + + timeoutInMinutes: 90 + condition: eq(variables['long_running_tests'], True) + dependsOn: + - 'Build_Target_Packages' + + pool: + vmImage: 'ubuntu-16.04' + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7 For Build Tools' + inputs: + versionSpec: '3.7' + + - script: | + pip install pathlib twine + displayName: 'Prep Environment' + + - script: | + cd ~/ + wget https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2 + tar xf pypy3-v6.0.0-linux64.tar.bz2 + cd $(Build.SourcesDirectory) + PATH=/home/vsts/pypy3-v6.0.0-linux64/bin/:$PATH + pypy3 -m ensurepip + python ./scripts/devops_tasks/setup_execute_tests.py -p pypy3 "$(build_targeting_string)" + displayName: 'Setup and Run Filtered Tests PyPy3' + continueOnError: true From f8a91a074de8ad86cb7d597bddf508278ffe3e65 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Tue, 12 Feb 2019 17:50:58 -0800 Subject: [PATCH 12/15] updated CG condition --- .azure-pipelines/client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 3f3928edfb00..a4ea38c5c517 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -66,7 +66,7 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR # builds should be sufficient. - condition: and(always(), ne(variables['Build.Reason'],'PullRequest')) + condition: and(succeedOrFailed(), ne(variables['Build.Reason'],'PullRequest')) displayName: 'Component Detection' - task: PublishBuildArtifacts@1 From d2624cd511233fd17485cefaa823b573bb356f88 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Tue, 12 Feb 2019 18:02:23 -0800 Subject: [PATCH 13/15] updated the condition CG --- .azure-pipelines/client.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index a4ea38c5c517..6d6322e1adb4 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -66,7 +66,7 @@ jobs: - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR # builds should be sufficient. - condition: and(succeedOrFailed(), ne(variables['Build.Reason'],'PullRequest')) + condition: and(succeededOrFailed(), ne(variables['Build.Reason'],'PullRequest')) displayName: 'Component Detection' - task: PublishBuildArtifacts@1 From 6dde85a6830d9099b692dc6c35ae2647ff967da7 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Wed, 13 Feb 2019 14:36:41 -0800 Subject: [PATCH 14/15] updates for standardization --- .azure-pipelines/client.yml | 58 ++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 6d6322e1adb4..413e6a54b2bb 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -10,7 +10,7 @@ variables: PythonVersion: '3.6' jobs: -- job: 'Build_Target_Packages' +- job: 'Build' pool: vmImage: 'ubuntu-16.04' @@ -41,7 +41,7 @@ jobs: - job: 'Analyze' dependsOn: - - 'Build_Target_Packages' + - 'Build' pool: vmImage: 'ubuntu-16.04' @@ -77,47 +77,47 @@ jobs: - job: 'Generic_Tests' dependsOn: - - 'Build_Target_Packages' + - 'Build' strategy: matrix: Linux_Python27: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '2.7' + OSName: 'Linux' + OSVmImage: 'ubuntu-16.04' + PythonVersion: '2.7' Linux_Python34: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.4' + OSName: 'Linux' + OSVmImage: 'ubuntu-16.04' + PythonVersion: '3.4' Linux_Python35: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.5' + OSName: 'Linux' + OSVmImage: 'ubuntu-16.04' + PythonVersion: '3.5' Linux_Python36: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.6' + OSName: 'Linux' + OSVmImage: 'ubuntu-16.04' + PythonVersion: '3.6' Linux_Python37: - os.name: 'Linux' - os.vmImage: 'ubuntu-16.04' - python.version: '3.7' + OSName: 'Linux' + OSVmImage: 'ubuntu-16.04' + PythonVersion: '3.7' Windows_Python35: - os.name: 'Windows' - os.vmImage: 'vs2017-win2016' - python.version: '3.5' + OSName: 'Windows' + OSVmImage: 'vs2017-win2016' + PythonVersion: '3.5' MacOS_Python27: - os.name: 'MacOS' - os.vmImage: 'macOS-10.13' - python.version: '2.7' + OSName: 'MacOS' + OSVmImage: 'macOS-10.13' + PythonVersion: '2.7' pool: - vmImage: '$(os.vmImage)' + vmImage: '$(OSVmImage)' steps: - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' + displayName: 'Use Python $(PythonVersion)' inputs: - versionSpec: '$(python.version)' + versionSpec: '$(PythonVersion)' - script: | pip install pathlib twine codecov beautifulsoup4 @@ -156,7 +156,7 @@ jobs: timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) dependsOn: - - 'Build_Target_Packages' + - 'Build' pool: vmImage: 'ubuntu-16.04' @@ -188,7 +188,7 @@ jobs: timeoutInMinutes: 90 condition: eq(variables['long_running_tests'], True) dependsOn: - - 'Build_Target_Packages' + - 'Build' pool: vmImage: 'ubuntu-16.04' From 7c017104467f464b287baa052aba4d8a2308baa1 Mon Sep 17 00:00:00 2001 From: Karishma Ghiya Date: Wed, 13 Feb 2019 17:23:12 -0800 Subject: [PATCH 15/15] cleanup --- .azure-pipelines/client.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/client.yml b/.azure-pipelines/client.yml index 413e6a54b2bb..3a8ead1d5c75 100644 --- a/.azure-pipelines/client.yml +++ b/.azure-pipelines/client.yml @@ -34,6 +34,7 @@ jobs: - script: | twine check $(Build.ArtifactStagingDirectory)/* displayName: 'Verify Readme' + - task: PublishBuildArtifacts@1 displayName: 'Publish Artifacts' condition: succeededOrFailed() @@ -81,32 +82,25 @@ jobs: strategy: matrix: - Linux_Python27: - OSName: 'Linux' + Linux_Python27: OSVmImage: 'ubuntu-16.04' PythonVersion: '2.7' - Linux_Python34: - OSName: 'Linux' + Linux_Python34: OSVmImage: 'ubuntu-16.04' PythonVersion: '3.4' - Linux_Python35: - OSName: 'Linux' + Linux_Python35: OSVmImage: 'ubuntu-16.04' PythonVersion: '3.5' Linux_Python36: - OSName: 'Linux' OSVmImage: 'ubuntu-16.04' PythonVersion: '3.6' Linux_Python37: - OSName: 'Linux' OSVmImage: 'ubuntu-16.04' PythonVersion: '3.7' Windows_Python35: - OSName: 'Windows' OSVmImage: 'vs2017-win2016' PythonVersion: '3.5' MacOS_Python27: - OSName: 'MacOS' OSVmImage: 'macOS-10.13' PythonVersion: '2.7'