Module releaser tool (Azure#1624)

* Ensures go.mod exists before doing actual work

* Fix wrong module name

* Add a new test scenario

* Some refinement on testcases for versioner

* Update version in version.go

* Only keep one line of tag in version.go

* Minor fix

* No need to print tag to stdout

* New test scenario

* Fix linter problem

* Return error from filepath.Abs

* Add test for update version.go file

* Remove method that ensures existence of go.mod file, and some redaudent
test cases

* Formatting code after versioning done

* Refine tests

* Fix a bug that when update applied to v2, the go.mod file does not
update as v2

* Clean up

* Purely file rename, nothing else changed

* Add functionality for run tool in root folder

* Add root command again

* Fix format

* Typo fix

* Add test for list all stage folders

* Add two functions for programmatically call

* Fix CI failure

* Add a new test case for more realistic

* Fix errors in the new test case

* Fix gofmt issues

* Add a new struct to cover this scenario

* Add test for go vet

* Add new test case for one line import

* Format stage folder first to avoid un-expected changes in code

* Fix a bug in regex used for multiline import statement

* Move getPkgs method to an internal package and exported

* Fix broken tests

* Add comment to silence golint error

* Init command

* Format change

* More minor tweak to satisfy golint

* Refine

* Update testcases. Do not bump patch version when files indentical

* Previous changes will break tests, revert

* Fix testcase

* Fix bad test case

* Add a new test case

* Avoid bumping version when files are identical

* Gofmt fix

* Initial commit of tool moduler

* Add copyright info in front of every new file

* Add profile building

* Fix CI failure

* Fix dry-run

* Do not need to update repo

* Refine

* Refine and add tests for moduler

* Update tests

* Remove profile related functionalities

* Fix bad test cases

tools/apidiff/exports/exports.go

@@ -37,6 +37,9 @@ type Content struct {
 	// the list of exported struct types.
 	// key is the exported name, value contains field information.
 	Structs map[string]Struct `json:"structs,omitempty"`
+
+	// the array of file names in this package
+	FileNames []string `json:"file_names,omitempty"`
 }
 
 // Const is a const definition.
@@ -78,6 +81,7 @@ func NewContent() Content {
 		Funcs:      make(map[string]Func),
 		Interfaces: make(map[string]Interface),
 		Structs:    make(map[string]Struct),
+		FileNames:  make([]string, 0),
 	}
 }
 

tools/apidiff/exports/package.go

@@ -109,6 +109,7 @@ func (pkg Package) GetExports() (c Content) {
 		}
 		return true
 	})
+	c.FileNames = pkg.FileNames()
 	return
 }
 
@@ -117,6 +118,16 @@ func (pkg Package) Name() string {
 	return pkg.p.Name
 }
 
+// FileNames returns the file names in this package
+func (pkg Package) FileNames() []string {
+	var filenames []string
+	pkg.f.Iterate(func(file *token.File) bool {
+		filenames = append(filenames, file.Name())
+		return true
+	})
+	return filenames
+}
+
 // Get loads the package in the specified directory and returns the exported
 // content.  It's a convenience wrapper around LoadPackage() and GetExports().
 func Get(pkgDir string) (Content, error) {

tools/apidiff/repo/repo.go

@@ -171,7 +171,7 @@ func (wt WorkingTree) CreateTag(name string) error {
 
 // ListTags calls "git tag -l <pattern>" to obtain the list of tags.
 // If there are no tags the returned slice will have zero length.
-// Tags are sorted in lexographic ascending order.
+// Tags are sorted in lexicographic ascending order.
 func (wt WorkingTree) ListTags(pattern string) ([]string, error) {
 	cmd := exec.Command("git", "tag", "-l", pattern)
 	cmd.Dir = wt.dir

tools/internal/modinfo/modinfo.go

@@ -16,6 +16,7 @@ package modinfo
 
 import (
 	"fmt"
+	"io/ioutil"
 	"path/filepath"
 	"regexp"
 	"sort"
@@ -33,6 +34,10 @@ var (
 	verSuffixRegex = regexp.MustCompile(`v\d+$`)
 )
 
+const (
+	versionFile = "version.go"
+)
+
 // HasVersionSuffix returns true if the specified path has a version suffix in the form vN.
 func HasVersionSuffix(path string) bool {
 	return verSuffixRegex.MatchString(path)
@@ -111,6 +116,7 @@ type Provider interface {
 	VersionSuffix() bool
 	NewModule() bool
 	GenerateReport() report.Package
+	HasChanges() (bool, error)
 }
 
 type module struct {
@@ -207,8 +213,63 @@ func (m module) GenerateReport() report.Package {
 	return report.Generate(m.lhs, m.rhs, false, false)
 }
 
+func (m module) HasChanges() (bool, error) {
+	if m.NewExports() || m.BreakingChanges() {
+		return true, nil
+	}
+	return hasChange(m.lhs, m.rhs)
+}
+
 // IsValidModuleVersion returns true if the provided string is a valid module version (e.g. v1.2.3).
 func IsValidModuleVersion(v string) bool {
 	r := regexp.MustCompile(`^v\d+\.\d+\.\d+$`)
 	return r.MatchString(v)
 }
+
+func hasChange(lhs, rhs exports.Content) (bool, error) {
+	// test file count
+	if len(lhs.FileNames) != len(rhs.FileNames) {
+		return true, nil
+	}
+	// test every file
+	for i, lPath := range lhs.FileNames {
+		rPath := rhs.FileNames[i]
+		lName := filepath.Base(lPath)
+		rName := filepath.Base(rPath)
+		// filename differs
+		if !strings.EqualFold(lName, rName) {
+			return true, nil
+		}
+		// ignore version.go files
+		if lName == versionFile {
+			continue
+		}
+		// same filename, check content
+		if same, err := checkFileContent(lPath, rPath); err != nil {
+			return true, err
+		} else if !same { // not same means has change
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func checkFileContent(filepath1, filepath2 string) (bool, error) {
+	content1, err := readFileContent(filepath1)
+	if err != nil {
+		return false, err
+	}
+	content2, err := readFileContent(filepath2)
+	if err != nil {
+		return false, err
+	}
+	return strings.EqualFold(content1, content2), nil
+}
+
+func readFileContent(path string) (string, error) {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("failed to read file '%s': %v", path, err)
+	}
+	return strings.ReplaceAll(string(b), "\r\n", "\n"), nil
+}

tools/internal/pkgs/pkgs.go

@@ -0,0 +1,126 @@
+// Copyright 2018 Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkgs
+
+import (
+	"errors"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// Pkg provides a package of a service
+type Pkg struct {
+	// the directory where the package relative to the root dir
+	Dest string
+	// the ast of the package
+	Package *ast.Package
+}
+
+// IsARMPkg returns if this package is ARM package (management plane)
+func (p Pkg) IsARMPkg() bool {
+	return strings.Index(p.Dest, "/mgmt/") > -1
+}
+
+// GetAPIVersion returns the api version of this package
+func (p Pkg) GetAPIVersion() (string, error) {
+	dest := p.Dest
+	if p.IsARMPkg() {
+		// management-plane
+		regex := regexp.MustCompile(`mgmt/(.+)/`)
+		groups := regex.FindStringSubmatch(dest)
+		if len(groups) < 2 {
+			return "", fmt.Errorf("cannot find api version in %s", dest)
+		}
+		versionString := groups[1]
+		return versionString, nil
+	}
+	// data-plane
+	regex := regexp.MustCompile(`/(\d{4}-\d{2}.*|v?\d+(\.\d+)?)/`)
+	groups := regex.FindStringSubmatch(dest)
+	if len(groups) < 2 {
+		return "", fmt.Errorf("cannot find api version in %s", dest)
+	}
+	versionString := groups[1]
+	if versionString == "" {
+		return "", fmt.Errorf("does not find api version in data plane package %s", dest)
+	}
+	return versionString, nil
+}
+
+// GetPkgs returns every package under the rootDir. Package for interfaces will be ignored.
+func GetPkgs(rootDir string) ([]Pkg, error) {
+	var pkgs []Pkg
+	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			// check if leaf dir
+			fi, err := ioutil.ReadDir(path)
+			if err != nil {
+				return err
+			}
+			hasSubDirs := false
+			interfacesDir := false
+			for _, f := range fi {
+				if f.IsDir() {
+					hasSubDirs = true
+					break
+				}
+				if f.Name() == "interfaces.go" {
+					interfacesDir = true
+				}
+			}
+			if !hasSubDirs {
+				fs := token.NewFileSet()
+				// with interfaces codegen the majority of leaf directories are now the
+				// *api packages. when this is the case parse from the parent directory.
+				if interfacesDir {
+					path = filepath.Dir(path)
+				}
+				packages, err := parser.ParseDir(fs, path, func(fileInfo os.FileInfo) bool {
+					return fileInfo.Name() != "interfaces.go"
+				}, parser.PackageClauseOnly)
+				if err != nil {
+					return nil
+				}
+				if len(packages) < 1 {
+					return errors.New("didn't find any packages which is unexpected")
+				}
+				if len(packages) > 1 {
+					return errors.New("found more than one package which is unexpected")
+				}
+				var p *ast.Package
+				for _, pkgs := range packages {
+					p = pkgs
+				}
+				// normalize directory separator to '/' character
+				pkgs = append(pkgs, Pkg{
+					Dest:    strings.ReplaceAll(path[len(rootDir):], "\\", "/"),
+					Package: p,
+				})
+			}
+		}
+		return nil
+	})
+	return pkgs, err
+}

tools/internal/pkgs/pkgs_test.go

@@ -0,0 +1,106 @@
+// Copyright 2018 Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pkgs
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const (
+	testRoot = "../../testpkgs"
+)
+
+var (
+	expected = map[string]string{
+		"/scenrioa/foo":                    "foo",
+		"/scenriob/foo":                    "foo",
+		"/scenriob/foo/v2":                 "foo",
+		"/scenrioc/mgmt/2019-10-11/foo":    "foo",
+		"/scenriod/mgmt/2019-10-11/foo":    "foo",
+		"/scenriod/mgmt/2019-10-11/foo/v2": "foo",
+		"/scenrioe/mgmt/2019-10-11/foo":    "foo",
+		"/scenrioe/mgmt/2019-10-11/foo/v2": "foo",
+		"/scenrioe/mgmt/2019-10-11/foo/v3": "foo",
+	}
+)
+
+func TestGetPkgs(t *testing.T) {
+	root, err := filepath.Abs(testRoot)
+	if err != nil {
+		t.Fatalf("failed to get absolute path: %+v", err)
+	}
+	pkgs, err := GetPkgs(root)
+	if err != nil {
+		t.Fatalf("failed to get packages: %+v", err)
+	}
+	if len(pkgs) != len(expected) {
+		t.Fatalf("expected %d packages, but got %d", len(expected), len(pkgs))
+	}
+	for _, pkg := range pkgs {
+		if pkgName, ok := expected[pkg.Dest]; !ok {
+			t.Fatalf("got pkg path '%s', but not found in expected", pkg.Dest)
+		} else if !strings.EqualFold(pkgName, pkg.Package.Name) {
+			t.Fatalf("expected package of '%s' in path '%s', but got '%s'", pkgName, pkg.Dest, pkg.Package.Name)
+		}
+	}
+}
+
+func TestPkg_GetApiVersion(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "/netapp/mgmt/2019-07-01/netapp",
+			expected: "2019-07-01",
+		},
+		{
+			input:    "/preview/resources/mgmt/2017-08-31-preview/managementgroups",
+			expected: "2017-08-31-preview",
+		},
+		{
+			input:    "/batch/2018-12-01.8.0/batch",
+			expected: "2018-12-01.8.0",
+		},
+		{
+			input:    "/cognitiveservices/v1.0/imagesearch",
+			expected: "v1.0",
+		},
+		{
+			input:    "/servicefabric/6.3/servicefabric",
+			expected: "6.3",
+		},
+		{
+			input:    "/keyvault/2015-06-01/keyvault",
+			expected: "2015-06-01",
+		},
+		{
+			input:    "/preview/datalake/analytics/2017-09-01-preview/job",
+			expected: "2017-09-01-preview",
+		},
+	}
+	for _, c := range cases {
+		p := Pkg{Dest: c.input}
+		api, err := p.GetAPIVersion()
+		if err != nil {
+			t.Fatalf("failed to get api version: %+v", err)
+		}
+		if !strings.EqualFold(api, c.expected) {
+			t.Fatalf("expected: %s, but got %s", c.expected, api)
+		}
+	}
+}