diff --git a/Makefile b/Makefile index 1ca0595..11451ac 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ clean:: ## Remove generated files # -- Test -------------------------------------------------------------- COVERFILE = coverage.out -COVERAGE = 91.0 +COVERAGE = 85.0 test: ## Run tests and generate a coverage file go test -coverprofile=$(COVERFILE) ./... diff --git a/go.mod b/go.mod index acdaca4..48496f2 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,17 @@ require ( github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 github.com/alecthomas/colour v0.1.0 // indirect github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 // indirect + github.com/anz-bank/sysl-examples v0.0.1 // indirect github.com/arr-ai/frozen v0.11.1 github.com/golang/protobuf v1.4.0 + github.com/google/go-github/v32 v32.1.0 github.com/mattn/go-isatty v0.0.12 // indirect + github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.4.2 + github.com/spf13/afero v1.3.4 github.com/stretchr/testify v1.5.1 go.opencensus.io v0.22.4 + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be google.golang.org/grpc v1.29.0 google.golang.org/protobuf v1.21.0 ) diff --git a/go.sum b/go.sum index bbb9be1..2b463be 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,10 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/anz-bank/sysl-examples v0.0.1 h1:Qn+aF2847YGUdJzeJG7D4oa7cuFXt9X1O/JRTS+dG6o= +github.com/anz-bank/sysl-examples v0.0.1/go.mod h1:AdokLChJxixvIBlAByw+kI13pAKawobfpcYCyPssvLQ= +github.com/anz-bank/sysl-examples v0.0.2 h1:szNx6secul+G1tyQaWTMKXTD2YByJ/ln2V85O34IhZk= +github.com/anz-bank/sysl-examples v0.0.2/go.mod h1:AdokLChJxixvIBlAByw+kI13pAKawobfpcYCyPssvLQ= github.com/arr-ai/frozen v0.11.1 h1:mX5fS6eeONZUNLFp/ZVEBaU+1RKUH8HgSUQflQm7YIA= github.com/arr-ai/frozen v0.11.1/go.mod h1:YEr4TubkrAoA3f9U1R4PcEVB5ocBUdiS3NKh03cbVts= github.com/arr-ai/hash v0.4.0 h1:VPIDl5nkhq8qntxmsNd00bdj+UVs2vRKFzzM7HZkR7Q= @@ -61,6 +65,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= +github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -69,6 +77,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -85,7 +94,9 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -111,6 +122,8 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/afero v1.3.4 h1:8q6vk3hthlpb2SouZcnBVKboxWQWMDNF38bwholZrJc= +github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -125,6 +138,8 @@ go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -135,9 +150,11 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -149,6 +166,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -160,6 +178,8 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -168,6 +188,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/mod/github.go b/mod/github.go new file mode 100644 index 0000000..5f39aaf --- /dev/null +++ b/mod/github.go @@ -0,0 +1,192 @@ +package mod + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/google/go-github/v32/github" + "github.com/pkg/errors" + "golang.org/x/oauth2" +) + +type githubMgr struct { + client *github.Client + cacheDir string +} + +func (d *githubMgr) Init(cacheDir, accessToken *string) error { + if accessToken == nil { + d.client = github.NewClient(nil) + } else { + // Authenticated clients can make up to 5,000 requests per hour. + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: *accessToken}, + ) + tc := oauth2.NewClient(context.Background(), ts) + + d.client = github.NewClient(tc) + } + + if cacheDir == nil { + return errors.New("cache directory cannot be empty") + } + d.cacheDir = *cacheDir + return nil +} + +func (d *githubMgr) Get(filename, ver string, m *Modules) (*Module, error) { + repoPath, err := getGitHubRepoPath(filename) + if err != nil { + return nil, err + } + ctx := context.Background() + var refOps *github.RepositoryContentGetOptions + if ver != "" { + refOps = &github.RepositoryContentGetOptions{Ref: ver} + } + + fileContent, _, _, err := d.client.Repositories.GetContents(ctx, repoPath.owner, repoPath.repo, repoPath.path, refOps) + if err != nil { + if _, ok := err.(*github.RateLimitError); ok { + return nil, errors.Wrap(err, + "\033[1;36mplease setup your GitHub access token\033[0m") + } + return nil, err + } + + content, err := fileContent.GetContent() + if err != nil { + return nil, err + } + if ver == "" || ver == MasterBranch { + ref, _, err := d.client.Git.GetRef(ctx, repoPath.owner, repoPath.repo, "heads/"+MasterBranch) + if err != nil { + return nil, err + } + ver = "v0.0.0-" + ref.GetObject().GetSHA()[:12] + } + + name := strings.Join([]string{"github.com", repoPath.owner, repoPath.repo}, "/") + dir := filepath.Join(d.cacheDir, "github.com", repoPath.owner, repoPath.repo) + dir = AppendVersion(dir, ver) + new := &Module{ + Name: name, + Dir: dir, + Version: ver, + } + + fname := filepath.Join(dir, repoPath.path) + if !fileExists(fname, false) { + err = writeFile(fname, []byte(content)) + if err != nil { + return nil, err + } + m.Add(new) + } + + return new, nil +} + +func (*githubMgr) Find(filename, ver string, m *Modules) *Module { + if ver == "" || ver == MasterBranch { + return nil + } + + for _, mod := range *m { + if hasPathPrefix(mod.Name, filename) { + if mod.Version == ver { + return mod + } + } + } + + return nil +} + +func (d *githubMgr) Load(m *Modules) error { + githubPath := filepath.Join(d.cacheDir, "github.com") + if !fileExists(githubPath, true) { + if err := os.MkdirAll(githubPath, 0770); err != nil { + return err + } + } + + githubDir, err := os.Open(githubPath) + if err != nil { + return err + } + + owners, err := githubDir.Readdirnames(-1) + if err != nil { + return err + } + + for _, owner := range owners { + ownerDir, err := os.Open(filepath.Join(githubPath, owner)) + if err != nil { + return err + } + repos, err := ownerDir.Readdirnames(-1) + if err != nil { + return err + } + for _, repo := range repos { + p, ver := ExtractVersion(repo) + name := filepath.Join("github.com", owner, p) + m.Add(&Module{ + Name: name, + Dir: filepath.Join(ownerDir.Name(), repo), + Version: ver, + }) + } + } + + return nil +} + +type githubRepoPath struct { + owner string + repo string + path string +} + +func getGitHubRepoPath(filename string) (*githubRepoPath, error) { + names := strings.FieldsFunc(filename, func(c rune) bool { + return c == '/' + }) + if len(names) < 4 { + return nil, fmt.Errorf("the imported module path %s is invalid", filename) + } + if names[0] != "github.com" { + return nil, errors.New("non-github.com repository is not supported under GitHub mode") + } + + owner := names[1] + repo := names[2] + path := path.Join(names[3:]...) + + return &githubRepoPath{ + owner: owner, + repo: repo, + path: path, + }, nil +} + +func writeFile(filename string, content []byte) error { + if err := os.MkdirAll(filepath.Dir(filename), 0770); err != nil { + return err + } + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + if _, err = file.Write(content); err != nil { + return err + } + return nil +} diff --git a/mod/github_test.go b/mod/github_test.go new file mode 100644 index 0000000..7f26c74 --- /dev/null +++ b/mod/github_test.go @@ -0,0 +1,90 @@ +package mod + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGitHubMgrInit(t *testing.T) { + githubmod := &githubMgr{} + dir := ".pkgcache" + err := githubmod.Init(&dir, nil) + assert.NoError(t, err) + + err = githubmod.Init(nil, nil) + assert.Error(t, err) +} + +func TestGitHubMgrGet(t *testing.T) { + githubmod := &githubMgr{} + dir := ".pkgcache" + err := githubmod.Init(&dir, nil) + assert.NoError(t, err) + testMods := Modules{} + + mod, err := githubmod.Get(RemoteDepsFile, "", &testMods) + assert.Nil(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + + mod, err = githubmod.Get(RemoteDepsFile, MasterBranch, &testMods) + assert.Nil(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + + mod, err = githubmod.Get(RemoteDepsFile, "v0.0.1", &testMods) + assert.Nil(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + assert.Equal(t, "v0.0.1", mod.Version) + + mod, err = githubmod.Get("github.com/anz-bank/wrong/path", "", &testMods) + assert.Error(t, err) + assert.Nil(t, mod) +} + +func TestGitHubMgrFind(t *testing.T) { + githubmod := &githubMgr{} + testMods := Modules{} + local := &Module{Name: "local", Version: "v0.0.1"} + mod1 := &Module{Name: "remote", Version: "v1.0.0-41f04d3bba15"} + mod2 := &Module{Name: "remote", Version: "v0.2.0"} + testMods.Add(local) + testMods.Add(mod1) + testMods.Add(mod2) + + assert.Equal(t, local, githubmod.Find("local/filename", "v0.0.1", &testMods)) + assert.Equal(t, local, githubmod.Find("local/filename2", "v0.0.1", &testMods)) + assert.Equal(t, local, githubmod.Find(".//local/filename", "v0.0.1", &testMods)) + assert.Equal(t, local, githubmod.Find("local", "v0.0.1", &testMods)) + assert.Nil(t, githubmod.Find("local2/filename", "v0.0.1", &testMods)) + + assert.Nil(t, githubmod.Find("local/filename", MasterBranch, &testMods)) + + assert.Equal(t, mod2, githubmod.Find("remote/filename", "v0.2.0", &testMods)) + assert.Nil(t, githubmod.Find("remote/filename", "v1.0.0", &testMods)) +} + +func TestGetGitHubRepoPath(t *testing.T) { + t.Parallel() + tests := []struct { + filename string + path *githubRepoPath + }{ + {"github.com/anz-bank/pkg", nil}, + {"github.com/anz-bank/pkg/", nil}, + {"github.com/anz-bank/pkg/deps.sysl", &githubRepoPath{"anz-bank", "pkg", "deps.sysl"}}, + {"github.com/anz-bank/pkg/nested/module/deps.sysl", &githubRepoPath{"anz-bank", "pkg", "nested/module/deps.sysl"}}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.filename, func(t *testing.T) { + t.Parallel() + p, err := getGitHubRepoPath(tt.filename) + if tt.path == nil { + assert.Error(t, err) + } else { + assert.Nil(t, err) + } + assert.Equal(t, tt.path, p) + }) + } +} diff --git a/mod/gomodules.go b/mod/gomodules.go new file mode 100644 index 0000000..7808d21 --- /dev/null +++ b/mod/gomodules.go @@ -0,0 +1,163 @@ +package mod + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type goModule struct { + Path string + Dir string + Version string +} + +type goModules struct{} + +func (d *goModules) Init(modName string) error { + err := runGo(context.Background(), ioutil.Discard, "mod", "init", modName) + if err != nil { + return errors.New(fmt.Sprintf("go mod init failed: %s", err.Error())) + } + + return nil +} + +func (d *goModules) Get(filename, ver string, m *Modules) (mod *Module, err error) { + if names := strings.Split(filename, "/"); len(names) > 0 { + for i := range names[1:] { + gogetPath := path.Join(names[:1+i]...) + gogetPath = AppendVersion(gogetPath, ver) + + err = goGet(gogetPath) + if err == nil { + err = d.Load(m) + if err != nil { + return nil, err + } + mod = d.Find(filename, ver, m) + if mod == nil { + return nil, fmt.Errorf("error finding module of file %s", filename) + } + return mod, nil + } + logrus.Debugf("go get %s error: %s\n", gogetPath, err.Error()) + } + } + + return nil, errors.New("no such module") +} + +func (*goModules) Find(filename, ver string, m *Modules) *Module { + for i, mod := range *m { + if hasPathPrefix(mod.Name, filename) { + if i == 0 && ver != "" && ver != MasterBranch { + logrus.Warn("importing files from current folder in remote way is incorrect: use local importing instead") + } + if i == 0 || ver == "" || ver == MasterBranch || ver == mod.Version { + return mod + } + } + } + + return nil +} + +func (*goModules) Load(m *Modules) error { + err := goDownload() + if err != nil { + return err + } + + b, err := goList() + if err != nil { + return err + } + + dec := json.NewDecoder(b) + for { + module := &goModule{} + if err := dec.Decode(module); err != nil { + if err == io.EOF { + break + } + return errors.Wrap(err, "failed to decode modules list") + } + + m.Add(&Module{ + Name: module.Path, + Dir: module.Dir, + Version: module.Version, + }) + } + + return nil +} + +func goGet(args ...string) error { + if err := runGo(context.Background(), logrus.StandardLogger().Out, append([]string{"get"}, args...)...); err != nil { // nolint:lll + return errors.Wrapf(err, "failed to get %q", args) + } + return nil +} + +func goDownload() error { + err := runGo(context.Background(), ioutil.Discard, "mod", "download") + if err != nil { + return errors.Wrap(err, "failed to download modules") + } + return nil +} + +func goList() (io.Reader, error) { + b := &bytes.Buffer{} + err := runGo(context.Background(), b, "list", "-m", "-json", "all") + if err != nil { + return b, errors.Wrap(err, "failed to list modules") + } + return b, nil +} + +func runGo(ctx context.Context, out io.Writer, args ...string) error { + cmd := exec.CommandContext(ctx, "go", args...) + + wd, err := os.Getwd() + if err != nil { + return errors.Errorf("get current working directory error: %s\n", err.Error()) + } + cmd.Dir = wd + + errbuf := new(bytes.Buffer) + cmd.Stderr = errbuf + cmd.Stdout = out + + logrus.Debugf("running command `go %v`\n", strings.Join(args, " ")) + if err := cmd.Run(); err != nil { + if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { + return nil + } + + _, ok := err.(*exec.ExitError) + if !ok { + return errors.Errorf("failed to execute 'go %v': %s %T", args, err, err) + } + + // Too old Go version + if strings.Contains(errbuf.String(), "flag provided but not defined") { + return nil + } + return errors.Errorf("go command failed: %s", errbuf) + } + + return nil +} diff --git a/mod/gomodules_test.go b/mod/gomodules_test.go new file mode 100644 index 0000000..e9e4f92 --- /dev/null +++ b/mod/gomodules_test.go @@ -0,0 +1,108 @@ +package mod + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" +) + +func TestModInit(t *testing.T) { + fs := afero.NewOsFs() + gomod := &goModules{} + + // assumes the test folder (cwd) is not a go module folder + removeFile(t, fs, "go.sum") + removeFile(t, fs, "go.mod") + + err := gomod.Init("test") + assert.NoError(t, err) + + removeFile(t, fs, "go.sum") + removeFile(t, fs, "go.mod") +} + +func TestModInitAlreadyExists(t *testing.T) { + fs := afero.NewOsFs() + gomod := &goModules{} + + // assumes the test folder (cwd) is not a go module folder + removeFile(t, fs, "go.sum") + removeFile(t, fs, "go.mod") + + err := gomod.Init("test") + assert.NoError(t, err) + + err = gomod.Init("test") + assert.Error(t, err) + + removeFile(t, fs, "go.sum") + removeFile(t, fs, "go.mod") +} + +func TestGoModulesGet(t *testing.T) { + gomod := &goModules{} + testMods := Modules{} + + mod, err := gomod.Get(RemoteDepsFile, "", &testMods) + assert.Nil(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + + mod, err = gomod.Get(RemoteDepsFile, MasterBranch, &testMods) + assert.Nil(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + + mod, err = gomod.Get(RemoteDepsFile, "v0.0.1", &testMods) + assert.Nil(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + assert.Equal(t, "v0.0.1", mod.Version) + + mod, err = gomod.Get("github.com/anz-bank/wrongpath", "", &testMods) + assert.Error(t, err) + assert.Nil(t, mod) +} + +func TestGoModulesFind(t *testing.T) { + gomod := &goModules{} + testMods := Modules{} + local := &Module{Name: "local"} + mod2 := &Module{Name: "remote", Version: "v0.2.0"} + testMods.Add(local) + testMods.Add(mod2) + + assert.Equal(t, local, gomod.Find("local/filename", "", &testMods)) + assert.Equal(t, local, gomod.Find("local/filename2", "", &testMods)) + assert.Equal(t, local, gomod.Find(".//local/filename", "", &testMods)) + assert.Equal(t, local, gomod.Find("local", "", &testMods)) + assert.Nil(t, gomod.Find("local2/filename", "", &testMods)) + + assert.Equal(t, local, gomod.Find("local/filename", MasterBranch, &testMods)) + assert.Equal(t, local, gomod.Find("local/filename", "v0.0.1", &testMods)) + + assert.Equal(t, mod2, gomod.Find("remote/filename", "v0.2.0", &testMods)) + assert.Nil(t, gomod.Find("remote/filename", "v1.0.0", &testMods)) +} + +func RemoveGomodFile(t *testing.T, fs afero.Fs) { + removeFile(t, fs, "go.mod") + removeFile(t, fs, "go.sum") +} + +func CreateGomodFile(t *testing.T, fs afero.Fs) { + gomod, err := fs.Create("go.mod") + assert.NoError(t, err) + defer gomod.Close() + _, err = gomod.WriteString("module github.com/anz-bank/pkg/mod") + assert.NoError(t, err) + err = gomod.Sync() + assert.NoError(t, err) +} + +func removeFile(t *testing.T, fs afero.Fs, file string) { + exists, err := afero.Exists(fs, file) + assert.NoError(t, err) + if exists { + err = fs.Remove(file) + assert.NoError(t, err) + } +} diff --git a/mod/module.go b/mod/module.go new file mode 100644 index 0000000..e952d10 --- /dev/null +++ b/mod/module.go @@ -0,0 +1,110 @@ +package mod + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +type Module struct { + // The name of module joined by forward slash(/). e.g. "github.com/anz-bank/foo" + Name string + // The absolute path to the module. + // e.g. "/Users/username/go/pkg/mod/github.com/anz-bank/foo@v1.1.0" on Linux and macOS + // "C:\Users\username\go\pkg\mod\github.com\anz-bank\foo@v1.1.0" on Windows + Dir string + // The version of the module. e.g. "v1.1.0" + Version string +} + +type Modules []*Module + +var modules Modules +var manager DependencyManager = &goModules{} +var mode ModeType + +type ModeType string + +const ( + GitHubMode ModeType = "github" + GoModulesMode ModeType = "go modules" +) +const MasterBranch = "master" + +type DependencyManager interface { + // Download external dependency to local directory + Get(filename, ver string, m *Modules) (*Module, error) + // Find dependency in m *Modules + Find(filename, ver string, m *Modules) *Module + // Load local cache into m *Modules + Load(m *Modules) error +} + +func (m *Modules) Add(v *Module) { + *m = append(*m, v) +} + +func (m *Modules) Len() int { + return len(*m) +} + +func Config(m ModeType, cacheDir, accessToken *string) error { + mode = m + switch mode { + case GitHubMode: + gh := &githubMgr{} + if err := gh.Init(cacheDir, accessToken); err != nil { + return err + } + manager = gh + case GoModulesMode: + if !fileExists("go.mod", false) { + return errors.New("no go.mod file, run `go mod init` first") + } + manager = &goModules{} + default: + return fmt.Errorf("unknown mode type %s", mode) + } + return nil +} + +func Retrieve(name string, ver string) (*Module, error) { + if modules.Len() == 0 { + if err := manager.Load(&modules); err != nil { + return nil, fmt.Errorf("error loading modules: %s", err.Error()) + } + } + + if ver != MasterBranch || (mode == GitHubMode && ver != "") { + mod := manager.Find(name, ver, &modules) + if mod != nil { + return mod, nil + } + } + + return manager.Get(name, ver, &modules) +} + +func hasPathPrefix(prefix, s string) bool { + prefix = filepath.Clean(prefix) + s = filepath.Clean(s) + + if len(s) > len(prefix) { + return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix + } + + return s == prefix +} + +func fileExists(filename string, isDir bool) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + if isDir { + return info.IsDir() + } + return !info.IsDir() +} diff --git a/mod/module_test.go b/mod/module_test.go new file mode 100644 index 0000000..b4b65da --- /dev/null +++ b/mod/module_test.go @@ -0,0 +1,136 @@ +package mod + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" +) + +const ( + SyslDepsFile = "github.com/anz-bank/sysl/tests/deps.sysl" + SyslRepo = "github.com/anz-bank/sysl" + RemoteDepsFile = "github.com/anz-bank/sysl-examples/demos/simple/simple.sysl" + RemoteRepo = "github.com/anz-bank/sysl-examples" +) + +func TestConfigGitHubMode(t *testing.T) { + err := Config(GitHubMode, nil, nil) + assert.Error(t, err) + + cacheDir := ".cache" + err = Config(GitHubMode, &cacheDir, nil) + assert.NoError(t, err) +} + +func TestConfigGoModulesMode(t *testing.T) { + fs := afero.NewOsFs() + CreateGomodFile(t, fs) + defer RemoveGomodFile(t, fs) + err := Config(GoModulesMode, nil, nil) + assert.NoError(t, err) +} +func TestConfigWrongMode(t *testing.T) { + err := Config("wrong", nil, nil) + assert.Error(t, err) +} + +func TestAdd(t *testing.T) { + var testMods Modules + testMods.Add(&Module{Name: "modulepath"}) + assert.Equal(t, 1, len(testMods)) + assert.Equal(t, &Module{Name: "modulepath"}, testMods[0]) +} + +func TestLen(t *testing.T) { + var testMods Modules + assert.Equal(t, 0, testMods.Len()) + testMods.Add(&Module{Name: "modulepath"}) + assert.Equal(t, 1, testMods.Len()) +} + +func TestRetrieveGoModules(t *testing.T) { + fs := afero.NewOsFs() + CreateGomodFile(t, fs) + defer RemoveGomodFile(t, fs) + + filename := SyslDepsFile + mod, err := Retrieve(filename, "") + assert.NoError(t, err) + assert.Equal(t, SyslRepo, mod.Name) + + filename = RemoteDepsFile + mod, err = Retrieve(filename, "") + assert.NoError(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + + mod, err = Retrieve(filename, "v0.0.1") + assert.NoError(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + assert.Equal(t, "v0.0.1", mod.Version) +} + +func TestRetrieveWithWrongPath(t *testing.T) { + fs := afero.NewOsFs() + CreateGomodFile(t, fs) + defer RemoveGomodFile(t, fs) + + wrongpath := "wrong_file_path/deps.sysl" + mod, err := Retrieve(wrongpath, "") + assert.Error(t, err) + assert.Nil(t, mod) +} + +func TestRetrieveGitHubMode(t *testing.T) { + mode = GitHubMode + defer func() { + mode = GoModulesMode + }() + + filename := SyslDepsFile + mod, err := Retrieve(filename, "") + assert.NoError(t, err) + assert.Equal(t, SyslRepo, mod.Name) + + filename = RemoteDepsFile + mod, err = Retrieve(filename, "") + assert.NoError(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + + mod, err = Retrieve(filename, "v0.0.1") + assert.NoError(t, err) + assert.Equal(t, RemoteRepo, mod.Name) + assert.Equal(t, "v0.0.1", mod.Version) +} + +func TestRetrieveWithWrongPathGitHubMode(t *testing.T) { + mode = GitHubMode + defer func() { + mode = GoModulesMode + }() + + wrongpath := "wrong_file_path/deps.sysl" + mod, err := Retrieve(wrongpath, "") + assert.Error(t, err) + assert.Nil(t, mod) +} + +func TestHasPathPrefix(t *testing.T) { + t.Parallel() + tests := []struct { + prefix string + }{ + {"github.com/anz-bank/sysl"}, + {"github.com/anz-bank/sysl/"}, + {"github.com/anz-bank/sysl/deps.sysl"}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.prefix, func(t *testing.T) { + t.Parallel() + assert.True(t, hasPathPrefix(tt.prefix, "github.com/anz-bank/sysl/deps.sysl")) + }) + } + + assert.False(t, hasPathPrefix("github.com/anz-bank/sysl2", "github.com/anz-bank/sysl/deps.sysl")) +} diff --git a/mod/util.go b/mod/util.go new file mode 100644 index 0000000..84d0884 --- /dev/null +++ b/mod/util.go @@ -0,0 +1,20 @@ +package mod + +import "strings" + +func ExtractVersion(path string) (newpath, ver string) { + newpath = path + s := strings.Split(path, "@") + if len(s) > 1 { + ver = s[len(s)-1] + newpath = path[:len(path)-len(ver)-1] + } + return +} + +func AppendVersion(path, ver string) string { + if ver == "" { + return path + } + return path + "@" + ver +} diff --git a/mod/util_test.go b/mod/util_test.go new file mode 100644 index 0000000..1d3a48b --- /dev/null +++ b/mod/util_test.go @@ -0,0 +1,27 @@ +package mod + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtractVersion(t *testing.T) { + path, ver := ExtractVersion("github.com/anz-bank/pkg@v0.1") + assert.Equal(t, "github.com/anz-bank/pkg", path) + assert.Equal(t, "v0.1", ver) + + path, ver = ExtractVersion("github.com/anz-bank/pkg/foo@v0.2") + assert.Equal(t, "github.com/anz-bank/pkg/foo", path) + assert.Equal(t, "v0.2", ver) + + path, ver = ExtractVersion("github.com/anz-bank/pkg/foo") + assert.Equal(t, "github.com/anz-bank/pkg/foo", path) + assert.Equal(t, "", ver) +} + +func TestAppendVersion(t *testing.T) { + assert.Equal(t, "github.com/anz-bank/pkg@v0.1", AppendVersion("github.com/anz-bank/pkg", "v0.1")) + assert.Equal(t, "github.com/anz-bank/pkg/foo@v0.2", AppendVersion("github.com/anz-bank/pkg/foo", "v0.2")) + assert.Equal(t, "github.com/anz-bank/pkg/foo", AppendVersion("github.com/anz-bank/pkg/foo", "")) +}