From ea1bb1d17405e879a02dfa3d27ae3032e9018f17 Mon Sep 17 00:00:00 2001 From: wanlindu Date: Thu, 18 May 2023 10:51:02 -0700 Subject: [PATCH 1/5] Update the prepare_prebuilt_workers to be able to specify repo. --- tools/README.md | 8 +-- tools/cmd/prepare_prebuilt_workers/main.go | 57 +++++++++++++--------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/tools/README.md b/tools/README.md index f2388018..0e23cae4 100644 --- a/tools/README.md +++ b/tools/README.md @@ -91,15 +91,15 @@ the following shows this process (building and pushing prebuilt images) for ```shell bin/prepare_prebuilt_workers \ - -l cxx:master \ - -l go:master \ + -l cxx:grpc/grpc:master \ + -l go:grpc/grpc-go:master \ -p "${image_registry}" \ -t "${tag}" \ -r containers/pre_built_workers ``` -This builds `cxx` and `go` images contain workers built from the commit/branch -we wish to test. +This builds `cxx` and `go` images contain workers built from the specific repo +and commit/branch we wish to test. The tool `prepare_prebuilt_workers` takes the following options: diff --git a/tools/cmd/prepare_prebuilt_workers/main.go b/tools/cmd/prepare_prebuilt_workers/main.go index efa02e9b..28d6abf9 100644 --- a/tools/cmd/prepare_prebuilt_workers/main.go +++ b/tools/cmd/prepare_prebuilt_workers/main.go @@ -37,11 +37,18 @@ import ( // Tests contains the values for fields that are accessible by // flags. type Tests struct { - preBuiltImagePrefix string - testTag string - dockerfileRoot string - buildOnly bool - languagesToGitref map[string]string + preBuiltImagePrefix string + testTag string + dockerfileRoot string + buildOnly bool + languagesToLanguageDetail map[string]LanguageDetail +} + +// LanguageDetail containers the details of each tested language. +type LanguageDetail struct { + Name string `json:"name"` + Repo string `json:"repo"` + Gitref string `json:"gitref"` } type langFlags []string @@ -72,7 +79,7 @@ func main() { flag.StringVar(&test.dockerfileRoot, "r", "", "root directory of Dockerfiles to build prebuilt images") - flag.Var(&languagesSelected, "l", "languages and its GITREF wish to run tests, example: cxx:") + flag.Var(&languagesSelected, "l", "languages, its repository and GITREF wish to run tests, example: cxx:grpc/grpc:") flag.Parse() @@ -94,7 +101,7 @@ func main() { log.Fatalf("Failed preparing prebuilt images: no language and its gitref pair specified, please provide languages and the GITREF as cxx:master") } - test.languagesToGitref = map[string]string{} + test.languagesToLanguageDetail = map[string]LanguageDetail{} converterToImageLanguage := map[string]string{ "c++": "cxx", "node_purejs": "node", @@ -105,41 +112,43 @@ func main() { for _, pair := range languagesSelected { split := strings.Split(pair, ":") - if len(split) != 2 || split[len(split)-1] == "" { - log.Fatalf("Input error in language and gitref selection, please follow the format as language:gitref, for example: c++:") + if len(split) != 3 || split[len(split)-1] == "" { + log.Fatalf("Input error in language and gitref selection, please follow the format as language:repository:gitref, for example: c++:grpc/grpc/") } - + var detail LanguageDetail lang := split[0] - gitref := split[1] - if convertedLang, ok := converterToImageLanguage[lang]; ok { - test.languagesToGitref[convertedLang] = gitref + detail.Name = convertedLang } else { - test.languagesToGitref[lang] = gitref + detail.Name = lang } + detail.Repo = split[1] + detail.Gitref = split[2] + test.languagesToLanguageDetail[detail.Name] = detail } - log.Println("Selected language : GITREF") - formattedMap, _ := json.MarshalIndent(test.languagesToGitref, "", " ") + log.Println("Selected language : REPOSITORY: GITREF") + formattedMap, _ := json.MarshalIndent(test.languagesToLanguageDetail, "", " ") log.Print(string(formattedMap)) var wg sync.WaitGroup - wg.Add(len(test.languagesToGitref)) + wg.Add(len(test.languagesToLanguageDetail)) uniqueCacheBreaker := time.Now().String() - for lang, gitRef := range test.languagesToGitref { - go func(lang string, gitRef string) { + for lang, detail := range test.languagesToLanguageDetail { + go func(lang string, detail LanguageDetail) { defer wg.Done() image := fmt.Sprintf("%s/%s:%s", test.preBuiltImagePrefix, lang, test.testTag) dockerfileLocation := fmt.Sprintf("%s/%s/", test.dockerfileRoot, lang) // build image - log.Println(fmt.Sprintf("building %s image", lang)) - // TODO(jtattermusch): allow setting REPOSITORY arg to allow building from user forks e.g. https://github.com/USER_NAME/grpc.git + log.Printf("building %s image\n", lang) buildCommandTimeoutSeconds := 30 * 60 // 30 mins should be enough for all languages - buildDockerImage := exec.Command("timeout", fmt.Sprintf("%ds", buildCommandTimeoutSeconds), "docker", "build", dockerfileLocation, "-t", image, "--build-arg", fmt.Sprintf("GITREF=%s", gitRef), "--build-arg", fmt.Sprintf("BREAK_CACHE=%s", uniqueCacheBreaker)) + cmd := fmt.Sprintf("docker build %s -t %s --build-arg REPOSITORY=%s --build-arg GITREF=%s --build-arg BREAK_CACHE=VAR", dockerfileLocation, image, detail.Repo, detail.Gitref) + log.Printf("Running: %s", cmd) + buildDockerImage := exec.Command("timeout", fmt.Sprintf("%ds", buildCommandTimeoutSeconds), "docker", "build", dockerfileLocation, "-t", image, "--build-arg", fmt.Sprintf("REPOSITORY=%s", detail.Repo), "--build-arg", fmt.Sprintf("GITREF=%s", detail.Gitref), "--build-arg", fmt.Sprintf("BREAK_CACHE=%s", uniqueCacheBreaker)) buildOutput, err := buildDockerImage.CombinedOutput() if err != nil { log.Printf("Failed building %s image. Dump of command's output will follow:\n", lang) @@ -152,7 +161,7 @@ func main() { if !test.buildOnly { // push image - log.Println(fmt.Sprintf("pushing %s image", lang)) + log.Printf("pushing %s image\n", lang) pushDockerImage := exec.Command("docker", "push", image) pushOutput, err := pushDockerImage.CombinedOutput() if err != nil { @@ -164,7 +173,7 @@ func main() { log.Println(string(pushOutput)) log.Printf("Succeeded pushing %s image to %s\n", lang, image) } - }(lang, gitRef) + }(lang, detail) } wg.Wait() From c46cf9e1da4350755f3beab9c1e217aa3b03508c Mon Sep 17 00:00:00 2001 From: wanlindu Date: Thu, 18 May 2023 18:12:49 -0700 Subject: [PATCH 2/5] Make prepare_prebuilt_workers.go backward-compatible. --- tools/README.md | 24 ++++++++-- tools/cmd/prepare_prebuilt_workers/main.go | 56 ++++++++++++---------- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/tools/README.md b/tools/README.md index 0e23cae4..46c831c9 100644 --- a/tools/README.md +++ b/tools/README.md @@ -91,20 +91,34 @@ the following shows this process (building and pushing prebuilt images) for ```shell bin/prepare_prebuilt_workers \ - -l cxx:grpc/grpc:master \ - -l go:grpc/grpc-go:master \ + -l cxx:master \ + -l go:master \ -p "${image_registry}" \ -t "${tag}" \ -r containers/pre_built_workers ``` -This builds `cxx` and `go` images contain workers built from the specific repo -and commit/branch we wish to test. +This builds `cxx` and `go` images contain workers built from the specific +commit/branch we wish to test. + +You may also specify the repo where the source code is pulled from by specifying +the repo, the following command would pull the source code from a forked repo +to build `cxx` and `go` images. + +```shell +bin/prepare_prebuilt_workers \ + -l cxx:/grpc:master \ + -l go:/grpc-go:master \ + -p "${image_registry}" \ + -t "${tag}" \ + -r containers/pre_built_workers +``` The tool `prepare_prebuilt_workers` takes the following options: - `-l`
Language and GITREF to benchmark. The language and its specific - GITREF wish to build workers from can be specified as `language:COMMIT_SHA`. + GITREF wish to build workers from can be specified as `language:COMMIT_SHA` or + `language:repo:COMMIT_SHA`. May be repeated. Valid input for language names are all in lower case: `csharp`, `c++`/`cxx`, `go`, `java`, `node`, `node_purejs`, `php7`. `php7_protobuf_c`, `python`, `python_asyncio` and `ruby`. diff --git a/tools/cmd/prepare_prebuilt_workers/main.go b/tools/cmd/prepare_prebuilt_workers/main.go index 28d6abf9..4f6c77f0 100644 --- a/tools/cmd/prepare_prebuilt_workers/main.go +++ b/tools/cmd/prepare_prebuilt_workers/main.go @@ -37,15 +37,15 @@ import ( // Tests contains the values for fields that are accessible by // flags. type Tests struct { - preBuiltImagePrefix string - testTag string - dockerfileRoot string - buildOnly bool - languagesToLanguageDetail map[string]LanguageDetail + preBuiltImagePrefix string + testTag string + dockerfileRoot string + buildOnly bool + languagesToLanguageSpec map[string]LanguageSpec } -// LanguageDetail containers the details of each tested language. -type LanguageDetail struct { +// LanguageSpec containers the specs of each tested language. +type LanguageSpec struct { Name string `json:"name"` Repo string `json:"repo"` Gitref string `json:"gitref"` @@ -79,7 +79,7 @@ func main() { flag.StringVar(&test.dockerfileRoot, "r", "", "root directory of Dockerfiles to build prebuilt images") - flag.Var(&languagesSelected, "l", "languages, its repository and GITREF wish to run tests, example: cxx:grpc/grpc:") + flag.Var(&languagesSelected, "l", "languages, its repository and GITREF wish to run tests, example: cxx: or cxx:grpc/grpc:") flag.Parse() @@ -101,7 +101,7 @@ func main() { log.Fatalf("Failed preparing prebuilt images: no language and its gitref pair specified, please provide languages and the GITREF as cxx:master") } - test.languagesToLanguageDetail = map[string]LanguageDetail{} + test.languagesToLanguageSpec = map[string]LanguageSpec{} converterToImageLanguage := map[string]string{ "c++": "cxx", "node_purejs": "node", @@ -112,32 +112,37 @@ func main() { for _, pair := range languagesSelected { split := strings.Split(pair, ":") - if len(split) != 3 || split[len(split)-1] == "" { + if len(split) < 2 || split[len(split)-1] == "" { log.Fatalf("Input error in language and gitref selection, please follow the format as language:repository:gitref, for example: c++:grpc/grpc/") } - var detail LanguageDetail + var spec LanguageSpec lang := split[0] if convertedLang, ok := converterToImageLanguage[lang]; ok { - detail.Name = convertedLang + spec.Name = convertedLang } else { - detail.Name = lang + spec.Name = lang } - detail.Repo = split[1] - detail.Gitref = split[2] - test.languagesToLanguageDetail[detail.Name] = detail + if len(split) == 3 { + spec.Repo = split[1] + spec.Gitref = split[2] + } else { + spec.Repo = "" + spec.Gitref = split[1] + } + test.languagesToLanguageSpec[spec.Name] = spec } log.Println("Selected language : REPOSITORY: GITREF") - formattedMap, _ := json.MarshalIndent(test.languagesToLanguageDetail, "", " ") + formattedMap, _ := json.MarshalIndent(test.languagesToLanguageSpec, "", " ") log.Print(string(formattedMap)) var wg sync.WaitGroup - wg.Add(len(test.languagesToLanguageDetail)) + wg.Add(len(test.languagesToLanguageSpec)) uniqueCacheBreaker := time.Now().String() - for lang, detail := range test.languagesToLanguageDetail { - go func(lang string, detail LanguageDetail) { + for lang, spec := range test.languagesToLanguageSpec { + go func(lang string, spec LanguageSpec) { defer wg.Done() image := fmt.Sprintf("%s/%s:%s", test.preBuiltImagePrefix, lang, test.testTag) @@ -146,9 +151,12 @@ func main() { // build image log.Printf("building %s image\n", lang) buildCommandTimeoutSeconds := 30 * 60 // 30 mins should be enough for all languages - cmd := fmt.Sprintf("docker build %s -t %s --build-arg REPOSITORY=%s --build-arg GITREF=%s --build-arg BREAK_CACHE=VAR", dockerfileLocation, image, detail.Repo, detail.Gitref) - log.Printf("Running: %s", cmd) - buildDockerImage := exec.Command("timeout", fmt.Sprintf("%ds", buildCommandTimeoutSeconds), "docker", "build", dockerfileLocation, "-t", image, "--build-arg", fmt.Sprintf("REPOSITORY=%s", detail.Repo), "--build-arg", fmt.Sprintf("GITREF=%s", detail.Gitref), "--build-arg", fmt.Sprintf("BREAK_CACHE=%s", uniqueCacheBreaker)) + buildDockerImage := exec.Command("timeout", fmt.Sprintf("%ds", buildCommandTimeoutSeconds), "docker", "build", dockerfileLocation, "-t", image, "--build-arg", fmt.Sprintf("GITREF=%s", spec.Gitref), "--build-arg", fmt.Sprintf("BREAK_CACHE=%s", uniqueCacheBreaker)) + if spec.Repo != "" { + //cmd = fmt.Sprintf("docker build %s -t %s --build-arg REPOSITORY=%s --build-arg GITREF=%s --build-arg BREAK_CACHE=VAR", dockerfileLocation, image, spec.Repo, spec.Gitref) + buildDockerImage.Args = append(buildDockerImage.Args, "--build-arg", fmt.Sprintf("REPOSITORY=%s", spec.Repo)) + } + log.Printf("Running command: %s", strings.Join(buildDockerImage.Args, " ")) buildOutput, err := buildDockerImage.CombinedOutput() if err != nil { log.Printf("Failed building %s image. Dump of command's output will follow:\n", lang) @@ -173,7 +181,7 @@ func main() { log.Println(string(pushOutput)) log.Printf("Succeeded pushing %s image to %s\n", lang, image) } - }(lang, detail) + }(lang, spec) } wg.Wait() From 5b5b1aefb5e70250579d44b918fc1ab98c47c713 Mon Sep 17 00:00:00 2001 From: wanlindu Date: Fri, 19 May 2023 14:19:12 -0700 Subject: [PATCH 3/5] Address comment. --- tools/README.md | 8 ++++---- tools/cmd/prepare_prebuilt_workers/main.go | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tools/README.md b/tools/README.md index 46c831c9..5249c0e8 100644 --- a/tools/README.md +++ b/tools/README.md @@ -102,13 +102,13 @@ This builds `cxx` and `go` images contain workers built from the specific commit/branch we wish to test. You may also specify the repo where the source code is pulled from by specifying -the repo, the following command would pull the source code from a forked repo -to build `cxx` and `go` images. +the repo. The following command pulls the source code from forked repos +`${USER}/grpc` and `${USER}/grpc-go` to build cxx and go images ```shell bin/prepare_prebuilt_workers \ - -l cxx:/grpc:master \ - -l go:/grpc-go:master \ + -l cxx:${USER}/grpc:master \ + -l go:${USER}/grpc-go:master \ -p "${image_registry}" \ -t "${tag}" \ -r containers/pre_built_workers diff --git a/tools/cmd/prepare_prebuilt_workers/main.go b/tools/cmd/prepare_prebuilt_workers/main.go index 4f6c77f0..55a922ab 100644 --- a/tools/cmd/prepare_prebuilt_workers/main.go +++ b/tools/cmd/prepare_prebuilt_workers/main.go @@ -110,10 +110,12 @@ func main() { } for _, pair := range languagesSelected { - split := strings.Split(pair, ":") + split := strings.SplitN(pair, ":", 3) - if len(split) < 2 || split[len(split)-1] == "" { - log.Fatalf("Input error in language and gitref selection, please follow the format as language:repository:gitref, for example: c++:grpc/grpc/") + // C++:master will be split to 2 items, c++:grpc/grpc:master will be + // split to 3 items. + if (len(split) == 0 || split[len(split)-1] == "" ){ + log.Fatalf("Input error in language and gitref selection. Please follow the format language:gitref or language:repository:gitref, for example c++:master or c++:grpc/grpc:master") } var spec LanguageSpec lang := split[0] @@ -148,12 +150,11 @@ func main() { image := fmt.Sprintf("%s/%s:%s", test.preBuiltImagePrefix, lang, test.testTag) dockerfileLocation := fmt.Sprintf("%s/%s/", test.dockerfileRoot, lang) - // build image + // Build image log.Printf("building %s image\n", lang) buildCommandTimeoutSeconds := 30 * 60 // 30 mins should be enough for all languages buildDockerImage := exec.Command("timeout", fmt.Sprintf("%ds", buildCommandTimeoutSeconds), "docker", "build", dockerfileLocation, "-t", image, "--build-arg", fmt.Sprintf("GITREF=%s", spec.Gitref), "--build-arg", fmt.Sprintf("BREAK_CACHE=%s", uniqueCacheBreaker)) if spec.Repo != "" { - //cmd = fmt.Sprintf("docker build %s -t %s --build-arg REPOSITORY=%s --build-arg GITREF=%s --build-arg BREAK_CACHE=VAR", dockerfileLocation, image, spec.Repo, spec.Gitref) buildDockerImage.Args = append(buildDockerImage.Args, "--build-arg", fmt.Sprintf("REPOSITORY=%s", spec.Repo)) } log.Printf("Running command: %s", strings.Join(buildDockerImage.Args, " ")) @@ -168,7 +169,7 @@ func main() { log.Printf("Succeeded building %s image: %s\n", lang, image) if !test.buildOnly { - // push image + // Push image log.Printf("pushing %s image\n", lang) pushDockerImage := exec.Command("docker", "push", image) pushOutput, err := pushDockerImage.CombinedOutput() From 17baa672d51ba74c592d669276ef1f5fed007011 Mon Sep 17 00:00:00 2001 From: wanlindu Date: Fri, 19 May 2023 15:40:15 -0700 Subject: [PATCH 4/5] Address comments. --- tools/README.md | 4 ++-- tools/cmd/prepare_prebuilt_workers/main.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/README.md b/tools/README.md index 5249c0e8..98380cc3 100644 --- a/tools/README.md +++ b/tools/README.md @@ -107,8 +107,8 @@ the repo. The following command pulls the source code from forked repos ```shell bin/prepare_prebuilt_workers \ - -l cxx:${USER}/grpc:master \ - -l go:${USER}/grpc-go:master \ + -l cxx:"${USER}"/grpc:master \ + -l go:"${USER}"/grpc-go:master \ -p "${image_registry}" \ -t "${tag}" \ -r containers/pre_built_workers diff --git a/tools/cmd/prepare_prebuilt_workers/main.go b/tools/cmd/prepare_prebuilt_workers/main.go index 55a922ab..853729cc 100644 --- a/tools/cmd/prepare_prebuilt_workers/main.go +++ b/tools/cmd/prepare_prebuilt_workers/main.go @@ -109,12 +109,12 @@ func main() { "python_asyncio": "python", } - for _, pair := range languagesSelected { - split := strings.SplitN(pair, ":", 3) + for _, s := range languagesSelected { + split := strings.SplitN(s, ":", 3) // C++:master will be split to 2 items, c++:grpc/grpc:master will be // split to 3 items. - if (len(split) == 0 || split[len(split)-1] == "" ){ + if len(split) < 2 || split[len(split)-1] == "" { log.Fatalf("Input error in language and gitref selection. Please follow the format language:gitref or language:repository:gitref, for example c++:master or c++:grpc/grpc:master") } var spec LanguageSpec From 0d1fc7d60fd06a9b1d00237f0f5161209665a47a Mon Sep 17 00:00:00 2001 From: wanlindu Date: Fri, 19 May 2023 17:04:14 -0700 Subject: [PATCH 5/5] Address comment. --- tools/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/README.md b/tools/README.md index 98380cc3..72eda5c6 100644 --- a/tools/README.md +++ b/tools/README.md @@ -107,8 +107,8 @@ the repo. The following command pulls the source code from forked repos ```shell bin/prepare_prebuilt_workers \ - -l cxx:"${USER}"/grpc:master \ - -l go:"${USER}"/grpc-go:master \ + -l "cxx:${USER}/grpc:master" \ + -l "go:${USER}/grpc-go:master" \ -p "${image_registry}" \ -t "${tag}" \ -r containers/pre_built_workers