diff --git a/Dockerfile b/Dockerfile index 437fb2737..8e034bf4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,56 @@ +# Support setting various labels on the final image +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.17-alpine3.16 as builder RUN apk add --no-cache make gcc musl-dev linux-headers git bash ADD . /go-ethereum -RUN cd /go-ethereum && make geth +RUN cd /go-ethereum && make geth-static # Pull Geth into a second stage deploy alpine container -FROM alpine:latest +FROM alpine:3.16 + +ARG BSC_USER=bsc +ARG BSC_USER_UID=1000 +ARG BSC_USER_GID=1000 + +ENV BSC_HOME=/bsc +ENV HOME=${BSC_HOME} +ENV DATA_DIR=/data + +ENV PACKAGES ca-certificates jq \ + bash bind-tools tini \ + grep curl sed + +RUN apk add --no-cache $PACKAGES \ + && rm -rf /var/cache/apk/* \ + && addgroup -g ${BSC_USER_GID} ${BSC_USER} \ + && adduser -u ${BSC_USER_UID} -G ${BSC_USER} --shell /bin/bash --no-create-home -D ${BSC_USER} \ + && addgroup ${BSC_USER} tty + +WORKDIR ${BSC_HOME} -RUN apk add --no-cache ca-certificates curl jq tini COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ +RUN mkdir -p ${DATA_DIR} \ + && chown -R ${BSC_USER_UID}:${BSC_USER_GID} ${BSC_HOME} ${DATA_DIR} + +VOLUME ${DATA_DIR} + +USER ${BSC_USER_UID}:${BSC_USER_GID} + +# rpc ws graphql EXPOSE 8545 8546 8547 30303 30303/udp -ENTRYPOINT ["geth"] \ No newline at end of file + +# Add some metadata labels to help programatic image consumption +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + +LABEL commit="$COMMIT" version="$VERSION" buildnum="$BUILDNUM" + +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/geth"] diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 483afad8c..f023d02d1 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -1,15 +1,56 @@ +# Support setting various labels on the final image +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.17-alpine3.16 as builder -RUN apk add --no-cache make gcc musl-dev linux-headers git +RUN apk add --no-cache make gcc musl-dev linux-headers git bash ADD . /go-ethereum -RUN cd /go-ethereum && make all +RUN cd /go-ethereum && make all-static # Pull all binaries into a second stage deploy alpine container -FROM alpine:latest +FROM alpine:3.16 + +ARG BSC_USER=bsc +ARG BSC_USER_UID=1000 +ARG BSC_USER_GID=1000 + +ENV BSC_HOME=/bsc +ENV HOME=${BSC_HOME} +ENV DATA_DIR=/data + +ENV PACKAGES ca-certificates jq \ + bash bind-tools tini \ + grep curl sed + +RUN apk add --no-cache $PACKAGES \ + && rm -rf /var/cache/apk/* \ + && addgroup -g ${BSC_USER_GID} ${BSC_USER} \ + && adduser -u ${BSC_USER_UID} -G ${BSC_USER} --shell /bin/bash --no-create-home -D ${BSC_USER} \ + && addgroup ${BSC_USER} tty + +WORKDIR ${BSC_HOME} -RUN apk add --no-cache ca-certificates COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/ -EXPOSE 8545 8546 30303 30303/udp +RUN mkdir -p ${DATA_DIR} \ + && chown -R ${BSC_USER_UID}:${BSC_USER_GID} ${BSC_HOME} ${DATA_DIR} + +VOLUME ${DATA_DIR} + +USER ${BSC_USER_UID}:${BSC_USER_GID} + +# rpc ws graphql +EXPOSE 8545 8546 8547 30303 30303/udp + +# Add some metadata labels to help programatic image consumption +ARG COMMIT="" +ARG VERSION="" +ARG BUILDNUM="" + +LABEL commit="$COMMIT" version="$VERSION" buildnum="$BUILDNUM" + +ENTRYPOINT ["/sbin/tini", "--"] \ No newline at end of file diff --git a/Makefile b/Makefile index 03f92e6f4..103240d35 100644 --- a/Makefile +++ b/Makefile @@ -17,9 +17,17 @@ geth: @echo "Done building." @echo "Run \"$(GOBIN)/geth\" to launch geth." +geth-static: + $(GORUN) build/ci.go install -static ./cmd/geth + @echo "Done building." + @echo "Run \"$(GOBIN)/geth\" to launch geth." + all: $(GORUN) build/ci.go install +all-static: + $(GORUN) build/ci.go install -static + android: $(GORUN) build/ci.go aar --local @echo "Done building." diff --git a/build/ci.go b/build/ci.go index df1cb50ab..69f993b02 100644 --- a/build/ci.go +++ b/build/ci.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build none // +build none /* @@ -23,20 +24,19 @@ Usage: go run build/ci.go Available commands are: - install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables - test [ -coverage ] [ packages... ] -- runs the tests - lint -- runs certain pre-selected linters - archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts - importkeys -- imports signing keys from env - debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package - nsis -- creates a Windows NSIS installer - aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive - xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework - xgo [ -alltools ] [ options ] -- cross builds according to options - purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore + install [ -arch architecture ] [-static] [ -cc compiler ] [ packages... ] -- builds packages and executables + test [ -coverage ] [ packages... ] -- runs the tests + lint -- runs certain pre-selected linters + archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts + importkeys -- imports signing keys from env + debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package + nsis -- creates a Windows NSIS installer + aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive + xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework + xgo [ -alltools ] [-static] [ options ] -- cross builds according to options + purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore For all commands, -n prevents execution of external programs (dry run mode). - */ package main @@ -54,6 +54,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" "time" @@ -128,19 +129,13 @@ var ( // Distros for which packages are created. // Note: vivid is unsupported because there is no golang-1.6 package for it. - // Note: wily is unsupported because it was officially deprecated on Launchpad. - // Note: yakkety is unsupported because it was officially deprecated on Launchpad. - // Note: zesty is unsupported because it was officially deprecated on Launchpad. - // Note: artful is unsupported because it was officially deprecated on Launchpad. - // Note: cosmic is unsupported because it was officially deprecated on Launchpad. - // Note: disco is unsupported because it was officially deprecated on Launchpad. - // Note: eoan is unsupported because it was officially deprecated on Launchpad. + // Note: the following Ubuntu releases have been officially deprecated on Launchpad: + // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy debDistroGoBoots = map[string]string{ "trusty": "golang-1.11", "xenial": "golang-go", "bionic": "golang-go", "focal": "golang-go", - "groovy": "golang-go", "hirsute": "golang-go", } @@ -152,7 +147,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.16.3" + dlgoVersion = "1.17.5" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -182,6 +177,8 @@ func main() { doLint(os.Args[2:]) case "archive": doArchive(os.Args[2:]) + case "docker": + doDocker(os.Args[2:]) case "debsrc": doDebianSource(os.Args[2:]) case "nsis": @@ -203,9 +200,10 @@ func main() { func doInstall(cmdline []string) { var ( - dlgo = flag.Bool("dlgo", false, "Download Go and build with it") - arch = flag.String("arch", "", "Architecture to cross build for") - cc = flag.String("cc", "", "C compiler to cross build with") + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Architecture to cross build for") + cc = flag.String("cc", "", "C compiler to cross build with") + staticlink = flag.Bool("static", false, "Static link build with") ) flag.CommandLine.Parse(cmdline) @@ -216,9 +214,12 @@ func doInstall(cmdline []string) { tc.Root = build.DownloadGo(csdb, dlgoVersion) } + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + // Configure the build. env := build.Env() - gobuild := tc.Go("build", buildFlags(env)...) + gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, // better disable it. This check isn't the best, it should probably @@ -251,7 +252,7 @@ func doInstall(cmdline []string) { } // buildFlags returns the go tool flags for building. -func buildFlags(env build.Environment) (flags []string) { +func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { var ld []string if env.Commit != "" { ld = append(ld, "-X", "main.gitCommit="+env.Commit) @@ -262,9 +263,26 @@ func buildFlags(env build.Environment) (flags []string) { if runtime.GOOS == "darwin" { ld = append(ld, "-s") } + if runtime.GOOS == "linux" { + staticflag := "" + if staticLinking { + staticflag = " -static" + // Under static linking, use of certain glibc features must be + // disabled to avoid shared library dependencies. + buildTags = append(buildTags, "osusergo", "netgo") + } + // Enforce the stacksize to 8M, which is the case on most platforms apart from + // alpine Linux. + extflag := "'-Wl,-z,stack-size=0x800000" + staticflag + "'" + ld = append(ld, "-extldflags", extflag) + } + if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + if len(buildTags) > 0 { + flags = append(flags, "-tags", strings.Join(buildTags, ",")) + } return flags } @@ -280,6 +298,7 @@ func doTest(cmdline []string) { coverage = flag.Bool("coverage", false, "Whether to record code coverage") verbose = flag.Bool("v", false, "Whether to log verbosely") timeout = flag.String("timeout", "10m", `Timeout of runing tests`) + race = flag.Bool("race", false, "Execute the race detector") ) flag.CommandLine.Parse(cmdline) @@ -303,6 +322,9 @@ func doTest(cmdline []string) { if *timeout != "" { gotest.Args = append(gotest.Args, []string{"-timeout", *timeout}...) } + if *race { + gotest.Args = append(gotest.Args, "-race") + } packages := []string{"./accounts/...", "./common/...", "./consensus/...", "./console/...", "./core/...", "./crypto/...", "./eth/...", "./ethclient/...", "./ethdb/...", "./event/...", "./graphql/...", "./les/...", @@ -334,10 +356,14 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.39.0" + const version = "1.48.0" csdb := build.MustLoadChecksums("build/checksums.txt") - base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) + arch := runtime.GOARCH + if arch == "arm" { + arch += "v" + os.Getenv("GOARM") + } + base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch) url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base) archivePath := filepath.Join(cachedir, base+".tar.gz") if err := csdb.DownloadFile(url, archivePath); err != nil { @@ -454,11 +480,177 @@ func maybeSkipArchive(env build.Environment) { os.Exit(0) } if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { - log.Printf("skipping archive creation because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) + log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag) os.Exit(0) } } +// Builds the docker images and optionally uploads them to Docker Hub. +func doDocker(cmdline []string) { + var ( + image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`) + manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`) + upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`) + ) + flag.CommandLine.Parse(cmdline) + + // Skip building and pushing docker images for PR builds + env := build.Env() + maybeSkipArchive(env) + + // Retrieve the upload credentials and authenticate + user := getenvBase64("DOCKER_HUB_USERNAME") + pass := getenvBase64("DOCKER_HUB_PASSWORD") + + if len(user) > 0 && len(pass) > 0 { + auther := exec.Command("docker", "login", "-u", string(user), "--password-stdin") + auther.Stdin = bytes.NewReader(pass) + build.MustRun(auther) + } + // Retrieve the version infos to build and push to the following paths: + // - ethereum/client-go:latest - Pushes to the master branch, Geth only + // - ethereum/client-go:stable - Version tag publish on GitHub, Geth only + // - ethereum/client-go:alltools-latest - Pushes to the master branch, Geth & tools + // - ethereum/client-go:alltools-stable - Version tag publish on GitHub, Geth & tools + // - ethereum/client-go:release-. - Version tag publish on GitHub, Geth only + // - ethereum/client-go:alltools-release-. - Version tag publish on GitHub, Geth & tools + // - ethereum/client-go:v.. - Version tag publish on GitHub, Geth only + // - ethereum/client-go:alltools-v.. - Version tag publish on GitHub, Geth & tools + var tags []string + + switch { + case env.Branch == "master": + tags = []string{"latest"} + case strings.HasPrefix(env.Tag, "v1."): + tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version} + } + // If architecture specific image builds are requested, build and push them + if *image { + build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".") + build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".") + + // Tag and upload the images to Docker Hub + for _, tag := range tags { + gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH) + toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH) + + // If the image already exists (non version tag), check the build + // number to prevent overwriting a newer commit if concurrent builds + // are running. This is still a tiny bit racey if two published are + // done at the same time, but that's extremely unlikely even on the + // master branch. + for _, img := range []string{gethImage, toolImage} { + if exec.Command("docker", "pull", img).Run() != nil { + continue // Generally the only failure is a missing image, which is good + } + buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() + if err != nil { + log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) + } + buildnum = bytes.TrimSpace(buildnum) + + if len(buildnum) > 0 && len(env.Buildnum) > 0 { + oldnum, err := strconv.Atoi(string(buildnum)) + if err != nil { + log.Fatalf("Failed to parse old image build number: %v", err) + } + newnum, err := strconv.Atoi(env.Buildnum) + if err != nil { + log.Fatalf("Failed to parse current build number: %v", err) + } + if oldnum > newnum { + log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum) + } else { + log.Printf("Updating %s from build %d to %d", img, oldnum, newnum) + } + } + } + build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage) + build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage) + build.MustRunCommand("docker", "push", gethImage) + build.MustRunCommand("docker", "push", toolImage) + } + } + // If multi-arch image manifest push is requested, assemble it + if len(*manifest) != 0 { + // Since different architectures are pushed by different builders, wait + // until all required images are updated. + var mismatch bool + for i := 0; i < 2; i++ { // 2 attempts, second is race check + mismatch = false // hope there's no mismatch now + + for _, tag := range tags { + for _, arch := range strings.Split(*manifest, ",") { + gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch) + toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch) + + for _, img := range []string{gethImage, toolImage} { + if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil { + log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out) + mismatch = true + break + } + buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput() + if err != nil { + log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum)) + } + buildnum = bytes.TrimSpace(buildnum) + + if string(buildnum) != env.Buildnum { + log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum) + mismatch = true + break + } + } + if mismatch { + break + } + } + if mismatch { + break + } + } + if mismatch { + // Build numbers mismatching, retry in a short time to + // avoid concurrent failes in both publisher images. If + // however the retry failed too, it means the concurrent + // builder is still crunching, let that do the publish. + if i == 0 { + time.Sleep(30 * time.Second) + } + continue + } + break + } + if mismatch { + log.Println("Relinquishing publish to other builder") + return + } + // Assemble and push the Geth manifest image + for _, tag := range tags { + gethImage := fmt.Sprintf("%s:%s", *upload, tag) + + var gethSubImages []string + for _, arch := range strings.Split(*manifest, ",") { + gethSubImages = append(gethSubImages, gethImage+"-"+arch) + } + build.MustRunCommand("docker", append([]string{"manifest", "create", gethImage}, gethSubImages...)...) + build.MustRunCommand("docker", "manifest", "push", gethImage) + } + // Assemble and push the alltools manifest image + for _, tag := range tags { + toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag) + + var toolSubImages []string + for _, arch := range strings.Split(*manifest, ",") { + toolSubImages = append(toolSubImages, toolImage+"-"+arch) + } + build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...) + build.MustRunCommand("docker", "manifest", "push", toolImage) + } + } +} + // Debian Packaging func doDebianSource(cmdline []string) { var ( @@ -782,10 +974,10 @@ func doWindowsInstaller(cmdline []string) { build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil) if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil { - log.Fatal("Failed to copy SimpleFC.dll: %v", err) + log.Fatalf("Failed to copy SimpleFC.dll: %v", err) } if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil { - log.Fatal("Failed to copy copyright note: %v", err) + log.Fatalf("Failed to copy copyright note: %v", err) } // Build the installer. This assumes that all the needed files have been previously // built (don't mix building and packaging to keep cross compilation complexity to a @@ -1047,7 +1239,8 @@ func newPodMetadata(env build.Environment, archive string) podMetadata { func doXgo(cmdline []string) { var ( - alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`) + alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`) + staticlink = flag.Bool("static", false, "Static link build with") ) flag.CommandLine.Parse(cmdline) env := build.Env() @@ -1056,8 +1249,11 @@ func doXgo(cmdline []string) { // Make sure xgo is available for cross compilation build.MustRun(tc.Install(GOBIN, "github.com/karalabe/xgo@latest")) + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + // If all tools building is requested, build everything the builder wants - args := append(buildFlags(env), flag.Args()...) + args := append(buildFlags(env, *staticlink, buildTags), flag.Args()...) if *alltools { args = append(args, []string{"--dest", GOBIN}...)