From a920de64d6dce0561e10791ba2170059e4d86011 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 23 Sep 2020 11:47:55 +0200 Subject: [PATCH 1/5] Remove librpm installation from auditbeat build It fails for some platforms, and the requested packages are already provided by the base images. --- x-pack/auditbeat/magefile.go | 89 ------------------------------------ 1 file changed, 89 deletions(-) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index f484cbb371d..6424bc73745 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -11,8 +11,6 @@ import ( "time" "github.com/magefile/mage/mg" - "github.com/magefile/mage/sh" - "github.com/pkg/errors" auditbeat "github.com/elastic/beats/v7/auditbeat/scripts/mage" devtools "github.com/elastic/beats/v7/dev-tools/mage" @@ -44,9 +42,6 @@ func Build() error { // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { - if d, ok := deps[devtools.Platform.Name]; ok { - mg.Deps(d) - } return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs()) } @@ -124,87 +119,3 @@ func ExportDashboard() error { func Dashboards() error { return devtools.KibanaDashboards(devtools.OSSBeatDir("module"), "module") } - -// ----------------------------------------------------------------------------- -// - Install the librpm-dev package -var ( - deps = map[string]func() error{ - "linux/386": installLinux386, - "linux/amd64": installLinuxAMD64, - "linux/arm64": installLinuxARM64, - "linux/armv5": installLinuxARMLE, - "linux/armv6": installLinuxARMLE, - "linux/armv7": installLinuxARMHF, - "linux/mips": installLinuxMIPS, - "linux/mipsle": installLinuxMIPSLE, - "linux/mips64le": installLinuxMIPS64LE, - "linux/ppc64le": installLinuxPPC64LE, - "linux/s390x": installLinuxS390X, - - //"linux/ppc64": installLinuxPpc64, - //"linux/mips64": installLinuxMips64, - } -) - -const ( - librpmDevPkgName = "librpm-dev" -) - -func installLinuxAMD64() error { - return installDependencies(librpmDevPkgName, "") -} - -func installLinuxARM64() error { - return installDependencies(librpmDevPkgName+":arm64", "arm64") -} - -func installLinuxARMHF() error { - return installDependencies(librpmDevPkgName+":armhf", "armhf") -} - -func installLinuxARMLE() error { - return installDependencies(librpmDevPkgName+":armel", "armel") -} - -func installLinux386() error { - return installDependencies(librpmDevPkgName+":i386", "i386") -} - -func installLinuxMIPS() error { - return installDependencies(librpmDevPkgName+":mips", "mips") -} - -func installLinuxMIPS64LE() error { - return installDependencies(librpmDevPkgName+":mips64el", "mips64el") -} - -func installLinuxMIPSLE() error { - return installDependencies(librpmDevPkgName+":mipsel", "mipsel") -} - -func installLinuxPPC64LE() error { - return installDependencies(librpmDevPkgName+":ppc64el", "ppc64el") -} - -func installLinuxS390X() error { - return installDependencies(librpmDevPkgName+":s390x", "s390x") -} - -func installDependencies(pkg, arch string) error { - if arch != "" { - err := sh.Run("dpkg", "--add-architecture", arch) - if err != nil { - return errors.Wrap(err, "error while adding architecture") - } - } - - // TODO: This is only for debian 7 and should be removed when move to a newer OS. This flag is - // going to be used unnecessary when building using non-debian7 images - // (like when making the linux/arm binaries) and we should remove it soonish. - // See https://github.com/elastic/beats/v7/issues/11750 for more details. - if err := sh.Run("apt-get", "update", "-o", "Acquire::Check-Valid-Until=false"); err != nil { - return err - } - - return sh.Run("apt-get", "install", "-y", "--no-install-recommends", pkg) -} From 359c0a9412988b6d4d21051ec91e220c701de7aa Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 23 Sep 2020 13:27:25 +0200 Subject: [PATCH 2/5] Revert "Remove librpm installation from auditbeat build" This reverts commit a920de64d6dce0561e10791ba2170059e4d86011. --- x-pack/auditbeat/magefile.go | 89 ++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index 6424bc73745..f484cbb371d 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -11,6 +11,8 @@ import ( "time" "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" auditbeat "github.com/elastic/beats/v7/auditbeat/scripts/mage" devtools "github.com/elastic/beats/v7/dev-tools/mage" @@ -42,6 +44,9 @@ func Build() error { // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { + if d, ok := deps[devtools.Platform.Name]; ok { + mg.Deps(d) + } return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs()) } @@ -119,3 +124,87 @@ func ExportDashboard() error { func Dashboards() error { return devtools.KibanaDashboards(devtools.OSSBeatDir("module"), "module") } + +// ----------------------------------------------------------------------------- +// - Install the librpm-dev package +var ( + deps = map[string]func() error{ + "linux/386": installLinux386, + "linux/amd64": installLinuxAMD64, + "linux/arm64": installLinuxARM64, + "linux/armv5": installLinuxARMLE, + "linux/armv6": installLinuxARMLE, + "linux/armv7": installLinuxARMHF, + "linux/mips": installLinuxMIPS, + "linux/mipsle": installLinuxMIPSLE, + "linux/mips64le": installLinuxMIPS64LE, + "linux/ppc64le": installLinuxPPC64LE, + "linux/s390x": installLinuxS390X, + + //"linux/ppc64": installLinuxPpc64, + //"linux/mips64": installLinuxMips64, + } +) + +const ( + librpmDevPkgName = "librpm-dev" +) + +func installLinuxAMD64() error { + return installDependencies(librpmDevPkgName, "") +} + +func installLinuxARM64() error { + return installDependencies(librpmDevPkgName+":arm64", "arm64") +} + +func installLinuxARMHF() error { + return installDependencies(librpmDevPkgName+":armhf", "armhf") +} + +func installLinuxARMLE() error { + return installDependencies(librpmDevPkgName+":armel", "armel") +} + +func installLinux386() error { + return installDependencies(librpmDevPkgName+":i386", "i386") +} + +func installLinuxMIPS() error { + return installDependencies(librpmDevPkgName+":mips", "mips") +} + +func installLinuxMIPS64LE() error { + return installDependencies(librpmDevPkgName+":mips64el", "mips64el") +} + +func installLinuxMIPSLE() error { + return installDependencies(librpmDevPkgName+":mipsel", "mipsel") +} + +func installLinuxPPC64LE() error { + return installDependencies(librpmDevPkgName+":ppc64el", "ppc64el") +} + +func installLinuxS390X() error { + return installDependencies(librpmDevPkgName+":s390x", "s390x") +} + +func installDependencies(pkg, arch string) error { + if arch != "" { + err := sh.Run("dpkg", "--add-architecture", arch) + if err != nil { + return errors.Wrap(err, "error while adding architecture") + } + } + + // TODO: This is only for debian 7 and should be removed when move to a newer OS. This flag is + // going to be used unnecessary when building using non-debian7 images + // (like when making the linux/arm binaries) and we should remove it soonish. + // See https://github.com/elastic/beats/v7/issues/11750 for more details. + if err := sh.Run("apt-get", "update", "-o", "Acquire::Check-Valid-Until=false"); err != nil { + return err + } + + return sh.Run("apt-get", "install", "-y", "--no-install-recommends", pkg) +} From b2065313c3a6249fecbcdb561d1cc8c08fd3c220 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 23 Sep 2020 13:33:02 +0200 Subject: [PATCH 3/5] Remove installation of librpm-dev only for armv7 --- x-pack/auditbeat/magefile.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index f484cbb371d..91bf26309d5 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -134,13 +134,14 @@ var ( "linux/arm64": installLinuxARM64, "linux/armv5": installLinuxARMLE, "linux/armv6": installLinuxARMLE, - "linux/armv7": installLinuxARMHF, "linux/mips": installLinuxMIPS, "linux/mipsle": installLinuxMIPSLE, "linux/mips64le": installLinuxMIPS64LE, "linux/ppc64le": installLinuxPPC64LE, "linux/s390x": installLinuxS390X, + //"linux/armv7": installLinuxARMHF, + //"linux/ppc64": installLinuxPpc64, //"linux/mips64": installLinuxMips64, } From 3d6886430e5b9bb8c8d14b5b4ad75bb3a871c9e0 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 23 Sep 2020 14:02:42 +0200 Subject: [PATCH 4/5] Force installation of some librpm-dev dependencies in arm architectures --- x-pack/auditbeat/magefile.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index 91bf26309d5..b0cbe4deb16 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -132,16 +132,15 @@ var ( "linux/386": installLinux386, "linux/amd64": installLinuxAMD64, "linux/arm64": installLinuxARM64, - "linux/armv5": installLinuxARMLE, - "linux/armv6": installLinuxARMLE, + "linux/armv5": installLinuxARMEL, + "linux/armv6": installLinuxARMEL, + "linux/armv7": installLinuxARMHF, "linux/mips": installLinuxMIPS, "linux/mipsle": installLinuxMIPSLE, "linux/mips64le": installLinuxMIPS64LE, "linux/ppc64le": installLinuxPPC64LE, "linux/s390x": installLinuxS390X, - //"linux/armv7": installLinuxARMHF, - //"linux/ppc64": installLinuxPpc64, //"linux/mips64": installLinuxMips64, } @@ -149,26 +148,30 @@ var ( const ( librpmDevPkgName = "librpm-dev" + + // Dependency of librpm-dev in ARM architectures, that needs to be explicitly + // installed to replace other conflicting packages pre-installed in the image. + libicuDevPkgName = "libicu-dev" ) func installLinuxAMD64() error { - return installDependencies(librpmDevPkgName, "") + return installDependencies("", librpmDevPkgName) } func installLinuxARM64() error { - return installDependencies(librpmDevPkgName+":arm64", "arm64") + return installDependencies("arm64", librpmDevPkgName+":arm64") } func installLinuxARMHF() error { - return installDependencies(librpmDevPkgName+":armhf", "armhf") + return installDependencies("armhf", librpmDevPkgName+":armhf", libicuDevPkgName+":armhf") } -func installLinuxARMLE() error { - return installDependencies(librpmDevPkgName+":armel", "armel") +func installLinuxARMEL() error { + return installDependencies("armel", librpmDevPkgName+":armel", libicuDevPkgName+":armel") } func installLinux386() error { - return installDependencies(librpmDevPkgName+":i386", "i386") + return installDependencies("i386", librpmDevPkgName+":i386") } func installLinuxMIPS() error { @@ -191,7 +194,10 @@ func installLinuxS390X() error { return installDependencies(librpmDevPkgName+":s390x", "s390x") } -func installDependencies(pkg, arch string) error { +func installDependencies(arch string, pkgs ...string) error { + if len(pkgs) == 0 { + return nil + } if arch != "" { err := sh.Run("dpkg", "--add-architecture", arch) if err != nil { @@ -207,5 +213,6 @@ func installDependencies(pkg, arch string) error { return err } - return sh.Run("apt-get", "install", "-y", "--no-install-recommends", pkg) + args := append([]string{"install", "-y", "--no-install-recommends"}, pkgs...) + return sh.Run("apt-get", args...) } From f04548d8ec2d37fe52627f46e75a4d3ffb5f20f3 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 23 Sep 2020 14:17:37 +0200 Subject: [PATCH 5/5] Typos --- x-pack/auditbeat/magefile.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/auditbeat/magefile.go b/x-pack/auditbeat/magefile.go index b0cbe4deb16..989f8e6d7b6 100644 --- a/x-pack/auditbeat/magefile.go +++ b/x-pack/auditbeat/magefile.go @@ -136,9 +136,9 @@ var ( "linux/armv6": installLinuxARMEL, "linux/armv7": installLinuxARMHF, "linux/mips": installLinuxMIPS, - "linux/mipsle": installLinuxMIPSLE, - "linux/mips64le": installLinuxMIPS64LE, - "linux/ppc64le": installLinuxPPC64LE, + "linux/mipsle": installLinuxMIPSEL, + "linux/mips64le": installLinuxMIPS64EL, + "linux/ppc64le": installLinuxPPC64EL, "linux/s390x": installLinuxS390X, //"linux/ppc64": installLinuxPpc64, @@ -175,23 +175,23 @@ func installLinux386() error { } func installLinuxMIPS() error { - return installDependencies(librpmDevPkgName+":mips", "mips") + return installDependencies("mips", librpmDevPkgName+":mips") } -func installLinuxMIPS64LE() error { - return installDependencies(librpmDevPkgName+":mips64el", "mips64el") +func installLinuxMIPS64EL() error { + return installDependencies("mips64el", librpmDevPkgName+":mips64el") } -func installLinuxMIPSLE() error { - return installDependencies(librpmDevPkgName+":mipsel", "mipsel") +func installLinuxMIPSEL() error { + return installDependencies("mispel", librpmDevPkgName+":mipsel") } -func installLinuxPPC64LE() error { - return installDependencies(librpmDevPkgName+":ppc64el", "ppc64el") +func installLinuxPPC64EL() error { + return installDependencies("ppc64el", librpmDevPkgName+":ppc64el") } func installLinuxS390X() error { - return installDependencies(librpmDevPkgName+":s390x", "s390x") + return installDependencies("s390x", librpmDevPkgName+":s390x") } func installDependencies(arch string, pkgs ...string) error {