Merge branch 'master' into 16029_zeek_ecs_1.4

.ci/packaging.groovy

@@ -8,9 +8,12 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/beats'
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    SNAPSHOT = "true"
   }
   options {
-    timeout(time: 2, unit: 'HOURS')
+    timeout(time: 3, unit: 'HOURS')
     buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
@@ -20,6 +23,7 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i)^\\/packaging$')
+    upstream('Beats/beats-beats-mbp/master')
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
@@ -31,66 +35,165 @@ pipeline {
       steps {
         deleteDir()
         gitCheckout(basedir: "${BASE_DIR}")
-        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
+        setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+        //stash allowEmpty: true, name: 'source', useDefaultExcludes: false
       }
     }
-    stage('Linux'){
-      agent { label 'ubuntu && immutable' }
-      options { skipDefaultCheckout() }
-      when {
-        beforeAgent true
-        expression { return params.linux }
-      }
-      environment {
-        SNAPSHOT = "true"
-        PLATFORMS = "+linux/armv7 +linux/ppc64le +linux/s390x +linux/mips64"
-        HOME = "${env.WORKSPACE}"
-      }
-      steps {
-        deleteDir()
-        unstash 'source'
-        dir("${BASE_DIR}"){
-          sh(label: 'Release', script: './dev-tools/jenkins_release.sh')
+    stage('Build Packages'){
+      matrix {
+        axes {
+          axis {
+            name 'BEATS_FOLDER'
+            values (
+              'auditbeat',
+              'filebeat',
+              'heartbeat',
+              'journalbeat',
+              'metricbeat',
+              'packetbeat',
+              'winlogbeat',
+              'x-pack/auditbeat',
+              'x-pack/elastic-agent',
+              'x-pack/dockerlogbeat',
+              'x-pack/filebeat',
+              'x-pack/functionbeat',
+              // 'x-pack/heartbeat',
+              // 'x-pack/journalbeat',
+              'x-pack/metricbeat',
+              // 'x-pack/packetbeat',
+              'x-pack/winlogbeat'
+            )
+          }
         }
-        publishPackages()
-      }
-    }
-    stage('Mac OS'){
-      agent { label 'macosx' }
-      options { skipDefaultCheckout() }
-      when {
-        beforeAgent true
-        expression { return params.linux }
-      }
-      environment {
-        KEYCHAIN = "/var/lib/jenkins/Library/Keychains/Elastic.keychain-db"
-        SNAPSHOT = "true"
-        PLATFORMS = "darwin"
-        APPLE_SIGNING_ENABLED = "true"
-        HOME = "${env.WORKSPACE}"
-      }
-      steps {
-        deleteDir()
-        unstash 'source'
-        withMaskEnv( vars: [
-            [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],
-        ]){
-          dir("${BASE_DIR}"){
-            sh(label: 'Release', script: './dev-tools/jenkins_release.sh')
+        stages {
+          stage('Package Linux'){
+            agent { label 'ubuntu && immutable' }
+            options { skipDefaultCheckout() }
+            when {
+              beforeAgent true
+              expression {
+                return params.linux
+              }
+            }
+            environment {
+              HOME = "${env.WORKSPACE}"
+              PLATFORMS = [
+                '+all',
+                'linux/amd64',
+                'linux/386',
+                'linux/arm64',
+                'linux/armv7',
+                'linux/ppc64le',
+                'linux/mips64',
+                'linux/s390x',
+                'windows/amd64',
+                'windows/386',
+                (params.macos ? '' : 'darwin/amd64'),
+              ].join(' ')
+            }
+            steps {
+              release()
+              pushCIDockerImages()
+            }
+          }
+          stage('Package Mac OS'){
+            agent { label 'macosx' }
+            options { skipDefaultCheckout() }
+            when {
+              beforeAgent true
+              expression {
+                return params.macos
+              }
+            }
+            environment {
+              HOME = "${env.WORKSPACE}"
+              PLATFORMS = [
+                '+all',
+                'darwin/amd64',
+              ].join(' ')
+            }
+            steps {
+              withMacOSEnv(){
+                release()
+              }
+            }
           }
-          publishPackages()
         }
       }
     }
   }
 }
 
-def publishPackages(){
+def pushCIDockerImages(){
+  sh(label: 'Push Docker image', script: '''
+    if [ -n "$(command -v docker)" ]; then
+      docker images || true
+    fi
+  ''')
+}
+
+def release(){
+  withBeatsEnv(){
+    dir("${env.BEATS_FOLDER}") {
+      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    }
+    publishPackages("${env.BEATS_FOLDER}")
+  }
+}
+
+def withMacOSEnv(Closure body){
+  withEnvMask( vars: [
+      [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],
+      [var: "KEYCHAIN", password: "/var/lib/jenkins/Library/Keychains/Elastic.keychain-db"],
+      [var: "APPLE_SIGNING_ENABLED", password: "true"],
+  ]){
+    body()
+  }
+}
+
+def publishPackages(baseDir){
   googleStorageUpload(bucket: "gs://${JOB_GCS_BUCKET}/snapshots",
     credentialsId: "${JOB_GCS_CREDENTIALS}",
-    pathPrefix: "${BASE_DIR}/build/distributions/",
-    pattern: "${BASE_DIR}/build/distributions/**/*",
+    pathPrefix: "${baseDir}/build/distributions/",
+    pattern: "${baseDir}/build/distributions/**/*",
     sharedPublicly: true,
     showInline: true
   )
 }
+
+def withBeatsEnv(Closure body) {
+  def os = goos()
+  def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64"
+
+  withEnv([
+    "HOME=${env.WORKSPACE}",
+    "GOPATH=${env.WORKSPACE}",
+    "GOROOT=${goRoot}",
+    "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}",
+    "MAGEFILE_CACHE=${WORKSPACE}/.magefile",
+    "PYTHON_ENV=${WORKSPACE}/python-env"
+  ]) {
+    deleteDir()
+    //unstash 'source'
+    gitCheckout(basedir: "${BASE_DIR}")
+    dir("${env.BASE_DIR}"){
+      sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh")
+      sh(label: "Install Mage", script: "make mage")
+      body()
+    }
+  }
+}
+
+def goos(){
+  def labels = env.NODE_LABELS
+
+  if (labels.contains('linux')) {
+    return 'linux'
+  } else if (labels.contains('windows')) {
+    return 'windows'
+  } else if (labels.contains('darwin')) {
+    return 'darwin'
+  }
+
+  error("Unhandled OS name in NODE_LABELS: " + labels)
+}

.ci/packer_cache.sh

@@ -7,7 +7,7 @@
 source /usr/local/bin/bash_standard_lib.sh
 
 # shellcheck disable=SC1091
-source ./script/common.bash
+source ./dev-tools/common.bash
 
 # Docker images used on Dockerfiles 2019-07-12
 # aerospike:3.9.0

.github/comments.yml

@@ -0,0 +1,5 @@
+---
+comments:
+  - label: needs_team
+    comment: |
+      This issue doesn't have a `Team:<team>` label.

.github/labels.yml

@@ -0,0 +1,5 @@
+---
+pulls:
+  opened:
+    add:
+      - needs_team

.github/relabel.yml

@@ -0,0 +1,4 @@
+---
+requiredLabels:
+  - missingLabel: needs_team
+    regex: Team:.+

.travis.yml

@@ -111,23 +111,22 @@ jobs:
     # Metricbeat
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="-C metricbeat unit-tests coverage-report"
+      env: TARGETS="-C metricbeat unit-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat integration-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat integration-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat update system-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat system-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
-
     - os: osx
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="TEST_ENVIRONMENT=0 -C metricbeat testsuite"
+      env: TARGETS="-C metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
@@ -137,6 +136,21 @@ jobs:
       stage: test
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat unit-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat integration-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat system-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
       env: TARGETS="-C x-pack/metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test

CHANGELOG-developer.next.asciidoc

@@ -39,6 +39,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - The newly generated Beats are using go modules to manage dependencies. {pull}16288[16288]
 - Extract Elasticsearch client logic from `outputs/elasticsearch` package into new `esclientleg` package. {pull}16150[16150]
 - Rename `queue.BufferConfig.Events` to `queue.BufferConfig.MaxEvents`. {pull}17622[17622]
+- Remove `queue.Feature` and replace `queue.RegisterType` with `queue.RegisterQueueType`. {pull}17666[17666]
 
 ==== Bugfixes
 
@@ -78,3 +79,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - New mage target for Functionbeat: generate pkg folder to make manager easier. {pull}15580[15880]
 - Add support for MODULE environment variable in `mage goIntegTest` in metricbeat to run integration tests for a single module. {pull}17147[17147]
 - Add support for a `TEST_TAGS` environment variable to add tags for tests selection following go build tags semantics, this environment variable is used by mage test targets to add build tags. Python tests can also be tagged with a decorator (`@beat.tag('sometag')`). {pull}16937[16937] {pull}17075[17075]
+- Add fields validation for histogram subfields. {pull}17759[17759]

CHANGELOG.asciidoc

@@ -30,6 +30,7 @@ https://github.com/elastic/beats/compare/v7.5.0...v7.5.1[View commits]
 - Fix docker network stats when multiple interfaces are configured. {issue}14586[14586] {pull}14825[14825]
 - Fix ListMetrics pagination in aws module. {issue}14926[14926] {pull}14942[14942]
 - Fix CPU count in docker/cpu in cases where no `online_cpus` are reported {pull}15070[15070]
+- Add domain state to kvm module {pull}17673[17673]
 
 [[release-notes-7.5.0]]
 === Beats version 7.5.0