Merge branch '7.12' into backport/7.12/pr-24455

.ci/packaging.groovy

@@ -429,7 +429,6 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
-    string(name: 'BEAT_VERSION', value: env.BEAT_VERSION),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),

.ci/scripts/install-go.sh

@@ -33,9 +33,9 @@ fi
 echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}"
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-${OS}-${GVM_ARCH_SUFFIX}"
 chmod +x "${GVM_CMD}"
 
-gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
+${GVM_CMD} "${GO_VERSION}" |cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
 
-eval $(gvm ${GO_VERSION})
+eval "$("${GVM_CMD}" "${GO_VERSION}")"

.ci/scripts/install-tools.bat

@@ -17,16 +17,18 @@ mkdir %WORKSPACE%\bin
 IF EXIST "%PROGRAMFILES(X86)%" (
     REM Force the gvm installation.
     SET GVM_BIN=gvm.exe
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.4/gvm-windows-amd64.exe
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
     IF ERRORLEVEL 1 (
         REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
         exit /b 1
     )
 ) ELSE (
     REM Windows 7 workers got a broken gvm installation.
-    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.4/gvm-windows-386.exe
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
     IF ERRORLEVEL 1 (
         REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
         exit /b 1
     )
 )
@@ -44,6 +46,7 @@ FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i
 go env
 IF ERRORLEVEL 1 (
     REM go is not configured correctly.
+    rmdir %WORKSPACE%\.gvm /s /q
     exit /b 1
 )
 

.go-version

@@ -1 +1 @@
-1.15.8
+1.15.9

CHANGELOG-developer.next.asciidoc

@@ -102,3 +102,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 
 - Update Go version to 1.15.7. {pull}22495[22495]
 - Update Go version to 1.15.8. {pull}23955[23955]
+- Update Go version to 1.15.9. {pull}24442[24442]

CHANGELOG.next.asciidoc

@@ -250,7 +250,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
 - Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904]
 - Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
+- Fix Netlow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110]
 - in httpjson input using encode_as "application/x-www-form-urlencoded" now sets Content-Type correctly {issue}24331[24331] {pull}24336[24336]
+- Fix default `scope` in `add_nomad_metadata`. {issue}24559[24559]
 
 *Heartbeat*
 
@@ -672,6 +674,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 
 - Upgrade to ECS 1.8.0. {pull}23783[23783]
+- Add `event.type: [connection]` to flow events and include `end` for final flows. {pull}24564[24564]
 
 *Functionbeat*
 

Jenkinsfile

@@ -157,7 +157,7 @@ VERSION=${env.VERSION}-SNAPSHOT""")
       dir("${BASE_DIR}"){
         notifyBuildResult(prComment: true,
                           slackComment: true, slackNotify: (isBranch() || isTag()),
-                          analyzeFlakey: !isTag(), flakyReportIdx: "reporter-beats-beats-${getIdSuffix()}")
+                          analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: getFlakyBranch()))
       }
     }
   }
@@ -166,11 +166,10 @@ VERSION=${env.VERSION}-SNAPSHOT""")
 /**
 * There are only two supported branches, master and 7.x
 */
-def getIdSuffix() {
+def getFlakyBranch() {
   if(isPR()) {
     return getBranchIndice(env.CHANGE_TARGET)
-  }
-  if(isBranch()) {
+  } else {
     return getBranchIndice(env.BRANCH_NAME)
   }
 }
@@ -665,7 +664,7 @@ def installTools(args) {
         git config --global user.name "beatsmachine"
       fi''')
   } else {
-    retryWithSleep(retries: 2, seconds: 5, backoff: true){ bat(label: "${stepHeader} - Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
+    retryWithSleep(retries: 3, seconds: 5, backoff: true){ bat(label: "${stepHeader} - Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
   }
 }
 
@@ -700,14 +699,19 @@ def archiveTestOutput(Map args = [:]) {
     if (isUnix()) {
       fixPermissions("${WORKSPACE}")
     }
-    cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
+    // Remove pycache directory and go vendors cache folders
+    if (isUnix()) {
+      dir('build') {
+        sh(label: 'Delete folders that are causing exceptions (See JENKINS-58421)', returnStatus: true,
+           script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;')
+      }
+    } else {
+      bat(label: 'Delete ve folder', returnStatus: true,
+          script: 'FOR /d /r . %%d IN ("ve") DO @IF EXIST "%%d" rmdir /s /q "%%d"')
+    }
+    cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py', returnStatus: true)
     dir('build') {
-      if (isUnix()) {
-        cmd(label: 'Delete folders that are causing exceptions (See JENKINS-58421)',
-            returnStatus: true,
-            script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;')
-      } else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') }
-        junit(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults)
+      junit(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults)
       if (args.upload) {
         tarAndUploadArtifacts(file: "test-build-artifacts-${args.id}.tgz", location: '.')
       }
@@ -732,10 +736,11 @@ def archiveTestOutput(Map args = [:]) {
 * disk space of the jenkins instance
 */
 def tarAndUploadArtifacts(Map args = [:]) {
-  tar(file: args.file, dir: args.location, archive: false, allowMissing: true)
+  def fileName = args.file.replaceAll('[^A-Za-z-0-9]','-')
+  tar(file: fileName, dir: args.location, archive: false, allowMissing: true)
   googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
                          credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
-                         pattern: "${args.file}",
+                         pattern: "${fileName}",
                          sharedPublicly: true)
 }
 

Vagrantfile

@@ -44,7 +44,7 @@ if (-Not (Test-Path $gopath_beats)) {
 if (-Not (Get-Command "gvm" -ErrorAction SilentlyContinue)) {
     echo "Installing gvm to manage go version"
     [Net.ServicePointManager]::SecurityProtocol = "tls12"
-    Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe
+    Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe
     C:\\Windows\\System32\\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression
     go version
 
@@ -119,7 +119,7 @@ def linuxGvmProvision(arch="amd64")
   return <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-linux-#{arch}
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-linux-#{arch}
   chmod +x ~/bin/gvm
   ~/bin/gvm #{GO_VERSION}
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

auditbeat/module/file_integrity/metricset_test.go

@@ -65,6 +65,8 @@ func TestData(t *testing.T) {
 }
 
 func TestActions(t *testing.T) {
+	skipOnCIForDarwinAMD64(t)
+
 	defer abtest.SetupDataDir(t)()
 
 	bucket, err := datastore.OpenBucket(bucketName)
@@ -173,6 +175,8 @@ func TestActions(t *testing.T) {
 }
 
 func TestExcludedFiles(t *testing.T) {
+	skipOnCIForDarwinAMD64(t)
+
 	defer abtest.SetupDataDir(t)()
 
 	bucket, err := datastore.OpenBucket(bucketName)
@@ -227,6 +231,8 @@ func TestExcludedFiles(t *testing.T) {
 }
 
 func TestIncludedExcludedFiles(t *testing.T) {
+	skipOnCIForDarwinAMD64(t)
+
 	defer abtest.SetupDataDir(t)()
 
 	bucket, err := datastore.OpenBucket(bucketName)
@@ -980,3 +986,9 @@ func getConfig(path ...string) map[string]interface{} {
 		"exclude_files": []string{`(?i)\.sw[nop]$`, `[/\\]\.git([/\\]|$)`},
 	}
 }
+
+func skipOnCIForDarwinAMD64(t testing.TB) {
+	if os.Getenv("BUILD_ID") != "" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+		t.Skip("Skip test on CI for darwin/amd64")
+	}
+}

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.8
+FROM golang:1.15.9
 
 RUN \
     apt-get update \

filebeat/docs/modules/netflow.asciidoc

@@ -72,6 +72,13 @@ details.
 monitor sequence numbers in the Netflow packets to detect an Exporting Process
 reset. See <<filebeat-input-netflow,netflow input>> for details.
 
+`var.internal_networks`:: A list of CIDR ranges describing the IP addresses that
+you consider internal. This is used in determining the values of
+`source.locality`, `destination.locality`, and `flow.locality`. The values
+can be either a CIDR value or one of the named ranges supported by the
+<<condition-network, `network`>> condition. The default value is `[private]`
+which classifies RFC 1918 (IPv4) and RFC 4193 (IPv6) addresses as internal.
+
 *`var.tags`*::
 
 A list of tags to include in events. Including `forwarded` indicates that the