diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index fbbfde9a764b..6c022a0f3573 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -429,7 +429,6 @@ def triggerE2ETests(String suite) { booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()), booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true), string(name: 'runTestsSuites', value: suite), - string(name: 'BEAT_VERSION', value: env.BEAT_VERSION), string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME), string(name: 'GITHUB_CHECK_REPO', value: env.REPO), string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT), diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh index 6e4afa0fb412..23832e21cb40 100755 --- a/.ci/scripts/install-go.sh +++ b/.ci/scripts/install-go.sh @@ -33,9 +33,9 @@ fi echo "UNMET DEP: Installing Go" mkdir -p "${HOME}/bin" -curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}" +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-${OS}-${GVM_ARCH_SUFFIX}" chmod +x "${GVM_CMD}" -gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} +${GVM_CMD} "${GO_VERSION}" |cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} -eval $(gvm ${GO_VERSION}) +eval "$("${GVM_CMD}" "${GO_VERSION}")" diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat index 1d29b5483187..5c826336dcb1 100755 --- a/.ci/scripts/install-tools.bat +++ b/.ci/scripts/install-tools.bat @@ -17,16 +17,18 @@ mkdir %WORKSPACE%\bin IF EXIST "%PROGRAMFILES(X86)%" ( REM Force the gvm installation. SET GVM_BIN=gvm.exe - curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.4/gvm-windows-amd64.exe + curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe IF ERRORLEVEL 1 ( REM gvm installation has failed. + del bin\gvm.exe /s /f /q exit /b 1 ) ) ELSE ( REM Windows 7 workers got a broken gvm installation. - curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.4/gvm-windows-386.exe + curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe IF ERRORLEVEL 1 ( REM gvm installation has failed. + del bin\gvm.exe /s /f /q exit /b 1 ) ) @@ -44,6 +46,7 @@ FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i go env IF ERRORLEVEL 1 ( REM go is not configured correctly. + rmdir %WORKSPACE%\.gvm /s /q exit /b 1 ) diff --git a/.go-version b/.go-version index 98e863cdf81f..30a88ea45a4f 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.15.8 +1.15.9 diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 920125d9d56d..55c44b49f4b0 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -102,3 +102,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Update Go version to 1.15.7. {pull}22495[22495] - Update Go version to 1.15.8. {pull}23955[23955] +- Update Go version to 1.15.9. {pull}24442[24442] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8723acf3ec83..eda9e956a018 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -250,7 +250,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920] - Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904] - Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167] +- Fix Netlow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110] - in httpjson input using encode_as "application/x-www-form-urlencoded" now sets Content-Type correctly {issue}24331[24331] {pull}24336[24336] +- Fix default `scope` in `add_nomad_metadata`. {issue}24559[24559] *Heartbeat* @@ -672,6 +674,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Upgrade to ECS 1.8.0. {pull}23783[23783] +- Add `event.type: [connection]` to flow events and include `end` for final flows. {pull}24564[24564] *Functionbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 983c7e48ce4a..0ee1bed19181 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -157,7 +157,7 @@ VERSION=${env.VERSION}-SNAPSHOT""") dir("${BASE_DIR}"){ notifyBuildResult(prComment: true, slackComment: true, slackNotify: (isBranch() || isTag()), - analyzeFlakey: !isTag(), flakyReportIdx: "reporter-beats-beats-${getIdSuffix()}") + analyzeFlakey: !isTag(), jobName: getFlakyJobName(withBranch: getFlakyBranch())) } } } @@ -166,11 +166,10 @@ VERSION=${env.VERSION}-SNAPSHOT""") /** * There are only two supported branches, master and 7.x */ -def getIdSuffix() { +def getFlakyBranch() { if(isPR()) { return getBranchIndice(env.CHANGE_TARGET) - } - if(isBranch()) { + } else { return getBranchIndice(env.BRANCH_NAME) } } @@ -665,7 +664,7 @@ def installTools(args) { git config --global user.name "beatsmachine" fi''') } else { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ bat(label: "${stepHeader} - Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + retryWithSleep(retries: 3, seconds: 5, backoff: true){ bat(label: "${stepHeader} - Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } } } @@ -700,14 +699,19 @@ def archiveTestOutput(Map args = [:]) { if (isUnix()) { fixPermissions("${WORKSPACE}") } - cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') + // Remove pycache directory and go vendors cache folders + if (isUnix()) { + dir('build') { + sh(label: 'Delete folders that are causing exceptions (See JENKINS-58421)', returnStatus: true, + script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;') + } + } else { + bat(label: 'Delete ve folder', returnStatus: true, + script: 'FOR /d /r . %%d IN ("ve") DO @IF EXIST "%%d" rmdir /s /q "%%d"') + } + cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py', returnStatus: true) dir('build') { - if (isUnix()) { - cmd(label: 'Delete folders that are causing exceptions (See JENKINS-58421)', - returnStatus: true, - script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;') - } else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') } - junit(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) + junit(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) if (args.upload) { tarAndUploadArtifacts(file: "test-build-artifacts-${args.id}.tgz", location: '.') } @@ -732,10 +736,11 @@ def archiveTestOutput(Map args = [:]) { * disk space of the jenkins instance */ def tarAndUploadArtifacts(Map args = [:]) { - tar(file: args.file, dir: args.location, archive: false, allowMissing: true) + def fileName = args.file.replaceAll('[^A-Za-z-0-9]','-') + tar(file: fileName, dir: args.location, archive: false, allowMissing: true) googleStorageUploadExt(bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}", credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", - pattern: "${args.file}", + pattern: "${fileName}", sharedPublicly: true) } diff --git a/Vagrantfile b/Vagrantfile index 81560b734341..be15c92214d8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -44,7 +44,7 @@ if (-Not (Test-Path $gopath_beats)) { if (-Not (Get-Command "gvm" -ErrorAction SilentlyContinue)) { echo "Installing gvm to manage go version" [Net.ServicePointManager]::SecurityProtocol = "tls12" - Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe + Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe C:\\Windows\\System32\\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression go version @@ -119,7 +119,7 @@ def linuxGvmProvision(arch="amd64") return <