Merge remote-tracking branch 'origin/master' into postgresql-7-12

elastic · Mar 16, 2021 · 3eaee22 · 3eaee22
2 parents 1b61205 + f34360e
commit 3eaee22
Show file tree

Hide file tree

Showing 155 changed files with 11,377 additions and 319 deletions.
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -13,7 +13,8 @@ pipeline {
     HOME = "${env.WORKSPACE}"
     KIND_VERSION = "v0.10.0"
     K8S_VERSION = "v1.20.2"
-
+    JOB_GCS_BUCKET = 'beats-ci-temp'
+    JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     ELASTIC_STACK_VERSION_PREV = "7.12.0-SNAPSHOT"
     ELASTIC_STACK_VERSION_PREV_PREV = "7.11.2-SNAPSHOT"
   }
@@ -35,6 +36,7 @@ pipeline {
       steps {
         deleteDir()
         gitCheckout(basedir: "${BASE_DIR}")
+        stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
       }
     }
     stage('Check Go sources') {
@@ -60,7 +62,7 @@ pipeline {
                   withNode(labels: 'ubuntu-20 && immutable', sleepMin: 10, sleepMax: 100) {
                     stage("${it}: check") {
                       deleteDir()
-                      gitCheckout(basedir: "${BASE_DIR}")
+                      unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
                       useElasticPackage()
                       try {
                         dir("${BASE_DIR}/packages/${it}") {

diff --git a/.ci/jobs/defaults.yml b/.ci/jobs/defaults.yml
@@ -16,5 +16,8 @@
     publishers:
       - email:
           recipients: infra-root+build@elastic.co
-    periodic-folder-trigger: 1w
+    # Webhook based rather than polling otherwise the GitHub API quota
+    # will be overkilled. For such, periodic-folder-trigger is not needed
+    # anymore, so we keep the comment below for clarity.
+    # periodic-folder-trigger: 1w
     prune-dead-branches: true
diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/elastic/elastic-package v0.0.0-20210310173719-3b8f76516ae3
+	github.com/elastic/elastic-package v0.0.0-20210315163716-d74415f39b4e
 	github.com/elastic/package-registry v0.17.0
 	github.com/magefile/mage v1.11.0
 	github.com/pkg/errors v0.9.1

diff --git a/go.sum b/go.sum
@@ -84,8 +84,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/elastic/elastic-package v0.0.0-20210310173719-3b8f76516ae3 h1:qgySP5yNiGirwIoYkSEM5aoXzysFL2Dt2HHu8B9q5RA=
-github.com/elastic/elastic-package v0.0.0-20210310173719-3b8f76516ae3/go.mod h1:zlwA6eqY17P630VjzQVkJAdSpXkIgdA3+xhcpcS/qis=
+github.com/elastic/elastic-package v0.0.0-20210315163716-d74415f39b4e h1:HFoLz6j2FZewc8lVSglpHPKP08Hjprix+1gNaF33/iY=
+github.com/elastic/elastic-package v0.0.0-20210315163716-d74415f39b4e/go.mod h1:zlwA6eqY17P630VjzQVkJAdSpXkIgdA3+xhcpcS/qis=
 github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M2y/YtBU2NC2M=
 github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=

diff --git a/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/access/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,78 @@
+config_version: "2"
+interval: {{interval}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: |-
+        {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+response.decode_as: application/x-ndjson
+response.split:
+  target: body.result._raw
+  type: string
+  delimiter: "\n"
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: message
+      target: json
+      add_error_key: true
+  - drop_event:
+      when:
+        not:
+          has_fields: ['json.result']
+  - fingerprint:
+      fields:
+        - json.result._cd
+        - json.result._indextime
+        - json.result._raw
+        - json.result._time
+        - json.result.host
+        - json.result.source
+      target_field: "@metadata._id"
+  - drop_fields:
+      fields: message
+  - rename:
+      fields:
+        - from: json.result._raw
+          to: message
+        - from: json.result.host
+          to: host.name
+        - from: json.result.source
+          to: file.path
+      ignore_missing: true
+      fail_on_error: false
+  - drop_fields:
+      fields: json
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.7.0
diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -40,6 +40,10 @@ processors:
     ignore_missing: true
     patterns:
     - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
+- remove:
+    field: event.created
+    ignore_missing: true
+    ignore_failure: true
 - rename:
     field: '@timestamp'
     target_field: event.created

diff --git a/packages/apache/data_stream/access/manifest.yml b/packages/apache/data_stream/access/manifest.yml
@@ -17,3 +17,29 @@ streams:
     template_path: log.yml.hbs
     title: Apache access logs
     description: Collect Apache access logs
+  - input: httpjson
+    title: Apache access logs via Splunk Enterprise REST API
+    description: Collect apache access logs via Splunk Enterprise REST API
+    enabled: false
+    template_path: httpjson.yml.hbs
+    vars:
+      - name: interval
+        type: text
+        title: Interval to query Splunk Enterprise REST API
+        description: Go Duration syntax (eg. 10s)
+        show_user: true
+        required: true
+        default: 10s
+      - name: search
+        type: text
+        title: Splunk search string
+        show_user: true
+        required: true
+        default: "search sourcetype=\"access*\""
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded
diff --git a/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs b/packages/apache/data_stream/error/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,78 @@
+config_version: 2
+interval: {{interval}}
+auth.basic.user: {{username}}
+auth.basic.password: {{password}}
+cursor:
+  index_earliest:
+    value: '[[.last_event.result.max_indextime]]'
+request.url: {{url}}/services/search/jobs/export
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.transforms:
+  - set:
+      target: url.params.search
+      value: |-
+        {{search}} | streamstats max(_indextime) AS max_indextime
+  - set:
+      target: url.params.output_mode
+      value: "json"
+  - set:
+      target: url.params.index_earliest
+      value: '[[ .cursor.index_earliest ]]'
+      default: '[[(now (parseDuration "-{{interval}}")).Unix]]'
+  - set:
+      target: url.params.index_latest
+      value: '[[(now).Unix]]'
+  - set:
+      target: header.Content-Type
+      value: application/x-www-form-urlencoded
+response.decode_as: application/x-ndjson
+response.split:
+  target: body.result._raw
+  type: string
+  delimiter: "\n"
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: message
+      target: json
+      add_error_key: true
+  - drop_event:
+      when:
+        not:
+          has_fields: ['json.result']
+  - fingerprint:
+      fields:
+        - json.result._cd
+        - json.result._indextime
+        - json.result._raw
+        - json.result._time
+        - json.result.host
+        - json.result.source
+      target_field: "@metadata._id"
+  - drop_fields:
+      fields: message
+  - rename:
+      fields:
+        - from: json.result._raw
+          to: message
+        - from: json.result.host
+          to: host.name
+        - from: json.result.source
+          to: file.path
+      ignore_missing: true
+      fail_on_error: false
+  - drop_fields:
+      fields: json
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.7.0
diff --git a/packages/apache/data_stream/error/manifest.yml b/packages/apache/data_stream/error/manifest.yml
@@ -16,3 +16,29 @@ streams:
     template_path: log.yml.hbs
     title: Apache error logs
     description: Collect Apache error logs
+  - input: httpjson
+    title: Apache error logs via Splunk Enterprise REST API
+    description: Collect apache error logs via Splunk Enterprise REST API
+    enabled: false
+    template_path: httpjson.yml.hbs
+    vars:
+      - name: interval
+        type: text
+        title: Interval to query Splunk Enterprise REST API
+        description: Go Duration syntax (eg. 10s)
+        show_user: true
+        required: true
+        default: 10s
+      - name: search
+        type: text
+        title: Splunk search string
+        show_user: true
+        required: true
+        default: search sourcetype=apache:error OR sourcetype=apache_error
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded
diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml
@@ -1,15 +1,15 @@
 format_version: 1.0.0
 name: apache
 title: Apache
-version: 0.3.4
+version: 0.4.0
 license: basic
 description: Apache Integration
 type: integration
 categories:
   - web
 release: experimental
 conditions:
-  kibana.version: '^7.11.0'
+  kibana.version: '^7.12.0'
 screenshots:
   - src: /img/apache-metrics-overview.png
     title: Apache metrics overview
@@ -32,6 +32,34 @@ policy_templates:
       - type: logfile
         title: Collect logs from Apache instances
         description: Collecting Apache access and error logs
+      - type: httpjson
+        title: Collect logs from third-party REST API (experimental)
+        description: Collect logs from third-party REST API (experimental)
+        vars:
+          - name: url
+            type: text
+            title: URL of Splunk Enterprise Server
+            description: i.e. scheme://host:port,  path is automatic
+            show_user: true
+            required: true
+            default: https://server.example.com:8089
+          - name: username
+            type: text
+            title: Splunk REST API Username
+            show_user: true
+            required: true
+          - name: password
+            type: password
+            title: Splunk REST API Password
+            required: true
+            show_user: true
+          - name: ssl
+            type: yaml
+            title: SSL Configuration
+            description: i.e. certificate_authorities, supported_protocols, verification_mode etc.
+            multi: false
+            required: false
+            show_user: false
       - type: apache/metrics
         title: Collect metrics from Apache instances
         description: Collecting Apache status metrics

diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.1"
+  changes:
+    - description: Ignore missing "json" field in ingest pipeline
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/791
 - version: "0.4.2"
   changes:
     - description: Updating package owner

diff --git a/...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json b/...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json
@@ -20,7 +20,7 @@
                 "ip": "127.0.0.1"
             },
             "event": {
-                "ingested": "2021-02-17T15:22:20.292967100Z",
+                "ingested": "2021-03-16T10:02:45.373411100Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-08T20:53:12Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"userName\":\"Bob\",\"newUserName\":\"Robert\"},\"responseElements\":null,\"requestID\":\"3a6b3260-739d-465e-9406-bcEXAMPLE\",\"eventID\":\"9150d546-3564-4262-8e62-110EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-08T20:53:12.000Z",
@@ -77,11 +77,8 @@
         {
             "event": {
                 "type": "info",
-                "ingested": "2021-02-17T15:22:20.292979700Z",
+                "ingested": "2021-03-16T10:02:45.373424100Z",
                 "kind": "event"
-            },
-            "error": {
-                "message": "field [json] not present as part of path [json]"
             }
         }
     ]