[Fortinet] updating fortinet ecs version and add event.original options

go.mod

@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/elastic/elastic-package v0.0.0-20210531144253-3ebb7c474f58
+	github.com/elastic/elastic-package v0.0.0-20210603142344-c2d0f7538a68
 	github.com/elastic/package-registry v0.17.0
 	github.com/magefile/mage v1.11.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -57,6 +57,7 @@ github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YH
 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
 github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk=
 github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
@@ -170,6 +171,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/creasty/defaults v1.5.1 h1:j8WexcS3d/t4ZmllX4GEkl4wIB/trOr035ajcLHCISM=
 github.com/creasty/defaults v1.5.1/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -199,8 +201,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/elastic/elastic-package v0.0.0-20210531144253-3ebb7c474f58 h1:s/eRZJyf9RnFE3Vbis8DHsuqZMASF7UjkpKtpl/4T2E=
-github.com/elastic/elastic-package v0.0.0-20210531144253-3ebb7c474f58/go.mod h1:B7wEN2PYd3e+UYhbNhloYNXQkoBLz84xXQgVSvipvPg=
+github.com/elastic/elastic-package v0.0.0-20210603142344-c2d0f7538a68 h1:m0/QMuIl9E+Q+P0bztltgmM/Bw2r/XhmXfFxExvRhSs=
+github.com/elastic/elastic-package v0.0.0-20210603142344-c2d0f7538a68/go.mod h1:B7wEN2PYd3e+UYhbNhloYNXQkoBLz84xXQgVSvipvPg=
 github.com/elastic/go-elasticsearch/v7 v7.9.0 h1:UEau+a1MiiE/F+UrDj60kqIHFWdzU1M2y/YtBU2NC2M=
 github.com/elastic/go-elasticsearch/v7 v7.9.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=
@@ -1086,6 +1088,7 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1118,6 +1121,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

packages/apache/data_stream/status/_dev/test/system/test-default-config.yml

@@ -1,3 +1,6 @@
+skip:
+  reason: apache/status metricset no longer adjusts fields as expected in fleet mode
+  link: https://github.com/elastic/beats/issues/26120
 vars:
   hosts:
     - http://{{Hostname}}:{{Port}}

packages/apache/docs/README.md

@@ -186,7 +186,7 @@ generated by the `mod_status` module.
 
 An example event for `status` looks as following:
 
-```$json
+```json
 {
     "@timestamp": "2020-12-03T16:31:04.445Z",
     "data_stream": {

packages/auditd/docs/README.md

@@ -16,7 +16,7 @@ This is the Auditd `log` dataset.
 
 An example event for `log` looks as following:
 
-```$json
+```json
 {
     "@timestamp": "2017-01-31T20:17:14.891Z",
     "destination": {

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.6.0"
+  changes:
+    - description: Update ECS version, add event.original and preparing for package GA
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1070
 - version: "0.5.6"
   changes:
     - description: Fix stack compatability
@@ -7,12 +12,12 @@
 - version: "0.5.5"
   changes:
     - description: Allow role_arn work with access keys for AWS
-      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      type: enhancement
       link: https://github.com/elastic/integrations/pull/979
 - version: "0.5.4"
   changes:
     - description: Rename s3 input to aws-s3.
-      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      type: enhancement
       link: https://github.com/elastic/integrations/pull/631
 - version: "0.5.3"
   changes:
@@ -42,15 +47,15 @@
 - version: "0.4.1"
   changes:
     - description: Correct sample event file.
-      type: bugfix # can be one of: enhancement, bugfix, breaking-change
+      type: bugfix
       link: https://github.com/elastic/integrations/pull/754
 - version: "0.4.0"
   changes:
     - description: Add changes to use ECS 1.8 fields.
-      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      type: enhancement
       link: https://github.com/elastic/integrations/pull/721
 - version: "0.0.3"
   changes:
     - description: initial release
-      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      type: enhancement
       link: https://github.com/elastic/integrations/pull/21

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json

@@ -5,6 +5,9 @@
                 "address": "127.0.0.1",
                 "ip": "127.0.0.1"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "cloud": {
                 "region": "us-east-2",
                 "account": {
@@ -13,7 +16,7 @@
             },
             "@timestamp": "2014-03-25T21:08:14.000Z",
             "ecs": {
-                "version": "1.9.0"
+                "version": "1.10.0"
             },
             "related": {
                 "user": [
@@ -22,7 +25,7 @@
                 ]
             },
             "event": {
-                "ingested": "2021-04-23T12:52:55.040800850Z",
+                "ingested": "2021-06-04T09:43:14.431910700Z",
                 "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-25T18:45:11Z\"}}},\"eventTime\":\"2014-03-25T21:08:14Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AddUserToGroup\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"AWSConsole\",\"requestParameters\":{\"userName\":\"Bob\",\"groupName\":\"admin\"},\"responseElements\":null}",
                 "provider": "iam.amazonaws.com",
                 "created": "2014-03-25T21:08:14.000Z",

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json

@@ -9,7 +9,7 @@
             },
             "@timestamp": "2019-10-02T22:12:29.000Z",
             "ecs": {
-                "version": "1.9.0"
+                "version": "1.10.0"
             },
             "source": {
                 "geo": {
@@ -33,7 +33,7 @@
                 "ip": "123.145.67.89"
             },
             "event": {
-                "ingested": "2021-04-23T12:52:55.085403815Z",
+                "ingested": "2021-06-04T09:43:14.571172Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"123.145.67.89\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"johndoe@example.com\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}",
                 "provider": "sts.amazonaws.com",
                 "created": "2019-10-02T22:12:29.000Z",
@@ -123,7 +123,10 @@
                     "name": "Spider"
                 },
                 "version": "1.16.248"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json

@@ -1,6 +1,13 @@
 {
     "expected": [
         {
+            "source": {
+                "address": "127.0.0.1",
+                "ip": "127.0.0.1"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
             "cloud": {
                 "region": "us-east-1",
                 "account": {
@@ -9,19 +16,15 @@
             },
             "@timestamp": "2020-01-09T00:09:33.000Z",
             "ecs": {
-                "version": "1.9.0"
+                "version": "1.10.0"
             },
             "related": {
                 "user": [
                     "Alice"
                 ]
             },
-            "source": {
-                "address": "127.0.0.1",
-                "ip": "127.0.0.1"
-            },
             "event": {
-                "ingested": "2021-04-23T12:52:55.138113495Z",
+                "ingested": "2021-06-04T09:43:14.663769400Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:09:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"AccessDeniedException\",\"errorMessage\":\"An unknown error occurred\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE\",\"eventID\":\"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-09T00:09:33.000Z",
@@ -66,6 +69,13 @@
             }
         },
         {
+            "source": {
+                "address": "127.0.0.1",
+                "ip": "127.0.0.1"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
             "cloud": {
                 "region": "us-east-1",
                 "account": {
@@ -74,19 +84,15 @@
             },
             "@timestamp": "2020-01-09T00:03:36.000Z",
             "ecs": {
-                "version": "1.9.0"
+                "version": "1.10.0"
             },
             "related": {
                 "user": [
                     "Alice"
                 ]
             },
-            "source": {
-                "address": "127.0.0.1",
-                "ip": "127.0.0.1"
-            },
             "event": {
-                "ingested": "2021-04-23T12:52:55.138120536Z",
+                "ingested": "2021-06-04T09:43:14.663789500Z",
                 "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:03:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5c16-4eda-9724-EXAMPLE\",\"eventID\":\"EXAMPLE-35a7-4c25-9fc7-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}",
                 "provider": "iam.amazonaws.com",
                 "created": "2020-01-09T00:03:36.000Z",