diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 3f9c2c11653c..881d12481446 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -47,7 +47,7 @@ Example config: cloudtrail: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -64,7 +64,7 @@ Example config: cloudwatch: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -81,7 +81,7 @@ Example config: ec2: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -98,7 +98,7 @@ Example config: elb: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -115,7 +115,7 @@ Example config: s3access: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -132,7 +132,7 @@ Example config: vpcflow: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 6701548032ab..ae388f679a14 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -102,6 +102,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Process CloudTrail logs # default is true, set to false to skip Cloudtrail logs # var.process_cloudtrail_logs: false @@ -154,6 +163,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -194,6 +212,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -234,6 +261,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -274,6 +310,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -314,6 +359,15 @@ filebeat.modules: # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/input/awss3/acker.go b/x-pack/filebeat/input/awss3/acker.go index ba80601997fe..db88c23f7d18 100644 --- a/x-pack/filebeat/input/awss3/acker.go +++ b/x-pack/filebeat/input/awss3/acker.go @@ -52,7 +52,7 @@ func (a *eventACKTracker) ACK() { // Wait waits for the number of pending ACKs to be zero. // Wait must be called sequentially only after every expected -// Add call are made. Failing to do so could reset the pendingACKs +// `Add` calls are made. Failing to do so could reset the pendingACKs // property to 0 and would results in Wait returning after additional // calls to `Add` are made without a corresponding `ACK` call. func (a *eventACKTracker) Wait() { diff --git a/x-pack/filebeat/input/awss3/config.go b/x-pack/filebeat/input/awss3/config.go index 9fef4a9fc600..4e887003477f 100644 --- a/x-pack/filebeat/input/awss3/config.go +++ b/x-pack/filebeat/input/awss3/config.go @@ -26,7 +26,7 @@ type config struct { FIPSEnabled bool `config:"fips_enabled"` MaxNumberOfMessages int `config:"max_number_of_messages"` QueueURL string `config:"queue_url"` - Bucket string `config:"bucket"` + BucketARN string `config:"bucket_arn"` BucketListInterval time.Duration `config:"bucket_list_interval"` NumberOfWorkers int `config:"number_of_workers"` AWSConfig awscommon.ConfigAWS `config:",inline"` @@ -49,20 +49,20 @@ func defaultConfig() config { } func (c *config) Validate() error { - if c.QueueURL == "" && c.Bucket == "" { - return fmt.Errorf("queue_url or bucket must provided") + if c.QueueURL == "" && c.BucketARN == "" { + return fmt.Errorf("queue_url or bucket_arn must provided") } - if c.QueueURL != "" && c.Bucket != "" { - return fmt.Errorf("queue_url <%v> and bucket <%v> "+ - "cannot be set at the same time", c.QueueURL, c.Bucket) + if c.QueueURL != "" && c.BucketARN != "" { + return fmt.Errorf("queue_url <%v> and bucket_arn <%v> "+ + "cannot be set at the same time", c.QueueURL, c.BucketARN) } - if c.Bucket != "" && c.BucketListInterval <= 0 { + if c.BucketARN != "" && c.BucketListInterval <= 0 { return fmt.Errorf("bucket_list_interval <%v> must be greater than 0", c.BucketListInterval) } - if c.Bucket != "" && c.NumberOfWorkers <= 0 { + if c.BucketARN != "" && c.NumberOfWorkers <= 0 { return fmt.Errorf("number_of_workers <%v> must be greater than 0", c.NumberOfWorkers) } diff --git a/x-pack/filebeat/input/awss3/config_test.go b/x-pack/filebeat/input/awss3/config_test.go index c0fd94e2aee6..9fdf4c1dffbc 100644 --- a/x-pack/filebeat/input/awss3/config_test.go +++ b/x-pack/filebeat/input/awss3/config_test.go @@ -29,7 +29,7 @@ func TestConfig(t *testing.T) { require.NoError(t, parserConf.Unpack(common.MustNewConfigFrom(""))) return config{ QueueURL: quequeURL, - Bucket: s3Bucket, + BucketARN: s3Bucket, APITimeout: 120 * time.Second, VisibilityTimeout: 300 * time.Second, SQSMaxReceiveCount: 5, @@ -69,7 +69,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "number_of_workers": 5, }, "", @@ -109,10 +109,10 @@ func TestConfig(t *testing.T) { "", "", common.MapStr{ - "queue_url": "", - "bucket": "", + "queue_url": "", + "bucket_arn": "", }, - "queue_url or bucket must provided", + "queue_url or bucket_arn must provided", nil, }, { @@ -120,10 +120,10 @@ func TestConfig(t *testing.T) { queueURL, s3Bucket, common.MapStr{ - "queue_url": queueURL, - "bucket": s3Bucket, + "queue_url": queueURL, + "bucket_arn": s3Bucket, }, - "queue_url and bucket cannot be set at the same time", + "queue_url and bucket_arn cannot be set at the same time", nil, }, { @@ -164,7 +164,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "bucket_list_interval": "0", }, "bucket_list_interval <0s> must be greater than 0", @@ -175,7 +175,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "number_of_workers": "0", }, "number_of_workers <0> must be greater than 0", @@ -231,7 +231,7 @@ func TestConfig(t *testing.T) { "", s3Bucket, common.MapStr{ - "bucket": s3Bucket, + "bucket_arn": s3Bucket, "expand_event_list_from_field": "Records", "content_type": "text/plain", }, diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index d12b2a24cee5..6662984e060e 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -138,7 +138,7 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error { } } - if in.config.Bucket != "" { + if in.config.BucketARN != "" { // Create S3 receiver and S3 notification processor. poller, err := in.createS3Lister(inputContext, client, persistentStore, states) if err != nil { @@ -203,7 +203,7 @@ func (in *s3Input) createS3Lister(ctx v2.Context, client beat.Client, persistent client: s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, s3ServiceName, in.awsConfig.Region, in.awsConfig)), } - log := ctx.Logger.With("s3_bucket", in.config.Bucket) + log := ctx.Logger.With("bucket_arn", in.config.BucketARN) log.Infof("number_of_workers is set to %v.", in.config.NumberOfWorkers) log.Infof("bucket_list_interval is set to %v.", in.config.BucketListInterval) log.Infof("AWS region is set to %v.", in.awsConfig.Region) @@ -223,7 +223,8 @@ func (in *s3Input) createS3Lister(ctx v2.Context, client beat.Client, persistent s3EventHandlerFactory, states, persistentStore, - in.config.Bucket, + in.config.BucketARN, + in.awsConfig.Region, in.config.NumberOfWorkers, in.config.BucketListInterval) diff --git a/x-pack/filebeat/input/awss3/input_benchmark_test.go b/x-pack/filebeat/input/awss3/input_benchmark_test.go index 0c7df7e012b0..00540479d5c5 100644 --- a/x-pack/filebeat/input/awss3/input_benchmark_test.go +++ b/x-pack/filebeat/input/awss3/input_benchmark_test.go @@ -277,7 +277,7 @@ func benchmarkInputS3(t *testing.T, numberOfWorkers int) testing.BenchmarkResult } s3EventHandlerFactory := newS3ObjectProcessorFactory(log.Named("s3"), metrics, s3API, client, conf.FileSelectors) - s3Poller := newS3Poller(logp.NewLogger(inputName), metrics, s3API, s3EventHandlerFactory, newStates(inputCtx), store, "bucket", numberOfWorkers, time.Second) + s3Poller := newS3Poller(logp.NewLogger(inputName), metrics, s3API, s3EventHandlerFactory, newStates(inputCtx), store, "bucket", "region", numberOfWorkers, time.Second) ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) diff --git a/x-pack/filebeat/input/awss3/input_integration_test.go b/x-pack/filebeat/input/awss3/input_integration_test.go index 1ce157c553f8..4b8a3115db69 100644 --- a/x-pack/filebeat/input/awss3/input_integration_test.go +++ b/x-pack/filebeat/input/awss3/input_integration_test.go @@ -75,7 +75,7 @@ func getTerraformOutputs(t *testing.T) terraformOutputData { func makeTestConfigS3(s3bucket string) *common.Config { return common.MustNewConfigFrom(fmt.Sprintf(`--- -bucket: aws:s3:::%s +bucket_arn: aws:s3:::%s number_of_workers: 1 file_selectors: - diff --git a/x-pack/filebeat/input/awss3/s3.go b/x-pack/filebeat/input/awss3/s3.go index f1c08d2fb81f..313a71211e70 100644 --- a/x-pack/filebeat/input/awss3/s3.go +++ b/x-pack/filebeat/input/awss3/s3.go @@ -41,6 +41,7 @@ type s3ObjectPayload struct { type s3Poller struct { numberOfWorkers int bucket string + region string bucketPollInterval time.Duration workerSem *sem s3 s3API @@ -60,6 +61,7 @@ func newS3Poller(log *logp.Logger, states *states, store *statestore.Store, bucket string, + awsRegion string, numberOfWorkers int, bucketPollInterval time.Duration) *s3Poller { if metrics == nil { @@ -68,6 +70,7 @@ func newS3Poller(log *logp.Logger, return &s3Poller{ numberOfWorkers: numberOfWorkers, bucket: bucket, + region: awsRegion, bucketPollInterval: bucketPollInterval, workerSem: newSem(numberOfWorkers), s3: s3, @@ -178,7 +181,9 @@ func (p *s3Poller) GetS3Objects(ctx context.Context, s3ObjectPayloadChan chan<- p.states.Update(state, "") event := s3EventV2{} + event.AWSRegion = p.region event.S3.Bucket.Name = bucketName + event.S3.Bucket.ARN = p.bucket event.S3.Object.Key = filename acker := newEventACKTracker(ctx) diff --git a/x-pack/filebeat/input/awss3/s3_objects.go b/x-pack/filebeat/input/awss3/s3_objects.go index f486fbac0321..3b5c37ff0f67 100644 --- a/x-pack/filebeat/input/awss3/s3_objects.go +++ b/x-pack/filebeat/input/awss3/s3_objects.go @@ -76,8 +76,8 @@ func (f *s3ObjectProcessorFactory) findReaderConfig(key string) *readerConfig { // match the S3 object key. func (f *s3ObjectProcessorFactory) Create(ctx context.Context, log *logp.Logger, ack *eventACKTracker, obj s3EventV2) s3ObjectHandler { log = log.With( - "s3_bucket", obj.S3.Bucket.Name, - "s3_object", obj.S3.Object.Key) + "bucket_arn", obj.S3.Bucket.Name, + "object_key", obj.S3.Object.Key) readerConfig := f.findReaderConfig(obj.S3.Object.Key) if readerConfig == nil { diff --git a/x-pack/filebeat/input/awss3/s3_test.go b/x-pack/filebeat/input/awss3/s3_test.go index a02f3a58495c..dc87356ba655 100644 --- a/x-pack/filebeat/input/awss3/s3_test.go +++ b/x-pack/filebeat/input/awss3/s3_test.go @@ -133,7 +133,7 @@ func TestS3Poller(t *testing.T) { Return(nil, errFakeConnectivityFailure) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockAPI, mockPublisher, nil) - receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, numberOfWorkers, pollInterval) + receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, "region", numberOfWorkers, pollInterval) require.Error(t, context.DeadlineExceeded, receiver.Poll(ctx)) assert.Equal(t, numberOfWorkers, receiver.workerSem.available) }) @@ -263,7 +263,7 @@ func TestS3Poller(t *testing.T) { Return(nil, errFakeConnectivityFailure) s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockAPI, mockPublisher, nil) - receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, numberOfWorkers, pollInterval) + receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, s3ObjProc, newStates(inputCtx), store, bucket, "region", numberOfWorkers, pollInterval) require.Error(t, context.DeadlineExceeded, receiver.Poll(ctx)) assert.Equal(t, numberOfWorkers, receiver.workerSem.available) }) diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index 881138ed9f45..c0bc21854fac 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -5,6 +5,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Process CloudTrail logs # default is true, set to false to skip Cloudtrail logs # var.process_cloudtrail_logs: false @@ -57,6 +66,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -97,6 +115,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -137,6 +164,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -177,6 +213,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -217,6 +262,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index f286b9251dc3..02abf381ea64 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -42,7 +42,7 @@ Example config: cloudtrail: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -59,7 +59,7 @@ Example config: cloudwatch: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -76,7 +76,7 @@ Example config: ec2: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -93,7 +93,7 @@ Example config: elb: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -110,7 +110,7 @@ Example config: s3access: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -127,7 +127,7 @@ Example config: vpcflow: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - #var.bucket: 'arn:aws:s3:::mybucket' + #var.bucket_arn: 'arn:aws:s3:::mybucket' #var.bucket_list_interval: 300s #var.number_of_workers: 5 #var.shared_credential_file: /etc/filebeat/aws_credentials diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml index 5f4b3da66d21..97cde2469aea 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 9d40124d846a..6d2c9cdebe02 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml index 415d3f42718b..7634f73d8d27 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml index 415d3f42718b..7634f73d8d27 100644 --- a/x-pack/filebeat/module/aws/ec2/manifest.yml +++ b/x-pack/filebeat/module/aws/ec2/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index 128fcbf735e7..128dc59791ed 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml index 52861fdba122..617e9a46bc13 100644 --- a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml @@ -2,8 +2,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index 415d3f42718b..7634f73d8d27 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 3c25c1a9254e..12c57c500926 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -4,8 +4,8 @@ type: aws-s3 {{ if .queue_url }} queue_url: {{ .queue_url }} {{ end }} -{{ if .bucket }} -bucket: {{ .bucket }} +{{ if .bucket_arn }} +bucket_arn: {{ .bucket_arn }} {{ end }} {{ if .number_of_workers }} diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index d3122493b8cd..cd16451fcba7 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -4,7 +4,7 @@ var: - name: input default: aws-s3 - name: queue_url - - name: bucket + - name: bucket_arn - name: number_of_workers - name: bucket_list_interval - name: shared_credential_file diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index 248090f69fe1..deb82259b711 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -8,6 +8,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Process CloudTrail logs # default is true, set to false to skip Cloudtrail logs # var.process_cloudtrail_logs: false @@ -60,6 +69,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -100,6 +118,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -140,6 +167,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -180,6 +216,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows @@ -220,6 +265,15 @@ # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + # AWS S3 bucket arn + #var.bucket_arn: 'arn:aws:s3:::mybucket' + + # Bucket list interval on S3 bucket + #var.bucket_list_interval: 300s + + # Number of workers on S3 bucket + #var.number_of_workers: 5 + # Filename of AWS credential file # If not set "$HOME/.aws/credentials" is used on Linux/Mac # "%UserProfile%\.aws\credentials" is used on Windows