diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml index 8bdc322f65a7..2ae688ffab77 100644 --- a/.ci/jobs/apm-beats-update.yml +++ b/.ci/jobs/apm-beats-update.yml @@ -48,7 +48,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml index 522abfa5e5c4..808123a225e9 100644 --- a/.ci/jobs/beats-tester.yml +++ b/.ci/jobs/beats-tester.yml @@ -44,7 +44,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml index b075d8bbdf24..6f59a9bcdf8e 100644 --- a/.ci/jobs/beats.yml +++ b/.ci/jobs/beats.yml @@ -46,7 +46,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml index 46303790610f..45175d169f69 100644 --- a/.ci/jobs/golang-crossbuild-mbp.yml +++ b/.ci/jobs/golang-crossbuild-mbp.yml @@ -31,7 +31,7 @@ before: true prune: true shallow-clone: true - depth: 4 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml index 0dce4d4672b8..fd6fb9f90c6b 100644 --- a/.ci/jobs/packaging.yml +++ b/.ci/jobs/packaging.yml @@ -44,7 +44,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a96a23db9d8f..279eda229a7c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -349,6 +349,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601] - Fix k8s scheduler compatibility issue. {pull}19699[19699] - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898 +- Add support for azure light metricset app_stats. {pull}20639[20639] - Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680] - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736] - The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772] @@ -603,6 +604,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Always attempt community_id processor on zeek module {pull}21155[21155] - Add related.hosts ecs field to all modules {pull}21160[21160] - Keep cursor state between httpjson input restarts {pull}20751[20751] +- Convert aws s3 to v2 input {pull}20005[20005] *Heartbeat* @@ -722,6 +724,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103] - Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889] - Request prometheus endpoints to be gzipped by default {pull}20766[20766] +- Add latency config parameter into aws module. {pull}20875[20875] - Release all kubernetes `state` metricsets as GA {pull}20901[20901] - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738] - Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985] @@ -729,6 +732,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124] - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255] - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137] +- Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486] +- Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381] *Packetbeat* diff --git a/Jenkinsfile b/Jenkinsfile index 119cea9b3abf..317a5c781e33 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,6 +24,7 @@ pipeline { PIPELINE_LOG_LEVEL = 'INFO' PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}" RUNBLD_DISABLE_NOTIFICATIONS = 'true' + SLACK_CHANNEL = "#beats-ci-builds" TERRAFORM_VERSION = "0.12.24" XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } @@ -121,7 +122,7 @@ pipeline { runbld(stashedTestReports: stashedTestReports, project: env.REPO) } cleanup { - notifyBuildResult(prComment: true) + notifyBuildResult(prComment: true, slackComment: true) } } } diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 2f720bf055b2..2278ea93735a 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -28,11 +28,13 @@ changeset: - "^\\.ci/scripts/.*" oss: - "^go.mod" + - "^pytest.ini" - "^dev-tools/.*" - "^libbeat/.*" - "^testing/.*" xpack: - "^go.mod" + - "^pytest.ini" - "^dev-tools/.*" - "^libbeat/.*" - "^testing/.*" diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index 89d0bfd20ca4..bf3167e71068 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -35,7 +35,7 @@ const ( Name = "auditbeat" // ecsVersion specifies the version of ECS that Auditbeat is implementing. - ecsVersion = "1.5.0" + ecsVersion = "1.6.0" ) // RootCmd for running auditbeat. diff --git a/dev-tools/dependencies-report b/dev-tools/dependencies-report index a2662a4ab9ac..928de5367ca4 100755 --- a/dev-tools/dependencies-report +++ b/dev-tools/dependencies-report @@ -48,7 +48,7 @@ go list -m -json all $@ | go run go.elastic.co/go-licence-detector \ # name,url,version,revision,license ubi8url='https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8' ubi8source='https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz' -ubilicense='Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf,https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz' +ubilicense='Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf' cat <> $outfile Red Hat Universal Base Image,$ubi8url,8,,$ubilicense,$ubi8source EOF diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 8a145ff87241..b66c11633671 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -26581,6 +26581,153 @@ type: keyword -- This key captures values or decorators used within a registry entry +type: keyword + +-- + +[float] +=== cisco.umbrella + +Fields for Cisco Umbrella. + + + +*`cisco.umbrella.identities`*:: ++ +-- +An array of the different identities related to the event. + + +type: keyword + +-- + +*`cisco.umbrella.categories`*:: ++ +-- +The security or content categories that the destination matches. + + +type: keyword + +-- + +*`cisco.umbrella.policy_identity_type`*:: ++ +-- +The first identity type matched with this request. Available in version 3 and above. + + +type: keyword + +-- + +*`cisco.umbrella.identity_types`*:: ++ +-- +The type of identity that made the request. For example, Roaming Computer or Network. + + +type: keyword + +-- + +*`cisco.umbrella.blocked_categories`*:: ++ +-- +The categories that resulted in the destination being blocked. Available in version 4 and above. + + +type: keyword + +-- + +*`cisco.umbrella.content_type`*:: ++ +-- +The type of web content, typically text/html. + + +type: keyword + +-- + +*`cisco.umbrella.sha_sha256`*:: ++ +-- +Hex digest of the response content. + + +type: keyword + +-- + +*`cisco.umbrella.av_detections`*:: ++ +-- +The detection name according to the antivirus engine used in file inspection. + + +type: keyword + +-- + +*`cisco.umbrella.puas`*:: ++ +-- +A list of all potentially unwanted application (PUA) results for the proxied file as returned by the antivirus scanner. + + +type: keyword + +-- + +*`cisco.umbrella.amp_disposition`*:: ++ +-- +The status of the files proxied and scanned by Cisco Advanced Malware Protection (AMP) as part of the Umbrella File Inspection feature; can be Clean, Malicious or Unknown. + + +type: keyword + +-- + +*`cisco.umbrella.amp_malware_name`*:: ++ +-- +If Malicious, the name of the malware according to AMP. + + +type: keyword + +-- + +*`cisco.umbrella.amp_score`*:: ++ +-- +The score of the malware from AMP. This field is not currently used and will be blank. + + +type: keyword + +-- + +*`cisco.umbrella.datacenter`*:: ++ +-- +The name of the Umbrella Data Center that processed the user-generated traffic. + + +type: keyword + +-- + +*`cisco.umbrella.origin_id`*:: ++ +-- +The unique identity of the network tunnel. + + type: keyword -- diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc index c12f818cacad..d087826245e4 100644 --- a/filebeat/docs/modules/cisco.asciidoc +++ b/filebeat/docs/modules/cisco.asciidoc @@ -10,13 +10,15 @@ This file is generated! See scripts/docs_collector.py == Cisco module -This is a module for Cisco network device's logs. It includes the following +This is a module for Cisco network device's logs and Cisco Umbrella. It includes the following filesets for receiving logs over syslog or read from a file: - `asa` fileset: supports Cisco ASA firewall logs. - `ftd` fileset: supports Cisco Firepower Threat Defense logs. - `ios` fileset: supports Cisco IOS router and switch logs. - `nexus` fileset: supports Cisco Nexus switch logs. +- `meraki` fileset: supports Cisco Meraki logs. +- `umbrella` fileset: supports Cisco Umbrella logs. Cisco ASA devices also support exporting flow records using NetFlow, which is supported by the {filebeat-ref}/filebeat-module-netflow.html[netflow module] in @@ -32,6 +34,8 @@ The module is by default configured to run via syslog on port 9001 for ASA and port 9002 for IOS. However it can also be configured to read from a file path. See the following example. +Cisco Umbrella publishes its logs in a compressed CSV format to a S3 bucket. + ["source","yaml",subs="attributes"] ----- - module: cisco @@ -379,6 +383,59 @@ will be found under `rsa.raw`. The default is false. :fileset_ex!: +[float] +==== `umbrella` fileset settings + +The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. + +To configure Cisco Umbrella to log to either your own S3 bucket or one that is managed by Cisco please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide.] + +This fileset supports all 4 log types: +- Proxy +- Cloud Firewall +- IP Logs +- DNS logs + +The Cisco Umbrella fileset depends on the original file path structure being followed. This structure is documented https://docs.umbrella.com/deployment-umbrella/docs/log-formats-and-versioning[Umbrella Log Formats and Versioning]: + +/--
/--
---.csv.gz +dnslogs/--/----.csv.gz + +When configuring the fileset, please ensure that the Queue URL is set to the root folder that includes each of these subfolders above. + +Example config: + +[source,yaml] +---- +- module: cisco + umbrella: + enabled: true + var.input: s3 + var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + var.access_key_id: 123456 + var.secret_access_key: PASSWORD +---- + +*`var.input`*:: + +The input from which messages are read. Can be S3 or file. + +*`var.queue_url`*:: + +The URL to the SQS queue if the input type is S3. + +*`var.access_key_id`*:: + +The ID for the access key used to read from the SQS queue. + +*`var.secret_access_key`*:: + +The secret token used for authenticating to the SQS queue. + +:has-dashboards!: + +:fileset_ex!: + [float] === Example dashboard diff --git a/filebeat/docs/reload-configuration.asciidoc b/filebeat/docs/reload-configuration.asciidoc index 10a706d4020e..14c4826b6d84 100644 --- a/filebeat/docs/reload-configuration.asciidoc +++ b/filebeat/docs/reload-configuration.asciidoc @@ -33,9 +33,11 @@ definitions. TIP: The first line of each external configuration file must be an input definition that starts with `- type`. Make sure you omit the line -+{beatname_lc}.config.inputs+ from this file. - -For example: ++{beatname_lc}.config.inputs+ from this file. All <> +must be specified within each external configuration file. Specifying these +configuration options at the global `filebeat.config.inputs` level is not supported. + +Example external configuration file: [source,yaml] ------------------------------------------------------------------------------ diff --git a/filebeat/input/filestream/config.go b/filebeat/input/filestream/config.go new file mode 100644 index 000000000000..93b232325945 --- /dev/null +++ b/filebeat/input/filestream/config.go @@ -0,0 +1,147 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package filestream + +import ( + "fmt" + "time" + + "github.com/dustin/go-humanize" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/reader/readfile" +) + +// Config stores the options of a file stream. +type config struct { + Paths []string `config:"paths"` + Close closerConfig `config:"close"` + FileWatcher *common.ConfigNamespace `config:"file_watcher"` + Reader readerConfig `config:"readers"` + FileIdentity *common.ConfigNamespace `config:"file_identity"` + CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` + CleanRemoved bool `config:"clean_removed"` + HarvesterLimit uint32 `config:"harvester_limit" validate:"min=0"` + IgnoreOlder time.Duration `config:"ignore_older"` +} + +type closerConfig struct { + OnStateChange stateChangeCloserConfig `config:"on_state_change"` + Reader readerCloserConfig `config:"reader"` +} + +type readerCloserConfig struct { + AfterInterval time.Duration + Inactive time.Duration + OnEOF bool +} + +type stateChangeCloserConfig struct { + CheckInterval time.Duration + Removed bool + Renamed bool +} + +// TODO should this be inline? +type readerConfig struct { + Backoff backoffConfig `config:"backoff"` + BufferSize int `config:"buffer_size"` + Encoding string `config:"encoding"` + ExcludeLines []match.Matcher `config:"exclude_lines"` + IncludeLines []match.Matcher `config:"include_lines"` + LineTerminator readfile.LineTerminator `config:"line_terminator"` + MaxBytes int `config:"message_max_bytes" validate:"min=0,nonzero"` + Tail bool `config:"seek_to_tail"` + + Parsers []*common.ConfigNamespace `config:"parsers"` // TODO multiline, json, syslog? +} + +type backoffConfig struct { + Init time.Duration `config:"init" validate:"nonzero"` + Max time.Duration `config:"max" validate:"nonzero"` +} + +func defaultConfig() config { + return config{ + Paths: []string{}, + Close: defaultCloserConfig(), + Reader: defaultReaderConfig(), + CleanInactive: 0, + CleanRemoved: true, + HarvesterLimit: 0, + IgnoreOlder: 0, + } +} + +func defaultCloserConfig() closerConfig { + return closerConfig{ + OnStateChange: stateChangeCloserConfig{ + CheckInterval: 5 * time.Second, + Removed: true, // TODO check clean_removed option + Renamed: false, + }, + Reader: readerCloserConfig{ + OnEOF: false, + Inactive: 0 * time.Second, + AfterInterval: 0 * time.Second, + }, + } +} + +func defaultReaderConfig() readerConfig { + return readerConfig{ + Backoff: backoffConfig{ + Init: 1 * time.Second, + Max: 10 * time.Second, + }, + BufferSize: 16 * humanize.KiByte, + LineTerminator: readfile.AutoLineTerminator, + MaxBytes: 10 * humanize.MiByte, + Tail: false, + Parsers: nil, + } +} + +func (c *config) Validate() error { + if len(c.Paths) == 0 { + return fmt.Errorf("no path is configured") + } + // TODO + //if c.CleanInactive != 0 && c.IgnoreOlder == 0 { + // return fmt.Errorf("ignore_older must be enabled when clean_inactive is used") + //} + + // TODO + //if c.CleanInactive != 0 && c.CleanInactive <= c.IgnoreOlder+c.ScanFrequency { + // return fmt.Errorf("clean_inactive must be > ignore_older + scan_frequency to make sure only files which are not monitored anymore are removed") + //} + + // TODO + //if c.JSON != nil && len(c.JSON.MessageKey) == 0 && + // c.Multiline != nil { + // return fmt.Errorf("When using the JSON decoder and multiline together, you need to specify a message_key value") + //} + + //if c.JSON != nil && len(c.JSON.MessageKey) == 0 && + // (len(c.IncludeLines) > 0 || len(c.ExcludeLines) > 0) { + // return fmt.Errorf("When using the JSON decoder and line filtering together, you need to specify a message_key value") + //} + + return nil +} diff --git a/filebeat/input/filestream/filestream.go b/filebeat/input/filestream/filestream.go new file mode 100644 index 000000000000..59f26ccca1b3 --- /dev/null +++ b/filebeat/input/filestream/filestream.go @@ -0,0 +1,225 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package filestream + +import ( + "context" + "errors" + "io" + "os" + "time" + + input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common/backoff" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/unison" +) + +var ( + ErrFileTruncate = errors.New("detected file being truncated") + ErrClosed = errors.New("reader closed") +) + +// logFile contains all log related data +type logFile struct { + file *os.File + log *logp.Logger + ctx context.Context + cancelReading context.CancelFunc + + closeInactive time.Duration + closeAfterInterval time.Duration + closeOnEOF bool + + offset int64 + lastTimeRead time.Time + backoff backoff.Backoff + tg unison.TaskGroup +} + +// newFileReader creates a new log instance to read log sources +func newFileReader( + log *logp.Logger, + canceler input.Canceler, + f *os.File, + config readerConfig, + closerConfig readerCloserConfig, +) (*logFile, error) { + offset, err := f.Seek(0, os.SEEK_CUR) + if err != nil { + return nil, err + } + + l := &logFile{ + file: f, + log: log, + closeInactive: closerConfig.Inactive, + closeAfterInterval: closerConfig.AfterInterval, + closeOnEOF: closerConfig.OnEOF, + offset: offset, + lastTimeRead: time.Now(), + backoff: backoff.NewExpBackoff(canceler.Done(), config.Backoff.Init, config.Backoff.Max), + tg: unison.TaskGroup{}, + } + + l.ctx, l.cancelReading = ctxtool.WithFunc(ctxtool.FromCanceller(canceler), func() { + err := l.tg.Stop() + if err != nil { + l.log.Errorf("Error while stopping filestream logFile reader: %v", err) + } + }) + + l.startFileMonitoringIfNeeded() + + return l, nil +} + +// Read reads from the reader and updates the offset +// The total number of bytes read is returned. +func (f *logFile) Read(buf []byte) (int, error) { + totalN := 0 + + for f.ctx.Err() == nil { + n, err := f.file.Read(buf) + if n > 0 { + f.offset += int64(n) + f.lastTimeRead = time.Now() + } + totalN += n + + // Read from source completed without error + // Either end reached or buffer full + if err == nil { + // reset backoff for next read + f.backoff.Reset() + return totalN, nil + } + + // Move buffer forward for next read + buf = buf[n:] + + // Checks if an error happened or buffer is full + // If buffer is full, cannot continue reading. + // Can happen if n == bufferSize + io.EOF error + err = f.errorChecks(err) + if err != nil || len(buf) == 0 { + return totalN, err + } + + f.log.Debugf("End of file reached: %s; Backoff now.", f.file.Name()) + f.backoff.Wait() + } + + return 0, ErrClosed +} + +func (f *logFile) startFileMonitoringIfNeeded() { + if f.closeInactive == 0 && f.closeAfterInterval == 0 { + return + } + + if f.closeInactive > 0 { + f.tg.Go(func(ctx unison.Canceler) error { + f.closeIfTimeout(ctx) + return nil + }) + } + + if f.closeAfterInterval > 0 { + f.tg.Go(func(ctx unison.Canceler) error { + f.closeIfInactive(ctx) + return nil + }) + } +} + +func (f *logFile) closeIfTimeout(ctx unison.Canceler) { + timer := time.NewTimer(f.closeAfterInterval) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + f.cancelReading() + return + } + } +} + +func (f *logFile) closeIfInactive(ctx unison.Canceler) { + // This can be made configureble if users need a more flexible + // cheking for inactive files. + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + age := time.Since(f.lastTimeRead) + if age > f.closeInactive { + f.cancelReading() + return + } + } + } +} + +// errorChecks determines the cause for EOF errors, and how the EOF event should be handled +// based on the config options. +func (f *logFile) errorChecks(err error) error { + if err != io.EOF { + f.log.Error("Unexpected state reading from %s; error: %s", f.file.Name(), err) + return err + } + + return f.handleEOF() +} + +func (f *logFile) handleEOF() error { + if f.closeOnEOF { + return io.EOF + } + + // Refetch fileinfo to check if the file was truncated. + // Errors if the file was removed/rotated after reading and before + // calling the stat function + info, statErr := f.file.Stat() + if statErr != nil { + f.log.Error("Unexpected error reading from %s; error: %s", f.file.Name(), statErr) + return statErr + } + + // check if file was truncated + if info.Size() < f.offset { + f.log.Debugf("File was truncated as offset (%d) > size (%d): %s", f.offset, info.Size(), f.file.Name()) + return ErrFileTruncate + } + + return nil +} + +// Close +func (f *logFile) Close() error { + f.cancelReading() + return f.file.Close() +} diff --git a/filebeat/input/filestream/fswatch.go b/filebeat/input/filestream/fswatch.go new file mode 100644 index 000000000000..d4bc1b5ea089 --- /dev/null +++ b/filebeat/input/filestream/fswatch.go @@ -0,0 +1,375 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package filestream + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/elastic/beats/v7/filebeat/input/file" + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/unison" +) + +const ( + recursiveGlobDepth = 8 + scannerName = "scanner" + watcherDebugKey = "file_watcher" +) + +var ( + watcherFactories = map[string]watcherFactory{ + scannerName: newScannerWatcher, + } +) + +type watcherFactory func(paths []string, cfg *common.Config) (loginp.FSWatcher, error) + +// fileScanner looks for files which match the patterns in paths. +// It is able to exclude files and symlinks. +type fileScanner struct { + paths []string + excludedFiles []match.Matcher + symlinks bool + + log *logp.Logger +} + +type fileWatcherConfig struct { + // Interval is the time between two scans. + Interval time.Duration + // Scanner is the configuration of the scanner. + Scanner fileScannerConfig +} + +// fileWatcher gets the list of files from a FSWatcher and creates events by +// comparing the files between its last two runs. +type fileWatcher struct { + interval time.Duration + prev map[string]os.FileInfo + scanner loginp.FSScanner + log *logp.Logger + events chan loginp.FSEvent +} + +func newFileWatcher(paths []string, ns *common.ConfigNamespace) (loginp.FSWatcher, error) { + if ns == nil { + return newScannerWatcher(paths, nil) + } + + watcherType := ns.Name() + f, ok := watcherFactories[watcherType] + if !ok { + return nil, fmt.Errorf("no such file watcher: %s", watcherType) + } + + return f(paths, ns.Config()) +} + +func newScannerWatcher(paths []string, c *common.Config) (loginp.FSWatcher, error) { + config := defaultFileWatcherConfig() + err := c.Unpack(&config) + if err != nil { + return nil, err + } + scanner, err := newFileScanner(paths, config.Scanner) + if err != nil { + return nil, err + } + return &fileWatcher{ + log: logp.NewLogger(watcherDebugKey), + interval: config.Interval, + prev: make(map[string]os.FileInfo, 0), + scanner: scanner, + events: make(chan loginp.FSEvent), + }, nil +} + +func defaultFileWatcherConfig() fileWatcherConfig { + return fileWatcherConfig{ + Interval: 10 * time.Second, + Scanner: defaultFileScannerConfig(), + } +} + +func (w *fileWatcher) Run(ctx unison.Canceler) { + defer close(w.events) + + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + w.watch(ctx) + } + } +} + +func (w *fileWatcher) watch(ctx unison.Canceler) { + w.log.Info("Start next scan") + + paths := w.scanner.GetFiles() + + newFiles := make(map[string]os.FileInfo) + + for path, info := range paths { + + prevInfo, ok := w.prev[path] + if !ok { + newFiles[path] = paths[path] + continue + } + + if prevInfo.ModTime() != info.ModTime() { + select { + case <-ctx.Done(): + return + case w.events <- writeEvent(path, info): + } + } + + // delete from previous state, as we have more up to date info + delete(w.prev, path) + } + + // remaining files are in the prev map are the ones that are missing + // either because they have been deleted or renamed + for removedPath, removedInfo := range w.prev { + for newPath, newInfo := range newFiles { + if os.SameFile(removedInfo, newInfo) { + select { + case <-ctx.Done(): + return + case w.events <- renamedEvent(removedPath, newPath, newInfo): + delete(newFiles, newPath) + goto CHECK_NEXT_REMOVED + } + } + } + + select { + case <-ctx.Done(): + return + case w.events <- deleteEvent(removedPath, removedInfo): + } + CHECK_NEXT_REMOVED: + } + + // remaining files in newFiles are new + for path, info := range newFiles { + select { + case <-ctx.Done(): + return + case w.events <- createEvent(path, info): + } + + } + + w.log.Debugf("Found %d paths", len(paths)) + w.prev = paths +} + +func createEvent(path string, fi os.FileInfo) loginp.FSEvent { + return loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: path, Info: fi} +} + +func writeEvent(path string, fi os.FileInfo) loginp.FSEvent { + return loginp.FSEvent{Op: loginp.OpWrite, OldPath: path, NewPath: path, Info: fi} +} + +func renamedEvent(oldPath, path string, fi os.FileInfo) loginp.FSEvent { + return loginp.FSEvent{Op: loginp.OpRename, OldPath: oldPath, NewPath: path, Info: fi} +} + +func deleteEvent(path string, fi os.FileInfo) loginp.FSEvent { + return loginp.FSEvent{Op: loginp.OpDelete, OldPath: path, NewPath: "", Info: fi} +} + +func (w *fileWatcher) Event() loginp.FSEvent { + return <-w.events +} + +type fileScannerConfig struct { + Paths []string + ExcludedFiles []match.Matcher + Symlinks bool + RecursiveGlob bool +} + +func defaultFileScannerConfig() fileScannerConfig { + return fileScannerConfig{ + Symlinks: false, + RecursiveGlob: true, + } +} + +func newFileScanner(paths []string, cfg fileScannerConfig) (loginp.FSScanner, error) { + fs := fileScanner{ + paths: paths, + excludedFiles: cfg.ExcludedFiles, + symlinks: cfg.Symlinks, + log: logp.NewLogger(scannerName), + } + err := fs.resolveRecursiveGlobs(cfg) + if err != nil { + return nil, err + } + err = fs.normalizeGlobPatterns() + if err != nil { + return nil, err + } + + return &fs, nil +} + +// resolveRecursiveGlobs expands `**` from the globs in multiple patterns +func (s *fileScanner) resolveRecursiveGlobs(c fileScannerConfig) error { + if !c.RecursiveGlob { + s.log.Debug("recursive glob disabled") + return nil + } + + s.log.Debug("recursive glob enabled") + var paths []string + for _, path := range s.paths { + patterns, err := file.GlobPatterns(path, recursiveGlobDepth) + if err != nil { + return err + } + if len(patterns) > 1 { + s.log.Debugf("%q expanded to %#v", path, patterns) + } + paths = append(paths, patterns...) + } + s.paths = paths + return nil +} + +// normalizeGlobPatterns calls `filepath.Abs` on all the globs from config +func (s *fileScanner) normalizeGlobPatterns() error { + var paths []string + for _, path := range s.paths { + pathAbs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("failed to get the absolute path for %s: %v", path, err) + } + paths = append(paths, pathAbs) + } + s.paths = paths + return nil +} + +// GetFiles returns a map of files and fileinfos which +// match the configured paths. +func (s *fileScanner) GetFiles() map[string]os.FileInfo { + pathInfo := map[string]os.FileInfo{} + + for _, path := range s.paths { + matches, err := filepath.Glob(path) + if err != nil { + s.log.Errorf("glob(%s) failed: %v", path, err) + continue + } + + for _, file := range matches { + if s.shouldSkipFile(file) { + continue + } + + // If symlink is enabled, it is checked that original is not part of same input + // If original is harvested by other input, states will potentially overwrite each other + if s.isOriginalAndSymlinkConfigured(file, pathInfo) { + continue + } + + fileInfo, err := os.Stat(file) + if err != nil { + s.log.Debug("stat(%s) failed: %s", file, err) + continue + } + pathInfo[file] = fileInfo + } + } + + return pathInfo +} + +func (s *fileScanner) shouldSkipFile(file string) bool { + if s.isFileExcluded(file) { + s.log.Debugf("Exclude file: %s", file) + return true + } + + fileInfo, err := os.Lstat(file) + if err != nil { + s.log.Debugf("lstat(%s) failed: %s", file, err) + return true + } + + if fileInfo.IsDir() { + s.log.Debugf("Skipping directory: %s", file) + return true + } + + isSymlink := fileInfo.Mode()&os.ModeSymlink > 0 + if isSymlink && !s.symlinks { + s.log.Debugf("File %s skipped as it is a symlink", file) + return true + } + + return false +} + +func (s *fileScanner) isOriginalAndSymlinkConfigured(file string, paths map[string]os.FileInfo) bool { + if s.symlinks { + fileInfo, err := os.Stat(file) + if err != nil { + s.log.Debugf("stat(%s) failed: %s", file, err) + return false + } + + for _, finfo := range paths { + if os.SameFile(finfo, fileInfo) { + s.log.Info("Same file found as symlink and original. Skipping file: %s (as it same as %s)", file, finfo.Name()) + return true + } + } + } + return false +} + +func (s *fileScanner) isFileExcluded(file string) bool { + return len(s.excludedFiles) > 0 && s.matchAny(s.excludedFiles, file) +} + +// matchAny checks if the text matches any of the regular expressions +func (s *fileScanner) matchAny(matchers []match.Matcher, text string) bool { + for _, m := range matchers { + if m.MatchString(text) { + return true + } + } + return false +} diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go new file mode 100644 index 000000000000..d6286a273eb0 --- /dev/null +++ b/filebeat/input/filestream/fswatch_test.go @@ -0,0 +1,272 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package filestream + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/logp" +) + +var ( + excludedFilePath = filepath.Join("testdata", "excluded_file") + includedFilePath = filepath.Join("testdata", "included_file") + directoryPath = filepath.Join("testdata", "unharvestable_dir") +) + +func TestFileScanner(t *testing.T) { + t.Skip("Flaky test: https://github.com/elastic/beats/issues/21489") + + testCases := map[string]struct { + paths []string + excludedFiles []match.Matcher + symlinks bool + expectedFiles []string + }{ + "select all files": { + paths: []string{excludedFilePath, includedFilePath}, + expectedFiles: []string{ + mustAbsPath(excludedFilePath), + mustAbsPath(includedFilePath), + }, + }, + "skip excluded files": { + paths: []string{excludedFilePath, includedFilePath}, + excludedFiles: []match.Matcher{ + match.MustCompile("excluded_file"), + }, + expectedFiles: []string{ + mustAbsPath(includedFilePath), + }, + }, + "skip directories": { + paths: []string{directoryPath}, + expectedFiles: []string{}, + }, + } + + setupFilesForScannerTest(t) + defer removeFilesOfScannerTest(t) + + for name, test := range testCases { + test := test + + t.Run(name, func(t *testing.T) { + cfg := fileScannerConfig{ + ExcludedFiles: test.excludedFiles, + Symlinks: test.symlinks, + RecursiveGlob: false, + } + fs, err := newFileScanner(test.paths, cfg) + if err != nil { + t.Fatal(err) + } + files := fs.GetFiles() + paths := make([]string, 0) + for p, _ := range files { + paths = append(paths, p) + } + assert.True(t, checkIfSameContents(test.expectedFiles, paths)) + }) + } +} + +func setupFilesForScannerTest(t *testing.T) { + err := os.MkdirAll(directoryPath, 0750) + if err != nil { + t.Fatal(t) + } + + for _, path := range []string{excludedFilePath, includedFilePath} { + f, err := os.Create(path) + if err != nil { + t.Fatalf("file %s, error %v", path, err) + } + f.Close() + } +} + +func removeFilesOfScannerTest(t *testing.T) { + err := os.RemoveAll("testdata") + if err != nil { + t.Fatal(err) + } +} + +// only handles sets +func checkIfSameContents(one, other []string) bool { + if len(one) != len(other) { + return false + } + + mustFind := len(one) + for _, oneElem := range one { + for _, otherElem := range other { + if oneElem == otherElem { + mustFind-- + } + } + } + return mustFind == 0 +} + +func TestFileWatchNewDeleteModified(t *testing.T) { + t.Skip("Flaky test: https://github.com/elastic/beats/issues/21489") + + oldTs := time.Now() + newTs := oldTs.Add(5 * time.Second) + testCases := map[string]struct { + prevFiles map[string]os.FileInfo + nextFiles map[string]os.FileInfo + expectedEvents []loginp.FSEvent + }{ + "one new file": { + prevFiles: map[string]os.FileInfo{}, + nextFiles: map[string]os.FileInfo{ + "new_path": testFileInfo{"new_path", 5, oldTs}, + }, + expectedEvents: []loginp.FSEvent{ + loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: "new_path", Info: testFileInfo{"new_path", 5, oldTs}}, + }, + }, + "one deleted file": { + prevFiles: map[string]os.FileInfo{ + "old_path": testFileInfo{"old_path", 5, oldTs}, + }, + nextFiles: map[string]os.FileInfo{}, + expectedEvents: []loginp.FSEvent{ + loginp.FSEvent{Op: loginp.OpDelete, OldPath: "old_path", NewPath: "", Info: testFileInfo{"old_path", 5, oldTs}}, + }, + }, + "one modified file": { + prevFiles: map[string]os.FileInfo{ + "path": testFileInfo{"path", 5, oldTs}, + }, + nextFiles: map[string]os.FileInfo{ + "path": testFileInfo{"path", 10, newTs}, + }, + expectedEvents: []loginp.FSEvent{ + loginp.FSEvent{Op: loginp.OpWrite, OldPath: "path", NewPath: "path", Info: testFileInfo{"path", 10, newTs}}, + }, + }, + "two modified files": { + prevFiles: map[string]os.FileInfo{ + "path1": testFileInfo{"path1", 5, oldTs}, + "path2": testFileInfo{"path2", 5, oldTs}, + }, + nextFiles: map[string]os.FileInfo{ + "path1": testFileInfo{"path1", 10, newTs}, + "path2": testFileInfo{"path2", 10, newTs}, + }, + expectedEvents: []loginp.FSEvent{ + loginp.FSEvent{Op: loginp.OpWrite, OldPath: "path1", NewPath: "path1", Info: testFileInfo{"path1", 10, newTs}}, + loginp.FSEvent{Op: loginp.OpWrite, OldPath: "path2", NewPath: "path2", Info: testFileInfo{"path2", 10, newTs}}, + }, + }, + "one modified file, one new file": { + prevFiles: map[string]os.FileInfo{ + "path1": testFileInfo{"path1", 5, oldTs}, + }, + nextFiles: map[string]os.FileInfo{ + "path1": testFileInfo{"path1", 10, newTs}, + "path2": testFileInfo{"path2", 10, newTs}, + }, + expectedEvents: []loginp.FSEvent{ + loginp.FSEvent{Op: loginp.OpWrite, OldPath: "path1", NewPath: "path1", Info: testFileInfo{"path1", 10, newTs}}, + loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: "path2", Info: testFileInfo{"path2", 10, newTs}}, + }, + }, + "one new file, one deleted file": { + prevFiles: map[string]os.FileInfo{ + "path_deleted": testFileInfo{"path_deleted", 5, oldTs}, + }, + nextFiles: map[string]os.FileInfo{ + "path_new": testFileInfo{"path_new", 10, newTs}, + }, + expectedEvents: []loginp.FSEvent{ + loginp.FSEvent{Op: loginp.OpDelete, OldPath: "path_deleted", NewPath: "", Info: testFileInfo{"path_deleted", 5, oldTs}}, + loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: "path_new", Info: testFileInfo{"path_new", 10, newTs}}, + }, + }, + } + + for name, test := range testCases { + test := test + + t.Run(name, func(t *testing.T) { + w := fileWatcher{ + log: logp.L(), + prev: test.prevFiles, + scanner: &mockScanner{test.nextFiles}, + events: make(chan loginp.FSEvent), + } + + go w.watch(context.Background()) + + for _, expectedEvent := range test.expectedEvents { + evt := w.Event() + assert.Equal(t, expectedEvent, evt) + } + }) + } +} + +type mockScanner struct { + files map[string]os.FileInfo +} + +func (m *mockScanner) GetFiles() map[string]os.FileInfo { + return m.files +} + +type testFileInfo struct { + path string + size int64 + time time.Time +} + +func (t testFileInfo) Name() string { return t.path } +func (t testFileInfo) Size() int64 { return t.size } +func (t testFileInfo) Mode() os.FileMode { return 0 } +func (t testFileInfo) ModTime() time.Time { return t.time } +func (t testFileInfo) IsDir() bool { return false } +func (t testFileInfo) Sys() interface{} { return nil } + +func mustAbsPath(path string) string { + p, err := filepath.Abs(path) + if err != nil { + panic(err) + } + return p +} + +func mustDuration(durStr string) time.Duration { + dur, err := time.ParseDuration(durStr) + if err != nil { + panic(err) + } + return dur +} diff --git a/filebeat/input/filestream/fswatch_test_non_windows.go b/filebeat/input/filestream/fswatch_test_non_windows.go new file mode 100644 index 000000000000..eecfeddf9308 --- /dev/null +++ b/filebeat/input/filestream/fswatch_test_non_windows.go @@ -0,0 +1,144 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package filestream + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestFileScannerSymlinks(t *testing.T) { + testCases := map[string]struct { + paths []string + excludedFiles []match.Matcher + symlinks bool + expectedFiles []string + }{ + // covers test_input.py/test_skip_symlinks + "skip symlinks": { + paths: []string{ + filepath.Join("testdata", "symlink_to_included_file"), + filepath.Join("testdata", "included_file"), + }, + symlinks: false, + expectedFiles: []string{ + mustAbsPath(filepath.Join("testdata", "included_file")), + }, + }, + "return a file once if symlinks are enabled": { + paths: []string{ + filepath.Join("testdata", "symlink_to_included_file"), + filepath.Join("testdata", "included_file"), + }, + symlinks: true, + expectedFiles: []string{ + mustAbsPath(filepath.Join("testdata", "included_file")), + }, + }, + } + + err := os.Symlink( + mustAbsPath(filepath.Join("testdata", "included_file")), + mustAbsPath(filepath.Join("testdata", "symlink_to_included_file")), + ) + if err != nil { + t.Fatal(err) + } + + for name, test := range testCases { + test := test + + t.Run(name, func(t *testing.T) { + cfg := fileScannerConfig{ + ExcludedFiles: test.excludedFiles, + Symlinks: true, + RecursiveGlob: false, + } + fs, err := newFileScanner(test.paths, cfg) + if err != nil { + t.Fatal(err) + } + files := fs.GetFiles() + paths := make([]string, 0) + for p, _ := range files { + paths = append(paths, p) + } + assert.Equal(t, test.expectedFiles, paths) + }) + } +} + +func TestFileWatcherRenamedFile(t *testing.T) { + testPath := mustAbsPath("first_name") + renamedPath := mustAbsPath("renamed") + + f, err := os.Create(testPath) + if err != nil { + t.Fatal(err) + } + f.Close() + fi, err := os.Stat(testPath) + if err != nil { + t.Fatal(err) + } + + cfg := fileScannerConfig{ + ExcludedFiles: nil, + Symlinks: false, + RecursiveGlob: false, + } + scanner, err := newFileScanner([]string{testPath, renamedPath}, cfg) + if err != nil { + t.Fatal(err) + } + w := fileWatcher{ + log: logp.L(), + scanner: scanner, + events: make(chan loginp.FSEvent), + } + + go w.watch(context.Background()) + assert.Equal(t, loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: testPath, Info: fi}, w.Event()) + + err = os.Rename(testPath, renamedPath) + if err != nil { + t.Fatal(err) + } + defer os.Remove(renamedPath) + fi, err = os.Stat(renamedPath) + if err != nil { + t.Fatal(err) + } + + go w.watch(context.Background()) + evt := w.Event() + + assert.Equal(t, loginp.OpRename, evt.Op) + assert.Equal(t, testPath, evt.OldPath) + assert.Equal(t, renamedPath, evt.NewPath) +} diff --git a/filebeat/input/filestream/identifier.go b/filebeat/input/filestream/identifier.go new file mode 100644 index 000000000000..736c66da2f03 --- /dev/null +++ b/filebeat/input/filestream/identifier.go @@ -0,0 +1,139 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package filestream + +import ( + "fmt" + "os" + + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/file" +) + +const ( + nativeName = "native" + pathName = "path" + inodeMarkerName = "inode_marker" + + DefaultIdentifierName = nativeName + identitySep = "::" +) + +var ( + identifierFactories = map[string]identifierFactory{ + nativeName: newINodeDeviceIdentifier, + pathName: newPathIdentifier, + inodeMarkerName: newINodeMarkerIdentifier, + } +) + +type identifierFactory func(*common.Config) (fileIdentifier, error) + +type fileIdentifier interface { + GetSource(loginp.FSEvent) fileSource + Name() string +} + +// fileSource implements the Source interface +// It is required to identify and manage file sources. +type fileSource struct { + info os.FileInfo + newPath string + oldPath string + + name string + identifierGenerator string +} + +// Name returns the registry identifier of the file. +func (f fileSource) Name() string { + return f.name +} + +// newFileIdentifier creates a new state identifier for a log input. +func newFileIdentifier(ns *common.ConfigNamespace) (fileIdentifier, error) { + if ns == nil { + return newINodeDeviceIdentifier(nil) + } + + identifierType := ns.Name() + f, ok := identifierFactories[identifierType] + if !ok { + return nil, fmt.Errorf("no such file_identity generator: %s", identifierType) + } + + return f(ns.Config()) +} + +type inodeDeviceIdentifier struct { + name string +} + +func newINodeDeviceIdentifier(_ *common.Config) (fileIdentifier, error) { + return &inodeDeviceIdentifier{ + name: nativeName, + }, nil +} + +func (i *inodeDeviceIdentifier) GetSource(e loginp.FSEvent) fileSource { + return fileSource{ + info: e.Info, + newPath: e.NewPath, + oldPath: e.OldPath, + name: pluginName + identitySep + i.name + identitySep + file.GetOSState(e.Info).String(), + identifierGenerator: i.name, + } +} + +func (i *inodeDeviceIdentifier) Name() string { + return i.name +} + +type pathIdentifier struct { + name string +} + +func newPathIdentifier(_ *common.Config) (fileIdentifier, error) { + return &pathIdentifier{ + name: pathName, + }, nil +} + +func (p *pathIdentifier) GetSource(e loginp.FSEvent) fileSource { + return fileSource{ + info: e.Info, + newPath: e.NewPath, + oldPath: e.OldPath, + name: pluginName + identitySep + p.name + identitySep + e.NewPath, + identifierGenerator: p.name, + } +} + +func (p *pathIdentifier) Name() string { + return p.name +} + +// mockIdentifier is used for testing +type MockIdentifier struct{} + +func (m *MockIdentifier) GetSource(e loginp.FSEvent) fileSource { + return fileSource{identifierGenerator: "mock"} +} + +func (m *MockIdentifier) Name() string { return "mock" } diff --git a/filebeat/input/filestream/identifier_inode_deviceid.go b/filebeat/input/filestream/identifier_inode_deviceid.go new file mode 100644 index 000000000000..25254d97fdca --- /dev/null +++ b/filebeat/input/filestream/identifier_inode_deviceid.go @@ -0,0 +1,108 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package filestream + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/file" + "github.com/elastic/beats/v7/libbeat/logp" +) + +type inodeMarkerIdentifier struct { + log *logp.Logger + name string + markerPath string + + markerFileLastModifitaion time.Time + markerTxt string +} + +func newINodeMarkerIdentifier(cfg *common.Config) (fileIdentifier, error) { + var config struct { + MarkerPath string `config:"path" validate:"required"` + } + err := cfg.Unpack(&config) + if err != nil { + return nil, fmt.Errorf("error while reading configuration of INode + marker file configuration: %v", err) + } + + fi, err := os.Stat(config.MarkerPath) + if err != nil { + return nil, fmt.Errorf("error while opening marker file at %s: %v", config.MarkerPath, err) + } + markerContent, err := ioutil.ReadFile(config.MarkerPath) + if err != nil { + return nil, fmt.Errorf("error while reading marker file at %s: %v", config.MarkerPath, err) + } + return &inodeMarkerIdentifier{ + log: logp.NewLogger("inode_marker_identifier_" + filepath.Base(config.MarkerPath)), + name: inodeMarkerName, + markerPath: config.MarkerPath, + markerFileLastModifitaion: fi.ModTime(), + markerTxt: string(markerContent), + }, nil +} + +func (i *inodeMarkerIdentifier) markerContents() string { + f, err := os.Open(i.markerPath) + if err != nil { + i.log.Errorf("Failed to open marker file %s: %v", i.markerPath, err) + return "" + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + i.log.Errorf("Failed to fetch file information for %s: %v", i.markerPath, err) + return "" + } + if i.markerFileLastModifitaion.Before(fi.ModTime()) { + contents, err := ioutil.ReadFile(i.markerPath) + if err != nil { + i.log.Errorf("Error while reading contents of marker file: %v", err) + return "" + } + i.markerTxt = string(contents) + } + + return i.markerTxt +} + +func (i *inodeMarkerIdentifier) GetSource(e loginp.FSEvent) fileSource { + osstate := file.GetOSState(e.Info) + return fileSource{ + info: e.Info, + newPath: e.NewPath, + oldPath: e.OldPath, + name: fmt.Sprintf("%s%s%s-%s", i.name, identitySep, osstate.InodeString(), i.markerContents()), + identifierGenerator: i.name, + } +} + +func (i *inodeMarkerIdentifier) Name() string { + return i.name +} diff --git a/filebeat/input/filestream/identifier_inode_deviceid_windows.go b/filebeat/input/filestream/identifier_inode_deviceid_windows.go new file mode 100644 index 000000000000..4ee8d8661248 --- /dev/null +++ b/filebeat/input/filestream/identifier_inode_deviceid_windows.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package filestream + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func newINodeMarkerIdentifier(cfg *common.Config) (fileIdentifier, error) { + return nil, fmt.Errorf("inode_deviceid is not supported on Windows") +} diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index 487a5f01c2aa..bcd143c1c5a4 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -29,6 +29,12 @@ import ( // are actively written by other applications. type filestream struct{} +type state struct { + Source string `json:"source" struct:"source"` + Offset int64 `json:"offset" struct:"offset"` + IdentifierName string `json:"identifier_name" struct:"identifier_name"` +} + const pluginName = "filestream" // Plugin creates a new filestream input plugin for creating a stateful input. diff --git a/filebeat/input/filestream/prospector.go b/filebeat/input/filestream/prospector.go index 257574b9ca1a..94670e18ce7f 100644 --- a/filebeat/input/filestream/prospector.go +++ b/filebeat/input/filestream/prospector.go @@ -18,19 +18,191 @@ package filestream import ( + "os" + "strings" + "time" + + "github.com/urso/sderr" + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/go-concert/unison" +) + +const ( + prospectorDebugKey = "file_prospector" ) // fileProspector implements the Prospector interface. // It contains a file scanner which returns file system events. // The FS events then trigger either new Harvester runs or updates // the statestore. -type fileProspector struct{} +type fileProspector struct { + filewatcher loginp.FSWatcher + identifier fileIdentifier + ignoreOlder time.Duration + cleanRemoved bool +} + +func newFileProspector( + paths []string, + ignoreOlder time.Duration, + fileWatcherNs, identifierNs *common.ConfigNamespace, +) (loginp.Prospector, error) { + + filewatcher, err := newFileWatcher(paths, fileWatcherNs) + if err != nil { + return nil, err + } + + identifier, err := newFileIdentifier(identifierNs) + if err != nil { + return nil, err + } + + return &fileProspector{ + filewatcher: filewatcher, + identifier: identifier, + ignoreOlder: ignoreOlder, + cleanRemoved: true, + }, nil +} +// Run starts the fileProspector which accepts FS events from a file watcher. func (p *fileProspector) Run(ctx input.Context, s *statestore.Store, hg *loginp.HarvesterGroup) { - panic("TODO: implement me") + log := ctx.Logger.With("prospector", prospectorDebugKey) + log.Debug("Starting prospector") + defer log.Debug("Prospector has stopped") + + if p.cleanRemoved { + p.cleanRemovedBetweenRuns(log, s) + } + + p.updateIdentifiersBetweenRuns(log, s) + + var tg unison.MultiErrGroup + + tg.Go(func() error { + p.filewatcher.Run(ctx.Cancelation) + return nil + }) + + tg.Go(func() error { + for ctx.Cancelation.Err() == nil { + fe := p.filewatcher.Event() + + if fe.Op == loginp.OpDone { + return nil + } + + src := p.identifier.GetSource(fe) + switch fe.Op { + case loginp.OpCreate: + log.Debugf("A new file %s has been found", fe.NewPath) + + if p.ignoreOlder > 0 { + now := time.Now() + if now.Sub(fe.Info.ModTime()) > p.ignoreOlder { + log.Debugf("Ignore file because ignore_older reached. File %s", fe.NewPath) + break + } + } + + hg.Run(ctx, src) + + case loginp.OpWrite: + log.Debugf("File %s has been updated", fe.NewPath) + + hg.Run(ctx, src) + + case loginp.OpDelete: + log.Debugf("File %s has been removed", fe.OldPath) + + if p.cleanRemoved { + log.Debugf("Remove state for file as file removed: %s", fe.OldPath) + + err := s.Remove(src.Name()) + if err != nil { + log.Errorf("Error while removing state from statestore: %v", err) + } + } + + case loginp.OpRename: + log.Debugf("File %s has been renamed to %s", fe.OldPath, fe.NewPath) + // TODO update state information in the store + + default: + log.Error("Unkown return value %v", fe.Op) + } + } + return nil + }) + + errs := tg.Wait() + if len(errs) > 0 { + log.Error("%s", sderr.WrapAll(errs, "running prospector failed")) + } +} + +func (p *fileProspector) cleanRemovedBetweenRuns(log *logp.Logger, s *statestore.Store) { + keyPrefix := pluginName + "::" + s.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + if !strings.HasPrefix(string(key), keyPrefix) { + return true, nil + } + + var st state + if err := dec.Decode(&st); err != nil { + log.Errorf("Failed to read regisry state for '%v', cursor state will be ignored. Error was: %+v", + key, err) + return true, nil + } + + _, err := os.Stat(st.Source) + if err != nil { + s.Remove(key) + } + + return true, nil + }) +} + +func (p *fileProspector) updateIdentifiersBetweenRuns(log *logp.Logger, s *statestore.Store) { + keyPrefix := pluginName + "::" + s.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + if !strings.HasPrefix(string(key), keyPrefix) { + return true, nil + } + + var st state + if err := dec.Decode(&st); err != nil { + log.Errorf("Failed to read regisry state for '%v', cursor state will be ignored. Error was: %+v", key, err) + return true, nil + } + + if st.IdentifierName == p.identifier.Name() { + return true, nil + } + + fi, err := os.Stat(st.Source) + if err != nil { + return true, nil + } + newKey := p.identifier.GetSource(loginp.FSEvent{NewPath: st.Source, Info: fi}).Name() + st.IdentifierName = p.identifier.Name() + + err = s.Set(newKey, st) + if err != nil { + log.Errorf("Failed to add updated state for '%v', cursor state will be ignored. Error was: %+v", key, err) + return true, nil + } + s.Remove(key) + + return true, nil + }) } func (p *fileProspector) Test() error { diff --git a/filebeat/module/apache/access/config/access.yml b/filebeat/module/apache/access/config/access.yml index 183de6298673..b39221031f39 100644 --- a/filebeat/module/apache/access/config/access.yml +++ b/filebeat/module/apache/access/config/access.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/apache/error/config/error.yml b/filebeat/module/apache/error/config/error.yml index 7aa0cdb7cf82..b8aa75eef7c4 100644 --- a/filebeat/module/apache/error/config/error.yml +++ b/filebeat/module/apache/error/config/error.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/auditd/log/config/log.yml b/filebeat/module/auditd/log/config/log.yml index 183de6298673..b39221031f39 100644 --- a/filebeat/module/auditd/log/config/log.yml +++ b/filebeat/module/auditd/log/config/log.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml index 7aa0cdb7cf82..b8aa75eef7c4 100644 --- a/filebeat/module/elasticsearch/audit/config/audit.yml +++ b/filebeat/module/elasticsearch/audit/config/audit.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/deprecation/config/log.yml b/filebeat/module/elasticsearch/deprecation/config/log.yml index f6c37b9e426d..aa6f7d220f06 100644 --- a/filebeat/module/elasticsearch/deprecation/config/log.yml +++ b/filebeat/module/elasticsearch/deprecation/config/log.yml @@ -15,4 +15,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/gc/config/gc.yml b/filebeat/module/elasticsearch/gc/config/gc.yml index 5f62e00c54c9..12b9a0f874df 100644 --- a/filebeat/module/elasticsearch/gc/config/gc.yml +++ b/filebeat/module/elasticsearch/gc/config/gc.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/server/config/log.yml b/filebeat/module/elasticsearch/server/config/log.yml index b0b9e3f55d01..45a7f84018db 100644 --- a/filebeat/module/elasticsearch/server/config/log.yml +++ b/filebeat/module/elasticsearch/server/config/log.yml @@ -15,4 +15,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml index 2d98286de6dd..4c861ea7dc7f 100644 --- a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml +++ b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml @@ -16,4 +16,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/haproxy/log/config/file.yml b/filebeat/module/haproxy/log/config/file.yml index c3e104f56d4d..ac3846087c31 100644 --- a/filebeat/module/haproxy/log/config/file.yml +++ b/filebeat/module/haproxy/log/config/file.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/haproxy/log/config/syslog.yml b/filebeat/module/haproxy/log/config/syslog.yml index 6fa56f8fe3c8..e919d732e74c 100644 --- a/filebeat/module/haproxy/log/config/syslog.yml +++ b/filebeat/module/haproxy/log/config/syslog.yml @@ -6,4 +6,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/icinga/debug/config/debug.yml b/filebeat/module/icinga/debug/config/debug.yml index 33c478508786..8b5c2ae7422d 100644 --- a/filebeat/module/icinga/debug/config/debug.yml +++ b/filebeat/module/icinga/debug/config/debug.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/icinga/main/config/main.yml b/filebeat/module/icinga/main/config/main.yml index 33c478508786..8b5c2ae7422d 100644 --- a/filebeat/module/icinga/main/config/main.yml +++ b/filebeat/module/icinga/main/config/main.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/icinga/startup/config/startup.yml b/filebeat/module/icinga/startup/config/startup.yml index a69343bd7969..acc8bd2b782d 100644 --- a/filebeat/module/icinga/startup/config/startup.yml +++ b/filebeat/module/icinga/startup/config/startup.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/iis/access/config/iis-access.yml b/filebeat/module/iis/access/config/iis-access.yml index b47970393973..2c845646a4e7 100644 --- a/filebeat/module/iis/access/config/iis-access.yml +++ b/filebeat/module/iis/access/config/iis-access.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/iis/error/config/iis-error.yml b/filebeat/module/iis/error/config/iis-error.yml index b47970393973..2c845646a4e7 100644 --- a/filebeat/module/iis/error/config/iis-error.yml +++ b/filebeat/module/iis/error/config/iis-error.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/kafka/log/config/log.yml b/filebeat/module/kafka/log/config/log.yml index a745c79562c9..f3cfc6875268 100644 --- a/filebeat/module/kafka/log/config/log.yml +++ b/filebeat/module/kafka/log/config/log.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/kibana/log/config/log.yml b/filebeat/module/kibana/log/config/log.yml index 1ad204c62fec..93daa1167916 100644 --- a/filebeat/module/kibana/log/config/log.yml +++ b/filebeat/module/kibana/log/config/log.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/logstash/log/config/log.yml b/filebeat/module/logstash/log/config/log.yml index af0e4c337356..407078dc1154 100644 --- a/filebeat/module/logstash/log/config/log.yml +++ b/filebeat/module/logstash/log/config/log.yml @@ -16,4 +16,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/logstash/slowlog/config/slowlog.yml b/filebeat/module/logstash/slowlog/config/slowlog.yml index 080a2c4310df..4c6abc278c69 100644 --- a/filebeat/module/logstash/slowlog/config/slowlog.yml +++ b/filebeat/module/logstash/slowlog/config/slowlog.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/mongodb/log/config/log.yml b/filebeat/module/mongodb/log/config/log.yml index 183de6298673..b39221031f39 100644 --- a/filebeat/module/mongodb/log/config/log.yml +++ b/filebeat/module/mongodb/log/config/log.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/mysql/error/config/error.yml b/filebeat/module/mysql/error/config/error.yml index 818cbab297da..8ceba377810c 100644 --- a/filebeat/module/mysql/error/config/error.yml +++ b/filebeat/module/mysql/error/config/error.yml @@ -16,4 +16,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml index 7b1a3bc28fd5..6893491d869b 100644 --- a/filebeat/module/mysql/slowlog/config/slowlog.yml +++ b/filebeat/module/mysql/slowlog/config/slowlog.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nats/log/config/log.yml b/filebeat/module/nats/log/config/log.yml index 183de6298673..b39221031f39 100644 --- a/filebeat/module/nats/log/config/log.yml +++ b/filebeat/module/nats/log/config/log.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml index 7aa0cdb7cf82..b8aa75eef7c4 100644 --- a/filebeat/module/nginx/access/config/nginx-access.yml +++ b/filebeat/module/nginx/access/config/nginx-access.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml index 797c45f6c384..173a43c5a1e1 100644 --- a/filebeat/module/nginx/error/config/nginx-error.yml +++ b/filebeat/module/nginx/error/config/nginx-error.yml @@ -14,4 +14,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml b/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml index 7aa0cdb7cf82..b8aa75eef7c4 100644 --- a/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml +++ b/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/postgresql/log/config/log.yml b/filebeat/module/postgresql/log/config/log.yml index f69d3e4d387a..eeac43780bfc 100644 --- a/filebeat/module/postgresql/log/config/log.yml +++ b/filebeat/module/postgresql/log/config/log.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/redis/log/config/log.yml b/filebeat/module/redis/log/config/log.yml index 9d6cae46b383..1c6b6d6c0843 100644 --- a/filebeat/module/redis/log/config/log.yml +++ b/filebeat/module/redis/log/config/log.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 183de6298673..b39221031f39 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/libbeat/common/transport/tlscommon/tls.go b/libbeat/common/transport/tlscommon/tls.go index 3616a9f07e44..ba44310727c5 100644 --- a/libbeat/common/transport/tlscommon/tls.go +++ b/libbeat/common/transport/tlscommon/tls.go @@ -66,7 +66,7 @@ func LoadCertificate(config *CertificateConfig) (*tls.Certificate, error) { return nil, err } - log.Debugf("tls", "loading certificate: %v and key %v", certificate, key) + log.Debugf("Loading certificate: %v and key %v", certificate, key) return &cert, nil } @@ -169,7 +169,7 @@ func LoadCertificateAuthorities(CAs []string) (*x509.CertPool, []error) { errors = append(errors, fmt.Errorf("%v adding %v to the list of known CAs", ErrNotACertificate, r)) continue } - log.Debugf("tls", "successfully loaded CA certificate: %v", r) + log.Debugf("Successfully loaded CA certificate: %v", r) } return roots, errors diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go index 9cd3eba55b8e..3028d531c1e5 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -34,6 +34,9 @@ var azureVMMetadataFetcher = provider{ azHeaders := map[string]string{"Metadata": "true"} azSchema := func(m map[string]interface{}) common.MapStr { out, _ := s.Schema{ + "account": s.Object{ + "id": c.Str("subscriptionId"), + }, "instance": s.Object{ "id": c.Str("vmId"), "name": c.Str("name"), diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go index 307ac60abada..a988cc8873f8 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -40,7 +40,8 @@ const azInstanceIdentityDocument = `{ "sku": "14.04.4-LTS", "version": "14.04.201605091", "vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", - "vmSize": "Standard_D3_v2" + "vmSize": "Standard_D3_v2", + "subscriptionId": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e" }` func initAzureTestServer() *httptest.Server { @@ -87,6 +88,9 @@ func TestRetrieveAzureMetadata(t *testing.T) { "machine": common.MapStr{ "type": "Standard_D3_v2", }, + "account": common.MapStr{ + "id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e", + }, "region": "eastus2", }, } diff --git a/libbeat/publisher/queue/diskqueue/config.go b/libbeat/publisher/queue/diskqueue/config.go index 6a165a665db2..b8ef456d03d4 100644 --- a/libbeat/publisher/queue/diskqueue/config.go +++ b/libbeat/publisher/queue/diskqueue/config.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "path/filepath" + "time" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgtype" @@ -61,16 +62,26 @@ type Settings struct { // A listener that should be sent ACKs when an event is successfully // written to disk. WriteToDiskListener queue.ACKListener + + // RetryInterval specifies how long to wait before retrying a fatal error + // writing to disk. If MaxRetryInterval is nonzero, subsequent retries will + // use exponential backoff up to the specified limit. + RetryInterval time.Duration + MaxRetryInterval time.Duration } // userConfig holds the parameters for a disk queue that are configurable // by the end user in the beats yml file. type userConfig struct { - Path string `config:"path"` - MaxSize cfgtype.ByteSize `config:"max_size" validate:"required"` - SegmentSize *cfgtype.ByteSize `config:"segment_size"` - ReadAheadLimit *int `config:"read_ahead"` - WriteAheadLimit *int `config:"write_ahead"` + Path string `config:"path"` + MaxSize cfgtype.ByteSize `config:"max_size" validate:"required"` + SegmentSize *cfgtype.ByteSize `config:"segment_size"` + + ReadAheadLimit *int `config:"read_ahead"` + WriteAheadLimit *int `config:"write_ahead"` + + RetryInterval *time.Duration `config:"retry_interval" validate:"positive"` + MaxRetryInterval *time.Duration `config:"max_retry_interval" validate:"positive"` } func (c *userConfig) Validate() error { @@ -96,6 +107,13 @@ func (c *userConfig) Validate() error { "Disk queue segment_size (%d) cannot be less than 1MB", *c.SegmentSize) } + if c.RetryInterval != nil && c.MaxRetryInterval != nil && + *c.MaxRetryInterval < *c.RetryInterval { + return fmt.Errorf( + "Disk queue max_retry_interval (%v) can't be less than retry_interval (%v)", + *c.MaxRetryInterval, *c.RetryInterval) + } + return nil } @@ -108,6 +126,9 @@ func DefaultSettings() Settings { ReadAheadLimit: 512, WriteAheadLimit: 2048, + + RetryInterval: 1 * time.Second, + MaxRetryInterval: 30 * time.Second, } } @@ -137,6 +158,13 @@ func SettingsForUserConfig(config *common.Config) (Settings, error) { settings.WriteAheadLimit = *userConfig.WriteAheadLimit } + if userConfig.RetryInterval != nil { + settings.RetryInterval = *userConfig.RetryInterval + } + if userConfig.MaxRetryInterval != nil { + settings.MaxRetryInterval = *userConfig.RetryInterval + } + return settings, nil } @@ -164,3 +192,17 @@ func (settings Settings) segmentPath(segmentID segmentID) string { func (settings Settings) maxSegmentOffset() segmentOffset { return segmentOffset(settings.MaxSegmentSize - segmentHeaderSize) } + +// Given a retry interval, nextRetryInterval returns the next higher level +// of backoff. +func (settings Settings) nextRetryInterval( + currentInterval time.Duration, +) time.Duration { + if settings.MaxRetryInterval > 0 { + currentInterval *= 2 + if currentInterval > settings.MaxRetryInterval { + currentInterval = settings.MaxRetryInterval + } + } + return currentInterval +} diff --git a/libbeat/publisher/queue/diskqueue/deleter_loop.go b/libbeat/publisher/queue/diskqueue/deleter_loop.go index 4e6852859487..3948d200cbca 100644 --- a/libbeat/publisher/queue/diskqueue/deleter_loop.go +++ b/libbeat/publisher/queue/diskqueue/deleter_loop.go @@ -57,6 +57,7 @@ func newDeleterLoop(settings Settings) *deleterLoop { } func (dl *deleterLoop) run() { + currentRetryInterval := dl.settings.RetryInterval for { request, ok := <-dl.requestChan if !ok { @@ -87,10 +88,14 @@ func (dl *deleterLoop) run() { // The delay can be interrupted if the request channel is closed, // indicating queue shutdown. select { - // TODO: make the retry interval configurable. - case <-time.After(time.Second): + case <-time.After(currentRetryInterval): case <-dl.requestChan: } + currentRetryInterval = + dl.settings.nextRetryInterval(currentRetryInterval) + } else { + // If we made progress, reset the retry interval. + currentRetryInterval = dl.settings.RetryInterval } dl.responseChan <- deleterLoopResponse{ results: results, diff --git a/libbeat/publisher/queue/diskqueue/segments.go b/libbeat/publisher/queue/diskqueue/segments.go index 5ce0dc499627..617b089110ed 100644 --- a/libbeat/publisher/queue/diskqueue/segments.go +++ b/libbeat/publisher/queue/diskqueue/segments.go @@ -207,10 +207,15 @@ func (segment *queueSegment) getWriter( // retry callback returns true. This is used for timed retries when // creating a queue segment from the writer loop. func (segment *queueSegment) getWriterWithRetry( - queueSettings Settings, retry func(error) bool, + queueSettings Settings, retry func(err error, firstTime bool) bool, ) (*os.File, error) { + firstTime := true file, err := segment.getWriter(queueSettings) - for err != nil && retry(err) { + for err != nil && retry(err, firstTime) { + // Set firstTime to false so the retry callback can perform backoff + // etc if needed. + firstTime = false + // Try again file, err = segment.getWriter(queueSettings) } diff --git a/libbeat/publisher/queue/diskqueue/util.go b/libbeat/publisher/queue/diskqueue/util.go index 60c529a99921..c54a26154e81 100644 --- a/libbeat/publisher/queue/diskqueue/util.go +++ b/libbeat/publisher/queue/diskqueue/util.go @@ -69,16 +69,32 @@ func writeErrorIsRetriable(err error) bool { // "wrapped" field in-place as long as it isn't captured by the callback. type callbackRetryWriter struct { wrapped io.Writer - retry func(error) bool + + // The retry callback is called with the error that was produced and whether + // this is the first (subsequent) error arising from this particular + // write call. + retry func(err error, firstTime bool) bool } func (w callbackRetryWriter) Write(p []byte) (int, error) { + // firstTime tracks whether the current error is the first subsequent error + // being passed to the retry callback. This is so that the callback can + // reset its internal counters in case it is using exponential backoff or + // a retry limit. + firstTime := true bytesWritten := 0 writer := w.wrapped n, err := writer.Write(p) for n < len(p) { - if err != nil && !w.retry(err) { - return bytesWritten + n, err + if err != nil { + shouldRetry := w.retry(err, firstTime) + firstTime = false + if !shouldRetry { + return bytesWritten + n, err + } + } else { + // If we made progress without an error, reset firstTime. + firstTime = true } // Advance p and try again. bytesWritten += n diff --git a/libbeat/publisher/queue/diskqueue/writer_loop.go b/libbeat/publisher/queue/diskqueue/writer_loop.go index b42e4573caba..ff1ff97616a6 100644 --- a/libbeat/publisher/queue/diskqueue/writer_loop.go +++ b/libbeat/publisher/queue/diskqueue/writer_loop.go @@ -82,6 +82,8 @@ type writerLoop struct { // The file handle corresponding to currentSegment. When currentSegment // changes, this handle is closed and a new one is created. outputFile *os.File + + currentRetryInterval time.Duration } func newWriterLoop(logger *logp.Logger, settings Settings) *writerLoop { @@ -91,6 +93,8 @@ func newWriterLoop(logger *logp.Logger, settings Settings) *writerLoop { requestChan: make(chan writerLoopRequest, 1), responseChan: make(chan writerLoopResponse), + + currentRetryInterval: settings.RetryInterval, } } @@ -215,14 +219,31 @@ outerLoop: return append(bytesWritten, curBytesWritten) } -// retryCallback is called (by way of retryCallbackWriter) when there is +func (wl *writerLoop) applyRetryBackoff() { + wl.currentRetryInterval = + wl.settings.nextRetryInterval(wl.currentRetryInterval) +} + +func (wl *writerLoop) resetRetryBackoff() { + wl.currentRetryInterval = wl.settings.RetryInterval +} + +// retryCallback is called (by way of callbackRetryWriter) when there is // an error writing to a segment file. It pauses for a configurable // interval and returns true if the operation should be retried (which // it always should, unless the queue is being closed). -func (wl *writerLoop) retryCallback(err error) bool { +func (wl *writerLoop) retryCallback(err error, firstTime bool) bool { + if firstTime { + // Reset any exponential backoff in the retry interval. + wl.resetRetryBackoff() + } if writeErrorIsRetriable(err) { return true } + // If this error isn't immediately retriable, increase the exponential + // backoff afterwards. + defer wl.applyRetryBackoff() + // If the error is not immediately retriable, log the error // and wait for the retry interval before trying again, but // abort if the queue is closed (indicated by the request channel @@ -230,8 +251,7 @@ func (wl *writerLoop) retryCallback(err error) bool { wl.logger.Errorf("Writing to segment %v: %v", wl.currentSegment.id, err) select { - case <-time.After(time.Second): - // TODO: use a configurable interval here + case <-time.After(wl.currentRetryInterval): return true case <-wl.requestChan: return false diff --git a/libbeat/tests/system/beat/common_tests.py b/libbeat/tests/system/beat/common_tests.py index c9cdbc52cc0f..920ee35e72e2 100644 --- a/libbeat/tests/system/beat/common_tests.py +++ b/libbeat/tests/system/beat/common_tests.py @@ -4,6 +4,12 @@ from beat.beat import INTEGRATION_TESTS +# Fail if the exported index pattern is larger than 10MiB +# This is to avoid problems with Kibana when the payload +# of the request to install the index pattern exceeds the +# default limit. +index_pattern_size_limit = 10 * 1024 * 1024 + class TestExportsMixin: @@ -56,7 +62,7 @@ def test_export_index_pattern(self): js = json.loads(output) assert "objects" in js size = len(output.encode('utf-8')) - assert size < 1024 * 1024, "Kibana index pattern must be less than 1MiB " \ + assert size < index_pattern_size_limit, "Kibana index pattern must be less than 10MiB " \ "to keep the Beat setup request size below " \ "Kibana's server.maxPayloadBytes." @@ -68,7 +74,7 @@ def test_export_index_pattern_migration(self): js = json.loads(output) assert "objects" in js size = len(output.encode('utf-8')) - assert size < 1024 * 1024, "Kibana index pattern must be less than 1MiB " \ + assert size < index_pattern_size_limit, "Kibana index pattern must be less than 10MiB " \ "to keep the Beat setup request size below " \ "Kibana's server.maxPayloadBytes." diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ae34419db2e7..26e83c190502 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4665,6 +4665,16 @@ type: keyword The subscription ID +type: keyword + +-- + +*`azure.application_id`*:: ++ +-- +The application ID + + type: keyword -- @@ -4696,17 +4706,44 @@ application insights -*`azure.app_insights.application_id`*:: +*`azure.app_insights.start_date`*:: + -- -The application ID +The start date -type: keyword +type: date -- -*`azure.app_insights.start_date`*:: +*`azure.app_insights.end_date`*:: ++ +-- +The end date + + +type: date + +-- + +*`azure.app_insights.metrics.*.*`*:: ++ +-- +The metrics + + +type: object + +-- + +[float] +=== app_state + +application state + + + +*`azure.app_state.start_date`*:: + -- The start date @@ -4716,7 +4753,7 @@ type: date -- -*`azure.app_insights.end_date`*:: +*`azure.app_state.end_date`*:: + -- The end date @@ -4726,13 +4763,183 @@ type: date -- -*`azure.app_insights.metrics.*.*`*:: +*`azure.app_state.requests_count.sum`*:: + -- -The metrics +Request count -type: object +type: float + +-- + +*`azure.app_state.requests_failed.sum`*:: ++ +-- +Request failed count + + +type: float + +-- + +*`azure.app_state.users_count.unique`*:: ++ +-- +User count + + +type: float + +-- + +*`azure.app_state.sessions_count.unique`*:: ++ +-- +Session count + + +type: float + +-- + +*`azure.app_state.users_authenticated.unique`*:: ++ +-- +Authenticated users count + + +type: float + +-- + +*`azure.app_state.browser_timings_network_duration.avg`*:: ++ +-- +Browser timings network duration + + +type: float + +-- + +*`azure.app_state.browser_timings_send_duration.avg`*:: ++ +-- +Browser timings send duration + + +type: float + +-- + +*`azure.app_state.browser_timings_receive_uration.avg`*:: ++ +-- +Browser timings receive duration + + +type: float + +-- + +*`azure.app_state.browser_timings_processing_duration.avg`*:: ++ +-- +Browser timings processing duration + + +type: float + +-- + +*`azure.app_state.browser_timings_total_duration.avg`*:: ++ +-- +Browser timings total duration + + +type: float + +-- + +*`azure.app_state.exceptions_count.sum`*:: ++ +-- +Exception count + + +type: float + +-- + +*`azure.app_state.exceptions_browser.sum`*:: ++ +-- +Exception count at browser level + + +type: float + +-- + +*`azure.app_state.exceptions_server.sum`*:: ++ +-- +Exception count at server level + + +type: float + +-- + +*`azure.app_state.performance_counters_memory_available_bytes.avg`*:: ++ +-- +Performance counters memory available bytes + + +type: float + +-- + +*`azure.app_state.performance_counters_process_private_bytes.avg`*:: ++ +-- +Performance counters process private bytes + + +type: float + +-- + +*`azure.app_state.performance_counters_process_cpu_percentage_total.avg`*:: ++ +-- +Performance counters process cpu percentage total + + +type: float + +-- + +*`azure.app_state.performance_counters_process_cpu_percentage.avg`*:: ++ +-- +Performance counters process cpu percentage + + +type: float + +-- + +*`azure.app_state.performance_counters_processiobytes_per_second.avg`*:: ++ +-- +Performance counters process IO bytes per second + + +type: float -- diff --git a/metricbeat/docs/images/metricbeat-azure-app-state-overview.png b/metricbeat/docs/images/metricbeat-azure-app-state-overview.png new file mode 100644 index 000000000000..cffc6e4ef038 Binary files /dev/null and b/metricbeat/docs/images/metricbeat-azure-app-state-overview.png differ diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 42d24c65ccd8..e02a7a814600 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -19,16 +19,27 @@ module. Please see <> for more details. [float] == Module-specific configuration notes +* *AWS Credentials* + The `aws` module requires AWS credentials configuration in order to make AWS API calls. Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN`, or use shared AWS credentials file. Please see <> for more details. +* *regions* + This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available AWS regions. +* *latency* + +Some AWS services send monitoring metrics to CloudWatch with a latency to +process larger than Metricbeat collection period. This case, please specify a +`latency` parameter so collection start time and end time will be shifted by the +given latency amount. + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index 4db38120041e..42d4d619c028 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -45,6 +45,10 @@ The Azure billing dashboards show relevant usage and forecast information: image::./images/metricbeat-azure-billing-overview.png[] +The Azure app_state dashboard shows relevant application insights information: + +image::./images/metricbeat-azure-app-state-overview.png[] + [float] === Module-specific configuration notes @@ -120,6 +124,10 @@ so the `period` for `billing` metricset should be `24h` or multiples of `24h`. === `app_insights` This metricset will collect application insights metrics, the `period` (interval) for the `app-insights` metricset is set by default at `300s`. +[float] +=== `app_state` +This metricset concentrate on the most relevant application insights metrics and provides a dashboard for visualization, the `period` (interval) for the `app_state` metricset is set by default at `300s`. + [float] [[azure-api-cost]] == Additional notes about metrics and costs @@ -242,6 +250,14 @@ metricbeat.modules: api_key: '' metrics: - id: ["requests/count", "requests/duration"] + +- module: azure + metricsets: + - app_state + enabled: true + period: 300s + application_id: '' + api_key: '' ---- [float] @@ -251,6 +267,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -271,6 +289,8 @@ The following metricsets are available: include::azure/app_insights.asciidoc[] +include::azure/app_state.asciidoc[] + include::azure/billing.asciidoc[] include::azure/compute_vm.asciidoc[] diff --git a/metricbeat/docs/modules/azure/app_state.asciidoc b/metricbeat/docs/modules/azure/app_state.asciidoc new file mode 100644 index 000000000000..10485dc818ea --- /dev/null +++ b/metricbeat/docs/modules/azure/app_state.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-app_state]] +[role="xpack"] +=== Azure app_state metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/app_state/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 2232cf3b0703..949657459dba 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -33,7 +33,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.10+| .10+| |<> beta[] +.11+| .11+| |<> beta[] +|<> beta[] |<> beta[] |<> |<> diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 786977cb2941..f852c153e5e8 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -192,11 +192,6 @@ $ kubectl --namespace=kube-system get ds/metricbeat NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE-SELECTOR AGE metricbeat 32 32 0 32 0 1m - -$ kubectl --namespace=kube-system get deploy/metricbeat - -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -metricbeat 1 1 1 1 1m ------------------------------------------------ Metrics should start flowing to Elasticsearch. diff --git a/metricbeat/module/kafka/broker.go b/metricbeat/module/kafka/broker.go index b9e78c7dacc2..2e558d7944a6 100644 --- a/metricbeat/module/kafka/broker.go +++ b/metricbeat/module/kafka/broker.go @@ -119,7 +119,7 @@ func (b *Broker) Connect() error { c, err := getClusterWideClient(b.Addr(), b.cfg) if err != nil { closeBroker(b.broker) - return fmt.Errorf("Could not get cluster client for advertised broker with address %v", b.Addr()) + return fmt.Errorf("getting cluster client for advertised broker with address %v: %w", b.Addr(), err) } b.client = c diff --git a/packetbeat/cmd/root.go b/packetbeat/cmd/root.go index 82ab41da374c..0a22e98a1a08 100644 --- a/packetbeat/cmd/root.go +++ b/packetbeat/cmd/root.go @@ -37,7 +37,7 @@ const ( Name = "packetbeat" // ecsVersion specifies the version of ECS that Packetbeat is implementing. - ecsVersion = "1.5.0" + ecsVersion = "1.6.0" ) // withECSVersion is a modifier that adds ecs.version to events. diff --git a/winlogbeat/cmd/root.go b/winlogbeat/cmd/root.go index 2cd26a9fe8e9..988ac6f9f6c2 100644 --- a/winlogbeat/cmd/root.go +++ b/winlogbeat/cmd/root.go @@ -37,7 +37,7 @@ const ( Name = "winlogbeat" // ecsVersion specifies the version of ECS that Winlogbeat is implementing. - ecsVersion = "1.5.0" + ecsVersion = "1.6.0" ) // withECSVersion is a modifier that adds ecs.version to events. diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 42c1a5d8cb47..969aa0a8e08c 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -5,11 +5,11 @@ when: - "@ci" ## special token regarding the changeset for the ci - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - - "/test auditbeat" + - "/test x-pack/auditbeat" labels: ## when PR labels matches any of those entries - - "auditbeat" + - "x-pack-auditbeat" parameters: ## when parameter was selected in the UI. - - "auditbeat" + - "x-pack-auditbeat" tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: @@ -22,7 +22,7 @@ stages: - "macosx" when: ## Override the top-level when. comments: - - "/test auditbeat for macos" + - "/test x-pack/auditbeat for macos" labels: - "macOS" parameters: diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 2ba08864ae85..278a9ea9cf45 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -28,3 +28,4 @@ - Add support for EQL based condition on inputs {pull}20994[20994] - Send `fleet.host.id` to Endpoint Security {pull}21042[21042] - Add `install` and `uninstall` subcommands {pull}21206[21206] +- Send updating state {pull}21461[21461] diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index 12a9c2427808..d1eaf197a886 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -200,11 +200,13 @@ func newManaged( } managedApplication.upgrader = upgrade.NewUpgrader( + agentInfo, cfg.Settings.DownloadConfig, log, []context.CancelFunc{managedApplication.cancelCtxFn}, reexec, - acker) + acker, + combinedReporter) actionDispatcher.MustRegister( &fleetapi.ActionPolicyChange{}, diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go index 28e93949fbf4..9db442d3655b 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go @@ -31,7 +31,7 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri return "", errors.New(err, "failed upgrade of agent binary") } - matches, err := verifier.Verify(agentName, version) + matches, err := verifier.Verify(agentName, version, agentArtifactName) if err != nil { return "", errors.New(err, "failed verification of agent binary") } diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go index cac36ef7922a..7aacf77ba634 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go @@ -20,6 +20,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -33,11 +34,13 @@ const ( // Upgrader performs an upgrade type Upgrader struct { + agentInfo *info.AgentInfo settings *artifact.Config log *logger.Logger closers []context.CancelFunc reexec reexecManager acker acker + reporter stateReporter upgradeable bool } @@ -50,14 +53,19 @@ type acker interface { Commit(ctx context.Context) error } +type stateReporter interface { + OnStateChange(id string, name string, s state.State) +} + // NewUpgrader creates an upgrader which is capable of performing upgrade operation -func NewUpgrader(settings *artifact.Config, log *logger.Logger, closers []context.CancelFunc, reexec reexecManager, a acker) *Upgrader { +func NewUpgrader(agentInfo *info.AgentInfo, settings *artifact.Config, log *logger.Logger, closers []context.CancelFunc, reexec reexecManager, a acker, r stateReporter) *Upgrader { return &Upgrader{ settings: settings, log: log, closers: closers, reexec: reexec, acker: a, + reporter: r, upgradeable: getUpgradable(), } } @@ -68,13 +76,22 @@ func (u *Upgrader) Upgradeable() bool { } // Upgrade upgrades running agent -func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error { +func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) (err error) { + // report failed + defer func() { + if err != nil { + u.reportFailure(ctx, a, err) + } + }() + if !u.upgradeable { return fmt.Errorf( "cannot be upgraded; must be installed with install sub-command and " + "running under control of the systems supervisor") } + u.reportUpdating(a.Version) + sourceURI, err := u.sourceURI(a.Version, a.SourceURI) archivePath, err := u.downloadArtifact(ctx, a.Version, sourceURI) if err != nil { @@ -91,7 +108,10 @@ func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error } if strings.HasPrefix(release.Commit(), newHash) { - return errors.New("upgrading to same version") + // not an error + u.ackAction(ctx, a) + u.log.Warn("upgrading to same version") + return nil } if err := copyActionStore(newHash); err != nil { @@ -132,11 +152,7 @@ func (u *Upgrader) Ack(ctx context.Context) error { return nil } - if err := u.acker.Ack(ctx, marker.Action); err != nil { - return err - } - - if err := u.acker.Commit(ctx); err != nil { + if err := u.ackAction(ctx, marker.Action); err != nil { return err } @@ -148,6 +164,7 @@ func (u *Upgrader) Ack(ctx context.Context) error { return ioutil.WriteFile(markerFile, markerBytes, 0600) } + func (u *Upgrader) sourceURI(version, retrievedURI string) (string, error) { if strings.HasSuffix(version, "-SNAPSHOT") && retrievedURI == "" { return "", errors.New("snapshot upgrade requires source uri", errors.TypeConfig) @@ -159,6 +176,50 @@ func (u *Upgrader) sourceURI(version, retrievedURI string) (string, error) { return u.settings.SourceURI, nil } +// ackAction is used for successful updates, it was either updated successfully or to the same version +// so we need to remove updating state and get prevent from receiving same update action again. +func (u *Upgrader) ackAction(ctx context.Context, action fleetapi.Action) error { + if err := u.acker.Ack(ctx, action); err != nil { + return err + } + + if err := u.acker.Commit(ctx); err != nil { + return err + } + + u.reporter.OnStateChange( + "", + agentName, + state.State{Status: state.Running}, + ) + + return nil +} + +// report failure is used when update process fails. action is acked so it won't be received again +// and state is changed to FAILED +func (u *Upgrader) reportFailure(ctx context.Context, action fleetapi.Action, err error) { + // ack action + u.acker.Ack(ctx, action) + + // report failure + u.reporter.OnStateChange( + "", + agentName, + state.State{Status: state.Failed, Message: err.Error()}, + ) +} + +// reportUpdating sets state of agent to updating. +func (u *Upgrader) reportUpdating(version string) { + // report failure + u.reporter.OnStateChange( + "", + agentName, + state.State{Status: state.Updating, Message: fmt.Sprintf("Update to version '%s' started", version)}, + ) +} + func rollbackInstall(hash string) { os.RemoveAll(filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash))) } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index 6a604554136a..a92766b9c07e 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -119,7 +119,9 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args if fromInstall { force = true } - if !force { + + // prompt only when it is not forced and is already enrolled + if !force && (cfg.Fleet != nil && cfg.Fleet.Enabled == true) { confirm, err := c.Confirm("This will replace your current settings. Do you want to continue?", true) if err != nil { return errors.New(err, "problem reading prompt response") diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index cc17733c6560..6e9b042fe929 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -143,7 +143,7 @@ var _ download.Downloader = &DummyDownloader{} type DummyVerifier struct{} -func (*DummyVerifier) Verify(p, v string) (bool, error) { +func (*DummyVerifier) Verify(p, v, _ string) (bool, error) { return true, nil } diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index fe33de852d1f..c4d895eb6eee 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -161,7 +161,7 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ ProgramSpec: program.Spec{ Name: metricsProcessName, Cmd: metricsProcessName, - Artifact: fmt.Sprintf("%s/%s", artifactPrefix, logsProcessName), + Artifact: fmt.Sprintf("%s/%s", artifactPrefix, metricsProcessName), }, Meta: map[string]interface{}{ configrequest.MetaConfigKey: mbConfig, diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go b/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go index 289693ca373a..97cf906cace3 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go @@ -66,7 +66,7 @@ func (o *operationVerify) Run(_ context.Context, application Application) (err e } }() - isVerified, err := o.verifier.Verify(o.program.BinaryName(), o.program.Version()) + isVerified, err := o.verifier.Verify(o.program.BinaryName(), o.program.Version(), o.program.ArtifactName()) if err != nil { return errors.New(err, fmt.Sprintf("operation '%s' failed to verify %s.%s", o.Name(), o.program.BinaryName(), o.program.Version()), diff --git a/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go index 33397a87e1ed..9d6c4477733c 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go @@ -29,11 +29,11 @@ func NewVerifier(verifiers ...download.Verifier) *Verifier { } // Verify checks the package from configured source. -func (e *Verifier) Verify(programName, version string) (bool, error) { +func (e *Verifier) Verify(programName, version, artifactName string) (bool, error) { var err error for _, v := range e.vv { - b, e := v.Verify(programName, version) + b, e := v.Verify(programName, version, artifactName) if e == nil { return b, nil } diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go index d934b20faefb..09462ef3f234 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go @@ -51,7 +51,7 @@ func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Veri // Verify checks downloaded package on preconfigured // location agains a key stored on elastic.co website. -func (v *Verifier) Verify(programName, version string) (bool, error) { +func (v *Verifier) Verify(programName, version, artifactName string) (bool, error) { filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch()) if err != nil { return false, errors.New(err, "retrieving package name") diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go index 4fd845482c5d..975d9ecb14d4 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go @@ -65,7 +65,7 @@ func TestFetchVerify(t *testing.T) { // first download verify should fail: // download skipped, as invalid package is prepared upfront // verify fails and cleans download - matches, err := verifier.Verify(programName, version) + matches, err := verifier.Verify(programName, version, artifactName) assert.NoError(t, err) assert.Equal(t, false, matches) @@ -88,7 +88,7 @@ func TestFetchVerify(t *testing.T) { _, err = os.Stat(hashTargetFilePath) assert.NoError(t, err) - matches, err = verifier.Verify(programName, version) + matches, err = verifier.Verify(programName, version, artifactName) assert.NoError(t, err) assert.Equal(t, true, matches) } @@ -162,7 +162,7 @@ func TestVerify(t *testing.T) { t.Fatal(err) } - isOk, err := testVerifier.Verify(beatName, version) + isOk, err := testVerifier.Verify(beatName, version, artifactName) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go index 0edb979a320e..fec1d991c880 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go @@ -110,7 +110,7 @@ func TestVerify(t *testing.T) { t.Fatal(err) } - isOk, err := testVerifier.Verify(beatName, version) + isOk, err := testVerifier.Verify(beatName, version, artifactName) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go index 9f2eacd93959..c0dfef9b30e2 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go @@ -59,7 +59,7 @@ func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Veri // Verify checks downloaded package on preconfigured // location agains a key stored on elastic.co website. -func (v *Verifier) Verify(programName, version string) (bool, error) { +func (v *Verifier) Verify(programName, version, artifactName string) (bool, error) { // TODO: think about verifying asc for prepacked beats filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch()) @@ -81,7 +81,7 @@ func (v *Verifier) Verify(programName, version string) (bool, error) { return isMatch, err } - return v.verifyAsc(programName, version) + return v.verifyAsc(programName, version, artifactName) } func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) { @@ -127,7 +127,7 @@ func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) { return expectedHash == computedHash, nil } -func (v *Verifier) verifyAsc(programName, version string) (bool, error) { +func (v *Verifier) verifyAsc(programName, version, artifactName string) (bool, error) { if len(v.pgpBytes) == 0 { // no pgp available skip verification process return true, nil @@ -143,7 +143,7 @@ func (v *Verifier) verifyAsc(programName, version string) (bool, error) { return false, errors.New(err, "retrieving package path") } - ascURI, err := v.composeURI(programName, filename) + ascURI, err := v.composeURI(filename, artifactName) if err != nil { return false, errors.New(err, "composing URI for fetching asc file", errors.TypeNetwork) } @@ -177,7 +177,7 @@ func (v *Verifier) verifyAsc(programName, version string) (bool, error) { } -func (v *Verifier) composeURI(programName, filename string) (string, error) { +func (v *Verifier) composeURI(filename, artifactName string) (string, error) { upstream := v.config.SourceURI if !strings.HasPrefix(upstream, "http") && !strings.HasPrefix(upstream, "file") && !strings.HasPrefix(upstream, "/") { // always default to https @@ -190,7 +190,7 @@ func (v *Verifier) composeURI(programName, filename string) (string, error) { return "", errors.New(err, "invalid upstream URI", errors.TypeNetwork, errors.M(errors.MetaKeyURI, upstream)) } - uri.Path = path.Join(uri.Path, "beats", programName, filename+ascSuffix) + uri.Path = path.Join(uri.Path, artifactName, filename+ascSuffix) return uri.String(), nil } diff --git a/x-pack/elastic-agent/pkg/artifact/download/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/verifier.go index 6aa4dc4abe47..491979514ead 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/verifier.go @@ -6,5 +6,5 @@ package download // Verifier is an interface verifying GPG key of a downloaded artifact type Verifier interface { - Verify(programName, version string) (bool, error) + Verify(programName, version, artifactName string) (bool, error) } diff --git a/x-pack/elastic-agent/pkg/core/state/state.go b/x-pack/elastic-agent/pkg/core/state/state.go index 6b7c8bd53dec..670cdc2a2f23 100644 --- a/x-pack/elastic-agent/pkg/core/state/state.go +++ b/x-pack/elastic-agent/pkg/core/state/state.go @@ -30,6 +30,8 @@ const ( Crashed // Restarting is status describing application is restarting. Restarting + // Updating is status describing application is updating. + Updating ) // State wraps the process state and application status. diff --git a/x-pack/elastic-agent/pkg/reporter/reporter.go b/x-pack/elastic-agent/pkg/reporter/reporter.go index c36708a837f7..3b128841b2a1 100644 --- a/x-pack/elastic-agent/pkg/reporter/reporter.go +++ b/x-pack/elastic-agent/pkg/reporter/reporter.go @@ -38,6 +38,8 @@ const ( EventSubTypeFailed = "FAILED" // EventSubTypeStopping is an event type indicating application is stopping. EventSubTypeStopping = "STOPPING" + // EventSubTypeUpdating is an event type indicating update process in progress. + EventSubTypeUpdating = "UPDATING" ) type agentInfo interface { @@ -127,6 +129,10 @@ func generateRecord(agentID string, id string, name string, s state.State) event case state.Restarting: subType = EventSubTypeStarting subTypeText = "RESTARTING" + case state.Updating: + subType = EventSubTypeUpdating + subTypeText = EventSubTypeUpdating + } err := errors.New( diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index cd8562560dac..4779b452f1d8 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" + "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" ) func Init(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin { @@ -29,5 +30,6 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 http_endpoint.Plugin(), httpjson.Plugin(log, store), o365audit.Plugin(log, store), + s3.Plugin(), } } diff --git a/x-pack/filebeat/input/s3/collector.go b/x-pack/filebeat/input/s3/collector.go new file mode 100644 index 000000000000..2976dd52a5ba --- /dev/null +++ b/x-pack/filebeat/input/s3/collector.go @@ -0,0 +1,635 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3 + +import ( + "bufio" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/awserr" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/s3iface" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/unison" +) + +type s3Collector struct { + cancellation context.Context + logger *logp.Logger + + config *config + visibilityTimeout int64 + + sqs sqsiface.ClientAPI + s3 s3iface.ClientAPI + publisher beat.Client +} + +type s3Info struct { + name string + key string + region string + arn string + expandEventListFromField string +} + +type bucket struct { + Name string `json:"name"` + Arn string `json:"arn"` +} + +type object struct { + Key string `json:"key"` +} + +type s3BucketObject struct { + bucket `json:"bucket"` + object `json:"object"` +} + +type sqsMessage struct { + Records []struct { + EventSource string `json:"eventSource"` + AwsRegion string `json:"awsRegion"` + EventName string `json:"eventName"` + S3 s3BucketObject `json:"s3"` + } `json:"Records"` +} + +type s3Context struct { + mux sync.Mutex + refs int + err error // first error witnessed or multi error + errC chan error +} + +var ( + // The maximum number of messages to return. Amazon SQS never returns more messages + // than this value (however, fewer messages might be returned). + maxNumberOfMessage uint8 = 10 + + // The duration (in seconds) for which the call waits for a message to arrive + // in the queue before returning. If a message is available, the call returns + // sooner than WaitTimeSeconds. If no messages are available and the wait time + // expires, the call returns successfully with an empty list of messages. + waitTimeSecond uint8 = 10 +) + +func (c *s3Collector) run() { + defer c.logger.Info("s3 input worker has stopped.") + c.logger.Info("s3 input worker has started.") + for c.cancellation.Err() == nil { + // receive messages from sqs + output, err := c.receiveMessage(c.sqs, c.visibilityTimeout) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + continue + } + c.logger.Error("SQS ReceiveMessageRequest failed: ", err) + continue + } + + if output == nil || len(output.Messages) == 0 { + c.logger.Debug("no message received from SQS") + continue + } + + // process messages received from sqs, get logs from s3 and create events + c.processor(c.config.QueueURL, output.Messages, c.visibilityTimeout, c.s3, c.sqs) + } +} + +func (c *s3Collector) processor(queueURL string, messages []sqs.Message, visibilityTimeout int64, svcS3 s3iface.ClientAPI, svcSQS sqsiface.ClientAPI) { + var grp unison.MultiErrGroup + numMessages := len(messages) + c.logger.Debugf("Processing %v messages", numMessages) + + // process messages received from sqs + for i := range messages { + i := i + errC := make(chan error) + grp.Go(func() (err error) { + return c.processMessage(svcS3, messages[i], errC) + }) + grp.Go(func() (err error) { + return c.processorKeepAlive(svcSQS, messages[i], queueURL, visibilityTimeout, errC) + }) + } + grp.Wait() +} + +func (c *s3Collector) processMessage(svcS3 s3iface.ClientAPI, message sqs.Message, errC chan error) error { + s3Infos, err := c.handleSQSMessage(message) + if err != nil { + c.logger.Error(fmt.Errorf("handleSQSMessage failed: %w", err)) + return err + } + c.logger.Debugf("handleSQSMessage succeed and returned %v sets of S3 log info", len(s3Infos)) + + // read from s3 object and create event for each log line + err = c.handleS3Objects(svcS3, s3Infos, errC) + if err != nil { + err = fmt.Errorf("handleS3Objects failed: %w", err) + c.logger.Error(err) + return err + } + c.logger.Debugf("handleS3Objects succeed") + return nil +} + +func (c *s3Collector) processorKeepAlive(svcSQS sqsiface.ClientAPI, message sqs.Message, queueURL string, visibilityTimeout int64, errC chan error) error { + for { + select { + case <-c.cancellation.Done(): + return nil + case err := <-errC: + if err != nil { + c.logger.Warn("Processing message failed, updating visibility timeout") + err := c.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) + if err != nil { + c.logger.Error(fmt.Errorf("SQS ChangeMessageVisibilityRequest failed: %w", err)) + } + c.logger.Infof("Message visibility timeout updated to %v", visibilityTimeout) + } else { + // When ACK done, message will be deleted. Or when message is + // not s3 ObjectCreated event related(handleSQSMessage function + // failed), it will be removed as well. + c.logger.Debug("Deleting message from SQS: ", *message.MessageId) + // only delete sqs message when errC is closed with no error + err := c.deleteMessage(queueURL, *message.ReceiptHandle, svcSQS) + if err != nil { + c.logger.Error(fmt.Errorf("deleteMessages failed: %w", err)) + } + } + return err + case <-time.After(time.Duration(visibilityTimeout/2) * time.Second): + c.logger.Warn("Half of the set visibilityTimeout passed, visibility timeout needs to be updated") + // If half of the set visibilityTimeout passed and this is + // still ongoing, then change visibility timeout. + err := c.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) + if err != nil { + c.logger.Error(fmt.Errorf("SQS ChangeMessageVisibilityRequest failed: %w", err)) + } + c.logger.Infof("Message visibility timeout updated to %v seconds", visibilityTimeout) + return err + } + } +} + +func (c *s3Collector) receiveMessage(svcSQS sqsiface.ClientAPI, visibilityTimeout int64) (*sqs.ReceiveMessageResponse, error) { + // receive messages from sqs + req := svcSQS.ReceiveMessageRequest( + &sqs.ReceiveMessageInput{ + QueueUrl: &c.config.QueueURL, + MessageAttributeNames: []string{"All"}, + MaxNumberOfMessages: awssdk.Int64(int64(maxNumberOfMessage)), + VisibilityTimeout: &visibilityTimeout, + WaitTimeSeconds: awssdk.Int64(int64(waitTimeSecond)), + }) + + // The Context will interrupt the request if the timeout expires. + sendCtx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + return req.Send(sendCtx) +} + +func (c *s3Collector) changeVisibilityTimeout(queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI, receiptHandle *string) error { + req := svcSQS.ChangeMessageVisibilityRequest(&sqs.ChangeMessageVisibilityInput{ + QueueUrl: &queueURL, + VisibilityTimeout: &visibilityTimeout, + ReceiptHandle: receiptHandle, + }) + + // The Context will interrupt the request if the timeout expires. + sendCtx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + _, err := req.Send(sendCtx) + return err +} + +func getRegionFromQueueURL(queueURL string) (string, error) { + // get region from queueURL + // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs + queueURLSplit := strings.Split(queueURL, ".") + if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" { + return queueURLSplit[1], nil + } + return "", fmt.Errorf("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}") +} + +// handle message +func (c *s3Collector) handleSQSMessage(m sqs.Message) ([]s3Info, error) { + msg := sqsMessage{} + err := json.Unmarshal([]byte(*m.Body), &msg) + if err != nil { + return nil, fmt.Errorf("json unmarshal sqs message body failed: %w", err) + } + + var s3Infos []s3Info + for _, record := range msg.Records { + if record.EventSource != "aws:s3" || !strings.HasPrefix(record.EventName, "ObjectCreated:") { + return nil, fmt.Errorf("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + } + // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" + filename, err := url.QueryUnescape(record.S3.object.Key) + if err != nil { + return nil, fmt.Errorf("url.QueryUnescape failed for '%s': %w", record.S3.object.Key, err) + } + + if len(c.config.FileSelectors) == 0 { + s3Infos = append(s3Infos, s3Info{ + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: c.config.ExpandEventListFromField, + }) + continue + } + + for _, fs := range c.config.FileSelectors { + if fs.Regex == nil { + continue + } + if fs.Regex.MatchString(filename) { + s3Infos = append(s3Infos, s3Info{ + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: fs.ExpandEventListFromField, + }) + break + } + } + } + return s3Infos, nil +} + +func (c *s3Collector) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC chan error) error { + s3Ctx := &s3Context{ + refs: 1, + errC: errC, + } + defer s3Ctx.done() + + for _, info := range s3Infos { + c.logger.Debugf("Processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key) + err := c.createEventsFromS3Info(svc, info, s3Ctx) + if err != nil { + err = fmt.Errorf("createEventsFromS3Info failed processing file from s3 bucket \"%s\" with name \"%s\": %w", info.name, info.key, err) + c.logger.Error(err) + s3Ctx.setError(err) + } + } + return nil +} + +func (c *s3Collector) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, s3Ctx *s3Context) error { + objectHash := s3ObjectHash(info) + + // Download the S3 object using GetObjectRequest. + s3GetObjectInput := &s3.GetObjectInput{ + Bucket: awssdk.String(info.name), + Key: awssdk.String(info.key), + } + req := svc.GetObjectRequest(s3GetObjectInput) + + // The Context will interrupt the request if the timeout expires. + ctx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + resp, err := req.Send(ctx) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + // If the SDK can determine the request or retry delay was canceled + // by a context the ErrCodeRequestCanceled error will be returned. + if awsErr.Code() == awssdk.ErrCodeRequestCanceled { + err = fmt.Errorf("s3 GetObjectRequest canceled for '%s' from S3 bucket '%s': %w", info.key, info.name, err) + c.logger.Error(err) + return err + } + + if awsErr.Code() == "NoSuchKey" { + c.logger.Warnf("Cannot find s3 file '%s' from S3 bucket '%s'", info.key, info.name) + return nil + } + } + return fmt.Errorf("s3 GetObjectRequest failed for '%s' from S3 bucket '%s': %w", info.key, info.name, err) + } + + defer resp.Body.Close() + + reader := bufio.NewReader(resp.Body) + + isS3ObjGzipped, err := isStreamGzipped(reader) + if err != nil { + err = fmt.Errorf("could not determine if S3 object is gzipped: %w", err) + c.logger.Error(err) + return err + } + + if isS3ObjGzipped { + gzipReader, err := gzip.NewReader(reader) + if err != nil { + err = fmt.Errorf("gzip.NewReader failed for '%s' from S3 bucket '%s': %w", info.key, info.name, err) + c.logger.Error(err) + return err + } + reader = bufio.NewReader(gzipReader) + gzipReader.Close() + } + + // Decode JSON documents when content-type is "application/json" or expand_event_list_from_field is given in config + if resp.ContentType != nil && *resp.ContentType == "application/json" || info.expandEventListFromField != "" { + decoder := json.NewDecoder(reader) + err := c.decodeJSON(decoder, objectHash, info, s3Ctx) + if err != nil { + err = fmt.Errorf("decodeJSONWithKey failed for '%s' from S3 bucket '%s': %w", info.key, info.name, err) + c.logger.Error(err) + return err + } + return nil + } + + // handle s3 objects that are not json content-type + offset := 0 + for { + log, err := readStringAndTrimDelimiter(reader) + if err == io.EOF { + // create event for last line + offset += len([]byte(log)) + event := createEvent(log, offset, info, objectHash, s3Ctx) + err = c.forwardEvent(event) + if err != nil { + err = fmt.Errorf("forwardEvent failed: %w", err) + c.logger.Error(err) + return err + } + return nil + } else if err != nil { + err = fmt.Errorf("readStringAndTrimDelimiter failed: %w", err) + c.logger.Error(err) + return err + } + + if log == "" { + break + } + + // create event per log line + offset += len([]byte(log)) + event := createEvent(log, offset, info, objectHash, s3Ctx) + err = c.forwardEvent(event) + if err != nil { + err = fmt.Errorf("forwardEvent failed: %w", err) + c.logger.Error(err) + return err + } + } + return nil +} + +func (c *s3Collector) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3Info, s3Ctx *s3Context) error { + offset := 0 + for { + var jsonFields interface{} + err := decoder.Decode(&jsonFields) + if jsonFields == nil { + return nil + } + + if err == io.EOF { + offsetNew, err := c.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) + if err != nil { + return err + } + offset = offsetNew + } else if err != nil { + // decode json failed, skip this log file + err = fmt.Errorf("decode json failed for '%s' from S3 bucket '%s', skipping this file: %w", s3Info.key, s3Info.name, err) + c.logger.Warn(err) + return nil + } + + offset, err = c.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) + if err != nil { + return err + } + } +} + +func (c *s3Collector) jsonFieldsType(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { + switch f := jsonFields.(type) { + case map[string][]interface{}: + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] + if !ok { + err := fmt.Errorf("key '%s' not found", s3Info.expandEventListFromField) + c.logger.Error(err) + return offset, err + } + for _, v := range textValues { + offset, err := c.convertJSONToEvent(v, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = fmt.Errorf("convertJSONToEvent failed for '%s' from S3 bucket '%s': %w", s3Info.key, s3Info.name, err) + c.logger.Error(err) + return offset, err + } + } + return offset, nil + } + case map[string]interface{}: + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] + if !ok { + err := fmt.Errorf("key '%s' not found", s3Info.expandEventListFromField) + c.logger.Error(err) + return offset, err + } + + valuesConverted := textValues.([]interface{}) + for _, textValue := range valuesConverted { + offsetNew, err := c.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = fmt.Errorf("convertJSONToEvent failed for '%s' from S3 bucket '%s': %w", s3Info.key, s3Info.name, err) + c.logger.Error(err) + return offset, err + } + offset = offsetNew + } + return offset, nil + } + + offset, err := c.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = fmt.Errorf("convertJSONToEvent failed for '%s' from S3 bucket '%s': %w", s3Info.key, s3Info.name, err) + c.logger.Error(err) + return offset, err + } + return offset, nil + } + return offset, nil +} + +func (c *s3Collector) convertJSONToEvent(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { + vJSON, _ := json.Marshal(jsonFields) + logOriginal := string(vJSON) + log := trimLogDelimiter(logOriginal) + offset += len([]byte(log)) + event := createEvent(log, offset, s3Info, objectHash, s3Ctx) + + err := c.forwardEvent(event) + if err != nil { + err = fmt.Errorf("forwardEvent failed: %w", err) + c.logger.Error(err) + return offset, err + } + return offset, nil +} + +func (c *s3Collector) forwardEvent(event beat.Event) error { + c.publisher.Publish(event) + return c.cancellation.Err() +} + +func (c *s3Collector) deleteMessage(queueURL string, messagesReceiptHandle string, svcSQS sqsiface.ClientAPI) error { + deleteMessageInput := &sqs.DeleteMessageInput{ + QueueUrl: awssdk.String(queueURL), + ReceiptHandle: awssdk.String(messagesReceiptHandle), + } + + req := svcSQS.DeleteMessageRequest(deleteMessageInput) + + // The Context will interrupt the request if the timeout expires. + ctx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + _, err := req.Send(ctx) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + return nil + } + return fmt.Errorf("SQS DeleteMessageRequest failed: %w", err) + } + return nil +} + +func trimLogDelimiter(log string) string { + return strings.TrimSuffix(log, "\n") +} + +func readStringAndTrimDelimiter(reader *bufio.Reader) (string, error) { + logOriginal, err := reader.ReadString('\n') + if err != nil { + return logOriginal, err + } + return trimLogDelimiter(logOriginal), nil +} + +func createEvent(log string, offset int, info s3Info, objectHash string, s3Ctx *s3Context) beat.Event { + s3Ctx.Inc() + + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: common.MapStr{ + "message": log, + "log": common.MapStr{ + "offset": int64(offset), + "file.path": constructObjectURL(info), + }, + "aws": common.MapStr{ + "s3": common.MapStr{ + "bucket": common.MapStr{ + "name": info.name, + "arn": info.arn}, + "object.key": info.key, + }, + }, + "cloud": common.MapStr{ + "provider": "aws", + "region": info.region, + }, + }, + Private: s3Ctx, + } + event.SetID(objectHash + "-" + fmt.Sprintf("%012d", offset)) + + return event +} + +func constructObjectURL(info s3Info) string { + return "https://" + info.name + ".s3-" + info.region + ".amazonaws.com/" + info.key +} + +// s3ObjectHash returns a short sha256 hash of the bucket arn + object key name. +func s3ObjectHash(s3Info s3Info) string { + h := sha256.New() + h.Write([]byte(s3Info.arn + s3Info.key)) + prefix := hex.EncodeToString(h.Sum(nil)) + return prefix[:10] +} + +func (c *s3Context) setError(err error) { + // only care about the last error for now + // TODO: add "Typed" error to error for context + c.mux.Lock() + defer c.mux.Unlock() + c.err = err +} + +func (c *s3Context) done() { + c.mux.Lock() + defer c.mux.Unlock() + c.refs-- + if c.refs == 0 { + c.errC <- c.err + close(c.errC) + } +} + +func (c *s3Context) Inc() { + c.mux.Lock() + defer c.mux.Unlock() + c.refs++ +} + +// isStreamGzipped determines whether the given stream of bytes (encapsulated in a buffered reader) +// represents gzipped content or not. A buffered reader is used so the function can peek into the byte +// stream without consuming it. This makes it convenient for code executed after this function call +// to consume the stream if it wants. +func isStreamGzipped(r *bufio.Reader) (bool, error) { + // Why 512? See https://godoc.org/net/http#DetectContentType + buf, err := r.Peek(512) + if err != nil && err != io.EOF { + return false, err + } + + switch http.DetectContentType(buf) { + case "application/x-gzip", "application/zip": + return true, nil + default: + return false, nil + } +} diff --git a/x-pack/filebeat/input/s3/input_test.go b/x-pack/filebeat/input/s3/collector_test.go similarity index 97% rename from x-pack/filebeat/input/s3/input_test.go rename to x-pack/filebeat/input/s3/collector_test.go index d1fab05cb3c6..510f94d40d56 100644 --- a/x-pack/filebeat/input/s3/input_test.go +++ b/x-pack/filebeat/input/s3/collector_test.go @@ -120,7 +120,7 @@ func TestHandleMessage(t *testing.T) { }, } - p := &s3Input{context: &channelContext{}} + p := &s3Collector{config: &config{}} for _, c := range casesPositive { t.Run(c.title, func(t *testing.T) { s3Info, err := p.handleSQSMessage(c.message) @@ -165,7 +165,8 @@ func TestHandleMessage(t *testing.T) { } func TestNewS3BucketReader(t *testing.T) { - p := &s3Input{context: &channelContext{}} + config := defaultConfig() + p := &s3Collector{cancellation: context.TODO(), config: &config} s3GetObjectInput := &s3.GetObjectInput{ Bucket: awssdk.String(info.name), Key: awssdk.String(info.key), @@ -174,7 +175,7 @@ func TestNewS3BucketReader(t *testing.T) { // The Context will interrupt the request if the timeout expires. var cancelFn func() - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) + ctx, cancelFn := context.WithTimeout(p.cancellation, p.config.APITimeout) defer cancelFn() resp, err := req.Send(ctx) @@ -201,7 +202,8 @@ func TestNewS3BucketReader(t *testing.T) { } func TestCreateEvent(t *testing.T) { - p := &s3Input{context: &channelContext{}} + config := defaultConfig() + p := &s3Collector{cancellation: context.TODO(), config: &config} errC := make(chan error) s3Context := &s3Context{ refs: 1, @@ -225,7 +227,7 @@ func TestCreateEvent(t *testing.T) { // The Context will interrupt the request if the timeout expires. var cancelFn func() - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) + ctx, cancelFn := context.WithTimeout(p.cancellation, p.config.APITimeout) defer cancelFn() resp, err := req.Send(ctx) diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/s3/config.go index f9780d822776..5f37a436d127 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/s3/config.go @@ -9,18 +9,16 @@ import ( "regexp" "time" - "github.com/elastic/beats/v7/filebeat/harvester" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) type config struct { - harvester.ForwarderConfig `config:",inline"` - QueueURL string `config:"queue_url" validate:"nonzero,required"` - VisibilityTimeout time.Duration `config:"visibility_timeout"` - AwsConfig awscommon.ConfigAWS `config:",inline"` - ExpandEventListFromField string `config:"expand_event_list_from_field"` - APITimeout time.Duration `config:"api_timeout"` - FileSelectors []FileSelectorCfg `config:"file_selectors"` + QueueURL string `config:"queue_url" validate:"nonzero,required"` + VisibilityTimeout time.Duration `config:"visibility_timeout"` + AwsConfig awscommon.ConfigAWS `config:",inline"` + ExpandEventListFromField string `config:"expand_event_list_from_field"` + APITimeout time.Duration `config:"api_timeout"` + FileSelectors []FileSelectorCfg `config:"file_selectors"` } // FileSelectorCfg defines type and configuration of FileSelectors @@ -32,9 +30,6 @@ type FileSelectorCfg struct { func defaultConfig() config { return config{ - ForwarderConfig: harvester.ForwarderConfig{ - Type: "s3", - }, VisibilityTimeout: 300 * time.Second, APITimeout: 120 * time.Second, } diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 83dc48428ee3..a6b56d039701 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -5,748 +5,120 @@ package s3 import ( - "bufio" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "fmt" - "io" - "net/http" - "net/url" - "strings" - "sync" - "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/awserr" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/s3iface" "github.com/aws/aws-sdk-go-v2/service/sqs" - "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/acker" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/feature" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/elastic/go-concert/ctxtool" ) const inputName = "s3" -var ( - // The maximum number of messages to return. Amazon SQS never returns more messages - // than this value (however, fewer messages might be returned). - maxNumberOfMessage int64 = 10 - - // The duration (in seconds) for which the call waits for a message to arrive - // in the queue before returning. If a message is available, the call returns - // sooner than WaitTimeSeconds. If no messages are available and the wait time - // expires, the call returns successfully with an empty list of messages. - waitTimeSecond int64 = 10 - - errOutletClosed = errors.New("input outlet closed") -) - -func init() { - err := input.Register(inputName, NewInput) - if err != nil { - panic(err) +func Plugin() v2.Plugin { + return v2.Plugin{ + Name: inputName, + Stability: feature.Beta, + Deprecated: false, + Info: "Collect logs from s3", + Manager: v2.ConfigureWith(configure), } } -// s3Input is a input for s3 -type s3Input struct { - outlet channel.Outleter // Output of received s3 logs. - config config - awsConfig awssdk.Config - logger *logp.Logger - close chan struct{} - workerOnce sync.Once // Guarantees that the worker goroutine is only started once. - context *channelContext - workerWg sync.WaitGroup // Waits on s3 worker goroutine. - stopOnce sync.Once -} - -type s3Info struct { - name string - key string - region string - arn string - expandEventListFromField string -} - -type bucket struct { - Name string `json:"name"` - Arn string `json:"arn"` -} - -type object struct { - Key string `json:"key"` -} - -type s3BucketObject struct { - bucket `json:"bucket"` - object `json:"object"` -} - -type sqsMessage struct { - Records []struct { - EventSource string `json:"eventSource"` - AwsRegion string `json:"awsRegion"` - EventName string `json:"eventName"` - S3 s3BucketObject `json:"s3"` - } `json:"Records"` -} - -type s3Context struct { - mux sync.Mutex - refs int - err error // first error witnessed or multi error - errC chan error -} - -// channelContext implements context.Context by wrapping a channel -type channelContext struct { - done <-chan struct{} -} - -func (c *channelContext) Deadline() (time.Time, bool) { return time.Time{}, false } -func (c *channelContext) Done() <-chan struct{} { return c.done } -func (c *channelContext) Err() error { - select { - case <-c.done: - return context.Canceled - default: - return nil - } -} -func (c *channelContext) Value(key interface{}) interface{} { return nil } - -// NewInput creates a new s3 input -func NewInput(cfg *common.Config, connector channel.Connector, context input.Context) (input.Input, error) { - cfgwarn.Beta("s3 input type is used") - logger := logp.NewLogger(inputName) - +func configure(cfg *common.Config) (v2.Input, error) { config := defaultConfig() if err := cfg.Unpack(&config); err != nil { - return nil, errors.Wrap(err, "failed unpacking config") - } - - out, err := connector.ConnectWith(cfg, beat.ClientConfig{ - ACKHandler: acker.ConnectionOnly( - acker.EventPrivateReporter(func(_ int, privates []interface{}) { - for _, private := range privates { - if s3Context, ok := private.(*s3Context); ok { - s3Context.done() - } - } - }), - ), - }) - if err != nil { return nil, err } - awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) - if err != nil { - return nil, errors.Wrap(err, "getAWSCredentials failed") - } - - closeChannel := make(chan struct{}) - p := &s3Input{ - outlet: out, - config: config, - awsConfig: awsConfig, - logger: logger, - close: closeChannel, - context: &channelContext{closeChannel}, - } - return p, nil -} - -// Run runs the input -func (p *s3Input) Run() { - p.workerOnce.Do(func() { - visibilityTimeout := int64(p.config.VisibilityTimeout.Seconds()) - p.logger.Infof("visibility timeout is set to %v seconds", visibilityTimeout) - p.logger.Infof("aws api timeout is set to %v", p.config.APITimeout) - - regionName, err := getRegionFromQueueURL(p.config.QueueURL) - if err != nil { - p.logger.Errorf("failed to get region name from queueURL: %v", p.config.QueueURL) - } - - awsConfig := p.awsConfig.Copy() - awsConfig.Region = regionName - - svcSQS := sqs.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "sqs", regionName, awsConfig)) - svcS3 := s3.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "s3", regionName, awsConfig)) - - p.workerWg.Add(1) - go p.run(svcSQS, svcS3, visibilityTimeout) - }) -} - -func (p *s3Input) run(svcSQS sqsiface.ClientAPI, svcS3 s3iface.ClientAPI, visibilityTimeout int64) { - defer p.workerWg.Done() - defer p.logger.Infof("s3 input worker for '%v' has stopped.", p.config.QueueURL) - - p.logger.Infof("s3 input worker has started. with queueURL: %v", p.config.QueueURL) - for p.context.Err() == nil { - // receive messages from sqs - output, err := p.receiveMessage(svcSQS, visibilityTimeout) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { - continue - } - p.logger.Error("SQS ReceiveMessageRequest failed: ", err) - time.Sleep(time.Duration(waitTimeSecond) * time.Second) - continue - } - - if output == nil || len(output.Messages) == 0 { - p.logger.Debug("no message received from SQS:", p.config.QueueURL) - continue - } - - // process messages received from sqs, get logs from s3 and create events - p.processor(p.config.QueueURL, output.Messages, visibilityTimeout, svcS3, svcSQS) - } -} - -// Stop stops the s3 input -func (p *s3Input) Stop() { - p.stopOnce.Do(func() { - defer p.outlet.Close() - close(p.close) - p.logger.Info("Stopping s3 input") - }) -} - -// Wait stops the s3 input. -func (p *s3Input) Wait() { - p.Stop() - p.workerWg.Wait() -} - -func (p *s3Input) processor(queueURL string, messages []sqs.Message, visibilityTimeout int64, svcS3 s3iface.ClientAPI, svcSQS sqsiface.ClientAPI) { - var wg sync.WaitGroup - numMessages := len(messages) - p.logger.Debugf("Processing %v messages", numMessages) - wg.Add(numMessages * 2) - - // process messages received from sqs - for i := range messages { - errC := make(chan error) - go p.processMessage(svcS3, messages[i], &wg, errC) - go p.processorKeepAlive(svcSQS, messages[i], queueURL, visibilityTimeout, &wg, errC) - } - wg.Wait() + return newInput(config) } -func (p *s3Input) processMessage(svcS3 s3iface.ClientAPI, message sqs.Message, wg *sync.WaitGroup, errC chan error) { - defer wg.Done() - - s3Infos, err := p.handleSQSMessage(message) - if err != nil { - p.logger.Error(errors.Wrap(err, "handleSQSMessage failed")) - return - } - p.logger.Debugf("handleSQSMessage succeed and returned %v sets of S3 log info", len(s3Infos)) - - // read from s3 object and create event for each log line - err = p.handleS3Objects(svcS3, s3Infos, errC) - if err != nil { - err = errors.Wrap(err, "handleS3Objects failed") - p.logger.Error(err) - return - } - p.logger.Debugf("handleS3Objects succeed") -} - -func (p *s3Input) processorKeepAlive(svcSQS sqsiface.ClientAPI, message sqs.Message, queueURL string, visibilityTimeout int64, wg *sync.WaitGroup, errC chan error) { - defer wg.Done() - for { - select { - case <-p.close: - return - case err := <-errC: - if err != nil { - p.logger.Warn("Processing message failed, updating visibility timeout") - err := p.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) - if err != nil { - p.logger.Error(errors.Wrap(err, "SQS ChangeMessageVisibilityRequest failed")) - } - p.logger.Infof("Message visibility timeout updated to %v", visibilityTimeout) - } else { - // When ACK done, message will be deleted. Or when message is - // not s3 ObjectCreated event related(handleSQSMessage function - // failed), it will be removed as well. - p.logger.Debug("Deleting message from SQS: ", *message.MessageId) - // only delete sqs message when errC is closed with no error - err := p.deleteMessage(queueURL, *message.ReceiptHandle, svcSQS) - if err != nil { - p.logger.Error(errors.Wrap(err, "deleteMessages failed")) - } - } - return - case <-time.After(time.Duration(visibilityTimeout/2) * time.Second): - p.logger.Warn("Half of the set visibilityTimeout passed, visibility timeout needs to be updated") - // If half of the set visibilityTimeout passed and this is - // still ongoing, then change visibility timeout. - err := p.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) - if err != nil { - p.logger.Error(errors.Wrap(err, "SQS ChangeMessageVisibilityRequest failed")) - } - p.logger.Infof("Message visibility timeout updated to %v seconds", visibilityTimeout) - } - } -} - -func (p *s3Input) receiveMessage(svcSQS sqsiface.ClientAPI, visibilityTimeout int64) (*sqs.ReceiveMessageResponse, error) { - // receive messages from sqs - req := svcSQS.ReceiveMessageRequest( - &sqs.ReceiveMessageInput{ - QueueUrl: &p.config.QueueURL, - MessageAttributeNames: []string{"All"}, - MaxNumberOfMessages: &maxNumberOfMessage, - VisibilityTimeout: &visibilityTimeout, - WaitTimeSeconds: &waitTimeSecond, - }) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - return req.Send(ctx) +// s3Input is a input for s3 +type s3Input struct { + config config } -func (p *s3Input) changeVisibilityTimeout(queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI, receiptHandle *string) error { - req := svcSQS.ChangeMessageVisibilityRequest(&sqs.ChangeMessageVisibilityInput{ - QueueUrl: &queueURL, - VisibilityTimeout: &visibilityTimeout, - ReceiptHandle: receiptHandle, - }) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - _, err := req.Send(ctx) - return err +func newInput(config config) (*s3Input, error) { + return &s3Input{config: config}, nil } -func getRegionFromQueueURL(queueURL string) (string, error) { - // get region from queueURL - // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs - queueURLSplit := strings.Split(queueURL, ".") - if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" { - return queueURLSplit[1], nil - } - return "", errors.New("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}") -} +func (in *s3Input) Name() string { return inputName } -// handle message -func (p *s3Input) handleSQSMessage(m sqs.Message) ([]s3Info, error) { - msg := sqsMessage{} - err := json.Unmarshal([]byte(*m.Body), &msg) +func (in *s3Input) Test(ctx v2.TestContext) error { + _, err := awscommon.GetAWSCredentials(in.config.AwsConfig) if err != nil { - return nil, errors.Wrap(err, "json unmarshal sqs message body failed") - } - - var s3Infos []s3Info - for _, record := range msg.Records { - if record.EventSource != "aws:s3" || !strings.HasPrefix(record.EventName, "ObjectCreated:") { - return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") - } - // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" - filename, err := url.QueryUnescape(record.S3.object.Key) - if err != nil { - return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) - } - - if len(p.config.FileSelectors) == 0 { - s3Infos = append(s3Infos, s3Info{ - region: record.AwsRegion, - name: record.S3.bucket.Name, - key: filename, - arn: record.S3.bucket.Arn, - expandEventListFromField: p.config.ExpandEventListFromField, - }) - continue - } - - for _, fs := range p.config.FileSelectors { - if fs.Regex == nil { - continue - } - if fs.Regex.MatchString(filename) { - s3Infos = append(s3Infos, s3Info{ - region: record.AwsRegion, - name: record.S3.bucket.Name, - key: filename, - arn: record.S3.bucket.Arn, - expandEventListFromField: fs.ExpandEventListFromField, - }) - break - } - } - } - return s3Infos, nil -} - -func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC chan error) error { - s3Ctx := &s3Context{ - refs: 1, - errC: errC, - } - defer s3Ctx.done() - - for _, info := range s3Infos { - p.logger.Debugf("Processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key) - err := p.createEventsFromS3Info(svc, info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "createEventsFromS3Info failed processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key) - p.logger.Error(err) - s3Ctx.setError(err) - } + return fmt.Errorf("getAWSCredentials failed: %w", err) } return nil } -func (p *s3Input) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, s3Ctx *s3Context) error { - objectHash := s3ObjectHash(info) - - // Download the S3 object using GetObjectRequest. - s3GetObjectInput := &s3.GetObjectInput{ - Bucket: awssdk.String(info.name), - Key: awssdk.String(info.key), - } - req := svc.GetObjectRequest(s3GetObjectInput) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - resp, err := req.Send(ctx) +func (in *s3Input) Run(ctx v2.Context, pipeline beat.Pipeline) error { + collector, err := in.createCollector(ctx, pipeline) if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - // If the SDK can determine the request or retry delay was canceled - // by a context the ErrCodeRequestCanceled error will be returned. - if awsErr.Code() == awssdk.ErrCodeRequestCanceled { - err = errors.Wrapf(err, "S3 GetObjectRequest canceled for '%s' from S3 bucket '%s'", info.key, info.name) - p.logger.Error(err) - return err - } - - if awsErr.Code() == "NoSuchKey" { - p.logger.Warnf("Cannot find s3 file '%s' from S3 bucket '%s'", info.key, info.name) - return nil - } - } - return errors.Wrapf(err, "S3 GetObjectRequest failed for '%s' from S3 bucket '%s'", info.key, info.name) - } - - defer resp.Body.Close() - - reader := bufio.NewReader(resp.Body) - - isS3ObjGzipped, err := isStreamGzipped(reader) - if err != nil { - err = errors.Wrap(err, "could not determine if S3 object is gzipped") - p.logger.Error(err) return err } - if isS3ObjGzipped { - gzipReader, err := gzip.NewReader(reader) - if err != nil { - err = errors.Wrapf(err, "gzip.NewReader failed for '%s' from S3 bucket '%s'", info.key, info.name) - p.logger.Error(err) - return err - } - reader = bufio.NewReader(gzipReader) - gzipReader.Close() - } - - // Decode JSON documents when content-type is "application/json" or expand_event_list_from_field is given in config - if resp.ContentType != nil && *resp.ContentType == "application/json" || info.expandEventListFromField != "" { - decoder := json.NewDecoder(reader) - err := p.decodeJSON(decoder, objectHash, info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "decodeJSONWithKey failed for '%s' from S3 bucket '%s'", info.key, info.name) - p.logger.Error(err) - return err - } - return nil - } - - // handle s3 objects that are not json content-type - offset := 0 - for { - log, err := readStringAndTrimDelimiter(reader) - if err == io.EOF { - // create event for last line - offset += len([]byte(log)) - event := createEvent(log, offset, info, objectHash, s3Ctx) - err = p.forwardEvent(event) - if err != nil { - err = errors.Wrap(err, "forwardEvent failed") - p.logger.Error(err) - return err - } - return nil - } else if err != nil { - err = errors.Wrap(err, "readStringAndTrimDelimiter failed") - p.logger.Error(err) - return err - } - - if log == "" { - break - } - - // create event per log line - offset += len([]byte(log)) - event := createEvent(log, offset, info, objectHash, s3Ctx) - err = p.forwardEvent(event) - if err != nil { - err = errors.Wrap(err, "forwardEvent failed") - p.logger.Error(err) - return err - } - } - return nil -} - -func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3Info, s3Ctx *s3Context) error { - offset := 0 - for { - var jsonFields interface{} - err := decoder.Decode(&jsonFields) - if jsonFields == nil { - return nil - } - - if err == io.EOF { - offset, err = p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) - if err != nil { - return err - } - } else if err != nil { - // decode json failed, skip this log file - err = errors.Wrapf(err, "decode json failed for '%s' from S3 bucket '%s', skipping this file", s3Info.key, s3Info.name) - p.logger.Warn(err) - return nil - } - - offsetNew, err := p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) - if err != nil { - return err - } - offset = offsetNew - } -} - -func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { - switch f := jsonFields.(type) { - case map[string][]interface{}: - if s3Info.expandEventListFromField != "" { - textValues, ok := f[s3Info.expandEventListFromField] - if !ok { - err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) - p.logger.Error(err) - return offset, err - } - for _, v := range textValues { - offset, err := p.convertJSONToEvent(v, offset, objectHash, s3Info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) - p.logger.Error(err) - return offset, err - } - } - return offset, nil - } - case map[string]interface{}: - if s3Info.expandEventListFromField != "" { - textValues, ok := f[s3Info.expandEventListFromField] - if !ok { - err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) - p.logger.Error(err) - return offset, err - } - - valuesConverted := textValues.([]interface{}) - for _, textValue := range valuesConverted { - offsetNew, err := p.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) - p.logger.Error(err) - return offset, err - } - offset = offsetNew - } - return offset, nil - } - - offset, err := p.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) - p.logger.Error(err) - return offset, err - } - return offset, nil - } - return offset, nil + defer collector.publisher.Close() + collector.run() + return ctx.Cancelation.Err() } -func (p *s3Input) convertJSONToEvent(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { - vJSON, err := json.Marshal(jsonFields) - logOriginal := string(vJSON) - log := trimLogDelimiter(logOriginal) - offset += len([]byte(log)) - event := createEvent(log, offset, s3Info, objectHash, s3Ctx) +func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3Collector, error) { + log := ctx.Logger.With("queue_url", in.config.QueueURL) - err = p.forwardEvent(event) + client, err := pipeline.ConnectWith(beat.ClientConfig{ + CloseRef: ctx.Cancelation, + ACKHandler: newACKHandler(), + }) if err != nil { - err = errors.Wrap(err, "forwardEvent failed") - p.logger.Error(err) - return offset, err - } - return offset, nil -} - -func (p *s3Input) forwardEvent(event beat.Event) error { - ok := p.outlet.OnEvent(event) - if !ok { - return errOutletClosed - } - return nil -} - -func (p *s3Input) deleteMessage(queueURL string, messagesReceiptHandle string, svcSQS sqsiface.ClientAPI) error { - deleteMessageInput := &sqs.DeleteMessageInput{ - QueueUrl: awssdk.String(queueURL), - ReceiptHandle: awssdk.String(messagesReceiptHandle), + return nil, err } - req := svcSQS.DeleteMessageRequest(deleteMessageInput) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - _, err := req.Send(ctx) + regionName, err := getRegionFromQueueURL(in.config.QueueURL) if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { - return nil - } - return errors.Wrapf(err, "SQS DeleteMessageRequest failed in queue %s", queueURL) + err := fmt.Errorf("getRegionFromQueueURL failed: %w", err) + log.Error(err) + return nil, err + } else { + log = log.With("region", regionName) } - return nil -} -func trimLogDelimiter(log string) string { - return strings.TrimSuffix(log, "\n") -} - -func readStringAndTrimDelimiter(reader *bufio.Reader) (string, error) { - logOriginal, err := reader.ReadString('\n') + awsConfig, err := awscommon.GetAWSCredentials(in.config.AwsConfig) if err != nil { - return logOriginal, err - } - return trimLogDelimiter(logOriginal), nil -} - -func createEvent(log string, offset int, info s3Info, objectHash string, s3Ctx *s3Context) beat.Event { - s3Ctx.Inc() - - event := beat.Event{ - Timestamp: time.Now().UTC(), - Fields: common.MapStr{ - "message": log, - "log": common.MapStr{ - "offset": int64(offset), - "file.path": constructObjectURL(info), - }, - "aws": common.MapStr{ - "s3": common.MapStr{ - "bucket": common.MapStr{ - "name": info.name, - "arn": info.arn}, - "object.key": info.key, - }, - }, - "cloud": common.MapStr{ - "provider": "aws", - "region": info.region, - }, - }, - Private: s3Ctx, - } - event.SetID(objectHash + "-" + fmt.Sprintf("%012d", offset)) - - return event -} - -func constructObjectURL(info s3Info) string { - return "https://" + info.name + ".s3-" + info.region + ".amazonaws.com/" + info.key -} - -// s3ObjectHash returns a short sha256 hash of the bucket arn + object key name. -func s3ObjectHash(s3Info s3Info) string { - h := sha256.New() - h.Write([]byte(s3Info.arn + s3Info.key)) - prefix := hex.EncodeToString(h.Sum(nil)) - return prefix[:10] -} - -func (c *s3Context) setError(err error) { - // only care about the last error for now - // TODO: add "Typed" error to error for context - c.mux.Lock() - defer c.mux.Unlock() - c.err = err -} - -func (c *s3Context) done() { - c.mux.Lock() - defer c.mux.Unlock() - c.refs-- - if c.refs == 0 { - c.errC <- c.err - close(c.errC) - } -} - -func (c *s3Context) Inc() { - c.mux.Lock() - defer c.mux.Unlock() - c.refs++ -} - -// isStreamGzipped determines whether the given stream of bytes (encapsulated in a buffered reader) -// represents gzipped content or not. A buffered reader is used so the function can peek into the byte -// stream without consuming it. This makes it convenient for code executed after this function call -// to consume the stream if it wants. -func isStreamGzipped(r *bufio.Reader) (bool, error) { - // Why 512? See https://godoc.org/net/http#DetectContentType - buf, err := r.Peek(512) - if err != nil && err != io.EOF { - return false, err - } - - switch http.DetectContentType(buf) { - case "application/x-gzip", "application/zip": - return true, nil - default: - return false, nil - } + return nil, fmt.Errorf("getAWSCredentials failed: %w", err) + } + awsConfig.Region = regionName + + visibilityTimeout := int64(in.config.VisibilityTimeout.Seconds()) + log.Infof("visibility timeout is set to %v seconds", visibilityTimeout) + log.Infof("aws api timeout is set to %v", in.config.APITimeout) + + return &s3Collector{ + cancellation: ctxtool.FromCanceller(ctx.Cancelation), + logger: log, + config: &in.config, + publisher: client, + visibilityTimeout: visibilityTimeout, + sqs: sqs.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "sqs", regionName, awsConfig)), + s3: s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "s3", regionName, awsConfig)), + }, nil +} + +func newACKHandler() beat.ACKer { + return acker.ConnectionOnly( + acker.EventPrivateReporter(func(_ int, privates []interface{}) { + for _, private := range privates { + if s3Context, ok := private.(*s3Context); ok { + s3Context.done() + } + } + }), + ) } diff --git a/x-pack/filebeat/input/s3/s3_integration_test.go b/x-pack/filebeat/input/s3/s3_integration_test.go index 1d6a400a7f19..cadb5da035bf 100644 --- a/x-pack/filebeat/input/s3/s3_integration_test.go +++ b/x-pack/filebeat/input/s3/s3_integration_test.go @@ -16,21 +16,20 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" - - "github.com/aws/aws-sdk-go-v2/aws" awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/s3iface" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" + "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" "github.com/elastic/beats/v7/libbeat/tests/resources" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/elastic/go-concert/unison" ) const ( @@ -77,228 +76,99 @@ func getConfigForTest(t *testing.T) config { return config } -func uploadSampleLogFile(t *testing.T, bucketName string, svcS3 s3iface.ClientAPI) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - file, err := os.Open(filePath) - if err != nil { - t.Fatalf("Failed to open file %v", filePath) - } - defer file.Close() - - s3PutObjectInput := s3.PutObjectInput{ - Bucket: aws.String(bucketName), - Key: aws.String(filepath.Base(filePath)), - Body: file, - } - req := svcS3.PutObjectRequest(&s3PutObjectInput) - output, err := req.Send(ctx) - if err != nil { - t.Fatalf("failed to put object into s3 bucket: %v", output) - } -} - -func collectOldMessages(t *testing.T, queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI) []string { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // receive messages from sqs - req := svcSQS.ReceiveMessageRequest( - &sqs.ReceiveMessageInput{ - QueueUrl: &queueURL, - MessageAttributeNames: []string{"All"}, - MaxNumberOfMessages: &maxNumberOfMessage, - VisibilityTimeout: &visibilityTimeout, - WaitTimeSeconds: &waitTimeSecond, - }) - - output, err := req.Send(ctx) - if err != nil { - t.Fatalf("failed to receive message from SQS: %v", output) - } - - var oldMessageHandles []string - for _, message := range output.Messages { - oldMessageHandles = append(oldMessageHandles, *message.ReceiptHandle) - } - - return oldMessageHandles -} - -func (input *s3Input) deleteAllMessages(t *testing.T, awsConfig awssdk.Config, queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI) error { - var messageReceiptHandles []string - init := true - for init || len(messageReceiptHandles) > 0 { - init = false - messageReceiptHandles = collectOldMessages(t, queueURL, visibilityTimeout, svcSQS) - for _, receiptHandle := range messageReceiptHandles { - err := input.deleteMessage(queueURL, receiptHandle, svcSQS) - if err != nil { - return err - } - } - } - return nil -} - func defaultTestConfig() *common.Config { return common.MustNewConfigFrom(map[string]interface{}{ "queue_url": os.Getenv("QUEUE_URL"), }) } -func runTest(t *testing.T, cfg *common.Config, run func(t *testing.T, input *s3Input, out *stubOutleter)) { - // Simulate input.Context from Filebeat input runner. - inputCtx := newInputContext() - defer close(inputCtx.Done) - - // Stub outlet for receiving events generated by the input. - eventOutlet := newStubOutlet() - defer eventOutlet.Close() - - connector := channel.ConnectorFunc(func(_ *common.Config, _ beat.ClientConfig) (channel.Outleter, error) { - return eventOutlet, nil - }) +func newV2Context() (v2.Context, func()) { + ctx, cancel := context.WithCancel(context.Background()) + return v2.Context{ + Logger: logp.NewLogger("s3_test"), + ID: "test_id", + Cancelation: ctx, + }, cancel +} - in, err := NewInput(cfg, connector, inputCtx) +func setupInput(t *testing.T, cfg *common.Config) (*s3Collector, chan beat.Event) { + inp, err := Plugin().Manager.Create(cfg) if err != nil { t.Fatal(err) } - s3Input := in.(*s3Input) - defer s3Input.Stop() - run(t, s3Input, eventOutlet) -} + ctx, cancel := newV2Context() + t.Cleanup(cancel) -func newInputContext() input.Context { - return input.Context{ - Done: make(chan struct{}), + client := pubtest.NewChanClient(0) + pipeline := pubtest.ConstClient(client) + collector, err := inp.(*s3Input).createCollector(ctx, pipeline) + if err != nil { + t.Fatal(err) } + return collector, client.Channel } -type stubOutleter struct { - sync.Mutex - cond *sync.Cond - done bool - Events []beat.Event -} - -func newStubOutlet() *stubOutleter { - o := &stubOutleter{} - o.cond = sync.NewCond(o) - return o -} - -func (o *stubOutleter) waitForEvents(numEvents int) ([]beat.Event, bool) { - o.Lock() - defer o.Unlock() - - for len(o.Events) < numEvents && !o.done { - o.cond.Wait() +func setupCollector(t *testing.T, cfg *common.Config, mock bool) (*s3Collector, chan beat.Event) { + collector, receiver := setupInput(t, cfg) + if mock { + svcS3 := &MockS3Client{} + svcSQS := &MockSQSClient{} + collector.sqs = svcSQS + collector.s3 = svcS3 + return collector, receiver } - size := numEvents - if size >= len(o.Events) { - size = len(o.Events) + config := getConfigForTest(t) + awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) + if err != nil { + t.Fatal("failed GetAWSCredentials with AWS Config: ", err) } - out := make([]beat.Event, size) - copy(out, o.Events) - return out, len(out) == numEvents + s3BucketRegion := os.Getenv("S3_BUCKET_REGION") + if s3BucketRegion == "" { + t.Log("S3_BUCKET_REGION is not set, default to us-west-1") + s3BucketRegion = "us-west-1" + } + awsConfig.Region = s3BucketRegion + awsConfig = awsConfig.Copy() + collector.sqs = sqs.New(awsConfig) + collector.s3 = s3.New(awsConfig) + return collector, receiver } -func (o *stubOutleter) Close() error { - o.Lock() - defer o.Unlock() - o.done = true - return nil -} - -func (o *stubOutleter) Done() <-chan struct{} { return nil } - -func (o *stubOutleter) OnEvent(event beat.Event) bool { - o.Lock() - defer o.Unlock() - o.Events = append(o.Events, event) - o.cond.Broadcast() - return !o.done +func runTest(t *testing.T, cfg *common.Config, mock bool, run func(t *testing.T, collector *s3Collector, receiver chan beat.Event)) { + collector, receiver := setupCollector(t, cfg, mock) + run(t, collector, receiver) } func TestS3Input(t *testing.T) { - inputConfig := defaultTestConfig() - config := getConfigForTest(t) - - runTest(t, inputConfig, func(t *testing.T, input *s3Input, out *stubOutleter) { - awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) - if err != nil { - - } - s3BucketRegion := os.Getenv("S3_BUCKET_REGION") - if s3BucketRegion == "" { - t.Log("S3_BUCKET_REGION is not set, default to us-west-1") - s3BucketRegion = "us-west-1" - } - awsConfig.Region = s3BucketRegion - input.awsConfig = awsConfig.Copy() - svcSQS := sqs.New(awsConfig) - - // remove old messages from SQS - err = input.deleteAllMessages(t, awsConfig, config.QueueURL, int64(config.VisibilityTimeout.Seconds()), svcSQS) - if err != nil { - t.Fatalf("failed to delete message: %v", err.Error()) - } - + runTest(t, defaultTestConfig(), false, func(t *testing.T, collector *s3Collector, receiver chan beat.Event) { // upload a sample log file for testing s3BucketNameEnv := os.Getenv("S3_BUCKET_NAME") if s3BucketNameEnv == "" { t.Fatal("failed to get S3_BUCKET_NAME") } - svcS3 := s3.New(awsConfig) - uploadSampleLogFile(t, s3BucketNameEnv, svcS3) - time.Sleep(30 * time.Second) - wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() - input.run(svcSQS, svcS3, 300) + collector.run() }() - events, ok := out.waitForEvents(2) - if !ok { - t.Fatalf("Expected 2 events, but got %d.", len(events)) - } - input.Stop() - - // check events - for i, event := range events { - bucketName, err := event.GetValue("aws.s3.bucket.name") - assert.NoError(t, err) - assert.Equal(t, s3BucketNameEnv, bucketName) - - objectKey, err := event.GetValue("aws.s3.object.key") - assert.NoError(t, err) - assert.Equal(t, fileName, objectKey) - - message, err := event.GetValue("message") - assert.NoError(t, err) - switch i { - case 0: - assert.Equal(t, "logline1\n", message) - case 1: - assert.Equal(t, "logline2\n", message) - } - } + event := <-receiver + bucketName, err := event.GetValue("aws.s3.bucket.name") + assert.NoError(t, err) + assert.Equal(t, s3BucketNameEnv, bucketName) - // delete messages from the queue - err = input.deleteAllMessages(t, awsConfig, config.QueueURL, int64(config.VisibilityTimeout.Seconds()), svcSQS) - if err != nil { - t.Fatalf("failed to delete message: %v", err.Error()) - } + objectKey, err := event.GetValue("aws.s3.object.key") + assert.NoError(t, err) + assert.Equal(t, fileName, objectKey) + + message, err := event.GetValue("message") + assert.NoError(t, err) + assert.Equal(t, "logline1\n", message) }) } @@ -354,41 +224,23 @@ func TestMockS3Input(t *testing.T) { "queue_url": "https://sqs.ap-southeast-1.amazonaws.com/123456/test", }) - runTest(t, cfg, func(t *testing.T, input *s3Input, out *stubOutleter) { - svcS3 := &MockS3Client{} - svcSQS := &MockSQSClient{} + runTest(t, cfg, true, func(t *testing.T, collector *s3Collector, receiver chan beat.Event) { + defer collector.cancellation.Done() + defer collector.publisher.Close() - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() - input.run(svcSQS, svcS3, 300) - }() + output, err := collector.receiveMessage(collector.sqs, collector.visibilityTimeout) + assert.NoError(t, err) - events, ok := out.waitForEvents(2) - if !ok { - t.Fatalf("Expected 2 events, but got %d.", len(events)) - } - input.Wait() - - // check events - for i, event := range events { - bucketName, err := event.GetValue("aws.s3.bucket.name") - assert.NoError(t, err) - assert.Equal(t, "test-s3-ks-2", bucketName) - - objectKey, err := event.GetValue("aws.s3.object.key") - assert.NoError(t, err) - assert.Equal(t, "server-access-logging2019-06-21-16-16-54", objectKey) - - message, err := event.GetValue("message") - assert.NoError(t, err) - switch i { - case 0: - assert.Equal(t, s3LogString1, message) - case 1: - assert.Equal(t, s3LogString2, message) - } - } + var grp unison.MultiErrGroup + errC := make(chan error) + defer close(errC) + grp.Go(func() (err error) { + return collector.processMessage(collector.s3, output.Messages[0], errC) + }) + + event := <-receiver + bucketName, err := event.GetValue("aws.s3.bucket.name") + assert.NoError(t, err) + assert.Equal(t, "test-s3-ks-2", bucketName) }) } diff --git a/x-pack/filebeat/module/activemq/audit/config/audit.yml b/x-pack/filebeat/module/activemq/audit/config/audit.yml index 3e0ec415541f..f40bf16116f5 100644 --- a/x-pack/filebeat/module/activemq/audit/config/audit.yml +++ b/x-pack/filebeat/module/activemq/audit/config/audit.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/activemq/log/config/log.yml b/x-pack/filebeat/module/activemq/log/config/log.yml index 17f6bd869f2d..55449d359f1a 100644 --- a/x-pack/filebeat/module/activemq/log/config/log.yml +++ b/x-pack/filebeat/module/activemq/log/config/log.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/file.yml b/x-pack/filebeat/module/aws/cloudtrail/config/file.yml index 5a56f210c79a..c4436629e213 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/file.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml index 2094f77c712f..7c455f64f268 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml @@ -58,4 +58,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/file.yml b/x-pack/filebeat/module/aws/cloudwatch/config/file.yml index 5a56f210c79a..c4436629e213 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/file.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml index 073eca58ab2b..5d2d75dc5d82 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml @@ -44,4 +44,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/ec2/config/file.yml b/x-pack/filebeat/module/aws/ec2/config/file.yml index 5a56f210c79a..c4436629e213 100644 --- a/x-pack/filebeat/module/aws/ec2/config/file.yml +++ b/x-pack/filebeat/module/aws/ec2/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/ec2/config/s3.yml b/x-pack/filebeat/module/aws/ec2/config/s3.yml index 073eca58ab2b..5d2d75dc5d82 100644 --- a/x-pack/filebeat/module/aws/ec2/config/s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/s3.yml @@ -44,4 +44,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/elb/config/file.yml b/x-pack/filebeat/module/aws/elb/config/file.yml index 498a79064572..e9470671e071 100644 --- a/x-pack/filebeat/module/aws/elb/config/file.yml +++ b/x-pack/filebeat/module/aws/elb/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml index 073eca58ab2b..5d2d75dc5d82 100644 --- a/x-pack/filebeat/module/aws/elb/config/s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/s3.yml @@ -44,4 +44,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/s3access/config/file.yml b/x-pack/filebeat/module/aws/s3access/config/file.yml index 498a79064572..e9470671e071 100644 --- a/x-pack/filebeat/module/aws/s3access/config/file.yml +++ b/x-pack/filebeat/module/aws/s3access/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml index 073eca58ab2b..5d2d75dc5d82 100644 --- a/x-pack/filebeat/module/aws/s3access/config/s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/s3.yml @@ -44,4 +44,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index c9e88b6a743a..d0c18047ca4b 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -173,4 +173,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index a45679591944..77c7ea3f5d07 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/activitylogs/config/file.yml b/x-pack/filebeat/module/azure/activitylogs/config/file.yml index 498a79064572..e9470671e071 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/file.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index 3633cc4e5dee..43a051f8dd32 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/file.yml b/x-pack/filebeat/module/azure/auditlogs/config/file.yml index 937446eb5238..8e77b860e994 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/file.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index dd8e1473a68f..b971be2783c1 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/file.yml b/x-pack/filebeat/module/azure/signinlogs/config/file.yml index 937446eb5238..8e77b860e994 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/file.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/cef/log/config/input.yml b/x-pack/filebeat/module/cef/log/config/input.yml index 49a2b1829be5..cc911a8611f4 100644 --- a/x-pack/filebeat/module/cef/log/config/input.yml +++ b/x-pack/filebeat/module/cef/log/config/input.yml @@ -28,4 +28,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml index 12440f8fffe2..f447d2aacdfc 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml @@ -23,4 +23,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc index 08dc160fab01..aaf285680e07 100644 --- a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc @@ -5,13 +5,15 @@ == Cisco module -This is a module for Cisco network device's logs. It includes the following +This is a module for Cisco network device's logs and Cisco Umbrella. It includes the following filesets for receiving logs over syslog or read from a file: - `asa` fileset: supports Cisco ASA firewall logs. - `ftd` fileset: supports Cisco Firepower Threat Defense logs. - `ios` fileset: supports Cisco IOS router and switch logs. - `nexus` fileset: supports Cisco Nexus switch logs. +- `meraki` fileset: supports Cisco Meraki logs. +- `umbrella` fileset: supports Cisco Umbrella logs. Cisco ASA devices also support exporting flow records using NetFlow, which is supported by the {filebeat-ref}/filebeat-module-netflow.html[netflow module] in @@ -27,6 +29,8 @@ The module is by default configured to run via syslog on port 9001 for ASA and port 9002 for IOS. However it can also be configured to read from a file path. See the following example. +Cisco Umbrella publishes its logs in a compressed CSV format to a S3 bucket. + ["source","yaml",subs="attributes"] ----- - module: cisco @@ -374,6 +378,59 @@ will be found under `rsa.raw`. The default is false. :fileset_ex!: +[float] +==== `umbrella` fileset settings + +The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. + +To configure Cisco Umbrella to log to either your own S3 bucket or one that is managed by Cisco please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide.] + +This fileset supports all 4 log types: +- Proxy +- Cloud Firewall +- IP Logs +- DNS logs + +The Cisco Umbrella fileset depends on the original file path structure being followed. This structure is documented https://docs.umbrella.com/deployment-umbrella/docs/log-formats-and-versioning[Umbrella Log Formats and Versioning]: + +/--
/--
---.csv.gz +dnslogs/--/----.csv.gz + +When configuring the fileset, please ensure that the Queue URL is set to the root folder that includes each of these subfolders above. + +Example config: + +[source,yaml] +---- +- module: cisco + umbrella: + enabled: true + var.input: s3 + var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + var.access_key_id: 123456 + var.secret_access_key: PASSWORD +---- + +*`var.input`*:: + +The input from which messages are read. Can be S3 or file. + +*`var.queue_url`*:: + +The URL to the SQS queue if the input type is S3. + +*`var.access_key_id`*:: + +The ID for the access key used to read from the SQS queue. + +*`var.secret_access_key`*:: + +The secret token used for authenticating to the SQS queue. + +:has-dashboards!: + +:fileset_ex!: + [float] === Example dashboard diff --git a/x-pack/filebeat/module/cisco/fields.go b/x-pack/filebeat/module/cisco/fields.go index fac83c30f27f..de57daee50fe 100644 --- a/x-pack/filebeat/module/cisco/fields.go +++ b/x-pack/filebeat/module/cisco/fields.go @@ -19,5 +19,5 @@ func init() { // AssetCisco returns asset data. // This is the base64 encoded gzipped contents of module/cisco. func AssetCisco() string { - return "eJzsvW1zIzeSIPx9fwXOEc+52yGzx+2XvfHN7oVWkte66W5rW93tfS4mogJEgSRGKKAaQJGif/0FEqgXVqFIiQKK6r3xB4ctkolEIpHI9/wW3dHtz4gwTeQ/IWSY4fRndOH/N6eaKFYaJsXP6F//CSGE3sq84hQtpEIrLHLOxNJ9HQlqNlLdoZyuGaGIy6We/RNCC0Z5rn/+J/i1/edbJHBB/ZozrHHzCUJmW9Kf0VLJquz8NYBG/c8vAB3QcVic356jX5iiG8z5rPPVGo32LzUeBdUaL2nG8h3IDpU7ut1ItfvJHnQQ+rCiHUw8bMRyKgxbMKpanAKo6GqxYPcPRIPe46K0p6Wp1kyKh+P4G/wdc78ewgtDFfr/LMIPRVRWitCMCUPVAhMag3K3ABM1MOFQzYqiBZcbJBWiayrMXrRyqg0T2MKPi9tlC/hJCKqK08z+Zwyk3uGCIrkAFM4JoVqjCymMkhy9YdrAYsissEEFNmRFc2RWTD8AS3+6laYqBa4WrsOLafiDW8+T80EYdg96MjQ7iz4G1wKXJc2z+sqUATx7fzwoYIzCQnNsaF7T7voG4TxXVOtH4LKS2jyYagtccZOBGP0ZLTDX9Kk42+UfgW0pVQhbLsXyqZhY0A/BZEe+RD7ILnc97jS7WJ3qSLvYP/Rcu3inONwuTgdP2KwUxSbjdE15HD3AwkMAD6RFgfkGK4peobk0ghqL6WLByAz9JkDmrKnafsvl5gzZf/XAFTKnCht6hlZsubKPDXzd/s9DtkWwoUuptjF2duFhNc/f+M5+sY9iraasmar0mf9Of39Gyb9jcYaoIXv3Q6QQlLgLGEVf+yjY56qroMG2MLzpezFhpCgzu2gAC73qs/NeHK4v3t7ALw8vSGQea0EL6qG0HtlnGrHy6eZdZ220s3ZIF8BlpiiRKtePQ+QJGj7Wmi0FzdHl+Q3qLx4kZVFgkWecCZphtawKKsx06PrlkV0eNctbE21JczTfwjXmkmCOcJUzYz/Zt516+/1H8IF7eOwr2T6HLeGNRNhxCmdUGKQr0IAXFefbhnvE3l2Uiq0Zp0s6k/yRPHzkWfy+ogJh0Cx1u7zVL8kKi2WtoXt9U/IcrTGv9nJ/uwlBN89wE4JuDm9iXiltZnL+d0r6UizdpVDUqQluWRD7gAfaYCWYWO690A5jNg3bdLG1SgC6vjwKXVIpRYXJLIzpZI9b1CML6GtKxQOwlWLBlpWi+WkQbtfv4H4YbbxengZfvKYKL+mTCH0y5DvEDuwDcy43NH8IhxcVx4ataUZkJaYTJkYazBGsaXX5Du4rZjTSTBDqhLqTNhusEbGquRVAChFOsepscNdHujBdbJ7uI/2FKVrKDVW1lXJJF1Ro+kwcp798uPyyHKcW4X84Tv/hOP2H4/SLdpyij5qiq4tb/9FMYDNj5T/8qUf6U0PkfMaO1gbdzuePYIF/OGEP47TLFn06/8NF+w8X7T9ctDsLHnTRakoqxUyIaYLelHbB3nLv8cZr+kDcWw8XXdlXen8U6gt3E6dEca+beNfGY1JHtfGuf7ttUnDqf8ZNOQxacMbZIx6uB2qD9ol1OraFvpeTFpgwHubmvXbc7dXF486lXggZiTYrRlZOSHqbU9EFVRq9WLSi8Qzdvnt7c4Zu///bM4SFVXR6YBdSmdXLGTpvgRMs0JwijFZY5SB+XWrUGcKoVNJIIvkZAlFWuKwquejLXKvkb7WhBdJyYSyQGbo2KKdCGrpjBHhJT3ClG9q7n/bfKbfN2YARfQLXrLHTZj3rQK6p2ihm7KOlKjrg1+EhHbhCew6qy0J1ZllrQG5WVDl/in/I0AprNKdUIDnXVK1pPtyf2kk1O7SZ4eXbu5XxuwVYC7yrsoyvPrZ+aImONqeXvWPet8K+W9U7lQ/WVrujW2vLVdoFXgguTeXpr/CmuThg8xFZUG03Le3nPdAIvZFLdEntw6bCG3GwWB+pY7dTwwVz02q7JDJgj3Bi6nuSuxtPpDAQwJMLxIQ2WJgaDR3E0bDiGATzvic4hJ238e0SCBsvTnHtW3PuT4zeUfM7M8I+A/70ZwPWaDarV7LiORJ0TZWVoDXflVhpit5Sgy1qGC2ULDpLvXgjl/rVDSZ31OiXA/CXTFFi+PasiU9h9J46YeE4XHTQnAUJObQ9HkbJgQnVo+QlLRUlYDBZTHK6YFZtkIIDWgbPuVXiyzBWhV72Ve24HOjP+K2/59eX37mYnvfy1Iq5+xa9xwQiyO681OAgYHcMlDbHLfA9exwlVoaRimMFv/cHOxvljAHoozglxBkDyOOcMnok62nP5PU/zmT/mdhV0xzI066vnP89g430j+XZYLfGxwi95Kgp6nTf54ibJVuq+/80zLTBhha0Fxx9JshB+lFGOO7d4WeCHhWm56F7JoitBl6nZ4IYE8chllZjqiXH8+W0nOJjpEdasi2oixjEsqFG9JqQndn5Yu0WsNgM9JCBkvA0K6KnhwygH7Aixqk4CLxOQkXR8aoEyefINdhmJPKhAAUfTT4yhVpdDWIO9f79n7a7Ru2FFMQ+DtjI527ZjoibNUsrDrvUvbDLsAUjuHuf38ilizfUGS2VyKkCZyn1gmqw9QW7pznSFLKudn68u4YeN1jqQxjAfrLB0hzCAPSjDmXoCYzvXzqOMQf7egRNHkeDQUg9CV/+KrXpikje50hNRc7Esv5Qh9im40P6cujLjmGwwY9GCXt9s/6hyeEfu+594g52b+SXStz1T6nJ+9P/u+QdRJ2TyIa+XHCOtK63LEcYLdmaisZJ9uUqApZEx/kv0log+XNU/r6MiMaoQ0OW20zRzwnOuhs8hAOGfft6syu3NLqBi3TmvdkGow/bkiKChxJkThFlZkUV+ngtzHc/IanQL1xi8/1rNMcauKgOkEE1Aah+B/Z9jLr7Be8bwqDpjM8I/oVgItwk1nG98hfvYJBqg9WgOjOa1tGRaJ1tdyl5ffNpR9/DUL/WP1JU57a4R9SjDen21HGqdsSDwhnFlgxqL9xvdrWVA3RIpX/tSYy4vvn0U4AE4ZwcFIEEDUZDKsd4fVpGHSqOx74+K4pzqiaJXf8KS6Hry6dESR2+3WApgDkuVvqsnWycZMn9bLhWtK5bRQsuijVdLiTnlBipvkQBbKl3gpwby3NMI+JIR3OL6Y6i+kb21Ra0h9DP0OIryPy5qKqF1JDsVkiB5tvBoSGk6OeKaiiC0qwo+dafk/0yJOpSTFZIs5yiF39CZqUq9PrHH19CaaimVDSr7KHEs1BeH0AJXUqhaTpSkC+GK1yJcO1TqIq5E3r2KusgBPQCz+WadojBRDCzshZv2iiKi9H7Q74YtjkxqWjOqr6eFoNQX4U0x8axwBaImb9Vr//03Z+1E+mvShCgNdJ/G+zmb9YefIO3VKHX6EoQXGoogpcCTMpHyfUQ9CcGPwK5laFVvn+N/sVu9wx9/z36F0SkgpYXcExu0TP037n5n/aLTKNdonwVPEIh80DR8DOxdcWGZgRzPsfkLq0G7JCrCwawcXaFJSIVeSmZMHV3kSCiwBwZVUomyk9r9UFdUsIwB4wBU22kspq12Dqtw36wxpzljjFCSCG0kJXI7QvDKSDPxNIrRweTF3dvxAByjFigvw57wkYjp7DlEufP5Z3z6CDN/qCooEYxErA6vCnc/TLYwu65r4WwffaxaTVauaiPbYZ+lRt7NEObkwkklTXGjER3lJYHiPYsXrwvhGhKQjHYmuVZnirqelVLniUVUDWroayqcna0twvXTJkKc2u07/jeRcDFwQpmzW6IlQMx3C78Vb++RMpKaw0OFSAaVktqmq8dpIRWiZKeTk6JumZ/HyVUklDQUPBfX9a+1/e0kIaiW8/vda+c+XZMUCLoNuICMV9A4MWvlOmSs5SZDc/anNdsoPY/C93MytyE/A63zr4BdZmm57raavFPyH+NCKMTLwvGTxCjt6ta4+jm4vzG676+KJcVpVR9jRfBE/nFpUFUz8P94ds0gCE+bL6GnCt115Sv2p+0BrvTc8Ayn6HXP/6ENkD3gmKBMOdhXwE49UFNav1HaEOV64GHoM0H1gZJ0SsX2SXiydXEL5uIgbuaImzrafe7VDkQDrKaKFkJyeVy2w/ELZgaaLEI/YjICitMjCOivdRbwB+c5gJVwuf08B2f+WhFbeyCbheoTxlE2BO7BIuisEqmFHUYQeHNqEwDydpTKzEBjdXFKIT3OUhCoMcjQNQGixyrHAmpCszZH6H8XqmKIH1yn+VwNIlkNR88SY8iUot1g8wrzhYUdhww8DUlUuQjCnZ73Jk2Kf0sezbEBJFFyakJMsCoExWDAm8U64nBTr2ZMidi5Fu7dpCdx1h5lzNH2a+QwqwiHVNbnxor56XNcspPRPgrkacguwX5hxSpuy3sEYt29VrFdOm1H/oUHoioZDf6HBl6b/zlQ2uqdKecIt+XBxY436cy25biWNtsy/SIVDnN072DPsnGP1O6WbHWMepMm+aL3fj68LVSspgB1AqK8jWhAismnVpfVNywbw2jCuGy5HX1S9vLpsACL0OluQhxCO/stPWpG0ohZr7WSG6Ei4wZXJR9z6DHGPpvKjlMPmJGI7Ji1rqROdUz9LbSBsykLlB7K7EZycvFhh55SHsF2GJh8V7TKTQhOOR6QUc7aAVFBXEMga1qnbM1y61mA/wQFmS3tSD70CNeeJP3JVOT7bA9TxcLurecyAzf1n2vjAR9zSLlmjPu9Y1GPPRRF86ZlcaNPJsNlmzSyWQVWwIVA0XuqRAb+se+KqBBfq5oNRkrWe52XNTKxw3WCJDIR/gGkPsuNlEjKgU7BE0g05aFSfD6LosUuJZZAlTLLIX2XMYURbtAX0eHmkBX6rwipzEhe+Zj8I0ZPJePenOOFZuH5NoxwYL2geh1Q4jtCMJkoMTHUKx1xVOHnUasKFkZIgv6yuHQGC+QlT1ogIksXzgS7BiQIwxC13TQDneyjdWr+yLATmRnn8snbfHioHege6WbShcLDeJOJSVswVrDJ6zd+lbuIzzldeX02UyBA2hcjCxvCyZqF1XugyxBvL3ZPNUhfNq10ruWoFTot1ufGst0nRDQ96sh3xe2N0AB7VRJ6lJqFlFwPIi3wJwWueswBan89d0d7cJTcTNsmH0qUSSqgipGHiuLgnuboIptz8a6lWzNzXBiyd3vwdbWVORS+YTZvTuT87+foHtNHdoNtDXvIpa+FnxAbitB9yPmJH3KXnVfDS+kr/r3YsZ7uVa4yS0W0iAMYyIskuEEWi6XWZ2ochKhXjPio4X6FD1TdmTfv0O6FXSt3h132MWqlJyRberbs0cu3AACvrm24NsRuRwctpSYgO8rTgGxsDiVwtD71Bprg9C1cP66th8qznNt/wWPKox6A4RCDWAOPM5uSmbWH9eZQBaMBS7rkZxNrxBsjGLzytCOhBjm6PsBn1Zb7z5/YdGhy/4EsadbLW7U6/Q3BwzBfn6Rnzvb0d8Cxi1UgFmC1Q0HdZvzpdZUzdAtdYdSaapmeEmhlbfPdF9IVeMwgF2DcXo7cUO33O87fSukQnMlN/az+q9e13Rm12g/6ev8BisT203XAI7tUfF3qj/Hd7o71czqTXilZEl9QDHVW3wuEOZUmSa7SLWL+r+58JYXH50mAJCEFFCYcySk+FbRkoIlsy/7AcyGKZ+cevhoY6+YZkDnK+YibHX4Z7CzDTMrryw7WY8uYcE5VJsIJMW3S2n/e89LAEpKFlAcE+4bd4KBrwABi6RcICsdDKN6hm5bmdIfbNCtrEqD8YUr56u0NWJcyahLtsm9+PWEx4jwSpuaIf3/DI4JfsK0PUlfE+39G1bxhU/HVaDJtR93w8IWvWvLlE4p+/qQ4WWxvAQsENZaEgb+UnsaQXsSDuwNu6M/I4zK1VYzgjnKmb47Q6WCmSgwSuzrsKKMFT6m9vKRD72rs1G4oAaGmWMNXbw0NHJwvQjq0flyJ2g/LK3ZmYqGhk+Tew9OpfF1zjDBw+TEN5FFWQ3vYIJjw2jDRC43Pp+WSEFoac6aTIpRYgy2uag436LPFebO+ZnLAjPhpYboLMTlyNPV9XrGUpf2bN2qhG+YuKO5rwWqE9GxBu+UN1DsJ181qM1Yvu/g+KArRFJR153s5NwSfQRq9H67PRVev5Xe84puh+16mqAzVQXrD3ZK7WL1awK2jv/3a9rfR9a0F4ynv+PNln+B1ZprrGheEYrqyBENu9s0VQzzLPCaJntEbmHJWm3uv4+dB9C+MKN+AUru9FEtB2J4jP3q9qFbYb1qbqhVCwNVhhVZuczfusamKTO8qCH1WoTZjTTLzLQiMPi+/v9hpSmy8lwgBjl3lYAR+fZP0AivRc0XELZD8Fxh5+HogxN+1bDP07N+sYgs5vU8XbnYebB82ah6xOsFA1+n9vR1tRFAYNzjN02ANHAlLtzqrifjuKfUWXDJXeMN+ZyX+foSvXOS5oVv3IDctD1f9GtxexnWq50D+hS+/I77+foSSOpL3hoxMfQe7EbkXBqg28LMMZGVBRumw0bqWm9T9rLfjer6Am2nLuz1Y48MR0586S7aSbnXlwc12Vj+uQOarEXstchbjXaGLlx9pu93yt0H+7VZQFDtfuO7r7w7bl6ZpnJTmuYxqgSn2lFGugdlI9EaK4bnfFAF6JoyMIFKjkcEgaZCJ+2PsnOgXVXVrTyzkspqGHV9IbPnfPvq+qavQyPfMtZ5FMbqso8cKPjgWsg20uKQRNfCoFu2FBiExQiLllKlbF779UB+WSa9qXU3CV0d4T8tIt3h05bLchlgnHe/fUBMEF7l1IozP8jWDcJ/cVUPML5xDhEHFqT3LOwXgcjc5LFNcE61T0sYM6bvrMp9BF6PKMXruDHf+afhPdN3e0KuRrHlkqp0I+zCJPvUjQV4HNyIZkX1SvLcco+z1Ucmje6E3ifwLAxj714qv3jvdIyXTTOO68twGcmDo/NEFmU2cd4VnIrPvYIxrs6/p6v5txYdKaA+deFmc+cVGbPSvFp6oqyxLuaNtJQKOg9YuV7jNzIlzg8iP4kCOOyqv4DZ5+4hspsYaY38wgpRjN5iUvdTDiu3VgRNasdI8W2toKr9UsjZmtGHWiuKdfTcYG2wqWIpzo0/CjN+MrPDLj6X94jlr8bfL/uyVlNgaDH6OGh87O6CxSJ8det3LPH0vQGTXw7n7h3znDEhq1gxzk4diV5Gv1NWksZ0Ogw8sj9EBpy6M+MOS5xzbuUe0hUhVOtFxdGVXR8RmVNtWaJu9hu2LJjI6X1kAnCmzXGa5xNlCywMppiqkZhTBfHNAivGIYMn4MFz8XexRBiI+K39bXBnIgEfyrlrLnQijdivjl40+ZwlVbr0RbdOwgxI5lWENiG+7vD0cqTI0Lm5hu9x6oQSp3w1SV7eV+W+bT/ETGiUU4MZDzgZ5rIynd+NbE3yyXMza48tbvLYAI/xh9TQouTJsnnOUU4X2IeAfOfLOobvszWtVrymiuMtFHIZ6R9X9CJwI+0HYHX7X9NFXQXufPXaMFNBY0YU3FhrGwwbNj31ukaNYnX8OwTHxjSBrCKyKOx9SsNGFw46Yp1k31LJNcud/6zuIldQPZoIlUtyfKDx8d6yXxhvtUbSzcsLqwb3JSQ9nUbW16unlfV/l/Mj/U5Hb+9/y7kPwIRvV8nSNc69hIRid/K3N9foeqBQddFI1rXWV5fsxyBiYVdTDbuMakg/xh/mc6vDyr0TEdlc5qkrvgYVd32lw+OCLC4j6tEqfrcEFzKYoPK84wL2pcMugbaJh7Aly5tQzogTr4htNQ7KwCO8/PGUvGbfZZXymaqne998dN1z6kAUJGvcU1J1vQgu9WtOQ+WtdRemfYkbEzhCgl7xfNch0lRX4jVmHA8DGahxhSOor1xQpUYmLbg7dIyvP17czRsrhW8A5QKwgy35dAPNlrMRiciKbF7l+Ta6f4YVWdQ6oA7cStPjGp3v9VLFh6iYjNjloFdil+lqioIEprvZq67nKq5yZprKurYvmscoNNiurdhwoqQNL+zfpMsSi03B9WRW+cWnK/TC10p8qrjVleeMQwEH5IFd3ZdS22++RN8OHQ2iH4W5E3IjdgwhTUkFzSzWu9BHJm0SPIELrp8WelFXub/zpUlv6BKTLfo4aq5xNlf4FEX5fuEdEjOBCszEQuGC7k3HKLGCqb3p+yTsKJc3sCx6J3OXHN22BexknQWQQge0L0gVsIRIZSHt9o17Rzfo10qAKflW5pSjF0ysZ9+cISbJGZrbf1H7Lyww32qmZ9+E44uGlNmC48Hk/Ng61K6Gf3GDYFHwdYGc3NbDr+Rib6MGI5Ni6v4693jWbRA0VZaRgwiti7hyt4fZp7e/Y0XRB5cA/M03n97+fv7+6ptvXM7tGivMRnlyI9VdzJLlgxfs93rBboRt1AmGRWwlwtfsxO1S0jwHmNjnYpvAhFlIRYVmJKYA6biSEmBcxPeCBOIDsYBmG8yGw4mf7B2A3uexgdrrE7tEXVfzRJfCzHNtVOzKd6jXTuYQ676l0d7RuuYjnZP02GKXdjDYQKXxxSZt3Yuvd7EgFmzU0VRvNZkj9titBrsRBbbZL+8JC+Wj+wk+3nFhkff6//vhqq3K7Cb/nYTF8o6P3iOyF8mTMEcdx92Hn5QTJG3tnGzHLn1hmoz2OssO+mS+BLfbgHMPR6brltVsingYFH0tMOOW1nUzlxsvM64vu7Vt0InLmoOGLgMtDMazCuuc68yqiEfs55jEa0i39tVHF7IoKtH3RA2wE8c1bnoqdu/ovfl3GtapG9z0cZr1U3G7xSL/NxmOmrW4GWzYMZLhydgNF95BTle6ZITJaFmiU1nwgP0GKzEMOjx31LUoykymEsa3797eoN+cH7VNSg0j8nnSVILb/3iDPldUjfRurbjIFO136kyb3NBxiG7R+7roLJjW1WjpJOJD2gUqY48RsEDLoxxHh6CaQHDsyXDz+AMaMMeqSHBaFmwC9wIuIxYgN0CrPNpU2h2Ycbtd7YDOselrhU+FO6eCrAqsYpWVNHC3JR6ML35y9AmTQTpVFJjZKjovELqIW0DVAF4sodVSArBy/vcEUEscfRKG6zgVnb0g6J6x2A+O79xWUKt6RkdaZJjAYJT45ScWthYRjfcO4PmyXP8g7s0q+vtOREaMynIdte96B7qFfFzk6QGA1xxHlxgio2LJRMSiyCHoFLnRIltkesMMiS4/RLbgcqNxET93pQtbmHU66AmiLkRkTKQUJ0yUVBXzbbSE9wHsktylAb7GPAWvsDIrlTQyix+SAujrHzLwOMaHzZPdTS6XWZ6C2BZw/Pw3IrIC32fGxHIb7AK2HM1pgkehYCIR0kykQ7rkOuNznsUOi+7A/lNC4NE7g3dgx+6F2IUdu6q3C/vHhLB/Sgj7nxPC/h8JYf85DWwjS47nNIVIaaDHN89EVlQclO/5NsE7WQMv7xLoJUXF2bIo02jfVsvEfBk7CclDZimUEk0/k/i+EZFpl5CY4AS1ImmsSQs4jTWpt7oqE8wiJaIpq05iqhpprOlB7xOIECONNcxSwQazJgnwSrB7gYXUlCRgwvVPliqJHoX1T7I0K4rzBG41WZQZ4Ql82BZwgiAJwFXzrYnvFrWQdRLIZZUliGkQxQwjmCcoINIZXlJBthGzrrqwBebbP2g+T4H3OoM2oEkgu3YwabB2ibVJoM+X5fqnND5onc2Z+XOSRmNEZ3FnxfUAKxldVOsk1xygUqLiV7lp5+OPNmurA5ialfPzx3eOOOCg9iUB7rrJx+sg14G9YJymsGF0tkhxiGwRszh7F3AK3UBnrIQkxSyJqGPl+odcm3LQzD8SbK1IEticLWgKM0aDo7mgOYtWMLoLm4k0XFLIvOJUE5mC2h44WyaQTbLUG2yizvzvQA9lkEcBrOiSaaNwfE9ICzuBxqdomYrUKhmtNXQiV4nkq8vMdyyeALpRFBcJFElXCpQK7XTK9WYlmc7chNn40LdY4SQMno8UwsaAvHbz7WPDZdpgEX3Oca7NvFKxhgXWUKmbFZQCahUd1/h6dF2THBssTG5YxB92fWyngX0wlzjPY98BlscOq9atgxK8RazIiJKySNKVyAJOYKaxIkuTHOk7HqUgc3kXvT1TqeO3LGWlLhWLDJRjw0wVPfuMM0HjtdhpoeqoE3UauFB8G9+txaXrepotuIz+nDfAE6T8W5s3utSxQBNIHGtDJ0A1em4Cl8skrCuWSS5wKVVsAVbMq2WKa1YwTVKIhUInYdgUcyAENdBcKTrc6DLcNYCOnfHnoMZOxxObTWwLJElFmXQDoKNbojK+ZiQVW2aBeVxPhrsRVMV/s8rMDeWNDjbqZOoWrBvxmoTJEhRu+pk4sYWBBxtbGpSZcyRFRxdrbT/MyCpWnf8ANL0vWfRAQElVsVRYmEHP3RiQN0kAx396XSeyjx97U0AjAFZymWFdRhwY0AWtcGyoimKeQr9TlAAdXNfRRMDjE9lCjtvCtQNZqjwBxvEdmTqBb1g733CCfABNYycCuIHHCYwTTT/HZ4BQg9ZoUBOYUpotEwheXcb2smlFUtwDRfLoirRWJNQVNwJgE2/EVhdmpaN31VwTEbtQIjgt9qlAXZPO2Ns3SxOfrRzQ+BG9ZqZnbLjbMnq31iqfJ8lDrxRP8BZWmqosZ7Gr3pOMragjQynIYIg2uIjtDV5nTGiDFwk0gzVTJoUavi5FgtZNRqpKxHSzhtqiBTqKnldGoveVQIOlm+yRhMPyPmHOcnShaM4MusAq990MNbR/D6PjJmclpNLYhFAAA0P0EfQ3IJKjUKlOkw/BRDrKXRUll1s6GCx4kH4LWUVr6v1AHrM0dD4jmHem6JLeowL3Gy20sVixrPrDQJIjyZmG4Qz16v7ooYES0lVZSmXQsPEoQpsVNogZVCq6GGOFJ6TlPmYIRYjw3upoUEBM+M7uI32hOROpJ/J3ULWrdfHUyMglNSuqZu339UpWgxcNIUHXVDXjiIxEJVaaorfUYJgI7u4qbkjw4o1c6lc3ruz1Jbr0I77OkFkFphRBM+D31I8+BrQFekfN78wIqsPnPGTqJMRbwMju5hbB4m6zmmJFVjMmWBA/mLk7QX/tnviEWRiQDPGK40rArN9lBXNc6ybu4QbuvX7te/aUvh13s6emCbefXzxi7NuDyCLWND2s8yosiz7QewO3YsxdMMU06hGB1A6uewcTqgUfmXgJ3XMTjgOH/rmaGqTo54pqs6dp9/HZyo/vle9UBhjL41Z1ErvvkWryTnfdKftwchhBbGzn79ChXf8c3HnM2f+H5xvaxa4va6EAa4d5A6yGeEm8j2Rh+7jMsabIpWs32KDBrWpOyf/iNPiKZhR8g7lUrn19kIwIYY00pTDuDO+fV6Ww0JhMMN530GHaLS1A7W2ZhlQKJqDtQ7qkqmBO3ZgK6XZJN5iDrRmnS4o4XVOOsNZsKdzBtfP6w6wPLZlPKL9h/T2cPj/JpGeLWSXY54r2xyTi8OXr4Htcx8TjpqDUGg3L3YUkUggKuRVow8xqTFAgFKgMaTR2RY8qL3q0aWHJCfKkeaK4XDKCObIYjJg+gMVpsYOlRsY0no525Wqrw+h10tk2spfVGvuBx5xhna1kcpvAGXGNuQazVNqhRlYqdkfwhPsBIHdpLLbwpvlBLIRTrGbnXEtriO/ct0sIlqNf/S9m6Fxsm/8bQDdgy2thEM5nRBZlZagKi+Ekbny7sXTm2Vf9s4AZizsHwszfqtd/+u7P1va97BxHTbGvgmh7Ps3iRswe6rjBW6rQPzc+Of3KowHIhW997Pqf9DwvWpx3uH7veRyZvHxItn3dH5hi15mhd799uLJ7p4o65wn4S3OmiaIlFmRrtUqvnvF+LggCCp2hD29/RtfCfP/6DF2/u7z6z5/Rx2thfvoBvdistkhQZlZUIbKS2o9Kk0pRYuBb3/30v/7by6+DFKFmlVDG9ekBMnVW4PA4Hp2Y+x55zW8dL17XSIWveP68kO7KpgOYH9kw7sEPfAjfnmLaWiefmDIV5ujN+bsgsn9IQdP5so7jjP8jBZ2FaWvR/WJEKGzksPCEI3iOb/Cec1hiQzf4BCPSgbtv0HmeK/DTOi4PodM8vaQoj41zPjUWcn3x9sa9SqPhsQLrCaMfO04lp6n6txtd31hURrxfloZHToKIQkO79jgNa00sc9O1phUQHXRxnjP7ZczbgG1nln/4nZuQAaxJCBdc+ht+ucsCA1TaXOsket1DnzSM3nkMb6QyjUgeCN0cAmxwAMxsD0tePTHt3X6YWNaPSb2tt2OEFzRkN07lxfXYgeWLtZaEWZXT+Y0GOg6ycllhsaSzxnQiUizYslI0R/MtwKQih6yhsJwpj2w9MCgaHdGWg4suEvQ74BF1/24JV3QHgKKFNDTzmd3x84zikzYXOsOZS8VPALo0Kg3wRQKWWCSoFuYprkOq/idlAqLiPKs9cenU8r4Fb/cx66/WdSacQIO9MiuqBDXow7akZ+hj/Yy9AQfY9+imdoANXoLfxjS1elTPBMrEiGlcI+394mcIcx5UJsr2i5DghhUk5q2psm8gE0YibeAxZwJ9vB4VKAQSZJPJq+gi2wKVZYKxbxawojp2Rq8Fm6DExb2IsVPRwd+eAFs3WiHjVCyjT4oEnK3ykVALHdFAncqDeScAIxCBdIIFwugXqTZY5cM53QidLyHZSyFsb/w95NLNqdlQKsKqZ+SuiY+NcUuDeTdU55BB0DIeMiMGO2TC57lCWkLBjBVLfsRGeItrjsUUcfwHOCjrBJGOi3KwwV2XZRtJWVsLdgkG7O7LEztSSQl0IVjH6wf3sIg9VoaRimOFoF80qpF4cXX/8xu5lItFePo7JZlZ0eTHu4PsB7ugu40dvK8s3hbd88qsqDA+WXwUbV3F7JzwsIQet+Q46h81VaMIy8oQOS2l/ZLjCN9WhFCtR3CGzuPHNUc7LvEE8EJWxV1KtUWBwoQBblMIpx0caQ9HK5UgwKdLKey7YuVWSDlsfogGitLurtbx+tGNvJsYua6lUDPAGc2b/Xg/TE8fZgJpZqqA/ERQXEC9iPZQV1gjnMvSvi5mRZlCciPaI3OEM/heClmM5NXCTA7NXIv6aZUIq9wzkVv5I5VuCIDRL4xTdO4Rmw3I8BBnr2g25u7kaMJ4s/+TpCuMkuDWZy3EpUJojwFCxKx3fwIhXL7era/XiE2J8YTQuUxZPRDY/Jyu8JrJCrRLIotSyYKNZCjSqZG7EnjOoYhsgS7248bEuhE7CZHsY7ijdaIgAjsYRh0ucwSCgfUb/FKfbueVbe/bKNu1ZZaVMP1yttgafQ5l4Bk5xqx/kBYE7/GSCqoYqbcEBIFEv35qATMreGpDs92QR3ZGvptpo8aDn/Wejmm7dbI9vd6/J69euLUS7itomjZGuGEF1VauO21P0ZKOBpH8KURrCnHwIKDx4BOPQT2QtY7p3X0y1vr+YXv6LtPRhpw+eGveYXxoh4O9wY5bgfAAYfDl7u71wd2pSc/OXbQoe1OHTy5aL9VpBMgBOd4IkC+XHb8/fGSxRhtMc2QPk49qUgkS8449QH5Myo4x9zZgxkaphxK0np86euVOZVZZQc1KniBKgnc8ycih4b82euDQS0nJpF6nPVGd95J7f61FZA9fJvKE/Ofsxz/9Cb14c3l+8xJdMm2YWFZMr2gOpfBBXLhcyuR9gfZFwiBbduHw8McMXxzJGFMysVdxX/2nPdUQBs2NAY98tKHPj7kuBNL+m7rfjuMPcArFTLEItUlvM8Uwj9WdrreR9zhnlXYrIKmQZgXjWDnxZMWmvUME3vVweRXcc83yKTuNdDPlP1pGqL2Ivb6Y7SVPV2dxLvbddQhr+ErDjv/XO4ngkwEveMcN7ZRl5GFXplQpEwMGIRsgtVRLLNgfe7KqRTpWeCixj6B0l6dGyL1gKlhLmqjrzy92OXgtXIsv17toJ6v5V4q5WRGsKCoVzWXBBA4W3HXE0w02jAqjD6bHczzlbt/gk27WtX6kZSLGtVfnayu4SqwMNENqt7pfrE7Y7MgLm4dI1AXNqcKG5lm0pLI9/GGFzy/1ik3w7EbJNcub5mH+e7gsuddUB4zhm//YZ21Xpw0rOO0mWT7RLpslfa8/sx3ZZnB4KGROrpmLnq/6ivtIC7hG6Yw5FPyxmie9B52p86NOJfQysFGno4LGijXSRion8S20ghoMq30N35rZb30d3n3B8pzT6aTcW1jvoXIucLwduXeUnKvHY0yz3Ru/WqfDkNjW0dkzVHJsj8y+z1IhKojalmNefkiFnMCefEAGnWpsy1+lNugtJismRky6HCeSHF/1af1RQKZ/qagVH1Y/ck3O9Ay9yXGJPsH/OP0ol8LVnf5t+HiiFV5TqzlxihX6XFG1RdCDUJdSaFprVOHiVLvfDH4zjbz0PfCIhaxY3QVSuO27vnzjeNZbmgDVloHe++aoD8UUpjyldZj1ebxuLb3TxMjahv7hZRqpSoigHavPmpfHRZ5dG6mRGjsPMfMWZvqDwGjDRC43GumSErZgxH5yFqoT9Hmywwtit+fwbXNu0AvoCEsFaZ8hCF2+7FALVQLe8Td0ickWfdS7jW+bCGzRL6SNnl1rV5jAYB957bumFqACtWrAZPZFHFC86QMQqP7fqTSFcp4h+Xa3nV6hHuvO69TrwI5hh0FG8785YrPT5PWObdVn+HrXey3rrmDr411Ah7uZxmHXBAx2z6ZNyHTHMDihcEOKw8XPUDYQcyTgaIUbbDmnCya8rx6EE3T1K3A50nQQsDuqUCwRbq0Dpqf+xRaMjc829d59L6WR3pSND9sYTFbFxC3w21WB4GhgHXWPI8mQlzkT8SaIRb0bdstQVJj28QwIqW7ZDhyLa6PdlvcHpnYOsE779h3AusSq5in757N2K5sVG7RSR/Z2WFvWJb8/aHsm+swS19ZCqm26A/+LLrH414MdY2pEdruo1+p56GmyZPnLK4B+YG8nU4kGu6r7re/f1SgXZFQYJctjREcuq/nAufAgHvdrWmubHihHABxddce09/BCFiUW2+Y+wrWDcfrOXllTZZ+hjImFDCsFWN+lrhE6ID96VmSN2Yam7Yq++JwqR+CXivMt+o8Kc7ZgNEeXUPfsnINBVDZ0nhEp79iJgu6/0zly67f2M+Zj2nz0brNtOLysDKjcR44wPXzX3zdL+Ck73h3tfPIz9GFbuq23ngNLHHeC44en6CKL2ky2h7bFwTki1Nc61La2j8wUrrpGudzFznkWS6lqbz+EmN+/GTnyTq+cyOxU06JMO4doDynsygc99zWaSspEmsguUnYdex6oxCbsmiQiwzpmtL8DWPly+siQK8UjHnMHasRTaYzRrFKxvCEdmJqqDC/j2ZQt6OjP0y7oqOmPu6A91ycQLPTeUAGqVXzjxMKPxs2NordStJcqE1ujcktMUUu4I3M/wLKgXr3y/33hUXjl/8PnNYXc/phTFc7O89s5YfTcbaYbPAePa2fU2mA7uR+IZk0qJhZUqZG463Dfk+yrq/gfJH3QPTsBknVf4kXnGAJXCsLaMumVCiwxGftdubi9ZbsPkEGsun/6Kx0maI0P/GTliqpp/BFWZ/cZTy8uYPTjS3QB64dRo8pM1CxlhM4XVPnhn3QnC3NPc16aNHTcIWTnwO2iX+tOp+i9J83+ONYr+fjWKOHTRrfsj7C3ht0lkinXf71Cgi6lYe4AyxXWIxOgNJm6rVDnKN3i48MF7VEnmwA1SHDp8VjdOL2uvwknpGi2nKKiYre/UTP18MPooGUrTZjWVXSlEyBDslQ6b93TYiiAIVUqqQ90cChd6XllF0e3EJzeJ50myZBoOoP7KPKLW0jt3P8YdaTncUg+XnruwXFchGrNs3XKF70fUvWO7CAyeWZZD1fR2zTqVIDZHfUWdaLmBl+140q6DxLI1h+QhnidVOj69vyvb2/QjX2n0G9iZPpKi22iSupjsP2wkWFsQQyRFSV3+ign8sOEcNoeZKGhc02/zqZFGKSB+hGErRTco+VSxQZNIU+g5Do8mq4go0YD4GywqSab8NnFco05yx0jBpDoC8LJulrvE4RAsTu61X2xHYnz6wTSyLBXxpQ6YzCDNgloOMoUBCH4GdwmthR15YtUzGwP3CgiiyJpn7gH4u3w8A6hcAn+hinK+5ZmbBfLhmORaX2qgbd2ZSfDf/e7rWu0gti6UuOslGyKtOoQwg4DBBgAUmFrAMhKVliIQeOM1O2m/KqAyEjMdqK2zc3D4mce/v7m/J1/9171lm8eFCNV3/cfvWcb03fZWvIqFQHO6znOws+5aSZj1+N8K8GMRi8cEvoldOuAwt56om4PPAKkg7vhVSJp9sbj+lEw49MFZrtFB2uqIFNgUXFEpCC0NNZQvnVnONJeYbNJKX0d4a3BXo/QtoiWUhkkLX1//bfzUApukOyx+U6q5fQJlv0Cgx0X6xy7ZifBRjH/fvXbzfUNeovvCybyZqx3+Fjt3iZPw9wZojiyLb+Nwe72batRn8Ili9HTs12VY7aYrmDz1EX49ZaTqx07zjIvla8vfZdej8VeDPl0h3LiXgH1jov/8nXDTWGOyIeaZOzbDf4Sa0KfKLvRj6sGK74J6hauuPcM6SqQoo41+os2Sorlv845JnecaUPzv7zyfztrPmViQUn4owVTdIN5UJHBc975DcIiR1qiEbZUdMm0UVtr2U8pLEpsVr5Zf4MD6uMwQBKcUlOh6QqhXb0WkarThbzRJxvMqTCdnJQabz+QcdZMU5v1Lv847mN453SBK24yuBM/owXmO6XIO1vazeB/10mOqCdFtiPj27I1o/BiwQgMEphTKpCcQ9+ITkOv5lw0fsRm+hf7wFaGt75xGVusRWJ1stCp2ySNSBSFN6igWuOl70tEpJXfMMAspEi+kUt0SYnMR8I+HlZ0H5Xr+RwxgamH8JTSCIow7YsmF4gJbbAwNRphG9+wox7xfPhOBVVxuIfMWrfG1Tm14wnQytq2MGH3d2YE1bo+/cNTEARdU9VtUFFipSl6Sw0GTd3X3DZLvXgjl/rVjUuqfTkAf+nTwVq1AqP31AkLx+Gig+ZIJxm6TuLCeVq0udDLtMqzP+O3/p5fX37nAy6u7VtrXUNPgHtMDOJy6c5r2NcGdgeTrD23wPf07twh+3t/sLNRzhiAPopTQpwxgDzOKaNHsp72TF7/40z2n4ldNc2BPO36yvnfs2Cvq2eD3TpVqPRpqCmaMiv26WRLdf+fhhnYfukK7p+GHK5yZjLoR/0c0ds1nJ4RYquIE3WjIsbEcYil1ZhqyfF8OS2nRw2LTUu2BaV56iKQ8bBFt22iayRJ84EeMlASnmZF9PSQAfQDVsQ4FaevM+8Pxg2Sz5FrsM1I5EMBCj6afGQKtdpHBxo1WjX793/a7hq1F1IQ+zhgI5+7ZTsibqBJXUJx2KXuhV3GJb907vMbufRjXX0VA/SSsyaIol5QDba+YPc0R5rCpN2dH++uoccNlvoQBrCfbLA0hzAA/ahDGXoC4/uXjmPMwb4eQZPH0SBii4U9fPlrnVfqOZL3OVJT0XQe5nKpQ2zT8SF9OfRlxzDY4EejhL2+Wf/Q9gMcue594g52b+SXStz1T6nJ+9P/u+RNXPvkadyXC86R1vWW5QijJVtT0TjJvlxFwJLoOP9FWgskf47K35cR0Rh1aMhymyn6OcFZd4OHcMCwb9/M78r3FLuBi3TmvdkGuwprgocSZE7r5NGP18J89xOSCv3CJTbfv95N8yJSLNiyUuP5Le2+j1F3v+B9Qxj0uZZNgmU8Qc+MseyYuproS3cwSLXBKk+m1O2fVO8Ukk87+h5GinI8TE1zrVX9I+rR9s0wgVN12+VDKrZkAvP6N7vaygE6pNK/9iRGXN98+ilAAhTsJosikKDBaEjlGK9Py6hDxfHY12dFcZ6wvH7HtIOl0PXlU6KkDt9usBTAHBcrfdZONk6y5H423OTgtooWXBRrulxIzqFv6pcogC31TpBzY3mOaUQc6erxcB1F9Y0cjrMYJ/QztPgKMn8uqmohtakL9+bbwaE1k7gsQM2Kkm/9OdkvQzIzxWSFNMspevEnZFaqQq9//PEl2mA/SqheZQ8lnoXy+gBK+Lk6yUhBvhiucENVap9C03fVXmUdhIBe4Llc0w4xWLhEpxZv2iiKi9H7Q74YtjkxqWjOjmqacIhQX4U0x8axwBaImbrvD4j0V65NaI30cJzV3xDUi2ypQq/RlSC41BXHTbOyR8n1EPQnBj8CuZWhVb5/jf7FbvcMff89+hdEpLL6sus5UA9T++/c/E/7RabRLlHC7S+EzOmztXXFhmYEcz7H5C596VNOhTT1aDSwKywR65oXME3GptIBcyRvZgQsAw23MQeM3Rx7I5XVrMXWaR32g04zihBSCC1kJXL7wnAYyKChI8DDkhd3b8QAcoxYoL8Oe8JGI6ew5RLnz+Wd8+ggzf6AYZSKkYDV4U3h7pfBFnbPfS2E7bOPTavRykV9bDP0q9zYoxnanEwgqawxZiS6o7Q8QLRn8eJ9IURzgymydcqB51e15IGxVG4+tYBJ/B27cM0UjEy9vtz1vYuAi6M70x2I4Xbhr/r1JVJWWmtwqAxni4xO/28okaye+eSU2J1HMpIvlyQUNBT8bfOr99ANv5nRTBTFfhDQiKC0/9SBmC8g8OJXynTJWeruJc/WnNcsVSHsE1Okj2sa9VB+h1tn34B6IpDnutpq8U/If40IoxMvg3FBk8ToYQSQVOjm4vzG674EC0seVpRS9TVeBE/kF5cGUT0P98dH91SBIR4adYuGpnzV/qQ12J2eA5b5DL3+8Se0AboXFAuEOQ/7Curq5wVq/UdoQxV1YLFBnGJtkBS9cpFdIp5cTfyyiRi4qynCtp52v0uVA+Egq4mSlZBcLrf9QNyCqYEWi9CPiKywwsQ4IlJoX2SxcBPcUSV8Tg/f8ZmPVtTGLuh2gfqUQYR90xasRVFYJVOKOoyg8GZUpoFk7amVmIDG6mIUwvscJCGVqiFqg0WOVY6EVAXm7I9Qfq9URZA+uc9yOJpED5uFt4dILdYNMq84W1DYccDA15RIkY8o2O1xZ9pM0NA+tCEmiCxKTk2QAUadqBgU+PFG09pgZU7EyLd27SA7j7HyLmeOsl8hRfROyPkgQeLJTQ9EfiLCX4k8BdktyD+kOFH3nHr1WsV06bUf+hQeiKhkN/ocwTBuP4Lct8Otscv35YEFzvepzLbtjwJ/OkhFiVQ5zdO9gz7Jxj9Tulmx1jHqTJvmi934+vC1UrKYAdQKivI1oQIrJp1aX1TcsG8NowrhsuR19Uvby6bAAi9DpbkIcQjv1PaiQ8rhqhEzX2skN8JFxgwuyr5n0GNcT00a3j6jEVkxa93InOoZeltpA2ZSF6jrnjWSl4sNPfKQ9gqwxcLivaZTaEJwyPWCjnZuaJogjiGwVa1ztma51WyAH8KC7LYWZB96xAtv8r5karIdtufpYkH3lhOZ4Vu3WW2FntXXLFLAoPt9oxEP/UC371qezQZLtt3VqtgSqIg+irOhf+yrAhrk54pWk7GS5W7HRa183GAYe1p1G3B10SwBuVijHhqiRlQKdgiaQKYtC5Pg9V0WKXAtswSollkK7bmMKYp2gcYa9dFCTaArdV6R05iQPfMx+MYMnstHvTnHis1Dcu2YYEH7QPS6IcR2BGEyUOJjKNa64idqmi8rQ2RBXzkcGuPFD3AZcAgWngQ7BuQIg9A1Vcykbg061n3ar+6LAMdGk/ZcPhMPbnOvdFPpYqFB3MmNum8Nn7B264I5Yz1VvK6cPpspcACNi5Hlg8mwzSTYIN6hKTIJD+HTrpXetQSlQr/d+tRYpuuEgL5fDdavT2isSlKXUrOIguNBvAXmtMjb7sLN3R3twlNxk6VrXfRIUSSqgipGHiuLgnubaPLzAyrZmpvhxJK734OtranIYU7yQbkl538/QfeaOrQrh9Npu4ilrwUfkBvmAe9FzEn6lL3qvhqdBOvFjPdyrXCTWyykQbiZpBZOoOVymdWJKicR6jUjPlqoT9EzZUf2/TukW0HX6mHb70bxl5yR7RTTdkbkwg0g4JtrC74dkcsVT5k3HSbg+8o3/w+LUykMvU+tsTYIXbejAurqqjzX9l/wqGJeIxRqAHPgcSYrLJY0E3STWhaMBS7pphPqByXEGMXmlaEdCTHM0dcOdautd5+/kaHEJY4m7BrK8cGEjkluDhiC/fwih0xXfwsYt1ABZglWNxzUbc6XWlM1Q7fUHUqlqZrhJYVW3j7TfSFVjcMAdg3G6e0Efo/c7zt9K6RCcyU39rP6r6Se42jNrtF+0tf5DVYmtpuuARzbo+LvlBxUh051pyTP2xmkia6ULKkPKKZ6i88Fwpwq02QXqXZR/zcX3vLio9MEAJKQAgpzjoQU3ypaUrBk9mU/TDEXZbePfmgaitPjXjEXYavDP4Od+aEaraxHl7DgHKpNBJLi26W0/73nJQAlJQsojgn3jTvBwFeAgEVSLhBMmGdUz9BtK1P6gw26lVVpML5w5XyVtkaMKxl1yTa5F7/NNBPCK21qhvT/Mzgm+AnT9iR9TbT3b1jFFz4dV4Em137cDQtb9K4tUzql7OtDhpfF8hKwQFhrSRj4S+1pBO1JOLA37I7+3BlkCIMLz1CpYCbKGaKGfB1WlLHCsQZWHwhiwVLUUKVRiTV08dLQyMFPk5ZFYaWY3AnaD0trqCF71T33HpxK4+ucYYKHyYlvIouyGt7BBMeG0YaJXG58Pq2fNnnWZFKMEmOwzUXF+RZ9rjB3zs9cFpj5Qbyw73ohLkeerq7XM9EA+8FoOCbuaO5rgepEdKzBO+UNFPvJVw1qM5bvOzg+6AqRVNR1Jzs5t0QfgRq9325Phddvpfe8otthu54m6ExVwfqDnVK7WP2anTF5+zXt7yNr2gvG09/xZsu/wGrNNVY0rwhFdeSIht1tbqZ+FnhNkz0itztj/PvvY+cBtC/MqF+Akjt9VMuBGB5jv7p96FZYr5obatXCQJVhRVYu87eusWnKDC9qSL0WYXYjzTIzrYj9VfP/w0pTZOW5QAxy7ipBOMXK/gka4bWo+QLCevJrXdh5OPrghF817PP0rF8sIot5M753sfNg+bJR9YjXa81Upaf29HW1EUBg3OM3TYA0cCUu3OquJ+O4p9RZcNMNrnVe5utLP4IbvfCNG+rZlK7o1+L2MqxXOwf0qQb8e/fz9WV3vmsjJobeg92InEsDdFuYOSaysmDDdNhIXettyl72u1FdX6Dt1IW9fmzhjO+Jxx1fNAuj68uDmmws/9wBTdYi9lrkrUY7QxeuPtP3O+Xug/3aLCCodr/x3VfeHTevTFO5KU3zGFWCU+0oI92DspFojRXDcz6oAnRNGZhAJccjgkBToZP2R9k50K6q6laeWUllNYy6vpDZc759dX3T16GRbxnrPApjddlHDhR8cC1kG2lxSKJrYdAtWwoMwmKERUupUjav/XogvyyT3tS6m4SujvCfFpHOXQYuy2WAcd799gExQXiVUyvO/CBb+/MZenF1j4uS05/RjXOIOLAgvWdhvwhE5iaPbYJzqn1awpgxfWdV7iPwekQpXseN+c4/De+ZvtsTcjWKLZdUpRthFybZp24swOMA2ulKUb2SPLfc42z1kUmjO6H3CTwLw9i7l8ov3jsd42XTjOP6MlxG8uDoPJFFmU2cdwWn4nOvYIyr8+/pav6tRUcKqE9dwLgZmVdkzErzaumJssa6mDfSUiroPGDleo3fyJQ4rPINVqfJ0Bt21bfSFfuHyG5ipDXyCytEMXqLSd1POazcWhE0qR0jxbe1gqr2SyFna0Yfaq0o1tFzg7XBpoqlODf+KMz4ycwOu/hc3iOWvxp/v+zLWk2BocXo46DxsbsLFovw1a3fscTT9wZMfjmcu3fMc8aErGLFODt1JHoZ/U5ZSRrT6TDwyP4QGXDqzow7LHHOuZV7SFeEUK0XFUdXdn1EZE61ZYm62W/YsmAip/eRCcCZNsdpnk+ULbAwmGKqRmJOFcQ3C6wYhwyegAfPxd/FEmEg4rf2t8GdiQR8KOeuudCJNGK/OnrR5HOWVOnSF906CTMgmVcR2oT4usPTy5EiQ+fmGr7HqRNKnPLVJHl5X5X7tv0QM6FRTg1mPOBkmMvKdH43sjXJJ8/NrD22uMljAzzGH1JDi5Iny+Y5RzldYB8C8p0v6xi+z9a0WvGaKo63UMhlpH9c0YvAjbQfgNXtf00XdRW489Vrw0wFjRlRcGOtbTBs2PTU6xo1itXx7xAcG9MEsorIorD3KQ0bXTjoiHWSfUsl1yx3/rO6i1xB9WgiVC7J8YHGx3vLfmG81RpJNy8vrBrcl5D0dBpZX6+eVtb/Xc6P9Dsdvb3/Lec+ABO+XSVL1zj3EhKK3cnf3lyj64FC1UUjWddaX12yH4OIhV1NNewyqiH9GH+Yz60OK/dORGRzmaeu+BpU3PWVDo8LsriMqEer+N0SXMhggsrzjgvYlw67BNomHsKWLG9COSNOvCK21TgoA4/w8sdT8pp9l1XKZ6qe7n3z0XXPqQNRkKxxT0nV9SK41K85DZW31l2Y9iVuTOAICXrF812HSFNdideYcTwMZKDGFY6gvnJBlRqZtODu0DG+/nhxN2+sFL4BlAvADrbk0w00W85GJCIrsnmV59vo/hlWZFHrgDpwK02Pa3S+10sVH6JiMmKXg16JXaarKQoSmO5mr7qeq7jKmWkq69q+aB6j0GC7tmLDiZI2vLB/ky5LLDYF15NZ5RefrtALXyvxqeJWV54zDgUckAd2dV9Kbb/5En07dDSIfhTmTsiN2DGENCUVNLNY70IfmbRJ8AQuuH5a6EVd5f7Olya9oUtMtujjqLnG2VzhUxTl+4V3SMwEKjATC4ULujcdo8QKpvam75Owo1zewLLoncxdcnTbFrCTdRZACh3QviBVwBIilYW02zfuHd2gXysBpuRbmVOOXjCxnn1zhpgkZ2hu/0Xtv7DAfKuZnn0Tji8aUmYLjgeT82PrULsa/sUNgkXB1wVyclsPv5KLvY0ajEyKqfvr3ONZt0HQVFlGDiK0LuLK3R5mn97+jhVFH1wC8DfffHr7+/n7q2++cTm3a6wwG+XJjVR3MUuWD16w3+sFuxG2UScYFrGVCF+zE7dLSfMcYGKfi20CE2YhFRWakZgCpONKSoBxEd8LEogPxAKabTAbDid+sncAep/HBmqvT+wSdV3NE10KM8+1UbEr36FeO5lDrPuWRntH65qPdE7SY4td2sFgA5XGF5u0dS++3sWCWLBRR1O91WSO2GO3GuxGFNhmv7wnLJSP7if4eMeFRd7r/++Hq7Yqs5v8dxIWyzs+eo/IXiRPwhx1HHcfflJOkLS1c7Idu/SFaTLa6yw76JP5EtxuA849HJmuW1azKeJhUPS1wIxbWtfNXG68zLi+7Na2QScuaw4augy0MBjPKqxzrjOrIh6xn2MSryHd2lcfXciiqETfEzXAThzXuOmp2L2j9+bfaVinbnDTx2nWT8XtFov832Q4atbiZrBhx0iGJ2M3XHgHOV3pkhEmo2WJTmXBA/YbrMQw6PDcUdeiKDOZShjfvnt7g35zftQ2KTWMyOdJUwlu/+MN+lxRNdK7teIiU7TfqTNtckPHIbpF7+uis2BaV6Olk4gPaReojD1GwAItj3IcHYJqAsGxJ8PN4w9owByrIsFpWbAJ3Au4jFiA3ACt8mhTaXdgxu12tQM6x6avFT4V7pwKsiqwilVW0sDdlngwvvjJ0SdMBulUUWBmq+i8QOgibgFVA3ixhFZLCcDK+d8TQC1x9EkYruNUdPaCoHvGYj84vnNbQa3qGR1pkWECg1Hil59Y2FpENN47gOfLcv2DuDer6O87ERkxKst11L7rHegW8nGRpwcAXnMcXWKIjIolExGLIoegU+RGi2yR6Q0zJLr8ENmCy43GRfzclS5sYdbpoCeIuhCRMZFSnDBRUlXMt9ES3gewS3KXBvga8xS8wsqsVNLILH5ICqCvf8jA4xgfNk92N7lcZnkKYlvA8fPfiMgKfJ8ZE8ttsAvYcjSnCR6FgolESDORDumS64zPeRY7LLoD+08JgUfvDN6BHbsXYhd27KreLuwfE8L+KSHsf04I+38khP3nNLCNLDme0xQipYEe3zwTWVFxUL7n2wTvZA28vEuglxQVZ8uiTKN9Wy0T82XsJCQPmaVQSjT9TOL7RkSmXUJighPUiqSxJi3gNNak3uqqTDCLlIimrDqJqWqksaYHvU8gQow01jBLBRvMmiTAK8HuBRZSU5KACdc/WaokehTWP8nSrCjOE7jVZFFmhCfwYVvACYIkAFfNtya+W9RC1kkgl1WWIKZBFDOMYJ6ggEhneEkF2UbMuurCFphv/6D5PAXe6wzagCaB7NrBpMHaJdYmgT5fluuf0vigdTZn5s9JGo0RncWdFdcDrGR0Ua2TXHOASomKX+WmnY8/2qytDmBqVs7PH9854oCD2pcEuOsmH6+DXAf2gnGawobR2SLFIbJFzOLsXcApdAOdsRKSFLMkoo6V6x9ybcpBM/9IsLUiSWBztqApzBgNjuaC5ixawegubCbScEkh84pTTWQKanvgbJlANslSb7CJOvO/Az2UQR4FsKJLpo3C8T0hLewEGp+iZSpSq2S01tCJXCWSry4z37F4AuhGUVwkUCRdKVAqtNMp15uVZDpzE2bjQ99ihZMweD5SCBsD8trNt48Nl2mDRfQ5x7k280rFGhZYQ6VuVlAKqFV0XOPr0XVNcmywMLlhEX/Y9bGdBvbBXOI8j30HWB47rFq3DkrwFrEiI0rKIklXIgs4gZnGiixNcqTveJSCzOVd9PZMpY7fspSVulQsMlCODTNV9OwzzgSN12KnhaqjTtRp4ELxbXy3Fpeu62m24DL6c94AT5Dyb23e6FLHAk0gcawNnQDV6LkJXC6TsK5YJrnApVSxBVgxr5YprlnBNEkhFgqdhGFTzIEQ1EBzpehwo8tw1wA6dsafgxo7HU9sNrEtkCQVZdINgI5uicr4mpFUbJkF5nE9Ge5GUBX/zSozN5Q3Otiok6lbsG7EaxImS1C46WfixBYGHmxsaVBmzpEUHV2stf0wI6tYdf4D0PS+ZNEDASVVxVJhYQY9d2NA3iQBHP/pdZ3IPn7sTQGNAFjJZYZ1GXFgQBe0wrGhKop5Cv1OUQJ0cF1HEwGPT2QLOW4L1w5kqfIEGMd3ZOoEvmHtfMMJ8gE0jZ0I4AYeJzBONP0cnwFCDVqjQU1gSmm2TCB4dRnby6YVSXEPFMmjK9JakVBX3AiATbwRW12YlY7eVXNNROxCieC02KcCdU06Y2/fLE18tnJA40f0mpmeseFuy+jdWqt8niQPvVI8wVtYaaqynMWuek8ytqKODKUggyHa4CK2N3idMaENXiTQDNZMmRRq+LoUCVo3GakqEdPNGmqLFugoel4Zid5XAg2WbrJHEg7L+4Q5y9GFojkz6AKr3Hcz1ND+PYyOm5yVkEpjE0IBDAzRR9DfgEiOQqU6TT4EE+kod1WUXG7pYLDgQfotZBWtqfcDeczS0PmMYN6Zokt6jwrcb7TQxmLFsuoPA0mOJGcahjPUq/ujhwZKSFdlKZVBw8ajCG1W2CBmUKnoYowVnpCW+5ghFCHCe6ujQQEx4Tu7j/SF5kyknsjfQdWu1sVTIyOX1KyomrXf1ytZDV40hARdU9WMIzISlVhpit5Sg2EiuLuruCHBizdyqV/duLLXl+jSj/g6Q2YVmFIEzYDfUz/6GNAW6B01vzMjqA6f85CpkxBvASO7m1sEi7vNaooVWc2YYEH8YObuBP21e+ITZmFAMsQrjisBs36XFcxxrZu4hxu49/q179lT+nbczZ6aJtx+fvGIsW8PIotY0/SwzquwLPpA7w3cijF3wRTTqEcEUju47h1MqBZ8ZOIldM9NOA4c+udqapCinyuqzZ6m3cdnKz++V75TGWAsj1vVSey+R6rJO911p+zDyWEEsbGdv0OHdv1zcOcxZ/8fnm9oF7u+rIUCrB3mDbAa4iXxPpKF7eMyx5oil67dYIMGt6o5Jf+L0+ArmlHwDeZSufb1QTIihDXSlMK4M7x/XpXCQmMywXjfQYdpt7QAtbdlGlIpmIC2D+mSqoI5dWMqpNsl3WAOtmacLinidE05wlqzpXAH187rD7M+tGQ+ofyG9fdw+vwkk54tZpVgnyvaH5OIw5evg+9xHROPm4JSazQsdxeSSCEo5FagDTOrMUGBUKAypNHYFT2qvOjRpoUlJ8iT5onicskI5shiMGL6ABanxQ6WGhnTeDralautDqPXSWfbyF5Wa+wHHnOGdbaSyW0CZ8Q15hrMUmmHGlmp2B3BE+4HgNylsdjCm+YHsRBOsZqdcy2tIb5z3y4hWI5+9b+YoXOxbf5vAN2ALa+FQTifEVmUlaEqLIaTuPHtxtKZZ1/1zwJmLO4cCDN/q17/6bs/W9v3snMcNcW+CqLt+TSLGzF7qOMGb6lC/9z45PQrjwYgF771set/0vO8aHHe4fq953Fk8vIh2fZ1f2CKXWeG3v324crunSrqnCfgL82ZJoqWWJCt1Sq9esb7uSAIKHSGPrz9GV0L8/3rM3T97vLqP39GH6+F+ekH9GKz2iJBmVlRhchKaj8qTSpFiYFvfffT//pvL78OUoSaVUIZ16cHyNRZgcPjeHRi7nvkNb91vHhdIxW+4vnzQrormw5gfmTDuAc/8CF8e4ppa518YspUmKM35++CyP4hBU3nyzqOM/6PFHQWpq1F94sRobCRw8ITjuA5vsF7zmGJDd3gE4xIB+6+Qed5rsBP67g8hE7z9JKiPDbO+dRYyPXF2xv3Ko2GxwqsJ4x+7DiVnKbq3250fWNRGfF+WRoeOQkiCg3t2uM0rDWxzE3XmlZAdNDFec7slzFvA7adWf7hd25CBrAmIVxw6W/45S4LDFBpc62T6HUPfdIweucxvJHKNCJ5IHRzCLDBATCzPSx59cS0d/thYlk/JvW23o4RXtCQ3TiVF9djB5Yv1loSZlVO5zca6DjIymWFxZLOGtOJSLFgy0rRHM23AJOKHLKGwnKmPLL1wKBodERbDi66SNDvgEfU/bslXNEdAIoW0tDMZ3bHzzOKT9pc6AxnLhU/AejSqDTAFwlYYpGgWpinuA6p+p+UCYiK86z2xKVTy/sWvN3HrL9a15lwAg32yqyoEtSgD9uSnqGP9TP2Bhxg36Ob2gE2eAl+G9PU6lE9EygTI6ZxjbT3i58hzHlQmSjbL0KCG1aQmLemyr6BTBiJtIHHnAn08XpUoBBIkE0mr6KLbAtUlgnGvlnAiurYGb0WbIISF/cixk5FB397AmzdaIWMU7GMPikScLbKR0ItdEQDdSoP5p0AjEAE0gkWCKNfpNpglQ/ndCN0voRkL4WwvfH3kEs3p2ZDqQirnpG7Jj42xi0N5t1QnUMGQct4yIwY7JAJn+cKaQkFM1Ys+REb4S2uORZTxPEf4KCsE0Q6LsrBBnddlm0kZW0t2CUYsLsvT+xIJSXQhWAdrx/cwyL2WBlGKo4Vgn7RqEbixdX9z2/kUi4W4envlGRmRZMf7w6yH+yC7jZ28L6yeFt0zyuzosL4ZPFRtHUVs3PCwxJ63JLjqH/UVI0iLCtD5LSU9kuOI3xbEUK1HsEZOo8f1xztuMQTwAtZFXcp1RYFChMGuE0hnHZwpD0crVSCAJ8upbDvipVbIeWw+SEaKEq7u1rH60c38m5i5LqWQs0AZzRv9uP9MD19mAmkmakC8hNBcQH1ItpDXWGNcC5L+7qYFWUKyY1oj8wRzuB7KWQxklcLMzk0cy3qp1UirHLPRG7lj1S6IQBGvzBO0blHbDYgw0OcvaLZmLuTownjzf5Pkq4wSoJbn7UQlwqhPQYIEbPe/QmEcPl6t75eIzYlxhNC5zJl9UBg83O6wmsmK9AuiSxKJQs2kqFIp0buSuA5hyKyBbrYjxsT60bsJESyj+GO1omCCOxgGHW4zBEIBtZv8Et9up1Xtr1vo2zXlllWwvTL2WJr9DmUgWfkGLP+QVoQvMdLKqhipN4SEAQS/fqpBcys4KkNzXZDHtkZ+W6mjRoPftZ7Oqbt1sn29Hr/nrx64dZKuK+gadoY4YYVVFu57rQ9RUs6GkTypxCtKcTBg4DGg088BvVA1jqmd/fJWOv7h+3pu0xHG3L64K15h/GhHQ72BjtuBcIDhMGXu7vXB3enJj07d9Gi7E0dPrlovVSnESAH5HgjQL5cdvz+8JHFGm0wzZE9TD6qSSVIzDv2APkxKTvG3NuAGRulHkrQen7q6JU7lVllBTUreYIoCd7xJCOHhv/a6IFDLyUlk3qd9kR13kvu/bUWkT18mcgT8p+zH//0J/TizeX5zUt0ybRhYlkxvaI5lMIHceFyKZP3BdoXCYNs2YXDwx8zfHEkY0zJxF7FffWf9lRDGDQ3Bjzy0YY+P+a6EEj7b+p+O44/wCkUM8Ui1Ca9zRTDPFZ3ut5G3uOcVdqtgKRCmhWMY+XEkxWb9g4ReNfD5VVwzzXLp+w00s2U/2gZofYi9vpitpc8XZ3Fudh31yGs4SsNO/5f7ySCTwa84B03tFOWkYddmVKlTAwYhGyA1FItsWB/7MmqFulY4aHEPoLSXZ4aIfeCqWAtaaKuP7/Y5eC1cC2+XO+inazmXynmZkWwoqhUNJcFEzhYcNcRTzfYMCqMPpgez/GUu32DT7pZ1/qRlokY116dr63gKrEy0Ayp3ep+sTphsyMvbB4iURc0pwobmmfRksr28IcVPr/UKzbBsxsl1yxvmof57+Gy5F5THTCGb/5jn7VdnTas4LSbZPlEu2yW9L3+zHZkm8HhoZA5uWYuer7qK+4jLeAapTPmUPDHap70HnSmzo86ldDLwEadjgoaK9ZIG6mcxLfQCmowrPY1fGtmv/V1ePcFy3NOp5Nyb2G9h8q5wPF25N5Rcq4ejzHNdm/8ap0OQ2JbR2fPUMmxPTL7PkuFqCBqW455+SEVcgJ78gEZdKqxLX+V2qC3mKyYGDHpcpxIcnzVp/VHAZn+paJWfFj9yDU50zP0Jscl+gT/4/SjXApXd/q34eOJVnhNrebEKVboc0XVFkEPQl1KoWmtUYWLU+1+M/jNNPLS98AjFrJidRdI4bbv+vKN41lvaQJUWwZ675ujPhRTmPKU1mHW5/G6tfROEyNrG/qHl2mkKiGCdqw+a14eF3l2baRGauw8xMxbmOkPAqMNE7ncaKRLStiCEfvJWahO0OfJDi+I3Z7Dt825QS+gIywVpH2GIHT5skMtVAl4x9/QJSZb9FHvNr5tIrBFv5A2enatXWECg33kte+aWoAK1KoBk9kXcUDxpg9AoPp/p9IUynmG5NvddnqFeqw7r1OvAzuGHQYZzf/miM1Ok9c7tlWf4etd77Wsu4Ktj3cBHe5mGoddEzDYPZs2IdMdw+CEwg0pDhc/Q9lAzJGAoxVusOWcLpjwvnoQTtDVr8DlSNNBwO6oQrFEuLUOmJ76F1swNj7b1Hv3vZRGelM2PmxjMFkVE7fAb1cFgqOBddQ9jiRDXuZMxJsgFvVu2C1DUWHaxzMgpLplO3Asro12W94fmNo5wDrt23cA6xKrmqfsn8/arWxWbNBKHdnbYW1Zl/z+oO2Z6DNLXFsLqbbpDvwvusTiXw92jKkR2e2iXqvnoafJkuUvrwD6gb2dTCUa7Krut75/V6NckFFhlCyPER25rOYD58KDeNyvaa1teqAcAXB01R3T3sMLWZRYbJv7CNcOxuk7e2VNlX2GMiYWMqwUYH2XukbogPzoWZE1Zhuativ64nOqHIFfKs636D8qzNmC0RxdQt2zcw4GUdnQeUakvGMnCrr/TufIrd/az5iPafPRu8224fCyMqByHznC9PBdf98s4afseHe088nP0Idt6bbeeg4scdwJjh+eoossajPZHtoWB+eIUF/rUNvaPjJTuOoa5XIXO+dZLKWqvf0QYn7/ZuTIO71yIrNTTYsy7RyiPaSwKx/03NdoKikTaSK7SNl17HmgEpuwa5KIDOuY0f4OYOXL6SNDrhSPeMwdqBFPpTFGs0rF8oZ0YGqqMryMZ1O2oKM/T7ugo6Y/7oL2XJ9AsNB7QwWoVvGNEws/Gjc3it5K0V6qTGyNyi0xRS3hjsz9AMuCevXK//eFR+GV/w+f1xRy+2NOVTg7z2/nhNFzt5lu8Bw8rp1Ra4Pt5H4gmjWpmFhQpUbirsN9T7KvruJ/kPRB9+wESNZ9iRedYwhcKQhry6RXKrDEZOx35eL2lu0+QAax6v7pr3SYoDU+8JOVK6qm8UdYnd1nPL24gNGPL9EFrB9GjSozUbOUETpfUOWHf9KdLMw9zXlp0tBxh5CdA7eLfq07naL3njT741iv5ONbo4RPG92yP8LeGnaXSKZc//UKCbqUhrkDLFdYj0yA0mTqtkKdo3SLjw8XtEedbALUIMGlx2N14/S6/iackKLZcoqKit3+Rs3Uww+jg5atNGFaV9GVToAMyVLpvHVPi6EAhlSppD7QwaF0peeVXRzdQnB6n3SaJEOi6Qzuo8gvbiG1c/9j1JGexyH5eOm5B8dxEao1z9YpX/R+SNU7soPI5JllPVxFb9OoUwFmd9Rb1ImaG3zVjivpPkggW39AGuJ1UqHr2/O/vr1BN/adQr+JkekrLbaJKqmPwfbDRoaxBTFEVpTc6aOcyA8Twml7kIWGzjX9OpsWYZAG6kcQtlJwj5ZLFRs0hTyBkuvwaLqCjBoNgLPBpppswmcXyzXmLHeMGECiLwgn62q9TxACxe7oVvfFdiTOrxNII8NeGVPqjMEM2iSg4ShTEITgZ3Cb2FLUlS9SMbM9cKOILIqkfeIeiLfDwzuEwiX4G6Yo71uasV0sG45FpvWpBt7alZ0M/93vtq7RCmLrSo2zUrIp0qpDCDsMEGAASIWtASArWWEhBo0zUreb8qsCIiMx24naNjcPi595+Pub83f+3XvVW755UIxUfd9/9J5tTN9la8mrVAQ4r+c4Cz/nppmMXY/zrQQzGr1wSOiX0K0DCnvribo98AiQDu6GV4mk2RuP60fBjE8XmO0WHaypgkyBRcURkYLQ0lhD+dad4Uh7hc0mpfR1hLcGez1C2yJaSmWQtPT99d/OQym4QbLH5jupltMnWPYLDHZcrHPsmp0EG8X8+9VvN9c36C2+L5jIm7He4WO1e5s8DXNniOLItvw2Brvbt61GfQqXLEZPz3ZVjtliuoLNUxfh11tOrnbsOMu8VL6+9F16PRZ7MeTTHcqJewXUOy7+y9cNN4U5Ih9qkrFvN/hLrAl9ouxGP64arPgmqFu44t4zpKtAijrW6C/aKCmW/zrnmNxxpg3N//LK/+2s+ZSJBSXhjxZM0Q3mQUUGz3nnNwiLHGmJRthS0SXTRm2tZT+lsCixWflm/Q0OqI/DAElwSk2FpiuEdvVaRKpOF/JGn2wwp8Ko7T/93wAAAP//Gm+GWA==" + return "eJzs/W1zHDeSII6/30+Bc8T/LDno1li2tTfe2b3gktKaN5LMFSV5/xcTUYFGobsxRAElANXN9qf/BRKoZ1STbAJFam/8wmGzuxOJRCKRz/k9uqb7XxBhmsh/Qsgww+kv6Mz/b041Uaw0TIpf0L/9E0IIvZN5xSlaSYU2WOScibX7OhLU7KS6RjndMkIRl2u9+CeEVozyXP/yT/Br+8/3SOCC+jUXWOPmE4TMvqS/oLWSVdn5awCN+p83AB3QcVicXp2iN0zRHeZ80flqjUb7lxqPgmqN1zRjeQ+yQ+Wa7ndS9T85gA5CHze0g4mHjVhOhWErRlWLUwAVXa1W7OaOaNAbXJT2tDTVmklxdxx/g79j7tdDeGWoQv8/i/BdEZWVIjRjwlC1woTGoNwVwEQNTDhUs6FoxeUOSYXolgpzEK2casMEtvDj4nbeAn4QgqriNLP/GQOp97igSK4AhVNCqNboTAqjJEdvmTawGDIbbFCBDdnQHJkN03fA0p9upalKgauF6/BiGv7g1vPkvBOG3YOeDc3OovfBtcBlSfOsvjJlAM/BH28VMEZhoTk2NK9pd3GJcJ4rqvU9cNlIbe5MtRWuuMlAjP6CVphr+lCc7fL3wLaUKoQtl2L9UEws6Ltg0pMvkQ+yy133O80uVo91pF3s73quXbxTHG4Xp1tP2GwUxSbjdEt5HD3AwkMAD6RFgfkOK4peoKU0ghqL6WrFyAL9JkDmbKnaf8/l7gTZfw3AFTKnCht6gjZsvbGPDXzd/s9dtkWwoWup9jF2duZhNc/f9M7e2EexVlO2TFX6xH9nuD+j5N+xOEHUkIP7IVIIStwFjKKvfRLsS9VV0GBbGN70g5gwUpSZXTSAhd4M2fkgDhdn7y7hl7cvSGQea0EL6q60nthnGrHy+fJ9Z23UWzukC+AyU5RIlev7IfIADR9rzdaC5uj89BINFw+SsiiwyDPOBM2wWlcFFWY+dP3yyC6PmuWtibamOVru4RpzSTBHuMqZsZ8c2k69/eEjeMc93PeVbJ/DlvBGIuw4hTMqDNIVaMCrivN9wz3i4C5KxbaM0zVdSH5PHj7yLH7fUIEwaJa6Xd7ql2SDxbrW0L2+KXmOtphXB7m/3YSguye4CUF3t29iWSltFnL5d0qGUizdpVDUqQluWRD7gAfaYSWYWB+80A5jNg/bdLG1SgC6OD8KXVIpRYXJLIz5ZI9b1CML6GtKxR2wlWLF1pWi+eMg3K7fwf12tPF2/Tj44i1VeE0fROhHQ75D7MA+MOdyR/O7cHhRcWzYlmZEVmI+YWKkwRzBmlaX7+C+YUYjzQShTqg7abPDGhGrmlsBpBDhFKvOBvs+0pXpYvNwH+kbpmgpd1TVVso5XVGh6RNxnL75eP51OU4twv9wnP7DcfoPx+lX7ThFnzRFr8+u/EcLgc2Clf/wpx7pTw2R8wk7Wht0O5/fgwX+4YS9Hac+Wwzp/A8X7T9ctP9w0fYWvNVFqympFDMhpgl6U9oFB8t9wDuv6QNxrzxc9Nq+0oejUF+5mzgligfdxH0bj0kd1ca7+O2qScGp/5k25TBowRln93i47qgN2ifW6dgW+kFOWmHCeJibD9pxV6/P7ncu9ULISLTbMLJxQtLbnIquqNLo2aoVjSfo6v27yxN09f+/OkFYWEVnAHYlldk8X6DTFjjBAi0pwmiDVQ7i16VGnSCMSiWNJJKfIBBlhcuqkquhzLVK/l4bWiAtV8YCWaALg3IqpKE9I8BLeoIr3dDe/XT4TrltLkaM6BO4Fo2dthhYB3JL1U4xYx8tVdERv44P6ZYrdOCguixUZ5a1BuRuQ5Xzp/iHDG2wRktKBZJLTdWW5uP9qV6q2W2bGV++g1uZvluAtcB9lWV69an1Q0t0tDm9HhzzoRUO3arBqXy0tto13VtbrtIu8EJwaSpPf4V3zcUBm4/Igmq7aWk/H4BG6K1co3NqHzYV3oiDxYZIHbudGi6Ym1bbJZEBe4QTU9+T3N14IoWBAJ5cISa0wcLUaOggjoYVxyCYDz3BIey8jW+XQNh4cYpr35pzf2L0nprfmRH2GfCnvxixRrNZvZEVz5GgW6qsBK35rsRKU/SOGmxRw2ilZNFZ6tlbudYvLjG5pkY/H4E/Z4oSw/cnTXwKow/UCQvH4aKD5iJIyLHtcTdKjkyoASXPaakoAYPJYpLTFbNqgxQc0DJ4ya0SX4axKvR6qGrH5UB/xu/8Pb84/8HF9LyXp1bM3bfoDSYQQXbnpUYHAbtjoLQ5boHv2eMosTKMVBwr+L0/2MUkZ4xAH8UpIc4YQZ7mlMkj2c57Ji//cSaHz8SumuZAHnZ95fLvGWxkeCxPBrstPkboJUdNUaf7PkXcLNlS3f+HYaYNNrSgg+DoE0EO0o8ywvHgDj8R9KgwAw/dE0FsM/I6PRHEmDgOsbQaUy05ni6n5RQfIz3Skm1FXcQglg01odeE7MzOF2u3gMVmpIeMlISHWREDPWQE/RYrYpqKo8DrLFQUHa9KkHyOXKNtRiIfClDw3uQjc6jV1SjmUO/f/2nfN2rPpCD2ccBGPnXLdkLcbFlacdil7pldhq0Ywd37/FauXbyhzmipRE4VOEupF1Sjra/YDc2RppB11ftxfw09bbDUhzCC/WCDpTmEEeh7HcrYExjfv3QcY472dQ+a3I8Go5B6Er78VWrTFZF8yJGaipyJdf2hDrFNx4f09dCXHcNgox9NEvbicvtTk8M/dd2HxB3t3sivlbjbV6nJ++r/XfKOos5JZMNQLjhHWtdbliOM1mxLReMk+3oVAUui4/wXaS2Q/Ckqf19HRGPSoSHLfabolwRn3Q0ewgHDvn292Wu3NLqEi3TivdkGo4/7kiKCxxJkSRFlZkMV+nQhzA+vkFToDZfY/PgSLbEGLqoDZFBNAKrfLfs+Rt39ivcNYdB0xmcE/0IwEW4W67he+at3MEi1w2pUnRlN6+hItM62u5S8uPzc0/cw1K8NjxTVuS3uEfVoQ7o9dZyqHfGgcEaxNYPaC/ebvrZyCx1S6V8HEiMuLj+/CpAgnJODIpCgwWhM5RivT8uoY8Xx2NdnQ3FO1Syx619hKXRx/pAoqcO3GywFMMfFSp+0k42TLLmfDdeK1kWraMFFsabLmeScEiPV1yiALfUeIefG8hzTiDjS0dxi2lNU38qh2oIOEPoJWnwFWT4VVbWQGpLdCinQcj86NIQU/VJRDUVQmhUl3/tzsl+GRF2KyQZpllP07E/IbFSFXv7883MoDdWUimaVA5R4EsrrHSihSyk0TUcK8tVwhSsRrn0KVbF0Qs9eZR2EgJ7hpdzSDjGYCGZW1uJNG0VxMXl/yFfDNo9MKpqzaqinxSDUNyHNsXEssBVi5m/Vyz/98GftRPqLEgRojfTfRrv5m7UH3+I9Veglei0ILjUUwUsBJuW95HoI+gODH4HcytAqP75E/2q3e4J+/BH9KyJSQcsLOCa36An6n9z8i/0i06hPlG+CRyhkHigafiK2rtjRjGDOl5hcp9WAHXJ1wQA2zq6wRKQiLyUTpu4uEkQUmCOjSslE+WmtPqhLShjmgDFgqo1UVrMWe6d12A+2mLPcMUYIKYRWshK5fWE4BeSZWHvl6Nbkxf6NGEGOEQv01+FA2GjiFPZc4vypvHMeHaTZHxQV1ChGAlaHN4W7XwZb2D33tRC2zz42rUYrV/WxLdCvcmePZmxzMoGkssaYkeia0vIWoj2JF+8rIZqSUAy2ZXmWp4q6vq4lz5oKqJrVUFZVOTva24VbpkyFuTXae753EXBxsIJZsxti5UAMtwt/1S/OkbLSWoNDBYiG1Zqa5mu3UkKrRElPj06Jumb/ECVUklDQWPBfnNe+1w+0kIaiK8/vda+c5X5KUCLoNuICMV9B4MWvlOmSs5SZDU/anNdspPY/Cd3MytyE/A63zr4BdZmm57raavFPyH+PCKMTLyvGHyFGb1e1xtHl2eml1319US4rSqmGGi+CJ/KrS4Oonob7w7dpAEN83HwNOVdq35Sv2p+0BrvTc8AyX6CXP79CO6B7QbFAmPOwrwCc+qAmtf4jtKPK9cBD0OYDa4OkGJSL9In46Gri103EwF1NEbb1tPtdqhwIB1lNlGyE5HK9HwbiVkyNtFiEfkZkgxUmxhHRXuo94A9Oc4Eq4XN6eM9nPllRG7ug2wXqUwYRDsQuwaIorJIpRR1GUHg3KdNAsg7USkxAY3UxCuF9DpIQ6PEIELXBIscqR0KqAnP2Ryi/V6oiSJ/cZzkcTSJZLUdP0r2I1GLdIPOCsxWFHQcMfE2JFPmEgt0ed6ZNSj/LgQ0xQWRRcmqCDDDpRMWgwBvFBmKwU2+mzCMx8pVdO8jOU6zc58xJ9iukMJtIx9TWp8bKeWmznPJHIvxrkacguwX5hxSpuy0cEIt29VrFdOm1H4cUHomoZDf6FBl6Y/zlQ1uqdKecIj+UBxY434cy257iWNtsy/SIVDnN072DPsnGP1O6WbHWMepMm+aL3fj6+LVSslgA1AqK8jWhAismnVpfVNyw7w2jCuGy5HX1S9vLpsACr0OluQhxCO/02vrUDaUQM99qJHfCRcYMLsqhZ9BjDP03lRwnHzGjEdkwa93InOoFeldpA2ZSF6i9ldhM5OViQ488pIMCbLWyeG/pHJoQHHK9oKMdtIKigjiGwFa1ztmW5VazAX4IC7KrWpB9HBAvvMmbkqnZdtiep4sF3VhOZIbv675XRoK+ZpFyzRkP+kYjHvqkC+fESuNGni1GSzbpZLKKLYGKkSL3UIgN/WNfFdAgv1S0mo2VLHc7Lmrl4w5rBEjkE3wDyP0Qm6gRlYIeQRPItHVhEry+6yIFrmWWANUyS6E9lzFFUR/oy+hQE+hKnVfkcUzIgfkYfGNGz+W93pxjxeZtcu2YYEH7QAy6IcR2BGEyUuJjKNa64qnDThNWlKwMkQV94XBojBfIyh41wESWLxwJegbkBIPQLR21w51tY/XqvgiwE9k55PJJW7w46h3oXumm0sVCg7hTSQlbsdbwCWu3vpX7BE95XTl9NlPgABoXI8vbgonaRZX7IEsQb282z3UIn/tWetcSlAr9duVTY5muEwKGfjXk+8IOBiigXpWkLqVmEQXHnXgLzGmRuw5TkMpf393JLjwVN+OG2Y8likRVUMXIfWVRcG8zVLEd2Fi3kq25GU4sufs92tqWilwqnzB7cGdy+fdH6F5Th3YDbc27iKWvBR+R20rQw4g5SZ+yV9034wvpq/69mPFerg1ucouFNAjDmAiLZDiBlst1VieqPIpQrxnx3kJ9jp4pPdn3H5BuBV2r++MOu1iVkjOyT317DsiFS0DAN9cWfD8hl4PDlhIT8EPFKSAWFqdSGHqTWmNtELoQzl/X9kPFea7tv+BRhVFvgFCoAcwtj7ObkpkNx3UmkAVTgct6JGfTKwQbo9iyMrQjIcY5+n7Ap9XWu89fWHTocjhB7OFWixv1Ov/NAUNwmF/k58529LeAcQsVYJZgdcNB3eZ8qS1VC3RF3aFUmqoFXlNo5e0z3VdS1TiMYNdgnN5O3NAt9/tO3wqp0FLJnf2s/qvXNZ3ZNdlP+iK/xMrEdtM1gGN7VPydGs7xne9ONbN6E14pWVIfUEz1Fp8KhDlVpskuUu2i/m8uvOXFR6cJACQhBRTmHAkpvle0pGDJHMp+ALNhzienHj7a2CumGdD5grkIWx3+Ge1sx8zGK8tO1qNzWHAJ1SYCSfH9Wtr/PvASgJKSBRTHhPvGnWDgC0DAIilXyEoHw6heoKtWpgwHG3Qrq9JgfObK+SptjRhXMuqSbXIvfj3hMSK80qZmSP8/o2OCnzBtT9LXRHv/hlV84dNpFWh27cfdsLBF79oypVPKvr3N8LJYngMWCGstCQN/qT2NoD0JB/aWXdNfEEblZq8ZwRzlTF+foFLBTBQYJfZtWFHGCh9Te3nPh97V2ShcUAPDzLGGLl4aGjm4XgT16HzZC9qPS2t6U9HQ+Gly78FjaXydM0zwMDnxTWRRVuM7mODYMNoxkcudz6clUhBampMmk2KSGKNtrirO9+hLhblzfuaywEx4qSE6C3E58XR1vZ6x1KUDW7cq4Vsmrmnua4HqRHSswTvlDRT7yTcNaguWHzo4PuoKkVTUdSc7ObfEEIEavd+uHguv30rveUVX43Y9TdCZqoINBzuldrH6NQFbx/+HNe0fI2vaK8bT3/Fmy29gteYaK5pXhKI6ckTD7jZNFcM8C7ymyR6RK1iyVpuH72PnAbQvzKRfgJJrfVTLgRgeY7+6feg2WG+aG2rVwkCVYUU2LvO3rrFpygzPakiDFmF2I80yC60IDL6v/39caYqsPBeIQc5dJWBEvv0TNMJrUfMFhO0QPFfYeXv0wQm/atzn6Um/WEQWy3qerlz1HixfNqru8XrBwNe5PX1dbQQQmPb4zRMgDVyJM7e668k47Sl1Flxy13hDPudlvjhH752keeYbNyA3bc8X/Vrcnof1aueAfgxffsf9fHEOJPUlb42YGHsP+hE5lwbotrBwTGRlwY7psJG61fuUvez7UV1foO3UhYN+7InhyIkv3Vk7Kffi/FZNNpZ/7hZN1iL2UuStRrtAZ64+0/c75e6Dw9osIKj63/jhG++OW1amqdyUpnmMKsGpdpSR7kHZSbTFiuElH1UBuqYMTKCS4wlBoKnQSfuj9A60q6q6lRdWUlkNo64vZPacr15cXA51aORbxjqPwlRd9pEDBe9cC9lGWhyS6EIYdMXWAoOwmGDRUqqUzWu/Hckvy6SXte4moasj/KdFpDt82nJZLgOM8/63j4gJwqucWnHmB9m6QfjPXtcDjC+dQ8SBBem9CPtFIDI3e2wTnFPt0xLGjOlrq3Ifgdc9SvE6bsz3/mn4wPT1gZCrUWy9pirdCLswyT53YwEeBzeiWVG9kTy33ONs9YlJo73Q+wyehXHs3UvlZx+cjvG8acZxcR4uI7lzdJ7IosxmzruCU/G5VzDG1fn3dLX83qIjBdSnrtxs7rwiU1aaV0sfKWusi3kjLaWCzgNWrtf4TUyJ84PIH0UBHHfVX8Hsc/cQ2U1MtEZ+ZoUoRu8wqfsph5VbK4JmtWOk+L5WUNVhKeRszehDrRXFOnpusDbYVLEU58YfhRl/NLPDLr6UN4jlL6bfL/uyVnNgaDH6NGp87O6CxSJ8det3LPH0vRGTn4/n7h3znDEhq1gxzk4diV5Hv1NWksZ0Oow8sj9FBpy6M2OPJU45t3IP6YoQqvWq4ui1XR8RmVNtWaJu9hu2LJjI6U1kAnCmzXGa5wNlCywMppiqkVhSBfHNAivGIYMn4MFz8XexRhiI+L39bXBnIgEfyqVrLvRIGrFfHT1r8jlLqnTpi26dhBmRzKsIbUJ83eHp+USRoXNzjd/j1AklTvlqkry8r8p9236ImdAopwYzHnAyLGVlOr+b2Jrks+dm1h5b3OSxAR7TD6mhRcmTZfOcopyusA8B+c6XdQzfZ2tarXhLFcd7KOQy0j+u6FngRtoPwOr2v6arugrc+eq1YaaCxowouLHWNhg3bHrodY0axer4dwiOjWkCWUVkUdj7lIaNzhx0xDrJvqWSW5Y7/1ndRa6gejIRKpfk+EDj/b1lbxhvtUbSzcsLqwY3JSQ9PY6sr1dPK+v/LpdH+p2O3t7/kUsfgAnfrpKla5x7DgnF7uSvLi/QxUih6qKRrGutry45jEHEwq6mGnYd1ZC+jz/M51aHlXsnIrKlzFNXfI0q7oZKh8cFWVwm1KNN/G4JLmQwQ+V5xwXsS4ddAm0TD2FrljehnAknXhHbahyVgUd4+eMpec2+yyrlM1VP97785Lrn1IEoSNa4oaTqehFc6teShspb6y5MhxI3ZnCEBL3ied8h0lRX4i1mHI8DGahxhSOor1xRpSYmLbg7dIyvP17czRsrhW8A5QKwoy35dAPN1osJiciKbFnl+T66f4YVWdQ6oA7cStPjGp0f9FLFh6iYjNjlYFBil+lqjoIEprvZq67nKq5yZprKurYvmscoNNiurdhwoqQNLxzepMsSi03B7WxW+dnn1+iZr5X4XHGrKy8ZhwIOyAN7fVNKbb/5HH0/djSIYRTmWsid6BlCmpIKmlls+9AnJm0SPIMLbpgWelZXub/3pUlv6RqTPfo0aa5xtlT4MYry/cI9EjOBCszESuGCHkzHKLGCqb3p+yT0lMtLWBa9l7lLjm7bAnayzgJIoVu0L0gVsIRIZSH1+8a9pzv0ayXAlHwnc8rRMya2i+9OEJPkBC3tv6j9FxaY7zXTi+/C8UVDymzF8Whyfmwdqq/hn10iWBR8XSAn9/XwK7k62KjByKSYur8uPZ51GwRNlWXkIELbIq7cHWD2+d3vWFH00SUAf/fd53e/n354/d13Lud2ixVmkzy5k+o6ZsnyrRfs93rBboRt0gmGRWwlwtfsxO1S0jwHmNjnYp/AhFlJRYVmJKYA6biSEmBcxPeCBOIDsYBmO8zGw4kf7B2A3uexgdrrE7tEXVfLRJfCLHNtVOzKd6jXTuYQ676l0d7RuuYjnZP02GKXdjDYSKXxxSZt3Yuvd7EgVmzS0VRvNZkj9titBrsRBbY5LO8JC+Wj+wne33Fhkff6/4fxqq3K7Cb/PQqL5R0fvUfkIJKPwhx1HPcQflLOkLTVO9mOXfrMNBntdZYd9Ml8Dm63EefeHpmuW1azOeJhUPS1woxbWtfNXC69zLg479a2QScuaw4aug60MJjOKqxzrjOrIh6xn2MSryHd2lcfncmiqMTQEzXCThzXuOmh2L2nN+Y/aFinbnDTx2nWD8XtCov832U4atbiZrBhx0iGB2M3XriHnK50yQiT0bJE57LgAfsdVmIcdHjqqGtRlJlMJYyv3r+7RL85P2qblBpG5MusqQRX//kWfamomujdWnGRKTrs1Jk2uaHjEN2jD3XRWTCtq9HSScSHtAtUxh4jYIGWRzmOboNqAsGxB8PN4w9owByrIsFpWbAJ3Au4jFiA3ACt8mhTaXsw43a76oHOsRlqhQ+Fu6SCbAqsYpWVNHD3JR6NL35w9AmTUTpVFJjZJjovELqKW0DVAF6todVSArBy+fcEUEscfRKG6zgVnb0g6J6x2A+O79xWUKt6RkdaZJjAYJT45ScWthYRjfcO4OW63P4kbswm+vtOREaMynIdte96B7qFfFzk6Q6AtxxHlxgio2LNRMSiyDHoFLnRIltlescMiS4/RLbicqdxET93pQtbmG066AmiLkRkTKQUJ0yUVBXLfbSE9xHsklynAb7FPAWvsDIrlTQyix+SAujbnzLwOMaHzZPdTS7XWZ6C2BZw/Pw3IrIC32TGxHIb9AFbjuY0waNQMJEIaSbSIV1ynfElz2KHRXuw/5QQePTO4B3YsXshdmHHrurtwv45IexXCWH/c0LY/ysh7D+ngW1kyfGSphApDfT45pnIioqD8r3cJ3gna+DldQK9pKg4WxdlGu3bapmYr2MnIXnILIVSoukXEt83IjLtEhITnKBWJI01aQGnsSb1XldlglmkRDRl1UlMVSONNT3oTQIRYqSxhlkq2GDWJAFeCXYjsJCakgRMuH1lqZLoUdi+kqXZUJwncKvJoswIT+DDtoATBEkArlruTXy3qIWsk0AuqyxBTIMoZhjBPEEBkc7wmgqyj5h11YUtMN//QfNlCry3GbQBTQLZtYNJg7VLrE0Cfbkut6/S+KB1tmTmz0kajRGdxZ0VNwCsZHRRrZNcc4BKiYpf5aadjz/arK0OYGo2zs8f3znigIPalwS46yYfr4NcB/aKcZrChtHZKsUhslXM4uw+4BS6gc5YCUmKWRJRx8rtT7k25aiZfyTYWpEksDlb0RRmjAZHc0FzFq1gtA+biTRcUsi84lQTmYLaHjhbJ5BNstQ7bKLO/O9AD2WQRwGs6Jppo3B8T0gLO4HGp2iZitQqGa01dCJXieSry8x3LJ4AulEUFwkUSVcKlArtdMr1biOZztyE2fjQ91jhJAyeTxTCxoC8dfPtY8Nl2mARfc5xrs2yUrGGBdZQqZsVlAJqFR3X+Hp0XZMcGyxMbljFH3Z9bKeBQzDXOM9j3wGWxw6r1q2DErxFrMiIkrJI0pXIAk5gprEiS5Mc6TsepSBzeR29PVOp47csZaUuFYsMlGPDTBU9+4wzQeO12Gmh6qgTdRq4UHwb363Fpet6mq24jP6cN8ATpPxbmze61LFAE0gca0MnQDV6bgKX6ySsK9ZJLnApVWwBViyrdYprVjBNUoiFQidh2BRzIAQ10FwpOtzoMtw1gI6d8eegxk7HE7tdbAskSUWZdAOgo1uiMr5mJBVbZ4F5XA+GuxNUxX+zyswN5Y0ONupk6hasG/GahMkSFG76mTixhYEHG1salJlzJEVHF2ttP8zIJlad/wg0vSlZ9EBASVWxVliYUc/dGJB3SQDHf3pdJ7JPnwZTQCMAVnKdYV1GHBjQBa1wbKiKYp5Cv1OUAB1c19FEwOMT2UKO28K1A1mqPAHG8R2ZOoFvWDvfcIJ8AE1jJwK4gccJjBNNv8RngFCD1mhQE5hSmq0TCF5dxvayaUVS3ANF8uiKtFYk1BU3AmATb8RWF2alo3fV3BIRu1AiOC32oUBdk87Y2zdrE5+tHND4Eb1mpmdsuPsyerfWKl8myUOvFE/wFlaaqixnsavek4ytqCNDKchgiDa4iO0N3mZMaINXCTSDLVMmhRq+LUWC1k1GqkrEdLOG2qIFOoqeVkaiD5VAo6Wb7JGEw/I+Y85ydKZozgw6wyr33Qw1tH8Po+MmZyWk0tSEUAADQ/QR9DcgkqNQqU6TD8FEOsq9Lkou93Q0WPBW+q1kFa2p9x15zNLQ+Yxg3pmia3qDCjxstNDGYsW6Gg4DSY4kZxqGM9Sr+6OHBkpIV2UplUHjxqMI7TbYIGZQqehqihUekJZ7nyEUIcJ7q6NBATHhO7tP9IXmTKSeyN9B1a7WxVMjI9fUbKhatN/XG1mNXjSEBN1S1YwjMhKVWGmK3lGDYSK4u6u4IcGzt3KtX1y6stfn6NyP+DpBZhOYUgTNgD9QP/oY0BboPTW/MyOoDp/zmKmTEG8FI7ubWwSLu81qihXZLJhgQfxg5u4M/bUH4hNmYUAyxAuOKwGzftcVzHGtm7iHG7gP+rUf2FP6dtzNnpom3H5+8YSxbw8ii1jTdLfOq7As+khvDNyKKXfBHNOoJwRSO7juPUyoFnxi4iV0z004Dhz652pqkKJfKqrNgabdx2cr379XvlMZYCyPW9VJ7KFHqsk77btTDuHkMILYWO/v0KFd/xLceczZ/7fPN7SLXZzXQgHWDvMGWA3xknjvycL2cVliTZFL126wQaNb1ZyS/8Xj4CuaUfAN5lK59vVBMiKENdKUwrgzfHhelcJCYzLDeN9Rh2m3tAC1t2UaUimYgHYI6ZKqgjl1Yy6k2yXdYA62ZZyuKeJ0SznCWrO1cAfXzusPsz60ZH5E+Q3rH+D05aNMeraYVYJ9qehwTCIOX74Ovsd1TDxuCkqt0bDcXUgihaCQW4F2zGymBAVCgcqQRmNX9KjyonubFpacIE+aJ4rLNSOYI4vBhOkDWDwudrDUxJjGx6NdudnrMHqddLadHGS1xn7gMWdYZxuZ3CZwRlxjrsEslXaokZWK3RE84X4AyF0aiy28aX4QC+EUq8Up19Ia4r37dg7BcvSr/8UCnYp9838j6AZseS0MwvmCyKKsDFVhMZzEjW83ls48+2Z4FjBjsXcgzPytevmnH/5sbd/zznHUFPsmiLbn0yxuxOyujhu8pwr9c+OT0y88GoBc+NbHrv9Jz/OixbnH9QfP48jk5dtk27fDgSl2nQV6/9vH13bvVFHnPAF/ac40UbTEguytVunVMz7MBUFAoRP08d0v6EKYH1+eoIv356//6xf06UKYVz+hZ7vNHgnKzIYqRDZS+1FpUilKDHzrh1f/+388/zZIEWo2CWXckB4gUxcFDo/j0Ym5757X/Mrx4kWNVPiK508L6a5sugXzIxvG3fmBD+E7UExb6+QzU6bCHL09fR9E9g8paDpf1nGc8X+loIswbS26X40IhY3cLjzhCJ7iG3zgHNbY0B1+hBHpwN2X6DTPFfhpHZeH0GmeXlKUx8Y5HxoLuTh7d+lepcnwWIH1jNGPnlPJaar+7UYXlxaVCe+XpeGRkyCi0NCuPU3DWhPL3HSteQVEB12c58x+GfM2YNuZ5R9+52ZkAGsSwgWX/oaf91lghEqba51Er7vrk4bRe4/hpVSmEckjoZtDgA0OgJn97ZJXz0x7tx8m1vVjUm/r3RThBQ3ZjXN5cT12YPlirSVhVuV0fqORjoOsXFZYrOmiMZ2IFCu2rhTN0XIPMKnIIWsoLGfKI1sPjIpGJ7Tl4KKrBP0OeETdv1vCFd0BoGghDc18Znf8PKP4pM2FznDmUvETgC6NSgN8lYAlVgmqhXmK65Cq/0mZgKg4z2pPXDq1fGjB230shqt1nQmPoMG+NhuqBDXo476kJ+hT/Yy9BQfYj+iydoCNXoLfpjS1elTPDMrEhGlcI+394icIcx5UJsr2i5DghhUk5m2psm8gE0YibeAxZwJ9upgUKAQSZJPJq+gi2wKVZYKxbxawojp2Rq8Fm6DExb2IsVPRwd+eAFs3WiHjVKyjT4oEnK3ykVALndBAncqDeScAIxCBdIIVwuiNVDus8vGcboRO15DspRC2N/4GcumW1OwoFWHVM3LXxPvGuKXBvBuqc8ggaBkPmRGjHTLh81whLaFgxoolP2IjvMUtx2KOOP4dHJR1gkjHRTnaYN9l2UZSttaCXYMB2395YkcqKYEuBNt4/eDuFrHHyjBScawQ9ItGNRLPXt/88lau5WoVnv5OSWY2NPnx9pD9aBd0t7GD92uLt0X3tDIbKoxPFp9EW1cxOyfcLaHHLTmN+idN1STCsjJEzktpv+Q0wlcVIVTrCZyh8/hxzdGOSzwBvJBVcddS7VGgMGGE2xzCqYcjHeBopRIE+HQphX1XrNwKKYfND9FIUervahuvH93Eu4mR61oKNQOc0bzZj/fDDPRhJpBmpgrITwTFBdSLaA91gzXCuSzt62I2lCkkd6I9Mkc4g2+kkMVEXi3M5NDMtaifV4mwyj0TuZU/UumGABi9YZyiU4/YYkSGuzh7RbMxdycnE8ab/T9KusIkCa581kJcKoT2GCBEzHr3BxDC5etd+XqN2JSYTghdypTVA4HNL+kGb5msQLsksiiVLNhEhiKdG7nXAi85FJGt0Nlh3JjYNmInIZJDDHtaJwoi0MMw6nCZIxAMrN/gl/p0O69se98m2a4ts6yEGZazxdbocygDz8gxZv2dtCB4j9dUUMVIvSUgCCT6DVMLmNnAUxua7YY8sgvyw0IbNR38rPd0TNutR9vTy8N78uqFWyvhvoKmaWOEG1ZQbeW60/YULelkEMmfQrSmELceBDQefOAxqDuy1jG9ux+NtX68255+yHS0Iad33pp3GN+2w9HeYMetQLiDMPh6d/fy1t2pWc/OXbQoe1O3n1y0XqrzCJBb5HgjQL5edvzx9iOLNdpgniO7m3xUs0qQmHfsDvJjVnaMubcRMzZKPZSgDfzU0St3KrPJCmo28hGiJLjnSUYODf+1yQOHXkpKJvU6HYjqfJDc+2stIgf4MpEn5L8WP//pT+jZ2/PTy+fonGnDxLpiekNzKIUP4sLlWibvC3QoEgbZsiuHhz9m+OJExpiSib2Kh+o/7amGMGhuDHjkow19vs91IZD239T9dhx/gFMoZopFqE16mymGeazudIONfMA5q7RbAUmFNCsYx8qJJys27R0i8K6Hy6vgnmuWz9lppJsp/8kyQu1FHPTFbC95ujqLU3HorkNYw1cadvy/3kkEn4x4wTtuaKcsIw+7MqVKmRgwCtkAqaVaY8H+OJBVLdKxwl2JfQSluzw1Qe4VU8Fa0kRdf97Y5eC1cC2+XO+iXlbzrxRzsyFYUVQqmsuCCRwsuOuIp0tsGBVG35oez/Gcu32LH3WzrvUjLRMxrr0631rBVWJloBlSu9XDYnXGZkde2NxFoq5oThU2NM+iJZUd4A8rfN7UKzbBs0sltyxvmof57+Gy5F5THTGGb/5jn7W+ThtWcNpNsnymXTZL+l5/Zj+xzeDwUMic3DIXPd8MFfeJFnCN0hlzKPh9NU96AzpT50edSuh1YKNORwWNFWukjVRO4ltoBTUYVvsWvrWw3/o2vPuC5Tmn80m5d7DeXeVc4Hg7cu8oOVePx5hnu5d+tU6HIbGvo7MnqOTYHpl9n6VCVBC1L6e8/JAKOYM9eYcMOtXYlr9KbdA7TDZMTJh0OU4kOb4Z0vqTgEz/UlErPqx+5Jqc6QV6m+MSfYb/cfpRLoWrO/3b+PFEG7ylVnPiFCv0paJqj6AHoS6l0LTWqMLFqXa/GfxmHnnpe+ARC1mxugukcNt3ffmm8ay3NAOqLQN98M1R74opTHlK6zAb8njdWrrXxMjahv7hZRqpSoigHatPmpfHRZ5dG6mJGjsPMfMWZvqDwGjHRC53GumSErZixH5yEqoT9Hmy4wtit+fwbXNu0DPoCEsFaZ8hCF0+71ALVQLe8bd0jckefdL9xrdNBLYYFtJGz661K8xgsE+89l1TC1CBWjVgMvsijije9AEIVP/3Kk2hnGdMvv620yvUU915nXod2DHsMMho/jdHbHaevN6prfoMX+96r2Xda9j6dBfQ8W7mcdg1AYP+2bQJme4YRicUbkhxe/EzlA3EHAk4WeEGW87pignvqwfhBF39ClxONB0E7I4qFEuEW+uAGah/sQVj47NNvXffS2miN2XjwzYGk00xcwv8dlUgOBpZR93jSDLkZclEvAliUe+G3TIUFaZ9PANCqlu2A8fi2mi35f2BqZ0jrNO+fbdgXWJV85T980m7ld2GjVqpI3s7rC3rkt/vtD0TfWaJa2sh1T7dgf9Fl1j8260dY2pE+l3Ua/U89DRZsvzlBUC/ZW+PphKNdlX3Wz+8q0kuyKgwSpbHiI5cVsuRc+FOPO7XtNY2vaUcAXB01R3z3sMzWZRY7Jv7CNcOxuk7e2VLlX2GMiZWMqwUYH2dukboFvkxsCJrzHY0bVf01ZdUOQJvKs736D8rzNmK0RydQ92zcw4GUdnRZUakvGaPFHT/nS6RW7+1nzGf0uajd5ttw+FlZUDlPnKE6e13/UOzhJ+y493Rzie/QB/3pdt66zmwxHEnOH14iq6yqM1kB2hbHJwjQn2rQ21rh8jM4aprlMs+ds6zWEpVe/shxPzh7cSRd3rlRGanmhZl2jlEB0hhV77Vc1+jqaRMpIn0kbLr2PNAJTZh1yQRGdYxo/0dwMqX00eGXCke8Zg7UCOeSmOMZpWK5Q3pwNRUZXgdz6ZsQUd/nvqgo6Y/9kF7rk8gWOiNoQJUq/jGiYUfjZsbRW+j6CBVJrZG5ZaYo5awJ3M/wrKgXr3w/33mUXjh/8PnNYXc/phTFc7O89t5xOi520w3eA4e186otdF2cj8QzZpUTKyoUhNx1/G+Z9lXV/G/lfRB9+wMSNZ9iVedYwhcKQhry6RXKrDEbOz32sXtLdt9hAxi1f3TX+k4QWt64CcrN1TN44+wOrvPeHp2BqMfn6MzWD+MGlVmpmYpE3Q+o8oP/6S9LMwDzXlp0tBxh5CdA7eLfqs7naIPnjT741iv5P1bo4RPG12xP8LeGnadSKZc/PU1EnQtDXMHWG6wnpgApcncbYU6R+kWnx4uaI862QSoUYLLgMfqxul1/U04IUWz9RwVFf3+Rs3Uw4+Tg5atNGFaV9GVToAMyVLpvHUPi6EAhlSppD7Q0aF0pedruzi6guD0Iek0S4ZE0xncR5GfXUFq5+HHqCM9j0Py/tLzAI7TIlRrnm1TvujDkKp3ZAeRyTPLeriK3qZRpwLMrqm3qBM1N/imHVfSfZBAtv6ENMTrpEIXV6d/fXeJLu07hX4TE9NXWmwTVVIfg+3HnQxjC2KIbCi51kc5ke8mhNP2IAsNnWv6dTYtwiAN1I8gbKXgAS2XKjZqCvkISq7Do+kKMmk0AM4Gm2q2CZ9dLLeYs9wxYgCJoSCcrav1IUEIFLumez0U25E4v04gjQx7Y0ypMwYzaJOAhqNMQRCCn8BtYmtRV75Ixcz+lhtFZFEk7RN3R7wdHt4hFC7B3zFF+dDSjO1i2XEsMq0fa+CtXdnJ8N/9busarSC2rtQ4KyWbI606hLDDAAEGgFTYGgCykg0WYtQ4I3W7Kb8qIDIRs52pbXPzsPiZh7+/PX3v370Xg+WbB8VINfT9R+/ZxvR1tpW8SkWA03qOs/BzbprJ2PU430owo9Ezh4R+Dt06oLC3nqg7AI8A6eBueJVImr31uH4SzPh0gUW/6GBLFWQKrCqOiBSElsYaylfuDCfaK+x2KaWvI7w12OsR2hbRUiqDpKXvr/9+GkrBDZI9Nt9JtZ4/wXJYYNBzsS6xa3YSbBTzH69/u7y4RO/wTcFE3oz1Dh+r3dvsaZi9IYoT2/LbGO3u0LYa9Slcshg9PdtVOWar+Qo2H7sIv95ycrWj5yzzUvni3Hfp9VgcxJDPdyiP3Cug3nHx375uuCnMEflYk4x9u8FfYk3oR8pu9OOqwYpvgrqFK+49QboKpKhjjf6ijZJi/W9Ljsk1Z9rQ/C8v/N9Omk+ZWFES/mjFFN1hHlRk8JJ3foOwyJGWaIItFV0zbdTeWvZzCosSm41v1t/ggIY4jJAEp9RcaLpCaFevRaTqdCFv9MkGcypMJyelxtsPZFw009QWg8s/jfsU3jld4YqbDO7EL2iFea8Uubelfgb/+05yRD0psh0Z35atGYVXK0ZgkMCSUoHkEvpGdBp6Neei8T02M7zYt2xlfOsbl7HFWiRWJwuduk3ShERReIcKqjVe+75ERFr5DQPMQorkW7lG55TIfCLs42FF91G5ns8RE5gGCM8pjaAI075ocoWY0AYLU6MRtvENO+oRz8fvVFAVh3vIrHVrXJ1TO54AbaxtCxN2f2dGUK3r0799CoKgW6q6DSpKrDRF76jBoKn7mttmqWdv5Vq/uHRJtc9H4M99OlirVmD0gTph4ThcdNCc6CRDt0lcOA+LNhd6nVZ59mf8zt/zi/MffMDFtX1rrWvoCXCDiUFcrt15jfvawO5gkrXnFvie7s8dsr/3B7uY5IwR6KM4JcQZI8jTnDJ5JNt5z+TlP87k8JnYVdMcyMOur1z+PQv2unoy2G1ThUofhpqiKbNiH062VPf/YZiB7Zeu4P5hyOEqZyaDftRPEb2+4fSEENtEnKgbFTEmjkMsrcZUS46ny2k5PWpYbFqyrSjNUxeBTIctum0TXSNJmo/0kJGS8DArYqCHjKDfYkVMU3H+OvPhYNwg+Ry5RtuMRD4UoOC9yUfmUKt9dKBRo1Wzf/+nfd+oPZOC2McBG/nULdsJcQNN6hKKwy51z+wyLvmlc5/fyrUf6+qrGKCXnDVBFPWCarT1FbuhOdIUJu32ftxfQ08bLPUhjGA/2GBpDmEE+l6HMvYExvcvHceYo33dgyb3o0HEFgsH+PLXOq/UcyQfcqSmouk8zOVah9im40P6eujLjmGw0Y8mCXtxuf2p7Qc4cd2HxB3t3sivlbjbV6nJ++r/XfImrn3yNB7KBedI63rLcoTRmm2paJxkX68iYEl0nP8irQWSP0Xl7+uIaEw6NGS5zxT9kuCsu8FDOGDYt2/m99r3FLuEi3TivdkGuwprgscSZEnr5NFPF8L88ApJhd5wic2PL/tpXkSKFVtXajq/pd33MeruV7xvCIM+1bJJsIxn6JkxlR1TVxN97Q4GqXZY5cmUusOT6p1C8rmn72GkKMfj1DTXWtU/oh5t3wwTOFW3XT6kYmsmMK9/09dWbqFDKv3rQGLExeXnVwESoGA3WRSBBA1GYyrHeH1aRh0rjse+PhuK84Tl9T3TDpZCF+cPiZI6fLvBUgBzXKz0STvZOMmS+9lwk4PbKlpwUazpciY5h76pX6MAttR7hJwby3NMI+JIV4+H6yiqb+V4nMU0oZ+gxVeQ5VNRVQupTV24t9yPDq2ZxGUBalaUfO/PyX4ZkpkpJhukWU7Rsz8hs1EVevnzz8/RDvtRQvUqByjxJJTXO1DCz9VJRgry1XCFG6pS+xSavqv2KusgBPQML+WWdojBwiU6tXjTRlFcTN4f8tWwzSOTiubsqKYJtxHqm5Dm2DgW2AoxU/f9AZH+wrUJrZEej7P6G4J6kT1V6CV6LQgudcVx06zsXnI9BP2BwY9AbmVolR9fon+12z1BP/6I/hURqay+7HoO1MPU/ic3/2K/yDTqEyXc/kLInD5ZW1fsaEYw50tMrtOXPuVUSFOPRgO7whKxrnkB02RqKh0wR/JmRsAy0HAbc8DYzbE3UlnNWuyd1mE/6DSjCCGF0EpWIrcvDIeBDBo6AtwtebF/I0aQY8QC/XU4EDaaOIU9lzh/Ku+cRwdp9gcMo1SMBKwObwp3vwy2sHvuayFsn31sWo1WrupjW6Bf5c4ezdjmZAJJZY0xI9E1peUtRHsSL95XQjQ3mCLbphx4/rqWPDCWys2nFjCJv2MXbpmCkakX533fuwi4OLoz3YEYbhf+ql+cI2WltQaHyni2yOT0/4YSyeqZH50S/XkkE/lySUJBY8HfNr/6AN3wmxnNRFHsBwFNCEr7Tx2I+QoCL36lTJecpe5e8mTNec1SFcI+MEX6uKZRd+V3uHX2DagnAnmuq60W/4T894gwOvEyGhc0S4weRgBJhS7PTi+97kuwsORhRSnVUONF8ER+dWkQ1dNwf3xyTxUY4qFRt2hsylftT1qD3ek5YJkv0MufX6Ed0L2gWCDMedhXUFc/r1DrP0I7qqgDiw3iFGuDpBiUi/SJ+Ohq4tdNxMBdTRG29bT7XaocCAdZTZRshORyvR8G4lZMjbRYhH5GZIMVJsYRkUL7IouFm+COKuFzenjPZz5ZURu7oNsF6lMGEQ5NW7AWRWGVTCnqMILCu0mZBpJ1oFZiAhqri1EI73OQhFSqhqgNFjlWORJSFZizP0L5vVIVQfrkPsvhaBLdbRbeASK1WDfIvOBsRWHHAQNfUyJFPqFgt8edaTNDQ/vQhpggsig5NUEGmHSiYlDgpxtNa4OVeSRGvrJrB9l5ipX7nDnJfoUU0Tsh56MEiQc3PRD5IxH+tchTkN2C/EOKR+qeU69eq5guvfbjkMIjEZXsRp8iGMbtR5D7drg1dvmhPLDA+T6U2fbDUeAPB6kokSqnebp30CfZ+GdKNyvWOkadadN8sRtfH79WShYLgFpBUb4mVGDFpFPri4ob9r1hVCFclryufml72RRY4HWoNBchDuGd2l50SDlcNWLmW43kTrjImMFFOfQMeozrqUnj22c0IhtmrRuZU71A7yptwEzqAnXdsybycrGhRx7SQQG2Wlm8t3QOTQgOuV7Q0c4NTRPEMQS2qnXOtiy3mg3wQ1iQXdWC7OOAeOFN3pRMzbbD9jxdLOjGciIzfO82q63Qs/qaRQoY9LBvNOKh39Ltu5Zni9GSbXe1KrYEKqKP4mzoH/uqgAb5paLVbKxkudtxUSsfdxjGnlbdBlxdNEtALtaoh4aoEZWCHkETyLR1YRK8vusiBa5llgDVMkuhPZcxRVEfaKxRHy3UBLpS5xV5HBNyYD4G35jRc3mvN+dYsXmbXDsmWNA+EINuCLEdQZiMlPgYirWu+CM1zZeVIbKgLxwOjfHiB7iMOAQLT4KeATnBIHRLFTOpW4NOdZ/2q/siwKnRpAOXz8yD29wr3VS6WGgQd3Kj7lvDJ6zdumDOVE8Vryunz2YKHEDjYmT5aDJsMwk2iHdoikzCQ/jct9K7lqBU6LcrnxrLdJ0QMPSrwfr1CU1VSepSahZRcNyJt8CcFnnbXbi5u5NdeCpusnSti+4pikRVUMXIfWVRcG8zTX6+QyVbczOcWHL3e7S1LRU5zEm+VW7J5d8foXtNHdqV4+m0XcTS14KPyA3zgA8i5iR9yl5130xOgvVixnu5NrjJLRbSINxMUgsn0HK5zupElUcR6jUj3luoz9EzpSf7/gPSraBr9bjtd6P4S87Ifo5pOxNy4RIQ8M21Bd9PyOWKp8ybDhPwQ+Wb/4fFqRSG3qTWWBuELtpRAXV1VZ5r+y94VDGvEQo1gLnlcSYbLNY0E3SXWhZMBS7prhPqByXEGMWWlaEdCTHO0dcOdautd5+/iaHEJY4m7BrK8dGEjlluDhiCw/wih0xXfwsYt1ABZglWNxzUbc6X2lK1QFfUHUqlqVrgNYVW3j7TfSVVjcMIdg3G6e0Efo/c7zt9K6RCSyV39rP6r6Se42jNrsl+0hf5JVYmtpuuARzbo+LvlBxVh851pyTP2xmkia6ULKkPKKZ6i08Fwpwq02QXqXZR/zcX3vLio9MEAJKQAgpzjoQU3ytaUrBkDmU/zDEXpd9HPzQNxelxL5iLsNXhn9HO/FCNVtajc1hwCdUmAknx/Vra/z7wEoCSkgUUx4T7xp1g4AtAwCIpVwgmzDOqF+iqlSnDwQbdyqo0GJ+5cr5KWyPGlYy6ZJvci99mmgnhlTY1Q/r/GR0T/IRpe5K+Jtr7N6ziC59Oq0Czaz/uhoUteteWKZ1S9u1thpfF8hywQFhrSRj4S+1pBO1JOLC37Jr+0hlkCIMLT1CpYCbKCaKGfBtWlLHCsQZW3xLEgqWooUqjEmvo4qWhkYOfJi2Lwkox2Qvaj0trqCEH1T33HjyWxtc5wwQPkxPfRBZlNb6DCY4Nox0Tudz5fFo/bfKkyaSYJMZom6uK8z36UmHunJ+5LDDzg3hh3/VCXE48XV2vZ6IB9qPRcExc09zXAtWJ6FiDd8obKPaTbxrUFiw/dHB81BUiqajrTnZybokhAjV6v109Fl6/ld7ziq7G7XqaoDNVBRsOdkrtYvVrdsbkHda0f4ysaa8YT3/Hmy2/gdWaa6xoXhGK6sgRDbvb3Ez9LPCaJntErnpj/IfvY+cBtC/MpF+Akmt9VMuBGB5jv7p96DZYb5obatXCQJVhRTYu87eusWnKDM9qSIMWYXYjzTILrYj9VfP/40pTZOW5QAxy7ipBOMXK/gka4bWo+QLCevJrXdh5e/TBCb9q3OfpSb9YRBbLZnzvqvdg+bJRdY/Xa8tUpef29HW1EUBg2uM3T4A0cCXO3OquJ+O0p9RZcPMNrnVe5otzP4IbPfONG+rZlK7o1+L2PKxXOwf0Yw349+7ni/PufNdGTIy9B/2InEsDdFtYOCaysmDHdNhI3ep9yl72/aiuL9B26sJBP7ZwxvfM447PmoXRxfmtmmws/9wtmqxF7KXIW412gc5cfabvd8rdB4e1WUBQ9b/xwzfeHbesTFO5KU3zGFWCU+0oI92DspNoixXDSz6qAnRNGZhAJccTgkBToZP2R+kdaFdVdSsvrKSyGkZdX8jsOV+9uLgc6tDIt4x1HoWpuuwjBwreuRayjbQ4JNGFMOiKrQUGYTHBoqVUKZvXfjuSX5ZJL2vdTUJXR/hPi0jnLgOX5TLAOO9/+4iYILzKqRVnfpCt/fkCPXt9g4uS01/QpXOIOLAgvRdhvwhE5maPbYJzqn1awpgxfW1V7iPwukcpXseN+d4/DR+Yvj4QcjWKrddUpRthFybZ524swOMA2ulGUb2RPLfc42z1iUmjvdD7DJ6FcezdS+VnH5yO8bxpxnFxHi4juXN0nsiizGbOu4JT8blXMMbV+fd0tfzeoiMF1KeuYNyMzCsyZaV5tfSRssa6mDfSUiroPGDleo3fxJQ4rPIdVo+ToTfuqm+lK/YPkd3ERGvkZ1aIYvQOk7qfcli5tSJoVjtGiu9rBVUdlkLO1ow+1FpRrKPnBmuDTRVLcW78UZjxRzM77OJLeYNY/mL6/bIvazUHhhajT6PGx+4uWCzCV7d+xxJP3xsx+fl47t4xzxkTsooV4+zUkeh19DtlJWlMp8PII/tTZMCpOzP2WOKUcyv3kK4IoVqvKo5e2/URkTnVliXqZr9hy4KJnN5EJgBn2hyneT5QtsDCYIqpGoklVRDfLLBiHDJ4Ah48F38Xa4SBiN/b3wZ3JhLwoVy65kKPpBH71dGzJp+zpEqXvujWSZgRybyK0CbE1x2enk8UGTo31/g9Tp1Q4pSvJsnL+6rct+2HmAmNcmow4wEnw1JWpvO7ia1JPntuZu2xxU0eG+Ax/ZAaWpQ8WTbPKcrpCvsQkO98Wcfwfbam1Yq3VHG8h0IuI/3jip4FbqT9AKxu/2u6qqvAna9eG2YqaMyIghtrbYNxw6aHXteoUayOf4fg2JgmkFVEFoW9T2nY6MxBR6yT7FsquWW585/VXeQKqicToXJJjg803t9b9obxVmsk3by8sGpwU0LS0+PI+nr1tLL+73J5pN/p6O39H7n0AZjw7SpZusa555BQ7E7+6vICXYwUqi4aybrW+uqSwxhELOxqqmHXUQ3p+/jDfG51WLl3IiJbyjx1xdeo4m6odHhckMVlQj3axO+W4EIGM1Sed1zAvnTYJdA28RC2ZnkTyplw4hWxrcZRGXiElz+ektfsu6xSPlP1dO/LT657Th2IgmSNG0qqrhfBpX4taai8te7CdChxYwZHSNArnvcdIk11Jd5ixvE4kIEaVziC+soVVWpi0oK7Q8f4+uPF3byxUvgGUC4AO9qSTzfQbL2YkIisyJZVnu+j+2dYkUWtA+rArTQ9rtH5QS9VfIiKyYhdDgYldpmu5ihIYLqbvep6ruIqZ6aprGv7onmMQoPt2ooNJ0ra8MLhTbossdgU3M5mlZ99fo2e+VqJzxW3uvKScSjggDyw1zel1Pabz9H3Y0eDGEZhroXciZ4hpCmpoJnFtg99YtImwTO44IZpoWd1lft7X5r0lq4x2aNPk+YaZ0uFH6Mo3y/cIzETqMBMrBQu6MF0jBIrmNqbvk9CT7m8hGXRe5m75Oi2LWAn6yyAFLpF+4JUAUuIVBZSv2/ce7pDv1YCTMl3MqccPWNiu/juBDFJTtDS/ovaf2GB+V4zvfguHF80pMxWHI8m58fWofoa/tklgkXB1wVycl8Pv5Krg40ajEyKqfvr0uNZt0HQVFlGDiK0LeLK3QFmn9/9jhVFH10C8HfffX73++mH199953Jut1hhNsmTO6muY5Ys33rBfq8X7EbYJp1gWMRWInzNTtwuJc1zgIl9LvYJTJiVVFRoRmIKkI4rKQHGRXwvSCA+EAtotsNsPJz4wd4B6H0eG6i9PrFL1HW1THQpzDLXRsWufId67WQOse5bGu0drWs+0jlJjy12aQeDjVQaX2zS1r34ehcLYsUmHU31VpM5Yo/darAbUWCbw/KesFA+up/g/R0XFnmv/38Yr9qqzG7y36OwWN7x0XtEDiL5KMxRx3EP4SflDElbvZPt2KXPTJPRXmfZQZ/M5+B2G3Hu7ZHpumU1myMeBkVfK8y4pXXdzOXSy4yL825tG3TisuagoetAC4PprMI65zqzKuIR+zkm8RrSrX310ZksikoMPVEj7MRxjZseit17emP+g4Z16gY3fZxm/VDcrrDI/12Go2YtbgYbdoxkeDB244V7yOlKl4wwGS1LdC4LHrDfYSXGQYenjroWRZnJVML46v27S/Sb86O2SalhRL7Mmkpw9Z9v0ZeKqonerRUXmaLDTp1pkxs6DtE9+lAXnQXTuhotnUR8SLtAZewxAhZoeZTj6DaoJhAcezDcPP6ABsyxKhKclgWbwL2Ay4gFyA3QKo82lbYHM263qx7oHJuhVvhQuEsqyKbAKlZZSQN3X+LR+OIHR58wGaVTRYGZbaLzAqGruAVUDeDVGlotJQArl39PALXE0SdhuI5T0dkLgu4Zi/3g+M5tBbWqZ3SkRYYJDEaJX35iYWsR0XjvAF6uy+1P4sZsor/vRGTEqCzXUfuud6BbyMdFnu4AeMtxdIkhMirWTEQsihyDTpEbLbJVpnfMkOjyQ2QrLncaF/FzV7qwhdmmg54g6kJExkRKccJESVWx3EdLeB/BLsl1GuBbzFPwCiuzUkkjs/ghKYC+/SkDj2N82DzZ3eRyneUpiG0Bx89/IyIr8E1mTCy3QR+w5WhOEzwKBROJkGYiHdIl1xlf8ix2WLQH+08JgUfvDN6BHbsXYhd27KreLuyfE8J+lRD2PyeE/b8Swv5zGthGlhwvaQqR0kCPb56JrKg4KN/LfYJ3sgZeXifQS4qKs3VRptG+rZaJ+Tp2EpKHzFIoJZp+IfF9IyLTLiExwQlqRdJYkxZwGmtS73VVJphFSkRTVp3EVDXSWNOD3iQQIUYaa5ilgg1mTRLglWA3AgupKUnAhNtXliqJHoXtK1maDcV5AreaLMqM8AQ+bAs4QZAE4Krl3sR3i1rIOgnkssoSxDSIYoYRzBMUEOkMr6kg+4hZV13YAvP9HzRfpsB7m0Eb0CSQXTuYNFi7xNok0JfrcvsqjQ9aZ0tm/pyk0RjRWdxZcQPASkYX1TrJNQeolKj4VW7a+fijzdrqAKZm4/z88Z0jDjiofUmAu27y8TrIdWCvGKcpbBidrVIcIlvFLM7uA06hG+iMlZCkmCURdazc/pRrU46a+UeCrRVJApuzFU1hxmhwNBc0Z9EKRvuwmUjDJYXMK041kSmo7YGzdQLZJEu9wybqzP8O9FAGeRTAiq6ZNgrH94S0sBNofIqWqUitktFaQydylUi+usx8x+IJoBtFcZFAkXSlQKnQTqdc7zaS6cxNmI0PfY8VTsLg+UQhbAzIWzffPjZcpg0W0ecc59osKxVrWGANlbpZQSmgVtFxja9H1zXJscHC5IZV/GHXx3YaOARzjfM89h1geeywat06KMFbxIqMKCmLJF2JLOAEZhorsjTJkb7jUQoyl9fR2zOVOn7LUlbqUrHIQDk2zFTRs884EzRei50Wqo46UaeBC8W38d1aXLqup9mKy+jPeQM8Qcq/tXmjSx0LNIHEsTZ0AlSj5yZwuU7CumKd5AKXUsUWYMWyWqe4ZgXTJIVYKHQShk0xB0JQA82VosONLsNdA+jYGX8Oaux0PLHbxbZAklSUSTcAOrolKuNrRlKxdRaYx/VguDtBVfw3q8zcUN7oYKNOpm7BuhGvSZgsQeGmn4kTWxh4sLGlQZk5R1J0dLHW9sOMbGLV+Y9A05uSRQ8ElFQVa4WFGfXcjQF5lwRw/KfXdSL79GkwBTQCYCXXGdZlxIEBXdAKx4aqKOYp9DtFCdDBdR1NBDw+kS3kuC1cO5ClyhNgHN+RqRP4hrXzDSfIB9A0diKAG3icwDjR9Et8Bgg1aI0GNYEppdk6geDVZWwvm1YkxT1QJI+uSGtFQl1xIwA28UZsdWFWOnpXzS0RsQslgtNiHwrUNemMvX2zNvHZygGNH9FrZnrGhrsvo3drrfJlkjz0SvEEb2GlqcpyFrvqPcnYijoylIIMhmiDi9je4G3GhDZ4lUAz2DJlUqjh21IkaN1kpKpETDdrqC1aoKPoaWUk+lAJNFq6yR5JOCzvM+YsR2eK5sygM6xy381QQ/v3MDpuclZCKk1NCAUwMEQfQX8DIjkKleo0+RBMpKPc66Lkck9HgwVvpd9KVtGaet+RxywNnc8I5p0puqY3qMDDRgttLFasq+EwkORIcqZhOEO9uj96aKCEdFWWUhk0bjyK0G6DDWIGlYqupljhAWm59xlCESK8tzoaFBATvrP7RF9ozkTqifwdVO1qXTw1MnJNzYaqRft9vZHV6EVDSNAtVc04IiNRiZWm6B01GCaCu7uKGxI8eyvX+sWlK3t9js79iK8TZDaBKUXQDPgD9aOPAW2B3lPzOzOC6vA5j5k6CfFWMLK7uUWwuNuspliRzYIJFsQPZu7O0F97ID5hFgYkQ7zguBIw63ddwRzXuol7uIH7oF/7gT2lb8fd7Klpwu3nF08Y+/Ygsog1TXfrvArLoo/0xsCtmHIXzDGNekIgtYPr3sOEasEnJl5C99yE48Chf66mBin6paLaHGjafXy28v175TuVAcbyuFWdxB56pJq807475RBODiOIjfX+Dh3a9S/Bncec/X/7fEO72MV5LRRg7TBvgNUQL4n3nixsH5cl1hS5dO0GGzS6Vc0p+V88Dr6iGQXfYC6Va18fJCNCWCNNKYw7w4fnVSksNCYzjPcddZh2SwtQe1umIZWCCWiHkC6pKphTN+ZCul3SDeZgW8bpmiJOt5QjrDVbC3dw7bz+MOtDS+ZHlN+w/gFOXz7KpGeLWSXYl4oOxyTi8OXr4Htcx8TjpqDUGg3L3YUkUggKuRVox8xmSlAgFKgMaTR2RY8qL7q3aWHJCfKkeaK4XDOCObIYTJg+gMXjYgdLTYxpfDzalZu9DqPXSWfbyUFWa+wHHnOGdbaRyW0CZ8Q15hrMUmmHGlmp2B3BE+4HgNylsdjCm+YHsRBOsVqcci2tId67b+cQLEe/+l8s0KnYN/83gm7AltfCIJwviCzKylAVFsNJ3Ph2Y+nMs2+GZwEzFnsHwszfqpd/+uHP1vY97xxHTbFvgmh7Ps3iRszu6rjBe6rQPzc+Of3CowHIhW997Pqf9DwvWpx7XH/wPI5MXr5Ntn07HJhi11mg9799fG33ThV1zhPwl+ZME0VLLMjeapVePePDXBAEFDpBH9/9gi6E+fHlCbp4f/76v35Bny6EefUTerbb7JGgzGyoQmQjtR+VJpWixMC3fnj1v//H82+DFKFmk1DGDekBMnVR4PA4Hp2Y++55za8cL17USIWveP60kO7KplswP7Jh3J0f+BC+A8W0tU4+M2UqzNHb0/dBZP+QgqbzZR3HGf9XCroI09ai+9WIUNjI7cITjuApvsEHzmGNDd3hRxiRDtx9iU7zXIGf1nF5CJ3m6SVFeWyc86GxkIuzd5fuVZoMjxVYzxj96DmVnKbq3250cWlRmfB+WRoeOQkiCg3t2tM0rDWxzE3XmldAdNDFec7slzFvA7adWf7hd25GBrAmIVxw6W/4eZ8FRqi0udZJ9Lq7PmkYvfcYXkplGpE8Ero5BNjgAJjZ3y559cy0d/thYl0/JvW23k0RXtCQ3TiXF9djB5Yv1loSZlVO5zca6TjIymWFxZouGtOJSLFi60rRHC33AJOKHLKGwnKmPLL1wKhodEJbDi66StDvgEfU/bslXNEdAIoW0tDMZ3bHzzOKT9pc6AxnLhU/AejSqDTAVwlYYpWgWpinuA6p+p+UCYiK86z2xKVTy4cWvN3HYrha15nwCBrsa7OhSlCDPu5LeoI+1c/YW3CA/YguawfY6CX4bUpTq0f1zKBMTJjGNdLeL36CMOdBZaJsvwgJblhBYt6WKvsGMmEk0gYecybQp4tJgUIgQTaZvIousi1QWSYY+2YBK6pjZ/RasAlKXNyLGDsVHfztCbB1oxUyTsU6+qRIwNkqHwm10AkN1Kk8mHcCMAIRSCdYIYzeSLXDKh/P6UbodA3JXgphe+NvIJduSc2OUhFWPSN3TbxvjFsazLuhOocMgpbxkBkx2iETPs8V0hIKZqxY8iM2wlvccizmiOPfwUFZJ4h0XJSjDfZdlm0kZWst2DUYsP2XJ3akkhLoQrCN1w/ubhF7rAwjFccKQb9oVCPx7PXNL2/lWq5W4envlGRmQ5Mfbw/Zj3ZBdxs7eL+2eFt0TyuzocL4ZPFJtHUVs3PC3RJ63JLTqH/SVE0iLCtD5LyU9ktOI3xVEUK1nsAZOo8f1xztuMQTwAtZFXct1R4FChNGuM0hnHo40gGOVipBgE+XUth3xcqtkHLY/BCNFKX+rrbx+tFNvJsYua6lUDPAGc2b/Xg/zEAfZgJpZqqA/ERQXEC9iPZQN1gjnMvSvi5mQ5lCcifaI3OEM/hGCllM5NXCTA7NXIv6eZUIq9wzkVv5I5VuCIDRG8YpOvWILUZkuIuzVzQbc3dyMmG82f+jpCtMkuDKZy3EpUJojwFCxKx3fwAhXL7ela/XiE2J6YTQpUxZPRDY/JJu8JbJCrRLIotSyYJNZCjSuZF7LfCSQxHZCp0dxo2JbSN2EiI5xLCndaIgAj0Mow6XOQLBwPoNfqlPt/PKtvdtku3aMstKmGE5W2yNPocy8IwcY9bfSQuC93hNBVWM1FsCgkCi3zC1gJkNPLWh2W7II7sgPyy0UdPBz3pPx7TderQ9vTy8J69euLUS7itomjZGuGEF1VauO21P0ZJOBpH8KURrCnHrQUDjwQceg7ojax3Tu/vRWOvHu+3ph0xHG3J65615h/FtOxztDXbcCoQ7CIOvd3cvb92dmvXs3EWLsjd1+8lF66U6jwC5RY43AuTrZccfbz+yWKMN5jmyu8lHNasEiXnH7iA/ZmXHmHsbMWOj1EMJ2sBPHb1ypzKbrKBmIx8hSoJ7nmTk0PBfmzxw6KWkZFKv04GozgfJvb/WInKALxN5Qv5r8fOf/oSevT0/vXyOzpk2TKwrpjc0h1L4IC5crmXyvkCHImGQLbtyePhjhi9OZIwpmdireKj+055qCIPmxoBHPtrQ5/tcFwJp/03db8fxBziFYqZYhNqkt5limMfqTjfYyAecs0q7FZBUSLOCcayceLJi094hAu96uLwK7rlm+ZydRrqZ8p8sI9RexEFfzPaSp6uzOBWH7jqENXylYcf/651E8MmIF7zjhnbKMvKwK1OqlIkBo5ANkFqqNRbsjwNZ1SIdK9yV2EdQustTE+ReMRWsJU3U9eeNXQ5eC9fiy/Uu6mU1/0oxNxuCFUWlorksmMDBgruOeLrEhlFh9K3p8RzPudu3+FE361o/0jIR49qr860VXCVWBpohtVs9LFZnbHbkhc1dJOqK5lRhQ/MsWlLZAf6wwudNvWITPLtUcsvypnmY/x4uS+411RFj+OY/9lnr67RhBafdJMtn2mWzpO/1Z/YT2wwOD4XMyS1z0fPNUHGfaAHXKJ0xh4LfV/OkN6AzdX7UqYReBzbqdFTQWLFG2kjlJL6FVlCDYbVv4VsL+61vw7svWJ5zOp+Uewfr3VXOBY63I/eOknP1eIx5tnvpV+t0GBL7Ojp7gkqO7ZHZ91kqRAVR+3LKyw+pkDPYk3fIoFONbfmr1Aa9w2TDxIRJl+NEkuObIa0/Ccj0LxW14sPqR67JmV6gtzku0Wf4H6cf5VK4utO/jR9PtMFbajUnTrFCXyqq9gh6EOpSCk1rjSpcnGr3m8Fv5pGXvgcesZAVq7tACrd915dvGs96SzOg2jLQB98c9a6YwpSntA6zIY/XraV7TYysbegfXqaRqoQI2rH6pHl5XOTZtZGaqLHzEDNvYaY/CIx2TORyp5EuKWErRuwnJ6E6QZ8nO74gdnsO3zbnBj2DjrBUkPYZgtDl8w61UCXgHX9L15js0Sfdb3zbRGCLYSFt9Oxau8IMBvvEa981tQAVqFUDJrMv4ojiTR+AQPV/r9IUynnG5OtvO71CPdWd16nXgR3DDoOM5n9zxGbnyeud2qrP8PWu91rWvYatT3cBHe9mHoddEzDon02bkOmOYXRC4YYUtxc/Q9lAzJGAkxVusOWcrpjwvnoQTtDVr8DlRNNBwO6oQrFEuLUOmIH6F1swNj7b1Hv3vZQmelM2PmxjMNkUM7fAb1cFgqORddQ9jiRDXpZMxJsgFvVu2C1DUWHaxzMgpLplO3Asro12W94fmNo5wjrt23cL1iVWNU/ZP5+0W9lt2KiVOrK3w9qyLvn9Ttsz0WeWuLYWUu3THfhfdInFv93aMaZGpN9FvVbPQ0+TJctfXgD0W/b2aCrRaFd1v/XDu5rkgowKo2R5jOjIZbUcORfuxON+TWtt01vKEQBHV90x7z08k0WJxb65j3DtYJy+s1e2VNlnKGNiJcNKAdbXqWuEbpEfAyuyxmxH03ZFX31JlSPwpuJ8j/6zwpytGM3ROdQ9O+dgEJUdXWZEymv2SEH33+kSufVb+xnzKW0+erfZNhxeVgZU7iNHmN5+1z80S/gpO94d7XzyC/RxX7qtt54DSxx3gtOHp+gqi9pMdoC2xcE5ItS3OtS2dojMHK66RrnsY+c8i6VUtbcfQswf3k4ceadXTmR2qmlRpp1DdIAUduVbPfc1mkrKRJpIHym7jj0PVGITdk0SkWEdM9rfAax8OX1kyJXiEY+5AzXiqTTGaFapWN6QDkxNVYbX8WzKFnT056kPOmr6Yx+05/oEgoXeGCpAtYpvnFj40bi5UfQ2ig5SZWJrVG6JOWoJezL3IywL6tUL/99nHoUX/j98XlPI7Y85VeHsPL+dR4yeu810g+fgce2MWhttJ/cD0axJxcSKKjURdx3ve5Z9dRX/W0kfdM/OgGTdl3jVOYbAlYKwtkx6pQJLzMZ+r13c3rLdR8ggVt0//ZWOE7SmB36yckPVPP4Iq7P7jKdnZzD68Tk6g/XDqFFlZmqWMkHnM6r88E/ay8I80JyXJg0ddwjZOXC76Le60yn64EmzP471St6/NUr4tNEV+yPsrWHXiWTKxV9fI0HX0jB3gOUG64kJUJrM3Vaoc5Ru8enhgvaok02AGiW4DHisbpxe19+EE1I0W89RUdHvb9RMPfw4OWjZShOmdRVd6QTIkCyVzlv3sBgKYEiVSuoDHR1KV3q+toujKwhOH5JOs2RINJ3BfRT52RWkdh5+jDrS8zgk7y89D+A4LUK15tk25Ys+DKl6R3YQmTyzrIer6G0adSrA7Jp6izpRc4Nv2nEl3QcJZOtPSEO8Tip0cXX613eX6NK+U+g3MTF9pcU2USX1Mdh+3MkwtiCGyIaSa32UE/luQjhtD7LQ0LmmX2fTIgzSQP0IwlYKHtByqWKjppCPoOQ6PJquIJNGA+BssKlmm/DZxXKLOcsdIwaQGArC2bpaHxKEQLFrutdDsR2J8+sE0siwN8aUOmMwgzYJaDjKFAQh+AncJrYWdeWLVMzsb7lRRBZF0j5xd8Tb4eEdQuES/B1TlA8tzdgulh3HItP6sQbe2pWdDP/d77au0Qpi60qNs1KyOdKqQwg7DBBgAEiFrQEgK9lgIUaNM1K3m/KrAiITMduZ2jY3D4ufefj729P3/t17MVi+eVCMVEPff/SebUxfZ1vJq1QEOK3nOAs/56aZjF2P860EMxo9c0jo59CtAwp764m6A/AIkA7uhleJpNlbj+snwYxPF1j0iw62VEGmwKriiEhBaGmsoXzlznCivcJul1L6OsJbg70eoW0RLaUySFr6/vrvp6EU3CDZY/OdVOv5EyyHBQY9F+sSu2YnwUYx//H6t8uLS/QO3xRM5M1Y7/Cx2r3NnobZG6I4sS2/jdHuDm2rUZ/CJYvR07NdlWO2mq9g87GL8OstJ1c7es4yL5Uvzn2XXo/FQQz5fIfyyL0C6h0X/+3rhpvCHJGPNcnYtxv8JdaEfqTsRj+uGqz4JqhbuOLeE6SrQIo61ugv2igp1v+25Jhcc6YNzf/ywv/tpPmUiRUl4Y9WTNEd5kFFBi955zcIixxpiSbYUtE100btrWU/p7Aosdn4Zv0NDmiIwwhJcErNhaYrhHb1WkSqThfyRp9sMKfCdHJSWo+7JnJRFUtFOe9a82FO7+HTT79/A1cAbuyZBYo+eaDdh3V8Twbt5tjAYpkmzgFUEDoVCCuFm/z7nK2gjNV01kGKcoj1+DOGutaQFuAdjpFQ+wjZK6RyrgrlquvakRGsrmUf6m0FNmRDdVB5lZyRfVZHDMeBwQegCs2BmmCk60zhUGlaDDJddyBZoNMtZhziZG32PfoRbjheym1Qy+rhHY3Gxnd9a1G3VC1w7psd1Bi/kQrRG1yUnJ6gDxIXTKyhrqAy0BaqnqgawnzJJbmmeRafQ4bcoKC8vi3C7nLGklqUPS4TR/DT4SPwTBiVc+oD2EF2PcA/sX/0CeaG3pgXG1PwED56gzO9wS9/fhUDm1/pDcrZmmpTy4N+14fwtcfbLKfGTf+Ndq4NRO8aIESqzlQYhIVhW6YqjahYM9EOWIHKFiZ06X4eFAMVjiM8kX3vwSnHOSqlJRBzRQFih4Xlwk43IvTs8tPpc8+gugnXlEreMKvBWbyxlRCmUqJT1tdsVBMsRH98b3MERZnlTJdSs5HW+hDxC/GMbtWhbvAFXQQwAlTdU3aabzG0QHiH+c4q35dK1uf47PTd5XO7wxKrhr/qt88Nhblojg2tKGRQ/Asi2F5cdMYpFicWLiNMVvCWfxLXQu6CR2wJUjgcxv67IylysWqXPxmNUvOr9Tn19N3lFHaaSBVNhACwISaQA2oxcAoRqBRNbbrTdev6FXuYO8a5pfSSYxEU4jk2mNDRWIAHoN2lX8MJ59hgdAbrOJHuiwF9HWilqfoe6vWdTqLwasVICF83wXBoOT8AXW8VNw9lE4p2s7pNJQTli3/6/wIAAP//a6iF4Q==" } diff --git a/x-pack/filebeat/module/cisco/umbrella/_meta/fields.yml b/x-pack/filebeat/module/cisco/umbrella/_meta/fields.yml new file mode 100644 index 000000000000..e45d12ef732b --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/_meta/fields.yml @@ -0,0 +1,61 @@ +- name: cisco.umbrella + type: group + description: > + Fields for Cisco Umbrella. + fields: + - name: identities + type: keyword + description: > + An array of the different identities related to the event. + - name: categories + type: keyword + description: > + The security or content categories that the destination matches. + - name: policy_identity_type + type: keyword + description: > + The first identity type matched with this request. Available in version 3 and above. + - name: identity_types + type: keyword + description: > + The type of identity that made the request. For example, Roaming Computer or Network. + - name: blocked_categories + type: keyword + description: > + The categories that resulted in the destination being blocked. Available in version 4 and above. + - name: content_type + type: keyword + description: > + The type of web content, typically text/html. + - name: sha_sha256 + type: keyword + description: > + Hex digest of the response content. + - name: av_detections + type: keyword + description: > + The detection name according to the antivirus engine used in file inspection. + - name: puas + type: keyword + description: > + A list of all potentially unwanted application (PUA) results for the proxied file as returned by the antivirus scanner. + - name: amp_disposition + type: keyword + description: > + The status of the files proxied and scanned by Cisco Advanced Malware Protection (AMP) as part of the Umbrella File Inspection feature; can be Clean, Malicious or Unknown. + - name: amp_malware_name + type: keyword + description: > + If Malicious, the name of the malware according to AMP. + - name: amp_score + type: keyword + description: > + The score of the malware from AMP. This field is not currently used and will be blank. + - name: datacenter + type: keyword + description: > + The name of the Umbrella Data Center that processed the user-generated traffic. + - name: origin_id + type: keyword + description: > + The unique identity of the network tunnel. diff --git a/x-pack/filebeat/module/cisco/umbrella/config/input.yml b/x-pack/filebeat/module/cisco/umbrella/config/input.yml new file mode 100644 index 000000000000..8b0ccde6e2eb --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/config/input.yml @@ -0,0 +1,23 @@ +{{ if eq .input "s3" }} + +type: s3 +queue_url: {{ .queue_url }} +access_key_id: {{ .access_key_id }} +secret_access_key: {{ .secret_access_key }} + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +processors: +- add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml new file mode 100644 index 000000000000..2a602ff23317 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -0,0 +1,246 @@ +description: Pipeline for parsing cisco umbrella logs +processors: +- set: + field: observer.vendor + value: Cisco +- set: + field: observer.product + value: Umbrella +- set: + field: event.ingested + value: "{{_ingest.timestamp}}" +- set: + field: event.original + value: "{{message}}" +############ +# DNS Logs # +############ +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - source.user.name + - cisco.umbrella.identities + - source.address + - destination.address + - cisco.umbrella.action + - dns.question.type + - dns.response_code + - destination.domain + - cisco.umbrella.categories + - cisco.umbrella.policy_identity_type + - cisco.umbrella.identity_types + - cisco.umbrella.blocked_categories + if: ctx?.log?.file?.path.contains('dnslogs') + +- set: + field: observer.type + value: dns + if: ctx?.log?.file?.path.contains('dnslogs') +########### +# IP Logs # +########### +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - source.user.name + - source.address + - source.port + - destination.address + - destination.port + - cisco.umbrella.categories + if: ctx?.log?.file?.path.contains('iplogs') + +- set: + field: observer.type + value: firewall + if: ctx?.log?.file?.path.contains('iplogs') + +############## +# Proxy Logs # +############## +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - cisco.umbrella.identities + - source.address + - source.nat.ip + - destination.address + - cisco.umbrella.content_type + - cisco.umbrella.verdict + - url.full + - http.request.referrer + - user_agent.original + - http.response.status_code + - http.request.bytes + - http.response.bytes + - http.response.body.bytes + - cisco.umbrella.sha_sha256 + - cisco.umbrella.categories + - cisco.umbrella.av_detections + - cisco.umbrella.puas + - cisco.umbrella.amp_disposition + - cisco.umbrella.amp_malware_name + - cisco.umbrella.amp_score + - cisco.umbrella.identity_types + - cisco.umbrella.blocked_categories + if: ctx?.log?.file?.path.contains('proxylogs') + +- set: + field: observer.type + value: proxy + if: ctx?.log?.file?.path.contains('proxylogs') + +####################### +# Cloud Firewall Logs # +####################### +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - cisco.umbrella.origin_id + - source.user.name + - cisco.umbrella.identity_types + - cisco.umbrella.direction + - network.transport + - source.bytes + - source.address + - source.port + - destination.address + - destination.port + - cisco.umbrella.datacenter + - cisco.umbrella.ruleid + - cisco.umbrella.verdict + if: ctx?.log?.file?.path.contains('cloudfirewalllogs') + +- set: + field: observer.type + value: firewall + if: ctx?.log?.file?.path.contains('cloudfirewalllogs') + +# Identifies is a field that includes any sort of username, device or other asset that is included in the request. +# Converting this to an array to make it easier to use in searches and visualizations +- split: + field: cisco.umbrella.identities + separator: "," + preserve_trailing: false + if: "ctx?.log?.file?.path.contains('dnslogs') && ctx?.cisco?.umbrella?.identities != null" + +###################### +# General ECS Fields # +###################### +# This field is always in UTC, so no timezone should need to be set +- date: + field: cisco.umbrella._tmp_time + target_field: "@timestamp" + formats: + - "yyyy-MM-dd HH:mm:ss" + +################## +# DNS ECS Fields # +################## +- set: + field: dns.type + value: query + if: ctx?.cisco?.umbrella?.action != null + +###################### +# Network ECS Fields # +###################### +- lowercase: + field: cisco.umbrella.direction + target_field: network.direction + if: ctx?.cisco?.umbrella?.direction != null + +################### +# Rule ECS Fields # +################### +- rename: + field: cisco.umbrella.ruleid + target_field: rule.id + if: ctx?.cisco?.umbrella?.ruleid != null + +#################### +# Event ECS Fields # +#################### +- set: + field: event.action + value: "dns-request-{{cisco.umbrella.action}}" + if: ctx?.cisco?.umbrella?.action != null +- set: + field: event.category + value: network + if: ctx?.cisco?.umbrella?.action != null +- append: + field: event.type + value: allowed + if: "ctx?.cisco?.umbrella?.action == 'Allowed' || ctx?.cisco?.umbrella?.verdict == 'ALLOWED' || ctx?.cisco?.umbrella?.verdict == 'ALLOW'" +- append: + field: event.type + value: denied + if: "ctx?.cisco?.umbrella?.action == 'Blocked' || ctx?.cisco?.umbrella?.verdict == 'BLOCKED' || ctx?.cisco?.umbrella?.verdict == 'BLOCK'" +- append: + field: event.type + value: connection + if: ctx?.cisco?.umbrella?.action != null + +# Converting address fields to either ip or domain +- grok: + field: source.address + patterns: + - "(?:%{IP:source.ip}|%{GREEDYDATA:source.domain})" + ignore_failure: true +- grok: + field: destination.address + patterns: + - "(?:%{IP:destination.ip}|%{GREEDYDATA:destination.domain})" + ignore_failure: true + +###################### +# Related ECS Fields # +###################### +- append: + field: related.user + value: "{{source.user.name}}" + if: ctx?.source?.user?.name != null +- append: + field: related.ip + value: "{{source.ip}}" + if: ctx?.source?.ip != null +- append: + field: related.ip + value: "{{source.nat.ip}}" + if: ctx?.source?.nat?.ip != null +- append: + field: related.ip + value: "{{destination.ip}}" + if: ctx?.destination?.ip != null +- append: + field: related.hosts + value: "{{source.domain}}" + if: ctx?.source?.domain != null +- append: + field: related.hosts + value: "{{destination.domain}}" + if: ctx?.destination?.domain != null +- append: + field: related.hash + value: "{{cisco.umbrella.sha_sha256}}" + if: ctx?.cisco?.umbrella?.sha_sha256 != null + +########### +# Cleanup # +########### +- remove: + field: + - cisco.umbrella._tmp_time + - cisco.umbrella.direction + - cisco.umbrella.action + - cisco.umbrella.verdict + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/cisco/umbrella/manifest.yml b/x-pack/filebeat/module/cisco/umbrella/manifest.yml new file mode 100644 index 000000000000..3a7150e714dd --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/manifest.yml @@ -0,0 +1,8 @@ +module_version: "1.0" + +var: + - name: tags + default: [cisco-umbrella, forwarded] + +ingest_pipeline: ingest/pipeline.yml +input: config/input.yml diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log new file mode 100644 index 000000000000..3e5f23fced24 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log @@ -0,0 +1,2 @@ +2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW +2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json new file mode 100644 index 000000000000..65aabab5a88d --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-07-23T18:03:46.000Z", + "cisco.umbrella.datacenter": "ams1.edc", + "cisco.umbrella.identity_types": "CDFW Tunnel Device", + "cisco.umbrella.origin_id": "[211039844]", + "destination.address": "146.112.255.129", + "destination.ip": "146.112.255.129", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW", + "event.type": [ + "allowed" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 0, + "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW", + "network.direction": "outbound", + "network.transport": "1", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "172.17.3.4", + "146.112.255.129" + ], + "related.user": [ + "Passive Monitor" + ], + "rule.id": "12", + "service.type": "cisco", + "source.address": "172.17.3.4", + "source.bytes": "84", + "source.ip": "172.17.3.4", + "source.user.name": "Passive Monitor" + }, + { + "@timestamp": "2020-07-23T18:03:46.000Z", + "cisco.umbrella.datacenter": "ams1.edc", + "cisco.umbrella.identity_types": "CDFW Tunnel Device", + "cisco.umbrella.origin_id": "[211039844]", + "destination.address": "146.112.255.129", + "destination.ip": "146.112.255.129", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK", + "event.type": [ + "denied" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 128, + "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK", + "network.direction": "inbound", + "network.transport": "1", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "172.17.3.4", + "146.112.255.129" + ], + "related.user": [ + "Passive Monitor" + ], + "rule.id": "12", + "service.type": "cisco", + "source.address": "172.17.3.4", + "source.bytes": "84", + "source.ip": "172.17.3.4", + "source.user.name": "Passive Monitor" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log new file mode 100644 index 000000000000..403c1c9df33b --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log @@ -0,0 +1,2 @@ +"2020-07-23 23:49:54","elasticuser","elasticuser2","some other identity","192.168.1.1","8.8.8.8","Allowed","1 (A)","NOERROR","elastic.co.","Software/Technology,Business Services,Application","Test Policy Name","SomeIdentityType","" +"2020-07-23 23:50:25","elasticuser","elasticuser2","some other identity","192.168.1.1","4.4.4.4","Blocked","1 (A)","NOERROR","elastic.co/something.","Chat,Instant Messaging,Block List,Application","Test Policy Name","SomeIdentityType","BlockedCategories" diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json new file mode 100644 index 000000000000..81b1478da273 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json @@ -0,0 +1,92 @@ +[ + { + "@timestamp": "2020-07-23T23:49:54.000Z", + "cisco.umbrella.blocked_categories": "SomeIdentityType", + "cisco.umbrella.categories": "elastic.co.", + "cisco.umbrella.identities": [ + "elasticuser2" + ], + "cisco.umbrella.identity_types": "Test Policy Name", + "cisco.umbrella.policy_identity_type": "Software/Technology,Business Services,Application", + "destination.address": "192.168.1.1", + "destination.domain": "NOERROR", + "destination.ip": "192.168.1.1", + "dns.question.type": "Allowed", + "dns.response_code": "1 (A)", + "dns.type": "query", + "event.action": "dns-request-8.8.8.8", + "event.category": "network", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:49:54\\\",\\\"elasticuser\\\",\\\"elasticuser2\\\",\\\"some other identity\\\",\\\"192.168.1.1\\\",\\\"8.8.8.8\\\",\\\"Allowed\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co.\\\",\\\"Software/Technology,Business Services,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"\\\"", + "event.type": [ + "connection" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 0, + "message": "\"2020-07-23 23:49:54\",\"elasticuser\",\"elasticuser2\",\"some other identity\",\"192.168.1.1\",\"8.8.8.8\",\"Allowed\",\"1 (A)\",\"NOERROR\",\"elastic.co.\",\"Software/Technology,Business Services,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"\"", + "observer.product": "Umbrella", + "observer.type": "dns", + "observer.vendor": "Cisco", + "related.hosts": [ + "some other identity", + "NOERROR" + ], + "related.ip": [ + "192.168.1.1" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "some other identity", + "source.domain": "some other identity", + "source.user.name": "elasticuser" + }, + { + "@timestamp": "2020-07-23T23:50:25.000Z", + "cisco.umbrella.blocked_categories": "SomeIdentityType", + "cisco.umbrella.categories": "elastic.co/something.", + "cisco.umbrella.identities": [ + "elasticuser2" + ], + "cisco.umbrella.identity_types": "Test Policy Name", + "cisco.umbrella.policy_identity_type": "Chat,Instant Messaging,Block List,Application", + "destination.address": "192.168.1.1", + "destination.domain": "NOERROR", + "destination.ip": "192.168.1.1", + "dns.question.type": "Blocked", + "dns.response_code": "1 (A)", + "dns.type": "query", + "event.action": "dns-request-4.4.4.4", + "event.category": "network", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:50:25\\\",\\\"elasticuser\\\",\\\"elasticuser2\\\",\\\"some other identity\\\",\\\"192.168.1.1\\\",\\\"4.4.4.4\\\",\\\"Blocked\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co/something.\\\",\\\"Chat,Instant Messaging,Block List,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"BlockedCategories\\\"", + "event.type": [ + "connection" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 232, + "message": "\"2020-07-23 23:50:25\",\"elasticuser\",\"elasticuser2\",\"some other identity\",\"192.168.1.1\",\"4.4.4.4\",\"Blocked\",\"1 (A)\",\"NOERROR\",\"elastic.co/something.\",\"Chat,Instant Messaging,Block List,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"BlockedCategories\"", + "observer.product": "Umbrella", + "observer.type": "dns", + "observer.vendor": "Cisco", + "related.hosts": [ + "some other identity", + "NOERROR" + ], + "related.ip": [ + "192.168.1.1" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "some other identity", + "source.domain": "some other identity", + "source.user.name": "elasticuser" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log new file mode 100644 index 000000000000..6200aeab3ae3 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log @@ -0,0 +1,2 @@ +"2020-08-26 20:32:46","elasticuser","192.168.1.1","0","8.8.8.8","0","Test Category" +"2020-08-26 20:32:45","elasticuser","192.168.1.1","61095","8.8.8.8","445","Test Category" diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log-expected.json new file mode 100644 index 000000000000..4d25464cb614 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log-expected.json @@ -0,0 +1,60 @@ +[ + { + "@timestamp": "2020-08-26T20:32:46.000Z", + "cisco.umbrella.categories": "Test Category", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "destination.port": "0", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-08-26 20:32:46\\\",\\\"elasticuser\\\",\\\"192.168.1.1\\\",\\\"0\\\",\\\"8.8.8.8\\\",\\\"0\\\",\\\"Test Category\\\"", + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 0, + "message": "\"2020-08-26 20:32:46\",\"elasticuser\",\"192.168.1.1\",\"0\",\"8.8.8.8\",\"0\",\"Test Category\"", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "8.8.8.8" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.port": "0", + "source.user.name": "elasticuser" + }, + { + "@timestamp": "2020-08-26T20:32:45.000Z", + "cisco.umbrella.categories": "Test Category", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "destination.port": "445", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-08-26 20:32:45\\\",\\\"elasticuser\\\",\\\"192.168.1.1\\\",\\\"61095\\\",\\\"8.8.8.8\\\",\\\"445\\\",\\\"Test Category\\\"", + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 84, + "message": "\"2020-08-26 20:32:45\",\"elasticuser\",\"192.168.1.1\",\"61095\",\"8.8.8.8\",\"445\",\"Test Category\"", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "8.8.8.8" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.port": "61095", + "source.user.name": "elasticuser" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log new file mode 100644 index 000000000000..bfe70c6839a8 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log @@ -0,0 +1,3 @@ +"2020-07-23 23:48:56","elasticuser, someotheruser","192.168.1.1","1.1.1.1","8.8.8.8","","ALLOWED","https://elastic.co/blog/ext_id=Anyclip","https://google.com/elastic","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36","200","850","","","","Business Services","AVDetectionName","Malicious","MalwareName","","","Roaming Computers","" +"2020-07-23 23:48:56","elasticuser, someotheruser","192.168.1.1","1.1.1.1","8.8.8.8","","BLOCKED","https://elastic.co/blog/ext_id=Anyclip","https://google.com/elastic","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36","200","850","","","","Business Services","AVDetectionName","Malicious","MalwareName","","","Roaming Computers","" +"2017-10-02 23:52:53","elasticuser","ActiveDirectoryUserName,ADSite,Network","192.192.192.135","1.1.1.91","","ALLOWED","http://google.com/the.js","www.google.com","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36","200","562","1489","","","","","","","","Networks" diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json new file mode 100644 index 000000000000..fd474d2d0293 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json @@ -0,0 +1,115 @@ +[ + { + "@timestamp": "2020-07-23T23:48:56.000Z", + "cisco.umbrella.amp_disposition": "MalwareName", + "cisco.umbrella.av_detections": "AVDetectionName", + "cisco.umbrella.categories": "Business Services", + "cisco.umbrella.identities": "elasticuser, someotheruser", + "cisco.umbrella.identity_types": "Roaming Computers", + "cisco.umbrella.puas": "Malicious", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:48:56\\\",\\\"elasticuser, someotheruser\\\",\\\"192.168.1.1\\\",\\\"1.1.1.1\\\",\\\"8.8.8.8\\\",\\\"\\\",\\\"ALLOWED\\\",\\\"https://elastic.co/blog/ext_id=Anyclip\\\",\\\"https://google.com/elastic\\\",\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\\\",\\\"200\\\",\\\"850\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"Business Services\\\",\\\"AVDetectionName\\\",\\\"Malicious\\\",\\\"MalwareName\\\",\\\"\\\",\\\"\\\",\\\"Roaming Computers\\\",\\\"\\\"", + "event.type": [ + "allowed" + ], + "fileset.name": "umbrella", + "http.request.bytes": "850", + "http.request.referrer": "https://google.com/elastic", + "http.response.status_code": "200", + "input.type": "log", + "log.offset": 0, + "message": "\"2020-07-23 23:48:56\",\"elasticuser, someotheruser\",\"192.168.1.1\",\"1.1.1.1\",\"8.8.8.8\",\"\",\"ALLOWED\",\"https://elastic.co/blog/ext_id=Anyclip\",\"https://google.com/elastic\",\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\",\"200\",\"850\",\"\",\"\",\"\",\"Business Services\",\"AVDetectionName\",\"Malicious\",\"MalwareName\",\"\",\"\",\"Roaming Computers\",\"\"", + "observer.product": "Umbrella", + "observer.type": "proxy", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "1.1.1.1", + "8.8.8.8" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.nat.ip": "1.1.1.1", + "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" + }, + { + "@timestamp": "2020-07-23T23:48:56.000Z", + "cisco.umbrella.amp_disposition": "MalwareName", + "cisco.umbrella.av_detections": "AVDetectionName", + "cisco.umbrella.categories": "Business Services", + "cisco.umbrella.identities": "elasticuser, someotheruser", + "cisco.umbrella.identity_types": "Roaming Computers", + "cisco.umbrella.puas": "Malicious", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:48:56\\\",\\\"elasticuser, someotheruser\\\",\\\"192.168.1.1\\\",\\\"1.1.1.1\\\",\\\"8.8.8.8\\\",\\\"\\\",\\\"BLOCKED\\\",\\\"https://elastic.co/blog/ext_id=Anyclip\\\",\\\"https://google.com/elastic\\\",\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\\\",\\\"200\\\",\\\"850\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"Business Services\\\",\\\"AVDetectionName\\\",\\\"Malicious\\\",\\\"MalwareName\\\",\\\"\\\",\\\"\\\",\\\"Roaming Computers\\\",\\\"\\\"", + "event.type": [ + "denied" + ], + "fileset.name": "umbrella", + "http.request.bytes": "850", + "http.request.referrer": "https://google.com/elastic", + "http.response.status_code": "200", + "input.type": "log", + "log.offset": 399, + "message": "\"2020-07-23 23:48:56\",\"elasticuser, someotheruser\",\"192.168.1.1\",\"1.1.1.1\",\"8.8.8.8\",\"\",\"BLOCKED\",\"https://elastic.co/blog/ext_id=Anyclip\",\"https://google.com/elastic\",\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\",\"200\",\"850\",\"\",\"\",\"\",\"Business Services\",\"AVDetectionName\",\"Malicious\",\"MalwareName\",\"\",\"\",\"Roaming Computers\",\"\"", + "observer.product": "Umbrella", + "observer.type": "proxy", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "1.1.1.1", + "8.8.8.8" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.nat.ip": "1.1.1.1", + "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" + }, + { + "@timestamp": "2017-10-02T23:52:53.000Z", + "cisco.umbrella.amp_score": "Networks", + "cisco.umbrella.identities": "elasticuser", + "destination.address": "1.1.1.91", + "destination.ip": "1.1.1.91", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2017-10-02 23:52:53\\\",\\\"elasticuser\\\",\\\"ActiveDirectoryUserName,ADSite,Network\\\",\\\"192.192.192.135\\\",\\\"1.1.1.91\\\",\\\"\\\",\\\"ALLOWED\\\",\\\"http://google.com/the.js\\\",\\\"www.google.com\\\",\\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\\\",\\\"200\\\",\\\"562\\\",\\\"1489\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"Networks\\\"", + "event.type": [ + "allowed" + ], + "fileset.name": "umbrella", + "http.request.bytes": "562", + "http.request.referrer": "www.google.com", + "http.response.bytes": "1489", + "http.response.status_code": "200", + "input.type": "log", + "log.offset": 798, + "message": "\"2017-10-02 23:52:53\",\"elasticuser\",\"ActiveDirectoryUserName,ADSite,Network\",\"192.192.192.135\",\"1.1.1.91\",\"\",\"ALLOWED\",\"http://google.com/the.js\",\"www.google.com\",\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\",\"200\",\"562\",\"1489\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"Networks\"", + "observer.product": "Umbrella", + "observer.type": "proxy", + "observer.vendor": "Cisco", + "related.hosts": [ + "ActiveDirectoryUserName,ADSite,Network" + ], + "related.ip": [ + "192.192.192.135", + "1.1.1.91" + ], + "service.type": "cisco", + "source.address": "ActiveDirectoryUserName,ADSite,Network", + "source.domain": "ActiveDirectoryUserName,ADSite,Network", + "source.nat.ip": "192.192.192.135", + "url.full": "http://google.com/the.js", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/coredns/log/config/coredns.yml b/x-pack/filebeat/module/coredns/log/config/coredns.yml index be7f27f551fb..4f9992ae2d47 100644 --- a/x-pack/filebeat/module/coredns/log/config/coredns.yml +++ b/x-pack/filebeat/module/coredns/log/config/coredns.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml index 689bd725530c..2af0eeca0922 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml @@ -23,4 +23,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml index be7f27f551fb..4f9992ae2d47 100644 --- a/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml +++ b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/googlecloud/audit/config/input.yml index 4c30e23b5e35..b5e392ee0b69 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/audit/config/input.yml @@ -34,4 +34,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml index d6579aa9f479..39648636c59e 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml @@ -35,4 +35,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml index cf89526bbe52..f19761956877 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml @@ -34,4 +34,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/gsuite/admin/config/config.yml b/x-pack/filebeat/module/gsuite/admin/config/config.yml index b5c62d3657f8..9ec19a836151 100644 --- a/x-pack/filebeat/module/gsuite/admin/config/config.yml +++ b/x-pack/filebeat/module/gsuite/admin/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/drive/config/config.yml b/x-pack/filebeat/module/gsuite/drive/config/config.yml index 5f1bd6ecbf3c..92e464164b0c 100644 --- a/x-pack/filebeat/module/gsuite/drive/config/config.yml +++ b/x-pack/filebeat/module/gsuite/drive/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/groups/config/config.yml b/x-pack/filebeat/module/gsuite/groups/config/config.yml index 46a3ed338d98..ba6890c80e40 100644 --- a/x-pack/filebeat/module/gsuite/groups/config/config.yml +++ b/x-pack/filebeat/module/gsuite/groups/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/login/config/config.yml b/x-pack/filebeat/module/gsuite/login/config/config.yml index b501012b3d2f..95ccc04ed2c8 100644 --- a/x-pack/filebeat/module/gsuite/login/config/config.yml +++ b/x-pack/filebeat/module/gsuite/login/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/saml/config/config.yml b/x-pack/filebeat/module/gsuite/saml/config/config.yml index 1e703737e0d1..8d1c7645fdae 100644 --- a/x-pack/filebeat/module/gsuite/saml/config/config.yml +++ b/x-pack/filebeat/module/gsuite/saml/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml index 773ab6201731..dc9e3d353e6b 100644 --- a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml b/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml index 2130bb419d23..bca14ecccaa2 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/iptables/log/config/input.yml b/x-pack/filebeat/module/iptables/log/config/input.yml index 6183661122a5..bcfde17c3c53 100644 --- a/x-pack/filebeat/module/iptables/log/config/input.yml +++ b/x-pack/filebeat/module/iptables/log/config/input.yml @@ -55,4 +55,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml index 3ff985b07f3a..26010fc478ab 100644 --- a/x-pack/filebeat/module/misp/threat/config/input.yml +++ b/x-pack/filebeat/module/misp/threat/config/input.yml @@ -37,4 +37,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/mssql/log/config/config.yml b/x-pack/filebeat/module/mssql/log/config/config.yml index 31990fb32c39..0ac3d9da2c23 100644 --- a/x-pack/filebeat/module/mssql/log/config/config.yml +++ b/x-pack/filebeat/module/mssql/log/config/config.yml @@ -14,4 +14,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/netflow/log/config/netflow.yml b/x-pack/filebeat/module/netflow/log/config/netflow.yml index b34160bd6b9c..f56ffeed2267 100644 --- a/x-pack/filebeat/module/netflow/log/config/netflow.yml +++ b/x-pack/filebeat/module/netflow/log/config/netflow.yml @@ -31,4 +31,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/o365/audit/config/input.yml b/x-pack/filebeat/module/o365/audit/config/input.yml index d41a5bb9aabb..ec30daef1278 100644 --- a/x-pack/filebeat/module/o365/audit/config/input.yml +++ b/x-pack/filebeat/module/o365/audit/config/input.yml @@ -62,4 +62,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index a544ab8dc656..487dfdf165e8 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/rabbitmq/log/config/log.yml b/x-pack/filebeat/module/rabbitmq/log/config/log.yml index bc46f2458c88..1f32bbf71cfd 100644 --- a/x-pack/filebeat/module/rabbitmq/log/config/log.yml +++ b/x-pack/filebeat/module/rabbitmq/log/config/log.yml @@ -18,4 +18,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/sophos/xg/config/config.yml b/x-pack/filebeat/module/sophos/xg/config/config.yml index 86c12e9ec080..8585ff4a19c6 100644 --- a/x-pack/filebeat/module/sophos/xg/config/config.yml +++ b/x-pack/filebeat/module/sophos/xg/config/config.yml @@ -27,7 +27,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - add_fields: target: '_conf' fields: diff --git a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml index 6b6fcf216f2f..5794bd0ca6f7 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml +++ b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml @@ -22,4 +22,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml index 8a79295724fb..f6caa1436897 100644 --- a/x-pack/filebeat/module/zeek/connection/config/connection.yml +++ b/x-pack/filebeat/module/zeek/connection/config/connection.yml @@ -102,4 +102,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml index 45010e089733..7d7f0c873534 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml @@ -58,4 +58,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml index f1a2f0ced3a1..b049d571c140 100644 --- a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml +++ b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml @@ -120,4 +120,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml index 7730d2b6d85a..67033a42cf21 100644 --- a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml +++ b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml @@ -68,4 +68,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml index 86a2022d695e..9e0f7e952e26 100644 --- a/x-pack/filebeat/module/zeek/dns/config/dns.yml +++ b/x-pack/filebeat/module/zeek/dns/config/dns.yml @@ -203,4 +203,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml index acc6defd4df2..67565ef21abb 100644 --- a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml +++ b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml @@ -57,4 +57,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/files/config/files.yml b/x-pack/filebeat/module/zeek/files/config/files.yml index 65c067609c99..0e0ce22e2bf7 100644 --- a/x-pack/filebeat/module/zeek/files/config/files.yml +++ b/x-pack/filebeat/module/zeek/files/config/files.yml @@ -42,4 +42,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml index 51a3c0535767..671a1162c665 100644 --- a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml +++ b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml @@ -86,4 +86,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml index 4c7c812d0ccb..8a43f07f0dd5 100644 --- a/x-pack/filebeat/module/zeek/http/config/http.yml +++ b/x-pack/filebeat/module/zeek/http/config/http.yml @@ -93,4 +93,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml index 5b73833ea352..80d565745f18 100644 --- a/x-pack/filebeat/module/zeek/intel/config/intel.yml +++ b/x-pack/filebeat/module/zeek/intel/config/intel.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/irc/config/irc.yml b/x-pack/filebeat/module/zeek/irc/config/irc.yml index 54aaa9d4f4b3..7e09f9fe376e 100644 --- a/x-pack/filebeat/module/zeek/irc/config/irc.yml +++ b/x-pack/filebeat/module/zeek/irc/config/irc.yml @@ -72,4 +72,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml index d656ad0ab6ac..a65bb88c0d00 100644 --- a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml +++ b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml @@ -73,4 +73,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml index 4c6e70d9f1cf..963c8469fe1a 100644 --- a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml +++ b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml @@ -72,4 +72,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml index 649d3f3ba970..07ccbfb76b47 100644 --- a/x-pack/filebeat/module/zeek/notice/config/notice.yml +++ b/x-pack/filebeat/module/zeek/notice/config/notice.yml @@ -104,4 +104,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml index c67f66b54b9f..135086e19f9d 100644 --- a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml +++ b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml @@ -86,4 +86,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml index 874a0fde6d94..933f829b7474 100644 --- a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml +++ b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml @@ -64,4 +64,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/pe/config/pe.yml b/x-pack/filebeat/module/zeek/pe/config/pe.yml index 3df430d7dc96..cfb8ea8c4b01 100644 --- a/x-pack/filebeat/module/zeek/pe/config/pe.yml +++ b/x-pack/filebeat/module/zeek/pe/config/pe.yml @@ -33,4 +33,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/radius/config/radius.yml b/x-pack/filebeat/module/zeek/radius/config/radius.yml index 66fccaa3f5ce..471275d5ece4 100644 --- a/x-pack/filebeat/module/zeek/radius/config/radius.yml +++ b/x-pack/filebeat/module/zeek/radius/config/radius.yml @@ -58,4 +58,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml index de71448fb1b0..071cc59fab94 100644 --- a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml +++ b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml @@ -88,4 +88,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml index 3adb14c55bf9..417e9ab4dcd8 100644 --- a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml +++ b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml @@ -73,4 +73,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/sip/config/sip.yml b/x-pack/filebeat/module/zeek/sip/config/sip.yml index 7aa30034de24..97885a1e2dcb 100644 --- a/x-pack/filebeat/module/zeek/sip/config/sip.yml +++ b/x-pack/filebeat/module/zeek/sip/config/sip.yml @@ -95,4 +95,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml index 763379a7d888..885669e4c9c0 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml @@ -101,4 +101,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml index c5f7c2e53e79..a3f02a7558f4 100644 --- a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml +++ b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml @@ -61,4 +61,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml index 624454ed1715..59a68e2054b4 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml @@ -57,4 +57,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml index 5b2f6595df2f..0af383a71879 100644 --- a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml +++ b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml index 0c7e05ce6db7..1fa024742be3 100644 --- a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml +++ b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml @@ -69,4 +69,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/socks/config/socks.yml b/x-pack/filebeat/module/zeek/socks/config/socks.yml index f834e5d1bccc..99613e4bfe3d 100644 --- a/x-pack/filebeat/module/zeek/socks/config/socks.yml +++ b/x-pack/filebeat/module/zeek/socks/config/socks.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml index c855d49dff2d..45f160cd99db 100644 --- a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml +++ b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml @@ -76,4 +76,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/stats/config/stats.yml b/x-pack/filebeat/module/zeek/stats/config/stats.yml index cdf243f7a454..45c22b6b3d2d 100644 --- a/x-pack/filebeat/module/zeek/stats/config/stats.yml +++ b/x-pack/filebeat/module/zeek/stats/config/stats.yml @@ -97,4 +97,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml index a89601cb717b..0d1b8fc05e50 100644 --- a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml +++ b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml @@ -57,4 +57,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml index 13a2a37cc695..741a0d7e4546 100644 --- a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml +++ b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml @@ -45,4 +45,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml index ac636e9e7c0a..243f83de760e 100644 --- a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml +++ b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml @@ -56,4 +56,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/weird/config/weird.yml b/x-pack/filebeat/module/zeek/weird/config/weird.yml index 5807f95927b3..4a4bab4dc332 100644 --- a/x-pack/filebeat/module/zeek/weird/config/weird.yml +++ b/x-pack/filebeat/module/zeek/weird/config/weird.yml @@ -56,4 +56,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zoom/webhook/config/webhook.yml b/x-pack/filebeat/module/zoom/webhook/config/webhook.yml index 207da5447e13..ac04136779cb 100644 --- a/x-pack/filebeat/module/zoom/webhook/config/webhook.yml +++ b/x-pack/filebeat/module/zoom/webhook/config/webhook.yml @@ -33,4 +33,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ff9bffda33ef..17c9234ea9f8 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -343,6 +343,14 @@ metricbeat.modules: metrics: - id: ["requests/count", "requests/duration"] +- module: azure + metricsets: + - app_state + enabled: true + period: 300s + application_id: '' + api_key: '' + #--------------------------------- Beat Module --------------------------------- - module: beat metricsets: diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 618ed4cd8548..6f604138505e 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -44,4 +44,8 @@ period: 24h metricsets: - s3_daily_storage +- module: aws + period: 1m + latency: 5m + metricsets: - s3_request diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index fe9aeea007f4..e4e55e82136a 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -11,16 +11,27 @@ module. Please see <> for more details. [float] == Module-specific configuration notes +* *AWS Credentials* + The `aws` module requires AWS credentials configuration in order to make AWS API calls. Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN`, or use shared AWS credentials file. Please see <> for more details. +* *regions* + This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available AWS regions. +* *latency* + +Some AWS services send monitoring metrics to CloudWatch with a latency to +process larger than Metricbeat collection period. This case, please specify a +`latency` parameter so collection start time and end time will be shifted by the +given latency amount. + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index f7b744c27cba..9786c4f7b386 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -27,6 +27,7 @@ import ( type Config struct { Period time.Duration `config:"period" validate:"nonzero,required"` Regions []string `config:"regions"` + Latency time.Duration `config:"latency"` AWSConfig awscommon.ConfigAWS `config:",inline"` TagsFilter []Tag `config:"tags_filter"` } @@ -37,6 +38,7 @@ type MetricSet struct { RegionsList []string Endpoint string Period time.Duration + Latency time.Duration AwsConfig *awssdk.Config AccountName string AccountID string @@ -87,6 +89,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { metricSet := MetricSet{ BaseMetricSet: base, Period: config.Period, + Latency: config.Latency, AwsConfig: &awsConfig, TagsFilter: config.TagsFilter, } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 42d68acb3af4..07e1f09acefe 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -141,7 +141,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // Check statistic method in config err := m.checkStatistics() diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 353ffd0e2369..393ddecb07ec 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -1357,7 +1357,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { Value: "test-ec2", }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1399,7 +1399,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { } resourceTypeTagFilters := map[string][]aws.Tag{} - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1447,7 +1447,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { Value: "foo", }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 36ad9a1ca026..4e0776072e68 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -88,7 +88,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index f8bd907b3f6c..b381dcac9433 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -79,7 +79,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 7c9d453baca7..53248284d418 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -69,7 +69,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/S3" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // GetMetricData for AWS S3 from Cloudwatch for _, regionName := range m.MetricSet.RegionsList { diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index afe53ac49ba1..63b93f6cdf48 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -69,7 +69,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/S3" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // GetMetricData for AWS S3 from Cloudwatch for _, regionName := range m.MetricSet.RegionsList { diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index 7bc6a8349d07..6dc0774f66df 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -67,7 +67,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/SQS" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 67e5809bc8e6..d18460838549 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -22,8 +22,13 @@ import ( ) // GetStartTimeEndTime function uses durationString to create startTime and endTime for queries. -func GetStartTimeEndTime(period time.Duration) (time.Time, time.Time) { +func GetStartTimeEndTime(period time.Duration, latency time.Duration) (time.Time, time.Time) { endTime := time.Now() + if latency != 0 { + // add latency if config is not 0 + endTime = endTime.Add(latency * -1) + } + // Set startTime double the period earlier than the endtime in order to // make sure GetMetricDataRequest gets the latest data point for each metric. return endTime.Add(period * -2), endTime diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index aef35f57e614..3c480d347dba 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -176,7 +176,7 @@ func TestGetListMetricsOutputWithWildcard(t *testing.T) { } func TestGetMetricDataPerRegion(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10 * time.Minute) + startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} var metricDataQueries []cloudwatch.MetricDataQuery @@ -205,7 +205,7 @@ func TestGetMetricDataPerRegion(t *testing.T) { } func TestGetMetricDataResults(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10 * time.Minute) + startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} metricInfo := cloudwatch.Metric{ diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 1f9ac04529e8..b06e466a01f5 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -102,3 +102,11 @@ api_key: '' metrics: - id: ["requests/count", "requests/duration"] + +- module: azure + metricsets: + - app_state + enabled: true + period: 300s + application_id: '' + api_key: '' diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 0f497af6fb44..f7215d4f991b 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -111,3 +111,11 @@ # api_key: '' # metrics: # - id: ["requests/count", "requests/duration"] + +#- module: azure +# metricsets: +# - app_state +# enabled: true +# period: 300s +# application_id: '' +# api_key: '' diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc index b0f76ecb6230..01f6389ab93e 100644 --- a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc @@ -37,6 +37,10 @@ The Azure billing dashboards show relevant usage and forecast information: image::./images/metricbeat-azure-billing-overview.png[] +The Azure app_state dashboard shows relevant application insights information: + +image::./images/metricbeat-azure-app-state-overview.png[] + [float] === Module-specific configuration notes @@ -112,6 +116,10 @@ so the `period` for `billing` metricset should be `24h` or multiples of `24h`. === `app_insights` This metricset will collect application insights metrics, the `period` (interval) for the `app-insights` metricset is set by default at `300s`. +[float] +=== `app_state` +This metricset concentrate on the most relevant application insights metrics and provides a dashboard for visualization, the `period` (interval) for the `app_state` metricset is set by default at `300s`. + [float] [[azure-api-cost]] == Additional notes about metrics and costs diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml index c6471dc108dc..a6476fcc957b 100644 --- a/x-pack/metricbeat/module/azure/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/_meta/fields.yml @@ -39,6 +39,10 @@ type: keyword description: > The subscription ID + - name: application_id + type: keyword + description: > + The application ID - name: dimensions.* type: object object_type: keyword diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-app-state-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-app-state-overview.json new file mode 100644 index 000000000000..b447a6b276e9 --- /dev/null +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-app-state-overview.json @@ -0,0 +1,1509 @@ +{ + "objects": [ + { + "attributes": { + "description": "Provides relevant app insights metrics for web applications", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "controlledBy": "1532342651170", + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "azure.app_state.application_id", + "negate": false, + "params": { + "query": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "azure.app_state.application_id": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "307a1ecd-284c-4f35-9a3c-d5b77c9a9c82", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "307a1ecd-284c-4f35-9a3c-d5b77c9a9c82", + "panelRefName": "panel_0", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Exceptions" + }, + "gridData": { + "h": 15, + "i": "654e745f-360d-4898-89b6-57f788c5f540", + "w": 20, + "x": 7, + "y": 0 + }, + "panelIndex": "654e745f-360d-4898-89b6-57f788c5f540", + "panelRefName": "panel_1", + "title": "Exceptions", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Available Memory" + }, + "gridData": { + "h": 15, + "i": "5adca737-559d-4b4f-9fa7-58841daa99c5", + "w": 21, + "x": 27, + "y": 0 + }, + "panelIndex": "5adca737-559d-4b4f-9fa7-58841daa99c5", + "panelRefName": "panel_2", + "title": "Available Memory", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "531cf244-45e0-43c3-9920-8f32397bd973", + "w": 8, + "x": 0, + "y": 15 + }, + "panelIndex": "531cf244-45e0-43c3-9920-8f32397bd973", + "panelRefName": "panel_3", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "b9242495-babc-48a7-9ad7-56c62b1dc117", + "w": 8, + "x": 8, + "y": 15 + }, + "panelIndex": "b9242495-babc-48a7-9ad7-56c62b1dc117", + "panelRefName": "panel_4", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "d311025a-f5c5-4e48-9f1c-710f59264c43", + "w": 8, + "x": 16, + "y": 15 + }, + "panelIndex": "d311025a-f5c5-4e48-9f1c-710f59264c43", + "panelRefName": "panel_5", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Requests" + }, + "gridData": { + "h": 15, + "i": "48974418-b1f7-4050-921e-a83771e125ae", + "w": 24, + "x": 24, + "y": 15 + }, + "panelIndex": "48974418-b1f7-4050-921e-a83771e125ae", + "panelRefName": "panel_6", + "title": "Requests", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Browser Send/Receive Duration" + }, + "gridData": { + "h": 15, + "i": "39d20db1-316a-4ff3-811a-5571cb4497c3", + "w": 24, + "x": 0, + "y": 30 + }, + "panelIndex": "39d20db1-316a-4ff3-811a-5571cb4497c3", + "panelRefName": "panel_7", + "title": "Browser Send/Receive Duration", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Browser Networking/Processing Duration" + }, + "gridData": { + "h": 15, + "i": "bc810208-0395-4c70-9057-d7307e064e43", + "w": 24, + "x": 24, + "y": 30 + }, + "panelIndex": "bc810208-0395-4c70-9057-d7307e064e43", + "panelRefName": "panel_8", + "title": "Browser Networking/Processing Duration", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Process CPU Usage" + }, + "gridData": { + "h": 15, + "i": "ecf6fbfa-ba65-481e-af85-07fd9d5feb5f", + "w": 24, + "x": 0, + "y": 45 + }, + "panelIndex": "ecf6fbfa-ba65-481e-af85-07fd9d5feb5f", + "panelRefName": "panel_9", + "title": "Process CPU Usage", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Process Private Bytes" + }, + "gridData": { + "h": 15, + "i": "40a1b80b-cd62-446d-91aa-a971bb3769e7", + "w": 24, + "x": 24, + "y": 45 + }, + "panelIndex": "40a1b80b-cd62-446d-91aa-a971bb3769e7", + "panelRefName": "panel_10", + "title": "Process Private Bytes", + "version": "7.9.2" + } + ], + "timeRestore": false, + "title": "[Metricbeat Azure] App State Overview", + "version": 1 + }, + "id": "d5fbd610-03d9-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "2e5183a0-03da-11eb-8034-63f2039e9d3f", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "1064f9a0-04a5-11eb-8034-63f2039e9d3f", + "name": "panel_1", + "type": "lens" + }, + { + "id": "76cc1d70-04a7-11eb-8034-63f2039e9d3f", + "name": "panel_2", + "type": "lens" + }, + { + "id": "a89c8fd0-03ec-11eb-8034-63f2039e9d3f", + "name": "panel_3", + "type": "lens" + }, + { + "id": "cb5ec410-03ed-11eb-8034-63f2039e9d3f", + "name": "panel_4", + "type": "lens" + }, + { + "id": "0df175c0-03ee-11eb-8034-63f2039e9d3f", + "name": "panel_5", + "type": "lens" + }, + { + "id": "f0678020-04a2-11eb-8034-63f2039e9d3f", + "name": "panel_6", + "type": "lens" + }, + { + "id": "e2704140-04a3-11eb-8034-63f2039e9d3f", + "name": "panel_7", + "type": "lens" + }, + { + "id": "0e74dee0-04a4-11eb-8034-63f2039e9d3f", + "name": "panel_8", + "type": "lens" + }, + { + "id": "cfa361a0-04a8-11eb-8034-63f2039e9d3f", + "name": "panel_9", + "type": "lens" + }, + { + "id": "2b54b2c0-04a8-11eb-8034-63f2039e9d3f", + "name": "panel_10", + "type": "lens" + } + ], + "type": "dashboard", + "updated_at": "2020-10-02T12:22:06.090Z", + "version": "WzMzMDEsMl0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "controlledBy": "1532342651170", + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "azure.application_id", + "negate": false, + "params": { + "query": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "azure.application_id": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "App State Filters [Metricbeat Azure]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "azure.application_id", + "id": "1532342651170", + "indexPatternRefName": "control_0_index_pattern", + "label": "Application ID", + "options": { + "multiselect": true, + "order": "desc", + "size": 10, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "azure.dimensions.request_url_host", + "id": "1601559750853", + "indexPatternRefName": "control_1_index_pattern", + "label": "Host URL", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "1532342651170", + "type": "list" + }, + { + "fieldName": "azure.dimensions.cloud_role_name", + "id": "1601640368472", + "indexPatternRefName": "control_2_index_pattern", + "label": "Name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "1532342651170", + "type": "list" + }, + { + "fieldName": "azure.dimensions.browser_timing_url_host", + "id": "1601640439434", + "indexPatternRefName": "control_3_index_pattern", + "label": "Browser URL Host", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "1532342651170", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "App State Filters [Metricbeat Azure]", + "type": "input_control_vis" + } + }, + "id": "2e5183a0-03da-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "visualization": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_1_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_2_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_3_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-10-02T12:22:51.232Z", + "version": "WzMzMTIsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"5788331a-267d-426a-a68e-94a5310af644\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.exception_type\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.exceptions_count.sum\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.exceptions_server.sum\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"9e183a5e-3dba-4929-b07e-2a3321f7926b\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.exceptions_browser.sum\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-4-5788331a-267d-426a-a68e-94a5310af644\\\":{\\\"label\\\":\\\"Type\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.exception_type\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"5788331a-267d-426a-a68e-94a5310af644\\\"},\\\"col-5-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Total\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.exceptions_count.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"col-6-e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\":{\\\"label\\\":\\\"Server \\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.exceptions_server.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\"},\\\"col-7-9e183a5e-3dba-4929-b07e-2a3321f7926b\\\":{\\\"label\\\":\\\"Browser\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.exceptions_browser.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"9e183a5e-3dba-4929-b07e-2a3321f7926b\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Total\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"5788331a-267d-426a-a68e-94a5310af644\" seriesType=\"area_stacked\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" accessors=\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\" accessors=\"9e183a5e-3dba-4929-b07e-2a3321f7926b\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Total\\\",\\\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\":\\\"Server \\\",\\\"9e183a5e-3dba-4929-b07e-2a3321f7926b\\\":\\\"Browser\\\",\\\"5788331a-267d-426a-a68e-94a5310af644\\\":\\\"Type\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "5788331a-267d-426a-a68e-94a5310af644", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "e5c93c50-bb0a-4609-a7ce-7003f2f9a20e", + "9e183a5e-3dba-4929-b07e-2a3321f7926b" + ], + "columns": { + "5788331a-267d-426a-a68e-94a5310af644": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Type", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.exception_type" + }, + "9e183a5e-3dba-4929-b07e-2a3321f7926b": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Browser", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.exceptions_browser.sum" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.exceptions_count.sum" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + }, + "e5c93c50-bb0a-4609-a7ce-7003f2f9a20e": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Server ", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.exceptions_server.sum" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "e5c93c50-bb0a-4609-a7ce-7003f2f9a20e", + "9e183a5e-3dba-4929-b07e-2a3321f7926b" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area_stacked", + "showGridlines": false, + "splitAccessor": "5788331a-267d-426a-a68e-94a5310af644", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area_stacked" + } + }, + "title": "App state Exceptions [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "1064f9a0-04a5-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:53:16.483Z", + "version": "WzI5NjMsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_memory_available_bytes.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-2-a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":{\\\"label\\\":\\\"Instance\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\"},\\\"col-3-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Available memory\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_memory_available_bytes.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"bytes\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"}}\" | lens_format_column format=\"bytes\" columnId=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" decimals=2}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Available memory\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\" seriesType=\"area\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Available memory\\\",\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":\\\"Instance\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "columns": { + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Instance", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.cloud_role_instance" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Available memory", + "operationType": "avg", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_memory_available_bytes.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Memory [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "76cc1d70-04a7-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T12:04:46.406Z", + "version": "WzMwNTcsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"82e648a8-6d9a-4ae0-9449-b802ce1ac723\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.users_count.unique\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\":{\\\"label\\\":\\\"Unique users\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.users_count.unique\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\"}}\"}\n| lens_metric_chart title=\"Unique users\" accessor=\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\" mode=\"full\"", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "82e648a8-6d9a-4ae0-9449-b802ce1ac723": { + "columnOrder": [ + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1" + ], + "columns": { + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique users", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.users_count.unique" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1", + "layerId": "82e648a8-6d9a-4ae0-9449-b802ce1ac723" + } + }, + "title": "App state Unique users [Metricbeat Azure]", + "visualizationType": "lnsMetric" + }, + "id": "a89c8fd0-03ec-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-01T13:47:34.093Z", + "version": "WzEzMzAsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"82e648a8-6d9a-4ae0-9449-b802ce1ac723\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.users_authenticated.unique\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\":{\\\"label\\\":\\\"Unique authenticated users\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.users_authenticated.unique\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\"}}\"}\n| lens_metric_chart title=\"Unique authenticated users\" accessor=\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\" mode=\"full\"", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "82e648a8-6d9a-4ae0-9449-b802ce1ac723": { + "columnOrder": [ + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1" + ], + "columns": { + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique authenticated users", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.users_authenticated.unique" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1", + "layerId": "82e648a8-6d9a-4ae0-9449-b802ce1ac723" + } + }, + "title": "App state Unique authenticated users [Metricbeat Azure]", + "visualizationType": "lnsMetric" + }, + "id": "cb5ec410-03ed-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-01T13:55:41.904Z", + "version": "WzE0MjAsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"82e648a8-6d9a-4ae0-9449-b802ce1ac723\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.sessions_count.unique\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\":{\\\"label\\\":\\\"Unique sessions\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.sessions_count.unique\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\"}}\"}\n| lens_metric_chart title=\"Unique sessions\" accessor=\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\" mode=\"full\"", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "82e648a8-6d9a-4ae0-9449-b802ce1ac723": { + "columnOrder": [ + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1" + ], + "columns": { + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique sessions", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.sessions_count.unique" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1", + "layerId": "82e648a8-6d9a-4ae0-9449-b802ce1ac723" + } + }, + "title": "App state Unique sessions [Metricbeat Azure]", + "visualizationType": "lnsMetric" + }, + "id": "0df175c0-03ee-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-01T13:57:33.596Z", + "version": "WzE0NjAsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"8864c98b-413a-484f-a61d-336a63ef3f13\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.request_url_host\\\",\\\"orderBy\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.requests_count.sum\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.requests_failed.sum\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-3-8864c98b-413a-484f-a61d-336a63ef3f13\\\":{\\\"label\\\":\\\"Host URL\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.request_url_host\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"8864c98b-413a-484f-a61d-336a63ef3f13\\\"},\\\"col-4-9ec4d260-e302-46c4-ac09-50ef54627894\\\":{\\\"label\\\":\\\"Total requests\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.requests_count.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\"},\\\"col-5-a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\":{\\\"label\\\":\\\"Failed requests\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.requests_failed.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Total requests\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"8864c98b-413a-484f-a61d-336a63ef3f13\" seriesType=\"area\" accessors=\"9ec4d260-e302-46c4-ac09-50ef54627894\" accessors=\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\" columnToLabel=\"{\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\":\\\"Total requests\\\",\\\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\":\\\"Failed requests\\\",\\\"8864c98b-413a-484f-a61d-336a63ef3f13\\\":\\\"Host URL\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "8864c98b-413a-484f-a61d-336a63ef3f13", + "9ec4d260-e302-46c4-ac09-50ef54627894", + "a47e59dc-fb62-42f8-90e1-236c7c5a073d" + ], + "columns": { + "8864c98b-413a-484f-a61d-336a63ef3f13": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Host URL", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "9ec4d260-e302-46c4-ac09-50ef54627894", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.request_url_host" + }, + "9ec4d260-e302-46c4-ac09-50ef54627894": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total requests", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.requests_count.sum" + }, + "a47e59dc-fb62-42f8-90e1-236c7c5a073d": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failed requests", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.requests_failed.sum" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "9ec4d260-e302-46c4-ac09-50ef54627894", + "a47e59dc-fb62-42f8-90e1-236c7c5a073d" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "8864c98b-413a-484f-a61d-336a63ef3f13", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Requests [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "f0678020-04a2-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:32:22.946Z", + "version": "WzI0MzUsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"orderBy\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_send_duration.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_total_duration.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"988e9976-3471-478c-89f6-11fd46458d7f\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_receive_duration.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-4-4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":{\\\"label\\\":\\\"Url Path\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\"},\\\"col-5-be6a3d8b-9428-480b-a7b3-071127726093\\\":{\\\"label\\\":\\\"Send duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_send_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"params\\\":{},\\\"customLabel\\\":true,\\\"id\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\"},\\\"col-6-6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\":{\\\"label\\\":\\\"Total duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_total_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\"},\\\"col-7-988e9976-3471-478c-89f6-11fd46458d7f\\\":{\\\"label\\\":\\\"Receive duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_receive_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"988e9976-3471-478c-89f6-11fd46458d7f\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Send duration\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"4d4c068a-0194-4d54-a1fa-3863c3df9331\" seriesType=\"bar\" accessors=\"be6a3d8b-9428-480b-a7b3-071127726093\" accessors=\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\" accessors=\"988e9976-3471-478c-89f6-11fd46458d7f\" columnToLabel=\"{\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\":\\\"Send duration\\\",\\\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\":\\\"Total duration\\\",\\\"988e9976-3471-478c-89f6-11fd46458d7f\\\":\\\"Receive duration\\\",\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":\\\"Url Path\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "be6a3d8b-9428-480b-a7b3-071127726093", + "6bc1fd35-168d-42d5-b9c8-7078896d8ce4", + "988e9976-3471-478c-89f6-11fd46458d7f" + ], + "columns": { + "4d4c068a-0194-4d54-a1fa-3863c3df9331": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Url Path", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "be6a3d8b-9428-480b-a7b3-071127726093", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.browser_timing_url_path" + }, + "6bc1fd35-168d-42d5-b9c8-7078896d8ce4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_total_duration.avg" + }, + "988e9976-3471-478c-89f6-11fd46458d7f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Receive duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_receive_duration.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + }, + "be6a3d8b-9428-480b-a7b3-071127726093": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Send duration", + "operationType": "avg", + "params": {}, + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_send_duration.avg" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "be6a3d8b-9428-480b-a7b3-071127726093", + "6bc1fd35-168d-42d5-b9c8-7078896d8ce4", + "988e9976-3471-478c-89f6-11fd46458d7f" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "bar", + "showGridlines": false, + "splitAccessor": "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar" + } + }, + "title": "App state Browser Send/Receive Duration [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "e2704140-04a3-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:39:09.012Z", + "version": "WzI2MzQsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"orderBy\\\":\\\"_key\\\",\\\"order\\\":\\\"asc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_network_duration.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"ab158cba-532f-47f8-8450-db883504dc0f\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_processing_duration.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-3-4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":{\\\"label\\\":\\\"Url Path\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"alphabetical\\\"},\\\"orderDirection\\\":\\\"asc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\"},\\\"col-4-b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\":{\\\"label\\\":\\\"Networking duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_network_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\"},\\\"col-5-ab158cba-532f-47f8-8450-db883504dc0f\\\":{\\\"label\\\":\\\"Processing duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_processing_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"ab158cba-532f-47f8-8450-db883504dc0f\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Networking duration\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"4d4c068a-0194-4d54-a1fa-3863c3df9331\" seriesType=\"bar\" accessors=\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\" accessors=\"ab158cba-532f-47f8-8450-db883504dc0f\" columnToLabel=\"{\\\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\":\\\"Networking duration\\\",\\\"ab158cba-532f-47f8-8450-db883504dc0f\\\":\\\"Processing duration\\\",\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":\\\"Url Path\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "b5a75764-e98b-434b-a0f0-5658a4aa1cf6", + "ab158cba-532f-47f8-8450-db883504dc0f" + ], + "columns": { + "4d4c068a-0194-4d54-a1fa-3863c3df9331": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Url Path", + "operationType": "terms", + "params": { + "orderBy": { + "type": "alphabetical" + }, + "orderDirection": "asc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.browser_timing_url_path" + }, + "ab158cba-532f-47f8-8450-db883504dc0f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Processing duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_processing_duration.avg" + }, + "b5a75764-e98b-434b-a0f0-5658a4aa1cf6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Networking duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_network_duration.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b5a75764-e98b-434b-a0f0-5658a4aa1cf6", + "ab158cba-532f-47f8-8450-db883504dc0f" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "bar", + "showGridlines": false, + "splitAccessor": "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar" + } + }, + "title": "App state Browser Networking/Processing Duration [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "0e74dee0-04a4-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:40:22.862Z", + "version": "WzI2ODQsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage_total.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"252dfd5f-26bd-4861-bb01-4b3530cadd95\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-3-a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":{\\\"label\\\":\\\"Instance\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\"},\\\"col-4-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Total CPU percentage \\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage_total.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"percent\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"col-5-252dfd5f-26bd-4861-bb01-4b3530cadd95\\\":{\\\"label\\\":\\\"CPU percentage\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"percent\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"customLabel\\\":true,\\\"id\\\":\\\"252dfd5f-26bd-4861-bb01-4b3530cadd95\\\"}}\" | lens_format_column format=\"percent\" columnId=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" decimals=2 | lens_format_column format=\"percent\" columnId=\"252dfd5f-26bd-4861-bb01-4b3530cadd95\" decimals=2}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Total CPU percentage \" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\" seriesType=\"area\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" accessors=\"252dfd5f-26bd-4861-bb01-4b3530cadd95\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Total CPU percentage \\\",\\\"252dfd5f-26bd-4861-bb01-4b3530cadd95\\\":\\\"CPU percentage\\\",\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":\\\"Instance\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "252dfd5f-26bd-4861-bb01-4b3530cadd95" + ], + "columns": { + "252dfd5f-26bd-4861-bb01-4b3530cadd95": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "CPU percentage", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_process_cpu_percentage.avg" + }, + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Instance", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.cloud_role_instance" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total CPU percentage ", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_process_cpu_percentage_total.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "252dfd5f-26bd-4861-bb01-4b3530cadd95" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Process CPU usage [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "cfa361a0-04a8-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T12:14:24.954Z", + "version": "WzMyNTMsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_process_private_bytes.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-2-a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":{\\\"label\\\":\\\"Instance\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\"},\\\"col-3-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Process private bytes\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_process_private_bytes.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"bytes\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"}}\" | lens_format_column format=\"bytes\" columnId=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" decimals=2}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Process private bytes\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\" seriesType=\"area\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Process private bytes\\\",\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":\\\"Instance\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "columns": { + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Instance", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.cloud_role_instance" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Process private bytes", + "operationType": "avg", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_process_private_bytes.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Process Private Bytes [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "2b54b2c0-04a8-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T12:09:49.291Z", + "version": "WzMxMjMsMl0=" + } + ], + "version": "7.9.2" +} diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go index ba8f35c7db68..1342170aec06 100644 --- a/x-pack/metricbeat/module/azure/add_metadata.go +++ b/x-pack/metricbeat/module/azure/add_metadata.go @@ -35,7 +35,7 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) { } } -func addCloudVMMetadata(event *mb.Event, vm VmResource) { +func addCloudVMMetadata(event *mb.Event, vm VmResource, subscriptionId string) { if vm.Name != "" { event.RootFields.Put("cloud.instance.name", vm.Name) event.RootFields.Put("host.name", vm.Name) @@ -47,4 +47,7 @@ func addCloudVMMetadata(event *mb.Event, vm VmResource) { if vm.Size != "" { event.RootFields.Put("cloud.machine.type", vm.Size) } + if subscriptionId != "" { + event.RootFields.Put("cloud.account.id", subscriptionId) + } } diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/data.json b/x-pack/metricbeat/module/azure/app_insights/_meta/data.json index 3d1f07c1ac1b..c9b35f399236 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/data.json +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/data.json @@ -1,16 +1,15 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "azure" : { - "app_insights" : { - "metrics" : { - "requests_failed" : { - "sum" : 182 - }, - "request_name" : "GET /favicon.ico" - }, - "start_date" : "2020-07-12T10:52:11.831Z", - "end_date" : "2020-07-12T12:52:11.831Z", - "application_id" : "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + "azure": { + "app_insights": { + "end_date": "2020-10-02T13:17:45.691Z", + "start_date": "2020-10-02T13:12:45.691Z" + }, + "application_id": "42cb59a9-d5be-400b-a5c4-69b0a00434fdf4", + "metrics": { + "requests_count": { + "sum": 0 + } } }, "cloud": { diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml b/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml index 40ab85608275..4e8fa91edbd6 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml @@ -4,10 +4,6 @@ description: > application insights fields: - - name: application_id - type: keyword - description: > - The application ID - name: start_date type: date description: > diff --git a/x-pack/metricbeat/module/azure/app_insights/app_insights.go b/x-pack/metricbeat/module/azure/app_insights/app_insights.go index 8ffe02eb860a..091a72f465ed 100644 --- a/x-pack/metricbeat/module/azure/app_insights/app_insights.go +++ b/x-pack/metricbeat/module/azure/app_insights/app_insights.go @@ -23,6 +23,7 @@ type Config struct { ApiKey string `config:"api_key" validate:"required"` Period time.Duration `config:"period" validate:"nonzero,required"` Metrics []Metric `config:"metrics" validate:"required"` + Namespace string `config:"namespace"` } // Metric struct used for configuration options @@ -70,7 +71,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err != nil { return errors.Wrap(err, "error retrieving metric values") } - events := EventsMapping(results, m.client.Config.ApplicationId) + events := EventsMapping(results, m.client.Config.ApplicationId, m.client.Config.Namespace) for _, event := range events { isOpen := report.Event(event) if !isOpen { diff --git a/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go b/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go index 3cb93663007a..c33f05c2de65 100644 --- a/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go +++ b/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go @@ -10,12 +10,20 @@ package app_insights import ( "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) +var metrics = []map[string]interface{}{{ + "id": "requests/count", +}} + func TestFetchMetricset(t *testing.T) { + config := test.GetConfigForInsights(t, "app_insights") + config["metrics"] = metrics metricSet := mbtest.NewReportingMetricSetV2Error(t, config) events, errs := mbtest.ReportingFetchV2Error(metricSet) if len(errs) > 0 { @@ -26,6 +34,8 @@ func TestFetchMetricset(t *testing.T) { } func TestData(t *testing.T) { + config := test.GetConfigForInsights(t, "app_insights") + config["metrics"] = metrics metricSet := mbtest.NewFetcher(t, config) metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/app_insights/client.go b/x-pack/metricbeat/module/azure/app_insights/client.go index b78f2257d3f2..49794da4d3fe 100644 --- a/x-pack/metricbeat/module/azure/app_insights/client.go +++ b/x-pack/metricbeat/module/azure/app_insights/client.go @@ -23,9 +23,6 @@ type Client struct { Log *logp.Logger } -type MetricValue struct { -} - // NewClient instantiates the an Azure monitoring client func NewClient(config Config) (*Client, error) { service, err := NewService(config) diff --git a/x-pack/metricbeat/module/azure/app_insights/data.go b/x-pack/metricbeat/module/azure/app_insights/data.go index df7efdbeaba9..1cd661cc3a0b 100644 --- a/x-pack/metricbeat/module/azure/app_insights/data.go +++ b/x-pack/metricbeat/module/azure/app_insights/data.go @@ -6,98 +6,217 @@ package app_insights import ( "fmt" + "regexp" "strings" + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" "github.com/Azure/go-autorest/autorest/date" - "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" ) -func EventsMapping(metricValues insights.ListMetricsResultsItem, applicationId string) []mb.Event { - var events []mb.Event - if metricValues.Value == nil { - return events - } - groupedAddProp := make(map[string][]insights.MetricsResultInfo) +const aggsRegex = "_(?:sum|count|unique|avg|min|max)$" + +// segmentNames list is used to filter out the dimension from the api response. Based on the body format it is not possible to detect what was the segment selected +var segmentNames = []string{ + "request_source", "request_name", "request_url_host", "request_url_path", "request_success", "request_result_code", "request_performance_bucket", "operation_name", "operation_synthetic", "operation_synthetic_source", "user_authenticated", "application_version", "client_type", "client_model", + "client_os", "client_city", "client_state_or_province", "client_country_or_region", "client_browser", "cloud_role_name", "cloud_role_instance", "custom_dimensions__ms_processedb_by_metric_extractors", "custom_dimensions_developer_mode", + "page_view_name", "page_view_url_path", "page_view_url_host", "page_view_performance_bucket", "custom_dimensions_ibiza_session_id", "custom_dimensions_part_instance", "browser_timing_name", "browser_timing_url_host", "browser_timing_url_path", "browser_timing_performance_bucket", + "trace_severity_level", "type", "custom_dimensions_agent_session", "custom_dimensions_agent_version", "custom_dimensions_machine_name", "custom_dimensions_running_mode", "custom_dimensions_source", "custom_dimensions_agent_assembly_version", "custom_dimensions_agent_process_session", + "custom_dimensions_hashed_machine_name", + "custom_dimensions_data_cube", "dependency_target", "dependency_type", "dependency_name", "dependency_success", "dependency_result_code", "dependency_performance_bucket", "custom_dimensions_container", "custom_dimensions_blob", "custom_dimensions_error_message", + "custom_event_name", "custom_dimensions_event_name", "custom_dimensions_page_title", "custom_dimensions_service_profiler_content", "custom_dimensions_executing_assembly_file_version", "custom_dimensions_service_profiler_version", "custom_dimensions_process_id", "custom_dimensions_request_id", + "custom_dimensions_running_session", "custom_dimensions_problem_id", "custom_dimensions_snapshot_context", "custom_dimensions_snapshot_version", "custom_dimensions_duration", "custom_dimensions_snapshot_id", "custom_dimensions_stamp_id", "custom_dimensions_de_optimization_id", + "custom_dimensions_method", "custom_dimensions_parent_process_id", "custom_dimensions_section", "custom_dimensions_configuration", "custom_dimensions_dump_folder", "custom_dimensions_reason", "custom_dimensions_extension_version", "custom_dimensions_site_name", + "availability_result_name", "availability_result_location", "availability_result_success", "custom_dimensions_full_test_result_available", "exception_problem_id", "exception_handled_at", "exception_type", "exception_assembly", "exception_method", "custom_dimensions_custom_perf_counter", + "exception_severity_level", "custom_dimensions_url", "custom_dimensions_ai.snapshot_stampid", "custom_dimensions_ai.snapshot_id", "custom_dimensions_ai.snapshot_version", "custom_dimensions_ai.snapshot_planid", "custom_dimensions__ms_example", "custom_dimensions_sa_origin_app_id", + "custom_dimensions_base_sdk_target_framework", "custom_dimensions_runtime_framework", "custom_dimensions__ms_aggregation_interval_ms", "custom_dimensions_problem_id", "custom_dimensions_operation_name", "custom_dimensions_request_success", "custom_dimensions__ms_metric_id", + "custom_dimensions_dependency_success", "custom_dimensions__ms_is_autocollected", "custom_dimensions_dependency_type", "performance_counter_name", "performance_counter_category", "performance_counter_counter", "performance_counter_instance", "custom_dimensions_counter_instance_name", +} + +type MetricValue struct { + SegmentName map[string]string + Value map[string]interface{} + Segments []MetricValue + Interval string + Start *date.Time + End *date.Time +} + +func mapMetricValues(metricValues insights.ListMetricsResultsItem) []MetricValue { + var mapped []MetricValue for _, item := range *metricValues.Value { + metricValue := MetricValue{ + Start: item.Body.Value.Start, + End: item.Body.Value.End, + Value: map[string]interface{}{}, + SegmentName: map[string]string{}, + } + metricValue.Interval = fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End) if item.Body != nil && item.Body.Value != nil { if item.Body.Value.AdditionalProperties != nil { - groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)] = - append(groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)], *item.Body.Value) - } else if item.Body.Value.Segments != nil { - for _, segment := range *item.Body.Value.Segments { - event, ok := createSegmentEvent(*item.Body.Value.Start, *item.Body.Value.End, segment, applicationId) - if ok { - events = append(events, event) + metrics := getAdditionalPropMetric(item.Body.Value.AdditionalProperties) + for key, metric := range metrics { + if isSegment(key) { + metricValue.SegmentName[key] = metric.(string) + } else { + metricValue.Value[key] = metric } } } + if item.Body.Value.Segments != nil { + for _, segment := range *item.Body.Value.Segments { + metVal := mapSegment(segment, metricValue.SegmentName) + metricValue.Segments = append(metricValue.Segments, metVal) + } + } + mapped = append(mapped, metricValue) } + } - if len(groupedAddProp) > 0 { - for _, val := range groupedAddProp { - event, ok := createEvent(val, applicationId) - if ok { - events = append(events, event) + return mapped +} + +func mapSegment(segment insights.MetricsSegmentInfo, parentSeg map[string]string) MetricValue { + metricValue := MetricValue{Value: map[string]interface{}{}, SegmentName: map[string]string{}} + if segment.AdditionalProperties != nil { + metrics := getAdditionalPropMetric(segment.AdditionalProperties) + for key, metric := range metrics { + if isSegment(key) { + metricValue.SegmentName[key] = metric.(string) + } else { + metricValue.Value[key] = metric + } + } + } + if len(parentSeg) > 0 { + for key, val := range parentSeg { + metricValue.SegmentName[key] = val + } + } + if segment.Segments != nil { + for _, segment := range *segment.Segments { + metVal := mapSegment(segment, metricValue.SegmentName) + metricValue.Segments = append(metricValue.Segments, metVal) + } + } + + return metricValue +} + +func isSegment(metric string) bool { + for _, seg := range segmentNames { + if metric == seg { + return true + } + } + return false +} + +func EventsMapping(metricValues insights.ListMetricsResultsItem, applicationId string, namespace string) []mb.Event { + var events []mb.Event + if metricValues.Value == nil { + return events + } + groupedAddProp := make(map[string][]MetricValue) + mValues := mapMetricValues(metricValues) + + var segValues []MetricValue + for _, mv := range mValues { + if len(mv.Segments) == 0 { + groupedAddProp[mv.Interval] = append(groupedAddProp[mv.Interval], mv) + } else { + segValues = append(segValues, mv) + } + } + + for _, val := range groupedAddProp { + event := createNoSegEvent(val, applicationId, namespace) + if len(event.MetricSetFields) > 0 { + events = append(events, event) + } + } + for _, val := range segValues { + for _, seg := range val.Segments { + lastSeg := getValue(seg) + for _, ls := range lastSeg { + events = append(events, createSegEvent(val, ls, applicationId, namespace)) } } } return events } -func createSegmentEvent(start date.Time, end date.Time, segment insights.MetricsSegmentInfo, applicationId string) (mb.Event, bool) { - metricList := common.MapStr{} - metrics := getMetric(segment.AdditionalProperties) - if len(metrics) == 0 { - return mb.Event{}, false +func getValue(metric MetricValue) []MetricValue { + var values []MetricValue + if metric.Segments == nil { + return []MetricValue{metric} + } + for _, met := range metric.Segments { + values = append(values, getValue(met)...) } - for key, metric := range metrics { + return values +} + +func createSegEvent(parentMetricValue MetricValue, metricValue MetricValue, applicationId string, namespace string) mb.Event { + metricList := common.MapStr{} + for key, metric := range metricValue.Value { metricList.Put(key, metric) } + if len(metricList) == 0 { + return mb.Event{} + } + event := createEvent(parentMetricValue.Start, parentMetricValue.End, applicationId, namespace, metricList) + if len(parentMetricValue.SegmentName) > 0 { + event.ModuleFields.Put("dimensions", parentMetricValue.SegmentName) + } + if len(metricValue.SegmentName) > 0 { + event.ModuleFields.Put("dimensions", metricValue.SegmentName) + } + return event +} + +func createEvent(start *date.Time, end *date.Time, applicationId string, namespace string, metricList common.MapStr) mb.Event { event := mb.Event{ - MetricSetFields: common.MapStr{ - "start_date": start, - "end_date": end, + ModuleFields: common.MapStr{ "application_id": applicationId, }, + MetricSetFields: common.MapStr{ + "start_date": start, + "end_date": end, + }, Timestamp: end.Time, } event.RootFields = common.MapStr{} event.RootFields.Put("cloud.provider", "azure") - event.MetricSetFields.Put("metrics", metricList) - return event, true + if namespace == "" { + event.ModuleFields.Put("metrics", metricList) + } else { + for key, metric := range metricList { + event.MetricSetFields.Put(key, metric) + } + } + return event } -func createEvent(values []insights.MetricsResultInfo, applicationId string) (mb.Event, bool) { +func createNoSegEvent(values []MetricValue, applicationId string, namespace string) mb.Event { metricList := common.MapStr{} for _, value := range values { - metrics := getMetric(value.AdditionalProperties) - for key, metric := range metrics { + for key, metric := range value.Value { metricList.Put(key, metric) } } if len(metricList) == 0 { - return mb.Event{}, false + return mb.Event{} } + return createEvent(values[0].Start, values[0].End, applicationId, namespace, metricList) - event := mb.Event{ - MetricSetFields: common.MapStr{ - "start_date": values[0].Start, - "end_date": values[0].End, - "application_id": applicationId, - }, - Timestamp: values[0].End.Time, - } - event.RootFields = common.MapStr{} - event.RootFields.Put("cloud.provider", "azure") - event.MetricSetFields.Put("metrics", metricList) - return event, true } -func getMetric(addProp map[string]interface{}) map[string]interface{} { +func getAdditionalPropMetric(addProp map[string]interface{}) map[string]interface{} { metricNames := make(map[string]interface{}) for key, val := range addProp { switch val.(type) { @@ -115,5 +234,19 @@ func getMetric(addProp map[string]interface{}) map[string]interface{} { } func cleanMetricNames(metric string) string { - return strings.Replace(metric, "/", "_", -1) + metric = strings.Replace(metric, "/", "_", -1) + metric = strings.Replace(metric, " ", "_", -1) + metric = azure.ReplaceUpperCase(metric) + obj := strings.Split(metric, ".") + for index := range obj { + // in some cases a trailing "_" is found + obj[index] = strings.TrimPrefix(obj[index], "_") + obj[index] = strings.TrimSuffix(obj[index], "_") + } + metric = strings.ToLower(strings.Join(obj, "_")) + aggsRegex := regexp.MustCompile(aggsRegex) + metric = aggsRegex.ReplaceAllStringFunc(metric, func(str string) string { + return strings.Replace(str, "_", ".", -1) + }) + return metric } diff --git a/x-pack/metricbeat/module/azure/app_insights/data_test.go b/x-pack/metricbeat/module/azure/app_insights/data_test.go index ebe4e7d98aa0..f65f9f84b1d2 100644 --- a/x-pack/metricbeat/module/azure/app_insights/data_test.go +++ b/x-pack/metricbeat/module/azure/app_insights/data_test.go @@ -39,20 +39,32 @@ func TestEventMapping(t *testing.T) { Value: &metrics, } applicationId := "abc" - events := EventsMapping(result, applicationId) + events := EventsMapping(result, applicationId, "") assert.Equal(t, len(events), 1) for _, event := range events { val1, _ := event.MetricSetFields.GetValue("start_date") assert.Equal(t, val1, &startDate) val2, _ := event.MetricSetFields.GetValue("end_date") assert.Equal(t, val2, &startDate) - val3, _ := event.MetricSetFields.GetValue("metrics.requests_count") + val3, _ := event.ModuleFields.GetValue("metrics.requests_count") assert.Equal(t, val3, common.MapStr{"sum": 12}) - val5, _ := event.MetricSetFields.GetValue("metrics.requests_failed") + val5, _ := event.ModuleFields.GetValue("metrics.requests_failed") assert.Equal(t, val5, common.MapStr{"sum": 10}) - val4, _ := event.MetricSetFields.GetValue("application_id") + val4, _ := event.ModuleFields.GetValue("application_id") assert.Equal(t, val4, applicationId) } } + +func TestCleanMetricNames(t *testing.T) { + ex := "customDimensions/ExecutingAssemblyFileVersion" + result := cleanMetricNames(ex) + assert.Equal(t, result, "custom_dimensions_executing_assembly_file_version") + ex = "customDimensions/_MS.AggregationIntervalMs" + result = cleanMetricNames(ex) + assert.Equal(t, result, "custom_dimensions__ms_aggregation_interval_ms") + ex = "customDimensions/_MS.IsAutocollected" + result = cleanMetricNames(ex) + assert.Equal(t, result, "custom_dimensions__ms_is_autocollected") +} diff --git a/x-pack/metricbeat/module/azure/app_state/_meta/data.json b/x-pack/metricbeat/module/azure/app_state/_meta/data.json new file mode 100644 index 000000000000..c4744db7f60e --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "app_state": { + "end_date": "2020-10-02T13:23:11.221Z", + "requests_count": { + "sum": 16 + }, + "start_date": "2020-10-01T13:23:11.221Z" + }, + "application_id": "42cb59a9-d5be-400b-a5c4-69b0a00434fdf4", + "dimensions": { + "request_name": "GET /auth", + "request_url_host": "demoapplogobs.azurewebsites.net" + } + }, + "cloud": { + "provider": "azure" + }, + "event": { + "dataset": "azure.app_state", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "app_state", + "period": 10000 + }, + "service": { + "type": "azure" + } +} diff --git a/x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc new file mode 100644 index 000000000000..4517ccfed0de --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc @@ -0,0 +1,12 @@ +This is the app_state metricset of the module azure. + +This metricset allows users to retrieve application insights metrics from specified applications. + +[float] +==== Config options to identify resources + +`application_id`:: (_[]string_) ID of the application. This is Application ID from the API Access settings blade in the Azure portal. + +`api_key`:: (_[]string_) The API key which will be generated, more on the steps here https://dev.applicationinsights.io/documentation/Authorization/API-key-and-App-ID. + + diff --git a/x-pack/metricbeat/module/azure/app_state/_meta/fields.yml b/x-pack/metricbeat/module/azure/app_state/_meta/fields.yml new file mode 100644 index 000000000000..fad928c090f6 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/_meta/fields.yml @@ -0,0 +1,86 @@ +- name: app_state + type: group + release: beta + description: > + application state + fields: + - name: start_date + type: date + description: > + The start date + - name: end_date + type: date + description: > + The end date + - name: requests_count.sum + type: float + description: > + Request count + - name: requests_failed.sum + type: float + description: > + Request failed count + - name: users_count.unique + type: float + description: > + User count + - name: sessions_count.unique + type: float + description: > + Session count + - name: users_authenticated.unique + type: float + description: > + Authenticated users count + - name: browser_timings_network_duration.avg + type: float + description: > + Browser timings network duration + - name: browser_timings_send_duration.avg + type: float + description: > + Browser timings send duration + - name: browser_timings_receive_uration.avg + type: float + description: > + Browser timings receive duration + - name: browser_timings_processing_duration.avg + type: float + description: > + Browser timings processing duration + - name: browser_timings_total_duration.avg + type: float + description: > + Browser timings total duration + - name: exceptions_count.sum + type: float + description: > + Exception count + - name: exceptions_browser.sum + type: float + description: > + Exception count at browser level + - name: exceptions_server.sum + type: float + description: > + Exception count at server level + - name: performance_counters_memory_available_bytes.avg + type: float + description: > + Performance counters memory available bytes + - name: performance_counters_process_private_bytes.avg + type: float + description: > + Performance counters process private bytes + - name: performance_counters_process_cpu_percentage_total.avg + type: float + description: > + Performance counters process cpu percentage total + - name: performance_counters_process_cpu_percentage.avg + type: float + description: > + Performance counters process cpu percentage + - name: performance_counters_processiobytes_per_second.avg + type: float + description: > + Performance counters process IO bytes per second diff --git a/x-pack/metricbeat/module/azure/app_state/app_state_integration_test.go b/x-pack/metricbeat/module/azure/app_state/app_state_integration_test.go new file mode 100644 index 000000000000..8762cae440d4 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/app_state_integration_test.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build azure + +package app_state + +import ( + "testing" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/app_insights" +) + +func TestFetchMetricset(t *testing.T) { + config := test.GetConfigForInsights(t, "app_state") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + +func TestData(t *testing.T) { + config := test.GetConfigForInsights(t, "app_state") + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/app_state/app_state_test.go b/x-pack/metricbeat/module/azure/app_state/app_state_test.go new file mode 100644 index 000000000000..a8dbab880812 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/app_state_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_state + +import ( + "os" + + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/app_state/manifest.yml b/x-pack/metricbeat/module/azure/app_state/manifest.yml new file mode 100644 index 000000000000..5c1d08a2ceb3 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/manifest.yml @@ -0,0 +1,29 @@ +default: false +input: + module: azure + metricset: app_insights + defaults: + namespace: app_state + metrics: + - id: ["requests/count", "requests/failed"] + segment: ["request/urlHost", "request/name"] + aggregation: ["sum"] + interval: "P5M" + - id: ["users/count", "sessions/count", "users/authenticated"] + segment: ["request/urlHost"] + aggregation: ["unique"] + interval: "P5M" + - id: ["browserTimings/networkDuration", "browserTimings/sendDuration", "browserTimings/receiveDuration", "browserTimings/processingDuration", "browserTimings/totalDuration"] + segment: ["browserTiming/urlHost", "browserTiming/urlPath"] + aggregation: ["avg"] + interval: "P5M" + top: 5 + - id: ["exceptions/count", "exceptions/browser", "exceptions/server"] + segment: ["exception/type", "cloud/roleName"] + aggregation: ["sum"] + interval: "P5M" + - id: ["performanceCounters/memoryAvailableBytes", "performanceCounters/processCpuPercentageTotal", "performanceCounters/processCpuPercentage", "performanceCounters/processIOBytesPerSecond", + "performanceCounters/processPrivateBytes"] + aggregation: ["avg"] + segment: ["cloud/roleName", "cloud/roleInstance"] + interval: "P5M" diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 1da5cfb63abf..c38407c76f14 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -2,62 +2,86 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm": { + "disk_read_bytes": { + "total": 0 + }, "disk_read_operations_per_sec": { - "avg": 3.3875 + "avg": 0 + }, + "disk_write_bytes": { + "total": 2969709.4 }, "disk_write_operations_per_sec": { - "avg": 0.6705 + "avg": 0.7809999999999999 }, "inbound_flows": { - "avg": 28.4 + "avg": 10 }, "inbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 + }, + "network_in": { + "total": 1478232 + }, + "network_in_total": { + "total": 1569665 + }, + "network_out": { + "total": 793344 + }, + "network_out_total": { + "total": 1074624 + }, + "os_disk_bandwidth_consumed_percentage": { + "avg": 0 + }, + "os_disk_iops_consumed_percentage": { + "avg": 0 }, "os_disk_queue_depth": { - "avg": 0.00125 + "avg": 0.002 }, "os_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "os_per_disk_qd": { - "avg": 0.00125 + "avg": 0.002 }, "os_per_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_per_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_per_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "outbound_flows": { - "avg": 28.4 + "avg": 10 }, "outbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 }, "per_disk_qd": { - "avg": 0.0025 + "avg": 0 }, "per_disk_read_bytes_per_sec": { - "avg": 51985.035 + "avg": 0 }, "per_disk_read_operations_per_sec": { - "avg": 2.92875 + "avg": 0 }, "per_disk_write_bytes_per_sec": { "avg": 0 @@ -66,26 +90,41 @@ "avg": 0 }, "percentage_cpu": { - "avg": 9.747 + "avg": 1.8719999999999999 + }, + "vm_cached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_cached_iops_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_iops_consumed_percentage": { + "avg": 0 } }, "namespace": "Microsoft.Compute/virtualMachines", "resource": { "group": "obs-infrastructure", - "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obslinux", - "name": "obslinux", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/testaz", + "name": "testaz", "type": "Microsoft.Compute/virtualMachines" }, - "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz" }, "machine": { - "type": "Basic_A0" + "type": "Standard_A1_v2" }, "provider": "azure", "region": "westeurope" @@ -97,10 +136,28 @@ }, "host": { "cpu": { - "pct": 0.09747 + "pct": 0.01872 }, - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 2969709.4 + } + }, + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz", + "network": { + "in": { + "bytes": 1569665, + "packets": 1478232 + }, + "out": { + "bytes": 1074624, + "packets": 793344 + } + } }, "metricset": { "name": "compute_vm", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index e8f59859d8b3..2d99d8ff6b0e 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -2,6 +2,12 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm_scaleset": { + "cpu_credits_consumed": { + "avg": 0.006999999999999999 + }, + "cpu_credits_remaining": { + "avg": 84.72 + }, "os_per_disk_qd": { "avg": 0 }, @@ -12,35 +18,34 @@ "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 1872.1200000000001 + "avg": 3846.531 }, "os_per_disk_write_operations_per_sec": { - "avg": 0.296 + "avg": 0.5519999999999999 } }, "namespace": "Microsoft.Compute/virtualMachineScaleSets", "resource": { - "group": "testgroup", - "id": "/subscriptions/70bd6e23-e3er3-4835-6785-db77b8eef364/resourceGroups/testgroup/providers/Microsoft.Compute/virtualMachineScaleSets/vmscaleset", - "name": "vmscaleset", - "tags": { - "environment": "staging", - "role": "allocator" - }, + "group": "obs-infrastructure", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", + "name": "obslinuxvmss", "type": "Microsoft.Compute/virtualMachineScaleSets" }, - "subscription_id": "70bd6e23-e3er3-4835-6785-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "machine": { - "type": "Standard_D4s_v3" + "type": "Standard_B1ls" }, "provider": "azure", - "region": "eastus2" + "region": "westeurope" }, "event": { "dataset": "azure.compute_vm_scaleset", @@ -48,7 +53,7 @@ "module": "azure" }, "host": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "metricset": { "name": "compute_vm_scaleset", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml b/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml index 9369a36b79e8..37b1293a2e50 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml @@ -12,8 +12,10 @@ input: - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" timegrain: "PT5M" + ignore_unsupported: true - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "SlotId" @@ -24,6 +26,7 @@ input: "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage", "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "VMName" @@ -40,9 +43,11 @@ input: metrics: - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "SlotId" @@ -53,6 +58,7 @@ input: "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage", "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "VMName" diff --git a/x-pack/metricbeat/module/azure/config.go b/x-pack/metricbeat/module/azure/config.go index 63bb5450b571..00f1af561267 100644 --- a/x-pack/metricbeat/module/azure/config.go +++ b/x-pack/metricbeat/module/azure/config.go @@ -41,7 +41,7 @@ type MetricConfig struct { Dimensions []DimensionConfig `config:"dimensions"` Timegrain string `config:"timegrain"` // namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if namespace is unsupported - // info messages will be logged instead + // info messages will be logged instead. Same situation with metrics, some are being removed from the API, we would like to make sure that does not affect the module IgnoreUnsupported bool `config:"ignore_unsupported"` } diff --git a/x-pack/metricbeat/module/azure/container_instance/manifest.yml b/x-pack/metricbeat/module/azure/container_instance/manifest.yml index a0e6cd5ec27d..c767e072a427 100644 --- a/x-pack/metricbeat/module/azure/container_instance/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_instance/manifest.yml @@ -10,21 +10,25 @@ input: metrics: - name: ["CpuUsage", "MemoryUsage"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "containerName" value: "*" - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" - resource_id: "" metrics: - name: ["CpuUsage", "MemoryUsage"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "containerName" value: "*" - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" diff --git a/x-pack/metricbeat/module/azure/container_service/manifest.yml b/x-pack/metricbeat/module/azure/container_service/manifest.yml index 1384a2688681..b7cf7639d428 100644 --- a/x-pack/metricbeat/module/azure/container_service/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_service/manifest.yml @@ -10,6 +10,7 @@ input: metrics: - name: ["kube_node_status_condition"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "node" @@ -20,9 +21,11 @@ input: value: "*" - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" - name: ["kube_pod_status_ready", "kube_pod_status_phase"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "pod" @@ -31,6 +34,7 @@ input: metrics: - name: ["kube_node_status_condition"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "node" @@ -41,9 +45,11 @@ input: value: "*" - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" - name: ["kube_pod_status_ready", "kube_pod_status_phase"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "pod" diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index bf77f6574162..eb7f1433142e 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -82,7 +82,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupDimValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } } @@ -90,7 +90,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupTimeValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } if client.Config.DefaultResourceType == "" { @@ -120,7 +120,7 @@ func managePropertyName(metric string) string { // create an object in case of ":" resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) // replace uppercases with underscores - resultMetricName = replaceUpperCase(resultMetricName) + resultMetricName = ReplaceUpperCase(resultMetricName) // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" obj := strings.Split(resultMetricName, ".") @@ -134,8 +134,8 @@ func managePropertyName(metric string) string { return resultMetricName } -// replaceUpperCase func will replace upper case with '_' -func replaceUpperCase(src string) string { +// ReplaceUpperCase func will replace upper case with '_' +func ReplaceUpperCase(src string) string { replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { var newStr string diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go index cdfad1965f8d..cca7edc02338 100644 --- a/x-pack/metricbeat/module/azure/data_test.go +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -47,10 +47,10 @@ func TestGetDimensionValue(t *testing.T) { } func TestReplaceUpperCase(t *testing.T) { - result := replaceUpperCase("TestReplaceUpper_Case") + result := ReplaceUpperCase("TestReplaceUpper_Case") assert.Equal(t, result, "Test_replace_upper_Case") // should not split on acronyms - result = replaceUpperCase("CPU_Percentage") + result = ReplaceUpperCase("CPU_Percentage") assert.Equal(t, result, "CPU_Percentage") } diff --git a/x-pack/metricbeat/module/azure/database_account/manifest.yml b/x-pack/metricbeat/module/azure/database_account/manifest.yml index 39086f6ff662..3436008db7ee 100644 --- a/x-pack/metricbeat/module/azure/database_account/manifest.yml +++ b/x-pack/metricbeat/module/azure/database_account/manifest.yml @@ -14,12 +14,14 @@ input: - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" value: "*" - name: ["TotalRequestUnits", "TotalRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" @@ -28,6 +30,7 @@ input: value: "*" - name: ["CassandraRequestCharges", "CassandraRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT1M" dimensions: - name: "DatabaseName" @@ -38,6 +41,7 @@ input: "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete", "TableTableThroughputUpdate","TableTableUpdate"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true dimensions: - name: "ResourceName" value: "*" @@ -49,12 +53,14 @@ input: - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" value: "*" - name: ["TotalRequestUnits", "TotalRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" @@ -63,6 +69,7 @@ input: value: "*" - name: ["CassandraRequestCharges", "CassandraRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT1M" dimensions: - name: "DatabaseName" @@ -73,6 +80,7 @@ input: "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete", "TableTableThroughputUpdate","TableTableUpdate"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true dimensions: - name: "ResourceName" value: "*" diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index 4c0ad95ad98f..6a308a5d2ae6 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUV8tu2zoQ3fsrBl4GSD7AiwsEt5suuuteGJNjhY1EEuQorfv1hR6kKVHyo1aKxIsAEcnzEM8MqUd4peMO8HfjaAPAiivawfa5/X+7AZDkhVOWldE7+G8DAP1cqI1sqnaJo4rQ0w5K3AAcFFXS77qJj6CxphN4++Ojbac609jhyQzDGCaFYlVT6VDpOBIgX+n40ziZPJ8F7n/fXwieexvETokZ3MDoyJvGCcoIUw9X0AUc8JaEOihKpU7tjiwfLY0Glh1fkBGktMvBHIATWbPUU4srcMfXkGNHw1j6p4dZWrP/QYInQ/3D4pywZEpRo7VKl8P87cP2NhN9bKKNTmwWmvavtziTmptjGqHAU0WCk9wENt/sI0Sh5P2cKSB8/ZIRSlWT9sro8T4t7NGF/bl2b85oHpVyIi4T3k/xTw836z5UBhcG/1b1t14MOOLGaZK5XLS2UNqr8oX9xf4T+/CeGK9TgNZWSmC3zTM857pSsnQcObi/R6SykvSl9J7RcSGR51vjzMAVvB1ovjZwkpZrM5KWy3zzYYUbmuE0tLBqK2wNDBqz5O5VVSldvkdoB2hALaHxWBJIYlTVtckVjXOkxXHdzM6iBkrriPFXIYyf7sbyPl3g/H+KFXszWXRck+aifbCuyxM4ZOAns0Y2WS7vJB5A88tQYO2SUHQVvGJ99vm60Bd6atLzXfAe4rPtYaiDwpJTRq7ehEOZ9fAwgY/HgBCmeZeoxTLvGZbzhoIbrFYtru4Q6mAhgw20B+NIoOfViQPwMnUfuJWPoqGVpmtjxzS1bZiKt3pyFo0//cYq/vl16qTyjIHCC6zIE38OJ1FuHJ/xphmVJtfeFBm1oI9tbVALQe0ZQ45K5dkdP4ehoPaMIU/uTX2WDRrE5h9/yLhHT8XQmT+ym6A1nCL5B6HRio27/p5aXnlLzYGnd9LlkvZsHJYfOiaDxPhe/wQAAP//ZxBMvQ==" + return "eJzkmU1v47YTxu/5FIMcA2w+gA9/IP+2hz0ULfpyJsbUWGEjkVxy6Kz76QuKkixLsizbspugOSywenme30MOJyLzBd5otwL8Ozh6AGDFBa3g8SX+//EBICMvnbKsjF7B/x4AID0LpclCEV9xVBB6WkGODwAbRUXmV9WDX0BjSXvx+MM7Gx91Jtj6yojDoUxXilVJuUOl2zuN5Bvt3o3LOtdHhdPPH68ELykGsVNyRLdxdORNcJIGht0MM+waHfCWpNoo6qL24x5E3lk6uHE88QmMBiW+DmYD3MEate5HXMC7HYahdhsYc//8NGpr1n+R5N6tdFFMgXUeESVaq3ReP//49HheiFQ2bYwKdlA08V9vcaRqzi7TVgo8FSS5UzeNmw/rVkKo7HrPriB8/XFgiNYWSuJifh29MbtMlaS9MvqwLI6UxIlymFsKE8gHnaMDNwBPj/jnp7O5N4XBIzcvpf45wYAjDk5TNsRFa4XSXuWv7E+2u7btr4lxHkF3mkd8ppqgZ3QsMuTxVjhyY0YvqkSH7zaepLOlHUlnx/3GqwXOaH79qoFFW18MUDOOlo7nw2Q3qZu+yX+9aBx9C+TZC2mC5mcfylHnsco4Yf1bUoZKeNp7g6qg7BbmSXmCIXhyTfig1bcwPvIXIPzpyU0Ye/JV17+J9+9J/GRuDPxKmuPioGxhhpeudvKb4Fk78+7JCVal0rkXmvjduDeRBVct3Gfc5kuR/T95Qe0FtRc0XrMAfbVS70Dnq/V7DpojSWpL4g5wtdV5fNYZGQtU53cZwL3deZhsGIu7EFZO03D0XVKld4NO/VOjPbE+O/71ON2QAJCb2YCCtlScQvLktjcnSiYTQJbcxrgStaQ0SbHFllQatxO4RVXguiCx3jH5Javp170tNLaQbKG1hcp2Pna9aIR1aot8L+raFWrXS6GlDcKSk6QZc0oL+W7s0gbYm6e1fW2Efwn+bGxlqhmL5MKTNDq7G/nXX1K1RD5I3oNdxloVhdL5LfYYtTSgjt86ceIzYlTF3A2qDM6RlrvRsbr4tGxUtZ1IR4zfhTS+Pw8Xz9APfa32CIYsOi5Js4gXlk25F4eB+D6sycJg93ulcS06PGLdf2XH/lNt+Rbc0KX6OrGRTNak+7GuN57cT9brIHYAZbLD0z24fsSbZZbkoSffnibIqj3coNTaZZ4cjtcbSg5YLLq4qpPOShYGso3txjiS6Hlx40b4uHUquIXPLupW2n237ZimtIFJbMveidfhH5QOKe5+arqnnAggvMSCPPHnSNLitvdHsmlGpckJpT3H39gfO1pNCw3tRCBHufLsdp8jUEM7ESjua9RnmaAadhAnQ8Y1ehJ1Z/7IaRrW5rfIIExptGLj5n+n5jO/UofC/W/S40vas3FxK/KBB7ZGbMf1nwAAAP//oQy/EA==" } diff --git a/x-pack/metricbeat/module/azure/module.yml b/x-pack/metricbeat/module/azure/module.yml index a51b202612b1..4170154e246d 100644 --- a/x-pack/metricbeat/module/azure/module.yml +++ b/x-pack/metricbeat/module/azure/module.yml @@ -4,5 +4,6 @@ metricsets: - container_instance - container_service - database_account + - app_state - compute_vm - compute_vm_scaleset diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc index 19254f4ada92..9957b6a2ddd8 100644 --- a/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc @@ -50,6 +50,9 @@ Metrics with dimensions are exported as flattened single dimensional metrics, ag `name`:: Dimension key `value`:: Dimension value. (Users can select * to return metric values for each dimension) +`ignore_unsupported`:: (_bool_) Namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if the namespace is unsupported. +The same will go for the metrics configured, some can be removed from Azure Monitor and it should not affect the state of the module. + Users can select the options to retrieve all metrics from a specific namespace using the following: ["source","yaml"] diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper.go b/x-pack/metricbeat/module/azure/monitor/client_helper.go index 82875f46de51..403b971496f7 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper.go @@ -78,20 +78,20 @@ func filterMetricNames(resourceId string, metricConfig azure.MetricConfig, metri } } else { // verify if configured metric names are valid, return log error event for the invalid ones, map only the valid metric names - supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metricConfig.Name, metricDefinitions) - if len(unsupportedMetricNames) > 0 { + supportedMetricNames, unsupportedMetricNames = filterConfiguredMetrics(metricConfig.Name, metricDefinitions) + if len(unsupportedMetricNames) > 0 && !metricConfig.IgnoreUnsupported { return nil, errors.Errorf("the metric names configured %s are not supported for the resource %s and namespace %s", strings.Join(unsupportedMetricNames, ","), resourceId, metricConfig.Namespace) } } - if len(supportedMetricNames) == 0 { + if len(supportedMetricNames) == 0 && !metricConfig.IgnoreUnsupported { return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceId, metricConfig.Namespace) } return supportedMetricNames, nil } -// filterSConfiguredMetrics will filter out any unsupported metrics based on the namespace selected -func filterSConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) { +// filterConfiguredMetrics will filter out any unsupported metrics based on the namespace selected +func filterConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) { var inRange []string var notInRange []string var allMetrics string diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go index a15ee0089b9d..bcbad0f4c265 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go @@ -108,7 +108,7 @@ func TestMapMetric(t *testing.T) { func TestFilterSConfiguredMetrics(t *testing.T) { selectedRange := []string{"TotalRequests", "Capacity", "CPUUsage"} - intersection, difference := filterSConfiguredMetrics(selectedRange, *MockMetricDefinitions()) + intersection, difference := filterConfiguredMetrics(selectedRange, *MockMetricDefinitions()) assert.Equal(t, intersection, []string{"TotalRequests", "Capacity"}) assert.Equal(t, difference, []string{"CPUUsage"}) } diff --git a/x-pack/metricbeat/module/azure/test/integration.go b/x-pack/metricbeat/module/azure/test/integration.go index 7187e3b93f70..39e8402110c1 100644 --- a/x-pack/metricbeat/module/azure/test/integration.go +++ b/x-pack/metricbeat/module/azure/test/integration.go @@ -40,3 +40,23 @@ func GetConfig(t *testing.T, metricSetName string) map[string]interface{} { "subscription_id": subId, } } + +// GetConfigForInsights function gets azure credentials for integration tests. +func GetConfigForInsights(t *testing.T, metricSetName string) map[string]interface{} { + t.Helper() + applicationId, ok := os.LookupEnv("AZURE_APPLICATION_ID") + if !ok { + t.Fatal("Could not find var AZURE_APPLICATION_ID") + } + apiKey, ok := os.LookupEnv("AZURE_API_KEY") + if !ok { + t.Fatal("Could not find var AZURE_API_KEY") + } + return map[string]interface{}{ + "module": "azure", + "period": "300s", + "metricsets": []string{metricSetName}, + "application_id": applicationId, + "api_key": apiKey, + } +} diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index d00532978859..cc3103643c7e 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -47,4 +47,8 @@ period: 24h metricsets: - s3_daily_storage +- module: aws + period: 1m + latency: 5m + metricsets: - s3_request diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index 23211f472061..10d00e003cfc 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -114,3 +114,11 @@ # api_key: '' # metrics: # - id: ["requests/count", "requests/duration"] + +#- module: azure +# metricsets: +# - app_state +# enabled: true +# period: 300s +# application_id: '' +# api_key: '' diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index ab7456b98b22..fd30546f70a8 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -1,15 +1,15 @@ when: branches: true ## for all the branches changeset: ## when PR contains any of those entries in the changeset - - "^x-pack/winlogbeat/.*" + - "^x-pack/packetbeat/.*" - "@ci" ## special token regarding the changeset for the ci - "@xpack" ## special token regarding the changeset for the xpack comments: ## when PR comment contains any of those entries - - "/test x-pack/winlogbeat" + - "/test x-pack/packetbeat" labels: ## when PR labels matches any of those entries - - "x-pack-winlogbeat" + - "x-pack-packetbeat" parameters: ## when parameter was selected in the UI. - - "x-pack-winlogbeat" + - "x-pack-packetbeat" tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: