Add AWS Cloudwatch Logs operator

cmd/stanza/go.mod

@@ -7,12 +7,12 @@ require (
 	github.com/observiq/stanza v0.14.0
 	github.com/observiq/stanza/operator/builtin/input/k8sevent v0.1.0
 	github.com/observiq/stanza/operator/builtin/input/windows v0.1.1
-	github.com/observiq/stanza/operator/builtin/output/elastic v0.1.0
+	github.com/observiq/stanza/operator/builtin/output/elastic v0.1.2
 	github.com/observiq/stanza/operator/builtin/output/googlecloud v0.1.3
-	github.com/observiq/stanza/operator/builtin/output/newrelic v0.1.0
-	github.com/observiq/stanza/operator/builtin/output/otlp v0.0.0
+	github.com/observiq/stanza/operator/builtin/output/newrelic v0.1.2
+	github.com/observiq/stanza/operator/builtin/output/otlp v0.1.0
 	github.com/observiq/stanza/operator/builtin/parser/syslog v0.1.5
-	github.com/observiq/stanza/operator/builtin/transformer/k8smetadata v0.1.0
+	github.com/observiq/stanza/operator/builtin/transformer/k8smetadata v0.1.2
 	github.com/spf13/cobra v1.1.3
 	github.com/stretchr/testify v1.7.0
 	go.etcd.io/bbolt v1.3.5

cmd/stanza/go.sum

@@ -159,6 +159,8 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:o
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.34.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.38.31 h1:iGTU2dctGX5SsFverLkQQzqLhcb56NnixoaOOZrXjJQ=
+github.com/aws/aws-sdk-go v1.38.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -641,6 +643,10 @@ github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYP
 github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
@@ -1247,8 +1253,9 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1336,6 +1343,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200821140526-fda516888d29/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

cmd/stanza/init_common.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	// Load packages when importing input operators
+	_ "github.com/observiq/stanza/operator/builtin/input/aws/cloudwatch"
 	_ "github.com/observiq/stanza/operator/builtin/input/azure/eventhub"
 	_ "github.com/observiq/stanza/operator/builtin/input/azure/loganalytics"
 	_ "github.com/observiq/stanza/operator/builtin/input/file"

docs/operators/aws_cloudwatch_input.md

@@ -0,0 +1,140 @@
+## `aws_cloudwatch_input` operator
+
+The `aws_cloudwatch_input` operator reads logs from AWS Cloudwatch Logs using [AWS's SDK](https://github.com/aws/aws-sdk-go).
+
+The `aws_cloudwatch_input` operator will use the `Timestamp` field of the event as the parsed entry's timestamp. All other fields are added to the entry's record.
+
+The `aws_cloudwatch_input` operator will use the following order to get credentials. 
+- Environment Variables (Details [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html))
+- Shared Credentials file (Details [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html)), 
+- Shared Configuration file (if SharedConfig is enabled details [here](https://docs.aws.amazon.com/sdkref/latest/guide/creds-config-files.html)) , 
+- EC2 Instance Metadata (credentials only details [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-metadata.html)). 
+
+You can provide `profile` to specify which credential set to use from a Shared Credentials file.
+
+### Configuration Fields
+
+| Field                     | Default                | Description                                                                                                 |
+| ---                       | ---                    | ---                                                                                                         |
+| `id`                      | `aws_cloudwatch_input` | A unique identifier for the operator.                                                                       |
+| `output`                  | Next in pipeline       | The connected operator(s) that will receive all outbound entries.                                           |
+| `log_group_name`          | required               | The Cloudwatch Logs Log Group Name.                                                                         |
+| `region`                  | required               | The AWS Region to be used.                                                                                  |
+| `log_stream_name_prefix`  |                        | The log stream name prefix to use. This will find any log stream name in the group with the starting prefix |
+| `log_stream_names`        |                        | An array of log stream names to get events from.                                                            |
+| `profile`                 |                        | Profile to use for authentication. Details on named profiles can be found [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) |
+| `event_limit`             | `10000`                | The maximum number of events to return per call.                                                            |
+| `poll_interval`           | `1m`                   | The duration between event calls.                                                                           |
+| `start_at`                | `end`                  | At startup, where to start reading events. Options are `beginning` or `end`                                 |
+
+### Log Stream Prefix
+
+The log_stream_prefix allows the use of "directives" such as `%Y` (4-digit year) and `%d` (2-digit zero-padded day). These directives are based on `strptime` directives. There are a limited set of the `strptime` directives. These directives are listed below. When directive is detected within the prefix it will replace the first occurance of directive with the data indicated in the description.
+
+#### Supported directives
+
+| Directive | Description                        |
+| :---:     | :---                               |
+| %Y        | Year, zero-padded                  |
+| %y        | Year, last two digits, zero-padded |
+| %m        | Month, zero-padded                 |
+| %q        | Month as a unpadded number         |
+| %b        | Abbreviated month name             |
+| %h        | Abbreviated month name             |
+| %B        | Full month name                    |
+| %d        | Day of the month, zero-padded      |
+| %g        | Day of the month, unpadded         |
+| %a        | Abbreviated weekday name           |
+| %A        | Full weekday name                  |
+
+### Example Configurations
+
+#### Simple AWS Cloudwatch Logs Example Input
+
+Configuration:
+
+```yaml
+pipeline:
+- type: aws_cloudwatch_input
+  LogGroupName: "/aws/lambda/service"
+  Region: us-east-2
+```
+
+### Simple AWS Cloudwatch Logs Example Output
+
+```json
+{
+  "timestamp": "2021-05-10T13:00:55.023-04:00",
+  "severity": 0,
+  "record": {
+    "event_id": "36142060744975733945009868546041203920891749688822923267",
+    "ingestion_time": 1620666055330,
+    "log_stream_name": "2021/05/10/[$LATEST]ff09d08f2836494690a1bd6b77365502",
+    "message": "REPORT RequestId: 291fe36c-116a-42fd-a563-a8615671bab9\tDuration: 4577.28 ms\tBilled Duration: 4578 ms\tMemory Size: 128 MB\tMax Memory Used: 68 MB\tInit Duration: 401.54 ms\t\n"
+  }
+}
+```
+
+#### Log Stream Prefix Directives Example Input
+
+Configuration:
+
+```yaml
+pipeline:
+- type: aws_cloudwatch_input
+  LogGroupName: "/aws/lambda/service"
+  Region: us-east-2
+  log_stream_name_prefix: "%Y/%m/%d"
+```
+
+### Log Stream Prefix Directives Example Output
+
+```json
+{
+  "timestamp": "2021-05-12T13:03:47.941-04:00",
+  "severity": 0,
+  "resource": {
+    "log_group_name": "/aws/lambda/service",
+    "log_stream_name": "2021/05/12/[$LATEST]0f36de8f623a491c9305990130201669",
+    "region": "us-east-2"
+  },
+  "record": {
+    "event_id": "36145918169946298276207227425947415243911741965970309123",
+    "ingestion_time": 1620839035104,
+    "message": "REPORT RequestId: d64685ba-913b-456f-acd7-d00021416e68\tDuration: 1852.30 ms\tBilled Duration: 1853 ms\tMemory Size: 128 MB\tMax Memory Used: 68 MB\t\n"
+  }
+}
+```
+
+#### Log Stream Names Example Input
+
+Configuration:
+
+```yaml
+pipeline:
+- type: aws_cloudwatch_input
+  LogGroupName: "/aws/lambda/service"
+  Region: us-east-2
+  log_stream_names:
+    - "2021/05/09/[$LATEST]62e990bb0e72460c95b1dcfc5d96adc5"
+    - "2021/05/08/[$LATEST]84d663604b6845e987d278272455ed95"
+```
+
+### Log Stream Names Example Output
+
+```json
+{
+  "timestamp": "2021-05-09T13:04:02.686-04:00",
+  "severity": 0,
+  "resource": {
+    "log_group_name": "/aws/lambda/service",
+    "log_stream_name": "2021/05/09/[$LATEST]62e990bb0e72460c95b1dcfc5d96adc5",
+    "region": "us-east-2"
+  },
+  "record": {
+    "event_id": "36140138145615327091022666253954182481286730645124743171",
+    "ingestion_time": 1620579849837,
+    "message": "REPORT RequestId: 346b9fa2-9117-4d41-89f8-071f0100213b\tDuration: 1865.27 ms\tBilled Duration: 1866 ms\tMemory Size: 128 MB\tMax Memory Used: 68 MB\t\n"
+  }
+}
+```

go.mod

@@ -5,6 +5,7 @@ go 1.14
 require (
 	github.com/Azure/azure-event-hubs-go/v3 v3.3.7
 	github.com/antonmedv/expr v1.8.2
+	github.com/aws/aws-sdk-go v1.38.31
 	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/cenkalti/backoff/v4 v4.0.2
 	github.com/jpillora/backoff v1.0.0

go.sum

@@ -53,6 +53,8 @@ github.com/Mottl/ctimefmt v0.0.0-20190803144728-fd2ac23a585a/go.mod h1:eyj2WSIdo
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/antonmedv/expr v1.8.2 h1:BfkVHGudYqq7jp3Ji33kTn+qZ9D19t/Mndg0ag/Ycq4=
 github.com/antonmedv/expr v1.8.2/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8=
+github.com/aws/aws-sdk-go v1.38.31 h1:iGTU2dctGX5SsFverLkQQzqLhcb56NnixoaOOZrXjJQ=
+github.com/aws/aws-sdk-go v1.38.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
 github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
 github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs=
@@ -79,6 +81,10 @@ github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
@@ -178,8 +184,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -192,6 +199,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPMJTiBfWycGh/u3UoO88=
 golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -229,6 +237,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=