diff --git a/examples/recombine/README.md b/examples/recombine/README.md new file mode 100644 index 00000000000..07212802898 --- /dev/null +++ b/examples/recombine/README.md @@ -0,0 +1,16 @@ +# Recombining logs + +This example showcases how the collector can read logs and recombine them. + +The example runs as a Docker Compose deployment. The collector is configured to send logs to Splunk Enterprise. + +Splunk is configured to receive data from the OpenTelemetry Collector using the HTTP Event collector. To learn more about HEC, visit [our guide](https://dev.splunk.com/enterprise/docs/dataapps/httpeventcollector/). + +To deploy the example, check out this git repository, open a terminal and in this directory type: +```bash +$> docker-compose up +``` + +Splunk will become available on port 18000. You can login on [http://localhost:18000](http://localhost:18000) with `admin` and `changeme`. + +Once logged in, visit the [search application](http://localhost:18000/en-US/app/search) to see just one log collected by Splunk, recombined from the 2 logs present in input.json. diff --git a/examples/recombine/docker-compose.yml b/examples/recombine/docker-compose.yml new file mode 100644 index 00000000000..0854dec1b48 --- /dev/null +++ b/examples/recombine/docker-compose.yml @@ -0,0 +1,34 @@ +version: "3" +services: + # Splunk Enterprise server: + splunk: + image: splunk/splunk:latest + container_name: splunk + environment: + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_HEC_TOKEN=00000000-0000-0000-0000-0000000000000 + - SPLUNK_PASSWORD=changeme + ports: + - 18000:8000 + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:8000'] + interval: 5s + timeout: 5s + retries: 20 + volumes: + - ./splunk.yml:/tmp/defaults/default.yml + - /opt/splunk/var + - /opt/splunk/etc + # OpenTelemetry Collector + otelcollector: + image: quay.io/signalfx/splunk-otel-collector:0.68.0 + container_name: otelcollector + command: ["--config=/etc/otel-collector-config.yml"] + volumes: + - ./otel-collector-config.yml:/etc/otel-collector-config.yml + - ./input.json:/input.json + depends_on: + splunk: + condition: service_healthy + ports: + - 18088:8088 \ No newline at end of file diff --git a/examples/recombine/input.json b/examples/recombine/input.json new file mode 100644 index 00000000000..659fbdbff83 --- /dev/null +++ b/examples/recombine/input.json @@ -0,0 +1,2 @@ +{"log":"foo","stream":"stdout","time":"2023-01-06T21:39:57.228483733Z"} +{"log":"bar\n","stream":"stdout","time":"2023-01-06T21:39:57.228495587Z"} diff --git a/examples/recombine/otel-collector-config.yml b/examples/recombine/otel-collector-config.yml new file mode 100644 index 00000000000..e87cf4de656 --- /dev/null +++ b/examples/recombine/otel-collector-config.yml @@ -0,0 +1,74 @@ +receivers: + filelog: + include: [ /input.json ] + start_at: beginning + include_file_name: false + include_file_path: true + max_concurrent_files: 1024 + max_log_size: 1MiB + operators: + - id: parser-docker + timestamp: + layout: '%Y-%m-%dT%H:%M:%S.%LZ' + parse_from: attributes.time + type: json_parser + - combine_field: attributes.log + combine_with: "" + id: docker-recombine + is_last_entry: attributes.log endsWith "\n" + source_identifier: attributes["log.file.path"] + type: recombine + - from: attributes.log + id: clean-up-log-record + to: body + type: move + +exporters: + splunk_hec/logs: + # Splunk HTTP Event Collector token. + token: "00000000-0000-0000-0000-0000000000000" + # URL to a Splunk instance to send data to. + endpoint: "https://splunk:8088/services/collector" + # Optional Splunk source: https://docs.splunk.com/Splexicon:Source + source: "output" + # Splunk index, optional name of the Splunk index targeted. + index: "logs" + # Maximum HTTP connections to use simultaneously when sending data. Defaults to 100. + max_connections: 20 + # Whether to disable gzip compression over HTTP. Defaults to false. + disable_compression: false + # HTTP timeout when sending data. Defaults to 10s. + timeout: 10s + tls: + # Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false. + # For this demo, we use a self-signed certificate on the Splunk docker instance, so this flag is set to true. + insecure_skip_verify: true + +processors: + batch: + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + file_storage/checkpoint: + directory: /output/ + timeout: 1s + compaction: + on_start: true + directory: /output/ + max_transaction_size: 65_536 + +service: + telemetry: + logs: + level: debug + extensions: [pprof, zpages, health_check, file_storage/checkpoint] + pipelines: + logs: + receivers: [filelog] + processors: [batch] + exporters: [splunk_hec/logs] diff --git a/examples/recombine/splunk.yml b/examples/recombine/splunk.yml new file mode 100644 index 00000000000..a7c67c12448 --- /dev/null +++ b/examples/recombine/splunk.yml @@ -0,0 +1,11 @@ +splunk: + conf: + indexes: + directory: /opt/splunk/etc/apps/search/local + content: + logs: + coldPath: $SPLUNK_DB/logs/colddb + datatype: event + homePath: $SPLUNK_DB/logs/db + maxTotalDataSizeMB: 512000 + thawedPath: $SPLUNK_DB/logs/thaweddb