Merge branch 'main' into flands/support-bundle

signalfx · Mar 30, 2021 · fec1252 · fec1252
2 parents ff3f051 + 8a52d52
commit fec1252
Show file tree

Hide file tree

Showing 48 changed files with 1,173 additions and 511 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -5,12 +5,24 @@ updates:
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "daily"
   - package-ecosystem: "gomod"
     directory: "/examples/prometheus-federation/prom-counter"
     schedule:
-      interval: "weekly"
+      interval: "daily"
   - package-ecosystem: "gomod"
     directory: "/examples/splunk-hec/logging"
     schedule:
-      interval: "weekly"
+      interval: "daily"
+  - package-ecosystem: "gomod"
+    directory: "/tests"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "pip"
+    directory: "/internal/buildscripts/packaging/release"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "pip"
+    directory: "/internal/buildscripts/packaging/tests"
+    schedule:
+      interval: "daily"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,22 @@
+# Changelog
+
+## Unreleased
+
+## v0.23.0
+
+This Splunk OpenTelemetry Collector release includes changes from the [opentelemetry-collector v0.23.0](https://github.com/open-telemetry/opentelemetry-collector/releases/tag/v0.23.0) and the [opentelemetry-collector-contrib v0.23.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.23.0) releases.
+
+## 🛑 Breaking changes 🛑
+
+- Renamed default config from `splunk_config_linux.yaml` to `gateway_config.yaml` (#170)
+
+## 💡 Enhancements 💡
+
+- Include smart agent bundle in amd64 deb/rpm packages (#177)
+- `smartagent` receiver: Add support for logs (#161) and traces (#192)
+
+## 🧰 Bug fixes 🧰
+
+- `smartagent` extension: Ensure propagation of collectd bundle dir (#180)
+- `smartagent` receiver: Fix logrus logger hook data race condition (#181)
+
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -11,13 +11,6 @@ In addition to this document, please review our [Code of
 Conduct](CODE_OF_CONDUCT.md). For any code of conduct questions or comments
 please email oss@splunk.com.
 
-
-## Reporting Security Issues
-
-Please *DO NOT* report security vulnerabilities with public GitHub issue
-reports. Please [report security issues here](
-https://www.splunk.com/en_us/product-security/report.html).
-
 ## Reporting Bugs/Feature Requests
 
 We welcome you to use the GitHub issue tracker to report bugs or suggest

diff --git a/Makefile b/Makefile
@@ -6,9 +6,6 @@ ADDLICENSE= addlicense
 ALL_DOC := $(shell find . \( -name "*.md" -o -name "*.yaml" \) \
                                 -type f | sort)
 
-# ALL_MODULES includes ./* dirs (excludes . dir)
-ALL_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort | egrep  '^./' )
-
 # All source code excluding any third party code and excluding the testbed.
 # This is the code that we want to run tests for and lint, staticcheck, etc.
 ALL_SRC := $(shell find . -name '*.go' \
@@ -107,6 +104,9 @@ checklicense:
 			echo "Check License finished successfully"; \
 		fi
 
+# ALL_MODULES includes ./* dirs (excludes . dir)
+ALL_GO_MODULES := $(shell find . -type f -name "go.mod" -exec dirname {} \; | sort | egrep  '^./' )
+ALL_PYTHON_DEPS := $(shell find . -type f \( -name "setup.py" -o -name "requirements.txt" \) -exec dirname {} \; | sort | egrep  '^./')
 DEPENDABOT_PATH=./.github/dependabot.yml
 .PHONY: gendependabot
 gendependabot:
@@ -115,10 +115,14 @@ gendependabot:
 	@echo "version: 2" >> ${DEPENDABOT_PATH}
 	@echo "updates:" >> ${DEPENDABOT_PATH}
 	@echo "Add entry for \"/\""
-	@echo "  - package-ecosystem: \"gomod\"\n    directory: \"/\"\n    schedule:\n      interval: \"weekly\"" >> ${DEPENDABOT_PATH}
-	@set -e; for dir in $(ALL_MODULES); do \
+	@echo "  - package-ecosystem: \"gomod\"\n    directory: \"/\"\n    schedule:\n      interval: \"daily\"" >> ${DEPENDABOT_PATH}
+	@set -e; for dir in $(ALL_GO_MODULES); do \
+		(echo "Add entry for \"$${dir:1}\"" && \
+		  echo "  - package-ecosystem: \"gomod\"\n    directory: \"$${dir:1}\"\n    schedule:\n      interval: \"daily\"" >> ${DEPENDABOT_PATH} ); \
+	done
+	@set -e; for dir in $(ALL_PYTHON_DEPS); do \
 		(echo "Add entry for \"$${dir:1}\"" && \
-		  echo "  - package-ecosystem: \"gomod\"\n    directory: \"$${dir:1}\"\n    schedule:\n      interval: \"weekly\"" >> ${DEPENDABOT_PATH} ); \
+		  echo "  - package-ecosystem: \"pip\"\n    directory: \"$${dir:1}\"\n    schedule:\n      interval: \"daily\"" >> ${DEPENDABOT_PATH} ); \
 	done
 
 .PHONY: misspell

diff --git a/README.md b/README.md
@@ -1,13 +1,60 @@
+---
+
+<p align="center">
+  <strong>
+    <a href="#getting-started">Getting Started</a>
+    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
+    <a href="CONTRIBUTING.md">Getting Involved</a>
+    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
+    <a href="docs/signalfx-smart-agent-migration.md">Migrating from Smart Agent</a>
+  </strong>
+</p>
+
+<p align="center">
+  <a href="https://goreportcard.com/report/github.com/signalfx/splunk-otel-collector">
+    <img alt="Go Report Card" src="https://goreportcard.com/badge/github.com/signalfx/splunk-otel-collector?style=for-the-badge">
+  </a>
+  <a href="https://circleci.com/gh/signalfx/splunk-otel-collector">
+    <img alt="Build Status" src="https://img.shields.io/circleci/build/github/signalfx/splunk-otel-collector?style=for-the-badge">
+  </a>
+  <a href="https://codecov.io/gh/signalfx/splunk-otel-collector/branch/main/">
+    <img alt="Codecov Status" src="https://img.shields.io/codecov/c/github/signalfx/splunk-otel-collector?style=for-the-badge">
+  </a>
+  <a href="https://github.com/signalfx/splunk-otel-collector/releases">
+    <img alt="GitHub release (latest by date including pre-releases)" src="https://img.shields.io/github/v/release/signalfx/splunk-otel-collector?include_prereleases&style=for-the-badge">
+  </a>
+  <img alt="Beta" src="https://img.shields.io/badge/status-beta-informational?style=for-the-badge">
+</p>
+
+<p align="center">
+  <strong>
+    <a href="docs/architecture.md">Architecture</a>
+    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
+    <a href="docs/components.md">Components</a>
+    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
+    <a href="docs/monitoring.md">Monitoring</a>
+    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
+    <a href="docs/security.md">Security</a>
+    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
+    <a href="docs/sizing.md">Sizing</a>
+    &nbsp;&nbsp;&bull;&nbsp;&nbsp;
+    <a href="docs/troubleshooting.md">Troubleshooting</a>
+  </strong>
+</p>
+
+---
+
 # Splunk Distribution of OpenTelemetry Collector
 
 The Splunk Distribution of [OpenTelemetry
 Collector](https://github.com/open-telemetry/opentelemetry-collector) provides
 a binary that can receive, process and export trace, metric and log data.
 
-**Installations that use this distribution can receive direct help from Splunk's support teams.**
-Customers are free to use the core OpenTelemetry OSS components (several do!) and we will provide best
-effort guidance to them for any issues that crop up, however only the Splunk distributions are in
-scope for official Splunk support and support-related SLAs.
+**Installations that use this distribution can receive direct help from
+Splunk's support teams.** Customers are free to use the core OpenTelemetry OSS
+components (several do!) and we will provide best effort guidance to them for
+any issues that crop up, however only the Splunk distributions are in scope for
+official Splunk support and support-related SLAs.
 
 The Collector currently supports:
 
@@ -64,112 +111,6 @@ The Collector is supported on and packaged for a variety of platforms including:
 
 You can consult additional use cases in the [examples](./examples) directory.
 
-## Supported Components
-
-The distribution offers support for the following components. Support is based
-on the [OpenTelemetry maturity
-matrix](https://github.com/open-telemetry/community/blob/47813530864b9fe5a5146f466a58bd2bb94edc72/maturity-matrix.yaml#L57).
-
-### Beta
-
-| Receivers        | Processors        | Exporters | Extensions    |
-| :--------------: | :--------:        | :-------: | :--------:    |
-| hostmetrics      | attributes        | file      | healthcheck   |
-| jaeger           | batch             | logging   | httpforwarder |
-| k8s_cluster      | filter            | otlp      | host_observer |
-| kubeletstats     | k8s_tagger        | sapm      | k8s_observer  |
-| opencensus       | memorylimiter     | signalfx  | pprof         |
-| otlp             | metrictransform   | splunkhec | zpages        |
-| receiver_creator | resource          |           |               |
-| sapm             | resourcedetection |           |               |
-| signalfx         | span              |           |               |
-| simpleprometheus |                   |           |               |
-| smartagent       |                   |           |               |
-| splunkhec        |                   |           |               |
-| zipkin           |                   |           |               |
-
-### Alpha
-
-| Receivers      | Processors | Exporters | Extensions |
-| :-------:      | :--------: | :-------: | :--------: |
-| carbon         |            |           |            |
-| collectd       |            |           |            |
-| fluentdforward |            |           |            |
-| statsd         |            |           |            |
-
-## Sizing
-
-The OpenTelemetry Collector can be scaled up or out as needed. Sizing is based
-on the amount of data per data source and requires 1 CPU core per:
-
-- Traces: 10,000 spans per second
-- Metrics: 20,000 data points per second
-
-If a Collector handles both trace and metric data then both must be accounted
-for when sizing. For example, 5K spans per second plus 10K data points per
-second would require 1 CPU core.
-
-The recommendation is to use a ratio of 1 CPU to 2 GB of memory. By default, the
-Collector is configured to use 512 MB of memory.
-
-> The Collector does not persist data to disk so no disk space is required.
-
-### Agent
-
-For Agent instances, scale up resources as needed. Typically only a single
-agent runs per application or host so properly sizing the agent is important.
-Multiple independent agents could be deployed on a given application or host
-depending on the use-case. For example, a privileged agent could be deployed
-alongside an unprivileged agent.
-
-### Gateway
-
-For Gateway instances, allocate at least a CPU core per Collector. Note that
-multiple Collectors can deployed behind a simple round-robin load balancer for
-availability and performance reasons. Each Collector runs independently, so
-scale increases linearly with the number of Collectors you deploy.
-
-The recommendation is to configure at least N+1 redundancy, which means a load
-balancer and a minimum of two Collector instances should be configured
-initially.
-
-## Monitoring
-
-The default configuration automatically scrapes the Collector's own metrics and
-sends the data using the `signalfx` exporter. A built-in dashboard provides
-information about the health and status of Collector instances.
-
-## Security
-
-Start by reviewing the [Collector security
-documentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security.md).
-
-For security information specific to this distribution, please review
-[security.md](docs/security.md).
-
-## Troubleshooting
-
-For troubleshooting information specific to this distribution, please review
-[troubleshooting.md](docs/troubleshooting.md).
-
-## Migrating from the SignalFx Smart Agent
-
-The Splunk Distribution of OpenTelemetry Collector is the next-generation
-agent and gateway for Splunk APM and Splunk Infrastructure Monitoring.
-As such, it is the replacement for the [SignalFx Smart
-Agent](https://github.com/signalfx/signalfx-agent).
-
-This distribution provides helpful components to assist current Smart Agent
-users in their transition to OpenTelemetry Collector and ensure no functionality
-loss.  The [Smart Agent
-Receiver](./internal/receiver/smartagentreceiver/README.md), its associated
-[extension](./internal/extension/smartagentextension/README.md), and other
-Collector components provide a means of integrating all Smart Agent metric
-monitors into your Collector pipelines.
-
-A detailed overview of our suggested migration practices is provided in the
-[migration guide](./docs/signalfx-smart-agent-migration.md).
-
 ## License
 
-[Apache Software License version 2.0](./LICENSE).
+[Apache Software License version 2.0](./LICENSE).
diff --git a/docs/components.md b/docs/components.md
@@ -0,0 +1,41 @@
+# Components
+
+The distribution offers support for the following components.
+
+## Beta
+
+These components are considered stable. While in beta, breaking changes may be
+introduced in a new release. In addition, any of these components may be
+removed prior to the 1.0 release.
+
+| Receivers        | Processors        | Exporters | Extensions    |
+| :--------------: | :--------:        | :-------: | :--------:    |
+| fluentforward    | attributes        | file      | healthcheck   |
+| hostmetrics      | batch             | logging   | httpforwarder |
+| jaeger           | filter            | otlp      | host_observer |
+| k8s_cluster      | k8s_tagger        | sapm      | k8s_observer  |
+| kubeletstats     | memorylimiter     | signalfx  | pprof         |
+| opencensus       | metrictransform   | splunk_hec | zpages        |
+| otlp             | resource          |           |               |
+| receiver_creator | resourcedetection |           |               |
+| sapm             | span              |           |               |
+| signalfx         |                   |           |               |
+| simpleprometheus |                   |           |               |
+| smartagent       |                   |           |               |
+| splunk_hec       |                   |           |               |
+| zipkin           |                   |           |               |
+
+## Alpha
+
+These components may or may not be stable. In addition, the may be limited in
+their capabilities (for example the Kafka receiver/exporter only offers tracing
+support at this time). While in alpha, breaking changes may be introduced in a
+new release. In addition, any of these components may be removed prior to the
+1.0 release.
+
+| Receivers      | Processors | Exporters | Extensions |
+| :-------:      | :--------: | :-------: | :--------: |
+| carbon         |            | kafka     |            |
+| collectd       |            |           |            |
+| kafka          |            |           |            |
+| statsd         |            |           |            |
diff --git a/docs/monitoring.md b/docs/monitoring.md
@@ -0,0 +1,11 @@
+# Monitoring
+
+The default configuration automatically scrapes the Collector's own metrics and
+sends the data using the `signalfx` exporter. A built-in dashboard provides
+information about the health and status of Collector instances.
+
+In addition, logs should be collected. For Log Observer customers, logs are
+automatically collected for the Collector and Journald processes.
+
+The Collector also offers
+[zpages](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/troubleshooting.md#zpages).
diff --git a/docs/release.md b/docs/release.md
@@ -55,7 +55,7 @@
    and back to the Github release.
    ```
    $ source venv/bin/activate  # if not already in virtualenv
-   $ ./internal/buildscripts/packaging/release/sign_release.py
+   $ ./internal/buildscripts/packaging/release/sign_release.py --stage release
    ```
    This may take 10+ minutes to complete.  The "Pre-release" label will then be
    removed from the Github release.  Run the script with `--help` for more

diff --git a/docs/security.md b/docs/security.md
@@ -1,5 +1,33 @@
 # Security
 
+Start by reviewing the [OpenTelemetry Collector security
+documentation](https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/security.md).
+
+## Reporting Security Issues
+
+Please *DO NOT* report security vulnerabilities via public GitHub issue
+reports. Please [report security issues here](
+https://www.splunk.com/en_us/product-security/report.html).
+
+## Dependencies
+
+This project relies on a variety of [external
+dependencies](https://github.com/signalfx/splunk-otel-collector/network/dependencies).
+These dependencies are monitored by
+[Dependabot](https://docs.github.com/en/code-security/supply-chain-security/configuring-dependabot-security-updates).
+Dependencies are [checked
+daily](https://github.com/signalfx/splunk-otel-collector/blob/main/.github/dependabot.yml)
+and associated pull requests are opened automatically. Upgrading to the [latest
+release](https://github.com/signalfx/splunk-otel-collector/releases)
+is recommended to ensure you have the latest security updates. If a security
+vulnerability is detected for a dependency of this project then either:
+
+- You are running an older release
+- A new release with the updates has not been cut yet
+- The updated dependency has not been merged likely due to some breaking change
+  (in this case, we will actively work to resolve the issue and open a tracking GitHub issues with details)
+- The dependency has not released an updated version with the patch
+
 ## Exposed endpoints
 
 By default, the Collector exposes the following endpoints:

diff --git a/docs/signalfx-smart-agent-migration.md b/docs/signalfx-smart-agent-migration.md
@@ -1,4 +1,15 @@
-# Migrating from the SignalFx Smart Agent to the Splunk Distribution of OpenTelemetry Collector
+# Migrating from the SignalFx Smart Agent
+
+The Splunk Distribution of OpenTelemetry Collector is the next-generation agent
+and gateway for Splunk Observability products. As such, it is the replacement
+for the [SignalFx Smart Agent](https://github.com/signalfx/signalfx-agent).
+This distribution provides helpful components to assist current Smart Agent
+users in their transition to OpenTelemetry Collector and ensure no functionality
+loss.  The [Smart Agent
+Receiver](./internal/receiver/smartagentreceiver/README.md), its associated
+[extension](./internal/extension/smartagentextension/README.md), and other
+Collector components provide a means of integrating all Smart Agent metric
+monitors into your Collector pipelines.
 
 The [SignalFx Smart Agent's](https://github.com/signalfx/signalfx-agent/blob/master/README.md)
 metric monitors allow real-time insights into how your target services and