From ebd7f2a7af30c20eaea651882d4284a90443a085 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Tue, 11 May 2021 21:58:15 -0700 Subject: [PATCH] Ansible role for Linux deployment Initial version of the Ansible role for Linux deployment with tests --- .github/workflows/ansible-role-test.yml | 60 +++++++++ .gitignore | 3 + .../roles/splunk-otel-collector/.yamllint | 12 ++ .../splunk-otel-collector/CONTRIBUTING.md | 75 +++++++++++ .../roles/splunk-otel-collector/README.md | 119 ++++++++++++++++++ .../splunk-otel-collector/defaults/main.yml | 35 ++++++ .../splunk-otel-collector/handlers/main.yml | 22 ++++ .../molecule/config/docker.yml | 17 +++ .../molecule/config/vagrant.yml | 18 +++ .../molecule/custom_vars/converge.yml | 19 +++ .../custom_vars/custom_collector_config.yml | 40 ++++++ .../custom_vars/custom_fluentd_config.conf | 22 ++++ .../molecule/custom_vars/molecule.yml | 1 + .../molecule/custom_vars/verify.yml | 95 ++++++++++++++ .../molecule/default/converge.yml | 11 ++ .../molecule/default/molecule.yml | 1 + .../molecule/default/verify.yml | 21 ++++ .../molecule/shared/verify_tasks.yml | 13 ++ .../molecule/without_fluentd/converge.yml | 11 ++ .../molecule/without_fluentd/molecule.yml | 1 + .../molecule/without_fluentd/verify.yml | 15 +++ .../tasks/apt_install_fluentd.yml | 31 +++++ .../tasks/apt_install_otel_collector.yml | 31 +++++ .../tasks/collector_service_owner.yml | 41 ++++++ .../tasks/linux_install.yml | 41 ++++++ .../tasks/linux_install_fluentd.yml | 117 +++++++++++++++++ .../splunk-otel-collector/tasks/main.yml | 17 +++ .../tasks/yum_install_fluentd.yml | 34 +++++ .../tasks/yum_install_otel_collector.yml | 21 ++++ .../templates/collector-service-owner.conf.j2 | 3 + .../templates/splunk-otel-collector.conf.j2 | 12 ++ .../templates/td-agent.conf.j2 | 2 + 32 files changed, 961 insertions(+) create mode 100644 .github/workflows/ansible-role-test.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/.yamllint create mode 100644 deployments/ansible/roles/splunk-otel-collector/CONTRIBUTING.md create mode 100644 deployments/ansible/roles/splunk-otel-collector/README.md create mode 100644 deployments/ansible/roles/splunk-otel-collector/defaults/main.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/handlers/main.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/config/docker.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/config/vagrant.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/converge.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_collector_config.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_fluentd_config.conf create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/molecule.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/verify.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/default/converge.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/default/molecule.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/default/verify.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/shared/verify_tasks.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/converge.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/molecule.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/verify.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_fluentd.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_otel_collector.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/collector_service_owner.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/linux_install.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/linux_install_fluentd.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/main.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_fluentd.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_otel_collector.yml create mode 100644 deployments/ansible/roles/splunk-otel-collector/templates/collector-service-owner.conf.j2 create mode 100644 deployments/ansible/roles/splunk-otel-collector/templates/splunk-otel-collector.conf.j2 create mode 100644 deployments/ansible/roles/splunk-otel-collector/templates/td-agent.conf.j2 diff --git a/.github/workflows/ansible-role-test.yml b/.github/workflows/ansible-role-test.yml new file mode 100644 index 0000000000..94604a1b82 --- /dev/null +++ b/.github/workflows/ansible-role-test.yml @@ -0,0 +1,60 @@ +on: + pull_request: + paths: + - 'deployments/ansible/roles/splunk-otel-collector/**' + +defaults: + run: + working-directory: 'deployments/ansible/roles/splunk-otel-collector' + +jobs: + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Check out the codebase. + uses: actions/checkout@v2 + + - name: Set up Python 3. + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install test dependencies. + run: pip3 install yamllint + + - name: Lint code. + run: yamllint . + + molecule-test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + distro: + - amazonlinux2 + - centos7 + - centos8 + - debian9 + - debian10 + - ubuntu2004 + + steps: + - name: Check out the codebase. + uses: actions/checkout@v2 + + - name: Set up Python 3. + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install test dependencies. + run: pip3 install ansible molecule[docker] docker + + - name: Run Molecule tests. + run: molecule --base-config ./molecule/config/docker.yml test --all + env: + PY_COLORS: '1' + ANSIBLE_FORCE_COLOR: '1' + MOLECULE_DISTRO: ${{ matrix.distro }} diff --git a/.gitignore b/.gitignore index 0b71eb2f19..66ccac0068 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ venv/ __pycache__ *.pyc .pytest_cache + +# Ansible Molecule +.cache diff --git a/deployments/ansible/roles/splunk-otel-collector/.yamllint b/deployments/ansible/roles/splunk-otel-collector/.yamllint new file mode 100644 index 0000000000..11e3d6d575 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/.yamllint @@ -0,0 +1,12 @@ +--- +# Based on ansible-lint config +extends: default + +rules: + line-length: + max: 120 + level: error + key-duplicates: enable + new-lines: + type: unix + truthy: disable diff --git a/deployments/ansible/roles/splunk-otel-collector/CONTRIBUTING.md b/deployments/ansible/roles/splunk-otel-collector/CONTRIBUTING.md new file mode 100644 index 0000000000..e18fc0630e --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/CONTRIBUTING.md @@ -0,0 +1,75 @@ +# Contributing Guidelines + +If you found a bug in the Ansible role, don't hesitate to submit an issue or a +pull request. + +## Local development + +Testing and validation of this Ansible role is based on +[Ansible Molecule](https://molecule.readthedocs.io/en/latest/). + +### MacOS or Windows setup + +Development of the role can be done on MacOS or Windows machines. Make sure the +following software is installed: + +- [VirtualBox](https://www.virtualbox.org/wiki/Downloads) +- [Vagrant](https://www.vagrantup.com/downloads) +- [Python3](https://www.python.org/downloads) +- Python packages: + - ansible + - molecule + - molecule-vagrant + +MacOS installation steps: + +```sh +brew tap hashicorp/tap +brew install virtualbox vagrant python3 +pip3 install ansible molecule molecule-vagrant +``` + +Use the following arguments with every molecule command +`--base-config ./molecule/config/vagrant.yml`. + +To setup test VMs: +```sh +molecule --base-config ./molecule/config/vagrant.yml create +``` + +To apply Molecule test playbooks: +```sh +molecule --base-config ./molecule/config/vagrant.yml converge +``` + +To run the full test suite: +```sh +molecule --base-config ./molecule/config/vagrant.yml test --all +``` + +### Linux setup + +Development on a Linux machine is simpler, all you need is docker. + +Linux installation steps: + +- Make sure [Python3](https://www.python.org/downloads) is installed +- `pip3 install ansible molecule[docker] docker` + +Use the following arguments with every molecule command +`--base-config ./molecule/config/docker.yml`. + +To setup test docker containers: +```sh +molecule --base-config ./molecule/config/docker.yml create +``` + +To apply Molecule playbooks: +```sh +molecule --base-config ./molecule/config/docker.yml converge +``` + +To run the full test suite: +```sh +molecule --base-config ./molecule/config/docker.yml test --all +``` diff --git a/deployments/ansible/roles/splunk-otel-collector/README.md b/deployments/ansible/roles/splunk-otel-collector/README.md new file mode 100644 index 0000000000..ba4fd48b79 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/README.md @@ -0,0 +1,119 @@ +# Role Name + +Ansible role that installs Splunk OpenTelemetry Connector configured to +collect metrics, traces and logs from Linux machines and send data to [Splunk +Observability Cloud](https://www.splunk.com/en_us/observability.html). + +## Prerequisites + +- [Splunk Access Token](https://docs.splunk.com/Observability/admin/authentication-tokens/org-tokens.html#admin-org-tokens) +- [Splunk Realm](https://dev.splunk.com/observability/docs/realms_in_endpoints/) +- [Double-check exposed ports](https://github.com/signalfx/splunk-otel-collector/blob/main/docs/security.md#exposed-endpoints) + to make sure your environment doesn't have conflicts. Ports can be changed in the collector's configuration. + +## Usage + +To install the role, clone the agent repo to your control host and add the +`splunk-otel-collector` role directory to its `roles_path` in `ansible.cfg`, +or use this document's directory as your working directory. + +To use this role, simply include the `splunk-otel-collector` role invocation +in your playbook. The following example shows how to use the role in a +playbook with minimal required configuration: + +```yaml +- name: Install Splunk OpenTelemetry Connector + hosts: all + become: yes + vars: + splunk_access_token: YOUR_ACCESS_TOKEN + splunk_realm: SPLUNK_REALM + tasks: + - name: "Include splunk-otel-collector" + include_role: + name: "splunk-otel-collector" +``` + +## Role Variables + +- `splunk_access_token` (**Required**): The Splunk access token to + authenticate requests. + +- `splunk_realm`: Which realm to send the data to. The `SPLUNK_REALM` + environment variable will be set with this value for the Splunk OTel + Collector service. (**default:** `us0`) + +- `splunk_ingest_url`: The Splunk ingest URL, e.g. + `https://ingest.us0.signalfx.com`. The `SPLUNK_INGEST_URL` environment + variable will be set with this value for the collector service. (**default:** + `https://ingest.{{ splunk_realm }}.signalfx.com`) + +- `splunk_api_url`: The Splunk API URL, e.g. `https://api.us0.signalfx.com`. + The `SPLUNK_API_URL` environment variable will be set with this value for the + collector service. (**default:** `https://api.{{ splunk_realm }}.signalfx.com`) + +- `collector_version`: Version of the collector package to install, e.g. + `0.26.0`. (**default:** `latest`) + +- `collector_config`: Splunk OTel Collector config YAML file. Can be set to + /etc/otel/collector/gateway_config.yaml to install the collector in gateway + mode. (**default:** `/etc/otel/collector/agent_config.yaml`) + +- `collector_config_source`: Source path to a Splunk OTel Collector config YAML + file on a control node that will be uploaded and set in place of + `collector_config` in remote nodes. Can be used to submit a custom collector + config, e.g. `./custom_collector_config.yaml`. (**default:** `""` meaning + that nothing will be copied and existing `collector_config` will be used) + +- `splunk_bundle_dir`: The path to the [Smart Agent bundle directory]( + https://github.com/signalfx/splunk-otel-collector/blob/main/internal/extension/smartagentextension/README.md). + The default path is provided by the collector package. If the specified path + is changed from the default value, the path should be an existing directory + on the node. The `SPLUNK_BUNDLE_DIR` environment variable will be set to + this value for the collector service. (**default:** + `/usr/lib/splunk-otel-collector/agent-bundle`) + +- `splunk_collectd_dir`: The path to the collectd config directory for the + Smart Agent bundle. The default path is provided by the collector package. + If the specified path is changed from the default value, the path should be + an existing directory on the node. The `SPLUNK_COLLECTD_DIR` environment + variable will be set to this value for the collector service. (**default:** + `/usr/lib/splunk-otel-collector/agent-bundle`) + +- `service_user` and `service_group` (Linux only): Set the user/group + ownership for the collector service. The user/group will be created if they + do not exist. (**default:** `splunk-otel-collector`) + +- `memory_total_mib`: Amount of memory in MiB allocated to the Splunk OTel + Collector. (**default:** `512`) + +- `ballast_size_mib`: Memory ballast size in MiB that will be set to the Splunk + OTel Collector. (**default:** 1/3 of `memory_total_mib`) + +- `install_fluentd`: Whether to install/manage fluentd and dependencies for log + collection. The dependencies include [capng_c]( + https://github.com/fluent-plugins-nursery/capng_c) for enabling + [Linux capabilities]( + https://docs.fluentd.org/deployment/linux-capability), + [fluent-plugin-systemd]( + https://github.com/fluent-plugin-systemd/fluent-plugin-systemd) for systemd + journal log collection, and the required libraries/development tools. + (**default:** `true`) + +- `td_agent_version`: Version of td-agent (fluentd package) that will be + installed (**default:** `3.3.0` for Debian jessie, `3.7.1` for Debian + stretch, and `4.1.1` for other distros`) + +- `fluentd_config`: Path to the fluentd config file on the remote host. + (**default:** `/etc/otel/collector/fluentd/fluent.conf`) + +- `fluentd_config_source`: Source path to a fluentd config file on a + control node that will be uploaded and set in place of `fluentd_config` on + remote nodes. Can be used to submit a custom fluentd config, + e.g. `./custom_fluentd_config.conf`. (**default:** `""` meaning + that nothing will be copied and existing `fluentd_config` will be used) + +## Contributing + +Check [Contributing guidelines](./CONTRIBUTING.md) if you see something that +needs to be improved in this Ansible role. \ No newline at end of file diff --git a/deployments/ansible/roles/splunk-otel-collector/defaults/main.yml b/deployments/ansible/roles/splunk-otel-collector/defaults/main.yml new file mode 100644 index 0000000000..f5d5b9d9b5 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/defaults/main.yml @@ -0,0 +1,35 @@ +--- +# defaults file for splunk-otel-collector role vars + +# splunk_access_token is REQUIRED variable +splunk_access_token: "" + +splunk_realm: us0 + +collector_version: latest + +# Set this variable to /etc/otel/collector/gateway_config.yaml in order to +# deploy splunk-otel-collector in gateway mode +collector_config: /etc/otel/collector/agent_config.yaml +# Set collector_config_source to copy a custom config from a local path +collector_config_source: "" + +service_user: splunk-otel-collector +service_group: splunk-otel-collector + +memory_total_mib: 512 +# 1/3 of memory_mib by default +ballast_size_mib: "" + +install_fluentd: true + +# Explicitly set version of td-agent, +# By default: 3.3.0 for Debian jessie, 3.7.1 for Debian stretch, +# and 4.1.1 for other distros. +td_agent_version: "" + +fluentd_config: /etc/otel/collector/fluentd/fluent.conf +fluentd_config_source: "" + +agent_bundle_dir: /usr/lib/splunk-otel-collector/agent-bundle +splunk_collectd_dir: /usr/lib/splunk-otel-collector/agent-bundle/run/collectd diff --git a/deployments/ansible/roles/splunk-otel-collector/handlers/main.yml b/deployments/ansible/roles/splunk-otel-collector/handlers/main.yml new file mode 100644 index 0000000000..727e9b6c7f --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/handlers/main.yml @@ -0,0 +1,22 @@ +--- +# handlers file for splunk-otel-collector + +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: yes + listen: + - reload otel-collector + - reload td-agent + +- name: Restart td-agent + ansible.builtin.service: + name: td-agent + state: restarted + when: install_fluentd + listen: "reload td-agent" + +- name: Restart Splunk OpenTelemetry Collector + ansible.builtin.service: + name: splunk-otel-collector + state: restarted + listen: "reload otel-collector" diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/config/docker.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/config/docker.yml new file mode 100644 index 0000000000..9aa154da79 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/config/docker.yml @@ -0,0 +1,17 @@ +--- +# molecule config based on docker for CI usage on a Linux system only + +driver: + name: docker +platforms: + - name: instance + image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos8}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + pre_build_image: true +provisioner: + name: ansible + playbooks: + converge: ${MOLECULE_PLAYBOOK:-converge.yml} diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/config/vagrant.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/config/vagrant.yml new file mode 100644 index 0000000000..d167125130 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/config/vagrant.yml @@ -0,0 +1,18 @@ +--- +# molecule config based on vagrant and virtualbox for local usage on any OS + +driver: + name: vagrant + provider: + name: virtualbox +platforms: + - name: focal64 + box: ubuntu/focal64 + - name: centos8 + box: centos/8 + - name: buster64 + box: debian/buster64 + - name: amazonlinux + box: bento/amazonlinux-2 +provisioner: + name: ansible diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/converge.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/converge.yml new file mode 100644 index 0000000000..87a678ca48 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/converge.yml @@ -0,0 +1,19 @@ +--- +- name: Converge scenario with custom variables + hosts: all + become: yes + vars: + splunk_access_token: fake-token + collector_version: 0.24.3 + collector_config: /etc/otel/collector/custom_config.yml + collector_config_source: ./custom_collector_config.yml + service_user: custom-user + service_group: custom-group + memory_total_mib: 256 + ballast_size_mib: 100 + fluentd_config: /etc/otel/collector/fluentd/custom_fluentd.conf + fluentd_config_source: ./custom_fluentd_config.conf + tasks: + - name: "Include splunk-otel-collector" + include_role: + name: "splunk-otel-collector" diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_collector_config.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_collector_config.yml new file mode 100644 index 0000000000..089ce552e3 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_collector_config.yml @@ -0,0 +1,40 @@ +--- +# Custom collector config for test purposes + +receivers: + fluentforward: + endpoint: 127.0.0.1:8006 + otlp: + protocols: + grpc: + http: + prometheus: + config: + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 10s + static_configs: + - targets: ['127.0.0.1:8888'] + +processors: + memory_limiter: + ballast_size_mib: ${SPLUNK_BALLAST_SIZE_MIB} + check_interval: 2s + limit_mib: ${SPLUNK_MEMORY_LIMIT_MIB} + +exporters: + logging/debug: + loglevel: debug + logging/info: + loglevel: info + +service: + pipelines: + metrics: + receivers: [otlp, prometheus] + processors: [memory_limiter] + exporters: [logging/info] + logs: + receivers: [fluentforward, otlp] + processors: [memory_limiter] + exporters: [logging/debug] diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_fluentd_config.conf b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_fluentd_config.conf new file mode 100644 index 0000000000..bf2b90a570 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/custom_fluentd_config.conf @@ -0,0 +1,22 @@ +# Custom fluentd config for test purposes + + + @type http + port 9880 + bind 0.0.0.0 + body_size_limit 32m + keepalive_timeout 10s + + + + @type forward + heartbeat_type udp + + host 127.0.0.1 + port 8006 + + + + + log_level info + diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/molecule.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/molecule.yml new file mode 100644 index 0000000000..5c1d4890b5 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/molecule.yml @@ -0,0 +1 @@ +# no default config provided, make sure to provide --base-config argument instead diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/verify.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/verify.yml new file mode 100644 index 0000000000..31c157a622 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/custom_vars/verify.yml @@ -0,0 +1,95 @@ +--- +- name: Verify scenario with custom variables + hosts: all + gather_facts: false + become: yes + tasks: + - name: Import shared verification tasks + ansible.builtin.import_tasks: ../shared/verify_tasks.yml + + - name: Assert td-agent service is running + assert: + that: ansible_facts.services['td-agent.service'].state == 'running' + + - name: Check custom_config.yaml + stat: + path: /etc/otel/collector/custom_config.yml + register: custom_collector_config + + - name: Assert custom_config.yaml is created + assert: + that: custom_collector_config.stat.exists + + - name: Assert custom_config.yaml is used + ansible.builtin.lineinfile: + line: SPLUNK_CONFIG=/etc/otel/collector/custom_config.yml + dest: /etc/otel/collector/splunk-otel-collector.conf + state: present + check_mode: yes + + - name: Populate package facts + ansible.builtin.package_facts: + + - name: Assert splunk-otel-collector version is set + assert: + that: ansible_facts.packages['splunk-otel-collector'][0].version == '0.24.3' + + - name: Assert custom service user is set + ansible.builtin.lineinfile: + line: User=custom-user + dest: /etc/systemd/system/splunk-otel-collector.service.d/service-owner.conf + state: present + check_mode: yes + + - name: Assert custom service group is set + ansible.builtin.lineinfile: + line: Group=custom-group + dest: /etc/systemd/system/splunk-otel-collector.service.d/service-owner.conf + state: present + check_mode: yes + + - name: Assert SPLUNK_MEMORY_TOTAL_MIB env var is set + ansible.builtin.lineinfile: + line: SPLUNK_MEMORY_TOTAL_MIB=256 + dest: /etc/otel/collector/splunk-otel-collector.conf + state: present + check_mode: yes + + - name: Look for ballast config in collector service output + ansible.builtin.shell: + cmd: 'journalctl -u splunk-otel-collector | grep "Set ballast to 100 MiB"' + register: result + changed_when: false + failed_when: not result.stdout + + - name: Check custom_fluentd.conf + stat: + path: /etc/otel/collector/fluentd/custom_fluentd.conf + register: custom_fluentd_config + + - name: Assert custom_fluentd.conf is created + assert: + that: custom_fluentd_config.stat.exists + + - name: Assert custom_fluentd.conf is used + ansible.builtin.lineinfile: + line: Environment=FLUENT_CONF=/etc/otel/collector/fluentd/custom_fluentd.conf + dest: /etc/systemd/system/td-agent.service.d/splunk-otel-collector.conf + state: present + check_mode: yes + + - name: Send a test log message + ansible.builtin.uri: + body: json={"log":"test logs pipeline"} + method: POST + url: http://localhost:9880/app.log + changed_when: false + + - name: Look for the test log message in collector service output + ansible.builtin.shell: + cmd: 'journalctl -u splunk-otel-collector | grep "Body: test logs pipeline"' + register: result + changed_when: false + until: result.stdout + retries: 20 + delay: 1 diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/default/converge.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/default/converge.yml new file mode 100644 index 0000000000..6b858bcd99 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge scenario with the default configuration + hosts: all + become: yes + vars: + splunk_access_token: fake-token + splunk_realm: fake-realm + tasks: + - name: "Include splunk-otel-collector" + include_role: + name: "splunk-otel-collector" diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/default/molecule.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/default/molecule.yml new file mode 100644 index 0000000000..5c1d4890b5 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/default/molecule.yml @@ -0,0 +1 @@ +# no default config provided, make sure to provide --base-config argument instead diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/default/verify.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/default/verify.yml new file mode 100644 index 0000000000..c4da349772 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/default/verify.yml @@ -0,0 +1,21 @@ +--- +- name: Verify scenario with the default configuration + hosts: all + gather_facts: false + tasks: + - name: Import shared verification tasks + ansible.builtin.import_tasks: ../shared/verify_tasks.yml + + - name: Assert SPLUNK_ACCESS_TOKEN env var is set + ansible.builtin.lineinfile: + line: SPLUNK_ACCESS_TOKEN=fake-token + dest: /etc/otel/collector/splunk-otel-collector.conf + state: present + check_mode: yes + + - name: Assert SPLUNK_REALM env var is set + ansible.builtin.lineinfile: + line: SPLUNK_REALM=fake-realm + dest: /etc/otel/collector/splunk-otel-collector.conf + state: present + check_mode: yes diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/shared/verify_tasks.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/shared/verify_tasks.yml new file mode 100644 index 0000000000..6bf3f588d8 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/shared/verify_tasks.yml @@ -0,0 +1,13 @@ +--- +# Verification tasks to make sure that both collector and fluentd services are running + +- name: Populate service facts + ansible.builtin.service_facts: + +- name: Assert splunk-otel-collector service is running + assert: + that: ansible_facts.services['splunk-otel-collector.service'].state == 'running' + +- name: Assert td-agent service is running + assert: + that: ansible_facts.services['td-agent.service'].state == 'running' diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/converge.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/converge.yml new file mode 100644 index 0000000000..392176d0b5 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: yes + vars: + splunk_access_token: fake-token + install_fluentd: no + tasks: + - name: "Include splunk-otel-collector" + include_role: + name: "splunk-otel-collector" diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/molecule.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/molecule.yml new file mode 100644 index 0000000000..5c1d4890b5 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/molecule.yml @@ -0,0 +1 @@ +# no default config provided, make sure to provide --base-config argument instead diff --git a/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/verify.yml b/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/verify.yml new file mode 100644 index 0000000000..e7631d3976 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/molecule/without_fluentd/verify.yml @@ -0,0 +1,15 @@ +--- +- name: Verify scenario without fluentd installation + hosts: all + gather_facts: false + tasks: + - name: Populate service facts + ansible.builtin.service_facts: + + - name: Assert splunk-otel-collector service is running + assert: + that: ansible_facts.services['splunk-otel-collector.service'].state == 'running' + + - name: Assert td-agent service is not installed + assert: + that: "'td-agent.service' not in ansible_facts.services" diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_fluentd.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_fluentd.yml new file mode 100644 index 0000000000..cf4b77d39c --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_fluentd.yml @@ -0,0 +1,31 @@ +--- +# Install FluentD agent from apt registry + +- name: Add an apt signing key for td-agent + ansible.builtin.apt_key: + url: "{{ td_agent_repo_base }}/GPG-KEY-td-agent" + state: present + +- name: Add td-agent repo to apt source list + ansible.builtin.apt_repository: + repo: "deb {{ td_agent_repo_base }}/{{ td_agent_major_version }}/{{ ansible_distribution | lower }}\ + /{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} contrib" + filename: td_agent.list + state: present + +- name: Install td-agent via apt package manager + ansible.builtin.apt: + name: td-agent={{ td_agent_version }} + state: present + update_cache: yes + notify: "reload td-agent" + +- name: Install FluentD Linux capability module apt dependencies + ansible.builtin.apt: + name: + - build-essential + - libcap-ng0 + - libcap-ng-dev + - pkg-config + state: present + update_cache: yes diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_otel_collector.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_otel_collector.yml new file mode 100644 index 0000000000..13f624e941 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/apt_install_otel_collector.yml @@ -0,0 +1,31 @@ +--- +# Install Splunk OpenTelemetry Collector from apt repository + +- name: Install apt dependencies for secure transport + ansible.builtin.apt: + name: + - apt-transport-https + - gnupg + state: present + update_cache: yes + +- name: Add an apt signing key for Splunk OpenTelemetry Collector + ansible.builtin.apt_key: + url: "{{ splunk_repo_base_url }}/otel-collector-deb/splunk-B3CD4420.gpg" + keyring: /etc/apt/trusted.gpg.d/splunk.gpg + +- name: Add Splunk OpenTelemetry Collector repo to apt source list + ansible.builtin.apt_repository: + repo: "deb {{ splunk_repo_base_url }}/otel-collector-deb release main" + filename: splunk-otel-collector.list + state: present + +- name: Install Splunk OpenTelemetry Collector via apt package manager + ansible.builtin.apt: + name: splunk-otel-collector{% if collector_version != "latest" %}={{ collector_version }}{% endif %} + state: "{% if collector_version != 'latest' %}present{% else %}latest{% endif %}" + force: yes + update_cache: yes + notify: + - "reload otel-collector" + - "reload td-agent" diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/collector_service_owner.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/collector_service_owner.yml new file mode 100644 index 0000000000..9668478ed7 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/collector_service_owner.yml @@ -0,0 +1,41 @@ +--- +# Ensure a custom system group and user created for Splunk OTel Collector service + +- name: Ensure service group created for Splunk Otel Collector + ansible.builtin.group: + name: "{{ service_group }}" + system: yes + +- name: Ensure service user created for Splunk Otel Collector + ansible.builtin.user: + name: "{{ service_user }}" + group: "{{ service_group }}" + system: yes + shell: /sbin/nologin + home: /etc/otel/collector + +- name: Set Otel Collector config directory + ansible.builtin.file: + path: /etc/otel/collector + state: directory + owner: "{{ service_user }}" + group: "{{ service_group }}" + +- name: Ensure systemd service directory for Splunk Otel Collector + ansible.builtin.file: + path: /etc/systemd/system/splunk-otel-collector.service.d + state: directory + +- name: Set systemd service owner for Splunk Otel Collector + ansible.builtin.template: + src: collector-service-owner.conf.j2 + dest: /etc/systemd/system/splunk-otel-collector.service.d/service-owner.conf + notify: "reload otel-collector" + +- name: Set Otel Collector agent bundle directory + ansible.builtin.file: + path: "{{ agent_bundle_dir }}" + state: directory + owner: "{{ service_user }}" + group: "{{ service_group }}" + recurse: yes diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/linux_install.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/linux_install.yml new file mode 100644 index 0000000000..74ab4b7bfc --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/linux_install.yml @@ -0,0 +1,41 @@ +--- +# splunk-otel-collector Linux installation + +- name: Set Linux packages sources + ansible.builtin.set_fact: + splunk_repo_base_url: https://splunk.jfrog.io/splunk + td_agent_repo_base: https://packages.treasuredata.com + +- name: Install Splunk OpenTelemetry Collector with apt package manager + ansible.builtin.import_tasks: apt_install_otel_collector.yml + when: ansible_os_family == "Debian" + +- name: Install Splunk OpenTelemetry Collector with yum package manager + ansible.builtin.import_tasks: yum_install_otel_collector.yml + when: ansible_os_family == "RedHat" + +- name: Set Splunk Otel Collector service owner + import_tasks: collector_service_owner.yml + +- name: Set up env file for Splunk OTel Collector service + ansible.builtin.template: + src: splunk-otel-collector.conf.j2 + dest: /etc/otel/collector/splunk-otel-collector.conf + owner: "{{ service_user }}" + group: "{{ service_group }}" + mode: 0600 + notify: "reload otel-collector" + +- name: Push custom config for Splunk OTel Collector, if provided + ansible.builtin.copy: + src: "{{ collector_config_source }}" + dest: "{{ collector_config }}" + owner: "{{ service_user }}" + group: "{{ service_group }}" + mode: 0644 + when: collector_config_source + notify: "reload otel-collector" + +- name: Install FluentD + import_tasks: linux_install_fluentd.yml + when: install_fluentd diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/linux_install_fluentd.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/linux_install_fluentd.yml new file mode 100644 index 0000000000..c867cfda05 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/linux_install_fluentd.yml @@ -0,0 +1,117 @@ +--- +# td-agent Linux installation + +- name: Set required td-agent version + ansible.builtin.set_fact: + td_agent_version: | + {%- if td_agent_version -%} + {{ td_agent_version }} + {%- elif ansible_distribution_release == "stretch" -%} + 3.7.1-0 + {%- elif ansible_distribution_release == "jessie" -%} + 3.3.0-1 + {%- elif ansible_os_family == "Debian" -%} + 4.1.1-1 + {%- else -%} + 4.1.1 + {%- endif -%} + +- name: Set required td-agent major version + ansible.builtin.set_fact: + td_agent_major_version: "{{ td_agent_version.split('.')[0] }}" + +- name: Install FluentD with apt package manager + ansible.builtin.import_tasks: apt_install_fluentd.yml + when: ansible_os_family == "Debian" + +- name: Install FluentD with yum package manager + ansible.builtin.import_tasks: yum_install_fluentd.yml + when: ansible_os_family == "RedHat" + +- name: Make sure td-agent can read provided config + ansible.builtin.file: + path: /etc/otel/collector/fluentd + state: directory + owner: td-agent + group: td-agent + recurse: yes + +- name: Push custom FluentD config, if provided + ansible.builtin.copy: + src: "{{ fluentd_config_source }}" + dest: "{{ fluentd_config }}" + owner: td-agent + group: td-agent + when: fluentd_config_source + notify: "reload td-agent" + +- name: Make sure td-agent.service.d directory exists + ansible.builtin.file: + path: /etc/systemd/system/td-agent.service.d + state: directory + +- name: Apply td-agent service config + ansible.builtin.template: + src: td-agent.conf.j2 + dest: /etc/systemd/system/td-agent.service.d/splunk-otel-collector.conf + owner: root + group: root + mode: 0644 + notify: "reload td-agent" + +- name: Check fluent-cap-ctl + stat: + path: /opt/td-agent/bin/fluent-cap-ctl + register: fluent_cap_ctl + +- name: Install capng_c fluentd plugin + community.general.gem: + executable: td-agent-gem + name: capng_c + state: present + user_install: no + version: "0.2.2" + when: fluent_cap_ctl.stat.exists + register: capng_c + notify: "reload td-agent" + +- name: Check installed Linux capabilities + ansible.builtin.command: + cmd: "/opt/td-agent/bin/fluent-cap-ctl --get -f /opt/td-agent/bin/ruby" + changed_when: false + register: cap_get + when: fluent_cap_ctl.stat.exists + +- name: Install Linux capabilities + ansible.builtin.command: + cmd: '/opt/td-agent/bin/fluent-cap-ctl --add "dac_override,dac_read_search" -f /opt/td-agent/bin/ruby' + when: > + fluent_cap_ctl.stat.exists and + ("dac_override" not in cap_get['stdout'] or "dac_read_search" not in cap_get['stdout']) + +- name: Determine available groups + getent: + database: group + split: ':' + when: not fluent_cap_ctl.stat.exists + +- name: Add td-agent user to systemd-journal and adm groups + ansible.builtin.user: + name: td-agent + append: yes + groups: + - "{{ item }}" + when: not fluent_cap_ctl.stat.exists and item in getent_group + with_items: + - systemd-journal + - adm + notify: "reload td-agent" + +- name: Install fluent-plugin-systemd + community.general.gem: + executable: td-agent-gem + name: fluent-plugin-systemd + state: present + user_install: no + version: "1.0.1" + notify: "reload td-agent" diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/main.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/main.yml new file mode 100644 index 0000000000..50a66a20e7 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: Make sure access token is provided + ansible.builtin.assert: + that: splunk_access_token + fail_msg: splunk_access_token variable must be provided + +- name: Make sure host OS is supported + ansible.builtin.assert: + that: ansible_os_family in ["Debian", "RedHat"] + fail_msg: "{{ ansible_os_family }} OS family currently is not supported" + +- name: Linux deployment + ansible.builtin.import_tasks: linux_install.yml + +- name: Linux deployment + ansible.builtin.import_tasks: linux_install_fluentd.yml + when: install_fluentd diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_fluentd.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_fluentd.yml new file mode 100644 index 0000000000..3b2ba49213 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_fluentd.yml @@ -0,0 +1,34 @@ +--- +# Install FluentD agent from yum repository + +- name: Set td-agent distro code + ansible.builtin.set_fact: + distro_code: "{{ 'amazon' if ansible_distribution == 'Amazon' else 'redhat' }}" + +- name: Add td-agent repo to yum source list + ansible.builtin.yum_repository: + name: td_agent + description: TreasureData Repository + baseurl: "{{ td_agent_repo_base }}/{{ td_agent_major_version }}/{{ distro_code }}\ + /{{ ansible_distribution_major_version }}/$basearch/" + gpgkey: "{{ td_agent_repo_base }}/GPG-KEY-td-agent" + gpgcheck: yes + enabled: yes + +- name: Install td-agent via yum package manager + ansible.builtin.yum: + name: td-agent-{{ td_agent_version }} + state: present + allow_downgrade: yes + update_cache: yes + notify: "reload td-agent" + +- name: Install FluentD Linux capability module yum dependencies + ansible.builtin.yum: + name: + - "@Development tools" + - libcap-ng + - libcap-ng-devel + - pkgconfig + state: present + update_cache: yes diff --git a/deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_otel_collector.yml b/deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_otel_collector.yml new file mode 100644 index 0000000000..e87fd1752e --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/tasks/yum_install_otel_collector.yml @@ -0,0 +1,21 @@ +--- +# Install Splunk OpenTelemetry Collector from yum repository + +- name: Add Splunk OpenTelemetry Collector repo to yum source list + ansible.builtin.yum_repository: + name: splunk-otel-collector + description: Splunk OpenTelemetry Collector Repository + baseurl: "{{ splunk_repo_base_url }}/otel-collector-rpm/release/$basearch/" + gpgkey: "{{ splunk_repo_base_url }}/otel-collector-rpm/splunk-B3CD4420.pub" + gpgcheck: yes + enabled: yes + +- name: Install Splunk OpenTelemetry Collector via yum package manager + ansible.builtin.yum: + name: splunk-otel-collector{% if collector_version != "latest" %}-{{ collector_version }}{% endif %} + state: "{% if collector_version != 'latest' %}present{% else %}latest{% endif %}" + allow_downgrade: yes + update_cache: yes + notify: + - "reload otel-collector" + - "reload td-agent" diff --git a/deployments/ansible/roles/splunk-otel-collector/templates/collector-service-owner.conf.j2 b/deployments/ansible/roles/splunk-otel-collector/templates/collector-service-owner.conf.j2 new file mode 100644 index 0000000000..ec7ccbf29f --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/templates/collector-service-owner.conf.j2 @@ -0,0 +1,3 @@ +[Service] +User={{ service_user }} +Group={{ service_group }} diff --git a/deployments/ansible/roles/splunk-otel-collector/templates/splunk-otel-collector.conf.j2 b/deployments/ansible/roles/splunk-otel-collector/templates/splunk-otel-collector.conf.j2 new file mode 100644 index 0000000000..4a1aa62f50 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/templates/splunk-otel-collector.conf.j2 @@ -0,0 +1,12 @@ +SPLUNK_CONFIG={{ collector_config }} +SPLUNK_ACCESS_TOKEN={{ splunk_access_token }} +SPLUNK_REALM={{ splunk_realm }} +SPLUNK_API_URL={{ splunk_api_url | default("https://api." + splunk_realm + ".signalfx.com") }} +SPLUNK_INGEST_URL={{ splunk_ingest_url | default("https://ingest." + splunk_realm + ".signalfx.com") }} +SPLUNK_TRACE_URL=${SPLUNK_INGEST_URL}/v2/trace +SPLUNK_HEC_URL=${SPLUNK_INGEST_URL}/v1/log +SPLUNK_HEC_TOKEN={{ splunk_access_token }} +SPLUNK_MEMORY_TOTAL_MIB={{ memory_total_mib }} +SPLUNK_BALLAST_SIZE_MIB={{ ballast_size_mib }} +SPLUNK_BUNDLE_DIR={{ agent_bundle_dir }} +SPLUNK_COLLECTD_DIR={{ splunk_collectd_dir }} diff --git a/deployments/ansible/roles/splunk-otel-collector/templates/td-agent.conf.j2 b/deployments/ansible/roles/splunk-otel-collector/templates/td-agent.conf.j2 new file mode 100644 index 0000000000..86a74dd9e0 --- /dev/null +++ b/deployments/ansible/roles/splunk-otel-collector/templates/td-agent.conf.j2 @@ -0,0 +1,2 @@ +[Service] +Environment=FLUENT_CONF={{ fluentd_config }}