From e35a087ce16e045cdfedcc5b424cf9e6abd7298e Mon Sep 17 00:00:00 2001 From: Jeff Cheng <83052155+jcheng-splunk@users.noreply.github.com> Date: Tue, 18 May 2021 09:59:53 -0400 Subject: [PATCH] Initial puppet module for linux (#405) --- .circleci/config.yml | 52 +++- deployments/puppet/.dockerignore | 1 + deployments/puppet/.fixtures.yml | 11 + deployments/puppet/.gitignore | 1 + deployments/puppet/Dockerfile | 31 +++ deployments/puppet/Gemfile | 127 +++++++++ deployments/puppet/Gemfile.lock | 172 ++++++++++++ deployments/puppet/Makefile | 32 +++ deployments/puppet/README.md | 65 +++++ deployments/puppet/RELEASE.md | 11 + deployments/puppet/Rakefile | 13 + deployments/puppet/lib/facter/local_groups.rb | 15 ++ deployments/puppet/lib/facter/local_users.rb | 15 ++ .../puppet/manifests/collector_debian_repo.pp | 19 ++ .../manifests/collector_service_owner.pp | 101 +++++++ .../puppet/manifests/collector_yum_repo.pp | 22 ++ .../puppet/manifests/fluentd_debian_repo.pp | 26 ++ .../puppet/manifests/fluentd_yum_repo.pp | 29 ++ deployments/puppet/manifests/init.pp | 255 ++++++++++++++++++ deployments/puppet/metadata.json | 68 +++++ deployments/puppet/release | 51 ++++ deployments/puppet/spec/classes/init_spec.rb | 24 ++ deployments/puppet/spec/spec_helper.rb | 15 ++ .../templates/collector_config.yaml.erb | 5 + .../packaging/tests/deployments/__init__.py | 0 .../tests/deployments/puppet/__init__.py | 0 .../images/deb/Dockerfile.debian-buster | 37 +++ .../images/deb/Dockerfile.debian-jessie | 37 +++ .../images/deb/Dockerfile.debian-stretch | 37 +++ .../images/deb/Dockerfile.ubuntu-bionic | 33 +++ .../puppet/images/deb/Dockerfile.ubuntu-focal | 33 +++ .../images/deb/Dockerfile.ubuntu-xenial | 33 +++ .../images/rpm/Dockerfile.amazonlinux-2 | 29 ++ .../puppet/images/rpm/Dockerfile.centos-7 | 29 ++ .../puppet/images/rpm/Dockerfile.centos-8 | 29 ++ .../images/rpm/Dockerfile.oraclelinux-7 | 29 ++ .../images/rpm/Dockerfile.oraclelinux-8 | 29 ++ .../tests/deployments/puppet/puppet_test.py | 134 +++++++++ .../packaging/tests/helpers/util.py | 29 +- 39 files changed, 1671 insertions(+), 8 deletions(-) create mode 100644 deployments/puppet/.dockerignore create mode 100644 deployments/puppet/.fixtures.yml create mode 100644 deployments/puppet/.gitignore create mode 100644 deployments/puppet/Dockerfile create mode 100644 deployments/puppet/Gemfile create mode 100644 deployments/puppet/Gemfile.lock create mode 100644 deployments/puppet/Makefile create mode 100644 deployments/puppet/README.md create mode 100644 deployments/puppet/RELEASE.md create mode 100644 deployments/puppet/Rakefile create mode 100644 deployments/puppet/lib/facter/local_groups.rb create mode 100644 deployments/puppet/lib/facter/local_users.rb create mode 100644 deployments/puppet/manifests/collector_debian_repo.pp create mode 100644 deployments/puppet/manifests/collector_service_owner.pp create mode 100644 deployments/puppet/manifests/collector_yum_repo.pp create mode 100644 deployments/puppet/manifests/fluentd_debian_repo.pp create mode 100644 deployments/puppet/manifests/fluentd_yum_repo.pp create mode 100644 deployments/puppet/manifests/init.pp create mode 100644 deployments/puppet/metadata.json create mode 100755 deployments/puppet/release create mode 100644 deployments/puppet/spec/classes/init_spec.rb create mode 100644 deployments/puppet/spec/spec_helper.rb create mode 100644 deployments/puppet/templates/collector_config.yaml.erb create mode 100644 internal/buildscripts/packaging/tests/deployments/__init__.py create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/__init__.py create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-buster create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-jessie create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-stretch create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-bionic create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-focal create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-xenial create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.amazonlinux-2 create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-7 create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-8 create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-7 create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-8 create mode 100644 internal/buildscripts/packaging/tests/deployments/puppet/puppet_test.py diff --git a/.circleci/config.yml b/.circleci/config.yml index a506b4d8fa..07190541ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -262,6 +262,10 @@ workflows: filters: tags: only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/ + - puppet-test: + filters: + tags: + only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/ jobs: setup-environment: @@ -481,7 +485,7 @@ jobs: name: Check for relevant changes command: | # Only run installer tests for main branch, tags, or if the branch has relevant changes - if [[ "$CIRCLE_BRANCH" != "main" ]] && [[ -z "${CIRCLE_TAG:-}" ]] && ! .circleci/scripts/changes-include internal/buildscripts/packaging; then + if [[ "$CIRCLE_BRANCH" != "main" ]] && [[ -z "${CIRCLE_TAG:-}" ]] && ! .circleci/scripts/changes-include internal/buildscripts/packaging/installer internal/buildscripts/packaging/tests; then echo "SKIP_TESTS=yes" >> $BASH_ENV fi - install_pytest @@ -537,3 +541,49 @@ jobs: $msi_path = Resolve-Path .\dist\splunk-otel-collector*.msi $env:VERIFY_ACCESS_TOKEN = "false" .\internal\buildscripts\packaging\installer\install.ps1 -access_token "testing123" -msi_path "$msi_path" -mode "<< parameters.mode >>" + + puppet-test: + machine: + image: ubuntu-1604:202007-01 + docker_layer_caching: true + parallelism: 2 + steps: + - checkout + - run: + name: Lint + command: | + # Only run lint for main branch, tags, or if the branch has relevant changes + if [[ "$CIRCLE_BRANCH" = "main" ]] || [[ -n "${CIRCLE_TAG:-}" ]] || .circleci/scripts/changes-include deployments/puppet; then + make -C deployments/puppet lint + fi + - run: + name: Rake spec + command: | + # Only run rake-spec for main branch, tags, or if the branch has relevant changes + if [[ "$CIRCLE_BRANCH" = "main" ]] || [[ -n "${CIRCLE_TAG:-}" ]] || .circleci/scripts/changes-include deployments/puppet; then + make -C deployments/puppet rake-spec + fi + - run: + name: Check for relevant changes + command: | + # Only run puppet tests for main branch, tags, or if the branch has relevant changes + if [[ "$CIRCLE_BRANCH" != "main" ]] && [[ -z "${CIRCLE_TAG:-}" ]] && ! .circleci/scripts/changes-include deployments/puppet internal/buildscripts/packaging/tests; then + echo "SKIP_TESTS=yes" >> $BASH_ENV + fi + - install_pytest + - run: + name: Test puppet module + command: | + [[ "$SKIP_TESTS" = "yes" ]] && exit 0 + mkdir -p ~/testresults + if [[ $CIRCLE_NODE_INDEX -eq 0 ]]; then + PACKAGE_TYPE=deb + else + PACKAGE_TYPE=rpm + fi + pytest -n2 --verbose -m $PACKAGE_TYPE \ + --junitxml=~/testresults/results.xml \ + --html=~/testresults/results.html \ + --self-contained-html \ + internal/buildscripts/packaging/tests/deployments/puppet/puppet_test.py + - save_pytest_results diff --git a/deployments/puppet/.dockerignore b/deployments/puppet/.dockerignore new file mode 100644 index 0000000000..7750a07db3 --- /dev/null +++ b/deployments/puppet/.dockerignore @@ -0,0 +1 @@ +pkg/* \ No newline at end of file diff --git a/deployments/puppet/.fixtures.yml b/deployments/puppet/.fixtures.yml new file mode 100644 index 0000000000..5580029d70 --- /dev/null +++ b/deployments/puppet/.fixtures.yml @@ -0,0 +1,11 @@ +fixtures: + forge_modules: + stdlib: + repo: "puppetlabs/stdlib" + ref: "4.24.0" + apt: + repo: "puppetlabs/apt" + ref: "7.0.0" + yum: + repo: "puppet/yum" + ref: "4.3.0" diff --git a/deployments/puppet/.gitignore b/deployments/puppet/.gitignore new file mode 100644 index 0000000000..cace5e84dc --- /dev/null +++ b/deployments/puppet/.gitignore @@ -0,0 +1 @@ +/pkg/ diff --git a/deployments/puppet/Dockerfile b/deployments/puppet/Dockerfile new file mode 100644 index 0000000000..2d1cb42c54 --- /dev/null +++ b/deployments/puppet/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:16.04 + +ENV PATH=$PATH:/opt/puppetlabs/bin:/opt/puppetlabs/pdk/bin + +WORKDIR /tmp +RUN apt update &&\ + apt install -y make wget vim gcc ruby ruby-dev libxml2 libxml2-dev libxslt1-dev git apt-transport-https ca-certificates g++ + +RUN wget https://apt.puppetlabs.com/puppet6-release-xenial.deb &&\ + dpkg -i puppet6-release-xenial.deb &&\ + rm *.deb &&\ + apt update &&\ + apt install -y puppet-agent + +RUN wget https://apt.puppet.com/puppet-tools-release-xenial.deb && \ + dpkg -i puppet-tools-release-xenial.deb && \ + apt-get update && \ + apt-get install -y pdk && \ + rm -f *.deb + +WORKDIR /etc/puppetlabs/code/modules/splunk_otel_collector +COPY ./Rakefile ./Gemfile ./Gemfile.lock ./ +RUN gem install bundler && bundle install + +RUN mkdir -p /root/.config/puppet && \ + echo "---\n\ +disabled: true" > /root/.config/puppet/analytics.yml + +ENV PATH=/opt/puppetlabs/bin:$PATH + +COPY ./ ./ diff --git a/deployments/puppet/Gemfile b/deployments/puppet/Gemfile new file mode 100644 index 0000000000..ed4388f323 --- /dev/null +++ b/deployments/puppet/Gemfile @@ -0,0 +1,127 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +def location_for(place_or_version, fake_version = nil) + if place_or_version =~ %r{\A(git[:@][^#]*)#(.*)} + [fake_version, { git: Regexp.last_match(1), branch: Regexp.last_match(2), require: false }].compact + elsif place_or_version =~ %r{\Afile:\/\/(.*)} + ['>= 0', { path: File.expand_path(Regexp.last_match(1)), require: false }] + else + [place_or_version, { require: false }] + end +end + +def gem_type(place_or_version) + if place_or_version =~ %r{\Agit[:@]} + :git + elsif !place_or_version.nil? && place_or_version.start_with?('file:') + :file + else + :gem + end +end + +ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments +minor_version = ruby_version_segments[0..1].join('.') + +group :development do + gem "fast_gettext", '1.1.0', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0') + gem "fast_gettext", require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0') + gem "json_pure", '<= 2.0.1', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0') + gem "json", '>= 2.3.0', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9') + gem "puppet-module-posix-default-r#{minor_version}", require: false, platforms: [:ruby] + gem "puppet-module-posix-dev-r#{minor_version}", require: false, platforms: [:ruby] + gem "puppet-module-win-default-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "puppet-module-win-dev-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "rspec_junit_formatter" +end + +puppet_version = ENV['PUPPET_GEM_VERSION'] +puppet_type = gem_type(puppet_version) +facter_version = ENV['FACTER_GEM_VERSION'] +hiera_version = ENV['HIERA_GEM_VERSION'] + +def puppet_older_than?(version) + puppet_version = ENV['PUPPET_GEM_VERSION'] + !puppet_version.nil? && + Gem::Version.correct?(puppet_version) && + Gem::Requirement.new("< #{version}").satisfied_by?(Gem::Version.new(puppet_version.dup)) +end + +gems = {} + +gems['puppet'] = location_for(puppet_version) + +# If facter or hiera versions have been specified via the environment +# variables, use those versions. If not, and if the puppet version is < 3.5.0, +# use known good versions of both for puppet < 3.5.0. +if facter_version + gems['facter'] = location_for(facter_version) +elsif puppet_type == :gem && puppet_older_than?('3.5.0') + gems['facter'] = ['>= 1.6.11', '<= 1.7.5', require: false] +end + +if hiera_version + gems['hiera'] = location_for(ENV['HIERA_GEM_VERSION']) +elsif puppet_type == :gem && puppet_older_than?('3.5.0') + gems['hiera'] = ['>= 1.0.0', '<= 1.3.0', require: false] +end + +if Gem.win_platform? && (puppet_type != :gem || puppet_older_than?('3.5.0')) + # For Puppet gems < 3.5.0 (tested as far back as 3.0.0) on Windows + if puppet_type == :gem + gems['ffi'] = ['1.9.0', require: false] + gems['minitar'] = ['0.5.4', require: false] + gems['win32-eventlog'] = ['0.5.3', '<= 0.6.5', require: false] + gems['win32-process'] = ['0.6.5', '<= 0.7.5', require: false] + gems['win32-security'] = ['~> 0.1.2', '<= 0.2.5', require: false] + gems['win32-service'] = ['0.7.2', '<= 0.8.8', require: false] + else + gems['ffi'] = ['~> 1.9.0', require: false] + gems['minitar'] = ['~> 0.5.4', require: false] + gems['win32-eventlog'] = ['~> 0.5', '<= 0.6.5', require: false] + gems['win32-process'] = ['~> 0.6', '<= 0.7.5', require: false] + gems['win32-security'] = ['~> 0.1', '<= 0.2.5', require: false] + gems['win32-service'] = ['~> 0.7', '<= 0.8.8', require: false] + end + + gems['win32-dir'] = ['~> 0.3', '<= 0.4.9', require: false] + + if RUBY_VERSION.start_with?('1.') + gems['win32console'] = ['1.3.2', require: false] + # sys-admin was removed in Puppet 3.7.0 and doesn't compile under Ruby 2.x + gems['sys-admin'] = ['1.5.6', require: false] + end + + # Puppet < 3.7.0 requires these. + # Puppet >= 3.5.0 gem includes these as requirements. + # The following versions are tested to work with 3.0.0 <= puppet < 3.7.0. + gems['win32-api'] = ['1.4.8', require: false] + gems['win32-taskscheduler'] = ['0.2.2', require: false] + gems['windows-api'] = ['0.4.3', require: false] + gems['windows-pr'] = ['1.2.3', require: false] +elsif Gem.win_platform? + # If we're using a Puppet gem on Windows which handles its own win32-xxx gem + # dependencies (>= 3.5.0), set the maximum versions (see PUP-6445). + gems['win32-dir'] = ['<= 0.4.9', require: false] + gems['win32-eventlog'] = ['<= 0.6.5', require: false] + gems['win32-process'] = ['<= 0.7.5', require: false] + gems['win32-security'] = ['<= 0.2.5', require: false] + gems['win32-service'] = ['<= 0.8.8', require: false] +end + +gems.each do |gem_name, gem_params| + gem gem_name, *gem_params +end + +# Evaluate Gemfile.local and ~/.gemfile if they exist +extra_gemfiles = [ + "#{__FILE__}.local", + File.join(Dir.home, '.gemfile'), +] + +extra_gemfiles.each do |gemfile| + if File.file?(gemfile) && File.readable?(gemfile) + eval(File.read(gemfile), binding) + end +end +# vim: syntax=ruby diff --git a/deployments/puppet/Gemfile.lock b/deployments/puppet/Gemfile.lock new file mode 100644 index 0000000000..e0f10934ed --- /dev/null +++ b/deployments/puppet/Gemfile.lock @@ -0,0 +1,172 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + ast (2.4.1) + coderay (1.1.3) + concurrent-ruby (1.1.6) + deep_merge (1.2.1) + diff-lcs (1.4.4) + docile (1.3.2) + domain_name (0.5.20190701) + unf (>= 0.0.5, < 1.0.0) + facter (4.0.30) + hocon (~> 1.3) + thor (>= 1.0.1, < 2.0) + facterdb (1.2.0) + facter + jgrep + fast_gettext (1.8.0) + hiera (3.6.0) + hocon (1.3.1) + http-accept (1.7.0) + http-cookie (1.0.3) + domain_name (~> 0.5) + httpclient (2.8.3) + jgrep (1.5.2) + json (2.3.1) + json-schema (2.8.1) + addressable (>= 2.4) + locale (2.1.3) + metaclass (0.0.4) + metadata-json-lint (2.4.0) + json-schema (~> 2.8) + spdx-licenses (~> 1.0) + method_source (1.0.0) + mime-types (3.3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2020.0512) + mocha (1.1.0) + metaclass (~> 0.0.1) + multi_json (1.15.0) + net-scp (3.0.0) + net-ssh (>= 2.6.5, < 7.0.0) + net-ssh (4.2.0) + net-telnet (0.2.0) + netrc (0.11.0) + parallel (1.19.2) + parallel_tests (2.32.0) + parallel + parser (2.7.1.4) + ast (~> 2.4.1) + pathspec (0.2.1) + powerpack (0.1.2) + pry (0.13.1) + coderay (~> 1.1) + method_source (~> 1.0) + public_suffix (4.0.5) + puppet (6.17.0) + concurrent-ruby (~> 1.0) + deep_merge (~> 1.0) + facter (> 2.0.1, < 5) + fast_gettext (~> 1.1) + hiera (>= 3.2.1, < 4) + httpclient (~> 2.8) + locale (~> 2.1) + multi_json (~> 1.10) + puppet-resource_api (~> 1.5) + semantic_puppet (~> 1.0) + puppet-blacksmith (4.1.2) + rest-client (~> 2.0) + puppet-lint (2.4.2) + puppet-module-posix-default-r2.3 (0.5.0) + puppet-module-posix-dev-r2.3 (0.0.8) + metadata-json-lint + mocha (< 1.2.0) + parallel_tests + pry + puppet-blacksmith (>= 3.4.0) + puppet-lint + puppetlabs_spec_helper (>= 1.2.1) + rainbow (< 2.2.0) + rspec-puppet (>= 2.3.2) + rspec-puppet-facts + rspec_junit_formatter (~> 0.2) + rubocop + rubocop-rspec (~> 1.15) + simplecov + specinfra (= 2.67.3) + puppet-resource_api (1.8.13) + hocon (>= 1.0) + puppet-syntax (2.6.1) + puppet (>= 5) + rake + puppetlabs_spec_helper (2.15.0) + mocha (~> 1.0) + pathspec (~> 0.2.1) + puppet-lint (~> 2.0) + puppet-syntax (>= 2.0, < 4) + rspec-puppet (~> 2.0) + rainbow (2.1.0) + rake (13.0.1) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + rspec (3.9.0) + rspec-core (~> 3.9.0) + rspec-expectations (~> 3.9.0) + rspec-mocks (~> 3.9.0) + rspec-core (3.9.2) + rspec-support (~> 3.9.3) + rspec-expectations (3.9.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.9.0) + rspec-mocks (3.9.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.9.0) + rspec-puppet (2.7.10) + rspec + rspec-puppet-facts (1.10.0) + facter + facterdb (>= 0.5.0) + json + puppet + rspec-support (3.9.3) + rspec_junit_formatter (0.4.1) + rspec-core (>= 2, < 4, != 2.12.0) + rubocop (0.49.1) + parallel (~> 1.10) + parser (>= 2.3.3.1, < 3.0) + powerpack (~> 0.1) + rainbow (>= 1.99.1, < 3.0) + ruby-progressbar (~> 1.7) + unicode-display_width (~> 1.0, >= 1.0.1) + rubocop-rspec (1.16.0) + rubocop (>= 0.49.0) + ruby-progressbar (1.10.1) + semantic_puppet (1.0.2) + sfl (2.3) + simplecov (0.17.1) + docile (~> 1.1) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.2) + spdx-licenses (1.2.0) + specinfra (2.67.3) + net-scp + net-ssh (>= 2.7, < 5.0) + net-telnet + sfl + thor (1.0.1) + unf (0.1.4) + unf_ext + unf_ext (0.0.7.7) + unicode-display_width (1.7.0) + +PLATFORMS + ruby + +DEPENDENCIES + fast_gettext + puppet + puppet-module-posix-default-r2.3 + puppet-module-posix-dev-r2.3 + puppet-module-win-default-r2.3 + puppet-module-win-dev-r2.3 + rspec_junit_formatter + +BUNDLED WITH + 2.1.4 diff --git a/deployments/puppet/Makefile b/deployments/puppet/Makefile new file mode 100644 index 0000000000..df39ce8219 --- /dev/null +++ b/deployments/puppet/Makefile @@ -0,0 +1,32 @@ +.PHONY: dev-image +dev-image: + docker build -t splunk-otel-connector-puppet-dev . + +.PHONY: run-dev-image +run-dev-image: + docker exec -it splunk-otel-collector-puppet-dev bash 2>/dev/null || \ + docker run \ + --rm \ + --name splunk-otel-collector-puppet-dev \ + -it \ + -v $(CURDIR):/etc/puppetlabs/code/modules/splunk_otel_collector \ + splunk-otel-connector-puppet-dev \ + bash + +.PHONY: lint +lint: dev-image + docker run \ + --rm \ + splunk-otel-connector-puppet-dev \ + puppet-lint --fail-on-warnings . + +.PHONY: rake-spec +rake-spec: dev-image + docker run \ + --rm \ + splunk-otel-connector-puppet-dev \ + rake spec + +.PHONY: release +release: + ./release diff --git a/deployments/puppet/README.md b/deployments/puppet/README.md new file mode 100644 index 0000000000..8fae54f156 --- /dev/null +++ b/deployments/puppet/README.md @@ -0,0 +1,65 @@ +# Splunk OpenTelemetry Connector Puppet Module + +This is a Puppet module that will install and configure the Splunk +OpenTelemetry Connector. + +Currently, the following Linux distributions and versions are supported: + +- Amazon Linux: 2 +- CentOS / Red Hat / Oracle: 7, 8 +- Debian: 8, 9, 10 +- Ubuntu: 16.04, 18.04, 20.04 + +> Note: `systemd` is required to be installed on the host for service +> management. + +To use this module, include the `splunk_otel_collector` class in your +manifests. For example, the simplest deployment definition with the +default parameters would be (replace `SPLUNK_ACCESS_TOKEN` with your Splunk +access token to authenticate requests and `SPLUNK_REALM` for the realm to send +the data to): + +```ruby +class { splunk_otel_collector: + splunk_access_token => 'SPLUNK_ACCESS_TOKEN', + splunk_realm => 'SPLUNK_REALM', +} +``` + +This class accepts the following parameters: + +| Name | Description | Default value | +| :--- | :---------- | :------------ | +| `splunk_access_token` | **Required**: The Splunk access token to authenticate requests. | None | +| `splunk_realm` | **Required**: Which realm to send the data to, e.g. `us0`. The Splunk ingest and API URLs will be inferred by this value. The `SPLUNK_REALM` environment variable will be set with this value for the collector service. | None | +| `splunk_ingest_url` | Set the Splunk ingest URL explicitly instead of the URL inferred by the `$splunk_realm` parameter. The `SPLUNK_INGEST_URL` environment variable will be set with this value for the collector service. | `https://ingest.${splunk_realm}.signalfx.com` | +| `splunk_api_url` | Set the Splunk API URL explicitly instead of the URL inferred by the `$splunk_realm` parameter. The `SPLUNK_API_URL` environment variable will be set with this value for the collector service. | `https://api.${splunk_realm}.signalfx.com` | +| `splunk_trace_url` | Set the Splunk trace endpoint URL explicitly instead of the URL inferred by the `$splunk_ingest_url` parameter. The `SPLUNK_TRACE_URL` environment variable will be set with this value for the collector service. | `${splunk_ingest_url}/v2/trace` | +| `splunk_hec_url` | Set the Splunk HEC endpoint URL explicitly instead of the URL inferred by the `$splunk_ingest_url` parameter. The `SPLUNK_HEC_URL` environment variable will be set with this value for the collector service. | `${splunk_ingest_url}/v1/log` | +| `splunk_hec_token` | Set the Splunk HEC authentication token if different than `$splunk_access_token`. The `SPLUNK_HEC_TOKEN` environment variable will be set with this value for the collector service. | `$splunk_access_token` | +| `splunk_bundle_dir` | The path to the [Smart Agent bundle directory](https://github.com/signalfx/splunk-otel-collector/blob/main/internal/extension/smartagentextension/README.md). The default path is provided by the collector package. If the specified path is changed from the default value, the path should be an existing directory on the node. The `SPLUNK_BUNDLE_DIR` environment variable will be set to this value for the collector service. | `/usr/lib/splunk-otel-collector/agent-bundle` | +| `splunk_collectd_dir` | The path to the collectd config directory for the Smart Agent bundle. The default path is provided by the collector package. If the specified path is changed from the default value, the path should be an existing directory on the node. The `SPLUNK_COLLECTD_DIR` environment variable will be set to this value for the collector service. | `${splunk_bundle_dir}/run/collectd` | +| `splunk_memory_total_mib` | Total memory in MIB to allocate to the collector; automatically calculates the ballast size. The `SPLUNK_MEMORY_TOTAL_MIB` environment variable will be set with this value for the collector service. | `512` | +| `splunk_ballast_size_mib` | Set the ballast size for the collector explicitly instead of the value calculated from the `$splunk_memory_total_mib` parameter. This should be set to 1/3 to 1/2 of configured memory. The `SPLUNK_BALLAST_SIZE_MIB` environment variable will be set with this value for the collector service. | None | +| `collector_config_source` | Source path to the collector config YAML file. This file will be copied to the `$collector_config_dest` path on the node. See the [source attribute](https://puppet.com/docs/puppet/latest/types/file.html#file-attribute-source) of the `file` resource for supported value types. The default source file is provided by the collector package. | `file:///etc/otel/collector/agent_config.yaml` | +| `collector_config_dest` | Destination path of the collector config file on the node. The `SPLUNK_CONFIG` environment variable will be set with this value for the collector service. | `/etc/otel/collector/agent_config.yaml` | +| `collector_version` | Version of the collector package to install, e.g., `0.25.0`. | `latest` | +| `service_user` and `$service_group` | **Linux only**: Set the user/group ownership for the collector service. The user/group will be created if they do not exist. | `splunk-otel-collector` | +| `with_fluentd` | Whether to install/manage fluentd and dependencies for log collection. The dependencies include [capng_c](https://github.com/fluent-plugins-nursery/capng_c) for enabling [Linux capabilities](https://docs.fluentd.org/deployment/linux-capability), [fluent-plugin-systemd](https://github.com/fluent-plugin-systemd/fluent-plugin-systemd) for systemd journal log collection, and the [required libraries/development tools](https://github.com/signalfx/splunk-otel-collector/blob/main/docs/getting-started/linux-installer.md#fluentd-configuration). | `true` | +| `fluentd_config_source` | Source path to the fluentd config file. This file will be copied to the `$fluentd_config_dest` path on the node. See the [source attribute](https://puppet.com/docs/puppet/latest/types/file.html#file-attribute-source) of the `file` resource for supported value types. The default source file is provided by the collector package. | `file:///etc/otel/collector/fluentd/fluent.conf` | +| `fluentd_config_dest` | Destination path to the fluentd config file on the node. | `/etc/otel/collector/fluentd/fluent.conf` | +| `manage_repo` | **Linux only**: In cases where the collector and fluentd apt/yum repositories are managed externally, set this to `false` to disable management of the repositories by this module. The repositories should provide the `splunk-otel-collector` and `td-agent` packages. **Note:** If set to `false`, the externally managed repositories should provide the `splunk-otel-collector` and `td-agent` packages. Also, the apt (`/etc/apt/sources.list.d/splunk-otel-collector.list`, `/etc/apt/sources.list.d/splunk-td-agent.list`) and yum (`/etc/yum.repos.d/splunk-otel-collector.repo`, `/etc/yum.repos.d/splunk-td-agent.repo`) repository definition files will be deleted if they exist in order to avoid any conflicts. | `true` | + +## Dependencies + +On Linux-based systems, the +[puppetlabs/stdlib](https://forge.puppet.com/puppetlabs/stdlib) module is +required. + +On Debian-based systems, the +[puppetlabs/apt](https://forge.puppet.com/puppetlabs/apt) module is required to +manage the collector and fluentd apt repositories. + +On RPM-based systems, the +[puppet/yum](https://forge.puppet.com/puppet/yum) module is required to +install the "Development Tools" package group as a dependency for fluentd. diff --git a/deployments/puppet/RELEASE.md b/deployments/puppet/RELEASE.md new file mode 100644 index 0000000000..1a3be96d89 --- /dev/null +++ b/deployments/puppet/RELEASE.md @@ -0,0 +1,11 @@ +# Release Process + +To release a new version of the module, run `./release` in this directory. You +will need access to the SignalFx account on the Puppet Forge website, and the +release script will give you instructions for what to do there. + +You should update the version in `metadata.json` to whatever is most appropriate +for semver and have that committed before running `./release`. + +The release script will try to make and push an annotated tag of the form +`puppet-vX.Y.Z` where `X.Y.Z` is the version in the `./metadata.json` file. diff --git a/deployments/puppet/Rakefile b/deployments/puppet/Rakefile new file mode 100644 index 0000000000..bdc83207bf --- /dev/null +++ b/deployments/puppet/Rakefile @@ -0,0 +1,13 @@ +require 'rspec' +require 'rspec-puppet/rake_task' +require 'puppetlabs_spec_helper/rake_tasks' + +begin + if Gem::Specification::find_by_name('puppet-lint') + require 'puppet-lint/tasks/puppet-lint' + PuppetLint.configuration.ignore_paths = ["spec/**/*.pp", "vendor/**/*.pp"] + task :default => [:rspec, :lint] + end +rescue Gem::LoadError + task :default => :rspec +end diff --git a/deployments/puppet/lib/facter/local_groups.rb b/deployments/puppet/lib/facter/local_groups.rb new file mode 100644 index 0000000000..afe9f1dcd6 --- /dev/null +++ b/deployments/puppet/lib/facter/local_groups.rb @@ -0,0 +1,15 @@ +# This custom fact pulls out all local groups from the /etc/group file +# and returns the collection as a comma-separated list. + +Facter.add(:local_groups) do + setcode do + groups = Array.new + if File.exists?("/etc/group") + File.open("/etc/group").each do |line| + next if line.match(/^\s|^#|^$/) + groups << line.split(':').first + end + end + groups.join(',') + end +end diff --git a/deployments/puppet/lib/facter/local_users.rb b/deployments/puppet/lib/facter/local_users.rb new file mode 100644 index 0000000000..d998b6a3cb --- /dev/null +++ b/deployments/puppet/lib/facter/local_users.rb @@ -0,0 +1,15 @@ +# This custom fact pulls out all local users from the /etc/passwd file +# and returns the collection as a comma-separated list. + +Facter.add(:local_users) do + setcode do + users = Array.new + if File.exists?("/etc/passwd") + File.open("/etc/passwd").each do |line| + next if line.match(/^\s|^#|^$/) + users << line.split(':').first + end + end + users.join(',') + end +end diff --git a/deployments/puppet/manifests/collector_debian_repo.pp b/deployments/puppet/manifests/collector_debian_repo.pp new file mode 100644 index 0000000000..2ca6ba8e2e --- /dev/null +++ b/deployments/puppet/manifests/collector_debian_repo.pp @@ -0,0 +1,19 @@ +# Installs the collector debian package repository config +class splunk_otel_collector::collector_debian_repo ($repo_url, $package_stage, $repo, $apt_gpg_key, $manage_repo) { + + if $manage_repo { + apt::source { 'splunk-otel-collector': + location => $repo_url, + release => $package_stage, + repos => $repo, + key => { + id => '58C33310B7A354C1279DB6695EFA01EDB3CD4420', + source => $apt_gpg_key, + }, + } + } else { + file { '/etc/apt/sources.list.d/splunk-otel-collector.list': + ensure => absent, + } + } +} diff --git a/deployments/puppet/manifests/collector_service_owner.pp b/deployments/puppet/manifests/collector_service_owner.pp new file mode 100644 index 0000000000..79b2c49323 --- /dev/null +++ b/deployments/puppet/manifests/collector_service_owner.pp @@ -0,0 +1,101 @@ +# Sets the user/group for the splunk-otel-collector service. +# If the user or group does not exist, they will be created. +class splunk_otel_collector::collector_service_owner ($service_name, $service_user, $service_group) { + + if !defined(Group[$service_group]) { + if $service_group == 'splunk-otel-collector' or $service_group in split($::local_groups, ',') { + group { $service_group: + noop => true, + } + } + else { + group { $service_group: + ensure => present, + system => true, + } + } + } + + if !defined(User[$service_user]) { + if $service_user == 'splunk-otel-collector' or $service_user in split($::local_users, ',') { + user { $service_user: + noop => true, } + } + else { + $shell = $::osfamily ? { + 'debian' => '/usr/sbin/nologin', + default => '/sbin/nologin', + } + user { $service_user: + ensure => present, + system => true, + shell => $shell, + groups => $service_group, + } + } + } + + case $::service_provider { + 'systemd': { + $tmpfile_path = "/etc/tmpfiles.d/${service_name}.conf" + $tmpfile_dir = $tmpfile_path.split('/')[0, - 2].join('/') + + $override_path = "/etc/systemd/system/${service_name}.service.d/service-owner.conf" + $override_dir = $override_path.split('/')[0, - 2].join('/') + + Package[$service_name] ~> Group[$service_group] ~> User[$service_user] + + ~> exec { 'systemctl stop splunk-otel-collector': + path => '/bin:/sbin:/usr/bin:/usr/sbin', + refreshonly => true, + } + + ~> file { [$tmpfile_dir, $override_dir]: + ensure => directory, + } + + ~> file { + $tmpfile_path: + ensure => file, + content => "D /run/${service_name} 0755 ${service_user} ${service_group} - -", + ; + $override_path: + ensure => file, + ; + } + + ~> file_line { + $override_path: + path => $override_path, + line => '[Service]', + match => '^[Service]', + ; + 'set-service-user': + path => $override_path, + line => "User=${service_user}", + match => '^User=', + after => '^[Service]', + require => File_Line[$override_path], + ; + 'set-service-group': + path => $override_path, + line => "Group=${service_group}", + match => '^Group=', + after => '^User=', + require => File_Line['set-service-user'], + ; + } + + ~> exec { ["systemd-tmpfiles --create --remove ${tmpfile_path}", 'systemctl daemon-reload']: + path => '/bin:/sbin:/usr/bin:/usr/sbin', + returns => [0], + refreshonly => true, + } + + ~> Service[$service_name] + } + default: { + fail('Only systemd is currently supported') + } + } +} diff --git a/deployments/puppet/manifests/collector_yum_repo.pp b/deployments/puppet/manifests/collector_yum_repo.pp new file mode 100644 index 0000000000..8b4d28ab52 --- /dev/null +++ b/deployments/puppet/manifests/collector_yum_repo.pp @@ -0,0 +1,22 @@ +# Installs the collector yum package repostitory for the given stage +class splunk_otel_collector::collector_yum_repo ($repo_url, $yum_gpg_key, $manage_repo) { + + if $manage_repo { + file { '/etc/yum.repos.d/splunk-otel-collector.repo': + content => @("EOH") + [splunk-otel-collector] + name=Splunk OpenTelemetry Collector + baseurl=${repo_url} + gpgcheck=1 + gpgkey=${yum_gpg_key} + enabled=1 + | EOH + , + mode => '0644', + } + } else { + file { '/etc/yum.repos.d/splunk-otel-collector.repo': + ensure => absent, + } + } +} diff --git a/deployments/puppet/manifests/fluentd_debian_repo.pp b/deployments/puppet/manifests/fluentd_debian_repo.pp new file mode 100644 index 0000000000..688112e086 --- /dev/null +++ b/deployments/puppet/manifests/fluentd_debian_repo.pp @@ -0,0 +1,26 @@ +# Installs the fluentd debian package repository config +class splunk_otel_collector::fluentd_debian_repo ($repo_url, $gpg_key_url, $version, $manage_repo) { + $distro = downcase($facts['os']['distro']['id']) + $codename = downcase($facts['os']['distro']['codename']) + $major_version = $version.split('\.')[0] + + if $manage_repo { + if $distro != 'ubuntu' and $distro != 'debian' { + fail("Your distribution '${distro}' is not currently supported") + } + + apt::source { 'splunk-td-agent': + location => "${repo_url}/${major_version}/${distro}/${codename}", + release => $codename, + repos => 'contrib', + key => { + id => 'BEE682289B2217F45AF4CC3F901F9177AB97ACBE', + source => $gpg_key_url, + }, + } + } else { + file { '/etc/apt/sources.list.d/splunk-td-agent.list': + ensure => 'absent', + } + } +} diff --git a/deployments/puppet/manifests/fluentd_yum_repo.pp b/deployments/puppet/manifests/fluentd_yum_repo.pp new file mode 100644 index 0000000000..3371eb7148 --- /dev/null +++ b/deployments/puppet/manifests/fluentd_yum_repo.pp @@ -0,0 +1,29 @@ +# Installs the fluentd yum package repostitory +class splunk_otel_collector::fluentd_yum_repo ($repo_url, $gpg_key_url, $version, $manage_repo) { + + if $manage_repo { + $os_name = $facts['os']['name'] ? { + 'Amazon' => 'amazon', + default => 'redhat', + } + $major_version = $version.split('\.')[0] + $url = "${repo_url}/${major_version}/${os_name}/\$releasever/\$basearch" + + file { '/etc/yum.repos.d/splunk-td-agent.repo': + content => @("EOH") + [td-agent] + name=TreasureData Repository + baseurl=${url} + gpgcheck=1 + gpgkey=${gpg_key_url} + enabled=1 + | EOH + , + mode => '0644', + } + } else { + file { '/etc/yum.repos.d/splunk-td-agent.repo': + ensure => 'absent', + } + } +} diff --git a/deployments/puppet/manifests/init.pp b/deployments/puppet/manifests/init.pp new file mode 100644 index 0000000000..15365e11bc --- /dev/null +++ b/deployments/puppet/manifests/init.pp @@ -0,0 +1,255 @@ +# Main class that installs and configures the agent +class splunk_otel_collector ( + $splunk_access_token = '', # required + $splunk_realm = '', # required + $splunk_ingest_url = "https://ingest.${splunk_realm}.signalfx.com", + $splunk_api_url = "https://api.${splunk_realm}.signalfx.com", + $splunk_trace_url = "${splunk_ingest_url}/v2/trace", + $splunk_hec_url = "${splunk_ingest_url}/v1/log", + $splunk_hec_token = $splunk_access_token, + $splunk_bundle_dir = '/usr/lib/splunk-otel-collector/agent-bundle', + $splunk_collectd_dir = "${splunk_bundle_dir}/run/collectd", + $splunk_memory_total_mib = '512', + $splunk_ballast_size_mib = '', + $collector_config_source = 'file:///etc/otel/collector/agent_config.yaml', + $collector_config_dest = '/etc/otel/collector/agent_config.yaml', + $collector_version = 'latest', + $package_stage = 'release', # collector package repository stage: release, beta, or test + $apt_repo_url = 'https://splunk.jfrog.io/splunk/otel-collector-deb', + $apt_repo = 'main', + $yum_repo_url = "https://splunk.jfrog.io/splunk/otel-collector-rpm/${package_stage}/\$basearch", + $service_user = 'splunk-otel-collector', # linux only + $service_group = 'splunk-otel-collector', # linux only + $apt_gpg_key = 'https://splunk.jfrog.io/splunk/otel-collector-deb/splunk-B3CD4420.gpg', + $yum_gpg_key = 'https://splunk.jfrog.io/splunk/otel-collector-rpm/splunk-B3CD4420.pub', + $with_fluentd = true, + $fluentd_repo_base = 'https://packages.treasuredata.com', + $fluentd_gpg_key = 'https://packages.treasuredata.com/GPG-KEY-td-agent', + $fluentd_version = '4.1.0', + $fluentd_version_jessie = '3.3.0-1', + $fluentd_version_stretch = '3.7.1-0', + $fluentd_config_source = 'file:///etc/otel/collector/fluentd/fluent.conf', + $fluentd_config_dest = '/etc/otel/collector/fluentd/fluent.conf', + $fluentd_capng_c_version = '<=0.2.2', + $fluentd_systemd_version = '<=1.0.2', + $manage_repo = true # linux only +) { + + if empty($splunk_access_token) { + fail('The splunk_access_token parameter is required') + } + + if empty($splunk_realm) { + fail('The splunk_realm parameter is required') + } + + if $::osfamily == 'windows' { + fail('Windows is not currently supported') + } else { + if $facts['service_provider'] != 'systemd' { + fail('Only systemd is currently supported') + } + $collector_config_dir = $collector_config_dest.split('/')[0, - 2].join('/') + } + + $collector_service_name = 'splunk-otel-collector' + + case $::osfamily { + 'debian': { + class { 'splunk_otel_collector::collector_debian_repo': + repo_url => $apt_repo_url, + package_stage => $package_stage, + repo => $apt_repo, + apt_gpg_key => $apt_gpg_key, + manage_repo => $manage_repo, + } + -> package { $collector_service_name: + ensure => $collector_version, + require => Exec['apt_update'], + } + } + 'redhat': { + class { 'splunk_otel_collector::collector_yum_repo': + repo_url => $yum_repo_url, + yum_gpg_key => $yum_gpg_key, + manage_repo => $manage_repo, + } + -> package { $collector_service_name: + ensure => $collector_version, + } + } + default: { + fail("Your OS (${::osfamily}) is not supported by the Splunk OpenTelemetry Connector") + } + } + + if $::osfamily != 'windows' { + $env_file_path = '/etc/otel/collector/splunk-otel-collector.conf' + $env_file_content = @("EOH") + SPLUNK_ACCESS_TOKEN=${splunk_access_token} + SPLUNK_API_URL=${splunk_api_url} + SPLUNK_BALLAST_SIZE_MIB=${splunk_ballast_size_mib} + SPLUNK_BUNDLE_DIR=${splunk_bundle_dir} + SPLUNK_COLLECTD_DIR=${splunk_collectd_dir} + SPLUNK_CONFIG=${collector_config_dest} + SPLUNK_HEC_TOKEN=${splunk_hec_token} + SPLUNK_HEC_URL=${splunk_hec_url} + SPLUNK_INGEST_URL=${splunk_ingest_url} + SPLUNK_MEMORY_TOTAL_MIB=${splunk_memory_total_mib} + SPLUNK_REALM=${splunk_realm} + SPLUNK_TRACE_URL=${splunk_trace_url} + | EOH + + class { 'splunk_otel_collector::collector_service_owner': + service_name => $collector_service_name, + service_user => $service_user, + service_group => $service_group, + } + + -> file { $env_file_path: + content => $env_file_content, + mode => '0600', + owner => $service_user, + group => $service_group, + } + + exec { 'create collector config directory': + command => "mkdir -p ${collector_config_dir}", + path => ['/bin', '/sbin', '/usr/bin', '/usr/sbin'], + unless => "test -d ${collector_config_dir}", + } + + -> file { $collector_config_dest: + source => $collector_config_source, + require => Package[$collector_service_name], + } + + service { $collector_service_name: + ensure => true, + enable => true, + subscribe => [Package[$collector_service_name], File[$collector_config_dest, $env_file_path]], + } + } + + if $with_fluentd { + $fluentd_service_name = 'td-agent' + $fluentd_config_dir = $fluentd_config_dest.split('/')[0, - 2].join('/') + $fluentd_config_override = '/etc/systemd/system/td-agent.service.d/splunk-otel-collector.conf' + $fluentd_config_override_dir= $fluentd_config_override.split('/')[0, - 2].join('/') + + case $::osfamily { + 'debian': { + package { ['build-essential', 'libcap-ng0', 'libcap-ng-dev', 'pkg-config']: + ensure => 'installed', + require => Exec['apt_update'], + } + case downcase($facts['os']['distro']['codename']) { + 'jessie': { + $version = $fluentd_version_jessie + } + 'stretch': { + $version = $fluentd_version_stretch + } + default: { + $version = "${fluentd_version}-1" + } + } + class { 'splunk_otel_collector::fluentd_debian_repo': + repo_url => $fluentd_repo_base, + gpg_key_url => $fluentd_gpg_key, + version => $version, + manage_repo => $manage_repo, + } + -> package { $fluentd_service_name: + ensure => $version, + require => Exec['apt_update'], + } + package { 'capng_c': + ensure => $fluentd_capng_c_version, + provider => gem, + command => '/usr/sbin/td-agent-gem', + require => Package[$fluentd_service_name, 'build-essential', 'libcap-ng0', 'libcap-ng-dev', 'pkg-config'], + } + package { 'fluent-plugin-systemd': + ensure => $fluentd_systemd_version, + provider => gem, + command => '/usr/sbin/td-agent-gem', + require => Package[$fluentd_service_name, 'build-essential', 'libcap-ng0', 'libcap-ng-dev', 'pkg-config'], + } + } + 'redhat': { + class { 'splunk_otel_collector::fluentd_yum_repo': + repo_url => $fluentd_repo_base, + gpg_key_url => $fluentd_gpg_key, + version => $fluentd_version, + manage_repo => $manage_repo, + } + -> package { $fluentd_service_name: + ensure => $fluentd_version, + } + yum::group { 'Development Tools': + ensure => 'present', + } + package { ['libcap-ng', 'libcap-ng-devel', 'pkgconfig']: + ensure => 'installed', + } + package { 'capng_c': + ensure => $fluentd_capng_c_version, + provider => gem, + command => '/usr/sbin/td-agent-gem', + require => [Package[$fluentd_service_name, 'libcap-ng', 'libcap-ng-devel', 'pkgconfig'], + Yum::Group['Development Tools']], + } + package { 'fluent-plugin-systemd': + ensure => $fluentd_systemd_version, + provider => gem, + command => '/usr/sbin/td-agent-gem', + require => [Package[$fluentd_service_name, 'libcap-ng', 'libcap-ng-devel', 'pkgconfig'], + Yum::Group['Development Tools']], + } + } + default: { + fail("Your OS (${::osfamily}) is not supported by the Splunk OpenTelemetry Connector") + } + } + + exec { 'create fluentd config directory': + command => "mkdir -p ${fluentd_config_dir}", + path => ['/bin', '/sbin', '/usr/bin', '/usr/sbin'], + unless => "test -d ${fluentd_config_dir}", + } + + -> file { $fluentd_config_dest: + source => $fluentd_config_source, + require => Package[$collector_service_name], + } + + file { $fluentd_config_override_dir: + ensure => 'directory', + } + + -> file { $fluentd_config_override: + content => @("EOH") + [Service] + Environment=FLUENT_CONF=${fluentd_config_dest} + | EOH + , + require => File[$fluentd_config_dest], + } + + # enable linux capabilities for fluentd + exec { 'fluent-cap-ctl': + command => '/opt/td-agent/bin/fluent-cap-ctl --add "dac_override,dac_read_search" -f /opt/td-agent/bin/ruby', + path => ['/bin', '/sbin', '/usr/bin', '/usr/sbin'], + require => Package['capng_c'], + onlyif => 'test -f /opt/td-agent/bin/fluent-cap-ctl', + } + + -> service { $fluentd_service_name: + ensure => true, + enable => true, + require => [Package[$fluentd_service_name, 'fluent-plugin-systemd'], Service[$collector_service_name]], + subscribe => File[$fluentd_config_dest, $fluentd_config_override], + } + } +} diff --git a/deployments/puppet/metadata.json b/deployments/puppet/metadata.json new file mode 100644 index 0000000000..f2c9fd6da0 --- /dev/null +++ b/deployments/puppet/metadata.json @@ -0,0 +1,68 @@ +{ + "name": "signalfx-splunk_otel_collector", + "version": "0.0.1", + "author": "Splunk, Inc.", + "summary": "This module installs the Splunk OpenTelemetry Connector via distro packages and configures it.", + "license": "Apache-2.0", + "source": "https://github.com/signalfx/splunk-otel-collector", + "project_page": "https://github.com/signalfx/splunk-otel-collector", + "issues_url": "https://github.com/signalfx/splunk-otel-collector/issues", + "dependencies": [ + { + "name": "puppetlabs/apt", + "version_requirement": ">= 4.1.0 <= 7.0.0" + }, + { + "name": "puppet/yum", + "version_requirement": "<= 4.3.0" + } + ], + "operatingsystem_support": [ + { + "operatingsystem": "CentOS", + "operatingsystemrelease": [ + "7", + "8" + ] + }, + { + "operatingsystem": "OracleLinux", + "operatingsystemrelease": [ + "7", + "8" + ] + }, + { + "operatingsystem": "RedHat", + "operatingsystemrelease": [ + "7", + "8" + ] + }, + { + "operatingsystem": "Debian", + "operatingsystemrelease": [ + "8", + "9", + "10" + ] + }, + { + "operatingsystem": "Ubuntu", + "operatingsystemrelease": [ + "16.04", + "18.04", + "20.04" + ] + } + ], + "requirements": [ + { + "name": "puppet", + "version_requirement": ">= 6.0.0" + } + ], + "pdk-version": "1.4.1", + "template-url": "file:///opt/puppetlabs/pdk/share/cache/pdk-templates.git", + "template-ref": "1.4.1-0-g52adbbb" +} diff --git a/deployments/puppet/release b/deployments/puppet/release new file mode 100755 index 0000000000..c1564ea02a --- /dev/null +++ b/deployments/puppet/release @@ -0,0 +1,51 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PUSH_TAG=${PUSH_TAG:-yes} + +version_from_metadata() { + jq -r '.version' $SCRIPT_DIR/metadata.json +} + +run_docker_command() { + docker run \ + --rm \ + -v $SCRIPT_DIR:/etc/puppetlabs/code/modules/splunk_otel_collector \ + splunk-otel-connector-puppet-dev \ + $@ +} + +current_version_exists() { + curl https://forgeapi.puppetlabs.com:443/v3/modules/signalfx-splunk_otel_collector | \ + jq -r '.releases[].version' | \ + grep $(version_from_metadata) > /dev/null || false +} + +if current_version_exists; then + echo "Version $(version_from_metadata) already exists in Puppet Forge. Please update the metadata.json with a new version" >&2 + exit 1 +fi + +if [ "$PUSH_TAG" = "yes" ]; then + tag=puppet-v$(version_from_metadata) + if ! git rev-parse $tag; then + git tag -a $tag + git push origin $tag + fi +fi + +make -C $SCRIPT_DIR dev-image + +run_docker_command pdk build + +expected_output=$SCRIPT_DIR/pkg/signalfx-splunk_otel_collector-$(version_from_metadata).tar.gz +if ! test -e $expected_output; then + echo "Module could not be built, expected output at $expected_output" >&2 + exit 1 +fi + +echo "Module was successfully built and is available in ./pkg/$(basename $expected_output)" +echo +echo "You must manually upload this file in the Puppet forge web UI by following the directions at https://puppet.com/docs/puppet/5.4/modules_publishing.html#upload-a-module-to-the-forge" diff --git a/deployments/puppet/spec/classes/init_spec.rb b/deployments/puppet/spec/classes/init_spec.rb new file mode 100644 index 0000000000..0f34a028a1 --- /dev/null +++ b/deployments/puppet/spec/classes/init_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +describe 'splunk_otel_collector' do + let(:title) { 'splunk_otel_collector' } + let(:params) { { 'splunk_access_token' => '' } } + + it "fails without access token" do + is_expected.to compile.and_raise_error(/splunk_access_token/) + end + + on_supported_os.each do |os, facts| + if os.include? "windows" + next + end + context "on #{os}" do + let(:params) { { 'splunk_access_token' => "testing", 'splunk_realm' => 'test' } } + let(:facts) do + facts + end + + it { is_expected.to compile.with_all_deps } + end + end +end diff --git a/deployments/puppet/spec/spec_helper.rb b/deployments/puppet/spec/spec_helper.rb new file mode 100644 index 0000000000..9bf07e128d --- /dev/null +++ b/deployments/puppet/spec/spec_helper.rb @@ -0,0 +1,15 @@ +require 'rspec-puppet' +require 'rspec-puppet-facts' +require 'puppetlabs_spec_helper/module_spec_helper' + +include RspecPuppetFacts + +fixture_path = File.join(File.dirname(File.expand_path(__FILE__)), 'fixtures') + +RSpec.configure do |c| + c.module_path = File.join(fixture_path, 'modules') + c.manifest_dir = File.join(fixture_path, 'manifests') + c.manifest = File.join(fixture_path, 'manifests', 'site.pp') + c.environmentpath = File.join(Dir.pwd, 'spec') + c.default_facts = { :osfamily => 'redhat', :service_provider => 'systemd' } +end diff --git a/deployments/puppet/templates/collector_config.yaml.erb b/deployments/puppet/templates/collector_config.yaml.erb new file mode 100644 index 0000000000..60c23831e0 --- /dev/null +++ b/deployments/puppet/templates/collector_config.yaml.erb @@ -0,0 +1,5 @@ +# Automatically generated by Puppet + +<% @collector_config.sort.each do |key, value| %> +<%= key %>: <%= value.to_json %> +<% end %> diff --git a/internal/buildscripts/packaging/tests/deployments/__init__.py b/internal/buildscripts/packaging/tests/deployments/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/__init__.py b/internal/buildscripts/packaging/tests/deployments/puppet/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-buster b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-buster new file mode 100644 index 0000000000..df6a2fca59 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-buster @@ -0,0 +1,37 @@ +FROM debian:buster + +RUN apt-get update &&\ + apt-get install -yq ca-certificates procps systemd wget apt-transport-https libcap2-bin curl gnupg lsb-release + +ARG PUPPET_RELEASE="6" +RUN wget https://apt.puppetlabs.com/puppet${PUPPET_RELEASE}-release-buster.deb && \ + dpkg -i puppet${PUPPET_RELEASE}-release-buster.deb && \ + apt-get update && \ + apt-get install -y puppet-agent + +ENV PATH=/opt/puppetlabs/bin:$PATH + +ENV container docker + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +# Keep it from wiping our scratch dir in /tmp/scratch +RUN rm -f /usr/lib/tmpfiles.d/tmp.conf; + +RUN systemctl set-default multi-user.target +ENV init /lib/systemd/systemd + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppetlabs-apt --version 7.0.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +VOLUME [ "/sys/fs/cgroup" ] + +CMD ["/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-jessie b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-jessie new file mode 100644 index 0000000000..9d91d84ed5 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-jessie @@ -0,0 +1,37 @@ +FROM debian:jessie + +RUN apt-get update &&\ + apt-get install -yq ca-certificates procps systemd wget apt-transport-https libcap2-bin curl gnupg lsb-release + +ARG PUPPET_RELEASE="6" +RUN wget https://apt.puppetlabs.com/puppet${PUPPET_RELEASE}-release-jessie.deb && \ + dpkg -i puppet${PUPPET_RELEASE}-release-jessie.deb && \ + apt-get update && \ + apt-get install -y puppet-agent + +ENV PATH=/opt/puppetlabs/bin:$PATH + +ENV container docker + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +# Keep it from wiping our scratch dir in /tmp/scratch +RUN rm -f /usr/lib/tmpfiles.d/tmp.conf; + +RUN systemctl set-default multi-user.target +ENV init /lib/systemd/systemd + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppetlabs-apt --version 7.0.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +VOLUME [ "/sys/fs/cgroup" ] + +CMD ["/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-stretch b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-stretch new file mode 100644 index 0000000000..09603750f7 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.debian-stretch @@ -0,0 +1,37 @@ +FROM debian:stretch + +RUN apt-get update &&\ + apt-get install -yq ca-certificates procps systemd wget apt-transport-https libcap2-bin curl gnupg lsb-release + +ARG PUPPET_RELEASE="6" +RUN wget https://apt.puppetlabs.com/puppet${PUPPET_RELEASE}-release-stretch.deb && \ + dpkg -i puppet${PUPPET_RELEASE}-release-stretch.deb && \ + apt-get update && \ + apt-get install -y puppet-agent + +ENV PATH=/opt/puppetlabs/bin:$PATH + +ENV container docker + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +# Keep it from wiping our scratch dir in /tmp/scratch +RUN rm -f /usr/lib/tmpfiles.d/tmp.conf; + +RUN systemctl set-default multi-user.target +ENV init /lib/systemd/systemd + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppetlabs-apt --version 7.0.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +VOLUME [ "/sys/fs/cgroup" ] + +ENTRYPOINT ["/lib/systemd/systemd"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-bionic b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-bionic new file mode 100644 index 0000000000..f1e64e53a0 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-bionic @@ -0,0 +1,33 @@ +FROM ubuntu:18.04 + +RUN apt-get update &&\ + apt-get install -yq ca-certificates procps systemd wget apt-transport-https libcap2-bin curl gnupg lsb-release + +ARG PUPPET_RELEASE="6" +RUN wget https://apt.puppetlabs.com/puppet${PUPPET_RELEASE}-release-bionic.deb && \ + dpkg -i puppet${PUPPET_RELEASE}-release-bionic.deb && \ + apt-get update && \ + apt-get install -y puppet-agent + +ENV PATH=/opt/puppetlabs/bin:$PATH + +ENV container docker + +RUN rm -f /lib/systemd/system/multi-user.target.wants/* \ + /etc/systemd/system/*.wants/* \ + /lib/systemd/system/local-fs.target.wants/* \ + /lib/systemd/system/sockets.target.wants/*udev* \ + /lib/systemd/system/sockets.target.wants/*initctl* \ + /lib/systemd/system/systemd-update-utmp* + +RUN systemctl set-default multi-user.target +ENV init /lib/systemd/systemd + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppetlabs-apt --version 7.0.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +VOLUME [ "/sys/fs/cgroup" ] + +ENTRYPOINT ["/lib/systemd/systemd"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-focal b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-focal new file mode 100644 index 0000000000..eb52244a8b --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-focal @@ -0,0 +1,33 @@ +FROM ubuntu:20.04 + +RUN apt-get update &&\ + apt-get install -yq ca-certificates procps systemd wget apt-transport-https libcap2-bin curl gnupg lsb-release + +ARG PUPPET_RELEASE="6" +RUN wget https://apt.puppetlabs.com/puppet${PUPPET_RELEASE}-release-focal.deb && \ + dpkg -i puppet${PUPPET_RELEASE}-release-focal.deb && \ + apt-get update && \ + apt-get install -y puppet-agent + +ENV PATH=/opt/puppetlabs/bin:$PATH + +ENV container docker + +RUN rm -f /lib/systemd/system/multi-user.target.wants/* \ + /etc/systemd/system/*.wants/* \ + /lib/systemd/system/local-fs.target.wants/* \ + /lib/systemd/system/sockets.target.wants/*udev* \ + /lib/systemd/system/sockets.target.wants/*initctl* \ + /lib/systemd/system/systemd-update-utmp* + +RUN systemctl set-default multi-user.target +ENV init /lib/systemd/systemd + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppetlabs-apt --version 7.0.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +VOLUME [ "/sys/fs/cgroup" ] + +ENTRYPOINT ["/lib/systemd/systemd"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-xenial b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-xenial new file mode 100644 index 0000000000..dad5c6e792 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/deb/Dockerfile.ubuntu-xenial @@ -0,0 +1,33 @@ +FROM ubuntu:16.04 + +RUN apt-get update &&\ + apt-get install -yq ca-certificates procps systemd wget apt-transport-https libcap2-bin curl gnupg lsb-release + +ARG PUPPET_RELEASE="6" +RUN wget https://apt.puppetlabs.com/puppet${PUPPET_RELEASE}-release-xenial.deb && \ + dpkg -i puppet${PUPPET_RELEASE}-release-xenial.deb && \ + apt-get update && \ + apt-get install -y puppet-agent + +ENV PATH=/opt/puppetlabs/bin:$PATH + +ENV container docker + +RUN rm -f /lib/systemd/system/multi-user.target.wants/* \ + /etc/systemd/system/*.wants/* \ + /lib/systemd/system/local-fs.target.wants/* \ + /lib/systemd/system/sockets.target.wants/*udev* \ + /lib/systemd/system/sockets.target.wants/*initctl* \ + /lib/systemd/system/systemd-update-utmp* + +RUN systemctl set-default multi-user.target +ENV init /lib/systemd/systemd + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppetlabs-apt --version 7.0.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +VOLUME [ "/sys/fs/cgroup" ] + +ENTRYPOINT ["/lib/systemd/systemd"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.amazonlinux-2 b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.amazonlinux-2 new file mode 100644 index 0000000000..7615a4d09a --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.amazonlinux-2 @@ -0,0 +1,29 @@ +FROM amazonlinux:2 + +ENV container docker + +RUN yum install -y systemd procps initscripts + +ARG PUPPET_RELEASE="6" +RUN rpm -Uvh https://yum.puppet.com/puppet${PUPPET_RELEASE}-release-el-7.noarch.rpm && \ + yum install -y puppet-agent + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +ENV PATH=/opt/puppetlabs/bin:$PATH + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppet-yum --version 4.3.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-7 b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-7 new file mode 100644 index 0000000000..ffd2dad33c --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-7 @@ -0,0 +1,29 @@ +FROM centos:7 + +ENV container docker + +RUN yum install -y systemd procps initscripts + +ARG PUPPET_RELEASE="6" +RUN rpm -Uvh https://yum.puppet.com/puppet${PUPPET_RELEASE}-release-el-7.noarch.rpm && \ + yum install -y puppet-agent + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +ENV PATH=/opt/puppetlabs/bin:$PATH + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppet-yum --version 4.3.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-8 b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-8 new file mode 100644 index 0000000000..10d25f0c74 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.centos-8 @@ -0,0 +1,29 @@ +FROM centos:8 + +ENV container docker + +RUN dnf install -y systemd procps initscripts + +ARG PUPPET_RELEASE="6" +RUN rpm -Uvh https://yum.puppet.com/puppet${PUPPET_RELEASE}-release-el-8.noarch.rpm && \ + dnf install -y puppet-agent + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +ENV PATH=/opt/puppetlabs/bin:$PATH + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppet-yum --version 4.3.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-7 b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-7 new file mode 100644 index 0000000000..fe8eb18146 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-7 @@ -0,0 +1,29 @@ +FROM oraclelinux:7 + +ENV container docker + +RUN yum install -y systemd procps initscripts + +ARG PUPPET_RELEASE="6" +RUN rpm -Uvh https://yum.puppet.com/puppet${PUPPET_RELEASE}-release-el-7.noarch.rpm && \ + yum install -y puppet-agent + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +ENV PATH=/opt/puppetlabs/bin:$PATH + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppet-yum --version 4.3.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-8 b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-8 new file mode 100644 index 0000000000..c8ffc1aa89 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/images/rpm/Dockerfile.oraclelinux-8 @@ -0,0 +1,29 @@ +FROM oraclelinux:8 + +ENV container docker + +RUN dnf install -y systemd procps initscripts + +ARG PUPPET_RELEASE="6" +RUN rpm -Uvh https://yum.puppet.com/puppet${PUPPET_RELEASE}-release-el-8.noarch.rpm && \ + dnf install -y puppet-agent + +RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i = \ + "systemd-tmpfiles-setup.service" ] || rm -f $i; done); \ + rm -f /lib/systemd/system/multi-user.target.wants/*;\ + rm -f /lib/systemd/system/local-fs.target.wants/*; \ + rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ + rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ + rm -f /lib/systemd/system/basic.target.wants/*;\ + rm -f /lib/systemd/system/anaconda.target.wants/*; + +VOLUME [ "/sys/fs/cgroup" ] + +ENV PATH=/opt/puppetlabs/bin:$PATH + +RUN puppet module install puppetlabs-stdlib --version 4.24.0 +RUN puppet module install puppet-yum --version 4.3.0 + +COPY deployments/puppet /etc/puppetlabs/code/environments/production/modules/splunk_otel_collector + +CMD ["/usr/sbin/init"] diff --git a/internal/buildscripts/packaging/tests/deployments/puppet/puppet_test.py b/internal/buildscripts/packaging/tests/deployments/puppet/puppet_test.py new file mode 100644 index 0000000000..b387ac1859 --- /dev/null +++ b/internal/buildscripts/packaging/tests/deployments/puppet/puppet_test.py @@ -0,0 +1,134 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import string +import tempfile + +from pathlib import Path + +import pytest + +from tests.helpers.util import ( + copy_file_into_container, + run_container_cmd, + run_distro_container, + service_is_running, + wait_for, + REPO_DIR, + SERVICE_NAME, + SERVICE_OWNER, +) + + +IMAGES_DIR = Path(__file__).parent.resolve() / "images" +DEB_DISTROS = [df.split(".")[-1] for df in glob.glob(str(IMAGES_DIR / "deb" / "Dockerfile.*"))] +RPM_DISTROS = [df.split(".")[-1] for df in glob.glob(str(IMAGES_DIR / "rpm" / "Dockerfile.*"))] +CONFIG_DIR = "/etc/otel/collector" +SPLUNK_ENV_PATH = f"{CONFIG_DIR}/splunk-otel-collector.conf" +SPLUNK_ACCESS_TOKEN = "testing123" +SPLUNK_REALM = "test" +SPLUNK_INGEST_URL = f"https://ingest.{SPLUNK_REALM}.signalfx.com" +SPLUNK_API_URL = f"https://api.{SPLUNK_REALM}.signalfx.com" + +CONFIG = string.Template( + f""" +class {{ splunk_otel_collector: + splunk_access_token => '{SPLUNK_ACCESS_TOKEN}', + splunk_realm => '{SPLUNK_REALM}', + collector_version => '$collector_version', + with_fluentd => $with_fluentd, +}} +""" +) + + +def run_puppet_apply(container, config): + with tempfile.NamedTemporaryFile(mode="w+") as fd: + print(config) + fd.write(config) + fd.flush() + copy_file_into_container(container, fd.name, "/root/test.pp") + + run_container_cmd(container, "puppet apply /root/test.pp") + + +def verify_env_file(container): + run_container_cmd(container, f"grep '^SPLUNK_ACCESS_TOKEN={SPLUNK_ACCESS_TOKEN}$' {SPLUNK_ENV_PATH}") + run_container_cmd(container, f"grep '^SPLUNK_API_URL={SPLUNK_API_URL}$' {SPLUNK_ENV_PATH}") + run_container_cmd(container, f"grep '^SPLUNK_HEC_TOKEN={SPLUNK_ACCESS_TOKEN}$' {SPLUNK_ENV_PATH}") + run_container_cmd(container, f"grep '^SPLUNK_HEC_URL={SPLUNK_INGEST_URL}/v1/log$' {SPLUNK_ENV_PATH}") + run_container_cmd(container, f"grep '^SPLUNK_INGEST_URL={SPLUNK_INGEST_URL}$' {SPLUNK_ENV_PATH}") + run_container_cmd(container, f"grep '^SPLUNK_REALM={SPLUNK_REALM}$' {SPLUNK_ENV_PATH}") + run_container_cmd(container, f"grep '^SPLUNK_TRACE_URL={SPLUNK_INGEST_URL}/v2/trace$' {SPLUNK_ENV_PATH}") + + +@pytest.mark.installer +@pytest.mark.parametrize( + "distro", + [pytest.param(distro, marks=pytest.mark.deb) for distro in DEB_DISTROS] + + [pytest.param(distro, marks=pytest.mark.rpm) for distro in RPM_DISTROS], + ) +@pytest.mark.parametrize("puppet_release", ["6", "7"]) +def test_puppet(distro, puppet_release): + if "jessie" in distro and puppet_release != "6": + pytest.skip(f"Puppet release version {puppet_release} not supported on debian jessie") + + if distro in DEB_DISTROS: + dockerfile = IMAGES_DIR / "deb" / f"Dockerfile.{distro}" + else: + dockerfile = IMAGES_DIR / "rpm" / f"Dockerfile.{distro}" + + buildargs = {"PUPPET_RELEASE": puppet_release} + with run_distro_container(distro, dockerfile=dockerfile, path=REPO_DIR, buildargs=buildargs) as container: + try: + for collector_version in ["0.24.0", "latest"]: + config = CONFIG.substitute(collector_version=collector_version, with_fluentd="true") + run_puppet_apply(container, config) + verify_env_file(container) + assert wait_for(lambda: service_is_running(container)) + assert container.exec_run("systemctl status td-agent").exit_code == 0 + finally: + run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager") + run_container_cmd(container, "journalctl -u td-agent --no-pager") + if container.exec_run("test -f /var/log/td-agent/td-agent.log").exit_code == 0: + run_container_cmd(container, "cat /var/log/td-agent/td-agent.log") + + +@pytest.mark.installer +@pytest.mark.parametrize( + "distro", + [pytest.param(distro, marks=pytest.mark.deb) for distro in DEB_DISTROS] + + [pytest.param(distro, marks=pytest.mark.rpm) for distro in RPM_DISTROS], + ) +@pytest.mark.parametrize("puppet_release", ["6", "7"]) +def test_puppet_without_fluentd(distro, puppet_release): + if "jessie" in distro and puppet_release != "6": + pytest.skip(f"Puppet release version {puppet_release} not supported on debian jessie") + + if distro in DEB_DISTROS: + dockerfile = IMAGES_DIR / "deb" / f"Dockerfile.{distro}" + else: + dockerfile = IMAGES_DIR / "rpm" / f"Dockerfile.{distro}" + + buildargs = {"PUPPET_RELEASE": puppet_release} + with run_distro_container(distro, dockerfile=dockerfile, path=REPO_DIR, buildargs=buildargs) as container: + try: + config = CONFIG.substitute(collector_version="latest", with_fluentd="false") + run_puppet_apply(container, config) + verify_env_file(container) + assert wait_for(lambda: service_is_running(container)) + assert container.exec_run("systemctl status td-agent").exit_code != 0 + finally: + run_container_cmd(container, f"journalctl -u {SERVICE_NAME} --no-pager") diff --git a/internal/buildscripts/packaging/tests/helpers/util.py b/internal/buildscripts/packaging/tests/helpers/util.py index e98ec27a66..52a0f7170c 100644 --- a/internal/buildscripts/packaging/tests/helpers/util.py +++ b/internal/buildscripts/packaging/tests/helpers/util.py @@ -1,3 +1,17 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import glob import os import tarfile @@ -28,18 +42,19 @@ def retry(function, exception, max_attempts=5, interval=5): @contextmanager -def run_distro_container(distro, timeout=DEFAULT_TIMEOUT): +def run_distro_container(distro, dockerfile=None, path=TESTS_DIR, buildargs=None, timeout=DEFAULT_TIMEOUT): client = docker.from_env() - assert distro in DEB_DISTROS + RPM_DISTROS, f"'{distro}' distro not supported!" + if not dockerfile: + if distro in DEB_DISTROS: + dockerfile = TESTS_DIR / "images" / "deb" / f"Dockerfile.{distro}" + else: + dockerfile = TESTS_DIR / "images" / "rpm" / f"Dockerfile.{distro}" - if distro in DEB_DISTROS: - dockerfile = TESTS_DIR / "images" / "deb" / f"Dockerfile.{distro}" - else: - dockerfile = TESTS_DIR / "images" / "rpm" / f"Dockerfile.{distro}" + assert os.path.isfile(str(dockerfile)), f"{dockerfile} not found!" image, _ = retry( - lambda: client.images.build(path=str(TESTS_DIR), dockerfile=str(dockerfile), pull=True, rm=True, forcerm=True), + lambda: client.images.build(path=str(path), dockerfile=str(dockerfile), pull=True, rm=True, forcerm=True, buildargs=buildargs), docker.errors.BuildError, )