From 8d45490140122d4b5d34382f3a155565e313bd55 Mon Sep 17 00:00:00 2001 From: Mason Morales Date: Tue, 20 Oct 2020 08:02:56 -0700 Subject: [PATCH] First commit --- README.md | 163 +++++++++++++++++- environments/development/group_vars/all.yml | 6 + environments/development/inventory.yml | 10 ++ environments/production/group_vars/all.yml | 7 + .../production/group_vars/clustermaster.yml | 7 + .../group_vars/deploymentserver.yml | 13 ++ .../group_vars/universalforwarder.yml | 6 + .../production/host_vars/my-shc-deployer.yml | 5 + environments/production/inventory.yml | 57 ++++++ playbooks/ansible.cfg | 27 +++ playbooks/splunk_app_install.yml | 8 + playbooks/splunk_install_or_upgrade.yml | 9 + playbooks/splunk_upgrade_full_stack.yml | 67 +++++++ roles/splunk/.yamllint | 33 ++++ roles/splunk/README.md | 35 ++++ roles/splunk/defaults/main.yml | 54 ++++++ roles/splunk/files/disable-thp.service | 9 + roles/splunk/files/rsync-filter | 9 + roles/splunk/files/splunk.secret | 1 + roles/splunk/files/splunk_ulimits.conf | 4 + roles/splunk/handlers/main.yml | 71 ++++++++ roles/splunk/meta/main.yml | 9 + roles/splunk/tasks/add_crashlog_script.yml | 18 ++ roles/splunk/tasks/add_diag_script.yml | 18 ++ roles/splunk/tasks/add_pstack_script.yml | 9 + roles/splunk/tasks/adhoc_clean_dispatch.yml | 10 ++ .../splunk/tasks/adhoc_configure_hostname.yml | 25 +++ roles/splunk/tasks/adhoc_decom_indexer.yml | 7 + roles/splunk/tasks/adhoc_fix_mongo.yml | 27 +++ .../tasks/adhoc_fix_server_certificate.yml | 12 ++ roles/splunk/tasks/adhoc_kill_splunkd.yml | 4 + roles/splunk/tasks/check_splunk.yml | 54 ++++++ roles/splunk/tasks/configure_apps.yml | 97 +++++++++++ .../splunk/tasks/configure_authentication.yml | 12 ++ roles/splunk/tasks/configure_bash.yml | 11 ++ .../tasks/configure_deploymentclient.yml | 12 ++ roles/splunk/tasks/configure_facl.yml | 33 ++++ roles/splunk/tasks/configure_license.yml | 14 ++ roles/splunk/tasks/configure_os.yml | 17 ++ roles/splunk/tasks/configure_splunk_boot.yml | 12 ++ .../tasks/configure_splunk_forwarder_meta.yml | 29 ++++ .../splunk/tasks/configure_splunk_secret.yml | 9 + roles/splunk/tasks/configure_thp.yml | 34 ++++ roles/splunk/tasks/configure_user-seed.yml | 20 +++ roles/splunk/tasks/download_and_unarchive.yml | 30 ++++ roles/splunk/tasks/install_apps.yml | 50 ++++++ roles/splunk/tasks/install_splunk.yml | 60 +++++++ roles/splunk/tasks/install_utilities.yml | 11 ++ roles/splunk/tasks/main.yml | 32 ++++ roles/splunk/tasks/post_install.yml | 32 ++++ roles/splunk/tasks/set_maintenance_mode.yml | 8 + roles/splunk/tasks/set_upgrade_state.yml | 8 + roles/splunk/tasks/slack_messenger.yml | 15 ++ roles/splunk/tasks/splunk_offline.yml | 6 + roles/splunk/tasks/splunk_restart.yml | 5 + roles/splunk/tasks/splunk_start.yml | 5 + roles/splunk/tasks/splunk_stop.yml | 5 + roles/splunk/tasks/upgrade_splunk.yml | 26 +++ roles/splunk/templates/authentication.conf.j2 | 4 + roles/splunk/templates/bash_profile.j2 | 14 ++ roles/splunk/templates/bashrc.j2 | 18 ++ .../splunk/templates/cleanup_crashlogs.sh.j2 | 4 + roles/splunk/templates/cleanup_diags.sh.j2 | 4 + .../splunk/templates/deploymentclient.conf.j2 | 5 + roles/splunk/templates/genpstacks.sh.j2 | 28 +++ roles/splunk/templates/splunk_facl.j2 | 5 + roles/splunk/templates/user-seed.conf.j2 | 3 + roles/splunk/vars/Debian.yml | 2 + roles/splunk/vars/RedHat.yml | 3 + roles/splunk/vars/main.yml | 8 + 70 files changed, 1481 insertions(+), 4 deletions(-) create mode 100644 environments/development/group_vars/all.yml create mode 100644 environments/development/inventory.yml create mode 100644 environments/production/group_vars/all.yml create mode 100644 environments/production/group_vars/clustermaster.yml create mode 100644 environments/production/group_vars/deploymentserver.yml create mode 100644 environments/production/group_vars/universalforwarder.yml create mode 100644 environments/production/host_vars/my-shc-deployer.yml create mode 100644 environments/production/inventory.yml create mode 100644 playbooks/ansible.cfg create mode 100644 playbooks/splunk_app_install.yml create mode 100644 playbooks/splunk_install_or_upgrade.yml create mode 100644 playbooks/splunk_upgrade_full_stack.yml create mode 100644 roles/splunk/.yamllint create mode 100644 roles/splunk/README.md create mode 100644 roles/splunk/defaults/main.yml create mode 100644 roles/splunk/files/disable-thp.service create mode 100644 roles/splunk/files/rsync-filter create mode 100644 roles/splunk/files/splunk.secret create mode 100644 roles/splunk/files/splunk_ulimits.conf create mode 100644 roles/splunk/handlers/main.yml create mode 100644 roles/splunk/meta/main.yml create mode 100644 roles/splunk/tasks/add_crashlog_script.yml create mode 100644 roles/splunk/tasks/add_diag_script.yml create mode 100644 roles/splunk/tasks/add_pstack_script.yml create mode 100644 roles/splunk/tasks/adhoc_clean_dispatch.yml create mode 100644 roles/splunk/tasks/adhoc_configure_hostname.yml create mode 100644 roles/splunk/tasks/adhoc_decom_indexer.yml create mode 100644 roles/splunk/tasks/adhoc_fix_mongo.yml create mode 100644 roles/splunk/tasks/adhoc_fix_server_certificate.yml create mode 100644 roles/splunk/tasks/adhoc_kill_splunkd.yml create mode 100644 roles/splunk/tasks/check_splunk.yml create mode 100644 roles/splunk/tasks/configure_apps.yml create mode 100644 roles/splunk/tasks/configure_authentication.yml create mode 100644 roles/splunk/tasks/configure_bash.yml create mode 100644 roles/splunk/tasks/configure_deploymentclient.yml create mode 100644 roles/splunk/tasks/configure_facl.yml create mode 100644 roles/splunk/tasks/configure_license.yml create mode 100644 roles/splunk/tasks/configure_os.yml create mode 100644 roles/splunk/tasks/configure_splunk_boot.yml create mode 100644 roles/splunk/tasks/configure_splunk_forwarder_meta.yml create mode 100644 roles/splunk/tasks/configure_splunk_secret.yml create mode 100644 roles/splunk/tasks/configure_thp.yml create mode 100644 roles/splunk/tasks/configure_user-seed.yml create mode 100644 roles/splunk/tasks/download_and_unarchive.yml create mode 100644 roles/splunk/tasks/install_apps.yml create mode 100644 roles/splunk/tasks/install_splunk.yml create mode 100644 roles/splunk/tasks/install_utilities.yml create mode 100644 roles/splunk/tasks/main.yml create mode 100644 roles/splunk/tasks/post_install.yml create mode 100644 roles/splunk/tasks/set_maintenance_mode.yml create mode 100644 roles/splunk/tasks/set_upgrade_state.yml create mode 100644 roles/splunk/tasks/slack_messenger.yml create mode 100644 roles/splunk/tasks/splunk_offline.yml create mode 100644 roles/splunk/tasks/splunk_restart.yml create mode 100644 roles/splunk/tasks/splunk_start.yml create mode 100644 roles/splunk/tasks/splunk_stop.yml create mode 100644 roles/splunk/tasks/upgrade_splunk.yml create mode 100644 roles/splunk/templates/authentication.conf.j2 create mode 100644 roles/splunk/templates/bash_profile.j2 create mode 100644 roles/splunk/templates/bashrc.j2 create mode 100644 roles/splunk/templates/cleanup_crashlogs.sh.j2 create mode 100644 roles/splunk/templates/cleanup_diags.sh.j2 create mode 100644 roles/splunk/templates/deploymentclient.conf.j2 create mode 100755 roles/splunk/templates/genpstacks.sh.j2 create mode 100644 roles/splunk/templates/splunk_facl.j2 create mode 100644 roles/splunk/templates/user-seed.conf.j2 create mode 100644 roles/splunk/vars/Debian.yml create mode 100644 roles/splunk/vars/RedHat.yml create mode 100644 roles/splunk/vars/main.yml diff --git a/README.md b/README.md index 9c43b725..8b0cbd54 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,161 @@ -# ansible-role-for-splunk -Ansible playbook repository for installing Splunk, upgrading Splunk, and installing apps/addons to Splunk +# ansible-role-for-splunk: An Ansible role for Splunk admins -## Coming Soon! ## +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)  +[![GitHub release](https://img.shields.io/github/v/tag/splunk/ansible-role-for-splunk?sort=semver&label=Version)](https://github.com/splunk/ansible-role-for-splunk/releases) -Check back for this release at .conf20 on October 20th and be sure to checkout [the related session](https://conf.splunk.com/learn/session-catalog.html?search=TRU1537C) +This repository contains an official Ansible role from Splunk that can manage Splunk Enterprise and Splunk Universal Forwarders on Linux-based platforms. Example playbooks and inventory files are also provided to help new Ansible users make the most out of this project. + +ansible-role-for-splunk is used by the Splunk@Splunk team to manage Splunk's corporate deployment of Splunk. + +---- + +## Table of Contents + +1. [Purpose](#purpose) +1. [Getting Started](#gettingstarted) +1. [Extended Documentation](#gettingstarted) +1. [Frequently Asked Questions](#faq) +1. [Support](#support) +1. [Contributing](#contributing) +1. [License](#license) + +---- + +## Purpose + +#### What is ansible-role-for-splunk? +ansible-role-for-splunk is a single Ansible role for managing Splunk deployments remotely over SSH. It supports all Splunk deployment roles (Universal Forwarder, Heavy Forwarder, Indexer, Search Head, Deployment Server, Cluster Master, SHC Deployer) as well as management of all apps and configurations (via git repositories). + +This codebase is used by the Splunk@Splunk team internally to manage our deployment, so it has been thoroughly vetted. For more information, checkout [our related .conf20 session](https://conf.splunk.com/learn/session-catalog.html?search=TRU1537C) for this project. + +#### Design Philosophy +A few different design philosophies have been applied in the development of this project. + +First, ansible-role-for-splunk was designed under the "Don't Repeat Yourself (DRY)" philosophy. This means that the project contains minimal code redundancy. If you want to fork this project and change any functionality, you only need to update the code in one place. + +Second, ansible-role-for-splunk was designed to be idempotent. This means that if the system is already in the desired state that Ansible expects, it will not make any changes. This even applies to our app management code. For example, if you want to upgrade an app on a search head, and your repository does not contain a local/ folder, Ansible will not touch the existing local/ folder on the search head. This is accomplished using the synchronize module. For more information on that, refer to the configure_apps.yml task description. + +Third, ansible-role-for-splunk was designed to manage all Splunk configurations as code. What do I mean by that? You're not going to find tasks for installing web certificates, disabling the REST port, templating indexes, or managing every Splunk configuration possible. Instead, you will find that we have a generic configure_apps.yml task which can deploy any version of any git repository to any path under $SPLUNK_HOME on the hosts in your inventory. We believe that having all configurations in git repositories is the best way to perform version control and configuration management for Splunk deployments. That said, we've made two exceptions: +1. Creation of the local splunk admin user. We are able to do this securely using ansible-vault to create the user-seed.conf during the initial installation, so we support that functionality, should you choose to use it. We are aware that this is not the only way to manage local user accounts in Splunk, so this functionality is disabled by default. +1. Configuring deploymentclient.conf for Deployment Server (DS) clients. We realize that some environments may have hundreds of clientNames configured and that creating a git repository for each variation would be pretty inefficient. Therefore, we support configuring deploymentclient.conf for your Ansible-managed forwarders using variables. The current version is based on a single template that supports only the clientName and targetUri keys. However, this can be easily extended with additional variables (or static content) of your choosing. + +## Getting Started +Getting started with this role will requires you to: +1. Have Ansible installed (minimum v2.7 but this role is compatible with newer versions through v2.11) +1. Setup your inventory correctly +1. Configure the appropriate variables to describe the desired state of your environment +1. Create a playbook or leverage one of the included example playbooks + +#### Ansible Setup +Ansible only needs to be installed on the host that you want to use to manage your Splunk deployments. We recommend having a dedicated server that is used only for Ansible orchestration, but technically you can run Ansible from any host, including your laptop, as long as you have the network connectivity required to SSH into the Splunk hosts that you want Ansible to manage. +* [Ansible Installation Guide](https://docs.ansible.com/ansible/devel/installation_guide/index.html) +* [Ansible User Guide](https://docs.ansible.com/ansible/devel/user_guide/index.html#getting-started) + +#### Inventory +The layout of your inventory is critical to the tasks included in ansible-role-for-splunk. The "role" of your host is determined by it being a member of one or more inventory groups that define its Splunk role. The following group names are currently supported: +* full +* uf +* clustermaster +* deploymentserver +* indexer +* licensemaster +* search +* shdeployer + +Note that in Ansible you may nest groups within groups, and we depend on this heavily to differentiate a full Splunk installation vs a Univerasl Forwarder (UF) installation. You will see examples of this within the sample inventory.yml files included in the "environments" folder of this project. + +You may also specify additional groups for provide further layers of abstraction nested within the aforementioned required groups. e.g. full -> indexer -> cluster_a | cluster_b | cluster_c + +#### Variables +As proper usage of this role requires a thorough understanding of variables, familiarity with [Ansible variable precedence](https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html#ansible-variable-precedence) is highly recommended. Almost all variables used in this role have been added to defaults/main.yml (lowest precendence) for reference. Default values of "unconfigured" are automatically ignored at the task level. + +A number of variables ship with this role, but many of them automatically configure themselves when the play is executed. For example, during the upgrade check, the desired version of Splunk that you want to be at is based solely upon the value of splunk_package_url_full or splunk_package_url_uf. We extract the version and build numbers from the URL automagically, and then compare those to the output of the "splunk version" command during the check_splunk.yml task to determine if an upgrade is required or not. + +That said, there are a few variables that you'll definitely need to configure to use this role with your environment: + splunk_uri_lm - The URI for your license master (e.g. https://my_license_master:8089) + ansible_user - The username that you want Ansible to connect as for SSH access + ansible_ssh_private_key_file - The file path to the private key that the Ansible user should use for SSH access authentication + +In addition, you may want to configure some of the optional variables that are mentioned in roles/splunk/defaults/main.yml to manage things like splunk.secret, send Slack notifications, automatically install useful scripts, additional Linux packages, etc. + +In order to use the app management functionality, you will need to configure the following additional variables: + git_server: ssh://git@git.mydomain.com + git_key: ~/.ssh/mygit.key + git_project: FOO + git_version: bar + git_apps: + - name: myapp + +You will find additional examples in the sample group_vars and host_vars files. Note that you may also specify git_server, git_key, git_project, and git_version within git_apps down to the repository (name) level. +Tip: If you only use one git server, you may want to define the git_server and related values in an all.yml group_var. + +**Configure admin user's password at install** + splunk_user_seed: true + splunk_admin_password: yourpassword + +**Note:** If you do not configure these 2 variables, new Splunk installations will be installed without an admin account present. This has no impact on upgrades to existing installations. + +#### Playbooks +The following example playbooks have been included in this project for your reference: +- **splunk_app_install.yml** - Install or upgrade apps on Splunk hosts using the configure_apps.yml task in the splunk role. Note that the apps you want to deploy should be defined in either host_vars or group_vars, along with a splunk_app_deploy_path. Refer to the documentation for app deployment for details. +- **splunk_install_or_upgrade.yml** - Install or upgrade Splunk (or Splunk UFs) on hosts using the check_splunk.yml task in the splunk role. +- **splunk_upgrade_full_stack.yml** - Example playbook that demonstrates how to upgrade an entire Splunk deployment with a single-site indexer cluster and a search head cluster using the splunk role. Note: This playbook does not upgrade forwarders, although you could easily add an extra play to do that. + +## Extended Documentation +This section contains additional reference documentation. +---- +#### Task File Descriptions + +- **add_crashlog_script.yml** - Installs a bash script and cron job that will automatically clean-up splunkd crash log files. By default, every night at midnight, it will find any crash logs that are more than 7 days old and will delete them. You may change how many days of crash logs are retained by editing the cleanup_crashlogs.sh.j2 template. +- **add_diag_script.yml** - Installs a bash script and cron job that will automatically clean-up splunk diag files. By default, every night at midnight, it will find any diags that are more than 30 days old and will delete them. You may change how many days of splunk diags are retained by editing the cleanup_diags.sh.j2 template. +- **add_pstack_script.yml** - Copies the genpstacks.sh script to $SPLUNK_HOME/genpstacks.sh. This file is useful to have on all of your Splunk servers for when Splunk Support asks you to capture pstacks. + +Note: Any task with an **adhoc** prefix means that it is intended to be used adhoc. These tasks are not included or used by others tasks. Instead, adhoc tasks tasks were developed for resolving various incidents that we've encountered, and rather than strip those automations out, we've opted to include them as bonus content. + +- **adhoc_clean_dispatch.yml** - This task is intended to be used for restoring service to search heads should the dispatch directory become full. You should need to use this task in a healthy environment, but it is at your disposal should the need arise. The task will stop splunk, remove all files in the dispatch directory, and then start splunk. +- **adhoc_configure_hostname** - Renames a Splunk host. First, it configure's the system hostname using the Ansible inventory_hostname, then it updates server.conf with ansible_hostname, and finally it updates inputs.conf with the ansible_hostname. All Splunk configuration changes are made using the lineinfile module, which acts to preserve other configurations that may exist in server.conf and/or inputs.conf. +- **adhoc_decom_indexer.yml** - Executes a splunk offline --enforce-counts command. This is useful when decommissioning one or more indexers from an indexer cluster. +- **adhoc_fix_mongo.yml** - Use when Splunk is in a stopped state to fix mongodb/kvstore issues. Ensures that permissions are set correctly on mongo's splunk.key file and deletes mongod.lock if it exists. +- **adhoc_fix_server_certificate.yml** - Use to delete an expired server.pem and generate a new one (default certs). Useful if your server.pem certificate has expired and you are using Splunk's default certificate for splunkd. Note that default certificates present a security risk and that their use should be avoided, if possible. +- **adhoc_kill_splunkd.yml** - Some releases of Splunk have a "feature" that leaves zombie splunkd processes after a 'splunk stop'. Use this task after a 'splunk stop' to make sure that it's really stopped. Useful for upgrades on some of the 7.x releases. +- **check_splunk.yml** - Check if Splunk is installed. If Splunk is not installed, it will be installed on the host. If Splunk is already installed, the task will execute a "splunk version" command on the host, and then compare the version and build number of Splunk to the version and build number of the expected version of Splunk. Note that the expected version of Splunk does not need to be statically defined; The expected Splunk version and build are automatically extracted from the value of splunk_package_url_full or splunk_package_url_uf using Jinja regex filters. This task will work for both the Universal Forwarder and full Splunk Enterprise packages. You define which host uses what package by organizing it under the appropriate group ('full' or 'uf') in your Ansible inventory. +- **configure_apps.yml** - This task can be called directly from a playbook in order to deploy apps or configurations (i.e. git repositories) to Splunk hosts. You may also include this task within install_splunk.yml to do a "install and deploy apps" all in one play (or simply add a second play to your playbook to call this task after check_splunk.yml). +- **configure_authentication.yml** - Uses the template identified by the splunk_authenticationconf variable to install an authentication.conf file to $SPLUNK_HOME/etc/system/local/authentication.conf. We are including this task here since Ansible is able to securely deploy an authentication.conf configuration by using ansible-vault to encrypt sensitive values such as ad_bind_password. If you are using a common splunk.secret file, you can omit this task and instead use configure_apps.yml to deploy an authentication.conf file from a Git repository containing an authentication.conf app with pre-hashed credentials. +- **configure_bash.yml** - Configures bashrc and bash_profile files for the splunk user. Please note that the templates included with this role will overwrite any existing files for the splunk user (if they exist). The templates will define a custom PS1 at the bash prompt, configure the $SPLUNK_HOME environment variable so that you can issue "splunk " without specifying the full path to the Splunk binary, and will enable auto-completion of Splunk CLI commands in bash. +- **configure_deploymentclient.yml** - Generates a new deploymentclient.conf file from the deploymentclient.conf.j2 template and installs it to $SPLUNK_HOME/etc/system/local/deploymentclient.conf. Note that this task requires two variables to be defined: clientName and splunk_uri_ds. Included automatically by install_splunk.yml. +- **configure_facl.yml** - Configure file system access control lists (FACLs) to allow the splunk user to read /var/log files and add the splunk user's group to /etc/audit/auditd.conf to read /var/log/audit/ directory. +- **configure_license.yml** - Configure the license master URI in server.conf for full Splunk installations when splunk_uri_lm has been defined. Note: This could also be accomplished using configure_apps.yml with a git repository. +- **configure_os.yml** - Increases ulimits for the splunk user and disables Transparent Huge Pages (THP) per Splunk implementation best practices. +- **configure_splunk_forwarder_meta.yml** - Configures a new indexed field called splunk_forwarder and sets its default value to the value of ansible_hostname. Note that you will need to install a fields.conf on your search head if you wish to use this custom indexed field. +- **configure_splunk_secret.yml** - Configures a common splunk.secret file from the files/authentication/splunk.secret so that pre-hashed passwords can be securely deployed. Note that changing splunk.secret will require re-encryption of any passwords that were encrypted using the previous splunk.secret since Splunk will no longer be able to decrypt them successfully. +- **configure_thp.yml** - Installs a custom systemd service that disables THP for RedHat|CentOS systems 6.0+. Note that this task is automatically called by the configure_os.yml task. +- **download_and_unarchive.yml** - Downloads the appropriate Splunk package to the Ansible hsot using the splunk_package_url (derived automatically from the values of the splunk_package_url_full and splunk_package_url_uf variables). The package is then installed to splunk_install_path (derived automatically in main.yml using the splunk_install_path and the package type). Calls handlers for enabling splunk boot-start (using init.d) and adding the configuration of ulimits to splunk's init.d file. +- **install_apps.yml** - Called by configure_apps.yml to perform app installation on the Splunk host. Do not call install_apps.yml directly! +- **install_splunk.yml** - Called by check_splunk.yml to install/upgrade Splunk and Splunk Universal Forwarders, as well as perform any initial configurations. This task is called by check_splunk.yml when the check determines that Splunk is not currently installed. This task will create the splunk user and splunk group, configure the bash profile for the splunk user (by calling configure_bash.yml), configure THP and ulimits (by calling configure_os.ym), download and install the appropriate Splunk package (by calling download_and_unarchive.yml), configure a common splunk.secret (by calling configure_splunk_secret.yml, if configure_secret is defined), create a deploymentclient.conf file with the splunk_ds_uri and clientName (by calling configure_deploymentclient.yml, if clientName is defined), install a user-seed.conf with a prehashed admin password (if used_seed is defined), and will then call the post_install.yml task. See post_install.yml entry for details on post-installation tasks. +- **install_utilities.yml** - Installs Linux packages that are useful for troubleshooting Splunk-related issues when install_utilities == true and linux_packages is defined with a list of packages to install. +- **main.yml** - This is the main task that will always be called when executing this role. This task sets the appropriate variables for full vs uf packages, sends a Slack notification about the play if the slack_token and slack_channel are defined, and then includes the task from the role to execute against, as defined by the value of the deployment_task variable. The deployment_task variable should be defined in your playbook(s). Refer to the included example playbooks to see this in action. +- **post_install.yml** - Executes post-installation tasks. Performs a touch on the .ui_login file which disables the first-time login prompt to change your password, ensures that splunk_home is owned by the correct user and group, and optionally configures three scripts to: cleanup crash logs and old diags (by calling add_crashlog_script.yml and add_diag_script.yml, respectively), and a pstack generation shell script for troubleshooting purposes (by calling add_pstack_script.yml). This task will install various Linux troubleshooting utilities (by calling install_utilities.yml) if install_utilities == true. +- **set_maintenance_mode.yml** - Enables or disables maintenance mode on a cluster master. Intended to be called by playbooks for indexer cluster upgrades/maintenance. Requires the "state" variable to be defined. Valid values: enabled, disabled +- **set_upgrade_state.yml** - Runs a splunk upgrade-{{ peer_state }} cluster-peers command on the cluster master. This task can be used for upgrading indexer clusters with new minor and maintenance releases of Splunk (assuming you are at Splunk v7.1.0 or higher). Refer to https://docs.splunk.com/Documentation/Splunk/latest/Indexer/Searchablerollingupgrade for more information. +- **splunk_offline.yml** - Runs a splunk offline CLI command. Useful for bringing down indexers non-intrusively by allowing searches to complete before stopping splunk. +- **splunk_restart.yml** - Restarts splunk via CLI command. Used when waiting for a handler to run at the end of the play would be inappapropriate. +- **splunk_start.yml** - Starts splunk via CLI command. This task will also accept the license and answers yes for any prompts. Used when waiting for a handler to run at the end of the play would be inappapropriate. +- **splunk_stop.yml** - Stops splunk via CLI command. Used when waiting for a handler to run at the end of the play would be inappapropriate. +- **upgrade_splunk.yml** - Called by check_splunk.yml. Performs an upgrade of an existing splunk installation. Configures .bash_profile and .bashrc for splunk user (by calling configure_bash.yml), disables THP and increases ulimits (by calling configure_os.yml), disables boot-start (this addresses a bug in 7.2.x -- it gets re-enabled by a hander at the end), kills any stale splunkd processes present (by calling kill_splunkd.yml). Note: You should NOT run the upgrade_splunk.yml task directly from a playbook. check_splunk.yml will call upgrade_splunk.yml if it determines that an upgrade is needed; It will then download and unarchive the new version of Splunk (by calling download_and_unarchive.yml), ensure that mongod is in a good stopped state (by calling fix_mongo.yml), and will then perform post-installation tasks using the post_install.yml task. + +## Frequently Asked Questions +What is the difference between this and splunk-ansible? +The splunk-ansible project was built for the docker-splunk project, which is a completely different use case. The way that docker-splunk works is by spinning-up an image that already has splunk-ansible inside of it, and then any arguments provided to Docker are passed into splunk-ansible so that it can run locally inside of the container to install and configure Splunk there. While it's a cool use case, we didn't feel that splunk-ansible met our needs as Splunk administrators to manage production Splunk deployments, so we wrote our own. + +## Support +Use the [GitHub issue tracker](https://github.com/splunk/splunk-ansible/issues) to submit bugs or request features. + +If you have questions or need support, you can: +* Post a question to [Splunk Answers](http://answers.splunk.com). +* Join the #ansible channel on [Splunk-Usergroups Slack](https://docs.splunk.com/Documentation/Community/1.0/community/Chat#Join_us_on_Slack). +* Please do note file cases in the Splunk support portal related to this project, as they will not be able to help you. + +## License +Copyright 2018-2020 Splunk. + +Distributed under the terms of the Apache 2.0 license, ansible-role-for-splunk is free and open-source software. diff --git a/environments/development/group_vars/all.yml b/environments/development/group_vars/all.yml new file mode 100644 index 00000000..4b4c280c --- /dev/null +++ b/environments/development/group_vars/all.yml @@ -0,0 +1,6 @@ +--- +splunk_uri_lm: https://my_license_master:8089 +ansible_user: my_ansible_ssh_user +ansible_ssh_private_key_file: ~/.ssh/my_ansible_ssh.key +git_server: ssh://git@mygithost:1234 +git_key: ~/.ssh/my-git-key diff --git a/environments/development/inventory.yml b/environments/development/inventory.yml new file mode 100644 index 00000000..b0176e99 --- /dev/null +++ b/environments/development/inventory.yml @@ -0,0 +1,10 @@ +--- +all: + children: + + full: + children: + + search: + hosts: + 10.202.37.251: diff --git a/environments/production/group_vars/all.yml b/environments/production/group_vars/all.yml new file mode 100644 index 00000000..36d3a7b9 --- /dev/null +++ b/environments/production/group_vars/all.yml @@ -0,0 +1,7 @@ +--- +splunk_uri_lm: https://my_license_master:8089 +ansible_user: my_ansible_ssh_user +ansible_ssh_private_key_file: ~/.ssh/my_ansible_ssh.key +git_server: ssh://git@mygithost:1234 +git_key: ~/.ssh/my-git-key +git_project: SOMEPROJECT diff --git a/environments/production/group_vars/clustermaster.yml b/environments/production/group_vars/clustermaster.yml new file mode 100644 index 00000000..6cfc3c06 --- /dev/null +++ b/environments/production/group_vars/clustermaster.yml @@ -0,0 +1,7 @@ +--- +# group vars for cluster master +splunk_app_deploy_path: etc/master-apps # default subdirectory in splunk_home that apps from git should be installed to, overridable in git_apps if you want to install to etc/apps +git_version: 1.0.0 +git_apps: + - name: my_outputs + - name: Splunk_TA_nix diff --git a/environments/production/group_vars/deploymentserver.yml b/environments/production/group_vars/deploymentserver.yml new file mode 100644 index 00000000..355f270a --- /dev/null +++ b/environments/production/group_vars/deploymentserver.yml @@ -0,0 +1,13 @@ +--- +# group_vars for deployment server +splunk_app_deploy_path: etc/deployment-apps +git_version: master +git_apps: + # Example of cloning a repository from a different git server and then deploying it to a path other than etc/deployment-apps + - name: my_outputs + git_version: 1.0.0 + git_server: ssh://git@anotherserver.com + git_key: ~/.ssh/anotherkey + git_project: SPLK + splunk_app_deploy_path: etc/apps + - name: disable_rest diff --git a/environments/production/group_vars/universalforwarder.yml b/environments/production/group_vars/universalforwarder.yml new file mode 100644 index 00000000..22bc9bdf --- /dev/null +++ b/environments/production/group_vars/universalforwarder.yml @@ -0,0 +1,6 @@ +--- +# group_vars for universal forwarders +splunk_uri_ds: my-ds:8089 +splunk_app_deploy_path: etc/apps # subdirectory in splunk_home that apps from git should be installed to by Ansible +git_apps: + - name: config_base diff --git a/environments/production/host_vars/my-shc-deployer.yml b/environments/production/host_vars/my-shc-deployer.yml new file mode 100644 index 00000000..051afd2c --- /dev/null +++ b/environments/production/host_vars/my-shc-deployer.yml @@ -0,0 +1,5 @@ +--- +deploy_target: my-shc-target-sh-uri +git_version: 1.0.0 # This is the golden repo branch for this host! It retires all ID/ED/SB/CM projects and repos +git_apps: + - name: myapp diff --git a/environments/production/inventory.yml b/environments/production/inventory.yml new file mode 100644 index 00000000..4fcfe4fc --- /dev/null +++ b/environments/production/inventory.yml @@ -0,0 +1,57 @@ +--- +# YAML format inventory with nested groups for variable inheritance +# Mason Morales, Splunk, Inc. +# References: +# https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html +# Read this: https://docs.ansible.com/ansible/2.9/plugins/inventory/yaml.html +# https://www.digitalocean.com/community/tutorials/how-to-manage-multistage-environments-with-ansible +# ########################################################################## +all: + children: + + full: + children: + + licensemaster: + hosts: + my-lm-host: + + dmc: + hosts: + my-dmc-host: + + shdeployer: + hosts: + my-shc-deployer: + + search: + children: + + west_coast_shc: + hosts: + sh-01: + sh-02: + sh-03: + + heavyforwarder: + hosts: + my-hf-01: + some-other-hf: + + standalone: + hosts: + my-standalone-splunk-server: + my-other-standalone-splunk-server: + + universalforwarder: + children: + + vmware: + hosts: + my-vm-ware-host-[001:100]: + vars: + clientName: vmware + + web: + hosts: + my-web-host-[01-05]: diff --git a/playbooks/ansible.cfg b/playbooks/ansible.cfg new file mode 100644 index 00000000..386165a3 --- /dev/null +++ b/playbooks/ansible.cfg @@ -0,0 +1,27 @@ +[defaults] +inventory_enabled = yaml, host_list, ini, script +internal_poll_interval = 0.001 +inventory = hosts +host_key_checking = False +timeout = 60 +Forks = 50 +transport = paramiko +# To use slack notification, make sure you do this first: easy_install prettytable +# callback_whitelist = slack, splunk + +[ssh_connection] +ssh_args = -o ControlMaster=auto -o ControlPersist=7200s -o PreferredAuthentications=publickey -o StrictHostKeyChecking=no +scp_if_ssh = True +timeout = 10 +retries = 5 +# Tip: Disable pipelining when you get "sudo: sorry, you must have a tty to run sudo\n" - disabled is MUCH slower but more compatible +pipelining=True + +#[callback_slack] +#webhook_url = https://hooks.slack.com/services/ +#channel = #channel_here +#username = splunk_ansible + +#[callback_splunk] +#url = https://my-hec-target/services/collector/event?channel= +#authtoken = diff --git a/playbooks/splunk_app_install.yml b/playbooks/splunk_app_install.yml new file mode 100644 index 00000000..e022c399 --- /dev/null +++ b/playbooks/splunk_app_install.yml @@ -0,0 +1,8 @@ +# Example playbook to install/upgrade apps on a Splunk host +- hosts: + - all + roles: + - ../roles/splunk + serial: 50 + vars: + - deployment_task: configure_apps.yml diff --git a/playbooks/splunk_install_or_upgrade.yml b/playbooks/splunk_install_or_upgrade.yml new file mode 100644 index 00000000..5a0ce9cc --- /dev/null +++ b/playbooks/splunk_install_or_upgrade.yml @@ -0,0 +1,9 @@ +--- +# Example playbook to perform either a splunk installation or upgrade +- hosts: + - all + roles: + - ../roles/splunk + serial: 50 + vars: + - deployment_task: check_splunk.yml diff --git a/playbooks/splunk_upgrade_full_stack.yml b/playbooks/splunk_upgrade_full_stack.yml new file mode 100644 index 00000000..c96f6792 --- /dev/null +++ b/playbooks/splunk_upgrade_full_stack.yml @@ -0,0 +1,67 @@ +--- +# Example playbook to properly upgrade a single-site indexer cluster and search head cluster using the splunk role from ansible-role-for-splunk +- hosts: + - clustermaster + become: yes + any_errors_fatal: true + max_fail_percentage: 0 + roles: + - ../roles/splunk + vars: + deployment_task: check_splunk.yml + +- hosts: + - licensemaster + - shdeployer + - deploymentserver + become: yes + any_errors_fatal: true + max_fail_percentage: 0 + roles: + - ../roles/splunk + vars: + deployment_task: check_splunk.yml + +- hosts: + - search + serial: 50 + become: yes + any_errors_fatal: true + max_fail_percentage: 0 + roles: + - ../roles/splunk + vars: + deployment_task: check_splunk.yml + +- hosts: + - clustermaster + become: yes + any_errors_fatal: true + max_fail_percentage: 0 + roles: + - ../roles/splunk + vars: + deployment_task: set_maintenance_mode.yml + state: enable + +- hosts: + - indexer + serial: 100 + become: yes + any_errors_fatal: true + max_fail_percentage: 0 + roles: + - ../roles/splunk + vars: + deployment_task: check_splunk.yml + +- hosts: + - clustermaster + become: yes + any_errors_fatal: true + max_fail_percentage: 0 + roles: + - ../roles/splunk + vars: + deployment_task: set_maintenance_mode.yml + state: disable diff --git a/roles/splunk/.yamllint b/roles/splunk/.yamllint new file mode 100644 index 00000000..88276760 --- /dev/null +++ b/roles/splunk/.yamllint @@ -0,0 +1,33 @@ +--- +# Based on ansible-lint config +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + colons: + max-spaces-after: -1 + level: error + commas: + max-spaces-after: -1 + level: error + comments: disable + comments-indentation: disable + document-start: disable + empty-lines: + max: 3 + level: error + hyphens: + level: error + indentation: disable + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable + truthy: disable diff --git a/roles/splunk/README.md b/roles/splunk/README.md new file mode 100644 index 00000000..f7474971 --- /dev/null +++ b/roles/splunk/README.md @@ -0,0 +1,35 @@ +Role Name +========= + +Ansible Starter Pack for Splunk Administration + +Requirements +------------ + +Ansible 2.7 or higher + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. +<> + +Dependencies +------------ + +prettytable if using the Slack plugin + +Example Playbook +---------------- + +See examples included in the playbooks directory. + +License +------- + +Apache 2.0 + +Author Information +------------------ + +Authored by Mason Morales, Splunk, Inc. This project is being open-sourced by Splunk for community usage. Please note that Splunk does not provide official support for this product. diff --git a/roles/splunk/defaults/main.yml b/roles/splunk/defaults/main.yml new file mode 100644 index 00000000..ae1631e9 --- /dev/null +++ b/roles/splunk/defaults/main.yml @@ -0,0 +1,54 @@ +--- +# defaults file for splunk role +# Anything that is undefined here should be configured. I recommend setting the values in your group_vars under an all.yml file. +slack_channel: undefined +slack_token: undefined +splunk_home: auto_determined # This gets set by main.yml but we have to define it here or Ansible will complain that it is undefined +splunk_package_url: auto_determined # This gets set by main.yml but we have to define it here or Ansible will complain that it is undefined +splunk_package_path: ~/ +splunk_package_url_full: https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=8.0.6&product=splunk&filename=splunk-8.0.6-152fb4b2bb96-Linux-x86_64.tgz&wget=true +#splunk_package_url_full: https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=8.0.4.1&product=splunk&filename=splunk-8.0.4.1-ab7a85abaa98-Linux-x86_64.tgz&wget=true +splunk_package_url_uf: https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version=8.0.4&product=universalforwarder&filename=splunkforwarder-8.0.4-767223ac207f-Linux-x86_64.tgz&wget=true +splunk_install_type: undefined # There are two ways to configure this. The easiest way is to nest hosts under either a "full" group or a "uf" group in your inventory and main.yml will handle it for you. Or, you can also set the value via a group_vars or host_vars file. +splunk_install_path: /opt # Base directory on the operating system to which splunk should be installed +splunk_nix_user: splunk +splunk_nix_group: splunk +splunk_uri_lm: undefined +splunk_uri_cm: undefined +splunk_uri_ds: undefined # e.g. mydeploymentserver.mydomain.com:8089 ; Note that you must also configure the clientName var under either group_vars or host_vars for deploymentclient.conf to be configured +clientName: undefined +splunk_admin_username: admin +splunk_admin_password: undefined # Use ansible-vault encrypt_string, e.g. ansible-vault encrypt_string --ask-vault-pass 'var_value_to_encrypt' --name 'var_name' +splunk_user_seed: false # Requires splunk_admin_password var to be set +splunk_configure_secret: false # If set to true, you need to update files/splunk.secret +# Although there are tasks for the following Splunk configurations in this role, they are not included in any tasks by default. You can add them to your install_splunk.yml if you would like to have Ansible manage any of these files +splunk_configure_authentication: false +ad_bind_password: undefined # Use ansible-vault encrypt_string, e.g. ansible-vault encrypt_string --ask-vault-pass 'var_value_to_encrypt' --name 'var_name' +splunk_authenticationconf: authentication.conf.j2 +splunk_use_initd: false +git_local_clone_path: ~/ # Base directory under which repositories for app deplyoment should be cloned to +git_server: undefined # e.g. ssh://git@mygithost:1234 - Note that this may be set in an all.yml group_var or inside the git_apps dictionary within host_vars +git_key: undefined # Path to SSH key for cloning repositories - Note that this may be set in an all.yml group_var or inside the git_apps dictionary within host_vars +git_project: undefined +git_version: master # Configure default version to clone, overridable inside the git_apps dictionary within host_vars +splunk_app_deploy_path: undefined # Path under $SPLUNK_HOME/ to deploy apps to - Note that this may be set in group_vars, host_vars, playbook vars, or inside the git_apps dictionary within host_vars +add_crashlog_script: false # Set to true to install a script and cron job to automatically cleanup splunk crash logs older than 7 days +add_diag_script: false # Set to true to install a script and cron job to automatically cleanup splunk diag files older than 30 days +add_pstack_script: false # Set to true to install a pstack generation script for troubleshooting purposes in $SPLUNK_HOLME/genpstacks.sh +install_utilities: false # Set to true to install the list of packages defined in the linux_packages var after installing splunk +linux_packages: + - nload + - iotop + - iftop + - sysstat + - telnet + - tcpdump + - htop + - atop + - lsof + - policycoreutils-python + - policycoreutils + - setroubleshoot + - nethogs + - gdb + - bind-utils diff --git a/roles/splunk/files/disable-thp.service b/roles/splunk/files/disable-thp.service new file mode 100644 index 00000000..e791cebe --- /dev/null +++ b/roles/splunk/files/disable-thp.service @@ -0,0 +1,9 @@ +[Unit] +Description=Disable Transparent Huge Pages (THP) + +[Service] +Type=simple +ExecStart=/bin/sh -c "echo 'never' > /sys/kernel/mm/transparent_hugepage/enabled && echo 'never' > /sys/kernel/mm/transparent_hugepage/defrag" + +[Install] +WantedBy=multi-user.target diff --git a/roles/splunk/files/rsync-filter b/roles/splunk/files/rsync-filter new file mode 100644 index 00000000..c79faad8 --- /dev/null +++ b/roles/splunk/files/rsync-filter @@ -0,0 +1,9 @@ +- /.git +- /*/.git +- /.gitignore +- /*/README.txt +- /*/README.MD +- /*/README.md +- /*/app.manifest +- /*/githash +- /*/package.json diff --git a/roles/splunk/files/splunk.secret b/roles/splunk/files/splunk.secret new file mode 100644 index 00000000..2bf67f51 --- /dev/null +++ b/roles/splunk/files/splunk.secret @@ -0,0 +1 @@ +# insert your common splunk.secret here and encrypt it with ansible-vault diff --git a/roles/splunk/files/splunk_ulimits.conf b/roles/splunk/files/splunk_ulimits.conf new file mode 100644 index 00000000..002d8f96 --- /dev/null +++ b/roles/splunk/files/splunk_ulimits.conf @@ -0,0 +1,4 @@ +splunk soft nofile 655350 +splunk hard nofile 819200 +splunk soft nproc 32768 +splunk hard nproc 51200 diff --git a/roles/splunk/handlers/main.yml b/roles/splunk/handlers/main.yml new file mode 100644 index 00000000..19d8d9bc --- /dev/null +++ b/roles/splunk/handlers/main.yml @@ -0,0 +1,71 @@ +--- +# handlers file for splunk role +# handlers are run in the order defined in this file +# thus "set ulimits in init.d" has been placed at the top to ensure it runs before splunk is started +- name: set ulimits in init.d + lineinfile: + path: /etc/init.d/splunk + insertafter: ^\s+echo Starting Splunk\.\.\.$ + line: "{{ item }}" + state: present + loop: + - ulimit -Hn 65536 + - ulimit -Sn 32768 + become: yes + ignore_errors: true + when: splunk_use_initd == "true" and ansible_os_family == "RedHat" and ansible_distribution_major_version|int > 6 + +- name: stop splunk + service: + name: splunk + state: stopped + become: yes + +- name: start splunk + shell: "{{ splunk_home }}/bin/splunk start --answer-yes --auto-ports --no-prompt --accept-license" + become: yes + become_user: "{{ splunk_nix_user }}" + +- name: apply indexer cluster bundle + shell: "{{ splunk_home }}/bin/splunk apply cluster-bundle --answer-yes --skip-validation -auth {{ splunk_auth }}" + become: yes + become_user: "{{ splunk_nix_user }}" + no_log: true + when: "'clustermaster' in group_names" + +- name: reload deployment server + shell: "{{ splunk_home }}/bin/splunk reload deploy-server -auth {{ splunk_auth }}" + become: yes + become_user: "{{ splunk_nix_user }}" + no_log: true + when: "'deploymentserver' in group_names" + +- name: apply shcluster-bundle + shell: "{{ splunk_home }}/bin/splunk apply shcluster-bundle -preserve-lookups true --answer-yes -auth {{ splunk_auth }} -target {{ deploy_target }}" + become: yes + become_user: "{{ splunk_nix_user }}" + no_log: true + when: "'shdeployer' in group_names and deploy_target is defined" + +- name: reload systemctl daemon + shell: systemctl daemon-reload + become: yes + when: ansible_connection!="docker" + +- name: restart splunk + shell: "{{ splunk_home }}/bin/splunk restart" + become: yes + become_user: "{{ splunk_nix_user }}" + +- name: restart splunk service + service: + name: splunk + state: restarted + become: yes + +- name: restart auditd service + service: + name: auditd + state: restarted + become: yes + ignore_errors: true diff --git a/roles/splunk/meta/main.yml b/roles/splunk/meta/main.yml new file mode 100644 index 00000000..480b1397 --- /dev/null +++ b/roles/splunk/meta/main.yml @@ -0,0 +1,9 @@ +galaxy_info: + author: Mason Morales + description: Ansible role for deploying Splunk + company: Splunk Inc. + license: Apache 2.0 + min_ansible_version: 2.7 + galaxy_tags: + - splunk +dependencies: [] diff --git a/roles/splunk/tasks/add_crashlog_script.yml b/roles/splunk/tasks/add_crashlog_script.yml new file mode 100644 index 00000000..9b6a64ae --- /dev/null +++ b/roles/splunk/tasks/add_crashlog_script.yml @@ -0,0 +1,18 @@ +--- +- name: Add cleanup_crashlogs.sh + template: + src: cleanup_crashlogs.sh.j2 + dest: "{{ splunk_home }}/cleanup_crashlogs.sh" + owner: root + group: root + mode: 0755 + become: yes + +- name: Configure cron to run cleanup_crashlogs.sh daily at midnight + cron: + name: "Run cleanup_crashlogs.sh" + state: present + job: "{{ splunk_home }}/cleanup_crashlogs.sh" + hour: 0 + minute: 0 + become: yes diff --git a/roles/splunk/tasks/add_diag_script.yml b/roles/splunk/tasks/add_diag_script.yml new file mode 100644 index 00000000..95aa1d24 --- /dev/null +++ b/roles/splunk/tasks/add_diag_script.yml @@ -0,0 +1,18 @@ +--- +- name: Install cleanup_diags.sh + template: + src: cleanup_diags.sh.j2 + dest: "{{ splunk_home }}/cleanup_diags.sh" + owner: root + group: root + mode: 0755 + become: yes + +- name: Configure cron to run cleanup_diags.sh daily at midnight + cron: + name: "Run cleanup_diags.sh" + state: present + job: "{{ splunk_home }}/cleanup_diags.sh" + hour: 0 + minute: 0 + become: yes diff --git a/roles/splunk/tasks/add_pstack_script.yml b/roles/splunk/tasks/add_pstack_script.yml new file mode 100644 index 00000000..a38b41e6 --- /dev/null +++ b/roles/splunk/tasks/add_pstack_script.yml @@ -0,0 +1,9 @@ +--- +- name: "Add genpstacks.sh to {{ splunk_home }}/genpstacks.sh for troubleshooting purposes" + template: + src: genpstacks.sh.j2 + dest: "{{ splunk_home }}/genpstacks.sh" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + mode: 0755 + become: yes diff --git a/roles/splunk/tasks/adhoc_clean_dispatch.yml b/roles/splunk/tasks/adhoc_clean_dispatch.yml new file mode 100644 index 00000000..7c0f65aa --- /dev/null +++ b/roles/splunk/tasks/adhoc_clean_dispatch.yml @@ -0,0 +1,10 @@ +--- +- name: Stop Splunk + include_tasks: splunk_stop.yml + +- name: Clean dispatch directory of all files + shell: "rm -rf {{ splunk_home }}/var/run/splunk/dispatch/*" + become: yes + +- name: Start Splunk + include_tasks: splunk_start.yml diff --git a/roles/splunk/tasks/adhoc_configure_hostname.yml b/roles/splunk/tasks/adhoc_configure_hostname.yml new file mode 100644 index 00000000..67aac76f --- /dev/null +++ b/roles/splunk/tasks/adhoc_configure_hostname.yml @@ -0,0 +1,25 @@ +--- +- name: Set the system's hostname to the inventory_hostname + hostname: + name: "{{ inventory_hostname }}" + become: yes + +- name: "Update serverName in {{ splunk_home }}/etc/system/local/server.conf" + lineinfile: + dest: "{{ splunk_home }}/etc/system/local/server.conf" + state: present + regexp: "^serverName = .*" + line: "serverName = {{ ansible_hostname }}" + become: yes + become_user: "{{ splunk_nix_user }}" + notify: restart splunk + +- name: "Update host in {{ splunk_home }}/etc/system/local/inputs.conf" + lineinfile: + dest: "{{ splunk_home }}/etc/system/local/inputs.conf" + state: present + regexp: "^host = .*" + line: "host = {{ ansible_hostname }}" + become: yes + become_user: "{{ splunk_nix_user }}" + notify: restart splunk diff --git a/roles/splunk/tasks/adhoc_decom_indexer.yml b/roles/splunk/tasks/adhoc_decom_indexer.yml new file mode 100644 index 00000000..f9f84df2 --- /dev/null +++ b/roles/splunk/tasks/adhoc_decom_indexer.yml @@ -0,0 +1,7 @@ +--- +- name: Execute splunk offline --enforce-counts + shell: "{{ splunk_home }}/bin/splunk offline --enforce-counts -auth {{ splunk_auth }}" + become: yes + become_user: "{{ splunk_nix_user }}" + no_log: true + when: "'indexer' in group_names" diff --git a/roles/splunk/tasks/adhoc_fix_mongo.yml b/roles/splunk/tasks/adhoc_fix_mongo.yml new file mode 100644 index 00000000..68e9dea2 --- /dev/null +++ b/roles/splunk/tasks/adhoc_fix_mongo.yml @@ -0,0 +1,27 @@ +--- +- name: Check permissions of splunk.key for mongo + file: + state: file + path: "{{ splunk_home }}/var/lib/splunk/kvstore/mongo/splunk.key" + mode: 0600 + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + +- name: Delete mongod.lock if present + file: + state: absent + path: "{{ splunk_home }}/var/lib/splunk/kvstore/mongo/mongod.lock" + become: yes + +- name: Update file permissions for journal + file: + state: directory + recurse: yes + path: "{{ splunk_home }}/var/lib/splunk/kvstore/mongo/journal" + mode: 0710 + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + +# Also consider adhoc_fix_server_certificate.yml if the cert is expired and you are using the default cert for kvstore diff --git a/roles/splunk/tasks/adhoc_fix_server_certificate.yml b/roles/splunk/tasks/adhoc_fix_server_certificate.yml new file mode 100644 index 00000000..83c0ac6f --- /dev/null +++ b/roles/splunk/tasks/adhoc_fix_server_certificate.yml @@ -0,0 +1,12 @@ +--- +# To check if cert is expired: splunk cmd openssl x509 -in /opt/splunk/etc/auth/server.pem -text -noout +# Note: Genereates new default certificate. Do not use this if you generate/sign your own splunkd certificate +- name: Remove old server.pem certificate and generate a new one + shell: "{{ item }}" + become: yes + become_user: "{{ splunk_nix_user }}" + loop: + - "rm {{ splunk_home }}/etc/auth/server.pem" + - "{{ splunk_home }}/bin/splunk createssl server-cert -d /opt/splunk/etc/auth -n server.pem" + notify: + - restart splunk diff --git a/roles/splunk/tasks/adhoc_kill_splunkd.yml b/roles/splunk/tasks/adhoc_kill_splunkd.yml new file mode 100644 index 00000000..9e8ae1bd --- /dev/null +++ b/roles/splunk/tasks/adhoc_kill_splunkd.yml @@ -0,0 +1,4 @@ +--- +- name: Kill any stale splunkd processes + shell: for i in $(ps -ef | grep splunkd -i | grep -v grep | awk '{print $2}'); do kill -9 $i; done + become: yes diff --git a/roles/splunk/tasks/check_splunk.yml b/roles/splunk/tasks/check_splunk.yml new file mode 100644 index 00000000..16f024d9 --- /dev/null +++ b/roles/splunk/tasks/check_splunk.yml @@ -0,0 +1,54 @@ +--- +- name: Check if Splunk is installed + stat: + path: "{{ splunk_home }}/bin/splunk" + follow: yes + register: splunkd_path + become: yes + +# If installed, check version, if version is good, don't install but continue +- name: Install Splunk if not installed + include_tasks: install_splunk.yml + when: splunkd_path.stat.exists == false + +# Configure the license for both fresh and old installs +- name: Configure license + include_tasks: configure_license.yml + +- name: Execute this block only if splunk is already installed + block: + # If Splunk is installed, get the current version + - name: Run splunk version command to check currently installed version + shell: "{{ splunk_home }}/bin/splunk version --answer-yes --auto-ports --no-prompt --accept-license" + register: current_version + become: yes + become_user: "{{ splunk_nix_user }}" + changed_when: false + + - name: "Checkpoint: Version" ########################## + debug: + msg: "The value of splunk_version is: {{ splunk_version }} and the current_version is: {{ current_version.stdout }}" + + - name: "Checkpoint: Package" + debug: + msg: "We will download the latest release from {{ splunk_package_url }}" + when: current_version.stdout != splunk_version + + - name: Check if Splunk needs to be stopped if we are not at the expected version + shell: "{{ splunk_home }}/bin/splunk status" + register: splunk_status + become: yes + become_user: "{{ splunk_nix_user }}" + when: current_version.stdout != splunk_version + changed_when: false + ignore_errors: true + + - name: Stop Splunk if not at expected version and not currently stopped + include_tasks: splunk_stop.yml + when: current_version.stdout != splunk_version and splunk_status.stdout != 'splunkd is not running.' + + - name: Upgrade Splunk if not at expected version + include_tasks: upgrade_splunk.yml + when: current_version.stdout != splunk_version +# Conditional for this block + when: splunkd_path.stat.exists == true diff --git a/roles/splunk/tasks/configure_apps.yml b/roles/splunk/tasks/configure_apps.yml new file mode 100644 index 00000000..417e5bff --- /dev/null +++ b/roles/splunk/tasks/configure_apps.yml @@ -0,0 +1,97 @@ +--- +# Configure default splunk_app_deploy_path based on which group the host is a member of +- name: Set default splunk_app_deploy_path for clustermaster hosts + set_fact: + splunk_app_deploy_path: etc/master-apps + when: "'clustermaster' in group_names" + +- name: Set default splunk_app_deploy_path for shdeployer hosts + set_fact: + splunk_app_deploy_path: etc/shcluster/apps + when: "'shdeployer' in group_names" + +- name: Set default splunk_app_deploy_path for deploymentserver hosts + set_fact: + splunk_app_deploy_path: etc/deployment-apps + when: "'deploymentserver' in group_names" + +- name: "Set default splunk_app_deploy_path all other hosts" + set_fact: + splunk_app_deploy_path: "etc/apps" + when: + - "'clustermaster' not in group_names" + - "'shdeployer' not in group_names" + - "'deploymentserver' not in group_names" + +# Prep the Ansible host to deploy apps then install them +- block: + - name: Create local directory for managing repos on the target host + file: + path: "{{ git_local_clone_path }}{{ ansible_nodename }}" + state: directory + delegate_to: localhost + changed_when: false + + - name: Install .rsync-filter to avoid copying undesirable files and folders to the target host later + copy: + src: rsync-filter + dest: "{{ git_local_clone_path }}{{ ansible_nodename }}/.rsync-filter" + delegate_to: localhost + changed_when: false + + - name: Download defined Git repos to local Ansible host + local_action: + module: git + accept_hostkey: yes + repo: "{{ item.git_server | default(git_server) }}/{{ item.git_project | default(git_project) }}/{{ item.name }}" + version: "{{ item.git_version | default(git_version) }}" + dest: "{{ git_local_clone_path }}{{ ansible_nodename }}/{{ item.name }}" + key_file: "{{ git_key }}" + force: yes + loop: "{{ git_apps }}" + ignore_errors: true + changed_when: false + check_mode: no + + - name: Ensure rsync is installed on target host + package: + name: rsync + state: present + update_cache: true + become: yes + + - name: "Comment out requiretty in sudoers as it causes synchronize to fail on target host" + lineinfile: + dest: "/etc/sudoers" + state: present + regexp: "^#?{{ item }}" + line: "#{{ item }}" + create: yes + owner: root + group: root + mode: 0440 + loop: + - "Defaults requiretty" + become: yes + + - name: Install apps + include_tasks: install_apps.yml + loop: "{{ git_apps }}" + vars: + app_dest: "{{ item.splunk_app_deploy_path | default(splunk_app_deploy_path) }}" + app_src: "{{ git_local_clone_path }}{{ ansible_nodename }}/{{ item.name }}" + + - name: Cleanup the local directory that was used to manage repos on the target host + file: + path: "{{ git_local_clone_path }}{{ ansible_nodename }}" + state: absent + delegate_to: localhost + changed_when: false + + # Conditional for block + when: + - git_apps is defined + - git_version is defined + - git_key != 'undefined' + - git_server != 'undefined' + - git_project != 'undefined' diff --git a/roles/splunk/tasks/configure_authentication.yml b/roles/splunk/tasks/configure_authentication.yml new file mode 100644 index 00000000..379bbd9c --- /dev/null +++ b/roles/splunk/tasks/configure_authentication.yml @@ -0,0 +1,12 @@ +--- +- name: Install authentication.conf for admins + template: + src: "{{ splunk_authenticationconf }}" + dest: "{{ splunk_home }}/etc/system/local/authentication.conf" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + when: + - splunk_authenticationconf is defined + - ad_bind_password != 'undefined' + notify: restart splunk diff --git a/roles/splunk/tasks/configure_bash.yml b/roles/splunk/tasks/configure_bash.yml new file mode 100644 index 00000000..149a3f46 --- /dev/null +++ b/roles/splunk/tasks/configure_bash.yml @@ -0,0 +1,11 @@ +--- +- name: Install .bashrc and .bash_profile files + template: + src: "{{ item.bashtemplate}}" + dest: "{{ splunk_home }}/{{ item.bashfilepath }}" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + loop: + - { bashtemplate: 'bashrc.j2', bashfilepath: '.bashrc' } + - { bashtemplate: 'bash_profile.j2', bashfilepath: '.bash_profile' } diff --git a/roles/splunk/tasks/configure_deploymentclient.yml b/roles/splunk/tasks/configure_deploymentclient.yml new file mode 100644 index 00000000..1a317402 --- /dev/null +++ b/roles/splunk/tasks/configure_deploymentclient.yml @@ -0,0 +1,12 @@ +--- +- name: Create deploymentclient.conf config + template: + src: deploymentclient.conf.j2 + dest: "{{ splunk_home }}/etc/system/local/deploymentclient.conf" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + notify: restart splunk + when: + - clientName != 'undefined' + - splunk_uri_ds != 'undefined' diff --git a/roles/splunk/tasks/configure_facl.yml b/roles/splunk/tasks/configure_facl.yml new file mode 100644 index 00000000..a233e24d --- /dev/null +++ b/roles/splunk/tasks/configure_facl.yml @@ -0,0 +1,33 @@ +--- +- name: Configure file access control list (facl) settings for splunk user + block: + - name: Set facl to allow splunk user to read /var/log + acl: + path: /var/log + entity: "{{ splunk_nix_user }}" + etype: user + permissions: rx + default: yes + recursive: yes + state: present + become: yes + + - name: Add logrotate script to enforce splunk user facls + template: + src: splunk_facl.j2 + dest: /etc/logrotate.d/splunk_facl + owner: root + group: root + become: yes + + - name: Allow splunk to read /var/log/audit.log + ini_file: + path: /etc/audit/auditd.conf + section: null + option: log_group + value: "{{ splunk_nix_group }}" + become: yes + notify: restart auditd service + ignore_errors: true + + when: splunk_nix_user != 'root' diff --git a/roles/splunk/tasks/configure_license.yml b/roles/splunk/tasks/configure_license.yml new file mode 100644 index 00000000..da4f9295 --- /dev/null +++ b/roles/splunk/tasks/configure_license.yml @@ -0,0 +1,14 @@ +--- +- name: Set license master_uri + ini_file: + path: "{{ splunk_home }}/etc/system/local/server.conf" + section: license + option: master_uri + value: "{{ splunk_uri_lm }}" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + notify: restart splunk + when: + - "'full' in group_names" + - splunk_uri_lm != 'undefined' diff --git a/roles/splunk/tasks/configure_os.yml b/roles/splunk/tasks/configure_os.yml new file mode 100644 index 00000000..a5fcc22d --- /dev/null +++ b/roles/splunk/tasks/configure_os.yml @@ -0,0 +1,17 @@ +--- +# Task to configure ulimits and THP for Splunk +- name: Include OS family-specific variables + include_vars: "{{ ansible_os_family }}.yml" + +- name: Set ulimits for splunk + copy: + src: splunk_ulimits.conf + dest: /etc/security/limits.d/splunk_ulimits.conf + owner: root + group: root + mode: 0644 + become: yes + +- name: Disable THP + include_tasks: configure_thp.yml + when: "'full' in group_names" diff --git a/roles/splunk/tasks/configure_splunk_boot.yml b/roles/splunk/tasks/configure_splunk_boot.yml new file mode 100644 index 00000000..871748c4 --- /dev/null +++ b/roles/splunk/tasks/configure_splunk_boot.yml @@ -0,0 +1,12 @@ +--- +- name: enable splunk boot-start via initd + shell: "{{ splunk_home }}/bin/splunk enable boot-start -user {{ splunk_nix_user }} -systemd-managed 0 --answer-yes --auto-ports --no-prompt --accept-license" + become: yes + when: splunk_use_initd == "true" + notify: + - set ulimits in init.d + +- name: enable splunk boot-start via systemd + shell: "{{ splunk_home }}/bin/splunk enable boot-start -user {{ splunk_nix_user }} -systemd-managed 1 --answer-yes --auto-ports --no-prompt --accept-license" + become: yes + when: splunk_use_initd == "false" diff --git a/roles/splunk/tasks/configure_splunk_forwarder_meta.yml b/roles/splunk/tasks/configure_splunk_forwarder_meta.yml new file mode 100644 index 00000000..9e8c1851 --- /dev/null +++ b/roles/splunk/tasks/configure_splunk_forwarder_meta.yml @@ -0,0 +1,29 @@ +--- +# This task creates a new indexed field in each event's metadata that identifies the heavy forwarder's hostname +- name: create splunk_forwarder reference in props.conf + ini_file: + path: "{{ splunk_home }}/etc/system/local/props.conf" + section: host::* + option: TRANSFORMS-splunk_forwarder_field + value: splunk_forwarder_field + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + notify: restart splunk + when: "'full' in group_names" + +- name: create splunk_forwarder field in transforms.conf + ini_file: + path: "{{ splunk_home }}/etc/system/local/transforms.conf" + section: splunk_forwarder_field + option: "{{ item.option }}" + value: "{{ item.value }}" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + notify: restart splunk + when: "'full' in group_names" + loop: + - { option: "REGEX", value: ".*" } + - { option: "FORMAT", value: "splunk_forwarder::{{ ansible_hostname }}" } + - { option: "WRITE_META", value: "true" } diff --git a/roles/splunk/tasks/configure_splunk_secret.yml b/roles/splunk/tasks/configure_splunk_secret.yml new file mode 100644 index 00000000..f6e78d45 --- /dev/null +++ b/roles/splunk/tasks/configure_splunk_secret.yml @@ -0,0 +1,9 @@ +- name: Install splunk.secret + copy: + src: splunk.secret + dest: "{{ splunk_home }}/etc/auth/splunk.secret" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + notify: restart splunk + when: splunk_configure_secret diff --git a/roles/splunk/tasks/configure_thp.yml b/roles/splunk/tasks/configure_thp.yml new file mode 100644 index 00000000..22804d6a --- /dev/null +++ b/roles/splunk/tasks/configure_thp.yml @@ -0,0 +1,34 @@ +--- +# Authored by Mason Morales for Splunk, Inc. +# References: +# https://blacksaildivision.com/how-to-disable-transparent-huge-pages-on-centos +# https://github.com/objectrocket/ansible-hadoop/blob/master/playbooks/roles/common/tasks/main.yml +- name: Install disable-thp service to systemd (CentOS/RHEL 7+) + copy: + src: disable-thp.service + dest: /etc/systemd/system/disable-thp.service + mode: 0755 + owner: root + group: root + become: yes + notify: + - reload systemctl daemon + # NOTE ansible_connection!="docker" is not included in the when expression + # because we want to test this task's functionality, we want it to run even though it does nothing for a container + # but the notified handlers above will be skipped by their when expressions + when: + - ansible_os_family == "RedHat" + - ansible_distribution_major_version|int > 6 + +- name: Disable THP service + service: + name: disable-thp + enabled: no + state: stopped + become: yes + # no (real) services in docker containers + # and no need to bother for molecule testing + when: + - ansible_connection!="docker" + - "'full' in group_names" + ignore_errors: yes diff --git a/roles/splunk/tasks/configure_user-seed.yml b/roles/splunk/tasks/configure_user-seed.yml new file mode 100644 index 00000000..c0b29010 --- /dev/null +++ b/roles/splunk/tasks/configure_user-seed.yml @@ -0,0 +1,20 @@ +--- +- name: "Check for existing {{ splunk_home }}/etc/passwd" + stat: + path: "{{ splunk_home }}/etc/passwd" + register: splunk_etc_passwd + become: yes + become_user: "{{ splunk_nix_user }}" + +- name: Create user-seed.conf file with splunk_admin_username and splunk_admin_password + template: + src: user-seed.conf.j2 + dest: "{{ splunk_home }}/etc/system/local/user-seed.conf" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + notify: restart splunk + become: yes + when: + - not splunk_etc_passwd.stat.exists + - splunk_user_seed + - splunk_admin_password != 'undefined' diff --git a/roles/splunk/tasks/download_and_unarchive.yml b/roles/splunk/tasks/download_and_unarchive.yml new file mode 100644 index 00000000..6b029cef --- /dev/null +++ b/roles/splunk/tasks/download_and_unarchive.yml @@ -0,0 +1,30 @@ +--- +- debug: + msg: "The splunk_package_url is set to: {{ splunk_package_url }}" + +- debug: + msg: "The filename that will be downloaded is {{ splunk_file }}" + +- debug: + msg: "The upgrade file will be downloaded to {{ splunk_package_path }}/{{ splunk_file }}" + +- name: "Download Splunk {{ splunk_install_type }} package" + local_action: + module: get_url + url: "{{ splunk_package_url }}" + dest: "{{ splunk_package_path }}/{{ splunk_file }}" + register: download_result + retries: 3 + delay: 10 + until: download_result is success + run_once: True + +- name: "Unarchive Splunk {{ splunk_install_type }} package to {{ splunk_home }}" + unarchive: + src: "{{ splunk_package_path }}/{{ splunk_file }}" + dest: "{{ splunk_install_path }}" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + become: yes + notify: + - start splunk diff --git a/roles/splunk/tasks/install_apps.yml b/roles/splunk/tasks/install_apps.yml new file mode 100644 index 00000000..2c814ff2 --- /dev/null +++ b/roles/splunk/tasks/install_apps.yml @@ -0,0 +1,50 @@ +--- +# This task MUST be called by configure_apps.yml to work correctly. Do NOT call this task directly via the deployment_task var! +- name: Set correct handler for master-apps + set_fact: + handler: "apply indexer cluster bundle" + when: app_dest == 'etc/master-apps' + +- name: Set correct handler for deployment-apps + set_fact: + handler: "reload deployment server" + when: app_dest == 'etc/deployment-apps' + +- name: Set correct handler for shcluster/apps + set_fact: + handler: "apply shcluster-bundle" + when: app_dest == 'etc/shcluster/apps' + +- name: "Set default restart splunk handler for all other paths (e.g. etc/auth)" + set_fact: + handler: "restart splunk" + when: + - app_dest != 'etc/shcluster/apps' + - app_dest != 'etc/deployment-apps' + - app_dest != 'etc/master-apps' + +# Note: By using the synchronize module, if the repo already exists on the target host, we are able to only update the diff while preserving the local/ folder +- name: "Synchronize {{ item.name }} repo from local Ansible host to {{ splunk_home }}/{{ app_dest }}/{{ item.name }} on remote host" + synchronize: + src: "{{ app_src }}" + dest: "{{ splunk_home }}/{{ app_dest }}/" + recursive: yes + delete: yes + checksum: yes + rsync_opts: + - "--prune-empty-dirs" + - "--itemize-changes" + - "--no-owner" + - "--no-group" + - "--no-times" + become: yes + become_user: "{{ splunk_nix_user }}" + notify: "{{ handler }}" + +- name: Ensure correct permissions are set + file: + path: "{{ splunk_home }}/{{ app_dest }}/{{ item.name }}" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + recurse: yes + become: yes diff --git a/roles/splunk/tasks/install_splunk.yml b/roles/splunk/tasks/install_splunk.yml new file mode 100644 index 00000000..79f3419e --- /dev/null +++ b/roles/splunk/tasks/install_splunk.yml @@ -0,0 +1,60 @@ +--- +# This task should be used for fresh installations of Splunk, refer to upgrade_splunk.yml for upgrades +- name: Block for non-root splunk user setup + block: + - name: Add nix splunk group + group: + name: "{{ splunk_nix_group }}" + state: present + become: yes + + - name: Add nix splunk user + user: + name: "{{ splunk_nix_user }}" + groups: "{{ splunk_nix_group }}" + home: "{{ splunk_home }}" + state: present + shell: /bin/bash + become: yes + + - name: Allow splunk user to read /var/log + include_tasks: configure_facl.yml + + - name: Configure .bash_profile and .bashrc for splunk user + include_tasks: configure_bash.yml + + when: splunk_nix_user != 'root' + +- name: Configure OS to disable THP and increase ulimits for splunk process + include_tasks: configure_os.yml + +- name: Include download and unarchive task + include_tasks: download_and_unarchive.yml + +- name: Include configure splunk.secret task to standardize splunk.secret + include_tasks: configure_splunk_secret.yml + when: splunk_configure_secret + +- name: Include configure deployment client task + include_tasks: configure_deploymentclient.yml + when: + - clientName != 'undefined' + - splunk_uri_ds != 'undefined' + +- name: Include configure user-seed task + include_tasks: configure_user-seed.yml + when: + - splunk_user_seed + - splunk_admin_password != 'undefined' + +- name: Include configure default authentication.conf for AD authentication and admin role mapping + include_tasks: configure_authentication.yml + when: + - splunk_configure_authentication + - ad_bind_password != 'undefined' + +- name: Include post-install tasks + include_tasks: post_install.yml + +- name: Enable boot start and accept license + include_tasks: configure_splunk_boot.yml diff --git a/roles/splunk/tasks/install_utilities.yml b/roles/splunk/tasks/install_utilities.yml new file mode 100644 index 00000000..126b01ac --- /dev/null +++ b/roles/splunk/tasks/install_utilities.yml @@ -0,0 +1,11 @@ +--- +# Todo: Use vars to define distribution-specific packages. For now, it'll just ignore errors for any package names that aren't available for your distro. +- name: Install useful packages for Linux troubleshooting + package: + name: "{{ item }}" + state: latest + update_cache: true + loop: "{{ linux_packages }}" + ignore_errors: yes + become: yes + when: install_utilities diff --git a/roles/splunk/tasks/main.yml b/roles/splunk/tasks/main.yml new file mode 100644 index 00000000..bfdb1e8a --- /dev/null +++ b/roles/splunk/tasks/main.yml @@ -0,0 +1,32 @@ +--- +- name: Configure vars for full package + tags: always + set_fact: + splunk_home: "{{ splunk_install_path }}/splunk" + splunk_package_url: "{{ splunk_package_url_full }}" + splunk_product: "Splunk" + splunk_install_type: full + changed_when: false + when: "'full' in group_names" + +- name: Configure vars for uf package + tags: always + set_fact: + splunk_home: "{{ splunk_install_path }}/splunkforwarder" + splunk_package_url: "{{ splunk_package_url_uf }}" + splunk_product: "Splunk Universal Forwarder" + splunk_install_type: uf + changed_when: false + when: "'uf' in group_names" + +- name: Send Slack messages + include_tasks: slack_messenger.yml + tags: always + when: + - slack_token != 'undefined' + - slack_channel != 'undefined' + +# Always set the deployment_task var to your desired task in the playbook! e.g. check_splunk.yml (install|upgrade) or configure_apps.yml (app/addon install|upgrade) +- name: Include the specified task file + tags: always + include_tasks: "{{ deployment_task }}" diff --git a/roles/splunk/tasks/post_install.yml b/roles/splunk/tasks/post_install.yml new file mode 100644 index 00000000..fe47331f --- /dev/null +++ b/roles/splunk/tasks/post_install.yml @@ -0,0 +1,32 @@ +--- +- name: Touch .ui_login file to disable first-time login prompt + file: + dest: "{{ splunk_home }}/etc/.ui_login" + state: touch + become: yes + become_user: "{{ splunk_nix_user }}" + when: "'full' in group_names" + +- name: "Ensure correct ownership for {{ splunk_home }}/etc" + file: + path: "{{ splunk_home }}/etc" + owner: "{{ splunk_nix_user }}" + group: "{{ splunk_nix_group }}" + recurse: yes + become: yes + +- name: Configure cron job to cleanup crash logs older than 7 days + include_tasks: add_crashlog_script.yml + when: add_crashlog_script + +- name: Configure cron job to cleanup diags older than 30 days + include_tasks: add_diag_script.yml + when: add_diag_script + +- name: Install pstack shell script for troubleshooting purposes + include_tasks: add_pstack_script.yml + when: add_pstack_script + +- name: Install additional utilities and troubleshooting tools + include_tasks: install_utilities.yml + when: install_utilities diff --git a/roles/splunk/tasks/set_maintenance_mode.yml b/roles/splunk/tasks/set_maintenance_mode.yml new file mode 100644 index 00000000..ee4534d5 --- /dev/null +++ b/roles/splunk/tasks/set_maintenance_mode.yml @@ -0,0 +1,8 @@ +--- +- name: "{{ state }} maintenance mode on cm" + shell: "{{ splunk_home }}/bin/splunk {{ state }} maintenance-mode --answer-yes --skip-validation -auth {{ splunk_auth }}" + become: yes + become_user: "{{ splunk_nix_user}}" + when: + - state is defined and splunk_auth is defined + - "'clustermaster' in group_names" diff --git a/roles/splunk/tasks/set_upgrade_state.yml b/roles/splunk/tasks/set_upgrade_state.yml new file mode 100644 index 00000000..c37c14e8 --- /dev/null +++ b/roles/splunk/tasks/set_upgrade_state.yml @@ -0,0 +1,8 @@ +--- +- name: "Run splunk upgrade-{{ peer_state }} cluster-peers on CM" + shell: "{{ splunk_home }}/bin/splunk upgrade-{{ peer_state }} cluster-peers -auth {{ splunk_auth }}" + become: yes + become_user: "{{ splunk_nix_user}}" + when: + - peer_state is defined and splunk_auth is defined + - "'clustermaster' in group_names" diff --git a/roles/splunk/tasks/slack_messenger.yml b/roles/splunk/tasks/slack_messenger.yml new file mode 100644 index 00000000..ff6fa973 --- /dev/null +++ b/roles/splunk/tasks/slack_messenger.yml @@ -0,0 +1,15 @@ +--- +# This task feeds messags to Slack about what Ansible is doing +- name: "Send Slack message to {{ slack_channel }} indicating Ansible play information" + local_action: + module: slack + msg: "{{ item }}" + token: "{{ slack_token }}" + channel: "{{ slack_channel }}" + ignore_errors: true + run_once: true + loop: + - "****************\n*Starting new Ansible play...*\n```Ansible Role: {{ role_name }}\nAnsible Task: {{ deployment_task }}\nTarget Host: {{ ansible_fqdn }}\nStart Time: {{ ansible_date_time.iso8601 }}```" + when: + - slack_token != 'undefined' + - slack_channel != 'undefined' diff --git a/roles/splunk/tasks/splunk_offline.yml b/roles/splunk/tasks/splunk_offline.yml new file mode 100644 index 00000000..3660c54e --- /dev/null +++ b/roles/splunk/tasks/splunk_offline.yml @@ -0,0 +1,6 @@ +--- +- name: splunk offline + shell: "{{ splunk_home }}/bin/splunk offline -auth {{ splunk_auth }}" + become: yes + become_user: "{{ splunk_nix_user }}" + no_log: true diff --git a/roles/splunk/tasks/splunk_restart.yml b/roles/splunk/tasks/splunk_restart.yml new file mode 100644 index 00000000..9f105899 --- /dev/null +++ b/roles/splunk/tasks/splunk_restart.yml @@ -0,0 +1,5 @@ +--- +- name: restart splunk + shell: "{{ splunk_home }}/bin/splunk restart" + become: yes + become_user: "{{ splunk_nix_user }}" diff --git a/roles/splunk/tasks/splunk_start.yml b/roles/splunk/tasks/splunk_start.yml new file mode 100644 index 00000000..1d3afc25 --- /dev/null +++ b/roles/splunk/tasks/splunk_start.yml @@ -0,0 +1,5 @@ +--- +- name: start splunk + shell: "{{ splunk_home }}/bin/splunk start --answer-yes --auto-ports --no-prompt --accept-license" + become: yes + become_user: "{{ splunk_nix_user }}" diff --git a/roles/splunk/tasks/splunk_stop.yml b/roles/splunk/tasks/splunk_stop.yml new file mode 100644 index 00000000..c85fcd99 --- /dev/null +++ b/roles/splunk/tasks/splunk_stop.yml @@ -0,0 +1,5 @@ +--- +- name: stop splunk + shell: "{{ splunk_home }}/bin/splunk stop" + become: yes + ignore_errors: yes diff --git a/roles/splunk/tasks/upgrade_splunk.yml b/roles/splunk/tasks/upgrade_splunk.yml new file mode 100644 index 00000000..2efe1891 --- /dev/null +++ b/roles/splunk/tasks/upgrade_splunk.yml @@ -0,0 +1,26 @@ +--- +# This task is called by check_splunk.yml to perform splunk upgrades. Do not run this task directly as check_splunk.yml will stop splunk first. +- name: Configure .bash_profile and .bashrc for splunk user + include_tasks: configure_bash.yml + when: splunk_nix_user != 'root' + +- name: Configure OS to disable THP and increase ulimits for splunk process + include_tasks: configure_os.yml + +- name: Disable boot-start to ensure restart is successful in 7.2.2+ (this will get re-enabled by a handler with the correct parameters at the end) + shell: "{{ splunk_home }}/bin/splunk disable boot-start -user {{ splunk_nix_user }}" + become: yes + ignore_errors: true + +- name: Include task to kill any stale splunkd processes + include_tasks: kill_splunkd.yml + +- name: Include download and unarchive task + include_tasks: download_and_unarchive.yml + +- name: Include mongod tasks to ensure it's in a good state + include_tasks: fix_mongo.yml + when: "'full' in group_names" + +- name: Include post-install tasks + include_tasks: post_install.yml diff --git a/roles/splunk/templates/authentication.conf.j2 b/roles/splunk/templates/authentication.conf.j2 new file mode 100644 index 00000000..2dc50d4d --- /dev/null +++ b/roles/splunk/templates/authentication.conf.j2 @@ -0,0 +1,4 @@ +# Place your authentication.conf here +# Suggest using a variable for this line: +# bindDNpassword = {{ ad_bind_password }} +# and encrypting ad_bind_password using ansible-vault encrypt_string, e.g. ansible-vault encrypt_string --ask-vault-pass 'var_value_to_encrypt' --name 'var_name' diff --git a/roles/splunk/templates/bash_profile.j2 b/roles/splunk/templates/bash_profile.j2 new file mode 100644 index 00000000..5dc72319 --- /dev/null +++ b/roles/splunk/templates/bash_profile.j2 @@ -0,0 +1,14 @@ +# .bash_profile + +# Get the aliases and functions +if [ -f ~/.bashrc ]; then + . ~/.bashrc +fi + +# User specific environment and startup programs + +PATH=$PATH:$HOME/bin + +export PATH + +export PS1='[\u@\H:$( pwd )]\$ ' diff --git a/roles/splunk/templates/bashrc.j2 b/roles/splunk/templates/bashrc.j2 new file mode 100644 index 00000000..196b7bcf --- /dev/null +++ b/roles/splunk/templates/bashrc.j2 @@ -0,0 +1,18 @@ +# .bashrc + +# Source global definitions +if [ -f /etc/bashrc ]; then + . /etc/bashrc +fi + +# Uncomment the following line if you don't like systemctl's auto-paging feature: +# export SYSTEMD_PAGER= + +# User specific aliases and functions +export PS1='[\u@\H:$( pwd )]\$ ' +export SPLUNK_HOME={{ splunk_home }} +export PATH=$SPLUNK_HOME/bin:$PATH + +if [ -f $SPLUNK_HOME/share/splunk/cli-command-completion.sh ]; then + . $SPLUNK_HOME/share/splunk/cli-command-completion.sh +fi diff --git a/roles/splunk/templates/cleanup_crashlogs.sh.j2 b/roles/splunk/templates/cleanup_crashlogs.sh.j2 new file mode 100644 index 00000000..5a252ef4 --- /dev/null +++ b/roles/splunk/templates/cleanup_crashlogs.sh.j2 @@ -0,0 +1,4 @@ +#!/bin/bash +# Delete crash logs older than 7 days +find {{ splunk_home }}/var/log/splunk -name "crash*.log" -mtime +7 -exec rm {} \; +exit diff --git a/roles/splunk/templates/cleanup_diags.sh.j2 b/roles/splunk/templates/cleanup_diags.sh.j2 new file mode 100644 index 00000000..c08b7a73 --- /dev/null +++ b/roles/splunk/templates/cleanup_diags.sh.j2 @@ -0,0 +1,4 @@ +#!/bin/bash +# Delete diags older than 30 days +find {{ splunk_home }} -name "diag*.tar.gz" -mtime +30 -exec rm {} \; +exit diff --git a/roles/splunk/templates/deploymentclient.conf.j2 b/roles/splunk/templates/deploymentclient.conf.j2 new file mode 100644 index 00000000..43b10d32 --- /dev/null +++ b/roles/splunk/templates/deploymentclient.conf.j2 @@ -0,0 +1,5 @@ +[deployment-client] +clientName = {{ clientName }} + +[target-broker:deploymentServer] +targetUri = {{ splunk_uri_ds }} diff --git a/roles/splunk/templates/genpstacks.sh.j2 b/roles/splunk/templates/genpstacks.sh.j2 new file mode 100755 index 00000000..3260d607 --- /dev/null +++ b/roles/splunk/templates/genpstacks.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash +# Script by Mason Morales, Splunk, Inc. +SPLUNK_HOME={{ splunk_home }} +OUTPUT_DIR=/tmp/splunk_pstacks +SAMPLES=100 +SAMPLE_PERIOD=3 +HOST=$(hostname) +USER={{ ansible_user }} +mkdir -p $OUTPUT_DIR +i=0 +echo "########## GENERATING PSTACKS, PLEASE WAIT.... THIS WILL TAKE 5 MINUTES TO COMPLETE. ##########" +while [ $i -lt $SAMPLES ] +do + pstack `head -1 $SPLUNK_HOME/var/run/splunk/splunkd.pid` > $OUTPUT_DIR/pstack_splunkd-$i-`date +%s`.out + let "i+=1" + sleep $SAMPLE_PERIOD +done +TIMESTAMP=$(date +"%Y%m%dT%H%M") +TARBALL=$HOST-pstacks-$TIMESTAMP.tar +echo "########## DONE, CREATING TARBALL.... ##########" +tar czvf $TARBALL $OUTPUT_DIR/ +echo "########## DONE, CLEANING UP.... ##########" +rm -rf $OUTPUT_DIR/*.out +# Test +mv $TARBALL /tmp/ +chown $USER /tmp/$TARBALL +echo "########## PSTACK CAPUTRE COMPLETE! ##########" +echo "Transfer command: scp $USER@$HOST:/tmp/$TARBALL ." diff --git a/roles/splunk/templates/splunk_facl.j2 b/roles/splunk/templates/splunk_facl.j2 new file mode 100644 index 00000000..63ee37dc --- /dev/null +++ b/roles/splunk/templates/splunk_facl.j2 @@ -0,0 +1,5 @@ +{ + postrotate + /usr/bin/setfacl -Rm u:{{ splunk_nix_user }}:rx /var/log + endscript +} diff --git a/roles/splunk/templates/user-seed.conf.j2 b/roles/splunk/templates/user-seed.conf.j2 new file mode 100644 index 00000000..998a9bc5 --- /dev/null +++ b/roles/splunk/templates/user-seed.conf.j2 @@ -0,0 +1,3 @@ +[user_info] +USERNAME = {{ splunk_admin_username }} +PASSWORD = {{ splunk_admin_password }} diff --git a/roles/splunk/vars/Debian.yml b/roles/splunk/vars/Debian.yml new file mode 100644 index 00000000..898f11e7 --- /dev/null +++ b/roles/splunk/vars/Debian.yml @@ -0,0 +1,2 @@ +--- +global_bashrc: /etc/bash.bashrc diff --git a/roles/splunk/vars/RedHat.yml b/roles/splunk/vars/RedHat.yml new file mode 100644 index 00000000..5f3e8ae3 --- /dev/null +++ b/roles/splunk/vars/RedHat.yml @@ -0,0 +1,3 @@ +--- +global_bashrc: /etc/bashrc +chk_config_cmd: chkconfig --add disable-thp diff --git a/roles/splunk/vars/main.yml b/roles/splunk/vars/main.yml new file mode 100644 index 00000000..3a3a4d50 --- /dev/null +++ b/roles/splunk/vars/main.yml @@ -0,0 +1,8 @@ +--- +# Automatically determine desired splunk version string +splunk_file: "{{ splunk_package_url | regex_search('(splunk(?:forwarder)?\\-\\d+[^/]*\\.tgz)') }}" +splunk_v: "{{ splunk_package_url | regex_search('(\\d+.\\d+.\\d+(?:.\\d+)?)') }}" +splunk_build: "{{ splunk_package_url | regex_search('\\d+.\\d+.\\d+(?:.\\d+)?\\-([^-]+)','\\1') | first }}" +# Create desired splunk version string (to compare with the output from the splunk version command for upgrades) +splunk_version: "{{ splunk_product }} {{ splunk_v }} (build {{ splunk_build }})" +splunk_auth: "{{ splunk_admin_username }}:{{ splunk_admin_password }}"