Install plugins

.circleci/config.yml

@@ -34,6 +34,22 @@ jobs:
           paths:
             - "*"
 
+  fetch-plugins:
+    executor: golang
+    steps:
+      - attach_workspace:
+          at: ./artifacts
+      - run:
+          name: Download Plugins Tarball
+          command: curl -fL https://github.com/observiq/carbon-plugins/releases/latest/download/carbon-plugins.tar.gz -o ./artifacts/plugins.tar.gz
+      - run:
+          name: Download Plugins Zip
+          command: curl -fL https://github.com/observiq/carbon-plugins/releases/latest/download/carbon-plugins.zip -o ./artifacts/plugins.zip
+      - persist_to_workspace:
+          root: ./artifacts
+          paths:
+            - "*"
+
   publish-github-release:
     docker:
       - image: cibuilds/github:0.10
@@ -380,9 +396,16 @@ workflows:
             - test-linux
             - test-macos
             - test-windows
+      - fetch-plugins:
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v\d+\.\d+\.\d+.*/
       - publish-github-release:
           requires:
             - build
+            - fetch-plugins
           filters:
             branches:
               ignore: /.*/
@@ -391,6 +414,7 @@ workflows:
       - publish-docker-release:
           requires:
             - build
+            - fetch-plugins
           filters:
             branches:
               ignore: /.*/

Dockerfile

@@ -1,11 +1,13 @@
 FROM ubuntu:bionic
 
-RUN mkdir -p /carbon_home/plugins
+RUN mkdir -p /carbon_home
 ENV CARBON_HOME=/carbon_home
 RUN echo "pipeline:\n" >> /carbon_home/config.yaml
 RUN apt-get update && apt-get install -y systemd ca-certificates
 
 COPY ./artifacts/carbon_linux_amd64 /carbon_home/carbon
+COPY ./artifacts/carbon-plugins.tar.gz /tmp/carbon-plugins.tar.gz
+RUN tar -zxvf /tmp/carbon-plugins.tar.gz -C /carbon_home/
 ENTRYPOINT /carbon_home/carbon \
   --config /carbon_home/config.yaml \
   --database /carbon_home/carbon.db \

scripts/unix-install.sh

@@ -7,7 +7,8 @@ set -e
 # Agent Constants
 SERVICE_NAME="carbon"
 BINARY_NAME="carbon"
-DOWNLOAD_BASE="https://github.com/observiq/carbon/releases/latest/download"
+DOWNLOAD_BASE="https://github.com/observiq/carbon/releases"
+PLUGINS_PACKAGE="carbon-plugins.tar.gz"
 
 # Script Constants
 PREREQS="curl hostname printf ps sed uname cut tar"
@@ -131,9 +132,9 @@ usage()
   increase_indent
   USAGE=$(cat <<EOF
 Usage:
-  $(fg_yellow '-d, --download-url')
-      Defines the download url of the agent.
-      If not provided, this will default to the standard location.
+  $(fg_yellow '-v, --version')
+      Defines the version of the agent.
+      If not provided, this will default to the latest version.
 
   $(fg_yellow '-i, --install-dir')
       Defines the install directory of the agent.
@@ -215,7 +216,7 @@ setup_installation()
 
     # Installation variables
     set_os
-    set_download_url
+    set_download_urls
     set_install_dir
     set_agent_home
 
@@ -247,30 +248,33 @@ set_os()
   esac
 }
 
-# This will set the url to use when downloading the agent. If provided,
-# it is set by the download-url flag or DOWNLOAD_URL env variable.
-# Otherwise, it is constructed from the download base, agent version,
-# and package name.
-set_download_url()
+# This will set the urls to use when downloading the agent and its plugins.
+# These urls are constructed based on the --version flag or CARBON_VERSION env variable.
+# If not specified, the version defaults to "latest".
+set_download_urls()
 {
-  if [ -z "$download_url" ] ; then
+  if [ -z "$version" ] ; then
     # shellcheck disable=SC2153
-    download_url="$DOWNLOAD_URL"
+    version=$CARBON_VERSION
   fi
 
-  if [ -z "$download_url" ] ; then
-    download_url="$DOWNLOAD_BASE/${BINARY_NAME}_${os}_amd64"
+  if [ -z "$version" ] ; then
+    agent_download_url="$DOWNLOAD_BASE/latest/download/${BINARY_NAME}_${os}_amd64"
+    plugins_download_url="$DOWNLOAD_BASE/latest/download/${PLUGINS_PACKAGE}"
+  else
+    agent_download_url="$DOWNLOAD_BASE/download/$version/${BINARY_NAME}_${os}_amd64"
+    plugins_download_url="$DOWNLOAD_BASE/download/$version/${PLUGINS_PACKAGE}"
   fi
 }
 
-# This will set the install directory of the agent. If provided,
-# it is set by the install-dir flag or INSTALL_DIR env variable.
-# Otherwise, it defaults to an OS specific value.
+# This will set the install directory of the agent.
+# It is set by the --install-dir flag or CARBON_INSTALL_DIR env variable.
+# If not specified, it defaults to an OS specific value.
 set_install_dir()
 {
   if [ -z "$install_dir" ]; then
     # shellcheck disable=SC2153
-    install_dir=$INSTALL_DIR
+    install_dir=$CARBON_INSTALL_DIR
   fi
 
   if [ -z "$install_dir" ]; then
@@ -292,24 +296,23 @@ set_agent_home()
   agent_home="$install_dir/observiq/carbon"
 }
 
-# This will set the user assigned to the agent service. If provided,
-# it is set by the service-user flag or SERVICE_USER env variable.
-# Otherwise, it defaults to root.
+# This will set the user assigned to the agent service.
+# It is set by the --service-user flag or CARBON_SERVICE_USER env variable.
+# If not specified, it defaults to root.
 set_service_user()
 {
   if [ -z "$service_user" ]; then
     # shellcheck disable=SC2153
-    service_user=$SERVICE_USER
+    service_user=$CARBON_SERVICE_USER
   fi
 
   if [ -z "$service_user" ] ; then
     service_user="root"
   fi
 }
 
-# This will set the location of the binary used to launch
-# the agent. This value cannot be overriden and is based
-# on the location of agent_home.
+# This will set the location of the binary used to launch the agent. 
+# This value cannot be overriden and is based on the location of agent_home.
 set_agent_binary()
 {
   agent_binary="$agent_home/$BINARY_NAME"
@@ -405,23 +408,28 @@ install_package()
   mkdir -p "$agent_home"
   succeeded
 
-  info "Creating plugins directory..."
-  mkdir -p "$agent_home/plugins"
-  succeeded
-
   info "Checking that service is not running..."
   stop_service
   succeeded
 
   info "Downloading binary..."
-  curl -L "$download_url" -o "$agent_binary" --progress-bar --fail || error_exit "$LINENO" "Failed to download package"
+  curl -L "$agent_download_url" -o "$agent_binary" --progress-bar --fail || error_exit "$LINENO" "Failed to download package"
   succeeded
 
   info "Setting permissions..."
   chmod +x "$agent_binary"
   ln -sf "$agent_binary" "/usr/local/bin/$BINARY_NAME"
   succeeded
 
+  info "Downloading plugins..."
+  mkdir -p ./tmp
+  curl -L "$plugins_download_url" -o ./tmp/plugins.tar.gz --progress-bar --fail || error_exit "$LINENO" "Failed to download plugins"
+  succeeded
+
+  info "Extracting plugins..."
+  tar -zxvf ./tmp/plugins.tar.gz -C "$agent_home"
+  rm -fr ./tmp
+
   success "Carbon installation complete!"
   decrease_indent
 }
@@ -1000,8 +1008,8 @@ main()
       case "$1" in
         -y|--accept-defaults)
           accept_defaults="yes" ; shift 1 ;;
-        -d|--download-url)
-          download_url=$2 ; shift 2 ;;
+        -v|--version)
+          version=$2 ; shift 2 ;;
         -i|--install-dir)
           install_dir=$2 ; shift 2 ;;
         -u|--service-user)

scripts/windows-install.ps1

@@ -3,7 +3,7 @@ new-module -name LogAgentInstall -scriptblock {
   # Constants
   $DEFAULT_WINDOW_TITLE = $host.ui.rawui.WindowTitle
   $DEFAULT_INSTALL_PATH = 'C:\'
-  $DOWNLOAD_BASE = "https://github.com/observiq/carbon/releases/latest/download"
+  $DOWNLOAD_BASE = "https://github.com/observiq/carbon/releases"
   $SERVICE_NAME = 'carbon'
   $INDENT_WIDTH = '  '
   $MIN_DOT_NET_VERSION = '4.5'
@@ -96,10 +96,10 @@ new-module -name LogAgentInstall -scriptblock {
     Remove-Indent
 
     Show-ColorText ''
-    Show-ColorText '-d, --download_url'
+    Show-ColorText '-v, --version'
     Add-Indent
-    Show-ColorText 'Defines the download url of the agent.' DarkCyan
-    Show-ColorText 'If not provided, this will default to the standard location.' DarkCyan
+    Show-ColorText 'Defines the version of the agent.' DarkCyan
+    Show-ColorText 'If not provided, this will default to the latest version.' DarkCyan
     Remove-Indent
 
     Show-ColorText ''
@@ -213,7 +213,7 @@ new-module -name LogAgentInstall -scriptblock {
     Show-Header "Configuring Installation Variables"
     Add-Indent
     Set-Defaults
-    Set-DownloadURL
+    Set-DownloadURLs
     Set-InstallDir
     Set-HomeDir
     Set-PluginDir
@@ -281,15 +281,22 @@ new-module -name LogAgentInstall -scriptblock {
     Remove-Indent
   }
 
-  # This will set download url for the agent. If not provided as a flag,
-  # this will be constructed from the agent version and download base.
-  function Set-DownloadURL {
-    Show-ColorText 'Configuring download url...'
+  # This will set the urls to use when downloading the agent and its plugins.
+  # These urls are constructed based on the --version flag.
+  # If not specified, the version defaults to "latest".
+  function Set-DownloadURLs {
+    Show-ColorText 'Configuring download urls...'
     Add-Indent
-    if ( !$script:download_url ) {
-      $script:download_url = "$DOWNLOAD_BASE/carbon_windows_amd64"
+    if ( !$script:version ) {
+      $script:agent_download_url = "$DOWNLOAD_BASE/latest/download/carbon_windows_amd64"
+      $script:plugins_download_url = "$DOWNLOAD_BASE/latest/download/carbon-plugins.zip"
     }
-    Show-ColorText "Using download url: " '' "$script:download_url" DarkCyan
+    else {
+      $script:agent_download_url = "$DOWNLOAD_BASE/download/$script:version/carbon_windows_amd64"
+      $script:plugins_download_url = "$DOWNLOAD_BASE/download/$script:version/carbon-plugins.zip"
+    }
+    Show-ColorText "Using agent download url: " '' "$script:agent_download_url" DarkCyan
+    Show-ColorText "Using plugins download url: " '' "$script:plugins_download_url" DarkCyan
     Remove-Indent
   }
 
@@ -403,10 +410,10 @@ new-module -name LogAgentInstall -scriptblock {
     Show-Header "Downloading Carbon Binary"
     Add-Indent
     Show-ColorText 'Downloading binary. Please wait...'
-    Show-ColorText "$INDENT_WIDTH$script:download_url" DarkCyan ' -> ' '' "$script:binary_location" DarkCyan
+    Show-ColorText "$INDENT_WIDTH$script:agent_download_url" DarkCyan ' -> ' '' "$script:binary_location" DarkCyan
     try {
       $WebClient = New-Object System.Net.WebClient
-      $WebClient.DownloadFile($script:download_url, $script:binary_location)
+      $WebClient.DownloadFile($script:agent_download_url, $script:binary_location)
       Complete
     }
     catch {
@@ -417,6 +424,68 @@ new-module -name LogAgentInstall -scriptblock {
     Remove-Indent
   }
 
+  # This will download and extract plugins to the plugins directory.
+  function Get-CarbonPlugins {
+    Show-Header "Downloading Carbon Plugins"
+    Add-Indent
+    Show-ColorText 'Downloading plugins. Please wait...'
+    Show-ColorText "$INDENT_WIDTH$script:plugins_download_url" DarkCyan ' -> ' '' "$script:agent_home\plugins.zip" DarkCyan
+
+    try {
+      New-Item -Path "$script:agent_home\download\tmp" -ItemType "directory"
+      Complete
+    }
+    catch {
+      Failed
+      $error_message = $_.Exception.Message -replace 'Exception calling.*?: ', ''
+      Exit-Error $MyInvocation.ScriptLineNumber "Failed to create tmp directory plugins: $error_message"
+    }
+
+    try {
+      $WebClient = New-Object System.Net.WebClient
+      $WebClient.DownloadFile($script:plugins_download_url, "$script:agent_home\download\plugins.zip")
+      Complete
+    }
+    catch {
+      Failed
+      $error_message = $_.Exception.Message -replace 'Exception calling.*?: ', ''
+      Exit-Error $MyInvocation.ScriptLineNumber "Failed to download plugins: $error_message"
+    }
+
+    try {
+      [System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
+      [System.IO.Compression.ZipFile]::ExtractToDirectory("$script:agent_home\download\plugins.zip", "$script:agent_home\download")
+      Complete
+    }
+    catch {
+      Failed
+      $error_message = $_.Exception.Message -replace 'Exception calling.*?: ', ''
+      Exit-Error $MyInvocation.ScriptLineNumber "Failed to expand plugins archive: $error_message"
+    }
+
+    try {
+      Copy-Item -Path "$script:agent_home\download\plugins\*.yaml" -Destination "$script:agent_home\plugins"
+      Complete
+    }
+    catch {
+      Failed
+      $error_message = $_.Exception.Message -replace 'Exception calling.*?: ', ''
+      Exit-Error $MyInvocation.ScriptLineNumber "Failed to relocate plugins: $error_message"
+    }
+
+    try {
+      Remove-Item -Path "$script:agent_home\download" -Recurse
+      Complete
+    }
+    catch {
+      Failed
+      $error_message = $_.Exception.Message -replace 'Exception calling.*?: ', ''
+      Exit-Error $MyInvocation.ScriptLineNumber "Failed to clean up download: $error_message"
+    }
+
+    Remove-Indent
+  }
+
   # This will remove the agent service.
   function Remove-AgentService {
     $service = Get-Service $SERVICE_NAME -ErrorAction SilentlyContinue
@@ -593,8 +662,8 @@ pipeline:
       [Alias('y', 'accept_defaults')]
       [string]$script:accept_defaults,
 
-      [Alias('d', 'download_url')]
-      [string]$script:download_url,
+      [Alias('v', 'version')]
+      [string]$script:version,
 
       [Alias('i', 'install_dir')]
       [string]$script:install_dir = '',
@@ -624,6 +693,7 @@ pipeline:
         Set-Permissions
         Remove-AgentService
         Get-CarbonBinary
+        Get-CarbonPlugins
         New-AgentConfig
         New-AgentService
         Complete-Install