Initial configuration files for 3 main applications and exporters build script

.build/tar/build_exporters.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# The use of >/dev/null is to supress stdout, but allow stderr to come thorugh
+# This is because installing the tools and building makes a lot of output
+
+echo "Installing tools needed"
+
+# Install git and maven for building the exporters
+sudo apt-get update >/dev/null
+sudo apt-get install -y git >/dev/null
+sudo apt-get install -y maven >/dev/null
+
+# check dist folder
+if [ ! -d "dist" ]
+then
+    echo "Not found: dist folder, creating the folder dist"
+    mkdir dist
+fi
+
+# Check for prometheus_exporters folder
+if [ ! -d "dist/prometheus_exporters" ]
+then
+    echo "Not found: prometheus_exporters folder, creating the folder"
+    mkdir dist/prometheus_exporters
+fi
+
+echo "Building Prometheus exporters"
+
+# Clone mysqld exporter, build the binary and put it in the folder
+echo "Cloning and building the MySQL exporter"
+
+git clone https://github.com/prometheus/mysqld_exporter.git -q
+cd mysqld_exporter
+make >/dev/null
+cd ..
+cp mysqld_exporter/mysqld_exporter dist/prometheus_exporters/mysqld_exporter
+rm -rf mysqld_exporter
+
+# Clone Apache exporter, build the binary and put it in the folder
+echo "Cloning and building the Apache exporter"
+
+git clone https://github.com/Lusitaniae/apache_exporter.git -q
+cd apache_exporter
+make >/dev/null
+cd ..
+cp apache_exporter/apache_exporter dist/prometheus_exporters/apache_exporter
+rm -rf apache_exporter
+
+# Clone JVM exporter, build the binary and put it in the folder
+echo "Cloning and building the JVM exporter"
+
+git clone https://github.com/prometheus/jmx_exporter.git -q
+cd jmx_exporter
+version=$(sed -n -e 's#.*<version>\(.*-SNAPSHOT\)</version>#\1#p' pom.xml)
+mvn package >/dev/null
+cd ..
+cp jmx_exporter/jmx_prometheus_httpserver/target/jmx_prometheus_httpserver-${version}-jar-with-dependencies.jar dist/prometheus_exporters/jmx_exporter.jar
+rm -rf jmx_exporter
+
+echo "Prometheus exporters built"

.build/tar/generate_tar.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
 CONFIG_FILE=${CONFIG_FILE:-config-example.yaml}
+EXPORTERS_DIRECTORY=prometheus_exporters
+EXPORTERS_CONFIG_DIRECTORY=exporter_configs
 
 echo "Start building tarball distribution file"
 
@@ -11,22 +13,18 @@ then
     exit 1
 fi
 
-# check dist folder
-if [ ! -d "dist" ]
-then
-    echo "Not found: dist folder, creating the folder dist"
-    mkdir dist
-fi
-
 # move the needed files into dist folder
 echo "Organizing files to be compressed"
 cp config/$CONFIG_FILE dist/
 cp bin/$OTELCOL_BINARY dist/
+cp -r config/$EXPORTERS_CONFIG_DIRECTORY dist/
 
 # compress the binary and the config into a .tar file
 echo "Compressing..."
-cd dist && tar -cvzf google-cloudops-opentelemetry-collector.tar.gz $OTELCOL_BINARY $CONFIG_FILE
+cd dist && tar -cvzf google-cloudops-opentelemetry-collector.tar.gz $OTELCOL_BINARY $CONFIG_FILE $EXPORTERS_DIRECTORY $EXPORTERS_CONFIG_DIRECTORY
 
 # remove the folders and files that were added
 echo "Cleaning up"
 rm $OTELCOL_BINARY $CONFIG_FILE
+rm -rf $EXPORTERS_DIRECTORY
+rm -rf $EXPORTERS_CONFIG_DIRECTORY

Makefile

@@ -28,8 +28,7 @@ ifeq ($(BUILD_IMAGE_NAME),)
 BUILD_IMAGE_NAME=otelopscol-build
 endif
 
-OTELCOL_BINARY=google-cloudops-opentelemetry-collector
-
+OTELCOL_BINARY=google-cloudops-opentelemetry-collector_$(GOOS)_$(GOARCH)$(EXTENSION)
 .EXPORT_ALL_VARIABLES:
 
 # --------------------------
@@ -38,7 +37,7 @@ OTELCOL_BINARY=google-cloudops-opentelemetry-collector
 
 .PHONY: build
 build:
-	go build -o ./bin/$(OTELCOL_BINARY)_$(GOOS)_$(GOARCH)$(EXTENSION) ./cmd/otelopscol
+	go build -o ./bin/$(OTELCOL_BINARY) ./cmd/otelopscol
 
 # googet (Windows)
 
@@ -57,14 +56,19 @@ package-goo:
 	goopack -output_dir ./dist <(envsubst < ./.build/googet/google-cloudops-opentelemetry-collector.goospec)
 	chmod -R 777 ./dist/
 
+# exporters
+.PHONY: build-exporters
+build-exporters:
+	bash ./.build/tar/build_exporters.sh
+
 # tarball
 
 .PHONY: build-tarball
-build-tarball: build package-tarball
+build-tarball: build build-exporters package-tarball
 
 .PHONY: package-tarball
 package-tarball:
-	./.build/tar/generate_tar.sh
+	bash ./.build/tar/generate_tar.sh
 	chmod -R 777 ./dist/
 
 # --------------------

cmd/otelopscol/components.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"github.com/Nicolas-MacBeth/opentelemetry-collector-contrib/receiver/prometheusexecreceiver" //update when merged
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/stackdriverexporter"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor"
 	"go.opentelemetry.io/collector/component"
@@ -39,7 +40,9 @@ func components() (config.Factories, error) {
 		errs = append(errs, err)
 	}
 
-	receivers := []component.ReceiverFactoryBase{}
+	receivers := []component.ReceiverFactoryBase{
+		&prometheusexecreceiver.Factory{},
+	}
 	for _, rcv := range factories.Receivers {
 		receivers = append(receivers, rcv)
 	}

config/config-mysql_apache_jvm.yaml

@@ -0,0 +1,83 @@
+# This config hopes to mimick the Cloud Monitoring Agent by collecting metrics from popular third-party services the Agent already supports
+# and formatting the metrics in the same way, so they appear similarly in Cloud Monitoring. The services targeted here are MySQL, Apache and JVM.
+# Please remove/comment-out the prometheus_exec configs that you don't need in the receivers section as well as the list in the service/pipelines/metrics/receivers section.
+# Documentation specific to each exporter is found in the comment directly above its configuration
+
+receivers:
+
+  # https://github.com/prometheus/mysqld_exporter
+  prometheus_exec/mysql:
+    exec: ./prometheus_exporters/mysqld_exporter --web.listen-address=":{{port}}" --no-collect.global_variables  # this last flag reduces the amount of generated metrics, can be removed to enable all (600+) MySQL metrics
+    port: 9104
+    env:
+    - name: DATA_SOURCE_NAME  # DATA_SOURCE_NAME and its value are needed to connect to the MySQL server, alternatively you could use a .my.cnf configuration file
+      value: username:password@(host:port)/db_name  # replace all value here with your MySQL server credentials
+
+  # https://github.com/Lusitaniae/apache_exporter
+  prometheus_exec/apache:
+    exec: ./prometheus_exporters/apache_exporter --telemetry.address=":{{port}}"
+    port: 9117
+
+  # https://github.com/prometheus/jmx_exporter
+  prometheus_exec/jvm:
+    exec: java -jar ./prometheus_exporters/jmx_exporter.jar {{port}} exporter_configs/jvm_config.yaml # see jvm_config.yaml for JMX configuration, notably the "hostPort" key
+    port: 9404
+
+processors:
+  filter:
+    metrics:
+      include:
+        match_type: regexp
+        metric_names: # Allowing all JVM and Apache metrics, but whitelisting MySQL ones to filter hundreds of unused ones
+                - mysql_global_status_commands_total
+                - mysql_global_status_threads_running
+                - mysql_global_status_handlers_total
+                - apache_.*
+                - jmx_.*
+                - java_.*
+                - promhttp_.*
+
+  metricstransform/mysql:
+    transforms:
+      - metric_name: mysql_global_status_commands_total
+        action: update
+        new_name: mysql/command_count
+        operations:
+          # change data type from double -> int64
+          - action: toggle_scalar_data_type
+      - metric_name: mysql_global_status_threads_running
+        action: update
+        new_name: mysql/thread_count
+      - metric_name: mysql_global_status_handlers_total
+        action: update
+        new_name: mysql/handler_count
+
+  metricstransform/java:
+    transforms:
+      - metric_name: java_lang_GarbageCollector_CollectionCount
+        action: update
+        new_name: jvm/gc/count
+
+  metricstransform/apache:
+    transforms:
+      - metric_name: promhttp_metric_handler_requests_total
+        action: update
+        new_name: apache/request_count
+        operations:
+          - action: aggregate_labels
+            label_set: []
+            aggregation_type: sum
+            # change data type from double -> int64
+          - action: toggle_scalar_data_type
+
+exporters:
+  stackdriver:  
+    project:  # stackdriver will use the default GCE credentials
+    #metric_prefix: agent.googleapis.com/
+
+service:
+  pipelines:
+    metrics:
+      receivers: [prometheus_exec/mysql, prometheus_exec/apache, prometheus_exec/jvm]  # Remember to remove unwanted receivers from this list
+      processors: [filter, metricstransform/mysql, metricstransform/java]
+      exporters: [stackdriver]

config/exporter_configs/jvm_config.yaml

@@ -0,0 +1,6 @@
+# Simple JMX exporter config file, please change host:port of your particular JVM configuration
+# Documentation: https://github.com/prometheus/jmx_exporter
+
+jmxUrl: service:jmx:rmi:///jndi/rmi://host:port/jmxrmi
+startDelaySeconds: 0
+ssl: false

go.mod

@@ -3,6 +3,7 @@ module github.com/GoogleCloudPlatform/opentelemetry-operations-collector
 go 1.14
 
 require (
+	github.com/Nicolas-MacBeth/opentelemetry-collector-contrib/receiver/prometheusexecreceiver v0.0.0-20200730172100-e63745f3601e // update when merged
 	github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/hashicorp/consul/api v1.4.0 // indirect

go.sum

@@ -58,6 +58,8 @@ github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/Nicolas-MacBeth/opentelemetry-collector-contrib/receiver/prometheusexecreceiver v0.0.0-20200730172100-e63745f3601e h1:yZLatx4UQSpOT4sZ4Ym3gO40OMedAieydLHw2vnUhSE=
+github.com/Nicolas-MacBeth/opentelemetry-collector-contrib/receiver/prometheusexecreceiver v0.0.0-20200730172100-e63745f3601e/go.mod h1:LQUXv42nhnFoqOeZopxsWTjWATwd2uk41+3UojST9dU=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI=
@@ -851,6 +853,7 @@ github.com/karrick/godirwalk v1.7.7/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46s
 github.com/karrick/godirwalk v1.7.8/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=