Simplify exporting dashboards for Devs (#7730)

A new `mage` command is introduced to simplify exporting dashboards and contributing them to Beats. Currently a contributor is required to know about folder hierarchy under which dashboards are stored and are versioned. To simply this a simple `mage` command is provided:

```
MODULE=mysql ID=the-dashboard-id mage ExportDashboard
```

The Dev must provide the module he wants to export the dashboard for and the dashboard id. The script will take care of extracting it from Kibana, creating directories and files. The current script only supports to export dashboards from localhost:5061 as no params are provided to change the host at the moment. Also this change is only for Metricbeat for now. Filebeat should also be added.

Additional changes:

* Create directories
* Pass module and dashboard id
* Add exit code 1 in case of export failure
* Improve error handling

CHANGELOG-developer.asciidoc

@@ -67,3 +67,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889]
 - Add `cfgwarn.CheckRemoved6xSetting(s)` to display a warning for options removed in 7.0. {pull}8909[8909]
 - Add docker image building to `mage.Package`. {pull}8898[8898]
+- Simplified exporting of dashboards. {pull}7730[7730]

auditbeat/magefile.go

@@ -103,6 +103,15 @@ func Fields() error {
 	return mage.GenerateFieldsYAML("module")
 }
 
+// ExportDashboard exports a dashboard and writes it into the correct directory
+//
+// Required ENV variables:
+// * MODULE: Name of the module
+// * ID: Dashboard id
+func ExportDashboard() error {
+	return mage.ExportDashboard()
+}
+
 // GoTestUnit executes the Go unit tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.

dev-tools/cmd/dashboards/export_dashboards.go

@@ -89,6 +89,7 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string,
 	for _, obj := range objects {
 		o := obj.(common.MapStr)
 
+		// All fields are optional, so errors are not catched
 		decodeValue(o, "attributes.uiStateJSON")
 		decodeValue(o, "attributes.visState")
 		decodeValue(o, "attributes.optionsJSON")
@@ -104,23 +105,27 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string,
 		return err
 	}
 
-	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0666)
+	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0644)
 	if !quiet {
 		log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out)
 	}
 	return err
 }
 
-func decodeValue(data common.MapStr, key string) {
+func decodeValue(data common.MapStr, key string) error {
 	v, err := data.GetValue(key)
 	if err != nil {
-		return
+		return err
 	}
 	s := v.(string)
 	var d interface{}
-	json.Unmarshal([]byte(s), &d)
+	err = json.Unmarshal([]byte(s), &d)
+	if err != nil {
+		return fmt.Errorf("error decoding %s: %v", key, err)
+	}
 
 	data.Put(key, d)
+	return nil
 }
 
 func ReadManifest(file string) ([]map[string]string, error) {

dev-tools/mage/dashboard.go

@@ -0,0 +1,53 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/magefile/mage/sh"
+)
+
+// ExportDashboard exports a dashboard from Kibana and writes it into the given module.
+func ExportDashboard() error {
+	module := EnvOr("MODULE", "")
+	if module == "" {
+		return fmt.Errorf("MODULE must be specified")
+	}
+
+	id := EnvOr("ID", "")
+	if id == "" {
+		return fmt.Errorf("Dashboad ID must be specified")
+	}
+
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+
+	// TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7
+	file := CWD("module", module, "_meta/kibana/6/dashboard", id+".json")
+
+	dashboardCmd := sh.RunCmd("go", "run",
+		filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"),
+		"-output", file, "-dashboard", id,
+	)
+
+	return dashboardCmd()
+}

docs/devguide/newdashboards.asciidoc

@@ -213,44 +213,24 @@ https://github.com/elastic/beats/tree/master/dev-tools/cmd/dashboards[dev-tools]
 for exporting Kibana 5.x dashboards. See the dev-tools
 https://github.com/elastic/beats/tree/master/dev-tools/README.md[readme] for more info.
 
-/////////////////////
-NOT YET IMPLEMENTED
-NOTE: You can make use of the Makefile from the Beat GitHub repository to export all the Kibana dashboards for a Beat
-from your Elasticsearch. If Elasticsearch is running on localhost, then you just need to run the following command from the Beat repository:
-
-[source,shell]
------------------------------
-make export-dashboards
------------------------------
-
-If Elasticsearch is running on a different host, then you can use the `ES_URL` variable:
-
-[source,shell]
-----------------------------
-ES_URL="http://192.168.3.206:9200" make export-dashboards
-----------------------------
-
-/////////////////////
-
 ==== Exporting Kibana 6.0 dashboards and newer
 
 The `dev-tools/cmd/export_dashboards.go` script helps you export your customized Kibana 6.0 dashboards and newer. You might need to export a single dashboard or all the dashboards available for a module or Beat.
 
-
 ===== Export a single Kibana dashboard
 
-You can export a single dashboard by passing the dashboard ID in the `-dashboard` flag.
-
-NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is
-`app/kibana#/dashboard/7fea2930-478e-11e7-b1f0-cb29bac6bf8b?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `7fea2930-478e-11e7-b1f0-cb29bac6bf8b`.
+To export a single dashboard for a module you can use the following command inside a Beat with modules:
 
 [source,shell]
 ---------------
-cd filebeat/module/redis/_meta/kibana/default/dashboard
-go run ../../../../../../../dev-tools/cmd/dashboards/export_dashboards.go -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -output Filebeat-redis.json
+MODULE=redis ID=AV4REOpp5NkDleZmzKkE mage exportDashboard
 ---------------
 
-This generates the `Filebeat-redis.json` file that contains the dashboard for the Redis module of Filebeat, including the dependencies (visualizations and searches).
+This generates a `AV4REOpp5NkDleZmzKkE.json` file inside dashboard directory in the redis module.
+It contains all dependencies like visualizations and searches.
+
+NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is
+`app/kibana#/dashboard/AV4REOpp5NkDleZmzKkE?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `AV4REOpp5NkDleZmzKkE`.
 
 ===== Export all module/Beat dashboards
 

filebeat/magefile.go

@@ -125,6 +125,15 @@ func GoTestIntegration(ctx context.Context) error {
 	return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs())
 }
 
+// ExportDashboard exports a dashboard and writes it into the correct directory
+//
+// Required ENV variables:
+// * MODULE: Name of the module
+// * ID: Dashboard id
+func ExportDashboard() error {
+	return mage.ExportDashboard()
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Filebeat.
 // - Include modules directory in packages (minus _meta and test files).

metricbeat/magefile.go

@@ -117,6 +117,15 @@ func GoTestIntegration(ctx context.Context) error {
 	return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs())
 }
 
+// ExportDashboard exports a dashboard and writes it into the correct directory
+//
+// Required ENV variables:
+// * MODULE: Name of the module
+// * ID: Dashboard id
+func ExportDashboard() error {
+	return mage.ExportDashboard()
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Metricbeat.
 // - Include modules.d directory in packages.