Skip to content

Commit

Permalink
Refactor idxmgmt supporter and export commands. (#11777)
Browse files Browse the repository at this point in the history
Remove methods from idxmgmt supporter interface for more flexibility. Adapt idxmgmt supporter to work with more generic client, allowing for reuse in export commands. Refactor export commands and allow printing to file for exporting ilm policies and templates. Improve beats creation and initialization process.
  • Loading branch information
simitt authored Apr 23, 2019
1 parent c5ca01b commit 12fe5f8
Show file tree
Hide file tree
Showing 37 changed files with 1,082 additions and 609 deletions.
1 change: 1 addition & 0 deletions CHANGELOG-developer.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
by `make` and `mage`. Example: `export PYTHON_EXE=python2.7`. {pull}11212[11212]
- Prometheus helper for metricbeat contains now `Namespace` field for `prometheus.MetricsMappings` {pull}11424[11424]
- Update Jinja2 version to 2.10.1. {pull}11817[11817]
- Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777]
15 changes: 4 additions & 11 deletions libbeat/cmd/export/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package export

import (
"fmt"
"os"

"github.com/spf13/cobra"
Expand All @@ -40,26 +39,20 @@ func GenExportConfigCmd(settings instance.Settings) *cobra.Command {
}

func exportConfig(settings instance.Settings) error {
b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
if err != nil {
return fmt.Errorf("error initializing beat: %s", err)
}

settings.DisableConfigResolver = true

err = b.InitWithSettings(settings)
b, err := instance.NewInitializedBeat(settings)
if err != nil {
return fmt.Errorf("error initializing beat: %s", err)
fatalfInitCmd(err)
}

var config map[string]interface{}
err = b.RawConfig.Unpack(&config)
if err != nil {
return fmt.Errorf("error unpacking config, error: %s", err)
fatalf("Error unpacking config: %+v.", err)
}
res, err := yaml.Marshal(config)
if err != nil {
return fmt.Errorf("Error converting config to YAML format, error: %s", err)
fatalf("Error converting config to YAML format: %+v.", err)
}

os.Stdout.Write(res)
Expand Down
23 changes: 6 additions & 17 deletions libbeat/cmd/export/dashboard.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package export

import (
"fmt"
"os"
"path/filepath"

"github.com/spf13/cobra"
Expand All @@ -40,15 +39,9 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command {
yml, _ := cmd.Flags().GetString("yml")
decode, _ := cmd.Flags().GetBool("decode")

b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
b, err := instance.NewInitializedBeat(settings)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating beat: %s\n", err)
os.Exit(1)
}
err = b.InitWithSettings(settings)
if err != nil {
fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
os.Exit(1)
fatalfInitCmd(err)
}

// Use empty config to use default configs if not set
Expand All @@ -58,16 +51,14 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command {

client, err := kibana.NewKibanaClient(b.Config.Kibana)
if err != nil {
fmt.Fprintf(os.Stderr, "Error creating Kibana client: %+v\n", err)
os.Exit(1)
fatalf("Error creating Kibana client: %+v.\n", err)
}

// Export dashboards from yml file
if yml != "" {
results, info, err := dashboards.ExportAllFromYml(client, yml)
if err != nil {
fmt.Fprintf(os.Stderr, "Error getting dashboards from yml: %+v\n", err)
os.Exit(1)
fatalf("Error exporting dashboards from yml: %+v.\n", err)
}
for i, r := range results {
if decode {
Expand All @@ -76,9 +67,8 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command {

err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(yml), client.GetVersion())
if err != nil {
fmt.Fprintf(os.Stderr, "Error saving dashboard '%s' to file '%s' : %+v\n",
fatalf("Error saving dashboard '%s' to file '%s' : %+v.\n",
info.Dashboards[i].ID, info.Dashboards[i].File, err)
os.Exit(1)
}
}
return
Expand All @@ -88,8 +78,7 @@ func GenDashboardCmd(settings instance.Settings) *cobra.Command {
if dashboard != "" {
result, err := dashboards.Export(client, dashboard)
if err != nil {
fmt.Fprintf(os.Stderr, "Error getting dashboard: %+v\n", err)
os.Exit(1)
fatalf("Error exporting dashboard: %+v.\n", err)
}

if decode {
Expand Down
112 changes: 112 additions & 0 deletions libbeat/cmd/export/export.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package export

import (
"fmt"
"os"
"path/filepath"

"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/idxmgmt"
"github.com/elastic/beats/libbeat/version"
)

type stdoutClient struct {
ver common.Version
f *os.File
}

type fileClient struct {
ver common.Version
dir string
}

func newIdxmgmtClient(dir string, ver string) idxmgmt.FileClient {
if dir == "" {
c, err := newStdoutClient(ver)
if err != nil {
fatalf("Error creating stdout writer: %+v.", err)
}
return c
}
c, err := newFileClient(dir, ver)
if err != nil {
fatalf("Error creating directory: %+v.", err)
}
return c
}

func newStdoutClient(ver string) (*stdoutClient, error) {
if ver == "" {
ver = version.GetDefaultVersion()
}
v, err := common.NewVersion(ver)
if err != nil {
return nil, err
}
return &stdoutClient{ver: *v, f: os.Stdout}, nil
}

func newFileClient(dir string, ver string) (*fileClient, error) {
if ver == "" {
ver = version.GetDefaultVersion()
}
path, err := filepath.Abs(dir)
if err != nil {
return nil, err
}
err = os.MkdirAll(path, os.ModePerm)
if err != nil {
return nil, err
}
return &fileClient{ver: *common.MustNewVersion(ver), dir: path}, nil
}

func (c *stdoutClient) GetVersion() common.Version {
return c.ver
}

func (c *stdoutClient) Write(_ string, body string) error {
_, err := c.f.WriteString(body)
return err
}

func (c *fileClient) GetVersion() common.Version {
return c.ver
}

func (c *fileClient) Write(name string, body string) error {
f, err := os.Create(filepath.Join(c.dir, fmt.Sprintf("%s.json", name)))
defer f.Close()
if err != nil {
return err
}
_, err = f.WriteString(body)
return err
}

func fatalf(msg string, vs ...interface{}) {
fmt.Fprintf(os.Stderr, msg, vs...)
fmt.Fprintln(os.Stderr)
os.Exit(1)
}

func fatalfInitCmd(err error) {
fatalf("Failed to initialize 'export' command: %+v.", err)
}
36 changes: 13 additions & 23 deletions libbeat/cmd/export/ilm_policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,10 @@
package export

import (
"fmt"
"os"

"github.com/spf13/cobra"

"github.com/elastic/beats/libbeat/cmd/instance"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/idxmgmt/ilm"
"github.com/elastic/beats/libbeat/idxmgmt"
)

// GenGetILMPolicyCmd is the command used to export the ilm policy.
Expand All @@ -34,30 +30,24 @@ func GenGetILMPolicyCmd(settings instance.Settings) *cobra.Command {
Use: "ilm-policy",
Short: "Export ILM policy",
Run: func(cmd *cobra.Command, args []string) {
b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
if err != nil {
fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
os.Exit(1)
}
err = b.InitWithSettings(settings)
if err != nil {
fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
os.Exit(1)
}
version, _ := cmd.Flags().GetString("es.version")
dir, _ := cmd.Flags().GetString("dir")

ilmFactory := settings.ILM
if ilmFactory == nil {
ilmFactory = ilm.DefaultSupport
}

ilm, err := ilmFactory(nil, b.Info, b.RawConfig)
b, err := instance.NewInitializedBeat(settings)
if err != nil {
fmt.Fprintf(os.Stderr, "Error initializing ILM support: %s\n", err)
fatalfInitCmd(err)
}

fmt.Println(common.MapStr(ilm.Policy().Body).StringToPrint())
clientHandler := idxmgmt.NewFileClientHandler(newIdxmgmtClient(dir, version))
idxManager := b.IdxSupporter.Manager(clientHandler, idxmgmt.BeatsAssets(b.Fields))
if err := idxManager.Setup(idxmgmt.LoadModeDisabled, idxmgmt.LoadModeEnabled); err != nil {
fatalf("Error exporting ilm-policy: %+v.", err)
}
},
}

genTemplateConfigCmd.Flags().String("es.version", settings.Version, "Elasticsearch version")
genTemplateConfigCmd.Flags().String("dir", "", "Specify directory for printing policy files. By default policies are printed to stdout.")

return genTemplateConfigCmd
}
17 changes: 6 additions & 11 deletions libbeat/cmd/export/index_pattern.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package export

import (
"log"
"os"

"github.com/spf13/cobra"
Expand All @@ -36,13 +35,9 @@ func GenIndexPatternConfigCmd(settings instance.Settings) *cobra.Command {
Run: func(cmd *cobra.Command, args []string) {
version, _ := cmd.Flags().GetString("es.version")

b, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version)
b, err := instance.NewInitializedBeat(settings)
if err != nil {
fatalf("Error initializing beat: %+v", err)
}
err = b.InitWithSettings(settings)
if err != nil {
fatalf("Error initializing beat: %+v", err)
fatalfInitCmd(err)
}

if version == "" {
Expand All @@ -52,21 +47,21 @@ func GenIndexPatternConfigCmd(settings instance.Settings) *cobra.Command {
// Index pattern generation
v, err := common.NewVersion(version)
if err != nil {
fatalf("Error creating version: %+v", err)
fatalf("Error creating version: %+v.", err)
}
indexPattern, err := kibana.NewGenerator(b.Info.IndexPrefix, b.Info.Beat, b.Fields, settings.Version, *v, b.Config.Migration.Enabled())
if err != nil {
log.Fatal(err)
fatalf("Error creating Kibana Generator: %+v.", err)
}

pattern, err := indexPattern.Generate()
if err != nil {
log.Fatalf("ERROR: %s", err)
fatalf("Error generating Index Pattern: %+v.", err)
}

_, err = os.Stdout.WriteString(pattern.StringToPrint() + "\n")
if err != nil {
fatalf("Error writing index pattern: %+v", err)
fatalf("Error writing Index Pattern: %+v.", err)
}
},
}
Expand Down
Loading

0 comments on commit 12fe5f8

Please sign in to comment.