Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move to a modules.d layout #4537

Merged
merged 14 commits into from
Jul 7, 2017
Merged
1 change: 1 addition & 0 deletions dev-tools/packer/platforms/binary/run.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ cp {{.beat_name}}-linux-{{.arch}} /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}
cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.yml
cp {{.beat_name}}-linux.reference.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.reference.yml
cp fields.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
cp -a modules.d-linux/ /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/modules.d || true

mkdir -p upload
tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}
Expand Down
40 changes: 23 additions & 17 deletions dev-tools/packer/platforms/centos/run.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -21,26 +21,32 @@ fi
RPM_VERSION=`echo ${VERSION} | sed 's/-/_/g'`

# create rpm
fpm --force -s dir -t rpm \
-n {{.beat_pkg_name}} -v ${RPM_VERSION} \
--architecture {{.rpm_arch}} \
--vendor "{{.beat_vendor}}" \
--license "{{.beat_license}}" \
--description "{{.beat_description}}" \
--url {{.beat_url}} \
--rpm-init /tmp/{{.beat_pkg_name}}.init \
--after-install /tmp/systemd-daemon-reload.sh \
--config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \
homedir/=/usr/share/{{.beat_name}} \
beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \
{{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
{{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
{{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \
fields.yml=/etc/{{.beat_name}}/fields.yml \
${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \
FPM_ARGS=(
--force -s dir -t rpm
-n {{.beat_pkg_name}} -v ${RPM_VERSION}
--architecture {{.rpm_arch}}
--vendor "{{.beat_vendor}}"
--license "{{.beat_license}}"
--description "{{.beat_description}}"
--url {{.beat_url}}
--rpm-init /tmp/{{.beat_pkg_name}}.init
--after-install /tmp/systemd-daemon-reload.sh
--config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
homedir/=/usr/share/{{.beat_name}}
beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
{{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
{{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
{{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
fields.yml=/etc/{{.beat_name}}/fields.yml
${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
)

if [ -d modules.d-linux ]; then
FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/)
fi

fpm "${FPM_ARGS[@]}"

# rename so that the filename respects semver rules
mkdir -p upload
Expand Down
1 change: 1 addition & 0 deletions dev-tools/packer/platforms/darwin/run.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_
cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.yml
cp {{.beat_name}}-darwin.reference.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.reference.yml
cp fields.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/
cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-darwin-x86_64/modules.d || true

mkdir -p upload
tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64
Expand Down
42 changes: 25 additions & 17 deletions dev-tools/packer/platforms/debian/run.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -18,24 +18,32 @@ if [ "$SNAPSHOT" = "yes" ]; then
fi

# create deb
fpm --force -s dir -t deb \
-n {{.beat_pkg_name}} -v ${VERSION} \
--vendor "{{.beat_vendor}}" \
--license "{{.beat_license}}" \
--architecture {{.deb_arch}} \
--description "{{.beat_description}}" \
--url {{.beat_url}} \
--deb-init /tmp/{{.beat_pkg_name}}.init \
--after-install /tmp/systemd-daemon-reload.sh \
--config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \
homedir/=/usr/share/{{.beat_name}} \
beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \
{{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
{{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
{{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \
fields.yml=/etc/{{.beat_name}}/fields.yml \
${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \
FPM_ARGS=(
--force -s dir -t deb
-n {{.beat_pkg_name}} -v ${VERSION}
--vendor "{{.beat_vendor}}"
--license "{{.beat_license}}"
--architecture {{.deb_arch}}
--description "{{.beat_description}}"
--url {{.beat_url}}
--deb-init /tmp/{{.beat_pkg_name}}.init
--after-install /tmp/systemd-daemon-reload.sh
--config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
homedir/=/usr/share/{{.beat_name}}
beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
{{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
{{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
{{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
fields.yml=/etc/{{.beat_name}}/fields.yml
${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
)

if [ -d modules.d-linux ]; then
FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/)
fi

fpm "${FPM_ARGS[@]}"

# move and rename to use the elastic conventions
mkdir -p upload
Expand Down
1 change: 1 addition & 0 deletions dev-tools/packer/platforms/windows/run.sh.j2
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ unix2dos {{.beat_name}}-win.yml
cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.yml
cp {{.beat_name}}-win.reference.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.reference.yml
cp fields.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/modules.d || true
cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/

Expand Down
7 changes: 6 additions & 1 deletion dev-tools/packer/xgo-scripts/before_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,22 @@ cp fields.yml $PREFIX/fields.yml
cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml
chmod 0600 $PREFIX/$BEAT_NAME-linux.yml
cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-linux.reference.yml
rm -rf $PREFIX/modules.d-linux
cp -a modules.d/ $PREFIX/modules.d-linux || true

# darwin
cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-darwin.yml
chmod 0600 $PREFIX/$BEAT_NAME-darwin.yml
cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-darwin.reference.yml
rm -rf $PREFIX/modules.d-darwin
cp -a modules.d/ $PREFIX/modules.d-darwin || true

# win
cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-win.yml
chmod 0600 $PREFIX/$BEAT_NAME-win.yml
cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-win.reference.yml
rm -rf $PREFIX/modules.d-win
cp -a modules.d/ $PREFIX/modules.d-win || true

# Runs beat specific tasks which should be done before building
PREFIX=$PREFIX make before-build
Expand All @@ -49,4 +55,3 @@ sed -i -e 's/:doc-branch/doc_branch/g' ${PREFIX}/package.yml

# Create README file
/go/bin/gotpl /templates/readme.md.j2 < ${PREFIX}/package.yml > ${PREFIX}/homedir/README.md

8 changes: 5 additions & 3 deletions filebeat/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ modules:
.PHONY: configs
configs: python-env
@cp ${ES_BEATS}/filebeat/_meta/common.p1.yml _meta/beat.yml
@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml
@cat ${ES_BEATS}/filebeat/_meta/common.p2.yml >> _meta/beat.yml
@cp ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml _meta/beat.reference.yml
@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
@cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml
@. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this need to activate the virtual environment? I thought you could invoke ${PYTHON_ENV}/bin/python because it's directly running a script (and it's slightly faster)? [virtualenv activate docs]

@cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml
@rm -rf modules.d && mkdir -p modules.d
@for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
@chmod go-w modules.d/*

# Collects all module docs
.PHONY: collect-docs
Expand Down
12 changes: 12 additions & 0 deletions filebeat/_meta/common.p2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,15 @@ filebeat.prospectors:
# Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
#multiline.match: after


#============================= Filebeat modules ===============================

filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml

# Set to true to enable config reloading
reload.enabled: false

# Period on which files under path should be checked for changes
#reload.period: 10s
33 changes: 33 additions & 0 deletions filebeat/cmd/modules.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package cmd

import (
"strings"

"github.com/pkg/errors"

"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/cfgfile"
"github.com/elastic/beats/libbeat/cmd"
)

func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
config, err := beat.BeatConfig()
if err != nil {
return nil, errors.Wrap(err, "initialization error")
}

glob, err := config.String("config.modules.path", -1)
if err != nil {
return nil, errors.Errorf("modules management requires 'filebeat.config.modules.path' setting")
}

if !strings.HasSuffix(glob, ".yml") {
return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe I'm taking the error too literally, but the check is for .yml and the error says it must end with *.yml which is a disconnect.

}

modulesManager, err := cfgfile.NewGlobManager(glob, ".yml", ".disabled")
if err != nil {
return nil, errors.Wrap(err, "initialization error")
}
return modulesManager, nil
}
1 change: 1 addition & 0 deletions filebeat/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,5 @@ func init() {
RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
RootCmd.ConfigTestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
}
99 changes: 12 additions & 87 deletions filebeat/filebeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,93 +7,6 @@
# You can find the full configuration reference here:
# https://www.elastic.co/guide/en/beats/filebeat/index.html


#========================== Modules configuration ============================
filebeat.modules:

#------------------------------- System Module -------------------------------
#- module: system
# Syslog
#syslog:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

# Authorization logs
#auth:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

#------------------------------- Apache2 Module ------------------------------
#- module: apache2
# Access logs
#access:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

# Error logs
#error:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

#------------------------------- Auditd Module -------------------------------
#- module: auditd
#log:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:


#-------------------------------- MySQL Module -------------------------------
#- module: mysql
# Error logs
#error:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

# Slow logs
#slowlog:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

#-------------------------------- Nginx Module -------------------------------
#- module: nginx
# Access logs
#access:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

# Error logs
#error:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:


# For more available modules and options, please see the filebeat.reference.yml sample
# configuration file.

Expand Down Expand Up @@ -150,6 +63,18 @@ filebeat.prospectors:
#multiline.match: after


#============================= Filebeat modules ===============================

filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml

# Set to true to enable config reloading
reload.enabled: false

# Period on which files under path should be checked for changes
#reload.period: 10s

#================================ General =====================================

# The name of the shipper that publishes the network data. It can be used to group
Expand Down
16 changes: 16 additions & 0 deletions filebeat/modules.d/apache2.yml.disabled
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#- module: apache2
# Access logs
#access:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

# Error logs
#error:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:
8 changes: 8 additions & 0 deletions filebeat/modules.d/auditd.yml.disabled
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#- module: auditd
#log:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

24 changes: 24 additions & 0 deletions filebeat/modules.d/icinga.yml.disabled
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#- module: icinga
# Main logs
#main:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

# Debug logs
#debug:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:

# Startup logs
#startup:
#enabled: true

# Set custom paths for the log files. If left empty,
# Filebeat will choose the paths depending on your OS.
#var.paths:
Loading