Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor Beat packaging and cross-building #7388

Merged
merged 6 commits into from
Jun 29, 2018
Merged

Refactor Beat packaging and cross-building #7388

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
eb52c4d
Add Windows 2016 to Vagrantfile
andrewkroh Jun 7, 2018
6d76c29
Add github.com/magefile/mage to vendor
andrewkroh Jun 7, 2018
93b679d
Remove dev-tools/packer and related deps
andrewkroh Jun 7, 2018
413db90
Add github.com/tsg/go-daemon to vendor
andrewkroh Jun 7, 2018
fe17304
Add golang.org/x/tools/go/vcs to dev-tools/vendor
andrewkroh Jun 7, 2018
23c4bc0
Refactor Beat packaging and cross-building
andrewkroh Jun 7, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG-developer.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ The list below covers the major changes between 6.3.0 and master only.
- Port fields.yml collector to Golang {pull}6911[6911]
- Dashboards under _meta/kibana are expected to be decoded. See https://github.com/elastic/beats/pull/7224 for a conversion script. {pull}7265[7265]
- Constructor `(github.com/elastic/beats/libbeat/output/codec/json).New` expects a new `escapeHTML` parameter. {pull}7445[7445]
- Packaging has been refactored and updates are required. See the PR for migration details. {pull}7388[7388]

==== Bugfixes

Expand Down
78 changes: 37 additions & 41 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
BUILD_DIR=$(CURDIR)/build
COVERAGE_DIR=$(BUILD_DIR)/coverage
BEATS=packetbeat filebeat winlogbeat metricbeat heartbeat auditbeat
BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat
PROJECTS=libbeat $(BEATS)
PROJECTS_ENV=libbeat filebeat metricbeat
SNAPSHOT?=yes
PYTHON_ENV?=$(BUILD_DIR)/python-env
VIRTUALENV_PARAMS?=
FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*"
Expand Down Expand Up @@ -62,6 +61,7 @@ clean:
@rm -rf build
@$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;)
@$(MAKE) -C generator clean
@-mage -clean 2> /dev/null

# Cleans up the vendor directory from unnecessary files
# This should always be run after updating the dependencies
Expand Down Expand Up @@ -109,51 +109,12 @@ lint:
@go get $(GOLINT_REPO) $(REVIEWDOG_REPO)
$(REVIEWDOG) $(REVIEWDOG_OPTIONS)

# Collects all dashboards and generates dashboard folder for https://github.com/elastic/beats-dashboards/tree/master/dashboards
.PHONY: beats-dashboards
beats-dashboards:
@mkdir -p build/dashboards
@$(foreach var,$(BEATS),cp -r $(var)/_meta/kibana.generated/ build/dashboards/$(var) || exit 1;)

# Builds the documents for each beat
.PHONY: docs
docs:
@$(foreach var,$(PROJECTS),BUILD_DIR=${BUILD_DIR} $(MAKE) -C $(var) docs || exit 1;)
sh ./script/build_docs.sh dev-guide github.com/elastic/beats/docs/devguide ${BUILD_DIR}

.PHONY: package-all
package-all: update beats-dashboards
@$(foreach var,$(BEATS),SNAPSHOT=$(SNAPSHOT) $(MAKE) -C $(var) package-all || exit 1;)

@echo "Start building the dashboards package"
@mkdir -p build/upload/
@BUILD_DIR=${BUILD_DIR} UPLOAD_DIR=${BUILD_DIR}/upload SNAPSHOT=$(SNAPSHOT) $(MAKE) -C dev-tools/packer package-dashboards ${BUILD_DIR}/upload/build_id.txt
@mv build/upload build/dashboards-upload

@# Copy build files over to top build directory
@mkdir -p build/upload/
@$(foreach var,$(BEATS),cp -r $(var)/build/upload/ build/upload/$(var) || exit 1;)
@cp -r build/dashboards-upload build/upload/dashboards
@# Run tests on the generated packages.
@go test ./dev-tools/package_test.go -files "${BUILD_DIR}/upload/*/*"

# Upload nightly builds to S3
.PHONY: upload-nightlies-s3
upload-nightlies-s3: all
aws s3 cp --recursive --acl public-read build/upload s3://beats-nightlies

# Run after building to sign packages and publish to APT and YUM repos.
.PHONY: package-upload
upload-package:
$(MAKE) -C dev-tools/packer deb-rpm-s3
# You must export AWS_ACCESS_KEY=<AWS access> and export AWS_SECRET_KEY=<secret>
# before running this make target.
dev-tools/packer/docker/deb-rpm-s3/deb-rpm-s3.sh

.PHONY: release-upload
upload-release:
aws s3 cp --recursive --acl public-read build/upload s3://download.elasticsearch.org/beats/

.PHONY: notice
notice: python-env
@echo "Generating NOTICE"
Expand All @@ -171,3 +132,38 @@ python-env:
.PHONY: test-apm
test-apm:
sh ./script/test_apm.sh

### Packaging targets ####

# Builds a snapshot release.
.PHONY: snapshot
snapshot:
@$(MAKE) SNAPSHOT=true release

# Builds a release.
.PHONY: release
release: beats-dashboards
@$(foreach var,$(BEATS),$(MAKE) -C $(var) release || exit 1;)
@$(foreach var,$(BEATS),mkdir -p build/distributions/$(var) && mv -f $(var)/build/distributions/* build/distributions/$(var)/ || exit 1;)

# Builds a snapshot release. The Go version defined in .go-version will be
# installed and used for the build.
.PHONY: release-manager-snapshot
release-manager-snapshot:
./dev-tools/run_with_go_ver $(MAKE) snapshot

# Builds a snapshot release. The Go version defined in .go-version will be
# installed and used for the build.
.PHONY: release-manager-release
release-manager-release:
./dev-tools/run_with_go_ver $(MAKE) release

# Installs the mage build tool from the vendor directory.
.PHONY: mage
mage:
@go install github.com/elastic/beats/vendor/github.com/magefile/mage

# Collects dashboards from all Beats and generates a zip file distribution.
.PHONY: beats-dashboards
beats-dashboards: mage update
@mage packageBeatDashboards
9 changes: 9 additions & 0 deletions NOTICE.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1396,6 +1396,15 @@ The above copyright notice and this permission notice shall be included in all c

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

--------------------------------------------------------------------
Dependency: github.com/magefile/mage
Revision: 5e51f9ad1ed0886c5d06a8c46a09703cfc4d9034
License type (autodetected): Apache-2.0
./vendor/github.com/magefile/mage/LICENSE:
--------------------------------------------------------------------
Apache License 2.0


--------------------------------------------------------------------
Dependency: github.com/mattn/go-colorable
Revision: 941b50ebc6efddf4c41c8e4537a5f68a4e686b24
Expand Down
10 changes: 10 additions & 0 deletions Vagrantfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,16 @@ Vagrant.configure(2) do |config|
c.vm.synced_folder ".", "/vagrant", type: "virtualbox"
end

# Windows Server 2016
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this is not directly related to this PR?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's only related in that I was testing these build changes on win2016 and we didn't have a machine for it yet.

config.vm.define "win2016", primary: true do |machine|
machine.vm.box = "elastic/windows-2016-x86_64"
machine.vm.provision "shell", inline: $winPsProvision

machine.vm.provider "virtualbox" do |v|
v.memory = 4096
end
end

end

# -*- mode: ruby -*-
Expand Down
50 changes: 3 additions & 47 deletions auditbeat/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
BEAT_NAME=auditbeat
BEAT_TITLE=Auditbeat
BEAT_DESCRIPTION=Audit the activities of users and processes on your system.
SYSTEM_TESTS=true
TEST_ENVIRONMENT?=true
GOX_OS?=linux windows ## @Building List of all OS to be supported by "make crosscompile".
Expand All @@ -12,49 +11,6 @@ FIELDS_FILE_PATH=module
# Path to the libbeat Makefile
include ${ES_BEATS}/libbeat/scripts/Makefile

# This is called by the beats packer before building starts
.PHONY: before-build
before-build:
@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
<(go run scripts/generate_config.go -os windows -concat) \
${ES_BEATS}/auditbeat/_meta/common.p2.yml \
${ES_BEATS}/libbeat/_meta/config.yml > \
${PREFIX}/${BEAT_NAME}-win.yml
@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
<(go run scripts/generate_config.go -os windows -concat -ref) \
${ES_BEATS}/libbeat/_meta/config.reference.yml > \
${PREFIX}/${BEAT_NAME}-win.reference.yml

@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
<(go run scripts/generate_config.go -os darwin -concat) \
${ES_BEATS}/auditbeat/_meta/common.p2.yml \
${ES_BEATS}/libbeat/_meta/config.yml > \
${PREFIX}/${BEAT_NAME}-darwin.yml
@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
<(go run scripts/generate_config.go -os darwin -concat -ref) \
${ES_BEATS}/libbeat/_meta/config.reference.yml > \
${PREFIX}/${BEAT_NAME}-darwin.reference.yml

@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
<(go run scripts/generate_config.go -os linux -arch amd64 -concat) \
${ES_BEATS}/auditbeat/_meta/common.p2.yml \
${ES_BEATS}/libbeat/_meta/config.yml > \
${PREFIX}/${BEAT_NAME}-linux.yml
@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
<(go run scripts/generate_config.go -os linux -concat -ref) \
${ES_BEATS}/libbeat/_meta/config.reference.yml > \
${PREFIX}/${BEAT_NAME}-linux.reference.yml

@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
<(go run scripts/generate_config.go -os linux -arch i386 -concat) \
${ES_BEATS}/auditbeat/_meta/common.p2.yml \
${ES_BEATS}/libbeat/_meta/config.yml > \
${PREFIX}/${BEAT_NAME}-linux-386.yml
@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
<(go run scripts/generate_config.go -os linux -concat -ref) \
${ES_BEATS}/libbeat/_meta/config.reference.yml > \
${PREFIX}/${BEAT_NAME}-linux-386.reference.yml

# Collects all dependencies and then calls update
.PHONY: collect
collect: fields collect-docs configs kibana
Expand All @@ -63,10 +19,10 @@ collect: fields collect-docs configs kibana
.PHONY: configs
configs: python-env
@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
<(go run scripts/generate_config.go -os ${DEV_OS} -concat) \
${ES_BEATS}/auditbeat/_meta/common.p2.yml > _meta/beat.yml
<(go run scripts/generate_config.go -os ${DEV_OS} -concat) \
${ES_BEATS}/auditbeat/_meta/common.p2.yml > _meta/beat.yml
@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
<(go run scripts/generate_config.go -os ${DEV_OS} -ref -concat) > _meta/beat.reference.yml
<(go run scripts/generate_config.go -os ${DEV_OS} -ref -concat) > _meta/beat.reference.yml

# Collects all module docs
.PHONY: collect-docs
Expand Down
Loading