-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Steve Hipwell <steve.hipwell@gmail.com>
- Loading branch information
1 parent
e497c99
commit 52f1b26
Showing
17 changed files
with
484 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
# Ignore everything | ||
* | ||
|
||
# Allow files and directories | ||
!fluent.conf | ||
!fluent.yaml | ||
!entrypoint.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
# All | ||
[*] | ||
charset = utf-8 | ||
indent_style = space | ||
indent_size = 2 | ||
trim_trailing_whitespace = true | ||
insert_final_newline = true | ||
end_of_line = lf | ||
max_line_length = off | ||
|
||
# Markdown uses whitespace for formatting | ||
[*.md] | ||
trim_trailing_whitespace = false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
* text=auto eol=lf | ||
*.{cmd,[cC][mM][dD]} text eol=crlf | ||
*.{bat,[bB][aA][tT]} text eol=crlf |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
version: 2 | ||
updates: | ||
- package-ecosystem: github-actions | ||
directory: / | ||
schedule: | ||
interval: daily |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
name: Merge | ||
|
||
on: | ||
push: | ||
branches: | ||
- main | ||
|
||
jobs: | ||
build: | ||
name: Build OCI Image | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
os: [alpine, debian] | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b | ||
|
||
- name: Run Hadolint | ||
uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 | ||
with: | ||
dockerfile: ./${{ matrix.os }}.dockerfile | ||
|
||
- name: Set up QEMU | ||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 | ||
|
||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 | ||
|
||
- name: Build | ||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 | ||
with: | ||
file: ./${{ matrix.os }}.dockerfile | ||
context: . | ||
platforms: linux/amd64,linux/arm64 | ||
push: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
name: Pull Request | ||
|
||
on: | ||
pull_request: | ||
branches: | ||
- main | ||
|
||
jobs: | ||
build: | ||
name: Build OCI Image | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
os: [alpine, debian] | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b | ||
|
||
- name: Run Hadolint | ||
uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 | ||
with: | ||
dockerfile: ./${{ matrix.os }}.dockerfile | ||
|
||
- name: Set up QEMU | ||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 | ||
|
||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 | ||
|
||
- name: Build | ||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 | ||
with: | ||
file: ./${{ matrix.os }}.dockerfile | ||
context: . | ||
platforms: linux/amd64 | ||
push: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,84 @@ | ||
name: Release | ||
|
||
on: | ||
push: | ||
tags: | ||
- v* | ||
|
||
jobs: | ||
release: | ||
name: Release OCI Image | ||
strategy: | ||
fail-fast: false | ||
matrix: | ||
os: [alpine, debian] | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b | ||
|
||
- name: Run Hadolint | ||
uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 | ||
with: | ||
dockerfile: ./${{ matrix.os }}.dockerfile | ||
|
||
- name: Docker meta | ||
id: metadata | ||
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a | ||
with: | ||
images: fluent/fluentd-aggregator,ghcr.io/fluent/fluentd-aggregator | ||
tags: | | ||
type=semver,pattern={{version}} | ||
type=semver,pattern={{major}}.{{minor}} | ||
type=semver,pattern={{major}} | ||
type=raw,value=latest,enable={{is_default_branch}} | ||
- name: Set up QEMU | ||
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 | ||
|
||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 | ||
|
||
- name: Login to GitHub Container Registry | ||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b | ||
with: | ||
registry: ghcr.io | ||
username: ${{ github.actor }} | ||
password: ${{ secrets.GITHUB_TOKEN }} | ||
|
||
- name: Login to DockerHub | ||
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b | ||
with: | ||
username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
|
||
- name: Build & push | ||
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 | ||
with: | ||
file: ./${{ matrix.os }}.dockerfile | ||
context: . | ||
platforms: linux/amd64,linux/arm64 | ||
push: true | ||
tags: ${{ steps.metadata.outputs.tags }} | ||
labels: ${{ steps.metadata.outputs.labels }} | ||
|
||
- name: Update Docker repo description | ||
uses: peter-evans/dockerhub-description@da890086d39c735e41d8823c8a95bde4302c3d64 | ||
with: | ||
username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
repository: fluent/fluentd-aggregator | ||
|
||
- name: Get changelog entry | ||
id: changelog | ||
uses: mindsers/changelog-reader-action@17b72bf66059b7ac310abdc0b511948903e85ea5 | ||
with: | ||
path: ./CHANGELOG.md | ||
version: ${{ steps.metadata.outputs.version }} | ||
|
||
- name: Create release | ||
uses: ncipollo/release-action@58ae73b360456532aafd58ee170c045abbeaee37 | ||
with: | ||
token: ${{ secrets.GITHUB_TOKEN }} | ||
allowUpdates: true | ||
body: ${{ steps.changelog.outputs.changes }} |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
failure-threshold: error | ||
ignored: | ||
- DL3008 | ||
- DL3018 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
MD013: false | ||
MD024: | ||
siblings_only: true | ||
MD028: false | ||
MD033: | ||
allowed_elements: | ||
- br | ||
- details | ||
- summary | ||
- sub | ||
- sup |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
# Fluentd Aggregator Docker Image Changelog | ||
|
||
All notable changes to this project will be documented in this file. | ||
|
||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), | ||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | ||
|
||
--- | ||
|
||
<!-- ## [vX.Y.Z] - UNRELEASED | ||
### Added | ||
### Changed | ||
### Updated | ||
### Fixed | ||
### Deprecated | ||
### Removed --> | ||
|
||
## [v0.0.1] - UNRELEASED | ||
|
||
### Added | ||
|
||
- Added initial version. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
# Fluentd Aggregator Docker Image | ||
|
||
[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/fluent/fluentd-aggregator?sort=semver)](https://hub.docker.com/r/fluent/fluentd-aggregator) | ||
![linux](https://img.shields.io/badge/os-linux-brightgreen) | ||
![amd64](https://img.shields.io/badge/arch-amd64-brightgreen) | ||
![arm64](https://img.shields.io/badge/arch-arm64-brightgreen) | ||
[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | ||
|
||
A [Fluentd](https://www.fluentd.org/) [OCI](https://opencontainers.org/) image to be used for log aggregation and based on the official [Fluentd Docker image](https://github.com/fluent/fluentd-docker-image) rebuilt as a multi-arch `linux/amd64` & `linux/arm64` image. | ||
|
||
## Aggregation Changes | ||
|
||
To optimise _Fluentd_ for log aggregation the default configuration file has been overwritten to allow logs to be forwarded and printed to stdout. Plugins have also been added to support the aggregation role. | ||
|
||
### Plugins | ||
|
||
The following plugins have been added to the base image. | ||
|
||
- [fluent-plugin-azure-loganalytics](https://github.com/yokawasa/fluent-plugin-azure-loganalytics) | ||
- [fluent-plugin-azurestorage-gen2](https://github.com/oleewere/fluent-plugin-azurestorage-gen2) | ||
- [fluent-plugin-cloudwatch-logs](https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-logs) | ||
- [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) | ||
- [fluent-plugin-datadog](https://github.com/DataDog/fluent-plugin-datadog) | ||
- [fluent-plugin-elasticsearch](https://docs.fluentd.org/output/elasticsearch) | ||
- [fluent-plugin-grafana-loki](https://github.com/grafana/loki/tree/main/clients/cmd/fluentd) | ||
- [fluent-plugin-kafka](https://github.com/fluent/fluent-plugin-kafka) | ||
- [fluent-plugin-opensearch](https://github.com/fluent/fluent-plugin-opensearch) | ||
- [fluent-plugin-prometheus](https://github.com/fluent/fluent-plugin-prometheus) | ||
- [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier) | ||
- [fluent-plugin-rewrite-tag-filter](https://github.com/fluent/fluent-plugin-rewrite-tag-filter) | ||
- [fluent-plugin-route](https://github.com/tagomoris/fluent-plugin-route) | ||
- [fluent-plugin-s3](https://docs.fluentd.org/output/s3) | ||
- [fluent-plugin-sqs](https://github.com/ixixi/fluent-plugin-sqs) | ||
|
||
## Usage | ||
|
||
This image is available at [Docker Hub](https://hub.docker.com/r/fluent/fluentd-aggregator); the image version matches the _Fluentd_ version that it's based on. | ||
|
||
This image can be pulled with the following commands. | ||
|
||
```shell | ||
docker pull fluent/fluentd-aggregator:latest | ||
|
||
docker pull ghcr.io/fluent/fluentd-aggregator:latest | ||
``` | ||
|
||
This image can be tested by running the following command and then forwarding logs. | ||
|
||
```shell | ||
docker run -p 24224:24224 fluent/fluentd-aggregator:latest | ||
``` | ||
|
||
## License | ||
|
||
[Apache License, Version 2.0](./LICENSE). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
FROM alpine:3.16 | ||
LABEL maintainer "Fluentd developers <fluentd@googlegroups.com>" | ||
LABEL description="Fluentd aggregator OCI image based on Fluentd v1.15.2" version="0.0.1" | ||
|
||
# Do not split this into multiple RUN! | ||
# Docker creates a layer for every RUN-Statement | ||
# therefore an 'apk delete' has no effect | ||
RUN apk update \ | ||
&& apk add --no-cache \ | ||
ca-certificates \ | ||
ruby ruby-irb ruby-etc ruby-webrick \ | ||
tini \ | ||
libcurl \ | ||
&& apk add --no-cache --virtual .build-deps \ | ||
build-base linux-headers \ | ||
ruby-dev gnupg \ | ||
&& echo 'gem: --no-document' >> /etc/gemrc \ | ||
&& gem install oj -v 3.13.19 \ | ||
&& gem install json -v 2.6.2 \ | ||
&& gem install async -v 1.30.3 \ | ||
&& gem install async-http -v 0.56.6 \ | ||
&& gem install fluentd -v 1.15.2 \ | ||
&& gem install bigdecimal -v 1.4.4 \ | ||
&& gem install fluent-plugin-azure-loganalytics -v 0.7.0 \ | ||
&& gem install fluent-plugin-azurestorage-gen2 -v 0.3.3 \ | ||
&& gem install fluent-plugin-cloudwatch-logs -v 0.14.3 \ | ||
&& gem install fluent-plugin-concat -v 2.5.0 \ | ||
&& gem install fluent-plugin-datadog -v 0.14.2 \ | ||
&& gem install fluent-plugin-elasticsearch -v 5.2.3 \ | ||
&& gem install fluent-plugin-grafana-loki -v 1.2.18 \ | ||
&& gem install fluent-plugin-kafka -v 0.18.1 \ | ||
&& gem install fluent-plugin-opensearch -v 1.0.8 \ | ||
&& gem install fluent-plugin-prometheus -v 2.0.3 \ | ||
&& gem install fluent-plugin-record-modifier -v 2.1.1 \ | ||
&& gem install fluent-plugin-rewrite-tag-filter -v 2.4.0 \ | ||
&& gem install fluent-plugin-route -v 1.0.0 \ | ||
&& gem install fluent-plugin-s3 -v 1.7.1 \ | ||
&& gem install fluent-plugin-sqs -v 3.0.0 \ | ||
&& apk del .build-deps \ | ||
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test | ||
|
||
RUN addgroup --system --gid 2000 fluent && adduser --system --ingroup fluent --uid 2000 fluent \ | ||
# for log storage (maybe shared with host) | ||
&& mkdir -p /fluentd/log \ | ||
&& mkdir -p /fluentd/state \ | ||
# configuration/plugins path (default: copied from .) | ||
&& mkdir -p /fluentd/etc /fluentd/plugins \ | ||
&& chown -R fluent /fluentd && chgrp -R fluent /fluentd | ||
|
||
COPY fluent.yaml /fluentd/etc/ | ||
COPY entrypoint.sh /bin/ | ||
|
||
ENV FLUENTD_CONF="fluent.yaml" | ||
|
||
ENV LD_PRELOAD="" | ||
EXPOSE 24224 | ||
|
||
USER fluent | ||
ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] | ||
CMD ["fluentd"] |
Oops, something went wrong.