diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..02f96d7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# Ignore everything +* + +# Allow files and directories +!fluent.conf +!fluent.yaml +!entrypoint.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..43580d3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +# All +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf +max_line_length = off + +# Markdown uses whitespace for formatting +[*.md] +trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..314766e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..253bcb7 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily diff --git a/.github/workflows/merge.yaml b/.github/workflows/merge.yaml new file mode 100644 index 0000000..5a0bcc7 --- /dev/null +++ b/.github/workflows/merge.yaml @@ -0,0 +1,37 @@ +name: Merge + +on: + push: + branches: + - main + +jobs: + build: + name: Build OCI Image + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Run Hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + with: + dockerfile: ./${{ matrix.os }}.dockerfile + + - name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 + + - name: Build + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: false diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..5c4d0be --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,37 @@ +name: Pull Request + +on: + pull_request: + branches: + - main + +jobs: + build: + name: Build OCI Image + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Run Hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + with: + dockerfile: ./${{ matrix.os }}.dockerfile + + - name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 + + - name: Build + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64 + push: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..51daa53 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,84 @@ +name: Release + +on: + push: + tags: + - v* + +jobs: + release: + name: Release OCI Image + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Run Hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + with: + dockerfile: ./${{ matrix.os }}.dockerfile + + - name: Docker meta + id: metadata + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a + with: + images: fluent/fluentd-aggregator,ghcr.io/fluent/fluentd-aggregator + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=latest,enable={{is_default_branch}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 + + - name: Login to GitHub Container Registry + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build & push + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + + - name: Update Docker repo description + uses: peter-evans/dockerhub-description@da890086d39c735e41d8823c8a95bde4302c3d64 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: fluent/fluentd-aggregator + + - name: Get changelog entry + id: changelog + uses: mindsers/changelog-reader-action@17b72bf66059b7ac310abdc0b511948903e85ea5 + with: + path: ./CHANGELOG.md + version: ${{ steps.metadata.outputs.version }} + + - name: Create release + uses: ncipollo/release-action@58ae73b360456532aafd58ee170c045abbeaee37 + with: + token: ${{ secrets.GITHUB_TOKEN }} + allowUpdates: true + body: ${{ steps.changelog.outputs.changes }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..442f183 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,3 @@ +failure-threshold: warning +ignored: + - DL3018 diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..a3c29f7 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,11 @@ +MD013: false +MD024: + siblings_only: true +MD028: false +MD033: + allowed_elements: + - br + - details + - summary + - sub + - sup diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a427fc2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +# Fluentd Aggregator Docker Image Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + + + +## [v0.0.1] - UNRELEASED + +### Added + +- Added initial version. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1384393 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# Fluentd Aggregator Docker Image + +[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/fluent/fluentd-aggregator?sort=semver)](https://hub.docker.com/r/fluent/fluentd-aggregator) +![linux](https://img.shields.io/badge/os-linux-brightgreen) +![amd64](https://img.shields.io/badge/arch-amd64-brightgreen) +![arm64](https://img.shields.io/badge/arch-arm64-brightgreen) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +A [Fluentd](https://www.fluentd.org/) [OCI](https://opencontainers.org/) image to be used for log aggregation and based on the official [Fluentd Docker image](https://github.com/fluent/fluentd-docker-image) rebuilt as a multi-arch `linux/amd64` & `linux/arm64` image. + +## Aggregation Changes + +To optimise _Fluentd_ for log aggregation the default configuration file has been overwritten to allow logs to be forwarded and printed to stdout. Plugins have also been added to support the aggregation role. + +### Plugins + +The following plugins have been added to the base image. + +- [fluent-plugin-azure-loganalytics](https://github.com/yokawasa/fluent-plugin-azure-loganalytics) +- [fluent-plugin-azurestorage-gen2](https://github.com/oleewere/fluent-plugin-azurestorage-gen2) +- [fluent-plugin-cloudwatch-logs](https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-logs) +- [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) +- [fluent-plugin-datadog](https://github.com/DataDog/fluent-plugin-datadog) +- [fluent-plugin-elasticsearch](https://docs.fluentd.org/output/elasticsearch) +- [fluent-plugin-grafana-loki](https://github.com/grafana/loki/tree/main/clients/cmd/fluentd) +- [fluent-plugin-kafka](https://github.com/fluent/fluent-plugin-kafka) +- [fluent-plugin-opensearch](https://github.com/fluent/fluent-plugin-opensearch) +- [fluent-plugin-prometheus](https://github.com/fluent/fluent-plugin-prometheus) +- [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier) +- [fluent-plugin-rewrite-tag-filter](https://github.com/fluent/fluent-plugin-rewrite-tag-filter) +- [fluent-plugin-route](https://github.com/tagomoris/fluent-plugin-route) +- [fluent-plugin-s3](https://docs.fluentd.org/output/s3) +- [fluent-plugin-sqs](https://github.com/ixixi/fluent-plugin-sqs) + +## Usage + +This image is available at [Docker Hub](https://hub.docker.com/r/fluent/fluentd-aggregator); the image version matches the _Fluentd_ version that it's based on. + +This image can be pulled with the following commands. + +```shell +docker pull fluent/fluentd-aggregator:latest + +docker pull ghcr.io/fluent/fluentd-aggregator:latest +``` + +This image can be tested by running the following command and then forwarding logs. + +```shell +docker run -p 24224:24224 fluent/fluentd-aggregator:latest +``` + +## License + +[Apache License, Version 2.0](./LICENSE). diff --git a/alpine.dockerfile b/alpine.dockerfile new file mode 100644 index 0000000..6f11a9a --- /dev/null +++ b/alpine.dockerfile @@ -0,0 +1,60 @@ +FROM alpine:3.16 +LABEL maintainer "Fluentd developers " +LABEL description="Fluentd aggregator OCI image based on Fluentd v1.15.2" version="0.0.1" + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apk delete' has no effect +RUN apk update \ + && apk add --no-cache \ + ca-certificates \ + ruby ruby-irb ruby-etc ruby-webrick \ + tini \ + libcurl \ + && apk add --no-cache --virtual .build-deps \ + build-base linux-headers \ + ruby-dev gnupg \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install oj -v 3.13.19 \ + && gem install json -v 2.6.2 \ + && gem install async -v 1.30.3 \ + && gem install async-http -v 0.56.6 \ + && gem install fluentd -v 1.15.2 \ + && gem install bigdecimal -v 1.4.4 \ + && gem install fluent-plugin-azure-loganalytics -v 0.7.0 \ + && gem install fluent-plugin-azurestorage-gen2 -v 0.3.3 \ + && gem install fluent-plugin-cloudwatch-logs -v 0.14.3 \ + && gem install fluent-plugin-concat -v 2.5.0 \ + && gem install fluent-plugin-datadog -v 0.14.2 \ + && gem install fluent-plugin-elasticsearch -v 5.2.3 \ + && gem install fluent-plugin-grafana-loki -v 1.2.18 \ + && gem install fluent-plugin-kafka -v 0.18.1 \ + && gem install fluent-plugin-opensearch -v 1.0.8 \ + && gem install fluent-plugin-prometheus -v 2.0.3 \ + && gem install fluent-plugin-record-modifier -v 2.1.1 \ + && gem install fluent-plugin-rewrite-tag-filter -v 2.4.0 \ + && gem install fluent-plugin-route -v 1.0.0 \ + && gem install fluent-plugin-s3 -v 1.7.1 \ + && gem install fluent-plugin-sqs -v 3.0.0 \ + && apk del .build-deps \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN addgroup --system --gid 2000 fluent && adduser --system --ingroup fluent --uid 2000 fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + && mkdir -p /fluentd/state \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent /fluentd && chgrp -R fluent /fluentd + +COPY fluent.yaml /fluentd/etc/ +COPY entrypoint.sh /bin/ + +ENV FLUENTD_CONF="fluent.yaml" + +ENV LD_PRELOAD="" +EXPOSE 24224 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/debian.dockerfile b/debian.dockerfile new file mode 100644 index 0000000..1a541ae --- /dev/null +++ b/debian.dockerfile @@ -0,0 +1,76 @@ +FROM ruby:3.1-slim-bullseye +LABEL maintainer "Fluentd developers " +LABEL description="Fluentd aggregator OCI image based on Fluentd v1.15.2" version="0.0.1" + +ENV TINI_VERSION=0.18.0 + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apt-get purge' has no effect +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + && buildDeps=" \ + make gcc g++ libc-dev \ + wget bzip2 gnupg dirmngr \ + " \ + && apt-get install -y --no-install-recommends $buildDeps \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install oj -v 3.13.19 \ + && gem install json -v 2.6.2 \ + && gem install async -v 1.30.3 \ + && gem install async-http -v 0.56.6 \ + && gem install fluentd -v 1.15.2 \ + && gem install fluent-plugin-azure-loganalytics -v 0.7.0 \ + && gem install fluent-plugin-azurestorage-gen2 -v 0.3.3 \ + && gem install fluent-plugin-cloudwatch-logs -v 0.14.3 \ + && gem install fluent-plugin-concat -v 2.5.0 \ + && gem install fluent-plugin-datadog -v 0.14.2 \ + && gem install fluent-plugin-elasticsearch -v 5.2.3 \ + && gem install fluent-plugin-grafana-loki -v 1.2.18 \ + && gem install fluent-plugin-kafka -v 0.18.1 \ + && gem install fluent-plugin-opensearch -v 1.0.8 \ + && gem install fluent-plugin-prometheus -v 2.0.3 \ + && gem install fluent-plugin-record-modifier -v 2.1.1 \ + && gem install fluent-plugin-rewrite-tag-filter -v 2.4.0 \ + && gem install fluent-plugin-route -v 1.0.0 \ + && gem install fluent-plugin-s3 -v 1.7.1 \ + && gem install fluent-plugin-sqs -v 3.0.0 \ + && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ + && wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini-$dpkgArch" \ + && wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini-$dpkgArch.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 6380DC428747F6C393FEACA59A84159D7001A4E5 \ + && gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \ + && rm -r /usr/local/bin/tini.asc \ + && chmod +x /usr/local/bin/tini \ + && tini -h \ + && wget -O /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \ + && cd /tmp && tar -xjf jemalloc-4.5.0.tar.bz2 && cd jemalloc-4.5.0/ \ + && ./configure && make \ + && mv lib/libjemalloc.so.2 /usr/lib \ + && apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN groupadd --system --gid 2000 fluent && useradd --system --gid fluent --uid 2000 fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + && mkdir -p /fluentd/state \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent /fluentd && chgrp -R fluent /fluentd + +COPY fluent.conf /fluentd/etc/ +COPY entrypoint.sh /bin/ + +ENV FLUENTD_CONF="fluent.conf" + +ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2" +EXPOSE 24224 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..d0f6b25 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +#source vars if file exists +DEFAULT=/etc/default/fluentd + +if [ -r $DEFAULT ]; then + set -o allexport + . $DEFAULT + set +o allexport +fi + +# If the user has supplied only arguments append them to `fluentd` command +if [ "${1#-}" != "$1" ]; then + set -- fluentd "$@" +fi + +# If user does not supply config file or plugins, use the default +if [ "$1" = "fluentd" ]; then + if ! echo $@ | grep -e ' \-c' -e ' \-\-config' ; then + set -- "$@" --config /fluentd/etc/${FLUENTD_CONF} + fi + + if ! echo $@ | grep -e ' \-p' -e ' \-\-plugin' ; then + set -- "$@" --plugin /fluentd/plugins + fi +fi + +exec "$@" diff --git a/fluent.conf b/fluent.conf new file mode 100644 index 0000000..bebe5be --- /dev/null +++ b/fluent.conf @@ -0,0 +1,18 @@ + + @type forward + @label @default + port 24224 + bind 0.0.0.0 + + + + + diff --git a/fluent.yaml b/fluent.yaml new file mode 100644 index 0000000..d5302af --- /dev/null +++ b/fluent.yaml @@ -0,0 +1,23 @@ +system: + root_dir: /fluentd/state + +config: + - source: + $type: forward + $label: '@default' + port: 24224 + bind: 0.0.0.0 + + - label: + $name: '@FLUENT_LOG' + config: + - match: + $tag: 'fluent.*' + $type: stdout + + - label: + $name: '@default' + config: + - match: + $tag: '**' + $type: stdout