From aff08e12c67c1a6bc9b65e3e6aa9f966cc826943 Mon Sep 17 00:00:00 2001 From: Steve Hipwell Date: Mon, 7 Mar 2022 18:53:09 +0000 Subject: [PATCH] feat: Add initial version Signed-off-by: Steve Hipwell --- .dockerignore | 7 ++ .editorconfig | 13 +++ .gitattributes | 3 + .github/CODEOWNERS | 0 .github/dependabot.yaml | 6 + .github/workflows/commit.yaml | 144 +++++++++++++++++++++++ .github/workflows/pull-request.yaml | 91 +++++++++++++++ .github/workflows/release.yaml | 170 ++++++++++++++++++++++++++++ .gitignore | 0 .hadolint.yaml | 4 + .markdownlint.yaml | 11 ++ CHANGELOG.md | 23 ++++ LICENSE | 25 ---- README.md | 56 +++++++++ alpine.dockerfile | 58 ++++++++++ debian.dockerfile | 74 ++++++++++++ entrypoint.sh | 28 +++++ fluent.conf | 18 +++ fluent.yaml | 23 ++++ 19 files changed, 729 insertions(+), 25 deletions(-) create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .github/CODEOWNERS create mode 100644 .github/dependabot.yaml create mode 100644 .github/workflows/commit.yaml create mode 100644 .github/workflows/pull-request.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .gitignore create mode 100644 .hadolint.yaml create mode 100644 .markdownlint.yaml create mode 100644 CHANGELOG.md create mode 100644 README.md create mode 100644 alpine.dockerfile create mode 100644 debian.dockerfile create mode 100644 entrypoint.sh create mode 100644 fluent.conf create mode 100644 fluent.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..02f96d7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# Ignore everything +* + +# Allow files and directories +!fluent.conf +!fluent.yaml +!entrypoint.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..43580d3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +# All +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true +end_of_line = lf +max_line_length = off + +# Markdown uses whitespace for formatting +[*.md] +trim_trailing_whitespace = false diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..314766e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..e69de29 diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..253bcb7 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml new file mode 100644 index 0000000..9b74f70 --- /dev/null +++ b/.github/workflows/commit.yaml @@ -0,0 +1,144 @@ +name: Commit + +on: + push: + branches: + - main + +jobs: + build: + name: Build OCI Image + permissions: write-all + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Install Cosign + uses: sigstore/cosign-installer@ced07f21fb1da67979f539bbc6304c16c0677e76 + + - name: Run Hadolint + id: hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + continue-on-error: true + with: + dockerfile: ./${{ matrix.os }}.dockerfile + format: sarif + output-file: hadolint.sarif + no-fail: false + + - name: Upload Hadolint SARIF report + uses: github/codeql-action/upload-sarif@e0e5ded33cabb451ae0a9768fc7b0410bad9ad44 + with: + category: hadolint + sarif_file: hadolint.sarif + + - name: Generate OCI image metadata + id: metadata + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a + with: + flavor: | + latest=false + images: | + ghcr.io/${{ github.repository }} + docker.io/${{ github.repository }} + tags: | + type=raw,value=${{ matrix.os }}-main + type=raw,value=main,enable=${{ matrix.os == 'alpine' }} + labels: | + org.opencontainers.image.description=Fluentd aggregator OCI image based on the default Fluentd OCI image. + org.opencontainers.image.authors=Fluentd developers + + - name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 + + - name: Build OCI image + id: build + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: true + push: false + + - name: Generate OCI image SBOM + uses: anchore/sbom-action@b5042e9d19d8b32849779bfe17673ff84aec702d + with: + image: "ghcr.io/${{ github.repository }}:${{ matrix.os }}-main" + dependency-snapshot: true + format: spdx-json + artifact-name: ${{ github.event.repository.name }}-sbom.spdx.json + output-file: ${{ github.event.repository.name }}-sbom.spdx.json + + - name: Scan OCI image SBOM with Grype + id: scan + uses: anchore/scan-action@d5aa5b6cb9414b0c7771438046ff5bcfa2854ed7 + continue-on-error: true + with: + sbom: ${{ github.event.repository.name }}-sbom.spdx.json + severity-cutoff: medium + output-format: sarif + fail-build: true + + - name: Upload Grype SARIF report + uses: github/codeql-action/upload-sarif@e0e5ded33cabb451ae0a9768fc7b0410bad9ad44 + with: + category: grype + sarif_file: ${{ steps.scan.outputs.sarif }} + + - name: Fail workflow + if: ${{ steps.hadolint.outcome == 'failure' && steps.scan.outcome == 'failure' }} + run: | + set -euo pipefail + echo "::error::Code scanning failed." + exit 1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build & push OCI image + id: build_push + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: false + push: true + + - name: Sign OCI image + env: + COSIGN_EXPERIMENTAL: true + run: | + set -euo pipefail + cosign sign --yes --recursive ghcr.io/${{ github.repository }}:main@${{ steps.build_push.outputs.digest }} + cosign sign --yes --recursive docker.io/${{ github.repository }}:main@${{ steps.build_push.outputs.digest }} diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml new file mode 100644 index 0000000..fd6a868 --- /dev/null +++ b/.github/workflows/pull-request.yaml @@ -0,0 +1,91 @@ +name: Pull Request + +on: + pull_request: + branches: + - main + +jobs: + build: + name: Build OCI Image + permissions: write-all + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Run Hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + with: + dockerfile: ./${{ matrix.os }}.dockerfile + format: sarif + output-file: hadolint.sarif + no-fail: true + + - name: Upload Hadolint SARIF report + uses: github/codeql-action/upload-sarif@e0e5ded33cabb451ae0a9768fc7b0410bad9ad44 + with: + category: hadolint + sarif_file: hadolint.sarif + + - name: Generate OCI image metadata + id: metadata + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a + with: + flavor: | + latest=false + images: | + ${{ github.repository }} + tags: | + type=raw,value=${{ matrix.os }}-local + labels: | + org.opencontainers.image.description=Fluentd aggregator OCI image based on the default Fluentd OCI image. + org.opencontainers.image.authors=Fluentd developers + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 + + - name: Build OCI image + id: build + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: true + push: false + + - name: Generate OCI image SBOM + uses: anchore/sbom-action@b5042e9d19d8b32849779bfe17673ff84aec702d + with: + image: "${{ github.repository }}:${{ matrix.os }}-local" + dependency-snapshot: true + format: spdx-json + artifact-name: ${{ github.event.repository.name }}-sbom.spdx.json + output-file: ${{ github.event.repository.name }}-sbom.spdx.json + + - name: Scan OCI image SBOM with Grype + id: scan + uses: anchore/scan-action@d5aa5b6cb9414b0c7771438046ff5bcfa2854ed7 + with: + sbom: ${{ github.event.repository.name }}-sbom.spdx.json + severity-cutoff: medium + output-format: sarif + fail-build: false + + - name: Upload Grype SARIF report + uses: github/codeql-action/upload-sarif@e0e5ded33cabb451ae0a9768fc7b0410bad9ad44 + with: + category: grype + sarif_file: ${{ steps.scan.outputs.sarif }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..a10cc09 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,170 @@ +name: Release + +on: + push: + tags: + - v* + +jobs: + release: + name: Build & Release OCI Image + permissions: write-all + strategy: + fail-fast: false + matrix: + os: [alpine, debian] + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Install Cosign + uses: sigstore/cosign-installer@ced07f21fb1da67979f539bbc6304c16c0677e76 + + - name: Run Hadolint + uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183 + continue-on-error: true + with: + dockerfile: ./${{ matrix.os }}.dockerfile + format: sarif + output-file: hadolint.sarif + no-fail: false + + - name: Upload Hadolint SARIF report + uses: github/codeql-action/upload-sarif@e0e5ded33cabb451ae0a9768fc7b0410bad9ad44 + with: + category: hadolint + sarif_file: hadolint.sarif + + - name: Generate OCI image metadata + id: metadata + uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a + with: + flavor: | + latest=false + images: | + ghcr.io/${{ github.repository }} + docker.io/${{ github.repository }} + tags: | + type=semver,pattern=${{ matrix.os }}-{{version}} + type=semver,pattern=${{ matrix.os }}-{{major}}.{{minor}} + type=semver,pattern=${{ matrix.os }}-{{major}} + type=raw,value=${{ matrix.os }} + type=semver,pattern={{version}},enable=${{ matrix.os == 'alpine' }} + type=semver,pattern={{major}}.{{minor}},enable=${{ matrix.os == 'alpine' }} + type=semver,pattern={{major}},enable=${{ matrix.os == 'alpine' }} + type=raw,value=latest,enable=${{ matrix.os == 'alpine' }} + labels: | + org.opencontainers.image.description=Fluentd aggregator OCI image based on the default Fluentd OCI image. + org.opencontainers.image.authors=Fluentd developers + + - name: Set up QEMU + uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 + + - name: Build OCI image + id: build + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: true + push: false + + - name: Generate OCI image SBOM + uses: anchore/sbom-action@b5042e9d19d8b32849779bfe17673ff84aec702d + with: + image: "ghcr.io/${{ github.repository }}:${{ matrix.os }}-${{ steps.metadata.outputs.version }}" + dependency-snapshot: true + format: spdx-json + artifact-name: ${{ github.event.repository.name }}-sbom.spdx.json + output-file: ${{ github.event.repository.name }}-sbom.spdx.json + + - name: Scan OCI image SBOM with Grype + id: scan + uses: anchore/scan-action@d5aa5b6cb9414b0c7771438046ff5bcfa2854ed7 + continue-on-error: true + with: + sbom: ${{ github.event.repository.name }}-sbom.spdx.json + severity-cutoff: medium + output-format: sarif + fail-build: true + + - name: Upload Grype SARIF report + uses: github/codeql-action/upload-sarif@e0e5ded33cabb451ae0a9768fc7b0410bad9ad44 + with: + category: grype + sarif_file: ${{ steps.scan.outputs.sarif }} + + - name: Fail workflow + if: ${{ steps.hadolint.outcome == 'failure' && steps.scan.outcome == 'failure' }} + run: | + set -euo pipefail + echo "::error::Code scanning failed." + exit 1 + + - name: Login to GitHub Container Registry + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub + uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build & push OCI image + id: build_push + uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94 + with: + file: ./${{ matrix.os }}.dockerfile + context: . + platforms: linux/amd64,linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + load: false + push: true + + - name: Sign OCI image + env: + COSIGN_EXPERIMENTAL: true + run: | + set -euo pipefail + cosign sign --yes --recursive ghcr.io/${{ github.repository }}:${{ steps.metadata.outputs.version }}@${{ steps.build_push.outputs.digest }} + cosign sign --yes --recursive docker.io/${{ github.repository }}:${{ steps.metadata.outputs.version }}@${{ steps.build_push.outputs.digest }} + + - name: Update Docker repository description + uses: peter-evans/dockerhub-description@da890086d39c735e41d8823c8a95bde4302c3d64 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ github.repository }} + + - name: Get changelog entry + id: changelog_reader + uses: mindsers/changelog-reader-action@17b72bf66059b7ac310abdc0b511948903e85ea5 + with: + path: ./CHANGELOG.md + version: v${{ steps.metadata.outputs.version }} + + - name: Create release + uses: ncipollo/release-action@d9356040b0dc694f6d006b6cabee4d3c024f9e72 + with: + token: ${{ secrets.GITHUB_TOKEN }} + allowUpdates: true + body: ${{ steps.changelog_reader.outputs.changes }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..65ea0e7 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,4 @@ +failure-threshold: warning +ignored: + - DL3008 + - DL3018 diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..a3c29f7 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,11 @@ +MD013: false +MD024: + siblings_only: true +MD028: false +MD033: + allowed_elements: + - br + - details + - summary + - sub + - sup diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a6a3d9c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Fluentd Aggregator Docker Image Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + + + +## [v0.0.1] - UNRELEASED + +### All Changes + +- Added initial version. diff --git a/LICENSE b/LICENSE index 261eeb9..d9a10c0 100644 --- a/LICENSE +++ b/LICENSE @@ -174,28 +174,3 @@ of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1a1650d --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +# Fluentd Aggregator Docker Image + +![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/fluent/fluentd-aggregator-docker-image?sort=semver) +[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/fluent/fluentd-aggregator?sort=semver)](https://hub.docker.com/r/fluent/fluentd-aggregator) +![linux](https://img.shields.io/badge/os-linux-brightgreen) +![amd64](https://img.shields.io/badge/arch-amd64-brightgreen) +![arm64](https://img.shields.io/badge/arch-arm64-brightgreen) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +A [Fluentd](https://www.fluentd.org/) [OCI](https://opencontainers.org/) image to be used for log aggregation and based on the official [Fluentd Docker image](https://github.com/fluent/fluentd-docker-image) rebuilt as a multi-arch `linux/amd64` & `linux/arm64` image. + +## Aggregation Changes + +To optimise _Fluentd_ for log aggregation the default `fluent.conf` file has been overwritten to allow logs to be forwarded and printed to `stdout`, an additional directory `/fluentd/state` has been created, and plugins have also been added to support the aggregation role. + +### Plugins + +The following plugins have been added to the base image, to see the specific version please look in the _Dockerfile_. + +- [fluent-plugin-azure-loganalytics](https://github.com/yokawasa/fluent-plugin-azure-loganalytics) +- [fluent-plugin-azurestorage-gen2](https://github.com/oleewere/fluent-plugin-azurestorage-gen2) +- [fluent-plugin-cloudwatch-logs](https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-logs) +- [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) +- [fluent-plugin-datadog](https://github.com/DataDog/fluent-plugin-datadog) +- [fluent-plugin-elasticsearch](https://docs.fluentd.org/output/elasticsearch) +- [fluent-plugin-grafana-loki](https://github.com/grafana/loki/tree/main/clients/cmd/fluentd) +- [fluent-plugin-kafka](https://github.com/fluent/fluent-plugin-kafka) +- [fluent-plugin-opensearch](https://github.com/fluent/fluent-plugin-opensearch) +- [fluent-plugin-prometheus](https://github.com/fluent/fluent-plugin-prometheus) +- [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier) +- [fluent-plugin-rewrite-tag-filter](https://github.com/fluent/fluent-plugin-rewrite-tag-filter) +- [fluent-plugin-route](https://github.com/tagomoris/fluent-plugin-route) +- [fluent-plugin-s3](https://docs.fluentd.org/output/s3) +- [fluent-plugin-sqs](https://github.com/ixixi/fluent-plugin-sqs) + +## Usage + +This image is available at [Docker Hub](https://hub.docker.com/r/fluent/fluentd-aggregator); the image version matches the _Fluentd_ version that it's based on. + +This image can be pulled with the following commands. + +```shell +docker pull fluent/fluentd-aggregator:latest + +docker pull ghcr.io/fluent/fluentd-aggregator:latest +``` + +This image can be tested by running the following command and then forwarding logs. + +```shell +docker run -p 24224:24224 fluent/fluentd-aggregator:latest +``` + +## License + +[Apache License, Version 2.0](./LICENSE). diff --git a/alpine.dockerfile b/alpine.dockerfile new file mode 100644 index 0000000..ad42f4d --- /dev/null +++ b/alpine.dockerfile @@ -0,0 +1,58 @@ +FROM alpine:3.16 + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apk delete' has no effect +RUN apk update \ + && apk add --no-cache \ + ca-certificates \ + ruby ruby-irb ruby-etc ruby-webrick \ + tini \ + libcurl \ + && apk add --no-cache --virtual .build-deps \ + build-base linux-headers \ + ruby-dev gnupg \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install oj -v 3.13.19 \ + && gem install json -v 2.6.2 \ + && gem install async -v 1.30.3 \ + && gem install async-http -v 0.56.6 \ + && gem install fluentd -v 1.15.2 \ + && gem install bigdecimal -v 1.4.4 \ + && gem install fluent-plugin-azure-loganalytics -v 0.7.0 \ + && gem install fluent-plugin-azurestorage-gen2 -v 0.3.3 \ + && gem install fluent-plugin-cloudwatch-logs -v 0.14.3 \ + && gem install fluent-plugin-concat -v 2.5.0 \ + && gem install fluent-plugin-datadog -v 0.14.2 \ + && gem install fluent-plugin-elasticsearch -v 5.2.3 \ + && gem install fluent-plugin-grafana-loki -v 1.2.18 \ + && gem install fluent-plugin-kafka -v 0.18.1 \ + && gem install fluent-plugin-opensearch -v 1.0.8 \ + && gem install fluent-plugin-prometheus -v 2.0.3 \ + && gem install fluent-plugin-record-modifier -v 2.1.1 \ + && gem install fluent-plugin-rewrite-tag-filter -v 2.4.0 \ + && gem install fluent-plugin-route -v 1.0.0 \ + && gem install fluent-plugin-s3 -v 1.7.1 \ + && gem install fluent-plugin-sqs -v 3.0.0 \ + && apk del .build-deps \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN addgroup --system --gid 2000 fluent && adduser --system --ingroup fluent --uid 2000 fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + && mkdir -p /fluentd/state \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent /fluentd && chgrp -R fluent /fluentd + +COPY fluent.yaml /fluentd/etc/ +COPY --chmod=755 entrypoint.sh /bin/ + +ENV FLUENTD_CONF="fluent.yaml" + +ENV LD_PRELOAD="" +EXPOSE 24224 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/debian.dockerfile b/debian.dockerfile new file mode 100644 index 0000000..f131cca --- /dev/null +++ b/debian.dockerfile @@ -0,0 +1,74 @@ +FROM ruby:3.1-slim-bullseye + +ENV TINI_VERSION=0.18.0 + +# Do not split this into multiple RUN! +# Docker creates a layer for every RUN-Statement +# therefore an 'apt-get purge' has no effect +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + && buildDeps=" \ + make gcc g++ libc-dev \ + wget bzip2 gnupg dirmngr \ + " \ + && apt-get install -y --no-install-recommends $buildDeps \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install oj -v 3.13.19 \ + && gem install json -v 2.6.2 \ + && gem install async -v 1.30.3 \ + && gem install async-http -v 0.56.6 \ + && gem install fluentd -v 1.15.2 \ + && gem install fluent-plugin-azure-loganalytics -v 0.7.0 \ + && gem install fluent-plugin-azurestorage-gen2 -v 0.3.3 \ + && gem install fluent-plugin-cloudwatch-logs -v 0.14.3 \ + && gem install fluent-plugin-concat -v 2.5.0 \ + && gem install fluent-plugin-datadog -v 0.14.2 \ + && gem install fluent-plugin-elasticsearch -v 5.2.3 \ + && gem install fluent-plugin-grafana-loki -v 1.2.18 \ + && gem install fluent-plugin-kafka -v 0.18.1 \ + && gem install fluent-plugin-opensearch -v 1.0.8 \ + && gem install fluent-plugin-prometheus -v 2.0.3 \ + && gem install fluent-plugin-record-modifier -v 2.1.1 \ + && gem install fluent-plugin-rewrite-tag-filter -v 2.4.0 \ + && gem install fluent-plugin-route -v 1.0.0 \ + && gem install fluent-plugin-s3 -v 1.7.1 \ + && gem install fluent-plugin-sqs -v 3.0.0 \ + && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ + && wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini-$dpkgArch" \ + && wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini-$dpkgArch.asc" \ + && export GNUPGHOME="$(mktemp -d)" \ + && gpg --batch --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 6380DC428747F6C393FEACA59A84159D7001A4E5 \ + && gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \ + && rm -r /usr/local/bin/tini.asc \ + && chmod +x /usr/local/bin/tini \ + && tini -h \ + && wget -O /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \ + && cd /tmp && tar -xjf jemalloc-4.5.0.tar.bz2 && cd jemalloc-4.5.0/ \ + && ./configure && make \ + && mv lib/libjemalloc.so.2 /usr/lib \ + && apt-get purge -y --auto-remove \ + -o APT::AutoRemove::RecommendsImportant=false \ + $buildDeps \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN groupadd --system --gid 2000 fluent && useradd --system --gid fluent --uid 2000 fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + && mkdir -p /fluentd/state \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent /fluentd && chgrp -R fluent /fluentd + +COPY fluent.conf /fluentd/etc/ +COPY --chmod=755 entrypoint.sh /bin/ + +ENV FLUENTD_CONF="fluent.conf" + +ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2" +EXPOSE 24224 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..d0f6b25 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +#source vars if file exists +DEFAULT=/etc/default/fluentd + +if [ -r $DEFAULT ]; then + set -o allexport + . $DEFAULT + set +o allexport +fi + +# If the user has supplied only arguments append them to `fluentd` command +if [ "${1#-}" != "$1" ]; then + set -- fluentd "$@" +fi + +# If user does not supply config file or plugins, use the default +if [ "$1" = "fluentd" ]; then + if ! echo $@ | grep -e ' \-c' -e ' \-\-config' ; then + set -- "$@" --config /fluentd/etc/${FLUENTD_CONF} + fi + + if ! echo $@ | grep -e ' \-p' -e ' \-\-plugin' ; then + set -- "$@" --plugin /fluentd/plugins + fi +fi + +exec "$@" diff --git a/fluent.conf b/fluent.conf new file mode 100644 index 0000000..8a664b4 --- /dev/null +++ b/fluent.conf @@ -0,0 +1,18 @@ + + @type forward + @label @DEFAULT + port 24224 + bind 0.0.0.0 + + + + + diff --git a/fluent.yaml b/fluent.yaml new file mode 100644 index 0000000..ed857c7 --- /dev/null +++ b/fluent.yaml @@ -0,0 +1,23 @@ +system: + root_dir: /fluentd/state + +config: + - source: + $type: forward + $label: "@DEFAULT" + port: 24224 + bind: 0.0.0.0 + + - label: + $name: "@FLUENT_LOG" + config: + - match: + $tag: "fluent.*" + $type: stdout + + - label: + $name: "@DEFAULT" + config: + - match: + $tag: "**" + $type: stdout