Skip to content

Commit

Permalink
feat: Add initial version
Browse files Browse the repository at this point in the history
Signed-off-by: Steve Hipwell <steve.hipwell@gmail.com>
  • Loading branch information
stevehipwell committed Sep 15, 2022
1 parent e497c99 commit 52f1b26
Show file tree
Hide file tree
Showing 17 changed files with 484 additions and 0 deletions.
7 changes: 7 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Ignore everything
*

# Allow files and directories
!fluent.conf
!fluent.yaml
!entrypoint.sh
13 changes: 13 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# All
[*]
charset = utf-8
indent_style = space
indent_size = 2
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf
max_line_length = off

# Markdown uses whitespace for formatting
[*.md]
trim_trailing_whitespace = false
3 changes: 3 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
* text=auto eol=lf
*.{cmd,[cC][mM][dD]} text eol=crlf
*.{bat,[bB][aA][tT]} text eol=crlf
6 changes: 6 additions & 0 deletions .github/dependabot.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: /
schedule:
interval: daily
37 changes: 37 additions & 0 deletions .github/workflows/merge.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: Merge

on:
push:
branches:
- main

jobs:
build:
name: Build OCI Image
strategy:
fail-fast: false
matrix:
os: [alpine, debian]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b

- name: Run Hadolint
uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183
with:
dockerfile: ./${{ matrix.os }}.dockerfile

- name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6

- name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94
with:
file: ./${{ matrix.os }}.dockerfile
context: .
platforms: linux/amd64,linux/arm64
push: false
37 changes: 37 additions & 0 deletions .github/workflows/pull-request.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: Pull Request

on:
pull_request:
branches:
- main

jobs:
build:
name: Build OCI Image
strategy:
fail-fast: false
matrix:
os: [alpine, debian]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b

- name: Run Hadolint
uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183
with:
dockerfile: ./${{ matrix.os }}.dockerfile

- name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6

- name: Build
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94
with:
file: ./${{ matrix.os }}.dockerfile
context: .
platforms: linux/amd64
push: false
84 changes: 84 additions & 0 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
name: Release

on:
push:
tags:
- v*

jobs:
release:
name: Release OCI Image
strategy:
fail-fast: false
matrix:
os: [alpine, debian]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b

- name: Run Hadolint
uses: hadolint/hadolint-action@f988afea3da57ee48710a9795b6bb677cc901183
with:
dockerfile: ./${{ matrix.os }}.dockerfile

- name: Docker meta
id: metadata
uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a
with:
images: fluent/fluentd-aggregator,ghcr.io/fluent/fluentd-aggregator
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
type=raw,value=latest,enable={{is_default_branch}}
- name: Set up QEMU
uses: docker/setup-qemu-action@8b122486cedac8393e77aa9734c3528886e4a1a8

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6

- name: Login to GitHub Container Registry
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Login to DockerHub
uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Build & push
uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94
with:
file: ./${{ matrix.os }}.dockerfile
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.metadata.outputs.tags }}
labels: ${{ steps.metadata.outputs.labels }}

- name: Update Docker repo description
uses: peter-evans/dockerhub-description@da890086d39c735e41d8823c8a95bde4302c3d64
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
repository: fluent/fluentd-aggregator

- name: Get changelog entry
id: changelog
uses: mindsers/changelog-reader-action@17b72bf66059b7ac310abdc0b511948903e85ea5
with:
path: ./CHANGELOG.md
version: ${{ steps.metadata.outputs.version }}

- name: Create release
uses: ncipollo/release-action@58ae73b360456532aafd58ee170c045abbeaee37
with:
token: ${{ secrets.GITHUB_TOKEN }}
allowUpdates: true
body: ${{ steps.changelog.outputs.changes }}
Empty file added .gitignore
Empty file.
4 changes: 4 additions & 0 deletions .hadolint.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
failure-threshold: error
ignored:
- DL3008
- DL3018
11 changes: 11 additions & 0 deletions .markdownlint.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
MD013: false
MD024:
siblings_only: true
MD028: false
MD033:
allowed_elements:
- br
- details
- summary
- sub
- sup
22 changes: 22 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Fluentd Aggregator Docker Image Changelog

All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

---

<!-- ## [vX.Y.Z] - UNRELEASED
### Added
### Changed
### Updated
### Fixed
### Deprecated
### Removed -->

## [v0.0.1] - UNRELEASED

### Added

- Added initial version.
55 changes: 55 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Fluentd Aggregator Docker Image

[![Docker Image Version (latest semver)](https://img.shields.io/docker/v/fluent/fluentd-aggregator?sort=semver)](https://hub.docker.com/r/fluent/fluentd-aggregator)
![linux](https://img.shields.io/badge/os-linux-brightgreen)
![amd64](https://img.shields.io/badge/arch-amd64-brightgreen)
![arm64](https://img.shields.io/badge/arch-arm64-brightgreen)
[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)

A [Fluentd](https://www.fluentd.org/) [OCI](https://opencontainers.org/) image to be used for log aggregation and based on the official [Fluentd Docker image](https://github.com/fluent/fluentd-docker-image) rebuilt as a multi-arch `linux/amd64` & `linux/arm64` image.

## Aggregation Changes

To optimise _Fluentd_ for log aggregation the default configuration file has been overwritten to allow logs to be forwarded and printed to stdout. Plugins have also been added to support the aggregation role.

### Plugins

The following plugins have been added to the base image.

- [fluent-plugin-azure-loganalytics](https://github.com/yokawasa/fluent-plugin-azure-loganalytics)
- [fluent-plugin-azurestorage-gen2](https://github.com/oleewere/fluent-plugin-azurestorage-gen2)
- [fluent-plugin-cloudwatch-logs](https://github.com/fluent-plugins-nursery/fluent-plugin-cloudwatch-logs)
- [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat)
- [fluent-plugin-datadog](https://github.com/DataDog/fluent-plugin-datadog)
- [fluent-plugin-elasticsearch](https://docs.fluentd.org/output/elasticsearch)
- [fluent-plugin-grafana-loki](https://github.com/grafana/loki/tree/main/clients/cmd/fluentd)
- [fluent-plugin-kafka](https://github.com/fluent/fluent-plugin-kafka)
- [fluent-plugin-opensearch](https://github.com/fluent/fluent-plugin-opensearch)
- [fluent-plugin-prometheus](https://github.com/fluent/fluent-plugin-prometheus)
- [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier)
- [fluent-plugin-rewrite-tag-filter](https://github.com/fluent/fluent-plugin-rewrite-tag-filter)
- [fluent-plugin-route](https://github.com/tagomoris/fluent-plugin-route)
- [fluent-plugin-s3](https://docs.fluentd.org/output/s3)
- [fluent-plugin-sqs](https://github.com/ixixi/fluent-plugin-sqs)

## Usage

This image is available at [Docker Hub](https://hub.docker.com/r/fluent/fluentd-aggregator); the image version matches the _Fluentd_ version that it's based on.

This image can be pulled with the following commands.

```shell
docker pull fluent/fluentd-aggregator:latest

docker pull ghcr.io/fluent/fluentd-aggregator:latest
```

This image can be tested by running the following command and then forwarding logs.

```shell
docker run -p 24224:24224 fluent/fluentd-aggregator:latest
```

## License

[Apache License, Version 2.0](./LICENSE).
60 changes: 60 additions & 0 deletions alpine.dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
FROM alpine:3.16
LABEL maintainer "Fluentd developers <fluentd@googlegroups.com>"
LABEL description="Fluentd aggregator OCI image based on Fluentd v1.15.2" version="0.0.1"

# Do not split this into multiple RUN!
# Docker creates a layer for every RUN-Statement
# therefore an 'apk delete' has no effect
RUN apk update \
&& apk add --no-cache \
ca-certificates \
ruby ruby-irb ruby-etc ruby-webrick \
tini \
libcurl \
&& apk add --no-cache --virtual .build-deps \
build-base linux-headers \
ruby-dev gnupg \
&& echo 'gem: --no-document' >> /etc/gemrc \
&& gem install oj -v 3.13.19 \
&& gem install json -v 2.6.2 \
&& gem install async -v 1.30.3 \
&& gem install async-http -v 0.56.6 \
&& gem install fluentd -v 1.15.2 \
&& gem install bigdecimal -v 1.4.4 \
&& gem install fluent-plugin-azure-loganalytics -v 0.7.0 \
&& gem install fluent-plugin-azurestorage-gen2 -v 0.3.3 \
&& gem install fluent-plugin-cloudwatch-logs -v 0.14.3 \
&& gem install fluent-plugin-concat -v 2.5.0 \
&& gem install fluent-plugin-datadog -v 0.14.2 \
&& gem install fluent-plugin-elasticsearch -v 5.2.3 \
&& gem install fluent-plugin-grafana-loki -v 1.2.18 \
&& gem install fluent-plugin-kafka -v 0.18.1 \
&& gem install fluent-plugin-opensearch -v 1.0.8 \
&& gem install fluent-plugin-prometheus -v 2.0.3 \
&& gem install fluent-plugin-record-modifier -v 2.1.1 \
&& gem install fluent-plugin-rewrite-tag-filter -v 2.4.0 \
&& gem install fluent-plugin-route -v 1.0.0 \
&& gem install fluent-plugin-s3 -v 1.7.1 \
&& gem install fluent-plugin-sqs -v 3.0.0 \
&& apk del .build-deps \
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test

RUN addgroup --system --gid 2000 fluent && adduser --system --ingroup fluent --uid 2000 fluent \
# for log storage (maybe shared with host)
&& mkdir -p /fluentd/log \
&& mkdir -p /fluentd/state \
# configuration/plugins path (default: copied from .)
&& mkdir -p /fluentd/etc /fluentd/plugins \
&& chown -R fluent /fluentd && chgrp -R fluent /fluentd

COPY fluent.yaml /fluentd/etc/
COPY entrypoint.sh /bin/

ENV FLUENTD_CONF="fluent.yaml"

ENV LD_PRELOAD=""
EXPOSE 24224

USER fluent
ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"]
CMD ["fluentd"]
Loading

0 comments on commit 52f1b26

Please sign in to comment.