From 6f0685d9f752c8f4bf324ef261ffbc933c7ca5c0 Mon Sep 17 00:00:00 2001 From: Sven Erik Jeroschewski Date: Fri, 26 May 2023 14:46:19 +0200 Subject: [PATCH] adds Dockerfile and build workflow for CSV provider --- .github/workflows/kuksa_csv_provider.yml | 119 +++++++++++++++++++++++ csv_provider/Dockerfile | 62 ++++++++++++ csv_provider/README.md | 14 +-- csv_provider/provider.py | 27 ++--- csv_provider/requirements.txt | 2 +- 5 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/kuksa_csv_provider.yml create mode 100644 csv_provider/Dockerfile diff --git a/.github/workflows/kuksa_csv_provider.yml b/.github/workflows/kuksa_csv_provider.yml new file mode 100644 index 00000000..b0d2ccd7 --- /dev/null +++ b/.github/workflows/kuksa_csv_provider.yml @@ -0,0 +1,119 @@ +# /******************************************************************************** +# * Copyright (c) 2022,2023 Contributors to the Eclipse Foundation +# * +# * See the NOTICE file(s) distributed with this work for additional +# * information regarding copyright ownership. +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License 2.0 which is available at +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * SPDX-License-Identifier: Apache-2.0 +# ********************************************************************************/ + +name: kuksa_csv_provider + +on: + push: + branches: [ main ] + pull_request: + paths: + - ".github/workflows/kuksa_csv_provider.yml" + - "csv_provider/**" + workflow_call: + workflow_dispatch: + +jobs: + checkrights: + uses: ./.github/workflows/check_push_rights.yml + secrets: inherit + + run-csv-provider-tests: + name: "Run csv provider linter" + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Run pylint (but accept errors for now) + run: | + cd csv_provider + # First just show, never fail + pylint --exit-zero provider.py + # Fail on errors and above + pylint -E provider.py + + build-csv-provider-image: + name: "Build csv provider image" + runs-on: self-hosted + needs: checkrights + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + # list of Docker images to use as base name for tags + images: | + ghcr.io/eclipse/kuksa.val.feeders/csv-provider + # generate Docker tags based on the following events/attributes + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + # only needed for runners without buildx setup, will be slow + #- name: Set up QEMU + # uses: docker/setup-qemu-action@v2 + + #- name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + + - name: Log in to the Container registry + if: needs.checkrights.outputs.have_secrets == 'true' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build CSV provider container and push to ghcr.io (and ttl.sh) + id: ghcr-build + if: ${{ needs.checkrights.outputs.have_secrets == 'true' && github.event_name != 'pull_request' }} + uses: docker/build-push-action@v3 + with: + platforms: | + linux/amd64 + linux/arm64 + file: ./csv_provider/Dockerfile + context: ./csv_provider/ + push: true + tags: | + ${{ steps.meta.outputs.tags }} + ttl.sh/kuksa.val/kuksa-csvprovider-${{github.sha}} + labels: ${{ steps.meta.outputs.labels }} + + - name: Build ephemeral CSV provider container and push to ttl.sh + if: ${{ needs.checkrights.outputs.have_secrets == 'false' || github.event_name == 'pull_request' }} + id: tmp-build + uses: docker/build-push-action@v3 + with: + platforms: | + linux/amd64 + linux/arm64 + file: ./csv_provider/Dockerfile + context: ./csv_provider/ + push: true + tags: "ttl.sh/kuksa.val/kuksa-csvprovider-${{github.sha}}" + labels: ${{ steps.meta.outputs.labels }} + + - name: Posting message + uses: ./.github/actions/post-container-location + with: + image: ttl.sh/kuksa.val/kuksa-csvprovider-${{github.sha}} diff --git a/csv_provider/Dockerfile b/csv_provider/Dockerfile new file mode 100644 index 00000000..c83d3259 --- /dev/null +++ b/csv_provider/Dockerfile @@ -0,0 +1,62 @@ +# /******************************************************************************** +# * Copyright (c) 2023 Contributors to the Eclipse Foundation +# * +# * See the NOTICE file(s) distributed with this work for additional +# * information regarding copyright ownership. +# * +# * This program and the accompanying materials are made available under the +# * terms of the Apache License 2.0 which is available at +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * SPDX-License-Identifier: Apache-2.0 +# ********************************************************************************/ + + +# Build stage, to create a Virtual Environent +FROM --platform=$TARGETPLATFORM python:3.10-alpine as builder + +ARG TARGETPLATFORM +ARG BUILDPLATFORM + +RUN echo "-- Running on $BUILDPLATFORM, building for $TARGETPLATFORM" + +RUN apk update && apk add alpine-sdk linux-headers + +COPY . / + +RUN python3 -m venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" + +RUN /opt/venv/bin/python3 -m pip install --upgrade pip \ + && pip3 install --no-cache-dir -r requirements.txt + + +RUN pip3 install wheel scons && pip3 install pyinstaller + +# By default we use certificates and tokens from kuksa_certificates, so they must be included +RUN pyinstaller --clean -F -s provider.py +# --debug=imports + +WORKDIR /dist + +WORKDIR /data +COPY ./signals.csv ./signals.csv + +# Runner stage, to copy in the virtual environment and the app +FROM alpine:3 + + +WORKDIR /dist + +COPY --from=builder /dist/* . +COPY --from=builder /data/ ./ + +ENV PATH="/dist:$PATH" + +# useful dumps about feeding values +ENV LOG_LEVEL="info" + +ENV PYTHONUNBUFFERED=yes + +ENTRYPOINT ["./provider"] diff --git a/csv_provider/README.md b/csv_provider/README.md index de4b21d4..23cf0a53 100644 --- a/csv_provider/README.md +++ b/csv_provider/README.md @@ -17,13 +17,13 @@ The provider uses the [kuksa_client]() Python implementation which you need to i ## Arguments You can start the provider with the following arguments on a command line: -| short argument | long argument | description | default value | -|---- | ---- | ----- | ----| -|-f| --file | This indicates the CSV-file containing the signals to update in the `kuksa.val` databroker. | signals.csv | -| -a | --address | This indicates the address of `kuksa.val` databroker to connect to. | 127.0.0.1 | -| -p | --port | This indicates the port of the `kuksa.val` databroker to connect to. | 55555 | -| -i | --infinite | If the flag is set, the provider loops over the file until stopped, otherwise the file gets processed once. | not present/False -| -l | --log | This sets the logging level. Possible values are: DEBUG, INFO, DEBUG, WARNING, ERROR, CRITICAL | WARNING +| short argument | long argument | environment variable | description | default value | +|---- | ---- | ---- |----- | ----| +|-f| --file | PROVIDER_SIGNALS_FILE | This indicates the CSV-file containing the signals to update in the `kuksa.val` databroker. | signals.csv | +| -a | --address | KUKSA_DATA_BROKER_ADDR | This indicates the address of `kuksa.val` databroker to connect to. | 127.0.0.1 | +| -p | --port | KUKSA_DATA_BROKER_PORT | This indicates the port of the `kuksa.val` databroker to connect to. | 55555 | +| -i | --infinite | PROVIDER_INFINITE | If the flag is set, the provider loops over the file until stopped, otherwise the file gets processed once. | not present/False +| -l | --log | PROVIDER_LOG_LEVEL | This sets the logging level. Possible values are: DEBUG, INFO, DEBUG, WARNING, ERROR, CRITICAL | WARNING ## CSV File An example CSV-files is available in [signals.csv](signals.csv) where an example line is: diff --git a/csv_provider/provider.py b/csv_provider/provider.py index 74e0da6e..3d01f883 100755 --- a/csv_provider/provider.py +++ b/csv_provider/provider.py @@ -17,6 +17,7 @@ import csv import argparse import logging +import os from kuksa_client.grpc import Datapoint from kuksa_client.grpc import DataEntry @@ -31,18 +32,22 @@ def init_argparse() -> argparse.ArgumentParser: usage="-a [BROKER ADDRESS] -p [BROKER PORT] -f [FILE]", description="This provider writes the content of a csv file to a kuksa.val databroker", ) - parser.add_argument("-a", "--address", default="127.0.0.1", help="This indicates the address" + - " of the kuksa.val databroker to connect to. The default value is 127.0.0.1") - parser.add_argument("-p", "--port", default="55555", help="This indicates the port" + - " of the kuksa.val databroker to connect to. The default value is 5555", type=int) - parser.add_argument("-f", "--file", default="signals.csv", help="This indicates the csv file" + - " containing the signals to update in the kuksa.val databroker." + - " The default value is signals.csv.") - parser.add_argument("-i", "--infinite", action=argparse.BooleanOptionalAction, + environment = os.environ + parser.add_argument("-a", "--address", default=environment.get("KUKSA_DATA_BROKER_ADDR", "127.0.0.1"), + help="This indicates the address of the kuksa.val databroker to connect to." + + " The default value is 127.0.0.1") + parser.add_argument("-p", "--port", default=environment.get('KUKSA_DATA_BROKER_PORT', "55555"), + help="This indicates the port of the kuksa.val databroker to connect to." + + " The default value is 55555", type=int) + parser.add_argument("-f", "--file", default=environment.get("PROVIDER_SIGNALS_FILE", "signals.csv"), + help="This indicates the csv file containing the signals to update in" + + " the kuksa.val databroker. The default value is signals.csv.") + parser.add_argument("-i", "--infinite", default=environment.get("PROVIDER_INFINITE"), + action=argparse.BooleanOptionalAction, help="If the flag is set, the provider loops" + "the file until stopped, otherwise the file gets processed once.") - parser.add_argument("-l", "--log", default="INFO", help="This sets the logging level." + - " The default value is WARNING.", + parser.add_argument("-l", "--log", default=environment.get("PROVIDER_LOG_LEVEL", "INFO"), + help="This sets the logging level. The default value is WARNING.", choices={"INFO", "DEBUG", "WARNING", "ERROR", "CRITICAL"}) return parser @@ -69,7 +74,7 @@ async def main(): except VSSClientError: logging.error("Could not connect to the kuksa.val databroker at %s:%s. " + "Make sure to set the correct connection details using --address and --port" + - "and that the kuksa.val databroker is running.", args.address, args.port) + " and that the kuksa.val databroker is running.", args.address, args.port) async def process_rows(client, rows): diff --git a/csv_provider/requirements.txt b/csv_provider/requirements.txt index 53296f8e..761178f8 100644 --- a/csv_provider/requirements.txt +++ b/csv_provider/requirements.txt @@ -1 +1 @@ -kuksa-client==0.3.x +kuksa-client==0.3.1