adds Dockerfile and build workflow for CSV provider

.github/workflows/kuksa_csv_provider.yml

@@ -0,0 +1,121 @@
+# /********************************************************************************
+# * Copyright (c) 2022,2023 Contributors to the Eclipse Foundation
+# *
+# * See the NOTICE file(s) distributed with this work for additional
+# * information regarding copyright ownership.
+# *
+# * This program and the accompanying materials are made available under the
+# * terms of the Apache License 2.0 which is available at
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * SPDX-License-Identifier: Apache-2.0
+# ********************************************************************************/
+
+name: kuksa_csv_provider
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths:
+      - ".github/workflows/kuksa_csv_provider.yml"
+      - "csv_provider/**"
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  checkrights:
+    uses: ./.github/workflows/check_push_rights.yml
+    secrets: inherit
+
+  run-csv-provider-tests:
+    name: "Run csv provider linter"
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+
+    - name: Run pylint (but accept errors for now)
+      run: |
+        cd csv_provider
+        pip3 install --no-cache-dir -r requirements.txt
+        pip3 install --no-cache-dir pylint
+        # First just show, never fail
+        pylint --exit-zero provider.py
+        # Fail on errors and above
+        pylint -E provider.py
+
+  build-csv-provider-image:
+    name: "Build csv provider image"
+    runs-on: self-hosted
+    needs: checkrights
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v4
+      with:
+        # list of Docker images to use as base name for tags
+        images: |
+          ghcr.io/eclipse/kuksa.val.feeders/csv-provider
+        # generate Docker tags based on the following events/attributes
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=semver,pattern={{version}}
+          type=semver,pattern={{major}}.{{minor}}
+          type=semver,pattern={{major}}
+
+    # only needed for runners without buildx setup, will be slow
+    #- name: Set up QEMU
+    #  uses: docker/setup-qemu-action@v2
+
+    #- name: Set up Docker Buildx
+    #  uses: docker/setup-buildx-action@v2
+
+    - name: Log in to the Container registry
+      if: needs.checkrights.outputs.have_secrets == 'true'
+      uses: docker/login-action@v2
+      with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build CSV provider container and push to ghcr.io (and ttl.sh)
+      id: ghcr-build
+      if: ${{ needs.checkrights.outputs.have_secrets == 'true' && github.event_name != 'pull_request' }}
+      uses: docker/build-push-action@v3
+      with:
+        platforms: |
+          linux/amd64
+          linux/arm64
+        file: ./csv_provider/Dockerfile
+        context: ./csv_provider/
+        push: true
+        tags: |
+          ${{ steps.meta.outputs.tags }}
+          ttl.sh/kuksa.val/kuksa-csvprovider-${{github.sha}}
+        labels: ${{ steps.meta.outputs.labels }}
+
+    - name: Build ephemeral CSV provider container and push to ttl.sh
+      if: ${{ needs.checkrights.outputs.have_secrets == 'false' || github.event_name == 'pull_request' }}
+      id: tmp-build
+      uses: docker/build-push-action@v3
+      with:
+        platforms: |
+          linux/amd64
+          linux/arm64
+        file: ./csv_provider/Dockerfile
+        context: ./csv_provider/
+        push: true
+        tags: "ttl.sh/kuksa.val/kuksa-csvprovider-${{github.sha}}"
+        labels: ${{ steps.meta.outputs.labels }}
+
+    - name: Posting message
+      uses: ./.github/actions/post-container-location
+      with:
+        image: ttl.sh/kuksa.val/kuksa-csvprovider-${{github.sha}}

csv_provider/Dockerfile

@@ -0,0 +1,60 @@
+# /********************************************************************************
+# * Copyright (c) 2023 Contributors to the Eclipse Foundation
+# *
+# * See the NOTICE file(s) distributed with this work for additional
+# * information regarding copyright ownership.
+# *
+# * This program and the accompanying materials are made available under the
+# * terms of the Apache License 2.0 which is available at
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * SPDX-License-Identifier: Apache-2.0
+# ********************************************************************************/
+
+
+# Build stage, to create a Virtual Environent
+FROM --platform=$TARGETPLATFORM python:3.10-alpine as builder
+
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+
+RUN echo "-- Running on $BUILDPLATFORM, building for $TARGETPLATFORM"
+
+RUN apk update && apk add alpine-sdk linux-headers
+
+COPY . /
+
+RUN python3 -m venv /opt/venv
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+RUN /opt/venv/bin/python3 -m pip install --upgrade pip \
+    && pip3 install --no-cache-dir -r requirements.txt
+
+
+RUN pip3 install wheel scons && pip3 install pyinstaller
+
+RUN pyinstaller --clean -F -s provider.py
+
+WORKDIR /dist
+
+WORKDIR /data
+COPY ./signals.csv ./signals.csv
+
+# Runner stage, to copy in the virtual environment and the app
+FROM alpine:3
+
+
+WORKDIR /dist
+
+COPY --from=builder /dist/* .
+COPY --from=builder /data/ ./
+
+ENV PATH="/dist:$PATH"
+
+# useful dumps about feeding values
+ENV LOG_LEVEL="info"
+
+ENV PYTHONUNBUFFERED=yes
+
+ENTRYPOINT ["./provider"]

csv_provider/README.md

@@ -17,13 +17,13 @@ The provider uses the [kuksa_client]() Python implementation which you need to i
 ## Arguments
 You can start the provider with the following arguments on a command line:
 
-| short argument | long argument | description | default value |
-|---- | ---- | ----- | ----|
-|-f| --file | This indicates the CSV-file containing the signals to update in the `kuksa.val` databroker. | signals.csv |
-| -a | --address | This indicates the address of `kuksa.val` databroker to connect to. | 127.0.0.1 |
-| -p | --port | This indicates the port of the `kuksa.val` databroker to connect to. | 55555 |
-| -i | --infinite | If the flag is set, the provider loops over the file until stopped, otherwise the file gets processed once. | not present/False
-| -l | --log | This sets the logging level. Possible values are: DEBUG, INFO, DEBUG, WARNING, ERROR, CRITICAL | WARNING
+| short argument | long argument | environment variable | description | default value |
+|---- | ---- | ---- |----- | ----|
+|-f| --file | PROVIDER_SIGNALS_FILE | This indicates the CSV-file containing the signals to update in the `kuksa.val` databroker. | signals.csv |
+| -a | --address | KUKSA_DATA_BROKER_ADDR | This indicates the address of `kuksa.val` databroker to connect to. | 127.0.0.1 |
+| -p | --port | KUKSA_DATA_BROKER_PORT | This indicates the port of the `kuksa.val` databroker to connect to. | 55555 |
+| -i | --infinite | PROVIDER_INFINITE | If the flag is set, the provider loops over the file until stopped, otherwise the file gets processed once. | not present/False
+| -l | --log | PROVIDER_LOG_LEVEL | This sets the logging level. Possible values are: DEBUG, INFO, DEBUG, WARNING, ERROR, CRITICAL | WARNING
 
 ## CSV File
 An example CSV-files is available in [signals.csv](signals.csv) where an example line is:

csv_provider/provider.py

@@ -12,11 +12,14 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 ########################################################################
+'''A provider accepting VSS-signals from a CSV-file
+ to write these signals into an Kuksa.val data broker'''
 
 import asyncio
 import csv
 import argparse
 import logging
+import os
 
 from kuksa_client.grpc import Datapoint
 from kuksa_client.grpc import DataEntry
@@ -27,36 +30,47 @@
 
 
 def init_argparse() -> argparse.ArgumentParser:
+    '''This inits the argument parser for the CSV-provider.'''
     parser = argparse.ArgumentParser(
         usage="-a [BROKER ADDRESS] -p [BROKER PORT] -f [FILE]",
         description="This provider writes the content of a csv file to a kuksa.val databroker",
     )
-    parser.add_argument("-a", "--address", default="127.0.0.1", help="This indicates the address" +
-                        " of the kuksa.val databroker to connect to. The default value is 127.0.0.1")
-    parser.add_argument("-p", "--port", default="55555", help="This indicates the port" +
-                        " of the kuksa.val databroker to connect to. The default value is 5555", type=int)
-    parser.add_argument("-f", "--file", default="signals.csv", help="This indicates the csv file" +
-                        " containing the signals to update in the kuksa.val databroker." +
-                        " The default value is signals.csv.")
-    parser.add_argument("-i", "--infinite", action=argparse.BooleanOptionalAction,
-                        help="If the flag is set, the provider loops" +
+    environment = os.environ
+    parser.add_argument("-a", "--address", default=environment.get("KUKSA_DATA_BROKER_ADDR",
+                                                                   "127.0.0.1"),
+                        help="This indicates the address of the kuksa.val databroker to connect to."
+                        " The default value is 127.0.0.1")
+    parser.add_argument("-p", "--port", default=environment.get('KUKSA_DATA_BROKER_PORT', "55555"),
+                        help="This indicates the port of the kuksa.val databroker to connect to."
+                        " The default value is 55555", type=int)
+    parser.add_argument("-f", "--file", default=environment.get("PROVIDER_SIGNALS_FILE",
+                                                                "signals.csv"),
+                        help="This indicates the csv file containing the signals to update in"
+                        " the kuksa.val databroker. The default value is signals.csv.")
+    parser.add_argument("-i", "--infinite", default=environment.get("PROVIDER_INFINITE"),
+                        action=argparse.BooleanOptionalAction,
+                        help="If the flag is set, the provider loops"
                         "the file until stopped, otherwise the file gets processed once.")
-    parser.add_argument("-l", "--log", default="INFO", help="This sets the logging level." +
-                        " The default value is WARNING.",
+    parser.add_argument("-l", "--log", default=environment.get("PROVIDER_LOG_LEVEL", "INFO"),
+                        help="This sets the logging level. The default value is WARNING.",
                         choices={"INFO", "DEBUG", "WARNING", "ERROR", "CRITICAL"})
     return parser
 
 
 async def main():
+    '''the main function as entry point for the CSV-provider'''
     parser = init_argparse()
     args = parser.parse_args()
     numeric_value = getattr(logging, args.log.upper(), None)
     if isinstance(numeric_value, int):
         logging.basicConfig(encoding='utf-8', level=numeric_value)
     try:
         async with VSSClient(args.address, args.port) as client:
-            csvfile = open(args.file, newline='')
-            signal_reader = csv.DictReader(csvfile, delimiter=',', quotechar='|', skipinitialspace=True)
+            csvfile = open(args.file, newline='', encoding="utf-8")
+            signal_reader = csv.DictReader(csvfile,
+                                           delimiter=',',
+                                           quotechar='|',
+                                           skipinitialspace=True)
             logging.info("Starting to apply the signals read from %s.", str(csvfile.name))
             if args.infinite:
                 backup = list(signal_reader)
@@ -67,12 +81,14 @@ async def main():
             else:
                 await process_rows(client, signal_reader)
     except VSSClientError:
-        logging.error("Could not connect to the kuksa.val databroker at %s:%s. " +
-                      "Make sure to set the correct connection details using --address and --port" +
-                      "and that the kuksa.val databroker is running.", args.address, args.port)
+        logging.error("Could not connect to the kuksa.val databroker at %s:%s."
+                      " Make sure to set the correct connection details using --address and --port"
+                      " and that the kuksa.val databroker is running.", args.address, args.port)
 
 
 async def process_rows(client, rows):
+    '''Processes a single row from the CSV-file and write the
+     recorded signal to the data broker through the client.'''
     for row in rows:
         entry = DataEntry(
             row['signal'],
@@ -88,11 +104,13 @@ async def process_rows(client, rows):
             updates = []
         try:
             await client.set(updates=updates)
-        except Exception as ex:
+        except VSSClientError as ex:
             logging.error("Error while updating %s\n%s", row['signal'], ex)
         try:
-            await asyncio.sleep(float(row['delay']))
-        except Exception:
-            logging.error("Error while waiting for '%s' seconds after updating %s.\n" +
-                          "Make sure to only use numbers for the delay value.", row['sleep'], row['delay'])
+            await asyncio.sleep(delay=float(row['delay']))
+        except ValueError:
+            logging.error("Error while waiting for %s seconds after updating %s to %s."
+                          " Make sure to only use numbers for the delay value.",
+                          row['delay'], row['signal'], row['value'])
+
 asyncio.run(main())

csv_provider/requirements.txt

@@ -1 +1 @@
-kuksa-client==0.3.x
+kuksa-client==0.3.1