Add DDS provider

.github/workflows/kuksa_dds_feeder.yml

@@ -0,0 +1,109 @@
+# /********************************************************************************
+# * Copyright (c) 2022 Contributors to the Eclipse Foundation
+# *
+# * See the NOTICE file(s) distributed with this work for additional
+# * information regarding copyright ownership.
+# *
+# * This program and the accompanying materials are made available under the
+# * terms of the Apache License 2.0 which is available at
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * SPDX-License-Identifier: Apache-2.0
+# ********************************************************************************/
+
+name: kuksa_dds_feeder
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/kuksa_dds_feeder.yml"
+      - "dds2val/**"
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  check-dds-feeder:
+    name: "Check DDS feeder"
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+
+    - name: Retrieve build binaries
+      uses: actions/download-artifact@v3
+      with:
+        path: ${{github.workspace}}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - id: repository-name-adjusted
+      name: Make repository name in lower case for docker upload.
+      uses: ASzc/change-string-case-action@v2
+      with:
+        string: ${{ github.repository }}
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
+    - name: build linux/amd64 docker image
+      id: image_build_amd64
+      uses: docker/build-push-action@v2
+      with:
+        pull: true
+        push: false
+        outputs: |
+          type=oci,dest=./dds2val_amd64.tar
+        context: ./dds2val
+        file: ./dds2val/Dockerfile
+        build-args: | 
+          TARGETPLATFORM=linux/amd64
+        tags: ${{ github.sha }}
+        labels: |
+          org.opencontainers.image.source=https://github.com/${{steps.repository-name-adjusted.outputs.lowercase}}
+
+    - name: Temporarily save linux/amd64 Docker image
+      uses: actions/upload-artifact@v3
+      with:
+        name: Container image
+        path: ${{github.workspace}}/dds2val_amd64.tar
+        retention-days: 1
+
+    - name: build linux/arm64 docker image
+      id: image_build_arm64
+      uses: docker/build-push-action@v2
+      with:
+        pull: true
+        push: false
+        outputs: |
+          type=oci,dest=./dds2val_arm64.tar
+        context: ./dds2val
+        file: ./dds2val/Dockerfile
+        build-args: | 
+          TARGETPLATFORM=linux/arm64
+          TARGETARCH=arm64
+        tags: ${{ github.sha }}
+        labels: |
+          org.opencontainers.image.source=https://github.com/${{steps.repository-name-adjusted.outputs.lowercase}}
+
+    - name: Temporarily save linux/arm64 Docker image
+      uses: actions/upload-artifact@v3
+      with:
+        name: Container image
+        path: ${{github.workspace}}/dds2val_arm64.tar
+        retention-days: 1
+
+    - name: Run dds tests
+      run: |
+        cd dds2val
+        pip3 install --no-cache-dir -r requirements/requirements.txt -r requirements/requirements-kml.txt -r requirements/requirements-test.txt
+        ./ddsproviderlib/idls/generate_py_dataclass.sh
+        python -m pytest tests/*

dds2val/Dockerfile

@@ -0,0 +1,76 @@
+# /********************************************************************************
+# * Copyright (c) 2023 Contributors to the Eclipse Foundation
+# *
+# * See the NOTICE file(s) distributed with this work for additional
+# * information regarding copyright ownership.
+# *
+# * This program and the accompanying materials are made available under the
+# * terms of the Apache License 2.0 which is available at
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * SPDX-License-Identifier: Apache-2.0
+# ********************************************************************************/
+
+
+# Build stage, to create a Virtual Environent
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+
+FROM --platform=$TARGETPLATFORM python:3.9-slim-bullseye as builder
+
+RUN echo "-- Running on $BUILDPLATFORM, building for $TARGETPLATFORM"
+
+RUN apt-get update -qqy && apt-get upgrade -qqy && apt-get install -qqy binutils g++ git python3-dev
+
+COPY . /
+
+RUN python3 -m venv --system-site-packages /opt/venv
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+RUN /opt/venv/bin/python3 -m pip install --upgrade pip
+RUN pip3 install wheel
+ARG TARGETARCH
+ENV TARGETARCH=$TARGETARCH
+RUN chmod u+x setup_arm_build.sh
+RUN bash setup_arm_build.sh
+ENV PATH="/cyclonedds/install/bin:$PATH"
+RUN pip3 install --no-cache-dir -r requirements/requirements.txt
+
+RUN pip3 install scons && pip3 install pyinstaller patchelf==0.17.0.0 staticx
+WORKDIR /ddsproviderlib/idls
+RUN ./generate_py_dataclass.sh
+
+WORKDIR /
+ENV PATH="/idls:$PATH"
+# The cyclonedds module uses a dynamic library which is not automatically discovered (--add-binary)
+RUN chmod u+x make_executable.sh
+RUN bash make_executable.sh
+
+WORKDIR /dist
+RUN staticx ddsprovider ddsprovider-exe
+
+# Runner stage, to copy in the virtual environment and the app
+FROM scratch
+
+LABEL org.opencontainers.image.source="https://github.com/eclipse/kuksa.val.feeders"
+
+LABEL org.opencontainers.image.licenses="APACHE-2.0"
+
+# needed as /dist/binary unpacks and runs from /tmp
+WORKDIR /tmp
+# optional volume mapping
+WORKDIR /conf
+
+WORKDIR /dist
+
+COPY --from=builder /dist/ddsprovider-exe .
+
+ENV PATH="/dist:$PATH"
+
+# useful dumps about feeding values
+ENV LOG_LEVEL="info"
+
+ENV PYTHONUNBUFFERED=yes
+
+ENTRYPOINT ["./ddsprovider-exe"]

dds2val/README.md

@@ -0,0 +1,74 @@
+# DDS Provider
+
+The DDS provider provides data from an DDS middleware/API. For further understanding of the DDS middleware/API see [this](https://www.dds-foundation.org/what-is-dds-3/). The DDS provider only works with the KUKSA databroker. The KUKSA C++ server is not supported.
+
+## How to build
+
+### local build
+
+1. `python3 -m venv env && source env/bin/activate`
+2. `pip install -r requirements.txt`
+3. `chmod u+x ddsproviderlib/idls/generate_py_dataclass.sh`
+4. `./ddsproviderlib/idls/generate_py_dataclass.sh`
+
+### build image (suggested)
+
+1. `docker build -f Dockerfile --progress=plain --build-arg TARGETPLATFORM=linux/amd64 -t ddsprovider:latest .`
+
+### KML replay
+
+1. `python3 -m venv env && source env/bin/activate`
+2. `pip install -r requirements-kml.txt`
+3. `cd kml && python3 dds_kmlreplay.py directions.kml`
+
+## How to run
+
+Choose from local build or contanerization via docker.
+These steps are necessary:
+
+1. Run an instance of databroker aka: `docker run -it --rm --net=host ghcr.io/eclipse/kuksa.val/databroker:master`
+2. Start the KML replay with an active local python virtual environment: `pip install requirements/requirements-kml.txt && cd kml && python3 dds_kmlreplay.py directions.kml`
+3. Start the DDS provider with either: `docker run --rm -it --net=host ddsprovider:latest` or with an active local python virtual environment: `python3 ddsprovider.py`
+
+## Configure the DDS provider
+
+Configuration for the DDS provider is solved through setting environment variables. Have a look at the table below.
+
+| Environment variable          | default value | description                                                                                                                                      |
+| ----------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ |
+| VEHICLEDATABROKER_DAPR_APP_ID | None          | DAPR ID for Vehicle App to look for. For more information to Vehicle Apps visit [Velocitas](https://eclipse-velocitas.github.io/velocitas-docs/) |
+| VDB_ADDRESS                   | 127.0.0.1     | Address where to look for (Vehicle) Databroker                                                                                                   |
+| DAPR_GRPC_PORT                | 55555         | Port where to look for (Vehicle) Databroker. If [DAPR](https://dapr.io/) gets used port of DAPR Sidecar where to look                            |
+| MAPPING_FILE                  | mapping.yml   | Place of mapping file from DDS to VSS                                                                                                            |
+
+## Overall sequence
+
+```mermaid
+sequenceDiagram
+title DDS on SDV
+box LightBlue Container-1
+participant databroker
+end
+
+dddsprovider -->> databroker : grpc_connect
+alt on connection
+    ddsprovider -->> databroker : register_datapoints
+    ddsprovider -->> DDS_network : subscribe to topics
+    ddspublisher1 -->> DDS_network : publish dds message
+    ddspublisher2 -->> DDS_network : publish dds message
+end
+box LightBlue Container-2
+participant ddspublisher1
+end
+box LightBlue Container-3
+participant ddspublisher2
+end
+alt on data reception
+    ddsprovider -->> databroker : update_datapoint
+end
+```
+
+## How to run the tests
+
+1. create virtual python environment (`python3 -m venv testEnv && source testEnv/bin/activate && pip install -r requirements/requirements.txt requirements/requirements-test.txt requirements/requirements-kml.txt`)
+2. terminal 2: `source testEnv/bin/activate && pytest --html=report.html --self-contained-html --cov=. tests/* --cov-report html --cov-report xml`

dds2val/ddsprovider.py

@@ -0,0 +1,53 @@
+import logging
+import os
+import signal
+import asyncio
+from pathlib import Path
+from ddsproviderlib import helper
+
+log = logging.getLogger("ddsprovider")
+
+async def main():
+    """Perform the main function activities."""
+    logging.basicConfig(level=logging.DEBUG)
+    log.setLevel(logging.DEBUG)
+
+    console_logger = logging.StreamHandler()
+    log.addHandler(console_logger)
+    log.info("Starting ddsprovider...")
+
+    if os.environ.get("VEHICLEDATABROKER_DAPR_APP_ID"):
+        grpc_metadata = (
+            ("dapr-app-id", os.environ.get("VEHICLEDATABROKER_DAPR_APP_ID")),
+        )
+    else:
+        grpc_metadata = None
+
+    databroker_address = os.environ.get("VDB_ADDRESS", "127.0.0.1:") + os.environ.get("DAPR_GRPC_PORT", "55555")
+
+    mappingfile = os.environ.get(
+        "MAPPING_FILE", str(Path(__file__).parent / "mapping.yml")
+    )
+
+    ddsprovider = helper.Ddsprovider()
+
+    # Handler for Ctrl-C and Kill signal
+    def signal_handler(signal_received, _frame):
+        log.info("Received signal %s, stopping", signal_received)
+        ddsprovider.stop()
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    await ddsprovider.start(
+        databroker_address=databroker_address,
+        grpc_metadata=grpc_metadata,
+        mappingfile=mappingfile,
+    )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    LOOP = asyncio.get_event_loop()
+    LOOP.add_signal_handler(signal.SIGTERM, LOOP.stop)
+    LOOP.run_until_complete(main())
+    LOOP.close()