Allow disabling of strict parsing of DBC files

The dbcfeeder now supports a new command line switch '--lax-dbc-parsing'
which can be used to disable strict parsing of DBC files.

This is helpful if the DBC file contains message length definitions
that do not match the signals' bit-offsets and lengths. Processing
DBC frames based on such DBC message definitions might still work,
so providing this switch might allow using the (erroneous) DBC
file without having to fix it first.

Signed-off-by: Kai Hudalla <kai.hudalla@bosch.io>

.github/workflows/kuksa_dbc_feeder.yml

@@ -79,7 +79,7 @@ jobs:
       - name: Run dbc tests
         run: |
           cd dbc2val
-          pip3 install --no-cache-dir -r requirements-dev.txt
+          pip3 install --no-cache-dir -r requirements.txt -r requirements-dev.txt
           python -m pytest
 
       - name: Run pylint (but accept errors for now)

dbc2val/Readme.md

@@ -49,6 +49,12 @@ $ python -V
 $ pip install -r requirements.txt
 ```
 
+4. If you want to run tests and linters, you will also need to install development dependencies
+
+```console
+$ pip install -r requirements-dev.txt
+```
+
 ## Steps for a local test with socket can or virtual socket can
 
 1. Use the argument --use-socketcan or you can remove the line with the dumpfile in `config/dbc_feeder.ini`

dbc2val/dbcfeeder.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 ########################################################################
-# Copyright (c) 2020 Robert Bosch GmbH
+# Copyright (c) 2020,2023 Robert Bosch GmbH
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -122,6 +122,7 @@ def start(
         mappingfile,
         candumpfile=None,
         use_j1939=False,
+        use_strict_parsing=False,
         grpc_metadata=None,
     ):
         log.info("Using mapping: {}".format(mappingfile))
@@ -133,11 +134,15 @@ def start(
                 rxqueue=self._can_queue,
                 dbcfile=dbcfile,
                 mapper=self._mapper,
+                use_strict_parsing=use_strict_parsing,
             )
         else:
             log.info("Using DBC reader")
             self._reader = dbcreader.DBCReader(
-                rxqueue=self._can_queue, dbcfile=dbcfile, mapper=self._mapper
+                rxqueue=self._can_queue,
+                dbcfile=dbcfile,
+                mapper=self._mapper,
+                use_strict_parsing=use_strict_parsing,
             )
 
         if candumpfile:
@@ -146,10 +151,12 @@ def start(
                 "Using virtual bus to replay CAN messages (channel: %s)",
                 canport,
             )
-            self._player = canplayer.CANplayer(dumpfile=candumpfile)
             self._reader.start_listening(
-                bustype="virtual", channel=canport, bitrate=500000
+                bustype="virtual",
+                channel=canport,
+                bitrate=500000
             )
+            self._player = canplayer.CANplayer(dumpfile=candumpfile)
             self._player.start_replaying(canport=canport)
         else:
 
@@ -192,7 +199,7 @@ def is_stopping(self):
         return self._shutdown
 
     def on_broker_connectivity_change(self, connectivity):
-        log.info("Connectivity changed to: %s", connectivity)
+        log.info("Connectivity to data broker changed to: %s", connectivity)
         if (
             connectivity == grpc.ChannelConnectivity.READY or
             connectivity == grpc.ChannelConnectivity.IDLE
@@ -364,7 +371,18 @@ def main(argv):
         choices=[server_type.value for server_type in ServerType],
         type=ServerType,
     )
-
+    parser.add_argument(
+        "--lax-dbc-parsing",
+        dest="strict",
+        help="""
+          Disable strict parsing of DBC files. This is helpful if the DBC file contains
+          message length definitions that do not match the signals' bit-offsets and lengths.
+          Processing DBC frames based on such DBC message definitions might still work, so
+          providing this switch might allow using the (erroneous) DBC file without having to
+          fix it first.
+          """,
+        action="store_false",
+    )
     args = parser.parse_args()
 
     config = parse_config(args.config)
@@ -494,6 +512,7 @@ def signal_handler(signal_received, frame):
         mappingfile=mappingfile,
         candumpfile=candumpfile,
         use_j1939=use_j1939,
+        use_strict_parsing=args.strict,
         grpc_metadata=grpc_metadata,
     )
 

dbc2val/dbcfeederlib/canplayer.py

@@ -1,5 +1,5 @@
 ########################################################################
-# Copyright (c) 2022 Contributors to the Eclipse Foundation
+# Copyright (c) 2022,2023 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -24,6 +24,21 @@
 
 
 class CANplayer:
+    """
+    Replay logged CAN messages from a file.
+
+    The format is determined from the file suffix which can be one of:
+      * .asc
+      * .blf
+      * .csv
+      * .db
+      * .log
+      * .trc
+
+    Gzip compressed files can be used as long as the original
+    files suffix is one of the above (e.g. filename.asc.gz).
+    """
+
     def __init__(self, dumpfile):
         self.run = True
         self.index = 0
@@ -33,7 +48,7 @@ def __init__(self, dumpfile):
 
         # open the file for reading can messages
         log.info("Replaying candump from {}".format(dumpfile))
-        log_reader = can.CanutilsLogReader(dumpfile)
+        log_reader = can.LogReader(dumpfile)
         # get all messages out of the dumpfile and store into array of can messages
         for msg in log_reader:
             # store the sum of messages
@@ -65,7 +80,6 @@ def txWorker(self):
             msg = can.Message(
                 arbitration_id=next_message.arbitration_id,
                 data=next_message.data,
-                is_extended_id=False,
                 timestamp=next_message.timestamp,
             )
             if msg:

dbc2val/dbcfeederlib/dbcreader.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 ########################################################################
-# Copyright (c) 2020 Robert Bosch GmbH
+# Copyright (c) 2020,2023 Robert Bosch GmbH
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,16 +24,17 @@
 import time
 import logging
 from dbcfeederlib import dbc2vssmapper
+from queue import Queue
 
 log = logging.getLogger(__name__)
 
 
 class DBCReader:
-    def __init__(self, rxqueue, dbcfile, mapper):
+    def __init__(self, rxqueue: Queue, dbcfile: str, mapper: str, use_strict_parsing: bool):
         self.queue = rxqueue
         self.mapper = mapper
         log.info("Reading DBC file {}".format(dbcfile))
-        self.db = cantools.database.load_file(dbcfile)
+        self.db = cantools.database.load_file(dbcfile, strict = use_strict_parsing)
         self.canidwl = self.get_whitelist()
         log.info("CAN ID whitelist={}".format(self.canidwl))
         self.parseErr = 0
@@ -86,6 +87,7 @@ def rxWorker(self):
         log.info("Starting Rx thread")
         while self.run:
             msg = self.bus.recv(timeout=1)
+            log.debug("processing message from CAN bus")
             if msg and msg.arbitration_id in self.canidwl:
                 try:
                     decode = self.db.decode_message(msg.arbitration_id, msg.data)