diff --git a/.github/workflows/kuksa_dbc_feeder.yml b/.github/workflows/kuksa_dbc_feeder.yml index c7c08847..fb7cccb9 100644 --- a/.github/workflows/kuksa_dbc_feeder.yml +++ b/.github/workflows/kuksa_dbc_feeder.yml @@ -79,7 +79,7 @@ jobs: - name: Run dbc tests run: | cd dbc2val - pip3 install --no-cache-dir -r requirements-dev.txt + pip3 install --no-cache-dir -r requirements.txt -r requirements-dev.txt python -m pytest - name: Run pylint (but accept errors for now) diff --git a/dbc2val/Readme.md b/dbc2val/Readme.md index a0bfaa21..bfd00ff8 100644 --- a/dbc2val/Readme.md +++ b/dbc2val/Readme.md @@ -49,6 +49,12 @@ $ python -V $ pip install -r requirements.txt ``` +4. If you want to run tests and linters, you will also need to install development dependencies + +```console +$ pip install -r requirements-dev.txt +``` + ## Steps for a local test with socket can or virtual socket can 1. Use the argument --use-socketcan or you can remove the line with the dumpfile in `config/dbc_feeder.ini` diff --git a/dbc2val/dbcfeeder.py b/dbc2val/dbcfeeder.py index 4b1f68b5..4ec26e3b 100755 --- a/dbc2val/dbcfeeder.py +++ b/dbc2val/dbcfeeder.py @@ -1,7 +1,7 @@ #!/usr/bin/env python ######################################################################## -# Copyright (c) 2020 Robert Bosch GmbH +# Copyright (c) 2020,2023 Robert Bosch GmbH # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -122,6 +122,7 @@ def start( mappingfile, candumpfile=None, use_j1939=False, + use_strict_parsing=False, grpc_metadata=None, ): log.info("Using mapping: {}".format(mappingfile)) @@ -133,11 +134,15 @@ def start( rxqueue=self._can_queue, dbcfile=dbcfile, mapper=self._mapper, + use_strict_parsing=use_strict_parsing, ) else: log.info("Using DBC reader") self._reader = dbcreader.DBCReader( - rxqueue=self._can_queue, dbcfile=dbcfile, mapper=self._mapper + rxqueue=self._can_queue, + dbcfile=dbcfile, + mapper=self._mapper, + use_strict_parsing=use_strict_parsing, ) if candumpfile: @@ -146,10 +151,12 @@ def start( "Using virtual bus to replay CAN messages (channel: %s)", canport, ) - self._player = canplayer.CANplayer(dumpfile=candumpfile) self._reader.start_listening( - bustype="virtual", channel=canport, bitrate=500000 + bustype="virtual", + channel=canport, + bitrate=500000 ) + self._player = canplayer.CANplayer(dumpfile=candumpfile) self._player.start_replaying(canport=canport) else: @@ -192,7 +199,7 @@ def is_stopping(self): return self._shutdown def on_broker_connectivity_change(self, connectivity): - log.info("Connectivity changed to: %s", connectivity) + log.info("Connectivity to data broker changed to: %s", connectivity) if ( connectivity == grpc.ChannelConnectivity.READY or connectivity == grpc.ChannelConnectivity.IDLE @@ -364,7 +371,18 @@ def main(argv): choices=[server_type.value for server_type in ServerType], type=ServerType, ) - + parser.add_argument( + "--lax-dbc-parsing", + dest="strict", + help=""" + Disable strict parsing of DBC files. This is helpful if the DBC file contains + message length definitions that do not match the signals' bit-offsets and lengths. + Processing DBC frames based on such DBC message definitions might still work, so + providing this switch might allow using the (erroneous) DBC file without having to + fix it first. + """, + action="store_false", + ) args = parser.parse_args() config = parse_config(args.config) @@ -494,6 +512,7 @@ def signal_handler(signal_received, frame): mappingfile=mappingfile, candumpfile=candumpfile, use_j1939=use_j1939, + use_strict_parsing=args.strict, grpc_metadata=grpc_metadata, ) diff --git a/dbc2val/dbcfeederlib/canplayer.py b/dbc2val/dbcfeederlib/canplayer.py index ca60758e..8bf82b18 100644 --- a/dbc2val/dbcfeederlib/canplayer.py +++ b/dbc2val/dbcfeederlib/canplayer.py @@ -1,5 +1,5 @@ ######################################################################## -# Copyright (c) 2022 Contributors to the Eclipse Foundation +# Copyright (c) 2022,2023 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -24,6 +24,21 @@ class CANplayer: + """ + Replay logged CAN messages from a file. + + The format is determined from the file suffix which can be one of: + * .asc + * .blf + * .csv + * .db + * .log + * .trc + + Gzip compressed files can be used as long as the original + files suffix is one of the above (e.g. filename.asc.gz). + """ + def __init__(self, dumpfile): self.run = True self.index = 0 @@ -33,7 +48,7 @@ def __init__(self, dumpfile): # open the file for reading can messages log.info("Replaying candump from {}".format(dumpfile)) - log_reader = can.CanutilsLogReader(dumpfile) + log_reader = can.LogReader(dumpfile) # get all messages out of the dumpfile and store into array of can messages for msg in log_reader: # store the sum of messages @@ -65,7 +80,6 @@ def txWorker(self): msg = can.Message( arbitration_id=next_message.arbitration_id, data=next_message.data, - is_extended_id=False, timestamp=next_message.timestamp, ) if msg: diff --git a/dbc2val/dbcfeederlib/dbcreader.py b/dbc2val/dbcfeederlib/dbcreader.py index e3612368..5dad41fc 100644 --- a/dbc2val/dbcfeederlib/dbcreader.py +++ b/dbc2val/dbcfeederlib/dbcreader.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 ######################################################################## -# Copyright (c) 2020 Robert Bosch GmbH +# Copyright (c) 2020,2023 Robert Bosch GmbH # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,16 +24,17 @@ import time import logging from dbcfeederlib import dbc2vssmapper +from queue import Queue log = logging.getLogger(__name__) class DBCReader: - def __init__(self, rxqueue, dbcfile, mapper): + def __init__(self, rxqueue: Queue, dbcfile: str, mapper: str, use_strict_parsing: bool): self.queue = rxqueue self.mapper = mapper log.info("Reading DBC file {}".format(dbcfile)) - self.db = cantools.database.load_file(dbcfile) + self.db = cantools.database.load_file(dbcfile, strict = use_strict_parsing) self.canidwl = self.get_whitelist() log.info("CAN ID whitelist={}".format(self.canidwl)) self.parseErr = 0 @@ -86,6 +87,7 @@ def rxWorker(self): log.info("Starting Rx thread") while self.run: msg = self.bus.recv(timeout=1) + log.debug("processing message from CAN bus") if msg and msg.arbitration_id in self.canidwl: try: decode = self.db.decode_message(msg.arbitration_id, msg.data) diff --git a/dbc2val/dbcfeederlib/j1939reader.py b/dbc2val/dbcfeederlib/j1939reader.py index e3ea6799..14cfe139 100644 --- a/dbc2val/dbcfeederlib/j1939reader.py +++ b/dbc2val/dbcfeederlib/j1939reader.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 ######################################################################## -# Copyright (c) 2020 Robert Bosch GmbH +# Copyright (c) 2020,2023 Robert Bosch GmbH # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,121 +19,31 @@ ######################################################################## # This script is to read CAN messages based on PGN - SAE J1939 -# Prior to using this script, j1939 and +# Prior to using this script, can-j1939 and # the relevant wheel-package should be installed first: -# $ pip3 install j1939 -# $ git clone https://github.com/benkfra/j1939.git -# $ cd j1939 -# $ pip install +# $ pip3 install can-j1939 import logging import time import cantools import j1939 +from dbcfeederlib import dbc2vssmapper +from queue import Queue log = logging.getLogger(__name__) -class J1939Reader(j1939.ControllerApplication): - # CA to produce messages - # This CA produces simulated sensor values and cyclically sends them to - # the bus with the PGN 0xFEF6 (Intake Exhaust Conditions 1) +class J1939Reader: - def __init__(self, rxqueue, dbcfile, mapper): - # compose the name descriptor for the new ca - name = j1939.Name( - arbitrary_address_capable=0, - industry_group=j1939.Name.IndustryGroup.Industrial, - vehicle_system_instance=1, - vehicle_system=1, - function=1, - function_instance=1, - ecu_instance=1, - manufacturer_code=666, - identity_number=1234567, - ) - device_address_preferred = 128 - # old fashion calling convention for compatibility with Python2 - j1939.ControllerApplication.__init__(self, name, device_address_preferred) - # adaptation + def __init__(self, rxqueue: Queue, dbcfile: str, mapper: str, use_strict_parsing: bool): self.queue = rxqueue - self.db = cantools.database.load_file(dbcfile) + self.db = cantools.database.load_file(dbcfile, strict = use_strict_parsing) self.mapper = mapper - self.canidwl = self.get_whitelist() - self.parseErr = 0 - self.run = True - - def start(self): - # Starts the CA - # (OVERLOADED function) - - # add our timer event - self._ecu.add_timer(0.500, self.timer_callback) - # call the super class function - j1939.ControllerApplication.start(self) + self.ecu = j1939.ElectronicControlUnit(); def stop(self): - j1939.ControllerApplication.stop(self) - - def timer_callback(self, cookie): - # Callback for sending the IEC1 message - # This callback is registered at the ECU timer event mechanism to be - # executed every 500ms. - # :param cookie: - # A cookie registered at 'add_timer'. May be None. - - # wait until we have our device_address - if self.state != j1939.ControllerApplication.State.NORMAL: - # returning true keeps the timer event active - return True - - pgn = j1939.ParameterGroupNumber(0, 0xFE, 0xF6) - data = [ - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_8, # Particulate Trap Inlet Pressure (SPN 81) - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_8, # Boost Pressure (SPN 102) - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_8, # Intake Manifold 1 Temperature (SPN 105) - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_8, # Air Inlet Pressure (SPN 106) - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_8, # Air Filter 1 Differential Pressure (SPN 107) - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_16_ARR[ - 0 - ], # Exhaust Gas Temperature (SPN 173) - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_16_ARR[1], - j1939.ControllerApplication.FieldValue.NOT_AVAILABLE_8, # Coolant Filter Differential Pressure (SPN 112) - ] - - # SPN 105, Range -40..+210 - # (Offset -40) - receiverTemperature = 30 - data[2] = receiverTemperature + 40 - - self.send_message(6, pgn.value, data) - - # returning true keeps the timer event active - return True - - def get_whitelist(self): - log.info("Collecting signals, generating CAN ID whitelist") - wl = [] - for entry in self.mapper.map(): - canid = self.get_canid_for_signal(entry[0]) - if canid is not None and canid not in wl: - wl.append(canid) - return wl - - def get_canid_for_signal(self, sig_to_find): - for msg in self.db.messages: - for signal in msg.signals: - if signal.name == sig_to_find: - id = msg.frame_id - log.info( - "Found signal {} in CAN frame id 0x{:02x}".format( - signal.name, id - ) - ) - return id - log.warning("Signal {} not found in DBC file".format(sig_to_find)) - return None + self.ecu.disconnect() def start_listening(self, *args, **kwargs): """Start listening to CAN bus @@ -150,22 +60,33 @@ def start_listening(self, *args, **kwargs): :param int bitrate: Bitrate in bit/s. """ - # create the ElectronicControlUnit (one ECU can hold multiple ControllerApplications) - ecu = j1939.ElectronicControlUnit() # Connect to the CAN bus - ecu.connect(*args, **kwargs) - - # add CA to the ECU - ecu.add_ca(controller_application=self) - self.start() + self.ecu.connect(*args, **kwargs) + self.ecu.subscribe(self.on_message) - def on_message(self, pgn, data): + def on_message(self, priority: int, pgn: int, sa: int, timestamp: int, data): message = self.identify_message(pgn) if message is not None: - signals = message._signals - for signal in signals: - self.put_signal_in_queue(signal, data) + log.debug("processing j1939 message [PGN: %s]", pgn) + try: + decode = message.decode(bytes(data), allow_truncated=True) + # log.debug("Decoded message: %s", str(decode)) + rxTime = time.time() + for k, v in decode.items(): + if k in self.mapper: + # Now time is defined per VSS signal, so handling needs to be different + for signal in self.mapper[k]: + if signal.time_condition_fulfilled(rxTime): + log.debug(f"Queueing {signal.vss_name}, triggered by {k}, raw value {v} ") + self.queue.put(dbc2vssmapper.VSSObservation(k, signal.vss_name, v, rxTime)) + else: + log.debug(f"Ignoring {signal.vss_name}, triggered by {k}, raw value {v} ") + except Exception: + log.warning( + "Error decoding message [PGN: {}]".format(message.name), + exc_info=True, + ) def identify_message(self, pgn): pgn_hex = hex(pgn)[2:] # only hex(pgn) without '0x' prefix @@ -175,83 +96,5 @@ def identify_message(self, pgn): ] # only hex(pgn) without '0x' prefix, priority and source address if pgn_hex == message_hex: return message + log.debug("no DBC mapping registered for j1939 message [PGN: %s]", pgn_hex) return None - - def put_signal_in_queue(self, signal, data): - name = signal._name - byte_order = signal._byte_order # 'little_endian' or 'big_endian' - scale = signal._scale - offset = signal._offset - data_type = type(data).__name__ - val = 0 - # When data_type is "list", `decode_signal` should be used. (Byte Level) - if data_type != "bytearray": - start_byte = int(signal._start / 8) # start from 0 - num_of_bytes = signal._length / 8 # most likely 1 or 2 - val = self.decode_signal( - start_byte, num_of_bytes, byte_order, scale, offset, data - ) - # When data_type is "bytearray", `decode_byte_array` should be used. (Bit Level) - else: - start_bit = signal._start - num_of_bits = signal._length - val = self.decode_byte_array( - start_bit, num_of_bits, byte_order, scale, offset, data - ) - if val < signal._minimum: - val = signal._minimum - elif val > signal._maximum: - val = signal._maximum - if name in self.mapper: - rxTime = time.time() - if self.mapper.minUpdateTimeElapsed(name, rxTime): - self.queue.put((name, val)) - - def decode_signal(self, start_byte, num_of_bytes, byte_order, scale, offset, data): - val = 0 - if num_of_bytes == 1: - raw_value = data[start_byte] - val = offset + raw_value * scale - else: - val = self.decode_2bytes(start_byte, byte_order, scale, offset, data) - return val - - def decode_2bytes(self, start_byte, byte_order, scale, offset, data): - start_data = data[start_byte] - end_data = data[start_byte + 1] - start_data_hex = hex(start_data)[2:] # without '0x' prefix - end_data_hex = hex(end_data)[2:] # without '0x' prefix - lit_end_hex_str = "" - # Little Endian - Intel, AMD - if byte_order == "little_endian": - lit_end_hex_str = "0x" + end_data_hex + start_data_hex - # Big Endian (a.k.a Endianness) - Motorola, IBM - else: - lit_end_hex_str = "0x" + start_data_hex + end_data_hex - raw_value = int(lit_end_hex_str, base=16) - val = offset + raw_value * scale - return val - - def decode_byte_array( - self, start_bit, num_of_bits, byte_order, scale, offset, data - ): - binary_str = "" - # temp_bit_array = [] - binstr = "" - # Little Endian - Intel, AMD - if byte_order == "little_endian": - for i in range(len(data)): - dec = data[i] - binstr = binstr + format(dec, "#010b")[2:][::-1] - # Big Endian (a.k.a Endianness) - Motorola, IBM - else: - for i in range(len(data)): - dec = data[i] - binstr = binstr + format(dec, "#010b")[2:] - # bit_array = list(reversed(temp_bit_array)) To call the smallest bit first - for i in range(0, num_of_bits): - binary_str = binstr[start_bit + i] + binary_str - binary_str = "0b" + binary_str - raw_value = int(binary_str, base=2) - val = offset + raw_value * scale - return val diff --git a/dbc2val/requirements-dev.txt b/dbc2val/requirements-dev.txt index 14f1acf5..5df9d719 100644 --- a/dbc2val/requirements-dev.txt +++ b/dbc2val/requirements-dev.txt @@ -1,12 +1,9 @@ -pyserial -cantools -python-can -pyyaml -j1939 -py_expression_eval -grpcio -types-PyYAML -types-protobuf +# Additional dependencies needed for build and test only +# +# For build and test install both dependencies in this file as well as dependencies in requirements.txt +# +# Example: +# pip3 install --no-cache-dir -r requirements.txt -r requirements-dev.txt +# pytest pylint -kuksa-client \ No newline at end of file diff --git a/dbc2val/requirements.in b/dbc2val/requirements.in index c2c5810d..1d19e2ee 100644 --- a/dbc2val/requirements.in +++ b/dbc2val/requirements.in @@ -15,6 +15,7 @@ # pip-compile requirements.in # +python-can ~= 4.1 pyserial ~= 3.5 cantools ~= 38.0 pyyaml ~= 6.0