Always use UTF-8 as per OpenLCB (Fix bobjacobsen#55). Change each int…

… list to bytearray (Resolves bobjacobsen#58).

doc/conf.py

@@ -30,7 +30,6 @@
 exclude_patterns = []
 
 
-
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 

examples/example_cdi_access.py

@@ -96,18 +96,19 @@ def printDatagram(memo):
 
 
 # accumulate the CDI information
-resultingCDI = []
+resultingCDI = bytearray()
 
 # callbacks to get results of memory read
 
+
 def memoryReadSuccess(memo):
     """Handle a successful read
     Invoked when the memory read successfully returns,
     this queues a new read until the entire CDI has been
     returned.  At that point, it invokes the XML processing below.
 
     Args:
-        memo (_type_): _description_
+        memo (MemoryReadMemo): _description_
     """
     # print("successful memory read: {}".format(memo.data))
 
@@ -121,15 +122,19 @@ def memoryReadSuccess(memo):
         memo.address = memo.address+64
         # and read again
         memoryService.requestMemoryRead(memo)
+        # The last packet is not yet reached, so don't parse (However,
+        #   parser.feed could be called for realtime processing).
     else :
         # and we're done!
         # save content
         resultingCDI += memo.data
         # concert resultingCDI to a string up to 1st zero
         cdiString = ""
-        for x in resultingCDI:
-            if x == 0 : break
-            cdiString += chr(x)
+        null_i = resultingCDI.find(b'\0')
+        terminate_i = len(resultingCDI)
+        if null_i > -1:
+            terminate_i = min(null_i, terminate_i)
+        cdiString = resultingCDI[:terminate_i].decode("utf-8")
         # print (cdiString)
 
         # and process that
@@ -158,25 +163,41 @@ def memoryReadFail(memo):
 class MyHandler(xml.sax.handler.ContentHandler):
     """XML SAX callbacks in a handler object"""
     def __init__(self):
-        self._charBuffer = []
+        self._charBuffer = bytearray()
 
     def startElement(self, name, attrs):
+        """_summary_
+
+        Args:
+            name (_type_): _description_
+            attrs (_type_): _description_
+        """
         print("Start: ", name)
         if attrs is not None and attrs :
             print("  Attributes: ", attrs.getNames())
 
     def endElement(self, name):
+        """_summary_
+
+        Args:
+            name (_type_): _description_
+        """
         print(name, "content:", self._flushCharBuffer())
         print("End: ", name)
         pass
 
     def _flushCharBuffer(self):
-        s = ''.join(self._charBuffer)
-        self._charBuffer = []
+        """Decode the buffer, clear it, and return all content.
+
+        Returns:
+            str: The content of the bytes buffer decoded as utf-8.
+        """
+        s = self._charBuffer.decode("utf-8")
+        self._charBuffer.clear()
         return s
 
     def characters(self, data):
-        self._charBuffer.append(data)
+        self._charBuffer.extend(data)
 
 
 handler = MyHandler()
@@ -188,6 +209,10 @@ def processXML(content) :
     Args:
         content (_type_): _description_
     """
+    # NOTE: The data is complete in this example since processXML is
+    #   only called when there is a null terminator, which indicates the
+    #   last packet was reached for the requested read.
+    #   - See memoryReadSuccess comments for details.
     xml.sax.parseString(content, handler)
     print("\nParser done")
 

examples/example_frame_interface.py

@@ -52,7 +52,7 @@ def printFrame(frame):
 canPhysicalLayerGridConnect.registerFrameReceivedListener(printFrame)
 
 # send an AME frame with arbitrary alias to provoke response
-frame = CanFrame(ControlFrame.AME.value, 1, [])
+frame = CanFrame(ControlFrame.AME.value, 1, bytearray())
 print("SL: {}".format(frame))
 canPhysicalLayerGridConnect.sendCanFrame(frame)
 

openlcb/__init__.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 import re
 
 
@@ -15,5 +16,27 @@ def only_hex_pairs(value):
 
 
 def emit_cast(value):
-    """Show type and value, such as for debug output."""
-    return "{}({})".format(type(value).__name__, repr(value))
+    """Get type and value, such as for debug output."""
+    repr_str = repr(value)
+    if repr_str.startswith(type(value).__name__):
+        return repr(value)  # type already included, such as bytearray(...)
+    return "{}({})".format(type(value).__name__, repr_str)
+
+
+def list_type_names(values):
+    """Get the type of several values, such as for debug output.
+    Args:
+        values (Union[list,tuple,dict,OrderedDict]): A collection where
+            each element's type is to be analyzed.
+    Returns:
+        list[str]: A list where each element is a type name. If
+            values argument is dict-like, each element is formatted as
+            "{key}: {type}".
+    """
+    if isinstance(values, (list, tuple)):
+        return [type(value).__name__ for value in values]
+    if isinstance(values, (dict, OrderedDict)):
+        return ["{}: {}".format(k, type(v).__name__) for k, v in values.items()]
+    raise TypeError("list_type_names is only implemented for"
+                    " list, tuple, dict, and OrderedDict, but got a(n) {}"
+                    .format(type(values).__name__))

openlcb/canbus/canframe.py

@@ -1,3 +1,5 @@
+import openlcb
+from collections import OrderedDict
 from openlcb.nodeid import NodeID
 
 
@@ -36,41 +38,98 @@ class CanFrame:
             mask = 0x07FF_F000).
     """
 
-    header = 0
-    data = []
+    ARG_LISTS = [
+        OrderedDict(N_cid=int, nodeID=NodeID, alias=int),
+        OrderedDict(header=int, data=bytearray),
+        OrderedDict(control=int, alias=int, data=bytearray),
+    ]
 
-    def __str__(self):
-        return "CanFrame header: 0x{:08X} {}".format(self.header, self.data)
+    @staticmethod
+    def constructor_help():
+        result = ""
+        for d in CanFrame.ARG_LISTS:
+            result += "("
+            for k, v in d.items():
+                result += "{}: {}, ".format(k, v.__name__)
+            result = result[:-2]  # remove last ", " from this ctor
+            result += "), "
+        return result[:-2]  # -1 to remove last ", " from list
 
-    # there are three ctor forms
+    def __str__(self):
+        return "CanFrame header: 0x{:08X} {}".format(
+            self.header,
+            list(self.data),  # cast to list to format bytearray(b'') as []
+        )
 
-    def __init__(self, arg1, arg2, arg3=[]):
+    def __init__(self, *args):
+        arg1 = None
+        arg2 = None
+        arg3 = None
+        if len(args) > 0:
+            arg1 = args[0]
+        if len(args) > 1:
+            arg2 = args[1]
+        if len(args) > 2:
+            arg3 = args[2]
+        else:
+            arg3 = bytearray()
+        # There are three ctor forms.
+        # - See "Args" in class for docstring.
+        self.header = 0
+        self.data = bytearray()
         # three arguments as N_cid, nodeID, alias
+        args_error = None
         if isinstance(arg2, NodeID):
+            # Other args' types will be enforced by doing math on them
+            #   (duck typing) in this case.
+            if len(args) < 3:
+                args_error = "Expected alias after NodeID"
             # cid must be 4 to 7 inclusive (100 to 111 binary)
             # precondition(4 <= cid && cid <= 7)
             cid = arg1
             nodeID = arg2
             alias = arg3
 
             nodeCode = ((nodeID.nodeId >> ((cid-4)*12)) & 0xFFF)
-            # ^ cid-4 results in 0 to 3. *12 results in 0 to 36 bit shift (nodeID size)
+            # ^ cid-4 results in 0 to 3. *12 results in 0 to 36 bit shift (nodeID size)  # noqa: E501
             self.header = ((cid << 12) | nodeCode) << 12 | (alias & 0xFFF) | 0x10_00_00_00  # noqa: E501
-            self.data = []
+            # self.data = bytearray()
 
         # two arguments as header, data
-        elif isinstance(arg2, list):
+        elif isinstance(arg2, bytearray):
+            if not isinstance(arg1, int):
+                args_error = "Expected int since 2nd argument is bytearray."
+            # Types of both args are enforced by this point.
             self.header = arg1
             self.data = arg2
+            if len(args) > 2:
+                args_error = "2nd argument is data, but got extra argument(s)"
+
+        # two arguments as header, data
+        elif isinstance(arg2, list):
+            args_error = ("Expected bytearray (formerly list[int])"
+                          " if data 2nd argument")
+            # self.header = arg1
+            # self.data = bytearray(arg2)
 
         # three arguments as control, alias, data
         elif isinstance(arg2, int):
+            # Types of all 3 are enforced by usage (duck typing) in this case.
             control = arg1
             alias = arg2
             self.header = (control << 12) | (alias & 0xFFF) | 0x10_00_00_00
+            if not isinstance(arg3, bytearray):
+                args_error = ("Expected bytearray (formerly list[int])"
+                              " 2nd if 1st argument is header int")
             self.data = arg3
         else:
-            print("could not decode NodeID ctor arguments")
+            args_error = "could not decode CanFrame arguments"
+
+        if args_error:
+            raise TypeError(
+                args_error.rstrip(".") + ". Valid constructors:"
+                + CanFrame.constructor_help() + ". Got: "
+                + openlcb.list_type_names(args))
 
     def __eq__(self, other):
         if other is None:

openlcb/canbus/canlink.py

@@ -110,7 +110,8 @@ def handleReceivedLinkRestarted(self, frame):  # CanFrame
         Args:
             frame (_type_): _description_
         """
-        msg = Message(MTI.Link_Layer_Restarted, NodeID(0), None, [])
+        msg = Message(MTI.Link_Layer_Restarted, NodeID(0), None,
+                      bytearray())
         self.fireListeners(msg)
 
     def defineAndReserveAlias(self):
@@ -154,9 +155,9 @@ def handleReceivedLinkDown(self, frame):  # CanFrame
     # invoked when the link layer comes up and down
     def linkStateChange(self, state):  # state is of the State enum
         if state == CanLink.State.Permitted:
-            msg = Message(MTI.Link_Layer_Up, NodeID(0), None, [])
+            msg = Message(MTI.Link_Layer_Up, NodeID(0), None, bytearray())
         else:
-            msg = Message(MTI.Link_Layer_Down, NodeID(0), None, [])
+            msg = Message(MTI.Link_Layer_Down, NodeID(0), None, bytearray())
         self.fireListeners(msg)
 
     def handleReceivedCID(self, frame):  # CanFrame
@@ -274,7 +275,7 @@ def handleReceivedData(self, frame):  # CanFrame
                 key = CanLink.AccumKey(mti, sourceID, destID)
                 if dgCode == 0x00A_000_000 or dgCode == 0x00B_000_000:
                     #    start of message, create the entry in the accumulator
-                    self.accumulator[key] = []
+                    self.accumulator[key] = bytearray()
                 else:
                     # not start frame
                     # check for never properly started, this is an error
@@ -327,7 +328,7 @@ def handleReceivedData(self, frame):  # CanFrame
                 key = CanLink.AccumKey(mti, sourceID, destID)
                 if (frame.data[0] & 0x20 == 0):
                     #    is start, create the entry in the accumulator
-                    self.accumulator[key] = []
+                    self.accumulator[key] = bytearray()
                 else:
                     # not start frame
                     # check for first bit set never seen
@@ -465,27 +466,27 @@ def segmentDatagramDataArray(self, data):
             data (_type_): _description_
 
         Returns:
-            _type_: _description_
+            list[bytearray]:  A list of one or more data segments.
         """
         nSegments = (len(data)+7) // 8
         # ^ the +7 is since integer division takes the floor value
         if nSegments == 0:
-            return [[]]
+            return [bytearray()]
 
         if nSegments == 1:
             return [data]
 
         #    multiple frames
-        retval = []
+        segments = []
         for i in range(0, nSegments-2+1):  # first entry of 2 has full data
-            nextEntry = (data[i*8:i*8+7+1]).copy()
-            retval.append(nextEntry)
+            nextEntry = data[i*8:i*8+7+1]
+            segments.append(nextEntry)
 
         #    add the last
-        lastEntry = (data[8*(nSegments-1):]).copy()
-        retval.append(lastEntry)
+        lastEntry = data[8*(nSegments-1):]
+        segments.append(lastEntry)
 
-        return retval
+        return segments
 
     def segmentAddressedDataArray(self, alias, data):
         '''Segment data into zero or more arrays
@@ -497,30 +498,30 @@ def segmentAddressedDataArray(self, alias, data):
             data (_type_): _description_
 
         Returns:
-            _type_: _description_
+            list[bytearray]: A list of one or more data segments.
         '''
         part0 = (alias >> 8) & 0xF
         part1 = alias & 0xFF
         nSegments = (len(data)+5) // 6  # the +5 is since integer division
         #   takes the floor value
         if nSegments == 0:
-            return [[part0, part1]]
+            return [bytearray([part0, part1])]
         if nSegments == 1:
-            return [[part0, part1]+data]
+            return [bytearray([part0, part1])+data]
 
         #    multiple frames
-        retval = []
+        segments = []
         for i in range(0, nSegments-2+1):  # first entry of 2 has full data
-            nextEntry = [part0 | 0x30, part1]+(data[i*6:i*6+5+1]).copy()
-            retval.append(nextEntry)
+            nextEntry = bytearray([part0 | 0x30, part1]) + data[i*6:i*6+5+1]
+            segments.append(nextEntry)
 
         #    add the last
-        lastEntry = [part0 | 0x20, part1]+(data[6*(nSegments-1):]).copy()
-        retval.append(lastEntry)
+        lastEntry = bytearray([part0 | 0x20, part1]) + data[6*(nSegments-1):]
+        segments.append(lastEntry)
         #    mark first (last already done above)
-        retval[0][0] &= ~0x20
+        segments[0][0] &= ~0x20
 
-        return retval
+        return segments
 
     #    MARK: common code
     def checkAndHandleAliasCollision(self, frame):