New pyln modules.

.gitignore

@@ -11,6 +11,7 @@
 *.po
 *.pyc
 .cppcheck-suppress
+.mypy_cache
 TAGS
 tags
 ccan/tools/configurator/configurator

contrib/pyln-proto/Makefile

@@ -0,0 +1,13 @@
+#! /usr/bin/make
+
+check:
+	pytest
+
+check-source: check-flake8 check-mypy
+
+check-flake8:
+	flake8 --ignore=E501,E731,W503
+
+# mypy . does not recurse.  I have no idea why...
+check-mypy:
+	mypy --ignore-missing-imports `find * -name '*.py'`

contrib/pyln-proto/pyln/proto/message/Makefile

@@ -0,0 +1,4 @@
+#! /usr/bin/make
+
+refresh:
+	for d in bolt*; do $(MAKE) -C $$d; done

contrib/pyln-proto/pyln/proto/message/__init__.py

@@ -0,0 +1,33 @@
+from .array_types import SizedArrayType, DynamicArrayType, EllipsisArrayType
+from .message import MessageNamespace, MessageType, Message, SubtypeType
+from .fundamental_types import split_field, FieldType
+
+__version__ = '0.0.1'
+
+__all__ = [
+    "MessageNamespace",
+    "MessageType",
+    "Message",
+    "SubtypeType",
+    "FieldType",
+    "split_field",
+    "SizedArrayType",
+    "DynamicArrayType",
+    "EllipsisArrayType",
+
+    # fundamental_types
+    'byte',
+    'u16',
+    'u32',
+    'u64',
+    'tu16',
+    'tu32',
+    'tu64',
+    'chain_hash',
+    'channel_id',
+    'sha256',
+    'point',
+    'short_channel_id',
+    'signature',
+    'bigsize',
+]

contrib/pyln-proto/pyln/proto/message/array_types.py

@@ -0,0 +1,184 @@
+from .fundamental_types import FieldType, IntegerType, split_field
+from typing import List, Optional, Dict, Tuple, TYPE_CHECKING, Any
+from io import BufferedIOBase
+if TYPE_CHECKING:
+    from .message import SubtypeType, TlvStreamType
+
+
+class ArrayType(FieldType):
+    """Abstract class for the different kinds of arrays.
+
+These are not in the namespace, but generated when a message says it
+wants an array of some type.
+
+    """
+    def __init__(self, outer: 'SubtypeType', name: str, elemtype: FieldType):
+        super().__init__("{}.{}".format(outer.name, name))
+        self.elemtype = elemtype
+
+    def val_from_str(self, s: str) -> Tuple[List[Any], str]:
+        # Simple arrays of bytes don't need commas
+        if self.elemtype.name == 'byte':
+            a, b = split_field(s)
+            return [b for b in bytes.fromhex(a)], b
+
+        if not s.startswith('['):
+            raise ValueError("array of {} must be wrapped in '[]': bad {}"
+                             .format(self.elemtype.name, s))
+        s = s[1:]
+        ret = []
+        while not s.startswith(']'):
+            val, s = self.elemtype.val_from_str(s)
+            ret.append(val)
+            if s[0] == ',':
+                s = s[1:]
+        return ret, s[1:]
+
+    def val_to_str(self, v: List[Any], otherfields: Dict[str, Any]) -> str:
+        if self.elemtype.name == 'byte':
+            return bytes(v).hex()
+
+        s = ','.join(self.elemtype.val_to_str(i, otherfields) for i in v)
+        return '[' + s + ']'
+
+    def write(self, io_out: BufferedIOBase, v: List[Any], otherfields: Dict[str, Any]) -> None:
+        for i in v:
+            self.elemtype.write(io_out, i, otherfields)
+
+    def read_arr(self, io_in: BufferedIOBase, otherfields: Dict[str, Any], arraysize: Optional[int]) -> List[Any]:
+        """arraysize None means take rest of io entirely and exactly"""
+        vals: List[Any] = []
+        while arraysize is None or len(vals) < arraysize:
+            # Throws an exception on partial read, so None means completely empty.
+            val = self.elemtype.read(io_in, otherfields)
+            if val is None:
+                if arraysize is not None:
+                    raise ValueError('{}: not enough remaining to read'
+                                     .format(self))
+                break
+
+            vals.append(val)
+
+        return vals
+
+
+class SizedArrayType(ArrayType):
+    """A fixed-size array"""
+    def __init__(self, outer: 'SubtypeType', name: str, elemtype: FieldType, arraysize: int):
+        super().__init__(outer, name, elemtype)
+        self.arraysize = arraysize
+
+    def val_to_str(self, v: List[Any], otherfields: Dict[str, Any]) -> str:
+        if len(v) != self.arraysize:
+            raise ValueError("Length of {} != {}", v, self.arraysize)
+        return super().val_to_str(v, otherfields)
+
+    def val_from_str(self, s: str) -> Tuple[List[Any], str]:
+        a, b = super().val_from_str(s)
+        if len(a) != self.arraysize:
+            raise ValueError("Length of {} != {}", s, self.arraysize)
+        return a, b
+
+    def write(self, io_out: BufferedIOBase, v: List[Any], otherfields: Dict[str, Any]) -> None:
+        if len(v) != self.arraysize:
+            raise ValueError("Length of {} != {}", v, self.arraysize)
+        return super().write(io_out, v, otherfields)
+
+    def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> List[Any]:
+        return super().read_arr(io_in, otherfields, self.arraysize)
+
+
+class EllipsisArrayType(ArrayType):
+    """This is used for ... fields at the end of a tlv: the array ends
+when the tlv ends"""
+    def __init__(self, tlv: 'TlvStreamType', name: str, elemtype: FieldType):
+        super().__init__(tlv, name, elemtype)
+
+    def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> List[Any]:
+        """Takes rest of bytestream"""
+        return super().read_arr(io_in, otherfields, None)
+
+    def only_at_tlv_end(self) -> bool:
+        """These only make sense at the end of a TLV"""
+        return True
+
+
+class LengthFieldType(FieldType):
+    """Special type to indicate this serves as a length field for others"""
+    def __init__(self, inttype: IntegerType):
+        if type(inttype) is not IntegerType:
+            raise ValueError("{} cannot be a length; not an integer!"
+                             .format(self.name))
+        super().__init__(inttype.name)
+        self.underlying_type = inttype
+        # You can be length for more than one field!
+        self.len_for: List[DynamicArrayType] = []
+
+    def is_optional(self) -> bool:
+        """This field value is always implies, never specified directly"""
+        return True
+
+    def add_length_for(self, field: 'DynamicArrayType') -> None:
+        assert isinstance(field.fieldtype, DynamicArrayType)
+        self.len_for.append(field)
+
+    def calc_value(self, otherfields: Dict[str, Any]) -> int:
+        """Calculate length value from field(s) themselves"""
+        if self.len_fields_bad('', otherfields):
+            raise ValueError("Lengths of fields {} not equal!"
+                             .format(self.len_for))
+
+        return len(otherfields[self.len_for[0].name])
+
+    def _maybe_calc_value(self, fieldname: str, otherfields: Dict[str, Any]) -> int:
+        # Perhaps we're just demarshalling from binary now, so we actually
+        # stored it.  Remove, and we'll calc from now on.
+        if fieldname in otherfields:
+            v = otherfields[fieldname]
+            del otherfields[fieldname]
+            return v
+        return self.calc_value(otherfields)
+
+    def val_to_str(self, _, otherfields: Dict[str, Any]) -> str:
+        return self.underlying_type.val_to_str(self.calc_value(otherfields),
+                                               otherfields)
+
+    def name_and_val(self, name: str, v: int) -> str:
+        """We don't print out length fields when printing out messages:
+they're implied by the length of other fields"""
+        return ''
+
+    def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> None:
+        """We store this, but it'll be removed from the fields as soon as it's used (i.e. by DynamicArrayType's val_from_bin)"""
+        return self.underlying_type.read(io_in, otherfields)
+
+    def write(self, io_out: BufferedIOBase, _, otherfields: Dict[str, Any]) -> None:
+        self.underlying_type.write(io_out, self.calc_value(otherfields),
+                                   otherfields)
+
+    def val_from_str(self, s: str):
+        raise ValueError('{} is implied, cannot be specified'.format(self))
+
+    def len_fields_bad(self, fieldname: str, otherfields: Dict[str, Any]) -> List[str]:
+        """fieldname is the name to return if this length is bad"""
+        mylen = None
+        for lens in self.len_for:
+            if mylen is not None:
+                if mylen != len(otherfields[lens.name]):
+                    return [fieldname]
+            # Field might be missing!
+            if lens.name in otherfields:
+                mylen = len(otherfields[lens.name])
+        return []
+
+
+class DynamicArrayType(ArrayType):
+    """This is used for arrays where another field controls the size"""
+    def __init__(self, outer: 'SubtypeType', name: str, elemtype: FieldType, lenfield: LengthFieldType):
+        super().__init__(outer, name, elemtype)
+        assert type(lenfield.fieldtype) is LengthFieldType
+        self.lenfield = lenfield
+
+    def read(self, io_in: BufferedIOBase, otherfields: Dict[str, Any]) -> List[Any]:
+        return super().read_arr(io_in, otherfields,
+                                self.lenfield.fieldtype._maybe_calc_value(self.lenfield.name, otherfields))

contrib/pyln-proto/pyln/proto/message/bolt1/Makefile

@@ -0,0 +1,7 @@
+#! /usr/bin/make
+
+SPECDIR := ../../../../../../../lightning-rfc
+
+csv.py: $(SPECDIR)/01-messaging.md Makefile
+	SPECNUM=`basename $< | sed 's/-.*//'`; (echo csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $< | sed 's/\(.*\)/    "\1",/'; echo ']') > $@
+	chmod a+x $@

contrib/pyln-proto/pyln/proto/message/bolt1/__init__.py

@@ -0,0 +1,16 @@
+from .csv import csv
+from .bolt import namespace
+import sys
+
+__version__ = '0.0.1'
+
+__all__ = [
+    'csv',
+    'namespace',
+]
+
+mod = sys.modules[__name__]
+for d in namespace.subtypes, namespace.tlvtypes, namespace.messagetypes:
+    for name in d:
+        setattr(mod, name, d[name])
+        __all__.append(name)

contrib/pyln-proto/pyln/proto/message/bolt1/bolt.py

@@ -0,0 +1,5 @@
+from pyln.proto.message import MessageNamespace
+from .csv import csv
+
+
+namespace = MessageNamespace(csv_lines=csv)

contrib/pyln-proto/pyln/proto/message/bolt1/csv.py

@@ -0,0 +1,35 @@
+csv = [
+    "msgtype,init,16",
+    "msgdata,init,gflen,u16,",
+    "msgdata,init,globalfeatures,byte,gflen",
+    "msgdata,init,flen,u16,",
+    "msgdata,init,features,byte,flen",
+    "msgdata,init,tlvs,init_tlvs,",
+    "tlvtype,init_tlvs,networks,1",
+    "tlvdata,init_tlvs,networks,chains,chain_hash,...",
+    "msgtype,error,17",
+    "msgdata,error,channel_id,channel_id,",
+    "msgdata,error,len,u16,",
+    "msgdata,error,data,byte,len",
+    "msgtype,ping,18",
+    "msgdata,ping,num_pong_bytes,u16,",
+    "msgdata,ping,byteslen,u16,",
+    "msgdata,ping,ignored,byte,byteslen",
+    "msgtype,pong,19",
+    "msgdata,pong,byteslen,u16,",
+    "msgdata,pong,ignored,byte,byteslen",
+    "tlvtype,n1,tlv1,1",
+    "tlvdata,n1,tlv1,amount_msat,tu64,",
+    "tlvtype,n1,tlv2,2",
+    "tlvdata,n1,tlv2,scid,short_channel_id,",
+    "tlvtype,n1,tlv3,3",
+    "tlvdata,n1,tlv3,node_id,point,",
+    "tlvdata,n1,tlv3,amount_msat_1,u64,",
+    "tlvdata,n1,tlv3,amount_msat_2,u64,",
+    "tlvtype,n1,tlv4,254",
+    "tlvdata,n1,tlv4,cltv_delta,u16,",
+    "tlvtype,n2,tlv1,0",
+    "tlvdata,n2,tlv1,amount_msat,tu64,",
+    "tlvtype,n2,tlv2,11",
+    "tlvdata,n2,tlv2,cltv_expiry,tu32,",
+]

contrib/pyln-proto/pyln/proto/message/bolt2/Makefile

@@ -0,0 +1,7 @@
+#! /usr/bin/make
+
+SPECDIR := ../../../../../../../lightning-rfc
+
+csv.py: $(SPECDIR)/02-peer-protocol.md Makefile
+	SPECNUM=`basename $< | sed 's/-.*//'`; (echo csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $< | sed 's/\(.*\)/    "\1",/'; echo ']') > $@
+	chmod a+x $@

contrib/pyln-proto/pyln/proto/message/bolt2/__init__.py

@@ -0,0 +1,16 @@
+from .csv import csv
+from .bolt import namespace
+import sys
+
+__version__ = '0.0.1'
+
+__all__ = [
+    'csv',
+    'namespace',
+]
+
+mod = sys.modules[__name__]
+for d in namespace.subtypes, namespace.tlvtypes, namespace.messagetypes:
+    for name in d:
+        setattr(mod, name, d[name])
+        __all__.append(name)

contrib/pyln-proto/pyln/proto/message/bolt2/bolt.py

@@ -0,0 +1,5 @@
+from pyln.proto.message import MessageNamespace
+from .csv import csv
+
+
+namespace = MessageNamespace(csv_lines=csv)