Merge pull request #15 from rustyrussell/guilt/channel_type

Fixes and channel_type tests!  (Needs pyln.proto upgrade to work though)

HACKING.md

@@ -86,7 +86,7 @@ Here's a short outline of the current expected methods for a Runner.
 - `get_keyset`: returns the node's KeySet (`revocation_base_secret`, `payment_base_secret`, `htlc_base_secret`, and `shachain_seed`)
 - `get_node_privkey`: Private key of the node. Used to generate the node id and establish a communication channel with the node under test.
 - `get_node_bitcoinkey`: Private key of the node under test's funding pubkey
-- `has_option`: Returns `None` if node does not support provided option, otherwise 'even' or 'odd' (required or supported). Example input: `option_data_loss_protect` or `option_anchor_outputs`
+- `has_option`: checks for features (e.g. `option_anchor_outputs`) in which cast it returns `None`, or "even" or "odd" (required or supported).  Also checks for non-feature-bit features, such as `supports_open_accept_channel_types` which returns `None` or "true".
 - `add_startup_flag`: Add flag to runner's startup.
 - `start`: Starts up / initializes the node under test.
 - `stop`: Stops the node under test and closes the connection.

lnprototest/__init__.py

@@ -15,14 +15,14 @@
 from .structure import Sequence, OneOf, AnyOrder, TryAll
 from .runner import Runner, Conn, remote_revocation_basepoint, remote_payment_basepoint, remote_delayed_payment_basepoint, remote_htlc_basepoint, remote_per_commitment_point, remote_per_commitment_secret, remote_funding_pubkey, remote_funding_privkey
 from .dummyrunner import DummyRunner
-from .namespace import peer_message_namespace, namespace, assign_namespace
+from .namespace import peer_message_namespace, namespace, assign_namespace, make_namespace
 from .bitfield import bitfield, has_bit, bitfield_len
 from .signature import SigType, Sig
 from .keyset import KeySet
 from .commit_tx import Commit, HTLC, UpdateCommit
 from .utils import Side, regtest_hash, privkey_expand
 from .funding import AcceptFunding, CreateFunding, CreateDualFunding, Funding, AddInput, AddOutput, FinalizeFunding, AddWitnesses
-from .proposals import dual_fund_csv
+from .proposals import dual_fund_csv, channel_type_csv
 
 __all__ = [
     "EventError",
@@ -65,6 +65,7 @@
     "peer_message_namespace",
     "namespace",
     "assign_namespace",
+    "make_namespace",
     "bitfield",
     "has_bit",
     "bitfield_len",
@@ -89,5 +90,5 @@
     "privkey_expand",
     "Wait",
     "dual_fund_csv",
-    "Wait",
+    "channel_type_csv",
 ]

lnprototest/backend/backend.py

@@ -5,6 +5,7 @@
 # https://creativecommons.org/publicdomain/zero/1.0/
 from abc import ABC, abstractmethod
 
+
 class Backend(ABC):
     """
     Generic implementation of Bitcoin backend

lnprototest/backend/bitcoind.py

@@ -10,7 +10,7 @@
 import logging
 
 from ephemeral_port_reserve import reserve
-from pyln.testing.utils import wait_for, SimpleBitcoinProxy
+from pyln.testing.utils import SimpleBitcoinProxy
 from .backend import Backend
 
 
@@ -41,7 +41,7 @@ def __init__(self, basedir: str):
             f.write("rpcport={}\n".format(self.port))
         self.rpc = SimpleBitcoinProxy(btc_conf_file=self.bitcoin_conf)
 
-    def version_compatibility(self):
+    def version_compatibility(self) -> None:
         """
         This method try to manage the compatibility between
         different version of Bitcoin Core implementation.
@@ -51,18 +51,19 @@ def version_compatibility(self):
         """
         if self.rpc is None:
             # Sanity check
-            raise Error("bitcoind not initialized")
+            raise ValueError("bitcoind not initialized")
 
         self.btc_version = self.rpc.getnetworkinfo()['version']
+        assert self.btc_version is not None
         logging.info("Bitcoin Core version {}".format(self.btc_version))
         if self.btc_version >= 210000:
             # Maintains the compatibility between wallet
             # different ln implementation can use the main wallet (?)
             self.rpc.createwallet("main")  # Automatically loads
 
-
     def start(self) -> None:
         self.proc = subprocess.Popen(self.cmd_line, stdout=subprocess.PIPE)
+        assert self.proc.stdout
 
         # Wait for it to startup.
         while b'Done loading' not in self.proc.stdout.readline():
@@ -84,4 +85,3 @@ def restart(self) -> None:
             self.stop()
             shutil.rmtree(os.path.join(self.bitcoin_dir, 'regtest'))
             self.start()
-

lnprototest/bitfield.py

@@ -30,7 +30,7 @@ def has_bit(bitfield: Union[List[int], str], bitnum: int) -> bool:
 
 def bitfield(*args: int) -> str:
     """Create a bitfield hex value with these bit numbers set"""
-    bytelen = max(args) + 8 // 8
+    bytelen = (max(args) + 8) // 8
     bfield = bytearray(bytelen)
     for bitnum in args:
         bfield[bytelen - 1 - bitnum // 8] |= (1 << (bitnum % 8))

lnprototest/clightning/clightning.py

@@ -62,8 +62,11 @@ def __init__(self, config: Any):
                               stdout=subprocess.PIPE, check=True).stdout.decode('utf-8').splitlines()
         self.options: Dict[str, str] = {}
         for o in opts:
-            k, v = o.split('/')
-            self.options[k] = v
+            if o.startswith("supports_"):
+                self.options[o] = "true"
+            else:
+                k, v = o.split('/')
+                self.options[k] = v
 
     def get_keyset(self) -> KeySet:
         return KeySet(revocation_base_secret='0000000000000000000000000000000000000000000000000000000000000011',
@@ -244,7 +247,7 @@ def init_rbf(self, event: Event, conn: Conn,
                                          min_witness_weight=110,
                                          locktime=0, excess_as_change=True)['psbt']
 
-        def _run_rbf(runner: Runner, conn: Conn):
+        def _run_rbf(runner: Runner, conn: Conn) -> Dict[str, Any]:
             bump = runner.rpc.openchannel_bump(channel_id, amount, initial_psbt)
             update = runner.rpc.openchannel_update(channel_id, bump['psbt'])
 

lnprototest/event.py

@@ -272,7 +272,7 @@ def message_match(self, runner: 'Runner', msg: Message) -> Optional[str]:
 
         ret = cmp_msg(msg, partmessage)
         if ret is None:
-            self.if_match(self, msg, runner=runner)
+            self.if_match(self, msg, runner)
             msg_to_stash(runner, self, msg)
         return ret
 
@@ -432,7 +432,7 @@ def action(self, runner: 'Runner') -> bool:
 
 
 class DualFundAccept(Event):
-    def __init__(self):
+    def __init__(self) -> None:
         super().__init__()
 
     def action(self, runner: 'Runner') -> bool:

lnprototest/funding.py

@@ -1,5 +1,5 @@
 # Support for funding txs.
-from typing import Tuple, Any, Optional, Union, Callable
+from typing import Tuple, Any, Optional, Union, Callable, Dict, List
 from .utils import Side, privkey_expand, regtest_hash
 from .event import Event, ResolvableInt, ResolvableStr
 from .namespace import namespace
@@ -42,8 +42,8 @@ def __init__(self,
                               privkey_expand(remote_node_privkey)]
         self.tx = None
         self.locktime = locktime
-        self.outputs = []
-        self.inputs = []
+        self.outputs: List[Dict[str, Any]] = []
+        self.inputs: List[Dict[str, Any]] = []
 
     def tx_hex(self) -> str:
         if not self.tx:
@@ -168,6 +168,7 @@ def our_witnesses(self) -> str:
         return val
 
     def sign_our_inputs(self) -> None:
+        assert self.tx is not None
         for idx, _in in enumerate(self.inputs):
             privkey = _in['privkey']
 
@@ -195,7 +196,8 @@ def sign_our_inputs(self) -> None:
                 else:
                     _in['sig'] = CTxInWitness(CScriptWitness([sig, inkey_pub.format()]))
 
-    def add_witnesses(self, witness_stack) -> str:
+    def add_witnesses(self, witness_stack: List[Dict[str, Any]]) -> str:
+        assert self.tx is not None
         wits = []
         for idx, _in in enumerate(self.inputs):
             if 'sig' in _in:
@@ -223,6 +225,7 @@ def build_tx(self) -> str:
         self.tx = CMutableTransaction([i['input'] for i in self.inputs],
                                       [o['output'] for o in self.outputs],
                                       nVersion=2, nLockTime=self.locktime)
+        assert self.tx is not None
         self.txid = self.tx.GetTxid().hex()
 
         # Set the output index for the funding output
@@ -712,7 +715,9 @@ def action(self, runner: Runner) -> bool:
 class AddWitnesses(Event):
     def __init__(self,
                  funding: ResolvableFunding,
-                 witness_stack: Any):  # FIXME what's the type here?
+                 witness_stack: Union[List[Dict[str, Any]],
+                                      Callable[['Runner', 'Event', str],
+                                               List[Dict[str, Any]]]]):
         self.funding = funding
         self.witness_stack = witness_stack
 

lnprototest/namespace.py

@@ -4,21 +4,26 @@
 import pyln.spec.bolt7
 from pyln.proto.message import MessageNamespace
 from .signature import SigType
+from typing import List
 
 
-def peer_message_namespace() -> MessageNamespace:
-    """Namespace containing all the peer messages"""
+def make_namespace(csv: List[str]) -> MessageNamespace:
+    """Load a namespace, replacing signature type"""
     ns = MessageNamespace()
     # We replace the fundamental signature type with our custom type,
     # then we load in all the csv files so they use it.
     ns.fundamentaltypes['signature'] = SigType()
-
-    ns.load_csv(pyln.spec.bolt1.csv
-                + pyln.spec.bolt2.csv
-                + pyln.spec.bolt7.csv)
+    ns.load_csv(csv)
     return ns
 
 
+def peer_message_namespace() -> MessageNamespace:
+    """Namespace containing all the peer messages"""
+    return make_namespace(pyln.spec.bolt1.csv
+                          + pyln.spec.bolt2.csv
+                          + pyln.spec.bolt7.csv)
+
+
 def namespace() -> MessageNamespace:
     return event_namespace
 

lnprototest/proposals.py

@@ -84,3 +84,14 @@
     "msgdata,ack_rbf,channel_id,channel_id,",
     "msgdata,ack_rbf,funding_satoshis,u64,",
 ]
+
+# This is https://github.com/lightningnetwork/lightning-rfc/pull/880
+channel_type_csv = [
+    "subtype,channel_type",
+    "subtypedata,channel_type,len,u16,",
+    "subtypedata,channel_type,features,byte,len",
+    "tlvtype,open_channel_tlvs,channel_types,1",
+    "tlvdata,open_channel_tlvs,channel_types,types,channel_type,...",
+    "tlvtype,accept_channel_tlvs,channel_type,1",
+    "tlvdata,accept_channel_tlvs,channel_type,type,channel_type,",
+]

tests/conftest.py

@@ -2,6 +2,9 @@
 import pytest
 import importlib
 import lnprototest
+import pyln.spec.bolt1
+import pyln.spec.bolt2
+import pyln.spec.bolt7
 from pyln.proto.message import MessageNamespace
 from typing import Any, Callable, Generator, List
 
@@ -33,7 +36,12 @@ def with_proposal(pytestconfig: Any) -> Generator[Callable[[List[str]], None], N
     """Use this to add additional messages to the namespace
        Useful for testing proposed (but not yet merged) spec mods"""
     def _setter(proposal_csv: List[str]) -> None:
-        lnprototest.assign_namespace(lnprototest.namespace() + MessageNamespace(proposal_csv))
+        # We merge *csv*, because then you can add tlv entries; merging
+        # namespaces with duplicate TLVs complains of a clash.
+        lnprototest.assign_namespace(lnprototest.make_namespace(pyln.spec.bolt1.csv
+                                                                + pyln.spec.bolt2.csv
+                                                                + pyln.spec.bolt7.csv
+                                                                + proposal_csv))
 
     yield _setter
 

tests/test_bolt2-20-open_channel_accepter.py

@@ -2,10 +2,11 @@
 # Variations on open_channel
 
 from hashlib import sha256
+from pyln.proto.message import Message
 from lnprototest import TryAll, Connect, Block, FundChannel, InitRbf, ExpectMsg, Msg, RawMsg, KeySet, CreateFunding, Commit, Runner, remote_funding_pubkey, remote_revocation_basepoint, remote_payment_basepoint, remote_htlc_basepoint, remote_per_commitment_point, remote_delayed_payment_basepoint, Side, msat, remote_funding_privkey, regtest_hash, bitfield, Event, DualFundAccept, OneOf, CreateDualFunding, EventError, Funding, privkey_expand, AddInput, AddOutput, FinalizeFunding, AddWitnesses, dual_fund_csv, ExpectError, Disconnect
 from lnprototest.stash import sent, rcvd, commitsig_to_send, commitsig_to_recv, funding_txid, funding_tx, funding, locking_script, get_member, witnesses
 from helpers import utxo, tx_spendable, funding_amount_for_utxo, pubkey_of, tx_out_for_index, privkey_for_index, utxo_amount
-from typing import Any, Callable
+from typing import Any, Callable, List
 
 
 def channel_id_v2(local_keyset: KeySet) -> Callable[[Runner, Event, str], str]:
@@ -44,7 +45,7 @@ def _channel_id_tmp(runner: Runner, event: Event, field: str) -> str:
     return _channel_id_tmp
 
 
-def odd_serial(event: Event, msg: Msg, runner: 'Runner') -> None:
+def odd_serial(event: Event, msg: Message, runner: 'Runner') -> None:
     """
         Test that a message's serial_id is odd.
         Note that the dummy runner will fail this test, so we skip for them
@@ -55,7 +56,7 @@ def odd_serial(event: Event, msg: Msg, runner: 'Runner') -> None:
         raise EventError(event, "Received **even** serial {}, expected odd".format(msg.fields['serial_id']))
 
 
-def even_serial(event: Event, msg: Msg, runner: 'Runner') -> None:
+def even_serial(event: Event, msg: Message, runner: 'Runner') -> None:
     if msg.fields['serial_id'] % 2 == 1:
         raise EventError(event, "Received **odd** serial {}, expected event".format(msg.fields['serial_id']))
 
@@ -74,7 +75,7 @@ def _agreed_funding(runner: Runner, event: Event, field: str) -> int:
                                     'ExpectMsg' if opener == Side.local else 'Msg',
                                     accept_msg + '.funding_satoshis')
 
-        return open_funding + accept_funding
+        return int(open_funding) + int(accept_funding)
     return _agreed_funding
 
 
@@ -112,8 +113,8 @@ def _change_amount(runner: Runner, event: Event, field: str) -> int:
             # p2wsh script is 34 bytes, all told
             weight += (8 + 34 + 1) * 4
 
-        fee = (weight * feerate) // 1000
-        change = input_amt - opening_amt - fee
+        fee = (weight * int(feerate)) // 1000
+        change = input_amt - int(opening_amt) - fee
 
         return change
 
@@ -1276,7 +1277,7 @@ def test_df_opener_accepter_underpays_fees(runner: Runner, with_proposal: Any) -
 def accepter_tx_creation(input_index: int, is_rbf: bool, funding_amt: int,
                          local_funding_privkey: str,
                          local_keyset: KeySet,
-                         runner: Runner):
+                         runner: Runner) -> List[Event]:
     """ Repeated tx construction protocols, for accepter tests """
     txid_in, tx_index_in, sats_in, spending_privkey, fee = utxo(input_index)
     fee = sats_in - funding_amt if is_rbf else fee
@@ -1413,7 +1414,7 @@ def accepter_tx_creation(input_index: int, is_rbf: bool, funding_amt: int,
 def opener_tx_creation(input_index: int, is_rbf: bool, funding_amt: int,
                        local_funding_privkey: str,
                        local_keyset: KeySet,
-                       runner: Runner):
+                       runner: Runner) -> List[Event]:
     """ Repeated tx construction protocols, for opener tests """
     txid_in, tx_index_in, sats_in, spending_privkey, fee = utxo(input_index)
     fee = sats_in - funding_amt if is_rbf else fee