JSON schemas in doc/ (only some commands so far)

Makefile

@@ -258,6 +258,11 @@ endif
 
 default: show-flags all-programs all-test-programs doc-all
 
+ifneq ($(SUPPRESS_GENERATION),1)
+FORCE = FORCE
+FORCE::
+endif
+
 show-flags: config.vars
 	@$(ECHO) "CC: $(CC) $(CFLAGS) -c -o"
 	@$(ECHO) "LD: $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o"
@@ -288,30 +293,30 @@ else
 # Git doesn't maintain timestamps, so we only regen if sources actually changed:
 # We place the SHA inside some generated files so we can tell if they need updating.
 # Usage: $(call SHA256STAMP_CHANGED)
-SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP://p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ]
-# Usage: $(call SHA256STAMP,commentprefix)
-SHA256STAMP = echo '$(1) SHA256STAMP:'`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64` >> $@
+SHA256STAMP_CHANGED = [ x"`sed -n 's/.*SHA256STAMP:\([a-f0-9]*\).*/\1/p' $@ 2>/dev/null`" != x"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`" ]
+# Usage: $(call SHA256STAMP,commentprefix,commentpostfix)
+SHA256STAMP = echo "$(1) SHA256STAMP:"`cat $(sort $(filter-out FORCE,$^)) | $(SHA256SUM) | cut -c1-64`"$(2)" >> $@
 endif
 
 # generate-wire.py --page [header|impl] hdrfilename wirename < csv > file
 %_wiregen.h: %_wire.csv $(WIRE_GEN_DEPS)
 	@if $(call SHA256STAMP_CHANGED); then if [ "$$NO_PYTHON" = 1 ]; then echo "Error: NO_PYTHON on $@"; exit 1; fi; \
-		$(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); \
+		$(call VERBOSE,"wiregen $@",tools/generate-wire.py --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
 	fi
 
 %_wiregen.c: %_wire.csv $(WIRE_GEN_DEPS)
 	@if $(call SHA256STAMP_CHANGED); then if [ "$$NO_PYTHON" = 1 ]; then echo "Error: NO_PYTHON on $@"; exit 1; fi; \
-		$(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); \
+		$(call VERBOSE,"wiregen $@",tools/generate-wire.py --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
 	fi
 
 %_printgen.h: %_wire.csv $(WIRE_GEN_DEPS)
 	@if $(call SHA256STAMP_CHANGED); then if [ "$$NO_PYTHON" = 1 ]; then echo "Error: NO_PYTHON on $@"; exit 1; fi; \
-		$(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); \
+		$(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page header $($@_args) $@ `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
 	fi
 
 %_printgen.c: %_wire.csv $(WIRE_GEN_DEPS)
 	@if $(call SHA256STAMP_CHANGED); then  if [ "$$NO_PYTHON" = 1 ]; then echo "Error: NO_PYTHON on $@"; exit 1; fi; \
-		$(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//)); \
+		$(call VERBOSE,"printgen $@",tools/generate-wire.py -s -P --page impl $($@_args) ${@:.c=.h} `basename $< .csv | sed 's/_exp_/_/'` < $< > $@ && $(call SHA256STAMP,//,)); \
 	fi
 
 include external/Makefile
@@ -536,11 +541,6 @@ ncc: ${TARGET_DIR}/libwally-core-build/src/libwallycore.la
 TAGS:
 	$(RM) TAGS; find * -name test -type d -prune -o -name '*.[ch]' -print -o -name '*.py' -print | xargs etags --append
 
-ifneq ($(SUPPRESS_GENERATION),1)
-FORCE = FORCE
-FORCE::
-endif
-
 ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
 
 ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr

contrib/pyln-client/pyln/client/lightning.py

@@ -349,6 +349,7 @@ def call(self, method, payload=None):
                     "enable": True
                 },
             })
+            # FIXME: Notification schema support?
             _, buf = self._readobj(sock, buf)
 
         request = {

contrib/pyln-testing/pyln/testing/fixtures.py

@@ -1,13 +1,17 @@
 from concurrent import futures
 from pyln.testing.db import SqliteDbProvider, PostgresDbProvider
 from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, DEVELOPER, LightningNode, TEST_DEBUG, Throttler
+from pyln.client import Millisatoshi
 from typing import Dict
 
+import json
+import jsonschema  # type: ignore
 import logging
 import os
 import pytest  # type: ignore
 import re
 import shutil
+import string
 import sys
 import tempfile
 
@@ -202,8 +206,145 @@ def throttler(test_base_dir):
     yield Throttler(test_base_dir)
 
 
+def _extra_validator():
+    """JSON Schema validator with additions for our specialized types"""
+    def is_hex(checker, instance):
+        """Hex string"""
+        if not checker.is_type(instance, "string"):
+            return False
+        return all(c in string.hexdigits for c in instance)
+
+    def is_u64(checker, instance):
+        """64-bit integer"""
+        if not checker.is_type(instance, "integer"):
+            return False
+        return instance >= 0 and instance < 2**64
+
+    def is_u32(checker, instance):
+        """32-bit integer"""
+        if not checker.is_type(instance, "integer"):
+            return False
+        return instance >= 0 and instance < 2**32
+
+    def is_u16(checker, instance):
+        """16-bit integer"""
+        if not checker.is_type(instance, "integer"):
+            return False
+        return instance >= 0 and instance < 2**16
+
+    def is_short_channel_id(checker, instance):
+        """Short channel id"""
+        if not checker.is_type(instance, "string"):
+            return False
+        parts = instance.split("x")
+        if len(parts) != 3:
+            return False
+        # May not be integers
+        try:
+            blocknum = int(parts[0])
+            txnum = int(parts[1])
+            outnum = int(parts[2])
+        except ValueError:
+            return False
+
+        # BOLT #7:
+        # ## Definition of `short_channel_id`
+        #
+        # The `short_channel_id` is the unique description of the funding transaction.
+        # It is constructed as follows:
+        # 1. the most significant 3 bytes: indicating the block height
+        # 2. the next 3 bytes: indicating the transaction index within the block
+        # 3. the least significant 2 bytes: indicating the output index that pays to the
+        #    channel.
+        return (blocknum >= 0 and blocknum < 2**24
+                and txnum >= 0 and txnum < 2**24
+                and outnum >= 0 and outnum < 2**16)
+
+    def is_pubkey(checker, instance):
+        """SEC1 encoded compressed pubkey"""
+        if not checker.is_type(instance, "hex"):
+            return False
+        if len(instance) != 66:
+            return False
+        return instance[0:2] == "02" or instance[0:2] == "03"
+
+    def is_pubkey32(checker, instance):
+        """x-only BIP-340 public key"""
+        if not checker.is_type(instance, "hex"):
+            return False
+        if len(instance) != 64:
+            return False
+        return True
+
+    def is_signature(checker, instance):
+        """DER encoded secp256k1 ECDSA signature"""
+        if not checker.is_type(instance, "hex"):
+            return False
+        if len(instance) > 72 * 2:
+            return False
+        return True
+
+    def is_bip340sig(checker, instance):
+        """Hex encoded secp256k1 Schnorr signature"""
+        if not checker.is_type(instance, "hex"):
+            return False
+        if len(instance) != 64 * 2:
+            return False
+        return True
+
+    def is_msat(checker, instance):
+        """String number ending in msat"""
+        return type(instance) is Millisatoshi
+
+    def is_txid(checker, instance):
+        """Bitcoin transaction ID"""
+        if not checker.is_type(instance, "hex"):
+            return False
+        return len(instance) == 64
+
+    type_checker = jsonschema.Draft7Validator.TYPE_CHECKER.redefine_many({
+        "hex": is_hex,
+        "u64": is_u64,
+        "u32": is_u32,
+        "u16": is_u16,
+        "pubkey": is_pubkey,
+        "msat": is_msat,
+        "txid": is_txid,
+        "signature": is_signature,
+        "bip340sig": is_bip340sig,
+        "pubkey32": is_pubkey32,
+        "short_channel_id": is_short_channel_id,
+    })
+
+    return jsonschema.validators.extend(jsonschema.Draft7Validator,
+                                        type_checker=type_checker)
+
+
+def _load_schema(filename):
+    """Load the schema from @filename and create a validator for it"""
+    with open(filename, 'r') as f:
+        return _extra_validator()(json.load(f))
+
+
+@pytest.fixture(autouse=True)
+def jsonschemas():
+    """Load schema files if they exist"""
+    try:
+        schemafiles = os.listdir('doc/schemas')
+    except FileNotFoundError:
+        schemafiles = []
+
+    schemas = {}
+    for fname in schemafiles:
+        if not fname.endswith('.schema.json'):
+            continue
+        schemas[fname.rpartition('.schema')[0]] = _load_schema(os.path.join('doc/schemas',
+                                                                            fname))
+    return schemas
+
+
 @pytest.fixture
-def node_factory(request, directory, test_name, bitcoind, executor, db_provider, teardown_checks, node_cls, throttler):
+def node_factory(request, directory, test_name, bitcoind, executor, db_provider, teardown_checks, node_cls, throttler, jsonschemas):
     nf = NodeFactory(
         request,
         test_name,
@@ -213,6 +354,7 @@ def node_factory(request, directory, test_name, bitcoind, executor, db_provider,
         db_provider=db_provider,
         node_cls=node_cls,
         throttler=throttler,
+        jsonschemas=jsonschemas,
     )
 
     yield nf

contrib/pyln-testing/pyln/testing/utils.py

@@ -601,7 +601,17 @@ class PrettyPrintingLightningRpc(LightningRpc):
     eyes. It has some overhead since we re-serialize the request and
     result to json in order to pretty print it.
 
+    Also validates (optional) schemas for us.
     """
+    def __init__(self, socket_path, executor=None, logger=logging,
+                 patch_json=True, jsonschemas={}):
+        super().__init__(
+            socket_path,
+            executor,
+            logger,
+            patch_json,
+        )
+        self.jsonschemas = jsonschemas
 
     def call(self, method, payload=None):
         id = self.next_id
@@ -615,6 +625,10 @@ def call(self, method, payload=None):
             "id": id,
             "result": res
         }, indent=2))
+
+        if method in self.jsonschemas:
+            self.jsonschemas[method].validate(res)
+
         return res
 
 
@@ -625,6 +639,7 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
                  allow_warning=False,
                  allow_bad_gossip=False,
                  db=None, port=None, disconnect=None, random_hsm=None, options=None,
+                 jsonschemas={},
                  **kwargs):
         self.bitcoin = bitcoind
         self.executor = executor
@@ -639,7 +654,7 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai
         self.rc = 0
 
         socket_path = os.path.join(lightning_dir, TEST_NETWORK, "lightning-rpc").format(node_id)
-        self.rpc = PrettyPrintingLightningRpc(socket_path, self.executor)
+        self.rpc = PrettyPrintingLightningRpc(socket_path, self.executor, jsonschemas=jsonschemas)
 
         self.daemon = LightningD(
             lightning_dir, bitcoindproxy=bitcoind.get_proxy(),
@@ -1196,7 +1211,7 @@ class NodeFactory(object):
     """A factory to setup and start `lightningd` daemons.
     """
     def __init__(self, request, testname, bitcoind, executor, directory,
-                 db_provider, node_cls, throttler):
+                 db_provider, node_cls, throttler, jsonschemas):
         if request.node.get_closest_marker("slow_test") and SLOW_MACHINE:
             self.valgrind = False
         else:
@@ -1211,6 +1226,7 @@ def __init__(self, request, testname, bitcoind, executor, directory,
         self.db_provider = db_provider
         self.node_cls = node_cls
         self.throttler = throttler
+        self.jsonschemas = jsonschemas
 
     def split_options(self, opts):
         """Split node options from cli options
@@ -1289,6 +1305,7 @@ def get_node(self, node_id=None, options=None, dbfile=None,
         node = self.node_cls(
             node_id, lightning_dir, self.bitcoind, self.executor, self.valgrind, db=db,
             port=port, options=options, may_fail=may_fail or expect_fail,
+            jsonschemas=self.jsonschemas,
             **kwargs
         )
 

contrib/pyln-testing/requirements.txt

@@ -9,3 +9,4 @@ pytest-timeout ~= 1.4.2
 pytest-xdist ~= 2.2.0
 pytest==6.1.*
 python-bitcoinlib==0.11.*
+jsonschema==3.2.*

doc/Makefile

@@ -86,10 +86,23 @@ MANPAGES := doc/lightning-cli.1 \
 
 doc-all: $(MANPAGES) doc/index.rst
 
+# Some manpages use a schema, so need that added.
+MARKDOWN_WITH_SCHEMA := $(shell grep -l GENERATE-FROM-SCHEMA $(MANPAGES:=.md))
+
+# These are hard to use in $(call) functions.
+LBRACKET=(
+RBRACKET=)
+
+$(MARKDOWN_WITH_SCHEMA): doc/lightning-%.7.md: doc/schemas/%.schema.json tools/fromschema.py
+	@if $(call SHA256STAMP_CHANGED); then $(call VERBOSE, "fromschema $@", tools/fromschema.py --markdownfile=$@ doc/schemas/$*.schema.json > $@.tmp && grep -v SHA256STAMP: $@.tmp > $@ && rm -f $@.tmp && $(call SHA256STAMP,[comment]: # $(LBRACKET),$(RBRACKET))); else touch $@; fi
+
+# mrkd doesn't format nested lists properly, so we fixup with sed (see doc/lightning-connect.7
+# and https://github.com/refi64/mrkd/issues/4
 $(MANPAGES): doc/%: doc/%.md
-	@if $(call SHA256STAMP_CHANGED); then $(call VERBOSE, "mrkd $<", mrkd $< $@ && $(call SHA256STAMP,\")); else touch $@; fi
+	@if $(call SHA256STAMP_CHANGED); then $(call VERBOSE, "mrkd $<", mrkd $< $@.tmp && sed -e 's/\(.\)\.RS$$/\1\n.RS/' < $@.tmp > $@ && rm $@.tmp && $(call SHA256STAMP,\\\",)); else touch $@; fi
 
 $(MANPAGES): $(FORCE)
+$(MARKDOWN_WITH_SCHEMA): $(FORCE)
 
 doc/protocol-%.svg: test/test_protocol
 	test/test_protocol --svg < test/commits/$*.script > $@

doc/lightning-addgossip.7.md

@@ -22,7 +22,9 @@ messages within error replies.
 RETURN VALUE
 ------------
 
+[comment]: # (GENERATE-FROM-SCHEMA-START)
 On success, an empty object is returned.
+[comment]: # (GENERATE-FROM-SCHEMA-END)
 
 AUTHOR
 ------
@@ -39,3 +41,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
+[comment]: # ( SHA256STAMP:f974a3848c4db5b73fffa969a741ef6619c9a375783fabe731882d84a6bbf5ff)