Add support for option_payment_metadata (#2063)

* Filter init, node and invoice features

We should explicitly filter features based on where they can be included
(`init`, `node_announcement` or `invoice`) as specified in Bolt 9.

* Add support for sending `payment_metadata`

Whenever we find a payment metadata field in an invoice, we send it in
the onion payload for the final recipient.

* Include `payment_metadata` in all invoices

We include a payment metadata in every invoice we generate. This lets us
see whether our payers support it or not, which is important data to have
before we make it mandatory and use it for storage-less invoices.

See lightning/bolts#912 for reference.

docs/release-notes/eclair-vnext.md

@@ -24,6 +24,15 @@ Eclair now supports the feature `option_onion_messages`. If this feature is enab
 It can also send onion messages with the `sendonionmessage` API.
 Messages sent to Eclair can be read with the websocket API.
 
+### Support for `option_payment_metadata`
+
+Eclair now supports the `option_payment_metadata` feature (see https://github.com/lightning/bolts/pull/912).
+This feature will let recipients generate "light" invoices that don't need to be stored locally until they're paid.
+This is particularly useful for payment hubs that generate a lot of invoices (e.g. to be displayed on a website) but expect only a fraction of them to actually be paid.
+
+Eclair includes a small `payment_metadata` field in all invoices it generates.
+This lets node operators verify that payers actually support that feature.
+
 ### API changes
 
 #### Timestamps

eclair-core/src/main/resources/reference.conf

@@ -58,6 +58,7 @@ eclair {
     option_shutdown_anysegwit = optional
     option_onion_messages = disabled
     option_channel_type = optional
+    option_payment_metadata = optional
     trampoline_payment = disabled
     keysend = disabled
   }

eclair-core/src/main/scala/fr/acinq/eclair/Features.scala

@@ -34,6 +34,8 @@ object FeatureSupport {
 
 trait Feature {
 
+  this: FeatureScope =>
+
   def rfcName: String
   def mandatory: Int
   def optional: Int = mandatory + 1
@@ -46,6 +48,15 @@ trait Feature {
   override def toString = rfcName
 
 }
+
+/** Feature scope as defined in Bolt 9. */
+sealed trait FeatureScope
+/** Feature that should be advertised in init messages. */
+trait InitFeature extends FeatureScope
+/** Feature that should be advertised in node announcements. */
+trait NodeFeature extends FeatureScope
+/** Feature that should be advertised in invoices. */
+trait InvoiceFeature extends FeatureScope
 // @formatter:on
 
 case class UnknownFeature(bitIndex: Int)
@@ -71,6 +82,13 @@ case class Features(activated: Map[Feature, FeatureSupport], unknown: Set[Unknow
     unknownFeaturesOk && knownFeaturesOk
   }
 
+  def initFeatures(): Features = Features(activated.collect { case (f: InitFeature, s) => (f: Feature, s) }, unknown)
+
+  def nodeAnnouncementFeatures(): Features = Features(activated.collect { case (f: NodeFeature, s) => (f: Feature, s) }, unknown)
+
+  // NB: we don't include unknown features in invoices, which means plugins cannot inject invoice features.
+  def invoiceFeatures(): Map[Feature with InvoiceFeature, FeatureSupport] = activated.collect { case (f: InvoiceFeature, s) => (f, s) }
+
   def toByteVector: ByteVector = {
     val activatedFeatureBytes = toByteVectorFromIndex(activated.map { case (feature, support) => feature.supportBit(support) }.toSet)
     val unknownFeatureBytes = toByteVectorFromIndex(unknown.map(_.bitIndex))
@@ -137,91 +155,96 @@ object Features {
     }
   }
 
-  case object OptionDataLossProtect extends Feature {
+  case object OptionDataLossProtect extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_data_loss_protect"
     val mandatory = 0
   }
 
-  case object InitialRoutingSync extends Feature {
+  case object InitialRoutingSync extends Feature with InitFeature {
     val rfcName = "initial_routing_sync"
     // reserved but not used as per lightningnetwork/lightning-rfc/pull/178
     val mandatory = 2
   }
 
-  case object OptionUpfrontShutdownScript extends Feature {
+  case object OptionUpfrontShutdownScript extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_upfront_shutdown_script"
     val mandatory = 4
   }
 
-  case object ChannelRangeQueries extends Feature {
+  case object ChannelRangeQueries extends Feature with InitFeature with NodeFeature {
     val rfcName = "gossip_queries"
     val mandatory = 6
   }
 
-  case object VariableLengthOnion extends Feature {
+  case object VariableLengthOnion extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "var_onion_optin"
     val mandatory = 8
   }
 
-  case object ChannelRangeQueriesExtended extends Feature {
+  case object ChannelRangeQueriesExtended extends Feature with InitFeature with NodeFeature {
     val rfcName = "gossip_queries_ex"
     val mandatory = 10
   }
 
-  case object StaticRemoteKey extends Feature {
+  case object StaticRemoteKey extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_static_remotekey"
     val mandatory = 12
   }
 
-  case object PaymentSecret extends Feature {
+  case object PaymentSecret extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "payment_secret"
     val mandatory = 14
   }
 
-  case object BasicMultiPartPayment extends Feature {
+  case object BasicMultiPartPayment extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "basic_mpp"
     val mandatory = 16
   }
 
-  case object Wumbo extends Feature {
+  case object Wumbo extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_support_large_channel"
     val mandatory = 18
   }
 
-  case object AnchorOutputs extends Feature {
+  case object AnchorOutputs extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_anchor_outputs"
     val mandatory = 20
   }
 
-  case object AnchorOutputsZeroFeeHtlcTx extends Feature {
+  case object AnchorOutputsZeroFeeHtlcTx extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_anchors_zero_fee_htlc_tx"
     val mandatory = 22
   }
 
-  case object ShutdownAnySegwit extends Feature {
+  case object ShutdownAnySegwit extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_shutdown_anysegwit"
     val mandatory = 26
   }
 
-  case object OnionMessages extends Feature {
+  case object OnionMessages extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_onion_messages"
     val mandatory = 38
   }
 
-  case object ChannelType extends Feature {
+  case object ChannelType extends Feature with InitFeature with NodeFeature {
     val rfcName = "option_channel_type"
     val mandatory = 44
   }
 
+  case object PaymentMetadata extends Feature with InvoiceFeature {
+    val rfcName = "option_payment_metadata"
+    val mandatory = 48
+  }
+
   // TODO: @t-bast: update feature bits once spec-ed (currently reserved here: https://github.com/lightningnetwork/lightning-rfc/issues/605)
   // We're not advertising these bits yet in our announcements, clients have to assume support.
   // This is why we haven't added them yet to `areSupported`.
-  case object TrampolinePayment extends Feature {
+  case object TrampolinePayment extends Feature with InitFeature with NodeFeature with InvoiceFeature {
     val rfcName = "trampoline_payment"
     val mandatory = 50
   }
 
-  case object KeySend extends Feature {
+  case object KeySend extends Feature with NodeFeature {
     val rfcName = "keysend"
     val mandatory = 54
   }
@@ -242,6 +265,7 @@ object Features {
     ShutdownAnySegwit,
     OnionMessages,
     ChannelType,
+    PaymentMetadata,
     TrampolinePayment,
     KeySend
   )

eclair-core/src/main/scala/fr/acinq/eclair/NodeParams.scala

@@ -110,7 +110,8 @@ case class NodeParams(nodeKeyManager: NodeKeyManager,
 
   def currentBlockHeight: Long = blockCount.get
 
-  def featuresFor(nodeId: PublicKey): Features = overrideFeatures.getOrElse(nodeId, features)
+  /** Returns the features that should be used in our init message with the given peer. */
+  def initFeaturesFor(nodeId: PublicKey): Features = overrideFeatures.getOrElse(nodeId, features).initFeatures()
 }
 
 object NodeParams extends Logging {

eclair-core/src/main/scala/fr/acinq/eclair/io/Switchboard.scala

@@ -92,7 +92,7 @@ class Switchboard(nodeParams: NodeParams, peerFactory: Switchboard.PeerFactory)
     case authenticated: PeerConnection.Authenticated =>
       // if this is an incoming connection, we might not yet have created the peer
       val peer = createOrGetPeer(authenticated.remoteNodeId, offlineChannels = Set.empty)
-      val features = nodeParams.featuresFor(authenticated.remoteNodeId)
+      val features = nodeParams.initFeaturesFor(authenticated.remoteNodeId)
       // if the peer is whitelisted, we sync with them, otherwise we only sync with peers with whom we have at least one channel
       val doSync = nodeParams.syncWhitelist.contains(authenticated.remoteNodeId) || (nodeParams.syncWhitelist.isEmpty && peersWithChannels.contains(authenticated.remoteNodeId))
       authenticated.peerConnection ! PeerConnection.InitializeConnection(peer, nodeParams.chainHash, features, doSync)

eclair-core/src/main/scala/fr/acinq/eclair/json/JsonSerializers.scala

@@ -335,6 +335,7 @@ object PaymentRequestSerializer extends MinimalSerializer({
         FeatureSupportSerializer +
         UnknownFeatureSerializer
     ))
+    val paymentMetadata = p.paymentMetadata.map(m => JField("paymentMetadata", JString(m.toHex))).toSeq
     val routingInfo = JField("routingInfo", Extraction.decompose(p.routingInfo)(
       DefaultFormats +
         ByteVector32Serializer +
@@ -344,12 +345,14 @@ object PaymentRequestSerializer extends MinimalSerializer({
         MilliSatoshiSerializer +
         CltvExpiryDeltaSerializer
     ))
-    val fieldList = List(JField("prefix", JString(p.prefix)),
+    val fieldList = List(
+      JField("prefix", JString(p.prefix)),
       JField("timestamp", JLong(p.timestamp.toLong)),
       JField("nodeId", JString(p.nodeId.toString())),
       JField("serialized", JString(PaymentRequest.write(p))),
       p.description.fold(string => JField("description", JString(string)), hash => JField("descriptionHash", JString(hash.toHex))),
       JField("paymentHash", JString(p.paymentHash.toString()))) ++
+      paymentMetadata ++
       expiry ++
       minFinalCltvExpiry ++
       amount :+

eclair-core/src/main/scala/fr/acinq/eclair/payment/Monitoring.scala

@@ -27,6 +27,7 @@ object Monitoring {
     val PaymentAmount = Kamon.histogram("payment.amount", "Payment amount (satoshi)")
     val PaymentFees = Kamon.histogram("payment.fees", "Payment fees (satoshi)")
     val PaymentParts = Kamon.histogram("payment.parts", "Number of HTLCs per payment (MPP)")
+    val PaymentHtlcReceived = Kamon.counter("payment.received", "Number of valid htlcs received")
     val PaymentFailed = Kamon.counter("payment.failed", "Number of failed payment")
     val PaymentError = Kamon.counter("payment.error", "Non-fatal errors encountered during payment attempts")
     val PaymentAttempt = Kamon.histogram("payment.attempt", "Number of attempts before a payment succeeds")
@@ -71,6 +72,7 @@ object Monitoring {
     val PaymentId = "paymentId"
     val ParentId = "parentPaymentId"
     val PaymentHash = "paymentHash"
+    val PaymentMetadataIncluded = "paymentMetadataIncluded"
 
     val Amount = "amount"
     val TotalAmount = "totalAmount"

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentEvents.scala

@@ -25,6 +25,7 @@ import fr.acinq.eclair.router.Announcements
 import fr.acinq.eclair.router.Router.{ChannelDesc, ChannelHop, Hop, Ignore}
 import fr.acinq.eclair.wire.protocol.{ChannelDisabled, ChannelUpdate, Node, TemporaryChannelFailure}
 import fr.acinq.eclair.{MilliSatoshi, ShortChannelId, TimestampMilli}
+import scodec.bits.ByteVector
 
 import java.util.UUID
 import scala.concurrent.duration.FiniteDuration
@@ -114,6 +115,8 @@ object PaymentReceived {
 
 }
 
+case class PaymentMetadataReceived(paymentHash: ByteVector32, paymentMetadata: ByteVector)
+
 case class PaymentSettlingOnChain(id: UUID, amount: MilliSatoshi, paymentHash: ByteVector32, timestamp: TimestampMilli = TimestampMilli.now()) extends PaymentEvent
 
 sealed trait PaymentFailure {

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentPacket.scala

@@ -117,7 +117,7 @@ object IncomingPaymentPacket {
     } else {
       // We merge contents from the outer and inner payloads.
       // We must use the inner payload's total amount and payment secret because the payment may be split between multiple trampoline payments (#reckless).
-      Right(FinalPacket(add, PaymentOnion.createMultiPartPayload(outerPayload.amount, innerPayload.totalAmount, outerPayload.expiry, innerPayload.paymentSecret)))
+      Right(FinalPacket(add, PaymentOnion.createMultiPartPayload(outerPayload.amount, innerPayload.totalAmount, outerPayload.expiry, innerPayload.paymentSecret, innerPayload.paymentMetadata)))
     }
   }
 
@@ -213,9 +213,12 @@ object OutgoingPaymentPacket {
    *         - the trampoline onion to include in final payload of a normal onion
    */
   def buildTrampolineToLegacyPacket(invoice: PaymentRequest, hops: Seq[NodeHop], finalPayload: PaymentOnion.FinalPayload): (MilliSatoshi, CltvExpiry, Sphinx.PacketAndSecrets) = {
-    val (firstAmount, firstExpiry, payloads) = hops.drop(1).reverse.foldLeft((finalPayload.amount, finalPayload.expiry, Seq[PaymentOnion.PerHopPayload](finalPayload))) {
+    // NB: the final payload will never reach the recipient, since the next-to-last node in the trampoline route will convert that to a non-trampoline payment.
+    // We use the smallest final payload possible, otherwise we may overflow the trampoline onion size.
+    val dummyFinalPayload = PaymentOnion.createSinglePartPayload(finalPayload.amount, finalPayload.expiry, finalPayload.paymentSecret, None)
+    val (firstAmount, firstExpiry, payloads) = hops.drop(1).reverse.foldLeft((finalPayload.amount, finalPayload.expiry, Seq[PaymentOnion.PerHopPayload](dummyFinalPayload))) {
       case ((amount, expiry, payloads), hop) =>
-        // The next-to-last trampoline hop must include invoice data to indicate the conversion to a legacy payment.
+        // The next-to-last node in the trampoline route must receive invoice data to indicate the conversion to a non-trampoline payment.
         val payload = if (payloads.length == 1) {
           PaymentOnion.createNodeRelayToNonTrampolinePayload(finalPayload.amount, finalPayload.totalAmount, finalPayload.expiry, hop.nextNodeId, invoice)
         } else {

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentRequest.scala

@@ -19,12 +19,11 @@ package fr.acinq.eclair.payment
 import fr.acinq.bitcoin.Crypto.{PrivateKey, PublicKey}
 import fr.acinq.bitcoin.{Base58, Base58Check, Bech32, Block, ByteVector32, ByteVector64, Crypto}
 import fr.acinq.eclair.payment.PaymentRequest._
-import fr.acinq.eclair.{CltvExpiryDelta, FeatureSupport, Features, MilliSatoshi, MilliSatoshiLong, NodeParams, ShortChannelId, TimestampSecond, randomBytes32}
+import fr.acinq.eclair.{CltvExpiryDelta, Feature, FeatureSupport, Features, InvoiceFeature, MilliSatoshi, MilliSatoshiLong, NodeParams, ShortChannelId, TimestampSecond, randomBytes32}
 import scodec.bits.{BitVector, ByteOrdering, ByteVector}
 import scodec.codecs.{list, ubyte}
 import scodec.{Codec, Err}
 
-import scala.concurrent.duration._
 import scala.util.{Failure, Success, Try}
 
 /**
@@ -67,6 +66,11 @@ case class PaymentRequest(prefix: String, amount: Option[MilliSatoshi], timestam
     case PaymentRequest.DescriptionHash(h) => Right(h)
   }.get
 
+  /**
+   * @return metadata about the payment (see option_payment_metadata).
+   */
+  lazy val paymentMetadata: Option[ByteVector] = tags.collectFirst { case m: PaymentRequest.PaymentMetadata => m.data }
+
   /**
    * @return the fallback address if any. It could be a script address, pubkey address, ..
    */
@@ -126,6 +130,11 @@ object PaymentRequest {
     Block.LivenetGenesisBlock.hash -> "lnbc"
   )
 
+  val defaultFeatures: Map[Feature with InvoiceFeature, FeatureSupport] = Map(
+    Features.VariableLengthOnion -> FeatureSupport.Mandatory,
+    Features.PaymentSecret -> FeatureSupport.Mandatory,
+  )
+
   def apply(chainHash: ByteVector32,
             amount: Option[MilliSatoshi],
             paymentHash: ByteVector32,
@@ -137,14 +146,16 @@ object PaymentRequest {
             extraHops: List[List[ExtraHop]] = Nil,
             timestamp: TimestampSecond = TimestampSecond.now(),
             paymentSecret: ByteVector32 = randomBytes32(),
-            features: PaymentRequestFeatures = PaymentRequestFeatures(Features.VariableLengthOnion.mandatory, Features.PaymentSecret.mandatory)): PaymentRequest = {
+            paymentMetadata: Option[ByteVector] = None,
+            features: PaymentRequestFeatures = PaymentRequestFeatures(defaultFeatures)): PaymentRequest = {
     require(features.requirePaymentSecret, "invoices must require a payment secret")
     val prefix = prefixes(chainHash)
     val tags = {
       val defaultTags = List(
         Some(PaymentHash(paymentHash)),
         Some(description.fold(Description, DescriptionHash)),
         Some(PaymentSecret(paymentSecret)),
+        paymentMetadata.map(PaymentMetadata),
         fallbackAddress.map(FallbackAddress(_)),
         expirySeconds.map(Expiry(_)),
         Some(MinFinalCltvExpiry(minFinalCltvExpiryDelta.toInt)),
@@ -193,7 +204,6 @@ object PaymentRequest {
   case class InvalidTag23(data: BitVector) extends InvalidTaggedField
   case class UnknownTag25(data: BitVector) extends UnknownTaggedField
   case class UnknownTag26(data: BitVector) extends UnknownTaggedField
-  case class UnknownTag27(data: BitVector) extends UnknownTaggedField
   case class UnknownTag28(data: BitVector) extends UnknownTaggedField
   case class UnknownTag29(data: BitVector) extends UnknownTaggedField
   case class UnknownTag30(data: BitVector) extends UnknownTaggedField
@@ -229,6 +239,11 @@ object PaymentRequest {
    */
   case class DescriptionHash(hash: ByteVector32) extends TaggedField
 
+  /**
+   * Additional metadata to attach to the payment.
+   */
+  case class PaymentMetadata(data: ByteVector) extends TaggedField
+
   /**
    * Fallback Payment that specifies a fallback payment address to be used if LN payment cannot be processed
    */
@@ -355,8 +370,8 @@ object PaymentRequest {
   }
 
   object PaymentRequestFeatures {
-    def apply(features: Int*): PaymentRequestFeatures = PaymentRequestFeatures(long2bits(features.foldLeft(0L) {
-      case (current, feature) => current + (1L << feature)
+    def apply(features: Map[Feature with InvoiceFeature, FeatureSupport]): PaymentRequestFeatures = PaymentRequestFeatures(long2bits(features.foldLeft(0L) {
+      case (current, (feature, support)) => current + (1L << feature.supportBit(support))
     }))
   }
 
@@ -429,7 +444,7 @@ object PaymentRequest {
       .typecase(24, dataCodec(bits).as[MinFinalCltvExpiry])
       .typecase(25, dataCodec(bits).as[UnknownTag25])
       .typecase(26, dataCodec(bits).as[UnknownTag26])
-      .typecase(27, dataCodec(bits).as[UnknownTag27])
+      .typecase(27, dataCodec(alignedBytesCodec(bytes)).as[PaymentMetadata])
       .typecase(28, dataCodec(bits).as[UnknownTag28])
       .typecase(29, dataCodec(bits).as[UnknownTag29])
       .typecase(30, dataCodec(bits).as[UnknownTag30])