Feature external signer

src/Nethermind/Nethermind.Blockchain.Test/Consensus/ClefSignerTests.cs

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Threading.Tasks;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Extensions;
+using Nethermind.Core.Test.Builders;
+using Nethermind.JsonRpc;
+using Nethermind.JsonRpc.Client;
+using NSubstitute;
+using NUnit.Framework;
+
+namespace Nethermind.Blockchain.Test.Consensus
+{
+    [TestFixture]
+    public class ClefSignerTests
+    {
+        [Test]
+        public async Task Sign_SigningHash_RequestHasCorrectParameters()
+        {
+            IJsonRpcClient client = Substitute.For<IJsonRpcClient>();
+            client.Post<string[]>("account_list").Returns(Task.FromResult<string[]?>([TestItem.AddressA!.ToString()]));
+            Task<string?> postMethod = client.Post<string>("account_signData", "text/plain", Arg.Any<string>(), Keccak.Zero);
+            var returnValue = (new byte[65]).ToHexString();
+            postMethod.Returns(returnValue);
+            ClefSigner sut = await ClefSigner.Create(client, 0);
+
+            var result = sut.Sign(Keccak.Zero);
+
+            Assert.That(new Signature(returnValue).Bytes, Is.EqualTo(result.Bytes));
+        }
+
+        [Test]
+        public async Task SignCliqueHeader_SigningCliqueHeader_RequestHasCorrectParameters()
+        {
+            IJsonRpcClient client = Substitute.For<IJsonRpcClient>();
+            client.Post<string[]>("account_list").Returns(Task.FromResult<string[]?>([TestItem.AddressA!.ToString()]));
+            Task<string?> postMethod = client.Post<string>("account_signData", "application/x-clique-header", Arg.Any<string>(), Arg.Any<string>());
+            var returnValue = (new byte[65]).ToHexString();
+            postMethod.Returns(returnValue);
+            ClefSigner sut = await ClefSigner.Create(client, 0);
+
+            var result = sut.SignCliqueHeader(Keccak.Zero.Bytes.ToArray());
+
+            Assert.That(new Signature(returnValue).Bytes, Is.EqualTo(result.Bytes));
+        }
+
+
+        [TestCase(0, 27)]
+        [TestCase(1, 28)]
+        public async Task SignCliqueHeader_RecoveryIdIsSetToCliqueValues_RecoveryIdIsAdjusted(byte recId, byte expected)
+        {
+            IJsonRpcClient client = Substitute.For<IJsonRpcClient>();
+            client.Post<string[]>("account_list").Returns(Task.FromResult<string[]?>([TestItem.AddressA!.ToString()]));
+            Task<string?> postMethod = client.Post<string>("account_signData", "application/x-clique-header", Arg.Any<string>(), Arg.Any<string>());
+            var returnValue = (new byte[65]);
+            returnValue[64] = recId;
+            postMethod.Returns(returnValue.ToHexString());
+            ClefSigner sut = await ClefSigner.Create(client, 0);
+
+            var result = sut.SignCliqueHeader(new byte[1]);
+
+            Assert.That(result.V, Is.EqualTo(expected));
+        }
+
+        [Test]
+        public async Task Create_SignerAddressSpecified_CorrectAddressIsSet()
+        {
+            IJsonRpcClient client = Substitute.For<IJsonRpcClient>();
+            client.Post<string[]>("account_list").Returns(Task.FromResult<string[]?>([TestItem.AddressA!.ToString(), TestItem.AddressB!.ToString()]));
+
+            ClefSigner sut = await ClefSigner.Create(client, 0, TestItem.AddressB);
+
+            Assert.That(sut.Address, Is.EqualTo(TestItem.AddressB));
+        }
+
+        [Test]
+        public void Create_SignerAddressDoesNotExists_ThrowInvalidOperationException()
+        {
+            IJsonRpcClient client = Substitute.For<IJsonRpcClient>();
+            client.Post<string[]>("account_list").Returns(Task.FromResult<string[]?>([TestItem.AddressA!.ToString(), TestItem.AddressB!.ToString()]));
+
+            Assert.That(async () => await ClefSigner.Create(client, 0, TestItem.AddressC), Throws.InstanceOf<InvalidOperationException>());
+        }
+
+        [Test]
+        public async Task SetSigner_TryingToASigner_ThrowInvalidOperationException()
+        {
+            IJsonRpcClient client = Substitute.For<IJsonRpcClient>();
+            client.Post<string[]>("account_list").Returns(Task.FromResult<string[]?>([TestItem.AddressA!.ToString()]));
+            ClefSigner sut = await ClefSigner.Create(client, 0);
+
+            Assert.That(() => sut.SetSigner(Build.A.PrivateKey.TestObject), Throws.InstanceOf<InvalidOperationException>());
+        }
+    }
+}

src/Nethermind/Nethermind.Consensus.Clique/CliqueSealer.cs

@@ -8,6 +8,7 @@
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Crypto;
+using Nethermind.JsonRpc;
 using Nethermind.Logging;
 
 [assembly: InternalsVisibleTo("Nethermind.Clique.Test")]
@@ -27,7 +28,6 @@ public CliqueSealer(ISigner signer, ICliqueConfig config, ISnapshotManager snaps
             _snapshotManager = snapshotManager ?? throw new ArgumentNullException(nameof(snapshotManager));
             _config = config ?? throw new ArgumentNullException(nameof(config));
             _signer = signer ?? throw new ArgumentNullException(nameof(signer));
-
             if (config.Epoch == 0) config.Epoch = Clique.DefaultEpochLength;
         }
 
@@ -64,7 +64,13 @@ public CliqueSealer(ISigner signer, ICliqueConfig config, ISnapshotManager snaps
 
             // Sign all the things!
             Hash256 headerHash = SnapshotManager.CalculateCliqueHeaderHash(header);
-            Signature signature = _signer.Sign(headerHash);
+
+            Signature signature;
+            if (_signer is ClefSigner clefSigner)
+                signature = clefSigner.SignCliqueHeader(SnapshotManager.CalculateCliqueRlp(header));
+            else
+                signature = _signer.Sign(headerHash);
+
             // Copy signature bytes (R and S)
             byte[] signatureBytes = signature.Bytes;
             Array.Copy(signatureBytes, 0, header.ExtraData, header.ExtraData.Length - Clique.ExtraSealLength, signatureBytes.Length);

src/Nethermind/Nethermind.Consensus.Clique/SnapshotManager.cs

@@ -31,6 +31,7 @@ public class SnapshotManager : ISnapshotManager
         private readonly IDb _blocksDb;
         private ulong _lastSignersCount = 0;
         private readonly LruCache<ValueHash256, Snapshot> _snapshotCache = new(Clique.InMemorySnapshots, "clique snapshots");
+        private static readonly HeaderDecoder _headerDecoder = new HeaderDecoder();
 
         public SnapshotManager(ICliqueConfig cliqueConfig, IDb blocksDb, IBlockTree blockTree, IEthereumEcdsa ecdsa, ILogManager logManager)
         {
@@ -85,6 +86,18 @@ public static Hash256 CalculateCliqueHeaderHash(BlockHeader blockHeader)
             return sigHash;
         }
 
+        public static byte[] CalculateCliqueRlp(BlockHeader header)
+        {
+            int extraSeal = 65;
+            int shortExtraLength = header.ExtraData.Length - extraSeal;
+            byte[] fullExtraData = header.ExtraData;
+            byte[] shortExtraData = header.ExtraData.Slice(0, shortExtraLength);
+            header.ExtraData = shortExtraData;
+            byte[] rlp = _headerDecoder.Encode(header).Bytes;
+            header.ExtraData = fullExtraData;
+            return rlp;
+        }
+
         private readonly object _snapshotCreationLock = new();
 
         public ulong GetLastSignersCount() => _lastSignersCount;

src/Nethermind/Nethermind.Consensus/IMiningConfig.cs

@@ -52,4 +52,9 @@ public interface IMiningConfig : IConfig
 
     [ConfigItem(HiddenFromDocs = true, DisabledForCli = true, DefaultValue = "null")]
     IBlocksConfig? BlocksConfig { get; }
+    [ConfigItem(
+    Description = "Url for an external signer like clef: https://github.com/ethereum/go-ethereum/blob/master/cmd/clef/tutorial.md",
+    HiddenFromDocs = false,
+    DefaultValue = "null")]
+    string Signer { get; set; }
 }

src/Nethermind/Nethermind.Consensus/MiningConfig.cs

@@ -70,4 +70,6 @@ public IBlocksConfig? BlocksConfig
             return _blocksConfig;
         }
     }
+
+    public string Signer { get; set; }
 }

src/Nethermind/Nethermind.Consensus/Nethermind.Consensus.csproj

@@ -10,6 +10,7 @@
     <ProjectReference Include="..\Nethermind.Core\Nethermind.Core.csproj" />
     <ProjectReference Include="..\Nethermind.Crypto\Nethermind.Crypto.csproj" />
     <ProjectReference Include="..\Nethermind.Evm\Nethermind.Evm.csproj" />
+    <ProjectReference Include="..\Nethermind.JsonRpc.Connectors\Nethermind.JsonRpc.Clients.csproj" />
     <ProjectReference Include="..\Nethermind.TxPool\Nethermind.TxPool.csproj" />
   </ItemGroup>
 

src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs

@@ -1,7 +1,9 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System;
 using System.IO;
+using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Api;
@@ -15,6 +17,10 @@
 using Nethermind.Core;
 using Nethermind.Db;
 using Nethermind.Db.Blooms;
+using Nethermind.JsonRpc;
+using Nethermind.JsonRpc.Client;
+using Nethermind.KeyStore.Config;
+using Nethermind.Network;
 using Nethermind.Serialization.Rlp;
 using Nethermind.State.Repositories;
 
@@ -31,7 +37,7 @@ public InitializeBlockTree(INethermindApi api)
             (_get, _set) = api.ForInit;
         }
 
-        public Task Execute(CancellationToken cancellationToken)
+        public async Task Execute(CancellationToken cancellationToken)
         {
             IInitConfig initConfig = _get.Config<IInitConfig>();
             IBloomConfig bloomConfig = _get.Config<IBloomConfig>();
@@ -68,11 +74,22 @@ public Task Execute(CancellationToken cancellationToken)
 
             ISigner signer = NullSigner.Instance;
             ISignerStore signerStore = NullSigner.Instance;
-            if (_get.Config<IMiningConfig>().Enabled)
+            IMiningConfig miningConfig = _get.Config<IMiningConfig>();
+            if (miningConfig.Enabled)
             {
-                Signer signerAndStore = new(_get.SpecProvider!.ChainId, _get.OriginalSignerKey!, _get.LogManager);
-                signer = signerAndStore;
-                signerStore = signerAndStore;
+                if (!string.IsNullOrEmpty(miningConfig.Signer))
+                {
+                    ClefSigner signerAndStore =
+                        await SetupExternalSigner(miningConfig.Signer, _get.SpecProvider!.ChainId, _get.Config<IKeyStoreConfig>().BlockAuthorAccount);
+                    signer = signerAndStore;
+                    signerStore = signerAndStore;
+                }
+                else
+                {
+                    Signer signerAndStore = new Signer(_get.SpecProvider!.ChainId, _get.OriginalSignerKey!, _get.LogManager);
+                    signer = signerAndStore;
+                    signerStore = signerAndStore;
+                }
             }
 
             _set.EngineSigner = signer;
@@ -109,7 +126,21 @@ public Task Execute(CancellationToken cancellationToken)
                 new ExitOnBlockNumberHandler(blockTree, _get.ProcessExit!, initConfig.ExitOnBlockNumber.Value, _get.LogManager);
             }
 
-            return Task.CompletedTask;
+        }
+
+        private async Task<ClefSigner> SetupExternalSigner(string urlSigner, ulong chainId, string blockAuthorAccount)
+        {
+            try
+            {
+                Address? address = string.IsNullOrEmpty(blockAuthorAccount) ? null : new Address(blockAuthorAccount);
+                BasicJsonRpcClient rpcClient = new(new Uri(urlSigner), _get.EthereumJsonSerializer, _get.LogManager, TimeSpan.FromSeconds(10));
+                _get.DisposeStack.Push(rpcClient);
+                return await ClefSigner.Create(rpcClient, chainId, address);
+            }
+            catch (HttpRequestException e)
+            {
+                throw new NetworkingException($"Remote signer at {urlSigner} did not respond.", NetworkExceptionType.TargetUnreachable, e);
+            }
         }
     }
 }

src/Nethermind/Nethermind.Init/Steps/SetupKeyStore.cs

@@ -6,6 +6,7 @@
 using System.Threading.Tasks;
 using Nethermind.Api;
 using Nethermind.Config;
+using Nethermind.Consensus;
 using Nethermind.Crypto;
 using Nethermind.KeyStore;
 using Nethermind.KeyStore.Config;
@@ -60,7 +61,11 @@ await Task.Run(() =>
                 INodeKeyManager nodeKeyManager = new NodeKeyManager(get.CryptoRandom, get.KeyStore, keyStoreConfig, get.LogManager, passwordProvider, get.FileSystem);
                 ProtectedPrivateKey? nodeKey = set.NodeKey = nodeKeyManager.LoadNodeKey();
 
-                set.OriginalSignerKey = nodeKeyManager.LoadSignerKey();
+                IMiningConfig miningConfig = get.Config<IMiningConfig>();
+                //Don't load the local key if an external signer is configured  
+                if (!miningConfig.Enabled && string.IsNullOrEmpty(miningConfig.Signer))
+                    set.OriginalSignerKey = nodeKeyManager.LoadSignerKey();
+
                 IPAddress ipAddress = networkConfig.ExternalIp is not null ? IPAddress.Parse(networkConfig.ExternalIp) : IPAddress.Loopback;
                 IEnode enode = set.Enode = new Enode(nodeKey.PublicKey, ipAddress, networkConfig.P2PPort);
 

src/Nethermind/Nethermind.JsonRpc/Client/BasicJsonRpcClient.cs → src/Nethermind/Nethermind.JsonRpc.Connectors/Client/BasicJsonRpcClient.cs

@@ -2,31 +2,31 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
-using System.Collections.Generic;
 using System.Data;
-using System.Linq;
-using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
-using System.Threading.Tasks;
 using Nethermind.Logging;
 using Nethermind.Serialization.Json;
 
 namespace Nethermind.JsonRpc.Client
 {
-    public class BasicJsonRpcClient : IJsonRpcClient
+    public class BasicJsonRpcClient : IJsonRpcClient, IDisposable
     {
         private readonly HttpClient _client;
         private readonly IJsonSerializer _jsonSerializer;
         private readonly ILogger _logger;
+        private bool disposedValue;
 
-        public BasicJsonRpcClient(Uri uri, IJsonSerializer jsonSerializer, ILogManager logManager)
+        public BasicJsonRpcClient(Uri uri, IJsonSerializer jsonSerializer, ILogManager logManager) :
+            this(uri, jsonSerializer, logManager, /*support long block traces better, default 100s might be too small*/ TimeSpan.FromMinutes(5))
+        { }
+        public BasicJsonRpcClient(Uri uri, IJsonSerializer jsonSerializer, ILogManager logManager, TimeSpan timeout)
         {
             _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager));
             _jsonSerializer = jsonSerializer;
 
             _client = new HttpClient { BaseAddress = uri };
-            _client.Timeout = TimeSpan.FromMinutes(5); // support long block traces better, default 100s might be too small
+            _client.Timeout = timeout;
             _client.DefaultRequestHeaders.Accept.Clear();
             _client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
             AddAuthorizationHeader();
@@ -70,6 +70,10 @@ public async Task<T> Post<T>(string method, params object[] parameters)
             {
                 throw;
             }
+            catch (TaskCanceledException)
+            {
+                throw;
+            }
             catch (Exception)
             {
                 throw new DataException($"Cannot deserialize {responseString}");
@@ -105,5 +109,23 @@ private void AddAuthorizationHeader()
 
         private static string Base64Encode(string plainText)
             => Convert.ToBase64String(Encoding.UTF8.GetBytes(plainText));
+
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!disposedValue)
+            {
+                if (disposing)
+                {
+                    _client?.Dispose();
+                }
+                disposedValue = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(disposing: true);
+            GC.SuppressFinalize(this);
+        }
     }
 }

src/Nethermind/Nethermind.JsonRpc.Connectors/Nethermind.JsonRpc.Clients.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>annotations</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Nethermind.Logging\Nethermind.Logging.csproj" />
+    <ProjectReference Include="..\Nethermind.Serialization.Json\Nethermind.Serialization.Json.csproj" />
+  </ItemGroup>
+
+</Project>