ETCM-[165, 166]: Publish the RLP and Crypto libraries to Sonatype. (#933

)

* ETCM-165: Publish the RLP and Crypto libraries to Sonatype.
ETCM-165: Accept SNAPSHOT in version number sent in Hello.
ETCM-165: Wait for unit tests before publishing.
ETCM-165: Check if the GPG key already exists.
ETCM-165: Update GPG.
ETCM-165: Try restarting the gpg agent.

* ETCM-165: Cross publish.

* ETCM-165: Update nix.

* ETCM-165: Publish bytes as well.

* ETCM-165: Remove obsolete nix file.

* ETCM-165: Add comment about Nix Flakes

* ETCM-165: Fix version regex in mantis.nix

* ETCM-165: Update dependency version hash.

* ETCM-165: Fix README to talk about the overlay.nix

* ETCM-165: Update the dependency sha after fixing the VM submodule version.

.buildkite/pipeline.nix

@@ -153,5 +153,15 @@ in
         nix-shell --run '$SBT benchmark:compile dist'
       '';
     };
+
+    publish = commonAttrs // {
+      dependsOn = [ test-crypto test-rlp test-unit ];
+      label = "Publishing libraries to Maven";
+      command = ''
+        nix-env -iA nixpkgs.gnupg && nix-shell --run '.buildkite/publish.sh'
+      '';
+      branches = "master develop";
+      timeoutInMinutes = 30;
+    };
   };
 }

.buildkite/publish.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+set -euv
+
+# The build agents have gpg 2.0.22, which doesn't have the `--pinentry-mode` option, but
+# the sbt-pgp plugin assumes that 2.x has it, and generates an invalid command.
+# We can either create a wrapper script that removes that option, or update gpg
+# somewhere in pipeline.nix
+
+# Force a restart of the agent becuase it may be out of sync with what Nix installed.
+gpgconf --kill all
+
+gpg --version
+
+
+# The build agent might have this key from before.
+GPG_EXISTS=$(gpg --list-keys "$GPG_KEY_ID" >&2 && echo "yes" || echo "no")
+
+if [[ "$GPG_EXISTS" == "no" ]]; then
+	echo "$GPG_KEY" | base64 --decode | gpg --batch --import
+fi
+# Local testing showed that without this the SBT plugin got "Bad passphrase".
+gpg --passphrase $GPG_PASSPHRASE --batch --yes -a -b LICENSE
+
+# https://github.com/olafurpg/sbt-ci-release#secrets
+export PGP_SECRET="$GPG_KEY"
+export PGP_PASSPHRASE="$GPG_PASSPHRASE"
+export SONATYPE_USERNAME="$OSS_USERNAME"
+export SONATYPE_PASSWORD="$OSS_PASSWORD"
+
+set +u
+
+#https://github.com/sbt/sbt/issues/3570
+export JAVA_OPTS="$JAVA_OPTS -Dsbt.gigahorse=false"
+
+# ci-release cannot be called on individual modules, but it looks like
+# with `publish / skip := true` in build.sbt for the default project,
+# without any aggregation, by default it would publish nothing, so
+# let's tell it here by using `sbt-ci-release` env vars.
+# NOTE: +rlp/publishSigned with the `+` would cross publish,
+# but it doesn't work because of scapegoat (see below).
+export CI_SNAPSHOT_RELEASE="; bytes/publishSigned; rlp/publishSigned; crypto/publishSigned"
+export CI_RELEASE=$CI_SNAPSHOT_RELEASE
+export CI_SONATYPE_RELEASE=$"; bytes/sonatypeBundleRelease; rlp/sonatypeBundleRelease; crypto/sonatypeBundleRelease"
+
+# Scala 2.12 has up to scapegoat 1.4.5, while Scala 2.13 starts with 1.4.7.
+# I couldn't make build.sbt vary the scapegoat version by the current cross build,
+# so as a workaround the combos are called here explicitly.
+function release {
+    SCALA_VERSION=$1
+    export SCAPEGOAT_VERSION=$2
+
+    sbt "++ $SCALA_VERSION ; ci-release"
+}
+
+function releaseAll {
+    release 2.12.10 1.4.5
+    release 2.13.4 1.4.7
+}
+
+if [[ "$BUILDKITE_BRANCH" == "develop" ]]; then
+
+    # Publish the -SNAPSHOT version.
+    releaseAll
+
+elif [[ "$BUILDKITE_BRANCH" == "master" ]]; then
+
+    # Remove the -SNAPSHOT from the version file, then publish and release.
+    sed -i 's/-SNAPSHOT//' version.sbt
+
+    # Whether ci-release does a release or a snapshot depends on whether it thinks the build is tagged; setting a dummy value.
+    # Check https://github.com/olafurpg/sbt-ci-release/blob/main/plugin/src/main/scala/com/geirsson/CiReleasePlugin.scala for the rules.
+	export CI_COMMIT_TAG=$(sbt -Dsbt.supershell=false -error "print version")
+
+    releaseAll
+
+else
+
+  echo "Skipping the publish step."
+
+fi

README.md

@@ -32,7 +32,7 @@ Possible networks: `etc`, `eth`, `mordor`, `testnet-internal`
 ### Command Line Interface
 
 `cli` is a tool that can be used to:
- 
+
  - generate a new private key
  ```
 ./bin/mantis cli generate-private-key
@@ -124,10 +124,12 @@ will need to be updated so that it includes the new dependency state.
 To do so, please run:
 ```
 ./update-nix.sh
-git add ./nix/pkgs/mantis.nix
+git add ./nix/overlay.nix
 git commit -m "Update nix-sbt sha"
 ```
 
+For this command to work you'll need the [Flakes](https://nixos.wiki/wiki/Flakes) feature enabled in your `nix` environment.
+
 *NOTE:* This should only be necessary when updating dependencies
 (For example, edits to build.sbt or project/plugins.sbt will likely need to be regenerated)
 
@@ -159,9 +161,9 @@ If a new certificate is required, create a new keystore with a certificate by ru
 1. Configure the certificate and password file to be used at `mantis.network.rpc.http.certificate` key on the `application.conf` file:
 
     keystore-path: path to the keystore storing the certificates (if generated through our script they are by default located in "./tls/mantisCA.p12")
-   
+
     keystore-type: type of certificate keystore being used (if generated through our script use "pkcs12")
-   
+
     password-file: path to the file with the password used for accessing the certificate keystore (if generated through our script they are by default located in "./tls/password")
 2. Enable TLS in specific config:
     - For JSON RPC: `mantis.network.rpc.http.mode=https`
@@ -171,9 +173,9 @@ If a new certificate is required, create a new keystore with a certificate by ru
 1. Configure the certificate and password file to be used at `mantis.network.rpc.http.certificate` key on the `faucet.conf` file:
 
    keystore-path: path to the keystore storing the certificates (if generated through our script they are by default located in "./tls/mantisCA.p12")
-   
+
    keystore-type: type of certificate keystore being used (if generated through our script use "pkcs12")
-   
+
    password-file: path to the file with the password used for accessing the certificate keystore (if generated through our script they are by default located in "./tls/password")
 2. Enable TLS in specific config:
    - For JSON RPC: `mantis.network.rpc.http.mode=https`
@@ -200,7 +202,7 @@ volumes:
 - $HOME/.mantis:/home/demiourgos728/.mantis/
 command: -Dconfig.file=./conf/sagano.conf
 ```
-   
+
 2. Create a wallet address. Run the following curl command, replacing `<password>` by a password of your choice:
 ```
 curl --request POST \
@@ -209,7 +211,7 @@ curl --request POST \
   --header 'Content-Type: application/json' \
   --data '{
 	"jsonrpc": "2.0",
-  "method": "personal_newAccount", 
+  "method": "personal_newAccount",
   "params": ["<password>"],
   "id": 1
 }'
@@ -244,7 +246,7 @@ curl --request POST \
   --header 'Content-Type: application/json' \
   --data '{
 	"jsonrpc": "2.0",
-  "method": "faucet_sendFunds", 
+  "method": "faucet_sendFunds",
   "params": ["<address>"],
   "id": 1
 }'

build.sbt

@@ -12,11 +12,29 @@ val nixBuild = sys.props.isDefinedAt("nix")
 // Enable dev mode: disable certain flags, etc.
 val mantisDev = sys.props.get("mantisDev").contains("true") || sys.env.get("MANTIS_DEV").contains("true")
 
+// Releasing. https://github.com/olafurpg/sbt-ci-release
+inThisBuild(List(
+  organization := "io.iohk",
+  homepage := Some(url("https://github.com/input-output-hk/mantis")),
+  scmInfo := Some(ScmInfo(url("https://github.com/input-output-hk/mantis"), "git@github.com:input-output-hk/mantis.git")),
+  licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0")),
+  developers := List()
+))
+
+// https://github.com/sbt/sbt/issues/3570
+updateOptions := updateOptions.value.withGigahorse(false)
+
+// artifact name will include scala version
+crossPaths := true
+
+val `scala-2.12` = "2.12.10"
+val `scala-2.13` = "2.13.4"
+val supportedScalaVersions = List(`scala-2.12`, `scala-2.13`)
+
 def commonSettings(projectName: String): Seq[sbt.Def.Setting[_]] = Seq(
   name := projectName,
   organization := "io.iohk",
-  version := "3.2.1",
-  scalaVersion := "2.13.4",
+  scalaVersion := `scala-2.13`,
   // Scalanet snapshots are published to Sonatype after each build.
   resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
   testOptions in Test += Tests
@@ -38,7 +56,14 @@ def commonSettings(projectName: String): Seq[sbt.Def.Setting[_]] = Seq(
   scalacOptions ~= (options => if (mantisDev) options.filterNot(_ == "-Xfatal-warnings") else options),
   Test / parallelExecution := true,
   testOptions in Test += Tests.Argument("-oDG"),
-  (scalastyleConfig in Test) := file("scalastyle-test-config.xml")
+  (scalastyleConfig in Test) := file("scalastyle-test-config.xml"),
+  // Only publish selected libraries.
+  skip in publish := true
+)
+
+val publishSettings = Seq(
+  publish / skip := false,
+  crossScalaVersions := supportedScalaVersions
 )
 
 // Adding an "it" config because in `Dependencies.scala` some are declared with `% "it,test"`
@@ -50,6 +75,7 @@ lazy val bytes = {
     .in(file("bytes"))
     .configs(Integration)
     .settings(commonSettings("mantis-bytes"))
+    .settings(publishSettings)
     .settings(
       libraryDependencies ++=
         Dependencies.akkaUtil ++
@@ -65,6 +91,7 @@ lazy val crypto = {
     .configs(Integration)
     .dependsOn(bytes)
     .settings(commonSettings("mantis-crypto"))
+    .settings(publishSettings)
     .settings(
       libraryDependencies ++=
         Dependencies.akkaUtil ++
@@ -81,6 +108,7 @@ lazy val rlp = {
     .configs(Integration)
     .dependsOn(bytes)
     .settings(commonSettings("mantis-rlp"))
+    .settings(publishSettings)
     .settings(
       libraryDependencies ++=
         Dependencies.akkaUtil ++
@@ -201,6 +229,9 @@ lazy val node = {
       batScriptExtraDefines += """call :add_java "-Dconfig.file=%APP_HOME%\conf\app.conf"""",
       batScriptExtraDefines += """call :add_java "-Dlogback.configurationFile=%APP_HOME%\conf\logback.xml""""
     )
+    .settings(
+      crossScalaVersions := List(`scala-2.13`)
+    )
 
   if (!nixBuild)
     node
@@ -252,5 +283,7 @@ addCommandAlias(
     |""".stripMargin
 )
 
-scapegoatVersion in ThisBuild := "1.4.7"
+// Scala 2.12 only has up to 1.4.5, while 2.13 only from 1.4.7
+// In theory we should be able to switch on `scalaVersion.value` but it doesn't seem to work.
+scapegoatVersion in ThisBuild := (sys.env.getOrElse("SCAPEGOAT_VERSION", "1.4.7"))
 scapegoatReports := Seq("xml")

bytes/src/main/scala/io/iohk/ethereum/utils/ByteStringUtils.scala

@@ -61,7 +61,7 @@ object ByteStringUtils {
 
   def concatByteStrings(elements: Iterator[ByteStringElement]): ByteString = {
     val builder = new mutable.ArrayBuilder.ofByte
-    elements.foreach(el => builder.addAll(el.asByteArray))
+    elements.foreach(el => builder ++= el.asByteArray)
     ByteString(builder.result())
   }
 

bytes/src/main/scala/io/iohk/ethereum/utils/Hex.scala

@@ -5,5 +5,7 @@ object Hex {
     bytes.map("%02x".format(_)).mkString
 
   def decode(hex: String): Array[Byte] =
-    hex.toSeq.sliding(2, 2).toArray.map(s => Integer.parseInt(s.unwrap, 16).toByte)
+    hex.toSeq.sliding(2, 2).toArray.map { s =>
+    	Integer.parseInt(s.mkString(""), 16).toByte
+    }
 }

nix/mantis.nix

@@ -19,8 +19,8 @@
 
 let
   version = let
-    buildSbt = builtins.readFile ../build.sbt;
-    captures = builtins.match ''.*version := "([^"]+)".*'' buildSbt;
+    versionSbt = builtins.readFile ../version.sbt;
+    captures = builtins.match ''.*version in ThisBuild := "([^"]+)".*'' versionSbt;
   in builtins.elemAt captures 0;
 
   PATH = lib.makeBinPath [ jre solc coreutils gawk gnused ];

nix/overlay.nix

@@ -3,7 +3,7 @@ rev: final: prev: {
 
   mantis = final.callPackage ./mantis.nix {
     src = ../.;
-    depsSha256 = "sha256-csNBHVOC2bNSOjLjWleiVeS5ts3qFS7V8DxBv2nVDqE=";
+    depsSha256 = "sha256-FpBrDvN0iR/eEpUunUIRRo6I9pWqpMyPRybEClCqTFs=";
   };
 
   mantis-hash = final.mantis.override {

project/plugins.sbt

@@ -11,3 +11,4 @@ addSbtPlugin("io.kamon" % "sbt-kanela-runner" % "2.0.5")
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2")
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1")
+addSbtPlugin("com.geirsson" % "sbt-ci-release" % "1.5.6")

src/test/scala/io/iohk/ethereum/utils/VersionInfoSpec.scala

@@ -8,7 +8,7 @@ class VersionInfoSpec extends AnyFlatSpec with Matchers {
 
   it should "match ethstats expected structure and preserve major and minor Java version" in {
     VersionInfo
-      .nodeName() should fullyMatch regex """mantis/v\d(\.\d+)*-[a-z0-9]{7}/[^/]+-[^/]+/[^/]+-.[^/]+-java-\d+\.\d+[._0-9]*"""
+      .nodeName() should fullyMatch regex """mantis/v\d(\.\d+)*(-SNAPSHOT)?-[a-z0-9]{7}/[^/]+-[^/]+/[^/]+-.[^/]+-java-\d+\.\d+[._0-9]*"""
   }
 
   it should "augment the name with an identity" in {

version.sbt

@@ -0,0 +1,6 @@
+// NOTE: This is replaced with `sed` during release,
+// but it could also be removed, and be determined
+// based on `git` tags by https://github.com/dwijnand/sbt-dynver,
+// which is a dependency of `sbt-ci-release`.
+
+version in ThisBuild := "3.2.1-SNAPSHOT"