Governance: Enhance CI/CD and preparation for hot reloads (#20)

* Refactor and introduce Code.status/0

* Require dialyxir in :prod env

* Elixirize erlangish code

* Use `npm ci` rather than `npm install` for release

* Modify Dockerfile to create CI and release images

* Add JobCache and JobConductor for heavy jobs

* Add docker powered CICD

* Modify ci job script

* Import formatter configuration from distillery

* Run mix format on rel/cofnig.ex

* Include erts in dev release

In order to generate hot upgrade erts must be included
https://hexdocs.pm/distillery/guides/upgrades_and_downgrades.html

* Randomize cookie for dev release

* Cleanup release configuration overlays

* Move calls to Application.app_dir/2 to runtime

* Add CICD tests

* Drop :src_branch configuration

* Hook docker baked CICD to uniris app

* Add CICD as a child to GovernanceSupervisor

* Add moduledoc to Governance.Code.CICD

* Configure git inside container

.dockerignore

@@ -10,6 +10,7 @@ erl_crash.dump
 .elixir_ls
 **/doc
 .
+.git/config
 
 /rel/artifacts
 
@@ -38,4 +39,4 @@ log_*
 
 /.idea/
 uniris-node.iml
-projectFilesBackup
+projectFilesBackup

.formatter.exs

@@ -5,5 +5,6 @@
     "{config,lib,test}/**/*.{ex,exs}",
     "apps/*/{lib,config,test}/**/*.{ex,exs}",
     "apps/*/mix.exs"
-  ]
+  ],
+  import_deps: [:distillery]
 ]

Dockerfile

@@ -1,45 +1,70 @@
-FROM bitwalker/alpine-elixir:latest
-
-# Install system requirements
-RUN apk add --no-cache --update
-    openssl \
-    build-base \
-    gcc \
-    git \
-    npm \
-    python3 \
-    wget
-
-# Install Libsodium
-RUN wget https://download.libsodium.org/libsodium/releases/LATEST.tar.gz && \
-    mkdir /opt/libsodium && \
-    tar zxvf LATEST.tar.gz -C /opt/libsodium && \
-    cd /opt/libsodium/libsodium-stable && \
-    ./configure && \
-    make && \
-    make install
+FROM elixir:alpine AS uniris-ci
 
-WORKDIR /opt/app
+# CI
+#  - compile
+#  - release
+#  - gen PLT
+
+# running CI with proposal should generate release upgrade
+#  - commit proposal
+#  - compile
+#  - run ci
+#  - generate release upgrade
+
+######### TODO
+# TESTNET
+#  - code
+#  - release
+
+# running TESTNET with release upgrade should ???
 
-ENV MIX_ENV prod
+RUN apk add --no-cache --update \
+  build-base bash gcc git npm python3 wget openssl libsodium-dev
+
+# Install hex and rebar
+RUN mix local.rebar --force \
+ && mix local.hex --if-missing --force
+
+WORKDIR /opt/code
 
 COPY . .
 
-# Install dependencies
-# Cache Elixir deps
-RUN mix deps.get --only prod
-RUN mix deps.compile
+RUN git config user.name uniris \
+ && git config user.email uniris@uniris.io \
+ && git remote add origin https://github.com/UNIRIS/uniris-node
 
-WORKDIR /opt/app/assets
+# Compile
+RUN mix deps.get \
+ && cd assets \
+ && npm ci \
+ && npm run deploy
 
-# Cache Node deps
-RUN npm install
+# Release
+RUN mix phx.digest \
+ && mix distillery.release
 
-# Compile JavaScript
-RUN npm run deploy
+# gen PLT
+RUN mix git_hooks.run pre_push
 
-WORKDIR /opt/app
+# Install
+RUN mkdir /opt/app \
+ && cd /opt/app \
+ && tar zxf /opt/code/_build/dev/rel/uniris_node/releases/*/uniris_node.tar.gz
+CMD /opt/app/bin/uniris_node foreground
 
-# Prepare the image
-RUN mix compile
-RUN mix phx.digest
+################################################################################
+
+FROM uniris-ci as build
+
+FROM alpine
+
+RUN apk add --no-cache --update bash git openssl libsodium
+
+COPY --from=build /opt/app /opt/app
+COPY --from=build /opt/code/.git /opt/code/.git
+
+WORKDIR /opt/code
+RUN git reset --hard
+
+WORKDIR /opt/app
+CMD /opt/app/bin/uniris_node foreground

config/config.exs

@@ -35,10 +35,6 @@ config :phoenix, :json_library, Jason
 
 config :uniris, :src_dir, File.cwd!()
 
-config :uniris,
-       :src_branch,
-       System.cmd("git", ["symbolic-ref", "--short", "HEAD"], []) |> elem(0) |> String.trim()
-
 config :uniris, Uniris.Crypto,
   supported_curves: [
     :ed25519,
@@ -78,6 +74,8 @@ config :uniris, UnirisWeb.Endpoint,
     layout: {UnirisWeb.LayoutView, "live.html"}
   ]
 
+config :uniris, Uniris.Governance.Code.CICD, impl: Uniris.Governance.Code.CICD.Docker
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

lib/uniris/crypto/ed25519/libsodium_port.ex

@@ -4,15 +4,15 @@ defmodule Uniris.Crypto.Ed25519.LibSodiumPort do
 
   require Logger
 
-  @libsodium_exec_file Application.app_dir(:uniris, "/priv/crypto/c_dist/libsodium")
-
   def start_link(opts \\ []) do
     GenServer.start_link(__MODULE__, opts, name: __MODULE__)
   end
 
   def init(_opts) do
+    libsodium = Application.app_dir(:uniris, "/priv/crypto/c_dist/libsodium")
+
     port =
-      Port.open({:spawn_executable, @libsodium_exec_file}, [
+      Port.open({:spawn_executable, libsodium}, [
         :binary,
         :exit_status,
         {:packet, 4}

lib/uniris/governance/code.ex

@@ -5,9 +5,8 @@ defmodule Uniris.Governance.Code do
 
   alias Uniris.Crypto
 
-  alias __MODULE__.CI
+  alias __MODULE__.CICD
   alias __MODULE__.Proposal
-  alias __MODULE__.TestNet
 
   alias Uniris.Governance.Pools
 
@@ -20,7 +19,6 @@ defmodule Uniris.Governance.Code do
   alias Uniris.Utils
 
   @src_dir Application.compile_env(:uniris, :src_dir)
-  @src_branch Application.compile_env(:uniris, :src_branch)
 
   @doc """
   List the source files from the master branch
@@ -61,7 +59,7 @@ defmodule Uniris.Governance.Code do
   Deploy the proposal into a dedicated testnet
   """
   @spec deploy_proposal_testnet(Proposal.t()) :: :ok
-  defdelegate deploy_proposal_testnet(prop), to: TestNet, as: :deploy_proposal
+  defdelegate deploy_proposal_testnet(prop), to: Utils.impl(CICD), as: :run_testnet!
 
   @doc """
   Ensure the code proposal is valid according to the defined rules:
@@ -80,12 +78,11 @@ defmodule Uniris.Governance.Code do
   end
 
   @doc """
-  Ensure the code proposal is an applicable on the given branch.
+  Ensure the code proposal is an applicable on the current branch.
   """
   @spec applicable_proposal?(Proposal.t()) :: boolean()
   def applicable_proposal?(
         %Proposal{changes: changes, address: address},
-        branch \\ @src_branch,
         src_dir \\ @src_dir
       ) do
     random = :crypto.strong_rand_bytes(4) |> Base.encode16()
@@ -95,11 +92,9 @@ defmodule Uniris.Governance.Code do
     cmd_options = [stderr_to_stdout: true, cd: src_dir]
     git = fn args -> System.cmd("git", args, cmd_options) end
 
-    x = byte_size(branch)
-
     res =
-      case git.(["symbolic-ref", "--short", "HEAD"]) do
-        {<<^branch::binary-size(x), _::binary>>, 0} ->
+      case status() do
+        {:clean, _} ->
           git.(["apply", "--check", prop_file])
 
         otherwise ->
@@ -111,6 +106,32 @@ defmodule Uniris.Governance.Code do
     match?({_, 0}, res)
   end
 
+  @doc """
+  Return tuple {state, branch_name} where state could be :clean or :dirty or
+  {:error, whatever}
+  """
+  @spec status() :: {:clean, String.t()} | {:dirty, String.t()} | {:error, any}
+  def status(src_dir \\ @src_dir) do
+    git = fn args -> System.cmd("git", args, stderr_to_stdout: true, cd: src_dir) end
+
+    case git.(["symbolic-ref", "--short", "HEAD"]) do
+      {branch, 0} ->
+        case git.(["status", "--porcelain"]) do
+          {"", 0} ->
+            {:clean, String.trim(branch)}
+
+          {_, 0} ->
+            {:dirty, String.trim(branch)}
+
+          otherwise ->
+            {:error, otherwise}
+        end
+
+      otherwise ->
+        {:error, otherwise}
+    end
+  end
+
   @doc """
   Return true if version2 is a direct successor of version1.
   Note that build and patch must not be set.
@@ -162,12 +183,9 @@ defmodule Uniris.Governance.Code do
   def succeessor_version?(%Version{}, %Version{}), do: false
 
   defp current_version do
-    {:ok, vsn} = :application.get_key(:uniris, :vsn)
-    List.to_string(vsn)
+    :uniris |> Application.spec(:vsn) |> List.to_string()
   end
 
   @spec list_proposal_CI_logs(binary()) :: Enumerable.t()
-  defdelegate list_proposal_CI_logs(address),
-    to: CI,
-    as: :list_logs
+  defdelegate list_proposal_CI_logs(address), to: Utils.impl(CICD), as: :get_log
 end