Merge pull request blockscout#3327 from poanetwork/vb-handle-errors-f…

…or-nonarchive-node Handle various indexer fetchers errors in setup with non-archive node
ForwardProtocol · Oct 2, 2020 · 73ea331 · 73ea331
2 parents 698622d + d9304c0
commit 73ea331
Show file tree

Hide file tree

Showing 8 changed files with 75 additions and 39 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -31,6 +31,7 @@
 - [#3256](https://github.com/poanetwork/blockscout/pull/3256) - Fix for invisible validator address at block page and wrong alert text color at xDai
 
 ### Chore
+- [#3327](https://github.com/poanetwork/blockscout/pull/3327) - Handle various indexer fetchers errors in setup with non-archive node
 - [#3325](https://github.com/poanetwork/blockscout/pull/3325) - Dark theme improvements
 - [#3316](https://github.com/poanetwork/blockscout/pull/3316), [#3317](https://github.com/poanetwork/blockscout/pull/3317) - xDai smile logo
 - [#3315](https://github.com/poanetwork/blockscout/pull/3315) - Environment variable to disable Bridge market cap updater

diff --git a/apps/explorer/lib/explorer/chain/address/coin_balance_daily.ex b/apps/explorer/lib/explorer/chain/address/coin_balance_daily.ex
@@ -64,7 +64,9 @@ defmodule Explorer.Chain.Address.CoinBalanceDaily do
     )
   end
 
-  def changeset(%__MODULE__{} = balance, params) do
+  def changeset(_, params) when is_nil(params), do: :ignore
+
+  def changeset(%__MODULE__{} = balance, params) when not is_nil(params) do
     balance
     |> cast(params, @allowed_fields)
     |> validate_required(@required_fields)

diff --git a/apps/explorer/lib/explorer/chain/import.ex b/apps/explorer/lib/explorer/chain/import.ex
@@ -168,6 +168,9 @@ defmodule Explorer.Chain.Import do
 
       %Changeset{valid?: true}, {:error, _} = error ->
         error
+
+      :ignore, error ->
+        {:error, error}
     end)
     |> case do
       {:ok, changes} -> {:ok, {runner, changes}}

diff --git a/apps/indexer/config/dev/geth.exs b/apps/indexer/config/dev/geth.exs
@@ -10,7 +10,7 @@ config :indexer,
       ),
     transport_options: [
       http: EthereumJSONRPC.HTTP.HTTPoison,
-      url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY",
+      url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"),
       http_options: [recv_timeout: :timer.minutes(1), timeout: :timer.minutes(1), hackney: [pool: :ethereum_jsonrpc]]
     ],
     variant: EthereumJSONRPC.Geth

diff --git a/apps/indexer/config/prod/geth.exs b/apps/indexer/config/prod/geth.exs
@@ -10,7 +10,7 @@ config :indexer,
       ),
     transport_options: [
       http: EthereumJSONRPC.HTTP.HTTPoison,
-      url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL") || "https://mainnet.infura.io/8lTvJTKmHPCHazkneJsY",
+      url: System.get_env("ETHEREUM_JSONRPC_HTTP_URL"),
       http_options: [recv_timeout: :timer.minutes(10), timeout: :timer.minutes(10), hackney: [pool: :ethereum_jsonrpc]]
     ],
     variant: EthereumJSONRPC.Geth

diff --git a/apps/indexer/lib/indexer/fetcher/coin_balance.ex b/apps/indexer/lib/indexer/fetcher/coin_balance.ex
@@ -151,25 +151,32 @@ defmodule Indexer.Fetcher.CoinBalance do
 
     block_timestamp_map =
       Enum.reduce(block_numbers, %{}, fn block_number, map ->
-        {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} =
-          EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments)
-
-        day = DateTime.to_date(timestamp)
-        Map.put(map, "#{block_number}", day)
+        case EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) do
+          {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} ->
+            day = DateTime.to_date(timestamp)
+            Map.put(map, "#{block_number}", day)
+
+          _ ->
+            %{}
+        end
       end)
 
     importable_balances_daily_params =
       params_list
       |> Enum.map(fn balance_param ->
-        day = Map.get(block_timestamp_map, "#{balance_param.block_number}")
-
-        incoming_balance_daily_param = %{
-          address_hash: balance_param.address_hash,
-          day: day,
-          value: balance_param.value
-        }
-
-        incoming_balance_daily_param
+        if Map.has_key?(block_timestamp_map, "#{balance_param.block_number}") do
+          day = Map.get(block_timestamp_map, "#{balance_param.block_number}")
+
+          incoming_balance_daily_param = %{
+            address_hash: balance_param.address_hash,
+            day: day,
+            value: balance_param.value
+          }
+
+          incoming_balance_daily_param
+        else
+          nil
+        end
       end)
 
     addresses_params = balances_params_to_address_params(importable_balances_params)
@@ -228,9 +235,9 @@ defmodule Indexer.Fetcher.CoinBalance do
   end
 
   defp run_fetched_balances(%FetchedBalances{errors: errors} = fetched_balances, _) do
-    {:ok, imported} = import_fetched_balances(fetched_balances)
-
-    Accounts.drop(imported[:addresses])
+    with {:ok, imported} <- import_fetched_balances(fetched_balances) do
+      Accounts.drop(imported[:addresses])
+    end
 
     retry(errors)
   end

diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex
@@ -109,7 +109,12 @@ defmodule Indexer.Fetcher.InternalTransaction do
         EthereumJSONRPC.fetch_block_internal_transactions(unique_numbers, json_rpc_named_arguments)
 
       _ ->
-        fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments)
+        try do
+          fetch_block_internal_transactions_by_transactions(unique_numbers, json_rpc_named_arguments)
+        rescue
+          error ->
+            {:error, error}
+        end
     end
     |> case do
       {:ok, internal_transactions_params} ->
@@ -156,8 +161,16 @@ defmodule Indexer.Fetcher.InternalTransaction do
         |> Chain.get_transactions_of_block_number()
         |> Enum.map(&params(&1))
         |> case do
-          [] -> {:ok, []}
-          transactions -> EthereumJSONRPC.fetch_internal_transactions(transactions, json_rpc_named_arguments)
+          [] ->
+            {:ok, []}
+
+          transactions ->
+            try do
+              EthereumJSONRPC.fetch_internal_transactions(transactions, json_rpc_named_arguments)
+            catch
+              :exit, error ->
+                {:error, error}
+            end
         end
         |> case do
           {:ok, internal_transactions} -> {:ok, internal_transactions ++ acc_list}

diff --git a/apps/indexer/lib/indexer/transform/address_coin_balances_daily.ex b/apps/indexer/lib/indexer/transform/address_coin_balances_daily.ex
@@ -64,19 +64,26 @@ defmodule Indexer.Transform.AddressCoinBalancesDaily do
 
     block_timestamp_map =
       Enum.reduce(block_numbers, %{}, fn block_number, map ->
-        {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} =
-          EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments)
-
-        day = DateTime.to_date(timestamp)
-        Map.put(map, "#{block_number}", day)
+        case EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) do
+          {:ok, %Blocks{blocks_params: [%{timestamp: timestamp}]}} ->
+            day = DateTime.to_date(timestamp)
+            Map.put(map, "#{block_number}", day)
+
+          _ ->
+            map
+        end
       end)
 
     logs_params
     |> Enum.into(acc, fn
       %{address_hash: address_hash, block_number: block_number}
       when is_binary(address_hash) and is_integer(block_number) ->
-        day = Map.get(block_timestamp_map, "#{block_number}")
-        %{address_hash: address_hash, day: day}
+        if Map.has_key?(block_timestamp_map, "#{block_number}") do
+          day = Map.get(block_timestamp_map, "#{block_number}")
+          %{address_hash: address_hash, day: day}
+        else
+          nil
+        end
 
       %{type: "pending"} ->
         nil
@@ -137,19 +144,22 @@ defmodule Indexer.Transform.AddressCoinBalancesDaily do
     # a transaction MUST have a `from_address_hash`
     json_rpc_named_arguments = Application.get_env(:explorer, :json_rpc_named_arguments)
 
-    {:ok, %Blocks{blocks_params: [%{timestamp: block_timestamp}]}} =
-      EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments)
+    case EthereumJSONRPC.fetch_blocks_by_range(block_number..block_number, json_rpc_named_arguments) do
+      {:ok, %Blocks{blocks_params: [%{timestamp: block_timestamp}]}} ->
+        day = DateTime.to_date(block_timestamp)
+        acc = MapSet.put(initial, %{address_hash: from_address_hash, day: day})
 
-    day = DateTime.to_date(block_timestamp)
-    acc = MapSet.put(initial, %{address_hash: from_address_hash, day: day})
+        # `to_address_hash` is optional
+        case transaction_params do
+          %{to_address_hash: to_address_hash} when is_binary(to_address_hash) ->
+            MapSet.put(acc, %{address_hash: to_address_hash, day: day})
 
-    # `to_address_hash` is optional
-    case transaction_params do
-      %{to_address_hash: to_address_hash} when is_binary(to_address_hash) ->
-        MapSet.put(acc, %{address_hash: to_address_hash, day: day})
+          _ ->
+            acc
+        end
 
       _ ->
-        acc
+        initial
     end
   end
 end