Replace Geolix with Locus (#2362)

This PR replaces geolix with locus to simplify self-hosted setup. locus can auto-update maxmind dbs which are recommended for self-hosters if they want city-level geolocation. locus is also a bit faster.

This PR also uses a test mmdb file from https://github.com/maxmind/MaxMind-DB for e2e geolocation tests without stubs.

.gitignore

@@ -63,6 +63,7 @@ plausible-report.xml
 
 # Geolocation databases
 /priv/geodb/*.mmdb
+/priv/geodb/*.mmdb.gz
 
 # Auto-generated tracker files
 /priv/tracker/js/*.js

config/.env.test

@@ -11,3 +11,4 @@ SELFHOST=false
 SITE_LIMIT=3
 HCAPTCHA_SITEKEY=test
 HCAPTCHA_SECRET=scottiger
+IP_GEOLOCATION_DB=test/priv/GeoLite2-City-Test.mmdb

config/runtime.exs

@@ -142,11 +142,13 @@ geolite2_country_db =
   get_var_from_path_or_env(
     config_dir,
     "GEOLITE2_COUNTRY_DB",
-    Application.app_dir(:plausible, "/priv/geodb/dbip-country.mmdb")
+    Application.app_dir(:plausible, "/priv/geodb/dbip-country.mmdb.gz")
   )
 
 ip_geolocation_db = get_var_from_path_or_env(config_dir, "IP_GEOLOCATION_DB", geolite2_country_db)
 geonames_source_file = get_var_from_path_or_env(config_dir, "GEONAMES_SOURCE_FILE")
+maxmind_license_key = get_var_from_path_or_env(config_dir, "MAXMIND_LICENSE_KEY")
+maxmind_edition = get_var_from_path_or_env(config_dir, "MAXMIND_EDITION", "GeoLite2-City")
 
 if System.get_env("DISABLE_AUTH") do
   require Logger
@@ -433,17 +435,38 @@ config :kaffy,
     ]
   ]
 
-if config_env() != :test do
-  config :geolix,
-    databases: [
-      %{
-        id: :geolocation,
-        adapter: Geolix.Adapter.MMDB2,
-        source: ip_geolocation_db,
-        result_as: :raw
-      }
-    ]
-end
+geo_opts =
+  cond do
+    maxmind_license_key ->
+      [
+        license_key: maxmind_license_key,
+        edition: maxmind_edition,
+        async: true
+      ]
+
+    ip_geolocation_db ->
+      [path: ip_geolocation_db]
+
+    true ->
+      raise """
+      Missing geolocation database configuration.
+
+      Please set the IP_GEOLOCATION_DB environment value to the location of
+      your IP geolocation .mmdb file:
+
+          IP_GEOLOCATION_DB=/etc/plausible/dbip-city.mmdb
+
+      Or authenticate with MaxMind by
+      configuring MAXMIND_LICENSE_KEY and (optionally) MAXMIND_EDITION environment
+      variables:
+
+          MAXMIND_LICENSE_KEY=LNpsJCCKPis6XvBP
+          MAXMIND_EDITION=GeoLite2-City # this is the default edition
+
+      """
+  end
+
+config :plausible, Plausible.Geo, geo_opts
 
 if geonames_source_file do
   config :location, :geonames_source_file, geonames_source_file

config/test.exs

@@ -24,45 +24,6 @@ config :plausible, :google,
 
 config :bamboo, :refute_timeout, 10
 
-geolix_sample_lookup = %{
-  city: %{geoname_id: 2_988_507, names: %{en: "Paris"}},
-  continent: %{code: "EU", geoname_id: 6_255_148, names: %{en: "Europe"}},
-  country: %{
-    geoname_id: 3_017_382,
-    is_in_european_union: true,
-    iso_code: "FR",
-    names: %{en: "France"}
-  },
-  ip_address: {2, 2, 2, 2},
-  location: %{
-    latitude: 48.8566,
-    longitude: 2.35222,
-    time_zone: "Europe/Paris",
-    weather_code: "FRXX0076"
-  },
-  postal: %{code: "75000"},
-  subdivisions: [
-    %{geoname_id: 3_012_874, iso_code: "IDF", names: %{en: "Île-de-France"}},
-    %{geoname_id: 2_968_815, iso_code: "75", names: %{en: "Paris"}}
-  ]
-}
-
-config :geolix,
-  databases: [
-    %{
-      id: :geolocation,
-      adapter: Geolix.Adapter.Fake,
-      data: %{
-        {1, 1, 1, 1} => %{country: %{iso_code: "US"}},
-        {2, 2, 2, 2} => geolix_sample_lookup,
-        {1, 1, 1, 1, 1, 1, 1, 1} => %{country: %{iso_code: "US"}},
-        {0, 0, 0, 0} => %{country: %{iso_code: "ZZ"}, city: %{geoname_id: 123_123}},
-        {0, 0, 0, 1} => %{country: %{iso_code: "XX"}, subdivisions: [%{iso_code: "IDF"}]},
-        {0, 0, 0, 2} => %{country: %{iso_code: "T1"}, subdivisions: [%{}, %{iso_code: "IDF"}]}
-      }
-    }
-  ]
-
 config :plausible,
   session_timeout: 0,
   http_impl: Plausible.HTTPClient.Mock,

lib/mix/tasks/download_country_database.ex

@@ -34,7 +34,7 @@ defmodule Mix.Tasks.DownloadCountryDatabase do
 
     if res.status_code == 200 do
       File.mkdir("priv/geodb")
-      File.write!("priv/geodb/dbip-country.mmdb", res.body)
+      File.write!("priv/geodb/dbip-country.mmdb.gz", res.body)
       Logger.info("Downloaded and saved the database successfully")
     else
       Logger.error("Unable to download and save the database. Response: #{inspect(res)}")

lib/plausible/application.ex

@@ -30,9 +30,14 @@ defmodule Plausible.Application do
     ]
 
     opts = [strategy: :one_for_one, name: Plausible.Supervisor]
+
     setup_sentry()
     setup_opentelemetry()
+
+    setup_geolocation()
     Location.load_all()
+    Plausible.Geo.await_loader()
+
     Supervisor.start_link(children, opts)
   end
 
@@ -119,4 +124,9 @@ defmodule Plausible.Application do
     OpentelemetryEcto.setup([:plausible, :clickhouse_repo])
     OpentelemetryOban.setup()
   end
+
+  defp setup_geolocation do
+    opts = Application.fetch_env!(:plausible, Plausible.Geo)
+    :ok = Plausible.Geo.load_db(opts)
+  end
 end

lib/plausible/geo.ex

@@ -0,0 +1,189 @@
+defmodule Plausible.Geo do
+  @moduledoc """
+  This module provides an API for fetching IP geolocation.
+  """
+
+  require Logger
+
+  @db :geolocation
+
+  @doc """
+  Starts the geodatabase loading process. Two modes are supported: local file
+  and MaxMind license key.
+
+  ## Options
+
+    * `:path` - the path to the .mmdb database local file. When present,
+      `:license_key` and `:edition` are not required.
+
+    * `:license_key` - the [license key](https://support.maxmind.com/hc/en-us/articles/4407111582235-Generate-a-License-Key)
+      from MaxMind to authenticate requests to MaxMind.
+
+    * `:edition` - the name of the MaxMind database to be downloaded from MaxMind
+      servers. Defaults to `GeoLite2-City`.
+
+    * `:async` - when used, configures the database loading to run
+      asynchronously.
+
+  ## Examples
+
+    Loading from a local file:
+
+      iex> load_db(path: "/etc/plausible/dbip-city.mmdb")
+      :ok
+
+    Downloading a MaxMind DB (this license key is no longer active):
+
+      iex> load_db(license_key: "LNpsJCCKPis6XvBP", edition: "GeoLite2-City", async: true)
+      :ok
+
+  """
+  def load_db(opts) do
+    cond do
+      license_key = opts[:license_key] ->
+        edition = opts[:edition] || "GeoLite2-City"
+        :ok = :locus.start_loader(@db, {:maxmind, edition}, license_key: license_key)
+
+      path = opts[:path] ->
+        :ok = :locus.start_loader(@db, path)
+
+      true ->
+        raise "failed to load geolocation db: need :path or :license_key to be provided"
+    end
+
+    unless opts[:async] do
+      {:ok, _version} = :locus.await_loader(@db)
+    end
+
+    :ok
+  end
+
+  @doc """
+  Waits for the database to start after calling `load_db/1` with the async option.
+  """
+  def await_loader, do: :locus.await_loader(@db)
+
+  @doc """
+  Returns geodatabase type.
+
+  Used for deciding whether to show the DB-IP disclaimer or not.
+
+  ## Examples
+
+    In the case of a DB-IP database:
+
+      iex> database_type()
+      "DBIP-City-Lite"
+
+    In the case of a MaxMind database:
+
+      iex> database_type()
+      "GeoLite2-City"
+
+  """
+  def database_type do
+    case :locus.get_info(@db, :metadata) do
+      {:ok, %{database_type: type}} -> type
+      _other -> nil
+    end
+  end
+
+  @doc """
+  Looks up geo info about an IP address.
+
+  ## Examples
+
+      iex> lookup("8.7.6.5")
+      %{
+        "city" => %{
+          "geoname_id" => 5349755,
+          "names" => %{
+            "de" => "Fontana",
+            "en" => "Fontana",
+            "ja" => "フォンタナ",
+            "ru" => "Фонтана"
+          }
+        },
+        "continent" => %{
+          "code" => "NA",
+          "geoname_id" => 6255149,
+          "names" => %{
+            "de" => "Nordamerika",
+            "en" => "North America",
+            "es" => "Norteamérica",
+            "fr" => "Amérique du Nord",
+            "ja" => "北アメリカ",
+            "pt-BR" => "América do Norte",
+            "ru" => "Северная Америка",
+            "zh-CN" => "北美洲"
+          }
+        },
+        "country" => %{
+          "geoname_id" => 6252001,
+          "iso_code" => "US",
+          "names" => %{
+            "de" => "Vereinigte Staaten",
+            "en" => "United States",
+            "es" => "Estados Unidos",
+            "fr" => "États Unis",
+            "ja" => "アメリカ",
+            "pt-BR" => "EUA",
+            "ru" => "США",
+            "zh-CN" => "美国"
+          }
+        },
+        "location" => %{
+          "accuracy_radius" => 50,
+          "latitude" => 34.1211,
+          "longitude" => -117.4362,
+          "metro_code" => 803,
+          "time_zone" => "America/Los_Angeles"
+        },
+        "postal" => %{"code" => "92336"},
+        "registered_country" => %{
+          "geoname_id" => 6252001,
+          "iso_code" => "US",
+          "names" => %{
+            "de" => "Vereinigte Staaten",
+            "en" => "United States",
+            "es" => "Estados Unidos",
+            "fr" => "États Unis",
+            "ja" => "アメリカ",
+            "pt-BR" => "EUA",
+            "ru" => "США",
+            "zh-CN" => "美国"
+          }
+        },
+        "subdivisions" => [
+          %{
+            "geoname_id" => 5332921,
+            "iso_code" => "CA",
+            "names" => %{
+              "de" => "Kalifornien",
+              "en" => "California",
+              "es" => "California",
+              "fr" => "Californie",
+              "ja" => "カリフォルニア州",
+              "pt-BR" => "Califórnia",
+              "ru" => "Калифорния",
+              "zh-CN" => "加州"
+            }
+          }
+        ]
+      }
+
+  """
+  def lookup(ip_address) do
+    case :locus.lookup(@db, ip_address) do
+      {:ok, entry} ->
+        entry
+
+      :not_found ->
+        nil
+
+      {:error, reason} ->
+        Logger.error("failed to lookup ip address: " <> inspect(reason))
+        nil
+    end
+  end
+end

lib/plausible/ingestion/event.ex

@@ -168,7 +168,7 @@ defmodule Plausible.Ingestion.Event do
   end
 
   defp put_geolocation(%__MODULE__{} = event) do
-    result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip)
+    result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip) || %{}
 
     update_attrs(event, result)
   end