test geolocation e2e with a test mmdb

config/.env.test

@@ -13,3 +13,4 @@ SELFHOST=false
 SITE_LIMIT=3
 HCAPTCHA_SITEKEY=test
 HCAPTCHA_SECRET=scottiger
+IP_GEOLOCATION_DB=test/priv/GeoLite2-City-Test.mmdb

config/config.exs

@@ -41,6 +41,4 @@ config :plausible, Plausible.Repo,
   connect_timeout: 300_000,
   handshake_timeout: 300_000
 
-config :plausible, Plausible.Geo, adapter: Plausible.Geo.Locus
-
 import_config "#{config_env()}.exs"

config/test.exs

@@ -27,5 +27,3 @@ config :bamboo, :refute_timeout, 10
 config :plausible,
   session_timeout: 0,
   http_impl: Plausible.HTTPClient.Mock
-
-config :plausible, Plausible.Geo, adapter: Plausible.Geo.Stub

lib/plausible/geo.ex

@@ -1,6 +1,62 @@
 defmodule Plausible.Geo do
   @moduledoc "Geolocation functions"
-  @adapter Application.compile_env!(:plausible, [__MODULE__, :adapter])
+  @db :geolocation
+
+  @doc """
+  Starts the geodatabase loading process. Two options are supported, local file and maxmind key.
+
+  Loading a local file:
+
+      iex> load_db(path: "/etc/plausible/dbip-city.mmdb")
+      :ok
+
+  Loading a maxmind db:
+
+      # this license key is no longer active
+      iex> load_db(license_key: "LNpsJCCKPis6XvBP", edition: "GeoLite2-City", async: true)
+      :ok
+
+  """
+  def load_db(opts) do
+    cond do
+      license_key = opts[:license_key] ->
+        edition = opts[:edition] || "GeoLite2-City"
+        :ok = :locus.start_loader(@db, {:maxmind, edition}, license_key: license_key)
+
+      path = opts[:path] ->
+        :ok = :locus.start_loader(@db, path)
+
+      true ->
+        raise "failed to load geolocation db: need :path or :license_key to be provided"
+    end
+
+    unless opts[:async] do
+      {:ok, _version} = :locus.await_loader(@db)
+    end
+
+    :ok
+  end
+
+  @doc """
+  Returns geodatabase type. Used for deciding whether to show the DBIP disclaimer.
+
+  Example:
+
+      # in the case of a dbip db
+      iex> database_type()
+      "DBIP-City-Lite"
+
+      # in the case of a maxmind db
+      iex> database_type()
+      "GeoLite2-City"
+
+  """
+  def database_type do
+    case :locus.get_info(@db, :metadata) do
+      {:ok, %{database_type: type}} -> type
+      _other -> nil
+    end
+  end
 
   @doc """
   Looks up geo info about an ip address.
@@ -88,43 +144,15 @@ defmodule Plausible.Geo do
 
   """
   def lookup(ip_address) do
-    @adapter.lookup(ip_address)
-  end
-
-  @doc """
-  Starts the geodatabase loading process. Two options are supported, local file and maxmind key.
-
-  Loading a local file:
+    case :locus.lookup(@db, ip_address) do
+      {:ok, entry} ->
+        entry
 
-      iex> load_db(path: "/etc/plausible/dbip-city.mmdb")
-      :ok
+      :not_found ->
+        nil
 
-  Loading a maxmind db:
-
-      # this license key is no longer active
-      iex> load_db(license_key: "LNpsJCCKPis6XvBP", edition: "GeoLite2-City", async: true)
-      :ok
-
-  """
-  def load_db(opts \\ []) do
-    @adapter.load_db(opts)
-  end
-
-  @doc """
-  Returns geodatabase type. Used for deciding whether to show the DBIP disclaimer.
-
-  Example:
-
-      # in the case of a dbip db
-      iex> database_type()
-      "DBIP-City-Lite"
-
-      # in the case of a maxmind db
-      iex> database_type()
-      "GeoLite2-City"
-
-  """
-  def database_type do
-    @adapter.database_type()
+      {:error, reason} ->
+        raise "failed to lookup ip address #{inspect(ip_address)}: " <> inspect(reason)
+    end
   end
 end

test/plausible/ingestion/event_test.exs

@@ -15,7 +15,7 @@ defmodule Plausible.Ingestion.EventTest do
   end
 
   @valid_request %Plausible.Ingestion.Request{
-    remote_ip: "2.2.2.2",
+    remote_ip: "2.125.160.216",
     user_agent:
       "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405",
     event_name: "pageview",
@@ -46,8 +46,8 @@ defmodule Plausible.Ingestion.EventTest do
              domain: "plausible-ingestion-event-basic.test",
              browser: "Safari",
              browser_version: "",
-             city_geoname_id: 2_988_507,
-             country_code: "FR",
+             city_geoname_id: 2_655_045,
+             country_code: "GB",
              hostname: "skywalker.test",
              "meta.key": [],
              "meta.value": [],
@@ -58,8 +58,8 @@ defmodule Plausible.Ingestion.EventTest do
              referrer: "m.facebook.test",
              referrer_source: "utm_source",
              screen_size: "Desktop",
-             subdivision1_code: "FR-IDF",
-             subdivision2_code: "FR-75",
+             subdivision1_code: "GB-ENG",
+             subdivision2_code: "GB-WBK",
              transferred_from: "",
              utm_campaign: "utm_campaign",
              utm_content: "utm_content",

test/plausible_web/controllers/api/external_controller_test.exs

@@ -630,7 +630,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       assert event.referrer == ""
     end
 
-    # Fake data is set up in config/test.exs
+    # Fake geo is loaded from test/priv/GeoLite2-City-Test.mmdb
     test "looks up location data from the ip address", %{conn: conn} do
       params = %{
         name: "pageview",
@@ -639,15 +639,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       }
 
       conn
-      |> put_req_header("x-forwarded-for", "2.2.2.2")
+      |> put_req_header("x-forwarded-for", "2.125.160.216")
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-20.com")
 
-      assert pageview.country_code == "FR"
-      assert pageview.subdivision1_code == "FR-IDF"
-      assert pageview.subdivision2_code == "FR-75"
-      assert pageview.city_geoname_id == 2_988_507
+      assert pageview.country_code == "GB"
+      assert pageview.subdivision1_code == "GB-ENG"
+      assert pageview.subdivision2_code == "GB-WBK"
+      assert pageview.city_geoname_id == 2_655_045
     end
 
     test "ignores unknown country code ZZ", %{conn: conn} do
@@ -674,7 +674,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       }
 
       conn
-      |> put_req_header("x-forwarded-for", "1.1.1.1:123")
+      |> put_req_header("x-forwarded-for", "216.160.83.56:123")
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-x-forwarded-for-port.com")
@@ -690,12 +690,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       }
 
       conn
-      |> put_req_header("x-forwarded-for", "1:1:1:1:1:1:1:1")
+      |> put_req_header("x-forwarded-for", "2001:218:1:1:1:1:1:1")
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-x-forwarded-for-ipv6.com")
 
-      assert pageview.country_code == "US"
+      assert pageview.country_code == "JP"
     end
 
     test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn} do
@@ -706,12 +706,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       }
 
       conn
-      |> put_req_header("x-forwarded-for", "[1:1:1:1:1:1:1:1]:123")
+      |> put_req_header("x-forwarded-for", "[2001:218:1:1:1:1:1:1]:123")
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-x-forwarded-for-ipv6-port.com")
 
-      assert pageview.country_code == "US"
+      assert pageview.country_code == "JP"
     end
 
     test "uses cloudflare's special header for client IP address if present", %{conn: conn} do
@@ -723,7 +723,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
 
       conn
       |> put_req_header("x-forwarded-for", "0.0.0.0")
-      |> put_req_header("cf-connecting-ip", "1.1.1.1")
+      |> put_req_header("cf-connecting-ip", "216.160.83.56")
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-cloudflare.com")
@@ -740,7 +740,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
 
       conn
       |> put_req_header("x-forwarded-for", "0.0.0.0")
-      |> put_req_header("b-forwarded-for", "1.1.1.1,9.9.9.9")
+      |> put_req_header("b-forwarded-for", "216.160.83.56,9.9.9.9")
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-bunny.com")
@@ -758,7 +758,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       }
 
       conn
-      |> put_req_header("forwarded", "by=0.0.0.0;for=1.1.1.1;host=somehost.com;proto=https")
+      |> put_req_header("forwarded", "by=0.0.0.0;for=216.160.83.56;host=somehost.com;proto=https")
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-forwarded.com")
@@ -776,13 +776,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
       conn
       |> put_req_header(
         "forwarded",
-        "by=0.0.0.0;for=\"[1:1:1:1:1:1:1:1]\",for=0.0.0.0;host=somehost.com;proto=https"
+        "by=0.0.0.0;for=\"[2001:218:1:1:1:1:1:1]\",for=0.0.0.0;host=somehost.com;proto=https"
       )
       |> post("/api/event", params)
 
       pageview = get_event("external-controller-test-forwarded-ipv6.com")
 
-      assert pageview.country_code == "US"
+      assert pageview.country_code == "JP"
     end
 
     test "URL is decoded", %{conn: conn} do

test/priv/README.md

@@ -0,0 +1 @@
+`GeoLite2-City-Test.mmdb` is downloaded from https://github.com/maxmind/MaxMind-DB