diff --git a/config/.env.test b/config/.env.test index 0f1a6f5d270a6..71044ee29bade 100644 --- a/config/.env.test +++ b/config/.env.test @@ -13,3 +13,4 @@ SELFHOST=false SITE_LIMIT=3 HCAPTCHA_SITEKEY=test HCAPTCHA_SECRET=scottiger +IP_GEOLOCATION_DB=test/priv/GeoLite2-City-Test.mmdb diff --git a/config/config.exs b/config/config.exs index 43e3ab4204330..369b11b00b264 100644 --- a/config/config.exs +++ b/config/config.exs @@ -41,6 +41,4 @@ config :plausible, Plausible.Repo, connect_timeout: 300_000, handshake_timeout: 300_000 -config :plausible, Plausible.Geo, adapter: Plausible.Geo.Locus - import_config "#{config_env()}.exs" diff --git a/config/test.exs b/config/test.exs index 8013847fef16a..02f33466f008a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -27,5 +27,3 @@ config :bamboo, :refute_timeout, 10 config :plausible, session_timeout: 0, http_impl: Plausible.HTTPClient.Mock - -config :plausible, Plausible.Geo, adapter: Plausible.Geo.Stub diff --git a/lib/plausible/geo.ex b/lib/plausible/geo.ex index e5762ddfa5bb2..d0afc7b2128a4 100644 --- a/lib/plausible/geo.ex +++ b/lib/plausible/geo.ex @@ -1,6 +1,62 @@ defmodule Plausible.Geo do @moduledoc "Geolocation functions" - @adapter Application.compile_env!(:plausible, [__MODULE__, :adapter]) + @db :geolocation + + @doc """ + Starts the geodatabase loading process. Two options are supported, local file and maxmind key. + + Loading a local file: + + iex> load_db(path: "/etc/plausible/dbip-city.mmdb") + :ok + + Loading a maxmind db: + + # this license key is no longer active + iex> load_db(license_key: "LNpsJCCKPis6XvBP", edition: "GeoLite2-City", async: true) + :ok + + """ + def load_db(opts) do + cond do + license_key = opts[:license_key] -> + edition = opts[:edition] || "GeoLite2-City" + :ok = :locus.start_loader(@db, {:maxmind, edition}, license_key: license_key) + + path = opts[:path] -> + :ok = :locus.start_loader(@db, path) + + true -> + raise "failed to load geolocation db: need :path or :license_key to be provided" + end + + unless opts[:async] do + {:ok, _version} = :locus.await_loader(@db) + end + + :ok + end + + @doc """ + Returns geodatabase type. Used for deciding whether to show the DBIP disclaimer. + + Example: + + # in the case of a dbip db + iex> database_type() + "DBIP-City-Lite" + + # in the case of a maxmind db + iex> database_type() + "GeoLite2-City" + + """ + def database_type do + case :locus.get_info(@db, :metadata) do + {:ok, %{database_type: type}} -> type + _other -> nil + end + end @doc """ Looks up geo info about an ip address. @@ -88,43 +144,15 @@ defmodule Plausible.Geo do """ def lookup(ip_address) do - @adapter.lookup(ip_address) - end - - @doc """ - Starts the geodatabase loading process. Two options are supported, local file and maxmind key. - - Loading a local file: + case :locus.lookup(@db, ip_address) do + {:ok, entry} -> + entry - iex> load_db(path: "/etc/plausible/dbip-city.mmdb") - :ok + :not_found -> + nil - Loading a maxmind db: - - # this license key is no longer active - iex> load_db(license_key: "LNpsJCCKPis6XvBP", edition: "GeoLite2-City", async: true) - :ok - - """ - def load_db(opts \\ []) do - @adapter.load_db(opts) - end - - @doc """ - Returns geodatabase type. Used for deciding whether to show the DBIP disclaimer. - - Example: - - # in the case of a dbip db - iex> database_type() - "DBIP-City-Lite" - - # in the case of a maxmind db - iex> database_type() - "GeoLite2-City" - - """ - def database_type do - @adapter.database_type() + {:error, reason} -> + raise "failed to lookup ip address #{inspect(ip_address)}: " <> inspect(reason) + end end end diff --git a/lib/plausible/geo/adapter.ex b/lib/plausible/geo/adapter.ex deleted file mode 100644 index c1c1f20ac12e1..0000000000000 --- a/lib/plausible/geo/adapter.ex +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Plausible.Geo.Adapter do - @moduledoc "Behaviour to be implemented by geolocation adapters" - - @type entry :: map - @type opts :: Keyword.t() - @type ip_address :: :inet.ip_address() | String.t() - - @callback load_db(opts) :: :ok - @callback database_type :: String.t() | nil - @callback lookup(ip_address) :: entry | nil -end diff --git a/lib/plausible/geo/locus.ex b/lib/plausible/geo/locus.ex deleted file mode 100644 index 6be51cea67ee6..0000000000000 --- a/lib/plausible/geo/locus.ex +++ /dev/null @@ -1,51 +0,0 @@ -defmodule Plausible.Geo.Locus do - @moduledoc false - require Logger - - @behaviour Plausible.Geo.Adapter - @db :geolocation - - @impl true - def load_db(opts) do - cond do - license_key = opts[:license_key] -> - edition = opts[:edition] || "GeoLite2-City" - :ok = :locus.start_loader(@db, {:maxmind, edition}, license_key: license_key) - - path = opts[:path] -> - :ok = :locus.start_loader(@db, path) - - true -> - raise "failed to load geolocation db: need :path or :license_key to be provided" - end - - unless opts[:async] do - {:ok, _version} = :locus.await_loader(@db) - end - - :ok - end - - @impl true - def database_type do - case :locus.get_info(@db, :metadata) do - {:ok, %{database_type: type}} -> type - _other -> nil - end - end - - @impl true - def lookup(ip_address) do - case :locus.lookup(@db, ip_address) do - {:ok, entry} -> - entry - - :not_found -> - nil - - {:error, reason} -> - Logger.error("failed to lookup ip address: " <> inspect(reason)) - nil - end - end -end diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs index e306bba2a171c..ef201367c3462 100644 --- a/test/plausible/ingestion/event_test.exs +++ b/test/plausible/ingestion/event_test.exs @@ -15,7 +15,7 @@ defmodule Plausible.Ingestion.EventTest do end @valid_request %Plausible.Ingestion.Request{ - remote_ip: "2.2.2.2", + remote_ip: "2.125.160.216", user_agent: "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405", event_name: "pageview", @@ -46,8 +46,8 @@ defmodule Plausible.Ingestion.EventTest do domain: "plausible-ingestion-event-basic.test", browser: "Safari", browser_version: "", - city_geoname_id: 2_988_507, - country_code: "FR", + city_geoname_id: 2_655_045, + country_code: "GB", hostname: "skywalker.test", "meta.key": [], "meta.value": [], @@ -58,8 +58,8 @@ defmodule Plausible.Ingestion.EventTest do referrer: "m.facebook.test", referrer_source: "utm_source", screen_size: "Desktop", - subdivision1_code: "FR-IDF", - subdivision2_code: "FR-75", + subdivision1_code: "GB-ENG", + subdivision2_code: "GB-WBK", transferred_from: "", utm_campaign: "utm_campaign", utm_content: "utm_content", diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index bc7d97d7d75de..ae421fa969547 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -630,7 +630,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert event.referrer == "" end - # Fake data is set up in config/test.exs + # Fake geo is loaded from test/priv/GeoLite2-City-Test.mmdb test "looks up location data from the ip address", %{conn: conn} do params = %{ name: "pageview", @@ -639,15 +639,15 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do } conn - |> put_req_header("x-forwarded-for", "2.2.2.2") + |> put_req_header("x-forwarded-for", "2.125.160.216") |> post("/api/event", params) pageview = get_event("external-controller-test-20.com") - assert pageview.country_code == "FR" - assert pageview.subdivision1_code == "FR-IDF" - assert pageview.subdivision2_code == "FR-75" - assert pageview.city_geoname_id == 2_988_507 + assert pageview.country_code == "GB" + assert pageview.subdivision1_code == "GB-ENG" + assert pageview.subdivision2_code == "GB-WBK" + assert pageview.city_geoname_id == 2_655_045 end test "ignores unknown country code ZZ", %{conn: conn} do @@ -674,7 +674,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do } conn - |> put_req_header("x-forwarded-for", "1.1.1.1:123") + |> put_req_header("x-forwarded-for", "216.160.83.56:123") |> post("/api/event", params) pageview = get_event("external-controller-test-x-forwarded-for-port.com") @@ -690,12 +690,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do } conn - |> put_req_header("x-forwarded-for", "1:1:1:1:1:1:1:1") + |> put_req_header("x-forwarded-for", "2001:218:1:1:1:1:1:1") |> post("/api/event", params) pageview = get_event("external-controller-test-x-forwarded-for-ipv6.com") - assert pageview.country_code == "US" + assert pageview.country_code == "JP" end test "works with ipv6 with a port number in x-forwarded-for", %{conn: conn} do @@ -706,12 +706,12 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do } conn - |> put_req_header("x-forwarded-for", "[1:1:1:1:1:1:1:1]:123") + |> put_req_header("x-forwarded-for", "[2001:218:1:1:1:1:1:1]:123") |> post("/api/event", params) pageview = get_event("external-controller-test-x-forwarded-for-ipv6-port.com") - assert pageview.country_code == "US" + assert pageview.country_code == "JP" end test "uses cloudflare's special header for client IP address if present", %{conn: conn} do @@ -723,7 +723,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("x-forwarded-for", "0.0.0.0") - |> put_req_header("cf-connecting-ip", "1.1.1.1") + |> put_req_header("cf-connecting-ip", "216.160.83.56") |> post("/api/event", params) pageview = get_event("external-controller-test-cloudflare.com") @@ -740,7 +740,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("x-forwarded-for", "0.0.0.0") - |> put_req_header("b-forwarded-for", "1.1.1.1,9.9.9.9") + |> put_req_header("b-forwarded-for", "216.160.83.56,9.9.9.9") |> post("/api/event", params) pageview = get_event("external-controller-test-bunny.com") @@ -758,7 +758,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do } conn - |> put_req_header("forwarded", "by=0.0.0.0;for=1.1.1.1;host=somehost.com;proto=https") + |> put_req_header("forwarded", "by=0.0.0.0;for=216.160.83.56;host=somehost.com;proto=https") |> post("/api/event", params) pageview = get_event("external-controller-test-forwarded.com") @@ -776,13 +776,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header( "forwarded", - "by=0.0.0.0;for=\"[1:1:1:1:1:1:1:1]\",for=0.0.0.0;host=somehost.com;proto=https" + "by=0.0.0.0;for=\"[2001:218:1:1:1:1:1:1]\",for=0.0.0.0;host=somehost.com;proto=https" ) |> post("/api/event", params) pageview = get_event("external-controller-test-forwarded-ipv6.com") - assert pageview.country_code == "US" + assert pageview.country_code == "JP" end test "URL is decoded", %{conn: conn} do diff --git a/test/priv/GeoLite2-City-Test.mmdb b/test/priv/GeoLite2-City-Test.mmdb new file mode 100644 index 0000000000000..ed9adc6547fd9 Binary files /dev/null and b/test/priv/GeoLite2-City-Test.mmdb differ diff --git a/test/priv/README.md b/test/priv/README.md new file mode 100644 index 0000000000000..eb99937238f82 --- /dev/null +++ b/test/priv/README.md @@ -0,0 +1 @@ +`GeoLite2-City-Test.mmdb` is downloaded from https://github.com/maxmind/MaxMind-DB diff --git a/test/support/geo_stub.ex b/test/support/geo_stub.ex deleted file mode 100644 index 5d81226a719bb..0000000000000 --- a/test/support/geo_stub.ex +++ /dev/null @@ -1,49 +0,0 @@ -defmodule Plausible.Geo.Stub do - @moduledoc false - @behaviour Plausible.Geo.Adapter - - sample_lookup = %{ - "city" => %{"geoname_id" => 2_988_507, "names" => %{"en" => "Paris"}}, - "continent" => %{"code" => "EU", "geoname_id" => 6_255_148, "names" => %{"en" => "Europe"}}, - "country" => %{ - "geoname_id" => 3_017_382, - "is_in_european_union" => true, - "iso_code" => "FR", - "names" => %{"en" => "France"} - }, - "location" => %{ - "latitude" => 48.8566, - "longitude" => 2.35222, - "time_zone" => "Europe/Paris", - "weather_code" => "FRXX0076" - }, - "postal" => %{"code" => "75000"}, - "subdivisions" => [ - %{"geoname_id" => 3_012_874, "iso_code" => "IDF", "names" => %{"en" => "Île-de-France"}}, - %{"geoname_id" => 2_968_815, "iso_code" => "75", "names" => %{"en" => "Paris"}} - ] - } - - @lut %{ - {1, 1, 1, 1} => %{"country" => %{"iso_code" => "US"}}, - {2, 2, 2, 2} => sample_lookup, - {1, 1, 1, 1, 1, 1, 1, 1} => %{"country" => %{"iso_code" => "US"}}, - {0, 0, 0, 0} => %{"country" => %{"iso_code" => "ZZ"}} - } - - @impl true - def lookup(ip_address) when is_tuple(ip_address) do - Map.get(@lut, ip_address) - end - - def lookup(ip_address) when is_binary(ip_address) do - {:ok, ip_address} = :inet.parse_address(to_charlist(ip_address)) - lookup(ip_address) - end - - @impl true - def load_db(_opts), do: :ok - - @impl true - def database_type, do: "DBIP-City-Lite" -end