From a623e72e7e995a321493e4459d0515322034684a Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chadwhitacre@sentry.io>
Date: Mon, 14 Dec 2020 11:56:04 -0500
Subject: [PATCH] Integrate with MaxMind out of the box (#766)

Integrate with MaxMind out of the box
---
 .gitignore                     |   3 +++
 docker-compose.yml             |  13 +++++++++++
 geoip/GeoLite2-City.mmdb.empty | Bin 0 -> 1055 bytes
 install.sh                     |   4 ++++
 install/geoip.sh               |  39 +++++++++++++++++++++++++++++++++
 nginx/nginx.conf               |   2 +-
 relay/config.example.yml       |   1 +
 sentry/sentry.conf.example.py  |   8 ++++++-
 8 files changed, 68 insertions(+), 2 deletions(-)
 create mode 100644 geoip/GeoLite2-City.mmdb.empty
 create mode 100755 install/geoip.sh

diff --git a/.gitignore b/.gitignore
index b8ee807d52..707622f425 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,3 +81,6 @@ sentry/requirements.txt
 relay/credentials.json
 relay/config.yml
 symbolicator/config.yml
+geoip/GeoIP.conf
+geoip/*.mmdb
+geoip/.geoipupdate.lock
diff --git a/docker-compose.yml b/docker-compose.yml
index 5909226ae4..160ae6294c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -31,6 +31,7 @@ x-sentry-defaults: &sentry_defaults
   volumes:
     - 'sentry-data:/data'
     - './sentry:/etc/sentry'
+    - './geoip:/geoip:ro'
 x-snuba-defaults: &snuba_defaults
   << : *restart_policy
   depends_on:
@@ -126,6 +127,14 @@ services:
       # If you have high volume and your search return incomplete results
       # You might want to change this to a higher value (and ensure your host has enough memory)
       MAX_MEMORY_USAGE_RATIO: 0.3
+  geoipupdate:
+    image: 'maxmindinc/geoipupdate:latest'
+    # Override the entrypoint in order to avoid using envvars for config.
+    # Futz with settings so we can keep mmdb and conf in same dir on host
+    # (image looks for them in separate dirs by default).
+    entrypoint: ['/usr/bin/geoipupdate', '-d', '/sentry', '-f', '/sentry/GeoIP.conf']
+    volumes:
+      - './geoip:/sentry'
   snuba-api:
     << : *snuba_defaults
   # Kafka consumer responsible for feeding events into Clickhouse
@@ -233,6 +242,10 @@ services:
         read_only: true
         source: ./relay
         target: /work/.relay
+      - type: bind
+        read_only: true
+        source: ./geoip
+        target: /geoip
     depends_on:
       - kafka
       - redis
diff --git a/geoip/GeoLite2-City.mmdb.empty b/geoip/GeoLite2-City.mmdb.empty
new file mode 100644
index 0000000000000000000000000000000000000000..94f6921fd1c885fac56dcb689bf4467b25d9d78d
GIT binary patch
literal 1055
zcmZ9`*;dm)0LJmpS~uKxRMfh!RK<NoK}A}kqPQZ)NiuB*(n(BGB)G4<hFtp;K7cpA
zftOzS3dS#I>d~H_-}(2OHXUFNupTG?i-Ebo3{a%=fQ7((_5zw6W$#6%J4|Hb5@0E?
z8dzq=<-iK|O1jGI`9s$5<664T9CZQ**&Be3X50jH0lR_CX4%5nN=tMb-A;GVopcw?
z{&5duFWpD?(*rcWQaA7kD04Xk#K2)-3^>B2haLs)0>^-Bz;P}o=t+8to~CE$S$dA1
zrx$21y+|(s13(|UpI)X{=vA8EFn7=_6-E^pVp#M#y+O0@-((B}w-~qSh&jDu;+~27
zzys3{fl>A&v*Z_e4A@+rP(dA_#&7|TA!+;Ob)dmTnSGow0r+NVGBgk{LbE>wBGcIe
z6VolmB=C$eMN>LWpVJrgC4EI-)9g`i7;ou2;6395{YcyQ&*I<Lmv6r;F=Z+3mEEBE
zbJS5<M5$d1qNYgfrWg+*ds0TR3iNE@|5K&^o_?-6E#-T*3<I}OEZE<NyDU#6!V$5w
zlQfjpfDDFIB73^~RgzXc8M~1RlkCAi`lKG~7-^`uWPVXnI+6O%q452*<jY!8a(tmD
p$`uv1FH{`Nrg}@%WsG7)2cERuprw<qGu24CLFC!7nwGPjzX7A6#-acK

literal 0
HcmV?d00001

diff --git a/install.sh b/install.sh
index 2ec1832167..44c4f3e4f4 100755
--- a/install.sh
+++ b/install.sh
@@ -325,6 +325,10 @@ if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then
   echo "Relay credentials written to $RELAY_CREDENTIALS_JSON"
 fi
 
+
+./install/geoip.sh
+
+
 if [[ "$MINIMIZE_DOWNTIME" ]]; then
   # Start the whole setup, except nginx and relay.
   $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$')
diff --git a/install/geoip.sh b/install/geoip.sh
new file mode 100755
index 0000000000..ec37b21660
--- /dev/null
+++ b/install/geoip.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi
+
+dc="docker-compose --no-ansi"
+dcr="$dc run --rm"
+
+
+install_geoip() {
+  local mmdb='geoip/GeoLite2-City.mmdb'
+  local conf='geoip/GeoIP.conf'
+  local result='Done'
+
+  echo "Setting up IP address geolocation ..."
+  if [[ ! -f "$mmdb" ]]; then
+    echo -n "Installing (empty) IP address geolocation database ... "
+    cp "$mmdb.empty" "$mmdb"
+    echo "done."
+  else
+    echo "IP address geolocation database already exists."
+  fi
+
+  if [[ ! -f "$conf" ]]; then
+    echo "IP address geolocation is not configured for updates."
+    echo "See https://develop.sentry.dev/self-hosted/geolocation/ for instructions."
+    result='Error'
+  else
+    echo "IP address geolocation is configured for updates."
+    echo "Updating IP address geolocation database ... "
+    $dcr geoipupdate
+    if [ $? -gt 0 ]; then
+      result='Error'
+    fi
+    echo "$result updating IP address geolocation database."
+  fi
+  echo "$result setting up IP address geolocation."
+}
+
+install_geoip
diff --git a/nginx/nginx.conf b/nginx/nginx.conf
index bd81a5e8df..e8d56443af 100644
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@@ -45,7 +45,7 @@ http {
 	proxy_set_header Connection '';
 	proxy_set_header Host $host;
 	proxy_set_header X-Real-IP $remote_addr;
-	proxy_set_header X-Forwarded-For $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 	proxy_set_header X-Forwarded-Proto $scheme;
 	proxy_set_header X-Request-Id $request_id;
 	proxy_read_timeout 30s;
diff --git a/relay/config.example.yml b/relay/config.example.yml
index 0488ba91a4..8538bd7d46 100644
--- a/relay/config.example.yml
+++ b/relay/config.example.yml
@@ -10,3 +10,4 @@ processing:
     - {name: "bootstrap.servers", value: "kafka:9092"}
     - {name: "message.max.bytes", value: 50000000} #50MB or bust
   redis: redis://redis:6379
+  geoip_path: "/geoip/GeoLite2-City.mmdb"
diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py
index ca217d117a..2d89b5677f 100644
--- a/sentry/sentry.conf.example.py
+++ b/sentry/sentry.conf.example.py
@@ -260,9 +260,15 @@ def get_internal_network():
     }
 )
 
+#######################
+# MaxMind Integration #
+#######################
+
+GEOIP_PATH_MMDB = '/geoip/GeoLite2-City.mmdb'
+
 #########################
 # Bitbucket Integration #
-########################
+#########################
 
 # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY'
 # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET'