Add Locations endpoints

infrastructure_files/download-geolite2.sh

@@ -22,41 +22,95 @@ then
     exit 1
 fi
 
-DATABASE_URL="https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
-SIGNATURE_URL="https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz.sha256"
-
-# Download the database and signature files
-echo "Downloading database file..."
-DATABASE_FILE=$(curl -s -u "$MM_ACCOUNT_ID":"$MM_LICENSE_KEY" -L -O -J "$DATABASE_URL" -w "%{filename_effective}")
-echo "Downloading signature file..."
-SIGNATURE_FILE=$(curl -s -u "$MM_ACCOUNT_ID":"$MM_LICENSE_KEY" -L -O -J "$SIGNATURE_URL" -w "%{filename_effective}")
-
-# Verify the signature
-echo "Verifying signature..."
-if sha256sum -c --status "$SIGNATURE_FILE"; then
-    echo "Signature is valid."
-else
-    echo "Signature is invalid. Aborting."
+if ! command -v sqlite3 &> /dev/null
+then
+    echo "sqlite3 is not installed or not in PATH, please install with your package manager. e.g. sudo apt install sqlite3" > /dev/stderr
     exit 1
 fi
 
-# Unpack the database file
-EXTRACTION_DIR=$(basename "$DATABASE_FILE" .tar.gz)
-echo "Unpacking $DATABASE_FILE..."
-mkdir -p "$EXTRACTION_DIR"
-tar -xzvf "$DATABASE_FILE"
-
-# Create a SHA256 signature file
-MMDB_FILE="GeoLite2-City.mmdb"
-cd "$EXTRACTION_DIR"
-sha256sum "$MMDB_FILE" > "$MMDB_FILE.sha256"
-echo "SHA256 signature created for $MMDB_FILE."
-cd -
-
-# Remove downloaded files
-rm "$DATABASE_FILE" "$SIGNATURE_FILE"
-
-# Done. Print next steps
-echo "Process completed successfully."
-echo "Now you can place $EXTRACTION_DIR/$MMDB_FILE to 'datadir' of management service."
-echo -e "Example:\n\tdocker compose cp $EXTRACTION_DIR/$MMDB_FILE management:/var/lib/netbird/"
+download_geolite_mmdb() {
+  DATABASE_URL="https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
+  SIGNATURE_URL="https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz.sha256"
+
+  # Download the database and signature files
+  echo "Downloading database file..."
+  DATABASE_FILE=$(curl -s -u "$MM_ACCOUNT_ID":"$MM_LICENSE_KEY" -L -O -J "$DATABASE_URL" -w "%{filename_effective}")
+  echo "Downloading signature file..."
+  SIGNATURE_FILE=$(curl -s -u "$MM_ACCOUNT_ID":"$MM_LICENSE_KEY" -L -O -J "$SIGNATURE_URL" -w "%{filename_effective}")
+
+  # Verify the signature
+  echo "Verifying signature..."
+  if sha256sum -c --status "$SIGNATURE_FILE"; then
+      echo "Signature is valid."
+  else
+      echo "Signature is invalid. Aborting."
+      exit 1
+  fi
+
+  # Unpack the database file
+  EXTRACTION_DIR=$(basename "$DATABASE_FILE" .tar.gz)
+  echo "Unpacking $DATABASE_FILE..."
+  mkdir -p "$EXTRACTION_DIR"
+  tar -xzvf "$DATABASE_FILE"
+
+  # Create a SHA256 signature file
+  MMDB_FILE="GeoLite2-City.mmdb"
+  cd "$EXTRACTION_DIR"
+  sha256sum "$MMDB_FILE" > "$MMDB_FILE.sha256"
+  echo "SHA256 signature created for $MMDB_FILE."
+  cd -
+
+  # Remove downloaded files
+  rm "$DATABASE_FILE" "$SIGNATURE_FILE"
+
+  # Done. Print next steps
+  echo "Process completed successfully."
+  echo "Now you can place $EXTRACTION_DIR/$MMDB_FILE to 'datadir' of management service."
+  echo -e "Example:\n\tdocker compose cp $EXTRACTION_DIR/$MMDB_FILE management:/var/lib/netbird/"
+}
+
+
+download_geolite_csv_and_create_sqlite_db() {
+  DATABASE_URL="https://download.maxmind.com/geoip/databases/GeoLite2-City-CSV/download?suffix=zip"
+  SIGNATURE_URL="https://download.maxmind.com/geoip/databases/GeoLite2-City-CSV/download?suffix=zip.sha256"
+
+
+  # Download the database file
+  echo "Downloading database file..."
+  DATABASE_FILE=$(curl -s -u "$MM_ACCOUNT_ID":"$MM_LICENSE_KEY" -L -O -J "$DATABASE_URL" -w "%{filename_effective}")
+  echo "Downloading signature file..."
+  SIGNATURE_FILE=$(curl -s -u "$MM_ACCOUNT_ID":"$MM_LICENSE_KEY" -L -O -J "$SIGNATURE_URL" -w "%{filename_effective}")
+
+  # Verify the signature
+  echo "Verifying signature..."
+  if sha256sum -c --status "$SIGNATURE_FILE"; then
+      echo "Signature is valid."
+  else
+      echo "Signature is invalid. Aborting."
+      exit 1
+  fi
+
+  # Unpack the database file
+  EXTRACTION_DIR=$(basename "$DATABASE_FILE" .zip)
+  DB_NAME="geonames.db"
+
+  unzip "$DATABASE_FILE"
+
+# Create SQLite database and import data from CSV
+sqlite3 "$DB_NAME" <<EOF
+.mode csv
+.import "$EXTRACTION_DIR/GeoLite2-City-Locations-en.csv" geonames
+EOF
+
+
+  # Remove downloaded and extracted files
+  rm -r -r "$EXTRACTION_DIR"
+  rm  "$DATABASE_FILE" "$SIGNATURE_FILE"
+
+  echo "SQLite database '$DB_NAME' created successfully."
+  echo "Now you can place $DB_NAME to 'datadir' of management service."
+  echo -e "Example:\n\tdocker compose cp $DB_NAME management:/var/lib/netbird/"
+}
+
+download_geolite_mmdb
+download_geolite_csv_and_create_sqlite_db

management/cmd/management.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/realip"
 	"github.com/netbirdio/management-integrations/integrations"
+
 	"github.com/netbirdio/netbird/encryption"
 	mgmtProto "github.com/netbirdio/netbird/management/proto"
 	"github.com/netbirdio/netbird/management/server"
@@ -230,7 +231,7 @@ var (
 				UserIDClaim:  config.HttpConfig.AuthUserIDClaim,
 				KeysLocation: config.HttpConfig.AuthKeysLocation,
 			}
-			httpAPIHandler, err := httpapi.APIHandler(accountManager, *jwtValidator, appMetrics, httpAPIAuthCfg)
+			httpAPIHandler, err := httpapi.APIHandler(accountManager, geo, *jwtValidator, appMetrics, httpAPIAuthCfg)
 			if err != nil {
 				return fmt.Errorf("failed creating HTTP API handler: %v", err)
 			}

management/server/geolocation/geolocation.go

@@ -22,6 +22,7 @@ type Geolocation struct {
 	mux                 *sync.RWMutex
 	sha256sum           []byte
 	db                  *maxminddb.Reader
+	locationDB          *SqliteStore
 	stopCh              chan struct{}
 	reloadCheckInterval time.Duration
 }
@@ -43,6 +44,16 @@ type Record struct {
 	} `maxminddb:"country"`
 }
 
+type City struct {
+	GeoNameID int `gorm:"column:geoname_id"`
+	CityName  string
+}
+
+type Country struct {
+	CountryISOCode string `gorm:"column:country_iso_code"`
+	CountryName    string
+}
+
 func NewGeolocation(datadir string) (*Geolocation, error) {
 	mmdbPath := path.Join(datadir, mmdbFileName)
 
@@ -56,11 +67,17 @@ func NewGeolocation(datadir string) (*Geolocation, error) {
 		return nil, err
 	}
 
+	locationDB, err := NewSqliteStore(datadir)
+	if err != nil {
+		return nil, err
+	}
+
 	geo := &Geolocation{
 		mmdbPath:            mmdbPath,
 		mux:                 &sync.RWMutex{},
 		sha256sum:           sha256sum,
 		db:                  db,
+		locationDB:          locationDB,
 		reloadCheckInterval: 60 * time.Second, // TODO: make configurable
 		stopCh:              make(chan struct{}),
 	}
@@ -115,6 +132,38 @@ func (gl *Geolocation) Lookup(ip net.IP) (*Record, error) {
 	return &record, nil
 }
 
+// GetAllCountries retrieves a list of all countries.
+func (gl *Geolocation) GetAllCountries() ([]Country, error) {
+	allCountries, err := gl.locationDB.GetAllCountries()
+	if err != nil {
+		return nil, err
+	}
+
+	countries := make([]Country, 0)
+	for _, country := range allCountries {
+		if country.CountryName != "" {
+			countries = append(countries, country)
+		}
+	}
+	return countries, nil
+}
+
+// GetCitiesByCountry retrieves a list of cities in a specific country based on the country's ISO code.
+func (gl *Geolocation) GetCitiesByCountry(countryISOCode string) ([]City, error) {
+	allCities, err := gl.locationDB.GetCitiesByCountry(countryISOCode)
+	if err != nil {
+		return nil, err
+	}
+
+	cities := make([]City, 0)
+	for _, city := range allCities {
+		if city.CityName != "" {
+			cities = append(cities, city)
+		}
+	}
+	return cities, nil
+}
+
 func (gl *Geolocation) Stop() error {
 	close(gl.stopCh)
 	if gl.db != nil {
@@ -129,6 +178,10 @@ func (gl *Geolocation) reloader() {
 		case <-gl.stopCh:
 			return
 		case <-time.After(gl.reloadCheckInterval):
+			if err := gl.locationDB.reload(); err != nil {
+				log.Errorf("geonames db reload failed: %s", err)
+			}
+
 			newSha256sum1, err := getSha256sum(gl.mmdbPath)
 			if err != nil {
 				log.Errorf("failed to calculate sha256 sum for '%s': %s", gl.mmdbPath, err)
@@ -149,7 +202,7 @@ func (gl *Geolocation) reloader() {
 				}
 				err = gl.reload(newSha256sum2)
 				if err != nil {
-					log.Errorf("reload failed: %s", err)
+					log.Errorf("mmdb reload failed: %s", err)
 				}
 			} else {
 				log.Debugf("No changes in '%s', no need to reload. Next check is in %.0f seconds.",
@@ -182,3 +235,14 @@ func (gl *Geolocation) reload(newSha256sum []byte) error {
 
 	return nil
 }
+
+func fileExists(filePath string) (bool, error) {
+	_, err := os.Stat(filePath)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, fmt.Errorf("%v does not exist", filePath)
+	}
+	return false, err
+}

management/server/geolocation/geolocation_test.go

@@ -4,10 +4,12 @@ import (
 	"net"
 	"os"
 	"path"
+	"sync"
 	"testing"
 
-	"github.com/netbirdio/netbird/util"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/netbirdio/netbird/util"
 )
 
 // from https://github.com/maxmind/MaxMind-DB/blob/main/test-data/GeoLite2-City-Test.mmdb
@@ -25,8 +27,14 @@ func TestGeoLite_Lookup(t *testing.T) {
 		}
 	}()
 
-	geo, err := NewGeolocation(tempDir)
+	db, err := openDB(mmdbPath)
 	assert.NoError(t, err)
+
+	geo := &Geolocation{
+		mux:    &sync.RWMutex{},
+		db:     db,
+		stopCh: make(chan struct{}),
+	}
 	assert.NotNil(t, geo)
 	defer func() {
 		err = geo.Stop()