Add ASN breakdown to outline-ss-server

README.md

@@ -1,13 +1,14 @@
 # Outline ss-server
 
-![Build Status](https://github.com/Jigsaw-Code/outline-ss-server/actions/workflows/go.yml/badge.svg)
+[![Build Status](https://github.com/Jigsaw-Code/outline-ss-server/actions/workflows/go.yml/badge.svg)](https://github.com/Jigsaw-Code/outline-ss-server/actions/workflows/go.yml?query=branch%3Amaster)
+
 [![Go Report Card](https://goreportcard.com/badge/github.com/Jigsaw-Code/outline-ss-server)](https://goreportcard.com/report/github.com/Jigsaw-Code/outline-ss-server)
 [![Go Reference](https://pkg.go.dev/badge/github.com/Jigsaw-Code/outline-ss-server.svg)](https://pkg.go.dev/github.com/Jigsaw-Code/outline-ss-server)
 
 [![Mattermost](https://badgen.net/badge/Mattermost/Outline%20Community/blue)](https://community.internetfreedomfestival.org/community/channels/outline-community)
 [![Reddit](https://badgen.net/badge/Reddit/r%2Foutlinevpn/orange)](https://www.reddit.com/r/outlinevpn/)
 
-This repository has the Shadowsocks service used by Outline servers. It was inspired by [go-shadowsocks2](https://github.com/shadowsocks/go-shadowsocks2), and adds a number of improvements to meet the needs of the Outline users.
+This repository has the Shadowsocks backend used by the [Outline Server](https://github.com/Jigsaw-Code/outline-server).
 
 The Outline Shadowsocks service allows for:
 - Multiple users on a single port.
@@ -20,8 +21,26 @@ The Outline Shadowsocks service allows for:
 
 ![Graphana Dashboard](https://user-images.githubusercontent.com/113565/44177062-419d7700-a0ba-11e8-9621-db519692ff6c.png "Graphana Dashboard")
 
+## How to run it
+
+Call the `outline-ss-server` command with the recommended flags, [as done by the official Outline Server](https://github.com/Jigsaw-Code/outline-server/blob/b2639d09c30a50479eddcd33b84432f57081be0c/src/shadowbox/server/outline_shadowsocks_server.ts#L91-L100):
+```
+outline-ss-server -replay_history=10000 -metrics=127.0.0.1:9091 -config=$CONFIG_YML  -ip_country_db=$COUNTRY_MMDB -ip_asn_db=$ASN_MMDB
+```
+
+Flags:
+- `replay_history`: Enables replay protection for the last 10000 connections.
+- `metrics`: Where the webserver exposing the Prometheus metrics will listen on. You should specify localhost so it's not accessible from outside the machine, unless you know what you are doing.
+- `config`: The config file with the access keys. See the config example.
+- `ip_country_db`: The IP-Country MMDB file to enable per-country metrics breakdown.
+- `ip_asn_db`: The IP-ASN MMDB file to enable per-country metrics breakdown.
+
+In the example, you can open https://127.0.0.1:9091 on your browser to see the exported Prometheus metrics.
+
+To fetch and update MMDB files from [DB-IP](https://db-ip.com), you can do something like the [update_mmdb.sh from the Outline Server](https://github.com/Jigsaw-Code/outline-server/blob/master/src/shadowbox/scripts/update_mmdb.sh).
+
 
-## Try it!
+## Full Working Example: Try It!
 
 Fetch dependencies for this demo:
 ```
@@ -47,7 +66,7 @@ $(go env GOPATH)/bin/prometheus --config.file=cmd/outline-ss-server/prometheus_e
 ### Run the SOCKS-to-Shadowsocks client
 On Terminal 3, start the SS client:
 ```
-go run github.com/shadowsocks/go-shadowsocks2 -c ss://chacha20-ietf-poly1305:Secret0@:9000 -verbose  -socks localhost:1080
+go run github.com/shadowsocks/go-shadowsocks2@latest -c ss://chacha20-ietf-poly1305:Secret0@:9000 -verbose  -socks localhost:1080
 ```
 
 ### Fetch a page over Shadowsocks
@@ -78,7 +97,7 @@ go run ./cmd/outline-ss-server -config cmd/outline-ss-server/config_example.yml
 
 Start the SS tunnel to redirect port 8000 -> localhost:5201 via the proxy on 9000:
 ```
-go run github.com/shadowsocks/go-shadowsocks2 -c ss://chacha20-ietf-poly1305:Secret0@:9000 -tcptun ":8000=localhost:5201" -udptun ":8000=localhost:5201" -verbose
+go run github.com/shadowsocks/go-shadowsocks2@latest -c ss://chacha20-ietf-poly1305:Secret0@:9000 -tcptun ":8000=localhost:5201" -udptun ":8000=localhost:5201" -verbose
 ```
 
 Test TCP upload (client -> server):

cmd/outline-ss-server/main.go

@@ -229,6 +229,7 @@ func main() {
 		ConfigFile    string
 		MetricsAddr   string
 		IPCountryDB   string
+		IPASNDB       string
 		natTimeout    time.Duration
 		replayHistory int
 		Verbose       bool
@@ -237,6 +238,7 @@ func main() {
 	flag.StringVar(&flags.ConfigFile, "config", "", "Configuration filename")
 	flag.StringVar(&flags.MetricsAddr, "metrics", "", "Address for the Prometheus metrics")
 	flag.StringVar(&flags.IPCountryDB, "ip_country_db", "", "Path to the ip-to-country mmdb file")
+	flag.StringVar(&flags.IPASNDB, "ip_asn_db", "", "Path to the ip-to-ASN mmdb file")
 	flag.DurationVar(&flags.natTimeout, "udptimeout", defaultNatTimeout, "UDP tunnel timeout")
 	flag.IntVar(&flags.replayHistory, "replay_history", 0, "Replay buffer size (# of handshakes)")
 	flag.BoolVar(&flags.Verbose, "verbose", false, "Enables verbose logging output")
@@ -268,16 +270,18 @@ func main() {
 		logger.Infof("Prometheus metrics available at http://%v/metrics", flags.MetricsAddr)
 	}
 
-	var ip2info *ipinfo.MMDBIPInfoMap
 	var err error
 	if flags.IPCountryDB != "" {
 		logger.Infof("Using IP-Country database at %v", flags.IPCountryDB)
-		ip2info, err := ipinfo.NewMMDBIPInfoMap(flags.IPCountryDB, "")
-		if err != nil {
-			logger.Fatalf("Could not open geoip database at %v: %v. Aborting", flags.IPCountryDB, err)
-		}
-		defer ip2info.Close()
 	}
+	if flags.IPASNDB != "" {
+		logger.Infof("Using IP-ASN database at %v", flags.IPASNDB)
+	}
+	ip2info, err := ipinfo.NewMMDBIPInfoMap(flags.IPCountryDB, flags.IPASNDB)
+	if err != nil {
+		logger.Fatalf("Could create IP info map: %v. Aborting", err)
+	}
+	defer ip2info.Close()
 
 	m := newPrometheusOutlineMetrics(ip2info, prometheus.DefaultRegisterer)
 	m.SetBuildInfo(version)

cmd/outline-ss-server/metrics.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"fmt"
 	"strconv"
 	"time"
 
@@ -81,13 +82,13 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus
 			Subsystem: "tcp",
 			Name:      "connections_opened",
 			Help:      "Count of open TCP connections",
-		}, []string{"location"}),
+		}, []string{"location", "asn"}),
 		tcpClosedConnections: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Namespace: "shadowsocks",
 			Subsystem: "tcp",
 			Name:      "connections_closed",
 			Help:      "Count of closed TCP connections",
-		}, []string{"location", "status", "access_key"}),
+		}, []string{"location", "asn", "status", "access_key"}),
 		tcpConnectionDurationMs: prometheus.NewHistogramVec(
 			prometheus.HistogramOpts{
 				Namespace: "shadowsocks",
@@ -114,7 +115,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus
 				Namespace: "shadowsocks",
 				Name:      "data_bytes_per_location",
 				Help:      "Bytes transferred by the proxy, per location",
-			}, []string{"dir", "proto", "location"}),
+			}, []string{"dir", "proto", "location", "asn"}),
 		timeToCipherMs: prometheus.NewHistogramVec(
 			prometheus.HistogramOpts{
 				Namespace: "shadowsocks",
@@ -128,7 +129,7 @@ func newPrometheusOutlineMetrics(ip2info ipinfo.IPInfoMap, registerer prometheus
 				Subsystem: "udp",
 				Name:      "packets_from_client_per_location",
 				Help:      "Packets received from the client, per location and status",
-			}, []string{"location", "status"}),
+			}, []string{"location", "asn", "status"}),
 		udpAddedNatEntries: prometheus.NewCounter(
 			prometheus.CounterOpts{
 				Namespace: "shadowsocks",
@@ -161,7 +162,7 @@ func (m *outlineMetrics) SetNumAccessKeys(numKeys int, ports int) {
 }
 
 func (m *outlineMetrics) AddOpenTCPConnection(clientInfo ipinfo.IPInfo) {
-	m.tcpOpenConnections.WithLabelValues(clientInfo.CountryCode.String()).Inc()
+	m.tcpOpenConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN)).Inc()
 }
 
 // addIfNonZero helps avoid the creation of series that are always zero.
@@ -171,32 +172,39 @@ func addIfNonZero(value int64, counterVec *prometheus.CounterVec, lvs ...string)
 	}
 }
 
+func asnLabel(asn int) string {
+	if asn == 0 {
+		return ""
+	}
+	return fmt.Sprint(asn)
+}
+
 func (m *outlineMetrics) AddClosedTCPConnection(clientInfo ipinfo.IPInfo, accessKey, status string, data metrics.ProxyMetrics, duration time.Duration) {
-	m.tcpClosedConnections.WithLabelValues(clientInfo.CountryCode.String(), status, accessKey).Inc()
+	m.tcpClosedConnections.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status, accessKey).Inc()
 	m.tcpConnectionDurationMs.WithLabelValues(status).Observe(duration.Seconds() * 1000)
 	addIfNonZero(data.ClientProxy, m.dataBytes, "c>p", "tcp", accessKey)
-	addIfNonZero(data.ClientProxy, m.dataBytesPerLocation, "c>p", "tcp", clientInfo.CountryCode.String())
+	addIfNonZero(data.ClientProxy, m.dataBytesPerLocation, "c>p", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 	addIfNonZero(data.ProxyTarget, m.dataBytes, "p>t", "tcp", accessKey)
-	addIfNonZero(data.ProxyTarget, m.dataBytesPerLocation, "p>t", "tcp", clientInfo.CountryCode.String())
+	addIfNonZero(data.ProxyTarget, m.dataBytesPerLocation, "p>t", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 	addIfNonZero(data.TargetProxy, m.dataBytes, "p<t", "tcp", accessKey)
-	addIfNonZero(data.TargetProxy, m.dataBytesPerLocation, "p<t", "tcp", clientInfo.CountryCode.String())
+	addIfNonZero(data.TargetProxy, m.dataBytesPerLocation, "p<t", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 	addIfNonZero(data.ProxyClient, m.dataBytes, "c<p", "tcp", accessKey)
-	addIfNonZero(data.ProxyClient, m.dataBytesPerLocation, "c<p", "tcp", clientInfo.CountryCode.String())
+	addIfNonZero(data.ProxyClient, m.dataBytesPerLocation, "c<p", "tcp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 }
 
 func (m *outlineMetrics) AddUDPPacketFromClient(clientInfo ipinfo.IPInfo, accessKey, status string, clientProxyBytes, proxyTargetBytes int) {
-	m.udpPacketsFromClientPerLocation.WithLabelValues(clientInfo.CountryCode.String(), status).Inc()
+	m.udpPacketsFromClientPerLocation.WithLabelValues(clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN), status).Inc()
 	addIfNonZero(int64(clientProxyBytes), m.dataBytes, "c>p", "udp", accessKey)
-	addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String())
+	addIfNonZero(int64(clientProxyBytes), m.dataBytesPerLocation, "c>p", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 	addIfNonZero(int64(proxyTargetBytes), m.dataBytes, "p>t", "udp", accessKey)
-	addIfNonZero(int64(proxyTargetBytes), m.dataBytesPerLocation, "p>t", "udp", clientInfo.CountryCode.String())
+	addIfNonZero(int64(proxyTargetBytes), m.dataBytesPerLocation, "p>t", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 }
 
 func (m *outlineMetrics) AddUDPPacketFromTarget(clientInfo ipinfo.IPInfo, accessKey, status string, targetProxyBytes, proxyClientBytes int) {
 	addIfNonZero(int64(targetProxyBytes), m.dataBytes, "p<t", "udp", accessKey)
-	addIfNonZero(int64(targetProxyBytes), m.dataBytesPerLocation, "p<t", "udp", clientInfo.CountryCode.String())
+	addIfNonZero(int64(targetProxyBytes), m.dataBytesPerLocation, "p<t", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 	addIfNonZero(int64(proxyClientBytes), m.dataBytes, "c<p", "udp", accessKey)
-	addIfNonZero(int64(proxyClientBytes), m.dataBytesPerLocation, "c<p", "udp", clientInfo.CountryCode.String())
+	addIfNonZero(int64(proxyClientBytes), m.dataBytesPerLocation, "c<p", "udp", clientInfo.CountryCode.String(), asnLabel(clientInfo.ASN))
 }
 
 func (m *outlineMetrics) AddUDPNatEntry() {

cmd/outline-ss-server/metrics_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/Jigsaw-Code/outline-ss-server/ipinfo"
 	"github.com/Jigsaw-Code/outline-ss-server/service/metrics"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
 )
 
 func TestMethodsDontPanic(t *testing.T) {
@@ -33,28 +34,33 @@ func TestMethodsDontPanic(t *testing.T) {
 	}
 	ssMetrics.SetBuildInfo("0.0.0-test")
 	ssMetrics.SetNumAccessKeys(20, 2)
-	ssMetrics.AddOpenTCPConnection(ipinfo.IPInfo{CountryCode: "US"})
-	ssMetrics.AddClosedTCPConnection(ipinfo.IPInfo{CountryCode: "US"}, "1", "OK", proxyMetrics, 10*time.Millisecond)
-	ssMetrics.AddUDPPacketFromClient(ipinfo.IPInfo{CountryCode: "US"}, "2", "OK", 10, 20)
-	ssMetrics.AddUDPPacketFromTarget(ipinfo.IPInfo{CountryCode: "US"}, "3", "OK", 10, 20)
+	ssMetrics.AddOpenTCPConnection(ipinfo.IPInfo{CountryCode: "US", ASN: 100})
+	ssMetrics.AddClosedTCPConnection(ipinfo.IPInfo{CountryCode: "US", ASN: 100}, "1", "OK", proxyMetrics, 10*time.Millisecond)
+	ssMetrics.AddUDPPacketFromClient(ipinfo.IPInfo{CountryCode: "US", ASN: 100}, "2", "OK", 10, 20)
+	ssMetrics.AddUDPPacketFromTarget(ipinfo.IPInfo{CountryCode: "US", ASN: 100}, "3", "OK", 10, 20)
 	ssMetrics.AddUDPNatEntry()
 	ssMetrics.RemoveUDPNatEntry()
 	ssMetrics.AddTCPProbe("ERR_CIPHER", "eof", 443, proxyMetrics.ClientProxy)
 	ssMetrics.AddTCPCipherSearch(true, 10*time.Millisecond)
 	ssMetrics.AddUDPCipherSearch(true, 10*time.Millisecond)
 }
 
+func TestASNLabel(t *testing.T) {
+	require.Equal(t, "", asnLabel(0))
+	require.Equal(t, "100", asnLabel(100))
+}
+
 func BenchmarkOpenTCP(b *testing.B) {
 	ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry())
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		ssMetrics.AddOpenTCPConnection(ipinfo.IPInfo{CountryCode: "ZZ"})
+		ssMetrics.AddOpenTCPConnection(ipinfo.IPInfo{CountryCode: "ZZ", ASN: 100})
 	}
 }
 
 func BenchmarkCloseTCP(b *testing.B) {
 	ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry())
-	clientInfo := ipinfo.IPInfo{CountryCode: "ZZ"}
+	clientInfo := ipinfo.IPInfo{CountryCode: "ZZ", ASN: 100}
 	accessKey := "key 1"
 	status := "OK"
 	data := metrics.ProxyMetrics{}
@@ -81,7 +87,7 @@ func BenchmarkProbe(b *testing.B) {
 
 func BenchmarkClientUDP(b *testing.B) {
 	ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry())
-	clientInfo := ipinfo.IPInfo{CountryCode: "ZZ"}
+	clientInfo := ipinfo.IPInfo{CountryCode: "ZZ", ASN: 100}
 	accessKey := "key 1"
 	status := "OK"
 	size := 1000
@@ -95,7 +101,7 @@ func BenchmarkClientUDP(b *testing.B) {
 
 func BenchmarkTargetUDP(b *testing.B) {
 	ssMetrics := newPrometheusOutlineMetrics(nil, prometheus.NewRegistry())
-	clientInfo := ipinfo.IPInfo{CountryCode: "ZZ"}
+	clientInfo := ipinfo.IPInfo{CountryCode: "ZZ", ASN: 100}
 	accessKey := "key 1"
 	status := "OK"
 	size := 1000