Use GitHub Actions to update datasets

.github/workflows/build.yml

@@ -0,0 +1,32 @@
+name: Build Dataset
+
+on:
+  push:
+    branches:
+      - master
+
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  build-ipasn:
+    name: Build IPASN
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+
+      - name: Download datasource
+        run: |
+          curl https://iptoasn.com/data/ip2asn-combined.tsv.gz | gunzip -c | tee data/ip2asn.tsv
+
+      - name: Build dataset
+        run: |
+          go run cmd/ipasn/ipasn.go
+
+      - uses: eine/tip@master
+        with:
+          tag: "nightly"
+          token: ${{ github.token }}
+          files: |
+            ./data/ipasn.mmdb

README.md

@@ -1,14 +1,24 @@
 # OpenData
 
-### Datacenter ASN list
-[datacenter.tsv](datacenter.tsv)
+## Datasets
 
-A TAB-separated list of ASNs known to belong to cloud, managed hosting, and colo facilities.
+### IPASN
 
-**Format:**
-`as_number` `as_description`
+IP-to-ASN map.
 
-### License
+#### MMDB
 
-[CC BY-SA 4.0](LICENSE)
+[ipasn.mmdb](releases/latest/download/ipasn.mmdb)
 
+**Fields:**
+`asn` `name`
+
+#### Sources
+
+- [IPtoASN](https://iptoasn.com/)
+
+## License
+
+Codes are licensed under [The MIT License](LICENSE).
+
+Datasets are licensed under [CC BY-SA 4.0](LICENSE-CC).

cmd/ipasn/ipasn.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2021 JoyMoe Interactive Entertainment Limited
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"encoding/csv"
+	"io"
+	"log"
+	"net"
+	"os"
+	"strconv"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+)
+
+func main() {
+	writer, err := mmdbwriter.New(mmdbwriter.Options{
+		RecordSize: 24,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	f, err := os.Open("data/ip2asn.tsv")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	r := csv.NewReader(f)
+	r.Comma = '\t'
+	r.LazyQuotes = true
+
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		start := net.ParseIP(record[0])
+		end := net.ParseIP(record[1])
+		asn, err := strconv.ParseUint(record[2], 10, 32)
+
+		if start == nil || end == nil || err != nil {
+			log.Printf("Invalid IP range: %s - %s %s", record[0], record[1], record[2])
+			continue
+		}
+
+		if asn == 0 {
+			continue
+		}
+
+		name := record[4]
+
+		if err := writer.InsertRange(start, end, mmdbtype.Map{
+			"asn":  mmdbtype.Uint32(asn),
+			"name": mmdbtype.String(name),
+		}); err != nil {
+			log.Printf("%s - %s %s", record[0], record[1], record[2])
+			log.Println(err)
+			continue
+		}
+	}
+
+	fh, err := os.Create("data/ipasn.mmdb")
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = writer.WriteTo(fh)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore