diff --git a/.github/workflows/check-regions.yml b/.github/workflows/check-regions.yml new file mode 100644 index 0000000..e47c822 --- /dev/null +++ b/.github/workflows/check-regions.yml @@ -0,0 +1,17 @@ +name: Check provider regions + +on: + schedule: + - cron: '5 4 * * 3' + +jobs: + region-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.19 + - name: region-check + run: go run cmd/regioncheck/regioncheck.go \ No newline at end of file diff --git a/Makefile b/Makefile index e92f305..48cbaa4 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ dev: docker-image: docker build -t $(DOCKER_IMAGE):$(VERSION) -f packaging/docker/Dockerfile . +lint: + docker run -t --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.53.3 golangci-lint run -v + test: go test ./... diff --git a/cmd/regioncheck/regioncheck.go b/cmd/regioncheck/regioncheck.go new file mode 100644 index 0000000..bc6310b --- /dev/null +++ b/cmd/regioncheck/regioncheck.go @@ -0,0 +1,194 @@ +package main + +import ( + "fmt" + "github.com/PuerkitoBio/goquery" + "github.com/sa7mon/s3scanner/collection" + "github.com/sa7mon/s3scanner/provider" + "log" + "net" + "net/http" + "os" + "regexp" + "sort" + "strings" + "sync" + "time" +) + +// eq compares sorted string slices. Once we move to Golang 1.21, use slices.Equal instead. +func eq(f []string, s []string) bool { + if len(f) != len(s) { + return false + } + for i := range f { + if f[i] != s[i] { + return false + } + } + return true +} + +// GetRegionsDO fetches regions from the DigitalOcean docs HTML page. +func GetRegionsDO() ([]string, error) { + requestURL := "https://docs.digitalocean.com/products/platform/availability-matrix/#other-product-availability" + res, err := http.Get(requestURL) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status) + } + + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + return nil, err + } + + regions := []string{} + doc.Find("h2#other-product-availability + table thead tr th").Each(func(i int, t *goquery.Selection) { + regions = append(regions, t.Text()) + }) + + spaces_supported := []bool{} + doc.Find("h2#other-product-availability + table tbody tr").Each(func(i int, t *goquery.Selection) { + // For each row, check the first cell for a value of "Spaces" + rowHeader := t.Find("td").First().Text() + if rowHeader == "Spaces" { + // For each cell in the "Spaces" row, check if the contents are not empty - meaning Spaces is supported + t.Find("td").Each(func(j int, v *goquery.Selection) { + supported := v.Text() != "" + spaces_supported = append(spaces_supported, supported) + }) + } + }) + + supported_regions := []string{} + for i := 0; i < len(regions); i++ { + if regions[i] == "Product" { + continue + } + if spaces_supported[i] { + supported_regions = append(supported_regions, strings.ToLower(regions[i])) + } + } + + // Return slice of region names + return supported_regions, nil +} + +// GetRegionsLinode fetches region names from Linode docs HTML page. Linode also provides this info via +// unauthenticated API (https://api.linode.com/v4/regions) but the region names do not include the trailing digit "-1". +func GetRegionsLinode() ([]string, error) { + requestURL := "https://www.linode.com/docs/products/storage/object-storage/" + res, err := http.Get(requestURL) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status) + } + + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + return nil, err + } + + regions := []string{} + doc.Find("h2#availability + p + table tbody tr td:nth-of-type(2)").Each(func(i int, t *goquery.Selection) { + regions = append(regions, t.Text()) + }) + + return regions, nil +} + +// GetRegionsDreamhost fetches subdomains of dream.io like 'objects-us-east-1.dream.io' via crt.sh since Dreamhost +// doesn't have a documentation page listing the regions. +func GetRegionsDreamhost() ([]string, error) { + var domainRe = regexp.MustCompile(`objects-([^\.]+)\.dream\.io`) + requestURL := "https://crt.sh/?q=.dream.io" + // Request the HTML page. + res, err := http.Get(requestURL) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status) + } + + // Load the HTML document + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + return nil, err + } + + certCNs := collection.StringSet{} + // For each cell in the Common Name column + doc.Find("body > table table tbody tr > td:nth-of-type(5)").Each(func(i int, t *goquery.Selection) { + matches := domainRe.FindAllStringSubmatch(t.Text(), -1) + for _, match := range matches { + if !strings.HasPrefix(match[1], "website-") { // regions like 'objects-website-us-east-1' are not for Object Storage + certCNs.Add(match[1]) + } + } + }) + + // crt.sh may return old or invalid SSL certs + // verify regions found resolve and respond on port 443 + resolvableRegions := []string{} + for _, s := range certCNs.Slice() { + timeout := 1 * time.Second + _, cerr := net.DialTimeout("tcp", fmt.Sprintf("objects-%s.dream.io:443", s), timeout) + if cerr == nil { + resolvableRegions = append(resolvableRegions, s) + } + + } + + return resolvableRegions, nil +} + +func main() { + wg := sync.WaitGroup{} + wg.Add(3) + + results := map[string][]string{} + errors := map[string]error{} + + go func(w *sync.WaitGroup) { + results["digitalocean"], errors["digitalocean"] = GetRegionsDO() + wg.Done() + }(&wg) + go func(w *sync.WaitGroup) { + results["dreamhost"], errors["dreamhost"] = GetRegionsDreamhost() + wg.Done() + }(&wg) + go func(w *sync.WaitGroup) { + results["linode"], errors["linode"] = GetRegionsLinode() + wg.Done() + }(&wg) + wg.Wait() + + exit := 0 + + for p, knownRegions := range provider.ProviderRegions { + if errors[p] != nil { + log.Printf("[%s]: %v\n", p, errors[p]) + continue + } + foundRegions := results[p] + sort.Strings(foundRegions) + sort.Strings(knownRegions) + + if !eq(foundRegions, knownRegions) { + log.Printf("[%s] regions differ! Existing: %v, found: %v", p, knownRegions, foundRegions) + exit = 1 + } else { + log.Printf("[%s} OK", p) + } + } + os.Exit(exit) +} diff --git a/cmd/regioncheck/regioncheck_test.go b/cmd/regioncheck/regioncheck_test.go new file mode 100644 index 0000000..3567320 --- /dev/null +++ b/cmd/regioncheck/regioncheck_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetRegionsDO(t *testing.T) { + r, err := GetRegionsDO() + assert.Nil(t, err) + assert.GreaterOrEqual(t, len(r), 1) + assert.Contains(t, r, "nyc3") +} + +func TestGetRegionsLinode(t *testing.T) { + r, err := GetRegionsLinode() + assert.Nil(t, err) + assert.GreaterOrEqual(t, len(r), 1) + assert.Contains(t, r, "us-east-1") +} + +func TestGetRegionsDreamhost(t *testing.T) { + dor, err := GetRegionsDreamhost() + assert.Nil(t, err) + assert.GreaterOrEqual(t, len(dor), 1) + assert.Contains(t, dor, "us-east-1") +} diff --git a/collection/collection.go b/collection/collection.go new file mode 100644 index 0000000..1ec43bd --- /dev/null +++ b/collection/collection.go @@ -0,0 +1,28 @@ +package collection + +// StringSet is a simple implementeation of the Set data structure. +// An empty struct allegedly takes zero memory +type StringSet map[string]struct{} + +func (ss StringSet) Add(s string) { + ss[s] = struct{}{} +} + +func (ss StringSet) Remove(s string) { + delete(ss, s) +} + +func (ss StringSet) Has(s string) bool { + _, ok := ss[s] + return ok +} + +func (ss StringSet) Slice() []string { + slice := make([]string, len(ss)) + i := 0 + for s := range ss { + slice[i] = s + i++ + } + return slice +} diff --git a/go.mod b/go.mod index 6f8abb0..90c86cc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/sa7mon/s3scanner go 1.19 require ( + github.com/PuerkitoBio/goquery v1.8.1 github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.39 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.83 @@ -18,6 +19,7 @@ require ( ) require ( + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.37 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect @@ -52,6 +54,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect golang.org/x/crypto v0.12.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 042da93..8ef3dfc 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,10 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= @@ -231,6 +235,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -243,6 +248,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -279,6 +285,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -310,7 +317,12 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -330,6 +342,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -367,11 +380,16 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -380,6 +398,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -432,6 +451,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/provider/digitalocean.go b/provider/digitalocean.go index 82a199e..7260102 100644 --- a/provider/digitalocean.go +++ b/provider/digitalocean.go @@ -10,7 +10,6 @@ import ( ) type providerDO struct { - regions []string clients *clientmap.ClientMap } @@ -60,18 +59,10 @@ func (pdo providerDO) Enumerate(b *bucket.Bucket) error { return nil } -func (pdo *providerDO) Regions() []string { - urls := make([]string, len(pdo.regions)) - for i, r := range pdo.regions { - urls[i] = fmt.Sprintf("https://%s.digitaloceanspaces.com", r) - } - return urls -} - func (pdo *providerDO) newClients() (*clientmap.ClientMap, error) { - clients := clientmap.WithCapacity(len(pdo.regions)) - for _, r := range pdo.Regions() { - client, err := newNonAWSClient(pdo, r) + clients := clientmap.WithCapacity(len(ProviderRegions[pdo.Name()])) + for _, r := range ProviderRegions[pdo.Name()] { + client, err := newNonAWSClient(pdo, fmt.Sprintf("https://%s.digitaloceanspaces.com", r)) if err != nil { return nil, err } @@ -87,7 +78,6 @@ func (pdo *providerDO) getRegionClient(region string) *s3.Client { func NewProviderDO() (*providerDO, error) { pdo := new(providerDO) - pdo.regions = []string{"nyc3", "sfo2", "sfo3", "ams3", "sgp1", "fra1", "syd1"} clients, err := pdo.newClients() if err != nil { diff --git a/provider/dreamhost.go b/provider/dreamhost.go index 8803001..3528583 100644 --- a/provider/dreamhost.go +++ b/provider/dreamhost.go @@ -3,14 +3,12 @@ package provider import ( "errors" "fmt" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/sa7mon/s3scanner/bucket" "github.com/sa7mon/s3scanner/provider/clientmap" ) type ProviderDreamhost struct { - regions []string clients *clientmap.ClientMap } @@ -64,18 +62,10 @@ func (p ProviderDreamhost) Enumerate(b *bucket.Bucket) error { return nil } -func (p ProviderDreamhost) Regions() []string { - urls := make([]string, len(p.regions)) - for i, r := range p.regions { - urls[i] = fmt.Sprintf("https://objects-%s.dream.io", r) - } - return urls -} - func (p *ProviderDreamhost) newClients() (*clientmap.ClientMap, error) { - clients := clientmap.WithCapacity(len(p.regions)) - for _, r := range p.Regions() { - client, err := newNonAWSClient(p, r) + clients := clientmap.WithCapacity(len(ProviderRegions[p.Name()])) + for _, r := range ProviderRegions[p.Name()] { + client, err := newNonAWSClient(p, fmt.Sprintf("https://objects-%s.dream.io", r)) if err != nil { return nil, err } @@ -87,7 +77,6 @@ func (p *ProviderDreamhost) newClients() (*clientmap.ClientMap, error) { func NewProviderDreamhost() (*ProviderDreamhost, error) { pd := new(ProviderDreamhost) - pd.regions = []string{"us-east-1"} clients, err := pd.newClients() if err != nil { diff --git a/provider/linode.go b/provider/linode.go index 067a443..7cb4ad8 100644 --- a/provider/linode.go +++ b/provider/linode.go @@ -10,13 +10,11 @@ import ( ) type providerLinode struct { - regions []string clients *clientmap.ClientMap } func NewProviderLinode() (*providerLinode, error) { pl := new(providerLinode) - pl.regions = []string{"us-east-1", "us-southeast-1", "eu-central-1", "ap-south-1"} clients, err := pl.newClients() if err != nil { @@ -60,9 +58,9 @@ func (pl *providerLinode) Enumerate(b *bucket.Bucket) error { } func (pl *providerLinode) newClients() (*clientmap.ClientMap, error) { - clients := clientmap.WithCapacity(len(pl.regions)) - for _, r := range pl.Regions() { - client, err := newNonAWSClient(pl, r) + clients := clientmap.WithCapacity(len(ProviderRegions[pl.Name()])) + for _, r := range ProviderRegions[pl.Name()] { + client, err := newNonAWSClient(pl, fmt.Sprintf("https://%s.linodeobjects.com", r)) if err != nil { return nil, err } @@ -85,14 +83,6 @@ func (*providerLinode) Name() string { return "linode" } -func (pl *providerLinode) Regions() []string { - urls := make([]string, len(pl.regions)) - for i, r := range pl.regions { - urls[i] = fmt.Sprintf("https://%s.linodeobjects.com", r) - } - return urls -} - func (*providerLinode) AddressStyle() int { return VirtualHostStyle } diff --git a/provider/providers.go b/provider/providers.go index 800927e..65c8782 100644 --- a/provider/providers.go +++ b/provider/providers.go @@ -43,6 +43,12 @@ var AllProviders = []string{ "aws", "custom", "digitalocean", "dreamhost", "gcp", "linode", } +var ProviderRegions = map[string][]string{ + "digitalocean": {"ams3", "blr1", "fra1", "nyc3", "sfo2", "sfo3", "sgp1", "syd1"}, + "dreamhost": {"us-east-1"}, + "linode": {"ap-south-1", "eu-central-1", "fr-par-1", "se-sto-1", "us-east-1", "us-iad-1", "us-ord-1", "us-southeast-1"}, +} + func NewProvider(name string) (StorageProvider, error) { var ( provider StorageProvider