-
-
Notifications
You must be signed in to change notification settings - Fork 374
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* add region check for DO * add linode region check * start adding dreamhost check * parse proper region names for dreamhost * simplify * add docs and refactor * move regions to provider interface * add region tests * exit(1) if regions differ and update regions * add workflow, temp remove region * put region back * simplify region getter * simplify region getters * verify dreamhost regions * lint * run workflow on schedule
- Loading branch information
Showing
11 changed files
with
307 additions
and
40 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| name: Check provider regions | ||
|
|
||
| on: | ||
| schedule: | ||
| - cron: '5 4 * * 3' | ||
|
|
||
| jobs: | ||
| region-check: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Set up Go | ||
| uses: actions/setup-go@v4 | ||
| with: | ||
| go-version: 1.19 | ||
| - name: region-check | ||
| run: go run cmd/regioncheck/regioncheck.go |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,194 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/PuerkitoBio/goquery" | ||
| "github.com/sa7mon/s3scanner/collection" | ||
| "github.com/sa7mon/s3scanner/provider" | ||
| "log" | ||
| "net" | ||
| "net/http" | ||
| "os" | ||
| "regexp" | ||
| "sort" | ||
| "strings" | ||
| "sync" | ||
| "time" | ||
| ) | ||
|
|
||
| // eq compares sorted string slices. Once we move to Golang 1.21, use slices.Equal instead. | ||
| func eq(f []string, s []string) bool { | ||
| if len(f) != len(s) { | ||
| return false | ||
| } | ||
| for i := range f { | ||
| if f[i] != s[i] { | ||
| return false | ||
| } | ||
| } | ||
| return true | ||
| } | ||
|
|
||
| // GetRegionsDO fetches regions from the DigitalOcean docs HTML page. | ||
| func GetRegionsDO() ([]string, error) { | ||
| requestURL := "https://docs.digitalocean.com/products/platform/availability-matrix/#other-product-availability" | ||
| res, err := http.Get(requestURL) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer res.Body.Close() | ||
| if res.StatusCode != 200 { | ||
| return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status) | ||
| } | ||
|
|
||
| doc, err := goquery.NewDocumentFromReader(res.Body) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| regions := []string{} | ||
| doc.Find("h2#other-product-availability + table thead tr th").Each(func(i int, t *goquery.Selection) { | ||
| regions = append(regions, t.Text()) | ||
| }) | ||
|
|
||
| spaces_supported := []bool{} | ||
| doc.Find("h2#other-product-availability + table tbody tr").Each(func(i int, t *goquery.Selection) { | ||
| // For each row, check the first cell for a value of "Spaces" | ||
| rowHeader := t.Find("td").First().Text() | ||
| if rowHeader == "Spaces" { | ||
| // For each cell in the "Spaces" row, check if the contents are not empty - meaning Spaces is supported | ||
| t.Find("td").Each(func(j int, v *goquery.Selection) { | ||
| supported := v.Text() != "" | ||
| spaces_supported = append(spaces_supported, supported) | ||
| }) | ||
| } | ||
| }) | ||
|
|
||
| supported_regions := []string{} | ||
| for i := 0; i < len(regions); i++ { | ||
| if regions[i] == "Product" { | ||
| continue | ||
| } | ||
| if spaces_supported[i] { | ||
| supported_regions = append(supported_regions, strings.ToLower(regions[i])) | ||
| } | ||
| } | ||
|
|
||
| // Return slice of region names | ||
| return supported_regions, nil | ||
| } | ||
|
|
||
| // GetRegionsLinode fetches region names from Linode docs HTML page. Linode also provides this info via | ||
| // unauthenticated API (https://api.linode.com/v4/regions) but the region names do not include the trailing digit "-1". | ||
| func GetRegionsLinode() ([]string, error) { | ||
| requestURL := "https://www.linode.com/docs/products/storage/object-storage/" | ||
| res, err := http.Get(requestURL) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer res.Body.Close() | ||
| if res.StatusCode != 200 { | ||
| return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status) | ||
| } | ||
|
|
||
| doc, err := goquery.NewDocumentFromReader(res.Body) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| regions := []string{} | ||
| doc.Find("h2#availability + p + table tbody tr td:nth-of-type(2)").Each(func(i int, t *goquery.Selection) { | ||
| regions = append(regions, t.Text()) | ||
| }) | ||
|
|
||
| return regions, nil | ||
| } | ||
|
|
||
| // GetRegionsDreamhost fetches subdomains of dream.io like 'objects-us-east-1.dream.io' via crt.sh since Dreamhost | ||
| // doesn't have a documentation page listing the regions. | ||
| func GetRegionsDreamhost() ([]string, error) { | ||
| var domainRe = regexp.MustCompile(`objects-([^\.]+)\.dream\.io`) | ||
| requestURL := "https://crt.sh/?q=.dream.io" | ||
| // Request the HTML page. | ||
| res, err := http.Get(requestURL) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer res.Body.Close() | ||
| if res.StatusCode != 200 { | ||
| return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status) | ||
| } | ||
|
|
||
| // Load the HTML document | ||
| doc, err := goquery.NewDocumentFromReader(res.Body) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| certCNs := collection.StringSet{} | ||
| // For each cell in the Common Name column | ||
| doc.Find("body > table table tbody tr > td:nth-of-type(5)").Each(func(i int, t *goquery.Selection) { | ||
| matches := domainRe.FindAllStringSubmatch(t.Text(), -1) | ||
| for _, match := range matches { | ||
| if !strings.HasPrefix(match[1], "website-") { // regions like 'objects-website-us-east-1' are not for Object Storage | ||
| certCNs.Add(match[1]) | ||
| } | ||
| } | ||
| }) | ||
|
|
||
| // crt.sh may return old or invalid SSL certs | ||
| // verify regions found resolve and respond on port 443 | ||
| resolvableRegions := []string{} | ||
| for _, s := range certCNs.Slice() { | ||
| timeout := 1 * time.Second | ||
| _, cerr := net.DialTimeout("tcp", fmt.Sprintf("objects-%s.dream.io:443", s), timeout) | ||
| if cerr == nil { | ||
| resolvableRegions = append(resolvableRegions, s) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| return resolvableRegions, nil | ||
| } | ||
|
|
||
| func main() { | ||
| wg := sync.WaitGroup{} | ||
| wg.Add(3) | ||
|
|
||
| results := map[string][]string{} | ||
| errors := map[string]error{} | ||
|
|
||
| go func(w *sync.WaitGroup) { | ||
| results["digitalocean"], errors["digitalocean"] = GetRegionsDO() | ||
| wg.Done() | ||
| }(&wg) | ||
| go func(w *sync.WaitGroup) { | ||
| results["dreamhost"], errors["dreamhost"] = GetRegionsDreamhost() | ||
| wg.Done() | ||
| }(&wg) | ||
| go func(w *sync.WaitGroup) { | ||
| results["linode"], errors["linode"] = GetRegionsLinode() | ||
| wg.Done() | ||
| }(&wg) | ||
| wg.Wait() | ||
|
|
||
| exit := 0 | ||
|
|
||
| for p, knownRegions := range provider.ProviderRegions { | ||
| if errors[p] != nil { | ||
| log.Printf("[%s]: %v\n", p, errors[p]) | ||
| continue | ||
| } | ||
| foundRegions := results[p] | ||
| sort.Strings(foundRegions) | ||
| sort.Strings(knownRegions) | ||
|
|
||
| if !eq(foundRegions, knownRegions) { | ||
| log.Printf("[%s] regions differ! Existing: %v, found: %v", p, knownRegions, foundRegions) | ||
| exit = 1 | ||
| } else { | ||
| log.Printf("[%s} OK", p) | ||
| } | ||
| } | ||
| os.Exit(exit) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "github.com/stretchr/testify/assert" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestGetRegionsDO(t *testing.T) { | ||
| r, err := GetRegionsDO() | ||
| assert.Nil(t, err) | ||
| assert.GreaterOrEqual(t, len(r), 1) | ||
| assert.Contains(t, r, "nyc3") | ||
| } | ||
|
|
||
| func TestGetRegionsLinode(t *testing.T) { | ||
| r, err := GetRegionsLinode() | ||
| assert.Nil(t, err) | ||
| assert.GreaterOrEqual(t, len(r), 1) | ||
| assert.Contains(t, r, "us-east-1") | ||
| } | ||
|
|
||
| func TestGetRegionsDreamhost(t *testing.T) { | ||
| dor, err := GetRegionsDreamhost() | ||
| assert.Nil(t, err) | ||
| assert.GreaterOrEqual(t, len(dor), 1) | ||
| assert.Contains(t, dor, "us-east-1") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package collection | ||
|
|
||
| // StringSet is a simple implementeation of the Set data structure. | ||
| // An empty struct allegedly takes zero memory | ||
| type StringSet map[string]struct{} | ||
|
|
||
| func (ss StringSet) Add(s string) { | ||
| ss[s] = struct{}{} | ||
| } | ||
|
|
||
| func (ss StringSet) Remove(s string) { | ||
| delete(ss, s) | ||
| } | ||
|
|
||
| func (ss StringSet) Has(s string) bool { | ||
| _, ok := ss[s] | ||
| return ok | ||
| } | ||
|
|
||
| func (ss StringSet) Slice() []string { | ||
| slice := make([]string, len(ss)) | ||
| i := 0 | ||
| for s := range ss { | ||
| slice[i] = s | ||
| i++ | ||
| } | ||
| return slice | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.