-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
allow URL to be passed in to -f/--file
This commit allows to pass URLs besides local files to the -f/--file switch. This is done by checking if the passed in resource string starts with an http:// or https://, and if it does, the URL validation is done followed by fetching the URL with 3 retry attempts with a gap of 1 second in between. Fix #55
- Loading branch information
Showing
4 changed files
with
155 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| package util | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io/ioutil" | ||
| "net/http" | ||
| "net/url" | ||
| "time" | ||
| ) | ||
|
|
||
| // This function validates the URL, then tries to fetch the data with retries | ||
| // and then reads and returns the data as []byte | ||
| // Returns an error if the URL is invalid, or fetching the data failed or | ||
| // if reading the response body fails. | ||
| func GetURLData(urlString string, attempts int) ([]byte, error) { | ||
| // Validate URL | ||
| _, err := url.ParseRequestURI(urlString) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("invalid URL: %v", urlString) | ||
| } | ||
|
|
||
| // Fetch the URL and store the response body | ||
| data, err := FetchURLWithRetries(urlString, attempts, 1*time.Second) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed fetching data from the URL %v: %s", urlString, err) | ||
| } | ||
|
|
||
| return data, nil | ||
| } | ||
|
|
||
| // Try to fetch the given url string, and make <attempts> attempts at it. | ||
| // Wait for <duration> time between each try. | ||
| // This returns the data from the response body []byte upon successful fetch | ||
| // The passed URL is not validated, so validate the URL before passing to this | ||
| // function | ||
| func FetchURLWithRetries(url string, attempts int, duration time.Duration) ([]byte, error) { | ||
| var data []byte | ||
| var err error | ||
|
|
||
| for i := 0; i < attempts; i++ { | ||
| var response *http.Response | ||
|
|
||
| // sleep for <duration> seconds before trying again | ||
| if i > 0 { | ||
| time.Sleep(duration) | ||
| } | ||
|
|
||
| // retry if http.Get fails | ||
| // if all the retries fail, then return statement at the end of the | ||
| // function will return this err received from http.Get | ||
| response, err = http.Get(url) | ||
| if err != nil { | ||
| continue | ||
| } | ||
| defer response.Body.Close() | ||
|
|
||
| // if the status code is not 200 OK, return an error | ||
| if response.StatusCode != http.StatusOK { | ||
| return nil, fmt.Errorf("unable to fetch %v, server returned status code %v", url, response.StatusCode) | ||
| } | ||
|
|
||
| // Read from the response body, ioutil.ReadAll will return []byte | ||
| data, err = ioutil.ReadAll(response.Body) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("reading from response body failed: %s", err) | ||
| } | ||
| break | ||
| } | ||
|
|
||
| return data, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| package util | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| const ( | ||
| retryAttempts = 3 | ||
| ) | ||
|
|
||
| func TestGetURLData(t *testing.T) { | ||
| tests := []struct { | ||
| succeed bool | ||
| url string | ||
| }{ | ||
| {true, "http://example.com"}, | ||
| {false, "invalid.url.^&*!@#"}, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(fmt.Sprintf("passing %v, expected to succeed: %v", tt.url, tt.succeed), func(t *testing.T) { | ||
| _, err := GetURLData(tt.url, retryAttempts) | ||
|
|
||
| // error occurred but test was expected to pass | ||
| if err != nil && tt.succeed { | ||
| t.Fatalf("GetURLData was expected to succeed for the URL: %v, but it failed with the error: %v", tt.url, err) | ||
| } | ||
|
|
||
| // no error occurred but test was expected to fail | ||
| if err == nil && !tt.succeed { | ||
| t.Fatalf("GetURLData was expected to fail for the URL: %v, but it passed", tt.url) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestFetchURLWithRetries(t *testing.T) { | ||
| tests := []struct { | ||
| succeed bool | ||
| url string | ||
| }{ | ||
| {true, "https://example.com/"}, // valid URL | ||
| {false, "https://invalid.example/"}, // test for no DNS resolution for URL | ||
| {false, ""}, // test for blank string | ||
| {false, "https://google.com/giveme404"}, // test for !200 status code | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(fmt.Sprintf("URL %v, expected output: %v", tt.url, tt.succeed), func(t *testing.T) { | ||
|
|
||
| _, err := FetchURLWithRetries(tt.url, retryAttempts, time.Second) | ||
|
|
||
| // error occurred but tt was expected to pass | ||
| if err != nil && tt.succeed { | ||
| t.Fatalf("FetchURLWithRetries was expected to succeed for the URL: %v, but it failed with the error: %v", tt.url, err) | ||
| } | ||
|
|
||
| // no error occurred but test was expected to fail | ||
| if err == nil && !tt.succeed { | ||
| t.Fatalf("FetchURLWithRetries was expected to fail for the URL: %v, but it passed", tt.url) | ||
| } | ||
| }) | ||
| } | ||
| } |