feat: allow insecure feed location

cmd/asset/config.default.toml

@@ -56,6 +56,8 @@ url = "https://hnrss.org/newest"
 type = "feed"
 # See doc for available filters
 filters = ["all", "today"]
+# Allow invalid certificates
+allow-insecure = false
 
 [pipes]
 

internal/goeland/fetch/feed.go

@@ -13,14 +13,15 @@ import (
 	"github.com/mmcdole/gofeed"
 	"github.com/slurdge/goeland/internal/goeland"
 	"github.com/slurdge/goeland/internal/goeland/httpget"
+	"github.com/slurdge/goeland/log"
 	"github.com/spf13/viper"
 )
 
 const minContentLen = 10
 
 var policy *bluemonday.Policy
 
-func fetchFeed(source *goeland.Source, feedLocation string, isFile bool) error {
+func fetchFeed(source *goeland.Source, feedLocation string, isFile bool, allowInsecure bool) error {
 	fp := gofeed.NewParser()
 	var feed *gofeed.Feed
 	if isFile {
@@ -34,7 +35,15 @@ func fetchFeed(source *goeland.Source, feedLocation string, isFile bool) error {
 			return fmt.Errorf("cannot parse file: %s", feedLocation)
 		}
 	} else {
-		body, err := httpget.GetHTTPRessource(feedLocation)
+		var body []byte
+		var err error
+		if !allowInsecure {
+			body, err = httpget.GetHTTPRessource(feedLocation)
+		} else {
+			log.Warningf("ignoring certificate security for url: %s\n", feedLocation)
+			body, err = httpget.GetHTTPRessourceInsecure(feedLocation)
+		}
+
 		if err != nil {
 			return fmt.Errorf("cannot open or parse url: %s (%v)", feedLocation, err)
 		}

internal/goeland/fetch/source.go

@@ -25,7 +25,8 @@ func FetchSource(config config.Provider, sourceName string, parents []string) (*
 	switch sourceType {
 	case "feed":
 		url := config.GetString(fmt.Sprintf("sources.%s.url", sourceName))
-		err = fetchFeed(source, url, !strings.HasPrefix(url, "http"))
+		allowInsecure := config.GetBool(fmt.Sprintf("sources.%s.allow-insecure", sourceName))
+		err = fetchFeed(source, url, !strings.HasPrefix(url, "http"), allowInsecure)
 		if err != nil {
 			log.Errorf("Cannot retrieve feed: %s error: %v", url, err)
 			return source, err

internal/goeland/httpget/httpget.go

@@ -3,36 +3,47 @@ package httpget
 import (
 	"crypto/tls"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 
 	"github.com/slurdge/goeland/version"
 )
 
 var userAgent string = "multiple:goeland:" + version.Version + " (commit id:" + version.GitCommit + ") (by /u/goelandrss)"
 var defaultClient http.Client
+var insecureClient http.Client
 
-// GetHTTPRessource gets the bytes corresponding to an URL
-func GetHTTPRessource(url string) (body []byte, err error) {
-	request, err := http.NewRequest("GET", url, nil)
+func GetHTTPRessourceGeneric(url string, client http.Client) (body []byte, err error) {
+	var request *http.Request
+	request, err = http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, err
 	}
 	request.Header.Set("User-Agent", userAgent)
 	request.Header.Set("Accept", "*/*")
 
-	resp, err := defaultClient.Do(request)
+	resp, err := client.Do(request)
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
-	body, err = ioutil.ReadAll(resp.Body)
+	body, err = io.ReadAll(resp.Body)
 	if resp.StatusCode != 200 {
 		return body, fmt.Errorf("received error code %d", resp.StatusCode)
 	}
 	return body, err
 }
 
+// GetHTTPRessource gets the bytes corresponding to an URL
+func GetHTTPRessource(url string) (body []byte, err error) {
+	return GetHTTPRessourceGeneric(url, defaultClient)
+}
+
+// GetHTTPRessource gets the bytes corresponding to an URL, without checking
+func GetHTTPRessourceInsecure(url string) (body []byte, err error) {
+	return GetHTTPRessourceGeneric(url, insecureClient)
+}
+
 func init() {
 	//this one is needed because of incompatibility between latest golang and reddit
 	defaultClient = http.Client{
@@ -44,4 +55,11 @@ func init() {
 			//ForceAttemptHTTP2: false,
 		},
 	}
+	insecureClient = http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
 }