Ignore specific status codes

algorithm/openssl-3.1.3-RSA/README.md

@@ -41,4 +41,4 @@ RSA の鍵生成 (素数 2 個の生成) アルゴリズムの紹介。
 
 [[FIPS186-4]]: National Institute of Standards and Technology (2013) Security Requirements for Cryptographic Modules. (Department of Commerce, Washington, D.C.), Federal Information Processing Standards Publications (FIPS PUBS) 186-4.
 
-[FIPS186-4]: https://www.omgwiki.org/dido/doku.php?id=dido:public:ra:xapend:xapend.b_stds:tech:nist:dss
+[FIPS186-4]: https://csrc.nist.gov/pubs/fips/186-4/final

check_links.go

@@ -11,25 +11,87 @@ import (
 	"regexp"
 	"strings"
 	"time"
+
+	"github.com/BurntSushi/toml"
 )
 
+const configFilePath = "./check_links_config.toml"
+
 var httpRegex = regexp.MustCompile("http://[-._%/[:alnum:]?:=+]+")
 var httpsRegex = regexp.MustCompile("https://[-._%/[:alnum:]?:=+]+")
 
-// config
-var retryCount = 5
+type Config struct {
+	RetryCount int `toml:"retry_count"`
+	// All text files' extensions
+	TextFileExtensions []string `toml:"text_file_extensions"`
+	Ignores            []Ignore `toml:"ignores"`
+}
+
+type Ignore struct {
+	URL                    string   `toml:"url"`
+	Codes                  []int    `toml:"codes"`
+	Reason                 string   `toml:"reason"`
+	ConsideredAlternatives []string `toml:"considered_alternatives"`
+}
+
+func (c *Config) Validate() error {
+	if len(c.TextFileExtensions) == 0 {
+		return errors.New("text_file_extensions cannot be empty")
+	}
+	for _, ignore := range c.Ignores {
+		if ignore.URL == "" {
+			return errors.New("url cannot be empty")
+		}
+		if len(ignore.Codes) == 0 {
+			return errors.New("codes cannot be empty")
+		}
+		if ignore.Reason == "" {
+			return errors.New("reason cannot be empty")
+		}
+		if len(ignore.ConsideredAlternatives) == 0 {
+			return errors.New("considered_alternatives cannot be empty")
+		}
+	}
+	return nil
+}
+
+func readConfig(configFilePath string) (*Config, error) {
+	var config Config
+	bytes, err := os.ReadFile(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+	toml.Decode(string(bytes), &config)
+	return &config, nil
+}
 
-func checkURLLiveness(url string, retryCount int) error {
+// if ignore != nil, ignore.Codes will be used instead of the 2xx criterion.
+func checkURLLiveness(url string, retryCount int, ignore *Ignore) error {
 	for i := 0; i < retryCount; i++ {
 		resp, err := http.Head(url)
 		if err != nil {
 			return err
 		}
-		if resp.StatusCode/100 == 2 {
-			// ok
-			return nil
+		if ignore != nil {
+			ok := false
+			for _, code := range ignore.Codes {
+				if resp.StatusCode == code {
+					ok = true
+					break
+				}
+			}
+			if ok {
+				// ok, but because ignore != nil, we need a log
+				log.Printf("ok: code = %d, url = %s, ignore = %v\n", resp.StatusCode, url, ignore)
+				return nil
+			}
+		} else {
+			if resp.StatusCode/100 == 2 {
+				// ok
+				return nil
+			}
 		}
-		log.Printf("code = %d, url = %s\n", resp.StatusCode, url)
+		log.Printf("code = %d, url = %s, ignore = %v\n", resp.StatusCode, url, ignore)
 		if i == retryCount-1 {
 			return errors.New("invalid status code")
 		} else {
@@ -40,7 +102,7 @@ func checkURLLiveness(url string, retryCount int) error {
 	return nil
 }
 
-func checkFile(path string) (err error) {
+func checkFile(path string, retryCount int, ignores map[string]*Ignore) (err error) {
 	content, err := os.ReadFile(path)
 	if err != nil {
 		return err
@@ -50,8 +112,9 @@ func checkFile(path string) (err error) {
 	var livenessErrors uint64 = 0
 	for _, v := range all {
 		url := string(v)
+		ignore := ignores[url]
 		log.Printf("%s: HTTP link: url = %s\n", path, url)
-		if thisError := checkURLLiveness(url, retryCount); thisError != nil {
+		if thisError := checkURLLiveness(url, retryCount, ignore); thisError != nil {
 			livenessErrors++
 			log.Printf("%s: not alive: url = %s, thiserror = %v\n", path, url, thisError)
 		}
@@ -60,7 +123,8 @@ func checkFile(path string) (err error) {
 	all = httpsRegex.FindAll(content, -1)
 	for _, v := range all {
 		url := string(v)
-		if thisError := checkURLLiveness(url, retryCount); thisError != nil {
+		ignore := ignores[url]
+		if thisError := checkURLLiveness(url, retryCount, ignore); thisError != nil {
 			livenessErrors++
 			log.Printf("%s: not alive: url = %s, thiserror = %v\n", path, url, thisError)
 		}
@@ -73,20 +137,18 @@ func checkFile(path string) (err error) {
 }
 
 func main() {
-	// All text files' extensions
-	extensions := []string{
-		".c",
-		".cpp",
-		".go",
-		".h",
-		".java",
-		".mod",
-		".md",
-		".py",
-		".rs",
-		".sh",
-		".txt",
+	config, err := readConfig(configFilePath)
+	if err != nil {
+		panic(err)
 	}
+	if err := config.Validate(); err != nil {
+		panic(err)
+	}
+	ignores := make(map[string]*Ignore)
+	for _, ignore := range config.Ignores {
+		ignores[ignore.URL] = &ignore
+	}
+
 	numErrors := 0
 	cmd := exec.Command("git", "ls-files")
 	output, err := cmd.Output()
@@ -108,14 +170,14 @@ func main() {
 		}
 		ext := filepath.Ext(path)
 		ok := false
-		for _, e := range extensions {
+		for _, e := range config.TextFileExtensions {
 			if ext == e {
 				ok = true
 				break
 			}
 		}
 		if ok {
-			if err := checkFile(path); err != nil {
+			if err := checkFile(path, config.RetryCount, ignores); err != nil {
 				numErrors++
 				log.Printf("%v\n", err)
 			}

check_links_config.toml

@@ -0,0 +1,23 @@
+retry_count = 5
+text_file_extensions = [
+    ".c",
+    ".cpp",
+    ".go",
+    ".h",
+    ".java",
+    ".mod",
+    ".md",
+    ".py",
+    ".rs",
+    ".sh",
+    ".txt",
+]
+[[ignores]]
+url = "https://csrc.nist.gov/pubs/fips/186-4/final"
+codes = [200, 404]
+reason = """
+This URL seems to sometimes return 404 to requests from GitHub Actions' runners,
+and the issue cannot be handled with retries."""
+considered_alternatives = [
+    "https://www.omgwiki.org/dido/doku.php?id=dido:public:ra:xapend:xapend.b_stds:tech:nist:dss", # as flaky as the original
+]

go.mod

@@ -1,3 +1,5 @@
 module main
 
 go 1.21
+
+require github.com/BurntSushi/toml v1.3.2

go.sum

@@ -0,0 +1,2 @@
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
+github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=

go.work

@@ -1,3 +1,4 @@
 go 1.21
 
+use .
 use ./algorithm/bip32