feat: use cloudflare snapshot as the primary url

debian_package_manager/internal/build/bazel.go

@@ -36,7 +36,9 @@ def repositories():
         name = "{{ printf "%s_%s_" $arch $distro }}{{ bazelify $pi.Name }}",
         package_name = "{{ $pi.Name }}",
         sha256 = "{{ $pi.SHA256 }}",
-        urls = ["{{ $pi.URL }}"],
+        urls = [{{ range $un,$url := $pi.URLs }}
+			"{{ $url }}",{{ end }}
+        ],
     )
 {{- end }}{{ end }}{{ end }}
 `

debian_package_manager/internal/build/bazel_test.go

@@ -17,29 +17,29 @@ func testdata() map[string]map[string]map[string]*deb.Package {
 			Name:    "xxxx",
 			SHA256:  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 			Version: deb.MustParseVersion("1"),
-			URL:     "https://example.com/xxxx-aaaa-1111.deb",
+			URLs:    []string{"https://example.com/xxxx-aaaa-1111.deb"},
 		},
 		"ssss": {
 			Name:    "ssss",
 			SHA256:  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 			Version: deb.MustParseVersion("2"),
-			URL:     "https://example.com/ssss-aaaa-1111.deb",
+			URLs:    []string{"https://example.com/ssss-aaaa-1111.deb"},
 		},
 	}
 	pkgInfo["cccc"]["2222"] = map[string]*deb.Package{
 		"xxxx": {
 			Name:    "xxxx",
 			SHA256:  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 			Version: deb.MustParseVersion("3"),
-			URL:     "https://example.com/xxxx-cccc-2222.deb",
+			URLs:    []string{"https://example.com/xxxx-cccc-2222.deb"},
 		},
 	}
 	pkgInfo["cccc"]["1111"] = map[string]*deb.Package{
 		"xxxx++": {
 			Name:    "xxxx++",
 			SHA256:  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
 			Version: deb.MustParseVersion("4"),
-			URL:     "https://example.com/xxxx++-cccc-1111.deb",
+			URLs:    []string{"https://example.com/xxxx++-cccc-1111.deb"},
 		},
 	}
 	return pkgInfo
@@ -60,25 +60,33 @@ def repositories():
         name = "aaaa_1111_ssss",
         package_name = "ssss",
         sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-        urls = ["https://example.com/ssss-aaaa-1111.deb"],
+        urls = [
+			"https://example.com/ssss-aaaa-1111.deb",
+        ],
     )
     debian_archive(
         name = "aaaa_1111_xxxx",
         package_name = "xxxx",
         sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-        urls = ["https://example.com/xxxx-aaaa-1111.deb"],
+        urls = [
+			"https://example.com/xxxx-aaaa-1111.deb",
+        ],
     )
     debian_archive(
         name = "cccc_1111_xxxxpp",
         package_name = "xxxx++",
         sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-        urls = ["https://example.com/xxxx++-cccc-1111.deb"],
+        urls = [
+			"https://example.com/xxxx++-cccc-1111.deb",
+        ],
     )
     debian_archive(
         name = "cccc_2222_xxxx",
         package_name = "xxxx",
         sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
-        urls = ["https://example.com/xxxx-cccc-2222.deb"],
+        urls = [
+			"https://example.com/xxxx-cccc-2222.deb",
+        ],
     )
 `
 	if diff := cmp.Diff(got.String(), want); diff != "" {

debian_package_manager/internal/build/build.go

@@ -75,11 +75,15 @@ func resolvePackages(pi *deb.PackageIndex, packages map[string]bool) (map[string
 		return nil, errors.Wrapf(err, "failed to xz open package index file %q", pi.URL)
 	}
 
-	poolRoot, err := url.Parse(pi.PoolRoot)
+	// CloudFlare mirror is the new mirror of snapshot.debian.org which is way faster as it
+	// caches everything across regions. This also takes presurre off of the origin server.
+	cloudflare, err := url.Parse(pi.PoolRoot)
 	if err != nil {
-		panic(err) // we shouldn't be generating bad urls
+		return nil, err
 	}
-	pkgs, err := deb.Parse(xzr, packages, poolRoot)
+	cloudflare.Host = "snapshot-cloudflare.debian.org"
+
+	pkgs, err := deb.Parse(xzr, packages, []string{cloudflare.String(), pi.PoolRoot})
 	return pkgs, errors.Wrapf(err, "parsing packages at %q", pi.URL)
 }
 

debian_package_manager/internal/deb/package.go

@@ -27,10 +27,10 @@ type Package struct {
 	Name    string
 	SHA256  string
 	Version Version
-	URL     string
+	URLs    []string
 }
 
-func NewPackage(name string, sha256hex string, version string, url string) (*Package, error) {
+func NewPackage(name string, sha256hex string, version string, urls []string) (*Package, error) {
 	if strings.TrimSpace(name) == "" {
 		return nil, errors.New("package name was empty")
 	}
@@ -50,7 +50,7 @@ func NewPackage(name string, sha256hex string, version string, url string) (*Pac
 		return nil, errors.Wrapf(err, "package version parsing error for %q", name)
 	}
 
-	return &Package{name, sha256hex, v, url}, nil
+	return &Package{name, sha256hex, v, urls}, nil
 }
 
 // Equivalent checks if these packages are the same file (even if they have different urls)

debian_package_manager/internal/deb/package_test.go

@@ -22,17 +22,17 @@ func TestPackage_Equivalent(t *testing.T) {
 		Name:    "goose",
 		SHA256:  "12345",
 		Version: MustParseVersion("1"),
-		URL:     "https://somewhere.com/package.deb",
+		URLs:    []string{"https://somewhere.com/package.deb"},
 	}
 	testCases := []struct {
 		pkg    *Package
 		equals bool
 	}{
 		{pkg: base, equals: true},
-		{pkg: &Package{"goose", "12345", MustParseVersion("1"), "https://somewhereelse.com/package.deb"}, equals: true},
-		{pkg: &Package{"moose", "12345", MustParseVersion("1"), "https://somewhere.com/package.deb"}, equals: false},
-		{pkg: &Package{"goose", "123456", MustParseVersion("1"), "https://somewhere.com/package.deb"}, equals: false},
-		{pkg: &Package{"goose", "123456", MustParseVersion("2"), "https://somewhere.com/package.deb"}, equals: false},
+		{pkg: &Package{"goose", "12345", MustParseVersion("1"), []string{"https://somewhereelse.com/package.deb"}}, equals: true},
+		{pkg: &Package{"moose", "12345", MustParseVersion("1"), []string{"https://somewhere.com/package.deb"}}, equals: false},
+		{pkg: &Package{"goose", "123456", MustParseVersion("1"), []string{"https://somewhere.com/package.deb"}}, equals: false},
+		{pkg: &Package{"goose", "123456", MustParseVersion("2"), []string{"https://somewhere.com/package.deb"}}, equals: false},
 	}
 	for _, tc := range testCases {
 		if base.Equivalent(tc.pkg) != tc.equals {

debian_package_manager/internal/deb/parser.go

@@ -88,17 +88,26 @@ func parse(r io.Reader, packages map[string]bool) ([]map[string]string, error) {
 	return allEntries, nil
 }
 
-func Parse(r io.Reader, packages map[string]bool, poolPrefix *url.URL) (map[string]*Package, error) {
+func Parse(r io.Reader, packages map[string]bool, poolprefixes []string) (map[string]*Package, error) {
 	allEntries, err := parse(r, packages)
 	if err != nil {
 		return nil, err
 	}
 	ps := map[string]*Package{}
 	for _, entry := range allEntries {
-		fileURL := *poolPrefix // make a copy of the pool root so we don't mutate it
-		fileURL.Path = path.Join(poolPrefix.Path, entry[filenameKey])
 
-		p, err := NewPackage(entry[packageKey], entry[sha256Key], entry[versionKey], fileURL.String())
+		urls := []string{}
+
+		for _, poolprefix := range poolprefixes {
+			url, err := url.Parse(poolprefix)
+			if err != nil {
+				return nil, err
+			}
+			url.Path = path.Join(url.Path, entry[filenameKey])
+			urls = append(urls, url.String())
+		}
+
+		p, err := NewPackage(entry[packageKey], entry[sha256Key], entry[versionKey], urls)
 		if err != nil {
 			return nil, err
 		}

debian_package_manager/internal/deb/parser_test.go

@@ -16,7 +16,6 @@
 package deb
 
 import (
-	"net/url"
 	"os"
 	"testing"
 
@@ -86,7 +85,7 @@ func TestParse_toPackageInfo(t *testing.T) {
 	want := map[string]*Package{
 		"zzuf": {
 			Name: "zzuf",
-			URL:  "https://somewhere/pool/main/z/zzuf/zzuf_0.13.svn20100215-4.1_amd64.deb",
+			URLs: []string{"https://somewhere/pool/main/z/zzuf/zzuf_0.13.svn20100215-4.1_amd64.deb"},
 			Version: Version{
 				UpstreamVersion: "0.13.svn20100215",
 				DebianRevision:  "4.1",
@@ -101,11 +100,7 @@ func TestParse_toPackageInfo(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			ppfx, err := url.Parse(pfx)
-			if err != nil {
-				t.Fatal(err)
-			}
-			got, err := Parse(packages, map[string]bool{"zzuf": true}, ppfx)
+			got, err := Parse(packages, map[string]bool{"zzuf": true}, []string{pfx})
 			if err != nil {
 				t.Fatal(err)
 			}