Add updater lib

backend/cmd/nebraska/controller.go

@@ -61,6 +61,7 @@ type controllerConfig struct {
 	oidcAuthConfig      *auth.OIDCAuthConfig
 	flatcarUpdatesURL   string
 	checkFrequency      time.Duration
+	syncerPkgsURL       string
 }
 
 func loggerWithUsername(l zerolog.Logger, c *gin.Context) zerolog.Logger {
@@ -85,12 +86,16 @@ func newController(conf *controllerConfig) (*controller, error) {
 		auth:         authenticator,
 	}
 
+	if conf.syncerPkgsURL == "" && conf.hostFlatcarPackages {
+		conf.syncerPkgsURL = conf.nebraskaURL + "/flatcar/"
+	}
+
 	if conf.enableSyncer {
 		syncerConf := &syncer.Config{
 			API:               conf.api,
 			HostPackages:      conf.hostFlatcarPackages,
 			PackagesPath:      conf.flatcarPackagesPath,
-			PackagesURL:       conf.nebraskaURL + "/flatcar/",
+			PackagesURL:       conf.syncerPkgsURL,
 			FlatcarUpdatesURL: conf.flatcarUpdatesURL,
 			CheckFrequency:    conf.checkFrequency,
 		}

backend/cmd/nebraska/nebraska.go

@@ -47,6 +47,7 @@ var (
 	hostFlatcarPackages   = flag.Bool("host-flatcar-packages", false, "Host Flatcar packages in Nebraska")
 	flatcarPackagesPath   = flag.String("flatcar-packages-path", "", "Path where Flatcar packages files should be stored")
 	nebraskaURL           = flag.String("nebraska-url", "http://localhost:8000", "nebraska URL (http://host:port - required when hosting Flatcar packages in nebraska)")
+	syncerPkgsURL         = flag.String("syncer-packages-url", "", "use this URL instead of the original one for packages created by the syncer; any {{ARCH}} and {{VERSION}} in the URL will be replaced by the original package's architecture and version, respectively. If this option is not used but the 'host-flatcar-packages' one is, then the URL will be nebraska-url/flatcar/ .")
 	httpLog               = flag.Bool("http-log", false, "Enable http requests logging")
 	httpStaticDir         = flag.String("http-static-dir", "../frontend/build", "Path to frontend static files")
 	authMode              = flag.String("auth-mode", "github", "authentication mode, available modes: noop, github, oidc")
@@ -208,6 +209,7 @@ func mainWithError() error {
 		oidcAuthConfig:      oidcAuthConfig,
 		flatcarUpdatesURL:   *flatcarUpdatesURL,
 		checkFrequency:      checkFrequency,
+		syncerPkgsURL:       *syncerPkgsURL,
 	}
 	ctl, err := newController(conf)
 	if err != nil {

backend/pkg/syncer/syncer.go

@@ -80,6 +80,12 @@ func New(conf *Config) (*Syncer, error) {
 		return nil, ErrInvalidAPIInstance
 	}
 
+	if conf.PackagesURL != "" {
+		if _, err := url.Parse(conf.PackagesURL); err != nil {
+			return nil, fmt.Errorf("invalid package url: %w", err)
+		}
+	}
+
 	s := &Syncer{
 		api:               conf.API,
 		hostPackages:      conf.HostPackages,
@@ -253,8 +259,13 @@ func (s *Syncer) processUpdate(descriptor channelDescriptor, update *omaha.Updat
 		url := update.URLs[0].CodeBase
 		filename := update.Manifest.Packages[0].Name
 
+		// Allow to override the URL if needed.
+		if s.packagesURL != "" {
+			url = strings.ReplaceAll(s.packagesURL, "{{VERSION}}", update.Manifest.Version)
+			url = strings.ReplaceAll(url, "{{ARCH}}", getArchString(descriptor.arch))
+		}
+
 		if s.hostPackages {
-			url = s.packagesURL
 			filename = fmt.Sprintf("flatcar-%s-%s.gz", getArchString(descriptor.arch), update.Manifest.Version)
 			if err := s.downloadPackage(update, filename); err != nil {
 				logger.Error().Err(err).Str("channel", descriptor.name).Str("arch", descriptor.arch.String()).Msg("processUpdate, downloading package")

backend/pkg/syncer/syncer_test.go

@@ -0,0 +1,282 @@
+package syncer
+
+import (
+	"log"
+	"os"
+	"testing"
+
+	"github.com/kinvolk/go-omaha/omaha"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/guregu/null.v4"
+
+	"github.com/kinvolk/nebraska/backend/pkg/api"
+)
+
+const (
+	defaultTestDbURL string = "postgres://postgres:nebraska@127.0.0.1:5432/nebraska_tests?sslmode=disable&connect_timeout=10"
+)
+
+func newAPI(t *testing.T) *api.API {
+	t.Helper()
+	a, err := api.NewForTest(api.OptionInitDB, api.OptionDisableUpdatesOnFailedRollout)
+
+	t.Logf("Failed to init DB: %v\n", err)
+	t.Log("These tests require PostgreSQL running and a tests database created, please adjust NEBRASKA_DB_URL as needed.")
+	require.NoError(t, err)
+
+	return a
+}
+
+func newForTest(t *testing.T, conf *Config) *Syncer {
+	t.Helper()
+	a := newAPI(t)
+
+	if conf.API == nil {
+		conf.API = a
+	}
+	s, err := New(conf)
+	require.NoError(t, err)
+
+	return s
+}
+
+func TestMain(m *testing.M) {
+	if os.Getenv("NEBRASKA_SKIP_TESTS") != "" {
+		return
+	}
+
+	if _, ok := os.LookupEnv("NEBRASKA_DB_URL"); !ok {
+		log.Printf("NEBRASKA_DB_URL not set, setting to default %q\n", defaultTestDbURL)
+		_ = os.Setenv("NEBRASKA_DB_URL", defaultTestDbURL)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestSyncer_NoAPI(t *testing.T) {
+	_, err := New(&Config{})
+	assert.ErrorIs(t, err, ErrInvalidAPIInstance)
+}
+
+func TestSyncer_InvalidPkgsURL(t *testing.T) {
+	a := newAPI(t)
+	t.Cleanup(func() {
+		a.Close()
+	})
+
+	tests := []struct {
+		url   string
+		isErr bool
+	}{
+		{
+			url:   "",
+			isErr: false,
+		},
+		{
+			url:   ":file",
+			isErr: true,
+		},
+		{
+			url:   "https://myphony.url",
+			isErr: false,
+		},
+		{
+			url:   "file:///my/file",
+			isErr: false,
+		},
+	}
+
+	for _, tc := range tests {
+		testCase := tc
+		t.Run(testCase.url, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := New(&Config{
+				API:         a,
+				PackagesURL: testCase.url,
+			})
+			if testCase.isErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestSyncer_Init(t *testing.T) {
+	syncer := newForTest(t, &Config{})
+	a := syncer.api
+	t.Cleanup(func() {
+		syncer.api.Close()
+	})
+
+	tApp, err := a.GetApp(flatcarAppID)
+	require.NoError(t, err)
+	tPkg, err := a.AddPackage(&api.Package{Type: api.PkgTypeFlatcar, URL: "http://sample.url/pkg", Version: "12.1.0", ApplicationID: tApp.ID, Arch: api.ArchAMD64})
+	require.NoError(t, err)
+	groupID, err := a.GetGroupID(flatcarAppID, "stable", tPkg.Arch)
+	require.NoError(t, err)
+
+	tGroup, err := a.GetGroup(groupID)
+	require.NoError(t, err)
+
+	tChannel := tGroup.Channel
+
+	tChannel.PackageID = null.StringFrom(tPkg.ID)
+
+	err = a.UpdateChannel(tChannel)
+	require.NoError(t, err)
+
+	err = syncer.initialize()
+	require.NoError(t, err)
+
+	desc := channelDescriptor{
+		name: tChannel.Name,
+		arch: tChannel.Arch,
+	}
+
+	version, ok := syncer.versions[desc]
+	assert.True(t, ok)
+	assert.Equal(t, tPkg.Version, version)
+}
+
+func createOmahaUpdate() *omaha.UpdateResponse {
+	return &omaha.UpdateResponse{
+		URLs: []*omaha.URL{
+			{CodeBase: "https://example.com"},
+		},
+		Manifest: &omaha.Manifest{
+			Version: "1.2.3",
+			Packages: []*omaha.Package{
+				{
+					Name: "updatepayload.tgz",
+					SHA1: "00000000000000000",
+				},
+			},
+			Actions: []*omaha.Action{
+				{},
+			},
+		},
+	}
+}
+
+func setupFlatcarAppStableGroup(t *testing.T, a *api.API) *api.Group {
+	t.Helper()
+	tApp, err := a.GetApp(flatcarAppID)
+	require.NoError(t, err)
+	tPkg, err := a.AddPackage(&api.Package{Type: api.PkgTypeFlatcar, URL: "http://sample.url/pkg", Version: "0.1.0", ApplicationID: tApp.ID, Arch: api.ArchAMD64})
+	require.NoError(t, err)
+	groupID, err := a.GetGroupID(flatcarAppID, "stable", tPkg.Arch)
+	require.NoError(t, err)
+
+	tGroup, err := a.GetGroup(groupID)
+	require.NoError(t, err)
+
+	tChannel := tGroup.Channel
+
+	tChannel.PackageID = null.StringFrom(tPkg.ID)
+
+	return tGroup
+}
+
+func TestSyncer_GetPackage(t *testing.T) {
+	syncer := newForTest(t, &Config{})
+	a := syncer.api
+	t.Cleanup(func() {
+		a.Close()
+	})
+
+	tGroup := setupFlatcarAppStableGroup(t, a)
+	tChannel := tGroup.Channel
+
+	err := syncer.initialize()
+	require.NoError(t, err)
+
+	update := createOmahaUpdate()
+
+	desc := channelDescriptor{
+		name: tChannel.Name,
+		arch: tChannel.Arch,
+	}
+	err = syncer.processUpdate(desc, update)
+	require.NoError(t, err)
+
+	// Get updated group
+	tGroup, err = a.GetGroup(tGroup.ID)
+	require.NoError(t, err)
+
+	assert.Equal(t, update.Manifest.Version, tGroup.Channel.Package.Version)
+	assert.Equal(t, update.URLs[0].CodeBase, tGroup.Channel.Package.URL)
+	assert.Equal(t, update.Manifest.Packages[0].Name, tGroup.Channel.Package.Filename.String)
+}
+
+func TestSyncer_GetPackageWithDiffURL(t *testing.T) {
+	conf := &Config{
+		PackagesURL: "https://my.super.different.packagesurl.io/bucket/",
+	}
+	syncer := newForTest(t, conf)
+	a := syncer.api
+	t.Cleanup(func() {
+		a.Close()
+	})
+
+	tGroup := setupFlatcarAppStableGroup(t, a)
+	tChannel := tGroup.Channel
+
+	err := syncer.initialize()
+	require.NoError(t, err)
+
+	update := createOmahaUpdate()
+
+	desc := channelDescriptor{
+		name: tChannel.Name,
+		arch: tChannel.Arch,
+	}
+	err = syncer.processUpdate(desc, update)
+	require.NoError(t, err)
+
+	// Get updated group
+	tGroup, err = a.GetGroup(tGroup.ID)
+	require.NoError(t, err)
+
+	assert.Equal(t, update.Manifest.Version, tGroup.Channel.Package.Version)
+	assert.Equal(t, conf.PackagesURL, tGroup.Channel.Package.URL)
+	assert.Equal(t, update.Manifest.Packages[0].Name, tGroup.Channel.Package.Filename.String)
+}
+
+func TestSyncer_GetPackageWithGeneratedURL(t *testing.T) {
+	baseURL := "https://my.super.different.packagesurl.io/bucket/"
+	conf := &Config{
+		PackagesURL: baseURL + "{{ARCH}}/{{VERSION}}",
+	}
+	syncer := newForTest(t, conf)
+	a := syncer.api
+	t.Cleanup(func() {
+		a.Close()
+	})
+
+	tGroup := setupFlatcarAppStableGroup(t, a)
+	tChannel := tGroup.Channel
+
+	err := syncer.initialize()
+	require.NoError(t, err)
+
+	update := createOmahaUpdate()
+
+	desc := channelDescriptor{
+		name: tChannel.Name,
+		arch: tChannel.Arch,
+	}
+	err = syncer.processUpdate(desc, update)
+	require.NoError(t, err)
+
+	// Get updated group
+	tGroup, err = a.GetGroup(tGroup.ID)
+	require.NoError(t, err)
+
+	assert.Equal(t, update.Manifest.Version, tGroup.Channel.Package.Version)
+	assert.Equal(t, baseURL+getArchString(tChannel.Arch)+"/"+tGroup.Channel.Package.Version, tGroup.Channel.Package.URL)
+	assert.Equal(t, update.Manifest.Packages[0].Name, tGroup.Channel.Package.Filename.String)
+}

docs/managing-updates.md

@@ -94,12 +94,29 @@ prefer to use it, you should pass the option `-enable-syncer=true` when running
 
 Notice that by default Nebraska only stores metadata about the Flatcar Container Linux updates, not the updates payload. This means that the updates served to your instances contain instructions to download the packages payload from the public Flatcar Container Linux update servers directly, so your servers need access to the Internet to download them.
 
+### Hosting synchronized packages
+
 It is also possible to host the Flatcar Container Linux packages payload in Nebraska. In this case, in addition to get the packages metadata, Nebraska will also download the package payload itself so that it can serve it to your instances when serving updates.
 
 This functionality is turned off by default. So to make Nebraska host the Flatcar Container Linux packages payload, the following options have to be passed to it:
 
     nebraska -host-flatcar-packages=true -flatcar-packages-path=/PATH/TO/STORE/PACKAGES -nebraska-url=http://your.Nebraska.host:port
 
+### Overriding synchronized packages' URLs
+
+Some users may choose to host their own packages elsewhere (i.e. without using the
+host function explained above), and thus it is desired to synchronize the packages
+from upstream but giving them a custom URL for the actual update download.
+
+This can be achieved by setting the `syncer-packages-url` CLI option. This should
+be set as a URL, and any `{{VERSION}}` and `{{ARCH}}` keywords in the URL will be
+replaced by the packages' version and arch, respectively.
+
+For example:
+```bash
+nebraska -enable-syncer=true -syncer-packages-url=https://mysepcialstorage.io/flatcar/{{ARCH}}/{{VERSION}}
+```
+
 ## Managing updates for your own applications
 
 In addition to managing updates for Flatcar Container Linux, you can use Nebraska for other applications as well.