main: add new --extra-repo flag

cmd/image-builder/describeimg_test.go

@@ -15,7 +15,7 @@ func TestDescribeImage(t *testing.T) {
 	restore := main.MockNewRepoRegistry(testrepos.New)
 	defer restore()
 
-	res, err := main.GetOneImage("", "centos-9", "tar", "x86_64")
+	res, err := main.GetOneImage("centos-9", "tar", "x86_64", nil)
 	assert.NoError(t, err)
 
 	var buf bytes.Buffer

cmd/image-builder/export_test.go

@@ -44,7 +44,7 @@ func MockOsStderr(new io.Writer) (restore func()) {
 
 func MockNewRepoRegistry(f func() (*reporegistry.RepoRegistry, error)) (restore func()) {
 	saved := newRepoRegistry
-	newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
+	newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) {
 		if dataDir != "" {
 			panic(fmt.Sprintf("cannot use custom dataDir %v in mock", dataDir))
 		}

cmd/image-builder/filters.go

@@ -10,19 +10,28 @@ import (
 	"github.com/osbuild/images/pkg/imagefilter"
 )
 
-func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) {
+func newImageFilterDefault(dataDir string, extraRepos []string) (*imagefilter.ImageFilter, error) {
 	fac := distrofactory.NewDefault()
-	repos, err := newRepoRegistry(dataDir)
+	repos, err := newRepoRegistry(dataDir, extraRepos)
 	if err != nil {
 		return nil, err
 	}
 
 	return imagefilter.New(fac, repos)
 }
 
+type repoOptions struct {
+	DataDir    string
+	ExtraRepos []string
+}
+
 // should this be moved to images:imagefilter?
-func getOneImage(dataDir, distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) {
-	imageFilter, err := newImageFilterDefault(dataDir)
+func getOneImage(distroName, imgTypeStr, archStr string, repoOpts *repoOptions) (*imagefilter.Result, error) {
+	if repoOpts == nil {
+		repoOpts = &repoOptions{}
+	}
+
+	imageFilter, err := newImageFilterDefault(repoOpts.DataDir, repoOpts.ExtraRepos)
 	if err != nil {
 		return nil, err
 	}

cmd/image-builder/filters_test.go

@@ -14,7 +14,6 @@ func TestGetOneImageHappy(t *testing.T) {
 	restore := main.MockNewRepoRegistry(testrepos.New)
 	defer restore()
 
-	dataDir := ""
 	for _, tc := range []struct {
 		distro, imgType, arch string
 	}{
@@ -23,7 +22,7 @@ func TestGetOneImageHappy(t *testing.T) {
 		{"distro:centos-9", "type:qcow2", "x86_64"},
 		{"distro:centos-9", "type:qcow2", "arch:x86_64"},
 	} {
-		res, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch)
+		res, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil)
 		assert.NoError(t, err)
 		assert.Equal(t, "centos-9", res.Distro.Name())
 		assert.Equal(t, "qcow2", res.ImgType.Name())
@@ -35,7 +34,6 @@ func TestGetOneImageError(t *testing.T) {
 	restore := main.MockNewRepoRegistry(testrepos.New)
 	defer restore()
 
-	dataDir := ""
 	for _, tc := range []struct {
 		distro, imgType, arch string
 		expectedErr           string
@@ -49,7 +47,7 @@ func TestGetOneImageError(t *testing.T) {
 			`cannot use globs in "centos*" when getting a single image`,
 		},
 	} {
-		_, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch)
+		_, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil)
 		assert.EqualError(t, err, tc.expectedErr)
 	}
 }

cmd/image-builder/list.go

@@ -4,8 +4,8 @@ import (
 	"github.com/osbuild/images/pkg/imagefilter"
 )
 
-func listImages(dataDir, output string, filterExprs []string) error {
-	imageFilter, err := newImageFilterDefault(dataDir)
+func listImages(dataDir string, extraRepos []string, output string, filterExprs []string) error {
+	imageFilter, err := newImageFilterDefault(dataDir, extraRepos)
 	if err != nil {
 		return err
 	}

cmd/image-builder/main.go

@@ -45,8 +45,12 @@ func cmdListImages(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	extraRepos, err := cmd.Flags().GetStringArray("extra-repo")
+	if err != nil {
+		return err
+	}
 
-	return listImages(dataDir, output, filter)
+	return listImages(dataDir, extraRepos, output, filter)
 }
 
 func ostreeImageOptions(cmd *cobra.Command) (*ostree.ImageOptions, error) {
@@ -79,6 +83,10 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	if err != nil {
 		return nil, err
 	}
+	extraRepos, err := cmd.Flags().GetStringArray("extra-repo")
+	if err != nil {
+		return nil, err
+	}
 	archStr, err := cmd.Flags().GetString("arch")
 	if err != nil {
 		return nil, err
@@ -129,7 +137,11 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	pbar.SetPulseMsgf("Manifest generation step")
 	pbar.SetMessagef("Building manifest for %s-%s", distroStr, imgTypeStr)
 
-	img, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr)
+	repoOpts := &repoOptions{
+		DataDir:    dataDir,
+		ExtraRepos: extraRepos,
+	}
+	img, err := getOneImage(distroStr, imgTypeStr, archStr, repoOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -146,7 +158,7 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 		RpmDownloader: rpmDownloader,
 		WithSBOM:      withSBOM,
 	}
-	err = generateManifest(dataDir, img, w, opts)
+	err = generateManifest(dataDir, extraRepos, img, w, opts)
 	return img, err
 }
 
@@ -276,7 +288,7 @@ func cmdDescribeImg(cmd *cobra.Command, args []string) error {
 		archStr = arch.Current().String()
 	}
 	imgTypeStr := args[0]
-	res, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr)
+	res, err := getOneImage(distroStr, imgTypeStr, archStr, &repoOptions{DataDir: dataDir})
 	if err != nil {
 		return err
 	}
@@ -304,6 +316,7 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support.
 		SilenceErrors: true,
 	}
 	rootCmd.PersistentFlags().String("datadir", "", `Override the default data directory for e.g. custom repositories/*.json data`)
+	rootCmd.PersistentFlags().StringArray("extra-repo", nil, `Add an extra repository during build (will *not* be gpg checked and not be part of the final image)`)
 	rootCmd.PersistentFlags().String("output-dir", "", `Put output into the specified directory`)
 	rootCmd.PersistentFlags().BoolP("verbose", "v", false, `Switch to verbose mode`)
 	rootCmd.SetOut(osStdout)

cmd/image-builder/main_test.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -15,6 +16,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	testrepos "github.com/osbuild/images/test/data/repositories"
 
@@ -599,3 +601,48 @@ func TestProgressFromCmd(t *testing.T) {
 		assert.Equal(t, tc.expectedProgress, fmt.Sprintf("%T", pbar))
 	}
 }
+
+func TestManifestExtraRepo(t *testing.T) {
+	if testing.Short() {
+		t.Skip("manifest generation takes a while")
+	}
+	if !hasDepsolveDnf() {
+		t.Skip("no osbuild-depsolve-dnf binary found")
+	}
+	if _, err := exec.LookPath("createrepo_c"); err != nil {
+		t.Skip("need createrepo_c to run this test")
+	}
+
+	localRepoDir := filepath.Join(t.TempDir(), "repo")
+	err := os.MkdirAll(localRepoDir, 0755)
+	assert.NoError(t, err)
+	err = exec.Command("cp", "-a", "../../test/data/rpm/dummy-1.0.0-0.noarch.rpm", localRepoDir).Run()
+	assert.NoError(t, err)
+	err = exec.Command("createrepo_c", localRepoDir).Run()
+	assert.NoError(t, err)
+
+	pkgHelloBlueprint := `{
+          "packages": [
+            {"name":"dummy"}
+          ]
+        }`
+	restore := main.MockOsArgs([]string{
+		"manifest",
+		"qcow2",
+		"--distro=centos-9",
+		fmt.Sprintf("--extra-repo=file://%s", localRepoDir),
+		"--blueprint", makeTestBlueprint(t, pkgHelloBlueprint),
+	})
+	defer restore()
+
+	var fakeStdout bytes.Buffer
+	restore = main.MockOsStdout(&fakeStdout)
+	defer restore()
+
+	err = main.Run()
+	require.NoError(t, err)
+
+	// our local repo got added
+	assert.Contains(t, fakeStdout.String(), `"path":"dummy-1.0.0-0.noarch.rpm"`)
+	assert.Contains(t, fakeStdout.String(), fmt.Sprintf(`"url":"file://%s"`, localRepoDir))
+}

cmd/image-builder/manifest.go

@@ -39,8 +39,8 @@ func sbomWriter(outputDir, filename string, content io.Reader) error {
 	return nil
 }
 
-func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
-	repos, err := newRepoRegistry(dataDir)
+func generateManifest(dataDir string, extraRepos []string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
+	repos, err := newRepoRegistry(dataDir, extraRepos)
 	if err != nil {
 		return err
 	}

cmd/image-builder/repos.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"fmt"
 	"io/fs"
+	"net/url"
 
 	"github.com/osbuild/images/data/repositories"
 	"github.com/osbuild/images/pkg/reporegistry"
+	"github.com/osbuild/images/pkg/rpmmd"
 )
 
 // defaultDataDirs contains the default search paths to look for
@@ -16,13 +19,74 @@ var defaultDataDirs = []string{
 	"/usr/share/image-builder/repositories",
 }
 
-var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
+type repoConfig struct {
+	DataDir    string
+	ExtraRepos []string
+}
+
+func parseExtraRepo(extraRepo string) ([]rpmmd.RepoConfig, error) {
+	// We want to eventually support more URIs repos here:
+	// - config:/path/to/repo.json
+	// - copr:@osbuild/osbuild (with full gpg retrival via the copr API)
+	// But for now just default to base-urls
+
+	baseURL, err := url.Parse(extraRepo)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse extra repo %w", err)
+	}
+	if baseURL.Scheme == "" {
+		return nil, fmt.Errorf(`scheme missing in %q, please prefix with e.g. file:`, extraRepo)
+	}
+
+	// TODO: to support gpg checking we will need to add signing keys.
+	// We will eventually add support for our own "repo.json" format
+	// which is rich enough to contain gpg keys (and more).
+	checkGPG := false
+	return []rpmmd.RepoConfig{
+		{
+			Id:           baseURL.String(),
+			Name:         baseURL.String(),
+			BaseURLs:     []string{baseURL.String()},
+			CheckGPG:     &checkGPG,
+			CheckRepoGPG: &checkGPG,
+		},
+	}, nil
+}
+
+func newRepoRegistryImpl(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) {
 	var dataDirs []string
 	if dataDir != "" {
 		dataDirs = []string{dataDir}
 	} else {
 		dataDirs = defaultDataDirs
 	}
 
-	return reporegistry.New(dataDirs, []fs.FS{repos.FS})
+	conf, err := reporegistry.LoadAllRepositories(dataDirs, []fs.FS{repos.FS})
+	if err != nil {
+		return nil, err
+	}
+
+	// XXX: this should probably go into manifestgen.Options as
+	// a new Options.ExtraRepoConf eventually (just like OverrideRepos)
+	for _, repo := range extraRepos {
+		// XXX: this loads the extra repo unconditionally to all
+		// distro/arch versions. we do not know in advance where
+		// it belongs to
+		extraRepo, err := parseExtraRepo(repo)
+		if err != nil {
+			return nil, err
+		}
+		for _, repoArchConfigs := range conf {
+			for arch := range repoArchConfigs {
+				archCfg := repoArchConfigs[arch]
+				archCfg = append(archCfg, extraRepo...)
+				repoArchConfigs[arch] = archCfg
+			}
+		}
+	}
+
+	return reporegistry.NewFromDistrosRepoConfigs(conf), nil
 }
+
+// this is a variable to make it overridable in tests
+var newRepoRegistry = newRepoRegistryImpl

cmd/image-builder/repos_test.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/osbuild/images/pkg/rpmmd"
+)
+
+func TestParseExtraRepoHappy(t *testing.T) {
+	checkGPG := false
+
+	cfg, err := parseExtraRepo("file:///path/to/repo")
+	assert.NoError(t, err)
+	assert.Equal(t, cfg, []rpmmd.RepoConfig{
+		{
+			Id:           "file:///path/to/repo",
+			Name:         "file:///path/to/repo",
+			BaseURLs:     []string{"file:///path/to/repo"},
+			CheckGPG:     &checkGPG,
+			CheckRepoGPG: &checkGPG,
+		},
+	})
+}
+
+func TestParseExtraRepoSad(t *testing.T) {
+	_, err := parseExtraRepo("/just/a/path")
+	assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:`)
+}

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/cheggaaa/pb/v3 v3.1.6
 	github.com/gobwas/glob v0.2.3
 	github.com/osbuild/bootc-image-builder/bib v0.0.0-20250205182004-b35eaa8a3a91
-	github.com/osbuild/images v0.116.0
+	github.com/osbuild/images v0.116.1-0.20250211125602-fa630cd456b7
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.10.0

go.sum

@@ -234,8 +234,8 @@ github.com/opencontainers/selinux v1.11.1 h1:nHFvthhM0qY8/m+vfhJylliSshm8G1jJ2jD
 github.com/opencontainers/selinux v1.11.1/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
 github.com/osbuild/bootc-image-builder/bib v0.0.0-20250205182004-b35eaa8a3a91 h1:80EI+8V4tOv+sMygoTP8YY+4IIkw1axuuvRHvz7S9HQ=
 github.com/osbuild/bootc-image-builder/bib v0.0.0-20250205182004-b35eaa8a3a91/go.mod h1:22erXOiwfznPLNE5Y8hp35fnE0vh5ZjWp/iIQsVnYTg=
-github.com/osbuild/images v0.116.0 h1:jByGxKjpC4DthZ++P13aRimn4ZmwigTJ8SL9T/bTwvM=
-github.com/osbuild/images v0.116.0/go.mod h1:TzlPk6joWb7YhmA9oYSNU4aGeaHwcKxp8nvgoNaR3Lw=
+github.com/osbuild/images v0.116.1-0.20250211125602-fa630cd456b7 h1:Fpf+nS1tanjlUaHwq+ukZ4InW9ju5k4f3UIEdsW1dro=
+github.com/osbuild/images v0.116.1-0.20250211125602-fa630cd456b7/go.mod h1:JriiHOSUAIv54w6bC0wXIZujmr1nifL/UwEYWv3dtb4=
 github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f h1:/UDgs8FGMqwnHagNDPGOlts35QkhAZ8by3DR7nMih7M=
 github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=