main: add new --extra-repo-path flag

This commit adds a  new flag `--extra-repo` that can be used
to point to a repository directory that is added to the base
repositories when depsolving.

This means that with a repo created with e.g. `createrepo_c` like
```console
$ mkdir repo
$ (cd repo && dnf download hello)
$ createrepo_c ./repo
```
and a blueprint like:
```toml
[[packages]]
name = "hello"
```
a manifest is generated that gets osbuild from the copr repo:
```console
$ image-builder  --extra-repo ./repo manifest qcow2 --distro centos-9 --blueprint ./bp.toml |jq|grep python3-osbuild
          "path": "hello-2.12.1-5.fc41.x86_64.rpm",
```
Note that this is part of the base repositories so anything with a
higher version number will get pulled from the extra-repo, even
system libraries or kernels. Note also that this repository does
not become part of the image so after the image build all rpms
from there are not updated (unless of course the normal repos
have higher versions of them).

Note as well that there is no safeguard right now against adding
extra repos for the wrong version of the distro, i.e. one could
add an extra repo build against/for fedora-42 on a fedora-40 image
which most likely will break with bad depsolve errors. But that
is okay, this option is meant for advanced users and testing.

cmd/image-builder/export_test.go

@@ -40,7 +40,7 @@ func MockOsStderr(new io.Writer) (restore func()) {
 
 func MockNewRepoRegistry(f func() (*reporegistry.RepoRegistry, error)) (restore func()) {
 	saved := newRepoRegistry
-	newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
+	newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) {
 		if dataDir != "" {
 			panic(fmt.Sprintf("cannot use custom dataDir %v in mock", dataDir))
 		}

cmd/image-builder/filters.go

@@ -10,9 +10,9 @@ import (
 	"github.com/osbuild/images/pkg/imagefilter"
 )
 
-func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) {
+func newImageFilterDefault(dataDir string, extraRepos []string) (*imagefilter.ImageFilter, error) {
 	fac := distrofactory.NewDefault()
-	repos, err := newRepoRegistry(dataDir)
+	repos, err := newRepoRegistry(dataDir, extraRepos)
 	if err != nil {
 		return nil, err
 	}
@@ -21,8 +21,8 @@ func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) {
 }
 
 // should this be moved to images:imagefilter?
-func getOneImage(dataDir, distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) {
-	imageFilter, err := newImageFilterDefault(dataDir)
+func getOneImage(dataDir string, extraRepos []string, distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) {
+	imageFilter, err := newImageFilterDefault(dataDir, extraRepos)
 	if err != nil {
 		return nil, err
 	}

cmd/image-builder/filters_test.go

@@ -23,7 +23,7 @@ func TestGetOneImageHappy(t *testing.T) {
 		{"distro:centos-9", "type:qcow2", "x86_64"},
 		{"distro:centos-9", "type:qcow2", "arch:x86_64"},
 	} {
-		res, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch)
+		res, err := main.GetOneImage(dataDir, nil, tc.distro, tc.imgType, tc.arch)
 		assert.NoError(t, err)
 		assert.Equal(t, "centos-9", res.Distro.Name())
 		assert.Equal(t, "qcow2", res.ImgType.Name())
@@ -49,7 +49,7 @@ func TestGetOneImageError(t *testing.T) {
 			`cannot use globs in "centos*" when getting a single image`,
 		},
 	} {
-		_, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch)
+		_, err := main.GetOneImage(dataDir, nil, tc.distro, tc.imgType, tc.arch)
 		assert.EqualError(t, err, tc.expectedErr)
 	}
 }

cmd/image-builder/list.go

@@ -4,8 +4,8 @@ import (
 	"github.com/osbuild/images/pkg/imagefilter"
 )
 
-func listImages(dataDir, output string, filterExprs []string) error {
-	imageFilter, err := newImageFilterDefault(dataDir)
+func listImages(dataDir string, extraRepos []string, output string, filterExprs []string) error {
+	imageFilter, err := newImageFilterDefault(dataDir, extraRepos)
 	if err != nil {
 		return err
 	}

cmd/image-builder/main.go

@@ -41,8 +41,12 @@ func cmdListImages(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	extraRepos, err := cmd.Flags().GetStringArray("extra-repo")
+	if err != nil {
+		return err
+	}
 
-	return listImages(dataDir, output, filter)
+	return listImages(dataDir, extraRepos, output, filter)
 }
 
 func ostreeImageOptions(cmd *cobra.Command) (*ostree.ImageOptions, error) {
@@ -75,6 +79,10 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	if err != nil {
 		return nil, err
 	}
+	extraRepos, err := cmd.Flags().GetStringArray("extra-repo")
+	if err != nil {
+		return nil, err
+	}
 	archStr, err := cmd.Flags().GetString("arch")
 	if err != nil {
 		return nil, err
@@ -126,7 +134,9 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 		return nil, err
 	}
 
-	img, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr)
+	// XXX: instead of passing dataDir, extraRepos as two strings,
+	// create a repoPaths (or similar) struct
+	img, err := getOneImage(dataDir, extraRepos, distroStr, imgTypeStr, archStr)
 	if err != nil {
 		return nil, err
 	}
@@ -143,7 +153,7 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 		RpmDownloader: rpmDownloader,
 		WithSBOM:      withSBOM,
 	}
-	err = generateManifest(dataDir, img, w, opts)
+	err = generateManifest(dataDir, extraRepos, img, w, opts)
 	return img, err
 }
 
@@ -222,6 +232,7 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support.
 		SilenceErrors: true,
 	}
 	rootCmd.PersistentFlags().String("datadir", "", `Override the default data directory for e.g. custom repositories/*.json data`)
+	rootCmd.PersistentFlags().StringArray("extra-repo", nil, `Add an extra repository during build (will not be part of the final image)`)
 	rootCmd.PersistentFlags().String("output-dir", "", `Put output into the specified directory`)
 	rootCmd.SetOut(osStdout)
 	rootCmd.SetErr(osStderr)

cmd/image-builder/main_test.go

@@ -7,13 +7,15 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
 
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	testrepos "github.com/osbuild/images/test/data/repositories"
 
@@ -543,3 +545,49 @@ func TestManifestIntegrationWithSBOMWithOutputDir(t *testing.T) {
 	assert.Equal(t, filepath.Join(outputDir, "centos-9-qcow2-x86_64.buildroot-build.spdx.json"), sboms[0])
 	assert.Equal(t, filepath.Join(outputDir, "centos-9-qcow2-x86_64.image-os.spdx.json"), sboms[1])
 }
+
+func TestManifestExtraRepo(t *testing.T) {
+	if testing.Short() {
+		t.Skip("manifest generation takes a while")
+	}
+	if !hasDepsolveDnf() {
+		t.Skip("no osbuild-depsolve-dnf binary found")
+	}
+	if _, err := exec.LookPath("createrepo_c"); err != nil {
+		t.Skip("need createrepo_c to run this test")
+	}
+
+	localRepoDir := filepath.Join(t.TempDir(), "repo")
+	err := os.MkdirAll(localRepoDir, 0755)
+	assert.NoError(t, err)
+	err = exec.Command("cp", "-a", "../../test/data/rpm/hello-2.12.1-5.fc41.x86_64.rpm", localRepoDir).Run()
+	assert.NoError(t, err)
+	err = exec.Command("createrepo_c", localRepoDir).Run()
+	assert.NoError(t, err)
+
+	pkgHelloBlueprint := `{
+          "packages": [
+            {"name":"hello"}
+          ]
+        }`
+	restore := main.MockOsArgs([]string{
+		"manifest",
+		"qcow2",
+		"--arch=x86_64",
+		"--distro=centos-9",
+		"--extra-repo", localRepoDir,
+		"--blueprint", makeTestBlueprint(t, pkgHelloBlueprint),
+	})
+	defer restore()
+
+	var fakeStdout bytes.Buffer
+	restore = main.MockOsStdout(&fakeStdout)
+	defer restore()
+
+	err = main.Run()
+	require.NoError(t, err)
+
+	// our local repo got added
+	assert.Contains(t, fakeStdout.String(), `"path":"hello-2.12.1-5.fc41.x86_64.rpm"`)
+	assert.Contains(t, fakeStdout.String(), fmt.Sprintf(`"url":"file:%s"`, localRepoDir))
+}

cmd/image-builder/manifest.go

@@ -39,8 +39,8 @@ func sbomWriter(outputDir, filename string, content io.Reader) error {
 	return nil
 }
 
-func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
-	repos, err := newRepoRegistry(dataDir)
+func generateManifest(dataDir string, extraRepos []string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
+	repos, err := newRepoRegistry(dataDir, extraRepos)
 	if err != nil {
 		return err
 	}

cmd/image-builder/repos.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
 
 	"github.com/osbuild/images/data/repositories"
 	"github.com/osbuild/images/pkg/reporegistry"
+	"github.com/osbuild/images/pkg/rpmmd"
 )
 
 // defaultDataDirs contains the default search paths to look for repository
@@ -19,7 +21,41 @@ var defaultDataDirs = []string{
 	"/usr/share/image-builder",
 }
 
-var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
+type repoConfig struct {
+	DataDir    string
+	ExtraRepos []string
+}
+
+func parseExtraRepo(extraRepo string) ([]rpmmd.RepoConfig, error) {
+	// XXX: we want to eventually support more repos here:
+	// - copr:@osbuild/osbuild (with full gpg retrival via the API)
+	// - https://example.com/repo/fedora-42-x86_64/ (with sslverify=True)
+	// But for now we only support local dirs
+	stat, err := os.Stat(extraRepo)
+	if err != nil {
+		return nil, fmt.Errorf("cannot add extra repository: %v", err)
+	}
+	if !stat.IsDir() {
+		return nil, fmt.Errorf("cannot add extra repository %v: must be a local directory", extraRepo)
+	}
+	extraRepo, err = filepath.Abs(extraRepo)
+	if err != nil {
+		return nil, err
+	}
+
+	checkGPG := false
+	return []rpmmd.RepoConfig{
+		{
+			Id:           extraRepo,
+			Name:         filepath.Base(extraRepo),
+			BaseURLs:     []string{fmt.Sprintf("file:%s", extraRepo)},
+			CheckGPG:     &checkGPG,
+			CheckRepoGPG: &checkGPG,
+		},
+	}, nil
+}
+
+var newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) {
 	var dataDirs []string
 	if dataDir != "" {
 		dataDirs = []string{dataDir}
@@ -39,5 +75,25 @@ var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// XXX: this should probably go into manifestgen.Options as
+	// a new Options.ExtraRepoConf eventually (just like OverrideRepos)
+	for _, repo := range extraRepos {
+		// XXX: this loads the extra repo unconditionally to all
+		// distro/arch versions. we do not know in advance where
+		// it belongs to
+		extraRepo, err := parseExtraRepo(repo)
+		if err != nil {
+			return nil, err
+		}
+		for _, repoArchConfigs := range conf {
+			for arch := range repoArchConfigs {
+				archCfg := repoArchConfigs[arch]
+				archCfg = append(archCfg, extraRepo...)
+				repoArchConfigs[arch] = archCfg
+			}
+		}
+	}
+
 	return reporegistry.NewFromDistrosRepoConfigs(conf), nil
 }