Use source path in determining if a source is a dir.

This is a minor behavior change where things that are normally treated
as a directory (build contexts, git repos, etc) may now actually be
considered a file when the source spec is pointing at a file in that
context.

This also fixes a case where we cannot point at patch files in a
directory-based source.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

frontend/gateway.go

@@ -2,10 +2,12 @@ package frontend
 
 import (
 	"context"
+	"io/fs"
 	"sync"
 	"sync/atomic"
 
 	"github.com/Azure/dalec"
+	"github.com/Azure/dalec/frontend/pkg/bkfs"
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/frontend/dockerui"
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
@@ -119,6 +121,9 @@ func SourceOptFromClient(ctx context.Context, c gwclient.Client) (dalec.SourceOp
 			}
 			return st, nil
 		},
+		GetFS: func(st *llb.State, opts ...llb.ConstraintsOpt) (fs.FS, error) {
+			return bkfs.FromState(ctx, st, c, opts...)
+		},
 	}, nil
 }
 

frontend/rpm/handle_buildroot.go

@@ -65,5 +65,5 @@ func SpecToBuildrootLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpt
 		return llb.Scratch(), err
 	}
 
-	return Dalec2SpecLLB(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES"), targetKey, "", opts...)
+	return Dalec2SpecLLB(spec, dalec.MergeAtPath(llb.Scratch(), sources, "SOURCES"), targetKey, "", sOpt, opts...)
 }

frontend/rpm/handle_sources.go

@@ -73,7 +73,12 @@ func Dalec2SourcesLLB(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts,
 	sorted := dalec.SortMapKeys(sources)
 	for _, k := range sorted {
 		st := sources[k]
-		if dalec.SourceIsDir(spec.Sources[k]) {
+
+		isDir, err := dalec.SourceIsDir(spec.Sources[k], sOpt, opts...)
+		if err != nil {
+			return nil, err
+		}
+		if isDir {
 			st = st.With(sourceTar(worker, k, withPG("Tar source: "+k)...))
 		}
 		out = append(out, st)

frontend/rpm/handle_spec.go

@@ -14,8 +14,8 @@ import (
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-func SpecHandler(ctx context.Context, client gwclient.Client, spec *dalec.Spec, targetKey string) (*gwclient.Result, error) {
-	st, err := Dalec2SpecLLB(spec, llb.Scratch(), targetKey, "")
+func SpecHandler(ctx context.Context, client gwclient.Client, spec *dalec.Spec, targetKey string, sOpt dalec.SourceOpts) (*gwclient.Result, error) {
+	st, err := Dalec2SpecLLB(spec, llb.Scratch(), targetKey, "", sOpt)
 	if err != nil {
 		return nil, err
 	}
@@ -33,7 +33,11 @@ func SpecHandler(ctx context.Context, client gwclient.Client, spec *dalec.Spec,
 func HandleSpec() gwclient.BuildFunc {
 	return func(ctx context.Context, client gwclient.Client) (*gwclient.Result, error) {
 		return frontend.BuildWithPlatform(ctx, client, func(ctx context.Context, client gwclient.Client, platform *ocispecs.Platform, spec *dalec.Spec, targetKey string) (gwclient.Reference, *dalec.DockerImageSpec, error) {
-			res, err := SpecHandler(ctx, client, spec, targetKey)
+			sOpt, err := frontend.SourceOptFromClient(ctx, client)
+			if err != nil {
+				return nil, nil, err
+			}
+			res, err := SpecHandler(ctx, client, spec, targetKey, sOpt)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -44,7 +48,7 @@ func HandleSpec() gwclient.BuildFunc {
 
 }
 
-func Dalec2SpecLLB(spec *dalec.Spec, in llb.State, targetKey, dir string, opts ...llb.ConstraintsOpt) (llb.State, error) {
+func Dalec2SpecLLB(spec *dalec.Spec, in llb.State, targetKey, dir string, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) {
 	if err := ValidateSpec(spec); err != nil {
 		return llb.Scratch(), fmt.Errorf("invalid spec: %w", err)
 	}
@@ -54,7 +58,7 @@ func Dalec2SpecLLB(spec *dalec.Spec, in llb.State, targetKey, dir string, opts .
 	buf.WriteString("# Automatically generated by " + info.Main.Path + "\n")
 	buf.WriteString("\n")
 
-	if err := WriteSpec(spec, targetKey, buf); err != nil {
+	if err := WriteSpec(spec, targetKey, buf, sOpt, opts...); err != nil {
 		return llb.Scratch(), err
 	}
 

frontend/rpm/template.go

@@ -11,6 +11,7 @@ import (
 	"text/template"
 
 	"github.com/Azure/dalec"
+	"github.com/moby/buildkit/client/llb"
 )
 
 const gomodsName = "__gomods"
@@ -49,7 +50,9 @@ BuildArch: noarch
 
 type specWrapper struct {
 	*dalec.Spec
-	Target string
+	Target     string
+	sOpts      dalec.SourceOpts
+	contraints []llb.ConstraintsOpt
 }
 
 func (w *specWrapper) Changelog() (fmt.Stringer, error) {
@@ -155,7 +158,10 @@ func (w *specWrapper) Sources() (fmt.Stringer, error) {
 	for idx, name := range keys {
 		src := w.Spec.Sources[name]
 		ref := name
-		isDir := dalec.SourceIsDir(src)
+		isDir, err := dalec.SourceIsDir(src, w.sOpts, w.contraints...)
+		if err != nil {
+			return nil, err
+		}
 
 		if isDir {
 			ref += ".tar.gz"
@@ -226,7 +232,10 @@ func (w *specWrapper) PrepareSources() (fmt.Stringer, error) {
 				return nil
 			}
 
-			isDir := dalec.SourceIsDir(src)
+			isDir, err := dalec.SourceIsDir(src, w.sOpts, w.contraints...)
+			if err != nil {
+				return err
+			}
 
 			if !isDir {
 				fmt.Fprintf(b, "cp -a \"%%{_sourcedir}/%s\" .\n", name)
@@ -569,8 +578,8 @@ func (w *specWrapper) Files() fmt.Stringer {
 }
 
 // WriteSpec generates an rpm spec from the provided [dalec.Spec] and distro target and writes it to the passed in writer
-func WriteSpec(spec *dalec.Spec, target string, w io.Writer) error {
-	s := &specWrapper{spec, target}
+func WriteSpec(spec *dalec.Spec, target string, w io.Writer, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) error {
+	s := &specWrapper{spec, target, sOpt, opts}
 
 	err := specTmpl.Execute(w, s)
 	if err != nil {

frontend/rpm/template_test.go

@@ -181,7 +181,12 @@ func TestTemplateSources(t *testing.T) {
 
 			s = s[len(expectedDoc):] // trim off the doc from the output
 			suffix := "\n"
-			if dalec.SourceIsDir(src) {
+
+			isDir, err := dalec.SourceIsDir(src, dalec.SourceOpts{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if isDir {
 				suffix = ".tar.gz\n"
 			}
 

frontend/windows/handle_container.go

@@ -56,7 +56,7 @@ func handleContainer(ctx context.Context, client gwclient.Client) (*gwclient.Res
 		pg := dalec.ProgressGroup("Build windows container: " + spec.Name)
 		worker := workerImg(sOpt, pg)
 
-		bin, err := buildBinaries(ctx, spec, worker, client, sOpt, targetKey)
+		bin, err := buildBinaries(ctx, spec, worker, client, sOpt, targetKey, pg)
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to build binary %w", err)
 		}

frontend/windows/handle_zip.go

@@ -34,7 +34,7 @@ func handleZip(ctx context.Context, client gwclient.Client) (*gwclient.Result, e
 		pg := dalec.ProgressGroup("Build windows container: " + spec.Name)
 		worker := workerImg(sOpt, pg)
 
-		bin, err := buildBinaries(ctx, spec, worker, client, sOpt, targetKey)
+		bin, err := buildBinaries(ctx, spec, worker, client, sOpt, targetKey, pg)
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to build binaries: %w", err)
 		}
@@ -94,8 +94,8 @@ func installBuildDeps(deps []string) llb.StateOption {
 	}
 }
 
-func withSourcesMounted(dst string, states map[string]llb.State, sources map[string]dalec.Source) llb.RunOption {
-	opts := make([]llb.RunOption, 0, len(states))
+func withSourcesMounted(dst string, states map[string]llb.State, sources map[string]dalec.Source, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	runOpts := make([]llb.RunOption, 0, len(states))
 
 	sorted := dalec.SortMapKeys(states)
 	files := []llb.State{}
@@ -107,26 +107,32 @@ func withSourcesMounted(dst string, states map[string]llb.State, sources map[str
 		// So we need to check for this.
 		src, ok := sources[k]
 
-		if ok && !dalec.SourceIsDir(src) {
-			files = append(files, state)
-			continue
+		if ok {
+			isDir, err := dalec.SourceIsDir(src, sOpt, opts...)
+			if err != nil {
+				return nil, err
+			}
+			if !isDir {
+				files = append(files, state)
+				continue
+			}
 		}
 
 		dirDst := filepath.Join(dst, k)
-		opts = append(opts, llb.AddMount(dirDst, state))
+		runOpts = append(runOpts, llb.AddMount(dirDst, state))
 	}
 
-	ordered := make([]llb.RunOption, 1, len(opts)+1)
+	ordered := make([]llb.RunOption, 1, len(runOpts)+1)
 	ordered[0] = llb.AddMount(dst, dalec.MergeAtPath(llb.Scratch(), files, "/"))
-	ordered = append(ordered, opts...)
+	ordered = append(ordered, runOpts...)
 
-	return dalec.WithRunOptions(ordered...)
+	return dalec.WithRunOptions(ordered...), nil
 }
 
-func buildBinaries(ctx context.Context, spec *dalec.Spec, worker llb.State, client gwclient.Client, sOpt dalec.SourceOpts, targetKey string) (llb.State, error) {
+func buildBinaries(ctx context.Context, spec *dalec.Spec, worker llb.State, client gwclient.Client, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.State, error) {
 	worker = worker.With(installBuildDeps(spec.GetBuildDeps(targetKey)))
 
-	sources, err := specToSourcesLLB(worker, spec, sOpt)
+	sources, err := specToSourcesLLB(worker, spec, sOpt, opts...)
 	if err != nil {
 		return llb.Scratch(), errors.Wrap(err, "could not generate sources")
 	}
@@ -136,12 +142,18 @@ func buildBinaries(ctx context.Context, spec *dalec.Spec, worker llb.State, clie
 	binaries := maps.Keys(spec.Artifacts.Binaries)
 	script := generateInvocationScript(binaries)
 
+	mountedSources, err := withSourcesMounted("/build", patched, spec.Sources, sOpt, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
 	st := worker.Run(
 		dalec.ShArgs(script.String()),
 		llb.Dir("/build"),
-		withSourcesMounted("/build", patched, spec.Sources),
 		llb.AddMount("/tmp/scripts", buildScript),
+		mountedSources,
 		llb.Network(llb.NetModeNone),
+		dalec.WithConstraints(opts...),
 	).AddMount(outputDir, llb.Scratch())
 
 	if signer, ok := spec.GetSigner(targetKey); ok {

source.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/fs"
 	"path/filepath"
 	"strings"
 
@@ -173,6 +174,7 @@ type SourceOpts struct {
 	Resolver   llb.ImageMetaResolver
 	Forward    ForwarderFunc
 	GetContext func(string, ...llb.LocalOption) (*llb.State, error)
+	GetFS      func(*llb.State, ...llb.ConstraintsOpt) (fs.FS, error)
 }
 
 func (s *Source) asState(name string, forMount bool, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error) {
@@ -265,7 +267,11 @@ func generateSourceFromImage(st llb.State, cmd *Command, sOpts SourceOpts, subPa
 			mountOpt = append(mountOpt, llb.SourcePath(src.Spec.Path))
 		}
 
-		if !SourceIsDir(src.Spec) {
+		isDir, err := SourceIsDir(src.Spec, sOpts, opts...)
+		if err != nil {
+			return llb.Scratch(), err
+		}
+		if !isDir {
 			mountOpt = append(mountOpt, llb.SourcePath(src.Dest))
 		}
 		baseRunOpts = append(baseRunOpts, llb.AddMount(src.Dest, srcSt, mountOpt...))
@@ -344,17 +350,59 @@ func WithCreateDestPath() llb.CopyOption {
 	})
 }
 
-func SourceIsDir(src Source) bool {
+// Used to determine if a given source which is that has a subpath specified is pointing
+// at a dir.
+// This allows dir-type sources to actually be a file source when the subpath points to a file.
+func isDirSubpath(src Source, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (bool, error) {
+	// fs.FS is not supposed to attempt to open paths with a leading "/"
+	// However this is perfectly valid in the dalec spec.
+	// We need to trim the path before passing to the fs.
+	p := filepath.Clean(src.Path)
+	p = strings.TrimPrefix("/", p)
+
+	if isRoot(p) {
+		return true, nil
+	}
+	if sOpt.GetFS == nil {
+		panic("missing llb->filesystem driver -- this is a bug in dalec, please report it")
+	}
+	st, err := getSource(src, "/", sOpt, opts...)
+	if err != nil {
+		return false, err
+	}
+
+	fs, err := sOpt.GetFS(&st, opts...)
+	if err != nil {
+		return false, errors.Wrap(err, "error reading source result")
+	}
+
+	f, err := fs.Open(p)
+	if err != nil {
+		return false, errors.Wrap(err, "error opening source path")
+	}
+	defer f.Close()
+
+	stat, err := f.Stat()
+	if err != nil {
+		return false, errors.Wrap(err, "error stating source path")
+	}
+	return stat.IsDir(), nil
+}
+
+func SourceIsDir(src Source, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (bool, error) {
 	switch {
 	case src.DockerImage != nil,
 		src.Git != nil,
 		src.Build != nil,
 		src.Context != nil:
-		return true
+		return isDirSubpath(src, sOpt, opts...)
 	case src.HTTP != nil:
-		return false
+		return false, nil
 	case src.Inline != nil:
-		return src.Inline.Dir != nil
+		if src.Inline.Dir != nil {
+			return isDirSubpath(src, sOpt, opts...)
+		}
+		return false, nil
 	default:
 		panic("unreachable")
 	}
@@ -502,13 +550,22 @@ func (s Source) Doc(name string) (io.Reader, error) {
 	return b, nil
 }
 
-func patchSource(worker, sourceState llb.State, sourceToState map[string]llb.State, patchNames []PatchSpec, opts ...llb.ConstraintsOpt) llb.State {
+func patchSource(worker llb.State, spec *Spec, sourceState llb.State, sourceToState map[string]llb.State, patchNames []PatchSpec, opts ...llb.ConstraintsOpt) llb.State {
 	for _, p := range patchNames {
 		patchState := sourceToState[p.Source]
 		// on each iteration, mount source state to /src to run `patch`, and
 		// set the state under /src to be the source state for the next iteration
+
+		sourcePath := p.Source
+		src := spec.Sources[p.Source]
+		if !isRoot(src.Path) {
+			// In some cases this may be a dir-backed source but with a subpath that points to a file
+			// In this case we'll have an llb.State with just that file in it.
+			sourcePath = filepath.Base(src.Path)
+		}
+
 		sourceState = worker.Run(
-			llb.AddMount("/patch", patchState, llb.Readonly, llb.SourcePath(p.Source)),
+			llb.AddMount("/patch", patchState, llb.Readonly, llb.SourcePath(sourcePath)),
 			llb.Dir("src"),
 			ShArgs(fmt.Sprintf("patch -p%d < /patch", *p.Strip)),
 			WithConstraints(opts...),
@@ -536,7 +593,7 @@ func PatchSources(worker llb.State, spec *Spec, sourceToState map[string]llb.Sta
 			continue
 		}
 		pg := llb.ProgressGroup(pgID, "Patch spec source: "+sourceName+" ", false)
-		states[sourceName] = patchSource(worker, sourceState, states, patches, pg, withConstraints(opts))
+		states[sourceName] = patchSource(worker, spec, sourceState, states, patches, pg, withConstraints(opts))
 	}
 
 	return states