Fix export cache issue

Resolve dagger#1551 and dagger#1020. We are never returning the result of solved operations so Buildkit could not cache the layer. This commit implements a simple system to forward operations' result to the main build to cache it. Signed-off-by: Vasek - Tom C <tom.chauveau@epitech.eu>
laashub-soa · Mar 31, 2022 · 19c0f99 · 19c0f99
1 parent 64cdadb
commit 19c0f99
Show file tree

Hide file tree

Showing 42 changed files with 144 additions and 103 deletions.
diff --git a/.github/workflows/dagger-ci.yml b/.github/workflows/dagger-ci.yml
@@ -2,7 +2,7 @@ name: "Dagger CI"
 
 on:
   push:
-    branches: [main]
+    branches: [ main ]
     paths:
       - '**.sh'
       - '**.bash'
@@ -14,7 +14,7 @@ on:
       - 'go.sum'
       - '.github/workflows/dagger-ci.yml'
   pull_request:
-    branches: [main]
+    branches: [ main ]
     paths:
       - '**.sh'
       - '**.bash'
@@ -33,11 +33,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v2
-      -
-        name: Dagger CI
+
+      - name: Dagger CI
         uses: dagger/dagger-for-github@v2
         with:
           workdir: ci

diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -2,7 +2,7 @@ name: "Test Integration"
 
 on:
   push:
-    branches: [main]
+    branches: [ main ]
     paths:
       - "**.sh"
       - "**.bash"
@@ -16,7 +16,7 @@ on:
       - "!docs/**"
 
   pull_request:
-    branches: [main]
+    branches: [ main ]
     paths:
       - "**.sh"
       - "**.bash"
@@ -67,9 +67,6 @@ jobs:
       - name: Test
         env:
           DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-        # TODO: https://github.com/dagger/dagger/pull/1341
-        #   DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-integration"
-        #   DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-integration"
         run: |
           env
-          make core-integration
+          make core-integration
diff --git a/.github/workflows/test-universe.yml b/.github/workflows/test-universe.yml
@@ -57,8 +57,5 @@ jobs:
         uses: crazy-max/ghaction-github-runtime@v1
 
       - name: Test
-        env:
-          DAGGER_CACHE_TO: "type=gha,mode=max,scope=test-universe"
-          DAGGER_CACHE_FROM: "type=gha,mode=max,scope=test-universe"
         run: |
           make universe-test
diff --git a/Makefile b/Makefile
@@ -93,4 +93,4 @@ web: # Run the website locally
 
 .PHONY: todo
 todo: # Find all TODO items
-	grep -r -A 1 "TODO:" $(CURDIR)
+	grep -r -A 1 "TODO:" $(CURDIR)
diff --git a/client/client.go b/client/client.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/containerd/containerd/platforms"
+	"github.com/google/uuid"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 
@@ -18,7 +18,6 @@ import (
 	// buildkit
 	bk "github.com/moby/buildkit/client"
 	_ "github.com/moby/buildkit/client/connhelper/dockercontainer" // import the container connection driver
-	"github.com/moby/buildkit/client/llb"
 	bkgw "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/session"
 
@@ -72,7 +71,7 @@ func New(ctx context.Context, host string, cfg Config) (*Client, error) {
 	}, nil
 }
 
-type DoFunc func(context.Context, solver.Solver) error
+type DoFunc func(context.Context, *solver.Solver) error
 
 // FIXME: return completed *Route, instead of *compiler.Value
 func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) error {
@@ -96,6 +95,19 @@ func (c *Client) Do(ctx context.Context, pctx *plancontext.Context, fn DoFunc) e
 	return eg.Wait()
 }
 
+func convertCacheOptionEntries(ims []bk.CacheOptionsEntry) []bkgw.CacheOptionsEntry {
+	convertIms := []bkgw.CacheOptionsEntry{}
+
+	for _, im := range ims {
+		convertIm := bkgw.CacheOptionsEntry{
+			Type:  im.Type,
+			Attrs: im.Attrs,
+		}
+		convertIms = append(convertIms, convertIm)
+	}
+	return convertIms
+}
+
 func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFunc, ch chan *bk.SolveStatus) error {
 	wg := sync.WaitGroup{}
 
@@ -156,29 +168,31 @@ func (c *Client) buildfn(ctx context.Context, pctx *plancontext.Context, fn DoFu
 
 	resp, err := c.c.Build(ctx, opts, "", func(ctx context.Context, gw bkgw.Client) (*bkgw.Result, error) {
 		s := solver.New(solver.Opts{
-			Control: c.c,
-			Gateway: gw,
-			Events:  eventsCh,
-			Auth:    auth,
-			NoCache: c.cfg.NoCache,
+			Control:      c.c,
+			Gateway:      gw,
+			Events:       eventsCh,
+			Auth:         auth,
+			NoCache:      c.cfg.NoCache,
+			CacheImports: convertCacheOptionEntries(opts.CacheImports),
 		})
 
 		// Close events channel
 		defer s.Stop()
 
 		// Compute output overlay
+		res := bkgw.NewResult()
 		if fn != nil {
-			if err := fn(ctx, s); err != nil {
+			err := fn(ctx, s)
+			if err != nil {
 				return nil, compiler.Err(err)
 			}
-		}
 
-		ref, err := s.Solve(ctx, llb.Scratch(), platforms.DefaultSpec())
-		if err != nil {
-			return nil, err
+			refs := s.References()
+			// Add functions layers
+			for _, ref := range refs {
+				res.AddRef(uuid.New().String(), ref)
+			}
 		}
-		res := bkgw.NewResult()
-		res.SetRef(ref)
 		return res, nil
 	}, buildCh)
 	if err != nil {

diff --git a/cmd/dagger/cmd/do.go b/cmd/dagger/cmd/do.go
@@ -67,7 +67,7 @@ var doCmd = &cobra.Command{
 			Value: target.String(),
 		})
 
-		err = cl.Do(ctx, p.Context(), func(ctx context.Context, s solver.Solver) error {
+		err = cl.Do(ctx, p.Context(), func(ctx context.Context, s *solver.Solver) error {
 			return p.Do(ctx, target, s)
 		})
 

diff --git a/docs/learn/tests/helpers.bash b/docs/learn/tests/helpers.bash
@@ -28,6 +28,12 @@ common_setup() {
     export DAGGER_SANDBOX
     dagger init --project "$DAGGER_SANDBOX"
 
+    if [ -n "$GITHUB_ACTIONS" ];
+    then
+        export DAGGER_CACHE_TO="type=gha,mode=max,scope=docs-tests-$BATS_TEST_NAME"
+        export DAGGER_CACHE_FROM="type=gha,scope=docs-tests-$BATS_TEST_NAME"
+    fi
+
     # allows the use of `sops`
     SOPS_AGE_KEY_FILE=~/.config/dagger/keys.txt
     export SOPS_AGE_KEY_FILE

diff --git a/pkg/universe.dagger.io/bats_helpers.bash b/pkg/universe.dagger.io/bats_helpers.bash
@@ -15,6 +15,13 @@ common_setup() {
     DAGGER_LOG_FORMAT="plain"
     export DAGGER_LOG_FORMAT
 
+    export DAGGER_LOG_LEVEL="debug"
+    if [ -n "$GITHUB_ACTIONS" ];
+    then
+        export DAGGER_CACHE_TO="type=gha,mode=max,scope=universe-tests-$BATS_TEST_NAME"
+        export DAGGER_CACHE_FROM="type=gha,scope=universe-tests-$BATS_TEST_NAME"
+    fi
+
     # cd into the directory containing the bats file
     cd "$BATS_TEST_DIRNAME" || exit 1
 }

diff --git a/pkg/universe.dagger.io/package.json b/pkg/universe.dagger.io/package.json
@@ -1,7 +1,7 @@
 {
   "license": "Apache-2.0",
   "scripts": {
-    "test": "bats --report-formatter junit --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')"
+    "test": "bats --report-formatter junit --print-output-on-failure --jobs 4 $(find . -type f -name '*.bats' -not -path '*/node_modules/*')"
   },
   "devDependencies": {
     "bats": "^1.5.0",

diff --git a/plan/plan.go b/plan/plan.go
@@ -168,7 +168,7 @@ func (p *Plan) prepare(ctx context.Context) error {
 }
 
 // Do executes an action in the plan
-func (p *Plan) Do(ctx context.Context, path cue.Path, s solver.Solver) error {
+func (p *Plan) Do(ctx context.Context, path cue.Path, s *solver.Solver) error {
 	ctx, span := otel.Tracer("dagger").Start(ctx, "plan.Up")
 	defer span.End()
 

diff --git a/plan/runner.go b/plan/runner.go
@@ -22,13 +22,13 @@ import (
 type Runner struct {
 	pctx   *plancontext.Context
 	target cue.Path
-	s      solver.Solver
+	s      *solver.Solver
 	tasks  sync.Map
 	mirror *compiler.Value
 	l      sync.Mutex
 }
 
-func NewRunner(pctx *plancontext.Context, target cue.Path, s solver.Solver) *Runner {
+func NewRunner(pctx *plancontext.Context, target cue.Path, s *solver.Solver) *Runner {
 	return &Runner{
 		pctx:   pctx,
 		target: target,

diff --git a/plan/task/clientcommand.go b/plan/task/clientcommand.go
@@ -23,7 +23,7 @@ func init() {
 type clientCommandTask struct {
 }
 
-func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t clientCommandTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	var opts struct {
 		Name string
 		Args []string

diff --git a/plan/task/clientenv.go b/plan/task/clientenv.go
@@ -19,7 +19,7 @@ func init() {
 type clientEnvTask struct {
 }
 
-func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t clientEnvTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	log.Ctx(ctx).Debug().Msg("loading environment variables")
 
 	fields, err := v.Fields()

diff --git a/plan/task/clientfilesystemread.go b/plan/task/clientfilesystemread.go
@@ -21,7 +21,7 @@ func init() {
 type clientFilesystemReadTask struct {
 }
 
-func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.Context, v *compiler.Value) error {
+func (t clientFilesystemReadTask) PreRun(_ context.Context, pctx *plancontext.Context, v *compiler.Value) error {
 	path, err := t.parsePath(v)
 	if err != nil {
 		return err
@@ -38,7 +38,7 @@ func (t clientFilesystemReadTask) PreRun(ctx context.Context, pctx *plancontext.
 	return nil
 }
 
-func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t clientFilesystemReadTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	path, err := t.parsePath(v)
 	if err != nil {
 		return nil, err
@@ -70,7 +70,7 @@ func (t clientFilesystemReadTask) parsePath(v *compiler.Value) (path string, err
 	return
 }
 
-func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (interface{}, error) {
+func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (interface{}, error) {
 	lg := log.Ctx(ctx)
 	contents := v.Lookup("contents")
 
@@ -97,7 +97,7 @@ func (t clientFilesystemReadTask) readContents(ctx context.Context, pctx *planco
 	return nil, fmt.Errorf("unsupported type %q", k)
 }
 
-func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
+func (t clientFilesystemReadTask) readFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) (*compiler.Value, error) {
 	var dir struct {
 		Include []string
 		Exclude []string

diff --git a/plan/task/clientfilesystemwrite.go b/plan/task/clientfilesystemwrite.go
@@ -21,7 +21,7 @@ func init() {
 type clientFilesystemWriteTask struct {
 }
 
-func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	path, err := v.Lookup("path").String()
 	if err != nil {
 		return nil, err
@@ -39,7 +39,7 @@ func (t clientFilesystemWriteTask) Run(ctx context.Context, pctx *plancontext.Co
 	return compiler.NewValue(), nil
 }
 
-func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
+func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
 	lg := log.Ctx(ctx)
 	contents := v.Lookup("contents")
 
@@ -79,7 +79,7 @@ func (t clientFilesystemWriteTask) writeContents(ctx context.Context, pctx *plan
 	return fmt.Errorf("unsupported type %q", k)
 }
 
-func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value, path string) error {
+func (t clientFilesystemWriteTask) writeFS(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value, path string) error {
 	contents, err := pctx.FS.FromValue(v)
 	if err != nil {
 		return err

diff --git a/plan/task/clientnetwork.go b/plan/task/clientnetwork.go
@@ -20,7 +20,7 @@ func init() {
 type clientNetwork struct {
 }
 
-func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t clientNetwork) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	lg := log.Ctx(ctx)
 
 	addr, err := v.Lookup("address").String()

diff --git a/plan/task/clientplatform.go b/plan/task/clientplatform.go
@@ -16,7 +16,7 @@ func init() {
 type clientPlatformTask struct {
 }
 
-func (t clientPlatformTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t clientPlatformTask) Run(_ context.Context, _ *plancontext.Context, _ *solver.Solver, _ *compiler.Value) (*compiler.Value, error) {
 	return compiler.NewValue().FillFields(map[string]interface{}{
 		"os":   runtime.GOOS,
 		"arch": runtime.GOARCH,

diff --git a/plan/task/copy.go b/plan/task/copy.go
@@ -16,7 +16,7 @@ func init() {
 type copyTask struct {
 }
 
-func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t *copyTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	var err error
 
 	input, err := pctx.FS.FromValue(v.Lookup("input"))

diff --git a/plan/task/decodesecret.go b/plan/task/decodesecret.go
@@ -20,7 +20,7 @@ func init() {
 type decodeSecretTask struct {
 }
 
-func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (c *decodeSecretTask) Run(ctx context.Context, pctx *plancontext.Context, _ *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	lg := log.Ctx(ctx)
 	lg.Debug().Msg("decoding secret")
 

diff --git a/plan/task/diff.go b/plan/task/diff.go
@@ -16,7 +16,7 @@ func init() {
 type diffTask struct {
 }
 
-func (t diffTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t *diffTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	lowerFS, err := pctx.FS.FromValue(v.Lookup("lower"))
 	if err != nil {
 		return nil, err

diff --git a/plan/task/dockerfile.go b/plan/task/dockerfile.go
@@ -28,7 +28,7 @@ func init() {
 type dockerfileTask struct {
 }
 
-func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s solver.Solver, v *compiler.Value) (*compiler.Value, error) {
+func (t *dockerfileTask) Run(ctx context.Context, pctx *plancontext.Context, s *solver.Solver, v *compiler.Value) (*compiler.Value, error) {
 	lg := log.Ctx(ctx)
 	auths, err := v.Lookup("auth").Fields()
 	if err != nil {