Merge branch 'master' into feature/s3_passthrough

.github/workflows/go.yaml

@@ -37,14 +37,47 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: '10.x'
-      - name: install UI dependencies
+      - name: Cache node_modules # TODO(ariels): Share with UI build
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: node_modules/
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('webui/package-lock.json') }}
+          restore-keys:
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Install UI dependencies
         run: npm install
         working-directory: ./webui
       - name: Setup Go
         uses: actions/setup-go@v2
         with:
           go-version: 1.15.2
         id: go
+
+      # No way to share code between workflows :-( If you change this, find and change the
+      # same code wherever "Find Go module and build caches" appears!
+      - name: Find Go module and build caches
+        run: |
+          echo GOMODCACHE=`go env GOMODCACHE` >> $GITHUB_ENV
+          echo GOCACHE=`go env GOCACHE` >> $GITHUB_ENV
+          cat $GITHUB_ENV
+      - name: Cache Go modules and builds
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-go-modules
+        with:
+          path: |
+            ${{ env.GOMODCACHE }}
+            ${{ env.GOCACHE }}
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.mod', 'go.sum') }}
+          restore-keys:
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
       - name: Run tests
         run: |
           make test

.github/workflows/nessie.yaml

@@ -39,6 +39,27 @@ jobs:
           go-version: 1.15.2
         id: go
 
+      # No way to share code between workflows :-( If you change this, find and change the
+      # same code wherever "Find Go module and build caches" appears!
+      - name: Find Go module and build caches
+        run: |
+          echo GOMODCACHE=`go env GOMODCACHE` >> $GITHUB_ENV
+          echo GOCACHE=`go env GOCACHE` >> $GITHUB_ENV
+          cat $GITHUB_ENV
+      - name: Cache Go modules and builds
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-go-modules
+        with:
+          path: |
+            ${{ env.GOMODCACHE }}
+            ${{ env.GOCACHE }}
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.mod', 'go.sum') }}
+          restore-keys:
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
       - name: Generate code
         run: |
           make gen

.gitignore

@@ -73,5 +73,6 @@ terraform.rc
 
 # generated mock files
 /graveler/committed/**/mock/
+/pyramid/mock/*
 
 .eslintcache

README.md

@@ -3,14 +3,18 @@
 </p>
 
 <p align="center">
-	<a href="https://raw.githubusercontent.com/treeverse/lakeFS/master/LICENSE" alt="License">
-		<img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
-	<a href="https://github.com/treeverse/lakeFS/actions?query=workflow%3AGo+branch%3Amaster++" alt="Go">
-		<img src="https://github.com/treeverse/lakeFS/workflows/Go/badge.svg?branch=master" /></a>
-	<a href="https://github.com/treeverse/lakeFS/actions?query=workflow%3ANode+branch%3Amaster++" alt="Node">
-		<img src="https://github.com/treeverse/lakeFS/workflows/Node/badge.svg?branch=master" /></a>
-	<a href="https://artifacthub.io/packages/search?repo=lakefs" alt="Artifact HUB">
-		<img src="https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/lakefs" /></a>
+	<a href="https://raw.githubusercontent.com/treeverse/lakeFS/master/LICENSE" >
+		<img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" alt="Apache License" /></a>
+	<a href="https://github.com/treeverse/lakeFS/actions?query=workflow%3AGo+branch%3Amaster++">
+		<img src="https://github.com/treeverse/lakeFS/workflows/Go/badge.svg?branch=master" alt="Go tests status" /></a>
+	<a href="https://github.com/treeverse/lakeFS/actions?query=workflow%3ANode+branch%3Amaster++" >
+		<img src="https://github.com/treeverse/lakeFS/workflows/Node/badge.svg?branch=master" alt="Node tests status" /></a>
+	<a href="https://github.com/treeverse/lakeFS/actions?query=workflow%3ANessie">
+		<img src="https://github.com/treeverse/lakeFS/workflows/Nessie/badge.svg" alt="nessie integration tests status" /></a>
+	<a href="https://artifacthub.io/packages/search?repo=lakefs">
+		<img src="https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/lakefs" alt="Artifact HUB" /></a>
+	<a href="CODE_OF_CONDUCT.md">
+		<img src="https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg" alt="code of conduct"></a>
 </p>
 
 

cache/refcnted.go → cache/reference_counted.go

@@ -38,6 +38,7 @@ type ParamsWithDisposal struct {
 // keyus have a String() method that is compatible with their equality, i.e. k1.GoString() ==
 // k2.GoString() implies k1 equals k2 in the sense of map comparison.
 type shardedCacheWithDisposal struct {
+	name   string
 	Seed   maphash.Seed
 	Shards []CacheWithDisposal
 }
@@ -50,7 +51,11 @@ func NewCacheWithDisposal(p ParamsWithDisposal) *shardedCacheWithDisposal {
 	for i := 0; i < p.Shards; i++ {
 		shards[i] = NewSingleThreadedCacheWithDisposal(p)
 	}
-	return &shardedCacheWithDisposal{Seed: maphash.MakeSeed(), Shards: shards}
+	return &shardedCacheWithDisposal{name: p.Name, Seed: maphash.MakeSeed(), Shards: shards}
+}
+
+func (s *shardedCacheWithDisposal) Name() string {
+	return s.name
 }
 
 func (s *shardedCacheWithDisposal) GetOrSet(k interface{}, setFn SetFn) (interface{}, Derefer, error) {

catalog/rocks/cataloger.go

@@ -12,7 +12,7 @@ import (
 )
 
 type cataloger struct {
-	EntryCatalog EntryCatalog
+	EntryCatalog *EntryCatalog
 	log          logging.Logger
 	dummyDedupCh chan *catalog.DedupReport
 	hooks        catalog.CatalogerHooks
@@ -655,7 +655,19 @@ func listDiffHelper(it EntryDiffIterator, limit int, after string) (catalog.Diff
 }
 
 func (c *cataloger) Merge(ctx context.Context, repository string, leftBranch string, rightBranch string, committer string, message string, metadata catalog.Metadata) (*catalog.MergeResult, error) {
-	panic("not implemented") // TODO: Implement
+	repositoryID := graveler.RepositoryID(repository)
+	leftRef := graveler.Ref(leftBranch)
+	rightBranchID := graveler.BranchID(rightBranch)
+	meta := graveler.Metadata(metadata)
+	commitID, err := c.EntryCatalog.Merge(ctx, repositoryID, leftRef, rightBranchID, committer, message, meta)
+	if err != nil {
+		return nil, err
+	}
+	return &catalog.MergeResult{
+		// TODO(barak): require implementation by graveler's merge
+		Summary:   map[catalog.DifferenceType]int{},
+		Reference: commitID.String(),
+	}, nil
 }
 
 func (c *cataloger) Hooks() *catalog.CatalogerHooks {

catalog/rocks/cataloger_test.go

@@ -91,7 +91,7 @@ func Test_cataloger_ListRepositories(t *testing.T) {
 				RepositoryIterator: NewFakeRepositoryIterator(gravelerData),
 			}
 			c := &cataloger{
-				EntryCatalog: &entryCatalog{
+				EntryCatalog: &EntryCatalog{
 					store: gravelerMock,
 				},
 			}
@@ -186,7 +186,7 @@ func TestCataloger_ListBranches(t *testing.T) {
 				BranchIterator: NewFakeBranchIterator(gravelerData),
 			}
 			c := &cataloger{
-				EntryCatalog: &entryCatalog{
+				EntryCatalog: &EntryCatalog{
 					store: gravelerMock,
 				},
 			}
@@ -309,7 +309,7 @@ func TestCataloger_ListEntries(t *testing.T) {
 				ListIterator: NewFakeValueIterator(gravelerData),
 			}
 			c := &cataloger{
-				EntryCatalog: &entryCatalog{
+				EntryCatalog: &EntryCatalog{
 					store: gravelerMock,
 				},
 			}