Merge remote-tracking branch 'origin/master' into 585-setup-bench-env

.env

@@ -1,2 +1,2 @@
-TAG=0.10.2
-LAKEFS_BLOCKSTORE_TYPE=local
+TAG=latest
+LAKEFS_BLOCKSTORE_TYPE=local

.github/workflows/nessie.yaml

@@ -110,6 +110,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.NESSIE_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.NESSIE_AWS_SECRET_ACCESS_KEY }}
         run: |
+            cd nessie/ops
             docker-compose ps -q postgres && docker-compose exec -T postgres pg_dumpall --username=lakefs | gzip | aws s3 cp - s3://nessie-system-testing/${{ github.run_number }}/dump.gz
       - name: Run lakeFS GS
         env:

README.md

@@ -43,23 +43,30 @@ For more information see the [Official Documentation](https://docs.lakefs.io).
 
 ## Getting Started
 
-#### Docker
+#### Docker (MacOS, Linux)
 
-1. Ensure you have Docker installed on your computer. The MacOS and Windows installations include Docker Compose by default.
+1. Ensure you have Docker & Docker Compose installed on your computer.
 
-2. Clone the repository:
+2. Run the following command:
 
    ```bash
-   git clone git@github.com:treeverse/lakeFS.git
+   curl https://compose.lakefs.io | docker-compose -f - up
    ```
 
-3. From the root of the cloned repository, run:
+3. Open [http://127.0.0.1:8000/setup](http://127.0.0.1:8000/setup) in your web browser to set up an initial admin user, used to login and send API requests.
 
-   ```bash
-   $ docker-compose up
-   ```
 
-4. Open [http://localhost:8000/setup](http://localhost:8000/setup) in your web browser to set up an initial admin user, used to login and send API requests.
+#### Docker (Windows)
+
+1. Ensure you have Docker installed
+
+2. Run the following command in PowerShell:
+
+   ```shell script
+   Invoke-WebRequest https://compose.lakefs.io | Select-Object -ExpandProperty Content | docker-compose -f - up
+   ``` 
+
+3. Open [http://127.0.0.1:8000/setup](http://127.0.0.1:8000/setup) in your web browser to set up an initial admin user, used to login and send API requests.
 
 #### Download the Binary
 
@@ -70,9 +77,9 @@ Binaries are available at [https://github.com/treeverse/lakeFS/releases](https:/
 
 #### Setting up a repository
 
-Please follow the [Guide to Get Started](https://docs.lakefs.io/quickstart.html#setting-up-a-repository) to set up your local lakeFS installation.
+Please follow the [Guide to Get Started](https://docs.lakefs.io/quickstart/repository) to set up your local lakeFS installation.
 
-For more detailed inforamation on how to setup lakeFS, please visit [the documentation](https://docs.lakefs.io)
+For more detailed information on how to set up lakeFS, please visit [the documentation](https://docs.lakefs.io)
 
 ## Community
 
@@ -83,11 +90,6 @@ Keep up to date and get lakeFS support via:
 - [YouTube](https://www.youtube.com/channel/UCZiDUd28ex47BTLuehb1qSA) (learn from video tutorials)
 - [Contact us](https://lakefs.io/contact-us/) (for anything)
 
-## Get ready for Hacktoberfest!
-
-Contribute to open-source projects throughout October 2020 by stepping up to Digital Ocean's annual tradition of hosting [Hacktoberfest](https://hacktoberfest.digitalocean.com/).  We _always_ welcome pull requests to lakeFS - but throughout October your pull requests to open source projects can get you some cool swag.  Check out our [contributing guide](https://docs.lakefs.io/contributing) and join our [slack channel](https://join.slack.com/t/lakefs/shared_invite/zt-g86mkroy-186GzaxR4xOar1i1Us0bzw) for help, more ideas, and discussions. 
-
-Not sure what to do?  We marked some issues that could get you started quickly on our [Hacktoberfest list](https://github.com/treeverse/lakeFS/issues?q=is%3Aissue+is%3Aopen+label%3Ahacktoberfest).
 
 ## More information
 

api/api_controller.go

@@ -2245,7 +2245,7 @@ func (c *Controller) ImportFromS3InventoryHandler() repositories.ImportFromS3Inv
 		if err == nil {
 			username = userModel.Username
 		}
-		importConfig := &onboard.ImporterConfig{
+		importConfig := &onboard.Config{
 			CommitUsername:     username,
 			InventoryURL:       params.ManifestURL,
 			Repository:         params.Repository,
@@ -2257,8 +2257,9 @@ func (c *Controller) ImportFromS3InventoryHandler() repositories.ImportFromS3Inv
 			return repositories.NewImportFromS3InventoryDefault(http.StatusInternalServerError).
 				WithPayload(responseErrorFrom(err))
 		}
-		var importStats *onboard.InventoryImportStats
-		if *params.DryRun {
+		var importStats *onboard.Stats
+		dryRun := swag.BoolValue(params.DryRun)
+		if dryRun {
 			importStats, err = importer.Import(deps.ctx, true)
 			if err != nil {
 				return repositories.NewImportFromS3InventoryDefault(http.StatusInternalServerError).
@@ -2288,7 +2289,7 @@ func (c *Controller) ImportFromS3InventoryHandler() repositories.ImportFromS3Inv
 			}
 		}
 		return repositories.NewImportFromS3InventoryCreated().WithPayload(&repositories.ImportFromS3InventoryCreatedBody{
-			IsDryRun:           *params.DryRun,
+			IsDryRun:           dryRun,
 			PreviousImportDate: importStats.PreviousImportDate.Unix(),
 			PreviousManifest:   importStats.PreviousInventoryURL,
 			AddedOrChanged:     int64(importStats.AddedOrChanged),

auth/service.go

@@ -86,15 +86,15 @@ func ListPaged(db db.Database, retType reflect.Type, params *model.PaginationPar
 	queryBuilder = queryBuilder.OrderBy(tokenColumnName)
 	if params != nil {
 		queryBuilder = queryBuilder.Where(sq.Gt{tokenColumnName: params.After})
-	}
-	if params.Amount >= 0 {
-		queryBuilder = queryBuilder.Limit(uint64(params.Amount) + 1)
+		if params.Amount >= 0 {
+			queryBuilder = queryBuilder.Limit(uint64(params.Amount) + 1)
+		}
 	}
 	query, args, err := queryBuilder.ToSql()
 	if err != nil {
 		return nil, nil, fmt.Errorf("convert to SQL: %w", err)
 	}
-	rows, err := db.Queryx(query, args...)
+	rows, err := db.Query(query, args...)
 	if err != nil {
 		return nil, nil, fmt.Errorf("query DB: %w", err)
 	}
@@ -106,7 +106,7 @@ func ListPaged(db db.Database, retType reflect.Type, params *model.PaginationPar
 		slice = reflect.Append(slice, value)
 	}
 	p := &model.Paginator{}
-	if params.Amount >= 0 && slice.Len() == params.Amount+1 {
+	if params != nil && params.Amount >= 0 && slice.Len() == params.Amount+1 {
 		// we have more pages
 		slice = slice.Slice(0, params.Amount)
 		p.Amount = params.Amount

auth/service_test.go

@@ -9,31 +9,53 @@ import (
 	"testing"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
+	sq "github.com/Masterminds/squirrel"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/go-test/deep"
-	"github.com/treeverse/lakefs/db"
-	"github.com/treeverse/lakefs/permissions"
-
 	"github.com/google/uuid"
-
-	sq "github.com/Masterminds/squirrel"
-
 	"github.com/ory/dockertest/v3"
 	"github.com/treeverse/lakefs/auth"
 	"github.com/treeverse/lakefs/auth/crypt"
 	"github.com/treeverse/lakefs/auth/model"
 	authparams "github.com/treeverse/lakefs/auth/params"
+	"github.com/treeverse/lakefs/db"
 	"github.com/treeverse/lakefs/logging"
+	"github.com/treeverse/lakefs/permissions"
 	"github.com/treeverse/lakefs/testutil"
 )
 
 var (
 	pool        *dockertest.Pool
 	databaseURI string
 	psql        = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)
+	someSecret  = []byte("some secret")
+
+	userPoliciesForTesting = []*model.Policy{
+		{
+			Statement: model.Statements{
+				{
+					Action:   []string{"auth:DeleteUser"},
+					Resource: "arn:lakefs:auth:::user/foobar",
+					Effect:   model.StatementEffectAllow,
+				},
+			},
+		},
+		{
+			Statement: model.Statements{
+				{
+					Action:   []string{"auth:*"},
+					Resource: "*",
+					Effect:   model.StatementEffectDeny,
+				},
+			},
+		},
+	}
 )
 
 func TestMain(m *testing.M) {
+	logrus.SetLevel(logrus.PanicLevel)
 	var err error
 	var closer func()
 	pool, err = dockertest.NewPool("")
@@ -46,15 +68,15 @@ func TestMain(m *testing.M) {
 	os.Exit(code)
 }
 
-func setupService(t *testing.T, opts ...testutil.GetDBOption) auth.Service {
+func setupService(t testing.TB, opts ...testutil.GetDBOption) auth.Service {
 	adb, _ := testutil.GetDB(t, databaseURI, opts...)
-	authService := auth.NewDBAuthService(adb, crypt.NewSecretStore([]byte("some secret")), authparams.ServiceCache{
+	authService := auth.NewDBAuthService(adb, crypt.NewSecretStore(someSecret), authparams.ServiceCache{
 		Enabled: false,
 	})
 	return authService
 }
 
-func userWithPolicies(t *testing.T, s auth.Service, policies []*model.Policy) string {
+func userWithPolicies(t testing.TB, s auth.Service, policies []*model.Policy) string {
 	userName := uuid.New().String()
 	err := s.CreateUser(&model.User{
 		Username: userName,
@@ -82,10 +104,10 @@ func userWithPolicies(t *testing.T, s auth.Service, policies []*model.Policy) st
 
 func TestDBAuthService_ListPaged(t *testing.T) {
 	const chars = "abcdefghijklmnopqrstuvwxyz"
-	db, _ := testutil.GetDB(t, databaseURI)
-	defer db.Close()
+	adb, _ := testutil.GetDB(t, databaseURI)
+	defer adb.Close()
 	type row struct{ A string }
-	if _, err := db.Exec(`CREATE TABLE test_pages (a text PRIMARY KEY)`); err != nil {
+	if _, err := adb.Exec(`CREATE TABLE test_pages (a text PRIMARY KEY)`); err != nil {
 		t.Fatalf("CREATE TABLE test_pages: %s", err)
 	}
 	insert := psql.Insert("test_pages")
@@ -96,7 +118,7 @@ func TestDBAuthService_ListPaged(t *testing.T) {
 	if err != nil {
 		t.Fatalf("create insert statement %v: %s", insert, err)
 	}
-	if _, err = db.Exec(insertSql, args...); err != nil {
+	if _, err = adb.Exec(insertSql, args...); err != nil {
 		t.Fatalf("%s [%v]: %s", insertSql, args, err)
 	}
 
@@ -109,13 +131,13 @@ func TestDBAuthService_ListPaged(t *testing.T) {
 			got := ""
 			for {
 				values, paginator, err := auth.ListPaged(
-					db, reflect.TypeOf(row{}), pagination, "A", psql.Select("a").From("test_pages"))
+					adb, reflect.TypeOf(row{}), pagination, "A", psql.Select("a").From("test_pages"))
 				if err != nil {
 					t.Errorf("ListPaged: %s", err)
 					break
 				}
 				if values == nil {
-					t.Errorf("expected values for pagination %+v but got just paginator %+v", pagination, paginator)
+					t.Fatalf("expected values for pagination %+v but got just paginator %+v", pagination, paginator)
 				}
 				letters := values.Interface().([]*row)
 				for _, c := range letters {
@@ -491,7 +513,6 @@ func TestDBAuthService_ListUsers(t *testing.T) {
 }
 
 func TestDBAuthService_ListUserCredentials(t *testing.T) {
-	const numCredentials = 5
 	const userName = "accredited"
 	s := setupService(t)
 	if err := s.CreateUser(&model.User{Username: userName}); err != nil {
@@ -569,8 +590,8 @@ func TestDbAuthService_GetUser(t *testing.T) {
 	const userName = "foo"
 	// Time should *not* have nanoseconds - otherwise we are comparing accuracy of golang
 	// and Postgres time storage.
-	time := time.Date(2222, 2, 22, 22, 22, 22, 0, time.UTC)
-	if err := s.CreateUser(&model.User{Username: userName, ID: -22, CreatedAt: time}); err != nil {
+	ts := time.Date(2222, 2, 22, 22, 22, 22, 0, time.UTC)
+	if err := s.CreateUser(&model.User{Username: userName, ID: -22, CreatedAt: ts}); err != nil {
 		t.Fatalf("CreateUser(%s): %s", userName, err)
 	}
 	user, err := s.GetUser(userName)
@@ -580,8 +601,8 @@ func TestDbAuthService_GetUser(t *testing.T) {
 	if user.Username != userName {
 		t.Errorf("GetUser(%s) returned user %+v with a different name", userName, user)
 	}
-	if user.CreatedAt.Sub(time) != 0 {
-		t.Errorf("expected user CreatedAt %s, got %+v", time, user.CreatedAt)
+	if user.CreatedAt.Sub(ts) != 0 {
+		t.Errorf("expected user CreatedAt %s, got %+v", ts, user.CreatedAt)
 	}
 	if user.ID == -22 {
 		t.Errorf("expected CreateUser ID:-22 to be dropped on server, got user %+v", user)
@@ -593,8 +614,8 @@ func TestDbAuthService_GetUserById(t *testing.T) {
 	const userName = "foo"
 	// Time should *not* have nanoseconds - otherwise we are comparing accuracy of golang
 	// and Postgres time storage.
-	time := time.Date(2222, 2, 22, 22, 22, 22, 0, time.UTC)
-	if err := s.CreateUser(&model.User{Username: userName, ID: -22, CreatedAt: time}); err != nil {
+	ts := time.Date(2222, 2, 22, 22, 22, 22, 0, time.UTC)
+	if err := s.CreateUser(&model.User{Username: userName, ID: -22, CreatedAt: ts}); err != nil {
 		t.Fatalf("CreateUser(%s): %s", userName, err)
 	}
 	user, err := s.GetUser(userName)
@@ -630,3 +651,44 @@ func TestDBAuthService_DeleteUser(t *testing.T) {
 		t.Errorf("GetUser(%s) after deletion: %s", userName, err)
 	}
 }
+
+func BenchmarkDBAuthService_ListEffectivePolicies(b *testing.B) {
+	// setup user with policies for benchmark
+	adb, _ := testutil.GetDB(b, databaseURI)
+	serviceWithoutCache := auth.NewDBAuthService(adb, crypt.NewSecretStore(someSecret), authparams.ServiceCache{
+		Enabled: false,
+	})
+	serviceWithCache := auth.NewDBAuthService(adb, crypt.NewSecretStore(someSecret), authparams.ServiceCache{
+		Enabled:        true,
+		Size:           1024,
+		TTL:            20 * time.Second,
+		EvictionJitter: 3 * time.Second,
+	})
+	serviceWithCacheLowTTL := auth.NewDBAuthService(adb, crypt.NewSecretStore(someSecret), authparams.ServiceCache{
+		Enabled:        true,
+		Size:           1024,
+		TTL:            1 * time.Millisecond,
+		EvictionJitter: 1 * time.Millisecond,
+	})
+	userName := userWithPolicies(b, serviceWithoutCache, userPoliciesForTesting)
+
+	b.Run("without_cache", func(b *testing.B) {
+		benchmarkListEffectivePolicies(b, serviceWithoutCache, userName)
+	})
+	b.Run("with_cache", func(b *testing.B) {
+		benchmarkListEffectivePolicies(b, serviceWithCache, userName)
+	})
+	b.Run("without_cache_low_ttl", func(b *testing.B) {
+		benchmarkListEffectivePolicies(b, serviceWithCacheLowTTL, userName)
+	})
+}
+
+func benchmarkListEffectivePolicies(b *testing.B, s *auth.DBAuthService, userName string) {
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		_, _, err := s.ListEffectivePolicies(userName, &model.PaginationParams{Amount: -1})
+		if err != nil {
+			b.Fatal("Failed to list effective policies", err)
+		}
+	}
+}