pull latest

.all-contributorsrc

@@ -0,0 +1,140 @@
+{
+  "projectName": "lakeFS",
+  "projectOwner": "treeverse",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "CONTRIBUTORS.md"
+  ],
+  "imageSize": 350,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "ozkatz",
+      "name": "Oz Katz",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/205955?v=4",
+      "profile": "https://lakefs.io/",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "nopcoder",
+      "name": "Barak Amar",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/8110?v=4",
+      "profile": "https://github.com/nopcoder",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tzahij",
+      "name": "tzahij",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/5455113?v=4",
+      "profile": "https://github.com/tzahij",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "guy-har",
+      "name": "guy-har",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/60321938?v=4",
+      "profile": "https://github.com/guy-har",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "arielshaqed",
+      "name": "arielshaqed",
+      "avatar_url": "https://avatars0.githubusercontent.com/u/7802932?v=4",
+      "profile": "https://github.com/arielshaqed",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "johnnyaug",
+      "name": "johnnyaug",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/3149885?v=4",
+      "profile": "https://github.com/johnnyaug",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "itaiad200",
+      "name": "itaiad200",
+      "avatar_url": "https://avatars1.githubusercontent.com/u/9861474?v=4",
+      "profile": "https://github.com/itaiad200",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "YaelRiv",
+      "name": "YaelRiv",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/67264175?v=4",
+      "profile": "https://github.com/YaelRiv",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "einato",
+      "name": "Einat Orr",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/56227795?v=4",
+      "profile": "https://github.com/einato",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "kzuri",
+      "name": "Devarsh Patel",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/65841886?v=4",
+      "profile": "https://github.com/kzuri",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "sourhub226",
+      "name": "Sourabh Sathe",
+      "avatar_url": "https://avatars0.githubusercontent.com/u/58329492?v=4",
+      "profile": "https://github.com/sourhub226",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "jengjeng",
+      "name": "Wittawat Patcharinsak",
+      "avatar_url": "https://avatars2.githubusercontent.com/u/6948085?v=4",
+      "profile": "https://github.com/jengjeng",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "d-harel",
+      "name": "d-harel",
+      "avatar_url": "https://avatars3.githubusercontent.com/u/11721947?v=4",
+      "profile": "https://github.com/d-harel",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "dependabot[bot]",
+      "name": "dependabot[bot]",
+      "avatar_url": "https://avatars0.githubusercontent.com/in/29110?v=4",
+      "profile": "https://github.com/apps/dependabot",
+      "contributions": [
+        "code"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7
+}

.github/workflows/benchmark.yaml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - master
+
 jobs:
   deploy-executor-image:
     name: Build and push benchmark executor Docker image
@@ -70,14 +71,17 @@ jobs:
           aws-region: us-east-1
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 0.13.4
       - name: Terraform init
         run: |
           cd benchmarks/terraform/
           terraform init
       - name: Terraform apply
         run: |
           cd benchmarks/terraform/
-          terraform apply --auto-approve -var="dockerReg=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com" -var="tag=${{ steps.version.outputs.tag }}"
+          terraform apply --auto-approve -var="dockerReg=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.us-east-1.amazonaws.com" -var="tag=${{ steps.version.outputs.tag }}" \
+            -var="build=${{ github.run_number }}" -var="grafana-username=${{ secrets.GRAFANA_USERNAME }}" -var="grafana-password=${{ secrets.GRAFANA_PASSWORD }}"
       - name: 'Terraform Output'
         id: terraform
         uses: hashicorp/terraform-github-actions@master
@@ -101,4 +105,3 @@ jobs:
           # keep the logs
           terraform state rm aws_cloudwatch_log_group.benchmark
           terraform destroy --auto-approve
-

.github/workflows/nessie.yaml

@@ -6,8 +6,23 @@ on:
       - master
 
 jobs:
+  check-secrets:
+    name: Check if secrets are available.
+    outputs:
+      secretsavailable: ${{ steps.enablejobs.outputs.secretsavailable }}
+    runs-on: ubuntu-latest
+    steps:
+      - id: enablejobs
+        env:
+          ENABLE_NEXT_JOBS: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        run: |
+          echo "Enable next jobs based on secrets existence: ${{ env.ENABLE_NEXT_JOBS != '' }}"
+          echo "::set-output name=secretsavailable::${{ env.ENABLE_NEXT_JOBS != '' }}"
+
   deploy-image:
     name: Build and push Docker image
+    needs: check-secrets
+    if: needs.check-secrets.outputs.secretsavailable == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -49,8 +64,15 @@ jobs:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY_LAKEFS: lakefs
         run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY_LAKEFS:${{ steps.version.outputs.tag }} --build-arg VERSION=${{ steps.version.outputs.tag }} .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY_LAKEFS:${{ steps.version.outputs.tag }}
+          set +e
+          describe_image="$( aws ecr describe-images --repository-name $ECR_REPOSITORY_LAKEFS --image-ids imageTag=${{ steps.version.outputs.tag }})"
+          if [ $? -eq 0 ]; then
+            echo "Image exists"
+          else
+            echo "Image doesn't exist"
+            docker build -t $ECR_REGISTRY/$ECR_REPOSITORY_LAKEFS:${{ steps.version.outputs.tag }} --build-arg VERSION=${{ steps.version.outputs.tag }} .
+            docker push $ECR_REGISTRY/$ECR_REPOSITORY_LAKEFS:${{ steps.version.outputs.tag }}
+          fi
 
   run-system:
     name: Run latest lakeFS app

CONTRIBUTORS.md

@@ -0,0 +1,28 @@
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://lakefs.io/"><img src="https://avatars2.githubusercontent.com/u/205955?v=4?s=350" width="350px;" alt=""/><br /><sub><b>Oz Katz</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=ozkatz" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/nopcoder"><img src="https://avatars1.githubusercontent.com/u/8110?v=4?s=350" width="350px;" alt=""/><br /><sub><b>Barak Amar</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=nopcoder" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tzahij"><img src="https://avatars1.githubusercontent.com/u/5455113?v=4?s=350" width="350px;" alt=""/><br /><sub><b>tzahij</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=tzahij" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/guy-har"><img src="https://avatars3.githubusercontent.com/u/60321938?v=4?s=350" width="350px;" alt=""/><br /><sub><b>guy-har</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=guy-har" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/arielshaqed"><img src="https://avatars0.githubusercontent.com/u/7802932?v=4?s=350" width="350px;" alt=""/><br /><sub><b>arielshaqed</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=arielshaqed" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/johnnyaug"><img src="https://avatars1.githubusercontent.com/u/3149885?v=4?s=350" width="350px;" alt=""/><br /><sub><b>johnnyaug</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=johnnyaug" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/itaiad200"><img src="https://avatars1.githubusercontent.com/u/9861474?v=4?s=350" width="350px;" alt=""/><br /><sub><b>itaiad200</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=itaiad200" title="Code">💻</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/YaelRiv"><img src="https://avatars2.githubusercontent.com/u/67264175?v=4?s=350" width="350px;" alt=""/><br /><sub><b>YaelRiv</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=YaelRiv" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/einato"><img src="https://avatars3.githubusercontent.com/u/56227795?v=4?s=350" width="350px;" alt=""/><br /><sub><b>Einat Orr</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=einato" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/kzuri"><img src="https://avatars2.githubusercontent.com/u/65841886?v=4?s=350" width="350px;" alt=""/><br /><sub><b>Devarsh Patel</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=kzuri" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/sourhub226"><img src="https://avatars0.githubusercontent.com/u/58329492?v=4?s=350" width="350px;" alt=""/><br /><sub><b>Sourabh Sathe</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=sourhub226" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/jengjeng"><img src="https://avatars2.githubusercontent.com/u/6948085?v=4?s=350" width="350px;" alt=""/><br /><sub><b>Wittawat Patcharinsak</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=jengjeng" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/d-harel"><img src="https://avatars3.githubusercontent.com/u/11721947?v=4?s=350" width="350px;" alt=""/><br /><sub><b>d-harel</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=d-harel" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/apps/dependabot"><img src="https://avatars0.githubusercontent.com/in/29110?v=4?s=350" width="350px;" alt=""/><br /><sub><b>dependabot[bot]</b></sub></a><br /><a href="https://github.com/treeverse/lakeFS/commits?author=dependabot[bot]" title="Code">💻</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->

Makefile

@@ -39,6 +39,18 @@ all: build
 clean:
 	@rm -rf $(API_BUILD_DIR) $(UI_BUILD_DIR) ddl/statik.go statik $(LAKEFS_BINARY_NAME) $(LAKECTL_BINARY_NAME)
 
+check-licenses: check-licenses-go-mod check-licenses-npm
+
+check-licenses-go-mod:
+	go get github.com/google/go-licenses
+	$(GOBINPATH)/go-licenses check ./cmd/$(LAKEFS_BINARY_NAME)
+	$(GOBINPATH)/go-licenses check ./cmd/$(LAKECTL_BINARY_NAME)
+
+check-licenses-npm:
+	go get github.com/senseyeio/diligent/cmd/diligent
+	# The -i arg is a workaround to ignore NPM scoped packages until https://github.com/senseyeio/diligent/issues/77 is fixed
+	$(GOBINPATH)/diligent check -w permissive -i ^@[^/]+?/[^/]+ $(UI_DIR)
+
 docs/assets/js/swagger.yml: swagger.yml
 	@cp swagger.yml docs/assets/js/swagger.yml
 

README.md

@@ -2,10 +2,16 @@
   <img src="logo_large.png"/>
 </p>
 
-[![Hacktoberfest](https://badgen.net/badge/hacktoberfest/friendly/pink)](docs/contributing.md)
-[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://raw.githubusercontent.com/treeverse/lakeFS/master/LICENSE)
-[![Go](https://github.com/treeverse/lakeFS/workflows/Go/badge.svg?branch=master)](https://github.com/treeverse/lakeFS/actions?query=workflow%3AGo+branch%3Amaster++)
-[![Node](https://github.com/treeverse/lakeFS/workflows/Node/badge.svg?branch=master)](https://github.com/treeverse/lakeFS/actions?query=workflow%3ANode+branch%3Amaster++)
+<p align="center">
+	<a href="docs/contributing.md" alt="Hacktoberfest">
+		<img src="https://badgen.net/badge/hacktoberfest/friendly/pink" /></a>
+	<a href="https://raw.githubusercontent.com/treeverse/lakeFS/master/LICENSE" alt="License">
+		<img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
+	<a href="https://github.com/treeverse/lakeFS/actions?query=workflow%3AGo+branch%3Amaster++" alt="Go">
+		<img src="https://github.com/treeverse/lakeFS/workflows/Go/badge.svg?branch=master" /></a>
+	<a href="https://github.com/treeverse/lakeFS/actions?query=workflow%3ANode+branch%3Amaster++" alt="Node">
+		<img src="https://github.com/treeverse/lakeFS/workflows/Node/badge.svg?branch=master" /></a>
+</p>
 
 
 ## What is lakeFS
@@ -18,60 +24,33 @@ lakeFS supports AWS S3 or Google Cloud Storage as its underlying storage service
 
 
 <p align="center">
-  <img src="docs/assets/img/wrapper.png"/>
+  <img src="docs/assets/img/wrapper.png" width="650px"/>
 </p>
 
 For more information see the [Official Documentation](https://docs.lakefs.io).
 
 <div style="background: #ffdddd border: 3px solid #dd4444; margine: 15px;">
 
-# A Hacktoberfest update
-
-Welcome Hacktoberfest participants!  We commit to actively seek, help, and merge your
-improvements to lakeFS.  We've labelled some [issues with the hacktoberfest
-label](https://github.com/treeverse/lakeFS/issues?q=is%3Aissue+is%3Aopen+label%3Ahacktoberfest).
-Please check out our [contributing guide](https://docs.lakefs.io/contributing).
-
-*We know you like badges, stickers, and T-shirts*, because **we like them too!** But, like many
-other open-source projects, we are seeing an influx of lower quality PRs.  During October will
-be unable to accept PRs if they:
-
-1. Only change punctuation or grammar, unless accompanied by an explanation or are clearly
-   better.
-1. Repeat an existing PR, or try to merge branches authored by other contributors that are under
-   active work.
-1. Do not affect generated code or documentation in any way.
-1. Are detrimental: do not compile or cause harm when run.
-1. Change text or code that should be changed upstream, such as licenses, code of conduct, or
-   React boilerplate.
-
-We shall close such PRs and label them `x/invalid`; Digital Ocean _will not count_ those PRs
-towards Hacktoberfest progress, so such PRs only waste your time and ours.
-
-You can **help us accept your PR** by adding a clear title and description to the PR and to
-commits in that PR.  "Fixes #1234" or "update README.md" are not as good as "Make lakeFS run 3x
-faster" or "Add update regarding Hacktoberfest".  Communication is key: If you are uncertain,
-please open a discussion: ask us on the PR or on the issue.
-
-Thanks!
-</div>
 
 ## Capabilities
 
 **Development Environment for Data**
 * **Experimentation** - try tools, upgrade versions and evaluate code changes in isolation. 
 * **Reproducibility** - go back to any point of time to a consistent version of your data lake.
+* [Learn more](https://github.com/treeverse/lakeFS/blob/master/docs/branching/recommendations.md#development-environment)
 
 **Continuous Data Integration**
 * **Ingest new data safely by enforcing best practices** - make sure new data sources adhere to your lake’s best practices such as format and schema enforcement, naming convention, etc.  
 * **Metadata validation** - prevent breaking changes from entering the production data environment.
+* [Learn more](https://github.com/treeverse/lakeFS/blob/master/docs/branching/recommendations.md#continuous-integration)
 
 **Continuous Data Deployment**
 * **Instantly revert changes to data** - if low quality data is exposed to your consumers, you can revert instantly to a former, consistent and correct snapshot of your data lake.
 * **Enforce cross collection consistency** - provide to consumers several collections of data that must be synchronized, in one atomic, revertible, action.
 * **Prevent data quality issues by enabling**
   - Testing of production data before exposing it to users / consumers.
   - Testing of intermediate results in your DAG to avoid cascading quality issues.
+* [Learn more](https://github.com/treeverse/lakeFS/blob/master/docs/branching/recommendations.md#continuous-deployment)
 
 ## Getting Started