chore: merge

Sefaria · Jan 18, 2023 · 1cf9aad · 1cf9aad
2 parents fc1bffb + 5c3aabb
commit 1cf9aad
Show file tree

Hide file tree

Showing 123 changed files with 3,362 additions and 4,139 deletions.
diff --git a/.github/workflows/continuous.yaml b/.github/workflows/continuous.yaml
@@ -44,10 +44,10 @@ jobs:
       - name: Format branch name
         id: branch-name
         run: |
-          echo "::set-output name=current_branch::"$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g')
+          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g') >> $GITHUB_OUTPUT
       - name: Get current date
         id: date
-        run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')"
+        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
       - name: Generate image metadata
         id: meta
         uses: docker/metadata-action@v3
@@ -98,10 +98,10 @@ jobs:
       - name: Format branch name
         id: branch-name
         run: |
-          echo "::set-output name=current_branch::"$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g')
+          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g') >> $GITHUB_OUTPUT
       - name: Get current date
         id: date
-        run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')"
+        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
       - name: Generate image metadata
         id: meta
         uses: docker/metadata-action@v3
@@ -117,7 +117,7 @@ jobs:
             latest=true
       - name: Set outputs
         id: get-sha
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: build and push
         uses: docker/build-push-action@v2
         with:
@@ -159,26 +159,34 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v2
       - name: Authenticate GHA Runner to GCP
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
+      - name: Setup GCloud
         uses: google-github-actions/setup-gcloud@v0
         with:
-          service_account_key: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
           project_id: ${{ secrets.DEV_PROJECT }}
+          install_components: 'gke-gcloud-auth-plugin'
       - name: Get branch name
         id: branch-raw
         uses: tj-actions/branch-names@v5.1
       - name: Format branch name
         id: branch-name
         run: |
-          echo "::set-output name=current_branch::"$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g')
+          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g') >> $GITHUB_OUTPUT
       - name: Set outputs
         id: get-sha
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Set up yq
         uses: frenck/action-setup-yq@v1
-      - name: setup helm
-        uses: azure/setup-helm@v1
+          #      - name: setup helm
+          #        uses: azure/setup-helm@v3
       - name: Authenticate GHA Runner To Target Cluster
-        run: gcloud container clusters get-credentials ${{secrets.DEV_GKE_CLUSTER}} --zone ${{secrets.DEV_GKE_REGION}} --project ${{secrets.DEV_GCP_PROJECT}}
+        uses: google-github-actions/get-gke-credentials@v0
+        with:
+          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
+          location: ${{secrets.DEV_GKE_REGION}}
+          project_id: ${{secrets.DEV_GCP_PROJECT}}
       - name: Deploy Sandbox
         run: ./build/ci/sandbox-helm-deploy.sh build/ci/values.yaml
         env:
@@ -195,78 +203,13 @@ jobs:
         uses: actions/checkout@v2
       - name: Set outputs
         id: get-sha
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Wait for test sandbox to become available 
         # https://gist.github.com/rgl/f90ff293d56dbb0a1e0f7e7e89a81f42
         run: ./build/ci/waitForSandbox.bash
         env:
           WAIT_DURATION: "3000"
           GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}"
-  selenium-sauce:
-    if: ${{ github.event_name == 'pull_request' }}
-    name: "Selenium"   # This name is referenced when slacking status
-    needs: sandbox-ready
-    runs-on: ubuntu-latest
-    steps:
-      # BEGIN SETUP
-      - name: Checkout Code
-        uses: actions/checkout@v2
-      - name: Set Up Python 3
-        uses: actions/setup-python@v1
-        with: 
-          python-version: '3.7'  
-      # Set Up Connection to Dev Mongo
-      # 1. Auth to cluster
-      # 2. Open encrypted connection
-      - uses: google-github-actions/setup-gcloud@v0
-        with:
-          service_account_key: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
-          project_id: ${{ secrets.DEV_PROJECT }}
-      - name: Authenticate GHA Runner To Target Cluster
-        run: gcloud container clusters get-credentials ${{secrets.DEV_GKE_CLUSTER}} --zone ${{secrets.DEV_GKE_REGION}} --project ${{secrets.DEV_GCP_PROJECT}}
-      - name: Establish MongoDB Connection
-        run: kubectl port-forward svc/mongo 27017:27017 &
-      - name: Await MongoDB Connection
-        run: sleep 20
-      - name: Confirm MongoDB Connection
-        # The following command returns a success code if mongo is available,
-        # and a failure code if it is not
-        run: curl localhost:27017
-      - name: Establish PostgreSQL Connection
-        run: kubectl port-forward svc/postgres 5432:5432 &
-      - name: Await MongoDB Connection
-        run: sleep 20
-      - name: Confirm PostgreSQL Connection
-        # if the PostgreSQL server is available, curl will fail with a '52' exit code
-        # if it is NOT available, it will fail with a '7' exit code
-        # run: curl localhost:5432; if [ $? -eq "52" ]; then exit 0; else exit 1; fi
-        run: exitCode=$(curl localhost:5432; echo $?); if [ $exitCode -eq 52 ]; then exit 0; else exit 1; fi
-      - name: Set outputs
-        id: get-sha
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      # END SETUP
-      - name: Install Python Packages
-        run: pip3 install -r ./requirements.txt
-      - name: Run Test Suite
-        run: env && python3 ./reader/browsertest/run_tests_on_github.py #  ; echo $?
-        shell: bash
-        env:
-          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
-          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
-          CI_RUN: "true"
-          CI_URL: https://sandbox-${{ steps.get-sha.outputs.sha_short }}.cauldron.sefaria.org
-          SEFARIA_TEST_PASS: ${{ secrets.SEFARIA_APP_PASS }}
-          SEFARIA_TEST_USER: ${{ secrets.SEFARIA_APP_USER }}
-          PYTHONPATH: "."
-          DJANGO_SETTINGS_MODULE: sefaria.settings
-          MONGO_DB_NAME: sefaria-sandbox-${{ steps.get-sha.outputs.sha_short }}
-          GITHUB_SHA_SORT: ${{ steps.get-sha.outputs.sha_short }}
-      - name: Authenticate GHA Runner to GCP
-        uses: google-github-actions/setup-gcloud@v0
-        with:
-          service_account_key: ${{ secrets.GCS_BUILD_UPLOAD_KEY }}
-          project_id: ${{ secrets.BUILD_UPLOAD_PROJECT }}
-        if: ${{ always() }}
   pytest-job:
     if: ${{ github.event_name == 'pull_request' }}
     name: "PyTest"  # This name is referenced when slacking status
@@ -276,25 +219,31 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      # Set Up Connection to Dev Mongo
-      # 1. Auth to cluster
-      # 2. Open encrypted connection
-      - uses: google-github-actions/setup-gcloud@v0
+      - name: Authenticate GHA Runner to GCP
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
+      - name: Setup GCloud
+        uses: google-github-actions/setup-gcloud@v0
         with:
-          service_account_key: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
           project_id: ${{ secrets.DEV_PROJECT }}
+          install_components: 'gke-gcloud-auth-plugin'
       - name: Authenticate GHA Runner To Target Cluster
-        run: gcloud container clusters get-credentials ${{secrets.DEV_GKE_CLUSTER}} --zone ${{secrets.DEV_GKE_REGION}} --project ${{secrets.DEV_GCP_PROJECT}}
+        uses: google-github-actions/get-gke-credentials@v0
+        with:
+          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
+          location: ${{secrets.DEV_GKE_REGION}}
+          project_id: ${{secrets.DEV_GCP_PROJECT}}
       - name: Set outputs
         id: get-sha
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Get branch name
         id: branch-raw
         uses: tj-actions/branch-names@v5.1
       - name: Format branch name
         id: branch-name
         run: |
-          echo "::set-output name=current_branch::"$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g')
+          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | awk '{print tolower($0)}' | sed 's/[^a-z0-9\.\-]//g') >> $GITHUB_OUTPUT
       - name: Start Job
         run: envsubst '${GITHUB_RUN_ID},${DEPLOY_ENV},${WEB_IMAGE_NAME},${WEB_IMAGE_TAG},${TIMESTAMP}' < ./build/ci/pyTestPod.yaml | kubectl apply -f -
         env:
@@ -317,7 +266,6 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ always() && github.event_name == 'pull_request' }}
     needs:
-      - selenium-sauce
       - pytest-job
     steps:
       - uses: actions/checkout@v2
@@ -337,26 +285,32 @@ jobs:
     name: "Clean up CI Sandbox"
     if: ${{ always() && github.event_name == 'pull_request' }}
     needs: 
-      - selenium-sauce
       - pytest-job
     runs-on: ubuntu-latest
     steps: 
       - name: Checkout
         uses: actions/checkout@v2
-      - uses: google-github-actions/setup-gcloud@v0
+      - name: Authenticate GHA Runner to GCP
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
+      - name: Setup GCloud
+        uses: google-github-actions/setup-gcloud@v0
         with:
-          service_account_key: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
           project_id: ${{ secrets.DEV_PROJECT }}
+          install_components: 'gke-gcloud-auth-plugin'
+      - name: Authenticate GHA Runner To Target Cluster
+        uses: google-github-actions/get-gke-credentials@v0
+        with:
+          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
+          location: ${{secrets.DEV_GKE_REGION}}
+          project_id: ${{secrets.DEV_GCP_PROJECT}}
       - name: Set outputs
         id: get-sha
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      - name: setup helm
-        uses: azure/setup-helm@v1
-      - name: Authenticate GHA Runner To Target Cluster
-        run: gcloud container clusters get-credentials ${{secrets.DEV_GKE_CLUSTER}} --zone ${{secrets.DEV_GKE_REGION}} --project ${{secrets.DEV_GCP_PROJECT}}
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: check helm
         id: get-helm
-        run: echo "::set-output name=count::$(helm list -n $NAMESPACE | grep $NAME | wc -l)"
+        run: echo "count=$(helm list -n $NAMESPACE | grep $NAME | wc -l)" >> $GITHUB_OUTPUT
         env:
           NAMESPACE: ${{ secrets.DEV_SANDBOX_NAMESPACE }}
           NAME: sandbox-${{ steps.get-sha.outputs.sha_short }}

diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml
@@ -43,6 +43,8 @@ jobs:
         with:
           # Fetch entire history. Required for chart-releaser; see https://github.com/helm/chart-releaser-action/issues/13#issuecomment-602063896
           fetch-depth: 0
+      - name: fetch head of branch to ensure semantic can run
+        run: git pull origin $GITHUB_REF
       - name: create release rules
         run: helm-chart/release-rules.sh 
       - uses: cycjimmy/semantic-release-action@v3
@@ -59,7 +61,7 @@ jobs:
       - name: Get appVersion
         id: appVersion
         run: |
-          echo "::set-output name=appVersion::$(git describe --match 'v*' --abbrev=0 HEAD --tags)"
+          echo "appVersion=$(git describe --match 'v*' --abbrev=0 HEAD --tags)" >> $GITHUB_OUTPUT
       - name: Download yq
         run: |
           wget -nv -nc -O yq https://github.com/mikefarah/yq/releases/download/v4.20.2/yq_linux_amd64

diff --git a/.github/workflows/production-deploy.yaml b/.github/workflows/production-deploy.yaml
@@ -23,9 +23,9 @@ jobs:
       - id: check
         run: |
           if [[ "${{ inputs.version }}" == "" ]]; then
-            echo "::set-output name=ref::${{ github.ref_name }}" 
+            echo "ref=${{ github.ref_name }}" >> $GITHUB_OUTPUT 
           else
-            echo "::set-output name=ref::v${{ inputs.version }}" 
+            echo "ref=v${{ inputs.version }}" >> $GITHUB_OUTPUT
           fi
   build-generic:
     needs: workflow-check
@@ -49,7 +49,7 @@ jobs:
           password: ${{ secrets.PROD_GKE_SA_KEY }}
       - name: Get current date
         id: date
-        run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')"
+        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
       - name: Generate image metadata
         id: meta
         uses: docker/metadata-action@v3
@@ -98,7 +98,7 @@ jobs:
           password: ${{ secrets.PROD_GKE_SA_KEY }}
       - name: Get current date
         id: date
-        run: echo "::set-output name=date::$(date +'%Y%m%d%H%M')"
+        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
       - name: Generate image metadata
         id: meta
         uses: docker/metadata-action@v3
@@ -136,18 +136,22 @@ jobs:
         with:
           ref: ${{ needs.workflow-check.outputs.ref }}
       - name: Authenticate GHA Runner to GCP
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: ${{ secrets.PROD_GKE_SA_KEY }}
+      - name: Setup GCloud
         uses: google-github-actions/setup-gcloud@v0
         with:
-          service_account_key: ${{ secrets.PROD_GKE_SA_KEY }}
           project_id: ${{ secrets.PROD_GKE_PROJECT }}
+          install_components: 'gke-gcloud-auth-plugin'
+      - name: Authenticate GHA Runner To Target Cluster
+        uses: google-github-actions/get-gke-credentials@v0
+        with:
+          cluster_name: ${{secrets.PROD_GKE_CLUSTER}}
+          location: ${{secrets.PROD_GKE_REGION}}
+          project_id: ${{secrets.PROD_GKE_PROJECT}}
       - name: Set up yq
         uses: frenck/action-setup-yq@v1
-      - name: setup helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.9.0
-      - name: Authenticate GHA Runner To Target Cluster
-        run: gcloud container clusters get-credentials ${{secrets.PROD_GKE_CLUSTER}} --zone ${{secrets.PROD_GKE_REGION}} --project ${{secrets.PROD_GKE_PROJECT}}
       - name: Deploy Production
         run: ./build/ci/production-helm-deploy.sh build/ci/production-values.yaml
         env:
@@ -156,3 +160,6 @@ jobs:
           NAMESPACE: "${{secrets.PROD_GKE_NAMESPACE}}"
           CHART_VERSION: 0.2.3
           IMAGE_NAME: "${{ secrets.IMAGE_NAME }}"
+          CHART_VERSION: 0.6.0
+
+
diff --git a/.github/workflows/weekly.yaml b/.github/workflows/weekly.yaml
@@ -5,5 +5,43 @@ on:
     - cron: '0 0 * 6 *'
 
 jobs:
-
+  build-mongo:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to GCR
+        uses: docker/login-action@v1
+        with:
+          registry: gcr.io
+          username: _json_key
+          password: ${{ secrets.DEV_CLOUDBUILD_SA_KEY }}
+      - name: Get current date
+        id: date
+        run: echo "date$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
+      - name: Generate image metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-mongo
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
+            type=sha
+          flavor: |
+            latest=true
+      - name: build and push
+        uses: docker/build-push-action@v2
+        with:
+          cache-from: type=registry, ref=sefaria-mongo/cache
+          cache-to: type=registry, ref=sefaria-mongo/cache, mode=max
+          context: .
+          push: true
+          file: ./build/standalone-db/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
 ...
diff --git a/README.mkd b/README.mkd
@@ -1,7 +1,4 @@
-<<<<<<< HEAD
 ## New Interfaces for Constitutional Texts
-=======
-## New Interfaces for Jewish Texts
 
 [Sefaria](https://www.sefaria.org) is creating interfaces, apps (like a source sheet builder), and infrastructure (like an API and a structured dataset) for Jewish texts and textual learning.
 
@@ -316,7 +313,6 @@ We're grateful to the following organizations for providing us with donated serv
 - [Google Apps for Work](https://apps.google.com/)
 - [PyCharm](https://www.jetbrains.com/pycharm/)
 - [Slack](https://slack.com)
->>>>>>> 0191e907cb5146ad9f4fd1147d887a79ed86ee11
 
 [ContextUS](https://www.contextus.org) is an living repository of American constitutional texts.
 The underlying code is derived from [The Sefaria Project](http://www.github.com/Sefaria/Sefaria-Project).