Feat/long running epoch testing

.github/workflows/get-sha.yml

@@ -0,0 +1,25 @@
+name: Set SHAs
+on:
+    workflow_call: {}
+
+jobs:
+   get-sha:
+    runs-on: ubuntu-latest
+    steps:
+     - name: Check SHA
+       uses: nrwl/nx-set-shas@v3
+       id: sha
+       with:
+        main-branch-name: 'master'
+
+   check_status:
+      needs: [get-sha]
+      runs-on: ubuntu-latest 
+      steps:
+        - name: output SHAs
+          run: |
+           if [${{needs.get-sha.result}} == 'success']; then
+                echo "get sha failed"
+                exit 1
+            fi
+            echo "All steps passed" 

.github/workflows/ci.yml → .github/workflows/intigration-test-long-running.yml

@@ -1,44 +1,20 @@
-name: Testing
+name: Intigration Testing Datil Long Running
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-      - staging/**
-      - feat/**
-      - feature/**
+  workflow_call:
+    outputs:
+      intigration-tests:
+        description: 'unit test results'
+        value: ${{jobs.integration-tests.result}}
+    inputs:
+        commit:
+          required: true
+          type: string
+  workflow_dispatch: {}
+
 jobs:
-  unit-tests:
-    runs-on: warp-ubuntu-latest-x64-16x
-    timeout-minutes: 30
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - uses: nrwl/nx-set-shas@v3
-        with:
-          main-branch-name: 'master'
-      - uses: actions/checkout@v3
-      - name: Use Node.js
-        uses: buildjet/setup-node@v3
-        with:
-          node-version: 20
-          cache: yarn
-          cache-dependency-path: ${{ github.workspace }}/yarn.lock
-      - name: Install Dependencies
-        run: yarn 
-      - name: Build
-        run: yarn build:dev
-      - name: Run Unit tests
-        run: yarn test:ci
   integration-tests:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
-    env:
-      DATIL_COMMIT_HASH: ae3c20e07eb933b61073689b95f56867c03de252
+    timeout-minutes: 40
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
@@ -50,7 +26,7 @@ jobs:
         with: 
           fetch-depth: 0
           repository: LIT-Protocol/lit-assets
-          ref: ${{env.DATIL_COMMIT_HASH}}
+          ref: ${{inputs.commit}}
           token: ${{secrets.GH_PAT}}
           path: ${{ github.workspace }}/lit-assets/
           submodules: false
@@ -76,7 +52,7 @@ jobs:
         run: docker pull ghcr.io/lit-protocol/shiva:latest 
       - name: Run Shiva Container
         id: shiva-runner
-        run: docker run -d -m 32g -p 8000:8000 -p 8545:8545 -p 7470:7470 -p 7471:7471 -p 7472:7472 -p 7473:7473 -p 7474:7474 -p 7475:7475 -v ${{github.workspace}}/lit-assets:/data -e GH_PAT=${{secrets.GH_PAT}} -e HASH=$DATIL_COMMIT_HASH -e IPFS_API_KEY=${{secrets.IPFS_API_KEY}} --name shiva ghcr.io/lit-protocol/shiva:latest
+        run: docker run -d -m 32g -p 8000:8000 -p 8545:8545 -p 7470:7470 -p 7471:7471 -p 7472:7472 -p 7473:7473 -p 7474:7474 -p 7475:7475 -v ${{github.workspace}}/lit-assets:/data -e GH_PAT=${{secrets.GH_PAT}} -e HASH=${{inputs.commit}} -e IPFS_API_KEY=${{secrets.IPFS_API_KEY}} --name shiva ghcr.io/lit-protocol/shiva:latest
       - name: Install project dependencies
         run: yarn 
       - uses: nrwl/nx-set-shas@v3
@@ -87,23 +63,17 @@ jobs:
         run: yarn build:dev
       - name: Copy ENV File
         run: cp .env.ci .env
-      - name: End to End Tests - Connection 
-        id: connection
+      - name: End to End Tests 
+        id: e2e-tests
         if: steps.build.outputs.exit_code == 0
-        run: yarn test:e2e -t 'Connections'
-      - name: End Tests Session Compaitiblity
-        id: session
-        if: steps.connection.outputs.exit_code == 0
-        run: yarn test:e2e -t 'SessionSigs'
-      - name: End to End Tests - PKP Ethers
-        if: steps.session.outputs.exit_code == 0
-        id: pkp
-        run: yarn test:e2e -t 'PKP Ethers'
+        run: yarn test:e2e:ci:epoch
       - name: Get Container Logs
         if: always()
         run: docker logs shiva
       - name: Post Pull Shiva Container
         id: container-stop
         if: steps.shiva-pull.outputs.exit_code == 0
         run: docker stop shiva && docker rm shiva
-
+      - name: Post Pull Shiva Image
+        if: steps.shiva-pull.outputs.exit_code == 0
+        run: docker rmi ghcr.io/lit-protocol/shiva 

.github/workflows/intigration-test.yml

@@ -0,0 +1,79 @@
+name: Intigration Testing Datil
+on:
+  workflow_call:
+    outputs:
+      intigration-tests:
+        description: 'unit test results'
+        value: ${{jobs.integration-tests.result}}
+    inputs:
+        commit:
+          required: true
+          type: string
+  workflow_dispatch: {}
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Checkout Lit Actions
+        uses: actions/checkout@v4
+        id: checkout
+        with: 
+          fetch-depth: 0
+          repository: LIT-Protocol/lit-assets
+          ref: ${{inputs.commit}}
+          token: ${{secrets.GH_PAT}}
+          path: ${{ github.workspace }}/lit-assets/
+          submodules: false
+          sparse-checkout: |
+           blockchain
+           rust/lit-node
+      - name: Check LA dir
+        run: ls -la ${{github.workspace}}/lit-assets
+      - name: Setup Lit Bockchain Dependencies
+        uses: buildjet/setup-node@v3
+        with:
+          node-version: 20
+          cache: npm 
+          cache-dependency-path: ${{ github.workspace }}/lit-assets/blockchain/contracts/package-lock.json
+      - name: Install Blockchain Deps
+        run: npm i
+        working-directory: ${{github.workspace}}/lit-assets/blockchain/contracts
+      - name: Docker login
+        id: login
+        run: docker login ghcr.io/ -u ${{secrets.GH_USER}} --password ${{secrets.GH_PAT}}
+      - name: Pull Shiva Container
+        id: shiva-pull
+        run: docker pull ghcr.io/lit-protocol/shiva:latest 
+      - name: Run Shiva Container
+        id: shiva-runner
+        run: docker run -d -m 32g -p 8000:8000 -p 8545:8545 -p 7470:7470 -p 7471:7471 -p 7472:7472 -p 7473:7473 -p 7474:7474 -p 7475:7475 -v ${{github.workspace}}/lit-assets:/data -e GH_PAT=${{secrets.GH_PAT}} -e HASH=${{inputs.commit}} -e IPFS_API_KEY=${{secrets.IPFS_API_KEY}} --name shiva ghcr.io/lit-protocol/shiva:latest
+      - name: Install project dependencies
+        run: yarn 
+      - uses: nrwl/nx-set-shas@v3
+        with:
+          main-branch-name: 'master'
+      - name: Build packages
+        id: build
+        run: yarn build:dev
+      - name: Copy ENV File
+        run: cp .env.ci .env
+      - name: End to End Tests 
+        id: e2e-tests
+        if: steps.build.outputs.exit_code == 0
+        run: yarn test:e2e:ci
+      - name: Get Container Logs
+        if: always()
+        run: docker logs shiva
+      - name: Post Pull Shiva Container
+        id: container-stop
+        if: steps.shiva-pull.outputs.exit_code == 0
+        run: docker stop shiva && docker rm shiva
+      - name: Post Pull Shiva Image
+        if: steps.shiva-pull.outputs.exit_code == 0
+        run: docker rmi ghcr.io/lit-protocol/shiva 

.github/workflows/lint.yml

@@ -1,9 +1,12 @@
 name: lint
 on:
-  pull_request:
-  push:
-    branches:
-      - master
+  workflow_call:
+    outputs:
+      lint_result:
+        description: 'lint result'
+        value: ${{jobs.linter.result}}
+
+  workflow_dispatch: {}
 jobs:
   linter:
     runs-on: ubuntu-latest
@@ -22,3 +25,18 @@ jobs:
         run: yarn install
       - name: Lint
         run: yarn nx format:check --all
+  check_status:
+    needs:
+      [
+        linter,
+      ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Status
+        working-directory: ${{ github.workspace }}
+        run: |
+          if [ ${{ needs.linter.result }} != 'success' ]; then
+              echo "Linting failed"
+              exit 1
+          fi
+          echo "All tests passed"

.github/workflows/list-changed-files.yml

@@ -0,0 +1,64 @@
+# This workflow is triggered by a PR being created or pushed to. It checks out the develop branch and gets the sha of the develop branch. It then checks for changes in the files and triggers the appropriate workflows based on the changes. It also checks the status of all the triggered workflows and fails the build if any of the workflows fail.  It runs the test against a new temporary branch which is created by merging the PR into develop.  This means that someone pushing to develop can break your PR even if you haven't changed anything!  This is a tradeoff to ensure that the tests are run against the latest code on develop.  If you want to run the tests against the code in your PR, you can run the tests locally.
+name: master-trigger
+on:
+  workflow_call:
+    outputs:
+      lit_core_changed:
+        description: 'lit_core_changed'
+        value: ${{ jobs.list_changed_files.outputs.lit_core_changed }}
+
+  workflow_dispatch: {}
+
+
+jobs:
+  get_base_commit_hash:
+    runs-on: ubuntu-latest
+    outputs:
+      base_sha: ${{ steps.base_sha.outputs.base_sha }}
+    steps:
+      - name: Get branch name
+        id: branch-name
+        uses: tj-actions/branch-names@v7
+
+      - name: Make sure we checked out develop, so we can get it's sha
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.branch-name.outputs.base_ref_branch }}
+
+      - name: Get sha of base branch
+        id: base_sha
+        run: |
+          export BASE_SHA=$(git rev-parse HEAD)
+          echo "Base sha is $BASE_SHA"
+          echo "base_sha=$BASE_SHA" >> $GITHUB_OUTPUT
+
+  list_changed_files:
+    runs-on: ubuntu-latest
+    needs: [get_base_commit_hash]
+    outputs:
+      lit_core_changed: ${{ steps.changed-files-yaml.outputs.lit_core_any_modified }}
+
+    steps:
+      - name: Checkout js-sdk
+        uses: actions/checkout@v4
+
+      - name: Print base sha
+        run: |
+          echo "Base sha is ${{ needs.get_base_commit_hash.outputs.base_sha }}"
+
+      - name: Get changed files
+        id: changed-files-yaml
+        uses: tj-actions/changed-files@v41
+        with:
+          base_sha: ${{ needs.get_base_commit_hash.outputs.base_sha }}
+          files_yaml: |
+            lit_core:
+              - packages/core/**
+            any:
+              - ./**
+
+      - name: List all changed files
+        run: |
+          for file in ${{ steps.changed-files-yaml.outputs.any_all_modified_files }}; do
+            echo "$file was changed"
+          done

.github/workflows/master-trigger.yml

@@ -0,0 +1,50 @@
+name: root-trigger 
+on:
+  pull_request:
+  workflow_dispatch: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  list_changed_files_and_trigger_workflows:
+    uses: ./.github/workflows/list-changed-files.yml
+    secrets: inherit
+  get_shas:
+    uses: ./.github/workflows/get-sha.yml
+    secrets: inherit
+  lint:
+    needs: [list_changed_files_and_trigger_workflows]
+    uses: ./.github/workflows/lint.yml
+    secrets: inherit
+  unit_tests:
+    uses: ./.github/workflows/unit-test.yml
+    secrets: inherit
+    needs: [lint, get_shas]
+  intigration_tests:
+      uses: ./.github/workflows/intigration-test.yml
+      secrets: inherit
+      if: ${{needs.list_changed_files_and_trigger_workflows.outputs.lit_core_changed == 'true'}} 
+      needs: [lint, get_shas, list_changed_files_and_trigger_workflows, unit_tests]
+      with:
+        commit: ${{vars.DATIL_COMMIT_HASH}}
+  intigration_tests_longrun_epoch:
+    uses: ./.github/workflows/intigration-test-long-running.yml
+    secrets: inherit
+    needs: [lint, get_shas, list_changed_files_and_trigger_workflows, unit_tests]
+    with:
+      commit: ${{vars.DATIL_COMMIT_HASH}} 
+  all_jobs:
+    if: ${{ always() }}
+    needs: [
+      lint,
+      unit_tests,
+      intigration_tests
+    ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check status
+        run: |
+          echo "Linter ${{needs.lint.result}}"
+          echo "Tests Unit ${{needs.unit_tests.result}}"
+          echo "Tests Intigration"