eval.yml

name: Run evals

on:
  push:
    # files:
    #   - 'test-eval/**'

permissions:
  pull-requests: write
  contents: read

jobs:
  eval:
    name: Run evals
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: "recursive"

      - name: Setup Node.js
        id: setup-node
        uses: actions/setup-node@v4
        with:
          node-version-file: .node-version

      - uses: pnpm/action-setup@v3
        with:
          version: 8

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - uses: actions/cache@v4
        name: Setup pnpm cache
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-

      - name: Install Dependencies
        id: install
        run: pnpm install
        working-directory: test-eval

      - name: Run Evals
        uses: ./
        with:
          api_key: ${{ secrets.BRAINTRUST_API_KEY }}
          root: test-eval
          runtime: node

      # - name: Start terminal session
      #   uses: mxschmitt/action-tmate@v3
      #   with:
      #     limit-access-to-actor: true