[FEAT] Add automatic evaluation workflow (#140)

.github/workflows/models-performance.yaml

@@ -0,0 +1,65 @@
+name: Models Performance
+
+permissions: write-all
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-models-performance:
+    runs-on: ubuntu-latest 
+    strategy:
+      fail-fast: false
+    env:
+      TIMEGPT_TOKEN: ${{ secrets.TIMEGPT_TOKEN }}
+      PLOTS_REPO_URL: https://github.com/Nixtla/nixtla/blob/docs-figs-model-performance
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@v2
+
+      - name: Set up environment
+        uses: mamba-org/setup-micromamba@v1 
+        with:
+          environment-file: environment.yml
+          create-args: python=3.9
+          cache-environment: true
+
+      - name: Install pip requirements
+        run: pip install ./ 
+
+      - name: Run evaluation 
+        run: python -m action_files.models_performance.main
+
+      - name: Upload images to new branch main
+        run: |
+          git config --global user.email azul@nixtla.io
+          git config --global user.name AzulGarza
+          git push https://$GITHUB_TOKEN@github.com/nixtla/nixtla.git --delete docs-figs-model-performance || true
+          git checkout -b docs-figs-model-performance 
+          git add -f "*.png"
+          git commit -m "[cd] update png images" || echo "No changes to commit"
+          git push https://$GITHUB_TOKEN@github.com/nixtla/nixtla.git HEAD:docs-figs-model-performance
+
+      - name: Upload results to the PR
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const fs = require('fs');
+            const markdown = fs.readFileSync('./action_files/models_performance/summary.md', 'utf8');
+            await github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: markdown
+            });                    
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+

.gitignore

@@ -22,3 +22,5 @@ nbs/.last_checked
 .venv
 .idea
 .env
+*/summary.md
+*/*.png

action_files/models_performance/experiments.yaml

@@ -0,0 +1,33 @@
+experiments:
+
+  - air-passengers:
+    - dataset_url: https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/air_passengers.csv
+    - time_col: timestamp
+    - target_col: value
+    - season_length: 12 # for benchmarks
+    - freq:
+      - MS
+    - h:
+      - 12
+      - 24
+
+  - electricity-multiple-series:
+    - dataset_url: https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/electricity-short.csv
+    - season_length: 24 # for benchmarks
+    - freq:
+    - freq:
+      - H
+    - h:
+      - 24
+      - 168
+
+  - electricity-multiple-series-with-ex-vars:
+    # we can perform evaluation 
+    # without the future exogenous variables
+    - dataset_url: https://raw.githubusercontent.com/Nixtla/transfer-learning-time-series/main/datasets/electricity-short-with-ex-vars.csv
+    - season_length: 24 # for benchmarks
+    - freq:
+      - H
+    - h:
+      - 24 
+      - 168