-
Notifications
You must be signed in to change notification settings - Fork 0
254 lines (222 loc) · 10.9 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
---
# SOURCE: https://github.com/openai/openai-python/blob/aed1e43745cd6358b4bafd3a39b3dfeee5e31a03/.github/workflows/ci.yml
name: CI
# SOURCE: https://github.com/tornadoweb/tornado/blob/f399f40fde0ae1b130646db783a6f79cc59231b2/.github/workflows/build.yml#L37
on: # yamllint disable-line rule:truthy
# https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#push
push:
branches:
# Run on release branches. This gives us a chance to detect rot in this
# configuration before pushing a tag (which we'd rather not have to undo).
- main
- feature/**
- '[0-9].*.x' # e.g., 4.14.x
- '[0-9][0-9].*.x' # e.g., 23.3.x
tags:
# The main purpose of this workflow is to build wheels for release tags.
# It runs automatically on tags matching this pattern and pushes to pypi.
- "v*"
# https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#pull_request
pull_request:
# # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch
# workflow_dispatch:
# # Allow this workflow to be run manually (pushing to testpypi instead of pypi)
workflow_dispatch:
inputs:
debug_enabled:
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
env:
python-version: '3.10'
MAIN_PYTHON_VERSION: "3.10"
jobs:
test:
name: test
runs-on: ubuntu-22.04
# # Service containers to run with `runner-job`
# services:
# # Label used to access the service container
# redis:
# # Docker Hub image
# image: redis
# # Set health checks to wait until redis has started
# # options: >-
# # --health-cmd "redis-cli ping"
# # --health-interval 10s
# # --health-timeout 5s
# # --health-retries 5
# ports:
# # Maps port 7600 on service container to the host
# - 7600:7600
if: github.repository == 'bossjones/goob_ai'
env:
OS: ubuntu-22.04
PYTHON: '3.10'
LANGCHAIN_ENDPOINT: "https://api.smith.langchain.com"
LANGCHAIN_TRACING_V2: true
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
# Update with your API URL if using a hosted instance of Langsmith.
LANGCHAIN_HUB_API_URL: "https://api.hub.langchain.com"
LANGCHAIN_HUB_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
LANGCHAIN_PROJECT: "goob-ci"
GOOB_AI_CONFIG_DISCORD_TOKEN: "foo"
GOOB_AI_CONFIG_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
steps:
- run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
- run: echo "🔎 The name of your branch is ${{ github.ref_name }} and your repository is ${{ github.repository }}."
- name: Set up Git repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Start Redis
id: redis-docker-compose-up
run: |
curl -SL https://github.com/docker/compose/releases/download/v2.29.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
docker compose up -d
docker ps -a
continue-on-error: true
- name: Start Chroma
id: chroma-docker-compose-up
run: |
mkdir -p ./src/goob_ai/data/chroma/vectorstorage || true
# git clone --depth 1 --branch feature-boss-chroma https://github.com/bossjones/chroma.git vendored_chroma
# cd vendored_chroma
docker compose up -d
docker ps -a
continue-on-error: true
- name: Checkout
uses: actions/checkout@v4
- name: get version python
shell: bash # use bash to run date command
run: echo "Py${{ env.python-version }}"
- name: Hash + Timestamp
shell: bash # use bash to run date command
run: 'echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ env.python-version }}-$(date -u "+%Y%m")" >> $GITHUB_ENV'
# # SOURCE: https://github.com/wakabame/study_pytorch/blob/327bee3de2c283e1592d806d9b909e8b4af6b9fe/.github/workflows/apply_lint_and_test.yaml#L24
# - name: Load cached venv
# id: cached-rye-dependencies
# uses: actions/cache@v3
# with:
# path: .venv
# key: venv-${{ runner.name }}-${{ runner.os }}-${{ hashFiles('.github/dependabot/requirements-dev.txt') }}
- name: "Set up Python ${{ env.python-version }}"
# if: steps.cached-rye-dependencies.outputs.cache-hit != 'true'
uses: actions/setup-python@v5
with:
python-version: "${{ env.python-version }}"
# cache: "pip"
# cache-dependency-path: .github/dependabot/requirements-dev.txt
- name: Install Rye
run: |
curl -sSf https://rye.astral.sh/get | bash
echo "$HOME/.rye/shims" >> $GITHUB_PATH
env:
RYE_VERSION: '0.37.0'
RYE_INSTALL_OPTION: '--yes'
- name: install tesseract and ffmpeg
run: |
sudo apt-get install -y tesseract-ocr tesseract-ocr-por libyaml-dev poppler-utils
sudo apt install ffmpeg -y
sudo apt-get install autoconf automake build-essential libtool python3-dev libsqlite3-dev -y
echo "install deps for llm_aided_ocr"
sudo apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev \
libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
xz-utils tk-dev libffi-dev liblzma-dev python3-openssl git
- name: Install dependencies
# if: steps.cached-rye-dependencies.outputs.cache-hit != 'true'
env:
LANGCHAIN_ENDPOINT: "https://api.smith.langchain.com"
LANGCHAIN_TRACING_V2: true
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
# Update with your API URL if using a hosted instance of Langsmith.
LANGCHAIN_HUB_API_URL: "https://api.hub.langchain.com"
LANGCHAIN_HUB_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
LANGCHAIN_PROJECT: "goob-ci"
GOOB_AI_CONFIG_DISCORD_TOKEN: "foo"
GOOB_AI_CONFIG_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
run: |
cp -f env.sample .env
rye config --set-bool behavior.use-uv=true
./scripts/retry -t 4 -- rye sync --all-features
# Allow debugging with tmate
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
with:
# Use registered public SSH key(s)
# If you have registered one or more public SSH keys with your GitHub profile, tmate will be started such that only those keys are authorized to connect, otherwise anybody can connect to the tmate session. If you want to require a public SSH key to be installed with the tmate session, no matter whether the user who started the workflow has registered any in their GitHub profile, you will need to configure the setting limit-access-to-actor to true, like so:
limit-access-to-actor: true
- name: Run tests and collect coverage
env:
LANGCHAIN_ENDPOINT: "https://api.smith.langchain.com"
LANGCHAIN_TRACING_V2: true
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
LANGCHAIN_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
# Update with your API URL if using a hosted instance of Langsmith.
LANGCHAIN_HUB_API_URL: "https://api.hub.langchain.com"
LANGCHAIN_HUB_API_KEY: ${{ secrets.LANGCHAIN_API_KEY }}
LANGCHAIN_PROJECT: "goob-ci"
GOOB_AI_CONFIG_DISCORD_TOKEN: "foo"
GOOB_AI_CONFIG_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
run: |
# cp -f env.sample .env
# rye config --set-bool behavior.use-uv=true
# ./scripts/retry -t 4 -- rye sync --all-features
rye run goobctl chroma generate
echo "rye run lint"
rye run lint
# rye run typecheck
echo "rye run python -c 'import goob_ai'"
rye run python -c 'import goob_ai'
echo "rye run test"
rye run test
sudo apt update
sudo apt-get install tree -y
tree
ls -lta
# SOURCE: https://til.simonwillison.net/github-actions/debug-tmate
# Starting a shell just for test failures on manual runs #
# I had a tricky test failure that I wanted to debug interactively. Here's a recipe for starting a tmate shell ONLY if the previous step failed, and only if the run was triggered manually (using workflow_dispatch) - because I don't want an accidental test opening up a shell and burning up my GitHub Actions minutes allowance.
# steps:
# - name: Run tests
# run: pytest
# - name: tmate session if tests fail
# if: failure() && github.event_name == 'workflow_dispatch'
# uses: mxschmitt/action-tmate@v3
- name: tmate session if tests fail
if: failure() && github.event_name == 'workflow_dispatch'
uses: mxschmitt/action-tmate@v3
with:
# Use registered public SSH key(s)
# If you have registered one or more public SSH keys with your GitHub profile, tmate will be started such that only those keys are authorized to connect, otherwise anybody can connect to the tmate session. If you want to require a public SSH key to be installed with the tmate session, no matter whether the user who started the workflow has registered any in their GitHub profile, you will need to configure the setting limit-access-to-actor to true, like so:
limit-access-to-actor: true
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4
with:
# token: ${{ secrets.CODECOV_TOKEN }}
flags: "${{ runner.os }},${{ runner.arch }},${{ env.python-version }}"
verbose: true
fail_ci_if_error: true
# directory: ./junit/
# files: ./test-results.xml,cov.xml
files: ./cov.xml
# flags: smart-tests
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Upload Test Results
if: '!cancelled()'
uses: actions/upload-artifact@84480863f228bb9747b473957fcc9e309aa96097
with:
name: test-results-${{ env.HASH }}
path: |
.coverage
tools\durations\${{ runner.os }}.json
junit/test-results.xml
cov.xml
retention-days: 1 # temporary, combined in aggregate below