Skip to content

4.0.0 (#85)

4.0.0 (#85) #254

# SPDX-FileCopyrightText: Nextcloud contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
name: Integration test
on:
pull_request:
push:
branches:
- main
- stable*
env:
APP_NAME: context_chat
concurrency:
group: integration-test-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
transcription:
runs-on: ubuntu-22.04
strategy:
# do not stop on another job's failure
fail-fast: false
matrix:
php-versions: [ '8.1' ]
databases: [ 'pgsql' ]
server-versions: [ 'master' ]
name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
env:
MYSQL_PORT: 4444
PGSQL_PORT: 4445
# use the same db for ccb and nextcloud
CCB_DB_URL: postgresql+psycopg://root:rootpassword@localhost:4445/nextcloud
services:
mysql:
image: mariadb:10.5
ports:
- 4444:3306/tcp
env:
MYSQL_ROOT_PASSWORD: rootpassword
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
postgres:
image: pgvector/pgvector:pg17
ports:
- 4445:5432/tcp
env:
POSTGRES_USER: root
POSTGRES_PASSWORD: rootpassword
POSTGRES_DB: nextcloud
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
steps:
- name: Checkout server
uses: actions/checkout@v4
with:
repository: nextcloud/server
ref: ${{ matrix.server-versions }}
- name: Checkout submodules
shell: bash
run: |
auth_header="$(git config --local --get http.https://github.com/.extraheader)"
git submodule sync --recursive
git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
tools: phpunit
extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip
- name: Checkout app
uses: actions/checkout@v4
with:
path: apps/${{ env.APP_NAME }}
- name: Checkout backend
uses: actions/checkout@v4
with:
repository: nextcloud/context_chat_backend
path: context_chat_backend/
- name: Get backend app version
id: appinfo
uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master
with:
filename: context_chat_backend/appinfo/info.xml
expression: "/info/version/text()"
- name: Read package.json node and npm engines version
uses: skjnldsv/read-package-engines-version-actions@v3
id: versions
with:
path: apps/${{ env.APP_NAME }}
fallbackNode: '^20'
fallbackNpm: '^10'
- name: Set up node ${{ steps.versions.outputs.nodeVersion }}
uses: actions/setup-node@v4
with:
node-version: ${{ steps.versions.outputs.nodeVersion }}
- name: Set up npm ${{ steps.versions.outputs.npmVersion }}
run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}"
- name: Install app
working-directory: apps/${{ env.APP_NAME }}
run: |
make all
composer install --no-dev
- name: Set up Nextcloud
if: ${{ matrix.databases != 'pgsql'}}
run: |
sleep 25
mkdir data
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
composer run serve &
- name: Set up Nextcloud
if: ${{ matrix.databases == 'pgsql'}}
run: |
sleep 25
mkdir data
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
composer run serve &
- name: Enable context_chat, app_api and testing
run: ./occ app:enable -vvv -f ${{ env.APP_NAME }} app_api testing
- name: Checkout documentation
uses: actions/checkout@v4
with:
repository: nextcloud/documentation
path: data/admin/files/documentation
- name: Prepare docs
run: |
cd data/admin/files
mv documentation/admin_manual .
cp -R documentation/developer_manual .
cd developer_manual
find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.md"' {} \;
cd ..
cp -R documentation/developer_manual ./developer_manual2
cd developer_manual2
find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.txt"' {} \;
cd ..
rm -rf documentation
- name: Setup python 3.11
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'
cache-dependency-path: context_chat_backend/requirements.txt
- name: Install and init backend
run: |
cd context_chat_backend
pip install --upgrade pip setuptools wheel
pip install -r requirements.txt
cp example.env .env
echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
python3 -u ./main.py > backend_logs 2>&1 &
echo $! > ../pid.txt # Save the process ID (PID)
- name: Register backend
run: |
set -x
./occ config:system:set debug --value true
./occ config:system:set loglevel --value 0
./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
- name: Scan files, baseline
run: |
./occ files:scan admin
./occ context_chat:scan admin -m text/plain
- name: Check python memory usage
run: |
ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
ps -p $(cat pid.txt) -o %mem --no-headers > initial_mem.txt
- name: Scan files
run: |
./occ files:scan admin
./occ context_chat:scan admin -m text/markdown &
./occ context_chat:scan admin -m text/x-rst
- name: Check python memory usage
run: |
ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
ps -p $(cat pid.txt) -o %mem --no-headers > after_scan_mem.txt
- name: Run the prompts
run: |
./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
- name: Check python memory usage
run: |
ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
ps -p $(cat pid.txt) -o %mem --no-headers > after_prompt_mem.txt
- name: Compare memory usage and detect leak
run: |
initial_mem=$(cat initial_mem.txt | tr -d ' ')
final_mem=$(cat after_scan_mem.txt | tr -d ' ')
echo "Initial Memory Usage: $initial_mem%"
echo "Memory Usage after scan: $final_mem%"
if (( $(echo "$final_mem > $initial_mem" | bc -l) )); then
echo "Memory usage has increased during scan. Possible memory leak detected!"
else
echo "Memory usage during scan is stable. No memory leak detected."
fi
- name: Compare memory usage and detect leak
run: |
initial_mem=$(cat after_scan_mem.txt | tr -d ' ')
final_mem=$(cat after_prompt_mem.txt | tr -d ' ')
echo "Initial Memory Usage: $initial_mem%"
echo "Memory Usage after prompt: $final_mem%"
if (( $(echo "$final_mem > $initial_mem" | bc -l) )); then
echo "Memory usage has increased during prompt. Possible memory leak detected!"
else
echo "Memory usage during prompt is stable. No memory leak detected."
fi
- name: Show logs
if: always()
run: |
tail data/nextcloud.log
echo '------------------------- BACKEND LOGS -------------------------'
[ -f context_chat_backend/backend_logs ] && cat context_chat_backend/backend_logs || echo "No backend logs"
echo '------------------------- EMBEDDING SERVER LOGS -------------------------'
[ -f context_chat_backend/embedding_model.log ] && cat context_chat_backend/embedding_model.log || echo "No embedding backend logs"