From 3a70023bf7129c69284dfea6f7fb973903e29c94 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 20 Nov 2023 09:51:17 +0100 Subject: [PATCH 01/24] ci: add integration test --- .github/workflows/integration-test.yml | 146 +++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 .github/workflows/integration-test.yml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000..5cbf702 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +name: Integration test + +on: + pull_request: + push: + branches: + - main + - stable* + +env: + APP_NAME: translate + +jobs: + transcription: + runs-on: ubuntu-latest + + strategy: + # do not stop on another job's failure + fail-fast: false + matrix: + php-versions: [ '8.1', '8.2', '8.3' ] + databases: [ 'sqlite' ] + server-versions: [ 'master' ] + + name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }} + + env: + MYSQL_PORT: 4444 + PGSQL_PORT: 4445 + + services: + mysql: + image: mariadb:10.5 + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 + postgres: + image: postgres + ports: + - 4445:5432/tcp + env: + POSTGRES_USER: root + POSTGRES_PASSWORD: rootpassword + POSTGRES_DB: nextcloud + options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 + + steps: + - name: Checkout server + uses: actions/checkout@v2 + with: + repository: nextcloud/server + ref: ${{ matrix.server-versions }} + + - name: Checkout submodules + shell: bash + run: | + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-versions }} + tools: phpunit + extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip + + - name: Checkout app + uses: actions/checkout@v2 + with: + path: apps/${{ env.APP_NAME }} + + - name: Checkout server + uses: actions/checkout@v2 + with: + repository: kyteinski/cwyd_backend + path: cwyd_backend/ + + - name: Read package.json node and npm engines version + uses: skjnldsv/read-package-engines-version-actions@v1.1 + id: versions + with: + path: apps/${{ env.APP_NAME }} + fallbackNode: '^16' + fallbackNpm: '^8' + + - name: Set up node ${{ steps.versions.outputs.nodeVersion }} + uses: actions/setup-node@v2 + with: + node-version: ${{ steps.versions.outputs.nodeVersion }} + + - name: Set up npm ${{ steps.versions.outputs.npmVersion }} + run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}" + + - name: Install app + working-directory: apps/${{ env.APP_NAME }} + run: | + make all + composer install --no-dev + + - name: Set up Nextcloud and install app + if: ${{ matrix.databases != 'pgsql'}} + run: | + sleep 25 + mkdir data + ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable -vvv -f ${{ env.APP_NAME }} + php -S localhost:8080 & + + - name: Set up Nextcloud and install app + if: ${{ matrix.databases == 'pgsql'}} + run: | + sleep 25 + mkdir data + ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable -vvv -f ${{ env.APP_NAME }} + php -S localhost:8080 & + + - name: Install + run: | + ./occ app:enable -vvv ${{ env.APP_NAME }} + + - name: Install and init backend + run: | + cd cwyd_backend + python3 -m venv .venv + . .venv/bin/activate + pip install --no-deps -r reqs.txt + cp example.env .env + echo "\nDISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env + ./main.py & + + - name: Install and init backend + run: | + occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http null http://localhost:8080 + occ app_api:app:register schackles manual_install --json-info "{\"appid\":\"schackles\",\"name\":\"Chat With Your Documents Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"localhost\",\"port\":10034,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" --force-scopes + + - name: Show log on failure + if: always() + run: | + tail data/nextcloud.log From 14d9c064099c23482e0d4cdb807184c0a2207cea Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 20 Nov 2023 09:59:30 +0100 Subject: [PATCH 02/24] fix typo --- .github/workflows/integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 5cbf702..7969c72 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -75,10 +75,10 @@ jobs: with: path: apps/${{ env.APP_NAME }} - - name: Checkout server + - name: Checkout backend uses: actions/checkout@v2 with: - repository: kyteinski/cwyd_backend + repository: kyteinsky/cwyd_backend path: cwyd_backend/ - name: Read package.json node and npm engines version From 97a6ed268b441f57a066774a9184bf78b76fc50f Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 20 Nov 2023 10:34:47 +0100 Subject: [PATCH 03/24] fix typo --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 7969c72..168175b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -11,7 +11,7 @@ on: - stable* env: - APP_NAME: translate + APP_NAME: cwyd jobs: transcription: From bb1e3dcc77cb92c7244ef9332ffdb8f0a24dda8b Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 20 Nov 2023 10:41:10 +0100 Subject: [PATCH 04/24] fix backend init --- .github/workflows/integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 168175b..f7eb74e 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -137,8 +137,8 @@ jobs: - name: Install and init backend run: | - occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http null http://localhost:8080 - occ app_api:app:register schackles manual_install --json-info "{\"appid\":\"schackles\",\"name\":\"Chat With Your Documents Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"localhost\",\"port\":10034,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" --force-scopes + ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http null http://localhost:8080 + ./occ app_api:app:register schackles manual_install --json-info "{\"appid\":\"schackles\",\"name\":\"Chat With Your Documents Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"localhost\",\"port\":10034,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" --force-scopes - name: Show log on failure if: always() From 5e2fbd8888493ef40d7e07b7c0bc9e8ec69fad6e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 20 Nov 2023 11:06:21 +0100 Subject: [PATCH 05/24] fix: install app_api --- .github/workflows/integration-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index f7eb74e..01f03ba 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -121,6 +121,10 @@ jobs: ./occ app:enable -vvv -f ${{ env.APP_NAME }} php -S localhost:8080 & + - name: Install app_api + run: | + ./occ app:enable -vvv app_api + - name: Install run: | ./occ app:enable -vvv ${{ env.APP_NAME }} From 2ae464e5ea8e40089f6d44cda75240665cf26d38 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 20 Nov 2023 11:26:35 +0100 Subject: [PATCH 06/24] fix: download model --- .github/workflows/integration-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 01f03ba..b2465a6 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -137,6 +137,7 @@ jobs: pip install --no-deps -r reqs.txt cp example.env .env echo "\nDISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env + curl https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf ./main.py & - name: Install and init backend From 4358b1cd96f63d243912e5025f765f73ac8690cc Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Mon, 20 Nov 2023 11:37:14 +0100 Subject: [PATCH 07/24] fix: download model --- .github/workflows/integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b2465a6..281f7e3 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -136,8 +136,8 @@ jobs: . .venv/bin/activate pip install --no-deps -r reqs.txt cp example.env .env - echo "\nDISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env - curl https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf + echo "DISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env + curl -L https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf ./main.py & - name: Install and init backend From 2c9530d1dc1e704cd36a1d52aa0e97aaea768951 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 21 Nov 2023 14:17:27 +0100 Subject: [PATCH 08/24] fix: CacheQueryBuilder usage --- lib/Service/StorageService.php | 14 ++++++++++---- tests/stub.phpstub | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/Service/StorageService.php b/lib/Service/StorageService.php index 1eb3715..6db1d0e 100644 --- a/lib/Service/StorageService.php +++ b/lib/Service/StorageService.php @@ -15,6 +15,7 @@ use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; use OCP\Files\IMimeTypeLoader; +use OCP\FilesMetadata\IFilesMetadataManager; use OCP\IDBConnection; use Psr\Log\LoggerInterface; @@ -40,7 +41,8 @@ public function __construct( private LoggerInterface $logger, private SystemConfig $systemConfig, private IMimeTypeLoader $mimeTypes, - private IUserMountCache $userMountCache) { + private IUserMountCache $userMountCache, + private IFilesMetadataManager $metadataManager) { } /** @@ -64,7 +66,7 @@ public function getMounts(): \Generator { $overrideRoot = $rootId; if (in_array($row['mount_provider_class'], self::HOME_MOUNT_TYPES)) { // Only crawl files, not cache or trashbin - $qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger); + $qb = $this->getCacheQueryBuilder(); try { /** @var array|false $root */ $root = $qb->selectFileCache() @@ -97,7 +99,7 @@ public function getMounts(): \Generator { * @return \Generator */ public function getFilesInMount(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator { - $qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger); + $qb = $this->getCacheQueryBuilder(); try { $result = $qb->selectFileCache() ->andWhere($qb->expr()->eq('filecache.fileid', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT))) @@ -117,7 +119,7 @@ public function getFilesInMount(int $storageId, int $rootId, int $lastFileId = 0 $mimeTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), self::MIME_TYPES); - $qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger); + $qb = $this->getCacheQueryBuilder(); try { $path = $root['path'] === '' ? '' : $root['path'] . '/'; @@ -159,4 +161,8 @@ public function getUsersForFileId(int $fileId): array { return $mountInfo->getUser()->getUID(); }, $mountInfos); } + + private function getCacheQueryBuilder(): CacheQueryBuilder { + return new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger, $this->metadataManager); + } } diff --git a/tests/stub.phpstub b/tests/stub.phpstub index 25f781f..5fcc1cb 100644 --- a/tests/stub.phpstub +++ b/tests/stub.phpstub @@ -23,7 +23,8 @@ namespace OC { namespace OC\Files\Cache { class CacheQueryBuilder extends \OCP\DB\QueryBuilder\IQueryBuilder { - public function __construct(\OCP\IDBCOnnection $db, \OC\SystemConfig $config, \Psr\Log\LoggerInterface $logger); + public function __construct(\OCP\IDBCOnnection $db, \OC\SystemConfig $config, \Psr\Log\LoggerInterface $logger, \OCP\FilesMetadata\IFilesMetadataManager $filesMetadataManager, + ); public function selectFileCache(string $alias = null, bool $joinExtendedCache = true):CacheQueryBuilder; public function whereStorageId(int $storageId):CacheQueryBuilder; public function whereFileId(int $fileId):CacheQueryBuilder; From 41fba6588bf9620cb7e89d22f12983482adb2f8c Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 22 Nov 2023 14:22:37 +0100 Subject: [PATCH 09/24] fix: integration-test.yml --- .github/workflows/integration-test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 281f7e3..032c70a 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -137,13 +137,15 @@ jobs: pip install --no-deps -r reqs.txt cp example.env .env echo "DISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env + echo "NEXTCLOUD_URL=http://localhost:8080" >> .env curl -L https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf ./main.py & - name: Install and init backend run: | ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http null http://localhost:8080 - ./occ app_api:app:register schackles manual_install --json-info "{\"appid\":\"schackles\",\"name\":\"Chat With Your Documents Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"localhost\",\"port\":10034,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" --force-scopes + # '&' because app:register has a bug that causes it to stall forever... + ./occ app_api:app:register schackles manual_install --json-info "{\"appid\":\"schackles\",\"name\":\"Chat With Your Documents Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"localhost\",\"port\":10034,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" --force-scopes & - name: Show log on failure if: always() From 656612c557b6b86329f0fdf119782e4734720c13 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 22 Nov 2023 14:23:01 +0100 Subject: [PATCH 10/24] fix: remove controller --- appinfo/routes.php | 2 -- lib/Controller/CwydController.php | 46 ------------------------------- 2 files changed, 48 deletions(-) delete mode 100644 lib/Controller/CwydController.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 7e14308..a85c98c 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -13,7 +13,5 @@ 'routes' => [ ['name' => 'config#setConfig', 'url' => '/config', 'verb' => 'PUT'], ['name' => 'config#setAdminConfig', 'url' => '/admin-config', 'verb' => 'PUT'], - - ['name' => 'cwyd#query', 'url' => '/query', 'verb' => 'POST'], ], ]; diff --git a/lib/Controller/CwydController.php b/lib/Controller/CwydController.php deleted file mode 100644 index 93c5095..0000000 --- a/lib/Controller/CwydController.php +++ /dev/null @@ -1,46 +0,0 @@ - - * @copyright Julien Veyssier 2022 - */ - -namespace OCA\Cwyd\Controller; - -use OCA\Cwyd\Service\LangRopeService; -use OCP\AppFramework\Controller; -use OCP\AppFramework\Http; -use OCP\AppFramework\Http\Attribute\NoAdminRequired; -use OCP\AppFramework\Http\DataResponse; -use OCP\IRequest; - -class CwydController extends Controller { - - public function __construct( - string $appName, - IRequest $request, - private LangRopeService $langRopeService, - private ?string $userId - ) { - parent::__construct($appName, $request); - } - - /** - * @param string $prompt - * @param int|null $n - * @param int|null $maxTokens - * @return DataResponse - */ - #[NoAdminRequired] - public function query(string $prompt, ?int $n = null, ?int $maxTokens = null): DataResponse { - $response = $this->langRopeService->query($this->userId, $prompt); - if (isset($response['error'])) { - return new DataResponse($response, Http::STATUS_BAD_REQUEST); - } - return new DataResponse($response); - } -} From b3b76e1be19792d5dcf01e6799d9958a0b02c6e9 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 22 Nov 2023 14:23:15 +0100 Subject: [PATCH 11/24] fix: increase cwyd timeout --- lib/AppInfo/Application.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index a20f74b..47f2982 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -28,7 +28,7 @@ class Application extends App implements IBootstrap { public const APP_ID = 'cwyd'; - public const CWYD_DEFAULT_REQUEST_TIMEOUT = 60 * 4; + public const CWYD_DEFAULT_REQUEST_TIMEOUT = 60 * 50; // max size per file + max size of the batch of files to be embedded in a single request public const CWYD_MAX_SIZE = 20 * 1024 * 1024; // 20MB public const CWYD_MAX_FILES = 100; From 27c4271cb91ded9ea50524cc344234c63a206b9e Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 22 Nov 2023 14:23:59 +0100 Subject: [PATCH 12/24] fix(TextProcessing\Providers) --- lib/TextProcessing/CwydProvider.php | 6 +++--- lib/TextProcessing/FreePromptProvider.php | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/TextProcessing/CwydProvider.php b/lib/TextProcessing/CwydProvider.php index fff7f81..40b9423 100644 --- a/lib/TextProcessing/CwydProvider.php +++ b/lib/TextProcessing/CwydProvider.php @@ -28,10 +28,10 @@ public function getName(): string { public function process(string $prompt): string { $response = $this->langRopeService->query($this->userId, $prompt); - if (isset($response['result']) && $response['result']) { - return $response['result']; + if (isset($response['error'])) { + throw new \RuntimeException('No result in Cwyd response. ' . $response['error']); } - throw new \Exception('No result in Cwyd response. ' . ($response['error'] ?? '')); + return $response['message'] ?? ''; } public function getTaskType(): string { diff --git a/lib/TextProcessing/FreePromptProvider.php b/lib/TextProcessing/FreePromptProvider.php index 1aa2544..1870206 100644 --- a/lib/TextProcessing/FreePromptProvider.php +++ b/lib/TextProcessing/FreePromptProvider.php @@ -29,10 +29,10 @@ public function getName(): string { public function process(string $prompt): string { $response = $this->langRopeService->query($this->userId, $prompt); - if (isset($response['result']) && $response['result']) { - return $response['result']; + if (isset($response['error'])) { + throw new \RuntimeException('No result in Cwyd response. ' . $response['error']); } - throw new \Exception('No result in Cwyd response. ' . ($response['error'] ?? '')); + return $response['message'] ?? ''; } public function getTaskType(): string { From 42a7cbdab971c36ce5a905e51f6921c921201b28 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 22 Nov 2023 14:24:58 +0100 Subject: [PATCH 13/24] fix(FileListener): Implement ShareDeletedEvent handler --- lib/Listener/FileListener.php | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/Listener/FileListener.php b/lib/Listener/FileListener.php index 42bc33f..d03d1b4 100644 --- a/lib/Listener/FileListener.php +++ b/lib/Listener/FileListener.php @@ -115,10 +115,39 @@ public function handle(Event $event): void { } $files = $this->storageService->getFilesInMount($mount->getNumericStorageId(), $node->getId(), 0, 0); foreach ($files as $fileId) { - $this->postInsert(current($this->rootFolder->getById($fileId)), false, true); + $node = current($this->rootFolder->getById($fileId)); + try { + $fileHandle = $node->fopen('r'); + } catch (LockedException|NotPermittedException $e) { + $this->logger->error('Could not open file ' . $node->getPath() . ' for reading', ['exception' => $e]); + return; + } + foreach ($userIds as $userId) { + try { + $source = new Source($userId, 'file: ' . $node->getId(), $fileHandle, $node->getMtime(), $node->getMimeType()); + } catch (InvalidPathException|NotFoundException $e) { + $this->logger->error('Could not find file ' . $node->getPath(), ['exception' => $e]); + break; + } + $this->langRopeService->deleteSources($userId, [$source]); + } } } else { - // TODO: remove index entry of $node for $userIds + try { + $fileHandle = $node->fopen('r'); + } catch (LockedException|NotPermittedException $e) { + $this->logger->error('Could not open file ' . $node->getPath() . ' for reading', ['exception' => $e]); + return; + } + foreach ($userIds as $userId) { + try { + $source = new Source($userId, 'file: ' . $node->getId(), $fileHandle, $node->getMtime(), $node->getMimeType()); + } catch (InvalidPathException|NotFoundException $e) { + $this->logger->error('Could not find file ' . $node->getPath(), ['exception' => $e]); + break; + } + $this->langRopeService->deleteSources($userId, [$source]); + } } } if ($event instanceof BeforeNodeDeletedEvent) { From 1d92ff75f3a94e2e0643706092ea988768946034 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 22 Nov 2023 14:28:07 +0100 Subject: [PATCH 14/24] fix(FileListener): Fix psalm issue --- lib/Listener/FileListener.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Listener/FileListener.php b/lib/Listener/FileListener.php index d03d1b4..b6dd436 100644 --- a/lib/Listener/FileListener.php +++ b/lib/Listener/FileListener.php @@ -108,7 +108,7 @@ public function handle(Event $event): void { */ $userIds = array_keys($accessList['users']); - if ($node->getType() === FileInfo::TYPE_FOLDER) { + if ($node instanceof File) { $mount = $node->getMountPoint(); if ($mount->getNumericStorageId() === null) { return; From 66d3d76f76c5ea7276a439a7c72e0d6ad40cd0b4 Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Wed, 22 Nov 2023 14:52:07 +0100 Subject: [PATCH 15/24] enh: Add prompt command Signed-off-by: Marcel Klehr --- .github/workflows/integration-test.yml | 25 ++++++++++- appinfo/info.xml | 1 + lib/Command/Prompt.php | 59 ++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 lib/Command/Prompt.php diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 032c70a..c9d9f91 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -13,6 +13,11 @@ on: env: APP_NAME: cwyd +concurrency: + group: integration-test-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + jobs: transcription: runs-on: ubuntu-latest @@ -21,7 +26,7 @@ jobs: # do not stop on another job's failure fail-fast: false matrix: - php-versions: [ '8.1', '8.2', '8.3' ] + php-versions: [ '8.1' ] databases: [ 'sqlite' ] server-versions: [ 'master' ] @@ -141,12 +146,28 @@ jobs: curl -L https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf ./main.py & - - name: Install and init backend + - name: Register backend run: | ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http null http://localhost:8080 # '&' because app:register has a bug that causes it to stall forever... ./occ app_api:app:register schackles manual_install --json-info "{\"appid\":\"schackles\",\"name\":\"Chat With Your Documents Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"localhost\",\"port\":10034,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" --force-scopes & + - name: Checkout documentation + uses: actions/checkout@v2 + with: + repository: nextcloud/documentation + path: data/admin/files/documentation + + - name: Prepare docs + run: | + cd data/admin/files/documentation + find . -name "*.rst" -exec bash -c 'mv "$1" "${1%.rst}".txt' - '{}' + + ./occ files:scan --all + + - name: Run prompt + run: | + ./occ cwyd:prompt admin "Which factors are taken into account for the Ethical AI Rating?" + - name: Show log on failure if: always() run: | diff --git a/appinfo/info.xml b/appinfo/info.xml index 950852a..be37a50 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -27,6 +27,7 @@ OCA\Cwyd\Command\ScanFiles + OCA\Cwyd\Command\Prompt