ci: add integration test

.github/workflows/integration-test.yml

@@ -0,0 +1,179 @@
+# SPDX-FileCopyrightText: Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: Integration test
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - stable*
+
+env:
+  APP_NAME: cwyd
+
+concurrency:
+  group: integration-test-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+
+jobs:
+  transcription:
+    runs-on: ubuntu-latest
+
+    strategy:
+      # do not stop on another job's failure
+      fail-fast: false
+      matrix:
+        php-versions: [ '8.1' ]
+        databases: [ 'sqlite' ]
+        server-versions: [ 'master' ]
+
+    name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
+
+    env:
+      MYSQL_PORT: 4444
+      PGSQL_PORT: 4445
+
+    services:
+      mysql:
+        image: mariadb:10.5
+        ports:
+          - 4444:3306/tcp
+        env:
+          MYSQL_ROOT_PASSWORD: rootpassword
+        options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
+      postgres:
+        image: postgres
+        ports:
+          - 4445:5432/tcp
+        env:
+          POSTGRES_USER: root
+          POSTGRES_PASSWORD: rootpassword
+          POSTGRES_DB: nextcloud
+        options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
+
+    steps:
+      - name: Checkout server
+        uses: actions/checkout@v2
+        with:
+          repository: nextcloud/server
+          ref: ${{ matrix.server-versions }}
+
+      - name: Checkout submodules
+        shell: bash
+        run: |
+          auth_header="$(git config --local --get http.https://github.com/.extraheader)"
+          git submodule sync --recursive
+          git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1
+
+      - name: Set up php ${{ matrix.php-versions }}
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php-versions }}
+          tools: phpunit
+          extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip
+
+      - name: Checkout app
+        uses: actions/checkout@v2
+        with:
+          path: apps/${{ env.APP_NAME }}
+
+      - name: Checkout backend
+        uses: actions/checkout@v2
+        with:
+          repository: kyteinsky/cwyd_backend
+          path: cwyd_backend/
+
+      - name: Read package.json node and npm engines version
+        uses: skjnldsv/read-package-engines-version-actions@v1.1
+        id: versions
+        with:
+          path: apps/${{ env.APP_NAME }}
+          fallbackNode: '^16'
+          fallbackNpm: '^8'
+
+      - name: Set up node ${{ steps.versions.outputs.nodeVersion }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ steps.versions.outputs.nodeVersion }}
+
+      - name: Set up npm ${{ steps.versions.outputs.npmVersion }}
+        run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}"
+
+      - name: Install app
+        working-directory: apps/${{ env.APP_NAME }}
+        run: |
+          make all
+          composer install --no-dev
+
+      - name: Set up Nextcloud and install app
+        if: ${{ matrix.databases != 'pgsql'}}
+        run: |
+          sleep 25
+          mkdir data
+          ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
+          ./occ app:enable -vvv -f ${{ env.APP_NAME }}
+          php -S localhost:8080 &
+
+      - name: Set up Nextcloud and install app
+        if: ${{ matrix.databases == 'pgsql'}}
+        run: |
+          sleep 25
+          mkdir data
+          ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
+          ./occ app:enable -vvv -f ${{ env.APP_NAME }}
+          php -S localhost:8080 &
+
+      - name: Install app_api
+        run: |
+          ./occ app:enable -vvv app_api
+
+      - name: Install
+        run: |
+          ./occ app:enable -vvv ${{ env.APP_NAME }}
+
+      - name: Install and init backend
+        run: |
+          cd cwyd_backend
+          python3 -m venv .venv
+          . .venv/bin/activate
+          pip install --no-deps -r reqs.txt
+          cp example.env .env
+          echo "DISABLE_CUSTOM_DOWNLOAD_URI=1" >> .env
+          echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
+          curl -L https://huggingface.co/TheBloke/dolphin-2.2.1-mistral-7B-GGUF/resolve/main/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf -o model_files/dolphin-2.2.1-mistral-7b.Q5_K_M.gguf
+          ./main.py &
+
+      - name: Register backend
+        run: |
+          ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http null http://localhost:8080
+          # '&' because app:register has a bug that causes it to stall forever...
+          ./occ app_api:app:register schackles manual_install --json-info "{\"appid\":\"schackles\",\"name\":\"Chat With Your Documents Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"localhost\",\"port\":10034,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" --force-scopes &
+
+      - name: Checkout documentation
+        uses: actions/checkout@v2
+        with:
+          repository: nextcloud/documentation
+          path: data/admin/files/documentation
+
+      - name: Prepare docs
+        run: |
+          cd data/admin/files/documentation
+          find ./ -depth -name "*.rst" -exec sh -c 'mv "$1" "${1%.rst}.txt"' _ {} \;
+          git status
+
+      - name: Scan files
+        run: |
+          ./occ files:scan --all
+          ./occ cwyd:scan admin
+
+      - name: Run prompt
+        run: |
+          ./occ cwyd:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
+
+      - name: Show log on failure
+        if: always()
+        run: |
+          tail data/nextcloud.log

appinfo/info.xml

@@ -27,6 +27,7 @@
 	</background-jobs>
 	<commands>
 		<command>OCA\Cwyd\Command\ScanFiles</command>
+		<command>OCA\Cwyd\Command\Prompt</command>
 	</commands>
     <!--settings>
         <admin>OCA\Cwyd\Settings\Admin</admin>

appinfo/routes.php

@@ -13,7 +13,5 @@
 	'routes' => [
 		['name' => 'config#setConfig', 'url' => '/config', 'verb' => 'PUT'],
 		['name' => 'config#setAdminConfig', 'url' => '/admin-config', 'verb' => 'PUT'],
-
-		['name' => 'cwyd#query', 'url' => '/query', 'verb' => 'POST'],
 	],
 ];

lib/AppInfo/Application.php

@@ -28,7 +28,7 @@ class Application extends App implements IBootstrap {
 
 	public const APP_ID = 'cwyd';
 
-	public const CWYD_DEFAULT_REQUEST_TIMEOUT = 60 * 4;
+	public const CWYD_DEFAULT_REQUEST_TIMEOUT = 60 * 50;
 	// max size per file + max size of the batch of files to be embedded in a single request
 	public const CWYD_MAX_SIZE = 20 * 1024 * 1024; // 20MB
 	public const CWYD_MAX_FILES = 100;

lib/Command/Prompt.php

@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * Nextcloud - Cwyd
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Julien Veyssier <julien-nc@posteo.net>
+ * @copyright Julien Veyssier 2023
+ */
+
+namespace OCA\Cwyd\Command;
+
+use OCA\Cwyd\TextProcessing\CwydTaskType;
+use OCP\TextProcessing\IManager;
+use OCP\TextProcessing\Task;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Prompt extends Command {
+
+	public function __construct(
+		private IManager $textProcessingManager,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this->setName('cwyd:prompt')
+			->setDescription('Prompt Chat with your documents')
+			->addArgument(
+				'user_id',
+				InputArgument::REQUIRED,
+				'The ID of the user to prompt the documents of'
+			)
+			->addArgument(
+				'prompt',
+				InputArgument::REQUIRED,
+				'The prompt'
+			);
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		$userId = $input->getArgument('user_id');
+		$prompt = $input->getArgument('prompt');
+		$task = new Task(CwydTaskType::class, $prompt, 'cwyd', $userId);
+
+		$this->textProcessingManager->runTask($task);
+		$output->writeln($task->getOutput());
+
+		return 0;
+	}
+}

lib/Listener/FileListener.php

@@ -108,17 +108,49 @@ public function handle(Event $event): void {
 			 */
 			$userIds = array_keys($accessList['users']);
 
-			if ($node->getType() === FileInfo::TYPE_FOLDER) {
+			if ($node instanceof Folder) {
 				$mount = $node->getMountPoint();
 				if ($mount->getNumericStorageId() === null) {
 					return;
 				}
 				$files = $this->storageService->getFilesInMount($mount->getNumericStorageId(), $node->getId(), 0, 0);
 				foreach ($files as $fileId) {
-					$this->postInsert(current($this->rootFolder->getById($fileId)), false, true);
+					$node = current($this->rootFolder->getById($fileId));
+					if (!$node instanceof File) {
+						continue;
+					}
+					try {
+						$fileHandle = $node->fopen('r');
+					} catch (LockedException|NotPermittedException $e) {
+						$this->logger->error('Could not open file ' . $node->getPath() . ' for reading', ['exception' => $e]);
+						return;
+					}
+					foreach ($userIds as $userId) {
+						try {
+							$source = new Source($userId, 'file: ' . $node->getId(), $fileHandle, $node->getMtime(), $node->getMimeType());
+						} catch (InvalidPathException|NotFoundException $e) {
+							$this->logger->error('Could not find file ' . $node->getPath(), ['exception' => $e]);
+							break;
+						}
+						$this->langRopeService->deleteSources($userId, [$source]);
+					}
 				}
 			} else {
-				// TODO: remove index entry of $node for $userIds
+				try {
+					$fileHandle = $node->fopen('r');
+				} catch (LockedException|NotPermittedException $e) {
+					$this->logger->error('Could not open file ' . $node->getPath() . ' for reading', ['exception' => $e]);
+					return;
+				}
+				foreach ($userIds as $userId) {
+					try {
+						$source = new Source($userId, 'file: ' . $node->getId(), $fileHandle, $node->getMtime(), $node->getMimeType());
+					} catch (InvalidPathException|NotFoundException $e) {
+						$this->logger->error('Could not find file ' . $node->getPath(), ['exception' => $e]);
+						break;
+					}
+					$this->langRopeService->deleteSources($userId, [$source]);
+				}
 			}
 		}
 		if ($event instanceof BeforeNodeDeletedEvent) {

lib/Service/LangRopeService.php

@@ -218,7 +218,7 @@ public function requestToExApp(
 			}
 
 			return $body;
-		} catch (\Exception $e) {
+		} catch (\Throwable $e) {
 			$this->logger->error(
 				sprintf('Error during request to ExApp %s: %s', $exApp->getAppid(), $e->getMessage()),
 				['exception' => $e]