feat: Use Taskprocessing TextToText provider as LLM

.github/workflows/integration-test.yml

@@ -42,7 +42,7 @@ jobs:
       matrix:
         php-versions: [ '8.1' ]
         databases: [ 'sqlite' ]
-        server-versions: [ 'master', 'stable28', 'stable29' ]
+        server-versions: [ 'master' ]
 
     name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
 
@@ -123,7 +123,7 @@ jobs:
           php -S localhost:8080 &
 
       - name: Enable context_chat and app_api
-        run: ./occ app:enable -vvv -f context_chat app_api
+        run: ./occ app:enable -vvv -f context_chat app_api testing
 
       - name: Checkout documentation
         uses: actions/checkout@v4
@@ -166,6 +166,8 @@ jobs:
 
       - name: Run the prompts
         run: |
+          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
           ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
 

appinfo/info.xml

@@ -25,7 +25,7 @@ Install the given apps for Context Chat to work as desired **in the given order*
 	<bugs>https://github.com/nextcloud/context_chat_backend/issues</bugs>
 	<repository type="git">https://github.com/nextcloud/context_chat_backend.git</repository>
 	<dependencies>
-		<nextcloud min-version="28" max-version="30"/>
+		<nextcloud min-version="30" max-version="30"/>
 	</dependencies>
 	<external-app>
 		<docker-install>

config.cpu.yaml

@@ -40,6 +40,8 @@ embedding:
       device: cpu
 
 llm:
+  nc_texttotext:
+
   llama:
     model_path: dolphin-2.2.1-mistral-7b.Q5_K_M.gguf
     n_batch: 512

config.gpu.yaml

@@ -40,6 +40,8 @@ embedding:
       device: cuda
 
 llm:
+  nc_texttotext:
+
   llama:
     model_path: dolphin-2.2.1-mistral-7b.Q5_K_M.gguf
     n_batch: 512

context_chat_backend/chain/query_proc.py

@@ -25,7 +25,7 @@ def get_pruned_query(llm: LLM, config: TConfig, query: str, template: str, text_
 		or llm_config.get('config', {}).get('max_new_tokens') \
 		or max(
 			llm_config.get('pipeline_kwargs', {}).get('config', {}).get('max_new_tokens', 0),
-			llm_config.get('pipeline_kwargs', {}).get('config', {}).get('max_length')
+			llm_config.get('pipeline_kwargs', {}).get('config', {}).get('max_length', 0)
 		) \
 		or 4096
 

context_chat_backend/controller.py

@@ -12,6 +12,7 @@
 from .config_parser import get_config
 from .download import background_init, ensure_models
 from .dyn_loader import EmbeddingModelLoader, LLMModelLoader, LoaderException, VectorDBLoader
+from .models import LlmException
 from .ocs_utils import AppAPIAuthMiddleware
 from .setup_functions import ensure_config_file, repair_run, setup_env_vars
 from .utils import JSONResponse, enabled_guard, update_progress, value_of
@@ -105,6 +106,11 @@ async def _(request: Request, exc: ValueError):
 	return JSONResponse(str(exc), 400)
 
 
+@app.exception_handler(LlmException)
+async def _(request: Request, exc: LlmException):
+	log_error(f'Llm Error: {request.url.path}:', exc)
+	return JSONResponse(str(exc), 400)
+
 # routes
 
 @app.get('/')

context_chat_backend/download.py

@@ -202,7 +202,8 @@ def background_init(app: FastAPI):
 	for model_type in ('embedding', 'llm'):
 		model_name = _get_model_name_or_path(config, model_type)
 		if model_name is None:
-			raise Exception(f'Error: Model name/path not found for {model_type}')
+			update_progress(app, progress := progress + 50)
+			continue
 
 		if not _download_model(model_name):
 			raise Exception(f'Error: Model download failed for {model_name}')
@@ -220,7 +221,7 @@ def ensure_models(app: FastAPI) -> bool:
 	for model_type in ('embedding', 'llm'):
 		model_name = _get_model_name_or_path(app.extra['CONFIG'], model_type)
 		if model_name is None:
-			return False
+			return True
 
 		if not _model_exists(model_name):
 			return False

context_chat_backend/models/__init__.py

@@ -4,53 +4,56 @@
 from langchain.llms.base import LLM
 from langchain.schema.embeddings import Embeddings
 
-_embedding_models = ['llama', 'hugging_face', 'instructor']
-_llm_models = ['llama', 'hugging_face', 'ctransformer']
+_embedding_models = ["llama", "hugging_face", "instructor"]
+_llm_models = ["nc_texttotext", "llama", "hugging_face", "ctransformer"]
 
 models = {
-	'embedding': _embedding_models,
-	'llm': _llm_models,
+    "embedding": _embedding_models,
+    "llm": _llm_models,
 }
 
-__all__ = ['init_model', 'load_model', 'models']
+__all__ = ["init_model", "load_model", "models", "LlmException"]
 
 
 def load_model(model_type: str, model_info: tuple[str, dict]) -> Embeddings | LLM | None:
-	model_name, model_config = model_info
+    model_name, model_config = model_info
 
-	try:
-		module = import_module(f'.{model_name}', 'context_chat_backend.models')
-	except Exception as e:
-		raise AssertionError(f'Error: could not load {model_name} model from context_chat_backend/models') from e
+    try:
+        module = import_module(f".{model_name}", "context_chat_backend.models")
+    except Exception as e:
+        raise AssertionError(f"Error: could not load {model_name} model from context_chat_backend/models") from e
 
-	if module is None or not hasattr(module, 'get_model_for'):
-		raise AssertionError(f'Error: could not load {model_name} model')
+    if module is None or not hasattr(module, "get_model_for"):
+        raise AssertionError(f"Error: could not load {model_name} model")
 
-	get_model_for = module.get_model_for
+    get_model_for = module.get_model_for
 
-	if not isinstance(get_model_for, Callable):
-		raise AssertionError(f'Error: {model_name} does not have a valid loader function')
+    if not isinstance(get_model_for, Callable):
+        raise AssertionError(f"Error: {model_name} does not have a valid loader function")
 
-	return get_model_for(model_type, model_config)
+    return get_model_for(model_type, model_config)
 
 
 def init_model(model_type: str, model_info: tuple[str, dict]):
-	'''
-	Initializes a given model. This function assumes that the model is implemented in a module with
-	the same name as the model in the models dir.
-	'''
-	model_name, _ = model_info
-	available_models = models.get(model_type, [])
+    """
+    Initializes a given model. This function assumes that the model is implemented in a module with
+    the same name as the model in the models dir.
+    """
+    model_name, _ = model_info
+    available_models = models.get(model_type, [])
 
-	if model_name not in available_models:
-		raise AssertionError(f'Error: {model_type}_model should be one of {available_models}')
+    if model_name not in available_models:
+        raise AssertionError(f"Error: {model_type}_model should be one of {available_models}")
 
-	try:
-		model = load_model(model_type, model_info)
-	except Exception as e:
-		raise AssertionError(f'Error: {model_name} failed to load') from e
+    try:
+        model = load_model(model_type, model_info)
+    except Exception as e:
+        raise AssertionError(f"Error: {model_name} failed to load") from e
 
-	if model_type == 'llm' and not isinstance(model, LLM):
-		raise AssertionError(f'Error: {model} does not implement "llm" type or has returned an invalid object')
+    if model_type == "llm" and not isinstance(model, LLM):
+        raise AssertionError(f'Error: {model} does not implement "llm" type or has returned an invalid object')
 
-	return model
+    return model
+
+
+class LlmException(Exception): ...

context_chat_backend/models/ctransformer.py

@@ -4,16 +4,16 @@
 
 
 def get_model_for(model_type: str, model_config: dict):
-	model_dir = getenv('MODEL_DIR', 'persistent_storage/model_files')
-	if str(model_config.get('model')).startswith('/'):
-		model_dir = ''
+    model_dir = getenv("MODEL_DIR", "persistent_storage/model_files")
+    if str(model_config.get("model")).startswith("/"):
+        model_dir = ""
 
-	model_path = path.join(model_dir, model_config.get('model', ''))
+    model_path = path.join(model_dir, model_config.get("model", ""))
 
-	if model_config is None:
-		return None
+    if model_config is None:
+        return None
 
-	if model_type == 'llm':
-		return CTransformers(**{ **model_config, 'model': model_path })
+    if model_type == "llm":
+        return CTransformers(**{**model_config, "model": model_path})
 
-	return None
+    return None

context_chat_backend/models/hugging_face.py

@@ -5,22 +5,22 @@
 
 
 def get_model_for(model_type: str, model_config: dict):
-	if model_config.get('model_path') is not None:
-		model_dir = getenv('MODEL_DIR', 'persistent_storage/model_files')
-		if str(model_config.get('model_path')).startswith('/'):
-			model_dir = ''
+    if model_config.get("model_path") is not None:
+        model_dir = getenv("MODEL_DIR", "persistent_storage/model_files")
+        if str(model_config.get("model_path")).startswith("/"):
+            model_dir = ""
 
-		model_path = path.join(model_dir, model_config.get('model_path', ''))
-	else:
-		model_path = model_config.get('model_id', '')
+        model_path = path.join(model_dir, model_config.get("model_path", ""))
+    else:
+        model_path = model_config.get("model_id", "")
 
-	if model_config is None:
-		return None
+    if model_config is None:
+        return None
 
-	if model_type == 'embedding':
-		return HuggingFaceEmbeddings(**model_config)
+    if model_type == "embedding":
+        return HuggingFaceEmbeddings(**model_config)
 
-	if model_type == 'llm':
-		return HuggingFacePipeline.from_model_id(**{ **model_config, 'model_id': model_path })
+    if model_type == "llm":
+        return HuggingFacePipeline.from_model_id(**{**model_config, "model_id": model_path})
 
-	return None
+    return None

context_chat_backend/models/instructor.py

@@ -2,10 +2,10 @@
 
 
 def get_model_for(model_type: str, model_config: dict):
-	if model_config is None:
-		return None
+    if model_config is None:
+        return None
 
-	if model_type == 'embedding':
-		return HuggingFaceInstructEmbeddings(**model_config)
+    if model_type == "embedding":
+        return HuggingFaceInstructEmbeddings(**model_config)
 
-	return None
+    return None

context_chat_backend/models/llama.py

@@ -5,19 +5,19 @@
 
 
 def get_model_for(model_type: str, model_config: dict):
-	model_dir = getenv('MODEL_DIR', 'persistent_storage/model_files')
-	if str(model_config.get('model_path')).startswith('/'):
-		model_dir = ''
+    model_dir = getenv("MODEL_DIR", "persistent_storage/model_files")
+    if str(model_config.get("model_path")).startswith("/"):
+        model_dir = ""
 
-	model_path = path.join(model_dir, model_config.get('model_path', ''))
+    model_path = path.join(model_dir, model_config.get("model_path", ""))
 
-	if model_config is None:
-		return None
+    if model_config is None:
+        return None
 
-	if model_type == 'embedding':
-		return LlamaCppEmbeddings(**{ **model_config, 'model_path': model_path })
+    if model_type == "embedding":
+        return LlamaCppEmbeddings(**{**model_config, "model_path": model_path})
 
-	if model_type == 'llm':
-		return LlamaCpp(**{ **model_config, 'model_path': model_path })
+    if model_type == "llm":
+        return LlamaCpp(**{**model_config, "model_path": model_path})
 
-	return None
+    return None