HydraConf, refactoring

.gitignore

@@ -13,4 +13,5 @@ __pycache__/
 .cache
 
 # due to using vscode
-.vscode/
+.vscode/
+outputs/

CICD/analyze.sh

@@ -0,0 +1,4 @@
+python -m pylint src
+python -m pyright src
+python -m black src --check
+python -m isort src --check-only

config/DevsDoCode-LLama-3-8b-Uncensored.yaml

@@ -1,17 +1,12 @@
 %YAML 1.1
 ---
-engine: !!str VLLMEngine
 
-model_name: !!str "DevsDoCode/LLama-3-8b-Uncensored"
+hf_model_name: !!str "Orenguteng/Llama-3-8B-Lexi-Uncensored-GGUF"
 tokenizer_name: "meta-llama/Meta-Llama-3-8B-Instruct"
 
-host: !!str "127.0.0.1"
-port: !!int 8000
+engine: !!str VLLMEngine
+engine_config:
 
-gui_title: !!str "LLama-3-8b-Uncensored"
-default_max_new_tokens: !!int 1000
-default_temperature: !!float 0.6
-default_top_p: !!float 0.95
-default_repetition_penalty: !!float 1.2
-default_frequency_penalty: !!float 0.0
-default_presence_penalty: !!float 0.0
+defaults:
+  - gui_config: default
+  - uvicorn_config: default

config/IlyaGusev-saiga_llama3_8b-4bits.yaml

@@ -1,18 +1,12 @@
 %YAML 1.1
 ---
-engine: !!str HFEngine
 
-model_name: !!str RichardErkhov/IlyaGusev_-_saiga_llama3_8b-4bits
+hf_model_name: !!str RichardErkhov/IlyaGusev_-_saiga_llama3_8b-4bits
 tokenizer_name: !!str RichardErkhov/IlyaGusev_-_saiga_llama3_8b-4bits
-# filename: !!str "*Q4_K_M.gguf"
 
-host: !!str "127.0.0.1"
-port: !!int 8000
-gui_title: !!str "LLama-3-8b-Uncensored"
+engine: !!str HFEngine
+engine_config:
 
-default_max_new_tokens: !!int 1000
-default_temperature: !!float 0.6
-default_top_p: !!float 0.95
-default_repetition_penalty: !!float 1.2
-default_frequency_penalty: !!float 0.0
-default_presence_penalty: !!float 0.0
+defaults:
+  - gui_config: default
+  - uvicorn_config: default

config/Llama-3-8B-Lexi-Uncensored.yaml

@@ -1,19 +1,14 @@
 %YAML 1.1
 ---
-engine: !!str LlamaCppEngine
 
-model_name: !!str "Orenguteng/Llama-3-8B-Lexi-Uncensored-GGUF"
+hf_model_name: !!str "Orenguteng/Llama-3-8B-Lexi-Uncensored-GGUF"
 tokenizer_name: !!str "meta-llama/Meta-Llama-3-8B-Instruct"
-filename: !!str "*Q4_K_M.gguf"
-n_gpu_layers: 0
 
-host: !!str "127.0.0.1"
-port: !!int 8000
-gui_title: !!str "LLama-3-8b-Uncensored-4bit-pruned"
+engine: !!str LlamaCppEngine
+engine_config:
+  filename: !!str "*Q4_K_M.gguf"
+  n_gpu_layers: 0
 
-default_max_new_tokens: !!int 1000
-default_temperature: !!float 0.6
-default_top_p: !!float 0.95
-default_repetition_penalty: !!float 1.2
-default_frequency_penalty: !!float 0.0
-default_presence_penalty: !!float 0.0
+defaults:
+  - gui_config: default
+  - uvicorn_config: default

config/Vikhr.yaml

@@ -1,18 +1,13 @@
 
 %YAML 1.1
 ---
-engine: !!str VLLMEngine
 
-model_name: !!str "Vikhrmodels/Vikhr-7B-instruct_0.2"
+hf_model_name: !!str "Vikhrmodels/Vikhr-7B-instruct_0.2"
 tokenizer_name: "Vikhrmodels/Vikhr-7B-instruct_0.2"
 
-host: !!str "127.0.0.1"
-port: !!int 8000
+engine: !!str VLLMEngine
+engine_config:
 
-gui_title: !!str "Vikhr-7B-instruct_0.2"
-default_max_new_tokens: !!int 1000
-default_temperature: !!float 0.6
-default_top_p: !!float 0.95
-default_repetition_penalty: !!float 1.2
-default_frequency_penalty: !!float 0.0
-default_presence_penalty: !!float 0.0
+defaults:
+  - gui_config: default
+  - uvicorn_config: default

config/gui_config/default.yaml

@@ -0,0 +1,7 @@
+default_sampling_config:
+  max_new_tokens: !!int 1000
+  temperature: !!float 0.6
+  top_p: !!float 0.95
+  repetition_penalty: !!float 1.2
+  frequency_penalty: !!float 0.0
+  presence_penalty: !!float 0.0

config/meta-llama-Meta-Llama-3-8B-Instruct.yaml

@@ -1,16 +1,12 @@
 %YAML 1.1
 ---
-engine: !!str VLLMEngine
 
-model_name: !!str "meta-llama/Meta-Llama-3-8B-Instruct"
+hf_model_name: !!str "meta-llama/Meta-Llama-3-8B-Instruct"
+tokenizer_name: !!str "meta-llama/Meta-Llama-3-8B-Instruct"
 
-host: !!str "127.0.0.1"
-port: !!int 8000
-gui_title: !!str "meta-llama/Meta-Llama-3-8B-Instruct"
+engine: !!str VLLMEngine
+engine_config:
 
-default_max_new_tokens: !!int 1000
-default_temperature: !!float 0.6
-default_top_p: !!float 0.95
-default_repetition_penalty: !!float 1.2
-default_frequency_penalty: !!float 0.0
-default_presence_penalty: !!float 0.0
+defaults:
+  - gui_config: default
+  - uvicorn_config: default

config/uvicorn_config/default.yaml

@@ -0,0 +1,4 @@
+host: !!str "127.0.0.1"
+port: !!int 8000
+log_level: !!str debug
+loop: !!str asyncio

src/AGISwarm/llm_instruct_ms/__main__.py

@@ -1,25 +1,30 @@
 """Main module for the LLM instruct microservice"""
 
-from argparse import ArgumentParser
+import os
+from pathlib import Path
 
+import hydra
 import uvicorn
 
 from .app import LLMInstructApp
-from .settings import LLMInstructSettings
+from .typing import LLMInstructConfig
 
 
-def main():
+@hydra.main(
+    config_path=str(Path(os.getcwd()) / "config"),
+)
+def main(config: LLMInstructConfig):
     """Main function"""
-    parser = ArgumentParser()
-    parser.add_argument("--config", type=str, help="Path to the configuration file")
 
-    args = parser.parse_args()
-    yaml_path = args.config
-
-    settings = LLMInstructSettings.from_yaml(yaml_path)
-    llm_instruct_app = LLMInstructApp(settings)
-    uvicorn.run(llm_instruct_app.app, host=settings.host, port=settings.port)
+    llm_instruct_app = LLMInstructApp(config)
+    uvicorn.run(
+        llm_instruct_app.app,
+        host=config.uvicorn_config.host,
+        port=config.uvicorn_config.port,
+        log_level=config.uvicorn_config.log_level,
+        loop=config.uvicorn_config.loop,
+    )
 
 
 if __name__ == "__main__":
-    main()
+    main()  # pylint: disable=no-value-for-parameter

src/AGISwarm/llm_instruct_ms/app/app.py → src/AGISwarm/llm_instruct_ms/app.py

@@ -2,28 +2,38 @@
 
 import uuid
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, cast
 
 from fastapi import APIRouter, FastAPI, WebSocket, WebSocketDisconnect
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 from jinja2 import Environment, FileSystemLoader
-from pydantic_settings import BaseSettings
+from omegaconf import OmegaConf
+from pydantic import BaseModel
 
-from ..llm_engines import EngineProtocol
-from ..settings import ENGINE_MAP, ENGINE_SAMPLING_PARAMS_MAP, LLMInstructSettings
+from .llm_engines import EngineProtocol
+from .typing import (
+    ENGINE_CONFIG_MAP,
+    ENGINE_MAP,
+    ENGINE_SAMPLING_PARAMS_MAP,
+    LLMInstructConfig,
+)
 
 
 class LLMInstructApp:  # pylint: disable=too-few-public-methods
     """Application factory"""
 
-    def __init__(self, settings: LLMInstructSettings):
-        self.settings = settings
+    def __init__(self, config: LLMInstructConfig):
+        self.config = config
         self.app = FastAPI()
-        self.llm: EngineProtocol[Any] = ENGINE_MAP[settings.engine](
-            **settings.engine_settings.model_dump()
+        if config.engine_config is None:
+            config.engine_config = ENGINE_CONFIG_MAP[config.engine]()
+        self.llm: EngineProtocol[Any] = ENGINE_MAP[config.engine](  # type: ignore
+            hf_model_name=config.hf_model_name,
+            tokenizer_name=config.tokenizer_name,
+            **cast(dict, OmegaConf.to_container(config.engine_config)),
         )
-        self.sampling_settings_cls = ENGINE_SAMPLING_PARAMS_MAP[settings.engine]
+        self.sampling_settings_cls = ENGINE_SAMPLING_PARAMS_MAP[config.engine]
         self._configure_routers()
 
     def _configure_routers(self):
@@ -52,7 +62,9 @@ async def get_root():  # type: ignore
             ) as f:
                 f.write(
                     template.render(
-                        **self.settings.model_dump(),
+                        OmegaConf.to_container(
+                            self.config.gui_config.default_sampling_config
+                        ),
                     )
                 )
             return FileResponse(Path(__file__).parent / "gui" / "current_index.html")
@@ -94,6 +106,9 @@ async def generate(websocket: WebSocket):  # type: ignore
                         elif response["response"] == "success":
                             reply += response["msg"]
                             await websocket.send_json(response)
+                        elif response["response"] == "abort":
+                            await websocket.send_json(response)
+                            break
                         else:
                             raise ValueError(
                                 f"Invalid response: {response['response']}"
@@ -111,7 +126,7 @@ async def generate(websocket: WebSocket):  # type: ignore
             finally:
                 await websocket.close()
 
-        class AbortRequest(BaseSettings):
+        class AbortRequest(BaseModel):
             """Abort request"""
 
             request_id: str

src/AGISwarm/llm_instruct_ms/app/gui/current_index.html → src/AGISwarm/llm_instruct_ms/gui/current_index.html

@@ -1,7 +1,7 @@
 <html>
 
 <head>
-    <title>LLama-3-8b-Uncensored</title>
+    <title>LLM Instruct Interface</title>
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
         integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
     <!--./style.css -->
@@ -15,7 +15,7 @@
     <div class="container">
         <div class="chat-container">
             <div class="chat-header">
-                <h1 id="title">LLama-3-8b-Uncensored</h1>
+                <h1 id="title"></h1>
                 <button id="menu-toggle" class="btn btn-secondary menu-toggle"><i class="fas fa-bars"></i></button>
             </div>
             <div id="chat-output" class="chat-output"></div>

src/AGISwarm/llm_instruct_ms/app/gui/jinja2.html → src/AGISwarm/llm_instruct_ms/gui/jinja2.html

@@ -1,7 +1,7 @@
 <html>
 
 <head>
-    <title>{{gui_title}}</title>
+    <title>LLM Instruct Interface</title>
     <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
         integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
     <!--./style.css -->
@@ -28,7 +28,7 @@ <h1 id="title">{{gui_title}}</h1>
             </div>
             <div class="input-container">
                 <textarea id="system_prompt" class="form-control" placeholder="Enter system prompt"
-                    value={{default_system_prompt}}></textarea>
+                    value={{system_prompt}}></textarea>
             </div>
         </div>
 
@@ -37,49 +37,49 @@ <h1 id="title">{{gui_title}}</h1>
             <div class="form-group">
                 <label for="max_new_tokens">Max tokens:</label>
                 <input type="number" id="max_new_tokens" class="form-control" placeholder="Enter max tokens"
-                    value="{{default_max_new_tokens}}">
+                    value="{{max_new_tokens}}">
             </div>
             <div class="form-group">
                 <span for="temperature">Temperature:</span>
                 <input type="number" id="temperature" class="form-control" min="1e-9" max="1.0" step="0.1"
-                    value="{{default_temperature}}">
+                    value="{{temperature}}">
 
             </div>
             <div class="form-group">
                 <span for="top_p">Top P:</span>
                 <input type="number" id="top_p" class="form-control" min="1e-9" max="1.0" step="0.1"
-                    value="{{default_top_p}}">
+                    value="{{top_p}}">
 
             </div>
             <div class="form-group">
                 <span for="repetition_penalty">Repetition Penalty:</span>
                 <input type="number" id="repetition_penalty" class="form-control" min="1e-9" max="2.0" step="0.1"
-                    value="{{default_repetition_penalty}}">
+                    value="{{repetition_penalty}}">
 
             </div>
             <div class="form-group">
                 <span for="frequency_penalty">Frequency Penalty:</span>
                 <input type="number" id="frequency_penalty" class="form-control" min="-2.0" max="2.0" step="0.1"
-                    value="{{default_frequency_penalty}}">
+                    value="{{frequency_penalty}}">
 
             </div>
             <div class="form-group">
                 <span for="presence_penalty">Presence Penalty:</span>
                 <input type="number" id="presence_penalty" class="form-control" min="-2.0" max="2.0" step="0.1"
-                    value="{{default_presence_penalty}}">
+                    value="{{presence_penalty}}">
 
             </div>
             <button type="button" class="btn btn-secondary" onclick="resetForm()">Reset</button>
         </div>
     </div>
     <script>
-        const DEFAULT_MAX_NEW_TOKENS = "{{default_max_new_tokens}}";
-        const DEFAULT_TEMPERATURE = "{{default_temperature}}";
-        const DEFAULT_TOP_P = "{{default_top_p}}";
-        const DEFAULT_REPETITION_PENALTY = "{{default_repetition_penalty}}";
-        const DEFAULT_FREQUENCY_PENALTY = "{{default_frequency_penalty}}";
-        const DEFAULT_PRESENCE_PENALTY = "{{default_presence_penalty}}";
-        const DEFAULT_SYSTEM_PROMPT = "{{default_system_prompt}}";
+        const DEFAULT_MAX_NEW_TOKENS = "{{max_new_tokens}}";
+        const DEFAULT_TEMPERATURE = "{{temperature}}";
+        const DEFAULT_TOP_P = "{{top_p}}";
+        const DEFAULT_REPETITION_PENALTY = "{{repetition_penalty}}";
+        const DEFAULT_FREQUENCY_PENALTY = "{{frequency_penalty}}";
+        const DEFAULT_PRESENCE_PENALTY = "{{presence_penalty}}";
+        const DEFAULT_SYSTEM_PROMPT = "{{system_prompt}}";
         const WEBSOCKET_URL = "/ws";
         const ABORT_URL = "/abort";
     </script>