fix mudler#1746 merge stuff

dave-gray101 · Mar 7, 2024 · fdbc871 · fdbc871
1 parent 570d9ee
commit fdbc871
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/core/backend/image.go b/core/backend/image.go
@@ -232,6 +232,13 @@ func (igbs *ImageGenerationBackendService) GenerateImage(request *schema.OpenAIR
 
 func imageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, backendConfig *config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error) {
 
+	threads := backendConfig.Threads
+	if threads == 0 && appConfig.Threads != 0 {
+		threads = appConfig.Threads
+	}
+
+	gRPCOpts := gRPCModelOpts(backendConfig)
+
 	opts := modelOpts(backendConfig, appConfig, []model.Option{
 		model.WithBackendString(backendConfig.Backend),
 		model.WithAssetDir(appConfig.AssetsDestination),

diff --git a/core/backend/llm.go b/core/backend/llm.go
@@ -64,13 +64,18 @@ func NewLLMBackendService(ml *model.ModelLoader, bcl *config.BackendConfigLoader
 func (llmbs *LLMBackendService) Inference(ctx context.Context, req *LLMRequest, bc *config.BackendConfig, enableTokenChannel bool) (
 	resultChannel <-chan utils.ErrorOr[*LLMResponse], tokenChannel <-chan utils.ErrorOr[*LLMResponse], err error) {
 
+	threads := bc.Threads
+	if threads == 0 && llmbs.appConfig.Threads != 0 {
+		threads = llmbs.appConfig.Threads
+	}
+
 	grpcOpts := gRPCModelOpts(bc)
 
 	var inferenceModel grpc.Backend
 
 	opts := modelOpts(bc, llmbs.appConfig, []model.Option{
 		model.WithLoadGRPCLoadModelOpts(grpcOpts),
-		model.WithThreads(uint32(bc.Threads)), // some models uses this to allocate threads during startup
+		model.WithThreads(uint32(threads)), // some models uses this to allocate threads during startup
 		model.WithAssetDir(llmbs.appConfig.AssetsDestination),
 		model.WithModel(bc.Model),
 		model.WithContext(llmbs.appConfig.Context),