Shut down the Ollama service if there are no Ollama models to evaluat…

…e, to avoid unnecessary processes being opened

Closes #255

cmd/eval-dev-quality/cmd/evaluate.go

@@ -302,7 +302,9 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 	{
 		models := map[string]model.Model{}
 		modelsSelected := map[string]model.Model{}
+		providersSelected := map[provider.Provider]bool{}
 		evaluationContext.ProviderForModel = map[model.Model]provider.Provider{}
+		providersShutdown := map[provider.Provider]func() error{}
 		for _, p := range provider.Providers {
 			command.logger.Printf("Checking provider %q for models", p.ID())
 
@@ -325,11 +327,7 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 				if err != nil {
 					command.logger.Panicf("ERROR: could not start services for provider %q: %s", p, err)
 				}
-				defer func() {
-					if err := shutdown(); err != nil {
-						command.logger.Panicf("ERROR: could not shutdown services of provider %q: %s", p, err)
-					}
-				}()
+				providersShutdown[p] = shutdown
 			}
 
 			ms, err := p.Models()
@@ -355,7 +353,29 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 		}
 		sort.Strings(command.Models)
 		for _, modelID := range command.Models {
-			modelsSelected[modelID] = models[modelID]
+			model := models[modelID]
+			modelsSelected[modelID] = model
+			if provider, ok := evaluationContext.ProviderForModel[model]; ok {
+				providersSelected[provider] = true
+			}
+		}
+
+		// Shut down services that are no longer needed.
+		for provider, shutdown := range providersShutdown {
+			if _, ok := providersSelected[provider]; !ok {
+				if err := shutdown(); err != nil {
+					command.logger.Panicf("ERROR: could not shutdown services of provider %q: %s", provider, err)
+				} else {
+					command.logger.Printf("Shutting down %s services since they are no longer needed", provider.ID())
+				}
+			} else {
+				currentProvider, currentShutdown := provider, shutdown
+				defer func() {
+					if err := currentShutdown(); err != nil {
+						command.logger.Panicf("ERROR: could not shutdown services of provider %q: %s", currentProvider, err)
+					}
+				}()
+			}
 		}
 
 		// Make the resolved selected models available in the command.

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -525,6 +525,31 @@ func TestEvaluateExecute(t *testing.T) {
 						filepath.Join("result-directory", "README.md"): nil,
 						filepath.Join("result-directory", string(evaluatetask.IdentifierWriteTests), "ollama_"+log.CleanModelNameForFileSystem(providertesting.OllamaTestModel), "golang", "golang", "plain.log"): nil,
 					},
+					ExpectedOutputValidate: func(t *testing.T, output, resultPath string) {
+						assert.NotContains(t, output, "Shutting down ollama services since they are no longer needed")
+					},
+				})
+			}
+			{
+				validate(t, &testCase{
+					Name: "Shutdown Ollama if no model needs it",
+
+					Arguments: []string{
+						"--language", "golang",
+						"--model", "symflower/symbolic-execution",
+						"--repository", filepath.Join("golang", "plain"),
+					},
+
+					ExpectedResultFiles: map[string]func(t *testing.T, filePath string, data string){
+						filepath.Join("result-directory", "categories.svg"): nil,
+						filepath.Join("result-directory", "evaluation.csv"): nil,
+						filepath.Join("result-directory", "evaluation.log"): nil,
+						filepath.Join("result-directory", "README.md"):      nil,
+						filepath.Join("result-directory", string(evaluatetask.IdentifierWriteTests), "symflower_symbolic-execution", "golang", "golang", "plain.log"): nil,
+					},
+					ExpectedOutputValidate: func(t *testing.T, output, resultPath string) {
+						assert.Contains(t, output, "Shutting down ollama services since they are no longer needed")
+					},
 				})
 			}
 		})
@@ -571,6 +596,9 @@ func TestEvaluateExecute(t *testing.T) {
 						filepath.Join("result-directory", "README.md"): nil,
 						filepath.Join("result-directory", string(evaluatetask.IdentifierWriteTests), "custom-ollama_"+log.CleanModelNameForFileSystem(providertesting.OllamaTestModel), "golang", "golang", "plain.log"): nil,
 					},
+					ExpectedOutputValidate: func(t *testing.T, output, resultPath string) {
+						assert.NotContains(t, output, "Shutting down ollama services since they are no longer needed")
+					},
 				})
 			}
 		})