Pull ollama models during evaluation

Closes #283

cmd/eval-dev-quality/cmd/evaluate.go

@@ -321,6 +321,11 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 		} else {
 			for _, model := range command.Models {
 				p := strings.SplitN(model, provider.ProviderModelSeparator, 2)[0]
+
+				if _, ok := providersSelected[p]; ok {
+					continue
+				}
+
 				if provider, ok := provider.Providers[p]; !ok {
 					command.logger.Panicf("Provider %q does not exist", p)
 				} else {
@@ -361,6 +366,18 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate.
 				}()
 			}
 
+			// Check if a provider has the ability to pull models and do so if necessary.
+			if puller, ok := p.(provider.Puller); ok {
+				command.logger.Printf("Pulling available models for provider %q", p.ID())
+				for _, modelID := range command.Models {
+					if strings.HasPrefix(modelID, p.ID()) {
+						if err := puller.Pull(command.logger, modelID); err != nil {
+							command.logger.Panicf("ERROR: could not pull model %q: %s", modelID, err)
+						}
+					}
+				}
+			}
+
 			ms, err := p.Models()
 			if err != nil {
 				command.logger.Panicf("ERROR: could not query models for provider %q: %s", p.ID(), err)

cmd/eval-dev-quality/cmd/evaluate_test.go

@@ -491,23 +491,9 @@ func TestEvaluateExecute(t *testing.T) {
 			}
 
 			{
-				var shutdown func() (err error)
-				defer func() { // Defer the shutdown in case there is a panic.
-					if shutdown != nil {
-						require.NoError(t, shutdown())
-					}
-				}()
 				validate(t, &testCase{
 					Name: "Pulled Model",
 
-					Before: func(t *testing.T, logger *log.Logger, resultPath string) {
-						var err error
-						shutdown, err = tools.OllamaStart(logger, tools.OllamaPath, tools.OllamaURL)
-						require.NoError(t, err)
-
-						require.NoError(t, tools.OllamaPull(logger, tools.OllamaPath, tools.OllamaURL, providertesting.OllamaTestModel))
-					},
-
 					Arguments: []string{
 						"--language", "golang",
 						"--model", "ollama/" + providertesting.OllamaTestModel,