diff --git a/pkg/model/loader.go b/pkg/model/loader.go
index 27ea78b34b4d..ffe67496df91 100644
--- a/pkg/model/loader.go
+++ b/pkg/model/loader.go
@@ -122,6 +122,7 @@ var knownModelsNameSuffixToSkip []string = []string{
 	".bin",
 	".partial",
 	".tar.gz",
+	".gguf",
 }
 
 const retryTimeout = time.Duration(2 * time.Minute)
diff --git a/pkg/model/loader_test.go b/pkg/model/loader_test.go
index 2c296c2ea152..16736614138b 100644
--- a/pkg/model/loader_test.go
+++ b/pkg/model/loader_test.go
@@ -68,6 +68,18 @@ var _ = Describe("ModelLoader", func() {
 			Expect(files).To(ContainElement("test.model"))
 			Expect(files).ToNot(ContainElement("README.md"))
 		})
+
+		It("should filter out .gguf files from the model listing", func() {
+			os.Create(filepath.Join(modelPath, "valid-model"))
+			os.Create(filepath.Join(modelPath, "model.gguf"))
+			os.Create(filepath.Join(modelPath, "another-model.gguf"))
+
+			files, err := modelLoader.ListFilesInModelPath()
+			Expect(err).To(BeNil())
+			Expect(files).To(ContainElement("valid-model"))
+			Expect(files).ToNot(ContainElement("model.gguf"))
+			Expect(files).ToNot(ContainElement("another-model.gguf"))
+		})
 	})
 
 	Context("LoadModel", func() {