diff --git a/composer.json b/composer.json
index c25a1e4fc..68c9c203a 100644
--- a/composer.json
+++ b/composer.json
@@ -17,7 +17,6 @@
     ],
     "require": {
         "php": "^7.2||^8.0",
-        "doctrine/annotations": "^1.13 || ^2.0",
         "doctrine/instantiator": "^1.0.3 || ^2.0",
         "doctrine/lexer": "^2.0 || ^3.0",
         "jms/metadata": "^2.6",
@@ -31,6 +30,7 @@
     },
     "require-dev": {
         "ext-pdo_sqlite": "*",
+        "doctrine/annotations": "^1.13 || ^2.0",
         "doctrine/coding-standard": "^12.0",
         "doctrine/orm": "~2.1",
         "doctrine/persistence": "^1.3.3|^2.0|^3.0",
@@ -51,6 +51,9 @@
         "symfony/yaml": "^3.3|^4.0|^5.0|^6.0|^7.0",
         "twig/twig": "~1.34|~2.4|^3.0"
     },
+    "conflict": {
+        "doctrine/annotations": "<1.13 || >=3.0"
+    },
     "autoload": {
         "psr-4": {
             "JMS\\Serializer\\": "src/"
diff --git a/src/Builder/CallbackDriverFactory.php b/src/Builder/CallbackDriverFactory.php
index 1bf2fc913..b35d4f5e1 100644
--- a/src/Builder/CallbackDriverFactory.php
+++ b/src/Builder/CallbackDriverFactory.php
@@ -12,15 +12,19 @@ final class CallbackDriverFactory implements DriverFactoryInterface
 {
     /**
      * @var callable
+     * @phpstan-var callable(array $metadataDirs, Reader|null $reader): DriverInterface
      */
     private $callback;
 
+    /**
+     * @phpstan-param callable(array $metadataDirs, Reader|null $reader): DriverInterface $callable
+     */
     public function __construct(callable $callable)
     {
         $this->callback = $callable;
     }
 
-    public function createDriver(array $metadataDirs, Reader $reader): DriverInterface
+    public function createDriver(array $metadataDirs, ?Reader $reader = null): DriverInterface
     {
         $driver = \call_user_func($this->callback, $metadataDirs, $reader);
         if (!$driver instanceof DriverInterface) {
diff --git a/src/Builder/DefaultDriverFactory.php b/src/Builder/DefaultDriverFactory.php
index 041ca4a71..b28e95c1a 100644
--- a/src/Builder/DefaultDriverFactory.php
+++ b/src/Builder/DefaultDriverFactory.php
@@ -5,6 +5,7 @@
 namespace JMS\Serializer\Builder;
 
 use Doctrine\Common\Annotations\Reader;
+use JMS\Serializer\Exception\RuntimeException;
 use JMS\Serializer\Expression\CompilableExpressionEvaluatorInterface;
 use JMS\Serializer\Metadata\Driver\AnnotationOrAttributeDriver;
 use JMS\Serializer\Metadata\Driver\DefaultValuePropertyDriver;
@@ -55,8 +56,12 @@ public function enableEnumSupport(bool $enableEnumSupport = true): void
         $this->enableEnumSupport = $enableEnumSupport;
     }
 
-    public function createDriver(array $metadataDirs, Reader $annotationReader): DriverInterface
+    public function createDriver(array $metadataDirs, ?Reader $annotationReader = null): DriverInterface
     {
+        if (empty($metadataDirs) && !interface_exists(Reader::class) && PHP_VERSION_ID < 80000) {
+            throw new RuntimeException(sprintf('To use "%s", either a list of metadata directories must be provided, the "doctrine/annotations" package installed, or use PHP 8.0 or later.', self::class));
+        }
+
         /*
          * Build the sorted list of metadata drivers based on the environment. The final order should be:
          *
@@ -65,7 +70,11 @@ public function createDriver(array $metadataDirs, Reader $annotationReader): Dri
          * - Annotations/Attributes Driver
          * - Null (Fallback) Driver
          */
-        $metadataDrivers = [new AnnotationOrAttributeDriver($this->propertyNamingStrategy, $this->typeParser, $this->expressionEvaluator, $annotationReader)];
+        $metadataDrivers = [];
+
+        if (PHP_VERSION_ID >= 80000 || $annotationReader instanceof Reader) {
+            $metadataDrivers[] = new AnnotationOrAttributeDriver($this->propertyNamingStrategy, $this->typeParser, $this->expressionEvaluator, $annotationReader);
+        }
 
         if (!empty($metadataDirs)) {
             $fileLocator = new FileLocator($metadataDirs);
diff --git a/src/Builder/DocBlockDriverFactory.php b/src/Builder/DocBlockDriverFactory.php
index 3188a01db..67f0ec527 100644
--- a/src/Builder/DocBlockDriverFactory.php
+++ b/src/Builder/DocBlockDriverFactory.php
@@ -26,7 +26,7 @@ public function __construct(DriverFactoryInterface $driverFactoryToDecorate, ?Pa
         $this->typeParser = $typeParser;
     }
 
-    public function createDriver(array $metadataDirs, Reader $annotationReader): DriverInterface
+    public function createDriver(array $metadataDirs, ?Reader $annotationReader = null): DriverInterface
     {
         $driver = $this->driverFactoryToDecorate->createDriver($metadataDirs, $annotationReader);
 
diff --git a/src/Builder/DriverFactoryInterface.php b/src/Builder/DriverFactoryInterface.php
index d7655e115..7bfade916 100644
--- a/src/Builder/DriverFactoryInterface.php
+++ b/src/Builder/DriverFactoryInterface.php
@@ -9,5 +9,5 @@
 
 interface DriverFactoryInterface
 {
-    public function createDriver(array $metadataDirs, Reader $annotationReader): DriverInterface;
+    public function createDriver(array $metadataDirs, ?Reader $annotationReader = null): DriverInterface;
 }
diff --git a/src/SerializerBuilder.php b/src/SerializerBuilder.php
index 47fa2d299..fece44a6f 100644
--- a/src/SerializerBuilder.php
+++ b/src/SerializerBuilder.php
@@ -552,9 +552,8 @@ public function setDocBlockTypeResolver(bool $docBlockTypeResolver): self
     public function build(): Serializer
     {
         $annotationReader = $this->annotationReader;
-        if (null === $annotationReader) {
-            $annotationReader = new AnnotationReader();
-            $annotationReader = $this->decorateAnnotationReader($annotationReader);
+        if (null === $annotationReader && class_exists(AnnotationReader::class)) {
+            $annotationReader = $this->decorateAnnotationReader(new AnnotationReader());
         }
 
         if (null === $this->driverFactory) {