diff --git a/lib/factory_bot_rails/railtie.rb b/lib/factory_bot_rails/railtie.rb
index 5512f23..e47fbc3 100644
--- a/lib/factory_bot_rails/railtie.rb
+++ b/lib/factory_bot_rails/railtie.rb
@@ -8,10 +8,12 @@
 
 module FactoryBotRails
   class Railtie < Rails::Railtie
-    config.factory_bot = ActiveSupport::OrderedOptions.new
-    config.factory_bot.definition_file_paths = FactoryBot.definition_file_paths
-    config.factory_bot.reject_primary_key_attributes = true
-    config.factory_bot.validator = FactoryBotRails::FactoryValidator.new
+    initializer "factory_bot.configuration" do
+      config.factory_bot = ActiveSupport::OrderedOptions.new
+      config.factory_bot.definition_file_paths = FactoryBot.definition_file_paths
+      config.factory_bot.reject_primary_key_attributes = true
+      config.factory_bot.validator = FactoryBotRails::FactoryValidator.new
+    end
 
     initializer "factory_bot.set_fixture_replacement" do
       Generator.new(config).run
@@ -21,13 +23,15 @@ class Railtie < Rails::Railtie
       FactoryBot.definition_file_paths = definition_file_paths
     end
 
-    ActiveSupport.on_load :active_record do
-      config = Rails.configuration.factory_bot
+    initializer "factory_bot.reject_primary_key_attributes" do
+      ActiveSupport.on_load :active_record do
+        config = Rails.configuration.factory_bot
 
-      if config.reject_primary_key_attributes
-        require "factory_bot_rails/factory_validator/active_record_validator"
+        if config.reject_primary_key_attributes
+          require "factory_bot_rails/factory_validator/active_record_validator"
 
-        config.validator.add_validator FactoryValidator::ActiveRecordValidator.new
+          config.validator.add_validator FactoryValidator::ActiveRecordValidator.new
+        end
       end
     end
 
diff --git a/spec/factory_bot_rails/railtie_spec.rb b/spec/factory_bot_rails/railtie_spec.rb
index 807be70..c927b92 100644
--- a/spec/factory_bot_rails/railtie_spec.rb
+++ b/spec/factory_bot_rails/railtie_spec.rb
@@ -47,4 +47,52 @@ def wait_for_rails_to_reload
       sleep 0.01
     end
   end
+
+  describe "initializer 'factory_bot.reject_primary_key_attributes'" do
+    before do
+      Rails.application.config.factory_bot.validator = FactoryBotRails::FactoryValidator.new
+    end
+
+    context "when config.reject_primary_key_attributes is true" do
+      before do
+        Rails.application.config.factory_bot.reject_primary_key_attributes = true
+        allow(Rails.application.config.factory_bot.validator).to receive(:add_validator)
+      end
+
+      it "adds the ActiveRecordValidator to the validator list" do
+        run_reject_primary_key_attributes_initializer
+
+        expect(Rails.application.config.factory_bot.validator).to have_received(:add_validator)
+      end
+    end
+
+    context "when config.reject_primary_key_attributes is false" do
+      before do
+        Rails.application.config.factory_bot.reject_primary_key_attributes = false
+        allow(Rails.application.config.factory_bot.validator).to receive(:add_validator)
+      end
+
+      it "does not add the ActiveRecordValidator to the validator list" do
+        run_reject_primary_key_attributes_initializer
+
+        expect(Rails.application.config.factory_bot.validator).not_to have_received(:add_validator)
+      end
+    end
+
+    def run_reject_primary_key_attributes_initializer
+      initializer = FactoryBotRails::Railtie.initializers.find do |i|
+        i.name == "factory_bot.reject_primary_key_attributes"
+      end
+      initializer.run(Rails.application)
+    end
+  end
+
+  describe "initializer order" do
+    it "runs 'factory_bot.configuration' before any other initializer" do
+      initializers = FactoryBotRails::Railtie.initializers.map(&:name)
+      first_initializer = initializers.first
+
+      expect(first_initializer).to eq("factory_bot.configuration")
+    end
+  end
 end