diff --git a/core/app/models/concerns/spree/user_address_book.rb b/core/app/models/concerns/spree/user_address_book.rb
index 72eb4702210..b1ea00d3e68 100644
--- a/core/app/models/concerns/spree/user_address_book.rb
+++ b/core/app/models/concerns/spree/user_address_book.rb
@@ -130,6 +130,7 @@ def mark_default_address(address)
     def remove_from_address_book(address_id)
       user_address = user_addresses.find_by(address_id: address_id)
       if user_address
+        remove_user_address_reference(address_id)
         user_address.update(archived: true, default: false)
       else
         false
@@ -145,5 +146,11 @@ def prepare_user_address(new_address)
       user_address.archived = false
       user_address
     end
+
+    def remove_user_address_reference(address_id)
+      self.bill_address_id = bill_address_id == address_id.to_i ? nil : bill_address_id
+      self.ship_address_id = ship_address_id == address_id.to_i ? nil : ship_address_id
+      save if changed?
+    end
   end
 end
diff --git a/core/spec/models/spree/concerns/user_address_book_spec.rb b/core/spec/models/spree/concerns/user_address_book_spec.rb
index 49dd20ef921..07f989d8648 100644
--- a/core/spec/models/spree/concerns/user_address_book_spec.rb
+++ b/core/spec/models/spree/concerns/user_address_book_spec.rb
@@ -221,6 +221,7 @@ module Spree
       let(:address1) { create(:address) }
       let(:address2) { create(:address, name: "Different") }
       let(:remove_id) { address1.id }
+
       subject { user.remove_from_address_book(remove_id) }
 
       before do
@@ -242,6 +243,49 @@ module Spree
       it "returns false if the addresses is not there" do
         expect(user.remove_from_address_book(0)).to be false
       end
+
+      context 'when user has previous order addresses' do
+        let(:order) { create(:order, ship_address: address1, bill_address: address2) }
+
+        before { user.persist_order_address(order) }
+
+        context 'when address does not match any user address references' do
+          let(:another_address) { create(:address) }
+
+          let(:remove_id) { another_address.id }
+
+          it 'leaves current user ship address' do
+            expect { subject }.not_to change(user, :ship_address_id).from(address1.id)
+          end
+
+          it 'leaves current user bill address' do
+            expect { subject }.not_to change(user, :bill_address_id).from(address2.id)
+          end
+        end
+
+        context 'when address matches user ship address' do
+          it 'removes the ship address reference from user' do
+            expect { subject }.to change(user, :ship_address_id).from(address1.id).to(nil)
+          end
+        end
+
+        context 'when address matches user bill address' do
+          let(:remove_id) { address2.id }
+
+          it 'removes the bill address reference from user' do
+            expect { subject }.to change(user, :bill_address_id).from(address2.id).to(nil)
+          end
+        end
+
+        context 'when address matches user bill and ship address' do
+          let(:order) { create(:order, ship_address: address1, bill_address: address1) }
+
+          it 'removes the address references from user' do
+            expect { subject }.to change(user, :ship_address_id).from(address1.id).to(nil)
+              .and change(user, :bill_address_id).from(address1.id).to(nil)
+          end
+        end
+      end
     end
 
     context "#persist_order_address" do