Check the remote schema_migrations table if the migration is not listed

If a column is added to MiqRegion, that causes replication to stop.  With
replication stopped we sit in the loop forever waiting for the MiqRegion
record for the remote region to be updated with the migrations_ran.  In
this case, we need to check the schema_migrations table on the remote
region directly.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1668800

lib/extensions/ar_migration.rb

@@ -64,7 +64,24 @@ def wait_for_remote_region_migration(wait_time = 1)
     private
 
     def wait_for_migration?
-      migrations_column_present? ? !region.migrations_ran&.include?(version) : false
+      migrations_column_present? ? !remote_region_migrated? : false
+    end
+
+    # NOTE: Check the local table first, but a migration can cause MiqRegion replication to stop (i.e. adding a new column to miq_regions).
+    # In that case, the global MiqRegion record for the remote region will not have the migration timestamp yet, so we need to check the
+    # schema_migrations table in the remote region directly.
+    def remote_region_migrated?
+      version_exists_in_replicated_region? || version_exists_on_remote_region?
+    end
+
+    def version_exists_in_replicated_region?
+      region.migrations_ran&.include?(version)
+    end
+
+    def version_exists_on_remote_region?
+      subscription.send(:with_remote_connection) do |connection|
+        connection.select_values("SELECT 1 FROM schema_migrations WHERE version = '#{version}' LIMIT 1")
+      end.any?
     end
 
     def migrations_column_present?

spec/lib/extensions/ar_migration_spec.rb

@@ -80,9 +80,10 @@
         end
       end
 
-      it "sleeps until the migration is added" do
+      it "sleeps until the migration is replicated up" do
         allow(subject).to receive(:restart_subscription)
         allow(subject.region).to receive(:reload)
+        allow(subscription).to receive(:with_remote_connection).and_return([])
 
         subject.region.update_attributes!(:migrations_ran => nil)
 
@@ -105,6 +106,26 @@
         t = t.join(5)
         expect(t.status).to be false
       end
+
+      it "when migrations_ran is nil, it checks the schema_migrations table on the remote region" do
+        expect(subject).not_to receive(:restart_subscription)
+        expect(subject.region).not_to receive(:reload)
+        expect(subscription).to receive(:with_remote_connection).and_return([version])
+
+        subject.region.update_attributes!(:migrations_ran => nil)
+
+        subject.wait_for_remote_region_migration(0)
+      end
+
+      it "when migrations_ran does not include that expected migration, it checks the schema_migrations on the remote region" do
+        expect(subject).not_to receive(:restart_subscription)
+        expect(subject.region).not_to receive(:reload)
+        expect(subscription).to receive(:with_remote_connection).and_return([version])
+
+        subject.region.update_attributes!(:migrations_ran => ["1234", "5678"])
+
+        subject.wait_for_remote_region_migration(0)
+      end
     end
   end
 end