Enhance persister serialization

app/models/manager_refresh/inventory/persister.rb

@@ -36,27 +36,13 @@ def self.supported_collections
   # @param json_data [String] input JSON data
   # @return [ManagerRefresh::Inventory::Persister] Persister object loaded from a passed JSON
   def self.from_json(json_data)
-    from_raw_data(JSON.parse(json_data))
+    from_hash(JSON.parse(json_data))
   end
 
   # Returns serialized Persisted object to JSON
   # @return [String] serialized Persisted object to JSON
   def to_json
-    JSON.dump(to_raw_data)
-  end
-
-  # Returns Persister object loaded from a passed YAML
-  #
-  # @param json_data [String] input JSON data
-  # @return [ManagerRefresh::Inventory::Persister] Persister object loaded from a passed YAML
-  def self.from_yaml(yaml_data)
-    from_raw_data(YAML.safe_load(yaml_data))
-  end
-
-  # Returns serialized Persisted object to YAML
-  # @return [String] serialized Persisted object to YAML
-  def to_yaml
-    YAML.dump(to_raw_data)
+    JSON.dump(to_hash)
   end
 
   # Creates method on class that lazy initializes an InventoryCollection
@@ -186,16 +172,11 @@ def add_remaining_inventory_collections(defaults, options = {})
   end
 
   # @return [Hash] entire Persister object serialized to hash
-  def to_raw_data
-    collections_data = collections.map do |key, collection|
-      next if collection.data.blank? && collection.manager_uuids.blank? && collection.all_manager_uuids.nil?
-
-      {
-        :name              => key,
-        :manager_uuids     => collection.manager_uuids,
-        :all_manager_uuids => collection.all_manager_uuids,
-        :data              => collection.to_raw_data
-      }
+  def to_hash
+    collections_data = collections.map do |_, collection|
+      next if collection.data.blank? && collection.targeted_scope.blank? && collection.all_manager_uuids.nil?
+
+      collection.to_hash
     end.compact
 
     {
@@ -212,27 +193,24 @@ class << self
     #
     # @param persister_data [Hash] serialized Persister object in hash
     # @return [ManagerRefresh::Inventory::Persister] Persister object built from serialized data
-    def from_raw_data(persister_data)
-      persister_data.transform_keys!(&:to_s)
-
+    def from_hash(persister_data)
       # Extract the specific Persister class
       persister_class = persister_data['class'].constantize
       unless persister_class < ManagerRefresh::Inventory::Persister
         raise "Persister class must inherit from a ManagerRefresh::Inventory::Persister"
       end
 
-      # TODO(lsmola) do we need a target in this case?
-      # Load the Persister object and fill the InventoryCollections with the data
-      persister = persister_class.new(ManageIQ::Providers::BaseManager.find(persister_data['ems_id']))
-      persister_data['collections'].each do |collection|
-        collection.transform_keys!(&:to_s)
+      ems = ManageIQ::Providers::BaseManager.find(persister_data['ems_id'])
+      persister = persister_class.new(
+        ems,
+        ManagerRefresh::TargetCollection.new(:manager => ems) # TODO(lsmola) we need to pass serialized targeted scope here
+      )
 
+      persister_data['collections'].each do |collection|
         inventory_collection = persister.collections[collection['name'].try(:to_sym)]
         raise "Unrecognized InventoryCollection name: #{inventory_collection}" if inventory_collection.blank?
 
-        inventory_collection.manager_uuids.merge(collection['manager_uuids'] || [])
-        inventory_collection.all_manager_uuids = collection['all_manager_uuids']
-        inventory_collection.from_raw_data(collection['data'], persister.collections)
+        inventory_collection.from_hash(collection, persister.collections)
       end
       persister
     end

app/models/manager_refresh/inventory_collection.rb

@@ -86,13 +86,12 @@ class InventoryCollection
              :each,
              :find_or_build,
              :find_or_build_by,
-             :from_raw_data,
-             :from_raw_value,
+             :from_hash,
              :index_proxy,
              :push,
              :size,
              :to_a,
-             :to_raw_data,
+             :to_hash,
              :to => :data_storage
 
     delegate :add_reference,

app/models/manager_refresh/inventory_collection/data_storage.rb

@@ -112,62 +112,14 @@ def to_a
         data
       end
 
-      # Reconstructs InventoryCollection from it's serialized form
-      #
-      # @param inventory_objects_data [Array[Hash]] Serialized array of InventoryObject objects as hashes
-      # @param available_inventory_collections [Array<ManagerRefresh::InventoryCollection>] List of available
-      #        InventoryCollection objects
-      def from_raw_data(inventory_objects_data, available_inventory_collections)
-        inventory_objects_data.each do |inventory_object_data|
-          hash = inventory_object_data.each_with_object({}) do |(key, value), result|
-            result[key.to_sym] = if value.kind_of?(Array)
-                                   value.map { |x| from_raw_value(x, available_inventory_collections) }
-                                 else
-                                   from_raw_value(value, available_inventory_collections)
-                                 end
-          end
-          build(hash)
-        end
-      end
-
-      # Transform serialized references into lazy InventoryObject objects
-      #
-      # @param value [Object, Hash] Serialized InventoryObject into Hash
-      # @param available_inventory_collections [Array<ManagerRefresh::InventoryCollection>] List of available
-      #        InventoryCollection objects
-      # @return [Object, ManagerRefresh::InventoryObjectLazy] Returns ManagerRefresh::InventoryObjectLazy object
-      #         if the serialized form was a reference, or return original value
-      def from_raw_value(value, available_inventory_collections)
-        if value.kind_of?(Hash) && (value['type'] || value[:type]) == "ManagerRefresh::InventoryObjectLazy"
-          value.transform_keys!(&:to_s)
-        end
-
-        if value.kind_of?(Hash) && value['type'] == "ManagerRefresh::InventoryObjectLazy"
-          inventory_collection = available_inventory_collections[value['inventory_collection_name'].try(:to_sym)]
-          raise "Couldn't build lazy_link #{value} the inventory_collection_name was not found" if inventory_collection.blank?
-          inventory_collection.lazy_find(value['ems_ref'], :key => value['key'], :default => value['default'])
-        else
-          value
-        end
+      def to_hash
+        ManagerRefresh::InventoryCollection::Serialization.new(inventory_collection).to_hash
       end
 
-      # Serializes InventoryCollection's data storage into Array of Hashes, which we can turn into JSON or YAML
-      #
-      # @return [Array<Hash>] Serialized InventoryCollection's data storage
-      def to_raw_data
-        data.map do |inventory_object|
-          inventory_object.data.transform_values do |value|
-            if inventory_object_lazy?(value)
-              value.to_raw_lazy_relation
-            elsif value.kind_of?(Array) && (inventory_object_lazy?(value.compact.first) || inventory_object?(value.compact.first))
-              value.compact.map(&:to_raw_lazy_relation)
-            elsif inventory_object?(value)
-              value.to_raw_lazy_relation
-            else
-              value
-            end
-          end
-        end
+      def from_hash(inventory_objects_data, available_inventory_collections)
+        ManagerRefresh::InventoryCollection::Serialization
+          .new(inventory_collection)
+          .from_hash(inventory_objects_data, available_inventory_collections)
       end
 
       private

app/models/manager_refresh/inventory_collection/index/proxy.rb

@@ -177,6 +177,10 @@ def assert_relation_keys(data, ref)
           end
         end
 
+        def assert_index_exists(ref)
+          raise "Index :#{ref} doesn't exist on #{inventory_collection}" if named_ref(ref).nil?
+        end
+
         def assert_index(manager_uuid, ref)
           # TODO(lsmola) do we need some production logging too? Maybe the refresh log level could drive this
           # Let' do this really slick development and test env, but disable for production, since the checks are pretty
@@ -186,13 +190,19 @@ def assert_index(manager_uuid, ref)
           if manager_uuid.kind_of?(ManagerRefresh::InventoryCollection::Reference)
             # ManagerRefresh::InventoryCollection::Reference has been already asserted, skip
           elsif manager_uuid.kind_of?(Hash)
+            # Test te index exists
+            assert_index_exists(ref)
+
             # Test we are sending all keys required for the index
             unless required_index_keys_present?(manager_uuid.keys, ref)
               raise "Finder has missing keys for index :#{ref}, missing indexes are: #{missing_keys(manager_uuid.keys, ref)}"
             end
             # Test that keys, that are relations, are nil or InventoryObject or InventoryObjectlazy class
             assert_relation_keys(manager_uuid, ref)
           else
+            # Test te index exists
+            assert_index_exists(ref)
+
             # Check that other value (possibly String or Integer)) has no composite index
             if named_ref(ref).size > 1
               right_format = "collection.find(#{named_ref(ref).map { |x| ":#{x} => 'X'" }.join(", ")}"

app/models/manager_refresh/inventory_collection/reference.rb

@@ -26,20 +26,6 @@ def primary?
         ref == :manager_ref
       end
 
-      # Returns serialized self into Hash
-      #
-      # @return [Hash] Serialized self into Hash
-      def to_hash
-      end
-
-      class << self
-        # Returns reference object built from serialized Hash
-        #
-        # @return [ManagerRefresh::InventoryCollection::Reference] Reference object built from serialized Hash
-        def from_hash
-        end
-      end
-
       class << self
         # Builds string uuid from passed Hash and keys
         #

app/models/manager_refresh/inventory_collection/references_storage.rb

@@ -1,7 +1,7 @@
 module ManagerRefresh
   class InventoryCollection
     class ReferencesStorage
-      # @return [Set] A set of InventoryObjects manager_uuids, which tells us which InventoryObjects were
+      # @return [Hash] A set of InventoryObjects manager_uuids, which tells us which InventoryObjects were
       #         referenced by other InventoryObjects using a lazy_find.
       attr_reader :references
 

app/models/manager_refresh/inventory_collection/serialization.rb

@@ -0,0 +1,130 @@
+module ManagerRefresh
+  class InventoryCollection
+    class Serialization
+      delegate :all_manager_uuids,
+               :build,
+               :targeted_scope,
+               :data,
+               :inventory_object_lazy?,
+               :inventory_object?,
+               :name,
+               :to => :inventory_collection
+
+      attr_reader :inventory_collection
+
+      # @param inventory_collection [ManagerRefresh::InventoryCollection] InventoryCollection object we want the storage
+      #        for
+      def initialize(inventory_collection)
+        @inventory_collection = inventory_collection
+      end
+
+      # Loads InventoryCollection data from it's serialized form into existing InventoryCollection object
+      #
+      # @param inventory_objects_data [Hash] Serialized InventoryCollection as Hash
+      # @param available_inventory_collections [Array<ManagerRefresh::InventoryCollection>] List of available
+      #        InventoryCollection objects
+      def from_hash(inventory_objects_data, available_inventory_collections)
+        inventory_objects_data['data'].each do |inventory_object_data|
+          build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
+        end
+
+        # TODO(lsmola) we need to remodel the targeted scope, to be able to serialize targeted InventoryCollections
+        # self.targeted_scope.merge!(inventory_objects_data['manager_uuids'] || [])
+        # self.all_manager_uuids = inventory_objects_data['all_manager_uuids']
+      end
+
+      # Serializes InventoryCollection's data storage into Array of Hashes
+      #
+      # @return [Hash] Serialized InventoryCollection object into Hash
+      def to_hash
+        {
+          :name              => name,
+          :manager_uuids     => targeted_scope.values.map { |x| data_to_hash(x) },
+          :all_manager_uuids => all_manager_uuids,
+          :data              => data.map { |x| data_to_hash(x.data) }
+        }
+      end
+
+      private
+
+      # Converts ManagerRefresh::InventoryObject or ManagerRefresh::InventoryObjectLazy into Hash
+      #
+      # @param value [ManagerRefresh::InventoryObject, ManagerRefresh::InventoryObjectLazy] InventoryObject or a lazy link
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [Hash] Serialized ManagerRefresh::InventoryObjectLazy
+      def lazy_relation_to_hash(value, depth = 0)
+        {
+          :type                      => "ManagerRefresh::InventoryObjectLazy",
+          :inventory_collection_name => value.inventory_collection.name,
+          :reference                 => data_to_hash(value.reference.full_reference, depth + 1),
+          :ref                       => value.reference.ref,
+          :key                       => value.try(:key),
+          :default                   => value.try(:default),
+        }
+      end
+
+      # Converts Hash to ManagerRefresh::InventoryObjectLazy
+      #
+      # @param value [Hash] Serialized InventoryObject or a lazy link
+      # @param available_inventory_collections [Array<ManagerRefresh::InventoryCollection>] List of available
+      #        InventoryCollection objects
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [ManagerRefresh::InventoryObjectLazy] Lazy link created from hash
+      def hash_to_lazy_relation(value, available_inventory_collections, depth = 0)
+        inventory_collection = available_inventory_collections[value['inventory_collection_name'].try(:to_sym)]
+        raise "Couldn't build lazy_link #{value} the inventory_collection_name was not found" if inventory_collection.blank?
+
+        inventory_collection.lazy_find(
+          hash_to_data(value['reference'], available_inventory_collections, depth + 1).symbolize_keys!,
+          :ref     => value['ref'].try(:to_sym),
+          :key     => value['key'].try(:to_sym),
+          :default => value['default']
+        )
+      end
+
+      # Converts Hash to attributes usable for building InventoryObject
+      #
+      # @param hash [Hash] Serialized InventoryObject data
+      # @param available_inventory_collections [Array<ManagerRefresh::InventoryCollection>] List of available
+      #        InventoryCollection objects
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [Hash] Hash with data usable for building InventoryObject
+      def hash_to_data(hash, available_inventory_collections, depth = 0)
+        raise "Nested lazy_relation of #{inventory_collection} is too deep, left processing: #{hash}" if depth > 20
+
+        hash.transform_values do |value|
+          if value.kind_of?(Hash) && value['type'] == "ManagerRefresh::InventoryObjectLazy"
+            hash_to_lazy_relation(value, available_inventory_collections, depth)
+          elsif value.kind_of?(Array) && value.first.kind_of?(Hash) && value.first['type'] == "ManagerRefresh::InventoryObjectLazy"
+            # TODO(lsmola) do we need to compact it sooner? What if first element is nil? On the other hand, we want to
+            # deprecate Vmthis HABTM assignment because it's not effective
+            value.compact.map { |x| hash_to_lazy_relation(x, available_inventory_collections, depth) }
+          else
+            value
+          end
+        end
+      end
+
+      # Transforms data of the InventoryObject or Reference to InventoryObject into Hash
+      #
+      # @param data [Hash] Data of the InventoryObject or Reference to InventoryObject
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [Hash] Serialized InventoryObject or Reference data into Hash
+      def data_to_hash(data, depth = 0)
+        raise "Nested lazy_relation of #{inventory_collection} is too deep, left processing: #{data}" if depth > 20
+
+        data.transform_values do |value|
+          if inventory_object_lazy?(value)
+            lazy_relation_to_hash(value, depth)
+          elsif value.kind_of?(Array) && (inventory_object_lazy?(value.compact.first) || inventory_object?(value.compact.first))
+            value.compact.map { |x| lazy_relation_to_hash(x, depth) }
+          elsif inventory_object?(value)
+            lazy_relation_to_hash(value, depth)
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

app/models/manager_refresh/inventory_object.rb

@@ -22,15 +22,6 @@ def manager_uuid
       reference.stringified_reference
     end
 
-    # @return [Hash] serialized InventoryObject into Lazy format
-    def to_raw_lazy_relation
-      {
-        :type                      => "ManagerRefresh::InventoryObjectLazy",
-        :inventory_collection_name => inventory_collection.name,
-        :ems_ref                   => manager_uuid,
-      }
-    end
-
     # @return [ManagerRefresh::InventoryObject] returns self
     def load
       self

app/models/manager_refresh/inventory_object_lazy.rb

@@ -37,17 +37,6 @@ def inspect
       "InventoryObjectLazy:('#{self}', #{inventory_collection}#{suffix})"
     end
 
-    # @return [Hash] serialized InventoryObjectLazy
-    def to_raw_lazy_relation
-      {
-        :type                      => "ManagerRefresh::InventoryObjectLazy",
-        :inventory_collection_name => inventory_collection.name,
-        :reference                 => reference.to_hash,
-        :key                       => key,
-        :default                   => default,
-      }
-    end
-
     # @return [ManagerRefresh::InventoryObject, Object] ManagerRefresh::InventoryObject instance or an attribute
     #         on key
     def load