Implement graph refresh

Add ems_refresh.openshift.inventory_object_refresh setting.
Run refresher_spec file for both old and graph refresh.
Graph refresh preserve images metadata with get_container_images=false
(except tag column).

app/models/manageiq/providers/openshift/container_manager/refresh_parser.rb

@@ -10,9 +10,34 @@ def ems_inv_to_hashes(inventory, options = Config::Options.new)
         get_templates(inventory)
         get_or_merge_openshift_images(inventory) if options.get_container_images
         EmsRefresh.log_inv_debug_trace(@data, "data:")
+
+        # Returning a hash triggers save_inventory_container code path.
         @data
       end
 
+      def initialize_inventory_collections(ems, options)
+        super
+        # get_container_images=false mode is a stopgap to speed up refresh by reducing functionality.  When it's flipped from true to false, we should at least retain existing metadata.
+        if options.get_container_images
+          initialize_custom_attributes_collections(ems.container_images, %w(labels docker_labels))
+        end
+      end
+
+      # We rely on Kubernetes initialize_inventory_collections setting up our collections too.
+      # TODO: does it mean kubernetes refresh would wipe out the openshift tables?
+      def ems_inv_populate_collections(inventory, options = Config::Options.new)
+        super
+        merge_projects_into_namespaces_graph(inventory)
+        get_routes_graph(inventory)
+        get_builds_graph(inventory)
+        get_build_pods_graph(inventory)
+        get_templates_graph(inventory)
+        # For now, we're reusing @data_index-reading-and-writing code path for images.
+        get_or_merge_openshift_images(inventory) if options.get_container_images
+      end
+
+      ## hashes -> save_inventory_container methods
+
       def get_or_merge_openshift_images(inventory)
         inventory["image"].each { |img| get_or_merge_openshift_image(img) }
       end
@@ -87,6 +112,98 @@ def get_templates(inventory)
         end
       end
 
+      ## InventoryObject refresh methods
+
+      def merge_projects_into_namespaces_graph(inventory)
+        collection = @inv_collections[:container_projects]
+
+        inventory["project"].each do |data|
+          h = parse_project(data)
+          # TODO: Assumes full refresh, and running after get_namespaces_graph.
+          #   Would be a problem with partial refresh.
+          namespace = collection.find(h.delete(:name), :ref => :by_name)
+          next if namespace.nil? # ignore openshift projects without an underlying kubernetes namespace
+          namespace.data.merge!(h)
+        end
+      end
+
+      def get_routes_graph(inventory)
+        collection = @inv_collections[:container_routes]
+
+        inventory["route"].each do |data|
+          h = parse_route(data)
+          h[:container_project] = lazy_find_project(:name => h[:namespace])
+          h[:container_service] = lazy_find_service(h.delete(:container_service_ref))
+          custom_attrs = h.extract!(:labels)
+          container_route = collection.build(h)
+
+          get_custom_attributes_graph(container_route, custom_attrs)
+        end
+      end
+
+      def get_builds_graph(inventory)
+        collection = @inv_collections[:container_builds]
+
+        inventory["build_config"].each do |data|
+          h = parse_build(data)
+          h[:container_project] = lazy_find_project(:name => h[:namespace])
+          custom_attrs = h.extract!(:labels)
+          container_build = collection.build(h)
+
+          get_custom_attributes_graph(container_build, custom_attrs)
+        end
+      end
+
+      def get_build_pods_graph(inventory)
+        collection = @inv_collections[:container_build_pods]
+
+        inventory["build"].each do |data|
+          h = parse_build_pod(data)
+          h[:container_build] = lazy_find_build(h.delete(:build_config_ref))
+          custom_attrs = h.extract!(:labels)
+          container_build_pod = collection.build(h)
+
+          get_custom_attributes_graph(container_build_pod, custom_attrs)
+        end
+      end
+
+      def get_templates_graph(inventory)
+        collection = @inv_collections[:container_templates]
+
+        inventory["template"].each do |data|
+          h = parse_template(data)
+          h[:container_project] = lazy_find_project(:name => h[:namespace])
+          parameters = h.delete(:container_template_parameters)
+          custom_attrs = h.extract!(:labels)
+
+          container_template = collection.build(h)
+
+          get_template_parameters_graph(container_template, parameters)
+          get_custom_attributes_graph(container_template, custom_attrs)
+        end
+      end
+
+      def get_template_parameters_graph(parent, parameters)
+        collection = @inv_collections[:container_template_parameters]
+
+        parameters.each do |h|
+          h[:container_template] = parent
+          collection.build(h)
+        end
+      end
+
+      def lazy_find_service(hash)
+        return nil if hash.nil?
+        @inv_collections[:container_services].lazy_find_by(hash, :ref => :by_namespace_and_name)
+      end
+
+      def lazy_find_build(hash)
+        return nil if hash.nil?
+        @inv_collections[:container_builds].lazy_find_by(hash, :ref => :by_namespace_and_name)
+      end
+
+      ## Shared parsing methods
+
       def parse_project(project_item)
         new_result = {:name => project_item.metadata.name}
         unless project_item.metadata.annotations.nil?

app/models/manageiq/providers/openshift/container_manager/refresher.rb

@@ -36,7 +36,12 @@ def parse_legacy_inventory(ems)
         entities = openshift_entities.merge(kube_entities)
         entities["additional_attributes"] = fetch_hawk_inv(ems) || {}
         EmsRefresh.log_inv_debug_trace(entities, "inv_hash:")
-        ManageIQ::Providers::Openshift::ContainerManager::RefreshParser.ems_inv_to_hashes(entities, refresher_options)
+
+        if refresher_options.inventory_object_refresh
+          ManageIQ::Providers::Openshift::ContainerManager::RefreshParser.ems_inv_to_inv_collections(ems, entities, refresher_options)
+        else
+          ManageIQ::Providers::Openshift::ContainerManager::RefreshParser.ems_inv_to_hashes(entities, refresher_options)
+        end
       end
     end
   end

config/settings.yml

@@ -12,6 +12,7 @@
 :ems_refresh:
   :openshift:
     :refresh_interval: 15.minutes
+    :inventory_object_refresh: false
     :get_container_images: true
 :workers:
   :worker_base:

spec/models/manageiq/providers/openshift/container_manager/refresher_spec.rb

@@ -1,4 +1,5 @@
-describe ManageIQ::Providers::Openshift::ContainerManager::Refresher do
+# instantiated at the end, for both classical and graph refresh
+shared_examples "openshift refresher VCR tests" do
   let(:all_images_count) { 40 } # including /oapi/v1/images data
   let(:pod_images_count) { 12 } # only images mentioned by pods
 
@@ -36,7 +37,7 @@ def normal_refresh
     end
   end
 
-  it "will perform a full refresh on openshift" do
+  def full_refresh_test
     2.times do
       @ems.reload
       normal_refresh
@@ -60,6 +61,10 @@ def normal_refresh
     end
   end
 
+  it "will perform a full refresh on openshift" do
+    full_refresh_test
+  end
+
   it "will skip additional attributes when hawk connection fails" do
     hostname = 'host2.example.com'
     token = 'theToken'
@@ -460,6 +465,24 @@ def assert_specific_used_container_image(metadata:)
     # TODO: for next recording, oc label some running, openshift-built image
     expect(@container_image.labels.count).to eq(0)
     expect(@container_image.docker_labels.count).to eq(metadata ? 19 : 0)
+    if metadata
+      expect(@container_image).to have_attributes(
+        :architecture   => "amd64",
+        :author         => nil,
+        :command        => ["/usr/libexec/s2i/run"],
+        :digest         => "sha256:9422207673100308c18bccead913007b76ca3ef48f3c6bb70ce5f19d497c1392",
+        :docker_version => "1.10.3",
+        :entrypoint     => ["container-entrypoint"],
+        :exposed_ports  => {"tcp"=>"8080"},
+        :image_ref      => "docker://172.30.190.81:5000/python-project/python-project@sha256:9422207673100308c18bccead913007b76ca3ef48f3c6bb70ce5f19d497c1392",
+        :registered_on  => "Thu, 08 Dec 2016 06:14:59 UTC +00:00",
+        :size           => 206_435_839,
+
+        # TODO: tag is set by both kubernetes and openshift parsers, so it
+        # regresses to kubernetes value with get_container_images=false.
+        #:tag            => "latest",
+      )
+    end
   end
 
   def assert_container_node_with_no_hawk_attributes
@@ -468,3 +491,29 @@ def assert_container_node_with_no_hawk_attributes
     expect(CustomAttribute.find_by(:name => "test_attr")).to be nil
   end
 end
+
+describe ManageIQ::Providers::Openshift::ContainerManager::Refresher do
+  context "classical refresh" do
+    before(:each) do
+      stub_settings_merge(
+        :ems_refresh => {:openshift => {:inventory_object_refresh => false}}
+      )
+
+      expect(ManageIQ::Providers::Openshift::ContainerManager::RefreshParser).not_to receive(:ems_inv_to_inv_collections)
+    end
+
+    include_examples "openshift refresher VCR tests"
+  end
+
+  context "graph refresh" do
+    before(:each) do
+      stub_settings_merge(
+        :ems_refresh => {:openshift => {:inventory_object_refresh => true}}
+      )
+
+      expect(ManageIQ::Providers::Openshift::ContainerManager::RefreshParser).not_to receive(:ems_inv_to_hashes)
+    end
+
+    include_examples "openshift refresher VCR tests"
+  end
+end