Merge branch 'master' into add_notification

Gemfile

@@ -65,7 +65,7 @@ gem "rugged",                         "~>0.25.0",      :require => false
 gem "simple-rss",                     "~>1.3.1",       :require => false
 gem "snmp",                           "~>1.2.0",       :require => false
 gem "sqlite3",                                         :require => false
-gem "trollop",                        "~>2.0",         :require => false
+gem "trollop",                        "~>2.1.3",       :require => false
 
 # Modified gems (forked on Github)
 gem "ruport",                         "=1.7.0",                       :git => "https://github.com/ManageIQ/ruport.git", :tag => "v1.7.0-3"

app/models/authenticator/base.rb

@@ -49,9 +49,8 @@ def authenticate(username, password, request = nil, options = {})
       user_or_taskid = nil
 
       begin
-        audit = {:event => audit_event, :userid => username}
-
         username = normalize_username(username)
+        audit = {:event => audit_event, :userid => username}
 
         authenticated = options[:authorize_only] || _authenticate(username, password, request)
         if authenticated
@@ -124,6 +123,7 @@ def authorize(taskid, username, *args)
           matching_groups = match_groups(groups_for(identity))
           userid, user = find_or_initialize_user(identity, username)
           update_user_attributes(user, userid, identity)
+          audit_new_user(audit, user) if user.new_record?
           user.miq_groups = matching_groups
 
           if matching_groups.empty?
@@ -195,6 +195,10 @@ def audit_event
       "authenticate_#{self.class.short_name}"
     end
 
+    def audit_new_user(audit, user)
+      audit_success(audit.merge(:message => "User creation successful for User: #{user.name} with ID: #{user.userid}"))
+    end
+
     def authorize?
       config[:"#{self.class.short_name}_role"] == true
     end

app/models/chargeback.rb

@@ -68,6 +68,8 @@ def self.report_row_key(consumption)
     elsif @options.group_by_tenant?
       tenant = @options.tenant_for(consumption)
       "#{tenant ? tenant.id : 'none'}_#{ts_key}"
+    elsif @options.group_by_date_only?
+      ts_key
     else
       default_key(consumption, ts_key)
     end
@@ -151,8 +153,19 @@ def new_chargeback_calculate_costs(consumption, rates)
     end
   end
 
+  def calculate_fixed_compute_metric(consumption)
+    return unless consumption.chargeback_fields_present
+
+    if @options.group_by_date_only?
+      self.fixed_compute_metric ||= 0
+      self.fixed_compute_metric += consumption.chargeback_fields_present
+    else
+      self.fixed_compute_metric = consumption.chargeback_fields_present
+    end
+  end
+
   def calculate_costs(consumption, rates)
-    self.fixed_compute_metric = consumption.chargeback_fields_present if consumption.chargeback_fields_present
+    calculate_fixed_compute_metric(consumption)
     self.class.try(:refresh_dynamic_metric_columns)
 
     rates.each do |rate|
@@ -185,7 +198,9 @@ def self.report_label_field
   def self.set_chargeback_report_options(rpt, group_by, header_for_tag, groupby_label, tz)
     rpt.cols = %w(start_date display_range)
 
-    static_cols       = group_by == "project" ? report_static_cols - ["image_name"] : report_static_cols
+    static_cols       = report_static_cols
+    static_cols      -= ["image_name"] if group_by == "project"
+    static_cols      -= ["vm_name"] if group_by == "date-only"
     static_cols       = group_by == "tag" ? [report_tag_field] : static_cols
     static_cols       = group_by == "label" ? [report_label_field] : static_cols
     static_cols       = group_by == "tenant" ? ['tenant_name'] : static_cols

app/models/chargeback/report_options.rb

@@ -19,6 +19,7 @@ class Chargeback
     :include_metrics,      # enable charging allocated resources with C & U
     :method_for_allocated_metrics,
     :group_by_tenant?,
+    :group_by_date_only?,
     :cumulative_rate_calculation,
   ) do
     def self.new_from_h(hash)
@@ -145,6 +146,10 @@ def group_by_tenant?
       self[:groupby] == 'tenant'
     end
 
+    def group_by_date_only?
+      self[:groupby] == 'date-only'
+    end
+
     private
 
     def tag_hash

app/models/container_label_tag_mapping.rb

@@ -23,7 +23,7 @@ class ContainerLabelTagMapping < ApplicationRecord
 
   require_nested :Mapper
 
-  TAG_PREFIXES = %w(amazon azure kubernetes).map { |name| "/managed/#{name}:" }.freeze
+  TAG_PREFIXES = %w(amazon azure kubernetes openstack).map { |name| "/managed/#{name}:" }.freeze
   validate :validate_tag_prefix
 
   # Return ContainerLabelTagMapping::Mapper instance that holds all current mappings,

app/models/custom_button.rb

@@ -105,6 +105,7 @@ def publish_event(source, target, args)
       :user_id    => args[:user_id],
       :group_id   => args[:miq_group_id],
       :tenant_id  => args[:tenant_id],
+      :timestamp  => Time.now.utc,
       :full_data  => {
         :args                 => args,
         :automate_entry_point => resource_action.ae_path,

app/models/ems_refresh.rb

@@ -130,7 +130,7 @@ def self.get_target_objects(target, single_id = nil)
       # Take care of both String or Class type being passed in
       target_class = target_class.to_s.constantize unless target_class.kind_of?(Class)
 
-      if ManagerRefresh::Inventory.persister_class_for(target_class).blank? &&
+      if ManageIQ::Providers::Inventory.persister_class_for(target_class).blank? &&
          [VmOrTemplate, Host, PhysicalServer, ExtManagementSystem, ManagerRefresh::Target].none? { |k| target_class <= k }
         _log.warn("Unknown target type: [#{target_class}].")
         next

app/models/ext_management_system.rb

@@ -78,6 +78,8 @@ def self.api_allowed_attributes
   has_many :miq_events,             :as => :target, :dependent => :destroy
   has_many :cloud_subnets, :foreign_key => :ems_id, :dependent => :destroy
 
+  has_many :vms_and_templates_advanced_settings, :through => :vms_and_templates, :source => :advanced_settings
+
   validates :name,     :presence => true, :uniqueness => {:scope => [:tenant_id]}
   validates :hostname, :presence => true, :if => :hostname_required?
   validate :hostname_uniqueness_valid?, :hostname_format_valid?, :if => :hostname_required?

app/models/manageiq/providers/base_manager/manager_refresher.rb

@@ -32,7 +32,7 @@ def parse_legacy(ems)
       # @param targets [Array] Array of targets which can be ManageIQ::Providers::BaseManager or ManagerRefresh::Target
       #        or ManagerRefresh::TargetCollection or ApplicationRecord we will be collecting data for.
       # @return [Array<Array>] Array of doubles [target, inventory] with target class from parameter and
-      #         ManagerRefresh::Inventory object
+      #         ManageIQ::Providers::Inventory object
       def collect_inventory_for_targets(ems, targets)
         targets_with_data = targets.collect do |target|
           target_name = target.try(:name) || target.try(:event_type)
@@ -54,7 +54,7 @@ def collect_inventory_for_targets(ems, targets)
       # Persister object. For legacy refresh we invoke parse_legacy.
       # @param ems [ManageIQ::Providers::BaseManager] Manager which targets we want to parse
       # @param _target [Array] Not used in new refresh or legacy refresh by default.
-      # @param inventory [ManagerRefresh::Inventory] Inventory object having Parsers, Collector and Persister objects
+      # @param inventory [ManageIQ::Providers::Inventory] Inventory object having Parsers, Collector and Persister objects
       #        that we need for parsing.
       # @return [Array<Hash> or ManagerRefresh::Persister] Returns parsed Array of hashes for legacy refresh, or
       #         Persister object containing parsed data for new refresh.

app/models/manageiq/providers/base_manager/refresher.rb

@@ -109,7 +109,7 @@ def refresh_targets_for_ems(ems, targets)
           Benchmark.realtime_block(:save_inventory) { save_inventory(ems, target, parsed) }
           _log.info "#{log_header} Refreshing target #{target.class} [#{target.name}] id [#{target.id}]...Complete"
 
-          if parsed.kind_of?(ManagerRefresh::Inventory::Persister)
+          if parsed.kind_of?(ManageIQ::Providers::Inventory::Persister)
             _log.info("#{log_header} ManagerRefresh Post Processing #{target.class} [#{target.name}] id [#{target.id}]...")
             # We have array of InventoryCollection, we want to use that data for post refresh
             Benchmark.realtime_block(:manager_refresh_post_processing) { manager_refresh_post_processing(ems, target, parsed) }
@@ -159,14 +159,14 @@ def parse_targeted_inventory(ems, target, collector)
         # new refreshers should override this method to parse inventory
         # TODO: remove this call after all refreshers support retrieving
         # inventory separate from parsing
-        if collector.kind_of?(ManagerRefresh::Inventory::Collector)
+        if collector.kind_of?(ManageIQ::Providers::Inventory::Collector)
           log_header = format_ems_for_logging(ems)
           _log.debug("#{log_header} Parsing inventory...")
           persister, = Benchmark.realtime_block(:parse_inventory) do
-            persister = ManagerRefresh::Inventory.persister_class_for(target.class).new(ems, target)
-            parser = ManagerRefresh::Inventory.parser_class_for(target.class).new
+            persister = ManageIQ::Providers::Inventory.persister_class_for(target.class).new(ems, target)
+            parser = ManageIQ::Providers::Inventory.parser_class_for(target.class).new
 
-            i = ManagerRefresh::Inventory.new(persister, collector, parser)
+            i = ManageIQ::Providers::Inventory.new(persister, collector, parser)
             i.parse
           end
           _log.debug("#{log_header} Parsing inventory...Complete")
@@ -181,9 +181,9 @@ def parse_targeted_inventory(ems, target, collector)
       #
       # @param ems [ManageIQ::Providers::BaseManager]
       # @param target [ManageIQ::Providers::BaseManager or ManagerRefresh::Target or ManagerRefresh::TargetCollection]
-      # @param parsed [Array<Hash> or ManagerRefresh::Inventory::Persister]
+      # @param parsed [Array<Hash> or ManageIQ::Providers::Inventory::Persister]
       def save_inventory(ems, target, parsed_hashes_or_persister)
-        if parsed_hashes_or_persister.kind_of?(ManagerRefresh::Inventory::Persister)
+        if parsed_hashes_or_persister.kind_of?(ManageIQ::Providers::Inventory::Persister)
           parsed_hashes_or_persister.persist!
         else
           EmsRefresh.save_ems_inventory(ems, parsed_hashes_or_persister, target)

app/models/manageiq/providers/embedded_ansible/inventory/collector/automation_manager.rb

@@ -1,3 +1,3 @@
-class ManageIQ::Providers::EmbeddedAnsible::Inventory::Collector::AutomationManager < ManagerRefresh::Inventory::Collector
+class ManageIQ::Providers::EmbeddedAnsible::Inventory::Collector::AutomationManager < ManageIQ::Providers::Inventory::Collector
   include ManageIQ::Providers::AnsibleTower::Shared::Inventory::Collector::AutomationManager
 end

app/models/manageiq/providers/embedded_ansible/inventory/collector/configuration_script_source.rb

@@ -1,3 +1,3 @@
-class ManageIQ::Providers::EmbeddedAnsible::Inventory::Collector::ConfigurationScriptSource < ManagerRefresh::Inventory::Collector
+class ManageIQ::Providers::EmbeddedAnsible::Inventory::Collector::ConfigurationScriptSource < ManageIQ::Providers::Inventory::Collector
   include ManageIQ::Providers::AnsibleTower::Shared::Inventory::Collector::ConfigurationScriptSource
 end

app/models/manageiq/providers/embedded_ansible/inventory/parser/automation_manager.rb

@@ -1,3 +1,3 @@
-class ManageIQ::Providers::EmbeddedAnsible::Inventory::Parser::AutomationManager < ManagerRefresh::Inventory::Parser
+class ManageIQ::Providers::EmbeddedAnsible::Inventory::Parser::AutomationManager < ManageIQ::Providers::Inventory::Parser
   include ManageIQ::Providers::AnsibleTower::Shared::Inventory::Parser::AutomationManager
 end

app/models/manageiq/providers/embedded_ansible/inventory/parser/configuration_script_source.rb

@@ -1,3 +1,3 @@
-class ManageIQ::Providers::EmbeddedAnsible::Inventory::Parser::ConfigurationScriptSource < ManagerRefresh::Inventory::Parser
+class ManageIQ::Providers::EmbeddedAnsible::Inventory::Parser::ConfigurationScriptSource < ManageIQ::Providers::Inventory::Parser
   include ManageIQ::Providers::AnsibleTower::Shared::Inventory::Parser::ConfigurationScriptSource
 end

app/models/manageiq/providers/inventory.rb

@@ -0,0 +1,61 @@
+module ManageIQ::Providers
+  class Inventory
+    require_nested :Collector
+    require_nested :Parser
+    require_nested :Persister
+
+    attr_accessor :collector, :parsers, :persister
+
+    # Based on the given provider/manager class, this returns correct parser class
+    #
+    # @param klass class of the Provider/Manager
+    # @return [Class] Correct class name of the Parser
+    def self.parser_class_for(klass)
+      provider_module = ManageIQ::Providers::Inflector.provider_module(klass)
+      "#{provider_module}::Inventory::Parser::#{klass.name.demodulize}".safe_constantize
+    rescue ManageIQ::Providers::Inflector::ObjectNotNamespacedError => _err
+      nil
+    end
+
+    # Based on the given provider/manager class, this returns correct persister class
+    #
+    # @param klass class of the Provider/Manager
+    # @return [Class] Correct class name of the persister
+    def self.persister_class_for(klass)
+      provider_module = ManageIQ::Providers::Inflector.provider_module(klass)
+      "#{provider_module}::Inventory::Persister::#{klass.name.demodulize}".safe_constantize
+    rescue ManageIQ::Providers::Inflector::ObjectNotNamespacedError => _err
+      nil
+    end
+
+    # @param persister [ManageIQ::Providers::Inventory::Persister] A Persister object
+    # @param collector [ManageIQ::Providers::Inventory::Collector] A Collector object
+    # @param parsers [ManageIQ::Providers::Inventory::Parser|Array] A Parser object or an array of
+    #   ManageIQ::Providers::Inventory::Parser objects
+    def initialize(persister, collector, parsers)
+      @collector = collector
+      @persister = persister
+      @parsers   = parsers.kind_of?(Array) ? parsers : [parsers]
+    end
+
+    # Invokes all associated parsers storing parsed data into persister.inventory_collections
+    #
+    # @return [ManageIQ::Providers::Inventory::Persister] persister object, to allow chaining
+    def parse
+      parsers.each do |parser|
+        parser.collector = collector
+        parser.persister = persister
+        parser.parse
+      end
+
+      persister
+    end
+
+    # Returns all InventoryCollections contained in persister
+    #
+    # @return [Array<ManagerRefresh::InventoryCollection>] List of InventoryCollections objects
+    def inventory_collections
+      parse.inventory_collections
+    end
+  end
+end

app/models/manageiq/providers/inventory/collector.rb

@@ -0,0 +1,17 @@
+class ManageIQ::Providers::Inventory::Collector
+  attr_reader :manager, :target
+
+  include Vmdb::Logging
+
+  # @param manager [ManageIQ::Providers::BaseManager] A manager object
+  # @param refresh_target [Object] A refresh Target object
+  def initialize(manager, refresh_target)
+    @manager = manager
+    @target  = refresh_target
+  end
+
+  # @return [Config::Options] Options for the manager type
+  def options
+    @options ||= Settings.ems_refresh[manager.class.ems_type]
+  end
+end

app/models/manageiq/providers/inventory/parser.rb

@@ -0,0 +1,10 @@
+class ManageIQ::Providers::Inventory::Parser
+  attr_accessor :collector
+  attr_accessor :persister
+
+  include Vmdb::Logging
+
+  def parse
+    raise NotImplementedError, _("must be implemented in a subclass")
+  end
+end