From 5fe910c1100fe278160a2223bb5490d8969b4f3f Mon Sep 17 00:00:00 2001 From: Alissa Bonas Date: Sun, 12 Apr 2015 16:57:33 +0300 Subject: [PATCH] Addition of OpenShift provider This change Contains: provider, parser, refresher, refresh worker, persistence of container_project and container_route and autofill of default port values for both containers providers in the UI --- lib/Gemfile | 1 + vmdb/app/controllers/ems_common.rb | 4 ++ .../ems_container_helper/textual_summary.rb | 15 +++++ vmdb/app/models/container_project.rb | 7 ++ vmdb/app/models/container_route.rb | 7 ++ vmdb/app/models/ems_kubernetes.rb | 60 +---------------- vmdb/app/models/ems_openshift.rb | 30 +++++++++ .../models/ems_refresh/parsers/openshift.rb | 44 ++++++++++++ .../refreshers/openshift_refresher.rb | 14 ++++ .../ems_refresh/save_inventory_container.rb | 35 +++++++++- .../miq_ems_refresh_worker_openshift.rb | 5 ++ .../worker_management/monitor/class_names.rb | 2 + .../models/mixins/container_provider_mixin.rb | 67 +++++++++++++++++++ vmdb/config/vmdb.tmpl.yml | 3 + ...0150414094834_create_container_projects.rb | 18 +++++ .../20150415131224_create_container_routes.rb | 21 ++++++ .../workers/ems_refresh_worker_openshift.rb | 4 ++ vmdb/spec/factories/ext_management_system.rb | 3 + .../spec/models/ext_management_system_spec.rb | 1 + 19 files changed, 283 insertions(+), 58 deletions(-) create mode 100644 vmdb/app/models/container_project.rb create mode 100644 vmdb/app/models/container_route.rb create mode 100644 vmdb/app/models/ems_openshift.rb create mode 100644 vmdb/app/models/ems_refresh/parsers/openshift.rb create mode 100644 vmdb/app/models/ems_refresh/refreshers/openshift_refresher.rb create mode 100644 vmdb/app/models/miq_ems_refresh_worker_openshift.rb create mode 100644 vmdb/app/models/mixins/container_provider_mixin.rb create mode 100644 vmdb/db/migrate/20150414094834_create_container_projects.rb create mode 100644 vmdb/db/migrate/20150415131224_create_container_routes.rb create mode 100644 vmdb/lib/workers/ems_refresh_worker_openshift.rb diff --git a/lib/Gemfile b/lib/Gemfile index 82a0c127d7e..ee4740dbaf2 100644 --- a/lib/Gemfile +++ b/lib/Gemfile @@ -34,6 +34,7 @@ gem "more_core_extensions", "~>1.2.0", :require => false gem "nokogiri", "~>1.5.0", :require => false gem "ovirt", "~>0.4.2", :require => false gem "kubeclient", ">=0.1.11", :require => false +gem "openshift_client", ">=0.0.3", :require => false gem "rest-client", :require => false, :git => "git://github.com/rest-client/rest-client.git", :ref => "08480eb86aef1e" gem "parallel", "~>0.5.21", :require => false gem "Platform", "=0.4.0", :require => false diff --git a/vmdb/app/controllers/ems_common.rb b/vmdb/app/controllers/ems_common.rb index 0f2ad67695d..b37c5e0aa75 100644 --- a/vmdb/app/controllers/ems_common.rb +++ b/vmdb/app/controllers/ems_common.rb @@ -702,6 +702,10 @@ def get_form_vars @edit[:new][:emstype] = params[:server_emstype] if ["openstack", "openstack_infra"].include?(params[:server_emstype]) @edit[:new][:port] = @ems.port ? @ems.port : 5000 + elsif params[:server_emstype] == EmsKubernetes.ems_type + @edit[:new][:port] = @ems.port ? @ems.port : EmsKubernetes.new.port + elsif params[:server_emstype] == EmsOpenshift.ems_type + @edit[:new][:port] = @ems.port ? @ems.port : EmsOpenshift.new.port else @edit[:new][:port] = nil end diff --git a/vmdb/app/helpers/ems_container_helper/textual_summary.rb b/vmdb/app/helpers/ems_container_helper/textual_summary.rb index e15465b908b..7ebd5779fe1 100644 --- a/vmdb/app/helpers/ems_container_helper/textual_summary.rb +++ b/vmdb/app/helpers/ems_container_helper/textual_summary.rb @@ -11,6 +11,9 @@ def textual_group_properties def textual_group_relationships items = %w(container_nodes container_services container_groups) + if @ems.kind_of?(EmsOpenshift) + items.concat(%w(container_routes container_projects)) + end items.collect { |m| send("textual_#{m}") }.flatten.compact end @@ -72,6 +75,18 @@ def textual_refresh_status } end + def textual_container_routes + count_of_routes = @ems.number_of(:container_routes) + label = "Container Routes" + {:label => label, :image => "container_route", :value => count_of_routes} + end + + def textual_container_projects + count_of_projects = @ems.number_of(:container_projects) + label = "Container Projects" + {:label => label, :image => "container_project", :value => count_of_projects} + end + def textual_container_nodes count_of_nodes = @ems.number_of(:container_nodes) label = "Container Nodes" diff --git a/vmdb/app/models/container_project.rb b/vmdb/app/models/container_project.rb new file mode 100644 index 00000000000..15c833204a5 --- /dev/null +++ b/vmdb/app/models/container_project.rb @@ -0,0 +1,7 @@ +class ContainerProject < ActiveRecord::Base + include CustomAttributeMixin + include ReportableMixin + belongs_to :ext_management_system, :foreign_key => "ems_id" + + has_many :labels, :class_name => CustomAttribute, :as => :resource, :conditions => {:section => "labels"} +end diff --git a/vmdb/app/models/container_route.rb b/vmdb/app/models/container_route.rb new file mode 100644 index 00000000000..142674cd762 --- /dev/null +++ b/vmdb/app/models/container_route.rb @@ -0,0 +1,7 @@ +class ContainerRoute < ActiveRecord::Base + include CustomAttributeMixin + include ReportableMixin + + belongs_to :ext_management_system, :foreign_key => "ems_id" + has_many :labels, :class_name => CustomAttribute, :as => :resource, :conditions => {:section => "labels"} +end diff --git a/vmdb/app/models/ems_kubernetes.rb b/vmdb/app/models/ems_kubernetes.rb index 2e826054ae8..d5788162a22 100644 --- a/vmdb/app/models/ems_kubernetes.rb +++ b/vmdb/app/models/ems_kubernetes.rb @@ -1,8 +1,5 @@ class EmsKubernetes < EmsContainer - has_many :container_nodes, :foreign_key => :ems_id, :dependent => :destroy - has_many :container_groups, :foreign_key => :ems_id, :dependent => :destroy - has_many :container_services, :foreign_key => :ems_id, :dependent => :destroy - has_many :container_replication_controllers, :foreign_key => :ems_id, :dependent => :destroy + include ContainerProviderMixin default_value_for :port, 6443 @@ -14,62 +11,11 @@ def self.description @description ||= "Kubernetes".freeze end - def self.raw_connect(hostname, port) - require 'kubeclient' - api_endpoint = raw_api_endpoint(hostname, port) - kube = Kubeclient::Client.new(api_endpoint) - # TODO: support real authentication using certificates - kube.ssl_options(:verify_ssl => OpenSSL::SSL::VERIFY_NONE) - kube - end - - def self.raw_api_endpoint(hostname, port) - URI::HTTPS.build(:host => hostname, :port => port.to_i) - end - - # UI methods for determining availability of fields - def supports_port? - true - end - - def api_endpoint - self.class.raw_api_endpoint(hostname, port) - end - - def connect(_options = {}) - self.class.raw_connect(hostname, port) + def self.raw_connect(hostname, port, _options = {}) + kubernetes_connect(hostname, port) end def self.event_monitor_class MiqEventCatcherKubernetes end - - def authentication_check - # TODO: support real authentication using certificates - [true, ""] - end - - def verify_credentials(_auth_type = nil, _options = {}) - # TODO: support real authentication using certificates - true - end - - def authentication_status_ok?(_type = nil) - # TODO: support real authentication using certificates - true - end - - # required by aggregate_hardware - def all_computer_system_ids - MiqPreloader.preload(container_nodes, :computer_system) - container_nodes.collect { |n| n.computer_system.id } - end - - def aggregate_logical_cpus(targets = nil) - aggregate_hardware(:computer_systems, :logical_cpus, targets) - end - - def aggregate_memory(targets = nil) - aggregate_hardware(:computer_systems, :memory_cpu, targets) - end end diff --git a/vmdb/app/models/ems_openshift.rb b/vmdb/app/models/ems_openshift.rb new file mode 100644 index 00000000000..9312c039e6f --- /dev/null +++ b/vmdb/app/models/ems_openshift.rb @@ -0,0 +1,30 @@ +class EmsOpenshift < EmsContainer + include ContainerProviderMixin + + has_many :container_routes, :foreign_key => :ems_id, :dependent => :destroy + has_many :container_projects, :foreign_key => :ems_id, :dependent => :destroy + + default_value_for :port, 8443 + + def self.ems_type + @ems_type ||= "openshift".freeze + end + + def self.description + @description ||= "OpenShift".freeze + end + + def self.raw_connect(hostname, port, service = nil) + service ||= "openshift" + send("#{service}_connect", hostname, port) + end + + def self.openshift_connect(hostname, port) + require 'openshift_client' + api_endpoint = raw_api_endpoint(hostname, port) + osclient = OpenshiftClient::Client.new(api_endpoint) + # TODO: support real authentication using certificates + osclient.ssl_options(:verify_ssl => OpenSSL::SSL::VERIFY_NONE) + osclient + end +end diff --git a/vmdb/app/models/ems_refresh/parsers/openshift.rb b/vmdb/app/models/ems_refresh/parsers/openshift.rb new file mode 100644 index 00000000000..32261c185a5 --- /dev/null +++ b/vmdb/app/models/ems_refresh/parsers/openshift.rb @@ -0,0 +1,44 @@ +module EmsRefresh + module Parsers + class Openshift < Kubernetes + def ems_inv_to_hashes(inventory) + super(inventory) + get_projects(inventory) + get_routes(inventory) + EmsRefresh.log_inv_debug_trace(@data, "data:") + @data + end + + def get_routes(inventory) + process_collection(inventory["route"], :container_routes) { |n| parse_route(n) } + end + + def get_projects(inventory) + process_collection(inventory["project"], :container_projects) { |n| parse_project(n) } + end + + def parse_project(project) + new_result = parse_base_item(project) + new_result.except!(:namespace) + new_result.merge!( + :labels => parse_labels(project), + :display_name => project.displayName + ) + new_result + end + + def parse_route(route) + new_result = parse_base_item(route) + new_result.merge!( + # TODO: persist tls + :host_name => route.host, + :labels => parse_labels(route), + # TODO: this part needs to be modified to service_id instead + :service_name => route.serviceName, + :path => route.path + ) + new_result + end + end + end +end diff --git a/vmdb/app/models/ems_refresh/refreshers/openshift_refresher.rb b/vmdb/app/models/ems_refresh/refreshers/openshift_refresher.rb new file mode 100644 index 00000000000..6a3b1da1c7d --- /dev/null +++ b/vmdb/app/models/ems_refresh/refreshers/openshift_refresher.rb @@ -0,0 +1,14 @@ +module EmsRefresh + module Refreshers + class OpenshiftRefresher < BaseRefresher + include EmsRefresherMixin + + def parse_inventory(ems, _targets = nil) + all_entities = ems.with_provider_connection(&:all_entities) + all_entities.merge!(ems.with_provider_connection(:service => EmsKubernetes.ems_type, &:all_entities)) + EmsRefresh.log_inv_debug_trace(all_entities, "inv_hash:") + EmsRefresh::Parsers::Openshift.ems_inv_to_hashes(all_entities) + end + end + end +end diff --git a/vmdb/app/models/ems_refresh/save_inventory_container.rb b/vmdb/app/models/ems_refresh/save_inventory_container.rb index cd1c12ee7a0..829b572146e 100644 --- a/vmdb/app/models/ems_refresh/save_inventory_container.rb +++ b/vmdb/app/models/ems_refresh/save_inventory_container.rb @@ -1,7 +1,8 @@ module EmsRefresh::SaveInventoryContainer def save_ems_container_inventory(ems, hashes, target = nil) target = ems if target.nil? - child_keys = [:container_nodes, :container_groups, :container_services, :container_replication_controllers] + child_keys = [:container_nodes, :container_groups, :container_services, + :container_replication_controllers, :container_routes, :container_projects] # Save and link other subsections child_keys.each do |k| @@ -11,6 +12,38 @@ def save_ems_container_inventory(ems, hashes, target = nil) ems.save! end + def save_container_projects_inventory(ems, hashes, target = nil) + return if hashes.nil? + target = ems if target.nil? + + ems.container_projects(true) + deletes = if target.kind_of?(ExtManagementSystem) + ems.container_projects.dup + else + [] + end + + save_inventory_multi(:container_projects, ems, hashes, deletes, [:ems_ref], + :labels) + store_ids_for_new_records(ems.container_projects, hashes, :ems_ref) + end + + def save_container_routes_inventory(ems, hashes, target = nil) + return if hashes.nil? + target = ems if target.nil? + + ems.container_routes(true) + deletes = if target.kind_of?(ExtManagementSystem) + ems.container_routes.dup + else + [] + end + + save_inventory_multi(:container_routes, ems, hashes, deletes, [:ems_ref], + :labels) + store_ids_for_new_records(ems.container_routes, hashes, :ems_ref) + end + def save_container_nodes_inventory(ems, hashes, target = nil) return if hashes.nil? target = ems if target.nil? diff --git a/vmdb/app/models/miq_ems_refresh_worker_openshift.rb b/vmdb/app/models/miq_ems_refresh_worker_openshift.rb new file mode 100644 index 00000000000..12c3f3e1cc3 --- /dev/null +++ b/vmdb/app/models/miq_ems_refresh_worker_openshift.rb @@ -0,0 +1,5 @@ +class MiqEmsRefreshWorkerOpenshift < MiqEmsRefreshWorker + def self.ems_class + EmsOpenshift + end +end diff --git a/vmdb/app/models/miq_server/worker_management/monitor/class_names.rb b/vmdb/app/models/miq_server/worker_management/monitor/class_names.rb index ba179517e7c..254e930433c 100644 --- a/vmdb/app/models/miq_server/worker_management/monitor/class_names.rb +++ b/vmdb/app/models/miq_server/worker_management/monitor/class_names.rb @@ -13,6 +13,7 @@ module MiqServer::WorkerManagement::Monitor::ClassNames MiqEmsRefreshWorkerForemanConfiguration MiqEmsRefreshWorkerForemanProvisioning MiqEmsRefreshWorkerKubernetes + MiqEmsRefreshWorkerOpenshift MiqEmsRefreshWorkerMicrosoft MiqEmsRefreshWorkerRedhat MiqEmsRefreshWorkerOpenstack @@ -60,6 +61,7 @@ module MiqServer::WorkerManagement::Monitor::ClassNames MiqEmsRefreshWorkerForemanConfiguration MiqEmsRefreshWorkerForemanProvisioning MiqEmsRefreshWorkerKubernetes + MiqEmsRefreshWorkerOpenshift MiqEmsRefreshWorkerMicrosoft MiqEmsRefreshWorkerRedhat MiqEmsRefreshWorkerOpenstack diff --git a/vmdb/app/models/mixins/container_provider_mixin.rb b/vmdb/app/models/mixins/container_provider_mixin.rb new file mode 100644 index 00000000000..a6be1d5603d --- /dev/null +++ b/vmdb/app/models/mixins/container_provider_mixin.rb @@ -0,0 +1,67 @@ +module ContainerProviderMixin + extend ActiveSupport::Concern + + included do + has_many :container_nodes, :foreign_key => :ems_id, :dependent => :destroy + has_many :container_groups, :foreign_key => :ems_id, :dependent => :destroy + has_many :container_services, :foreign_key => :ems_id, :dependent => :destroy + has_many :container_replication_controllers, :foreign_key => :ems_id, :dependent => :destroy + end + + module ClassMethods + def raw_api_endpoint(hostname, port) + URI::HTTPS.build(:host => hostname, :port => port.to_i) + end + + def kubernetes_connect(hostname, port) + require 'kubeclient' + api_endpoint = raw_api_endpoint(hostname, port) + kubeclient = Kubeclient::Client.new(api_endpoint) + # TODO: support real authentication using certificates + kubeclient.ssl_options(:verify_ssl => OpenSSL::SSL::VERIFY_NONE) + kubeclient + end + end + + # UI methods for determining availability of fields + def supports_port? + true + end + + def api_endpoint + self.class.raw_api_endpoint(hostname, port) + end + + def connect(options = {}) + self.class.raw_connect(hostname, port, options[:service]) + end + + def authentication_check + # TODO: support real authentication using certificates + [true, ""] + end + + def verify_credentials(_auth_type = nil, _options = {}) + # TODO: support real authentication using certificates + true + end + + def authentication_status_ok?(_type = nil) + # TODO: support real authentication using certificates + true + end + + # required by aggregate_hardware + def all_computer_system_ids + MiqPreloader.preload(container_nodes, :computer_system) + container_nodes.collect { |n| n.computer_system.id } + end + + def aggregate_logical_cpus(targets = nil) + aggregate_hardware(:computer_systems, :logical_cpus, targets) + end + + def aggregate_memory(targets = nil) + aggregate_hardware(:computer_systems, :memory_cpu, targets) + end +end diff --git a/vmdb/config/vmdb.tmpl.yml b/vmdb/config/vmdb.tmpl.yml index f921f1ef8c9..22d102e4a21 100644 --- a/vmdb/config/vmdb.tmpl.yml +++ b/vmdb/config/vmdb.tmpl.yml @@ -77,6 +77,8 @@ ems_refresh: full_refresh_threshold: 100 kubernetes: :refresh_interval: 15.minutes + openshift: + :refresh_interval: 15.minutes raise_vm_snapshot_complete_if_created_within: 15.minutes refresh_interval: 24.hours scvmm: @@ -352,6 +354,7 @@ workers: :ems_refresh_worker_foreman_configuration: {} :ems_refresh_worker_foreman_provisioning: {} :ems_refresh_worker_kubernetes: {} + :ems_refresh_worker_openshift: {} :ems_refresh_worker_microsoft: {} :ems_refresh_worker_redhat: {} :ems_refresh_worker_openstack: {} diff --git a/vmdb/db/migrate/20150414094834_create_container_projects.rb b/vmdb/db/migrate/20150414094834_create_container_projects.rb new file mode 100644 index 00000000000..721f64f7f01 --- /dev/null +++ b/vmdb/db/migrate/20150414094834_create_container_projects.rb @@ -0,0 +1,18 @@ +class CreateContainerProjects < ActiveRecord::Migration + def up + create_table :container_projects do |t| + t.string :ems_ref + t.string :name + t.timestamp :creation_timestamp + t.string :resource_version + t.string :display_name + t.belongs_to :ems, :type => :bigint + end + add_index :container_projects, :ems_id + end + + def down + remove_index :container_projects, :ems_id + drop_table :container_projects + end +end diff --git a/vmdb/db/migrate/20150415131224_create_container_routes.rb b/vmdb/db/migrate/20150415131224_create_container_routes.rb new file mode 100644 index 00000000000..6f6d9013925 --- /dev/null +++ b/vmdb/db/migrate/20150415131224_create_container_routes.rb @@ -0,0 +1,21 @@ +class CreateContainerRoutes < ActiveRecord::Migration + def up + create_table :container_routes do |t| + t.string :ems_ref + t.string :name + t.timestamp :creation_timestamp + t.string :resource_version + t.string :namespace + t.string :host_name + t.string :service_name + t.string :path + t.belongs_to :ems, :type => :bigint + end + add_index :container_routes, :ems_id + end + + def down + remove_index :container_routes, :ems_id + drop_table :container_routes + end +end diff --git a/vmdb/lib/workers/ems_refresh_worker_openshift.rb b/vmdb/lib/workers/ems_refresh_worker_openshift.rb new file mode 100644 index 00000000000..89d0a734258 --- /dev/null +++ b/vmdb/lib/workers/ems_refresh_worker_openshift.rb @@ -0,0 +1,4 @@ +require 'workers/ems_refresh_worker' + +class EmsRefreshWorkerOpenshift < EmsRefreshWorker +end diff --git a/vmdb/spec/factories/ext_management_system.rb b/vmdb/spec/factories/ext_management_system.rb index 29a4039d3bb..5ab85ed74c2 100644 --- a/vmdb/spec/factories/ext_management_system.rb +++ b/vmdb/spec/factories/ext_management_system.rb @@ -95,6 +95,9 @@ factory :ems_kubernetes, :class => "EmsKubernetes", :parent => :ems_container do end + factory :ems_openshift, :class => "EmsOpenshift", :parent => :ems_container do + end + # Leaf classes for configuration_manager factory :configuration_manager_foreman, :class => "ConfigurationManagerForeman", :parent => :configuration_manager do diff --git a/vmdb/spec/models/ext_management_system_spec.rb b/vmdb/spec/models/ext_management_system_spec.rb index 8169e3eb657..30ebafd761a 100644 --- a/vmdb/spec/models/ext_management_system_spec.rb +++ b/vmdb/spec/models/ext_management_system_spec.rb @@ -14,6 +14,7 @@ foreman_configuration foreman_provisioning kubernetes + openshift openstack openstack_infra rhevm